Run linters on merge requests
[ghc.git] / .gitlab-ci.yml
1 variables:
2   GIT_SSL_NO_VERIFY: "1"
3
4   # Commit of ghc/ci-images repository from which to pull Docker images
5   DOCKER_REV: cefaee3c742af193e0f7783f87edb0d35374515c
6
7 before_script:
8   - python3 .gitlab/fix-submodules.py
9   - git submodule sync --recursive
10   - git submodule update --init --recursive
11   - git checkout .gitmodules
12   - "git fetch https://gitlab.haskell.org/ghc/ghc-performance-notes.git refs/notes/perf:refs/notes/perf || true"
13
14 stages:
15   - lint       # Source linting
16   - build      # A quick smoke-test to weed out broken commits
17   - full-build # Build all the things
18   - cleanup    # See Note [Cleanup on Windows]
19   - packaging  # Source distribution, etc.
20   - hackage    # head.hackage testing
21
22 .only-default: &only-default
23   only:
24     - master
25     - /ghc-[0-9]+\.[0-9]+/
26     - merge_requests
27     - tags
28
29 ############################################################
30 # Runner Tags
31 ############################################################
32 #
33 # * x86_64-linux: Any Docker-capable x86_64 Linux machine
34 # * aarch64-linux: Any Docker-capable AArch64 Linux machine
35 # * x86_64-windows: A x86_64 Windows machine
36 # * lint: Any Docker-capable x86_64 Linux machine; distinct from
37 #         x86_64-linux to ensure low-latency availability.
38 #
39
40
41 ############################################################
42 # Linting
43 ############################################################
44
45 ghc-linters:
46   stage: lint
47   image: "registry.gitlab.haskell.org/ghc/ci-images/linters:$DOCKER_REV"
48   script:
49     - git fetch origin
50     - git checkout "$CI_MERGE_REQUEST_TARGET_BRANCH_NAME"
51     - git reset --hard "origin/$CI_MERGE_REQUEST_TARGET_BRANCH_NAME"
52     - echo "$CI_MERGE_REQUEST_ID"
53     - echo "$CI_MERGE_REQUEST_TARGET_BRANCH_NAME"
54     - echo "$CI_COMMIT_SHA"
55     - base="$(git merge-base $CI_MERGE_REQUEST_TARGET_BRANCH_NAME $CI_COMMIT_SHA)"
56     - echo $base
57     #    - validate-commit-msg .git $(git rev-list $base..$CI_COMMIT_SHA)
58     - validate-whitespace .git $(git rev-list $base..$CI_COMMIT_SHA)
59     - .gitlab/linters/check-makefiles.py $base $CI_COMMIT_SHA
60     - .gitlab/linters/check-cpp.py $base $CI_COMMIT_SHA
61   dependencies: []
62   tags:
63     - lint
64   only:
65     refs:
66       - merge_requests
67
68 # We allow the submodule checker to fail when run on merge requests (to
69 # accomodate, e.g., haddock changes not yet upstream) but not on `master` or
70 # Marge jobs.
71 .lint-submods:
72   stage: lint
73   image: "registry.gitlab.haskell.org/ghc/ci-images/linters:$DOCKER_REV"
74   script:
75     - submodchecker .git $(git rev-list $base..$CI_COMMIT_SHA)
76   dependencies: []
77   tags:
78     - lint
79
80 lint-submods:
81   extends: .lint-submods
82   only:
83     refs:
84       - master
85       - /ghc-[0-9]+\.[0-9]+/
86       - wip/marge_bot_batch_merge_job
87
88 lint-submods-mr:
89   extends: .lint-submods
90   allow_failure: true
91   only:
92     refs:
93       - merge_requests
94
95 ############################################################
96 # Validation via Pipelines (hadrian)
97 ############################################################
98
99 .validate-hadrian:
100   <<: *only-default
101   allow_failure: true
102   script:
103     - cabal update
104     - git clean -xdf && git submodule foreach git clean -xdf
105     - bash .circleci/prepare-system.sh
106     - if [[ -d ./cabal-cache ]]; then cp -R ./.cabal-cache ~/.cabal-cache; fi
107     - ./boot
108     - ./configure $CONFIGURE_ARGS
109     - hadrian/build.cabal.sh -j`mk/detect-cpu-count.sh` --docs=no-sphinx binary-dist
110     - mv _build/bindist/ghc*.tar.xz ghc.tar.xz
111   cache:
112     key: hadrian
113     paths:
114       - cabal-cache
115   artifacts:
116     when: always
117     paths:
118       - ghc.tar.xz
119
120 validate-x86_64-linux-deb8-hadrian:
121   extends: .validate-hadrian
122   stage: build
123   image: "registry.gitlab.haskell.org/ghc/ci-images/x86_64-linux-deb8:$DOCKER_REV"
124   before_script:
125     # workaround for docker permissions
126     - sudo chown ghc:ghc -R .
127     - python3 .gitlab/fix-submodules.py
128     - git submodule sync --recursive
129     - git submodule update --init --recursive
130     - git checkout .gitmodules
131     - "git fetch https://gitlab.haskell.org/ghc/ghc-performance-notes.git refs/notes/perf:refs/notes/perf || true"
132   tags:
133     - x86_64-linux
134
135 hadrian-ghc-in-ghci:
136   <<: *only-default
137   stage: build
138   image: ghcci/x86_64-linux-deb8:0.1
139   before_script:
140     # workaround for docker permissions
141     - sudo chown ghc:ghc -R .
142     - python3 .gitlab/fix-submodules.py
143     - git submodule sync --recursive
144     - git submodule update --init --recursive
145     - git checkout .gitmodules
146   tags:
147     - x86_64-linux
148   script:
149     - cabal update
150     - git clean -xdf && git submodule foreach git clean -xdf
151     - bash .circleci/prepare-system.sh
152     - if [[ -d ./cabal-cache ]]; then cp -R ./.cabal-cache ~/.cabal-cache; fi
153     - ./boot
154     - ./configure $CONFIGURE_ARGS
155     # Load ghc-in-ghci then immediately exit
156     - echo ":q" | hadrian/ghci.sh
157   cache:
158     key: hadrian-ghci
159     paths:
160       - cabal-cache
161
162 ############################################################
163 # Validation via Pipelines (make)
164 ############################################################
165
166 .validate:
167   <<: *only-default
168   variables:
169     TEST_TYPE: test
170   before_script:
171     - git clean -xdf && git submodule foreach git clean -xdf
172   script:
173     - ./boot
174     - ./configure $CONFIGURE_ARGS
175     - |
176       THREADS=`mk/detect-cpu-count.sh`
177       make V=0 -j$THREADS WERROR=-Werror
178     - |
179       make binary-dist TAR_COMP_OPTS="-1"
180     - |
181       # Prepare to push git notes.
182       METRICS_FILE=$(mktemp)
183       git config user.email "ben+ghc-ci@smart-cactus.org"
184       git config user.name "GHC GitLab CI"
185     - |
186       THREADS=`mk/detect-cpu-count.sh`
187       make $TEST_TYPE THREADS=$THREADS JUNIT_FILE=../../junit.xml METRICS_FILE=$METRICS_FILE
188     - |
189       # Push git notes.
190       METRICS_FILE=$METRICS_FILE .gitlab/push-test-metrics.sh
191   dependencies: []
192   artifacts:
193     reports:
194       junit: junit.xml
195     expire_in: 2 week
196     paths:
197       - ghc-*.tar.xz
198       - junit.xml
199
200 #################################
201 # x86_64-darwin
202 #################################
203
204 validate-x86_64-darwin:
205   extends: .validate
206   stage: full-build
207   tags:
208     - x86_64-darwin
209   variables:
210     GHC_VERSION: 8.6.3
211     BIN_DIST_PREP_TAR_COMP: "bindistprep/ghc-x86_64-apple-darwin.tar.xz"
212     MACOSX_DEPLOYMENT_TARGET: "10.7"
213     # Only Sierra and onwards supports clock_gettime. See #12858
214     ac_cv_func_clock_gettime: "no"
215     LANG: "en_US.UTF-8"
216     CONFIGURE_ARGS: --with-intree-gmp
217     TEST_ENV: "x86_64-darwin"
218   before_script:
219     - git clean -xdf && git submodule foreach git clean -xdf
220     - python3 .gitlab/fix-submodules.py
221     - git submodule sync --recursive
222     - git submodule update --init --recursive
223     - git checkout .gitmodules
224     - "git fetch https://gitlab.haskell.org/ghc/ghc-performance-notes.git refs/notes/perf:refs/notes/perf || true"
225
226     - bash .gitlab/darwin-init.sh
227     - PATH="`pwd`/toolchain/bin:$PATH"
228   after_script:
229     - cp -Rf $HOME/.cabal cabal-cache
230   artifacts:
231     when: always
232     expire_in: 2 week
233   cache:
234     key: darwin
235     paths:
236       - cabal-cache
237       - toolchain
238
239 .validate-linux:
240   extends: .validate
241   tags:
242     - x86_64-linux
243   before_script:
244     - git clean -xdf && git submodule foreach git clean -xdf
245     - python3 .gitlab/fix-submodules.py
246     - git submodule sync --recursive
247     - git submodule update --init --recursive
248     - git checkout .gitmodules
249     - "git fetch https://gitlab.haskell.org/ghc/ghc-performance-notes.git refs/notes/perf:refs/notes/perf || true"
250
251     - bash .circleci/prepare-system.sh
252     # workaround for docker permissions
253     - sudo chown ghc:ghc -R .
254   after_script:
255     - cp -Rf $HOME/.cabal cabal-cache
256   cache:
257     key: linux
258     paths:
259       - cabal-cache
260       - toolchain
261
262 #################################
263 # aarch64-linux-deb9
264 #################################
265
266 .build-aarch64-linux-deb9:
267   extends: .validate-linux
268   stage: full-build
269   image: "registry.gitlab.haskell.org/ghc/ci-images/aarch64-linux-deb9:$DOCKER_REV"
270   allow_failure: true
271   variables:
272     TEST_ENV: "aarch64-linux-deb9"
273     BIN_DIST_PREP_TAR_COMP: "bindistprep/ghc-aarch64-linux-deb9.tar.xz"
274   cache:
275     key: linux-aarch64-deb9
276   tags:
277     - aarch64-linux
278
279 validate-aarch64-linux-deb9:
280   extends: .build-aarch64-linux-deb9
281   artifacts:
282     when: always
283     expire_in: 2 week
284
285 nightly-aarch64-linux-deb9:
286   extends: .build-aarch64-linux-deb9
287   artifacts:
288     expire_in: 2 year
289   variables:
290     TEST_TYPE: slowtest
291   only:
292     variables:
293       - $NIGHTLY
294
295 #################################
296 # i386-linux-deb9
297 #################################
298
299 .build-i386-linux-deb9:
300   extends: .validate-linux
301   stage: full-build
302   image: "registry.gitlab.haskell.org/ghc/ci-images/i386-linux-deb9:$DOCKER_REV"
303   variables:
304     TEST_ENV: "i386-linux-deb9"
305     BIN_DIST_PREP_TAR_COMP: "bindistprep/ghc-i386-deb9-linux.tar.xz"
306   cache:
307     key: linux-i386-deb9
308
309 validate-i386-linux-deb9:
310   extends: .build-i386-linux-deb9
311   artifacts:
312     when: always
313     expire_in: 2 week
314
315 nightly-i386-linux-deb9:
316   extends: .build-i386-linux-deb9
317   variables:
318     TEST_TYPE: slowtest
319   artifacts:
320     when: always
321     expire_in: 2 week
322   only:
323     variables:
324       - $NIGHTLY
325
326 #################################
327 # x86_64-linux-deb9
328 #################################
329
330 .build-x86_64-linux-deb9:
331   extends: .validate-linux
332   stage: full-build
333   image: "registry.gitlab.haskell.org/ghc/ci-images/x86_64-linux-deb9:$DOCKER_REV"
334   variables:
335     TEST_ENV: "x86_64-linux-deb9"
336     BIN_DIST_PREP_TAR_COMP: "bindistprep/ghc-x86_64-deb9-linux.tar.xz"
337   cache:
338     key: linux-x86_64-deb9
339
340 validate-x86_64-linux-deb9:
341   extends: .build-x86_64-linux-deb9
342   artifacts:
343     when: always
344     expire_in: 2 week
345
346 nightly-x86_64-linux-deb9:
347   extends: .build-x86_64-linux-deb9
348   artifacts:
349     expire_in: 2 year
350   variables:
351     TEST_TYPE: slowtest
352   only:
353     variables:
354       - $NIGHTLY
355
356 # N.B. Has DEBUG assertions enabled in stage2
357 validate-x86_64-linux-deb9-debug:
358   extends: .build-x86_64-linux-deb9
359   stage: build
360   variables:
361     BUILD_FLAVOUR: validate
362     TEST_ENV: "x86_64-linux-deb9-debug"
363
364 validate-x86_64-linux-deb9-llvm:
365   extends: .build-x86_64-linux-deb9
366   stage: full-build
367   variables:
368     BUILD_FLAVOUR: perf-llvm
369     TEST_ENV: "x86_64-linux-deb9-llvm"
370
371 validate-x86_64-linux-deb9-integer-simple:
372   extends: .build-x86_64-linux-deb9
373   stage: full-build
374   variables:
375     INTEGER_LIBRARY: integer-simple
376     TEST_ENV: "x86_64-linux-deb9-integer-simple"
377     BIN_DIST_PREP_TAR_COMP: "bindistprep/ghc-x86_64-deb9-linux-integer-simple.tar.xz"
378
379 nightly-x86_64-linux-deb9-integer-simple:
380   extends: .build-x86_64-linux-deb9
381   stage: full-build
382   variables:
383     INTEGER_LIBRARY: integer-simple
384     TEST_ENV: "x86_64-linux-deb9-integer-simple"
385     TEST_TYPE: slowtest
386   artifacts:
387     expire_in: 2 year
388   only:
389     variables:
390       - $NIGHTLY
391
392 validate-x86_64-linux-deb9-unreg:
393   extends: .build-x86_64-linux-deb9
394   stage: full-build
395   variables:
396     CONFIGURE_ARGS: --enable-unregisterised
397     TEST_ENV: "x86_64-linux-deb9-unreg"
398
399
400 #################################
401 # x86_64-linux-deb8
402 #################################
403
404 release-x86_64-linux-deb8:
405   extends: .validate-linux
406   stage: full-build
407   image: "registry.gitlab.haskell.org/ghc/ci-images/x86_64-linux-deb8:$DOCKER_REV"
408   variables:
409     TEST_ENV: "x86_64-linux-deb8"
410     BIN_DIST_PREP_TAR_COMP: "bindistprep/ghc-x86_64-deb8-linux.tar.xz"
411   only:
412     - tags
413   cache:
414     key: linux-x86_64-deb8
415   artifacts:
416     when: always
417     expire_in: 2 week
418
419
420 #################################
421 # x86_64-linux-fedora27
422 #################################
423
424 validate-x86_64-linux-fedora27:
425   extends: .validate-linux
426   stage: full-build
427   image: "registry.gitlab.haskell.org/ghc/ci-images/x86_64-linux-fedora27:$DOCKER_REV"
428   variables:
429     TEST_ENV: "x86_64-linux-fedora27"
430     BIN_DIST_PREP_TAR_COMP: "bindistprep/ghc-x86_64-fedora27-linux.tar.xz"
431   cache:
432     key: linux-x86_64-fedora27
433   artifacts:
434     when: always
435     expire_in: 2 week
436
437 ############################################################
438 # Validation via Pipelines (Windows)
439 ############################################################
440
441 .build-windows:
442   <<: *only-default
443   before_script:
444     - git clean -xdf
445     - git submodule foreach git clean -xdf
446
447     # Use a local temporary directory to ensure that concurrent builds don't
448     # interfere with one another
449     - |
450       mkdir tmp
451       set TMP=%cd%\tmp
452       set TEMP=%cd%\tmp
453
454     - set PATH=C:\msys64\usr\bin;%PATH%
455     - python .gitlab/fix-submodules.py
456     - git submodule sync --recursive
457     - git submodule update --init --recursive
458     - git checkout .gitmodules
459     - "git fetch https://gitlab.haskell.org/ghc/ghc-performance-notes.git refs/notes/perf:refs/notes/perf || true"
460     - bash .gitlab/win32-init.sh
461   after_script:
462     - rd /s /q tmp
463     - robocopy /np /nfl /ndl /e "%APPDATA%\cabal" cabal-cache
464     - bash -c 'make clean || true'
465   dependencies: []
466   variables:
467     FORCE_SYMLINKS: 1
468     LANG: "en_US.UTF-8"
469   cache:
470     paths:
471       - cabal-cache
472       - ghc-8.6.2
473       - ghc-tarballs
474
475 .build-windows-hadrian:
476   extends: .build-windows
477   stage: full-build
478   variables:
479     GHC_VERSION: "8.6.2"
480   script:
481     - |
482       python boot
483       bash -c './configure --enable-tarballs-autodownload GHC=`pwd`/toolchain/bin/ghc HAPPY=`pwd`/toolchain/bin/happy ALEX=`pwd`/toolchain/bin/alex'
484     - bash -c "PATH=`pwd`/toolchain/bin:$PATH hadrian/build.cabal.sh -j`mk/detect-cpu-count.sh` --flavour=Quick --docs=no-sphinx binary-dist"
485     - mv _build/bindist/ghc*.tar.xz ghc.tar.xz
486     # FIXME: Testsuite disabled due to #16156.
487     # - bash -c 'make V=0 test THREADS=`mk/detect-cpu-count.sh` JUNIT_FILE=../../junit.xml'
488   tags:
489     - x86_64-windows
490   artifacts:
491     when: always
492     paths:
493       - ghc.tar.xz
494
495 validate-x86_64-windows-hadrian:
496   extends: .build-windows-hadrian
497   variables:
498     MSYSTEM: MINGW64
499   cache:
500     key: x86_64-windows-hadrian
501
502 nightly-i386-windows-hadrian:
503   extends: .build-windows-hadrian
504   variables:
505     MSYSTEM: MINGW32
506   only:
507     variables:
508       - $NIGHTLY
509   cache:
510     key: i386-windows-hadrian
511
512 .build-windows-make:
513   extends: .build-windows
514   stage: full-build
515   # due to #16084
516   allow_failure: true
517   variables:
518     GHC_VERSION: "8.6.2"
519     BIN_DIST_PREP_TAR_COMP: "bindistprep/ghc-x86_64-mingw32.tar.xz"
520   script:
521     - |
522       python boot
523       bash -c './configure --enable-tarballs-autodownload GHC=`pwd`/toolchain/bin/ghc HAPPY=`pwd`/toolchain/bin/happy ALEX=`pwd`/toolchain/bin/alex'
524     - bash -c "echo include mk/flavours/quick.mk > mk/build.mk"
525     - bash -c "PATH=`pwd`/toolchain/bin:$PATH make -j`mk/detect-cpu-count.sh`"
526     - bash -c "PATH=`pwd`/toolchain/bin:$PATH make binary-dist TAR_COMP_OPTS=-1"
527     - bash -c 'make V=0 test THREADS=`mk/detect-cpu-count.sh` JUNIT_FILE=../../junit.xml'
528   tags:
529     - x86_64-windows
530   artifacts:
531     when: always
532     expire_in: 2 week
533     reports:
534       junit: junit.xml
535     paths:
536       - ghc-*.tar.xz
537       - junit.xml
538
539 validate-x86_64-windows:
540   extends: .build-windows-make
541   variables:
542     MSYSTEM: MINGW64
543   cache:
544     key: x86_64-windows
545
546 release-i386-windows:
547   extends: .build-windows-make
548   only:
549     - tags
550   variables:
551     MSYSTEM: MINGW32
552   cache:
553     key: i386-windows
554
555 nightly-i386-windows:
556   extends: .build-windows-make
557   only:
558     variables:
559       - $NIGHTLY
560   variables:
561     MSYSTEM: MINGW32
562   cache:
563     key: i386-windows
564
565 ############################################################
566 # Cleanup
567 ############################################################
568
569 # Note [Cleaning up after shell executor]
570 # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
571 #
572 # As noted in [1], gitlab-runner's shell executor doesn't clean up its working
573 # directory after builds. Unfortunately, we are forced to use the shell executor
574 # on Windows. To avoid running out of disk space we add a stage at the end of
575 # the build to remove the \GitLabRunner\builds directory. Since we only run a
576 # single build at a time on Windows this should be safe.
577 #
578 # [1] https://gitlab.com/gitlab-org/gitlab-runner/issues/3856
579
580 # See Note [Cleanup after shell executor]
581 cleanup-windows:
582   <<: *only-default
583   stage: cleanup
584   tags:
585     - x86_64-windows
586   when: always
587   dependencies: []
588   before_script:
589     - echo "Time to clean up"
590   script:
591     - echo "Let's go"
592   after_script:
593     - set "BUILD_DIR=%CI_PROJECT_DIR%"
594     - set "BUILD_DIR=%BUILD_DIR:/=\%"
595     - echo "Cleaning %BUILD_DIR%"
596     - cd \GitLabRunner
597     # This is way more complicated than it should be:
598     # See https://stackoverflow.com/questions/1965787
599     - del %BUILD_DIR%\* /F /Q
600     - for /d %%p in (%BUILD_DIR%\*) do rd /Q /S "%%p"
601     - exit /b 0
602
603 # See Note [Cleanup after shell executor]
604 cleanup-darwin:
605   <<: *only-default
606   stage: cleanup
607   tags:
608     - x86_64-darwin
609   when: always
610   dependencies: []
611   before_script:
612     - echo "Time to clean up"
613   script:
614     - echo "Let's go"
615   after_script:
616     - BUILD_DIR=$CI_PROJECT_DIR
617     - echo "Cleaning $BUILD_DIR"
618     - cd $HOME
619     - rm -Rf $BUILD_DIR/*
620     - exit 0
621
622 ############################################################
623 # Packaging
624 ############################################################
625
626 doc-tarball:
627   <<: *only-default
628   stage: packaging
629   tags:
630     - x86_64-linux
631   image: ghcci/x86_64-linux-deb9:0.2
632   dependencies:
633     - validate-x86_64-linux-deb9
634     - validate-x86_64-windows
635   artifacts:
636     paths:
637       - haddock.html.tar.xz
638       - libraries.html.tar.xz
639       - users_guide.html.tar.xz
640       - index.html
641       - "*.pdf"
642   script:
643     - rm -Rf docs
644     - distrib/mkDocs/mkDocs ghc-x86_64-deb9-linux.tar.xz ghc-x86_64-mingw32.tar.xz
645     - ls -lh
646     - mv docs/*.tar.xz .
647
648 source-tarball:
649   stage: packaging
650   tags:
651     - x86_64-linux
652   image: ghcci/x86_64-linux-deb9:0.2
653   only:
654     - tags
655   artifacts:
656     paths:
657       - ghc-*.tar.xz
658       - version
659   script:
660     - mk/get-win32-tarballs.sh download all
661     - ./boot
662     - ./configure
663     - make sdist
664     - mv sdistprep/*.xz  .
665     - make show VALUE=version > version
666
667
668 ############################################################
669 # Testing via head.hackage
670 ############################################################
671
672 # Triggering jobs in the ghc/head.hackage project requires that we have a job
673 # token for that repository. Furthermore the head.hackage CI job must have
674 # access to an unprivileged access token with the ability to query the ghc/ghc
675 # project such that it can find the job ID of the fedora27 job for the current
676 # pipeline.
677
678 .hackage:
679   <<: *only-default
680   stage: hackage
681   image: ghcci/x86_64-linux-deb9:0.2
682   tags:
683     - x86_64-linux
684   dependencies: []
685   variables:
686     HEAD_HACKAGE_PROJECT_ID: "78"
687   script:
688     - bash .gitlab/start-head.hackage.sh
689
690 hackage:
691   extends: .hackage
692   when: manual
693
694 hackage-label:
695   extends: .hackage
696   only:
697     variables:
698       - $CI_MERGE_REQUEST_LABELS =~ /.*user-facing.*/
699
700 nightly-hackage:
701   extends: .hackage
702   only:
703     variables:
704       - $NIGHTLY