ci: Add some descriptions of the stages
[ghc.git] / .gitlab-ci.yml
1 variables:
2   GIT_SSL_NO_VERIFY: "1"
3
4   # Commit of ghc/ci-images repository from which to pull Docker images
5   DOCKER_REV: 07bd4acf4f91b56e22389d7b7f6c13486a7b63de
6
7 before_script:
8   - python3 .gitlab/fix-submodules.py
9   - git submodule sync --recursive
10   - git submodule update --init --recursive
11   - git checkout .gitmodules
12   - "git fetch https://gitlab.haskell.org/ghc/ghc-performance-notes.git refs/notes/perf:refs/notes/perf || true"
13
14 stages:
15   - lint       # Source linting
16   - build      # A quick smoke-test to weed out broken commits
17   - full-build # Build all the things
18   - cleanup    # See Note [Cleanup on Windows]
19   - packaging  # Source distribution, etc.
20   - hackage    # head.hackage testing
21
22 ############################################################
23 # Runner Tags
24 ############################################################
25 #
26 # * x86_64-linux: Any Docker-capable x86_64 Linux machine
27 # * aarch64-linux: Any Docker-capable AArch64 Linux machine
28 # * x86_64-windows: A x86_64 Windows machine
29 # * lint: Any Docker-capable x86_64 Linux machine; distinct from
30 #         x86_64-linux to ensure low-latency availability.
31 #
32
33
34 ############################################################
35 # Linting
36 ############################################################
37
38 ghc-linters:
39   stage: lint
40   image: "registry.gitlab.haskell.org/ghc/ci-images/linters:$DOCKER_REV"
41   script:
42     - |
43       if [ -n "$CI_MERGE_REQUEST_ID" ]; then
44         base="$(git merge-base $CI_MERGE_REQUEST_BRANCH_NAME HEAD)"
45         validate-commit-msg .git $(git rev-list $base..$CI_COMMIT_SHA)
46         submodchecker .git $(git rev-list $base..$CI_COMMIT_SHA)
47         validate-whitespace .git $(git rev-list $base..$CI_COMMIT_SHA)
48         .gitlab/linters/check-makefiles.py $base $CI_COMMIT_SHA
49         .gitlab/linters/check-cpp.py $base $CI_COMMIT_SHA
50       fi
51   dependencies: []
52   tags:
53     - lint
54
55 ############################################################
56 # Validation via Pipelines (hadrian)
57 ############################################################
58
59 .validate-hadrian:
60   allow_failure: true
61   script:
62     - cabal update
63     - git clean -xdf && git submodule foreach git clean -xdf
64     - bash .circleci/prepare-system.sh
65     - if [[ -d ./cabal-cache ]]; then cp -R ./.cabal-cache ~/.cabal-cache; fi
66     - ./boot
67     - ./configure $CONFIGURE_ARGS
68     - hadrian/build.cabal.sh -j`mk/detect-cpu-count.sh` --docs=no-sphinx binary-dist
69     - mv _build/bindist/ghc*.tar.xz ghc.tar.xz
70   cache:
71     key: hadrian
72     paths:
73       - cabal-cache
74   artifacts:
75     when: always
76     paths:
77       - ghc.tar.xz
78
79 validate-x86_64-linux-deb8-hadrian:
80   extends: .validate-hadrian
81   stage: build
82   image: "registry.gitlab.haskell.org/ghc/ci-images/x86_64-linux-deb8:$DOCKER_REV"
83   before_script:
84     # workaround for docker permissions
85     - sudo chown ghc:ghc -R .
86     - python3 .gitlab/fix-submodules.py
87     - git submodule sync --recursive
88     - git submodule update --init --recursive
89     - git checkout .gitmodules
90     - "git fetch https://gitlab.haskell.org/ghc/ghc-performance-notes.git refs/notes/perf:refs/notes/perf || true"
91   tags:
92     - x86_64-linux
93
94 hadrian-ghc-in-ghci:
95   stage: build
96   image: ghcci/x86_64-linux-deb8:0.1
97   before_script:
98     # workaround for docker permissions
99     - sudo chown ghc:ghc -R .
100     - python3 .gitlab/fix-submodules.py
101     - git submodule sync --recursive
102     - git submodule update --init --recursive
103     - git checkout .gitmodules
104   tags:
105     - x86_64-linux
106   script:
107     - cabal update
108     - git clean -xdf && git submodule foreach git clean -xdf
109     - bash .circleci/prepare-system.sh
110     - if [[ -d ./cabal-cache ]]; then cp -R ./.cabal-cache ~/.cabal-cache; fi
111     - ./boot
112     - ./configure $CONFIGURE_ARGS
113     # Load ghc-in-ghci then immediately exit
114     - echo ":q" | hadrian/ghci.sh
115   cache:
116     key: hadrian-ghci
117     paths:
118       - cabal-cache
119
120 ############################################################
121 # Validation via Pipelines (make)
122 ############################################################
123
124 .validate:
125   variables:
126     TEST_TYPE: test
127   before_script:
128     - git clean -xdf && git submodule foreach git clean -xdf
129   script:
130     - ./boot
131     - ./configure $CONFIGURE_ARGS
132     - |
133       THREADS=`mk/detect-cpu-count.sh`
134       make V=0 -j$THREADS WERROR=-Werror
135     - |
136       make binary-dist TAR_COMP_OPTS="-1"
137     - |
138       # Prepare to push git notes.
139       METRICS_FILE=$(mktemp)
140       git config user.email "ben+ghc-ci@smart-cactus.org"
141       git config user.name "GHC GitLab CI"
142     - |
143       THREADS=`mk/detect-cpu-count.sh`
144       make $TEST_TYPE THREADS=$THREADS JUNIT_FILE=../../junit.xml METRICS_FILE=$METRICS_FILE
145     - |
146       # Push git notes.
147       METRICS_FILE=$METRICS_FILE .gitlab/push-test-metrics.sh
148   dependencies: []
149   artifacts:
150     reports:
151       junit: junit.xml
152     expire_in: 2 week
153     paths:
154       - ghc-*.tar.xz
155       - junit.xml
156
157 #################################
158 # x86_64-darwin
159 #################################
160
161 validate-x86_64-darwin:
162   extends: .validate
163   stage: full-build
164   tags:
165     - x86_64-darwin
166   variables:
167     GHC_VERSION: 8.6.3
168     BIN_DIST_PREP_TAR_COMP: "bindistprep/ghc-x86_64-apple-darwin.tar.xz"
169     MACOSX_DEPLOYMENT_TARGET: "10.7"
170     # Only Sierra and onwards supports clock_gettime. See #12858
171     ac_cv_func_clock_gettime: "no"
172     LANG: "en_US.UTF-8"
173     CONFIGURE_ARGS: --with-intree-gmp
174     TEST_ENV: "x86_64-darwin"
175   before_script:
176     - git clean -xdf && git submodule foreach git clean -xdf
177     - python3 .gitlab/fix-submodules.py
178     - git submodule sync --recursive
179     - git submodule update --init --recursive
180     - git checkout .gitmodules
181     - "git fetch https://gitlab.haskell.org/ghc/ghc-performance-notes.git refs/notes/perf:refs/notes/perf || true"
182
183     - bash .gitlab/darwin-init.sh
184     - PATH="`pwd`/toolchain/bin:$PATH"
185   after_script:
186     - cp -Rf $HOME/.cabal cabal-cache
187   artifacts:
188     when: always
189     expire_in: 2 week
190   cache:
191     key: darwin
192     paths:
193       - cabal-cache
194       - toolchain
195
196 .validate-linux:
197   extends: .validate
198   tags:
199     - x86_64-linux
200   before_script:
201     - git clean -xdf && git submodule foreach git clean -xdf
202     - python3 .gitlab/fix-submodules.py
203     - git submodule sync --recursive
204     - git submodule update --init --recursive
205     - git checkout .gitmodules
206     - "git fetch https://gitlab.haskell.org/ghc/ghc-performance-notes.git refs/notes/perf:refs/notes/perf || true"
207
208     - bash .circleci/prepare-system.sh
209     # workaround for docker permissions
210     - sudo chown ghc:ghc -R .
211   after_script:
212     - cp -Rf $HOME/.cabal cabal-cache
213   cache:
214     key: linux
215     paths:
216       - cabal-cache
217       - toolchain
218
219 #################################
220 # aarch64-linux-deb9
221 #################################
222
223 .build-aarch64-linux-deb9:
224   extends: .validate-linux
225   stage: full-build
226   image: "registry.gitlab.haskell.org/ghc/ci-images/aarch64-linux-deb9:$DOCKER_REV"
227   allow_failure: true
228   variables:
229     TEST_ENV: "aarch64-linux-deb9"
230     BIN_DIST_PREP_TAR_COMP: "bindistprep/ghc-aarch64-linux-deb9.tar.xz"
231   cache:
232     key: linux-aarch64-deb9
233   tags:
234     - aarch64-linux
235
236 validate-aarch64-linux-deb9:
237   extends: .build-aarch64-linux-deb9
238   artifacts:
239     when: always
240     expire_in: 2 week
241
242 nightly-aarch64-linux-deb9:
243   extends: .build-aarch64-linux-deb9
244   artifacts:
245     expire_in: 2 year
246   variables:
247     TEST_TYPE: slowtest
248   only:
249     variables:
250       - $NIGHTLY
251
252 #################################
253 # i386-linux-deb9
254 #################################
255
256 .build-i386-linux-deb9:
257   extends: .validate-linux
258   stage: full-build
259   image: "registry.gitlab.haskell.org/ghc/ci-images/i386-linux-deb9:$DOCKER_REV"
260   variables:
261     TEST_ENV: "i386-linux-deb9"
262     BIN_DIST_PREP_TAR_COMP: "bindistprep/ghc-i386-deb9-linux.tar.xz"
263   cache:
264     key: linux-i386-deb9
265
266 validate-i386-linux-deb9:
267   extends: .build-i386-linux-deb9
268   artifacts:
269     when: always
270     expire_in: 2 week
271
272 nightly-i386-linux-deb9:
273   extends: .build-i386-linux-deb9
274   variables:
275     TEST_TYPE: slowtest
276   artifacts:
277     when: always
278     expire_in: 2 week
279   only:
280     variables:
281       - $NIGHTLY
282
283 #################################
284 # x86_64-linux-deb9
285 #################################
286
287 .build-x86_64-linux-deb9:
288   extends: .validate-linux
289   stage: full-build
290   image: "registry.gitlab.haskell.org/ghc/ci-images/x86_64-linux-deb9:$DOCKER_REV"
291   variables:
292     TEST_ENV: "x86_64-linux-deb9"
293     BIN_DIST_PREP_TAR_COMP: "bindistprep/ghc-x86_64-deb9-linux.tar.xz"
294   cache:
295     key: linux-x86_64-deb9
296
297 validate-x86_64-linux-deb9:
298   extends: .build-x86_64-linux-deb9
299   artifacts:
300     when: always
301     expire_in: 2 week
302
303 nightly-x86_64-linux-deb9:
304   extends: .build-x86_64-linux-deb9
305   artifacts:
306     expire_in: 2 year
307   variables:
308     TEST_TYPE: slowtest
309   only:
310     variables:
311       - $NIGHTLY
312
313 # N.B. Has DEBUG assertions enabled in stage2
314 validate-x86_64-linux-deb9-debug:
315   extends: .build-x86_64-linux-deb9
316   stage: build
317   variables:
318     BUILD_FLAVOUR: validate
319     TEST_ENV: "x86_64-linux-deb9-debug"
320
321 validate-x86_64-linux-deb9-llvm:
322   extends: .build-x86_64-linux-deb9
323   stage: full-build
324   variables:
325     BUILD_FLAVOUR: perf-llvm
326     TEST_ENV: "x86_64-linux-deb9-llvm"
327
328 validate-x86_64-linux-deb9-integer-simple:
329   extends: .build-x86_64-linux-deb9
330   stage: full-build
331   variables:
332     INTEGER_LIBRARY: integer-simple
333     TEST_ENV: "x86_64-linux-deb9-integer-simple"
334     BIN_DIST_PREP_TAR_COMP: "bindistprep/ghc-x86_64-deb9-linux-integer-simple.tar.xz"
335
336 nightly-x86_64-linux-deb9-integer-simple:
337   extends: .build-x86_64-linux-deb9
338   stage: full-build
339   variables:
340     INTEGER_LIBRARY: integer-simple
341     TEST_ENV: "x86_64-linux-deb9-integer-simple"
342     TEST_TYPE: slowtest
343   artifacts:
344     expire_in: 2 year
345   only:
346     variables:
347       - $NIGHTLY
348
349 validate-x86_64-linux-deb9-unreg:
350   extends: .build-x86_64-linux-deb9
351   stage: full-build
352   variables:
353     CONFIGURE_ARGS: --enable-unregisterised
354     TEST_ENV: "x86_64-linux-deb9-unreg"
355
356
357 #################################
358 # x86_64-linux-deb8
359 #################################
360
361 release-x86_64-linux-deb8:
362   extends: .validate-linux
363   stage: full-build
364   image: "registry.gitlab.haskell.org/ghc/ci-images/x86_64-linux-deb8:$DOCKER_REV"
365   variables:
366     TEST_ENV: "x86_64-linux-deb8"
367     BIN_DIST_PREP_TAR_COMP: "bindistprep/ghc-x86_64-deb8-linux.tar.xz"
368   only:
369     - tags
370   cache:
371     key: linux-x86_64-deb8
372   artifacts:
373     when: always
374     expire_in: 2 week
375
376
377 #################################
378 # x86_64-linux-fedora27
379 #################################
380
381 validate-x86_64-linux-fedora27:
382   extends: .validate-linux
383   stage: full-build
384   image: "registry.gitlab.haskell.org/ghc/ci-images/x86_64-linux-fedora27:$DOCKER_REV"
385   variables:
386     TEST_ENV: "x86_64-linux-fedora27"
387     BIN_DIST_PREP_TAR_COMP: "bindistprep/ghc-x86_64-fedora27-linux.tar.xz"
388   cache:
389     key: linux-x86_64-fedora27
390   artifacts:
391     when: always
392     expire_in: 2 week
393
394 ############################################################
395 # Validation via Pipelines (Windows)
396 ############################################################
397
398 .build-windows:
399   before_script:
400     - git clean -xdf
401     - git submodule foreach git clean -xdf
402
403     # Use a local temporary directory to ensure that concurrent builds don't
404     # interfere with one another
405     - |
406       mkdir tmp
407       set TMP=%cd%\tmp
408       set TEMP=%cd%\tmp
409
410     - set PATH=C:\msys64\usr\bin;%PATH%
411     - python .gitlab/fix-submodules.py
412     - git submodule sync --recursive
413     - git submodule update --init --recursive
414     - git checkout .gitmodules
415     - "git fetch https://gitlab.haskell.org/ghc/ghc-performance-notes.git refs/notes/perf:refs/notes/perf || true"
416     - bash .gitlab/win32-init.sh
417   after_script:
418     - rd /s /q tmp
419     - robocopy /np /nfl /ndl /e "%APPDATA%\cabal" cabal-cache
420     - bash -c 'make clean || true'
421   dependencies: []
422   variables:
423     FORCE_SYMLINKS: 1
424     LANG: "en_US.UTF-8"
425   cache:
426     paths:
427       - cabal-cache
428       - ghc-8.6.2
429       - ghc-tarballs
430
431 .build-windows-hadrian:
432   extends: .build-windows
433   stage: full-build
434   variables:
435     GHC_VERSION: "8.6.2"
436   script:
437     - |
438       python boot
439       bash -c './configure --enable-tarballs-autodownload GHC=`pwd`/toolchain/bin/ghc HAPPY=`pwd`/toolchain/bin/happy ALEX=`pwd`/toolchain/bin/alex'
440     - bash -c "PATH=`pwd`/toolchain/bin:$PATH hadrian/build.cabal.sh -j`mk/detect-cpu-count.sh` --flavour=Quick --docs=no-sphinx binary-dist"
441     - mv _build/bindist/ghc*.tar.xz ghc.tar.xz
442     # FIXME: Testsuite disabled due to #16156.
443     # - bash -c 'make V=0 test THREADS=`mk/detect-cpu-count.sh` JUNIT_FILE=../../junit.xml'
444   tags:
445     - x86_64-windows
446   artifacts:
447     when: always
448     paths:
449       - ghc.tar.xz
450
451 validate-x86_64-windows-hadrian:
452   extends: .build-windows-hadrian
453   variables:
454     MSYSTEM: MINGW64
455   cache:
456     key: x86_64-windows-hadrian
457
458 nightly-i386-windows-hadrian:
459   extends: .build-windows-hadrian
460   variables:
461     MSYSTEM: MINGW32
462   only:
463     variables:
464       - $NIGHTLY
465   cache:
466     key: i386-windows-hadrian
467
468 .build-windows-make:
469   extends: .build-windows
470   stage: full-build
471   # due to #16084
472   allow_failure: true
473   variables:
474     GHC_VERSION: "8.6.2"
475     BIN_DIST_PREP_TAR_COMP: "bindistprep/ghc-x86_64-mingw32.tar.xz"
476   script:
477     - |
478       python boot
479       bash -c './configure --enable-tarballs-autodownload GHC=`pwd`/toolchain/bin/ghc HAPPY=`pwd`/toolchain/bin/happy ALEX=`pwd`/toolchain/bin/alex'
480     - bash -c "echo include mk/flavours/quick.mk > mk/build.mk"
481     - bash -c "PATH=`pwd`/toolchain/bin:$PATH make -j`mk/detect-cpu-count.sh`"
482     - bash -c "PATH=`pwd`/toolchain/bin:$PATH make binary-dist TAR_COMP_OPTS=-1"
483     - bash -c 'make V=0 test THREADS=`mk/detect-cpu-count.sh` JUNIT_FILE=../../junit.xml'
484   tags:
485     - x86_64-windows
486   artifacts:
487     when: always
488     expire_in: 2 week
489     reports:
490       junit: junit.xml
491     paths:
492       - ghc-*.tar.xz
493       - junit.xml
494
495 validate-x86_64-windows:
496   extends: .build-windows-make
497   variables:
498     MSYSTEM: MINGW64
499   cache:
500     key: x86_64-windows
501
502 release-i386-windows:
503   extends: .build-windows-make
504   only:
505     - tags
506   variables:
507     MSYSTEM: MINGW32
508   cache:
509     key: i386-windows
510
511 nightly-i386-windows:
512   extends: .build-windows-make
513   only:
514     variables:
515       - $NIGHTLY
516   variables:
517     MSYSTEM: MINGW32
518   cache:
519     key: i386-windows
520
521 ############################################################
522 # Cleanup
523 ############################################################
524
525 # Note [Cleaning up after shell executor]
526 # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
527 #
528 # As noted in [1], gitlab-runner's shell executor doesn't clean up its working
529 # directory after builds. Unfortunately, we are forced to use the shell executor
530 # on Windows. To avoid running out of disk space we add a stage at the end of
531 # the build to remove the \GitLabRunner\builds directory. Since we only run a
532 # single build at a time on Windows this should be safe.
533 #
534 # [1] https://gitlab.com/gitlab-org/gitlab-runner/issues/3856
535
536 # See Note [Cleanup after shell executor]
537 cleanup-windows:
538   stage: cleanup
539   tags:
540     - x86_64-windows
541   when: always
542   dependencies: []
543   before_script:
544     - echo "Time to clean up"
545   script:
546     - echo "Let's go"
547   after_script:
548     - set "BUILD_DIR=%CI_PROJECT_DIR%"
549     - set "BUILD_DIR=%BUILD_DIR:/=\%"
550     - echo "Cleaning %BUILD_DIR%"
551     - cd \GitLabRunner
552     # This is way more complicated than it should be:
553     # See https://stackoverflow.com/questions/1965787
554     - del %BUILD_DIR%\* /F /Q
555     - for /d %%p in (%BUILD_DIR%\*) do rd /Q /S "%%p"
556     - exit /b 0
557
558 # See Note [Cleanup after shell executor]
559 cleanup-darwin:
560   stage: cleanup
561   tags:
562     - x86_64-darwin
563   when: always
564   dependencies: []
565   before_script:
566     - echo "Time to clean up"
567   script:
568     - echo "Let's go"
569   after_script:
570     - BUILD_DIR=$CI_PROJECT_DIR
571     - echo "Cleaning $BUILD_DIR"
572     - cd $HOME
573     - rm -Rf $BUILD_DIR/*
574     - exit 0
575
576 ############################################################
577 # Packaging
578 ############################################################
579
580 doc-tarball:
581   stage: packaging
582   tags:
583     - x86_64-linux
584   image: ghcci/x86_64-linux-deb9:0.2
585   dependencies:
586     - validate-x86_64-linux-deb9
587     - validate-x86_64-windows
588   artifacts:
589     paths:
590       - haddock.html.tar.xz
591       - libraries.html.tar.xz
592       - users_guide.html.tar.xz
593       - index.html
594       - "*.pdf"
595   script:
596     - rm -Rf docs
597     - distrib/mkDocs/mkDocs ghc-x86_64-deb9-linux.tar.xz ghc-x86_64-mingw32.tar.xz
598     - ls -lh
599     - mv docs/*.tar.xz .
600
601 source-tarball:
602   stage: packaging
603   tags:
604     - x86_64-linux
605   image: ghcci/x86_64-linux-deb9:0.2
606   only:
607     - tags
608   artifacts:
609     paths:
610       - ghc-*.tar.xz
611       - version
612   script:
613     - mk/get-win32-tarballs.sh download all
614     - ./boot
615     - ./configure
616     - make sdist
617     - mv sdistprep/*.xz  .
618     - make show VALUE=version > version
619
620
621 ############################################################
622 # Testing via head.hackage
623 ############################################################
624
625 # Triggering jobs in the ghc/head.hackage project requires that we have a job
626 # token for that repository. Furthermore the head.hackage CI job must have
627 # access to an unprivileged access token with the ability to query the ghc/ghc
628 # project such that it can find the job ID of the fedora27 job for the current
629 # pipeline.
630
631 .hackage:
632   stage: hackage
633   image: ghcci/x86_64-linux-deb9:0.2
634   tags:
635     - x86_64-linux
636   dependencies: []
637   variables:
638     HEAD_HACKAGE_PROJECT_ID: "78"
639   script:
640     - bash .gitlab/start-head.hackage.sh
641
642 hackage:
643   extends: .hackage
644   when: manual
645
646 hackage-label:
647   extends: .hackage
648   only:
649     variables:
650       - $CI_MERGE_REQUEST_LABELS =~ /.*user-facing.*/
651
652 nightly-hackage:
653   extends: .hackage
654   only:
655     variables:
656       - $NIGHTLY