gitlab-ci: Bump docker images
[ghc.git] / .gitlab-ci.yml
1 variables:
2   GIT_SSL_NO_VERIFY: "1"
3
4   # Commit of ghc/ci-images repository from which to pull Docker images
5   DOCKER_REV: 7cbd823abe7eddd9c52a2e93062035b1c863605f
6
7 before_script:
8   - python3 .gitlab/fix-submodules.py
9   - git submodule sync --recursive
10   - git submodule update --init --recursive
11   - git checkout .gitmodules
12   - "git fetch https://gitlab.haskell.org/ghc/ghc-performance-notes.git refs/notes/perf:refs/notes/perf || true"
13
14 stages:
15   - lint
16   - build
17   - full-build
18   - cleanup # See Note [Cleanup on Windows]
19   - packaging
20   - hackage
21
22 ############################################################
23 # Runner Tags
24 ############################################################
25 #
26 # * x86_64-linux: Any Docker-capable x86_64 Linux machine
27 # * aarch64-linux: Any Docker-capable AArch64 Linux machine
28 # * x86_64-windows: A x86_64 Windows machine
29 # * lint: Any Docker-capable x86_64 Linux machine; distinct from
30 #         x86_64-linux to ensure low-latency availability.
31 #
32
33
34 ############################################################
35 # Linting
36 ############################################################
37
38 ghc-linters:
39   stage: lint
40   image: "registry.gitlab.haskell.org/ghc/ci-images/linters:$DOCKER_REV"
41   script:
42     - |
43       if [ -n "$CI_MERGE_REQUEST_ID" ]; then
44         base="$(git merge-base $CI_MERGE_REQUEST_BRANCH_NAME HEAD)"
45         validate-commit-msg .git $(git rev-list $base..$CI_COMMIT_SHA)
46         submodchecker .git $(git rev-list $base..$CI_COMMIT_SHA)
47         validate-whitespace .git $(git rev-list $base..$CI_COMMIT_SHA)
48         .gitlab/linters/check-makefiles.py $base $CI_COMMIT_SHA
49         .gitlab/linters/check-cpp.py $base $CI_COMMIT_SHA
50       fi
51   dependencies: []
52   tags:
53     - lint
54
55 ############################################################
56 # Validation via Pipelines (hadrian)
57 ############################################################
58
59 .validate-hadrian:
60   allow_failure: true
61   script:
62     - cabal update
63     - git clean -xdf && git submodule foreach git clean -xdf
64     - bash .circleci/prepare-system.sh
65     - if [[ -d ./cabal-cache ]]; then cp -R ./.cabal-cache ~/.cabal-cache; fi
66     - ./boot
67     - ./configure $CONFIGURE_ARGS
68     - hadrian/build.cabal.sh -j`mk/detect-cpu-count.sh` --docs=no-sphinx binary-dist
69     - mv _build/bindist/ghc*.tar.xz ghc.tar.xz
70   cache:
71     key: hadrian
72     paths:
73       - cabal-cache
74   artifacts:
75     when: always
76     paths:
77       - ghc.tar.xz
78
79 validate-x86_64-linux-deb8-hadrian:
80   extends: .validate-hadrian
81   stage: build
82   image: "registry.gitlab.haskell.org/ghc/ci-images/x86_64-linux-deb8:$DOCKER_REV"
83   before_script:
84     # workaround for docker permissions
85     - sudo chown ghc:ghc -R .
86     - python3 .gitlab/fix-submodules.py
87     - git submodule sync --recursive
88     - git submodule update --init --recursive
89     - git checkout .gitmodules
90     - "git fetch https://gitlab.haskell.org/ghc/ghc-performance-notes.git refs/notes/perf:refs/notes/perf || true"
91   tags:
92     - x86_64-linux
93
94 hadrian-ghc-in-ghci:
95   stage: build
96   image: ghcci/x86_64-linux-deb8:0.1
97   before_script:
98     # workaround for docker permissions
99     - sudo chown ghc:ghc -R .
100     - python3 .gitlab/fix-submodules.py
101     - git submodule sync --recursive
102     - git submodule update --init --recursive
103     - git checkout .gitmodules
104   tags:
105     - x86_64-linux
106   script:
107     - cabal update
108     - git clean -xdf && git submodule foreach git clean -xdf
109     - bash .circleci/prepare-system.sh
110     - if [[ -d ./cabal-cache ]]; then cp -R ./.cabal-cache ~/.cabal-cache; fi
111     - ./boot
112     - ./configure $CONFIGURE_ARGS
113     # Load ghc-in-ghci then immediately exit
114     - echo ":q" | hadrian/ghci.sh
115   cache:
116     key: hadrian-ghci
117     paths:
118       - cabal-cache
119
120 ############################################################
121 # Validation via Pipelines (make)
122 ############################################################
123
124 .validate:
125   variables:
126     TEST_TYPE: test
127   before_script:
128     - git clean -xdf && git submodule foreach git clean -xdf
129   script:
130     - ./boot
131     - ./configure $CONFIGURE_ARGS
132     - |
133       THREADS=`mk/detect-cpu-count.sh`
134       make V=0 -j$THREADS WERROR=-Werror
135     - |
136       make binary-dist TAR_COMP_OPTS="-1"
137     - |
138       # Prepare to push git notes.
139       METRICS_FILE=$(mktemp)
140       git config user.email "ben+ghc-ci@smart-cactus.org"
141       git config user.name "GHC GitLab CI"
142     - |
143       THREADS=`mk/detect-cpu-count.sh`
144       make $TEST_TYPE THREADS=$THREADS JUNIT_FILE=../../junit.xml METRICS_FILE=$METRICS_FILE
145     - |
146       # Push git notes.
147       METRICS_FILE=$METRICS_FILE .gitlab/push-test-metrics.sh
148   dependencies: []
149   artifacts:
150     reports:
151       junit: junit.xml
152     expire_in: 2 week
153     paths:
154       - ghc-*.tar.xz
155       - junit.xml
156
157 #################################
158 # x86_64-darwin
159 #################################
160
161 validate-x86_64-darwin:
162   extends: .validate
163   stage: full-build
164   tags:
165     - x86_64-darwin
166   variables:
167     GHC_VERSION: 8.6.3
168     BIN_DIST_PREP_TAR_COMP: "bindistprep/ghc-x86_64-apple-darwin.tar.xz"
169     MACOSX_DEPLOYMENT_TARGET: "10.7"
170     # Only Sierra and onwards supports clock_gettime. See #12858
171     ac_cv_func_clock_gettime: "no"
172     LANG: "en_US.UTF-8"
173     CONFIGURE_ARGS: --with-intree-gmp
174     TEST_ENV: "x86_64-darwin"
175   before_script:
176     - git clean -xdf && git submodule foreach git clean -xdf
177     - python3 .gitlab/fix-submodules.py
178     - git submodule sync --recursive
179     - git submodule update --init --recursive
180     - git checkout .gitmodules
181     - "git fetch https://gitlab.haskell.org/ghc/ghc-performance-notes.git refs/notes/perf:refs/notes/perf || true"
182
183     - bash .gitlab/darwin-init.sh
184     - PATH="`pwd`/toolchain/bin:$PATH"
185   after_script:
186     - cp -Rf $HOME/.cabal cabal-cache
187   artifacts:
188     when: always
189     expire_in: 2 week
190   cache:
191     key: darwin
192     paths:
193       - cabal-cache
194       - toolchain
195
196 .validate-linux:
197   extends: .validate
198   tags:
199     - x86_64-linux
200   before_script:
201     - git clean -xdf && git submodule foreach git clean -xdf
202     - python3 .gitlab/fix-submodules.py
203     - git submodule sync --recursive
204     - git submodule update --init --recursive
205     - git checkout .gitmodules
206     - "git fetch https://gitlab.haskell.org/ghc/ghc-performance-notes.git refs/notes/perf:refs/notes/perf || true"
207
208     - bash .circleci/prepare-system.sh
209     # workaround for docker permissions
210     - sudo chown ghc:ghc -R .
211   after_script:
212     - cp -Rf $HOME/.cabal cabal-cache
213   cache:
214     key: linux
215     paths:
216       - cabal-cache
217       - toolchain
218
219 #################################
220 # aarch64-linux-deb9
221 #################################
222
223 .build-aarch64-linux-deb9:
224   extends: .validate-linux
225   stage: full-build
226   image: "registry.gitlab.haskell.org/ghc/ci-images/aarch64-linux-deb9:$DOCKER_REV"
227   allow_failure: true
228   variables:
229     TEST_ENV: "aarch64-linux-deb9"
230     BIN_DIST_PREP_TAR_COMP: "bindistprep/ghc-aarch64-linux-deb9.tar.xz"
231   cache:
232     key: linux-aarch64-deb9
233   tags:
234     - aarch64-linux
235
236 validate-aarch64-linux-deb9:
237   extends: .build-aarch64-linux-deb9
238   artifacts:
239     when: always
240     expire_in: 2 week
241
242 nightly-aarch64-linux-deb9:
243   extends: .build-aarch64-linux-deb9
244   artifacts:
245     expire_in: 2 year
246   variables:
247     TEST_TYPE: slowtest
248   only:
249     variables:
250       - $NIGHTLY
251
252 #################################
253 # i386-linux-deb9
254 #################################
255
256 .build-i386-linux-deb9:
257   extends: .validate-linux
258   stage: full-build
259   image: "registry.gitlab.haskell.org/ghc/ci-images/i386-linux-deb9:$DOCKER_REV"
260   variables:
261     TEST_ENV: "i386-linux-deb9"
262     BIN_DIST_PREP_TAR_COMP: "bindistprep/ghc-i386-deb9-linux.tar.xz"
263   cache:
264     key: linux-i386-deb9
265
266 validate-i386-linux-deb9:
267   extends: .build-i386-linux-deb9
268   artifacts:
269     when: always
270     expire_in: 2 week
271
272 nightly-i386-linux-deb9:
273   extends: .build-i386-linux-deb9
274   variables:
275     TEST_TYPE: slowtest
276   artifacts:
277     when: always
278     expire_in: 2 week
279   only:
280     variables:
281       - $NIGHTLY
282
283 #################################
284 # x86_64-linux-deb9
285 #################################
286
287 .build-x86_64-linux-deb9:
288   extends: .validate-linux
289   stage: build
290   image: "registry.gitlab.haskell.org/ghc/ci-images/x86_64-linux-deb9:$DOCKER_REV"
291   variables:
292     TEST_ENV: "x86_64-linux-deb9"
293     BIN_DIST_PREP_TAR_COMP: "bindistprep/ghc-x86_64-deb9-linux.tar.xz"
294   cache:
295     key: linux-x86_64-deb9
296
297 validate-x86_64-linux-deb9:
298   extends: .build-x86_64-linux-deb9
299   artifacts:
300     when: always
301     expire_in: 2 week
302
303 nightly-x86_64-linux-deb9:
304   extends: .build-x86_64-linux-deb9
305   artifacts:
306     expire_in: 2 year
307   variables:
308     TEST_TYPE: slowtest
309   only:
310     variables:
311       - $NIGHTLY
312
313 # N.B. Has DEBUG assertions enabled in stage2
314 validate-x86_64-linux-deb9-debug:
315   extends: .build-x86_64-linux-deb9
316   variables:
317     BUILD_FLAVOUR: validate
318     TEST_ENV: "x86_64-linux-deb9-debug"
319
320 validate-x86_64-linux-deb9-llvm:
321   extends: .build-x86_64-linux-deb9
322   stage: full-build
323   variables:
324     BUILD_FLAVOUR: perf-llvm
325     TEST_ENV: "x86_64-linux-deb9-llvm"
326
327 validate-x86_64-linux-deb9-integer-simple:
328   extends: .build-x86_64-linux-deb9
329   stage: full-build
330   variables:
331     INTEGER_LIBRARY: integer-simple
332     TEST_ENV: "x86_64-linux-deb9-integer-simple"
333     BIN_DIST_PREP_TAR_COMP: "bindistprep/ghc-x86_64-deb9-linux-integer-simple.tar.xz"
334
335 nightly-x86_64-linux-deb9-integer-simple:
336   extends: .build-x86_64-linux-deb9
337   stage: full-build
338   variables:
339     INTEGER_LIBRARY: integer-simple
340     TEST_ENV: "x86_64-linux-deb9-integer-simple"
341     TEST_TYPE: slowtest
342   artifacts:
343     expire_in: 2 year
344   only:
345     variables:
346       - $NIGHTLY
347
348 validate-x86_64-linux-deb9-unreg:
349   extends: .build-x86_64-linux-deb9
350   stage: full-build
351   variables:
352     CONFIGURE_ARGS: --enable-unregisterised
353     TEST_ENV: "x86_64-linux-deb9-unreg"
354
355
356 #################################
357 # x86_64-linux-deb8
358 #################################
359
360 release-x86_64-linux-deb8:
361   extends: .validate-linux
362   stage: full-build
363   image: "registry.gitlab.haskell.org/ghc/ci-images/x86_64-linux-deb8:$DOCKER_REV"
364   variables:
365     TEST_ENV: "x86_64-linux-deb8"
366     BIN_DIST_PREP_TAR_COMP: "bindistprep/ghc-x86_64-deb8-linux.tar.xz"
367   only:
368     - tags
369   cache:
370     key: linux-x86_64-deb8
371   artifacts:
372     when: always
373     expire_in: 2 week
374
375
376 #################################
377 # x86_64-linux-fedora27
378 #################################
379
380 validate-x86_64-linux-fedora27:
381   extends: .validate-linux
382   stage: full-build
383   image: "registry.gitlab.haskell.org/ghc/ci-images/x86_64-linux-fedora27:$DOCKER_REV"
384   variables:
385     TEST_ENV: "x86_64-linux-fedora27"
386     BIN_DIST_PREP_TAR_COMP: "bindistprep/ghc-x86_64-fedora27-linux.tar.xz"
387   cache:
388     key: linux-x86_64-fedora27
389   artifacts:
390     when: always
391     expire_in: 2 week
392
393 ############################################################
394 # Validation via Pipelines (Windows)
395 ############################################################
396
397 .build-windows:
398   before_script:
399     - git clean -xdf
400     - git submodule foreach git clean -xdf
401
402     # Use a local temporary directory to ensure that concurrent builds don't
403     # interfere with one another
404     - |
405       mkdir tmp
406       set TMP=%cd%\tmp
407       set TEMP=%cd%\tmp
408
409     - set PATH=C:\msys64\usr\bin;%PATH%
410     - python .gitlab/fix-submodules.py
411     - git submodule sync --recursive
412     - git submodule update --init --recursive
413     - git checkout .gitmodules
414     - "git fetch https://gitlab.haskell.org/ghc/ghc-performance-notes.git refs/notes/perf:refs/notes/perf || true"
415     - bash .gitlab/win32-init.sh
416   after_script:
417     - rd /s /q tmp
418     - robocopy /np /nfl /ndl /e "%APPDATA%\cabal" cabal-cache
419     - bash -c 'make clean || true'
420   dependencies: []
421   variables:
422     FORCE_SYMLINKS: 1
423     LANG: "en_US.UTF-8"
424   cache:
425     paths:
426       - cabal-cache
427       - ghc-8.6.2
428       - ghc-tarballs
429
430 .build-windows-hadrian:
431   extends: .build-windows
432   stage: full-build
433   variables:
434     GHC_VERSION: "8.6.2"
435   script:
436     - |
437       python boot
438       bash -c './configure --enable-tarballs-autodownload GHC=`pwd`/toolchain/bin/ghc HAPPY=`pwd`/toolchain/bin/happy ALEX=`pwd`/toolchain/bin/alex'
439     - bash -c "PATH=`pwd`/toolchain/bin:$PATH hadrian/build.cabal.sh -j`mk/detect-cpu-count.sh` --flavour=Quick --docs=no-sphinx binary-dist"
440     - mv _build/bindist/ghc*.tar.xz ghc.tar.xz
441     # FIXME: Testsuite disabled due to #16156.
442     # - bash -c 'make V=0 test THREADS=`mk/detect-cpu-count.sh` JUNIT_FILE=../../junit.xml'
443   tags:
444     - x86_64-windows
445   artifacts:
446     when: always
447     paths:
448       - ghc.tar.xz
449
450 validate-x86_64-windows-hadrian:
451   extends: .build-windows-hadrian
452   variables:
453     MSYSTEM: MINGW64
454   cache:
455     key: x86_64-windows-hadrian
456
457 nightly-i386-windows-hadrian:
458   extends: .build-windows-hadrian
459   variables:
460     MSYSTEM: MINGW32
461   only:
462     variables:
463       - $NIGHTLY
464   cache:
465     key: i386-windows-hadrian
466
467 .build-windows-make:
468   extends: .build-windows
469   stage: full-build
470   # due to #16084
471   allow_failure: true
472   variables:
473     GHC_VERSION: "8.6.2"
474     BIN_DIST_PREP_TAR_COMP: "bindistprep/ghc-x86_64-mingw32.tar.xz"
475   script:
476     - |
477       python boot
478       bash -c './configure --enable-tarballs-autodownload GHC=`pwd`/toolchain/bin/ghc HAPPY=`pwd`/toolchain/bin/happy ALEX=`pwd`/toolchain/bin/alex'
479     - bash -c "echo include mk/flavours/quick.mk > mk/build.mk"
480     - bash -c "PATH=`pwd`/toolchain/bin:$PATH make -j`mk/detect-cpu-count.sh`"
481     - bash -c "PATH=`pwd`/toolchain/bin:$PATH make binary-dist TAR_COMP_OPTS=-1"
482     - bash -c 'make V=0 test THREADS=`mk/detect-cpu-count.sh` JUNIT_FILE=../../junit.xml'
483   tags:
484     - x86_64-windows
485   artifacts:
486     when: always
487     expire_in: 2 week
488     reports:
489       junit: junit.xml
490     paths:
491       - ghc-*.tar.xz
492       - junit.xml
493
494 validate-x86_64-windows:
495   extends: .build-windows-make
496   variables:
497     MSYSTEM: MINGW64
498   cache:
499     key: x86_64-windows
500
501 release-i386-windows:
502   extends: .build-windows-make
503   only:
504     - tags
505   variables:
506     MSYSTEM: MINGW32
507   cache:
508     key: i386-windows
509
510 nightly-i386-windows:
511   extends: .build-windows-make
512   only:
513     variables:
514       - $NIGHTLY
515   variables:
516     MSYSTEM: MINGW32
517   cache:
518     key: i386-windows
519
520 ############################################################
521 # Cleanup
522 ############################################################
523
524 # Note [Cleaning up after shell executor]
525 # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
526 #
527 # As noted in [1], gitlab-runner's shell executor doesn't clean up its working
528 # directory after builds. Unfortunately, we are forced to use the shell executor
529 # on Windows. To avoid running out of disk space we add a stage at the end of
530 # the build to remove the \GitLabRunner\builds directory. Since we only run a
531 # single build at a time on Windows this should be safe.
532 #
533 # [1] https://gitlab.com/gitlab-org/gitlab-runner/issues/3856
534
535 # See Note [Cleanup after shell executor]
536 cleanup-windows:
537   stage: cleanup
538   tags:
539     - x86_64-windows
540   when: always
541   dependencies: []
542   before_script:
543     - echo "Time to clean up"
544   script:
545     - echo "Let's go"
546   after_script:
547     - set "BUILD_DIR=%CI_PROJECT_DIR%"
548     - set "BUILD_DIR=%BUILD_DIR:/=\%"
549     - echo "Cleaning %BUILD_DIR%"
550     - cd \GitLabRunner
551     # This is way more complicated than it should be:
552     # See https://stackoverflow.com/questions/1965787
553     - del %BUILD_DIR%\* /F /Q
554     - for /d %%p in (%BUILD_DIR%\*) do rd /Q /S "%%p"
555     - exit /b 0
556
557 # See Note [Cleanup after shell executor]
558 cleanup-darwin:
559   stage: cleanup
560   tags:
561     - x86_64-darwin
562   when: always
563   dependencies: []
564   before_script:
565     - echo "Time to clean up"
566   script:
567     - echo "Let's go"
568   after_script:
569     - BUILD_DIR=$CI_PROJECT_DIR
570     - echo "Cleaning $BUILD_DIR"
571     - cd $HOME
572     - rm -Rf $BUILD_DIR/*
573     - exit 0
574
575 ############################################################
576 # Packaging
577 ############################################################
578
579 doc-tarball:
580   stage: packaging
581   tags:
582     - x86_64-linux
583   image: ghcci/x86_64-linux-deb9:0.2
584   dependencies:
585     - validate-x86_64-linux-deb9
586     - validate-x86_64-windows
587   artifacts:
588     paths:
589       - haddock.html.tar.xz
590       - libraries.html.tar.xz
591       - users_guide.html.tar.xz
592       - index.html
593       - "*.pdf"
594   script:
595     - rm -Rf docs
596     - distrib/mkDocs/mkDocs ghc-x86_64-deb9-linux.tar.xz ghc-x86_64-mingw32.tar.xz
597     - ls -lh
598     - mv docs/*.tar.xz .
599
600 source-tarball:
601   stage: packaging
602   tags:
603     - x86_64-linux
604   image: ghcci/x86_64-linux-deb9:0.2
605   only:
606     - tags
607   artifacts:
608     paths:
609       - ghc-*.tar.xz
610       - version
611   script:
612     - mk/get-win32-tarballs.sh download all
613     - ./boot
614     - ./configure
615     - make sdist
616     - mv sdistprep/*.xz  .
617     - make show VALUE=version > version
618
619
620 ############################################################
621 # Testing via head.hackage
622 ############################################################
623
624 # Triggering jobs in the ghc/head.hackage project requires that we have a job
625 # token for that repository. Furthermore the head.hackage CI job must have
626 # access to an unprivileged access token with the ability to query the ghc/ghc
627 # project such that it can find the job ID of the fedora27 job for the current
628 # pipeline.
629
630 .hackage:
631   stage: hackage
632   image: ghcci/x86_64-linux-deb9:0.2
633   tags:
634     - x86_64-linux
635   dependencies: []
636   variables:
637     HEAD_HACKAGE_PROJECT_ID: "78"
638   script:
639     - bash .gitlab/start-head.hackage.sh
640
641 hackage:
642   extends: .hackage
643   when: manual
644
645 hackage-label:
646   extends: .hackage
647   only:
648     variables:
649       - $CI_MERGE_REQUEST_LABELS =~ /.*user-facing.*/
650
651 nightly-hackage:
652   extends: .hackage
653   only:
654     variables:
655       - $NIGHTLY