cca7551b0e430c58f920011d4dce0ecf1877dcbe
[ghc.git] / .gitlab-ci.yml
1 variables:
2   GIT_SSL_NO_VERIFY: "1"
3
4   # Commit of ghc/ci-images repository from which to pull Docker images
5   DOCKER_REV: 6d19c3adc1f5c28c82aed8c5b1ac40931ac60f3f
6
7 before_script:
8   - python3 .gitlab/fix-submodules.py
9   - git submodule sync --recursive
10   - git submodule update --init --recursive
11   - git checkout .gitmodules
12   - "git fetch https://gitlab.haskell.org/ghc/ghc-performance-notes.git refs/notes/perf:refs/notes/perf || true"
13
14 stages:
15   - lint
16   - build
17   - full-build
18   - cleanup # See Note [Cleanup on Windows]
19   - packaging
20
21 ############################################################
22 # Runner Tags
23 ############################################################
24 #
25 # * x86_64-linux: Any Docker-capable x86_64 Linux machine
26 # * aarch64-linux: Any Docker-capable AArch64 Linux machine
27 # * x86_64-windows: A x86_64 Windows machine
28 # * lint: Any Docker-capable x86_64 Linux machine; distinct from
29 #         x86_64-linux to ensure low-latency availability.
30 #
31
32
33 ############################################################
34 # Linting
35 ############################################################
36
37 ghc-linters:
38   stage: lint
39   image: "registry.gitlab.haskell.org/ghc/ci-images/linters:$DOCKER_REV"
40   script:
41     - |
42       if [ -n "$CI_MERGE_REQUEST_ID" ]; then
43         base="$(git merge-base $CI_MERGE_REQUEST_BRANCH_NAME HEAD)"
44         validate-commit-msg .git $(git rev-list $base..$CI_COMMIT_SHA)
45         submodchecker .git $(git rev-list $base..$CI_COMMIT_SHA)
46         validate-whitespace .git $(git rev-list $base..$CI_COMMIT_SHA)
47         .gitlab/linters/check-makefiles.py $base $CI_COMMIT_SHA
48         .gitlab/linters/check-cpp.py $base $CI_COMMIT_SHA
49       fi
50   dependencies: []
51   tags:
52     - lint
53
54 ############################################################
55 # Validation via Pipelines (hadrian)
56 ############################################################
57
58 .validate-hadrian:
59   allow_failure: true
60   script:
61     - cabal update
62     - git clean -xdf && git submodule foreach git clean -xdf
63     - bash .circleci/prepare-system.sh
64     - if [[ -d ./cabal-cache ]]; then cp -R ./.cabal-cache ~/.cabal-cache; fi
65     - ./boot
66     - ./configure $CONFIGURE_ARGS
67     - hadrian/build.cabal.sh -j`mk/detect-cpu-count.sh` --docs=no-sphinx binary-dist
68     - mv _build/bindist/ghc*.tar.xz ghc.tar.xz
69   cache:
70     key: hadrian
71     paths:
72       - cabal-cache
73   artifacts:
74     when: always
75     paths:
76       - ghc.tar.xz
77
78 validate-x86_64-linux-deb8-hadrian:
79   extends: .validate-hadrian
80   stage: build
81   image: "registry.gitlab.haskell.org/ghc/ci-images/x86_64-linux-deb8:$DOCKER_REV"
82   before_script:
83     # workaround for docker permissions
84     - sudo chown ghc:ghc -R .
85     - python3 .gitlab/fix-submodules.py
86     - git submodule sync --recursive
87     - git submodule update --init --recursive
88     - git checkout .gitmodules
89     - "git fetch https://gitlab.haskell.org/ghc/ghc-performance-notes.git refs/notes/perf:refs/notes/perf || true"
90   tags:
91     - x86_64-linux
92
93 hadrian-ghc-in-ghci:
94   stage: build
95   image: ghcci/x86_64-linux-deb8:0.1
96   before_script:
97     # workaround for docker permissions
98     - sudo chown ghc:ghc -R .
99     - python3 .gitlab/fix-submodules.py
100     - git submodule sync --recursive
101     - git submodule update --init --recursive
102     - git checkout .gitmodules
103   tags:
104     - x86_64-linux
105   script:
106     - cabal update
107     - git clean -xdf && git submodule foreach git clean -xdf
108     - bash .circleci/prepare-system.sh
109     - if [[ -d ./cabal-cache ]]; then cp -R ./.cabal-cache ~/.cabal-cache; fi
110     - ./boot
111     - ./configure $CONFIGURE_ARGS
112     # Load ghc-in-ghci then immediately exit
113     - echo ":q" | hadrian/ghci.sh
114   cache:
115     key: hadrian-ghci
116     paths:
117       - cabal-cache
118
119 ############################################################
120 # Validation via Pipelines (make)
121 ############################################################
122
123 .validate:
124   variables:
125     TEST_TYPE: test
126   before_script:
127     - git clean -xdf && git submodule foreach git clean -xdf
128   script:
129     - ./boot
130     - ./configure $CONFIGURE_ARGS
131     - |
132       THREADS=`mk/detect-cpu-count.sh`
133       make V=0 -j$THREADS WERROR=-Werror
134     - |
135       make binary-dist TAR_COMP_OPTS="-1"
136     - |
137       # Prepare to push git notes.
138       METRICS_FILE=$(mktemp)
139       git config user.email "ben+ghc-ci@smart-cactus.org"
140       git config user.name "GHC GitLab CI"
141     - |
142       THREADS=`mk/detect-cpu-count.sh`
143       make $TEST_TYPE THREADS=$THREADS JUNIT_FILE=../../junit.xml METRICS_FILE=$METRICS_FILE
144     - |
145       # Push git notes.
146       METRICS_FILE=$METRICS_FILE .gitlab/push-test-metrics.sh
147   dependencies: []
148   artifacts:
149     reports:
150       junit: junit.xml
151     expire_in: 2 week
152     paths:
153       - ghc-*.tar.xz
154       - junit.xml
155
156 #################################
157 # x86_64-darwin
158 #################################
159
160 validate-x86_64-darwin:
161   extends: .validate
162   stage: full-build
163   tags:
164     - x86_64-darwin
165   variables:
166     GHC_VERSION: 8.6.3
167     BIN_DIST_PREP_TAR_COMP: "bindistprep/ghc-x86_64-apple-darwin.tar.xz"
168     MACOSX_DEPLOYMENT_TARGET: "10.7"
169     # Only Sierra and onwards supports clock_gettime. See #12858
170     ac_cv_func_clock_gettime: "no"
171     LANG: "en_US.UTF-8"
172     CONFIGURE_ARGS: --with-intree-gmp
173     TEST_ENV: "x86_64-darwin"
174   before_script:
175     - git clean -xdf && git submodule foreach git clean -xdf
176     - python3 .gitlab/fix-submodules.py
177     - git submodule sync --recursive
178     - git submodule update --init --recursive
179     - git checkout .gitmodules
180     - "git fetch https://gitlab.haskell.org/ghc/ghc-performance-notes.git refs/notes/perf:refs/notes/perf || true"
181
182     - bash .gitlab/darwin-init.sh
183     - PATH="`pwd`/toolchain/bin:$PATH"
184   after_script:
185     - cp -Rf $HOME/.cabal cabal-cache
186   artifacts:
187     when: always
188     expire_in: 2 week
189   cache:
190     key: darwin
191     paths:
192       - cabal-cache
193       - toolchain
194
195 .validate-linux:
196   extends: .validate
197   tags:
198     - x86_64-linux
199   before_script:
200     - git clean -xdf && git submodule foreach git clean -xdf
201     - python3 .gitlab/fix-submodules.py
202     - git submodule sync --recursive
203     - git submodule update --init --recursive
204     - git checkout .gitmodules
205     - "git fetch https://gitlab.haskell.org/ghc/ghc-performance-notes.git refs/notes/perf:refs/notes/perf || true"
206
207     - bash .circleci/prepare-system.sh
208     # workaround for docker permissions
209     - sudo chown ghc:ghc -R .
210   after_script:
211     - cp -Rf $HOME/.cabal cabal-cache
212   cache:
213     key: linux
214     paths:
215       - cabal-cache
216       - toolchain
217
218 #################################
219 # aarch64-linux-deb9
220 #################################
221
222 .build-aarch64-linux-deb9:
223   extends: .validate-linux
224   stage: full-build
225   image: "registry.gitlab.haskell.org/ghc/ci-images/aarch64-linux-deb9:$DOCKER_REV"
226   allow_failure: true
227   variables:
228     TEST_ENV: "aarch64-linux-deb9"
229     BIN_DIST_PREP_TAR_COMP: "bindistprep/ghc-aarch64-linux-deb9.tar.xz"
230   cache:
231     key: linux-aarch64-deb9
232   tags:
233     - aarch64-linux
234
235 validate-aarch64-linux-deb9:
236   extends: .build-aarch64-linux-deb9
237   artifacts:
238     when: always
239     expire_in: 2 week
240
241 nightly-aarch64-linux-deb9:
242   extends: .build-aarch64-linux-deb9
243   artifacts:
244     expire_in: 2 year
245   variables:
246     TEST_TYPE: slowtest
247   only:
248     variables:
249       - $NIGHTLY
250
251 #################################
252 # i386-linux-deb9
253 #################################
254
255 .build-i386-linux-deb9:
256   extends: .validate-linux
257   stage: full-build
258   image: "registry.gitlab.haskell.org/ghc/ci-images/i386-linux-deb9:$DOCKER_REV"
259   variables:
260     TEST_ENV: "i386-linux-deb9"
261     BIN_DIST_PREP_TAR_COMP: "bindistprep/ghc-i386-deb9-linux.tar.xz"
262   cache:
263     key: linux-i386-deb9
264
265 validate-i386-linux-deb9:
266   extends: .build-i386-linux-deb9
267   artifacts:
268     when: always
269     expire_in: 2 week
270
271 nightly-i386-linux-deb9:
272   extends: .build-i386-linux-deb9
273   variables:
274     TEST_TYPE: slowtest
275   artifacts:
276     when: always
277     expire_in: 2 week
278   only:
279     variables:
280       - $NIGHTLY
281
282 #################################
283 # x86_64-linux-deb9
284 #################################
285
286 .build-x86_64-linux-deb9:
287   extends: .validate-linux
288   stage: build
289   image: "registry.gitlab.haskell.org/ghc/ci-images/x86_64-linux-deb9:$DOCKER_REV"
290   variables:
291     TEST_ENV: "x86_64-linux-deb9"
292     BIN_DIST_PREP_TAR_COMP: "bindistprep/ghc-x86_64-deb9-linux.tar.xz"
293   cache:
294     key: linux-x86_64-deb9
295
296 validate-x86_64-linux-deb9:
297   extends: .build-x86_64-linux-deb9
298   artifacts:
299     when: always
300     expire_in: 2 week
301
302 nightly-x86_64-linux-deb9:
303   extends: .build-x86_64-linux-deb9
304   artifacts:
305     expire_in: 2 year
306   variables:
307     TEST_TYPE: slowtest
308   only:
309     variables:
310       - $NIGHTLY
311
312 # N.B. Has DEBUG assertions enabled in stage2
313 validate-x86_64-linux-deb9-debug:
314   extends: .build-x86_64-linux-deb9
315   variables:
316     BUILD_FLAVOUR: validate
317     TEST_ENV: "x86_64-linux-deb9-debug"
318
319 validate-x86_64-linux-deb9-llvm:
320   extends: .build-x86_64-linux-deb9
321   stage: full-build
322   variables:
323     BUILD_FLAVOUR: perf-llvm
324     TEST_ENV: "x86_64-linux-deb9-llvm"
325
326 validate-x86_64-linux-deb9-integer-simple:
327   extends: .build-x86_64-linux-deb9
328   stage: full-build
329   variables:
330     INTEGER_LIBRARY: integer-simple
331     TEST_ENV: "x86_64-linux-deb9-integer-simple"
332     BIN_DIST_PREP_TAR_COMP: "bindistprep/ghc-x86_64-deb9-linux-integer-simple.tar.xz"
333
334 nightly-x86_64-linux-deb9-integer-simple:
335   extends: .build-x86_64-linux-deb9
336   stage: full-build
337   variables:
338     INTEGER_LIBRARY: integer-simple
339     TEST_ENV: "x86_64-linux-deb9-integer-simple"
340     TEST_TYPE: slowtest
341   artifacts:
342     expire_in: 2 year
343   only:
344     variables:
345       - $NIGHTLY
346
347 validate-x86_64-linux-deb9-unreg:
348   extends: .build-x86_64-linux-deb9
349   stage: full-build
350   variables:
351     CONFIGURE_ARGS: --enable-unregisterised
352     TEST_ENV: "x86_64-linux-deb9-unreg"
353
354
355 #################################
356 # x86_64-linux-deb8
357 #################################
358
359 release-x86_64-linux-deb8:
360   extends: .validate-linux
361   stage: full-build
362   image: "registry.gitlab.haskell.org/ghc/ci-images/x86_64-linux-deb8:$DOCKER_REV"
363   variables:
364     TEST_ENV: "x86_64-linux-deb8"
365     BIN_DIST_PREP_TAR_COMP: "bindistprep/ghc-x86_64-deb8-linux.tar.xz"
366   only:
367     - tags
368   cache:
369     key: linux-x86_64-deb8
370   artifacts:
371     when: always
372     expire_in: 2 week
373
374
375 #################################
376 # x86_64-linux-fedora27
377 #################################
378
379 validate-x86_64-linux-fedora27:
380   extends: .validate-linux
381   stage: full-build
382   image: "registry.gitlab.haskell.org/ghc/ci-images/x86_64-linux-fedora27:$DOCKER_REV"
383   variables:
384     TEST_ENV: "x86_64-linux-fedora27"
385     BIN_DIST_PREP_TAR_COMP: "bindistprep/ghc-x86_64-fedora27-linux.tar.xz"
386   cache:
387     key: linux-x86_64-fedora27
388   artifacts:
389     when: always
390     expire_in: 2 week
391
392 ############################################################
393 # Validation via Pipelines (Windows)
394 ############################################################
395
396 .validate-windows:
397   before_script:
398     - git clean -xdf
399     - git submodule foreach git clean -xdf
400
401     # Use a local temporary directory to ensure that concurrent builds don't
402     # interfere with one another
403     - |
404       mkdir tmp
405       set TMP=%cd%\tmp
406       set TEMP=%cd%\tmp
407
408     - set PATH=C:\msys64\usr\bin;%PATH%
409     - python .gitlab/fix-submodules.py
410     - git submodule sync --recursive
411     - git submodule update --init --recursive
412     - git checkout .gitmodules
413     - "git fetch https://gitlab.haskell.org/ghc/ghc-performance-notes.git refs/notes/perf:refs/notes/perf || true"
414     - bash .gitlab/win32-init.sh
415   after_script:
416     - rd /s /q tmp
417     - robocopy /np /nfl /ndl /e "%APPDATA%\cabal" cabal-cache
418     - bash -c 'make clean || true'
419   dependencies: []
420   variables:
421     FORCE_SYMLINKS: 1
422   cache:
423     paths:
424       - cabal-cache
425       - ghc-8.6.2
426       - ghc-tarballs
427
428 validate-x86_64-windows-hadrian:
429   extends: .validate-windows
430   stage: full-build
431   variables:
432     GHC_VERSION: "8.6.2"
433     LANG: "en_US.UTF-8"
434   script:
435     - |
436       set MSYSTEM=MINGW64
437       python boot
438       bash -c './configure --enable-tarballs-autodownload GHC=`pwd`/toolchain/bin/ghc HAPPY=`pwd`/toolchain/bin/happy ALEX=`pwd`/toolchain/bin/alex'
439     - bash -c "PATH=`pwd`/toolchain/bin:$PATH hadrian/build.cabal.sh -j`mk/detect-cpu-count.sh` --flavour=Quick --docs=no-sphinx binary-dist"
440     - mv _build/bindist/ghc*.tar.xz ghc.tar.xz
441     # FIXME: Testsuite disabled due to #16156.
442     # - bash -c 'make V=0 test THREADS=`mk/detect-cpu-count.sh` JUNIT_FILE=../../junit.xml'
443   cache:
444     key: x86_64-windows
445   tags:
446     - x86_64-windows
447   artifacts:
448     when: always
449     paths:
450       - ghc.tar.xz
451
452 validate-x86_64-windows:
453   extends: .validate-windows
454   stage: full-build
455   # due to #16084
456   allow_failure: true
457   variables:
458     GHC_VERSION: "8.6.2"
459     LANG: "en_US.UTF-8"
460     BIN_DIST_PREP_TAR_COMP: "bindistprep/ghc-x86_64-mingw32.tar.xz"
461   script:
462     - |
463       set MSYSTEM=MINGW64
464       python boot
465       bash -c './configure --enable-tarballs-autodownload GHC=`pwd`/toolchain/bin/ghc HAPPY=`pwd`/toolchain/bin/happy ALEX=`pwd`/toolchain/bin/alex'
466     - bash -c "echo include mk/flavours/quick.mk > mk/build.mk"
467     - bash -c "PATH=`pwd`/toolchain/bin:$PATH make -j`mk/detect-cpu-count.sh`"
468     - bash -c "PATH=`pwd`/toolchain/bin:$PATH make binary-dist TAR_COMP_OPTS=-1"
469     - bash -c 'make V=0 test THREADS=`mk/detect-cpu-count.sh` JUNIT_FILE=../../junit.xml'
470   cache:
471     key: x86_64-windows
472   tags:
473     - x86_64-windows
474   artifacts:
475     when: always
476     reports:
477       junit: junit.xml
478     paths:
479       - ghc-*.tar.xz
480       - junit.xml
481
482 ############################################################
483 # Cleanup
484 ############################################################
485
486 # Note [Cleaning up after shell executor]
487 # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
488 #
489 # As noted in [1], gitlab-runner's shell executor doesn't clean up its working
490 # directory after builds. Unfortunately, we are forced to use the shell executor
491 # on Windows. To avoid running out of disk space we add a stage at the end of
492 # the build to remove the \GitLabRunner\builds directory. Since we only run a
493 # single build at a time on Windows this should be safe.
494 #
495 # [1] https://gitlab.com/gitlab-org/gitlab-runner/issues/3856
496
497 # See Note [Cleanup after shell executor]
498 cleanup-windows:
499   stage: cleanup
500   tags:
501     - x86_64-windows
502   when: always
503   dependencies: []
504   before_script:
505     - echo "Time to clean up"
506   script:
507     - echo "Let's go"
508   after_script:
509     - set "BUILD_DIR=%CI_PROJECT_DIR%"
510     - set "BUILD_DIR=%BUILD_DIR:/=\%"
511     - echo "Cleaning %BUILD_DIR%"
512     - cd \GitLabRunner
513     # This is way more complicated than it should be:
514     # See https://stackoverflow.com/questions/1965787
515     - del %BUILD_DIR%\* /F /Q
516     - for /d %%p in (%BUILD_DIR%\*) do rd /Q /S "%%p"
517     - exit /b 0
518
519 # See Note [Cleanup after shell executor]
520 cleanup-darwin:
521   stage: cleanup
522   tags:
523     - x86_64-darwin
524   when: always
525   dependencies: []
526   before_script:
527     - echo "Time to clean up"
528   script:
529     - echo "Let's go"
530   after_script:
531     - BUILD_DIR=$CI_PROJECT_DIR
532     - echo "Cleaning $BUILD_DIR"
533     - cd $HOME
534     - rm -Rf $BUILD_DIR/*
535     - exit 0
536
537 ############################################################
538 # Packaging
539 ############################################################
540
541 doc-tarball:
542   stage: packaging
543   tags:
544     - x86_64-linux
545   image: ghcci/x86_64-linux-deb9:0.2
546   dependencies:
547     - validate-x86_64-linux-deb9
548     - validate-x86_64-windows
549   artifacts:
550     paths:
551       - haddock.html.tar.xz
552       - libraries.html.tar.xz
553       - users_guide.html.tar.xz
554       - index.html
555       - "*.pdf"
556   script:
557     - rm -Rf docs
558     - distrib/mkDocs/mkDocs ghc-x86_64-deb9-linux.tar.xz ghc-x86_64-mingw32.tar.xz
559     - ls -lh
560     - mv docs/*.tar.xz .
561
562 source-tarball:
563   stage: packaging
564   tags:
565     - x86_64-linux
566   image: ghcci/x86_64-linux-deb9:0.2
567   only:
568     - tags
569   artifacts:
570     paths:
571       - ghc-*.tar.xz
572       - version
573   script:
574     - mk/get-win32-tarballs.sh download all
575     - ./boot
576     - ./configure
577     - make sdist
578     - mv sdistprep/*.xz  .
579     - make show VALUE=version > version