gitlab-ci: Implement head.hackage jobs
[ghc.git] / .gitlab-ci.yml
1 variables:
2   GIT_SSL_NO_VERIFY: "1"
3
4   # Commit of ghc/ci-images repository from which to pull Docker images
5   DOCKER_REV: 6d19c3adc1f5c28c82aed8c5b1ac40931ac60f3f
6
7 before_script:
8   - python3 .gitlab/fix-submodules.py
9   - git submodule sync --recursive
10   - git submodule update --init --recursive
11   - git checkout .gitmodules
12   - "git fetch https://gitlab.haskell.org/ghc/ghc-performance-notes.git refs/notes/perf:refs/notes/perf || true"
13
14 stages:
15   - lint
16   - build
17   - full-build
18   - cleanup # See Note [Cleanup on Windows]
19   - packaging
20   - hackage
21
22 ############################################################
23 # Runner Tags
24 ############################################################
25 #
26 # * x86_64-linux: Any Docker-capable x86_64 Linux machine
27 # * aarch64-linux: Any Docker-capable AArch64 Linux machine
28 # * x86_64-windows: A x86_64 Windows machine
29 # * lint: Any Docker-capable x86_64 Linux machine; distinct from
30 #         x86_64-linux to ensure low-latency availability.
31 #
32
33
34 ############################################################
35 # Linting
36 ############################################################
37
38 ghc-linters:
39   stage: lint
40   image: "registry.gitlab.haskell.org/ghc/ci-images/linters:$DOCKER_REV"
41   script:
42     - |
43       if [ -n "$CI_MERGE_REQUEST_ID" ]; then
44         base="$(git merge-base $CI_MERGE_REQUEST_BRANCH_NAME HEAD)"
45         validate-commit-msg .git $(git rev-list $base..$CI_COMMIT_SHA)
46         submodchecker .git $(git rev-list $base..$CI_COMMIT_SHA)
47         validate-whitespace .git $(git rev-list $base..$CI_COMMIT_SHA)
48         .gitlab/linters/check-makefiles.py $base $CI_COMMIT_SHA
49         .gitlab/linters/check-cpp.py $base $CI_COMMIT_SHA
50       fi
51   dependencies: []
52   tags:
53     - lint
54
55 ############################################################
56 # Validation via Pipelines (hadrian)
57 ############################################################
58
59 .validate-hadrian:
60   allow_failure: true
61   script:
62     - cabal update
63     - git clean -xdf && git submodule foreach git clean -xdf
64     - bash .circleci/prepare-system.sh
65     - if [[ -d ./cabal-cache ]]; then cp -R ./.cabal-cache ~/.cabal-cache; fi
66     - ./boot
67     - ./configure $CONFIGURE_ARGS
68     - hadrian/build.cabal.sh -j`mk/detect-cpu-count.sh` --docs=no-sphinx binary-dist
69     - mv _build/bindist/ghc*.tar.xz ghc.tar.xz
70   cache:
71     key: hadrian
72     paths:
73       - cabal-cache
74   artifacts:
75     when: always
76     paths:
77       - ghc.tar.xz
78
79 validate-x86_64-linux-deb8-hadrian:
80   extends: .validate-hadrian
81   stage: build
82   image: "registry.gitlab.haskell.org/ghc/ci-images/x86_64-linux-deb8:$DOCKER_REV"
83   before_script:
84     # workaround for docker permissions
85     - sudo chown ghc:ghc -R .
86     - python3 .gitlab/fix-submodules.py
87     - git submodule sync --recursive
88     - git submodule update --init --recursive
89     - git checkout .gitmodules
90     - "git fetch https://gitlab.haskell.org/ghc/ghc-performance-notes.git refs/notes/perf:refs/notes/perf || true"
91   tags:
92     - x86_64-linux
93
94 hadrian-ghc-in-ghci:
95   stage: build
96   image: ghcci/x86_64-linux-deb8:0.1
97   before_script:
98     # workaround for docker permissions
99     - sudo chown ghc:ghc -R .
100     - python3 .gitlab/fix-submodules.py
101     - git submodule sync --recursive
102     - git submodule update --init --recursive
103     - git checkout .gitmodules
104   tags:
105     - x86_64-linux
106   script:
107     - cabal update
108     - git clean -xdf && git submodule foreach git clean -xdf
109     - bash .circleci/prepare-system.sh
110     - if [[ -d ./cabal-cache ]]; then cp -R ./.cabal-cache ~/.cabal-cache; fi
111     - ./boot
112     - ./configure $CONFIGURE_ARGS
113     # Load ghc-in-ghci then immediately exit
114     - echo ":q" | hadrian/ghci.sh
115   cache:
116     key: hadrian-ghci
117     paths:
118       - cabal-cache
119
120 ############################################################
121 # Validation via Pipelines (make)
122 ############################################################
123
124 .validate:
125   variables:
126     TEST_TYPE: test
127   before_script:
128     - git clean -xdf && git submodule foreach git clean -xdf
129   script:
130     - ./boot
131     - ./configure $CONFIGURE_ARGS
132     - |
133       THREADS=`mk/detect-cpu-count.sh`
134       make V=0 -j$THREADS WERROR=-Werror
135     - |
136       make binary-dist TAR_COMP_OPTS="-1"
137     - |
138       # Prepare to push git notes.
139       METRICS_FILE=$(mktemp)
140       git config user.email "ben+ghc-ci@smart-cactus.org"
141       git config user.name "GHC GitLab CI"
142     - |
143       THREADS=`mk/detect-cpu-count.sh`
144       make $TEST_TYPE THREADS=$THREADS JUNIT_FILE=../../junit.xml METRICS_FILE=$METRICS_FILE
145     - |
146       # Push git notes.
147       METRICS_FILE=$METRICS_FILE .gitlab/push-test-metrics.sh
148   dependencies: []
149   artifacts:
150     reports:
151       junit: junit.xml
152     expire_in: 2 week
153     paths:
154       - ghc-*.tar.xz
155       - junit.xml
156
157 #################################
158 # x86_64-darwin
159 #################################
160
161 validate-x86_64-darwin:
162   extends: .validate
163   stage: full-build
164   tags:
165     - x86_64-darwin
166   variables:
167     GHC_VERSION: 8.6.3
168     BIN_DIST_PREP_TAR_COMP: "bindistprep/ghc-x86_64-apple-darwin.tar.xz"
169     MACOSX_DEPLOYMENT_TARGET: "10.7"
170     # Only Sierra and onwards supports clock_gettime. See #12858
171     ac_cv_func_clock_gettime: "no"
172     LANG: "en_US.UTF-8"
173     CONFIGURE_ARGS: --with-intree-gmp
174     TEST_ENV: "x86_64-darwin"
175   before_script:
176     - git clean -xdf && git submodule foreach git clean -xdf
177     - python3 .gitlab/fix-submodules.py
178     - git submodule sync --recursive
179     - git submodule update --init --recursive
180     - git checkout .gitmodules
181     - "git fetch https://gitlab.haskell.org/ghc/ghc-performance-notes.git refs/notes/perf:refs/notes/perf || true"
182
183     - bash .gitlab/darwin-init.sh
184     - PATH="`pwd`/toolchain/bin:$PATH"
185   after_script:
186     - cp -Rf $HOME/.cabal cabal-cache
187   artifacts:
188     when: always
189     expire_in: 2 week
190   cache:
191     key: darwin
192     paths:
193       - cabal-cache
194       - toolchain
195
196 .validate-linux:
197   extends: .validate
198   tags:
199     - x86_64-linux
200   before_script:
201     - git clean -xdf && git submodule foreach git clean -xdf
202     - python3 .gitlab/fix-submodules.py
203     - git submodule sync --recursive
204     - git submodule update --init --recursive
205     - git checkout .gitmodules
206     - "git fetch https://gitlab.haskell.org/ghc/ghc-performance-notes.git refs/notes/perf:refs/notes/perf || true"
207
208     - bash .circleci/prepare-system.sh
209     # workaround for docker permissions
210     - sudo chown ghc:ghc -R .
211   after_script:
212     - cp -Rf $HOME/.cabal cabal-cache
213   cache:
214     key: linux
215     paths:
216       - cabal-cache
217       - toolchain
218
219 #################################
220 # aarch64-linux-deb9
221 #################################
222
223 .build-aarch64-linux-deb9:
224   extends: .validate-linux
225   stage: full-build
226   image: "registry.gitlab.haskell.org/ghc/ci-images/aarch64-linux-deb9:$DOCKER_REV"
227   allow_failure: true
228   variables:
229     TEST_ENV: "aarch64-linux-deb9"
230     BIN_DIST_PREP_TAR_COMP: "bindistprep/ghc-aarch64-linux-deb9.tar.xz"
231   cache:
232     key: linux-aarch64-deb9
233   tags:
234     - aarch64-linux
235
236 validate-aarch64-linux-deb9:
237   extends: .build-aarch64-linux-deb9
238   artifacts:
239     when: always
240     expire_in: 2 week
241
242 nightly-aarch64-linux-deb9:
243   extends: .build-aarch64-linux-deb9
244   artifacts:
245     expire_in: 2 year
246   variables:
247     TEST_TYPE: slowtest
248   only:
249     variables:
250       - $NIGHTLY
251
252 #################################
253 # i386-linux-deb9
254 #################################
255
256 .build-i386-linux-deb9:
257   extends: .validate-linux
258   stage: full-build
259   image: "registry.gitlab.haskell.org/ghc/ci-images/i386-linux-deb9:$DOCKER_REV"
260   variables:
261     TEST_ENV: "i386-linux-deb9"
262     BIN_DIST_PREP_TAR_COMP: "bindistprep/ghc-i386-deb9-linux.tar.xz"
263   cache:
264     key: linux-i386-deb9
265
266 validate-i386-linux-deb9:
267   extends: .build-i386-linux-deb9
268   artifacts:
269     when: always
270     expire_in: 2 week
271
272 nightly-i386-linux-deb9:
273   extends: .build-i386-linux-deb9
274   variables:
275     TEST_TYPE: slowtest
276   artifacts:
277     when: always
278     expire_in: 2 week
279   only:
280     variables:
281       - $NIGHTLY
282
283 #################################
284 # x86_64-linux-deb9
285 #################################
286
287 .build-x86_64-linux-deb9:
288   extends: .validate-linux
289   stage: build
290   image: "registry.gitlab.haskell.org/ghc/ci-images/x86_64-linux-deb9:$DOCKER_REV"
291   variables:
292     TEST_ENV: "x86_64-linux-deb9"
293     BIN_DIST_PREP_TAR_COMP: "bindistprep/ghc-x86_64-deb9-linux.tar.xz"
294   cache:
295     key: linux-x86_64-deb9
296
297 validate-x86_64-linux-deb9:
298   extends: .build-x86_64-linux-deb9
299   artifacts:
300     when: always
301     expire_in: 2 week
302
303 nightly-x86_64-linux-deb9:
304   extends: .build-x86_64-linux-deb9
305   artifacts:
306     expire_in: 2 year
307   variables:
308     TEST_TYPE: slowtest
309   only:
310     variables:
311       - $NIGHTLY
312
313 # N.B. Has DEBUG assertions enabled in stage2
314 validate-x86_64-linux-deb9-debug:
315   extends: .build-x86_64-linux-deb9
316   variables:
317     BUILD_FLAVOUR: validate
318     TEST_ENV: "x86_64-linux-deb9-debug"
319
320 validate-x86_64-linux-deb9-llvm:
321   extends: .build-x86_64-linux-deb9
322   stage: full-build
323   variables:
324     BUILD_FLAVOUR: perf-llvm
325     TEST_ENV: "x86_64-linux-deb9-llvm"
326
327 validate-x86_64-linux-deb9-integer-simple:
328   extends: .build-x86_64-linux-deb9
329   stage: full-build
330   variables:
331     INTEGER_LIBRARY: integer-simple
332     TEST_ENV: "x86_64-linux-deb9-integer-simple"
333     BIN_DIST_PREP_TAR_COMP: "bindistprep/ghc-x86_64-deb9-linux-integer-simple.tar.xz"
334
335 nightly-x86_64-linux-deb9-integer-simple:
336   extends: .build-x86_64-linux-deb9
337   stage: full-build
338   variables:
339     INTEGER_LIBRARY: integer-simple
340     TEST_ENV: "x86_64-linux-deb9-integer-simple"
341     TEST_TYPE: slowtest
342   artifacts:
343     expire_in: 2 year
344   only:
345     variables:
346       - $NIGHTLY
347
348 validate-x86_64-linux-deb9-unreg:
349   extends: .build-x86_64-linux-deb9
350   stage: full-build
351   variables:
352     CONFIGURE_ARGS: --enable-unregisterised
353     TEST_ENV: "x86_64-linux-deb9-unreg"
354
355
356 #################################
357 # x86_64-linux-deb8
358 #################################
359
360 release-x86_64-linux-deb8:
361   extends: .validate-linux
362   stage: full-build
363   image: "registry.gitlab.haskell.org/ghc/ci-images/x86_64-linux-deb8:$DOCKER_REV"
364   variables:
365     TEST_ENV: "x86_64-linux-deb8"
366     BIN_DIST_PREP_TAR_COMP: "bindistprep/ghc-x86_64-deb8-linux.tar.xz"
367   only:
368     - tags
369   cache:
370     key: linux-x86_64-deb8
371   artifacts:
372     when: always
373     expire_in: 2 week
374
375
376 #################################
377 # x86_64-linux-fedora27
378 #################################
379
380 validate-x86_64-linux-fedora27:
381   extends: .validate-linux
382   stage: full-build
383   image: "registry.gitlab.haskell.org/ghc/ci-images/x86_64-linux-fedora27:$DOCKER_REV"
384   variables:
385     TEST_ENV: "x86_64-linux-fedora27"
386     BIN_DIST_PREP_TAR_COMP: "bindistprep/ghc-x86_64-fedora27-linux.tar.xz"
387   cache:
388     key: linux-x86_64-fedora27
389   artifacts:
390     when: always
391     expire_in: 2 week
392
393 ############################################################
394 # Validation via Pipelines (Windows)
395 ############################################################
396
397 .validate-windows:
398   before_script:
399     - git clean -xdf
400     - git submodule foreach git clean -xdf
401
402     # Use a local temporary directory to ensure that concurrent builds don't
403     # interfere with one another
404     - |
405       mkdir tmp
406       set TMP=%cd%\tmp
407       set TEMP=%cd%\tmp
408
409     - set PATH=C:\msys64\usr\bin;%PATH%
410     - python .gitlab/fix-submodules.py
411     - git submodule sync --recursive
412     - git submodule update --init --recursive
413     - git checkout .gitmodules
414     - "git fetch https://gitlab.haskell.org/ghc/ghc-performance-notes.git refs/notes/perf:refs/notes/perf || true"
415     - bash .gitlab/win32-init.sh
416   after_script:
417     - rd /s /q tmp
418     - robocopy /np /nfl /ndl /e "%APPDATA%\cabal" cabal-cache
419     - bash -c 'make clean || true'
420   dependencies: []
421   variables:
422     FORCE_SYMLINKS: 1
423   cache:
424     paths:
425       - cabal-cache
426       - ghc-8.6.2
427       - ghc-tarballs
428
429 validate-x86_64-windows-hadrian:
430   extends: .validate-windows
431   stage: full-build
432   variables:
433     GHC_VERSION: "8.6.2"
434     LANG: "en_US.UTF-8"
435   script:
436     - |
437       set MSYSTEM=MINGW64
438       python boot
439       bash -c './configure --enable-tarballs-autodownload GHC=`pwd`/toolchain/bin/ghc HAPPY=`pwd`/toolchain/bin/happy ALEX=`pwd`/toolchain/bin/alex'
440     - bash -c "PATH=`pwd`/toolchain/bin:$PATH hadrian/build.cabal.sh -j`mk/detect-cpu-count.sh` --flavour=Quick --docs=no-sphinx binary-dist"
441     - mv _build/bindist/ghc*.tar.xz ghc.tar.xz
442     # FIXME: Testsuite disabled due to #16156.
443     # - bash -c 'make V=0 test THREADS=`mk/detect-cpu-count.sh` JUNIT_FILE=../../junit.xml'
444   cache:
445     key: x86_64-windows
446   tags:
447     - x86_64-windows
448   artifacts:
449     when: always
450     paths:
451       - ghc.tar.xz
452
453 validate-x86_64-windows:
454   extends: .validate-windows
455   stage: full-build
456   # due to #16084
457   allow_failure: true
458   variables:
459     GHC_VERSION: "8.6.2"
460     LANG: "en_US.UTF-8"
461     BIN_DIST_PREP_TAR_COMP: "bindistprep/ghc-x86_64-mingw32.tar.xz"
462   script:
463     - |
464       set MSYSTEM=MINGW64
465       python boot
466       bash -c './configure --enable-tarballs-autodownload GHC=`pwd`/toolchain/bin/ghc HAPPY=`pwd`/toolchain/bin/happy ALEX=`pwd`/toolchain/bin/alex'
467     - bash -c "echo include mk/flavours/quick.mk > mk/build.mk"
468     - bash -c "PATH=`pwd`/toolchain/bin:$PATH make -j`mk/detect-cpu-count.sh`"
469     - bash -c "PATH=`pwd`/toolchain/bin:$PATH make binary-dist TAR_COMP_OPTS=-1"
470     - bash -c 'make V=0 test THREADS=`mk/detect-cpu-count.sh` JUNIT_FILE=../../junit.xml'
471   cache:
472     key: x86_64-windows
473   tags:
474     - x86_64-windows
475   artifacts:
476     when: always
477     reports:
478       junit: junit.xml
479     paths:
480       - ghc-*.tar.xz
481       - junit.xml
482
483 ############################################################
484 # Cleanup
485 ############################################################
486
487 # Note [Cleaning up after shell executor]
488 # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
489 #
490 # As noted in [1], gitlab-runner's shell executor doesn't clean up its working
491 # directory after builds. Unfortunately, we are forced to use the shell executor
492 # on Windows. To avoid running out of disk space we add a stage at the end of
493 # the build to remove the \GitLabRunner\builds directory. Since we only run a
494 # single build at a time on Windows this should be safe.
495 #
496 # [1] https://gitlab.com/gitlab-org/gitlab-runner/issues/3856
497
498 # See Note [Cleanup after shell executor]
499 cleanup-windows:
500   stage: cleanup
501   tags:
502     - x86_64-windows
503   when: always
504   dependencies: []
505   before_script:
506     - echo "Time to clean up"
507   script:
508     - echo "Let's go"
509   after_script:
510     - set "BUILD_DIR=%CI_PROJECT_DIR%"
511     - set "BUILD_DIR=%BUILD_DIR:/=\%"
512     - echo "Cleaning %BUILD_DIR%"
513     - cd \GitLabRunner
514     # This is way more complicated than it should be:
515     # See https://stackoverflow.com/questions/1965787
516     - del %BUILD_DIR%\* /F /Q
517     - for /d %%p in (%BUILD_DIR%\*) do rd /Q /S "%%p"
518     - exit /b 0
519
520 # See Note [Cleanup after shell executor]
521 cleanup-darwin:
522   stage: cleanup
523   tags:
524     - x86_64-darwin
525   when: always
526   dependencies: []
527   before_script:
528     - echo "Time to clean up"
529   script:
530     - echo "Let's go"
531   after_script:
532     - BUILD_DIR=$CI_PROJECT_DIR
533     - echo "Cleaning $BUILD_DIR"
534     - cd $HOME
535     - rm -Rf $BUILD_DIR/*
536     - exit 0
537
538 ############################################################
539 # Packaging
540 ############################################################
541
542 doc-tarball:
543   stage: packaging
544   tags:
545     - x86_64-linux
546   image: ghcci/x86_64-linux-deb9:0.2
547   dependencies:
548     - validate-x86_64-linux-deb9
549     - validate-x86_64-windows
550   artifacts:
551     paths:
552       - haddock.html.tar.xz
553       - libraries.html.tar.xz
554       - users_guide.html.tar.xz
555       - index.html
556       - "*.pdf"
557   script:
558     - rm -Rf docs
559     - distrib/mkDocs/mkDocs ghc-x86_64-deb9-linux.tar.xz ghc-x86_64-mingw32.tar.xz
560     - ls -lh
561     - mv docs/*.tar.xz .
562
563 source-tarball:
564   stage: packaging
565   tags:
566     - x86_64-linux
567   image: ghcci/x86_64-linux-deb9:0.2
568   only:
569     - tags
570   artifacts:
571     paths:
572       - ghc-*.tar.xz
573       - version
574   script:
575     - mk/get-win32-tarballs.sh download all
576     - ./boot
577     - ./configure
578     - make sdist
579     - mv sdistprep/*.xz  .
580     - make show VALUE=version > version
581
582
583 ############################################################
584 # Testing via head.hackage
585 ############################################################
586
587 # Triggering jobs in the ghc/head.hackage project requires that we have a job
588 # token for that repository. Furthermore the head.hackage CI job must have
589 # access to an unprivileged access token with the ability to query the ghc/ghc
590 # project such that it can find the job ID of the fedora27 job for the current
591 # pipeline.
592
593 .hackage:
594   stage: hackage
595   image: ghcci/x86_64-linux-deb9:0.2
596   tags:
597     - x86_64-linux
598   dependencies: []
599   variables:
600     HEAD_HACKAGE_PROJECT_ID: "78"
601   script:
602     - bash .gitlab/start-head.hackage.sh
603
604 hackage:
605   extends: .hackage
606   when: manual
607
608 hackage-label:
609   extends: .hackage
610   only:
611     variables:
612       - $CI_MERGE_REQUEST_LABELS =~ /.*user-facing.*/
613
614 nightly-hackage:
615   extends: .hackage
616   only:
617     variables:
618       - $NIGHTLY