Hadrian: build (and retrieve) binary distributions in CI
[ghc.git] / .gitlab-ci.yml
1 variables:
2   GIT_SSL_NO_VERIFY: "1"
3
4   # Commit of ghc/ci-images repository from which to pull Docker images
5   DOCKER_REV: 6d19c3adc1f5c28c82aed8c5b1ac40931ac60f3f
6
7 before_script:
8   - python3 .gitlab/fix-submodules.py
9   - git submodule sync --recursive
10   - git submodule update --init --recursive
11   - git checkout .gitmodules
12   - "git fetch https://gitlab.haskell.org/ghc/ghc-performance-notes.git refs/notes/perf:refs/notes/perf || true"
13
14 stages:
15   - lint
16   - build
17   - full-build
18   - cleanup # See Note [Cleanup on Windows]
19
20 ############################################################
21 # Runner Tags
22 ############################################################
23 #
24 # * x86_64-linux: Any Docker-capable x86_64 Linux machine
25 # * aarch64-linux: Any Docker-capable AArch64 Linux machine
26 # * x86_64-windows: A x86_64 Windows machine
27 # * lint: Any Docker-capable x86_64 Linux machine; distinct from
28 #         x86_64-linux to ensure low-latency availability.
29 #
30
31
32 ############################################################
33 # Linting
34 ############################################################
35
36 ghc-linters:
37   stage: lint
38   image: "registry.gitlab.haskell.org/ghc/ci-images/linters:$DOCKER_REV"
39   script:
40     - |
41       if [ -n "$CI_MERGE_REQUEST_ID" ]; then
42         base="$(git merge-base $CI_MERGE_REQUEST_BRANCH_NAME HEAD)"
43         validate-commit-msg .git $(git rev-list $base..$CI_COMMIT_SHA)
44         submodchecker .git $(git rev-list $base..$CI_COMMIT_SHA)
45         validate-whitespace .git $(git rev-list $base..$CI_COMMIT_SHA)
46         .gitlab/linters/check-makefiles.py $base $CI_COMMIT_SHA
47         .gitlab/linters/check-cpp.py $base $CI_COMMIT_SHA
48       fi
49   dependencies: []
50   tags:
51     - lint
52
53 ############################################################
54 # Validation via Pipelines (hadrian)
55 ############################################################
56
57 .validate-hadrian:
58   allow_failure: true
59   script:
60     - cabal update
61     - git clean -xdf && git submodule foreach git clean -xdf
62     - bash .circleci/prepare-system.sh
63     - if [[ -d ./cabal-cache ]]; then cp -R ./.cabal-cache ~/.cabal-cache; fi
64     - ./boot
65     - ./configure $CONFIGURE_ARGS
66     - hadrian/build.cabal.sh -j`mk/detect-cpu-count.sh` --docs=no-sphinx binary-dist
67     - mv _build/bindist/ghc*.tar.xz ghc.tar.xz
68   cache:
69     key: hadrian
70     paths:
71       - cabal-cache
72   artifacts:
73     when: always
74     paths:
75       - ghc.tar.xz
76
77 validate-x86_64-linux-deb8-hadrian:
78   extends: .validate-hadrian
79   stage: build
80   image: "registry.gitlab.haskell.org/ghc/ci-images/x86_64-linux-deb8:$DOCKER_REV"
81   before_script:
82     # workaround for docker permissions
83     - sudo chown ghc:ghc -R .
84     - python3 .gitlab/fix-submodules.py
85     - git submodule sync --recursive
86     - git submodule update --init --recursive
87     - git checkout .gitmodules
88     - "git fetch https://gitlab.haskell.org/ghc/ghc-performance-notes.git refs/notes/perf:refs/notes/perf || true"
89   tags:
90     - x86_64-linux
91
92 hadrian-ghc-in-ghci:
93   stage: build
94   image: ghcci/x86_64-linux-deb8:0.1
95   before_script:
96     # workaround for docker permissions
97     - sudo chown ghc:ghc -R .
98     - python3 .gitlab/fix-submodules.py
99     - git submodule sync --recursive
100     - git submodule update --init --recursive
101     - git checkout .gitmodules
102   tags:
103     - x86_64-linux
104   script:
105     - cabal update
106     - git clean -xdf && git submodule foreach git clean -xdf
107     - bash .circleci/prepare-system.sh
108     - if [[ -d ./cabal-cache ]]; then cp -R ./.cabal-cache ~/.cabal-cache; fi
109     - ./boot
110     - ./configure $CONFIGURE_ARGS
111     # Load ghc-in-ghci then immediately exit
112     - echo ":q" | hadrian/ghci.sh
113   cache:
114     key: hadrian-ghci
115     paths:
116       - cabal-cache
117
118 ############################################################
119 # Validation via Pipelines (make)
120 ############################################################
121
122 .validate:
123   variables:
124     TEST_TYPE: test
125   before_script:
126     - git clean -xdf && git submodule foreach git clean -xdf
127   script:
128     - ./boot
129     - ./configure $CONFIGURE_ARGS
130     - |
131       THREADS=`mk/detect-cpu-count.sh`
132       make V=0 -j$THREADS WERROR=-Werror
133     - |
134       make binary-dist TAR_COMP_OPTS="-1"
135       mv ghc-*.tar.xz ghc.tar.xz
136     - |
137       # Prepare to push git notes.
138       METRICS_FILE=$(mktemp)
139       git config user.email "ben+ghc-ci@smart-cactus.org"
140       git config user.name "GHC GitLab CI"
141     - |
142       THREADS=`mk/detect-cpu-count.sh`
143       make $TEST_TYPE THREADS=$THREADS JUNIT_FILE=../../junit.xml METRICS_FILE=$METRICS_FILE
144     - |
145       # Push git notes.
146       METRICS_FILE=$METRICS_FILE .gitlab/push-test-metrics.sh
147   dependencies: []
148   artifacts:
149     reports:
150       junit: junit.xml
151     expire_in: 2 week
152     paths:
153       - ghc.tar.xz
154       - junit.xml
155
156 #################################
157 # x86_64-darwin
158 #################################
159
160 validate-x86_64-darwin:
161   extends: .validate
162   stage: full-build
163   tags:
164     - x86_64-darwin
165   variables:
166     GHC_VERSION: 8.6.3
167     MACOSX_DEPLOYMENT_TARGET: "10.7"
168     # Only Sierra and onwards supports clock_gettime. See #12858
169     ac_cv_func_clock_gettime: "no"
170     LANG: "en_US.UTF-8"
171     CONFIGURE_ARGS: --with-intree-gmp
172     TEST_ENV: "x86_64-darwin"
173   before_script:
174     - git clean -xdf && git submodule foreach git clean -xdf
175     - python3 .gitlab/fix-submodules.py
176     - git submodule sync --recursive
177     - git submodule update --init --recursive
178     - git checkout .gitmodules
179     - "git fetch https://gitlab.haskell.org/ghc/ghc-performance-notes.git refs/notes/perf:refs/notes/perf || true"
180
181     - bash .gitlab/darwin-init.sh
182     - PATH="`pwd`/toolchain/bin:$PATH"
183   after_script:
184     - cp -Rf $HOME/.cabal cabal-cache
185   artifacts:
186     when: always
187     expire_in: 2 week
188   cache:
189     key: darwin
190     paths:
191       - cabal-cache
192       - toolchain
193
194 .validate-linux:
195   extends: .validate
196   tags:
197     - x86_64-linux
198   before_script:
199     - git clean -xdf && git submodule foreach git clean -xdf
200     - python3 .gitlab/fix-submodules.py
201     - git submodule sync --recursive
202     - git submodule update --init --recursive
203     - git checkout .gitmodules
204     - "git fetch https://gitlab.haskell.org/ghc/ghc-performance-notes.git refs/notes/perf:refs/notes/perf || true"
205
206     - bash .circleci/prepare-system.sh
207     # workaround for docker permissions
208     - sudo chown ghc:ghc -R .
209   after_script:
210     - cp -Rf $HOME/.cabal cabal-cache
211   cache:
212     key: linux
213     paths:
214       - cabal-cache
215       - toolchain
216
217 #################################
218 # aarch64-linux-deb9
219 #################################
220
221 .build-aarch64-linux-deb9:
222   extends: .validate-linux
223   stage: full-build
224   image: "registry.gitlab.haskell.org/ghc/ci-images/aarch64-linux-deb9:$DOCKER_REV"
225   allow_failure: true
226   variables:
227     TEST_ENV: "aarch64-linux-deb9"
228   cache:
229     key: linux-aarch64-deb9
230   tags:
231     - aarch64-linux
232
233 validate-aarch64-linux-deb9:
234   extends: .build-aarch64-linux-deb9
235   artifacts:
236     when: always
237     expire_in: 2 week
238
239 nightly-aarch64-linux-deb9:
240   extends: .build-aarch64-linux-deb9
241   artifacts:
242     expire_in: 2 year
243   variables:
244     TEST_TYPE: slowtest
245   only:
246     variables:
247       - $NIGHTLY
248
249 #################################
250 # i386-linux-deb9
251 #################################
252
253 .build-i386-linux-deb9:
254   extends: .validate-linux
255   stage: full-build
256   image: "registry.gitlab.haskell.org/ghc/ci-images/i386-linux-deb9:$DOCKER_REV"
257   variables:
258     TEST_ENV: "i386-linux-deb9"
259   cache:
260     key: linux-i386-deb9
261
262 validate-i386-linux-deb9:
263   extends: .build-i386-linux-deb9
264   artifacts:
265     when: always
266     expire_in: 2 week
267
268 nightly-i386-linux-deb9:
269   extends: .build-i386-linux-deb9
270   variables:
271     TEST_TYPE: slowtest
272   artifacts:
273     when: always
274     expire_in: 2 week
275   only:
276     variables:
277       - $NIGHTLY
278
279 #################################
280 # x86_64-linux-deb9
281 #################################
282
283 .build-x86_64-linux-deb9:
284   extends: .validate-linux
285   stage: build
286   image: "registry.gitlab.haskell.org/ghc/ci-images/x86_64-linux-deb9:$DOCKER_REV"
287   variables:
288     TEST_ENV: "x86_64-linux-deb9"
289   cache:
290     key: linux-x86_64-deb9
291
292 validate-x86_64-linux-deb9:
293   extends: .build-x86_64-linux-deb9
294   artifacts:
295     when: always
296     expire_in: 2 week
297
298 nightly-x86_64-linux-deb9:
299   extends: .build-x86_64-linux-deb9
300   artifacts:
301     expire_in: 2 year
302   variables:
303     TEST_TYPE: slowtest
304   only:
305     variables:
306       - $NIGHTLY
307
308 # N.B. Has DEBUG assertions enabled in stage2
309 validate-x86_64-linux-deb9-debug:
310   extends: .build-x86_64-linux-deb9
311   variables:
312     BUILD_FLAVOUR: validate
313     TEST_ENV: "x86_64-linux-deb9-debug"
314
315 validate-x86_64-linux-deb9-llvm:
316   extends: .build-x86_64-linux-deb9
317   stage: full-build
318   variables:
319     BUILD_FLAVOUR: perf-llvm
320     TEST_ENV: "x86_64-linux-deb9-llvm"
321
322 validate-x86_64-linux-deb9-integer-simple:
323   extends: .build-x86_64-linux-deb9
324   stage: full-build
325   variables:
326     INTEGER_LIBRARY: integer-simple
327     TEST_ENV: "x86_64-linux-deb9-integer-simple"
328
329 nightly-x86_64-linux-deb9-integer-simple:
330   extends: .build-x86_64-linux-deb9
331   stage: full-build
332   variables:
333     INTEGER_LIBRARY: integer-simple
334     TEST_ENV: "x86_64-linux-deb9-integer-simple"
335     TEST_TYPE: slowtest
336   artifacts:
337     expire_in: 2 year
338   only:
339     variables:
340       - $NIGHTLY
341
342 validate-x86_64-linux-deb9-unreg:
343   extends: .build-x86_64-linux-deb9
344   stage: full-build
345   variables:
346     CONFIGURE_ARGS: --enable-unregisterised
347     TEST_ENV: "x86_64-linux-deb9-unreg"
348
349
350 #################################
351 # x86_64-linux-deb8
352 #################################
353
354 release-x86_64-linux-deb8:
355   extends: .validate-linux
356   stage: full-build
357   image: "registry.gitlab.haskell.org/ghc/ci-images/x86_64-linux-deb8:$DOCKER_REV"
358   variables:
359     TEST_ENV: "x86_64-linux-deb8"
360   only:
361     - tags
362   cache:
363     key: linux-x86_64-deb8
364   artifacts:
365     when: always
366     expire_in: 2 week
367
368
369 #################################
370 # x86_64-linux-fedora27
371 #################################
372
373 release-x86_64-linux-fedora27:
374   extends: .validate-linux
375   stage: full-build
376   image: "registry.gitlab.haskell.org/ghc/ci-images/x86_64-linux-fedora27:$DOCKER_REV"
377   variables:
378     TEST_ENV: "x86_64-linux-fedora27"
379   only:
380     - tags
381   cache:
382     key: linux-x86_64-fedora27
383   artifacts:
384     when: always
385     expire_in: 2 week
386
387 ############################################################
388 # Validation via Pipelines (Windows)
389 ############################################################
390
391 .validate-windows:
392   before_script:
393     - git clean -xdf
394     - git submodule foreach git clean -xdf
395
396     # Use a local temporary directory to ensure that concurrent builds don't
397     # interfere with one another
398     - |
399       mkdir tmp
400       set TMP=%cd%\tmp
401       set TEMP=%cd%\tmp
402
403     - set PATH=C:\msys64\usr\bin;%PATH%
404     - python .gitlab/fix-submodules.py
405     - git submodule sync --recursive
406     - git submodule update --init --recursive
407     - git checkout .gitmodules
408     - "git fetch https://gitlab.haskell.org/ghc/ghc-performance-notes.git refs/notes/perf:refs/notes/perf || true"
409     - bash .gitlab/win32-init.sh
410   after_script:
411     - rd /s /q tmp
412     - robocopy /np /nfl /ndl /e "%APPDATA%\cabal" cabal-cache
413     - bash -c 'make clean || true'
414   dependencies: []
415   variables:
416     FORCE_SYMLINKS: 1
417   cache:
418     paths:
419       - cabal-cache
420       - ghc-8.6.2
421       - ghc-tarballs
422
423 validate-x86_64-windows-hadrian:
424   extends: .validate-windows
425   stage: full-build
426   variables:
427     GHC_VERSION: "8.6.2"
428     LANG: "en_US.UTF-8"
429   script:
430     - |
431       set MSYSTEM=MINGW64
432       python boot
433       bash -c './configure --enable-tarballs-autodownload GHC=`pwd`/toolchain/bin/ghc HAPPY=`pwd`/toolchain/bin/happy ALEX=`pwd`/toolchain/bin/alex'
434     - bash -c "PATH=`pwd`/toolchain/bin:$PATH hadrian/build.cabal.sh -j`mk/detect-cpu-count.sh` --flavour=Quick --docs=no-sphinx binary-dist"
435     - mv _build/bindist/ghc*.tar.xz ghc.tar.xz
436     # FIXME: Testsuite disabled due to #16156.
437     # - bash -c 'make V=0 test THREADS=`mk/detect-cpu-count.sh` JUNIT_FILE=../../junit.xml'
438   cache:
439     key: x86_64-windows
440   tags:
441     - x86_64-windows
442   artifacts:
443     when: always
444     paths:
445       - ghc.tar.xz
446
447 validate-x86_64-windows:
448   extends: .validate-windows
449   stage: full-build
450   # due to #16084
451   allow_failure: true
452   variables:
453     GHC_VERSION: "8.6.2"
454     LANG: "en_US.UTF-8"
455   script:
456     - |
457       set MSYSTEM=MINGW64
458       python boot
459       bash -c './configure --enable-tarballs-autodownload GHC=`pwd`/toolchain/bin/ghc HAPPY=`pwd`/toolchain/bin/happy ALEX=`pwd`/toolchain/bin/alex'
460     - bash -c "echo include mk/flavours/quick.mk > mk/build.mk"
461     - bash -c "PATH=`pwd`/toolchain/bin:$PATH make -j`mk/detect-cpu-count.sh`"
462     - |
463       bash -c "make binary-dist TAR_COMP_OPTS=-1"
464       mv ghc-*.tar.xz ghc.tar.xz
465     - bash -c 'make V=0 test THREADS=`mk/detect-cpu-count.sh` JUNIT_FILE=../../junit.xml'
466   cache:
467     key: x86_64-windows
468   tags:
469     - x86_64-windows
470   artifacts:
471     when: always
472     reports:
473       junit: junit.xml
474     paths:
475       - ghc.tar.xz
476       - junit.xml
477
478 ############################################################
479 # Cleanup
480 ############################################################
481
482 # Note [Cleaning up after shell executor]
483 # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
484 #
485 # As noted in [1], gitlab-runner's shell executor doesn't clean up its working
486 # directory after builds. Unfortunately, we are forced to use the shell executor
487 # on Windows. To avoid running out of disk space we add a stage at the end of
488 # the build to remove the \GitLabRunner\builds directory. Since we only run a
489 # single build at a time on Windows this should be safe.
490 #
491 # [1] https://gitlab.com/gitlab-org/gitlab-runner/issues/3856
492
493 # See Note [Cleanup after shell executor]
494 cleanup-windows:
495   stage: cleanup
496   tags:
497     - x86_64-windows
498   when: always
499   dependencies: []
500   before_script:
501     - echo "Time to clean up"
502   script:
503     - echo "Let's go"
504   after_script:
505     - set "BUILD_DIR=%CI_PROJECT_DIR%"
506     - set "BUILD_DIR=%BUILD_DIR:/=\%"
507     - echo "Cleaning %BUILD_DIR%"
508     - cd \GitLabRunner
509     # This is way more complicated than it should be:
510     # See https://stackoverflow.com/questions/1965787
511     - del %BUILD_DIR%\* /F /Q
512     - for /d %%p in (%BUILD_DIR%\*) do rd /Q /S "%%p"
513     - exit /b 0
514
515 # See Note [Cleanup after shell executor]
516 cleanup-darwin:
517   stage: cleanup
518   tags:
519     - x86_64-darwin
520   when: always
521   dependencies: []
522   before_script:
523     - echo "Time to clean up"
524   script:
525     - echo "Let's go"
526   after_script:
527     - BUILD_DIR=$CI_PROJECT_DIR
528     - echo "Cleaning $BUILD_DIR"
529     - cd $HOME
530     - rm -Rf $BUILD_DIR/*
531     - exit 0
532