Hadrian: track mingw, ship it in bindists, more robust install script
[ghc.git] / .gitlab-ci.yml
1 variables:
2   GIT_SSL_NO_VERIFY: "1"
3
4   # Commit of ghc/ci-images repository from which to pull Docker images
5   DOCKER_REV: 6d19c3adc1f5c28c82aed8c5b1ac40931ac60f3f
6
7 before_script:
8   - python3 .gitlab/fix-submodules.py
9   - git submodule sync --recursive
10   - git submodule update --init --recursive
11   - git checkout .gitmodules
12   - "git fetch https://gitlab.haskell.org/ghc/ghc-performance-notes.git refs/notes/perf:refs/notes/perf || true"
13
14 stages:
15   - lint
16   - build
17   - full-build
18   - cleanup # See Note [Cleanup on Windows]
19
20 ############################################################
21 # Runner Tags
22 ############################################################
23 #
24 # * x86_64-linux: Any Docker-capable x86_64 Linux machine
25 # * aarch64-linux: Any Docker-capable AArch64 Linux machine
26 # * x86_64-windows: A x86_64 Windows machine
27 # * lint: Any Docker-capable x86_64 Linux machine; distinct from
28 #         x86_64-linux to ensure low-latency availability.
29 #
30
31
32 ############################################################
33 # Linting
34 ############################################################
35
36 ghc-linters:
37   stage: lint
38   image: "registry.gitlab.haskell.org/ghc/ci-images/linters:$DOCKER_REV"
39   script:
40     - |
41       if [ -n "$CI_MERGE_REQUEST_ID" ]; then
42         base="$(git merge-base $CI_MERGE_REQUEST_BRANCH_NAME HEAD)"
43         validate-commit-msg .git $(git rev-list $base..$CI_COMMIT_SHA)
44         submodchecker .git $(git rev-list $base..$CI_COMMIT_SHA)
45         validate-whitespace .git $(git rev-list $base..$CI_COMMIT_SHA)
46         .gitlab/linters/check-makefiles.py $base $CI_COMMIT_SHA
47         .gitlab/linters/check-cpp.py $base $CI_COMMIT_SHA
48       fi
49   dependencies: []
50   tags:
51     - lint
52
53 ############################################################
54 # Validation via Pipelines (hadrian)
55 ############################################################
56
57 .validate-hadrian:
58   allow_failure: true
59   script:
60     - cabal update
61     - git clean -xdf && git submodule foreach git clean -xdf
62     - bash .circleci/prepare-system.sh
63     - if [[ -d ./cabal-cache ]]; then cp -R ./.cabal-cache ~/.cabal-cache; fi
64     - ./boot
65     - ./configure $CONFIGURE_ARGS
66     - hadrian/build.cabal.sh -j`mk/detect-cpu-count.sh`
67     - hadrian/build.cabal.sh -j`mk/detect-cpu-count.sh` --docs=no-sphinx docs
68   cache:
69     key: hadrian
70     paths:
71       - cabal-cache
72
73 validate-x86_64-linux-deb8-hadrian:
74   extends: .validate-hadrian
75   stage: build
76   image: "registry.gitlab.haskell.org/ghc/ci-images/x86_64-linux-deb8:$DOCKER_REV"
77   before_script:
78     # workaround for docker permissions
79     - sudo chown ghc:ghc -R .
80     - python3 .gitlab/fix-submodules.py
81     - git submodule sync --recursive
82     - git submodule update --init --recursive
83     - git checkout .gitmodules
84     - "git fetch https://gitlab.haskell.org/ghc/ghc-performance-notes.git refs/notes/perf:refs/notes/perf || true"
85   tags:
86     - x86_64-linux
87
88 ############################################################
89 # Validation via Pipelines (make)
90 ############################################################
91
92 .validate:
93   variables:
94     TEST_TYPE: test
95   before_script:
96     - git clean -xdf && git submodule foreach git clean -xdf
97   script:
98     - ./boot
99     - ./configure $CONFIGURE_ARGS
100     - |
101       THREADS=`mk/detect-cpu-count.sh`
102       make V=0 -j$THREADS WERROR=-Werror 
103     - |
104       make binary-dist TAR_COMP_OPTS="-1"
105       mv ghc-*.tar.xz ghc.tar.xz
106     - |
107       # Prepare to push git notes.
108       METRICS_FILE=$(mktemp)
109       git config user.email "ben+ghc-ci@smart-cactus.org"
110       git config user.name "GHC GitLab CI"
111     - |
112       THREADS=`mk/detect-cpu-count.sh`
113       make $TEST_TYPE THREADS=$THREADS JUNIT_FILE=../../junit.xml METRICS_FILE=$METRICS_FILE
114     - |
115       # Push git notes.
116       METRICS_FILE=$METRICS_FILE .gitlab/push-test-metrics.sh
117   dependencies: []
118   artifacts:
119     reports:
120       junit: junit.xml
121     expire_in: 2 week
122     paths:
123       - ghc.tar.xz
124       - junit.xml
125
126 #################################
127 # x86_64-darwin
128 #################################
129
130 validate-x86_64-darwin:
131   extends: .validate
132   stage: full-build
133   tags:
134     - x86_64-darwin
135   variables:
136     GHC_VERSION: 8.6.3
137     MACOSX_DEPLOYMENT_TARGET: "10.7"
138     # Only Sierra and onwards supports clock_gettime. See #12858
139     ac_cv_func_clock_gettime: "no"
140     LANG: "en_US.UTF-8"
141     CONFIGURE_ARGS: --with-intree-gmp
142     TEST_ENV: "x86_64-darwin"
143   before_script:
144     - git clean -xdf && git submodule foreach git clean -xdf
145     - python3 .gitlab/fix-submodules.py
146     - git submodule sync --recursive
147     - git submodule update --init --recursive
148     - git checkout .gitmodules
149     - "git fetch https://gitlab.haskell.org/ghc/ghc-performance-notes.git refs/notes/perf:refs/notes/perf || true"
150
151     - bash .gitlab/darwin-init.sh
152     - PATH="`pwd`/toolchain/bin:$PATH"
153   after_script:
154     - cp -Rf $HOME/.cabal cabal-cache
155   artifacts:
156     when: always
157     expire_in: 2 week
158   cache:
159     key: darwin
160     paths:
161       - cabal-cache
162       - toolchain
163
164 .validate-linux:
165   extends: .validate
166   tags:
167     - x86_64-linux
168   before_script:
169     - git clean -xdf && git submodule foreach git clean -xdf
170     - python3 .gitlab/fix-submodules.py
171     - git submodule sync --recursive
172     - git submodule update --init --recursive
173     - git checkout .gitmodules
174     - "git fetch https://gitlab.haskell.org/ghc/ghc-performance-notes.git refs/notes/perf:refs/notes/perf || true"
175
176     - bash .circleci/prepare-system.sh
177     # workaround for docker permissions
178     - sudo chown ghc:ghc -R .
179   after_script:
180     - cp -Rf $HOME/.cabal cabal-cache
181   cache:
182     key: linux
183     paths:
184       - cabal-cache
185       - toolchain
186
187 #################################
188 # aarch64-linux-deb9
189 #################################
190
191 .build-aarch64-linux-deb9:
192   extends: .validate-linux
193   stage: full-build
194   image: "registry.gitlab.haskell.org/ghc/ci-images/aarch64-linux-deb9:$DOCKER_REV"
195   allow_failure: true
196   variables:
197     TEST_ENV: "aarch64-linux-deb9"
198   cache:
199     key: linux-aarch64-deb9
200   tags:
201     - aarch64-linux
202
203 validate-aarch64-linux-deb9:
204   extends: .build-aarch64-linux-deb9
205   artifacts:
206     when: always
207     expire_in: 2 week
208
209 nightly-aarch64-linux-deb9:
210   extends: .build-aarch64-linux-deb9
211   artifacts:
212     expire_in: 2 year
213   variables:
214     TEST_TYPE: slowtest
215   only:
216     variables:
217       - $NIGHTLY
218
219 #################################
220 # i386-linux-deb9
221 #################################
222
223 .build-i386-linux-deb9:
224   extends: .validate-linux
225   stage: full-build
226   image: "registry.gitlab.haskell.org/ghc/ci-images/i386-linux-deb9:$DOCKER_REV"
227   allow_failure: true
228   variables:
229     TEST_ENV: "i386-linux-deb9"
230   cache:
231     key: linux-i386-deb9
232
233 validate-i386-linux-deb9:
234   extends: .build-i386-linux-deb9
235   artifacts:
236     when: always
237     expire_in: 2 week
238
239 nightly-i386-linux-deb9:
240   extends: .build-i386-linux-deb9
241   variables:
242     TEST_TYPE: slowtest
243   artifacts:
244     when: always
245     expire_in: 2 week
246   only:
247     variables:
248       - $NIGHTLY
249
250 #################################
251 # x86_64-linux-deb9
252 #################################
253
254 .build-x86_64-linux-deb9:
255   extends: .validate-linux
256   stage: build
257   image: "registry.gitlab.haskell.org/ghc/ci-images/x86_64-linux-deb9:$DOCKER_REV"
258   variables:
259     TEST_ENV: "x86_64-linux-deb9"
260   cache:
261     key: linux-x86_64-deb9
262
263 validate-x86_64-linux-deb9:
264   extends: .build-x86_64-linux-deb9
265   artifacts:
266     when: always
267     expire_in: 2 week
268
269 nightly-x86_64-linux-deb9:
270   extends: .build-x86_64-linux-deb9
271   artifacts:
272     expire_in: 2 year
273   variables:
274     TEST_TYPE: slowtest
275   only:
276     variables:
277       - $NIGHTLY
278
279 # N.B. Has DEBUG assertions enabled in stage2
280 validate-x86_64-linux-deb9-debug:
281   extends: .build-x86_64-linux-deb9
282   variables:
283     BUILD_FLAVOUR: validate
284     TEST_ENV: "x86_64-linux-deb9-debug"
285
286 validate-x86_64-linux-deb9-llvm:
287   extends: .build-x86_64-linux-deb9
288   stage: full-build
289   variables:
290     BUILD_FLAVOUR: perf-llvm
291     TEST_ENV: "x86_64-linux-deb9-llvm"
292
293 validate-x86_64-linux-deb9-integer-simple:
294   extends: .build-x86_64-linux-deb9
295   stage: full-build
296   variables:
297     INTEGER_LIBRARY: integer-simple
298     TEST_ENV: "x86_64-linux-deb9-integer-simple"
299
300 nightly-x86_64-linux-deb9-integer-simple:
301   extends: .build-x86_64-linux-deb9
302   stage: full-build
303   variables:
304     INTEGER_LIBRARY: integer-simple
305     TEST_ENV: "x86_64-linux-deb9-integer-simple"
306     TEST_TYPE: slowtest
307   artifacts:
308     expire_in: 2 year
309   only:
310     variables:
311       - $NIGHTLY
312
313 validate-x86_64-linux-deb9-unreg:
314   extends: .build-x86_64-linux-deb9
315   stage: full-build
316   variables:
317     CONFIGURE_ARGS: --enable-unregisterised
318     TEST_ENV: "x86_64-linux-deb9-unreg"
319
320
321 #################################
322 # x86_64-linux-deb8
323 #################################
324
325 release-x86_64-linux-deb8:
326   extends: .validate-linux
327   stage: full-build
328   image: "registry.gitlab.haskell.org/ghc/ci-images/x86_64-linux-deb8:$DOCKER_REV"
329   variables:
330     TEST_ENV: "x86_64-linux-deb8"
331   only:
332     - tags
333   cache:
334     key: linux-x86_64-deb8
335   artifacts:
336     when: always
337     expire_in: 2 week
338
339
340 #################################
341 # x86_64-linux-fedora27
342 #################################
343
344 release-x86_64-linux-fedora27:
345   extends: .validate-linux
346   stage: full-build
347   image: "registry.gitlab.haskell.org/ghc/ci-images/x86_64-linux-fedora27:$DOCKER_REV"
348   variables:
349     TEST_ENV: "x86_64-linux-fedora27"
350   only:
351     - tags
352   cache:
353     key: linux-x86_64-fedora27
354   artifacts:
355     when: always
356     expire_in: 2 week
357
358 ############################################################
359 # Validation via Pipelines (Windows)
360 ############################################################
361
362 .validate-windows:
363   before_script:
364     - git clean -xdf
365     - git submodule foreach git clean -xdf
366
367     # Use a local temporary directory to ensure that concurrent builds don't
368     # interfere with one another
369     - |
370       mkdir tmp
371       set TMP=%cd%\tmp
372       set TEMP=%cd%\tmp
373
374     - set PATH=C:\msys64\usr\bin;%PATH%
375     - python .gitlab/fix-submodules.py
376     - git submodule sync --recursive
377     - git submodule update --init --recursive
378     - git checkout .gitmodules
379     - "git fetch https://gitlab.haskell.org/ghc/ghc-performance-notes.git refs/notes/perf:refs/notes/perf || true"
380     - bash .gitlab/win32-init.sh
381   after_script:
382     - rd /s /q tmp
383     - robocopy /np /nfl /ndl /e "%APPDATA%\cabal" cabal-cache
384     - bash -c 'make clean || true'
385   dependencies: []
386   variables:
387     FORCE_SYMLINKS: 1
388   cache:
389     paths:
390       - cabal-cache
391       - ghc-8.6.2
392       - ghc-tarballs
393
394 validate-x86_64-windows-hadrian:
395   extends: .validate-windows
396   stage: full-build
397   variables:
398     GHC_VERSION: "8.6.2"
399     LANG: "en_US.UTF-8"
400   script:
401     - |
402       set MSYSTEM=MINGW64
403       python boot
404       bash -c './configure --enable-tarballs-autodownload GHC=`pwd`/toolchain/bin/ghc HAPPY=`pwd`/toolchain/bin/happy ALEX=`pwd`/toolchain/bin/alex'
405     - bash -c "PATH=`pwd`/toolchain/bin:$PATH hadrian/build.cabal.sh -j`mk/detect-cpu-count.sh` --flavour=Quick"
406     - bash -c "PATH=`pwd`/toolchain/bin:$PATH hadrian/build.cabal.sh -j`mk/detect-cpu-count.sh` --flavour=Quick --docs=no-sphinx docs
407     # FIXME: Bindist disabled due to #16073
408     #- bash -c "PATH=`pwd`/toolchain/bin:$PATH hadrian/build.cabal.sh binary-dist"
409     # FIXME: Testsuite disabled due to #16156.
410     #- bash -c 'make V=0 test THREADS=`mk/detect-cpu-count.sh` JUNIT_FILE=../../junit.xml'
411   cache:
412     key: x86_64-windows
413   tags:
414     - x86_64-windows
415
416 validate-x86_64-windows:
417   extends: .validate-windows
418   stage: full-build
419   # due to #16084
420   allow_failure: true
421   variables:
422     GHC_VERSION: "8.6.2"
423     LANG: "en_US.UTF-8"
424   script:
425     - |
426       set MSYSTEM=MINGW64
427       python boot
428       bash -c './configure --enable-tarballs-autodownload GHC=`pwd`/toolchain/bin/ghc HAPPY=`pwd`/toolchain/bin/happy ALEX=`pwd`/toolchain/bin/alex'
429     - bash -c "echo include mk/flavours/quick.mk > mk/build.mk"
430     - bash -c "PATH=`pwd`/toolchain/bin:$PATH make -j`mk/detect-cpu-count.sh`"
431     - |
432       bash -c "make binary-dist TAR_COMP_OPTS=-1"
433       mv ghc-*.tar.xz ghc.tar.xz
434     - bash -c 'make V=0 test THREADS=`mk/detect-cpu-count.sh` JUNIT_FILE=../../junit.xml'
435   cache:
436     key: x86_64-windows
437   tags:
438     - x86_64-windows
439   artifacts:
440     when: always
441     reports:
442       junit: junit.xml
443     paths:
444       - ghc.tar.xz
445       - junit.xml
446
447 ############################################################
448 # Cleanup
449 ############################################################
450
451 # Note [Cleaning up after shell executor]
452 # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
453 #
454 # As noted in [1], gitlab-runner's shell executor doesn't clean up its working
455 # directory after builds. Unfortunately, we are forced to use the shell executor
456 # on Windows. To avoid running out of disk space we add a stage at the end of
457 # the build to remove the \GitLabRunner\builds directory. Since we only run a
458 # single build at a time on Windows this should be safe.
459 #
460 # [1] https://gitlab.com/gitlab-org/gitlab-runner/issues/3856
461
462 # See Note [Cleanup after shell executor]
463 cleanup-windows:
464   stage: cleanup
465   tags:
466     - x86_64-windows
467   when: always
468   dependencies: []
469   before_script:
470     - echo "Time to clean up"
471   script:
472     - echo "Let's go"
473   after_script:
474     - set "BUILD_DIR=%CI_PROJECT_DIR%"
475     - set "BUILD_DIR=%BUILD_DIR:/=\%"
476     - echo "Cleaning %BUILD_DIR%"
477     - cd \GitLabRunner
478     # This is way more complicated than it should be:
479     # See https://stackoverflow.com/questions/1965787
480     - del %BUILD_DIR%\* /F /Q
481     - for /d %%p in (%BUILD_DIR%\*) do rd /Q /S "%%p"
482     - exit /b 0
483
484 # See Note [Cleanup after shell executor]
485 cleanup-darwin:
486   stage: cleanup
487   tags:
488     - x86_64-darwin
489   when: always
490   dependencies: []
491   before_script:
492     - echo "Time to clean up"
493   script:
494     - echo "Let's go"
495   after_script:
496     - BUILD_DIR=$CI_PROJECT_DIR
497     - echo "Cleaning $BUILD_DIR"
498     - cd $HOME
499     - rm -Rf $BUILD_DIR/*
500     - exit 0
501