Hadrian: Allow passing CABFLAGS into build.cabal.sh
[ghc.git] / .gitlab-ci.yml
1 variables:
2   GIT_SSL_NO_VERIFY: "1"
3
4   # Commit of ghc/ci-images repository from which to pull Docker images
5   DOCKER_REV: 6d19c3adc1f5c28c82aed8c5b1ac40931ac60f3f
6
7 before_script:
8   - python3 .gitlab/fix-submodules.py
9   - git submodule sync --recursive
10   - git submodule update --init --recursive
11   - git checkout .gitmodules
12   - "git fetch https://gitlab.haskell.org/ghc/ghc-performance-notes.git refs/notes/perf:refs/notes/perf || true"
13
14 stages:
15   - lint
16   - build
17   - full-build
18   - cleanup # See Note [Cleanup on Windows]
19
20 ############################################################
21 # Runner Tags
22 ############################################################
23 #
24 # * x86_64-linux: Any Docker-capable x86_64 Linux machine
25 # * aarch64-linux: Any Docker-capable AArch64 Linux machine
26 # * x86_64-windows: A x86_64 Windows machine
27 # * lint: Any Docker-capable x86_64 Linux machine; distinct from
28 #         x86_64-linux to ensure low-latency availability.
29 #
30
31
32 ############################################################
33 # Linting
34 ############################################################
35
36 ghc-linters:
37   stage: lint
38   image: "registry.gitlab.haskell.org/ghc/ci-images/linters:$DOCKER_REV"
39   script:
40     - |
41       if [ -n "$CI_MERGE_REQUEST_ID" ]; then
42         base="$(git merge-base $CI_MERGE_REQUEST_BRANCH_NAME HEAD)"
43         validate-commit-msg .git $(git rev-list $base..$CI_COMMIT_SHA)
44         submodchecker .git $(git rev-list $base..$CI_COMMIT_SHA)
45         validate-whitespace .git $(git rev-list $base..$CI_COMMIT_SHA)
46         .gitlab/linters/check-makefiles.py $base $CI_COMMIT_SHA
47         .gitlab/linters/check-cpp.py $base $CI_COMMIT_SHA
48       fi
49   dependencies: []
50   tags:
51     - lint
52
53 ############################################################
54 # Validation via Pipelines (hadrian)
55 ############################################################
56
57 .validate-hadrian:
58   allow_failure: true
59   script:
60     - cabal update
61     - git clean -xdf && git submodule foreach git clean -xdf
62     - bash .circleci/prepare-system.sh
63     - if [[ -d ./cabal-cache ]]; then cp -R ./.cabal-cache ~/.cabal-cache; fi
64     - ./boot
65     - ./configure $CONFIGURE_ARGS
66     - hadrian/build.cabal.sh -j`mk/detect-cpu-count.sh`
67     - hadrian/build.cabal.sh -j`mk/detect-cpu-count.sh` --docs=no-sphinx docs
68   cache:
69     key: hadrian
70     paths:
71       - cabal-cache
72
73 validate-x86_64-linux-deb8-hadrian:
74   extends: .validate-hadrian
75   stage: build
76   image: "registry.gitlab.haskell.org/ghc/ci-images/x86_64-linux-deb8:$DOCKER_REV"
77   before_script:
78     # workaround for docker permissions
79     - sudo chown ghc:ghc -R .
80     - python3 .gitlab/fix-submodules.py
81     - git submodule sync --recursive
82     - git submodule update --init --recursive
83     - git checkout .gitmodules
84     - "git fetch https://gitlab.haskell.org/ghc/ghc-performance-notes.git refs/notes/perf:refs/notes/perf || true"
85   tags:
86     - x86_64-linux
87
88 ############################################################
89 # Validation via Pipelines (make)
90 ############################################################
91
92 .validate:
93   variables:
94     TEST_TYPE: test
95   before_script:
96     - git clean -xdf && git submodule foreach git clean -xdf
97   script:
98     - ./boot
99     - ./configure $CONFIGURE_ARGS
100     - |
101       THREADS=`mk/detect-cpu-count.sh`
102       make V=0 -j$THREADS WERROR=-Werror 
103     - |
104       make binary-dist TAR_COMP_OPTS="-1"
105       mv ghc-*.tar.xz ghc.tar.xz
106     - |
107       # Prepare to push git notes.
108       METRICS_FILE=$(mktemp)
109       git config user.email "ben+ghc-ci@smart-cactus.org"
110       git config user.name "GHC GitLab CI"
111     - |
112       THREADS=`mk/detect-cpu-count.sh`
113       make $TEST_TYPE THREADS=$THREADS JUNIT_FILE=../../junit.xml METRICS_FILE=$METRICS_FILE
114     - |
115       # Push git notes.
116       METRICS_FILE=$METRICS_FILE .gitlab/push-test-metrics.sh
117   dependencies: []
118   artifacts:
119     reports:
120       junit: junit.xml
121     expire_in: 2 week
122     paths:
123       - ghc.tar.xz
124       - junit.xml
125
126 #################################
127 # x86_64-darwin
128 #################################
129
130 validate-x86_64-darwin:
131   extends: .validate
132   stage: full-build
133   tags:
134     - x86_64-darwin
135   variables:
136     GHC_VERSION: 8.6.3
137     MACOSX_DEPLOYMENT_TARGET: "10.7"
138     # Only Sierra and onwards supports clock_gettime. See #12858
139     ac_cv_func_clock_gettime: "no"
140     LANG: "en_US.UTF-8"
141     CONFIGURE_ARGS: --with-intree-gmp
142     TEST_ENV: "x86_64-darwin"
143   before_script:
144     - git clean -xdf && git submodule foreach git clean -xdf
145     - python3 .gitlab/fix-submodules.py
146     - git submodule sync --recursive
147     - git submodule update --init --recursive
148     - git checkout .gitmodules
149     - "git fetch https://gitlab.haskell.org/ghc/ghc-performance-notes.git refs/notes/perf:refs/notes/perf || true"
150
151     - bash .gitlab/darwin-init.sh
152     - PATH="`pwd`/toolchain/bin:$PATH"
153   after_script:
154     - cp -Rf $HOME/.cabal cabal-cache
155   artifacts:
156     when: always
157     expire_in: 2 week
158   cache:
159     key: darwin
160     paths:
161       - cabal-cache
162       - toolchain
163
164 .validate-linux:
165   extends: .validate
166   tags:
167     - x86_64-linux
168   before_script:
169     - git clean -xdf && git submodule foreach git clean -xdf
170     - python3 .gitlab/fix-submodules.py
171     - git submodule sync --recursive
172     - git submodule update --init --recursive
173     - git checkout .gitmodules
174     - "git fetch https://gitlab.haskell.org/ghc/ghc-performance-notes.git refs/notes/perf:refs/notes/perf || true"
175
176     - bash .circleci/prepare-system.sh
177     # workaround for docker permissions
178     - sudo chown ghc:ghc -R .
179   after_script:
180     - cp -Rf $HOME/.cabal cabal-cache
181   cache:
182     key: linux
183     paths:
184       - cabal-cache
185       - toolchain
186
187 #################################
188 # aarch64-linux-deb9
189 #################################
190
191 .build-aarch64-linux-deb9:
192   extends: .validate-linux
193   stage: full-build
194   image: "registry.gitlab.haskell.org/ghc/ci-images/aarch64-linux-deb9:$DOCKER_REV"
195   allow_failure: true
196   variables:
197     TEST_ENV: "aarch64-linux-deb9"
198   cache:
199     key: linux-aarch64-deb9
200   tags:
201     - aarch64-linux
202
203 validate-aarch64-linux-deb9:
204   extends: .build-aarch64-linux-deb9
205   artifacts:
206     when: always
207     expire_in: 2 week
208
209 nightly-aarch64-linux-deb9:
210   extends: .build-aarch64-linux-deb9
211   artifacts:
212     expire_in: 2 year
213   variables:
214     TEST_TYPE: slowtest
215   only:
216     variables:
217       - $NIGHTLY
218
219 #################################
220 # i386-linux-deb9
221 #################################
222
223 .build-i386-linux-deb9:
224   extends: .validate-linux
225   stage: full-build
226   image: "registry.gitlab.haskell.org/ghc/ci-images/i386-linux-deb9:$DOCKER_REV"
227   variables:
228     TEST_ENV: "i386-linux-deb9"
229   cache:
230     key: linux-i386-deb9
231
232 validate-i386-linux-deb9:
233   extends: .build-i386-linux-deb9
234   artifacts:
235     when: always
236     expire_in: 2 week
237
238 nightly-i386-linux-deb9:
239   extends: .build-i386-linux-deb9
240   variables:
241     TEST_TYPE: slowtest
242   artifacts:
243     when: always
244     expire_in: 2 week
245   only:
246     variables:
247       - $NIGHTLY
248
249 #################################
250 # x86_64-linux-deb9
251 #################################
252
253 .build-x86_64-linux-deb9:
254   extends: .validate-linux
255   stage: build
256   image: "registry.gitlab.haskell.org/ghc/ci-images/x86_64-linux-deb9:$DOCKER_REV"
257   variables:
258     TEST_ENV: "x86_64-linux-deb9"
259   cache:
260     key: linux-x86_64-deb9
261
262 validate-x86_64-linux-deb9:
263   extends: .build-x86_64-linux-deb9
264   artifacts:
265     when: always
266     expire_in: 2 week
267
268 nightly-x86_64-linux-deb9:
269   extends: .build-x86_64-linux-deb9
270   artifacts:
271     expire_in: 2 year
272   variables:
273     TEST_TYPE: slowtest
274   only:
275     variables:
276       - $NIGHTLY
277
278 # N.B. Has DEBUG assertions enabled in stage2
279 validate-x86_64-linux-deb9-debug:
280   extends: .build-x86_64-linux-deb9
281   variables:
282     BUILD_FLAVOUR: validate
283     TEST_ENV: "x86_64-linux-deb9-debug"
284
285 validate-x86_64-linux-deb9-llvm:
286   extends: .build-x86_64-linux-deb9
287   stage: full-build
288   variables:
289     BUILD_FLAVOUR: perf-llvm
290     TEST_ENV: "x86_64-linux-deb9-llvm"
291
292 validate-x86_64-linux-deb9-integer-simple:
293   extends: .build-x86_64-linux-deb9
294   stage: full-build
295   variables:
296     INTEGER_LIBRARY: integer-simple
297     TEST_ENV: "x86_64-linux-deb9-integer-simple"
298
299 nightly-x86_64-linux-deb9-integer-simple:
300   extends: .build-x86_64-linux-deb9
301   stage: full-build
302   variables:
303     INTEGER_LIBRARY: integer-simple
304     TEST_ENV: "x86_64-linux-deb9-integer-simple"
305     TEST_TYPE: slowtest
306   artifacts:
307     expire_in: 2 year
308   only:
309     variables:
310       - $NIGHTLY
311
312 validate-x86_64-linux-deb9-unreg:
313   extends: .build-x86_64-linux-deb9
314   stage: full-build
315   variables:
316     CONFIGURE_ARGS: --enable-unregisterised
317     TEST_ENV: "x86_64-linux-deb9-unreg"
318
319
320 #################################
321 # x86_64-linux-deb8
322 #################################
323
324 release-x86_64-linux-deb8:
325   extends: .validate-linux
326   stage: full-build
327   image: "registry.gitlab.haskell.org/ghc/ci-images/x86_64-linux-deb8:$DOCKER_REV"
328   variables:
329     TEST_ENV: "x86_64-linux-deb8"
330   only:
331     - tags
332   cache:
333     key: linux-x86_64-deb8
334   artifacts:
335     when: always
336     expire_in: 2 week
337
338
339 #################################
340 # x86_64-linux-fedora27
341 #################################
342
343 release-x86_64-linux-fedora27:
344   extends: .validate-linux
345   stage: full-build
346   image: "registry.gitlab.haskell.org/ghc/ci-images/x86_64-linux-fedora27:$DOCKER_REV"
347   variables:
348     TEST_ENV: "x86_64-linux-fedora27"
349   only:
350     - tags
351   cache:
352     key: linux-x86_64-fedora27
353   artifacts:
354     when: always
355     expire_in: 2 week
356
357 ############################################################
358 # Validation via Pipelines (Windows)
359 ############################################################
360
361 .validate-windows:
362   before_script:
363     - git clean -xdf
364     - git submodule foreach git clean -xdf
365
366     # Use a local temporary directory to ensure that concurrent builds don't
367     # interfere with one another
368     - |
369       mkdir tmp
370       set TMP=%cd%\tmp
371       set TEMP=%cd%\tmp
372
373     - set PATH=C:\msys64\usr\bin;%PATH%
374     - python .gitlab/fix-submodules.py
375     - git submodule sync --recursive
376     - git submodule update --init --recursive
377     - git checkout .gitmodules
378     - "git fetch https://gitlab.haskell.org/ghc/ghc-performance-notes.git refs/notes/perf:refs/notes/perf || true"
379     - bash .gitlab/win32-init.sh
380   after_script:
381     - rd /s /q tmp
382     - robocopy /np /nfl /ndl /e "%APPDATA%\cabal" cabal-cache
383     - bash -c 'make clean || true'
384   dependencies: []
385   variables:
386     FORCE_SYMLINKS: 1
387   cache:
388     paths:
389       - cabal-cache
390       - ghc-8.6.2
391       - ghc-tarballs
392
393 validate-x86_64-windows-hadrian:
394   extends: .validate-windows
395   stage: full-build
396   variables:
397     GHC_VERSION: "8.6.2"
398     LANG: "en_US.UTF-8"
399   script:
400     - |
401       set MSYSTEM=MINGW64
402       python boot
403       bash -c './configure --enable-tarballs-autodownload GHC=`pwd`/toolchain/bin/ghc HAPPY=`pwd`/toolchain/bin/happy ALEX=`pwd`/toolchain/bin/alex'
404     - bash -c "PATH=`pwd`/toolchain/bin:$PATH hadrian/build.cabal.sh -j`mk/detect-cpu-count.sh` --flavour=Quick"
405     - bash -c "PATH=`pwd`/toolchain/bin:$PATH hadrian/build.cabal.sh -j`mk/detect-cpu-count.sh` --flavour=Quick --docs=no-sphinx docs
406     # FIXME: Bindist disabled due to #16073
407     #- bash -c "PATH=`pwd`/toolchain/bin:$PATH hadrian/build.cabal.sh binary-dist"
408     # FIXME: Testsuite disabled due to #16156.
409     #- bash -c 'make V=0 test THREADS=`mk/detect-cpu-count.sh` JUNIT_FILE=../../junit.xml'
410   cache:
411     key: x86_64-windows
412   tags:
413     - x86_64-windows
414
415 validate-x86_64-windows:
416   extends: .validate-windows
417   stage: full-build
418   # due to #16084
419   allow_failure: true
420   variables:
421     GHC_VERSION: "8.6.2"
422     LANG: "en_US.UTF-8"
423   script:
424     - |
425       set MSYSTEM=MINGW64
426       python boot
427       bash -c './configure --enable-tarballs-autodownload GHC=`pwd`/toolchain/bin/ghc HAPPY=`pwd`/toolchain/bin/happy ALEX=`pwd`/toolchain/bin/alex'
428     - bash -c "echo include mk/flavours/quick.mk > mk/build.mk"
429     - bash -c "PATH=`pwd`/toolchain/bin:$PATH make -j`mk/detect-cpu-count.sh`"
430     - |
431       bash -c "make binary-dist TAR_COMP_OPTS=-1"
432       mv ghc-*.tar.xz ghc.tar.xz
433     - bash -c 'make V=0 test THREADS=`mk/detect-cpu-count.sh` JUNIT_FILE=../../junit.xml'
434   cache:
435     key: x86_64-windows
436   tags:
437     - x86_64-windows
438   artifacts:
439     when: always
440     reports:
441       junit: junit.xml
442     paths:
443       - ghc.tar.xz
444       - junit.xml
445
446 ############################################################
447 # Cleanup
448 ############################################################
449
450 # Note [Cleaning up after shell executor]
451 # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
452 #
453 # As noted in [1], gitlab-runner's shell executor doesn't clean up its working
454 # directory after builds. Unfortunately, we are forced to use the shell executor
455 # on Windows. To avoid running out of disk space we add a stage at the end of
456 # the build to remove the \GitLabRunner\builds directory. Since we only run a
457 # single build at a time on Windows this should be safe.
458 #
459 # [1] https://gitlab.com/gitlab-org/gitlab-runner/issues/3856
460
461 # See Note [Cleanup after shell executor]
462 cleanup-windows:
463   stage: cleanup
464   tags:
465     - x86_64-windows
466   when: always
467   dependencies: []
468   before_script:
469     - echo "Time to clean up"
470   script:
471     - echo "Let's go"
472   after_script:
473     - set "BUILD_DIR=%CI_PROJECT_DIR%"
474     - set "BUILD_DIR=%BUILD_DIR:/=\%"
475     - echo "Cleaning %BUILD_DIR%"
476     - cd \GitLabRunner
477     # This is way more complicated than it should be:
478     # See https://stackoverflow.com/questions/1965787
479     - del %BUILD_DIR%\* /F /Q
480     - for /d %%p in (%BUILD_DIR%\*) do rd /Q /S "%%p"
481     - exit /b 0
482
483 # See Note [Cleanup after shell executor]
484 cleanup-darwin:
485   stage: cleanup
486   tags:
487     - x86_64-darwin
488   when: always
489   dependencies: []
490   before_script:
491     - echo "Time to clean up"
492   script:
493     - echo "Let's go"
494   after_script:
495     - BUILD_DIR=$CI_PROJECT_DIR
496     - echo "Cleaning $BUILD_DIR"
497     - cd $HOME
498     - rm -Rf $BUILD_DIR/*
499     - exit 0
500