3bca1febbfb3f5a659ca944e9a33720a375992c9
[ghc.git] / .gitlab-ci.yml
1 variables:
2   GIT_SSL_NO_VERIFY: "1"
3
4 before_script:
5   - python3 .gitlab/fix-submodules.py
6   - git submodule sync --recursive
7   - git submodule update --init --recursive
8   - git checkout .gitmodules
9   - "git fetch https://gitlab.haskell.org/ghc/ghc-performance-notes.git refs/notes/perf:refs/notes/perf || true"
10
11 stages:
12   - lint
13   - build
14   - full-build
15   - cleanup # See Note [Cleanup on Windows]
16
17 ############################################################
18 # Runner Tags
19 ############################################################
20 #
21 # * x86_64-linux: Any Docker-capable x86_64 Linux machine
22 # * aarch64-linux: Any Docker-capable AArch64 Linux machine
23 # * x86_64-windows: A x86_64 Windows machine
24 # * lint: Any Docker-capable x86_64 Linux machine; distinct from
25 #         x86_64-linux to ensure low-latency availability.
26 #
27
28
29 ############################################################
30 # Linting
31 ############################################################
32
33 ghc-linters:
34   stage: lint
35   image: ghcci/linters:0.1
36   script:
37     - |
38       if [ -n "$CI_MERGE_REQUEST_ID" ]; then
39         base="$(git merge-base $CI_MERGE_REQUEST_BRANCH_NAME HEAD)"
40         validate-commit-msg .git $(git rev-list $base..$CI_COMMIT_SHA)
41         submodchecker .git $(git rev-list $base..$CI_COMMIT_SHA)
42         validate-whitespace .git $(git rev-list $base..$CI_COMMIT_SHA)
43         .gitlab/linters/check-makefiles.py $base $CI_COMMIT_SHA
44         .gitlab/linters/check-cpp.py $base $CI_COMMIT_SHA
45       fi
46   dependencies: []
47   tags:
48     - lint
49
50 ############################################################
51 # Validation via Pipelines (hadrian)
52 ############################################################
53
54 .validate-hadrian:
55   allow_failure: true
56   script:
57     - cabal update
58     - git clean -xdf && git submodule foreach git clean -xdf
59     - bash .circleci/prepare-system.sh
60     - if [[ -d ./cabal-cache ]]; then cp -R ./.cabal-cache ~/.cabal-cache; fi
61     - ./boot
62     - ./configure $CONFIGURE_ARGS
63     - hadrian/build.cabal.sh -j`mk/detect-cpu-count.sh`
64     - hadrian/build.cabal.sh -j`mk/detect-cpu-count.sh` --docs=no-sphinx docs
65   cache:
66     key: hadrian
67     paths:
68       - cabal-cache
69
70 validate-x86_64-linux-deb8-hadrian:
71   extends: .validate-hadrian
72   stage: build
73   image: ghcci/x86_64-linux-deb8:0.1
74   before_script:
75     # workaround for docker permissions
76     - sudo chown ghc:ghc -R .
77     - python3 .gitlab/fix-submodules.py
78     - git submodule sync --recursive
79     - git submodule update --init --recursive
80     - git checkout .gitmodules
81     - "git fetch https://gitlab.haskell.org/ghc/ghc-performance-notes.git refs/notes/perf:refs/notes/perf || true"
82   tags:
83     - x86_64-linux
84
85 ############################################################
86 # Validation via Pipelines (make)
87 ############################################################
88
89 .validate:
90   variables:
91     TEST_TYPE: test
92   before_script:
93     - git clean -xdf && git submodule foreach git clean -xdf
94   script:
95     - ./boot
96     - ./configure $CONFIGURE_ARGS
97     - |
98       THREADS=`mk/detect-cpu-count.sh`
99       make V=0 -j$THREADS WERROR=-Werror 
100     - |
101       make binary-dist TAR_COMP_OPTS="-1"
102       mv ghc-*.tar.xz ghc.tar.xz
103     - |
104       # Prepare to push git notes.
105       METRICS_FILE=$(mktemp)
106       git config user.email "ben+ghc-ci@smart-cactus.org"
107       git config user.name "GHC GitLab CI"
108     - |
109       THREADS=`mk/detect-cpu-count.sh`
110       make $TEST_TYPE THREADS=$THREADS JUNIT_FILE=../../junit.xml METRICS_FILE=$METRICS_FILE
111     - |
112       # Push git notes.
113       METRICS_FILE=$METRICS_FILE .gitlab/push-test-metrics.sh
114   dependencies: []
115   artifacts:
116     reports:
117       junit: junit.xml
118     expire_in: 2 week
119     paths:
120       - ghc.tar.xz
121       - junit.xml
122
123 validate-x86_64-darwin:
124   extends: .validate
125   stage: full-build
126   tags:
127     - x86_64-darwin
128   variables:
129     GHC_VERSION: 8.6.3
130     MACOSX_DEPLOYMENT_TARGET: "10.7"
131     # Only Sierra and onwards supports clock_gettime. See #12858
132     ac_cv_func_clock_gettime: "no"
133     LANG: "en_US.UTF-8"
134     CONFIGURE_ARGS: --with-intree-gmp
135     TEST_ENV: "x86_64-darwin"
136   before_script:
137     - git clean -xdf && git submodule foreach git clean -xdf
138     - python3 .gitlab/fix-submodules.py
139     - git submodule sync --recursive
140     - git submodule update --init --recursive
141     - git checkout .gitmodules
142     - "git fetch https://gitlab.haskell.org/ghc/ghc-performance-notes.git refs/notes/perf:refs/notes/perf || true"
143
144     - bash .gitlab/darwin-init.sh
145     - PATH="`pwd`/toolchain/bin:$PATH"
146   after_script:
147     - cp -Rf $HOME/.cabal cabal-cache
148   artifacts:
149     when: always
150     expire_in: 2 week
151   cache:
152     key: darwin
153     paths:
154       - cabal-cache
155       - toolchain
156
157 .validate-linux:
158   extends: .validate
159   tags:
160     - x86_64-linux
161   before_script:
162     - git clean -xdf && git submodule foreach git clean -xdf
163     - python3 .gitlab/fix-submodules.py
164     - git submodule sync --recursive
165     - git submodule update --init --recursive
166     - git checkout .gitmodules
167     - "git fetch https://gitlab.haskell.org/ghc/ghc-performance-notes.git refs/notes/perf:refs/notes/perf || true"
168
169     - bash .circleci/prepare-system.sh
170     # workaround for docker permissions
171     - sudo chown ghc:ghc -R .
172   after_script:
173     - cp -Rf $HOME/.cabal cabal-cache
174   cache:
175     key: linux
176     paths:
177       - cabal-cache
178       - toolchain
179
180 validate-aarch64-linux-deb9:
181   extends: .validate-linux
182   stage: full-build
183   image: ghcci/aarch64-linux-deb9:0.1
184   allow_failure: true
185   variables:
186     TEST_ENV: "aarch64-linux-deb9"
187   artifacts:
188     when: always
189     expire_in: 2 week
190   cache:
191     key: linux-aarch64-deb9
192   tags:
193     - aarch64-linux
194
195 nightly-aarch64-linux-deb9:
196   extends: validate-aarch64-linux-deb9
197   stage: full-build
198   artifacts:
199     expire_in: 2 year
200   variables:
201     TEST_TYPE: slowtest
202   only:
203     variables:
204       - $NIGHTLY
205
206 validate-i386-linux-deb9:
207   extends: .validate-linux
208   stage: full-build
209   image: ghcci/i386-linux-deb9:0.1
210   allow_failure: true
211   variables:
212     TEST_ENV: "i386-linux-deb9"
213   artifacts:
214     when: always
215     expire_in: 2 week
216   cache:
217     key: linux-i386-deb9
218
219 nightly-i386-linux-deb9:
220   extends: .validate-linux
221   stage: full-build
222   image: ghcci/i386-linux-deb9:0.1
223   allow_failure: true
224   variables:
225     TEST_TYPE: slowtest
226     TEST_ENV: "i386-linux-deb9"
227   artifacts:
228     when: always
229     expire_in: 2 week
230   only:
231     variables:
232       - $NIGHTLY
233   cache:
234     key: linux-i386-deb9
235
236 # N.B. Has DEBUG assertions enabled in stage2
237 validate-x86_64-linux-deb9-debug:
238   extends: .validate-linux
239   stage: build
240   image: ghcci/x86_64-linux-deb9:0.2
241   variables:
242     BUILD_FLAVOUR: validate
243     TEST_ENV: "x86_64-linux-deb9-debug"
244   cache:
245     key: linux-x86_64-deb9
246
247 validate-x86_64-linux-deb9:
248   extends: .validate-linux
249   stage: build
250   image: ghcci/x86_64-linux-deb9:0.2
251   variables:
252     TEST_ENV: "x86_64-linux-deb9"
253   artifacts:
254     when: always
255     expire_in: 2 week
256   cache:
257     key: linux-x86_64-deb9
258
259 nightly-x86_64-linux-deb9:
260   extends: validate-x86_64-linux-deb9
261   stage: build
262   artifacts:
263     expire_in: 2 year
264   variables:
265     TEST_TYPE: slowtest
266   only:
267     variables:
268       - $NIGHTLY
269
270 validate-x86_64-linux-deb9-llvm:
271   extends: .validate-linux
272   stage: full-build
273   image: ghcci/x86_64-linux-deb9:0.2
274   variables:
275     BUILD_FLAVOUR: perf-llvm
276     TEST_ENV: "x86_64-linux-deb9-llvm"
277   cache:
278     key: linux-x86_64-deb9
279
280 release-x86_64-linux-deb8:
281   extends: .validate-linux
282   stage: full-build
283   image: ghcci/x86_64-linux-deb8:0.1
284   variables:
285     TEST_ENV: "x86_64-linux-deb8"
286   only:
287     - tags
288   cache:
289     key: linux-x86_64-deb8
290   artifacts:
291     when: always
292     expire_in: 2 week
293
294 release-x86_64-linux-fedora27:
295   extends: .validate-linux
296   stage: full-build
297   image: ghcci/x86_64-linux-fedora27:0.1
298   variables:
299     TEST_ENV: "x86_64-linux-fedora27"
300   only:
301     - tags
302   cache:
303     key: linux-x86_64-fedora27
304   artifacts:
305     when: always
306     expire_in: 2 week
307
308 validate-x86_64-linux-deb9-integer-simple:
309   extends: .validate-linux
310   stage: full-build
311   variables:
312     INTEGER_LIBRARY: integer-simple
313     TEST_ENV: "x86_64-linux-deb9-integer-simple"
314   image: ghcci/x86_64-linux-deb9:0.2
315   cache:
316     key: linux-x86_64-deb9
317
318 nightly-x86_64-linux-deb9-integer-simple:
319   extends: validate-x86_64-linux-deb9-integer-simple
320   stage: full-build
321   artifacts:
322     expire_in: 2 year
323   variables:
324     TEST_TYPE: slowtest
325   only:
326     variables:
327       - $NIGHTLY
328
329 validate-x86_64-linux-deb9-unreg:
330   extends: .validate-linux
331   stage: full-build
332   variables:
333     CONFIGURE_ARGS: --enable-unregisterised
334     TEST_ENV: "x86_64-linux-deb9-unreg"
335   image: ghcci/x86_64-linux-deb9:0.2
336   cache:
337     key: linux-x86_64-deb9
338
339 ############################################################
340 # Validation via Pipelines (Windows)
341 ############################################################
342
343 .validate-windows:
344   before_script:
345     - git clean -xdf
346     - git submodule foreach git clean -xdf
347
348     # Use a local temporary directory to ensure that concurrent builds don't
349     # interfere with one another
350     - |
351       mkdir tmp
352       set TMP=%cd%\tmp
353       set TEMP=%cd%\tmp
354
355     - set PATH=C:\msys64\usr\bin;%PATH%
356     - python .gitlab/fix-submodules.py
357     - git submodule sync --recursive
358     - git submodule update --init --recursive
359     - git checkout .gitmodules
360     - "git fetch https://gitlab.haskell.org/ghc/ghc-performance-notes.git refs/notes/perf:refs/notes/perf || true"
361     - bash .gitlab/win32-init.sh
362   after_script:
363     - rd /s /q tmp
364     - robocopy /np /nfl /ndl /e "%APPDATA%\cabal" cabal-cache
365     - bash -c 'make clean || true'
366   dependencies: []
367   variables:
368     FORCE_SYMLINKS: 1
369   cache:
370     paths:
371       - cabal-cache
372       - ghc-8.6.2
373       - ghc-tarballs
374
375 validate-x86_64-windows-hadrian:
376   extends: .validate-windows
377   stage: full-build
378   variables:
379     GHC_VERSION: "8.6.2"
380     LANG: "en_US.UTF-8"
381   script:
382     - |
383       set MSYSTEM=MINGW64
384       python boot
385       bash -c './configure --enable-tarballs-autodownload GHC=`pwd`/toolchain/bin/ghc HAPPY=`pwd`/toolchain/bin/happy ALEX=`pwd`/toolchain/bin/alex'
386       mkdir -p _build
387       cp -R inplace/mingw _build/mingw
388     - bash -c "PATH=`pwd`/toolchain/bin:$PATH hadrian/build.cabal.sh -j`mk/detect-cpu-count.sh` --flavour=Quick"
389     - bash -c "PATH=`pwd`/toolchain/bin:$PATH hadrian/build.cabal.sh -j`mk/detect-cpu-count.sh` --flavour=Quick --docs=no-sphinx docs
390     # FIXME: Bindist disabled due to #16073
391     #- bash -c "PATH=`pwd`/toolchain/bin:$PATH hadrian/build.cabal.sh binary-dist"
392     # FIXME: Testsuite disabled due to #16156.
393     #- bash -c 'make V=0 test THREADS=`mk/detect-cpu-count.sh` JUNIT_FILE=../../junit.xml'
394   cache:
395     key: x86_64-windows
396   tags:
397     - x86_64-windows
398
399 validate-x86_64-windows:
400   extends: .validate-windows
401   stage: full-build
402   # due to #16084
403   allow_failure: true
404   variables:
405     GHC_VERSION: "8.6.2"
406     LANG: "en_US.UTF-8"
407   script:
408     - |
409       set MSYSTEM=MINGW64
410       python boot
411       bash -c './configure --enable-tarballs-autodownload GHC=`pwd`/toolchain/bin/ghc HAPPY=`pwd`/toolchain/bin/happy ALEX=`pwd`/toolchain/bin/alex'
412     - bash -c "echo include mk/flavours/quick.mk > mk/build.mk"
413     - bash -c "PATH=`pwd`/toolchain/bin:$PATH make -j`mk/detect-cpu-count.sh`"
414     - |
415       bash -c "make binary-dist TAR_COMP_OPTS=-1"
416       mv ghc-*.tar.xz ghc.tar.xz
417     - bash -c 'make V=0 test THREADS=`mk/detect-cpu-count.sh` JUNIT_FILE=../../junit.xml'
418   cache:
419     key: x86_64-windows
420   tags:
421     - x86_64-windows
422   artifacts:
423     when: always
424     reports:
425       junit: junit.xml
426     paths:
427       - ghc.tar.xz
428       - junit.xml
429
430 # Note [Cleaning up after shell executor]
431 # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
432 #
433 # As noted in [1], gitlab-runner's shell executor doesn't clean up its working
434 # directory after builds. Unfortunately, we are forced to use the shell executor
435 # on Windows. To avoid running out of disk space we add a stage at the end of
436 # the build to remove the \GitLabRunner\builds directory. Since we only run a
437 # single build at a time on Windows this should be safe.
438 #
439 # [1] https://gitlab.com/gitlab-org/gitlab-runner/issues/3856
440
441 # See Note [Cleanup after shell executor]
442 cleanup-windows:
443   stage: cleanup
444   tags:
445     - x86_64-windows
446   when: always
447   dependencies: []
448   before_script:
449     - echo "Time to clean up"
450   script:
451     - echo "Let's go"
452   after_script:
453     - set "BUILD_DIR=%CI_PROJECT_DIR%"
454     - set "BUILD_DIR=%BUILD_DIR:/=\%"
455     - echo "Cleaning %BUILD_DIR%"
456     - cd \GitLabRunner
457     # This is way more complicated than it should be:
458     # See https://stackoverflow.com/questions/1965787
459     - del %BUILD_DIR%\* /F /Q
460     - for /d %%p in (%BUILD_DIR%\*) do rd /Q /S "%%p"
461     - exit /b 0
462
463 # See Note [Cleanup after shell executor]
464 cleanup-darwin:
465   stage: cleanup
466   tags:
467     - x86_64-darwin
468   when: always
469   dependencies: []
470   before_script:
471     - echo "Time to clean up"
472   script:
473     - echo "Let's go"
474   after_script:
475     - BUILD_DIR=$CI_PROJECT_DIR
476     - echo "Cleaning $BUILD_DIR"
477     - cd $HOME
478     - rm -Rf $BUILD_DIR/*
479     - exit 0
480