gitlab-ci: Give deb9-unreg job a distinct cache key
[ghc.git] / .gitlab-ci.yml
1 variables:
2   GIT_SSL_NO_VERIFY: "1"
3   # Commit of ghc/ci-images repository from which to pull Docker images
4   DOCKER_REV: 6d19c3adc1f5c28c82aed8c5b1ac40931ac60f3f
5
6 before_script:
7   - python3 .gitlab/fix-submodules.py
8   - git submodule sync --recursive
9   - git submodule update --init --recursive
10   - git checkout .gitmodules
11   - "git fetch https://gitlab.haskell.org/ghc/ghc-performance-notes.git refs/notes/perf:refs/notes/perf || true"
12
13 stages:
14   - lint
15   - build
16   - full-build
17   - cleanup # See Note [Cleanup on Windows]
18
19 ############################################################
20 # Runner Tags
21 ############################################################
22 #
23 # * x86_64-linux: Any Docker-capable x86_64 Linux machine
24 # * aarch64-linux: Any Docker-capable AArch64 Linux machine
25 # * x86_64-windows: A x86_64 Windows machine
26 # * lint: Any Docker-capable x86_64 Linux machine; distinct from
27 #         x86_64-linux to ensure low-latency availability.
28 #
29
30
31 ############################################################
32 # Linting
33 ############################################################
34
35 ghc-linters:
36   stage: lint
37   image: "registry.gitlab.haskell.org/ghc/ci-images/linters:$DOCKER_REV"
38   script:
39     - |
40       if [ -n "$CI_MERGE_REQUEST_ID" ]; then
41         base="$(git merge-base $CI_MERGE_REQUEST_BRANCH_NAME HEAD)"
42         validate-commit-msg .git $(git rev-list $base..$CI_COMMIT_SHA)
43         submodchecker .git $(git rev-list $base..$CI_COMMIT_SHA)
44         validate-whitespace .git $(git rev-list $base..$CI_COMMIT_SHA)
45         .gitlab/linters/check-makefiles.py $base $CI_COMMIT_SHA
46         .gitlab/linters/check-cpp.py $base $CI_COMMIT_SHA
47       fi
48   dependencies: []
49   tags:
50     - lint
51
52 ############################################################
53 # Validation via Pipelines (hadrian)
54 ############################################################
55
56 .validate-hadrian:
57   allow_failure: true
58   script:
59     - cabal update
60     - git clean -xdf && git submodule foreach git clean -xdf
61     - bash .circleci/prepare-system.sh
62     - if [[ -d ./cabal-cache ]]; then cp -R ./.cabal-cache ~/.cabal-cache; fi
63     - ./boot
64     - ./configure $CONFIGURE_ARGS
65     - hadrian/build.cabal.sh -j`mk/detect-cpu-count.sh`
66     - hadrian/build.cabal.sh -j`mk/detect-cpu-count.sh` --docs=no-sphinx docs
67   cache:
68     key: hadrian
69     paths:
70       - cabal-cache
71
72 validate-x86_64-linux-deb8-hadrian:
73   extends: .validate-hadrian
74   stage: build
75   image: "registry.gitlab.haskell.org/ghc/ci-images/x86_64-linux-deb8:$DOCKER_REV"
76   before_script:
77     # workaround for docker permissions
78     - sudo chown ghc:ghc -R .
79     - python3 .gitlab/fix-submodules.py
80     - git submodule sync --recursive
81     - git submodule update --init --recursive
82     - git checkout .gitmodules
83     - "git fetch https://gitlab.haskell.org/ghc/ghc-performance-notes.git refs/notes/perf:refs/notes/perf || true"
84   tags:
85     - x86_64-linux
86
87 ############################################################
88 # Validation via Pipelines (make)
89 ############################################################
90
91 .validate:
92   variables:
93     TEST_TYPE: test
94   before_script:
95     - git clean -xdf && git submodule foreach git clean -xdf
96   script:
97     - ./boot
98     - ./configure $CONFIGURE_ARGS
99     - |
100       THREADS=`mk/detect-cpu-count.sh`
101       make V=0 -j$THREADS WERROR=-Werror 
102     - |
103       make binary-dist TAR_COMP_OPTS="-1"
104       mv ghc-*.tar.xz ghc.tar.xz
105     - |
106       # Prepare to push git notes.
107       METRICS_FILE=$(mktemp)
108       git config user.email "ben+ghc-ci@smart-cactus.org"
109       git config user.name "GHC GitLab CI"
110     - |
111       THREADS=`mk/detect-cpu-count.sh`
112       make $TEST_TYPE THREADS=$THREADS JUNIT_FILE=../../junit.xml METRICS_FILE=$METRICS_FILE
113     - |
114       # Push git notes.
115       METRICS_FILE=$METRICS_FILE .gitlab/push-test-metrics.sh
116   dependencies: []
117   artifacts:
118     reports:
119       junit: junit.xml
120     expire_in: 2 week
121     paths:
122       - ghc.tar.xz
123       - junit.xml
124
125 validate-x86_64-darwin:
126   extends: .validate
127   stage: full-build
128   tags:
129     - x86_64-darwin
130   variables:
131     GHC_VERSION: 8.6.3
132     MACOSX_DEPLOYMENT_TARGET: "10.7"
133     # Only Sierra and onwards supports clock_gettime. See #12858
134     ac_cv_func_clock_gettime: "no"
135     LANG: "en_US.UTF-8"
136     CONFIGURE_ARGS: --with-intree-gmp
137     TEST_ENV: "x86_64-darwin"
138   before_script:
139     - git clean -xdf && git submodule foreach git clean -xdf
140     - python3 .gitlab/fix-submodules.py
141     - git submodule sync --recursive
142     - git submodule update --init --recursive
143     - git checkout .gitmodules
144     - "git fetch https://gitlab.haskell.org/ghc/ghc-performance-notes.git refs/notes/perf:refs/notes/perf || true"
145
146     - bash .gitlab/darwin-init.sh
147     - PATH="`pwd`/toolchain/bin:$PATH"
148   after_script:
149     - cp -Rf $HOME/.cabal cabal-cache
150   artifacts:
151     when: always
152     expire_in: 2 week
153   cache:
154     key: darwin
155     paths:
156       - cabal-cache
157       - toolchain
158
159 .validate-linux:
160   extends: .validate
161   tags:
162     - x86_64-linux
163   before_script:
164     - git clean -xdf && git submodule foreach git clean -xdf
165     - python3 .gitlab/fix-submodules.py
166     - git submodule sync --recursive
167     - git submodule update --init --recursive
168     - git checkout .gitmodules
169     - "git fetch https://gitlab.haskell.org/ghc/ghc-performance-notes.git refs/notes/perf:refs/notes/perf || true"
170
171     - bash .circleci/prepare-system.sh
172     # workaround for docker permissions
173     - sudo chown ghc:ghc -R .
174   after_script:
175     - cp -Rf $HOME/.cabal cabal-cache
176   cache:
177     key: linux
178     paths:
179       - cabal-cache
180       - toolchain
181
182 validate-aarch64-linux-deb9:
183   extends: .validate-linux
184   stage: full-build
185   image: "registry.gitlab.haskell.org/ghc/ci-images/aarch64-linux-deb9:$DOCKER_REV"
186   allow_failure: true
187   variables:
188     TEST_ENV: "aarch64-linux-deb9"
189   artifacts:
190     when: always
191     expire_in: 2 week
192   cache:
193     key: linux-aarch64-deb9
194   tags:
195     - aarch64-linux
196
197 nightly-aarch64-linux-deb9:
198   extends: validate-aarch64-linux-deb9
199   stage: full-build
200   artifacts:
201     expire_in: 2 year
202   variables:
203     TEST_TYPE: slowtest
204   only:
205     variables:
206       - $NIGHTLY
207
208 validate-i386-linux-deb9:
209   extends: .validate-linux
210   stage: full-build
211   image: "registry.gitlab.haskell.org/ghc/ci-images/i386-linux-deb9:$DOCKER_REV"
212   allow_failure: true
213   variables:
214     TEST_ENV: "i386-linux-deb9"
215   artifacts:
216     when: always
217     expire_in: 2 week
218   cache:
219     key: linux-i386-deb9
220
221 nightly-i386-linux-deb9:
222   extends: .validate-linux
223   stage: full-build
224   image: "registry.gitlab.haskell.org/ghc/ci-images/i386-linux-deb9:$DOCKER_REV"
225   allow_failure: true
226   variables:
227     TEST_TYPE: slowtest
228     TEST_ENV: "i386-linux-deb9"
229   artifacts:
230     when: always
231     expire_in: 2 week
232   only:
233     variables:
234       - $NIGHTLY
235   cache:
236     key: linux-i386-deb9
237
238 # N.B. Has DEBUG assertions enabled in stage2
239 validate-x86_64-linux-deb9-debug:
240   extends: .validate-linux
241   stage: build
242   image: "registry.gitlab.haskell.org/ghc/ci-images/x86_64-linux-deb9:$DOCKER_REV"
243   variables:
244     BUILD_FLAVOUR: validate
245     TEST_ENV: "x86_64-linux-deb9-debug"
246   cache:
247     key: linux-x86_64-deb9
248
249 validate-x86_64-linux-deb9:
250   extends: .validate-linux
251   stage: build
252   image: "registry.gitlab.haskell.org/ghc/ci-images/x86_64-linux-deb9:$DOCKER_REV"
253   variables:
254     TEST_ENV: "x86_64-linux-deb9"
255   artifacts:
256     when: always
257     expire_in: 2 week
258   cache:
259     key: linux-x86_64-deb9
260
261 release-x86_64-linux-deb9-dwarf:
262   extends: .validate-linux
263   stage: build
264   image: "registry.gitlab.haskell.org/ghc/ci-images/x86_64-linux-deb9:$DOCKER_REV"
265   allow_failure: true
266   variables:
267     CONFIGURE_ARGS: "--enable-dwarf-unwind"
268     BUILD_FLAVOUR: dwarf
269     TEST_ENV: "x86_64-linux-deb9"
270   artifacts:
271     when: always
272     expire_in: 2 week
273   cache:
274     key: linux-x86_64-deb9
275
276 nightly-x86_64-linux-deb9:
277   extends: validate-x86_64-linux-deb9
278   stage: build
279   artifacts:
280     expire_in: 2 year
281   variables:
282     TEST_TYPE: slowtest
283   only:
284     variables:
285       - $NIGHTLY
286
287 validate-x86_64-linux-deb9-llvm:
288   extends: .validate-linux
289   stage: full-build
290   image: "registry.gitlab.haskell.org/ghc/ci-images/x86_64-linux-deb9:$DOCKER_REV"
291   variables:
292     BUILD_FLAVOUR: perf-llvm
293     TEST_ENV: "x86_64-linux-deb9-llvm"
294   cache:
295     key: linux-x86_64-deb9
296
297 release-x86_64-linux-deb8:
298   extends: .validate-linux
299   stage: full-build
300   image: "registry.gitlab.haskell.org/ghc/ci-images/x86_64-linux-deb8:$DOCKER_REV"
301   variables:
302     TEST_ENV: "x86_64-linux-deb8"
303   only:
304     - tags
305   cache:
306     key: linux-x86_64-deb8
307   artifacts:
308     when: always
309     expire_in: 2 week
310
311 release-x86_64-linux-fedora27:
312   extends: .validate-linux
313   stage: full-build
314   image: "registry.gitlab.haskell.org/ghc/ci-images/x86_64-linux-fedora27:$DOCKER_REV"
315   variables:
316     TEST_ENV: "x86_64-linux-fedora27"
317   only:
318     - tags
319   cache:
320     key: linux-x86_64-fedora27
321   artifacts:
322     when: always
323     expire_in: 2 week
324
325 validate-x86_64-linux-deb9-integer-simple:
326   extends: .validate-linux
327   stage: full-build
328   variables:
329     INTEGER_LIBRARY: integer-simple
330     TEST_ENV: "x86_64-linux-deb9-integer-simple"
331   image: "registry.gitlab.haskell.org/ghc/ci-images/x86_64-linux-deb9:$DOCKER_REV"
332   cache:
333     key: linux-x86_64-deb9
334
335 nightly-x86_64-linux-deb9-integer-simple:
336   extends: validate-x86_64-linux-deb9-integer-simple
337   stage: full-build
338   artifacts:
339     expire_in: 2 year
340   variables:
341     TEST_TYPE: slowtest
342   only:
343     variables:
344       - $NIGHTLY
345
346 validate-x86_64-linux-deb9-unreg:
347   extends: .validate-linux
348   stage: full-build
349   variables:
350     CONFIGURE_ARGS: --enable-unregisterised
351     TEST_ENV: "x86_64-linux-deb9-unreg"
352   image: "registry.gitlab.haskell.org/ghc/ci-images/x86_64-linux-deb9:$DOCKER_REV"
353   cache:
354     key: linux-x86_64-deb9-unreg
355
356 ############################################################
357 # Validation via Pipelines (Windows)
358 ############################################################
359
360 .validate-windows:
361   before_script:
362     - git clean -xdf
363     - git submodule foreach git clean -xdf
364
365     # Use a local temporary directory to ensure that concurrent builds don't
366     # interfere with one another
367     - |
368       mkdir tmp
369       set TMP=%cd%\tmp
370       set TEMP=%cd%\tmp
371
372     - set PATH=C:\msys64\usr\bin;%PATH%
373     - python .gitlab/fix-submodules.py
374     - git submodule sync --recursive
375     - git submodule update --init --recursive
376     - git checkout .gitmodules
377     - "git fetch https://gitlab.haskell.org/ghc/ghc-performance-notes.git refs/notes/perf:refs/notes/perf || true"
378     - bash .gitlab/win32-init.sh
379   after_script:
380     - rd /s /q tmp
381     - robocopy /np /nfl /ndl /e "%APPDATA%\cabal" cabal-cache
382     - bash -c 'make clean || true'
383   dependencies: []
384   variables:
385     FORCE_SYMLINKS: 1
386   cache:
387     paths:
388       - cabal-cache
389       - ghc-8.6.2
390       - ghc-tarballs
391
392 validate-x86_64-windows-hadrian:
393   extends: .validate-windows
394   stage: full-build
395   variables:
396     GHC_VERSION: "8.6.2"
397     LANG: "en_US.UTF-8"
398   script:
399     - |
400       set MSYSTEM=MINGW64
401       python boot
402       bash -c './configure --enable-tarballs-autodownload GHC=`pwd`/toolchain/bin/ghc HAPPY=`pwd`/toolchain/bin/happy ALEX=`pwd`/toolchain/bin/alex'
403       mkdir -p _build
404       cp -R inplace/mingw _build/mingw
405     - bash -c "PATH=`pwd`/toolchain/bin:$PATH hadrian/build.cabal.sh -j`mk/detect-cpu-count.sh` --flavour=Quick"
406     - bash -c "PATH=`pwd`/toolchain/bin:$PATH hadrian/build.cabal.sh -j`mk/detect-cpu-count.sh` --flavour=Quick --docs=no-sphinx docs
407     # FIXME: Bindist disabled due to #16073
408     #- bash -c "PATH=`pwd`/toolchain/bin:$PATH hadrian/build.cabal.sh binary-dist"
409     # FIXME: Testsuite disabled due to #16156.
410     #- bash -c 'make V=0 test THREADS=`mk/detect-cpu-count.sh` JUNIT_FILE=../../junit.xml'
411   cache:
412     key: x86_64-windows
413   tags:
414     - x86_64-windows
415
416 validate-x86_64-windows:
417   extends: .validate-windows
418   stage: full-build
419   # due to #16084
420   allow_failure: true
421   variables:
422     GHC_VERSION: "8.6.2"
423     LANG: "en_US.UTF-8"
424   script:
425     - |
426       set MSYSTEM=MINGW64
427       python boot
428       bash -c './configure --enable-tarballs-autodownload GHC=`pwd`/toolchain/bin/ghc HAPPY=`pwd`/toolchain/bin/happy ALEX=`pwd`/toolchain/bin/alex'
429     - bash -c "echo include mk/flavours/quick.mk > mk/build.mk"
430     - bash -c "PATH=`pwd`/toolchain/bin:$PATH make -j`mk/detect-cpu-count.sh`"
431     - |
432       bash -c "make binary-dist TAR_COMP_OPTS=-1"
433       mv ghc-*.tar.xz ghc.tar.xz
434     - bash -c 'make V=0 test THREADS=`mk/detect-cpu-count.sh` JUNIT_FILE=../../junit.xml'
435   cache:
436     key: x86_64-windows
437   tags:
438     - x86_64-windows
439   artifacts:
440     when: always
441     reports:
442       junit: junit.xml
443     paths:
444       - ghc.tar.xz
445       - junit.xml
446
447 # Note [Cleaning up after shell executor]
448 # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
449 #
450 # As noted in [1], gitlab-runner's shell executor doesn't clean up its working
451 # directory after builds. Unfortunately, we are forced to use the shell executor
452 # on Windows. To avoid running out of disk space we add a stage at the end of
453 # the build to remove the \GitLabRunner\builds directory. Since we only run a
454 # single build at a time on Windows this should be safe.
455 #
456 # [1] https://gitlab.com/gitlab-org/gitlab-runner/issues/3856
457
458 # See Note [Cleanup after shell executor]
459 cleanup-windows:
460   stage: cleanup
461   tags:
462     - x86_64-windows
463   when: always
464   dependencies: []
465   before_script:
466     - echo "Time to clean up"
467   script:
468     - echo "Let's go"
469   after_script:
470     - set "BUILD_DIR=%CI_PROJECT_DIR%"
471     - set "BUILD_DIR=%BUILD_DIR:/=\%"
472     - echo "Cleaning %BUILD_DIR%"
473     - cd \GitLabRunner
474     # This is way more complicated than it should be:
475     # See https://stackoverflow.com/questions/1965787
476     - del %BUILD_DIR%\* /F /Q
477     - for /d %%p in (%BUILD_DIR%\*) do rd /Q /S "%%p"
478     - exit /b 0
479
480 # See Note [Cleanup after shell executor]
481 cleanup-darwin:
482   stage: cleanup
483   tags:
484     - x86_64-darwin
485   when: always
486   dependencies: []
487   before_script:
488     - echo "Time to clean up"
489   script:
490     - echo "Let's go"
491   after_script:
492     - BUILD_DIR=$CI_PROJECT_DIR
493     - echo "Cleaning $BUILD_DIR"
494     - cd $HOME
495     - rm -Rf $BUILD_DIR/*
496     - exit 0
497