gitlab-ci: Pull docker images from ghc/ci-images registry
[ghc.git] / .gitlab-ci.yml
1 variables:
2   GIT_SSL_NO_VERIFY: "1"
3   # Commit of ghc/ci-images repository from which to pull Docker images
4   DOCKER_REV: 2757db588256b0f96dd44d602f0857ae8f86a704
5
6 before_script:
7   - python3 .gitlab/fix-submodules.py
8   - git submodule sync --recursive
9   - git submodule update --init --recursive
10   - git checkout .gitmodules
11   - "git fetch https://gitlab.haskell.org/ghc/ghc-performance-notes.git refs/notes/perf:refs/notes/perf || true"
12
13 stages:
14   - lint
15   - build
16   - full-build
17   - cleanup # See Note [Cleanup on Windows]
18
19 ############################################################
20 # Runner Tags
21 ############################################################
22 #
23 # * x86_64-linux: Any Docker-capable x86_64 Linux machine
24 # * aarch64-linux: Any Docker-capable AArch64 Linux machine
25 # * x86_64-windows: A x86_64 Windows machine
26 # * lint: Any Docker-capable x86_64 Linux machine; distinct from
27 #         x86_64-linux to ensure low-latency availability.
28 #
29
30
31 ############################################################
32 # Linting
33 ############################################################
34
35 ghc-linters:
36   stage: lint
37   image: "registry.gitlab.haskell.org/ghc/ci-images/linters:$DOCKER_REV"
38   script:
39     - |
40       if [ -n "$CI_MERGE_REQUEST_ID" ]; then
41         base="$(git merge-base $CI_MERGE_REQUEST_BRANCH_NAME HEAD)"
42         validate-commit-msg .git $(git rev-list $base..$CI_COMMIT_SHA)
43         submodchecker .git $(git rev-list $base..$CI_COMMIT_SHA)
44         validate-whitespace .git $(git rev-list $base..$CI_COMMIT_SHA)
45         .gitlab/linters/check-makefiles.py $base $CI_COMMIT_SHA
46         .gitlab/linters/check-cpp.py $base $CI_COMMIT_SHA
47       fi
48   dependencies: []
49   tags:
50     - lint
51
52 ############################################################
53 # Validation via Pipelines (hadrian)
54 ############################################################
55
56 .validate-hadrian:
57   allow_failure: true
58   script:
59     - cabal update
60     - git clean -xdf && git submodule foreach git clean -xdf
61     - bash .circleci/prepare-system.sh
62     - if [[ -d ./cabal-cache ]]; then cp -R ./.cabal-cache ~/.cabal-cache; fi
63     - ./boot
64     - ./configure $CONFIGURE_ARGS
65     - hadrian/build.cabal.sh -j`mk/detect-cpu-count.sh`
66     - hadrian/build.cabal.sh -j`mk/detect-cpu-count.sh` --docs=no-sphinx docs
67   cache:
68     key: hadrian
69     paths:
70       - cabal-cache
71
72 validate-x86_64-linux-deb8-hadrian:
73   extends: .validate-hadrian
74   stage: build
75   image: "registry.gitlab.haskell.org/ghc/ci-images/x86_64-linux-deb8:$DOCKER_REV"
76   before_script:
77     # workaround for docker permissions
78     - sudo chown ghc:ghc -R .
79     - python3 .gitlab/fix-submodules.py
80     - git submodule sync --recursive
81     - git submodule update --init --recursive
82     - git checkout .gitmodules
83     - "git fetch https://gitlab.haskell.org/ghc/ghc-performance-notes.git refs/notes/perf:refs/notes/perf || true"
84   tags:
85     - x86_64-linux
86
87 ############################################################
88 # Validation via Pipelines (make)
89 ############################################################
90
91 .validate:
92   variables:
93     TEST_TYPE: test
94   before_script:
95     - git clean -xdf && git submodule foreach git clean -xdf
96   script:
97     - ./boot
98     - ./configure $CONFIGURE_ARGS
99     - |
100       THREADS=`mk/detect-cpu-count.sh`
101       make V=0 -j$THREADS WERROR=-Werror 
102     - |
103       make binary-dist TAR_COMP_OPTS="-1"
104       mv ghc-*.tar.xz ghc.tar.xz
105     - |
106       # Prepare to push git notes.
107       METRICS_FILE=$(mktemp)
108       git config user.email "ben+ghc-ci@smart-cactus.org"
109       git config user.name "GHC GitLab CI"
110     - |
111       THREADS=`mk/detect-cpu-count.sh`
112       make $TEST_TYPE THREADS=$THREADS JUNIT_FILE=../../junit.xml METRICS_FILE=$METRICS_FILE
113     - |
114       # Push git notes.
115       METRICS_FILE=$METRICS_FILE .gitlab/push-test-metrics.sh
116   dependencies: []
117   artifacts:
118     reports:
119       junit: junit.xml
120     expire_in: 2 week
121     paths:
122       - ghc.tar.xz
123       - junit.xml
124
125 validate-x86_64-darwin:
126   extends: .validate
127   stage: full-build
128   tags:
129     - x86_64-darwin
130   variables:
131     GHC_VERSION: 8.6.3
132     MACOSX_DEPLOYMENT_TARGET: "10.7"
133     # Only Sierra and onwards supports clock_gettime. See #12858
134     ac_cv_func_clock_gettime: "no"
135     LANG: "en_US.UTF-8"
136     CONFIGURE_ARGS: --with-intree-gmp
137     TEST_ENV: "x86_64-darwin"
138   before_script:
139     - git clean -xdf && git submodule foreach git clean -xdf
140     - python3 .gitlab/fix-submodules.py
141     - git submodule sync --recursive
142     - git submodule update --init --recursive
143     - git checkout .gitmodules
144     - "git fetch https://gitlab.haskell.org/ghc/ghc-performance-notes.git refs/notes/perf:refs/notes/perf || true"
145
146     - bash .gitlab/darwin-init.sh
147     - PATH="`pwd`/toolchain/bin:$PATH"
148   after_script:
149     - cp -Rf $HOME/.cabal cabal-cache
150   artifacts:
151     when: always
152     expire_in: 2 week
153   cache:
154     key: darwin
155     paths:
156       - cabal-cache
157       - toolchain
158
159 .validate-linux:
160   extends: .validate
161   tags:
162     - x86_64-linux
163   before_script:
164     - git clean -xdf && git submodule foreach git clean -xdf
165     - python3 .gitlab/fix-submodules.py
166     - git submodule sync --recursive
167     - git submodule update --init --recursive
168     - git checkout .gitmodules
169     - "git fetch https://gitlab.haskell.org/ghc/ghc-performance-notes.git refs/notes/perf:refs/notes/perf || true"
170
171     - bash .circleci/prepare-system.sh
172     # workaround for docker permissions
173     - sudo chown ghc:ghc -R .
174   after_script:
175     - cp -Rf $HOME/.cabal cabal-cache
176   cache:
177     key: linux
178     paths:
179       - cabal-cache
180       - toolchain
181
182 validate-aarch64-linux-deb9:
183   extends: .validate-linux
184   stage: full-build
185   image: "registry.gitlab.haskell.org/ghc/ci-images/aarch64-linux-deb9:$DOCKER_REV"
186   allow_failure: true
187   variables:
188     TEST_ENV: "aarch64-linux-deb9"
189   artifacts:
190     when: always
191     expire_in: 2 week
192   cache:
193     key: linux-aarch64-deb9
194   tags:
195     - aarch64-linux
196
197 nightly-aarch64-linux-deb9:
198   extends: validate-aarch64-linux-deb9
199   stage: full-build
200   artifacts:
201     expire_in: 2 year
202   variables:
203     TEST_TYPE: slowtest
204   only:
205     variables:
206       - $NIGHTLY
207
208 validate-i386-linux-deb9:
209   extends: .validate-linux
210   stage: full-build
211   image: "registry.gitlab.haskell.org/ghc/ci-images/i386-linux-deb9:$DOCKER_REV"
212   allow_failure: true
213   variables:
214     TEST_ENV: "i386-linux-deb9"
215   artifacts:
216     when: always
217     expire_in: 2 week
218   cache:
219     key: linux-i386-deb9
220
221 nightly-i386-linux-deb9:
222   extends: .validate-linux
223   stage: full-build
224   image: "registry.gitlab.haskell.org/ghc/ci-images/i386-linux-deb9:$DOCKER_REV"
225   allow_failure: true
226   variables:
227     TEST_TYPE: slowtest
228     TEST_ENV: "i386-linux-deb9"
229   artifacts:
230     when: always
231     expire_in: 2 week
232   only:
233     variables:
234       - $NIGHTLY
235   cache:
236     key: linux-i386-deb9
237
238 # N.B. Has DEBUG assertions enabled in stage2
239 validate-x86_64-linux-deb9-debug:
240   extends: .validate-linux
241   stage: build
242   image: "registry.gitlab.haskell.org/ghc/ci-images/x86_64-linux-deb9:$DOCKER_REV"
243   variables:
244     BUILD_FLAVOUR: validate
245     TEST_ENV: "x86_64-linux-deb9-debug"
246   cache:
247     key: linux-x86_64-deb9
248
249 validate-x86_64-linux-deb9:
250   extends: .validate-linux
251   stage: build
252   image: "registry.gitlab.haskell.org/ghc/ci-images/x86_64-linux-deb9:$DOCKER_REV"
253   variables:
254     TEST_ENV: "x86_64-linux-deb9"
255   artifacts:
256     when: always
257     expire_in: 2 week
258   cache:
259     key: linux-x86_64-deb9
260
261 nightly-x86_64-linux-deb9:
262   extends: validate-x86_64-linux-deb9
263   stage: build
264   artifacts:
265     expire_in: 2 year
266   variables:
267     TEST_TYPE: slowtest
268   only:
269     variables:
270       - $NIGHTLY
271
272 validate-x86_64-linux-deb9-llvm:
273   extends: .validate-linux
274   stage: full-build
275   image: "registry.gitlab.haskell.org/ghc/ci-images/x86_64-linux-deb9:$DOCKER_REV"
276   variables:
277     BUILD_FLAVOUR: perf-llvm
278     TEST_ENV: "x86_64-linux-deb9-llvm"
279   cache:
280     key: linux-x86_64-deb9
281
282 release-x86_64-linux-deb8:
283   extends: .validate-linux
284   stage: full-build
285   image: "registry.gitlab.haskell.org/ghc/ci-images/x86_64-linux-deb8:$DOCKER_REV"
286   variables:
287     TEST_ENV: "x86_64-linux-deb8"
288   only:
289     - tags
290   cache:
291     key: linux-x86_64-deb8
292   artifacts:
293     when: always
294     expire_in: 2 week
295
296 release-x86_64-linux-fedora27:
297   extends: .validate-linux
298   stage: full-build
299   image: "registry.gitlab.haskell.org/ghc/ci-images/x86_64-linux-fedora27:$DOCKER_REV"
300   variables:
301     TEST_ENV: "x86_64-linux-fedora27"
302   only:
303     - tags
304   cache:
305     key: linux-x86_64-fedora27
306   artifacts:
307     when: always
308     expire_in: 2 week
309
310 validate-x86_64-linux-deb9-integer-simple:
311   extends: .validate-linux
312   stage: full-build
313   variables:
314     INTEGER_LIBRARY: integer-simple
315     TEST_ENV: "x86_64-linux-deb9-integer-simple"
316   image: "registry.gitlab.haskell.org/ghc/ci-images/x86_64-linux-deb9:$DOCKER_REV"
317   cache:
318     key: linux-x86_64-deb9
319
320 nightly-x86_64-linux-deb9-integer-simple:
321   extends: validate-x86_64-linux-deb9-integer-simple
322   stage: full-build
323   artifacts:
324     expire_in: 2 year
325   variables:
326     TEST_TYPE: slowtest
327   only:
328     variables:
329       - $NIGHTLY
330
331 validate-x86_64-linux-deb9-unreg:
332   extends: .validate-linux
333   stage: full-build
334   variables:
335     CONFIGURE_ARGS: --enable-unregisterised
336     TEST_ENV: "x86_64-linux-deb9-unreg"
337   image: "registry.gitlab.haskell.org/ghc/ci-images/x86_64-linux-deb9:$DOCKER_REV"
338   cache:
339     key: linux-x86_64-deb9
340
341 ############################################################
342 # Validation via Pipelines (Windows)
343 ############################################################
344
345 .validate-windows:
346   before_script:
347     - git clean -xdf
348     - git submodule foreach git clean -xdf
349
350     # Use a local temporary directory to ensure that concurrent builds don't
351     # interfere with one another
352     - |
353       mkdir tmp
354       set TMP=%cd%\tmp
355       set TEMP=%cd%\tmp
356
357     - set PATH=C:\msys64\usr\bin;%PATH%
358     - python .gitlab/fix-submodules.py
359     - git submodule sync --recursive
360     - git submodule update --init --recursive
361     - git checkout .gitmodules
362     - "git fetch https://gitlab.haskell.org/ghc/ghc-performance-notes.git refs/notes/perf:refs/notes/perf || true"
363     - bash .gitlab/win32-init.sh
364   after_script:
365     - rd /s /q tmp
366     - robocopy /np /nfl /ndl /e "%APPDATA%\cabal" cabal-cache
367     - bash -c 'make clean || true'
368   dependencies: []
369   variables:
370     FORCE_SYMLINKS: 1
371   cache:
372     paths:
373       - cabal-cache
374       - ghc-8.6.2
375       - ghc-tarballs
376
377 validate-x86_64-windows-hadrian:
378   extends: .validate-windows
379   stage: full-build
380   variables:
381     GHC_VERSION: "8.6.2"
382     LANG: "en_US.UTF-8"
383   script:
384     - |
385       set MSYSTEM=MINGW64
386       python boot
387       bash -c './configure --enable-tarballs-autodownload GHC=`pwd`/toolchain/bin/ghc HAPPY=`pwd`/toolchain/bin/happy ALEX=`pwd`/toolchain/bin/alex'
388       mkdir -p _build
389       cp -R inplace/mingw _build/mingw
390     - bash -c "PATH=`pwd`/toolchain/bin:$PATH hadrian/build.cabal.sh -j`mk/detect-cpu-count.sh` --flavour=Quick"
391     - bash -c "PATH=`pwd`/toolchain/bin:$PATH hadrian/build.cabal.sh -j`mk/detect-cpu-count.sh` --flavour=Quick --docs=no-sphinx docs
392     # FIXME: Bindist disabled due to #16073
393     #- bash -c "PATH=`pwd`/toolchain/bin:$PATH hadrian/build.cabal.sh binary-dist"
394     # FIXME: Testsuite disabled due to #16156.
395     #- bash -c 'make V=0 test THREADS=`mk/detect-cpu-count.sh` JUNIT_FILE=../../junit.xml'
396   cache:
397     key: x86_64-windows
398   tags:
399     - x86_64-windows
400
401 validate-x86_64-windows:
402   extends: .validate-windows
403   stage: full-build
404   # due to #16084
405   allow_failure: true
406   variables:
407     GHC_VERSION: "8.6.2"
408     LANG: "en_US.UTF-8"
409   script:
410     - |
411       set MSYSTEM=MINGW64
412       python boot
413       bash -c './configure --enable-tarballs-autodownload GHC=`pwd`/toolchain/bin/ghc HAPPY=`pwd`/toolchain/bin/happy ALEX=`pwd`/toolchain/bin/alex'
414     - bash -c "echo include mk/flavours/quick.mk > mk/build.mk"
415     - bash -c "PATH=`pwd`/toolchain/bin:$PATH make -j`mk/detect-cpu-count.sh`"
416     - |
417       bash -c "make binary-dist TAR_COMP_OPTS=-1"
418       mv ghc-*.tar.xz ghc.tar.xz
419     - bash -c 'make V=0 test THREADS=`mk/detect-cpu-count.sh` JUNIT_FILE=../../junit.xml'
420   cache:
421     key: x86_64-windows
422   tags:
423     - x86_64-windows
424   artifacts:
425     when: always
426     reports:
427       junit: junit.xml
428     paths:
429       - ghc.tar.xz
430       - junit.xml
431
432 # Note [Cleaning up after shell executor]
433 # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
434 #
435 # As noted in [1], gitlab-runner's shell executor doesn't clean up its working
436 # directory after builds. Unfortunately, we are forced to use the shell executor
437 # on Windows. To avoid running out of disk space we add a stage at the end of
438 # the build to remove the \GitLabRunner\builds directory. Since we only run a
439 # single build at a time on Windows this should be safe.
440 #
441 # [1] https://gitlab.com/gitlab-org/gitlab-runner/issues/3856
442
443 # See Note [Cleanup after shell executor]
444 cleanup-windows:
445   stage: cleanup
446   tags:
447     - x86_64-windows
448   when: always
449   dependencies: []
450   before_script:
451     - echo "Time to clean up"
452   script:
453     - echo "Let's go"
454   after_script:
455     - set "BUILD_DIR=%CI_PROJECT_DIR%"
456     - set "BUILD_DIR=%BUILD_DIR:/=\%"
457     - echo "Cleaning %BUILD_DIR%"
458     - cd \GitLabRunner
459     # This is way more complicated than it should be:
460     # See https://stackoverflow.com/questions/1965787
461     - del %BUILD_DIR%\* /F /Q
462     - for /d %%p in (%BUILD_DIR%\*) do rd /Q /S "%%p"
463     - exit /b 0
464
465 # See Note [Cleanup after shell executor]
466 cleanup-darwin:
467   stage: cleanup
468   tags:
469     - x86_64-darwin
470   when: always
471   dependencies: []
472   before_script:
473     - echo "Time to clean up"
474   script:
475     - echo "Let's go"
476   after_script:
477     - BUILD_DIR=$CI_PROJECT_DIR
478     - echo "Cleaning $BUILD_DIR"
479     - cd $HOME
480     - rm -Rf $BUILD_DIR/*
481     - exit 0
482