Set builder env
[ghc.git] / .gitlab-ci.yml
1 variables:
2   GIT_SSL_NO_VERIFY: "1"
3
4 before_script:
5   - python3 .gitlab/fix-submodules.py
6   - git submodule sync --recursive
7   - git submodule update --init --recursive
8   - git checkout .gitmodules
9   - "git fetch https://gitlab.haskell.org/ghc/ghc-performance-notes.git refs/notes/perf:refs/notes/perf || true"
10
11 stages:
12   - lint
13   - build
14   - full-build
15   - cleanup # See Note [Cleanup on Windows]
16
17 ############################################################
18 # Runner Tags
19 ############################################################
20 #
21 # * x86_64-linux: Any Docker-capable x86_64 Linux machine
22 # * aarch64-linux: Any Docker-capable AArch64 Linux machine
23 # * x86_64-windows: A x86_64 Windows machine
24 # * lint: Any Docker-capable x86_64 Linux machine; distinct from
25 #         x86_64-linux to ensure low-latency availability.
26 #
27
28
29 ############################################################
30 # Linting
31 ############################################################
32
33 ghc-linters:
34   stage: lint
35   image: ghcci/linters:0.1
36   script:
37     - |
38       if [ -n "$CI_MERGE_REQUEST_ID" ]; then
39         base="$(git merge-base $CI_MERGE_REQUEST_BRANCH_NAME HEAD)"
40         validate-commit-msg .git $(git rev-list $base..$CI_COMMIT_SHA)
41         submodchecker .git $(git rev-list $base..$CI_COMMIT_SHA)
42         validate-whitespace .git $(git rev-list $base..$CI_COMMIT_SHA)
43         .gitlab/linters/check-makefiles.py $base $CI_COMMIT_SHA
44         .gitlab/linters/check-cpp.py $base $CI_COMMIT_SHA
45       fi
46   dependencies: []
47   tags:
48     - lint
49
50 ############################################################
51 # Validation via Pipelines (hadrian)
52 ############################################################
53
54 .validate-hadrian:
55   allow_failure: true
56   script:
57     - cabal update
58     - git clean -xdf && git submodule foreach git clean -xdf
59     - bash .circleci/prepare-system.sh
60     - if [[ -d ./cabal-cache ]]; then cp -R ./.cabal-cache ~/.cabal-cache; fi
61     - ./boot
62     - ./configure $CONFIGURE_ARGS
63     - hadrian/build.cabal.sh -j`mk/detect-cpu-count.sh`
64     - hadrian/build.cabal.sh -j`mk/detect-cpu-count.sh` docs-haddock
65   cache:
66     key: hadrian
67     paths:
68       - cabal-cache
69
70 validate-x86_64-linux-deb8-hadrian:
71   extends: .validate-hadrian
72   stage: build
73   image: ghcci/x86_64-linux-deb8:0.1
74   before_script:
75     # workaround for docker permissions
76     - sudo chown ghc:ghc -R .
77     - python3 .gitlab/fix-submodules.py
78     - git submodule sync --recursive
79     - git submodule update --init --recursive
80     - git checkout .gitmodules
81     - "git fetch https://gitlab.haskell.org/ghc/ghc-performance-notes.git refs/notes/perf:refs/notes/perf || true"
82   tags:
83     - x86_64-linux
84
85 ############################################################
86 # Validation via Pipelines (make)
87 ############################################################
88
89 .validate:
90   variables:
91     TEST_TYPE: test
92   before_script:
93     - git clean -xdf && git submodule foreach git clean -xdf
94   script:
95     - ./boot
96     - ./configure $CONFIGURE_ARGS
97     - |
98       THREADS=`mk/detect-cpu-count.sh`
99       make V=0 -j$THREADS WERROR=-Werror 
100     - |
101       make binary-dist TAR_COMP_OPTS="-1"
102       mv ghc-*.tar.xz ghc.tar.xz
103     - |
104       # Prepare to push git notes.
105       METRICS_FILE=$(mktemp)
106       git config user.email "ben+ghc-ci@smart-cactus.org"
107       git config user.name "GHC GitLab CI"
108     - |
109       THREADS=`mk/detect-cpu-count.sh`
110       make $TEST_TYPE THREADS=$THREADS JUNIT_FILE=../../junit.xml METRICS_FILE=$METRICS_FILE
111     - |
112       # Push git notes.
113       METRICS_FILE=$METRICS_FILE .gitlab/push-test-metrics.sh
114   dependencies: []
115   artifacts:
116     reports:
117       junit: junit.xml
118     expire_in: 2 week
119     paths:
120       - ghc.tar.xz
121       - junit.xml
122
123 validate-x86_64-darwin:
124   extends: .validate
125   stage: full-build
126   tags:
127     - x86_64-darwin
128   variables:
129     GHC_VERSION: 8.6.3
130     MACOSX_DEPLOYMENT_TARGET: "10.7"
131     # Only Sierra and onwards supports clock_gettime. See #12858
132     ac_cv_func_clock_gettime: "no"
133     LANG: "en_US.UTF-8"
134     CONFIGURE_ARGS: --with-intree-gmp
135     TEST_ENV: "x86_64-darwin"
136   before_script:
137     - git clean -xdf && git submodule foreach git clean -xdf
138     - python3 .gitlab/fix-submodules.py
139     - git submodule sync --recursive
140     - git submodule update --init --recursive
141     - git checkout .gitmodules
142     - "git fetch https://gitlab.haskell.org/ghc/ghc-performance-notes.git refs/notes/perf:refs/notes/perf || true"
143
144     - bash .gitlab/darwin-init.sh
145     - PATH="`pwd`/toolchain/bin:$PATH"
146   after_script:
147     - cp -Rf $HOME/.cabal cabal-cache
148   artifacts:
149     when: always
150     expire_in: 2 week
151   cache:
152     key: darwin
153     paths:
154       - cabal-cache
155       - toolchain
156
157 .validate-linux:
158   extends: .validate
159   tags:
160     - x86_64-linux
161   before_script:
162     - git clean -xdf && git submodule foreach git clean -xdf
163     - python3 .gitlab/fix-submodules.py
164     - git submodule sync --recursive
165     - git submodule update --init --recursive
166     - git checkout .gitmodules
167     - "git fetch https://gitlab.haskell.org/ghc/ghc-performance-notes.git refs/notes/perf:refs/notes/perf || true"
168
169     - bash .circleci/prepare-system.sh
170     # workaround for docker permissions
171     - sudo chown ghc:ghc -R .
172   after_script:
173     - cp -Rf $HOME/.cabal cabal-cache
174   cache:
175     key: linux
176     paths:
177       - cabal-cache
178       - toolchain
179
180 validate-aarch64-linux-deb9:
181   extends: .validate-linux
182   stage: full-build
183   image: ghcci/aarch64-linux-deb9:0.1
184   allow_failure: true
185   variables:
186     TEST_ENV: "aarch64-linux-deb9"
187   artifacts:
188     when: always
189     expire_in: 2 week
190   cache:
191     key: linux-aarch64-deb9
192   tags:
193     - aarch64-linux
194
195 nightly-aarch64-linux-deb9:
196   extends: validate-aarch64-linux-deb9
197   stage: full-build
198   artifacts:
199     expire_in: 2 year
200   variables:
201     TEST_TYPE: slowtest
202   only:
203     variables:
204       - $NIGHTLY
205
206 validate-i386-linux-deb9:
207   extends: .validate-linux
208   stage: full-build
209   image: ghcci/i386-linux-deb9:0.1
210   allow_failure: true
211   variables:
212     TEST_ENV: "i386-linux-deb9"
213   artifacts:
214     when: always
215     expire_in: 2 week
216   cache:
217     key: linux-i386-deb9
218
219 nightly-i386-linux-deb9:
220   extends: .validate-linux
221   stage: full-build
222   image: ghcci/i386-linux-deb9:0.1
223   allow_failure: true
224   variables:
225     TEST_TYPE: slowtest
226     TEST_ENV: "i386-linux-deb9"
227   artifacts:
228     when: always
229     expire_in: 2 week
230   only:
231     variables:
232       - $NIGHTLY
233   cache:
234     key: linux-i386-deb9
235
236 # N.B. Has DEBUG assertions enabled in stage2
237 validate-x86_64-linux-deb9-debug:
238   extends: .validate-linux
239   stage: build
240   image: ghcci/x86_64-linux-deb9:0.2
241   variables:
242     BUILD_FLAVOUR: devel2
243     TEST_ENV: "x86_64-linux-deb9-debug"
244   cache:
245     key: linux-x86_64-deb9
246
247 validate-x86_64-linux-deb9:
248   extends: .validate-linux
249   stage: build
250   image: ghcci/x86_64-linux-deb9:0.2
251   variables:
252     TEST_ENV: "x86_64-linux-deb9"
253   artifacts:
254     when: always
255     expire_in: 2 week
256   cache:
257     key: linux-x86_64-deb9
258
259 nightly-x86_64-linux-deb9:
260   extends: validate-x86_64-linux-deb9
261   stage: build
262   artifacts:
263     expire_in: 2 year
264   variables:
265     TEST_TYPE: slowtest
266   only:
267     variables:
268       - $NIGHTLY
269
270 validate-x86_64-linux-deb9-llvm:
271   extends: .validate-linux
272   stage: full-build
273   image: ghcci/x86_64-linux-deb9:0.2
274   variables:
275     BUILD_FLAVOUR: perf-llvm
276     TEST_ENV: "x86_64-linux-deb9-llvm"
277   cache:
278     key: linux-x86_64-deb9
279
280 validate-x86_64-linux-deb8:
281   extends: .validate-linux
282   stage: full-build
283   image: ghcci/x86_64-linux-deb8:0.1
284   variables:
285     TEST_ENV: "x86_64-linux-deb8"
286   cache:
287     key: linux-x86_64-deb8
288   artifacts:
289     when: always
290     expire_in: 2 week
291
292 validate-x86_64-linux-fedora27:
293   extends: .validate-linux
294   stage: full-build
295   image: ghcci/x86_64-linux-fedora27:0.1
296   variables:
297     TEST_ENV: "x86_64-linux-fedora27"
298   cache:
299     key: linux-x86_64-fedora27
300   artifacts:
301     when: always
302     expire_in: 2 week
303
304 validate-x86_64-linux-deb9-integer-simple:
305   extends: .validate-linux
306   stage: full-build
307   variables:
308     INTEGER_LIBRARY: integer-simple
309     TEST_ENV: "x86_64-linux-deb9-integer-simple"
310   image: ghcci/x86_64-linux-deb9:0.2
311   cache:
312     key: linux-x86_64-deb9
313
314 nightly-x86_64-linux-deb9-integer-simple:
315   extends: validate-x86_64-linux-deb9-integer-simple
316   stage: full-build
317   artifacts:
318     expire_in: 2 year
319   variables:
320     TEST_TYPE: slowtest
321   only:
322     variables:
323       - $NIGHTLY
324
325 validate-x86_64-linux-deb9-unreg:
326   extends: .validate-linux
327   stage: full-build
328   variables:
329     CONFIGURE_ARGS: --enable-unregisterised
330     TEST_ENV: "x86_64-linux-deb9-unreg"
331   image: ghcci/x86_64-linux-deb9:0.2
332   cache:
333     key: linux-x86_64-deb9
334
335 ############################################################
336 # Validation via Pipelines (Windows)
337 ############################################################
338
339 .validate-windows:
340   before_script:
341     - git clean -xdf
342     - git submodule foreach git clean -xdf
343
344     # Use a local temporary directory to ensure that concurrent builds don't
345     # interfere with one another
346     - |
347       mkdir tmp
348       set TMP=%cd%\tmp
349       set TEMP=%cd%\tmp
350
351     - set PATH=C:\msys64\usr\bin;%PATH%
352     - python .gitlab/fix-submodules.py
353     - git submodule sync --recursive
354     - git submodule update --init --recursive
355     - git checkout .gitmodules
356     - "git fetch https://gitlab.haskell.org/ghc/ghc-performance-notes.git refs/notes/perf:refs/notes/perf || true"
357     - bash .gitlab/win32-init.sh
358   after_script:
359     - rd /s /q tmp
360     - robocopy /np /nfl /ndl /e "%APPDATA%\cabal" cabal-cache
361     - bash -c 'make clean || true'
362   dependencies: []
363   variables:
364     FORCE_SYMLINKS: 1
365   cache:
366     paths:
367       - cabal-cache
368       - ghc-8.6.2
369       - ghc-tarballs
370
371 validate-x86_64-windows-hadrian:
372   extends: .validate-windows
373   stage: full-build
374   variables:
375     GHC_VERSION: "8.6.2"
376     LANG: "en_US.UTF-8"
377   script:
378     - |
379       set MSYSTEM=MINGW64
380       python boot
381       bash -c './configure --enable-tarballs-autodownload GHC=`pwd`/toolchain/bin/ghc HAPPY=`pwd`/toolchain/bin/happy ALEX=`pwd`/toolchain/bin/alex'
382       mkdir -p _build
383       cp -R inplace/mingw _build/mingw
384     - bash -c "PATH=`pwd`/toolchain/bin:$PATH hadrian/build.cabal.sh -j`mk/detect-cpu-count.sh` --flavour=Quick"
385     # FIXME: Bindist disabled due to #16073
386     #- bash -c "PATH=`pwd`/toolchain/bin:$PATH hadrian/build.cabal.sh binary-dist"
387     # FIXME: Testsuite disabled due to #16156.
388     #- bash -c 'make V=0 test THREADS=`mk/detect-cpu-count.sh` JUNIT_FILE=../../junit.xml'
389   cache:
390     key: x86_64-windows
391   tags:
392     - x86_64-windows
393
394 validate-x86_64-windows:
395   extends: .validate-windows
396   stage: full-build
397   # due to #16084
398   allow_failure: true
399   variables:
400     GHC_VERSION: "8.6.2"
401     LANG: "en_US.UTF-8"
402   script:
403     - |
404       set MSYSTEM=MINGW64
405       python boot
406       bash -c './configure --enable-tarballs-autodownload GHC=`pwd`/toolchain/bin/ghc HAPPY=`pwd`/toolchain/bin/happy ALEX=`pwd`/toolchain/bin/alex'
407     - bash -c "echo include mk/flavours/quick.mk > mk/build.mk"
408     - bash -c "PATH=`pwd`/toolchain/bin:$PATH make -j`mk/detect-cpu-count.sh`"
409     - |
410       bash -c "make binary-dist TAR_COMP_OPTS=-1"
411       mv ghc-*.tar.xz ghc.tar.xz
412     - bash -c 'make V=0 test THREADS=`mk/detect-cpu-count.sh` JUNIT_FILE=../../junit.xml'
413   cache:
414     key: x86_64-windows
415   tags:
416     - x86_64-windows
417   artifacts:
418     when: always
419     reports:
420       junit: junit.xml
421     paths:
422       - ghc.tar.xz
423       - junit.xml
424
425 # Note [Cleaning up after shell executor]
426 # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
427 #
428 # As noted in [1], gitlab-runner's shell executor doesn't clean up its working
429 # directory after builds. Unfortunately, we are forced to use the shell executor
430 # on Windows. To avoid running out of disk space we add a stage at the end of
431 # the build to remove the \GitLabRunner\builds directory. Since we only run a
432 # single build at a time on Windows this should be safe.
433 #
434 # [1] https://gitlab.com/gitlab-org/gitlab-runner/issues/3856
435
436 # See Note [Cleanup after shell executor]
437 cleanup-windows:
438   stage: cleanup
439   tags:
440     - x86_64-windows
441   when: always
442   dependencies: []
443   before_script:
444     - echo "Time to clean up"
445   script:
446     - echo "Let's go"
447   after_script:
448     - set "BUILD_DIR=%CI_PROJECT_DIR%"
449     - set "BUILD_DIR=%BUILD_DIR:/=\%"
450     - echo "Cleaning %BUILD_DIR%"
451     - cd \GitLabRunner
452     # This is way more complicated than it should be:
453     # See https://stackoverflow.com/questions/1965787
454     - del %BUILD_DIR%\* /F /Q
455     - for /d %%p in (%BUILD_DIR%\*) do rd /Q /S "%%p"
456     - exit /b 0
457
458 # See Note [Cleanup after shell executor]
459 cleanup-darwin:
460   stage: cleanup
461   tags:
462     - x86_64-darwin
463   when: always
464   dependencies: []
465   before_script:
466     - echo "Time to clean up"
467   script:
468     - echo "Let's go"
469   after_script:
470     - BUILD_DIR=$CI_PROJECT_DIR
471     - echo "Cleaning $BUILD_DIR"
472     - cd $HOME
473     - rm -Rf $BUILD_DIR/*
474     - exit 0
475