Make CI via Hadrian build docs
[ghc.git] / .gitlab-ci.yml
1 variables:
2   GIT_SSL_NO_VERIFY: "1"
3
4 before_script:
5   - python3 .gitlab/fix-submodules.py
6   - git submodule sync --recursive
7   - git submodule update --init --recursive
8   - git checkout .gitmodules
9
10 stages:
11   - lint
12   - build
13   - full-build
14   - cleanup # See Note [Cleanup on Windows]
15
16 ############################################################
17 # Runner Tags
18 ############################################################
19 #
20 # * x86_64-linux: Any Docker-capable x86_64 Linux machine
21 # * aarch64-linux: Any Docker-capable AArch64 Linux machine
22 # * x86_64-windows: A x86_64 Windows machine
23 # * lint: Any Docker-capable x86_64 Linux machine; distinct from
24 #         x86_64-linux to ensure low-latency availability.
25 #
26
27
28 ############################################################
29 # Linting
30 ############################################################
31
32 ghc-linters:
33   stage: lint
34   image: ghcci/linters:0.1
35   script:
36     - |
37       if [ -n "$CI_MERGE_REQUEST_ID" ]; then
38         base="$(git merge-base $CI_MERGE_REQUEST_BRANCH_NAME HEAD)"
39         validate-commit-msg .git $(git rev-list $base..$CI_COMMIT_SHA)
40         submodchecker .git $(git rev-list $base..$CI_COMMIT_SHA)
41         validate-whitespace .git $(git rev-list $base..$CI_COMMIT_SHA)
42         .gitlab/linters/check-makefiles.py $base $CI_COMMIT_SHA
43         .gitlab/linters/check-cpp.py $base $CI_COMMIT_SHA
44       fi
45   dependencies: []
46   tags:
47     - lint
48
49 ############################################################
50 # Validation via Pipelines (hadrian)
51 ############################################################
52
53 .validate-hadrian:
54   allow_failure: true
55   script:
56     - git clean -xdf && git submodule foreach git clean -xdf
57     - bash .circleci/prepare-system.sh
58     - if [[ -d ./cabal-cache ]]; then cp -R ./.cabal-cache ~/.cabal-cache; fi
59     - ./boot
60     - ./configure $CONFIGURE_ARGS
61     - hadrian/build.cabal.sh -j`mk/detect-cpu-count.sh`
62     - hadrian/build.cabal.sh -j`mk/detect-cpu-count.sh` docs-haddock
63   cache:
64     key: hadrian
65     paths:
66       - cabal-cache
67
68 validate-x86_64-linux-deb8-hadrian:
69   extends: .validate-hadrian
70   stage: build
71   image: ghcci/x86_64-linux-deb8:0.1
72   before_script:
73     # workaround for docker permissions
74     - sudo chown ghc:ghc -R .
75     - python3 .gitlab/fix-submodules.py
76     - git submodule sync --recursive
77     - git submodule update --init --recursive
78     - git checkout .gitmodules
79   tags:
80     - x86_64-linux
81
82 ############################################################
83 # Validation via Pipelines (make)
84 ############################################################
85
86 .validate:
87   variables:
88     TEST_TYPE: test
89   before_script:
90     - git clean -xdf && git submodule foreach git clean -xdf
91   script:
92     - ./boot
93     - ./configure $CONFIGURE_ARGS
94     - |
95       THREADS=`mk/detect-cpu-count.sh`
96       make V=0 -j$THREADS WERROR=-Werror 
97     - |
98       make binary-dist TAR_COMP_OPTS="-1"
99       mv ghc-*.tar.xz ghc.tar.xz
100     - |
101       THREADS=`mk/detect-cpu-count.sh`
102       make $TEST_TYPE THREADS=$THREADS JUNIT_FILE=../../junit.xml
103   dependencies: []
104   artifacts:
105     reports:
106       junit: junit.xml
107     expire_in: 2 week
108     paths:
109       - ghc.tar.xz
110       - junit.xml
111
112 validate-x86_64-darwin:
113   extends: .validate
114   stage: full-build
115   tags:
116     - x86_64-darwin
117   variables:
118     GHC_VERSION: 8.6.3
119     MACOSX_DEPLOYMENT_TARGET: "10.7"
120     # Only Sierra and onwards supports clock_gettime. See #12858
121     ac_cv_func_clock_gettime: "no"
122     LANG: "en_US.UTF-8"
123     CONFIGURE_ARGS: --with-intree-gmp
124   before_script:
125     - git clean -xdf && git submodule foreach git clean -xdf
126     - python3 .gitlab/fix-submodules.py
127     - git submodule sync --recursive
128     - git submodule update --init --recursive
129     - git checkout .gitmodules
130
131     - bash .gitlab/darwin-init.sh
132     - PATH="`pwd`/toolchain/bin:$PATH"
133   after_script:
134     - cp -Rf $HOME/.cabal cabal-cache
135   artifacts:
136     when: always
137     expire_in: 2 week
138   cache:
139     key: darwin
140     paths:
141       - cabal-cache
142       - toolchain
143
144 .validate-linux:
145   extends: .validate
146   tags:
147     - x86_64-linux
148   before_script:
149     - git clean -xdf && git submodule foreach git clean -xdf
150     - python3 .gitlab/fix-submodules.py
151     - git submodule sync --recursive
152     - git submodule update --init --recursive
153     - git checkout .gitmodules
154
155     - bash .circleci/prepare-system.sh
156     # workaround for docker permissions
157     - sudo chown ghc:ghc -R .
158   after_script:
159     - cp -Rf $HOME/.cabal cabal-cache
160   cache:
161     key: linux
162     paths:
163       - cabal-cache
164       - toolchain
165
166 validate-aarch64-linux-deb9:
167   extends: .validate-linux
168   stage: full-build
169   image: ghcci/aarch64-linux-deb9:0.1
170   allow_failure: true
171   artifacts:
172     when: always
173     expire_in: 2 week
174   cache:
175     key: linux-aarch64-deb9
176   tags:
177     - aarch64-linux
178
179 nightly-aarch64-linux-deb9:
180   extends: validate-aarch64-linux-deb9
181   stage: full-build
182   artifacts:
183     expire_in: 2 year
184   variables:
185     TEST_TYPE: slowtest
186   only:
187     variables:
188       - $NIGHTLY
189
190 validate-i386-linux-deb9:
191   extends: .validate-linux
192   stage: full-build
193   image: ghcci/i386-linux-deb9:0.1
194   allow_failure: true
195   artifacts:
196     when: always
197     expire_in: 2 week
198   cache:
199     key: linux-i386-deb9
200
201 nightly-i386-linux-deb9:
202   extends: .validate-linux
203   stage: full-build
204   image: ghcci/i386-linux-deb9:0.1
205   allow_failure: true
206   variables:
207     TEST_TYPE: slowtest
208   artifacts:
209     when: always
210     expire_in: 2 week
211   only:
212     variables:
213       - $NIGHTLY
214   cache:
215     key: linux-i386-deb9
216
217 # N.B. Has DEBUG assertions enabled in stage2
218 validate-x86_64-linux-deb9-debug:
219   extends: .validate-linux
220   stage: build
221   image: ghcci/x86_64-linux-deb9:0.2
222   variables:
223     BUILD_FLAVOUR: devel2
224   cache:
225     key: linux-x86_64-deb9
226
227 validate-x86_64-linux-deb9:
228   extends: .validate-linux
229   stage: build
230   image: ghcci/x86_64-linux-deb9:0.2
231   artifacts:
232     when: always
233     expire_in: 2 week
234   cache:
235     key: linux-x86_64-deb9
236
237 nightly-x86_64-linux-deb9:
238   extends: validate-x86_64-linux-deb9
239   stage: build
240   artifacts:
241     expire_in: 2 year
242   variables:
243     TEST_TYPE: slowtest
244   only:
245     variables:
246       - $NIGHTLY
247
248 validate-x86_64-linux-deb9-llvm:
249   extends: .validate-linux
250   stage: full-build
251   image: ghcci/x86_64-linux-deb9:0.2
252   variables:
253     BUILD_FLAVOUR: perf-llvm
254   cache:
255     key: linux-x86_64-deb9
256
257 validate-x86_64-linux-deb8:
258   extends: .validate-linux
259   stage: full-build
260   image: ghcci/x86_64-linux-deb8:0.1
261   cache:
262     key: linux-x86_64-deb8
263   artifacts:
264     when: always
265     expire_in: 2 week
266
267 validate-x86_64-linux-fedora27:
268   extends: .validate-linux
269   stage: full-build
270   image: ghcci/x86_64-linux-fedora27:0.1
271   cache:
272     key: linux-x86_64-fedora27
273   artifacts:
274     when: always
275     expire_in: 2 week
276
277 validate-x86_64-linux-deb9-integer-simple:
278   extends: .validate-linux
279   stage: full-build
280   variables:
281     INTEGER_LIBRARY: integer-simple
282   image: ghcci/x86_64-linux-deb9:0.2
283   cache:
284     key: linux-x86_64-deb9
285
286 nightly-x86_64-linux-deb9-integer-simple:
287   extends: validate-x86_64-linux-deb9-integer-simple
288   stage: full-build
289   artifacts:
290     expire_in: 2 year
291   variables:
292     TEST_TYPE: slowtest
293   only:
294     variables:
295       - $NIGHTLY
296
297 validate-x86_64-linux-deb9-unreg:
298   extends: .validate-linux
299   stage: full-build
300   variables:
301     CONFIGURE_ARGS: --enable-unregisterised
302   image: ghcci/x86_64-linux-deb9:0.2
303   cache:
304     key: linux-x86_64-deb9
305
306 ############################################################
307 # Validation via Pipelines (Windows)
308 ############################################################
309
310 .validate-windows:
311   before_script:
312     - git clean -xdf
313     - git submodule foreach git clean -xdf
314
315     # Use a local temporary directory to ensure that concurrent builds don't
316     # interfere with one another
317     - |
318       mkdir tmp
319       set TMP=%cd%\tmp
320       set TEMP=%cd%\tmp
321
322     - set PATH=C:\msys64\usr\bin;%PATH%
323     - python .gitlab/fix-submodules.py
324     - git submodule sync --recursive
325     - git submodule update --init --recursive
326     - git checkout .gitmodules
327     - bash .gitlab/win32-init.sh
328   after_script:
329     - rd /s /q tmp
330     - robocopy /np /nfl /ndl /e "%APPDATA%\cabal" cabal-cache
331     - bash -c 'make clean || true'
332   dependencies: []
333   cache:
334     paths:
335       - cabal-cache
336       - ghc-8.6.2
337       - ghc-tarballs
338
339 validate-x86_64-windows-hadrian:
340   extends: .validate-windows
341   stage: full-build
342   variables:
343     GHC_VERSION: "8.6.2"
344     LANG: "en_US.UTF-8"
345   script:
346     - |
347       set MSYSTEM=MINGW64
348       python boot
349       bash -c './configure --enable-tarballs-autodownload GHC=`pwd`/toolchain/bin/ghc HAPPY=`pwd`/toolchain/bin/happy ALEX=`pwd`/toolchain/bin/alex'
350       mkdir -p _build
351       cp -R inplace/mingw _build/mingw
352     - bash -c "PATH=`pwd`/toolchain/bin:$PATH hadrian/build.cabal.sh -j`mk/detect-cpu-count.sh` --flavour=Quick"
353     # FIXME: Bindist disabled due to #16073
354     #- bash -c "PATH=`pwd`/toolchain/bin:$PATH hadrian/build.cabal.sh binary-dist"
355     # FIXME: Testsuite disabled due to #16156.
356     #- bash -c 'make V=0 test THREADS=`mk/detect-cpu-count.sh` JUNIT_FILE=../../junit.xml'
357   cache:
358     key: x86_64-windows
359   tags:
360     - x86_64-windows
361
362 validate-x86_64-windows:
363   extends: .validate-windows
364   stage: full-build
365   # due to #16084
366   allow_failure: true
367   variables:
368     GHC_VERSION: "8.6.2"
369     LANG: "en_US.UTF-8"
370   script:
371     - |
372       set MSYSTEM=MINGW64
373       python boot
374       bash -c './configure --enable-tarballs-autodownload GHC=`pwd`/toolchain/bin/ghc HAPPY=`pwd`/toolchain/bin/happy ALEX=`pwd`/toolchain/bin/alex'
375     - bash -c "echo include mk/flavours/quick.mk > mk/build.mk"
376     - bash -c "PATH=`pwd`/toolchain/bin:$PATH make -j`mk/detect-cpu-count.sh`"
377     - |
378       bash -c "make binary-dist TAR_COMP_OPTS=-1"
379       mv ghc-*.tar.xz ghc.tar.xz
380     - bash -c 'make V=0 test THREADS=`mk/detect-cpu-count.sh` JUNIT_FILE=../../junit.xml'
381   cache:
382     key: x86_64-windows
383   tags:
384     - x86_64-windows
385   artifacts:
386     when: always
387     reports:
388       junit: junit.xml
389     paths:
390       - ghc.tar.xz
391       - junit.xml
392
393 # Note [Cleaning up after shell executor]
394 # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
395 #
396 # As noted in [1], gitlab-runner's shell executor doesn't clean up its working
397 # directory after builds. Unfortunately, we are forced to use the shell executor
398 # on Windows. To avoid running out of disk space we add a stage at the end of
399 # the build to remove the \GitLabRunner\builds directory. Since we only run a
400 # single build at a time on Windows this should be safe.
401 #
402 # [1] https://gitlab.com/gitlab-org/gitlab-runner/issues/3856
403
404 # See Note [Cleanup after shell executor]
405 cleanup-windows:
406   stage: cleanup
407   tags:
408     - x86_64-windows
409   when: always
410   dependencies: []
411   before_script:
412     - echo "Time to clean up"
413   script:
414     - echo "Let's go"
415   after_script:
416     - set "BUILD_DIR=%CI_PROJECT_DIR%"
417     - set "BUILD_DIR=%BUILD_DIR:/=\%"
418     - echo "Cleaning %BUILD_DIR%"
419     - cd \GitLabRunner
420     # This is way more complicated than it should be:
421     # See https://stackoverflow.com/questions/1965787
422     - del %BUILD_DIR%\* /F /Q
423     - for /d %%p in (%BUILD_DIR%\*) do rd /Q /S "%%p"
424     - exit /b 0
425
426 # See Note [Cleanup after shell executor]
427 cleanup-darwin:
428   stage: cleanup
429   tags:
430     - x86_64-darwin
431   when: always
432   before_script:
433     - echo "Time to clean up"
434   script:
435     - echo "Let's go"
436   after_script:
437     - BUILD_DIR=$CI_PROJECT_DIR
438     - echo "Cleaning $BUILD_DIR"
439     - cd $HOME
440     - rm -Rf $BUILD_DIR/*
441     - exit 0
442