gitlab-ci: Generate documentation tarball
[ghc.git] / .gitlab-ci.yml
1 variables:
2   GIT_SSL_NO_VERIFY: "1"
3
4   # Commit of ghc/ci-images repository from which to pull Docker images
5   DOCKER_REV: 6d19c3adc1f5c28c82aed8c5b1ac40931ac60f3f
6
7 before_script:
8   - python3 .gitlab/fix-submodules.py
9   - git submodule sync --recursive
10   - git submodule update --init --recursive
11   - git checkout .gitmodules
12   - "git fetch https://gitlab.haskell.org/ghc/ghc-performance-notes.git refs/notes/perf:refs/notes/perf || true"
13
14 stages:
15   - lint
16   - build
17   - full-build
18   - cleanup # See Note [Cleanup on Windows]
19   - packaging
20
21 ############################################################
22 # Runner Tags
23 ############################################################
24 #
25 # * x86_64-linux: Any Docker-capable x86_64 Linux machine
26 # * aarch64-linux: Any Docker-capable AArch64 Linux machine
27 # * x86_64-windows: A x86_64 Windows machine
28 # * lint: Any Docker-capable x86_64 Linux machine; distinct from
29 #         x86_64-linux to ensure low-latency availability.
30 #
31
32
33 ############################################################
34 # Linting
35 ############################################################
36
37 ghc-linters:
38   stage: lint
39   image: "registry.gitlab.haskell.org/ghc/ci-images/linters:$DOCKER_REV"
40   script:
41     - |
42       if [ -n "$CI_MERGE_REQUEST_ID" ]; then
43         base="$(git merge-base $CI_MERGE_REQUEST_BRANCH_NAME HEAD)"
44         validate-commit-msg .git $(git rev-list $base..$CI_COMMIT_SHA)
45         submodchecker .git $(git rev-list $base..$CI_COMMIT_SHA)
46         validate-whitespace .git $(git rev-list $base..$CI_COMMIT_SHA)
47         .gitlab/linters/check-makefiles.py $base $CI_COMMIT_SHA
48         .gitlab/linters/check-cpp.py $base $CI_COMMIT_SHA
49       fi
50   dependencies: []
51   tags:
52     - lint
53
54 ############################################################
55 # Validation via Pipelines (hadrian)
56 ############################################################
57
58 .validate-hadrian:
59   allow_failure: true
60   script:
61     - cabal update
62     - git clean -xdf && git submodule foreach git clean -xdf
63     - bash .circleci/prepare-system.sh
64     - if [[ -d ./cabal-cache ]]; then cp -R ./.cabal-cache ~/.cabal-cache; fi
65     - ./boot
66     - ./configure $CONFIGURE_ARGS
67     - hadrian/build.cabal.sh -j`mk/detect-cpu-count.sh` --docs=no-sphinx binary-dist
68     - mv _build/bindist/ghc*.tar.xz ghc.tar.xz
69   cache:
70     key: hadrian
71     paths:
72       - cabal-cache
73   artifacts:
74     when: always
75     paths:
76       - ghc.tar.xz
77
78 validate-x86_64-linux-deb8-hadrian:
79   extends: .validate-hadrian
80   stage: build
81   image: "registry.gitlab.haskell.org/ghc/ci-images/x86_64-linux-deb8:$DOCKER_REV"
82   before_script:
83     # workaround for docker permissions
84     - sudo chown ghc:ghc -R .
85     - python3 .gitlab/fix-submodules.py
86     - git submodule sync --recursive
87     - git submodule update --init --recursive
88     - git checkout .gitmodules
89     - "git fetch https://gitlab.haskell.org/ghc/ghc-performance-notes.git refs/notes/perf:refs/notes/perf || true"
90   tags:
91     - x86_64-linux
92
93 hadrian-ghc-in-ghci:
94   stage: build
95   image: ghcci/x86_64-linux-deb8:0.1
96   before_script:
97     # workaround for docker permissions
98     - sudo chown ghc:ghc -R .
99     - python3 .gitlab/fix-submodules.py
100     - git submodule sync --recursive
101     - git submodule update --init --recursive
102     - git checkout .gitmodules
103   tags:
104     - x86_64-linux
105   script:
106     - cabal update
107     - git clean -xdf && git submodule foreach git clean -xdf
108     - bash .circleci/prepare-system.sh
109     - if [[ -d ./cabal-cache ]]; then cp -R ./.cabal-cache ~/.cabal-cache; fi
110     - ./boot
111     - ./configure $CONFIGURE_ARGS
112     # Load ghc-in-ghci then immediately exit
113     - echo ":q" | hadrian/ghci.sh
114   cache:
115     key: hadrian-ghci
116     paths:
117       - cabal-cache
118
119 ############################################################
120 # Validation via Pipelines (make)
121 ############################################################
122
123 .validate:
124   variables:
125     TEST_TYPE: test
126   before_script:
127     - git clean -xdf && git submodule foreach git clean -xdf
128   script:
129     - ./boot
130     - ./configure $CONFIGURE_ARGS
131     - |
132       THREADS=`mk/detect-cpu-count.sh`
133       make V=0 -j$THREADS WERROR=-Werror
134     - |
135       make binary-dist TAR_COMP_OPTS="-1"
136     - |
137       # Prepare to push git notes.
138       METRICS_FILE=$(mktemp)
139       git config user.email "ben+ghc-ci@smart-cactus.org"
140       git config user.name "GHC GitLab CI"
141     - |
142       THREADS=`mk/detect-cpu-count.sh`
143       make $TEST_TYPE THREADS=$THREADS JUNIT_FILE=../../junit.xml METRICS_FILE=$METRICS_FILE
144     - |
145       # Push git notes.
146       METRICS_FILE=$METRICS_FILE .gitlab/push-test-metrics.sh
147   dependencies: []
148   artifacts:
149     reports:
150       junit: junit.xml
151     expire_in: 2 week
152     paths:
153       - ghc-*.tar.xz
154       - junit.xml
155
156 #################################
157 # x86_64-darwin
158 #################################
159
160 validate-x86_64-darwin:
161   extends: .validate
162   stage: full-build
163   tags:
164     - x86_64-darwin
165   variables:
166     GHC_VERSION: 8.6.3
167     BIN_DIST_PREP_TAR_COMP: "bindistprep/ghc-x86_64-apple-darwin.tar.xz"
168     MACOSX_DEPLOYMENT_TARGET: "10.7"
169     # Only Sierra and onwards supports clock_gettime. See #12858
170     ac_cv_func_clock_gettime: "no"
171     LANG: "en_US.UTF-8"
172     CONFIGURE_ARGS: --with-intree-gmp
173     TEST_ENV: "x86_64-darwin"
174   before_script:
175     - git clean -xdf && git submodule foreach git clean -xdf
176     - python3 .gitlab/fix-submodules.py
177     - git submodule sync --recursive
178     - git submodule update --init --recursive
179     - git checkout .gitmodules
180     - "git fetch https://gitlab.haskell.org/ghc/ghc-performance-notes.git refs/notes/perf:refs/notes/perf || true"
181
182     - bash .gitlab/darwin-init.sh
183     - PATH="`pwd`/toolchain/bin:$PATH"
184   after_script:
185     - cp -Rf $HOME/.cabal cabal-cache
186   artifacts:
187     when: always
188     expire_in: 2 week
189   cache:
190     key: darwin
191     paths:
192       - cabal-cache
193       - toolchain
194
195 .validate-linux:
196   extends: .validate
197   tags:
198     - x86_64-linux
199   before_script:
200     - git clean -xdf && git submodule foreach git clean -xdf
201     - python3 .gitlab/fix-submodules.py
202     - git submodule sync --recursive
203     - git submodule update --init --recursive
204     - git checkout .gitmodules
205     - "git fetch https://gitlab.haskell.org/ghc/ghc-performance-notes.git refs/notes/perf:refs/notes/perf || true"
206
207     - bash .circleci/prepare-system.sh
208     # workaround for docker permissions
209     - sudo chown ghc:ghc -R .
210   after_script:
211     - cp -Rf $HOME/.cabal cabal-cache
212   cache:
213     key: linux
214     paths:
215       - cabal-cache
216       - toolchain
217
218 #################################
219 # aarch64-linux-deb9
220 #################################
221
222 .build-aarch64-linux-deb9:
223   extends: .validate-linux
224   stage: full-build
225   image: "registry.gitlab.haskell.org/ghc/ci-images/aarch64-linux-deb9:$DOCKER_REV"
226   allow_failure: true
227   variables:
228     TEST_ENV: "aarch64-linux-deb9"
229     BIN_DIST_PREP_TAR_COMP: "bindistprep/ghc-aarch64-linux-deb9.tar.xz"
230   cache:
231     key: linux-aarch64-deb9
232   tags:
233     - aarch64-linux
234
235 validate-aarch64-linux-deb9:
236   extends: .build-aarch64-linux-deb9
237   artifacts:
238     when: always
239     expire_in: 2 week
240
241 nightly-aarch64-linux-deb9:
242   extends: .build-aarch64-linux-deb9
243   artifacts:
244     expire_in: 2 year
245   variables:
246     TEST_TYPE: slowtest
247   only:
248     variables:
249       - $NIGHTLY
250
251 #################################
252 # i386-linux-deb9
253 #################################
254
255 .build-i386-linux-deb9:
256   extends: .validate-linux
257   stage: full-build
258   image: "registry.gitlab.haskell.org/ghc/ci-images/i386-linux-deb9:$DOCKER_REV"
259   variables:
260     TEST_ENV: "i386-linux-deb9"
261     BIN_DIST_PREP_TAR_COMP: "bindistprep/ghc-i386-deb9-linux.tar.xz"
262   cache:
263     key: linux-i386-deb9
264
265 validate-i386-linux-deb9:
266   extends: .build-i386-linux-deb9
267   artifacts:
268     when: always
269     expire_in: 2 week
270
271 nightly-i386-linux-deb9:
272   extends: .build-i386-linux-deb9
273   variables:
274     TEST_TYPE: slowtest
275   artifacts:
276     when: always
277     expire_in: 2 week
278   only:
279     variables:
280       - $NIGHTLY
281
282 #################################
283 # x86_64-linux-deb9
284 #################################
285
286 .build-x86_64-linux-deb9:
287   extends: .validate-linux
288   stage: build
289   image: "registry.gitlab.haskell.org/ghc/ci-images/x86_64-linux-deb9:$DOCKER_REV"
290   variables:
291     TEST_ENV: "x86_64-linux-deb9"
292     BIN_DIST_PREP_TAR_COMP: "bindistprep/ghc-x86_64-deb9-linux.tar.xz"
293   cache:
294     key: linux-x86_64-deb9
295
296 validate-x86_64-linux-deb9:
297   extends: .build-x86_64-linux-deb9
298   artifacts:
299     when: always
300     expire_in: 2 week
301
302 nightly-x86_64-linux-deb9:
303   extends: .build-x86_64-linux-deb9
304   artifacts:
305     expire_in: 2 year
306   variables:
307     TEST_TYPE: slowtest
308   only:
309     variables:
310       - $NIGHTLY
311
312 # N.B. Has DEBUG assertions enabled in stage2
313 validate-x86_64-linux-deb9-debug:
314   extends: .build-x86_64-linux-deb9
315   variables:
316     BUILD_FLAVOUR: validate
317     TEST_ENV: "x86_64-linux-deb9-debug"
318
319 validate-x86_64-linux-deb9-llvm:
320   extends: .build-x86_64-linux-deb9
321   stage: full-build
322   variables:
323     BUILD_FLAVOUR: perf-llvm
324     TEST_ENV: "x86_64-linux-deb9-llvm"
325
326 validate-x86_64-linux-deb9-integer-simple:
327   extends: .build-x86_64-linux-deb9
328   stage: full-build
329   variables:
330     INTEGER_LIBRARY: integer-simple
331     TEST_ENV: "x86_64-linux-deb9-integer-simple"
332     BIN_DIST_PREP_TAR_COMP: "bindistprep/ghc-x86_64-deb9-linux-integer-simple.tar.xz"
333
334 nightly-x86_64-linux-deb9-integer-simple:
335   extends: .build-x86_64-linux-deb9
336   stage: full-build
337   variables:
338     INTEGER_LIBRARY: integer-simple
339     TEST_ENV: "x86_64-linux-deb9-integer-simple"
340     TEST_TYPE: slowtest
341   artifacts:
342     expire_in: 2 year
343   only:
344     variables:
345       - $NIGHTLY
346
347 validate-x86_64-linux-deb9-unreg:
348   extends: .build-x86_64-linux-deb9
349   stage: full-build
350   variables:
351     CONFIGURE_ARGS: --enable-unregisterised
352     TEST_ENV: "x86_64-linux-deb9-unreg"
353
354
355 #################################
356 # x86_64-linux-deb8
357 #################################
358
359 release-x86_64-linux-deb8:
360   extends: .validate-linux
361   stage: full-build
362   image: "registry.gitlab.haskell.org/ghc/ci-images/x86_64-linux-deb8:$DOCKER_REV"
363   variables:
364     TEST_ENV: "x86_64-linux-deb8"
365     BIN_DIST_PREP_TAR_COMP: "bindistprep/ghc-x86_64-deb8-linux.tar.xz"
366   only:
367     - tags
368   cache:
369     key: linux-x86_64-deb8
370   artifacts:
371     when: always
372     expire_in: 2 week
373
374
375 #################################
376 # x86_64-linux-fedora27
377 #################################
378
379 release-x86_64-linux-fedora27:
380   extends: .validate-linux
381   stage: full-build
382   image: "registry.gitlab.haskell.org/ghc/ci-images/x86_64-linux-fedora27:$DOCKER_REV"
383   variables:
384     TEST_ENV: "x86_64-linux-fedora27"
385     BIN_DIST_PREP_TAR_COMP: "bindistprep/ghc-x86_64-fedora27-linux.tar.xz"
386   only:
387     - tags
388   cache:
389     key: linux-x86_64-fedora27
390   artifacts:
391     when: always
392     expire_in: 2 week
393
394 ############################################################
395 # Validation via Pipelines (Windows)
396 ############################################################
397
398 .validate-windows:
399   before_script:
400     - git clean -xdf
401     - git submodule foreach git clean -xdf
402
403     # Use a local temporary directory to ensure that concurrent builds don't
404     # interfere with one another
405     - |
406       mkdir tmp
407       set TMP=%cd%\tmp
408       set TEMP=%cd%\tmp
409
410     - set PATH=C:\msys64\usr\bin;%PATH%
411     - python .gitlab/fix-submodules.py
412     - git submodule sync --recursive
413     - git submodule update --init --recursive
414     - git checkout .gitmodules
415     - "git fetch https://gitlab.haskell.org/ghc/ghc-performance-notes.git refs/notes/perf:refs/notes/perf || true"
416     - bash .gitlab/win32-init.sh
417   after_script:
418     - rd /s /q tmp
419     - robocopy /np /nfl /ndl /e "%APPDATA%\cabal" cabal-cache
420     - bash -c 'make clean || true'
421   dependencies: []
422   variables:
423     FORCE_SYMLINKS: 1
424   cache:
425     paths:
426       - cabal-cache
427       - ghc-8.6.2
428       - ghc-tarballs
429
430 validate-x86_64-windows-hadrian:
431   extends: .validate-windows
432   stage: full-build
433   variables:
434     GHC_VERSION: "8.6.2"
435     LANG: "en_US.UTF-8"
436   script:
437     - |
438       set MSYSTEM=MINGW64
439       python boot
440       bash -c './configure --enable-tarballs-autodownload GHC=`pwd`/toolchain/bin/ghc HAPPY=`pwd`/toolchain/bin/happy ALEX=`pwd`/toolchain/bin/alex'
441     - bash -c "PATH=`pwd`/toolchain/bin:$PATH hadrian/build.cabal.sh -j`mk/detect-cpu-count.sh` --flavour=Quick --docs=no-sphinx binary-dist"
442     - mv _build/bindist/ghc*.tar.xz ghc.tar.xz
443     # FIXME: Testsuite disabled due to #16156.
444     # - bash -c 'make V=0 test THREADS=`mk/detect-cpu-count.sh` JUNIT_FILE=../../junit.xml'
445   cache:
446     key: x86_64-windows
447   tags:
448     - x86_64-windows
449   artifacts:
450     when: always
451     paths:
452       - ghc.tar.xz
453
454 validate-x86_64-windows:
455   extends: .validate-windows
456   stage: full-build
457   # due to #16084
458   allow_failure: true
459   variables:
460     GHC_VERSION: "8.6.2"
461     LANG: "en_US.UTF-8"
462     BIN_DIST_PREP_TAR_COMP: "bindistprep/ghc-x86_64-mingw32.tar.xz"
463   script:
464     - |
465       set MSYSTEM=MINGW64
466       python boot
467       bash -c './configure --enable-tarballs-autodownload GHC=`pwd`/toolchain/bin/ghc HAPPY=`pwd`/toolchain/bin/happy ALEX=`pwd`/toolchain/bin/alex'
468     - bash -c "echo include mk/flavours/quick.mk > mk/build.mk"
469     - bash -c "PATH=`pwd`/toolchain/bin:$PATH make -j`mk/detect-cpu-count.sh`"
470     - bash -c "PATH=`pwd`/toolchain/bin:$PATH make binary-dist TAR_COMP_OPTS=-1"
471     - bash -c 'make V=0 test THREADS=`mk/detect-cpu-count.sh` JUNIT_FILE=../../junit.xml'
472   cache:
473     key: x86_64-windows
474   tags:
475     - x86_64-windows
476   artifacts:
477     when: always
478     reports:
479       junit: junit.xml
480     paths:
481       - ghc-*.tar.xz
482       - junit.xml
483
484 ############################################################
485 # Cleanup
486 ############################################################
487
488 # Note [Cleaning up after shell executor]
489 # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
490 #
491 # As noted in [1], gitlab-runner's shell executor doesn't clean up its working
492 # directory after builds. Unfortunately, we are forced to use the shell executor
493 # on Windows. To avoid running out of disk space we add a stage at the end of
494 # the build to remove the \GitLabRunner\builds directory. Since we only run a
495 # single build at a time on Windows this should be safe.
496 #
497 # [1] https://gitlab.com/gitlab-org/gitlab-runner/issues/3856
498
499 # See Note [Cleanup after shell executor]
500 cleanup-windows:
501   stage: cleanup
502   tags:
503     - x86_64-windows
504   when: always
505   dependencies: []
506   before_script:
507     - echo "Time to clean up"
508   script:
509     - echo "Let's go"
510   after_script:
511     - set "BUILD_DIR=%CI_PROJECT_DIR%"
512     - set "BUILD_DIR=%BUILD_DIR:/=\%"
513     - echo "Cleaning %BUILD_DIR%"
514     - cd \GitLabRunner
515     # This is way more complicated than it should be:
516     # See https://stackoverflow.com/questions/1965787
517     - del %BUILD_DIR%\* /F /Q
518     - for /d %%p in (%BUILD_DIR%\*) do rd /Q /S "%%p"
519     - exit /b 0
520
521 # See Note [Cleanup after shell executor]
522 cleanup-darwin:
523   stage: cleanup
524   tags:
525     - x86_64-darwin
526   when: always
527   dependencies: []
528   before_script:
529     - echo "Time to clean up"
530   script:
531     - echo "Let's go"
532   after_script:
533     - BUILD_DIR=$CI_PROJECT_DIR
534     - echo "Cleaning $BUILD_DIR"
535     - cd $HOME
536     - rm -Rf $BUILD_DIR/*
537     - exit 0
538
539 ############################################################
540 # Packaging
541 ############################################################
542
543 doc-tarball:
544   stage: packaging
545   tags:
546     - x86_64-linux
547   image: ghcci/x86_64-linux-deb9:0.2
548   dependencies:
549     - validate-x86_64-linux-deb9
550     - validate-x86_64-windows
551   artifacts:
552     paths:
553       - haddock.html.tar.xz
554       - libraries.html.tar.xz
555       - users_guide.html.tar.xz
556       - index.html
557       - "*.pdf"
558   script:
559     - rm -Rf docs
560     - distrib/mkDocs/mkDocs ghc-x86_64-deb9-linux.tar.xz ghc-x86_64-mingw32.tar.xz
561     - ls -lh
562     - mv docs/*.tar.xz .
563