gitlab-ci: Cleanup Windows builds
[ghc.git] / .gitlab-ci.yml
index 8ddabfc..162c19c 100644 (file)
@@ -1,10 +1,29 @@
 variables:
   GIT_SSL_NO_VERIFY: "1"
-  GIT_SUBMODULE_STRATEGY: "recursive"
+
+before_script:
+  - python3 .gitlab/fix-submodules.py
+  - git submodule sync --recursive
+  - git submodule update --init --recursive
+  - git checkout .gitmodules
 
 stages:
   - lint
   - build
+  - full-build
+  - cleanup # See Note [Cleanup on Windows]
+
+############################################################
+# Runner Tags
+############################################################
+#
+# * x86_64-linux: Any Docker-capable x86_64 Linux machine
+# * aarch64-linux: Any Docker-capable AArch64 Linux machine
+# * x86_64-windows: A x86_64 Windows machine
+# * lint: Any Docker-capable x86_64 Linux machine; distinct from
+#         x86_64-linux to ensure low-latency availability.
+#
+
 
 ############################################################
 # Linting
@@ -23,15 +42,17 @@ ghc-linters:
         .gitlab/linters/check-makefiles.py $base $CI_COMMIT_SHA
         .gitlab/linters/check-cpp.py $base $CI_COMMIT_SHA
       fi
+  tags:
+    - lint
 
 ############################################################
 # Validation via Pipelines (hadrian)
 ############################################################
 
 .validate-hadrian:
-  stage: build
   allow_failure: true
   script:
+    - git clean -xdf && git submodule foreach git clean -xdf
     - bash .circleci/prepare-system.sh
     - if [[ -d ./cabal-cache ]]; then cp -R ./.cabal-cache ~/.cabal-cache; fi
     - ./boot
@@ -44,10 +65,15 @@ ghc-linters:
 
 validate-x86_64-linux-deb8-hadrian:
   extends: .validate-hadrian
+  stage: build
   image: ghcci/x86_64-linux-deb8:0.1
   before_script:
     # workaround for docker permissions
     - sudo chown ghc:ghc -R .
+    - python3 .gitlab/fix-submodules.py
+    - git submodule sync --recursive
+    - git submodule update --init --recursive
+    - git checkout .gitmodules
   tags:
     - x86_64-linux
 
@@ -56,11 +82,11 @@ validate-x86_64-linux-deb8-hadrian:
 ############################################################
 
 .validate:
-  stage: build
   variables:
     TEST_TYPE: test
+  before_script:
+    - git clean -xdf && git submodule foreach git clean -xdf
   script:
-    - make clean || true
     - ./boot
     - ./configure $CONFIGURE_ARGS
     - |
@@ -82,6 +108,7 @@ validate-x86_64-linux-deb8-hadrian:
 
 validate-x86_64-darwin:
   extends: .validate
+  stage: full-build
   allow_failure: true
   tags:
     - x86_64-darwin
@@ -90,11 +117,16 @@ validate-x86_64-darwin:
     MACOSX_DEPLOYMENT_TARGET: "10.7"
     # Only Sierra and onwards supports clock_gettime. See #12858
     ac_cv_func_clock_gettime: "no"
+    LANG: "en_US.UTF-8"
   before_script:
+    - git clean -xdf && git submodule foreach git clean -xdf
+    - python3 .gitlab/fix-submodules.py
+    - git submodule sync --recursive
+    - git submodule update --init --recursive
+    - git checkout .gitmodules
+
     - bash .gitlab/darwin-init.sh
     - PATH="`pwd`/toolchain/bin:$PATH"
-      # Disable sphinx PDF output as MacTeX apparently doesn't provide xelatex
-    - echo "BUILD_SPHINX_PDF=NO" >> mk/build.mk
     - echo "libraries/integer-gmp_CONFIGURE_OPTS += --configure-option=--with-intree-gmp" >> mk/build.mk
   after_script:
     - cp -Rf $HOME/.cabal cabal-cache
@@ -112,6 +144,12 @@ validate-x86_64-darwin:
   tags:
     - x86_64-linux
   before_script:
+    - git clean -xdf && git submodule foreach git clean -xdf
+    - python3 .gitlab/fix-submodules.py
+    - git submodule sync --recursive
+    - git submodule update --init --recursive
+    - git checkout .gitmodules
+
     - bash .circleci/prepare-system.sh
     # workaround for docker permissions
     - sudo chown ghc:ghc -R .
@@ -125,6 +163,7 @@ validate-x86_64-darwin:
 
 validate-aarch64-linux-deb9:
   extends: .validate-linux
+  stage: full-build
   image: ghcci/aarch64-linux-deb9:0.1
   allow_failure: true
   artifacts:
@@ -137,6 +176,7 @@ validate-aarch64-linux-deb9:
 
 nightly-aarch64-linux-deb9:
   extends: validate-aarch64-linux-deb9
+  stage: full-build
   artifacts:
     expire_in: 2 year
   variables:
@@ -145,8 +185,36 @@ nightly-aarch64-linux-deb9:
     variables:
       - $NIGHTLY
 
+validate-i386-linux-deb9:
+  extends: .validate-linux
+  stage: full-build
+  image: ghcci/i386-linux-deb9:0.1
+  allow_failure: true
+  artifacts:
+    when: always
+    expire_in: 2 week
+  cache:
+    key: linux-i386-deb9
+
+nightly-i386-linux-deb9:
+  extends: .validate-linux
+  stage: full-build
+  image: ghcci/i386-linux-deb9:0.1
+  allow_failure: true
+  variables:
+    TEST_TYPE: slowtest
+  artifacts:
+    when: always
+    expire_in: 2 week
+  only:
+    variables:
+      - $NIGHTLY
+  cache:
+    key: linux-i386-deb9
+
 validate-x86_64-linux-deb9:
   extends: .validate-linux
+  stage: build
   image: ghcci/x86_64-linux-deb9:0.2
   artifacts:
     when: always
@@ -156,6 +224,7 @@ validate-x86_64-linux-deb9:
 
 nightly-x86_64-linux-deb9:
   extends: validate-x86_64-linux-deb9
+  stage: build
   artifacts:
     expire_in: 2 year
   variables:
@@ -166,6 +235,7 @@ nightly-x86_64-linux-deb9:
 
 validate-x86_64-linux-deb9-llvm:
   extends: .validate-linux
+  stage: full-build
   allow_failure: true
   image: ghcci/x86_64-linux-deb9:0.2
   variables:
@@ -175,6 +245,7 @@ validate-x86_64-linux-deb9-llvm:
 
 validate-x86_64-linux-deb8:
   extends: .validate-linux
+  stage: full-build
   image: ghcci/x86_64-linux-deb8:0.1
   cache:
     key: linux-x86_64-deb8
@@ -184,6 +255,7 @@ validate-x86_64-linux-deb8:
 
 validate-x86_64-linux-fedora27:
   extends: .validate-linux
+  stage: full-build
   image: ghcci/x86_64-linux-fedora27:0.1
   cache:
     key: linux-x86_64-fedora27
@@ -193,7 +265,7 @@ validate-x86_64-linux-fedora27:
 
 validate-x86_64-linux-deb9-integer-simple:
   extends: .validate-linux
-  allow_failure: true
+  stage: full-build
   variables:
     INTEGER_LIBRARY: integer-simple
   image: ghcci/x86_64-linux-deb9:0.2
@@ -202,6 +274,7 @@ validate-x86_64-linux-deb9-integer-simple:
 
 nightly-x86_64-linux-deb9-integer-simple:
   extends: validate-x86_64-linux-deb9-integer-simple
+  stage: full-build
   artifacts:
     expire_in: 2 year
   variables:
@@ -212,6 +285,7 @@ nightly-x86_64-linux-deb9-integer-simple:
 
 validate-x86_64-linux-deb9-unreg:
   extends: .validate-linux
+  stage: full-build
   variables:
     CONFIGURE_ARGS: --enable-unregisterised
   image: ghcci/x86_64-linux-deb9:0.2
@@ -222,58 +296,108 @@ validate-x86_64-linux-deb9-unreg:
 # Validation via Pipelines (Windows)
 ############################################################
 
+.validate-windows:
+  before_script:
+    - git clean -xdf
+    - git submodule foreach git clean -xdf
+
+    # Use a local temporary directory to ensure that concurrent builds don't
+    # interfere with one another
+    - |
+      mkdir tmp
+      set TMP=%cd%\tmp
+      set TEMP=%cd%\tmp
+
+    - set PATH=C:\msys64\usr\bin;%PATH%
+    - python .gitlab/fix-submodules.py
+    - git submodule sync --recursive
+    - git submodule update --init --recursive
+    - git checkout .gitmodules
+    - bash .gitlab/win32-init.sh
+  after_script:
+    - rd /s /q tmp
+    - robocopy /np /nfl /ndl /e "%APPDATA%\cabal" cabal-cache
+    - bash -c 'make clean || true'
+  cache:
+    paths:
+      - cabal-cache
+      - ghc-8.6.2
+      - ghc-tarballs
+
 validate-x86_64-windows-hadrian:
-  stage: build
+  extends: .validate-windows
+  stage: full-build
+  # due to #16073
+  allow_failure: true
   variables:
     GHC_VERSION: "8.6.2"
+    LANG: "en_US.UTF-8"
   script:
-    - PATH=C:\msys64\usr\bin;%PATH%
-    - bash .gitlab/win32-init.sh
     - |
       set MSYSTEM=MINGW64
       python boot
-      bash -c './configure --with-ghc=`pwd`/toolchain/bin/ghc --enable-tarballs-autodownload HappyCmd=`pwd`/toolchain/bin/happy AlexCmd=`pwd`/toolchain/bin/alex'
+      bash -c './configure --enable-tarballs-autodownload GHC=`pwd`/toolchain/bin/ghc HAPPY=`pwd`/toolchain/bin/happy ALEX=`pwd`/toolchain/bin/alex'
       mkdir -p _build
       cp -R inplace/mingw _build/mingw
     # FIXME: --no-lint due to #15950
     - bash -c "PATH=`pwd`/toolchain/bin:$PATH hadrian/build.cabal.sh -j`mk/detect-cpu-count.sh` --flavour=Quick --no-lint"
     - bash -c "PATH=`pwd`/toolchain/bin:$PATH hadrian/build.cabal.sh binary-dist"
-    - mv ghc-*.tar.xz ghc.tar.xz
-    - bash -c 'make V=0 -j`mk/detect-cpu-count.sh` HADDOCK_DOCS=NO test THREADS=$THREADS JUNIT_FILE=../../junit.xml'
-    - cp -Rf $APPDATA/cabal cabal-cache
+    - bash -c 'make V=0 test THREADS=`mk/detect-cpu-count.sh` JUNIT_FILE=../../junit.xml'
   cache:
-    key: windows-hadrian
-    paths:
-      - cabal-cache
-      - ghc-8.6.2
-      - ghc-tarballs
+    key: x86_64-windows
   tags:
     - x86_64-windows
 
 validate-x86_64-windows:
-  stage: build
+  extends: .validate-windows
+  stage: full-build
+  # due to #16084
+  allow_failure: true
   variables:
     GHC_VERSION: "8.6.2"
+    LANG: "en_US.UTF-8"
   script:
-    - PATH=C:\msys64\usr\bin;%PATH%
-    - bash .gitlab/win32-init.sh
     - |
       set MSYSTEM=MINGW64
       python boot
-      bash -c './configure --with-ghc=`pwd`/toolchain/bin/ghc --enable-tarballs-autodownload HappyCmd=`pwd`/toolchain/bin/happy AlexCmd=`pwd`/toolchain/bin/alex'
+      bash -c './configure --enable-tarballs-autodownload GHC=`pwd`/toolchain/bin/ghc HAPPY=`pwd`/toolchain/bin/happy ALEX=`pwd`/toolchain/bin/alex'
+    - bash -c "echo include mk/flavours/quick.mk > mk/build.mk"
     - bash -c "PATH=`pwd`/toolchain/bin:$PATH make -j`mk/detect-cpu-count.sh`"
-    - mv ghc-*.tar.xz ghc.tar.xz
-    - bash -c 'make V=0 -j`mk/detect-cpu-count.sh` HADDOCK_DOCS=NO test THREADS=$THREADS JUNIT_FILE=../../junit.xml'
-    - cp -Rf $APPDATA/cabal cabal-cache
+    - bash -c 'make V=0 test THREADS=`mk/detect-cpu-count.sh` JUNIT_FILE=../../junit.xml'
   cache:
-    key: windows
-    paths:
-      - cabal-cache
-      - ghc-8.6.2
-      - ghc-tarballs
+    key: x86_64-windows
   tags:
     - x86_64-windows
 
+# Note [Cleanup on Windows]
+# ~~~~~~~~~~~~~~~~~~~~~~~~~
+#
+# As noted in [1], gitlab-runner's shell executor doesn't clean up its working
+# directory after builds. Unfortunately, we are forced to use the shell executor
+# on Windows. To avoid running out of disk space we add a stage at the end of
+# the build to remove the \GitLabRunner\builds directory. Since we only run a
+# single build at a time on Windows this should be safe.
+#
+# [1] https://gitlab.com/gitlab-org/gitlab-runner/issues/3856
+
+# See Note [Cleanup on Windows]
+cleanup-windows:
+  stage: cleanup
+  tags:
+    - x86_64-windows
+  when: always
+  before_script:
+    - echo "Time to clean up"
+  script:
+    - echo "Let's go"
+  after_script:
+    - set "BUILD_DIR=%CI_PROJECT_DIR%"
+    - set "BUILD_DIR=%BUILD_DIR:/=\%"
+    - echo "Cleaning %BUILD_DIR%"
+    - cd \GitLabRunner
+    - rmdir /S /Q %BUILD_DIR%/*
+    - exit /b 0
+
 ############################################################
 # Validation via CircleCI
 ############################################################
@@ -298,18 +422,8 @@ validate-x86_64-windows:
 # This requires updating the maximum artifacts size limit in Gitlab to
 # something like 200MB.
 
-circleci-validate-x86_64-linux-deb8:
-  extends: .circleci
-  script: ".gitlab/circle-ci-job.sh validate-x86_64-linux-deb8"
-
-circleci-validate-i386-linux-deb8:
-  extends: .circleci
-  script: ".gitlab/circle-ci-job.sh validate-i386-linux-deb8"
-
-# circleci-validate-x86_64-freebsd:
-#   extends: .circleci
-#   script: ".gitlab/circle-ci-job.sh validate-x86_64-freebsd"
-
-circleci-validate-x86_64-darwin:
+.circleci-validate-x86_64-darwin:
   extends: .circleci
+  stage: full-build
+  allow_failure: true
   script: ".gitlab/circle-ci-job.sh validate-x86_64-darwin"