testsuite: Ensure T5423 flushes C output buffer
[ghc.git] / .gitlab-ci.yml
index 07a1d47..6eb9d65 100644 (file)
@@ -1,17 +1,40 @@
 variables:
   GIT_SSL_NO_VERIFY: "1"
 
+  # Commit of ghc/ci-images repository from which to pull Docker images
+  DOCKER_REV: 88e952f165f48cfb956ac9a2486a9263aa4f777c
+
+  # Sequential version number capturing the versions of all tools fetched by
+  # .gitlab/win32-init.sh.
+  WINDOWS_TOOLCHAIN_VERSION: 1
+
+  # Disable shallow clones; they break our linting rules
+  GIT_DEPTH: 0
+
 before_script:
   - python3 .gitlab/fix-submodules.py
   - git submodule sync --recursive
   - git submodule update --init --recursive
   - git checkout .gitmodules
+  - "git fetch https://gitlab.haskell.org/ghc/ghc-performance-notes.git refs/notes/perf:refs/notes/perf || true"
 
 stages:
-  - lint
-  - build
-  - full-build
-  - cleanup # See Note [Cleanup on Windows]
+  - lint       # Source linting
+  - build      # A quick smoke-test to weed out broken commits
+  - full-build # Build all the things
+  - cleanup    # See Note [Cleanup after the shell executor]
+  - packaging  # Source distribution, etc.
+  - hackage    # head.hackage testing
+  - deploy     # push documentation
+
+# N.B.Don't run on wip/ branches, instead on run on merge requests.
+.only-default: &only-default
+  only:
+    - master
+    - /ghc-[0-9]+\.[0-9]+/
+    - merge_requests
+    - tags
+    - web
 
 ############################################################
 # Runner Tags
@@ -31,43 +54,152 @@ stages:
 
 ghc-linters:
   stage: lint
-  image: ghcci/linters:0.1
+  image: "registry.gitlab.haskell.org/ghc/ci-images/linters:$DOCKER_REV"
   script:
-    - |
-      if [ -n "$CI_MERGE_REQUEST_ID" ]; then
-        base="$(git merge-base $CI_MERGE_REQUEST_BRANCH_NAME HEAD)"
-        validate-commit-msg .git $(git rev-list $base..$CI_COMMIT_SHA)
-        submodchecker .git $(git rev-list $base..$CI_COMMIT_SHA)
-        validate-whitespace .git $(git rev-list $base..$CI_COMMIT_SHA)
-        .gitlab/linters/check-makefiles.py $base $CI_COMMIT_SHA
-        .gitlab/linters/check-cpp.py $base $CI_COMMIT_SHA
-      fi
+    - git fetch "$CI_MERGE_REQUEST_PROJECT_URL" $CI_MERGE_REQUEST_TARGET_BRANCH_NAME
+    - base="$(git merge-base FETCH_HEAD $CI_COMMIT_SHA)"
+    - "echo Linting changes between $base..$CI_COMMIT_SHA"
+    #    - validate-commit-msg .git $(git rev-list $base..$CI_COMMIT_SHA)
+    - validate-whitespace .git $(git rev-list $base..$CI_COMMIT_SHA)
+    - .gitlab/linters/check-makefiles.py $base $CI_COMMIT_SHA
+    - .gitlab/linters/check-cpp.py $base $CI_COMMIT_SHA
+    - .gitlab/linters/check-version-number.sh
+  dependencies: []
+  tags:
+    - lint
+  only:
+    refs:
+      - merge_requests
+
+lint-linters:
+  <<: *only-default
+  stage: lint
+  image: "registry.gitlab.haskell.org/ghc/ci-images/linters:$DOCKER_REV"
+  script:
+    - mypy .gitlab/linters/*.py
+  dependencies: []
+  tags:
+    - lint
+
+lint-testsuite:
+  <<: *only-default
+  stage: lint
+  image: "registry.gitlab.haskell.org/ghc/ci-images/x86_64-linux-deb9:$DOCKER_REV"
+  script:
+    - make -Ctestsuite list_broken TEST_HC=ghc
+  dependencies: []
+  tags:
+    - lint
+
+# We allow the submodule checker to fail when run on merge requests (to
+# accomodate, e.g., haddock changes not yet upstream) but not on `master` or
+# Marge jobs.
+.lint-submods:
+  <<: *only-default
+  stage: lint
+  image: "registry.gitlab.haskell.org/ghc/ci-images/linters:$DOCKER_REV"
+  script:
+    - git fetch "$CI_MERGE_REQUEST_PROJECT_URL" $CI_MERGE_REQUEST_TARGET_BRANCH_NAME
+    - base="$(git merge-base FETCH_HEAD $CI_COMMIT_SHA)"
+    - "echo Linting submodule changes between $base..$CI_COMMIT_SHA"
+    - git submodule foreach git remote update
+    - submodchecker .git $(git rev-list $base..$CI_COMMIT_SHA)
+  dependencies: []
+  tags:
+    - lint
+
+lint-submods-marge:
+  extends: .lint-submods
+  only:
+    refs:
+      - merge_requests
+    variables:
+      - $CI_MERGE_REQUEST_LABELS =~ /.*wip/marge_bot_batch_merge_job.*/
+
+lint-submods-mr:
+  extends: .lint-submods
+  # Allow failure since any necessary submodule patches may not be upstreamed
+  # yet.
+  allow_failure: true
+  only:
+    refs:
+      - merge_requests
+  except:
+    variables:
+      - $CI_MERGE_REQUEST_LABELS =~ /.*wip/marge_bot_batch_merge_job.*/
+
+lint-submods-branch:
+  extends: .lint-submods
+  script:
+    - "echo Linting submodule changes between $CI_COMMIT_BEFORE_SHA..$CI_COMMIT_SHA"
+    - git submodule foreach git remote update
+    - submodchecker .git $(git rev-list $CI_COMMIT_BEFORE_SHA..$CI_COMMIT_SHA)
+  only:
+    refs:
+      - master
+      - /ghc-[0-9]+\.[0-9]+/
+
+.lint-changelogs:
+  <<: *only-default
+  stage: lint
+  image: "registry.gitlab.haskell.org/ghc/ci-images/linters:$DOCKER_REV"
   dependencies: []
   tags:
     - lint
+  script:
+    - |
+      grep TBA libraries/*/changelog.md && (
+          echo "Error: Found \"TBA\"s in changelogs."
+          exit 1
+      )
+
+lint-changelogs:
+  extends: .lint-changelogs
+  # Allow failure since this isn't a final release.
+  allow_failure: true
+  only:
+    refs:
+      - /ghc-[0-9]+\.[0-9]+/
+
+lint-release-changelogs:
+  extends: .lint-changelogs
+  only:
+    refs:
+      - /ghc-[0-9]+\.[0-9]+\.[0-9]+-.*/
+
 
 ############################################################
 # Validation via Pipelines (hadrian)
 ############################################################
 
 .validate-hadrian:
-  allow_failure: true
+  <<: *only-default
   script:
+    - cabal update
     - git clean -xdf && git submodule foreach git clean -xdf
-    - bash .circleci/prepare-system.sh
+    - .gitlab/prepare-system.sh
     - if [[ -d ./cabal-cache ]]; then cp -R ./.cabal-cache ~/.cabal-cache; fi
     - ./boot
     - ./configure $CONFIGURE_ARGS
-    - hadrian/build.cabal.sh -j`mk/detect-cpu-count.sh`
+    - hadrian/build.cabal.sh -j`mk/detect-cpu-count.sh` --docs=no-sphinx binary-dist
+    - hadrian/build.cabal.sh -j`mk/detect-cpu-count.sh` --docs=no-sphinx test --summary-junit=./junit.xml
+    - mv _build/bindist/ghc*.tar.xz ghc.tar.xz
   cache:
     key: hadrian
     paths:
       - cabal-cache
+  artifacts:
+    reports:
+      junit: junit.xml
+    expire_in: 2 week
+    paths:
+      - ghc.tar.xz
+      - junit.xml
 
-validate-x86_64-linux-deb8-hadrian:
+validate-x86_64-linux-deb9-hadrian:
   extends: .validate-hadrian
   stage: build
-  image: ghcci/x86_64-linux-deb8:0.1
+  image: "registry.gitlab.haskell.org/ghc/ci-images/x86_64-linux-deb9:$DOCKER_REV"
   before_script:
     # workaround for docker permissions
     - sudo chown ghc:ghc -R .
@@ -75,14 +207,44 @@ validate-x86_64-linux-deb8-hadrian:
     - git submodule sync --recursive
     - git submodule update --init --recursive
     - git checkout .gitmodules
+    - "git fetch https://gitlab.haskell.org/ghc/ghc-performance-notes.git refs/notes/perf:refs/notes/perf || true"
   tags:
     - x86_64-linux
 
+hadrian-ghc-in-ghci:
+  <<: *only-default
+  stage: build
+  image: "registry.gitlab.haskell.org/ghc/ci-images/x86_64-linux-deb9:$DOCKER_REV"
+  before_script:
+    # workaround for docker permissions
+    - sudo chown ghc:ghc -R .
+    - python3 .gitlab/fix-submodules.py
+    - git submodule sync --recursive
+    - git submodule update --init --recursive
+    - git checkout .gitmodules
+  tags:
+    - x86_64-linux
+  script:
+    - cabal update
+    - cd hadrian; cabal new-build --project-file=ci.project; cd ..
+    - git clean -xdf && git submodule foreach git clean -xdf
+    - .gitlab/prepare-system.sh
+    - if [[ -d ./cabal-cache ]]; then cp -R ./.cabal-cache ~/.cabal-cache; fi
+    - ./boot
+    - ./configure $CONFIGURE_ARGS
+    # Load ghc-in-ghci then immediately exit and check the modules loaded
+    - echo ":q" | hadrian/ghci.sh | tail -n2 | grep "Ok,"
+  cache:
+    key: hadrian-ghci
+    paths:
+      - cabal-cache
+
 ############################################################
 # Validation via Pipelines (make)
 ############################################################
 
 .validate:
+  <<: *only-default
   variables:
     TEST_TYPE: test
   before_script:
@@ -92,22 +254,35 @@ validate-x86_64-linux-deb8-hadrian:
     - ./configure $CONFIGURE_ARGS
     - |
       THREADS=`mk/detect-cpu-count.sh`
-      make V=0 -j$THREADS WERROR=-Werror 
+      make V=0 -j$THREADS WERROR=-Werror
+    - |
+      make bindisttest
     - |
       make binary-dist TAR_COMP_OPTS="-1"
-      mv ghc-*.tar.xz ghc.tar.xz
+    - |
+      # Prepare to push git notes.
+      METRICS_FILE=$(mktemp)
+      git config user.email "ben+ghc-ci@smart-cactus.org"
+      git config user.name "GHC GitLab CI"
     - |
       THREADS=`mk/detect-cpu-count.sh`
-      make $TEST_TYPE THREADS=$THREADS JUNIT_FILE=../../junit.xml
+      make $TEST_TYPE THREADS=$THREADS JUNIT_FILE=../../junit.xml METRICS_FILE=$METRICS_FILE
+    - |
+      # Push git notes.
+      METRICS_FILE=$METRICS_FILE .gitlab/push-test-metrics.sh
   dependencies: []
   artifacts:
     reports:
       junit: junit.xml
     expire_in: 2 week
     paths:
-      - ghc.tar.xz
+      - ghc-*.tar.xz
       - junit.xml
 
+#################################
+# x86_64-darwin
+#################################
+
 validate-x86_64-darwin:
   extends: .validate
   stage: full-build
@@ -115,17 +290,20 @@ validate-x86_64-darwin:
     - x86_64-darwin
   variables:
     GHC_VERSION: 8.6.3
+    BIN_DIST_PREP_TAR_COMP: "bindistprep/ghc-x86_64-apple-darwin.tar.xz"
     MACOSX_DEPLOYMENT_TARGET: "10.7"
     # Only Sierra and onwards supports clock_gettime. See #12858
     ac_cv_func_clock_gettime: "no"
     LANG: "en_US.UTF-8"
     CONFIGURE_ARGS: --with-intree-gmp
+    TEST_ENV: "x86_64-darwin"
   before_script:
     - git clean -xdf && git submodule foreach git clean -xdf
     - python3 .gitlab/fix-submodules.py
     - git submodule sync --recursive
     - git submodule update --init --recursive
     - git checkout .gitmodules
+    - "git fetch https://gitlab.haskell.org/ghc/ghc-performance-notes.git refs/notes/perf:refs/notes/perf || true"
 
     - bash .gitlab/darwin-init.sh
     - PATH="`pwd`/toolchain/bin:$PATH"
@@ -140,6 +318,47 @@ validate-x86_64-darwin:
       - cabal-cache
       - toolchain
 
+# Disabled because of OS X CI capacity
+.validate-x86_64-darwin-hadrian:
+  <<: *only-default
+  stage: full-build
+  tags:
+    - x86_64-darwin
+  variables:
+    GHC_VERSION: 8.6.3
+    MACOSX_DEPLOYMENT_TARGET: "10.7"
+    ac_cv_func_clock_gettime: "no"
+    LANG: "en_US.UTF-8"
+    CONFIGURE_ARGS: --with-intree-gmp
+    TEST_ENV: "x86_64-darwin"
+  before_script:
+    - git clean -xdf && git submodule foreach git clean -xdf
+    - python3 .gitlab/fix-submodules.py
+    - git submodule sync --recursive
+    - git submodule update --init --recursive
+    - git checkout .gitmodules
+    - "git fetch https://gitlab.haskell.org/ghc/ghc-performance-notes.git refs/notes/perf:refs/notes/perf || true"
+
+    - bash .gitlab/darwin-init.sh
+    - PATH="`pwd`/toolchain/bin:$PATH"
+  script:
+    - cabal update
+    - ./boot
+    - ./configure $CONFIGURE_ARGS
+    - hadrian/build.cabal.sh -j`mk/detect-cpu-count.sh` --docs=no-sphinx binary-dist
+    - hadrian/build.cabal.sh -j`mk/detect-cpu-count.sh` --docs=no-sphinx test --summary-junit=./junit.xml
+    - mv _build/bindist/ghc*.tar.xz ghc.tar.xz
+  after_script:
+    - cp -Rf $HOME/.cabal cabal-cache
+  artifacts:
+    when: always
+    expire_in: 2 week
+    reports:
+      junit: junit.xml
+    paths:
+      - ghc.tar.xz
+      - junit.xml
+
 .validate-linux:
   extends: .validate
   tags:
@@ -150,8 +369,14 @@ validate-x86_64-darwin:
     - git submodule sync --recursive
     - git submodule update --init --recursive
     - git checkout .gitmodules
+    - "git fetch https://gitlab.haskell.org/ghc/ghc-performance-notes.git refs/notes/perf:refs/notes/perf || true"
+      # Build hyperlinked sources for documentation when building releases
+    - |
+      if [[ -n "$CI_COMMIT_TAG" ]]; then
+        echo "EXTRA_HADDOCK_OPTS += --hyperlinked-source --quickjump" >> mk/build.mk
+      fi
 
-    - bash .circleci/prepare-system.sh
+    - .gitlab/prepare-system.sh
     # workaround for docker permissions
     - sudo chown ghc:ghc -R .
   after_script:
@@ -162,22 +387,33 @@ validate-x86_64-darwin:
       - cabal-cache
       - toolchain
 
-validate-aarch64-linux-deb9:
+#################################
+# aarch64-linux-deb9
+#################################
+
+.build-aarch64-linux-deb9:
   extends: .validate-linux
   stage: full-build
-  image: ghcci/aarch64-linux-deb9:0.1
+  image: "registry.gitlab.haskell.org/ghc/ci-images/aarch64-linux-deb9:$DOCKER_REV"
   allow_failure: true
-  artifacts:
-    when: always
-    expire_in: 2 week
+  variables:
+    TEST_ENV: "aarch64-linux-deb9"
+    BIN_DIST_PREP_TAR_COMP: "bindistprep/ghc-aarch64-linux-deb9.tar.xz"
+    # Inexplicably makeindex fails
+    BUILD_SPHINX_PDF: "NO"
   cache:
     key: linux-aarch64-deb9
   tags:
     - aarch64-linux
 
+validate-aarch64-linux-deb9:
+  extends: .build-aarch64-linux-deb9
+  artifacts:
+    when: always
+    expire_in: 2 week
+
 nightly-aarch64-linux-deb9:
-  extends: validate-aarch64-linux-deb9
-  stage: full-build
+  extends: .build-aarch64-linux-deb9
   artifacts:
     expire_in: 2 year
   variables:
@@ -186,22 +422,28 @@ nightly-aarch64-linux-deb9:
     variables:
       - $NIGHTLY
 
-validate-i386-linux-deb9:
+#################################
+# i386-linux-deb9
+#################################
+
+.build-i386-linux-deb9:
   extends: .validate-linux
   stage: full-build
-  image: ghcci/i386-linux-deb9:0.1
-  allow_failure: true
+  image: "registry.gitlab.haskell.org/ghc/ci-images/i386-linux-deb9:$DOCKER_REV"
+  variables:
+    TEST_ENV: "i386-linux-deb9"
+    BIN_DIST_PREP_TAR_COMP: "bindistprep/ghc-i386-deb9-linux.tar.xz"
+  cache:
+    key: linux-i386-deb9
+
+validate-i386-linux-deb9:
+  extends: .build-i386-linux-deb9
   artifacts:
     when: always
     expire_in: 2 week
-  cache:
-    key: linux-i386-deb9
 
 nightly-i386-linux-deb9:
-  extends: .validate-linux
-  stage: full-build
-  image: ghcci/i386-linux-deb9:0.1
-  allow_failure: true
+  extends: .build-i386-linux-deb9
   variables:
     TEST_TYPE: slowtest
   artifacts:
@@ -210,22 +452,30 @@ nightly-i386-linux-deb9:
   only:
     variables:
       - $NIGHTLY
-  cache:
-    key: linux-i386-deb9
 
-validate-x86_64-linux-deb9:
+#################################
+# x86_64-linux-deb9
+#################################
+
+.build-x86_64-linux-deb9:
   extends: .validate-linux
-  stage: build
-  image: ghcci/x86_64-linux-deb9:0.2
+  stage: full-build
+  image: "registry.gitlab.haskell.org/ghc/ci-images/x86_64-linux-deb9:$DOCKER_REV"
+  variables:
+    TEST_ENV: "x86_64-linux-deb9"
+    BIN_DIST_PREP_TAR_COMP: "bindistprep/ghc-x86_64-deb9-linux.tar.xz"
+  cache:
+    key: linux-x86_64-deb9
+
+# Disabled to reduce CI load
+.validate-x86_64-linux-deb9:
+  extends: .build-x86_64-linux-deb9
   artifacts:
     when: always
     expire_in: 2 week
-  cache:
-    key: linux-x86_64-deb9
 
 nightly-x86_64-linux-deb9:
-  extends: validate-x86_64-linux-deb9
-  stage: build
+  extends: .build-x86_64-linux-deb9
   artifacts:
     expire_in: 2 year
   variables:
@@ -234,69 +484,144 @@ nightly-x86_64-linux-deb9:
     variables:
       - $NIGHTLY
 
-validate-x86_64-linux-deb9-llvm:
-  extends: .validate-linux
-  stage: full-build
-  image: ghcci/x86_64-linux-deb9:0.2
+# N.B. Has DEBUG assertions enabled in stage2
+validate-x86_64-linux-deb9-debug:
+  extends: .build-x86_64-linux-deb9
+  stage: build
   variables:
-    BUILD_FLAVOUR: perf-llvm
-  cache:
-    key: linux-x86_64-deb9
-
-validate-x86_64-linux-deb8:
-  extends: .validate-linux
-  stage: full-build
-  image: ghcci/x86_64-linux-deb8:0.1
-  cache:
-    key: linux-x86_64-deb8
+    BUILD_FLAVOUR: validate
+    TEST_TYPE: slowtest
+    TEST_ENV: "x86_64-linux-deb9-debug"
   artifacts:
     when: always
     expire_in: 2 week
 
-validate-x86_64-linux-fedora27:
-  extends: .validate-linux
+# Disabled to alleviate CI load
+.validate-x86_64-linux-deb9-llvm:
+  extends: .build-x86_64-linux-deb9
   stage: full-build
-  image: ghcci/x86_64-linux-fedora27:0.1
-  cache:
-    key: linux-x86_64-fedora27
-  artifacts:
-    when: always
-    expire_in: 2 week
+  variables:
+    BUILD_FLAVOUR: perf-llvm
+    TEST_ENV: "x86_64-linux-deb9-llvm"
+
+nightly-x86_64-linux-deb9-llvm:
+  extends: .build-x86_64-linux-deb9
+  stage: full-build
+  variables:
+    BUILD_FLAVOUR: perf-llvm
+    TEST_ENV: "x86_64-linux-deb9-llvm"
+  only:
+    variables:
+      - $NIGHTLY
 
 validate-x86_64-linux-deb9-integer-simple:
-  extends: .validate-linux
+  extends: .build-x86_64-linux-deb9
   stage: full-build
   variables:
     INTEGER_LIBRARY: integer-simple
-  image: ghcci/x86_64-linux-deb9:0.2
-  cache:
-    key: linux-x86_64-deb9
+    TEST_ENV: "x86_64-linux-deb9-integer-simple"
+    BIN_DIST_PREP_TAR_COMP: "bindistprep/ghc-x86_64-deb9-linux-integer-simple.tar.xz"
 
 nightly-x86_64-linux-deb9-integer-simple:
-  extends: validate-x86_64-linux-deb9-integer-simple
+  extends: .build-x86_64-linux-deb9
   stage: full-build
-  artifacts:
-    expire_in: 2 year
   variables:
+    INTEGER_LIBRARY: integer-simple
+    TEST_ENV: "x86_64-linux-deb9-integer-simple"
     TEST_TYPE: slowtest
+  artifacts:
+    expire_in: 2 year
   only:
     variables:
       - $NIGHTLY
 
 validate-x86_64-linux-deb9-unreg:
-  extends: .validate-linux
+  extends: .build-x86_64-linux-deb9
   stage: full-build
   variables:
     CONFIGURE_ARGS: --enable-unregisterised
-  image: ghcci/x86_64-linux-deb9:0.2
+    TEST_ENV: "x86_64-linux-deb9-unreg"
+
+release-x86_64-linux-deb9-dwarf:
+  extends: .validate-linux
+  stage: build
+  image: "registry.gitlab.haskell.org/ghc/ci-images/x86_64-linux-deb9:$DOCKER_REV"
+  allow_failure: true
+  variables:
+    CONFIGURE_ARGS: "--enable-dwarf-unwind"
+    BUILD_FLAVOUR: dwarf
+    TEST_ENV: "x86_64-linux-deb9"
+  artifacts:
+    when: always
+    expire_in: 2 week
   cache:
     key: linux-x86_64-deb9
 
+
+#################################
+# x86_64-linux-deb8
+#################################
+
+release-x86_64-linux-deb8:
+  extends: .validate-linux
+  stage: full-build
+  image: "registry.gitlab.haskell.org/ghc/ci-images/x86_64-linux-deb8:$DOCKER_REV"
+  variables:
+    TEST_ENV: "x86_64-linux-deb8"
+    BIN_DIST_PREP_TAR_COMP: "bindistprep/ghc-x86_64-deb8-linux.tar.xz"
+  only:
+    - tags
+  cache:
+    key: linux-x86_64-deb8
+  artifacts:
+    when: always
+    expire_in: 2 week
+
+#################################
+# x86_64-linux-centos7
+#################################
+
+release-x86_64-linux-centos7:
+  extends: .validate-linux
+  stage: full-build
+  image: "registry.gitlab.haskell.org/ghc/ci-images/x86_64-linux-centos7:$DOCKER_REV"
+  variables:
+    # The sphinx release shipped with Centos 7 fails to build out documentation
+    BUILD_SPHINX_HTML: "NO"
+    BUILD_SPHINX_PDF: "NO"
+    TEST_ENV: "x86_64-linux-centos7"
+    BIN_DIST_PREP_TAR_COMP: "bindistprep/ghc-x86_64-centos7-linux.tar.xz"
+  only:
+    - tags
+  cache:
+    key: linux-x86_64-centos7
+  artifacts:
+    when: always
+    expire_in: 2 week
+
+#################################
+# x86_64-linux-fedora27
+#################################
+
+validate-x86_64-linux-fedora27:
+  extends: .validate-linux
+  stage: full-build
+  image: "registry.gitlab.haskell.org/ghc/ci-images/x86_64-linux-fedora27:$DOCKER_REV"
+  variables:
+    TEST_ENV: "x86_64-linux-fedora27"
+    BIN_DIST_PREP_TAR_COMP: "bindistprep/ghc-x86_64-fedora27-linux.tar.xz"
+  cache:
+    key: linux-x86_64-fedora27
+  artifacts:
+    when: always
+    expire_in: 2 week
+
 ############################################################
 # Validation via Pipelines (Windows)
 ############################################################
 
-.validate-windows:
+.build-windows:
+  <<: *only-default
   before_script:
     - git clean -xdf
     - git submodule foreach git clean -xdf
@@ -313,111 +638,164 @@ validate-x86_64-linux-deb9-unreg:
     - git submodule sync --recursive
     - git submodule update --init --recursive
     - git checkout .gitmodules
+    - "git fetch https://gitlab.haskell.org/ghc/ghc-performance-notes.git refs/notes/perf:refs/notes/perf || true"
     - bash .gitlab/win32-init.sh
   after_script:
     - rd /s /q tmp
     - robocopy /np /nfl /ndl /e "%APPDATA%\cabal" cabal-cache
     - bash -c 'make clean || true'
   dependencies: []
+  variables:
+    FORCE_SYMLINKS: 1
+    LANG: "en_US.UTF-8"
   cache:
     paths:
       - cabal-cache
-      - ghc-8.6.2
+      - ghc-8.6.5
       - ghc-tarballs
 
-validate-x86_64-windows-hadrian:
-  extends: .validate-windows
+.build-windows-hadrian:
+  extends: .build-windows
   stage: full-build
   variables:
-    GHC_VERSION: "8.6.2"
-    LANG: "en_US.UTF-8"
+    GHC_VERSION: "8.6.5"
+  # due to #16574 this currently fails
+  allow_failure: true
   script:
     - |
-      set MSYSTEM=MINGW64
       python boot
       bash -c './configure --enable-tarballs-autodownload GHC=`pwd`/toolchain/bin/ghc HAPPY=`pwd`/toolchain/bin/happy ALEX=`pwd`/toolchain/bin/alex'
-      mkdir -p _build
-      cp -R inplace/mingw _build/mingw
-    - bash -c "PATH=`pwd`/toolchain/bin:$PATH hadrian/build.cabal.sh -j`mk/detect-cpu-count.sh` --flavour=Quick"
-    # FIXME: Bindist disabled due to #16073
-    #- bash -c "PATH=`pwd`/toolchain/bin:$PATH hadrian/build.cabal.sh binary-dist"
+    - bash -c "PATH=`pwd`/toolchain/bin:$PATH hadrian/build.cabal.sh -j`mk/detect-cpu-count.sh` --flavour=Quick --docs=no-sphinx binary-dist"
+    - mv _build/bindist/ghc*.tar.xz ghc.tar.xz
     # FIXME: Testsuite disabled due to #16156.
-    #- bash -c 'make V=0 test THREADS=`mk/detect-cpu-count.sh` JUNIT_FILE=../../junit.xml'
-  cache:
-    key: x86_64-windows
+    # - bash -c 'make V=0 test THREADS=`mk/detect-cpu-count.sh` JUNIT_FILE=../../junit.xml'
   tags:
     - x86_64-windows
+  artifacts:
+    when: always
+    paths:
+      - ghc.tar.xz
 
-validate-x86_64-windows:
-  extends: .validate-windows
+validate-x86_64-windows-hadrian:
+  extends: .build-windows-hadrian
+  variables:
+    MSYSTEM: MINGW64
+  cache:
+    key: "x86_64-windows-hadrian-$WINDOWS_TOOLCHAIN_VERSION"
+
+nightly-i386-windows-hadrian:
+  extends: .build-windows-hadrian
+  variables:
+    MSYSTEM: MINGW32
+  only:
+    variables:
+      - $NIGHTLY
+  cache:
+    key: "i386-windows-hadrian-$WINDOWS_TOOLCHAIN_VERSION"
+
+.build-windows-make:
+  extends: .build-windows
   stage: full-build
   # due to #16084
   allow_failure: true
   variables:
-    GHC_VERSION: "8.6.2"
-    LANG: "en_US.UTF-8"
+    BUILD_FLAVOUR: "quick"
+    GHC_VERSION: "8.6.5"
+    BIN_DIST_PREP_TAR_COMP: "bindistprep/ghc-x86_64-mingw32.tar.xz"
   script:
     - |
-      set MSYSTEM=MINGW64
       python boot
-      bash -c './configure --enable-tarballs-autodownload GHC=`pwd`/toolchain/bin/ghc HAPPY=`pwd`/toolchain/bin/happy ALEX=`pwd`/toolchain/bin/alex'
-    - bash -c "echo include mk/flavours/quick.mk > mk/build.mk"
+      bash -c './configure --enable-tarballs-autodownload GHC=`pwd`/toolchain/bin/ghc HAPPY=`pwd`/toolchain/bin/happy ALEX=`pwd`/toolchain/bin/alex $CONFIGURE_ARGS'
+    - bash -c "echo include mk/flavours/${BUILD_FLAVOUR}.mk > mk/build.mk"
+    - bash -c "echo 'GhcLibHcOpts+=-haddock' >> mk/build.mk"
     - bash -c "PATH=`pwd`/toolchain/bin:$PATH make -j`mk/detect-cpu-count.sh`"
-    - |
-      bash -c "make binary-dist TAR_COMP_OPTS=-1"
-      mv ghc-*.tar.xz ghc.tar.xz
+    - bash -c "PATH=`pwd`/toolchain/bin:$PATH make bindisttest"
+    - bash -c "PATH=`pwd`/toolchain/bin:$PATH make binary-dist TAR_COMP_OPTS=-1"
     - bash -c 'make V=0 test THREADS=`mk/detect-cpu-count.sh` JUNIT_FILE=../../junit.xml'
-  cache:
-    key: x86_64-windows
   tags:
     - x86_64-windows
   artifacts:
     when: always
+    expire_in: 2 week
     reports:
       junit: junit.xml
     paths:
-      - ghc.tar.xz
+      - ghc-*.tar.xz
       - junit.xml
 
+validate-x86_64-windows:
+  extends: .build-windows-make
+  variables:
+    MSYSTEM: MINGW64
+    CONFIGURE_ARGS: "--target=x86_64-unknown-mingw32"
+  cache:
+    key: "x86_64-windows-$WINDOWS_TOOLCHAIN_VERSION"
+
+# Normal Windows validate builds are profiled; that won't do for releases.
+release-x86_64-windows:
+  extends: validate-x86_64-windows
+  variables:
+    MSYSTEM: MINGW64
+    BUILD_FLAVOUR: "perf"
+    CONFIGURE_ARGS: "--target=x86_64-unknown-mingw32"
+  only:
+    - tags
+
+release-i386-windows:
+  extends: .build-windows-make
+  only:
+    - tags
+  variables:
+    MSYSTEM: MINGW32
+    BUILD_FLAVOUR: "perf"
+    CONFIGURE_ARGS: "--target=i386-unknown-mingw32"
+    # Due to #15934
+    BUILD_PROF_LIBS: "NO"
+  cache:
+    key: "i386-windows-$WINDOWS_TOOLCHAIN_VERSION"
+
+nightly-i386-windows:
+  extends: .build-windows-make
+  only:
+    variables:
+      - $NIGHTLY
+  variables:
+    MSYSTEM: MINGW32
+    CONFIGURE_ARGS: "--target=i386-unknown-mingw32"
+    # Due to #15934
+    BUILD_PROF_LIBS: "NO"
+  cache:
+    key: "i386-windows-$WINDOWS_TOOLCHAIN_VERSION"
+
+############################################################
+# Cleanup
+############################################################
+
 # Note [Cleaning up after shell executor]
 # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 #
 # As noted in [1], gitlab-runner's shell executor doesn't clean up its working
 # directory after builds. Unfortunately, we are forced to use the shell executor
-# on Windows. To avoid running out of disk space we add a stage at the end of
-# the build to remove the \GitLabRunner\builds directory. Since we only run a
-# single build at a time on Windows this should be safe.
+# on Darwin. To avoid running out of disk space we add a stage at the end of
+# the build to remove the /.../GitLabRunner/builds directory. Since we only run a
+# single build at a time on Darwin this should be safe.
+#
+# We used to have a similar cleanup job on Windows as well however it ended up
+# being quite fragile as we have multiple Windows builders yet there is no
+# guarantee that the cleanup job is run on the same machine as the build itself
+# was run. Consequently we were forced to instead handle cleanup with a separate
+# cleanup cron job on Windows.
 #
 # [1] https://gitlab.com/gitlab-org/gitlab-runner/issues/3856
 
 # See Note [Cleanup after shell executor]
-cleanup-windows:
-  stage: cleanup
-  tags:
-    - x86_64-windows
-  when: always
-  dependencies: []
-  before_script:
-    - echo "Time to clean up"
-  script:
-    - echo "Let's go"
-  after_script:
-    - set "BUILD_DIR=%CI_PROJECT_DIR%"
-    - set "BUILD_DIR=%BUILD_DIR:/=\%"
-    - echo "Cleaning %BUILD_DIR%"
-    - cd \GitLabRunner
-    # This is way more complicated than it should be:
-    # See https://stackoverflow.com/questions/1965787
-    - del %BUILD_DIR%\* /F /Q
-    - for /d %%p in (%BUILD_DIR%\*) do rd /Q /S "%%p"
-    - exit /b 0
-
-# See Note [Cleanup after shell executor]
 cleanup-darwin:
+  <<: *only-default
   stage: cleanup
   tags:
     - x86_64-darwin
   when: always
+  dependencies: []
   before_script:
     - echo "Time to clean up"
   script:
@@ -429,3 +807,128 @@ cleanup-darwin:
     - rm -Rf $BUILD_DIR/*
     - exit 0
 
+############################################################
+# Packaging
+############################################################
+
+doc-tarball:
+  <<: *only-default
+  stage: packaging
+  tags:
+    - x86_64-linux
+  image: ghcci/x86_64-linux-deb9:0.2
+  dependencies:
+    - validate-x86_64-linux-deb9-debug
+    - validate-x86_64-windows
+  variables:
+    LINUX_BINDIST: "ghc-x86_64-deb9-linux.tar.xz"
+    WINDOWS_BINDIST: "ghc-x86_64-mingw32.tar.xz"
+  # Due to Windows allow_failure
+  allow_failure: true
+  artifacts:
+    paths:
+      - haddock.html.tar.xz
+      - libraries.html.tar.xz
+      - users_guide.html.tar.xz
+      - index.html
+      - "*.pdf"
+  script:
+    - |
+      if [ ! -f "$LINUX_BINDIST" ]; then
+        echo "Error: $LINUX_BINDIST does not exist. Did the Debian 9 job fail?"
+        exit 1
+      fi
+      if [ ! -f "$WINDOWS_BINDIST" ]; then
+        echo "Error: $WINDOWS_BINDIST does not exist. Did the 64-bit Windows job fail?"
+        exit 1
+      fi
+    - rm -Rf docs
+    - bash -ex distrib/mkDocs/mkDocs $LINUX_BINDIST $WINDOWS_BINDIST
+    - ls -lh
+    - mv docs/*.tar.xz docs/index.html .
+
+source-tarball:
+  stage: packaging
+  tags:
+    - x86_64-linux
+  image: ghcci/x86_64-linux-deb9:0.2
+  dependencies: []
+  only:
+    - tags
+  artifacts:
+    paths:
+      - ghc-*.tar.xz
+      - version
+  script:
+    - mk/get-win32-tarballs.sh download all
+    - ./boot
+    - ./configure
+    - make sdist
+    - mv sdistprep/*.xz  .
+    - make show VALUE=version > version
+
+
+############################################################
+# Testing via head.hackage
+############################################################
+
+# Triggering jobs in the ghc/head.hackage project requires that we have a job
+# token for that repository. Furthermore the head.hackage CI job must have
+# access to an unprivileged access token with the ability to query the ghc/ghc
+# project such that it can find the job ID of the fedora27 job for the current
+# pipeline.
+
+.hackage:
+  <<: *only-default
+  stage: hackage
+  image: ghcci/x86_64-linux-deb9:0.2
+  tags:
+    - x86_64-linux
+  dependencies: []
+  variables:
+    HEAD_HACKAGE_PROJECT_ID: "78"
+  script:
+    - bash .gitlab/start-head.hackage.sh
+
+hackage:
+  extends: .hackage
+  when: manual
+
+hackage-label:
+  extends: .hackage
+  only:
+    variables:
+      - $CI_MERGE_REQUEST_LABELS =~ /.*user-facing.*/
+
+nightly-hackage:
+  extends: .hackage
+  only:
+    variables:
+      - $NIGHTLY
+
+pages:
+  stage: deploy
+  dependencies:
+    - doc-tarball
+  image: ghcci/x86_64-linux-deb9:0.2
+  # Due to Windows allow_failure
+  allow_failure: true
+  tags:
+    - x86_64-linux
+  script:
+    - mkdir -p public/doc
+    - tar -xf haddock.html.tar.xz -C public/doc
+    - tar -xf libraries.html.tar.xz -C public/doc
+    - tar -xf users_guide.html.tar.xz -C public/doc
+    - |
+      cat >public/index.html <<EOF
+      <!DOCTYPE HTML>
+      <meta charset="UTF-8">
+      <meta http-equiv="refresh" content="1; url=doc/">
+      EOF
+    - cp -f index.html public/doc
+  only:
+    - master
+  artifacts:
+    paths:
+      - public