Add GitLab CI
authorBen Gamari <ben@smart-cactus.org>
Sat, 24 Nov 2018 00:20:00 +0000 (19:20 -0500)
committerBen Gamari <ben@smart-cactus.org>
Sat, 1 Dec 2018 13:57:37 +0000 (08:57 -0500)
.circleci/config.yml
.circleci/images/i386-linux/Dockerfile
.circleci/images/x86_64-freebsd/Dockerfile
.circleci/images/x86_64-linux-fedora/Dockerfile
.circleci/images/x86_64-linux/Dockerfile
.gitlab-ci.yml
.gitlab/darwin-init.sh [new file with mode: 0644]
.gitlab/fix-submodules.py [new file with mode: 0644]
.gitlab/win32-init.sh [new file with mode: 0644]

index 4b8d7a4..68ab2c4 100644 (file)
@@ -111,7 +111,7 @@ jobs:
   "validate-x86_64-linux":
     resource_class: xlarge
     docker:
-      - image: ghcci/x86_64-linux:0.0.7
+      - image: ghcci/x86_64-linux:0.0.8
     environment:
       <<: *buildenv
       GHC_COLLECTOR_FLAVOR: x86_64-linux
@@ -185,7 +185,7 @@ jobs:
   "validate-hadrian-x86_64-linux":
     resource_class: xlarge
     docker:
-      - image: ghcci/x86_64-linux:0.0.7
+      - image: ghcci/x86_64-linux:0.0.8
     environment:
       <<: *buildenv
     steps:
@@ -200,7 +200,7 @@ jobs:
   "validate-x86_64-linux-unreg":
     resource_class: xlarge
     docker:
-      - image: ghcci/x86_64-linux:0.0.7
+      - image: ghcci/x86_64-linux:0.0.8
     environment:
       <<: *buildenv
       TEST_ENV: x86_64-linux-unreg
@@ -220,7 +220,7 @@ jobs:
   "validate-x86_64-linux-llvm":
     resource_class: xlarge
     docker:
-      - image: ghcci/x86_64-linux:0.0.7
+      - image: ghcci/x86_64-linux:0.0.8
     environment:
       <<: *buildenv
       BUILD_FLAVOUR: perf-llvm
@@ -250,7 +250,7 @@ jobs:
   "validate-x86_64-linux-debug":
     resource_class: xlarge
     docker:
-      - image: ghcci/x86_64-linux:0.0.7
+      - image: ghcci/x86_64-linux:0.0.8
     environment:
       BUILD_FLAVOUR: devel2
       <<: *buildenv
@@ -272,7 +272,7 @@ jobs:
   "validate-i386-linux":
     resource_class: xlarge
     docker:
-      - image: ghcci/i386-linux:0.0.6
+      - image: ghcci/i386-linux:0.0.7
     environment:
       <<: *buildenv
       GHC_COLLECTOR_FLAVOR: i386-linux
@@ -295,7 +295,7 @@ jobs:
   "validate-x86_64-fedora":
     resource_class: xlarge
     docker:
-      - image: ghcci/x86_64-linux-fedora:0.0.16
+      - image: ghcci/x86_64-linux-fedora:0.0.17
     environment:
       <<: *buildenv
       GHC_COLLECTOR_FLAVOR: x86_64-fedora
@@ -318,7 +318,7 @@ jobs:
   "slow-validate-x86_64-linux":
     resource_class: xlarge
     docker:
-      - image: ghcci/x86_64-linux:0.0.7
+      - image: ghcci/x86_64-linux:0.0.8
     environment:
       <<: *buildenv
       GHC_COLLECTOR_FLAVOR: x86_64-linux
index 47d19cc..c8e5181 100644 (file)
@@ -8,8 +8,9 @@ RUN apt-get update -qq
 
 # Core build utilities
 RUN apt-get install -qy zlib1g-dev libtinfo-dev libsqlite3-0 libsqlite3-dev \
-    ca-certificates g++ git curl make automake autoconf gcc \
-    perl python3 texinfo xz-utils lbzip2 patch openssh-client sudo
+    ca-certificates g++ git make automake autoconf gcc \
+    perl python3 texinfo xz-utils lbzip2 patch openssh-client sudo \
+    jq wget curl
 
 # Documentation tools
 RUN apt-get install -qy python3-sphinx texlive-xetex texlive-latex-extra
index 9c51cc0..1098d7a 100644 (file)
@@ -18,6 +18,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
   software-properties-common \
   sudo \
   wget \
+  jq \
   xz-utils
 
 COPY build-toolchain.sh /tmp/
index 0ed546c..6fad77f 100644 (file)
@@ -5,8 +5,9 @@ ENV LANG C.UTF-8
 # Core build utilities
 RUN dnf -y install coreutils binutils which git make \
     automake autoconf gcc perl python3 texinfo xz lbzip2 \
-    patch openssh-clients sudo curl zlib-devel sqlite \
-    ncurses-compat-libs gmp-devel ncurses-devel gcc-c++ findutils
+    patch openssh-clients sudo zlib-devel sqlite \
+    ncurses-compat-libs gmp-devel ncurses-devel gcc-c++ findutils \
+    curl wget jq
 
 # Documentation tools
 RUN dnf -y install python3-sphinx \
index 5360645..0491ad9 100644 (file)
@@ -8,8 +8,9 @@ RUN apt-get update -qq
 
 # Core build utilities
 RUN apt-get install -qy zlib1g-dev libtinfo-dev libsqlite3-0 libsqlite3-dev \
-    ca-certificates g++ git curl make automake autoconf gcc \
-    perl python3 texinfo xz-utils lbzip2 patch openssh-client sudo
+    ca-certificates g++ git make automake autoconf gcc \
+    perl python3 texinfo xz-utils lbzip2 patch openssh-client sudo \
+    jq wget curl
 
 # Documentation tools
 RUN apt-get install -qy python3-sphinx texlive-xetex texlive-latex-extra
index 52ee2f2..482e71a 100644 (file)
@@ -1,43 +1,56 @@
-# image: ghcci/x86_64-linux:0.0.4
 variables:
   GIT_SSL_NO_VERIFY: "1"
 
 before_script:
-  - mkdir -p ~/.ssh
-  - chmod 700 ~/.ssh
-  #- git config --global user.email "user@example.com"
-  #- git config --global user.name "User name"
-  - echo "$SSH_KNOWN_HOSTS" >> ~/.ssh/known_hosts
-  - chmod 644 ~/.ssh/known_hosts
-  #- sudo apt-get update -y
-  #- sudo apt-get install jq wget -y
-  #- 'which ssh-agent || sudo apt-get install openssh-client -y'
-  - eval $(ssh-agent -s)
-  ## Add the SSH key stored in SSH_PRIVATE_KEY variable to the agent store
-  ## We're using tr to fix line endings which makes ed25519 keys work
-  ## without extra base64 encoding.
-  ## https://gitlab.com/gitlab-examples/ssh-private-key/issues/1#note_48526556
-  ##
-  - echo "$SSH_PRIVATE_KEY" | tr -d '\r' | ssh-add - > /dev/null
+  - python .gitlab/fix-submodules.py
+  - git submodule sync --recursive
+  - git submodule update --init --recursive
+  - git checkout .gitmodules
 
-# All validation jobs keep the bindists and test results are artifacts,
-# when we get far enough to generate them.
-#
-# This requires updating the maximum artifacts size limit in Gitlab to
-# something like 200MB.
+############################################################
+# Validation via Pipelines (hadrian)
+############################################################
 
-validate-x86_64-linux:
-  script: ".gitlab/circle-ci-job.sh validate-x86_64-linux"
-  artifacts:
-    when: always
-    reports:
-      junit: junit.xml
+.validate-hadrian:
+  script:
+    - bash .circleci/prepare-system.sh
+    - if [[ -d ./cabal-cache ]]; then cp -R ./.cabal-cache ~/.cabal-cache; fi
+    - ./boot
+    - ./configure
+    - hadrian/build.cabal.sh -j`mk/detect-cpu-count.sh`
+  cache:
+    key: hadrian
     paths:
-      - ghc.tar.xz
-      - junit.xml
+      - cabal-cache
+  image: ghcci/x86_64-linux:0.0.8
 
-validate-i386-linux:
-  script: ".gitlab/circle-ci-job.sh validate-i386-linux"
+validate-x86_64-linux-hadrian:
+  extends: .validate-hadrian
+  before_script:
+    # workaround for docker permissions
+    - sudo chown ghc:ghc -R .
+  tags:
+    - x86_64-linux
+
+############################################################
+# Validation via Pipelines (make)
+############################################################
+
+.validate:
+  script:
+    - make clean || true
+    - ./boot
+    - ./configure
+    - |
+      THREADS=`mk/detect-cpu-count.sh`
+      make V=0 -j$THREADS HADDOCK_DOCS=NO
+    - |
+      make binary-dist TAR_COMP_OPTS="-1"
+      mv ghc-*.tar.xz ghc.tar.xz
+    - |
+      THREADS=`mk/detect-cpu-count.sh`
+      make test THREADS=$THREADS JUNIT_FILE=../../junit.xml
+  image: ghcci/x86_64-linux:0.0.8
   artifacts:
     when: always
     reports:
@@ -46,18 +59,109 @@ validate-i386-linux:
       - ghc.tar.xz
       - junit.xml
 
-# validate-x86_64-freebsd:
-#   script: ".gitlab/circle-ci-job.sh validate-x86_64-freebsd"
-#   artifacts:
-#     when: always
-#     reports:
-#       junit: junit.xml
-#     paths:
-#       - ghc.tar.xz
-#       - junit.xml
-
 validate-x86_64-darwin:
-  script: ".gitlab/circle-ci-job.sh validate-x86_64-darwin"
+  extends: .validate
+  tags:
+    - x86_64-darwin
+  variables:
+    GHC_VERSION: 8.6.2
+    MACOSX_DEPLOYMENT_TARGET: "10.7"
+    # Only Sierra and onwards supports clock_gettime. See #12858
+    ac_cv_func_clock_gettime: "no"
+  before_script:
+    - python .gitlab/fix-submodules.py
+    - git submodule sync --recursive
+    - git submodule update --init --recursive
+    - git checkout .gitmodules
+
+    - bash .gitlab/darwin-init.sh
+    - PATH="`pwd`/toolchain/bin:$PATH"
+      # Disable sphinx PDF output as MacTeX apparently doesn't provide xelatex
+    - echo "BUILD_SPHINX_PDF=NO" >> mk/build.mk
+  after_script:
+    - cp -Rf $HOME/.cabal cabal-cache
+  cache:
+    key: darwin
+    paths:
+      - cabal-cache
+      - toolchain
+
+validate-x86_64-linux:
+  extends: .validate
+  tags:
+    - x86_64-linux
+  before_script:
+    - python .gitlab/fix-submodules.py
+    - git submodule sync --recursive
+    - git submodule update --init --recursive
+    - git checkout .gitmodules
+
+    - bash .circleci/prepare-system.sh
+    # workaround for docker permissions
+    - sudo chown ghc:ghc -R .
+  after_script:
+    - cp -Rf $HOME/.cabal cabal-cache
+  cache:
+    key: darwin
+    paths:
+      - cabal-cache
+      - toolchain
+
+############################################################
+# Validation via Pipelines (make, Windows)
+############################################################
+
+validate-x86_64-windows:
+  variables:
+    GHC_VERSION: "8.6.2"
+  script:
+    - bash -e .gitlab/win32-init.sh
+    - |
+      set MSYSTEM=MINGW64
+      python boot
+      bash -c './configure --with-ghc=`pwd`/toolchain/bin/ghc --enable-tarballs-autodownload HappyCmd=`pwd`/toolchain/bin/happy AlexCmd=`pwd`/toolchain/bin/alex'
+      mkdir -p _build
+      cp -R inplace/mingw _build/mingw
+    - bash -c "PATH=`pwd`/toolchain/bin:$PATH hadrian/build.cabal.sh -j`mk/detect-cpu-count.sh` --flavour=Quick"
+    - bash -c "PATH=`pwd`/toolchain/bin:$PATH hadrian/build.cabal.sh binary-dist"
+    - mv ghc-*.tar.xz ghc.tar.xz
+    - bash -c 'make V=0 -j`mk/detect-cpu-count.sh` HADDOCK_DOCS=NO test THREADS=$THREADS JUNIT_FILE=../../junit.xml'
+    - cp -Rf $APPDATA/cabal cabal-cache
+  cache:
+    key: windows
+    paths:
+      - cabal-cache
+      - ghc-8.6.2
+      - ghc-tarballs
+  tags:
+    - x86_64-windows
+
+
+############################################################
+# Validation via CircleCI
+############################################################
+
+.circleci:
+  image: ghcci/x86_64-linux:0.0.8
+  tags:
+    - circleci
+  before_script:
+    - mkdir -p ~/.ssh
+    - chmod 700 ~/.ssh
+    #- git config --global user.email "user@example.com"
+    #- git config --global user.name "User name"
+    - echo "$SSH_KNOWN_HOSTS" >> ~/.ssh/known_hosts
+    - chmod 644 ~/.ssh/known_hosts
+    #- sudo apt-get update -y
+    #- sudo apt-get install jq wget -y
+    #- 'which ssh-agent || sudo apt-get install openssh-client -y'
+    - eval $(ssh-agent -s)
+    ## Add the SSH key stored in SSH_PRIVATE_KEY variable to the agent store
+    ## We're using tr to fix line endings which makes ed25519 keys work
+    ## without extra base64 encoding.
+    ## https://gitlab.com/gitlab-examples/ssh-private-key/issues/1#note_48526556
+
+    - echo "$SSH_PRIVATE_KEY" | tr -d '\r' | ssh-add - > /dev/null
   artifacts:
     when: always
     reports:
@@ -66,54 +170,24 @@ validate-x86_64-darwin:
       - ghc.tar.xz
       - junit.xml
 
-validate-hadrian-x86_64-linux:
-  script: ".gitlab/circle-ci-job.sh validate-hadrian-x86_64-linux"
-  # the testsuite doesn't pass with hadrian yet...
-  #
-  # artifacts:
-  #   when: always
-  #   reports:
-  #     junit: junit.xml
-  #   paths:
-  #     - ghc.tar.xz
-  #     - junit.xml
-
-# validate-x86_64-linux-unreg:
-#   script: ".gitlab/circle-ci-job.sh validate-x86_64-linux-unreg"
-#   artifacts:
-#     when: always
-#     reports:
-#       junit: junit.xml
-#     paths:
-#       - ghc.tar.xz
-#       - junit.xml
-
-# validate-x86_64-linux-llvm:
-#   script: ".gitlab/circle-ci-job.sh validate-x86_64-linux-llvm"
-#   artifacts:
-#     when: always
-#     reports:
-#       junit: junit.xml
-#     paths:
-#       - ghc.tar.xz
-#       - junit.xml
-
-# validate-x86_64-linux-debug:
-#   script: ".gitlab/circle-ci-job.sh validate-x86_64-linux-debug"
-#   artifacts:
-#     when: always
-#     reports:
-#       junit: junit.xml
-#     paths:
-#       - ghc.tar.xz
-#       - junit.xml
-
-# slow-validate-x86_64-linux:
-#   script: ".gitlab/circle-ci-job.sh slow-validate-x86_64-linux"
-#   artifacts:
-#     when: always
-#     reports:
-#       junit: junit.xml
-#     paths:
-#       - ghc.tar.xz
-#       - junit.xml
+# All validation jobs keep the bindists and test results are artifacts,
+# when we get far enough to generate them.
+#
+# This requires updating the maximum artifacts size limit in Gitlab to
+# something like 200MB.
+
+circleci-validate-x86_64-linux:
+  extends: .circleci
+  script: ".gitlab/circle-ci-job.sh validate-x86_64-linux"
+
+circleci-validate-i386-linux:
+  extends: .circleci
+  script: ".gitlab/circle-ci-job.sh validate-i386-linux"
+
+# circleci-validate-x86_64-freebsd:
+#   extends: .circleci
+#   script: ".gitlab/circle-ci-job.sh validate-x86_64-freebsd"
+
+circleci-validate-x86_64-darwin:
+  extends: .circleci
+  script: ".gitlab/circle-ci-job.sh validate-x86_64-darwin"
diff --git a/.gitlab/darwin-init.sh b/.gitlab/darwin-init.sh
new file mode 100644 (file)
index 0000000..5561ca8
--- /dev/null
@@ -0,0 +1,38 @@
+#!/bin/bash
+
+set -e
+
+toolchain=`pwd`/toolchain
+PATH="$toolchain/bin:$PATH"
+
+if [ -d "`pwd`/cabal-cache" ]; then
+    cp -Rf cabal-cache $HOME/.cabal
+fi
+
+if [ ! -e $toolchain/bin/ghc ]; then
+    mkdir -p tmp
+    cd tmp
+    curl https://downloads.haskell.org/~ghc/$GHC_VERSION/ghc-$GHC_VERSION-x86_64-apple-darwin.tar.xz | tar -xJ
+    cd ghc-$GHC_VERSION
+    ./configure --prefix=$toolchain
+    make install
+    cd ../..
+    rm -Rf tmp
+fi
+
+if [ ! -e $toolchain/bin/cabal ]; then
+    cabal_tarball="https://www.haskell.org/cabal/release/cabal-install-2.2.0.0/cabal-install-2.2.0.0-x86_64-apple-darwin-sierra.tar.gz"
+    curl $cabal_tarball | tar -xz
+    mv cabal $toolchain/bin
+fi
+
+if [ ! -e $toolchain/bin/happy ]; then
+    cabal update
+    cabal new-install happy --symlink-bindir=$toolchain/bin
+fi
+
+if [ ! -e $toolchain/bin/alex ]; then
+    cabal update
+    cabal new-install alex --symlink-bindir=$toolchain/bin
+fi
+
diff --git a/.gitlab/fix-submodules.py b/.gitlab/fix-submodules.py
new file mode 100644 (file)
index 0000000..51c9b06
--- /dev/null
@@ -0,0 +1,8 @@
+#!/usr/bin/python
+
+import re
+
+x = open('.gitmodules').read()
+x = re.sub(r"url *= *\.\.", "url = https://git.haskell.org", x)
+open('.gitmodules', 'w').write(x)
+
diff --git a/.gitlab/win32-init.sh b/.gitlab/win32-init.sh
new file mode 100644 (file)
index 0000000..9141404
--- /dev/null
@@ -0,0 +1,34 @@
+#!/bin/bash
+
+set -e
+
+toolchain=`pwd`/toolchain
+PATH="$toolchain/bin:$PATH"
+
+if [ -d "`pwd`/cabal-cache" ]; then
+    cp -Rf cabal-cache $APPDATA/cabal
+fi
+
+if [ ! -e $toolchain/bin/ghc ]; then
+    curl https://downloads.haskell.org/~ghc/$GHC_VERSION/ghc-$GHC_VERSION-x86_64-unknown-mingw32.tar.xz | tar -xJ
+    mv ghc-$GHC_VERSION toolchain
+fi
+
+if [ ! -e $toolchain/bin/cabal ]; then
+    curl https://www.haskell.org/cabal/release/cabal-install-2.2.0.0/cabal-install-2.2.0.0-i386-unknown-mingw32.zip > /tmp/cabal.zip
+    unzip /tmp/cabal.zip
+    mv cabal.exe $toolchain/bin
+fi
+
+if [ ! -e $toolchain/bin/happy ]; then
+    cabal update
+    cabal install happy
+    cp $APPDATA/cabal/bin/happy $toolchain/bin
+fi
+
+if [ ! -e $toolchain/bin/alex ]; then
+    cabal update
+    cabal install alex
+    cp $APPDATA/cabal/bin/alex $toolchain/bin
+fi
+