Windows: Update the mirror script to generate hashes and use mirror fallback
[ghc.git] / mk / get-win32-tarballs.sh
1 #!/usr/bin/env bash
2
3 tarball_dir='ghc-tarballs'
4 missing_files=0
5
6 # see #12502
7 if test -z "$FIND"; then FIND="find"; fi
8
9 fail() {
10 echo >&2
11 echo "$1" >&2
12 exit 1
13 }
14
15 download_file() {
16 local file_url="$1"
17 local dest_file="$2"
18 local description="$3"
19 local extra_curl_opts="$4"
20 local backup_url="$5"
21 local dest_dir="$(dirname $dest_file)"
22
23 if ! test -f "${dest_file}"
24 then
25 local curl_cmd="curl -f -L ${file_url} -o ${dest_file} --create-dirs -# ${extra_curl_opts}"
26 if test -n "${backup_url}"; then
27 local curl_cmd_bnk="curl -f -L ${backup_url} -o ${dest_file} --create-dirs -# ${extra_curl_opts}"
28 else
29 local curl_cmd_bnk="true"
30 fi
31
32 if test "$download" = "0"
33 then
34 echo "ERROR: Missing ${description}" >&2
35 echo "${file_url}"
36 missing_files=1
37 return
38 else
39 echo "Downloading ${description} to ${dest_dir}..."
40 $curl_cmd || echo "Checking repo.msys2.org instead of Haskell.org..." && $curl_cmd_bnk || {
41 rm -f "${dest_file}"
42 fail "ERROR: Download failed."
43 exit 1
44 }
45 fi
46 fi
47
48 local sig_file="${dest_file}.sig"
49 if test "$sigs" = "1" -a ! -f "$sig_file"
50 then
51 echo "Downloading ${description} (signature) to ${dest_dir}..."
52 local curl_cmd="curl -f -L ${file_url}.sig -o ${sig_file} --create-dirs -# ${extra_curl_opts}"
53 if test -n "${backup_url}"; then
54 local curl_cmd_bnk="curl -f -L ${backup_url} -o ${sig_file} --create-dirs -# ${extra_curl_opts}"
55 else
56 local curl_cmd_bnk="true"
57 fi
58 $curl_cmd || echo "Checking repo.msys2.org instead of Haskell.org..." && $curl_cmd_bnk || {
59 rm -f "${dest_file}.sig"
60 fail "ERROR: Download failed."
61 exit 1
62 }
63 fi
64
65 if test "$verify" = "1"
66 then
67 grep "${dest_file}$" mk/win32-tarballs.md5sum | md5sum --quiet -c - ||
68 fail "ERROR: ${description} appears to be corrupted, please delete it and try again."
69 fi
70 }
71
72 download_mingw() {
73 local mingw_base_url_primary="https://downloads.haskell.org/~ghc/mingw"
74 local mingw_base_url_secondary="http://repo.msys2.org/mingw"
75
76 if test "$mingw_arch" = "sources"
77 then
78 mingw_url_tmp=`echo "$1" | sed -e 's/-any\.pkg\.tar\.xz/\.src\.tar\.gz/' \
79 -e 's/-sources-/-/' \
80 -e 's/-libwinpthread-git-/-winpthreads-git-/' `
81 local mingw_url="${mingw_base_url_primary}/${mingw_url_tmp}"
82 local mingw_url_backup="${mingw_base_url_secondary}/${mingw_url_tmp}"
83 else
84 local mingw_url="${mingw_base_url_primary}/$1"
85 local mingw_url_backup="${mingw_base_url_secondary}/$1"
86 fi
87
88 local mingw_toolchain="$(basename $mingw_url)"
89 local mingw_w64="${tarball_dir}/${tarball_dest_dir}/${mingw_toolchain}"
90
91 download_file "${mingw_url}" "${mingw_w64}" "${mingw_toolchain}" "" "${mingw_url_backup}"
92
93 # Mark the tree as needing updates by deleting the folder
94 if test -d inplace/mingw && test inplace/mingw -ot "$mingw_w64" ; then
95 echo "In-tree MinGW-w64 tree requires updates..."
96 rm -rf inplace/mingw
97 fi
98 }
99
100 download_tarballs() {
101 local package_prefix="mingw-w64"
102 local format_url="/${mingw_arch}/${package_prefix}-${mingw_arch}"
103
104 download_mingw "${format_url}-crt-git-5.0.0.4795.e3d96cb1-1-any.pkg.tar.xz"
105 download_mingw "${format_url}-winpthreads-git-5.0.0.4761.02bea78-1-any.pkg.tar.xz"
106 download_mingw "${format_url}-headers-git-5.0.0.4797.31e66d7e-1-any.pkg.tar.xz"
107 download_mingw "${format_url}-libwinpthread-git-5.0.0.4761.02bea78-1-any.pkg.tar.xz"
108 download_mingw "${format_url}-zlib-1.2.11-1-any.pkg.tar.xz"
109 download_mingw "${format_url}-isl-0.17.1-1-any.pkg.tar.xz"
110 download_mingw "${format_url}-mpfr-3.1.5.p1-1-any.pkg.tar.xz"
111 download_mingw "${format_url}-gmp-6.1.2-1-any.pkg.tar.xz"
112 download_mingw "${format_url}-binutils-2.28-1-any.pkg.tar.xz"
113 download_mingw "${format_url}-libidn-1.33-1-any.pkg.tar.xz"
114 download_mingw "${format_url}-gcc-6.3.0-2-any.pkg.tar.xz"
115
116 # Upstream is unfortunately quite inconsistent in naming
117 if test "$mingw_arch" != "sources"; then
118 download_mingw "${format_url}-mpc-1.0.3-2-any.pkg.tar.xz"
119 download_mingw "${format_url}-gcc-libs-6.3.0-2-any.pkg.tar.xz"
120 else
121 local format_url="${mingw_base_url}/${mingw_arch}/${package_prefix}"
122 download_mingw "${format_url}-i686-mpc-1.0.3-2.src.tar.gz"
123 download_mingw "${format_url}-x86_64-mpc-1.0.3-2.src.tar.gz"
124 fi
125
126 download_file "https://downloads.haskell.org/~ghc/mingw/ghc-perl-1.tar.gz" "ghc-tarballs/perl/ghc-perl-1.tar.gz" "Windows Perl binary distributions"
127
128 if ! test "$missing_files" = "0"
129 then
130 exit 2
131 fi
132 }
133
134 download_i386() {
135 mingw_arch="i686"
136 tarball_dest_dir="mingw-w64/x86"
137 download_tarballs
138 }
139
140 download_x86_64() {
141 mingw_arch="x86_64"
142 tarball_dest_dir="mingw-w64/x86_64"
143 download_tarballs
144 }
145
146 download_sources() {
147 mingw_arch="sources"
148 tarball_dest_dir="mingw-w64/sources"
149 download_tarballs
150 }
151
152 sync_binaries_and_sources() {
153 gpg --recv-key 5F92EFC1A47D45A1
154
155 # ensure sources are downloaded
156 sigs=1
157 download_i386
158 download_x86_64
159 verify=0
160 download_sources
161
162 for f in $($FIND ghc-tarballs/mingw-w64 -iname '*.sig'); do
163 echo "Verifying $f"
164 gpg --verify $f
165 done
166
167 md5sum `$FIND ghc-tarballs -type f -a -not -iname '*.sig'` >| mk/win32-tarballs.md5sum
168 chmod -R ugo+rX ghc-tarballs
169
170 rsync -av ghc-tarballs/mingw-w64/* downloads.haskell.org:public_html/mingw
171 for f in $($FIND ghc-tarballs/mingw-w64); do
172 curl -XPURGE http://downloads.haskell.org/~ghc/mingw/$f
173 done
174 }
175
176 show_hashes_for_binaries() {
177 $FIND ghc-tarballs/ -iname "*.*" | xargs md5sum | grep -v "\.sig" | sed -s "s/\*//"
178 }
179
180 usage() {
181 echo "$0 - Download GHC mingw toolchain tarballs"
182 echo
183 echo "Usage: $0 <action> [<arch>]"
184 echo
185 echo "Where <action> is one of,"
186 echo ""
187 echo " download download the necessary tarballs for the given architecture"
188 echo " fetch download the necessary tarballs for the given architecture but doesn't verify their md5."
189 echo " verify verify the existence and correctness of the necessary tarballs"
190 echo " hash generate md5 hashes for inclusion in win32-tarballs.md5sum"
191 echo " sync upload packages downloaded with 'fetch mirror' to haskell.org"
192 echo ""
193 echo "and <arch> is one of i386, x86_64,all or mirror (which includes sources)"
194 }
195
196 case $1 in
197 download)
198 download=1
199 verify=1
200 sigs=0
201 ;;
202 fetch)
203 download=1
204 verify=0
205 ;;
206 verify)
207 download=0
208 verify=1
209 ;;
210 sync)
211 download=1
212 verify=0
213 sync=1
214 ;;
215 hash)
216 show_hashes_for_binaries
217 exit 1
218 ;;
219 *)
220 usage
221 exit 1
222 ;;
223 esac
224
225 case $2 in
226 i386)
227 download_i386
228 ;;
229 x86_64)
230 download_x86_64
231 ;;
232 all)
233 download_i386
234 download_x86_64
235 ;;
236 mirror)
237 sigs=1
238 download_i386
239 download_x86_64
240 verify=0
241 download_sources
242 show_hashes_for_binaries
243 ;;
244 *)
245 if test "$sync" = "1"; then
246 sync_binaries_and_sources
247 else
248 usage
249 exit 1
250 fi
251 ;;
252 esac