Hadrian: disable cloud build cache for symlinks #16800
[ghc.git] / boot
1 #!/usr/bin/env python3
2
3 import glob
4 import os
5 import os.path
6 import sys
7 import argparse
8 from textwrap import dedent
9 import subprocess
10 import re
11
12 cwd = os.getcwd()
13
14 parser = argparse.ArgumentParser()
15 parser.add_argument('--validate', action='store_true', help='Run in validate mode')
16 parser.add_argument('--hadrian', action='store_true', help='Do not assume the make base build system')
17 args = parser.parse_args()
18
19 # Packages whose libraries aren't in the submodule root
20 EXCEPTIONS = {
21     'libraries/containers/': 'libraries/containers/containers/'
22 }
23
24 def print_err(s):
25     print(dedent(s), file=sys.stderr)
26
27 def die(mesg):
28     print_err(mesg)
29     sys.exit(1)
30
31 def check_for_url_rewrites():
32     if os.path.isdir('.git') and \
33        subprocess.check_output('git config remote.origin.url'.split()).find(b'github.com') != -1 and \
34        subprocess.call(['git', 'config', '--get-regexp', '^url.*github.com/.*/packages-.insteadOf']) != 0:
35         # If we cloned from github, make sure the url rewrites are set.
36         # Otherwise 'git submodule update --init' prints confusing errors.
37         die("""\
38             It seems you cloned this repository from GitHub. But your git config files
39             don't contain the url rewrites that are needed to make this work (GitHub
40             doesn't support '/' in repository names, so we use a different naming scheme
41             for the submodule repositories there).
42
43             Please run the following commands first:
44
45               git config --global url."git://github.com/ghc/packages-".insteadOf     git://github.com/ghc/packages/
46               git config --global url."http://github.com/ghc/packages-".insteadOf    http://github.com/ghc/packages/
47               git config --global url."https://github.com/ghc/packages-".insteadOf   https://github.com/ghc/packages/
48               git config --global url."ssh://git\@github.com/ghc/packages-".insteadOf ssh://git\@github.com/ghc/packages/
49               git config --global url."git\@github.com:/ghc/packages-".insteadOf      git\@github.com:/ghc/packages/
50
51             And then:
52
53               git submodule update --init
54               ./boot
55
56             Or start over, and clone the GHC repository from the haskell server:
57
58               git clone --recursive git@gitlab.haskell.org:ghc/ghc.git
59
60             For more information, see:
61               * https://gitlab.haskell.org/ghc/ghc/wikis/newcomers or
62               * https://gitlab.haskell.org/ghc/ghc/wikis/building/getting-the-sources#cloning-from-github
63         """)
64
65 def check_boot_packages():
66     # Check that we have all boot packages.
67     import re
68     for l in open('packages', 'r'):
69         if l.startswith('#'):
70             continue
71
72         parts = [part for part in l.split(' ') if part]
73         if len(parts) != 4:
74             die("Error: Bad line in packages file: " + l)
75
76         dir_ = parts[0]
77         tag = parts[1]
78
79         # If tag is not "-" then it is an optional repository, so its
80         # absence isn't an error.
81         if tag == '-':
82             # We would like to just check for a .git directory here,
83             # but in an lndir tree we avoid making .git directories,
84             # so it doesn't exist. We therefore require that every repo
85             # has a LICENSE file instead.
86             license_path = os.path.join(EXCEPTIONS.get(dir_+'/', dir_), 'LICENSE')
87             if not os.path.isfile(license_path):
88                 die("""\
89                     Error: %s doesn't exist
90                     Maybe you haven't run 'git submodule update --init'?
91                     """ % license_path)
92
93 # Create libraries/*/{ghc.mk,GNUmakefile}
94 def boot_pkgs():
95     library_dirs = []
96
97     for package in glob.glob("libraries/*/"):
98         packages_file = os.path.join(package, 'ghc-packages')
99         print(package)
100         if os.path.isfile(packages_file):
101             for subpkg in open(packages_file, 'r'):
102                 library_dirs.append(os.path.join(package, subpkg.strip()))
103         elif package in EXCEPTIONS:
104             library_dirs.append(EXCEPTIONS[package])
105         else:
106             library_dirs.append(package)
107
108     for package in library_dirs:
109         if package[-1] == '/':
110             # drop trailing '/'
111             package = package[:-1]
112
113         dir_ = os.path.relpath(package, 'libraries')
114         cabals = glob.glob(os.path.join(package, '*.cabal.in'))
115         if len(cabals) == 0:
116             cabals = glob.glob(os.path.join(package, '*.cabal'))
117
118         if len(cabals) > 1:
119             die('Too many .cabal files in %s' % package)
120         elif len(cabals) == 1:
121             cabal = cabals[0]
122
123             if os.path.isfile(cabal):
124                 # strip both .cabal and .in
125                 pkg = os.path.splitext(os.path.splitext(os.path.basename(cabal))[0])[0]
126                 top = os.path.join(*['..'] * len(os.path.normpath(package).split(os.path.sep)))
127
128                 ghc_mk = os.path.join(package, 'ghc.mk')
129                 print('Creating %s' % ghc_mk)
130                 with open(ghc_mk, 'w') as f:
131                     f.write(dedent(
132                         """\
133                         {package}_PACKAGE = {pkg}
134                         {package}_dist-install_GROUP = libraries
135                         $(if $(filter {dir},$(PACKAGES_STAGE0)),$(eval $(call build-package,{package},dist-boot,0)))
136                         $(if $(filter {dir},$(PACKAGES_STAGE1)),$(eval $(call build-package,{package},dist-install,1)))
137                         $(if $(filter {dir},$(PACKAGES_STAGE2)),$(eval $(call build-package,{package},dist-install,2)))
138                         """.format(package = package,
139                                 pkg = pkg,
140                                 dir = dir_)))
141
142                 makefile = os.path.join(package, 'GNUmakefile')
143                 with open(makefile, 'w') as f:
144                     f.write(dedent(
145                         """\
146                         dir = {package}
147                         TOP = {top}
148                         include $(TOP)/mk/sub-makefile.mk
149                         FAST_MAKE_OPTS += stage=0
150                         """.format(package = package, top = top)
151                     ))
152
153
154 def autoreconf():
155     # Run autoreconf on everything that needs it.
156     processes = {}
157     if os.name == 'nt':
158         # Get the normalized ACLOCAL_PATH for Windows
159         # This is necessary since on Windows this will be a Windows
160         # path, which autoreconf doesn't know doesn't know how to handle.
161         ac_local = os.getenv('ACLOCAL_PATH', '')
162         ac_local_arg = re.sub(r';', r':', ac_local)
163         ac_local_arg = re.sub(r'\\', r'/', ac_local_arg)
164         ac_local_arg = re.sub(r'(\w):/', r'/\1/', ac_local_arg)
165         reconf_cmd = 'ACLOCAL_PATH=%s autoreconf' % ac_local_arg
166     else:
167         reconf_cmd = 'autoreconf'
168
169     for dir_ in ['.'] + glob.glob('libraries/*/'):
170         if os.path.isfile(os.path.join(dir_, 'configure.ac')):
171             print("Booting %s" % dir_)
172             processes[dir_] = subprocess.Popen(['sh', '-c', reconf_cmd], cwd=dir_)
173
174     # Wait for all child processes to finish.
175     fail = False
176     for k,v in processes.items():
177         code = v.wait()
178         if code != 0:
179             print_err('autoreconf in %s failed with exit code %d' % (k, code))
180             fail = True
181
182     if fail:
183         sys.exit(1)
184
185 def check_build_mk():
186     if not args.validate and not os.path.isfile("mk/build.mk"):
187         print(dedent(
188             """
189             WARNING: You don't have a mk/build.mk file.
190
191             By default a standard GHC build will be done, which uses optimisation
192             and builds the profiling libraries. This will take a long time, so may
193             not be what you want if you are developing GHC or the libraries, rather
194             than simply building it to use it.
195
196             For information on creating a mk/build.mk file, please see:
197                 https://gitlab.haskell.org/ghc/ghc/wikis/building/using#build-configuration
198             """))
199
200 check_for_url_rewrites()
201 check_boot_packages()
202 if not args.hadrian:
203     boot_pkgs()
204 autoreconf()
205 if not args.hadrian:
206     check_build_mk()