Hadrian: build (and retrieve) binary distributions in CI
[ghc.git] / hadrian / src / Builder.hs
1 {-# LANGUAGE InstanceSigs #-}
2 module Builder (
3 -- * Data types
4 ArMode (..), CcMode (..), ConfigurationInfo (..), GhcMode (..),
5 GhcPkgMode (..), HaddockMode (..), SphinxMode (..), TarMode (..),
6 Builder (..),
7
8 -- * Builder properties
9 builderProvenance, systemBuilderPath, builderPath, isSpecified, needBuilder,
10 runBuilder, runBuilderWith, runBuilderWithCmdOptions, getBuilderPath,
11 builderEnvironment,
12
13 -- * Ad hoc builder invocation
14 applyPatch
15 ) where
16
17 import Development.Shake.Classes
18 import GHC.Generics
19 import qualified Hadrian.Builder as H
20 import Hadrian.Builder hiding (Builder)
21 import Hadrian.Builder.Ar
22 import Hadrian.Builder.Sphinx
23 import Hadrian.Builder.Tar
24 import Hadrian.Oracles.Path
25 import Hadrian.Oracles.TextFile
26 import Hadrian.Utilities
27
28 import Base
29 import Context
30 import Oracles.Flag
31 import Oracles.Setting
32 import Packages
33
34 -- | C compiler can be used in two different modes:
35 -- * Compile or preprocess a source file.
36 -- * Extract source dependencies by passing @-MM@ command line argument.
37 data CcMode = CompileC | FindCDependencies deriving (Eq, Generic, Show)
38
39 instance Binary CcMode
40 instance Hashable CcMode
41 instance NFData CcMode
42
43 -- | GHC can be used in four different modes:
44 -- * Compile a Haskell source file.
45 -- * Compile a C source file.
46 -- * Extract source dependencies by passing @-M@ command line argument.
47 -- * Link object files & static libraries into an executable.
48 data GhcMode = CompileHs
49 | CompileCWithGhc
50 | FindHsDependencies
51 | LinkHs
52 | ToolArgs
53 deriving (Eq, Generic, Show)
54
55 instance Binary GhcMode
56 instance Hashable GhcMode
57 instance NFData GhcMode
58
59 -- | To configure a package we need two pieces of information, which we choose
60 -- to record separately for convenience.
61 --
62 -- * Command line arguments to be passed to the setup script.
63 --
64 -- * Package configuration flags that enable/disable certain package features.
65 -- Here is an example from "Settings.Packages":
66 --
67 -- > package rts
68 -- > ? builder (Cabal Flags)
69 -- > ? any (wayUnit Profiling) rtsWays
70 -- > ? arg "profiling"
71 --
72 -- This instructs package configuration functions (such as 'configurePackage')
73 -- to enable the @profiling@ Cabal flag when processing @rts.cabal@ and
74 -- building RTS with profiling information.
75 data ConfigurationInfo = Setup | Flags deriving (Eq, Generic, Show)
76
77 instance Binary ConfigurationInfo
78 instance Hashable ConfigurationInfo
79 instance NFData ConfigurationInfo
80
81 -- TODO: Do we really need all these modes? Why do we need 'Dependencies'? We
82 -- can extract dependencies using the Cabal library. Note: we used to also have
83 -- the @Init@ mode for initialising a new package database but we've deleted it.
84 -- | 'GhcPkg' can initialise a package database and register packages in it.
85 data GhcPkgMode = Copy -- ^ Copy a package from one database to another.
86 | Dependencies -- ^ Compute package dependencies.
87 | Unregister -- ^ Unregister a package.
88 | Update -- ^ Update a package.
89 deriving (Eq, Generic, Show)
90
91 instance Binary GhcPkgMode
92 instance Hashable GhcPkgMode
93 instance NFData GhcPkgMode
94
95 -- | Haddock can be used in two different modes:
96 -- * Generate documentation for a single package
97 -- * Generate an index page for a collection of packages
98 data HaddockMode = BuildPackage | BuildIndex deriving (Eq, Generic, Show)
99
100 instance Binary HaddockMode
101 instance Hashable HaddockMode
102 instance NFData HaddockMode
103
104 -- | A 'Builder' is a (usually external) command invoked in a separate process
105 -- via 'cmd'. Here are some examples:
106 -- * 'Alex' is a lexical analyser generator that builds @Lexer.hs@ from @Lexer.x@.
107 -- * 'Ghc' 'Stage0' is the bootstrapping Haskell compiler used in 'Stage0'.
108 -- * 'Ghc' @StageN@ (N > 0) is the GHC built in stage (N - 1) and used in @StageN@.
109 --
110 -- The 'Cabal' builder is unusual in that it does not correspond to an external
111 -- program but instead relies on the Cabal library for package configuration.
112 data Builder = Alex
113 | Ar ArMode Stage
114 | Autoreconf FilePath
115 | DeriveConstants
116 | Cabal ConfigurationInfo Stage
117 | Cc CcMode Stage
118 | Configure FilePath
119 | GenApply
120 | GenPrimopCode
121 | Ghc GhcMode Stage
122 | GhcPkg GhcPkgMode Stage
123 | Haddock HaddockMode
124 | Happy
125 | Hpc
126 | Hp2Ps
127 | HsCpp
128 | Hsc2Hs Stage
129 | Ld Stage
130 | Make FilePath
131 | Nm
132 | Objdump
133 | Patch
134 | Python
135 | Ranlib
136 | RunTest
137 | Sphinx SphinxMode
138 | Tar TarMode
139 | Unlit
140 | Xelatex
141 deriving (Eq, Generic, Show)
142
143 instance Binary Builder
144 instance Hashable Builder
145 instance NFData Builder
146
147 -- | Some builders are built by this very build system, in which case
148 -- 'builderProvenance' returns the corresponding build 'Context' (which includes
149 -- 'Stage' and GHC 'Package').
150 builderProvenance :: Builder -> Maybe Context
151 builderProvenance = \case
152 DeriveConstants -> context Stage0 deriveConstants
153 GenApply -> context Stage0 genapply
154 GenPrimopCode -> context Stage0 genprimopcode
155 Ghc _ Stage0 -> Nothing
156 Ghc _ stage -> context (pred stage) ghc
157 GhcPkg _ Stage0 -> Nothing
158 GhcPkg _ s -> context (pred s) ghcPkg
159 Haddock _ -> context Stage1 haddock
160 Hpc -> context Stage1 hpcBin
161 Hp2Ps -> context Stage0 hp2ps
162 Hsc2Hs _ -> context Stage0 hsc2hs
163 Unlit -> context Stage0 unlit
164 _ -> Nothing
165 where
166 context s p = Just $ vanillaContext s p
167
168 instance H.Builder Builder where
169 builderPath :: Builder -> Action FilePath
170 builderPath builder = case builderProvenance builder of
171 Nothing -> systemBuilderPath builder
172 Just context -> programPath context
173
174 runtimeDependencies :: Builder -> Action [FilePath]
175 runtimeDependencies = \case
176 Autoreconf dir -> return [dir -/- "configure.ac"]
177 Configure dir -> return [dir -/- "configure"]
178
179 Ghc _ Stage0 -> generatedGhcDependencies Stage0
180 Ghc _ stage -> do
181 root <- buildRoot
182 win <- windowsHost
183 touchyPath <- programPath (vanillaContext Stage0 touchy)
184 unlitPath <- builderPath Unlit
185 ghcdeps <- ghcDeps stage
186 ghcgens <- generatedGhcDependencies stage
187 return $ [ unlitPath ]
188 ++ ghcdeps
189 ++ ghcgens
190 ++ [ touchyPath | win ]
191 ++ [ root -/- mingwStamp | win ]
192 -- proxy for the entire mingw toolchain that
193 -- we have in inplace/mingw initially, and then at
194 -- root -/- mingw.
195
196 Hsc2Hs stage -> (\p -> [p]) <$> templateHscPath stage
197 Make dir -> return [dir -/- "Makefile"]
198 Haddock _ -> haddockDeps Stage1 -- Haddock currently runs in Stage1
199 _ -> return []
200
201 -- query the builder for some information.
202 -- contrast this with runBuilderWith, which returns @Action ()@
203 -- this returns the @stdout@ from running the builder.
204 -- For now this only implements asking @ghc-pkg@ about package
205 -- dependencies.
206 askBuilderWith :: Builder -> BuildInfo -> Action String
207 askBuilderWith builder BuildInfo {..} = case builder of
208 GhcPkg Dependencies _ -> do
209 let input = fromSingleton msgIn buildInputs
210 msgIn = "[askBuilder] Exactly one input file expected."
211 needBuilder builder
212 path <- H.builderPath builder
213 need [path]
214 Stdout stdout <- cmd [path] ["--no-user-package-db", "field", input, "depends"]
215 return stdout
216 _ -> error $ "Builder " ++ show builder ++ " can not be asked!"
217
218 runBuilderWith :: Builder -> BuildInfo -> Action ()
219 runBuilderWith builder BuildInfo {..} = do
220 path <- builderPath builder
221 withResources buildResources $ do
222 verbosity <- getVerbosity
223 let input = fromSingleton msgIn buildInputs
224 msgIn = "[runBuilderWith] Exactly one input file expected."
225 output = fromSingleton msgOut buildOutputs
226 msgOut = "[runBuilderWith] Exactly one output file expected."
227 -- Suppress stdout depending on the Shake's verbosity setting.
228 echo = EchoStdout (verbosity >= Loud)
229 -- Capture stdout and write it to the output file.
230 captureStdout = do
231 Stdout stdout <- cmd [path] buildArgs
232 writeFileChanged output stdout
233 case builder of
234 Ar Pack _ -> do
235 useTempFile <- flag ArSupportsAtFile
236 if useTempFile then runAr path buildArgs
237 else runArWithoutTempFile path buildArgs
238
239 Ar Unpack _ -> cmd echo [Cwd output] [path] buildArgs
240
241 Autoreconf dir -> cmd echo [Cwd dir] ["sh", path] buildArgs
242 Configure dir -> do
243 -- Inject /bin/bash into `libtool`, instead of /bin/sh,
244 -- otherwise Windows breaks. TODO: Figure out why.
245 bash <- bashPath
246 let env = AddEnv "CONFIG_SHELL" bash
247 cmd echo env [Cwd dir] ["sh", path] buildOptions buildArgs
248
249 HsCpp -> captureStdout
250 GenApply -> captureStdout
251
252 GenPrimopCode -> do
253 stdin <- readFile' input
254 Stdout stdout <- cmd (Stdin stdin) [path] buildArgs
255 writeFileChanged output stdout
256
257 Make dir -> cmd echo path ["-C", dir] buildArgs
258
259 Xelatex -> do
260 unit $ cmd [Cwd output] [path] buildArgs
261 unit $ cmd [Cwd output] [path] buildArgs
262 unit $ cmd [Cwd output] [path] buildArgs
263 unit $ cmd [Cwd output] ["makeindex"] (input -<.> "idx")
264 unit $ cmd [Cwd output] [path] buildArgs
265 unit $ cmd [Cwd output] [path] buildArgs
266
267 GhcPkg Copy _ -> do
268 Stdout pkgDesc <- cmd [path]
269 [ "--expand-pkgroot"
270 , "--no-user-package-db"
271 , "describe"
272 , input -- the package name
273 ]
274 cmd (Stdin pkgDesc) [path] (buildArgs ++ ["-"])
275
276 GhcPkg Unregister _ -> do
277 Exit _ <- cmd echo [path] (buildArgs ++ [input])
278 return ()
279
280 _ -> cmd echo [path] buildArgs
281
282 -- TODO: Some builders are required only on certain platforms. For example,
283 -- 'Objdump' is only required on OpenBSD and AIX. Add support for platform
284 -- specific optional builders as soon as we can reliably test this feature.
285 -- See https://github.com/snowleopard/hadrian/issues/211.
286 isOptional :: Builder -> Bool
287 isOptional = \case
288 Objdump -> True
289 _ -> False
290
291 -- | Determine the location of a system 'Builder'.
292 systemBuilderPath :: Builder -> Action FilePath
293 systemBuilderPath builder = case builder of
294 Alex -> fromKey "alex"
295 Ar _ Stage0 -> fromKey "system-ar"
296 Ar _ _ -> fromKey "ar"
297 Autoreconf _ -> stripExe =<< fromKey "autoreconf"
298 Cc _ Stage0 -> fromKey "system-cc"
299 Cc _ _ -> fromKey "cc"
300 -- We can't ask configure for the path to configure!
301 Configure _ -> return "configure"
302 Ghc _ Stage0 -> fromKey "system-ghc"
303 GhcPkg _ Stage0 -> fromKey "system-ghc-pkg"
304 Happy -> fromKey "happy"
305 HsCpp -> fromKey "hs-cpp"
306 Ld _ -> fromKey "ld"
307 Make _ -> fromKey "make"
308 Nm -> fromKey "nm"
309 Objdump -> fromKey "objdump"
310 Patch -> fromKey "patch"
311 Python -> fromKey "python"
312 Ranlib -> fromKey "ranlib"
313 RunTest -> fromKey "python"
314 Sphinx _ -> fromKey "sphinx-build"
315 Tar _ -> fromKey "tar"
316 Xelatex -> fromKey "xelatex"
317 _ -> error $ "No entry for " ++ show builder ++ inCfg
318 where
319 inCfg = " in " ++ quote configFile ++ " file."
320 fromKey key = do
321 let unpack = fromMaybe . error $ "Cannot find path to builder "
322 ++ quote key ++ inCfg ++ " Did you skip configure?"
323 path <- unpack <$> lookupValue configFile key
324 if null path
325 then do
326 unless (isOptional builder) . error $ "Non optional builder "
327 ++ quote key ++ " is not specified" ++ inCfg
328 return "" -- TODO: Use a safe interface.
329 else do
330 win <- windowsHost
331 fullPath <- lookupInPath path
332 case (win, hasExtension fullPath) of
333 (False, _ ) -> return fullPath
334 (True , True ) -> fixAbsolutePathOnWindows fullPath
335 (True , False) -> fixAbsolutePathOnWindows fullPath <&> (<.> exe)
336
337 -- Without this function, on Windows we can observe a bad builder path
338 -- for 'autoreconf'. If the relevant system.config field is set to
339 -- /usr/bin/autoreconf in the file, the path that we read
340 -- is C:/msys64/usr/bin/autoreconf.exe. A standard msys2 set up happens
341 -- to have an executable named 'autoreconf' there, without the 'exe'
342 -- extension. Hence this function.
343 stripExe s = do
344 let sNoExt = dropExtension s
345 exists <- doesFileExist s
346 if exists then return s else return sNoExt
347
348
349 -- | Was the path to a given system 'Builder' specified in configuration files?
350 isSpecified :: Builder -> Action Bool
351 isSpecified = fmap (not . null) . systemBuilderPath
352
353 -- | Apply a patch by executing the 'Patch' builder in a given directory.
354 applyPatch :: FilePath -> FilePath -> Action ()
355 applyPatch dir patch = do
356 let file = dir -/- patch
357 needBuilder Patch
358 path <- builderPath Patch
359 putBuild $ "| Apply patch " ++ file
360 quietly $ cmd [Cwd dir, FileStdin file] [path, "-p0"]