Switched out optparse for argparse in runtests.py
[ghc.git] / testsuite / mk / test.mk
1 # -----------------------------------------------------------------------------
2 # Examples of use:
3 #
4 # make -- run all the tests in the current directory
5 # make verbose -- as make test, but up the verbosity
6 # make accept -- run the tests, accepting the current output
7 #
8 # The following variables may be set on the make command line:
9 #
10 # TEST -- specific test to run
11 # TESTS -- specific tests to run (same as $TEST really)
12 # EXTRA_HC_OPTS -- extra flags to send to the Haskell compiler
13 # EXTRA_RUNTEST_OPTS -- extra flags to give the test driver
14 # CONFIG -- use a different configuration file
15 # COMPILER -- select a configuration file from config/
16 # THREADS -- run n tests at once
17 #
18 # -----------------------------------------------------------------------------
19
20 # export the value of $MAKE for invocation in tests/driver/
21 export MAKE
22
23 RUNTESTS = $(TOP)/driver/runtests.py
24 COMPILER = ghc
25 CONFIGDIR = $(TOP)/config
26 CONFIG = $(CONFIGDIR)/$(COMPILER)
27
28 ifeq "$(GhcUnregisterised)" "YES"
29 # Otherwise C backend generates many warnings about
30 # imcompatible proto casts for GCC's buitins:
31 # memcpy, printf, strlen.
32 EXTRA_HC_OPTS += -optc-fno-builtin
33 endif
34
35 # TEST_HC_OPTS is passed to every invocation of TEST_HC
36 # in nested Makefiles
37 TEST_HC_OPTS = -dcore-lint -dcmm-lint -no-user-$(GhcPackageDbFlag) -rtsopts $(EXTRA_HC_OPTS)
38
39 ifeq "$(MinGhcVersion711)" "YES"
40 # Don't warn about missing specialisations. They can only occur with `-O`, but
41 # we want tests to produce the same output for all test ways.
42 TEST_HC_OPTS += -fno-warn-missed-specialisations
43 TEST_HC_OPTS += -fshow-warning-groups
44 endif
45
46 ifeq "$(MinGhcVersion801)" "YES"
47 # Turn off any VT800 codes in the output or they wreak havoc on the
48 # testsuite output.
49 TEST_HC_OPTS += -fdiagnostics-color=never
50 TEST_HC_OPTS += -fno-diagnostics-show-caret
51 endif
52
53 # Add the no-debug-output last as it is often convenient to copy the test invocation
54 # removing this line.
55 TEST_HC_OPTS += -dno-debug-output
56
57 TEST_HC_OPTS_INTERACTIVE = $(TEST_HC_OPTS) --interactive -v0 -ignore-dot-ghci -fno-ghci-history
58
59
60 RUNTEST_OPTS =
61
62 ifeq "$(filter $(TargetOS_CPP), cygwin32 mingw32)" ""
63 exeext =
64 else
65 exeext = .exe
66 endif
67
68 ifneq "$(filter $(TargetOS_CPP),cygwin32 mingw32)" ""
69 dllext = .dll
70 else ifeq "$(TargetOS_CPP)" "darwin"
71 dllext = .dylib
72 else
73 dllext = .so
74 endif
75
76 RUNTEST_OPTS += -e "ghc_compiler_always_flags='$(TEST_HC_OPTS)'"
77
78 RUNTEST_OPTS += -e config.compiler_debugged=$(GhcDebugged)
79
80 ifeq "$(GhcWithNativeCodeGen)" "YES"
81 RUNTEST_OPTS += -e ghc_with_native_codegen=1
82 else
83 RUNTEST_OPTS += -e ghc_with_native_codegen=0
84 endif
85
86 GHC_PRIM_LIBDIR := $(subst library-dirs: ,,$(shell "$(GHC_PKG)" field ghc-prim library-dirs --simple-output))
87 HAVE_VANILLA := $(shell if [ -f $(subst \,/,$(GHC_PRIM_LIBDIR))/GHC/PrimopWrappers.hi ]; then echo YES; else echo NO; fi)
88 HAVE_DYNAMIC := $(shell if [ -f $(subst \,/,$(GHC_PRIM_LIBDIR))/GHC/PrimopWrappers.dyn_hi ]; then echo YES; else echo NO; fi)
89 HAVE_PROFILING := $(shell if [ -f $(subst \,/,$(GHC_PRIM_LIBDIR))/GHC/PrimopWrappers.p_hi ]; then echo YES; else echo NO; fi)
90
91 ifeq "$(HAVE_VANILLA)" "YES"
92 RUNTEST_OPTS += -e config.have_vanilla=True
93 else
94 RUNTEST_OPTS += -e config.have_vanilla=False
95 endif
96
97 ifeq "$(HAVE_DYNAMIC)" "YES"
98 RUNTEST_OPTS += -e config.have_dynamic=True
99 else
100 RUNTEST_OPTS += -e config.have_dynamic=False
101 endif
102
103 ifeq "$(HAVE_PROFILING)" "YES"
104 RUNTEST_OPTS += -e config.have_profiling=True
105 else
106 RUNTEST_OPTS += -e config.have_profiling=False
107 endif
108
109 ifeq "$(filter thr, $(GhcRTSWays))" "thr"
110 RUNTEST_OPTS += -e ghc_with_threaded_rts=1
111 else
112 RUNTEST_OPTS += -e ghc_with_threaded_rts=0
113 endif
114
115 ifeq "$(filter dyn, $(GhcRTSWays))" "dyn"
116 RUNTEST_OPTS += -e ghc_with_dynamic_rts=1
117 else
118 RUNTEST_OPTS += -e ghc_with_dynamic_rts=0
119 endif
120
121 ifeq "$(GhcWithInterpreter)" "NO"
122 RUNTEST_OPTS += -e config.have_interp=False
123 else ifeq "$(GhcStage)" "1"
124 RUNTEST_OPTS += -e config.have_interp=False
125 else
126 RUNTEST_OPTS += -e config.have_interp=True
127 endif
128
129 ifeq "$(GhcUnregisterised)" "YES"
130 RUNTEST_OPTS += -e config.unregisterised=True
131 else
132 RUNTEST_OPTS += -e config.unregisterised=False
133 endif
134
135 ifeq "$(GhcDynamicByDefault)" "YES"
136 RUNTEST_OPTS += -e config.ghc_dynamic_by_default=True
137 CABAL_MINIMAL_BUILD = --enable-shared --disable-library-vanilla
138 else
139 RUNTEST_OPTS += -e config.ghc_dynamic_by_default=False
140 CABAL_MINIMAL_BUILD = --enable-library-vanilla --disable-shared
141 endif
142
143 ifeq "$(GhcDynamic)" "YES"
144 RUNTEST_OPTS += -e config.ghc_dynamic=True
145 CABAL_PLUGIN_BUILD = --enable-shared --disable-library-vanilla
146 else
147 RUNTEST_OPTS += -e config.ghc_dynamic=False
148 CABAL_PLUGIN_BUILD = --enable-library-vanilla --disable-shared
149 endif
150
151 ifeq "$(GhcWithSMP)" "YES"
152 RUNTEST_OPTS += -e ghc_with_smp=1
153 else
154 RUNTEST_OPTS += -e ghc_with_smp=0
155 endif
156
157 ifeq "$(LLC)" ""
158 RUNTEST_OPTS += -e ghc_with_llvm=0
159 else ifeq "$(TargetARCH_CPP)" "powerpc"
160 RUNTEST_OPTS += -e ghc_with_llvm=0
161 else ifneq "$(LLC)" "llc"
162 # If we have a real detected value for LLVM, then it really ought to work
163 RUNTEST_OPTS += -e ghc_with_llvm=1
164 else
165 RUNTEST_OPTS += -e ghc_with_llvm=0
166 endif
167
168 ifeq "$(WINDOWS)" "YES"
169 RUNTEST_OPTS += -e windows=True
170 else
171 RUNTEST_OPTS += -e windows=False
172 endif
173
174 ifeq "$(DARWIN)" "YES"
175 RUNTEST_OPTS += -e darwin=True
176 else
177 RUNTEST_OPTS += -e darwin=False
178 endif
179
180 ifeq "$(IN_TREE_COMPILER)" "YES"
181 RUNTEST_OPTS += -e config.in_tree_compiler=True
182 else
183 RUNTEST_OPTS += -e config.in_tree_compiler=False
184 endif
185
186 ifneq "$(THREADS)" ""
187 RUNTEST_OPTS += --threads=$(THREADS)
188 endif
189
190 ifneq "$(VERBOSE)" ""
191 RUNTEST_OPTS += --verbose=$(VERBOSE)
192 endif
193
194 ifeq "$(SKIP_PERF_TESTS)" "YES"
195 RUNTEST_OPTS += --skip-perf-tests
196 endif
197
198 ifeq "$(CLEANUP)" "0"
199 RUNTEST_OPTS += -e config.cleanup=False
200 else ifeq "$(CLEANUP)" "NO"
201 RUNTEST_OPTS += -e config.cleanup=False
202 else
203 RUNTEST_OPTS += -e config.cleanup=True
204 endif
205
206 ifeq "$(LOCAL)" "0"
207 # See Note [Running tests in /tmp].
208 RUNTEST_OPTS += -e config.local=False
209 else ifeq "$(LOCAL)" "NO"
210 RUNTEST_OPTS += -e config.local=False
211 else
212 RUNTEST_OPTS += -e config.local=True
213 endif
214
215 RUNTEST_OPTS += \
216 --rootdir=. \
217 --config-file=$(CONFIG) \
218 -e 'config.confdir="$(CONFIGDIR)"' \
219 -e 'config.platform="$(TARGETPLATFORM)"' \
220 -e 'config.os="$(TargetOS_CPP)"' \
221 -e 'config.arch="$(TargetARCH_CPP)"' \
222 -e 'config.wordsize="$(WORDSIZE)"' \
223 -e 'config.timeout=int($(TIMEOUT)) or config.timeout' \
224 -e 'config.exeext="$(exeext)"' \
225 -e 'config.top="$(TOP_ABS)"'
226
227 # Wrap non-empty program paths in quotes, because they may contain spaces. Do
228 # it here, so we don't have to (and don't forget to do it) in the .T test
229 # scripts (search for '{compiler}' or '{hpc}'). This may or may not be a good
230 # idea.
231 # Use `--config` instead of `-e`, because `-e` (which calls Python's `eval`
232 # function) would require another pair of (escaped) quotes, which interfers
233 # with MinGW's magic path handling (see #10449, and
234 # http://www.mingw.org/wiki/Posix_path_conversion).
235 # We use double instead of single quotes, which may or may not be important
236 # when using msys2 (#9626, #10441).
237 quote_path = $(if $1,"$1")
238 RUNTEST_OPTS += \
239 --config 'compiler=$(call quote_path,$(TEST_HC))' \
240 --config 'ghc_pkg=$(call quote_path,$(GHC_PKG))' \
241 --config 'haddock=$(call quote_path,$(HADDOCK))' \
242 --config 'hp2ps=$(call quote_path,$(HP2PS_ABS))' \
243 --config 'hpc=$(call quote_path,$(HPC))' \
244 --config 'gs=$(call quote_path,$(GS))' \
245 --config 'timeout_prog=$(call quote_path,$(TIMEOUT_PROGRAM))'
246
247 RUNTEST_OPTS += -e "config.stage=$(GhcStage)"
248
249 ifneq "$(SUMMARY_FILE)" ""
250 RUNTEST_OPTS += \
251 --summary-file "$(SUMMARY_FILE)"
252 endif
253 ifeq "$(NO_PRINT_SUMMARY)" "YES"
254 RUNTEST_OPTS += \
255 --no-print-summary
256 endif
257
258 RUNTEST_OPTS += \
259 $(EXTRA_RUNTEST_OPTS)
260
261 ifeq "$(list_broken)" "YES"
262 set_list_broken = -e config.list_broken=True
263 else
264 set_list_broken =
265 endif
266
267 # See Note [validate and testsuite speed] in toplevel Makefile.
268 ifneq "$(SPEED)" ""
269 setspeed = -e config.speed="$(SPEED)"
270 else ifeq "$(fast)" "YES"
271 # Backward compatibility. Maybe some people are running 'make accept fast=YES'?
272 setspeed = -e config.speed=2
273 else
274 setspeed =
275 endif
276
277 ifeq "$(accept)" "YES"
278 setaccept = -e config.accept=1
279 else
280 setaccept =
281 endif
282
283 .PHONY: all boot test verbose accept fast slow list_broken
284
285 all: test
286
287 TIMEOUT_PROGRAM = $(TOP)/timeout/install-inplace/bin/timeout$(exeext)
288
289 boot: $(TIMEOUT_PROGRAM)
290
291 $(TIMEOUT_PROGRAM) :
292 @echo "Looks like you don't have timeout, building it first..."
293 $(MAKE) -C $(TOP)/timeout all
294
295 # Use a '+' to make sure that any sub-MAKEs that python spawns can
296 # communicate with the topmake.
297 # See Note [Communicating options and variables to a submake]
298 test: $(TIMEOUT_PROGRAM)
299 +PYTHON="$(PYTHON)" "$(PYTHON)" $(RUNTESTS) $(RUNTEST_OPTS) \
300 $(patsubst %, --only=%, $(TEST)) \
301 $(patsubst %, --only=%, $(TESTS)) \
302 $(patsubst %, --way=%, $(WAY)) \
303 $(patsubst %, --skipway=%, $(SKIPWAY)) \
304 $(set_list_broken) \
305 $(setspeed) \
306 $(setaccept)
307
308 verbose: test
309
310 accept:
311 $(MAKE) accept=YES
312
313 fast fasttest:
314 # See Note [validate and testsuite speed] in toplevel Makefile.
315 $(MAKE) SPEED=2
316
317 slow slowtest:
318 $(MAKE) SPEED=0
319
320 list_broken:
321 $(MAKE) list_broken=YES
322
323 # Note [Communicating options and variables to a submake]
324 #
325 # Consider the following scenario:
326 # * A test foo is defined as
327 # test('foo', [], run_command, ['$MAKE footarget'])
328 # * A user calls 'make -j24 TEST=foo'
329 #
330 # What happens is something like this:
331 # * make (topmake) reads all options and variables given on the commandline
332 # and adds them to the variable MAKEFLAGS [1]. This variable is exported by
333 # default [1], so submakes can use them.
334 # * The 'test' target calls 'python ..'
335 # * Python calls 'make footarget' (submake)
336 #
337 # **First question**: what happens to the '-j24' option when calling make
338 # recursively?
339 #
340 # From
341 # https://www.gnu.org/software/make/manual/html_node/Variables_002fRecursion.html:
342 #
343 # "The '-j' option is a special case (see Parallel Execution). If you set
344 # it to some numeric value 'N' and your operating system supports it (most
345 # any UNIX system will; others typically won't), the parent make and all the
346 # sub-makes will communicate to ensure that there are only 'N' jobs running
347 # at the same time between them all."
348 #
349 # In our scenario, the user will actually see the following warning [2]:
350 #
351 # 'warning: jobserver unavailable: using -j1. Add '+' to parent make rule.'
352 #
353 # The problem is that topmake and submake don't know about eachother, since
354 # python is in between. To let them communicate, we have to use the '+'
355 # option, by calling '+python' instead of 'python' [2]. This works,
356 # magically, and fixes #11569.
357 #
358 # **Second question**: can't we just unexport MAKEFLAGS, instead of using
359 # that '+' trick? The testsuite driver (python) mangages parallelism by
360 # itself already, so '-j24' doesn't do the right thing anyway. You have to
361 # use 'make test THREADS=24'. Unexporting MAKEFLAGS would mean ignoring
362 # any '-j' flags passed to make (either from the user calling 'make -j'
363 # explicitly or from having MAKEFLAGS=-j set in the shell, see #11569).
364 #
365 # This almost works, except when calling 'make fast/slow/accept TEST_HC=ghc'
366 # instead of just 'make test'. These targets call 'make test FAST=YES'
367 # recursively (and 'make test' calls python, as before).
368 #
369 # The problem is that in boilerplate.mk we try to override the variable
370 # TEST_HC (See Note [The TEST_HC variable]). Somewhere somehow this
371 # information (of us wanting to update TEST_HC) gets lost in the process,
372 # resulting in the final TEST_HC always getting set to the inplace compiler.
373 # It seems possible to remedy this yet again by exporting TEST_HC explicitly,
374 # but I didn't understand nor test it thoroughly (what about the other
375 # variables we override, see calls to canonicalise), and the '+' trick seems
376 # to work at least equally well (just don't run something like
377 # 'make test fast slow accept').
378 #
379 # Tests:
380 # * `make TEST=T3307 -j2` should not show a warning.
381 # * `make TEST=tc001 TEST_HC=ghc fast` should not use the inplace compiler.
382 #
383 # [1] https://www.gnu.org/software/make/manual/html_node/Variables_002fRecursion.html
384 # [2] https://www.gnu.org/software/make/manual/html_node/Error-Messages.html