1bac4fd091f81cbf0f399babe2a6ff0ac0db688c
[ghc.git] / testsuite / mk / test.mk
1 # -----------------------------------------------------------------------------
2 # Examples of use:
3 #
4 # make -- run all the tests in the current directory
5 # make verbose -- as make test, but up the verbosity
6 # make accept -- run the tests, accepting the current output
7 #
8 # The following variables may be set on the make command line:
9 #
10 # TEST -- specific test to run
11 # TESTS -- specific tests to run (same as $TEST really)
12 # EXTRA_HC_OPTS -- extra flags to send to the Haskell compiler
13 # EXTRA_RUNTEST_OPTS -- extra flags to give the test driver
14 # CONFIG -- use a different configuration file
15 # COMPILER -- select a configuration file from config/
16 # THREADS -- run n tests at once
17 # PLATFORM -- if accepting a result, accept it for the current platform.
18 # OS -- if accepting a result, accept it for all wordsizes of the
19 # current os.
20 #
21 # -----------------------------------------------------------------------------
22
23 # export the value of $MAKE for invocation in tests/driver/
24 export MAKE
25
26 RUNTESTS = $(TOP)/driver/runtests.py
27 COMPILER = ghc
28 CONFIG = $(TOP)/config/$(COMPILER)
29
30 ifeq "$(GhcUnregisterised)" "YES"
31 # Otherwise C backend generates many warnings about
32 # imcompatible proto casts for GCC's buitins:
33 # memcpy, printf, strlen.
34 EXTRA_HC_OPTS += -optc-fno-builtin
35 endif
36
37 # TEST_HC_OPTS is passed to every invocation of TEST_HC
38 # in nested Makefiles
39 TEST_HC_OPTS = -dcore-lint -dstg-lint -dcmm-lint \
40 -no-user-$(GhcPackageDbFlag) -rtsopts $(EXTRA_HC_OPTS)
41
42 ifeq "$(MinGhcVersion711)" "YES"
43 # Don't warn about missing specialisations. They can only occur with `-O`, but
44 # we want tests to produce the same output for all test ways.
45 TEST_HC_OPTS += -fno-warn-missed-specialisations
46 TEST_HC_OPTS += -fshow-warning-groups
47 endif
48
49 ifeq "$(MinGhcVersion801)" "YES"
50 # Turn off any VT800 codes in the output or they wreak havoc on the
51 # testsuite output.
52 TEST_HC_OPTS += -fdiagnostics-color=never
53 TEST_HC_OPTS += -fno-diagnostics-show-caret
54 endif
55
56 # See Trac #15278.
57 TEST_HC_OPTS += -Werror=compat
58
59 # Add the no-debug-output last as it is often convenient to copy the test invocation
60 # removing this line.
61 TEST_HC_OPTS += -dno-debug-output
62
63 TEST_HC_OPTS_INTERACTIVE = $(TEST_HC_OPTS) --interactive -v0 -ignore-dot-ghci -fno-ghci-history
64
65
66 RUNTEST_OPTS =
67
68 ifeq "$(filter $(TargetOS_CPP), cygwin32 mingw32)" ""
69 exeext =
70 else
71 exeext = .exe
72 endif
73
74 ifneq "$(filter $(TargetOS_CPP),cygwin32 mingw32)" ""
75 dllext = .dll
76 else ifeq "$(TargetOS_CPP)" "darwin"
77 dllext = .dylib
78 else
79 dllext = .so
80 endif
81
82 RUNTEST_OPTS += -e "ghc_compiler_always_flags='$(TEST_HC_OPTS)'"
83
84 RUNTEST_OPTS += -e config.compiler_debugged=$(GhcDebugged)
85
86 ifeq "$(GhcWithNativeCodeGen)" "YES"
87 RUNTEST_OPTS += -e ghc_with_native_codegen=True
88 else
89 RUNTEST_OPTS += -e ghc_with_native_codegen=False
90 endif
91
92 GHC_PRIM_LIBDIR := $(subst library-dirs: ,,$(shell "$(GHC_PKG)" field ghc-prim library-dirs --simple-output))
93 HAVE_VANILLA := $(shell if [ -f $(subst \,/,$(GHC_PRIM_LIBDIR))/GHC/PrimopWrappers.hi ]; then echo YES; else echo NO; fi)
94 HAVE_DYNAMIC := $(shell if [ -f $(subst \,/,$(GHC_PRIM_LIBDIR))/GHC/PrimopWrappers.dyn_hi ]; then echo YES; else echo NO; fi)
95 HAVE_PROFILING := $(shell if [ -f $(subst \,/,$(GHC_PRIM_LIBDIR))/GHC/PrimopWrappers.p_hi ]; then echo YES; else echo NO; fi)
96 HAVE_GDB := $(shell if gdb --version > /dev/null 2> /dev/null; then echo YES; else echo NO; fi)
97 HAVE_READELF := $(shell if readelf --version > /dev/null 2> /dev/null; then echo YES; else echo NO; fi)
98
99 ifeq "$(HAVE_VANILLA)" "YES"
100 RUNTEST_OPTS += -e config.have_vanilla=True
101 else
102 RUNTEST_OPTS += -e config.have_vanilla=False
103 endif
104
105 ifeq "$(HAVE_DYNAMIC)" "YES"
106 RUNTEST_OPTS += -e config.have_dynamic=True
107 else
108 RUNTEST_OPTS += -e config.have_dynamic=False
109 endif
110
111 ifeq "$(HAVE_PROFILING)" "YES"
112 RUNTEST_OPTS += -e config.have_profiling=True
113 else
114 RUNTEST_OPTS += -e config.have_profiling=False
115 endif
116
117 ifeq "$(filter thr, $(GhcRTSWays))" "thr"
118 RUNTEST_OPTS += -e ghc_with_threaded_rts=True
119 else
120 RUNTEST_OPTS += -e ghc_with_threaded_rts=False
121 endif
122
123 ifeq "$(filter dyn, $(GhcRTSWays))" "dyn"
124 RUNTEST_OPTS += -e ghc_with_dynamic_rts=True
125 else
126 RUNTEST_OPTS += -e ghc_with_dynamic_rts=False
127 endif
128
129 ifeq "$(GhcWithInterpreter)" "NO"
130 RUNTEST_OPTS += -e config.have_interp=False
131 else ifeq "$(GhcStage)" "1"
132 RUNTEST_OPTS += -e config.have_interp=False
133 else
134 RUNTEST_OPTS += -e config.have_interp=True
135 endif
136
137 ifeq "$(GhcUnregisterised)" "YES"
138 RUNTEST_OPTS += -e config.unregisterised=True
139 else
140 RUNTEST_OPTS += -e config.unregisterised=False
141 endif
142
143 ifeq "$(HAVE_GDB)" "YES"
144 RUNTEST_OPTS += -e config.have_gdb=True
145 else
146 RUNTEST_OPTS += -e config.have_gdb=False
147 endif
148
149 ifeq "$(HAVE_READELF)" "YES"
150 RUNTEST_OPTS += -e config.have_readelf=True
151 else
152 RUNTEST_OPTS += -e config.have_readelf=False
153 endif
154
155 ifeq "$(GhcDynamicByDefault)" "YES"
156 RUNTEST_OPTS += -e config.ghc_dynamic_by_default=True
157 CABAL_MINIMAL_BUILD = --enable-shared --disable-library-vanilla
158 else
159 RUNTEST_OPTS += -e config.ghc_dynamic_by_default=False
160 CABAL_MINIMAL_BUILD = --enable-library-vanilla --disable-shared
161 endif
162
163 ifeq "$(GhcDynamic)" "YES"
164 RUNTEST_OPTS += -e config.ghc_dynamic=True
165 CABAL_PLUGIN_BUILD = --enable-shared --disable-library-vanilla
166 else
167 RUNTEST_OPTS += -e config.ghc_dynamic=False
168 CABAL_PLUGIN_BUILD = --enable-library-vanilla --disable-shared
169 endif
170
171 ifeq "$(GhcWithSMP)" "YES"
172 RUNTEST_OPTS += -e ghc_with_smp=True
173 else
174 RUNTEST_OPTS += -e ghc_with_smp=False
175 endif
176
177 ifeq "$(LLC)" ""
178 RUNTEST_OPTS += -e ghc_with_llvm=False
179 else ifeq "$(TargetARCH_CPP)" "powerpc"
180 RUNTEST_OPTS += -e ghc_with_llvm=False
181 else ifneq "$(LLC)" "llc"
182 # If we have a real detected value for LLVM, then it really ought to work
183 RUNTEST_OPTS += -e ghc_with_llvm=True
184 else
185 RUNTEST_OPTS += -e ghc_with_llvm=False
186 endif
187
188 ifeq "$(WINDOWS)" "YES"
189 RUNTEST_OPTS += -e windows=True
190 else
191 RUNTEST_OPTS += -e windows=False
192 endif
193
194 ifeq "$(DARWIN)" "YES"
195 RUNTEST_OPTS += -e darwin=True
196 else
197 RUNTEST_OPTS += -e darwin=False
198 endif
199
200 ifeq "$(IN_TREE_COMPILER)" "YES"
201 RUNTEST_OPTS += -e config.in_tree_compiler=True
202 else
203 RUNTEST_OPTS += -e config.in_tree_compiler=False
204 endif
205
206 ifneq "$(THREADS)" ""
207 RUNTEST_OPTS += --threads=$(THREADS)
208 endif
209
210 ifneq "$(VERBOSE)" ""
211 RUNTEST_OPTS += --verbose=$(VERBOSE)
212 endif
213
214 ifeq "$(SKIP_PERF_TESTS)" "YES"
215 RUNTEST_OPTS += --skip-perf-tests
216 endif
217
218 ifeq "$(ONLY_PERF_TESTS)" "YES"
219 RUNTEST_OPTS += --only-perf-tests
220 endif
221
222 ifneq "$(TEST_ENV)" ""
223 RUNTEST_OPTS += --test-env="$(TEST_ENV)"
224 endif
225
226 ifeq "$(CLEANUP)" "0"
227 RUNTEST_OPTS += -e config.cleanup=False
228 else ifeq "$(CLEANUP)" "NO"
229 RUNTEST_OPTS += -e config.cleanup=False
230 else
231 RUNTEST_OPTS += -e config.cleanup=True
232 endif
233
234 ifeq "$(LOCAL)" "0"
235 # See Note [Running tests in /tmp].
236 RUNTEST_OPTS += -e config.local=False
237 else ifeq "$(LOCAL)" "NO"
238 RUNTEST_OPTS += -e config.local=False
239 else
240 RUNTEST_OPTS += -e config.local=True
241 endif
242
243 RUNTEST_OPTS += -e 'config.integer_backend="$(INTEGER_LIBRARY)"'
244
245 RUNTEST_OPTS += \
246 --rootdir=. \
247 --config-file=$(CONFIG) \
248 -e 'config.platform="$(TARGETPLATFORM)"' \
249 -e 'config.os="$(TargetOS_CPP)"' \
250 -e 'config.arch="$(TargetARCH_CPP)"' \
251 -e 'config.wordsize="$(WORDSIZE)"' \
252 -e 'config.timeout=int($(TIMEOUT)) or config.timeout' \
253 -e 'config.exeext="$(exeext)"' \
254 -e 'config.top="$(TOP_ABS)"'
255
256 # Wrap non-empty program paths in quotes, because they may contain spaces. Do
257 # it here, so we don't have to (and don't forget to do it) in the .T test
258 # scripts (search for '{compiler}' or '{hpc}'). This may or may not be a good
259 # idea.
260 # Use `--config` instead of `-e`, because `-e` (which calls Python's `eval`
261 # function) would require another pair of (escaped) quotes, which interfers
262 # with MinGW's magic path handling (see #10449, and
263 # http://www.mingw.org/wiki/Posix_path_conversion).
264 # We use double instead of single quotes, which may or may not be important
265 # when using msys2 (#9626, #10441).
266 quote_path = $(if $1,"$1")
267 RUNTEST_OPTS += \
268 --config 'compiler=$(call quote_path,$(TEST_HC))' \
269 --config 'ghc_pkg=$(call quote_path,$(GHC_PKG))' \
270 --config 'haddock=$(call quote_path,$(HADDOCK))' \
271 --config 'hp2ps=$(call quote_path,$(HP2PS_ABS))' \
272 --config 'hpc=$(call quote_path,$(HPC))' \
273 --config 'gs=$(call quote_path,$(GS))' \
274 --config 'timeout_prog=$(call quote_path,$(TIMEOUT_PROGRAM))'
275
276 RUNTEST_OPTS += -e "config.stage=$(GhcStage)"
277
278 ifneq "$(METRICS_FILE)" ""
279 RUNTEST_OPTS += \
280 --metrics-file "$(METRICS_FILE)"
281 endif
282 ifneq "$(JUNIT_FILE)" ""
283 RUNTEST_OPTS += \
284 --junit "$(JUNIT_FILE)"
285 endif
286 ifneq "$(SUMMARY_FILE)" ""
287 RUNTEST_OPTS += \
288 --summary-file "$(SUMMARY_FILE)"
289 endif
290 ifeq "$(NO_PRINT_SUMMARY)" "YES"
291 RUNTEST_OPTS += \
292 --no-print-summary
293 endif
294
295 RUNTEST_OPTS += \
296 $(EXTRA_RUNTEST_OPTS)
297
298 ifeq "$(list_broken)" "YES"
299 set_list_broken = -e config.list_broken=True
300 else
301 set_list_broken =
302 endif
303
304 # See Note [validate and testsuite speed] in toplevel Makefile.
305 ifneq "$(SPEED)" ""
306 setspeed = -e config.speed="$(SPEED)"
307 else ifeq "$(fast)" "YES"
308 # Backward compatibility. Maybe some people are running 'make accept fast=YES'?
309 setspeed = -e config.speed=2
310 else
311 setspeed =
312 endif
313
314 ifeq "$(accept)" "YES"
315 setaccept = -e config.accept=True
316
317 ifeq "$(PLATFORM)" "YES"
318 setaccept += -e config.accept_platform=True
319 endif
320
321 ifeq "$(OS)" "YES"
322 setaccept += -e config.accept_os=True
323 endif
324
325 else
326 setaccept =
327 endif
328
329 .PHONY: all boot test verbose accept fast slow list_broken
330
331 all: test
332
333 TIMEOUT_PROGRAM = $(TOP)/timeout/install-inplace/bin/timeout$(exeext)
334
335 boot: $(TIMEOUT_PROGRAM)
336
337 $(TIMEOUT_PROGRAM) :
338 @echo "Looks like you don't have timeout, building it first..."
339 $(MAKE) -C $(TOP)/timeout all
340
341 # Use a '+' to make sure that any sub-MAKEs that python spawns can
342 # communicate with the topmake.
343 # See Note [Communicating options and variables to a submake]
344 test: $(TIMEOUT_PROGRAM)
345 +PYTHON="$(PYTHON)" "$(PYTHON)" $(RUNTESTS) $(RUNTEST_OPTS) \
346 $(patsubst %, --only=%, $(TEST)) \
347 $(patsubst %, --only=%, $(TESTS)) \
348 $(patsubst %, --way=%, $(WAY)) \
349 $(patsubst %, --skipway=%, $(SKIPWAY)) \
350 $(set_list_broken) \
351 $(setspeed) \
352 $(setaccept)
353
354 verbose: test
355
356 accept:
357 $(MAKE) accept=YES
358
359 fast fasttest:
360 # See Note [validate and testsuite speed] in toplevel Makefile.
361 $(MAKE) SPEED=2
362
363 slow slowtest:
364 $(MAKE) SPEED=0
365
366 list_broken:
367 $(MAKE) list_broken=YES
368
369 # Note [Communicating options and variables to a submake]
370 #
371 # Consider the following scenario:
372 # * A test foo is defined as
373 # test('foo', [], run_command, ['$MAKE footarget'])
374 # * A user calls 'make -j24 TEST=foo'
375 #
376 # What happens is something like this:
377 # * make (topmake) reads all options and variables given on the commandline
378 # and adds them to the variable MAKEFLAGS [1]. This variable is exported by
379 # default [1], so submakes can use them.
380 # * The 'test' target calls 'python ..'
381 # * Python calls 'make footarget' (submake)
382 #
383 # **First question**: what happens to the '-j24' option when calling make
384 # recursively?
385 #
386 # From
387 # https://www.gnu.org/software/make/manual/html_node/Variables_002fRecursion.html:
388 #
389 # "The '-j' option is a special case (see Parallel Execution). If you set
390 # it to some numeric value 'N' and your operating system supports it (most
391 # any UNIX system will; others typically won't), the parent make and all the
392 # sub-makes will communicate to ensure that there are only 'N' jobs running
393 # at the same time between them all."
394 #
395 # In our scenario, the user will actually see the following warning [2]:
396 #
397 # 'warning: jobserver unavailable: using -j1. Add '+' to parent make rule.'
398 #
399 # The problem is that topmake and submake don't know about eachother, since
400 # python is in between. To let them communicate, we have to use the '+'
401 # option, by calling '+python' instead of 'python' [2]. This works,
402 # magically, and fixes #11569.
403 #
404 # **Second question**: can't we just unexport MAKEFLAGS, instead of using
405 # that '+' trick? The testsuite driver (python) mangages parallelism by
406 # itself already, so '-j24' doesn't do the right thing anyway. You have to
407 # use 'make test THREADS=24'. Unexporting MAKEFLAGS would mean ignoring
408 # any '-j' flags passed to make (either from the user calling 'make -j'
409 # explicitly or from having MAKEFLAGS=-j set in the shell, see #11569).
410 #
411 # This almost works, except when calling 'make fast/slow/accept TEST_HC=ghc'
412 # instead of just 'make test'. These targets call 'make test FAST=YES'
413 # recursively (and 'make test' calls python, as before).
414 #
415 # The problem is that in boilerplate.mk we try to override the variable
416 # TEST_HC (See Note [The TEST_HC variable]). Somewhere somehow this
417 # information (of us wanting to update TEST_HC) gets lost in the process,
418 # resulting in the final TEST_HC always getting set to the inplace compiler.
419 # It seems possible to remedy this yet again by exporting TEST_HC explicitly,
420 # but I didn't understand nor test it thoroughly (what about the other
421 # variables we override, see calls to canonicalise), and the '+' trick seems
422 # to work at least equally well (just don't run something like
423 # 'make test fast slow accept').
424 #
425 # Tests:
426 # * `make TEST=T3307 -j2` should not show a warning.
427 # * `make TEST=tc001 TEST_HC=ghc fast` should not use the inplace compiler.
428 #
429 # [1] https://www.gnu.org/software/make/manual/html_node/Variables_002fRecursion.html
430 # [2] https://www.gnu.org/software/make/manual/html_node/Error-Messages.html