testsuite: Add testcase for #17206
[ghc.git] / testsuite / mk / test.mk
1 # -----------------------------------------------------------------------------
2 # Examples of use:
3 #
4 # make -- run all the tests in the current directory
5 # make verbose -- as make test, but up the verbosity
6 # make accept -- run the tests, accepting the current output
7 #
8 # The following variables may be set on the make command line:
9 #
10 # TEST -- specific test to run
11 # TESTS -- specific tests to run (same as $TEST really)
12 # EXTRA_HC_OPTS -- extra flags to send to the Haskell compiler
13 # EXTRA_RUNTEST_OPTS -- extra flags to give the test driver
14 # CONFIG -- use a different configuration file
15 # COMPILER -- select a configuration file from config/
16 # THREADS -- run n tests at once
17 # PLATFORM -- if accepting a result, accept it for the current platform.
18 # OS -- if accepting a result, accept it for all wordsizes of the
19 # current os.
20 #
21 # -----------------------------------------------------------------------------
22
23 # export the value of $MAKE for invocation in tests/driver/
24 export MAKE
25
26 RUNTESTS = $(TOP)/driver/runtests.py
27 COMPILER = ghc
28 CONFIG = $(TOP)/config/$(COMPILER)
29
30 ifeq "$(GhcUnregisterised)" "YES"
31 # Otherwise C backend generates many warnings about
32 # imcompatible proto casts for GCC's buitins:
33 # memcpy, printf, strlen.
34 EXTRA_HC_OPTS += -optc-fno-builtin
35 endif
36
37 # TEST_HC_OPTS is passed to every invocation of TEST_HC
38 # in nested Makefiles
39 TEST_HC_OPTS = -dcore-lint -dstg-lint -dcmm-lint \
40 -no-user-$(GhcPackageDbFlag) -rtsopts $(EXTRA_HC_OPTS)
41
42 ifeq "$(MinGhcVersion711)" "YES"
43 # Don't warn about missing specialisations. They can only occur with `-O`, but
44 # we want tests to produce the same output for all test ways.
45 TEST_HC_OPTS += -fno-warn-missed-specialisations
46 TEST_HC_OPTS += -fshow-warning-groups
47 endif
48
49 ifeq "$(MinGhcVersion801)" "YES"
50 # Turn off any VT800 codes in the output or they wreak havoc on the
51 # testsuite output.
52 TEST_HC_OPTS += -fdiagnostics-color=never
53 TEST_HC_OPTS += -fno-diagnostics-show-caret
54 endif
55
56 # See #15278.
57 TEST_HC_OPTS += -Werror=compat
58
59 # Add the no-debug-output last as it is often convenient to copy the test invocation
60 # removing this line.
61 TEST_HC_OPTS += -dno-debug-output
62
63 TEST_HC_OPTS_INTERACTIVE = $(TEST_HC_OPTS) --interactive -v0 -ignore-dot-ghci -fno-ghci-history
64
65
66 RUNTEST_OPTS =
67
68 ifeq "$(filter $(TargetOS_CPP), cygwin32 mingw32)" ""
69 exeext =
70 else
71 exeext = .exe
72 endif
73
74 ifneq "$(filter $(TargetOS_CPP),cygwin32 mingw32)" ""
75 dllext = .dll
76 else ifeq "$(TargetOS_CPP)" "darwin"
77 dllext = .dylib
78 else
79 dllext = .so
80 endif
81
82 RUNTEST_OPTS += -e "ghc_compiler_always_flags='$(TEST_HC_OPTS)'"
83
84 ifeq "$(GhcDebugged)" "YES"
85 RUNTEST_OPTS += -e "config.compiler_debugged=True"
86 else
87 RUNTEST_OPTS += -e "config.compiler_debugged=False"
88 endif
89
90 ifeq "$(GhcWithNativeCodeGen)" "YES"
91 RUNTEST_OPTS += -e ghc_with_native_codegen=True
92 else
93 RUNTEST_OPTS += -e ghc_with_native_codegen=False
94 endif
95
96 GHC_PRIM_LIBDIR := $(subst library-dirs: ,,$(shell "$(GHC_PKG)" field ghc-prim library-dirs --simple-output))
97 HAVE_VANILLA := $(shell if [ -f $(subst \,/,$(GHC_PRIM_LIBDIR))/GHC/PrimopWrappers.hi ]; then echo YES; else echo NO; fi)
98 HAVE_DYNAMIC := $(shell if [ -f $(subst \,/,$(GHC_PRIM_LIBDIR))/GHC/PrimopWrappers.dyn_hi ]; then echo YES; else echo NO; fi)
99 HAVE_PROFILING := $(shell if [ -f $(subst \,/,$(GHC_PRIM_LIBDIR))/GHC/PrimopWrappers.p_hi ]; then echo YES; else echo NO; fi)
100 HAVE_GDB := $(shell if gdb --version > /dev/null 2> /dev/null; then echo YES; else echo NO; fi)
101 HAVE_READELF := $(shell if readelf --version > /dev/null 2> /dev/null; then echo YES; else echo NO; fi)
102
103 ifeq "$(HAVE_VANILLA)" "YES"
104 RUNTEST_OPTS += -e config.have_vanilla=True
105 else
106 RUNTEST_OPTS += -e config.have_vanilla=False
107 endif
108
109 ifeq "$(HAVE_DYNAMIC)" "YES"
110 RUNTEST_OPTS += -e config.have_dynamic=True
111 else
112 RUNTEST_OPTS += -e config.have_dynamic=False
113 endif
114
115 ifeq "$(HAVE_PROFILING)" "YES"
116 RUNTEST_OPTS += -e config.have_profiling=True
117 else
118 RUNTEST_OPTS += -e config.have_profiling=False
119 endif
120
121 ifeq "$(filter thr, $(GhcRTSWays))" "thr"
122 RUNTEST_OPTS += -e ghc_with_threaded_rts=True
123 else
124 RUNTEST_OPTS += -e ghc_with_threaded_rts=False
125 endif
126
127 ifeq "$(filter dyn, $(GhcRTSWays))" "dyn"
128 RUNTEST_OPTS += -e ghc_with_dynamic_rts=True
129 else
130 RUNTEST_OPTS += -e ghc_with_dynamic_rts=False
131 endif
132
133 ifeq "$(GhcWithInterpreter)" "NO"
134 RUNTEST_OPTS += -e config.have_interp=False
135 else ifeq "$(GhcStage)" "1"
136 RUNTEST_OPTS += -e config.have_interp=False
137 else
138 RUNTEST_OPTS += -e config.have_interp=True
139 endif
140
141 ifeq "$(GhcUnregisterised)" "YES"
142 RUNTEST_OPTS += -e config.unregisterised=True
143 else
144 RUNTEST_OPTS += -e config.unregisterised=False
145 endif
146
147 ifeq "$(HAVE_GDB)" "YES"
148 RUNTEST_OPTS += -e config.have_gdb=True
149 else
150 RUNTEST_OPTS += -e config.have_gdb=False
151 endif
152
153 ifeq "$(HAVE_READELF)" "YES"
154 RUNTEST_OPTS += -e config.have_readelf=True
155 else
156 RUNTEST_OPTS += -e config.have_readelf=False
157 endif
158
159 ifeq "$(GhcDynamicByDefault)" "YES"
160 RUNTEST_OPTS += -e config.ghc_dynamic_by_default=True
161 CABAL_MINIMAL_BUILD = --enable-shared --disable-library-vanilla
162 else
163 RUNTEST_OPTS += -e config.ghc_dynamic_by_default=False
164 CABAL_MINIMAL_BUILD = --enable-library-vanilla --disable-shared
165 endif
166
167 ifeq "$(GhcDynamic)" "YES"
168 RUNTEST_OPTS += -e config.ghc_dynamic=True
169 CABAL_PLUGIN_BUILD = --enable-shared --disable-library-vanilla
170 else
171 RUNTEST_OPTS += -e config.ghc_dynamic=False
172 CABAL_PLUGIN_BUILD = --enable-library-vanilla --disable-shared
173 endif
174
175 ifeq "$(GhcWithSMP)" "YES"
176 RUNTEST_OPTS += -e ghc_with_smp=True
177 else
178 RUNTEST_OPTS += -e ghc_with_smp=False
179 endif
180
181 ifeq "$(LLC)" ""
182 RUNTEST_OPTS += -e ghc_with_llvm=False
183 else ifeq "$(TargetARCH_CPP)" "powerpc"
184 RUNTEST_OPTS += -e ghc_with_llvm=False
185 else ifneq "$(LLC)" "llc"
186 # If we have a real detected value for LLVM, then it really ought to work
187 RUNTEST_OPTS += -e ghc_with_llvm=True
188 else
189 RUNTEST_OPTS += -e ghc_with_llvm=False
190 endif
191
192 ifeq "$(WINDOWS)" "YES"
193 RUNTEST_OPTS += -e windows=True
194 else
195 RUNTEST_OPTS += -e windows=False
196 endif
197
198 ifeq "$(DARWIN)" "YES"
199 RUNTEST_OPTS += -e darwin=True
200 else
201 RUNTEST_OPTS += -e darwin=False
202 endif
203
204 ifeq "$(IN_TREE_COMPILER)" "YES"
205 RUNTEST_OPTS += -e config.in_tree_compiler=True
206 else
207 RUNTEST_OPTS += -e config.in_tree_compiler=False
208 endif
209
210 ifneq "$(THREADS)" ""
211 RUNTEST_OPTS += --threads=$(THREADS)
212 endif
213
214 ifneq "$(VERBOSE)" ""
215 RUNTEST_OPTS += --verbose=$(VERBOSE)
216 endif
217
218 ifeq "$(SKIP_PERF_TESTS)" "YES"
219 RUNTEST_OPTS += --skip-perf-tests
220 endif
221
222 ifeq "$(ONLY_PERF_TESTS)" "YES"
223 RUNTEST_OPTS += --only-perf-tests
224 endif
225
226 ifneq "$(TEST_ENV)" ""
227 RUNTEST_OPTS += --test-env="$(TEST_ENV)"
228 endif
229
230 ifeq "$(CLEANUP)" "0"
231 RUNTEST_OPTS += -e config.cleanup=False
232 else ifeq "$(CLEANUP)" "NO"
233 RUNTEST_OPTS += -e config.cleanup=False
234 else
235 RUNTEST_OPTS += -e config.cleanup=True
236 endif
237
238 ifeq "$(LOCAL)" "0"
239 # See Note [Running tests in /tmp].
240 RUNTEST_OPTS += -e config.local=False
241 else ifeq "$(LOCAL)" "NO"
242 RUNTEST_OPTS += -e config.local=False
243 else
244 RUNTEST_OPTS += -e config.local=True
245 endif
246
247 # Some tests in ext-interp fail when ghc-stage2 is built using LLVM. See #16087
248 ifeq "$(findstring llvm,$(BUILD_FLAVOUR))" ""
249 RUNTEST_OPTS += -e config.ghc_built_by_llvm=False
250 else
251 RUNTEST_OPTS += -e config.ghc_built_by_llvm=True
252 endif
253
254 RUNTEST_OPTS += \
255 --rootdir=. \
256 --config-file=$(CONFIG) \
257 -e 'config.platform="$(TARGETPLATFORM)"' \
258 -e 'config.os="$(TargetOS_CPP)"' \
259 -e 'config.arch="$(TargetARCH_CPP)"' \
260 -e 'config.wordsize="$(WORDSIZE)"' \
261 -e 'config.timeout=int($(TIMEOUT)) or config.timeout' \
262 -e 'config.exeext="$(exeext)"' \
263 -e 'config.top="$(TOP_ABS)"'
264
265 # Wrap non-empty program paths in quotes, because they may contain spaces. Do
266 # it here, so we don't have to (and don't forget to do it) in the .T test
267 # scripts (search for '{compiler}' or '{hpc}'). This may or may not be a good
268 # idea.
269 # Use `--config` instead of `-e`, because `-e` (which calls Python's `eval`
270 # function) would require another pair of (escaped) quotes, which interfers
271 # with MinGW's magic path handling (see #10449, and
272 # http://www.mingw.org/wiki/Posix_path_conversion).
273 # We use double instead of single quotes, which may or may not be important
274 # when using msys2 (#9626, #10441).
275 quote_path = $(if $1,"$1")
276 RUNTEST_OPTS += \
277 --config 'compiler=$(call quote_path,$(TEST_HC))' \
278 --config 'ghc_pkg=$(call quote_path,$(GHC_PKG))' \
279 --config 'haddock=$(call quote_path,$(HADDOCK))' \
280 --config 'hp2ps=$(call quote_path,$(HP2PS_ABS))' \
281 --config 'hpc=$(call quote_path,$(HPC))' \
282 --config 'gs=$(call quote_path,$(GS))' \
283 --config 'timeout_prog=$(call quote_path,$(TIMEOUT_PROGRAM))'
284
285 RUNTEST_OPTS += --config 'stats_files_dir=$(TOP)/tests/perf/haddock'
286
287 RUNTEST_OPTS += -e "config.stage=$(GhcStage)"
288
289 ifneq "$(METRICS_FILE)" ""
290 RUNTEST_OPTS += \
291 --metrics-file "$(METRICS_FILE)"
292 endif
293 ifneq "$(JUNIT_FILE)" ""
294 RUNTEST_OPTS += \
295 --junit "$(JUNIT_FILE)"
296 endif
297 ifneq "$(SUMMARY_FILE)" ""
298 RUNTEST_OPTS += \
299 --summary-file "$(SUMMARY_FILE)"
300 endif
301 ifeq "$(NO_PRINT_SUMMARY)" "YES"
302 RUNTEST_OPTS += \
303 --no-print-summary
304 endif
305
306 RUNTEST_OPTS += \
307 $(EXTRA_RUNTEST_OPTS)
308
309 ifeq "$(list_broken)" "YES"
310 set_list_broken = -e config.list_broken=True
311 else
312 set_list_broken =
313 endif
314
315 # See Note [validate and testsuite speed] in toplevel Makefile.
316 ifneq "$(SPEED)" ""
317 setspeed = -e config.speed="$(SPEED)"
318 else ifeq "$(fast)" "YES"
319 # Backward compatibility. Maybe some people are running 'make accept fast=YES'?
320 setspeed = -e config.speed=2
321 else
322 setspeed =
323 endif
324
325 ifeq "$(accept)" "YES"
326 setaccept = -e config.accept=True
327
328 ifeq "$(PLATFORM)" "YES"
329 setaccept += -e config.accept_platform=True
330 endif
331
332 ifeq "$(OS)" "YES"
333 setaccept += -e config.accept_os=True
334 endif
335
336 else
337 setaccept =
338 endif
339
340 .PHONY: all boot test verbose accept fast slow list_broken
341
342 all: test
343
344 TIMEOUT_PROGRAM = $(TOP)/timeout/install-inplace/bin/timeout$(exeext)
345
346 boot: $(TIMEOUT_PROGRAM)
347
348 $(TIMEOUT_PROGRAM) :
349 @echo "Looks like you don't have timeout, building it first..."
350 $(MAKE) -C $(TOP)/timeout all
351
352 # Use a '+' to make sure that any sub-MAKEs that python spawns can
353 # communicate with the topmake.
354 # See Note [Communicating options and variables to a submake]
355 test: $(TIMEOUT_PROGRAM)
356 +PYTHON="$(PYTHON)" "$(PYTHON)" $(RUNTESTS) $(RUNTEST_OPTS) \
357 $(patsubst %, --only=%, $(TEST)) \
358 $(patsubst %, --only=%, $(TESTS)) \
359 $(patsubst %, --way=%, $(WAY)) \
360 $(patsubst %, --skipway=%, $(SKIPWAY)) \
361 $(set_list_broken) \
362 $(setspeed) \
363 $(setaccept)
364
365 verbose: test
366
367 accept:
368 $(MAKE) accept=YES
369
370 fast fasttest:
371 # See Note [validate and testsuite speed] in toplevel Makefile.
372 $(MAKE) SPEED=2
373
374 slow slowtest:
375 $(MAKE) SPEED=0
376
377 list_broken:
378 $(MAKE) list_broken=YES
379
380 # Note [Communicating options and variables to a submake]
381 #
382 # Consider the following scenario:
383 # * A test foo is defined as
384 # test('foo', [], run_command, ['$MAKE footarget'])
385 # * A user calls 'make -j24 TEST=foo'
386 #
387 # What happens is something like this:
388 # * make (topmake) reads all options and variables given on the commandline
389 # and adds them to the variable MAKEFLAGS [1]. This variable is exported by
390 # default [1], so submakes can use them.
391 # * The 'test' target calls 'python ..'
392 # * Python calls 'make footarget' (submake)
393 #
394 # **First question**: what happens to the '-j24' option when calling make
395 # recursively?
396 #
397 # From
398 # https://www.gnu.org/software/make/manual/html_node/Variables_002fRecursion.html:
399 #
400 # "The '-j' option is a special case (see Parallel Execution). If you set
401 # it to some numeric value 'N' and your operating system supports it (most
402 # any UNIX system will; others typically won't), the parent make and all the
403 # sub-makes will communicate to ensure that there are only 'N' jobs running
404 # at the same time between them all."
405 #
406 # In our scenario, the user will actually see the following warning [2]:
407 #
408 # 'warning: jobserver unavailable: using -j1. Add '+' to parent make rule.'
409 #
410 # The problem is that topmake and submake don't know about eachother, since
411 # python is in between. To let them communicate, we have to use the '+'
412 # option, by calling '+python' instead of 'python' [2]. This works,
413 # magically, and fixes #11569.
414 #
415 # **Second question**: can't we just unexport MAKEFLAGS, instead of using
416 # that '+' trick? The testsuite driver (python) mangages parallelism by
417 # itself already, so '-j24' doesn't do the right thing anyway. You have to
418 # use 'make test THREADS=24'. Unexporting MAKEFLAGS would mean ignoring
419 # any '-j' flags passed to make (either from the user calling 'make -j'
420 # explicitly or from having MAKEFLAGS=-j set in the shell, see #11569).
421 #
422 # This almost works, except when calling 'make fast/slow/accept TEST_HC=ghc'
423 # instead of just 'make test'. These targets call 'make test FAST=YES'
424 # recursively (and 'make test' calls python, as before).
425 #
426 # The problem is that in boilerplate.mk we try to override the variable
427 # TEST_HC (See Note [The TEST_HC variable]). Somewhere somehow this
428 # information (of us wanting to update TEST_HC) gets lost in the process,
429 # resulting in the final TEST_HC always getting set to the inplace compiler.
430 # It seems possible to remedy this yet again by exporting TEST_HC explicitly,
431 # but I didn't understand nor test it thoroughly (what about the other
432 # variables we override, see calls to canonicalise), and the '+' trick seems
433 # to work at least equally well (just don't run something like
434 # 'make test fast slow accept').
435 #
436 # Tests:
437 # * `make TEST=T3307 -j2` should not show a warning.
438 # * `make TEST=tc001 TEST_HC=ghc fast` should not use the inplace compiler.
439 #
440 # [1] https://www.gnu.org/software/make/manual/html_node/Variables_002fRecursion.html
441 # [2] https://www.gnu.org/software/make/manual/html_node/Error-Messages.html