Update Trac ticket URLs to point to GitLab
[ghc.git] / testsuite / mk / test.mk
1 # -----------------------------------------------------------------------------
2 # Examples of use:
3 #
4 # make -- run all the tests in the current directory
5 # make verbose -- as make test, but up the verbosity
6 # make accept -- run the tests, accepting the current output
7 #
8 # The following variables may be set on the make command line:
9 #
10 # TEST -- specific test to run
11 # TESTS -- specific tests to run (same as $TEST really)
12 # EXTRA_HC_OPTS -- extra flags to send to the Haskell compiler
13 # EXTRA_RUNTEST_OPTS -- extra flags to give the test driver
14 # CONFIG -- use a different configuration file
15 # COMPILER -- select a configuration file from config/
16 # THREADS -- run n tests at once
17 # PLATFORM -- if accepting a result, accept it for the current platform.
18 # OS -- if accepting a result, accept it for all wordsizes of the
19 # current os.
20 #
21 # -----------------------------------------------------------------------------
22
23 # export the value of $MAKE for invocation in tests/driver/
24 export MAKE
25
26 RUNTESTS = $(TOP)/driver/runtests.py
27 COMPILER = ghc
28 CONFIG = $(TOP)/config/$(COMPILER)
29
30 ifeq "$(GhcUnregisterised)" "YES"
31 # Otherwise C backend generates many warnings about
32 # imcompatible proto casts for GCC's buitins:
33 # memcpy, printf, strlen.
34 EXTRA_HC_OPTS += -optc-fno-builtin
35 endif
36
37 # TEST_HC_OPTS is passed to every invocation of TEST_HC
38 # in nested Makefiles
39 TEST_HC_OPTS = -dcore-lint -dstg-lint -dcmm-lint \
40 -no-user-$(GhcPackageDbFlag) -rtsopts $(EXTRA_HC_OPTS)
41
42 ifeq "$(MinGhcVersion711)" "YES"
43 # Don't warn about missing specialisations. They can only occur with `-O`, but
44 # we want tests to produce the same output for all test ways.
45 TEST_HC_OPTS += -fno-warn-missed-specialisations
46 TEST_HC_OPTS += -fshow-warning-groups
47 endif
48
49 ifeq "$(MinGhcVersion801)" "YES"
50 # Turn off any VT800 codes in the output or they wreak havoc on the
51 # testsuite output.
52 TEST_HC_OPTS += -fdiagnostics-color=never
53 TEST_HC_OPTS += -fno-diagnostics-show-caret
54 endif
55
56 # See #15278.
57 TEST_HC_OPTS += -Werror=compat
58
59 # Add the no-debug-output last as it is often convenient to copy the test invocation
60 # removing this line.
61 TEST_HC_OPTS += -dno-debug-output
62
63 TEST_HC_OPTS_INTERACTIVE = $(TEST_HC_OPTS) --interactive -v0 -ignore-dot-ghci -fno-ghci-history
64
65
66 RUNTEST_OPTS =
67
68 ifeq "$(filter $(TargetOS_CPP), cygwin32 mingw32)" ""
69 exeext =
70 else
71 exeext = .exe
72 endif
73
74 ifneq "$(filter $(TargetOS_CPP),cygwin32 mingw32)" ""
75 dllext = .dll
76 else ifeq "$(TargetOS_CPP)" "darwin"
77 dllext = .dylib
78 else
79 dllext = .so
80 endif
81
82 RUNTEST_OPTS += -e "ghc_compiler_always_flags='$(TEST_HC_OPTS)'"
83
84 RUNTEST_OPTS += -e config.compiler_debugged=$(GhcDebugged)
85
86 ifeq "$(GhcWithNativeCodeGen)" "YES"
87 RUNTEST_OPTS += -e ghc_with_native_codegen=True
88 else
89 RUNTEST_OPTS += -e ghc_with_native_codegen=False
90 endif
91
92 GHC_PRIM_LIBDIR := $(subst library-dirs: ,,$(shell "$(GHC_PKG)" field ghc-prim library-dirs --simple-output))
93 HAVE_VANILLA := $(shell if [ -f $(subst \,/,$(GHC_PRIM_LIBDIR))/GHC/PrimopWrappers.hi ]; then echo YES; else echo NO; fi)
94 HAVE_DYNAMIC := $(shell if [ -f $(subst \,/,$(GHC_PRIM_LIBDIR))/GHC/PrimopWrappers.dyn_hi ]; then echo YES; else echo NO; fi)
95 HAVE_PROFILING := $(shell if [ -f $(subst \,/,$(GHC_PRIM_LIBDIR))/GHC/PrimopWrappers.p_hi ]; then echo YES; else echo NO; fi)
96 HAVE_GDB := $(shell if gdb --version > /dev/null 2> /dev/null; then echo YES; else echo NO; fi)
97 HAVE_READELF := $(shell if readelf --version > /dev/null 2> /dev/null; then echo YES; else echo NO; fi)
98
99 ifeq "$(HAVE_VANILLA)" "YES"
100 RUNTEST_OPTS += -e config.have_vanilla=True
101 else
102 RUNTEST_OPTS += -e config.have_vanilla=False
103 endif
104
105 ifeq "$(HAVE_DYNAMIC)" "YES"
106 RUNTEST_OPTS += -e config.have_dynamic=True
107 else
108 RUNTEST_OPTS += -e config.have_dynamic=False
109 endif
110
111 ifeq "$(HAVE_PROFILING)" "YES"
112 RUNTEST_OPTS += -e config.have_profiling=True
113 else
114 RUNTEST_OPTS += -e config.have_profiling=False
115 endif
116
117 ifeq "$(filter thr, $(GhcRTSWays))" "thr"
118 RUNTEST_OPTS += -e ghc_with_threaded_rts=True
119 else
120 RUNTEST_OPTS += -e ghc_with_threaded_rts=False
121 endif
122
123 ifeq "$(filter dyn, $(GhcRTSWays))" "dyn"
124 RUNTEST_OPTS += -e ghc_with_dynamic_rts=True
125 else
126 RUNTEST_OPTS += -e ghc_with_dynamic_rts=False
127 endif
128
129 ifeq "$(GhcWithInterpreter)" "NO"
130 RUNTEST_OPTS += -e config.have_interp=False
131 else ifeq "$(GhcStage)" "1"
132 RUNTEST_OPTS += -e config.have_interp=False
133 else
134 RUNTEST_OPTS += -e config.have_interp=True
135 endif
136
137 ifeq "$(GhcUnregisterised)" "YES"
138 RUNTEST_OPTS += -e config.unregisterised=True
139 else
140 RUNTEST_OPTS += -e config.unregisterised=False
141 endif
142
143 ifeq "$(HAVE_GDB)" "YES"
144 RUNTEST_OPTS += -e config.have_gdb=True
145 else
146 RUNTEST_OPTS += -e config.have_gdb=False
147 endif
148
149 ifeq "$(HAVE_READELF)" "YES"
150 RUNTEST_OPTS += -e config.have_readelf=True
151 else
152 RUNTEST_OPTS += -e config.have_readelf=False
153 endif
154
155 ifeq "$(GhcDynamicByDefault)" "YES"
156 RUNTEST_OPTS += -e config.ghc_dynamic_by_default=True
157 CABAL_MINIMAL_BUILD = --enable-shared --disable-library-vanilla
158 else
159 RUNTEST_OPTS += -e config.ghc_dynamic_by_default=False
160 CABAL_MINIMAL_BUILD = --enable-library-vanilla --disable-shared
161 endif
162
163 ifeq "$(GhcDynamic)" "YES"
164 RUNTEST_OPTS += -e config.ghc_dynamic=True
165 CABAL_PLUGIN_BUILD = --enable-shared --disable-library-vanilla
166 else
167 RUNTEST_OPTS += -e config.ghc_dynamic=False
168 CABAL_PLUGIN_BUILD = --enable-library-vanilla --disable-shared
169 endif
170
171 ifeq "$(GhcWithSMP)" "YES"
172 RUNTEST_OPTS += -e ghc_with_smp=True
173 else
174 RUNTEST_OPTS += -e ghc_with_smp=False
175 endif
176
177 ifeq "$(LLC)" ""
178 RUNTEST_OPTS += -e ghc_with_llvm=False
179 else ifeq "$(TargetARCH_CPP)" "powerpc"
180 RUNTEST_OPTS += -e ghc_with_llvm=False
181 else ifneq "$(LLC)" "llc"
182 # If we have a real detected value for LLVM, then it really ought to work
183 RUNTEST_OPTS += -e ghc_with_llvm=True
184 else
185 RUNTEST_OPTS += -e ghc_with_llvm=False
186 endif
187
188 ifeq "$(WINDOWS)" "YES"
189 RUNTEST_OPTS += -e windows=True
190 else
191 RUNTEST_OPTS += -e windows=False
192 endif
193
194 ifeq "$(DARWIN)" "YES"
195 RUNTEST_OPTS += -e darwin=True
196 else
197 RUNTEST_OPTS += -e darwin=False
198 endif
199
200 ifeq "$(IN_TREE_COMPILER)" "YES"
201 RUNTEST_OPTS += -e config.in_tree_compiler=True
202 else
203 RUNTEST_OPTS += -e config.in_tree_compiler=False
204 endif
205
206 ifneq "$(THREADS)" ""
207 RUNTEST_OPTS += --threads=$(THREADS)
208 endif
209
210 ifneq "$(VERBOSE)" ""
211 RUNTEST_OPTS += --verbose=$(VERBOSE)
212 endif
213
214 ifeq "$(SKIP_PERF_TESTS)" "YES"
215 RUNTEST_OPTS += --skip-perf-tests
216 endif
217
218 ifeq "$(ONLY_PERF_TESTS)" "YES"
219 RUNTEST_OPTS += --only-perf-tests
220 endif
221
222 ifneq "$(TEST_ENV)" ""
223 RUNTEST_OPTS += --test-env="$(TEST_ENV)"
224 endif
225
226 ifeq "$(CLEANUP)" "0"
227 RUNTEST_OPTS += -e config.cleanup=False
228 else ifeq "$(CLEANUP)" "NO"
229 RUNTEST_OPTS += -e config.cleanup=False
230 else
231 RUNTEST_OPTS += -e config.cleanup=True
232 endif
233
234 ifeq "$(LOCAL)" "0"
235 # See Note [Running tests in /tmp].
236 RUNTEST_OPTS += -e config.local=False
237 else ifeq "$(LOCAL)" "NO"
238 RUNTEST_OPTS += -e config.local=False
239 else
240 RUNTEST_OPTS += -e config.local=True
241 endif
242
243 # Some tests in ext-interp fail when ghc-stage2 is built using LLVM. See #16087
244 ifeq "$(findstring llvm,$(BUILD_FLAVOUR))" ""
245 RUNTEST_OPTS += -e config.ghc_built_by_llvm=False
246 else
247 RUNTEST_OPTS += -e config.ghc_built_by_llvm=True
248 endif
249
250 RUNTEST_OPTS += \
251 --rootdir=. \
252 --config-file=$(CONFIG) \
253 -e 'config.platform="$(TARGETPLATFORM)"' \
254 -e 'config.os="$(TargetOS_CPP)"' \
255 -e 'config.arch="$(TargetARCH_CPP)"' \
256 -e 'config.wordsize="$(WORDSIZE)"' \
257 -e 'config.timeout=int($(TIMEOUT)) or config.timeout' \
258 -e 'config.exeext="$(exeext)"' \
259 -e 'config.top="$(TOP_ABS)"'
260
261 # Wrap non-empty program paths in quotes, because they may contain spaces. Do
262 # it here, so we don't have to (and don't forget to do it) in the .T test
263 # scripts (search for '{compiler}' or '{hpc}'). This may or may not be a good
264 # idea.
265 # Use `--config` instead of `-e`, because `-e` (which calls Python's `eval`
266 # function) would require another pair of (escaped) quotes, which interfers
267 # with MinGW's magic path handling (see #10449, and
268 # http://www.mingw.org/wiki/Posix_path_conversion).
269 # We use double instead of single quotes, which may or may not be important
270 # when using msys2 (#9626, #10441).
271 quote_path = $(if $1,"$1")
272 RUNTEST_OPTS += \
273 --config 'compiler=$(call quote_path,$(TEST_HC))' \
274 --config 'ghc_pkg=$(call quote_path,$(GHC_PKG))' \
275 --config 'haddock=$(call quote_path,$(HADDOCK))' \
276 --config 'hp2ps=$(call quote_path,$(HP2PS_ABS))' \
277 --config 'hpc=$(call quote_path,$(HPC))' \
278 --config 'gs=$(call quote_path,$(GS))' \
279 --config 'timeout_prog=$(call quote_path,$(TIMEOUT_PROGRAM))'
280
281 RUNTEST_OPTS += -e "config.stage=$(GhcStage)"
282
283 ifneq "$(METRICS_FILE)" ""
284 RUNTEST_OPTS += \
285 --metrics-file "$(METRICS_FILE)"
286 endif
287 ifneq "$(JUNIT_FILE)" ""
288 RUNTEST_OPTS += \
289 --junit "$(JUNIT_FILE)"
290 endif
291 ifneq "$(SUMMARY_FILE)" ""
292 RUNTEST_OPTS += \
293 --summary-file "$(SUMMARY_FILE)"
294 endif
295 ifeq "$(NO_PRINT_SUMMARY)" "YES"
296 RUNTEST_OPTS += \
297 --no-print-summary
298 endif
299
300 RUNTEST_OPTS += \
301 $(EXTRA_RUNTEST_OPTS)
302
303 ifeq "$(list_broken)" "YES"
304 set_list_broken = -e config.list_broken=True
305 else
306 set_list_broken =
307 endif
308
309 # See Note [validate and testsuite speed] in toplevel Makefile.
310 ifneq "$(SPEED)" ""
311 setspeed = -e config.speed="$(SPEED)"
312 else ifeq "$(fast)" "YES"
313 # Backward compatibility. Maybe some people are running 'make accept fast=YES'?
314 setspeed = -e config.speed=2
315 else
316 setspeed =
317 endif
318
319 ifeq "$(accept)" "YES"
320 setaccept = -e config.accept=True
321
322 ifeq "$(PLATFORM)" "YES"
323 setaccept += -e config.accept_platform=True
324 endif
325
326 ifeq "$(OS)" "YES"
327 setaccept += -e config.accept_os=True
328 endif
329
330 else
331 setaccept =
332 endif
333
334 .PHONY: all boot test verbose accept fast slow list_broken
335
336 all: test
337
338 TIMEOUT_PROGRAM = $(TOP)/timeout/install-inplace/bin/timeout$(exeext)
339
340 boot: $(TIMEOUT_PROGRAM)
341
342 $(TIMEOUT_PROGRAM) :
343 @echo "Looks like you don't have timeout, building it first..."
344 $(MAKE) -C $(TOP)/timeout all
345
346 # Use a '+' to make sure that any sub-MAKEs that python spawns can
347 # communicate with the topmake.
348 # See Note [Communicating options and variables to a submake]
349 test: $(TIMEOUT_PROGRAM)
350 +PYTHON="$(PYTHON)" "$(PYTHON)" $(RUNTESTS) $(RUNTEST_OPTS) \
351 $(patsubst %, --only=%, $(TEST)) \
352 $(patsubst %, --only=%, $(TESTS)) \
353 $(patsubst %, --way=%, $(WAY)) \
354 $(patsubst %, --skipway=%, $(SKIPWAY)) \
355 $(set_list_broken) \
356 $(setspeed) \
357 $(setaccept)
358
359 verbose: test
360
361 accept:
362 $(MAKE) accept=YES
363
364 fast fasttest:
365 # See Note [validate and testsuite speed] in toplevel Makefile.
366 $(MAKE) SPEED=2
367
368 slow slowtest:
369 $(MAKE) SPEED=0
370
371 list_broken:
372 $(MAKE) list_broken=YES
373
374 # Note [Communicating options and variables to a submake]
375 #
376 # Consider the following scenario:
377 # * A test foo is defined as
378 # test('foo', [], run_command, ['$MAKE footarget'])
379 # * A user calls 'make -j24 TEST=foo'
380 #
381 # What happens is something like this:
382 # * make (topmake) reads all options and variables given on the commandline
383 # and adds them to the variable MAKEFLAGS [1]. This variable is exported by
384 # default [1], so submakes can use them.
385 # * The 'test' target calls 'python ..'
386 # * Python calls 'make footarget' (submake)
387 #
388 # **First question**: what happens to the '-j24' option when calling make
389 # recursively?
390 #
391 # From
392 # https://www.gnu.org/software/make/manual/html_node/Variables_002fRecursion.html:
393 #
394 # "The '-j' option is a special case (see Parallel Execution). If you set
395 # it to some numeric value 'N' and your operating system supports it (most
396 # any UNIX system will; others typically won't), the parent make and all the
397 # sub-makes will communicate to ensure that there are only 'N' jobs running
398 # at the same time between them all."
399 #
400 # In our scenario, the user will actually see the following warning [2]:
401 #
402 # 'warning: jobserver unavailable: using -j1. Add '+' to parent make rule.'
403 #
404 # The problem is that topmake and submake don't know about eachother, since
405 # python is in between. To let them communicate, we have to use the '+'
406 # option, by calling '+python' instead of 'python' [2]. This works,
407 # magically, and fixes #11569.
408 #
409 # **Second question**: can't we just unexport MAKEFLAGS, instead of using
410 # that '+' trick? The testsuite driver (python) mangages parallelism by
411 # itself already, so '-j24' doesn't do the right thing anyway. You have to
412 # use 'make test THREADS=24'. Unexporting MAKEFLAGS would mean ignoring
413 # any '-j' flags passed to make (either from the user calling 'make -j'
414 # explicitly or from having MAKEFLAGS=-j set in the shell, see #11569).
415 #
416 # This almost works, except when calling 'make fast/slow/accept TEST_HC=ghc'
417 # instead of just 'make test'. These targets call 'make test FAST=YES'
418 # recursively (and 'make test' calls python, as before).
419 #
420 # The problem is that in boilerplate.mk we try to override the variable
421 # TEST_HC (See Note [The TEST_HC variable]). Somewhere somehow this
422 # information (of us wanting to update TEST_HC) gets lost in the process,
423 # resulting in the final TEST_HC always getting set to the inplace compiler.
424 # It seems possible to remedy this yet again by exporting TEST_HC explicitly,
425 # but I didn't understand nor test it thoroughly (what about the other
426 # variables we override, see calls to canonicalise), and the '+' trick seems
427 # to work at least equally well (just don't run something like
428 # 'make test fast slow accept').
429 #
430 # Tests:
431 # * `make TEST=T3307 -j2` should not show a warning.
432 # * `make TEST=tc001 TEST_HC=ghc fast` should not use the inplace compiler.
433 #
434 # [1] https://www.gnu.org/software/make/manual/html_node/Variables_002fRecursion.html
435 # [2] https://www.gnu.org/software/make/manual/html_node/Error-Messages.html