Testsuite: make CLEANUP=1 the default (#9758)
[ghc.git] / testsuite / mk / test.mk
1 # -----------------------------------------------------------------------------
2 # Examples of use:
3 #
4 # make -- run all the tests in the current directory
5 # make verbose -- as make test, but up the verbosity
6 # make accept -- run the tests, accepting the current output
7 #
8 # The following variables may be set on the make command line:
9 #
10 # TEST -- specific test to run
11 # TESTS -- specific tests to run (same as $TEST really)
12 # EXTRA_HC_OPTS -- extra flags to send to the Haskell compiler
13 # EXTRA_RUNTEST_OPTS -- extra flags to give the test driver
14 # CONFIG -- use a different configuration file
15 # COMPILER -- select a configuration file from config/
16 # THREADS -- run n tests at once
17 #
18 # -----------------------------------------------------------------------------
19
20 # export the value of $MAKE for invocation in tests/driver/
21 export MAKE
22
23 RUNTESTS = $(TOP)/driver/runtests.py
24 COMPILER = ghc
25 CONFIGDIR = $(TOP)/config
26 CONFIG = $(CONFIGDIR)/$(COMPILER)
27
28 ifeq "$(GhcUnregisterised)" "YES"
29 # Otherwise C backend generates many warnings about
30 # imcompatible proto casts for GCC's buitins:
31 # memcpy, printf, strlen.
32 EXTRA_HC_OPTS += -optc-fno-builtin
33 endif
34
35 # TEST_HC_OPTS is passed to every invocation of TEST_HC
36 # in nested Makefiles
37 TEST_HC_OPTS = -fforce-recomp -dcore-lint -dcmm-lint -dno-debug-output -no-user-$(GhcPackageDbFlag) -rtsopts $(EXTRA_HC_OPTS)
38
39 # The warning suppression flag below is a temporary kludge. While working with
40 # tests that contain tabs, please de-tab them so this flag can be eventually
41 # removed. See
42 # http://ghc.haskell.org/trac/ghc/wiki/Commentary/CodingStyle#TabsvsSpaces
43 # for details
44 #
45 TEST_HC_OPTS += -fno-warn-tabs
46
47 ifeq "$(MinGhcVersion711)" "YES"
48 # Don't warn about missing specialisations. They can only occur with `-O`, but
49 # we want tests to produce the same output for all test ways.
50 TEST_HC_OPTS += -fno-warn-missed-specialisations
51 TEST_HC_OPTS += -fshow-warning-groups
52 endif
53
54 RUNTEST_OPTS =
55
56 ifeq "$(filter $(TargetOS_CPP), cygwin32 mingw32)" ""
57 exeext =
58 else
59 exeext = .exe
60 endif
61
62 ifneq "$(filter $(TargetOS_CPP),cygwin32 mingw32)" ""
63 dllext = .dll
64 else ifeq "$(TargetOS_CPP)" "darwin"
65 dllext = .dylib
66 else
67 dllext = .so
68 endif
69
70 RUNTEST_OPTS += -e ghc_compiler_always_flags="'$(TEST_HC_OPTS)'"
71
72 RUNTEST_OPTS += -e ghc_debugged=$(GhcDebugged)
73
74 ifeq "$(GhcWithNativeCodeGen)" "YES"
75 RUNTEST_OPTS += -e ghc_with_native_codegen=1
76 else
77 RUNTEST_OPTS += -e ghc_with_native_codegen=0
78 endif
79
80 GHC_PRIM_LIBDIR := $(subst library-dirs: ,,$(shell "$(GHC_PKG)" field ghc-prim library-dirs --simple-output))
81 HAVE_VANILLA := $(shell if [ -f $(subst \,/,$(GHC_PRIM_LIBDIR))/GHC/PrimopWrappers.hi ]; then echo YES; else echo NO; fi)
82 HAVE_DYNAMIC := $(shell if [ -f $(subst \,/,$(GHC_PRIM_LIBDIR))/GHC/PrimopWrappers.dyn_hi ]; then echo YES; else echo NO; fi)
83 HAVE_PROFILING := $(shell if [ -f $(subst \,/,$(GHC_PRIM_LIBDIR))/GHC/PrimopWrappers.p_hi ]; then echo YES; else echo NO; fi)
84
85 ifeq "$(HAVE_VANILLA)" "YES"
86 RUNTEST_OPTS += -e ghc_with_vanilla=1
87 else
88 RUNTEST_OPTS += -e ghc_with_vanilla=0
89 endif
90
91 ifeq "$(HAVE_DYNAMIC)" "YES"
92 RUNTEST_OPTS += -e ghc_with_dynamic=1
93 else
94 RUNTEST_OPTS += -e ghc_with_dynamic=0
95 endif
96
97 ifeq "$(HAVE_PROFILING)" "YES"
98 RUNTEST_OPTS += -e ghc_with_profiling=1
99 else
100 RUNTEST_OPTS += -e ghc_with_profiling=0
101 endif
102
103 ifeq "$(filter thr, $(GhcRTSWays))" "thr"
104 RUNTEST_OPTS += -e ghc_with_threaded_rts=1
105 else
106 RUNTEST_OPTS += -e ghc_with_threaded_rts=0
107 endif
108
109 ifeq "$(filter dyn, $(GhcRTSWays))" "dyn"
110 RUNTEST_OPTS += -e ghc_with_dynamic_rts=1
111 else
112 RUNTEST_OPTS += -e ghc_with_dynamic_rts=0
113 endif
114
115 ifeq "$(GhcWithInterpreter)" "NO"
116 RUNTEST_OPTS += -e ghc_with_interpreter=0
117 else ifeq "$(GhcStage)" "1"
118 RUNTEST_OPTS += -e ghc_with_interpreter=0
119 else
120 RUNTEST_OPTS += -e ghc_with_interpreter=1
121 endif
122
123 ifeq "$(GhcUnregisterised)" "YES"
124 RUNTEST_OPTS += -e ghc_unregisterised=1
125 else
126 RUNTEST_OPTS += -e ghc_unregisterised=0
127 endif
128
129 ifeq "$(GhcDynamicByDefault)" "YES"
130 RUNTEST_OPTS += -e ghc_dynamic_by_default=True
131 CABAL_MINIMAL_BUILD = --enable-shared --disable-library-vanilla
132 else
133 RUNTEST_OPTS += -e ghc_dynamic_by_default=False
134 CABAL_MINIMAL_BUILD = --enable-library-vanilla --disable-shared
135 endif
136
137 ifeq "$(GhcDynamic)" "YES"
138 RUNTEST_OPTS += -e ghc_dynamic=True
139 CABAL_PLUGIN_BUILD = --enable-shared --disable-library-vanilla
140 else
141 RUNTEST_OPTS += -e ghc_dynamic=False
142 CABAL_PLUGIN_BUILD = --enable-library-vanilla --disable-shared
143 endif
144
145 ifeq "$(GhcWithSMP)" "YES"
146 RUNTEST_OPTS += -e ghc_with_smp=1
147 else
148 RUNTEST_OPTS += -e ghc_with_smp=0
149 endif
150
151 ifeq "$(LLC)" ""
152 RUNTEST_OPTS += -e ghc_with_llvm=0
153 else ifeq "$(TargetARCH_CPP)" "powerpc"
154 RUNTEST_OPTS += -e ghc_with_llvm=0
155 else ifneq "$(LLC)" "llc"
156 # If we have a real detected value for LLVM, then it really ought to work
157 RUNTEST_OPTS += -e ghc_with_llvm=1
158 else
159 RUNTEST_OPTS += -e ghc_with_llvm=0
160 endif
161
162 ifeq "$(WINDOWS)" "YES"
163 RUNTEST_OPTS += -e windows=True
164 else
165 RUNTEST_OPTS += -e windows=False
166 endif
167
168 ifeq "$(DARWIN)" "YES"
169 RUNTEST_OPTS += -e darwin=True
170 else
171 RUNTEST_OPTS += -e darwin=False
172 endif
173
174 ifeq "$(IN_TREE_COMPILER)" "YES"
175 RUNTEST_OPTS += -e in_tree_compiler=True
176 else
177 RUNTEST_OPTS += -e in_tree_compiler=False
178 endif
179
180 ifneq "$(THREADS)" ""
181 RUNTEST_OPTS += --threads=$(THREADS)
182 endif
183
184 ifneq "$(VERBOSE)" ""
185 RUNTEST_OPTS += --verbose=$(VERBOSE)
186 endif
187
188 ifeq "$(SKIP_PERF_TESTS)" "YES"
189 RUNTEST_OPTS += --skip-perf-tests
190 endif
191
192 ifeq "$(CLEANUP)" "0"
193 RUNTEST_OPTS += -e cleanup=False
194 else ifeq "$(CLEANUP)" "NO"
195 RUNTEST_OPTS += -e cleanup=False
196 else
197 RUNTEST_OPTS += -e cleanup=True
198 endif
199
200 ifneq "$(CLEAN_ONLY)" ""
201 RUNTEST_OPTS += -e clean_only=True
202 else
203 RUNTEST_OPTS += -e clean_only=False
204 endif
205
206 ifneq "$(CHECK_FILES_WRITTEN)" ""
207 RUNTEST_OPTS += --check-files-written
208 endif
209
210 RUNTEST_OPTS += \
211 --rootdir=. \
212 --configfile=$(CONFIG) \
213 -e 'config.confdir="$(CONFIGDIR)"' \
214 -e 'config.platform="$(TARGETPLATFORM)"' \
215 -e 'config.os="$(TargetOS_CPP)"' \
216 -e 'config.arch="$(TargetARCH_CPP)"' \
217 -e 'config.wordsize="$(WORDSIZE)"' \
218 -e 'config.timeout=int($(TIMEOUT)) or config.timeout' \
219 -e 'config.exeext="$(exeext)"' \
220 -e 'config.top="$(TOP_ABS)"'
221
222 # Wrap non-empty program paths in quotes, because they may contain spaces. Do
223 # it here, so we don't have to (and don't forget to do it) in the .T test
224 # scripts (search for '{compiler}' or '{hpc}'). This may or may not be a good
225 # idea.
226 # Use `--config` instead of `-e`, because `-e` (which calls Python's `eval`
227 # function) would require another pair of (escaped) quotes, which interfers
228 # with MinGW's magic path handling (see #10449, and
229 # http://www.mingw.org/wiki/Posix_path_conversion).
230 # We use double instead of single quotes, which may or may not be important
231 # when using msys2 (#9626, #10441).
232 quote_path = $(if $1,"$1")
233 RUNTEST_OPTS += \
234 --config 'compiler=$(call quote_path,$(TEST_HC))' \
235 --config 'ghc_pkg=$(call quote_path,$(GHC_PKG))' \
236 --config 'haddock=$(call quote_path,$(HADDOCK))' \
237 --config 'hp2ps=$(call quote_path,$(HP2PS_ABS))' \
238 --config 'hpc=$(call quote_path,$(HPC))' \
239 --config 'gs=$(call quote_path,$(GS))' \
240 --config 'timeout_prog=$(call quote_path,$(TIMEOUT_PROGRAM))'
241
242 ifneq "$(SUMMARY_FILE)" ""
243 RUNTEST_OPTS += \
244 --summary-file "$(SUMMARY_FILE)"
245 endif
246 ifeq "$(NO_PRINT_SUMMARY)" "YES"
247 RUNTEST_OPTS += \
248 --no-print-summary 1
249 endif
250
251 RUNTEST_OPTS += \
252 $(EXTRA_RUNTEST_OPTS)
253
254 ifeq "$(list_broken)" "YES"
255 set_list_broken = -e config.list_broken=True
256 else
257 set_list_broken =
258 endif
259
260 # See Note [validate and testsuite speed] in toplevel Makefile.
261 ifneq "$(SPEED)" ""
262 setspeed = -e config.speed="$(SPEED)"
263 else ifeq "$(fast)" "YES"
264 # Backward compatibility. Maybe some people are running 'make accept fast=YES'?
265 setspeed = -e config.speed=2
266 else
267 setspeed =
268 endif
269
270 ifeq "$(accept)" "YES"
271 setaccept = -e config.accept=1
272 else
273 setaccept =
274 endif
275
276 .PHONY: all boot test verbose accept fast slow list_broken
277
278 all: test
279
280 TIMEOUT_PROGRAM = $(TOP)/timeout/install-inplace/bin/timeout$(exeext)
281
282 boot: $(TIMEOUT_PROGRAM)
283
284 $(TIMEOUT_PROGRAM) :
285 @echo "Looks like you don't have timeout, building it first..."
286 $(MAKE) -C $(TOP)/timeout all
287
288 # Use a '+' to make sure that any sub-MAKEs that python spawns can
289 # communicate with the topmake.
290 # See Note [Communicating options and variables to a submake]
291 test: $(TIMEOUT_PROGRAM)
292 +$(PYTHON) $(RUNTESTS) $(RUNTEST_OPTS) \
293 $(patsubst %, --only=%, $(TEST)) \
294 $(patsubst %, --only=%, $(TESTS)) \
295 $(patsubst %, --way=%, $(WAY)) \
296 $(patsubst %, --skipway=%, $(SKIPWAY)) \
297 $(set_list_broken) \
298 $(setspeed) \
299 $(setaccept)
300
301 verbose: test
302
303 accept:
304 $(MAKE) accept=YES
305
306 fast:
307 # See Note [validate and testsuite speed] in toplevel Makefile.
308 $(MAKE) SPEED=2
309
310 slow:
311 $(MAKE) SPEED=0
312
313 list_broken:
314 $(MAKE) list_broken=YES
315
316 # Note [Communicating options and variables to a submake]
317 #
318 # Consider the following scenario:
319 # * A test foo is defined as
320 # test('foo', [], run_command, ['$MAKE footarget'])
321 # * A user calls 'make -j24 TEST=foo'
322 #
323 # What happens is something like this:
324 # * make (topmake) reads all options and variables given on the commandline
325 # and adds them to the variable MAKEFLAGS [1]. This variable is exported by
326 # default [1], so submakes can use them.
327 # * The 'test' target calls 'python ..'
328 # * Python calls 'make footarget' (submake)
329 #
330 # **First question**: what happens to the '-j24' option when calling make
331 # recursively?
332 #
333 # From
334 # https://www.gnu.org/software/make/manual/html_node/Variables_002fRecursion.html:
335 #
336 # "The '-j' option is a special case (see Parallel Execution). If you set
337 # it to some numeric value 'N' and your operating system supports it (most
338 # any UNIX system will; others typically won't), the parent make and all the
339 # sub-makes will communicate to ensure that there are only 'N' jobs running
340 # at the same time between them all."
341 #
342 # In our scenario, the user will actually see the following warning [2]:
343 #
344 # 'warning: jobserver unavailable: using -j1. Add '+' to parent make rule.'
345 #
346 # The problem is that topmake and submake don't know about eachother, since
347 # python is in between. To let them communicate, we have to use the '+'
348 # option, by calling '+python' instead of 'python' [2]. This works,
349 # magically, and fixes #11569.
350 #
351 # **Second question**: can't we just unexport MAKEFLAGS, instead of using
352 # that '+' trick? The testsuite driver (python) mangages parallelism by
353 # itself already, so '-j24' doesn't do the right thing anyway. You have to
354 # use 'make test THREADS=24'. Unexporting MAKEFLAGS would mean ignoring
355 # any '-j' flags passed to make (either from the user calling 'make -j'
356 # explicitly or from having MAKEFLAGS=-j set in the shell, see #11569).
357 #
358 # This almost works, except when calling 'make fast/slow/accept TEST_HC=ghc'
359 # instead of just 'make test'. These targets call 'make test FAST=YES'
360 # recursively (and 'make test' calls python, as before).
361 #
362 # The problem is that in boilerplate.mk we try to override the variable
363 # TEST_HC (See Note [The TEST_HC variable]). Somewhere somehow this
364 # information (of us wanting to update TEST_HC) gets lost in the process,
365 # resulting in the final TEST_HC always getting set to the inplace compiler.
366 # It seems possible to remedy this yet again by exporting TEST_HC explicitly,
367 # but I didn't understand nor test it thoroughly (what about the other
368 # variables we override, see calls to canonicalise), and the '+' trick seems
369 # to work at least equally well (just don't run something like
370 # 'make test fast slow accept').
371 #
372 # Tests:
373 # * `make TEST=T3307 -j2` should not show a warning.
374 # * `make TEST=tc001 TEST_HC=ghc fast` should not use the inplace compiler.
375 #
376 # [1] https://www.gnu.org/software/make/manual/html_node/Variables_002fRecursion.html
377 # [2] https://www.gnu.org/software/make/manual/html_node/Error-Messages.html