Add NUMA support for Windows
[ghc.git] / rts / win32 / OSMem.c
1 /* -----------------------------------------------------------------------------
2 *
3 * (c) The University of Glasgow 2006-2007
4 *
5 * OS-specific memory management
6 *
7 * ---------------------------------------------------------------------------*/
8
9 #include "Rts.h"
10 #include "sm/OSMem.h"
11 #include "sm/HeapAlloc.h"
12 #include "RtsUtils.h"
13
14 #include <windows.h>
15
16 typedef struct alloc_rec_ {
17 char* base; // non-aligned base address, directly from VirtualAlloc
18 W_ size; // Size in bytes
19 struct alloc_rec_* next;
20 } alloc_rec;
21
22 typedef struct block_rec_ {
23 char* base; // base address, non-MBLOCK-aligned
24 W_ size; // size in bytes
25 struct block_rec_* next;
26 } block_rec;
27
28 /* allocs are kept in ascending order, and are the memory regions as
29 returned by the OS as we need to have matching VirtualAlloc and
30 VirtualFree calls.
31
32 If USE_LARGE_ADDRESS_SPACE is defined, this list will contain only
33 one element.
34 */
35 static alloc_rec* allocs = NULL;
36
37 /* free_blocks are kept in ascending order, and adjacent blocks are merged */
38 static block_rec* free_blocks = NULL;
39
40 /* Mingw-w64 does not currently have this in their header. So we have to import it.*/
41 typedef LPVOID(WINAPI *VirtualAllocExNumaProc)(HANDLE, LPVOID, SIZE_T, DWORD, DWORD, DWORD);
42
43 /* Cache NUMA API call. */
44 VirtualAllocExNumaProc VirtualAllocExNuma;
45
46 void
47 osMemInit(void)
48 {
49 allocs = NULL;
50 free_blocks = NULL;
51
52 /* Resolve and cache VirtualAllocExNuma. */
53 if (osNumaAvailable() && RtsFlags.GcFlags.numa)
54 {
55 VirtualAllocExNuma = (VirtualAllocExNumaProc)GetProcAddress(GetModuleHandleW(L"kernel32"), "VirtualAllocExNuma");
56 if (!VirtualAllocExNuma)
57 {
58 sysErrorBelch(
59 "osBindMBlocksToNode: VirtualAllocExNuma does not exist. How did you get this far?");
60 }
61 }
62 }
63
64 static
65 alloc_rec*
66 allocNew(uint32_t n) {
67 alloc_rec* rec;
68 rec = (alloc_rec*)stgMallocBytes(sizeof(alloc_rec),"getMBlocks: allocNew");
69 rec->size = ((W_)n+1)*MBLOCK_SIZE;
70 rec->base =
71 VirtualAlloc(NULL, rec->size, MEM_RESERVE, PAGE_READWRITE);
72 if(rec->base==0) {
73 stgFree((void*)rec);
74 rec=0;
75 if (GetLastError() == ERROR_NOT_ENOUGH_MEMORY) {
76
77 errorBelch("Out of memory");
78 stg_exit(EXIT_HEAPOVERFLOW);
79 } else {
80 sysErrorBelch(
81 "getMBlocks: VirtualAlloc MEM_RESERVE %d blocks failed", n);
82 }
83 } else {
84 alloc_rec temp;
85 temp.base=0; temp.size=0; temp.next=allocs;
86
87 alloc_rec* it;
88 it=&temp;
89 for(; it->next!=0 && it->next->base<rec->base; it=it->next) ;
90 rec->next=it->next;
91 it->next=rec;
92
93 allocs=temp.next;
94 }
95 return rec;
96 }
97
98 static
99 void
100 insertFree(char* alloc_base, W_ alloc_size) {
101 block_rec temp;
102 block_rec* it;
103 block_rec* prev;
104
105 temp.base=0; temp.size=0; temp.next=free_blocks;
106 it = free_blocks;
107 prev = &temp;
108 for( ; it!=0 && it->base<alloc_base; prev=it, it=it->next) {}
109
110 if(it!=0 && alloc_base+alloc_size == it->base) {
111 if(prev->base + prev->size == alloc_base) { /* Merge it, alloc, prev */
112 prev->size += alloc_size + it->size;
113 prev->next = it->next;
114 stgFree(it);
115 } else { /* Merge it, alloc */
116 it->base = alloc_base;
117 it->size += alloc_size;
118 }
119 } else if(prev->base + prev->size == alloc_base) { /* Merge alloc, prev */
120 prev->size += alloc_size;
121 } else { /* Merge none */
122 block_rec* rec;
123 rec = (block_rec*)stgMallocBytes(sizeof(block_rec),
124 "getMBlocks: insertFree");
125 rec->base=alloc_base;
126 rec->size=alloc_size;
127 rec->next = it;
128 prev->next=rec;
129 }
130 free_blocks=temp.next;
131 }
132
133 static
134 void*
135 findFreeBlocks(uint32_t n) {
136 void* ret=0;
137 block_rec* it;
138 block_rec temp;
139 block_rec* prev;
140
141 W_ required_size;
142 it=free_blocks;
143 required_size = n*MBLOCK_SIZE;
144 temp.next=free_blocks; temp.base=0; temp.size=0;
145 prev=&temp;
146 /* TODO: Don't just take first block, find smallest sufficient block */
147 for( ; it!=0 && it->size<required_size; prev=it, it=it->next ) {}
148 if(it!=0) {
149 if( (((W_)it->base) & MBLOCK_MASK) == 0) { /* MBlock aligned */
150 ret = (void*)it->base;
151 if(it->size==required_size) {
152 prev->next=it->next;
153 stgFree(it);
154 } else {
155 it->base += required_size;
156 it->size -=required_size;
157 }
158 } else {
159 char* need_base;
160 block_rec* next;
161 int new_size;
162 need_base =
163 (char*)(((W_)it->base) & ((W_)~MBLOCK_MASK)) + MBLOCK_SIZE;
164 next = (block_rec*)stgMallocBytes(
165 sizeof(block_rec)
166 , "getMBlocks: findFreeBlocks: splitting");
167 new_size = need_base - it->base;
168 next->base = need_base +required_size;
169 next->size = it->size - (new_size+required_size);
170 it->size = new_size;
171 next->next = it->next;
172 it->next = next;
173 ret=(void*)need_base;
174 }
175 }
176 free_blocks=temp.next;
177 return ret;
178 }
179
180 /* VirtualAlloc MEM_COMMIT can't cross boundaries of VirtualAlloc MEM_RESERVE,
181 so we might need to do many VirtualAlloc MEM_COMMITs. We simply walk the
182 (ordered) allocated blocks. */
183 static void
184 commitBlocks(char* base, W_ size) {
185 alloc_rec* it;
186 it=allocs;
187 for( ; it!=0 && (it->base+it->size)<=base; it=it->next ) {}
188 for( ; it!=0 && size>0; it=it->next ) {
189 W_ size_delta;
190 void* temp;
191 size_delta = it->size - (base-it->base);
192 if(size_delta>size) size_delta=size;
193 temp = VirtualAlloc(base, size_delta, MEM_COMMIT, PAGE_READWRITE);
194 if(temp==0) {
195 sysErrorBelch("getMBlocks: VirtualAlloc MEM_COMMIT failed");
196 stg_exit(EXIT_HEAPOVERFLOW);
197 }
198 size-=size_delta;
199 base+=size_delta;
200 }
201 }
202
203 void *
204 osGetMBlocks(uint32_t n) {
205 void* ret;
206 ret = findFreeBlocks(n);
207 if(ret==0) {
208 alloc_rec* alloc;
209 alloc = allocNew(n);
210 /* We already belch in allocNew if it fails */
211 if (alloc == 0) {
212 stg_exit(EXIT_FAILURE);
213 } else {
214 insertFree(alloc->base, alloc->size);
215 ret = findFreeBlocks(n);
216 }
217 }
218
219 if(ret!=0) {
220 /* (In)sanity tests */
221 if (((W_)ret & MBLOCK_MASK) != 0) {
222 barf("getMBlocks: misaligned block returned");
223 }
224
225 commitBlocks(ret, (W_)MBLOCK_SIZE*n);
226 }
227
228 return ret;
229 }
230
231 static void decommitBlocks(char *addr, W_ nBytes)
232 {
233 alloc_rec *p;
234
235 p = allocs;
236 while ((p != NULL) && (addr >= (p->base + p->size))) {
237 p = p->next;
238 }
239 while (nBytes > 0) {
240 if ((p == NULL) || (p->base > addr)) {
241 errorBelch("Memory to be freed isn't allocated\n");
242 stg_exit(EXIT_FAILURE);
243 }
244 if (p->base + p->size >= addr + nBytes) {
245 if (!VirtualFree(addr, nBytes, MEM_DECOMMIT)) {
246 sysErrorBelch("osFreeMBlocks: VirtualFree MEM_DECOMMIT failed");
247 stg_exit(EXIT_FAILURE);
248 }
249 nBytes = 0;
250 }
251 else {
252 W_ bytesToFree = p->base + p->size - addr;
253 if (!VirtualFree(addr, bytesToFree, MEM_DECOMMIT)) {
254 sysErrorBelch("osFreeMBlocks: VirtualFree MEM_DECOMMIT failed");
255 stg_exit(EXIT_FAILURE);
256 }
257 addr += bytesToFree;
258 nBytes -= bytesToFree;
259 p = p->next;
260 }
261 }
262 }
263
264 void osFreeMBlocks(void *addr, uint32_t n)
265 {
266 W_ nBytes = (W_)n * MBLOCK_SIZE;
267
268 insertFree(addr, nBytes);
269 decommitBlocks(addr, nBytes);
270 }
271
272 void osReleaseFreeMemory(void)
273 {
274 alloc_rec *prev_a, *a;
275 alloc_rec head_a;
276 block_rec *prev_fb, *fb;
277 block_rec head_fb;
278 char *a_end, *fb_end;
279
280 /* go through allocs and free_blocks in lockstep, looking for allocs
281 that are completely free, and uncommit them */
282
283 head_a.base = 0;
284 head_a.size = 0;
285 head_a.next = allocs;
286 head_fb.base = 0;
287 head_fb.size = 0;
288 head_fb.next = free_blocks;
289 prev_a = &head_a;
290 a = allocs;
291 prev_fb = &head_fb;
292 fb = free_blocks;
293
294 while (a != NULL) {
295 a_end = a->base + a->size;
296 /* If a is freeable then there is a single freeblock in fb that
297 covers it. The end of this free block must be >= the end of
298 a, so skip anything in fb that ends before a. */
299 while (fb != NULL && fb->base + fb->size < a_end) {
300 prev_fb = fb;
301 fb = fb->next;
302 }
303
304 if (fb == NULL) {
305 /* If we have nothing left in fb, then neither a nor
306 anything later in the list is freeable, so we are done. */
307 break;
308 }
309 else {
310 fb_end = fb->base + fb->size;
311 /* We have a candidate fb. But does it really cover a? */
312 if (fb->base <= a->base) {
313 /* Yes, the alloc is within the free block. Now we need
314 to know if it sticks out at either end. */
315 if (fb_end == a_end) {
316 if (fb->base == a->base) {
317 /* fb and a are identical, so just free fb */
318 prev_fb->next = fb->next;
319 stgFree(fb);
320 fb = prev_fb->next;
321 }
322 else {
323 /* fb begins earlier, so truncate it to not include a */
324 fb->size = a->base - fb->base;
325 }
326 }
327 else {
328 /* fb ends later, so we'll make fb just be the part
329 after a. First though, if it also starts earlier,
330 we make a new free block record for the before bit. */
331 if (fb->base != a->base) {
332 block_rec *new_fb;
333
334 new_fb =
335 (block_rec *)stgMallocBytes(sizeof(block_rec),
336 "osReleaseFreeMemory");
337 new_fb->base = fb->base;
338 new_fb->size = a->base - fb->base;
339 new_fb->next = fb;
340 prev_fb->next = new_fb;
341 }
342 fb->size = fb_end - a_end;
343 fb->base = a_end;
344 }
345 /* Now we can free the alloc */
346 prev_a->next = a->next;
347 if(!VirtualFree((void *)a->base, 0, MEM_RELEASE)) {
348 sysErrorBelch("freeAllMBlocks: VirtualFree MEM_RELEASE "
349 "failed");
350 stg_exit(EXIT_FAILURE);
351 }
352 stgFree(a);
353 a = prev_a->next;
354 }
355 else {
356 /* Otherwise this alloc is not freeable, so go on to the
357 next one */
358 prev_a = a;
359 a = a->next;
360 }
361 }
362 }
363
364 allocs = head_a.next;
365 free_blocks = head_fb.next;
366 }
367
368 void
369 osFreeAllMBlocks(void)
370 {
371 {
372 block_rec* next;
373 block_rec* it;
374 next=0;
375 it = free_blocks;
376 for(; it!=0; ) {
377 next = it->next;
378 stgFree(it);
379 it=next;
380 }
381 }
382 {
383 alloc_rec* next;
384 alloc_rec* it;
385 next=0;
386 it=allocs;
387 for(; it!=0; ) {
388 if(!VirtualFree((void*)it->base, 0, MEM_RELEASE)) {
389 sysErrorBelch("freeAllMBlocks: VirtualFree MEM_RELEASE failed");
390 stg_exit(EXIT_FAILURE);
391 }
392 next = it->next;
393 stgFree(it);
394 it=next;
395 }
396 }
397 }
398
399 size_t getPageSize (void)
400 {
401 static size_t pagesize = 0;
402
403 if (pagesize == 0) {
404 SYSTEM_INFO sSysInfo;
405 GetSystemInfo(&sSysInfo);
406 pagesize = sSysInfo.dwPageSize;
407 }
408
409 return pagesize;
410 }
411
412 /* Returns 0 if physical memory size cannot be identified */
413 StgWord64 getPhysicalMemorySize (void)
414 {
415 static StgWord64 physMemSize = 0;
416 if (!physMemSize) {
417 MEMORYSTATUSEX status;
418 status.dwLength = sizeof(status);
419 if (!GlobalMemoryStatusEx(&status)) {
420 #if defined(DEBUG)
421 errorBelch("warning: getPhysicalMemorySize: cannot get physical "
422 "memory size");
423 #endif
424 return 0;
425 }
426 physMemSize = status.ullTotalPhys;
427 }
428 return physMemSize;
429 }
430
431 void setExecutable (void *p, W_ len, rtsBool exec)
432 {
433 DWORD dwOldProtect = 0;
434 if (VirtualProtect (p, len,
435 exec ? PAGE_EXECUTE_READWRITE : PAGE_READWRITE,
436 &dwOldProtect) == 0)
437 {
438 sysErrorBelch("setExecutable: failed to protect 0x%p; old protection: "
439 "%lu\n", p, (unsigned long)dwOldProtect);
440 stg_exit(EXIT_FAILURE);
441 }
442 }
443
444 #ifdef USE_LARGE_ADDRESS_SPACE
445
446 static void* heap_base = NULL;
447
448 void *osReserveHeapMemory (void *startAddress, W_ *len)
449 {
450 void *start;
451
452 heap_base = VirtualAlloc(startAddress, *len + MBLOCK_SIZE,
453 MEM_RESERVE, PAGE_READWRITE);
454 if (heap_base == NULL) {
455 if (GetLastError() == ERROR_NOT_ENOUGH_MEMORY) {
456 errorBelch("out of memory");
457 } else {
458 sysErrorBelch(
459 "osReserveHeapMemory: VirtualAlloc MEM_RESERVE %llu bytes \
460 at address %p bytes failed",
461 len + MBLOCK_SIZE, startAddress);
462 }
463 stg_exit(EXIT_FAILURE);
464 }
465
466 // VirtualFree MEM_RELEASE must always match a
467 // previous MEM_RESERVE call, in address and size
468 // so we necessarily leak some address space here,
469 // before and after the aligned area
470 // It is not a huge problem because we never commit
471 // that memory
472 start = MBLOCK_ROUND_UP(heap_base);
473
474 return start;
475 }
476
477 void osCommitMemory (void *at, W_ size)
478 {
479 void *temp;
480 temp = VirtualAlloc(at, size, MEM_COMMIT, PAGE_READWRITE);
481 if (temp == NULL) {
482 sysErrorBelch("osCommitMemory: VirtualAlloc MEM_COMMIT failed");
483 stg_exit(EXIT_FAILURE);
484 }
485 }
486
487 void osDecommitMemory (void *at, W_ size)
488 {
489 if (!VirtualFree(at, size, MEM_DECOMMIT)) {
490 sysErrorBelch("osDecommitMemory: VirtualFree MEM_DECOMMIT failed");
491 stg_exit(EXIT_FAILURE);
492 }
493 }
494
495 void osReleaseHeapMemory (void)
496 {
497 VirtualFree(heap_base, 0, MEM_RELEASE);
498 }
499
500 #endif
501
502 rtsBool osNumaAvailable(void)
503 {
504 return osNumaNodes() > 1;
505 }
506
507 uint32_t osNumaNodes(void)
508 {
509 /* Cache the amount of NUMA values. */
510 static ULONG numNumaNodes = 0;
511
512 /* Cache the amount of NUMA nodes. */
513 if (!numNumaNodes && !GetNumaHighestNodeNumber(&numNumaNodes))
514 {
515 numNumaNodes = 1;
516 }
517
518 return numNumaNodes;
519 }
520
521 StgWord osNumaMask(void)
522 {
523 StgWord numaMask;
524 if (!GetNumaNodeProcessorMask(0, &numaMask))
525 {
526 return 1;
527 }
528 return numaMask;
529 }
530
531 void osBindMBlocksToNode(
532 void *addr,
533 StgWord size,
534 uint32_t node)
535 {
536 if (osNumaAvailable())
537 {
538 void* temp;
539 if (RtsFlags.GcFlags.numa) {
540 /* Note [base memory]
541 I would like to use addr here to specify the base
542 memory of allocation. The problem is that the address
543 we are requesting is too high. I can't figure out if it's
544 because of my NUMA-emulation or a bug in the code.
545
546 On windows also -xb is broken, it does nothing so that can't
547 be used to tweak it (see #12577). So for now, just let the OS decide.
548 */
549 temp = VirtualAllocExNuma(
550 GetCurrentProcess(),
551 NULL, // addr? See base memory
552 size,
553 MEM_RESERVE | MEM_COMMIT,
554 PAGE_READWRITE,
555 node
556 );
557
558 if (!temp) {
559 if (GetLastError() == ERROR_NOT_ENOUGH_MEMORY) {
560 errorBelch("out of memory");
561 }
562 else {
563 sysErrorBelch(
564 "osBindMBlocksToNode: VirtualAllocExNuma MEM_RESERVE %llu bytes "
565 "at address %p bytes failed",
566 size, addr);
567 }
568 stg_exit(EXIT_FAILURE);
569 }
570 }
571 }
572 }