Update README with a new port
[libffi.git] / src / x86 / ffi64.c
1 /* -----------------------------------------------------------------------
2 ffi64.c - Copyright (c) 2013 The Written Word, Inc.
3 Copyright (c) 2011 Anthony Green
4 Copyright (c) 2008, 2010 Red Hat, Inc.
5 Copyright (c) 2002, 2007 Bo Thorsen <bo@suse.de>
6
7 x86-64 Foreign Function Interface
8
9 Permission is hereby granted, free of charge, to any person obtaining
10 a copy of this software and associated documentation files (the
11 ``Software''), to deal in the Software without restriction, including
12 without limitation the rights to use, copy, modify, merge, publish,
13 distribute, sublicense, and/or sell copies of the Software, and to
14 permit persons to whom the Software is furnished to do so, subject to
15 the following conditions:
16
17 The above copyright notice and this permission notice shall be included
18 in all copies or substantial portions of the Software.
19
20 THE SOFTWARE IS PROVIDED ``AS IS'', WITHOUT WARRANTY OF ANY KIND,
21 EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
22 MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
23 NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
24 HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
25 WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
26 OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
27 DEALINGS IN THE SOFTWARE.
28 ----------------------------------------------------------------------- */
29
30 #include <ffi.h>
31 #include <ffi_common.h>
32
33 #include <stdlib.h>
34 #include <stdarg.h>
35 #include <stdint.h>
36 #include "internal64.h"
37
38 #ifdef __x86_64__
39
40 #define MAX_GPR_REGS 6
41 #define MAX_SSE_REGS 8
42
43 #if defined(__INTEL_COMPILER)
44 #include "xmmintrin.h"
45 #define UINT128 __m128
46 #else
47 #if defined(__SUNPRO_C)
48 #include <sunmedia_types.h>
49 #define UINT128 __m128i
50 #else
51 #define UINT128 __int128_t
52 #endif
53 #endif
54
55 union big_int_union
56 {
57 UINT32 i32;
58 UINT64 i64;
59 UINT128 i128;
60 };
61
62 struct register_args
63 {
64 /* Registers for argument passing. */
65 UINT64 gpr[MAX_GPR_REGS];
66 union big_int_union sse[MAX_SSE_REGS];
67 UINT64 rax; /* ssecount */
68 UINT64 r10; /* static chain */
69 };
70
71 extern void ffi_call_unix64 (void *args, unsigned long bytes, unsigned flags,
72 void *raddr, void (*fnaddr)(void)) FFI_HIDDEN;
73
74 /* All reference to register classes here is identical to the code in
75 gcc/config/i386/i386.c. Do *not* change one without the other. */
76
77 /* Register class used for passing given 64bit part of the argument.
78 These represent classes as documented by the PS ABI, with the
79 exception of SSESF, SSEDF classes, that are basically SSE class,
80 just gcc will use SF or DFmode move instead of DImode to avoid
81 reformatting penalties.
82
83 Similary we play games with INTEGERSI_CLASS to use cheaper SImode moves
84 whenever possible (upper half does contain padding). */
85 enum x86_64_reg_class
86 {
87 X86_64_NO_CLASS,
88 X86_64_INTEGER_CLASS,
89 X86_64_INTEGERSI_CLASS,
90 X86_64_SSE_CLASS,
91 X86_64_SSESF_CLASS,
92 X86_64_SSEDF_CLASS,
93 X86_64_SSEUP_CLASS,
94 X86_64_X87_CLASS,
95 X86_64_X87UP_CLASS,
96 X86_64_COMPLEX_X87_CLASS,
97 X86_64_MEMORY_CLASS
98 };
99
100 #define MAX_CLASSES 4
101
102 #define SSE_CLASS_P(X) ((X) >= X86_64_SSE_CLASS && X <= X86_64_SSEUP_CLASS)
103
104 /* x86-64 register passing implementation. See x86-64 ABI for details. Goal
105 of this code is to classify each 8bytes of incoming argument by the register
106 class and assign registers accordingly. */
107
108 /* Return the union class of CLASS1 and CLASS2.
109 See the x86-64 PS ABI for details. */
110
111 static enum x86_64_reg_class
112 merge_classes (enum x86_64_reg_class class1, enum x86_64_reg_class class2)
113 {
114 /* Rule #1: If both classes are equal, this is the resulting class. */
115 if (class1 == class2)
116 return class1;
117
118 /* Rule #2: If one of the classes is NO_CLASS, the resulting class is
119 the other class. */
120 if (class1 == X86_64_NO_CLASS)
121 return class2;
122 if (class2 == X86_64_NO_CLASS)
123 return class1;
124
125 /* Rule #3: If one of the classes is MEMORY, the result is MEMORY. */
126 if (class1 == X86_64_MEMORY_CLASS || class2 == X86_64_MEMORY_CLASS)
127 return X86_64_MEMORY_CLASS;
128
129 /* Rule #4: If one of the classes is INTEGER, the result is INTEGER. */
130 if ((class1 == X86_64_INTEGERSI_CLASS && class2 == X86_64_SSESF_CLASS)
131 || (class2 == X86_64_INTEGERSI_CLASS && class1 == X86_64_SSESF_CLASS))
132 return X86_64_INTEGERSI_CLASS;
133 if (class1 == X86_64_INTEGER_CLASS || class1 == X86_64_INTEGERSI_CLASS
134 || class2 == X86_64_INTEGER_CLASS || class2 == X86_64_INTEGERSI_CLASS)
135 return X86_64_INTEGER_CLASS;
136
137 /* Rule #5: If one of the classes is X87, X87UP, or COMPLEX_X87 class,
138 MEMORY is used. */
139 if (class1 == X86_64_X87_CLASS
140 || class1 == X86_64_X87UP_CLASS
141 || class1 == X86_64_COMPLEX_X87_CLASS
142 || class2 == X86_64_X87_CLASS
143 || class2 == X86_64_X87UP_CLASS
144 || class2 == X86_64_COMPLEX_X87_CLASS)
145 return X86_64_MEMORY_CLASS;
146
147 /* Rule #6: Otherwise class SSE is used. */
148 return X86_64_SSE_CLASS;
149 }
150
151 /* Classify the argument of type TYPE and mode MODE.
152 CLASSES will be filled by the register class used to pass each word
153 of the operand. The number of words is returned. In case the parameter
154 should be passed in memory, 0 is returned. As a special case for zero
155 sized containers, classes[0] will be NO_CLASS and 1 is returned.
156
157 See the x86-64 PS ABI for details.
158 */
159 static size_t
160 classify_argument (ffi_type *type, enum x86_64_reg_class classes[],
161 size_t byte_offset)
162 {
163 switch (type->type)
164 {
165 case FFI_TYPE_UINT8:
166 case FFI_TYPE_SINT8:
167 case FFI_TYPE_UINT16:
168 case FFI_TYPE_SINT16:
169 case FFI_TYPE_UINT32:
170 case FFI_TYPE_SINT32:
171 case FFI_TYPE_UINT64:
172 case FFI_TYPE_SINT64:
173 case FFI_TYPE_POINTER:
174 do_integer:
175 {
176 size_t size = byte_offset + type->size;
177
178 if (size <= 4)
179 {
180 classes[0] = X86_64_INTEGERSI_CLASS;
181 return 1;
182 }
183 else if (size <= 8)
184 {
185 classes[0] = X86_64_INTEGER_CLASS;
186 return 1;
187 }
188 else if (size <= 12)
189 {
190 classes[0] = X86_64_INTEGER_CLASS;
191 classes[1] = X86_64_INTEGERSI_CLASS;
192 return 2;
193 }
194 else if (size <= 16)
195 {
196 classes[0] = classes[1] = X86_64_INTEGER_CLASS;
197 return 2;
198 }
199 else
200 FFI_ASSERT (0);
201 }
202 case FFI_TYPE_FLOAT:
203 if (!(byte_offset % 8))
204 classes[0] = X86_64_SSESF_CLASS;
205 else
206 classes[0] = X86_64_SSE_CLASS;
207 return 1;
208 case FFI_TYPE_DOUBLE:
209 classes[0] = X86_64_SSEDF_CLASS;
210 return 1;
211 #if FFI_TYPE_LONGDOUBLE != FFI_TYPE_DOUBLE
212 case FFI_TYPE_LONGDOUBLE:
213 classes[0] = X86_64_X87_CLASS;
214 classes[1] = X86_64_X87UP_CLASS;
215 return 2;
216 #endif
217 case FFI_TYPE_STRUCT:
218 {
219 const size_t UNITS_PER_WORD = 8;
220 size_t words = (type->size + UNITS_PER_WORD - 1) / UNITS_PER_WORD;
221 ffi_type **ptr;
222 int i;
223 enum x86_64_reg_class subclasses[MAX_CLASSES];
224
225 /* If the struct is larger than 32 bytes, pass it on the stack. */
226 if (type->size > 32)
227 return 0;
228
229 for (i = 0; i < words; i++)
230 classes[i] = X86_64_NO_CLASS;
231
232 /* Zero sized arrays or structures are NO_CLASS. We return 0 to
233 signalize memory class, so handle it as special case. */
234 if (!words)
235 {
236 case FFI_TYPE_VOID:
237 classes[0] = X86_64_NO_CLASS;
238 return 1;
239 }
240
241 /* Merge the fields of structure. */
242 for (ptr = type->elements; *ptr != NULL; ptr++)
243 {
244 size_t num;
245
246 byte_offset = ALIGN (byte_offset, (*ptr)->alignment);
247
248 num = classify_argument (*ptr, subclasses, byte_offset % 8);
249 if (num == 0)
250 return 0;
251 for (i = 0; i < num; i++)
252 {
253 size_t pos = byte_offset / 8;
254 classes[i + pos] =
255 merge_classes (subclasses[i], classes[i + pos]);
256 }
257
258 byte_offset += (*ptr)->size;
259 }
260
261 if (words > 2)
262 {
263 /* When size > 16 bytes, if the first one isn't
264 X86_64_SSE_CLASS or any other ones aren't
265 X86_64_SSEUP_CLASS, everything should be passed in
266 memory. */
267 if (classes[0] != X86_64_SSE_CLASS)
268 return 0;
269
270 for (i = 1; i < words; i++)
271 if (classes[i] != X86_64_SSEUP_CLASS)
272 return 0;
273 }
274
275 /* Final merger cleanup. */
276 for (i = 0; i < words; i++)
277 {
278 /* If one class is MEMORY, everything should be passed in
279 memory. */
280 if (classes[i] == X86_64_MEMORY_CLASS)
281 return 0;
282
283 /* The X86_64_SSEUP_CLASS should be always preceded by
284 X86_64_SSE_CLASS or X86_64_SSEUP_CLASS. */
285 if (classes[i] == X86_64_SSEUP_CLASS
286 && classes[i - 1] != X86_64_SSE_CLASS
287 && classes[i - 1] != X86_64_SSEUP_CLASS)
288 {
289 /* The first one should never be X86_64_SSEUP_CLASS. */
290 FFI_ASSERT (i != 0);
291 classes[i] = X86_64_SSE_CLASS;
292 }
293
294 /* If X86_64_X87UP_CLASS isn't preceded by X86_64_X87_CLASS,
295 everything should be passed in memory. */
296 if (classes[i] == X86_64_X87UP_CLASS
297 && (classes[i - 1] != X86_64_X87_CLASS))
298 {
299 /* The first one should never be X86_64_X87UP_CLASS. */
300 FFI_ASSERT (i != 0);
301 return 0;
302 }
303 }
304 return words;
305 }
306 case FFI_TYPE_COMPLEX:
307 {
308 ffi_type *inner = type->elements[0];
309 switch (inner->type)
310 {
311 case FFI_TYPE_INT:
312 case FFI_TYPE_UINT8:
313 case FFI_TYPE_SINT8:
314 case FFI_TYPE_UINT16:
315 case FFI_TYPE_SINT16:
316 case FFI_TYPE_UINT32:
317 case FFI_TYPE_SINT32:
318 case FFI_TYPE_UINT64:
319 case FFI_TYPE_SINT64:
320 goto do_integer;
321
322 case FFI_TYPE_FLOAT:
323 classes[0] = X86_64_SSE_CLASS;
324 if (byte_offset % 8)
325 {
326 classes[1] = X86_64_SSESF_CLASS;
327 return 2;
328 }
329 return 1;
330 case FFI_TYPE_DOUBLE:
331 classes[0] = classes[1] = X86_64_SSEDF_CLASS;
332 return 2;
333 #if FFI_TYPE_LONGDOUBLE != FFI_TYPE_DOUBLE
334 case FFI_TYPE_LONGDOUBLE:
335 classes[0] = X86_64_COMPLEX_X87_CLASS;
336 return 1;
337 #endif
338 }
339 }
340 }
341 abort();
342 }
343
344 /* Examine the argument and return set number of register required in each
345 class. Return zero iff parameter should be passed in memory, otherwise
346 the number of registers. */
347
348 static size_t
349 examine_argument (ffi_type *type, enum x86_64_reg_class classes[MAX_CLASSES],
350 _Bool in_return, int *pngpr, int *pnsse)
351 {
352 size_t n;
353 int i, ngpr, nsse;
354
355 n = classify_argument (type, classes, 0);
356 if (n == 0)
357 return 0;
358
359 ngpr = nsse = 0;
360 for (i = 0; i < n; ++i)
361 switch (classes[i])
362 {
363 case X86_64_INTEGER_CLASS:
364 case X86_64_INTEGERSI_CLASS:
365 ngpr++;
366 break;
367 case X86_64_SSE_CLASS:
368 case X86_64_SSESF_CLASS:
369 case X86_64_SSEDF_CLASS:
370 nsse++;
371 break;
372 case X86_64_NO_CLASS:
373 case X86_64_SSEUP_CLASS:
374 break;
375 case X86_64_X87_CLASS:
376 case X86_64_X87UP_CLASS:
377 case X86_64_COMPLEX_X87_CLASS:
378 return in_return != 0;
379 default:
380 abort ();
381 }
382
383 *pngpr = ngpr;
384 *pnsse = nsse;
385
386 return n;
387 }
388
389 /* Perform machine dependent cif processing. */
390
391 extern ffi_status
392 ffi_prep_cif_machdep_efi64(ffi_cif *cif);
393
394 ffi_status
395 ffi_prep_cif_machdep (ffi_cif *cif)
396 {
397 int gprcount, ssecount, i, avn, ngpr, nsse, flags;
398 enum x86_64_reg_class classes[MAX_CLASSES];
399 size_t bytes, n, rtype_size;
400 ffi_type *rtype;
401
402 if (cif->abi == FFI_EFI64)
403 return ffi_prep_cif_machdep_efi64(cif);
404 if (cif->abi != FFI_UNIX64)
405 return FFI_BAD_ABI;
406
407 gprcount = ssecount = 0;
408
409 rtype = cif->rtype;
410 rtype_size = rtype->size;
411 switch (rtype->type)
412 {
413 case FFI_TYPE_VOID:
414 flags = UNIX64_RET_VOID;
415 break;
416 case FFI_TYPE_UINT8:
417 flags = UNIX64_RET_UINT8;
418 break;
419 case FFI_TYPE_SINT8:
420 flags = UNIX64_RET_SINT8;
421 break;
422 case FFI_TYPE_UINT16:
423 flags = UNIX64_RET_UINT16;
424 break;
425 case FFI_TYPE_SINT16:
426 flags = UNIX64_RET_SINT16;
427 break;
428 case FFI_TYPE_UINT32:
429 flags = UNIX64_RET_UINT32;
430 break;
431 case FFI_TYPE_INT:
432 case FFI_TYPE_SINT32:
433 flags = UNIX64_RET_SINT32;
434 break;
435 case FFI_TYPE_UINT64:
436 case FFI_TYPE_SINT64:
437 flags = UNIX64_RET_INT64;
438 break;
439 case FFI_TYPE_POINTER:
440 flags = (sizeof(void *) == 4 ? UNIX64_RET_UINT32 : UNIX64_RET_INT64);
441 break;
442 case FFI_TYPE_FLOAT:
443 flags = UNIX64_RET_XMM32;
444 break;
445 case FFI_TYPE_DOUBLE:
446 flags = UNIX64_RET_XMM64;
447 break;
448 case FFI_TYPE_LONGDOUBLE:
449 flags = UNIX64_RET_X87;
450 break;
451 case FFI_TYPE_STRUCT:
452 n = examine_argument (cif->rtype, classes, 1, &ngpr, &nsse);
453 if (n == 0)
454 {
455 /* The return value is passed in memory. A pointer to that
456 memory is the first argument. Allocate a register for it. */
457 gprcount++;
458 /* We don't have to do anything in asm for the return. */
459 flags = UNIX64_RET_VOID | UNIX64_FLAG_RET_IN_MEM;
460 }
461 else
462 {
463 _Bool sse0 = SSE_CLASS_P (classes[0]);
464
465 if (rtype_size == 4 && sse0)
466 flags = UNIX64_RET_XMM32;
467 else if (rtype_size == 8)
468 flags = sse0 ? UNIX64_RET_XMM64 : UNIX64_RET_INT64;
469 else
470 {
471 _Bool sse1 = n == 2 && SSE_CLASS_P (classes[1]);
472 if (sse0 && sse1)
473 flags = UNIX64_RET_ST_XMM0_XMM1;
474 else if (sse0)
475 flags = UNIX64_RET_ST_XMM0_RAX;
476 else if (sse1)
477 flags = UNIX64_RET_ST_RAX_XMM0;
478 else
479 flags = UNIX64_RET_ST_RAX_RDX;
480 flags |= rtype_size << UNIX64_SIZE_SHIFT;
481 }
482 }
483 break;
484 case FFI_TYPE_COMPLEX:
485 switch (rtype->elements[0]->type)
486 {
487 case FFI_TYPE_UINT8:
488 case FFI_TYPE_SINT8:
489 case FFI_TYPE_UINT16:
490 case FFI_TYPE_SINT16:
491 case FFI_TYPE_INT:
492 case FFI_TYPE_UINT32:
493 case FFI_TYPE_SINT32:
494 case FFI_TYPE_UINT64:
495 case FFI_TYPE_SINT64:
496 flags = UNIX64_RET_ST_RAX_RDX | (rtype_size << UNIX64_SIZE_SHIFT);
497 break;
498 case FFI_TYPE_FLOAT:
499 flags = UNIX64_RET_XMM64;
500 break;
501 case FFI_TYPE_DOUBLE:
502 flags = UNIX64_RET_ST_XMM0_XMM1 | (16 << UNIX64_SIZE_SHIFT);
503 break;
504 #if FFI_TYPE_LONGDOUBLE != FFI_TYPE_DOUBLE
505 case FFI_TYPE_LONGDOUBLE:
506 flags = UNIX64_RET_X87_2;
507 break;
508 #endif
509 default:
510 return FFI_BAD_TYPEDEF;
511 }
512 break;
513 default:
514 return FFI_BAD_TYPEDEF;
515 }
516
517 /* Go over all arguments and determine the way they should be passed.
518 If it's in a register and there is space for it, let that be so. If
519 not, add it's size to the stack byte count. */
520 for (bytes = 0, i = 0, avn = cif->nargs; i < avn; i++)
521 {
522 if (examine_argument (cif->arg_types[i], classes, 0, &ngpr, &nsse) == 0
523 || gprcount + ngpr > MAX_GPR_REGS
524 || ssecount + nsse > MAX_SSE_REGS)
525 {
526 long align = cif->arg_types[i]->alignment;
527
528 if (align < 8)
529 align = 8;
530
531 bytes = ALIGN (bytes, align);
532 bytes += cif->arg_types[i]->size;
533 }
534 else
535 {
536 gprcount += ngpr;
537 ssecount += nsse;
538 }
539 }
540 if (ssecount)
541 flags |= UNIX64_FLAG_XMM_ARGS;
542
543 cif->flags = flags;
544 cif->bytes = ALIGN (bytes, 8);
545
546 return FFI_OK;
547 }
548
549 static void
550 ffi_call_int (ffi_cif *cif, void (*fn)(void), void *rvalue,
551 void **avalue, void *closure)
552 {
553 enum x86_64_reg_class classes[MAX_CLASSES];
554 char *stack, *argp;
555 ffi_type **arg_types;
556 int gprcount, ssecount, ngpr, nsse, i, avn, flags;
557 struct register_args *reg_args;
558
559 /* Can't call 32-bit mode from 64-bit mode. */
560 FFI_ASSERT (cif->abi == FFI_UNIX64);
561
562 /* If the return value is a struct and we don't have a return value
563 address then we need to make one. Otherwise we can ignore it. */
564 flags = cif->flags;
565 if (rvalue == NULL)
566 {
567 if (flags & UNIX64_FLAG_RET_IN_MEM)
568 rvalue = alloca (cif->rtype->size);
569 else
570 flags = UNIX64_RET_VOID;
571 }
572
573 /* Allocate the space for the arguments, plus 4 words of temp space. */
574 stack = alloca (sizeof (struct register_args) + cif->bytes + 4*8);
575 reg_args = (struct register_args *) stack;
576 argp = stack + sizeof (struct register_args);
577
578 reg_args->r10 = (uintptr_t) closure;
579
580 gprcount = ssecount = 0;
581
582 /* If the return value is passed in memory, add the pointer as the
583 first integer argument. */
584 if (flags & UNIX64_FLAG_RET_IN_MEM)
585 reg_args->gpr[gprcount++] = (unsigned long) rvalue;
586
587 avn = cif->nargs;
588 arg_types = cif->arg_types;
589
590 for (i = 0; i < avn; ++i)
591 {
592 size_t n, size = arg_types[i]->size;
593
594 n = examine_argument (arg_types[i], classes, 0, &ngpr, &nsse);
595 if (n == 0
596 || gprcount + ngpr > MAX_GPR_REGS
597 || ssecount + nsse > MAX_SSE_REGS)
598 {
599 long align = arg_types[i]->alignment;
600
601 /* Stack arguments are *always* at least 8 byte aligned. */
602 if (align < 8)
603 align = 8;
604
605 /* Pass this argument in memory. */
606 argp = (void *) ALIGN (argp, align);
607 memcpy (argp, avalue[i], size);
608 argp += size;
609 }
610 else
611 {
612 /* The argument is passed entirely in registers. */
613 char *a = (char *) avalue[i];
614 int j;
615
616 for (j = 0; j < n; j++, a += 8, size -= 8)
617 {
618 switch (classes[j])
619 {
620 case X86_64_NO_CLASS:
621 case X86_64_SSEUP_CLASS:
622 break;
623 case X86_64_INTEGER_CLASS:
624 case X86_64_INTEGERSI_CLASS:
625 /* Sign-extend integer arguments passed in general
626 purpose registers, to cope with the fact that
627 LLVM incorrectly assumes that this will be done
628 (the x86-64 PS ABI does not specify this). */
629 switch (arg_types[i]->type)
630 {
631 case FFI_TYPE_SINT8:
632 reg_args->gpr[gprcount] = (SINT64) *((SINT8 *) a);
633 break;
634 case FFI_TYPE_SINT16:
635 reg_args->gpr[gprcount] = (SINT64) *((SINT16 *) a);
636 break;
637 case FFI_TYPE_SINT32:
638 reg_args->gpr[gprcount] = (SINT64) *((SINT32 *) a);
639 break;
640 default:
641 reg_args->gpr[gprcount] = 0;
642 memcpy (&reg_args->gpr[gprcount], a, size);
643 }
644 gprcount++;
645 break;
646 case X86_64_SSE_CLASS:
647 case X86_64_SSEDF_CLASS:
648 reg_args->sse[ssecount++].i64 = *(UINT64 *) a;
649 break;
650 case X86_64_SSESF_CLASS:
651 reg_args->sse[ssecount++].i32 = *(UINT32 *) a;
652 break;
653 default:
654 abort();
655 }
656 }
657 }
658 }
659 reg_args->rax = ssecount;
660
661 ffi_call_unix64 (stack, cif->bytes + sizeof (struct register_args),
662 flags, rvalue, fn);
663 }
664
665 extern void
666 ffi_call_efi64(ffi_cif *cif, void (*fn)(void), void *rvalue, void **avalue);
667
668 void
669 ffi_call (ffi_cif *cif, void (*fn)(void), void *rvalue, void **avalue)
670 {
671 if (cif->abi == FFI_EFI64)
672 return ffi_call_efi64(cif, fn, rvalue, avalue);
673 ffi_call_int (cif, fn, rvalue, avalue, NULL);
674 }
675
676 extern void
677 ffi_call_go_efi64(ffi_cif *cif, void (*fn)(void), void *rvalue,
678 void **avalue, void *closure);
679
680 void
681 ffi_call_go (ffi_cif *cif, void (*fn)(void), void *rvalue,
682 void **avalue, void *closure)
683 {
684 if (cif->abi == FFI_EFI64)
685 ffi_call_go_efi64(cif, fn, rvalue, avalue, closure);
686 ffi_call_int (cif, fn, rvalue, avalue, closure);
687 }
688
689
690 extern void ffi_closure_unix64(void) FFI_HIDDEN;
691 extern void ffi_closure_unix64_sse(void) FFI_HIDDEN;
692
693 extern ffi_status
694 ffi_prep_closure_loc_efi64(ffi_closure* closure,
695 ffi_cif* cif,
696 void (*fun)(ffi_cif*, void*, void**, void*),
697 void *user_data,
698 void *codeloc);
699
700 ffi_status
701 ffi_prep_closure_loc (ffi_closure* closure,
702 ffi_cif* cif,
703 void (*fun)(ffi_cif*, void*, void**, void*),
704 void *user_data,
705 void *codeloc)
706 {
707 static const unsigned char trampoline[16] = {
708 /* leaq -0x7(%rip),%r10 # 0x0 */
709 0x4c, 0x8d, 0x15, 0xf9, 0xff, 0xff, 0xff,
710 /* jmpq *0x3(%rip) # 0x10 */
711 0xff, 0x25, 0x03, 0x00, 0x00, 0x00,
712 /* nopl (%rax) */
713 0x0f, 0x1f, 0x00
714 };
715 void (*dest)(void);
716 char *tramp = closure->tramp;
717
718 if (cif->abi == FFI_EFI64)
719 return ffi_prep_closure_loc_efi64(closure, cif, fun, user_data, codeloc);
720 if (cif->abi != FFI_UNIX64)
721 return FFI_BAD_ABI;
722
723 if (cif->flags & UNIX64_FLAG_XMM_ARGS)
724 dest = ffi_closure_unix64_sse;
725 else
726 dest = ffi_closure_unix64;
727
728 memcpy (tramp, trampoline, sizeof(trampoline));
729 *(UINT64 *)(tramp + 16) = (uintptr_t)dest;
730
731 closure->cif = cif;
732 closure->fun = fun;
733 closure->user_data = user_data;
734
735 return FFI_OK;
736 }
737
738 int FFI_HIDDEN
739 ffi_closure_unix64_inner(ffi_cif *cif,
740 void (*fun)(ffi_cif*, void*, void**, void*),
741 void *user_data,
742 void *rvalue,
743 struct register_args *reg_args,
744 char *argp)
745 {
746 void **avalue;
747 ffi_type **arg_types;
748 long i, avn;
749 int gprcount, ssecount, ngpr, nsse;
750 int flags;
751
752 avn = cif->nargs;
753 flags = cif->flags;
754 avalue = alloca(avn * sizeof(void *));
755 gprcount = ssecount = 0;
756
757 if (flags & UNIX64_FLAG_RET_IN_MEM)
758 {
759 /* On return, %rax will contain the address that was passed
760 by the caller in %rdi. */
761 void *r = (void *)(uintptr_t)reg_args->gpr[gprcount++];
762 *(void **)rvalue = r;
763 rvalue = r;
764 flags = (sizeof(void *) == 4 ? UNIX64_RET_UINT32 : UNIX64_RET_INT64);
765 }
766
767 arg_types = cif->arg_types;
768 for (i = 0; i < avn; ++i)
769 {
770 enum x86_64_reg_class classes[MAX_CLASSES];
771 size_t n;
772
773 n = examine_argument (arg_types[i], classes, 0, &ngpr, &nsse);
774 if (n == 0
775 || gprcount + ngpr > MAX_GPR_REGS
776 || ssecount + nsse > MAX_SSE_REGS)
777 {
778 long align = arg_types[i]->alignment;
779
780 /* Stack arguments are *always* at least 8 byte aligned. */
781 if (align < 8)
782 align = 8;
783
784 /* Pass this argument in memory. */
785 argp = (void *) ALIGN (argp, align);
786 avalue[i] = argp;
787 argp += arg_types[i]->size;
788 }
789 /* If the argument is in a single register, or two consecutive
790 integer registers, then we can use that address directly. */
791 else if (n == 1
792 || (n == 2 && !(SSE_CLASS_P (classes[0])
793 || SSE_CLASS_P (classes[1]))))
794 {
795 /* The argument is in a single register. */
796 if (SSE_CLASS_P (classes[0]))
797 {
798 avalue[i] = &reg_args->sse[ssecount];
799 ssecount += n;
800 }
801 else
802 {
803 avalue[i] = &reg_args->gpr[gprcount];
804 gprcount += n;
805 }
806 }
807 /* Otherwise, allocate space to make them consecutive. */
808 else
809 {
810 char *a = alloca (16);
811 int j;
812
813 avalue[i] = a;
814 for (j = 0; j < n; j++, a += 8)
815 {
816 if (SSE_CLASS_P (classes[j]))
817 memcpy (a, &reg_args->sse[ssecount++], 8);
818 else
819 memcpy (a, &reg_args->gpr[gprcount++], 8);
820 }
821 }
822 }
823
824 /* Invoke the closure. */
825 fun (cif, rvalue, avalue, user_data);
826
827 /* Tell assembly how to perform return type promotions. */
828 return flags;
829 }
830
831 extern void ffi_go_closure_unix64(void) FFI_HIDDEN;
832 extern void ffi_go_closure_unix64_sse(void) FFI_HIDDEN;
833
834 extern ffi_status
835 ffi_prep_go_closure_efi64(ffi_go_closure* closure, ffi_cif* cif,
836 void (*fun)(ffi_cif*, void*, void**, void*));
837
838 ffi_status
839 ffi_prep_go_closure (ffi_go_closure* closure, ffi_cif* cif,
840 void (*fun)(ffi_cif*, void*, void**, void*))
841 {
842 if (cif->abi == FFI_EFI64)
843 return ffi_prep_go_closure_efi64(closure, cif, fun);
844 if (cif->abi != FFI_UNIX64)
845 return FFI_BAD_ABI;
846
847 closure->tramp = (cif->flags & UNIX64_FLAG_XMM_ARGS
848 ? ffi_go_closure_unix64_sse
849 : ffi_go_closure_unix64);
850 closure->cif = cif;
851 closure->fun = fun;
852
853 return FFI_OK;
854 }
855
856 #endif /* __x86_64__ */