a03061b5a7fb595e1d6af64969b2016a9107551d
[libffi.git] / src / x86 / ffi64.c
1 /* -----------------------------------------------------------------------
2 ffi64.c - Copyright (c) 2013 The Written Word, Inc.
3 Copyright (c) 2011 Anthony Green
4 Copyright (c) 2008, 2010 Red Hat, Inc.
5 Copyright (c) 2002, 2007 Bo Thorsen <bo@suse.de>
6
7 x86-64 Foreign Function Interface
8
9 Permission is hereby granted, free of charge, to any person obtaining
10 a copy of this software and associated documentation files (the
11 ``Software''), to deal in the Software without restriction, including
12 without limitation the rights to use, copy, modify, merge, publish,
13 distribute, sublicense, and/or sell copies of the Software, and to
14 permit persons to whom the Software is furnished to do so, subject to
15 the following conditions:
16
17 The above copyright notice and this permission notice shall be included
18 in all copies or substantial portions of the Software.
19
20 THE SOFTWARE IS PROVIDED ``AS IS'', WITHOUT WARRANTY OF ANY KIND,
21 EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
22 MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
23 NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
24 HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
25 WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
26 OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
27 DEALINGS IN THE SOFTWARE.
28 ----------------------------------------------------------------------- */
29
30 #include <ffi.h>
31 #include <ffi_common.h>
32
33 #include <stdlib.h>
34 #include <stdarg.h>
35 #include <stdint.h>
36 #include "internal64.h"
37
38 #ifdef __x86_64__
39
40 #define MAX_GPR_REGS 6
41 #define MAX_SSE_REGS 8
42
43 #if defined(__INTEL_COMPILER)
44 #include "xmmintrin.h"
45 #define UINT128 __m128
46 #else
47 #if defined(__SUNPRO_C)
48 #include <sunmedia_types.h>
49 #define UINT128 __m128i
50 #else
51 #define UINT128 __int128_t
52 #endif
53 #endif
54
55 union big_int_union
56 {
57 UINT32 i32;
58 UINT64 i64;
59 UINT128 i128;
60 };
61
62 struct register_args
63 {
64 /* Registers for argument passing. */
65 UINT64 gpr[MAX_GPR_REGS];
66 union big_int_union sse[MAX_SSE_REGS];
67 UINT64 rax; /* ssecount */
68 UINT64 r10; /* static chain */
69 };
70
71 extern void ffi_call_unix64 (void *args, unsigned long bytes, unsigned flags,
72 void *raddr, void (*fnaddr)(void)) FFI_HIDDEN;
73
74 /* All reference to register classes here is identical to the code in
75 gcc/config/i386/i386.c. Do *not* change one without the other. */
76
77 /* Register class used for passing given 64bit part of the argument.
78 These represent classes as documented by the PS ABI, with the
79 exception of SSESF, SSEDF classes, that are basically SSE class,
80 just gcc will use SF or DFmode move instead of DImode to avoid
81 reformatting penalties.
82
83 Similary we play games with INTEGERSI_CLASS to use cheaper SImode moves
84 whenever possible (upper half does contain padding). */
85 enum x86_64_reg_class
86 {
87 X86_64_NO_CLASS,
88 X86_64_INTEGER_CLASS,
89 X86_64_INTEGERSI_CLASS,
90 X86_64_SSE_CLASS,
91 X86_64_SSESF_CLASS,
92 X86_64_SSEDF_CLASS,
93 X86_64_SSEUP_CLASS,
94 X86_64_X87_CLASS,
95 X86_64_X87UP_CLASS,
96 X86_64_COMPLEX_X87_CLASS,
97 X86_64_MEMORY_CLASS
98 };
99
100 #define MAX_CLASSES 4
101
102 #define SSE_CLASS_P(X) ((X) >= X86_64_SSE_CLASS && X <= X86_64_SSEUP_CLASS)
103
104 /* x86-64 register passing implementation. See x86-64 ABI for details. Goal
105 of this code is to classify each 8bytes of incoming argument by the register
106 class and assign registers accordingly. */
107
108 /* Return the union class of CLASS1 and CLASS2.
109 See the x86-64 PS ABI for details. */
110
111 static enum x86_64_reg_class
112 merge_classes (enum x86_64_reg_class class1, enum x86_64_reg_class class2)
113 {
114 /* Rule #1: If both classes are equal, this is the resulting class. */
115 if (class1 == class2)
116 return class1;
117
118 /* Rule #2: If one of the classes is NO_CLASS, the resulting class is
119 the other class. */
120 if (class1 == X86_64_NO_CLASS)
121 return class2;
122 if (class2 == X86_64_NO_CLASS)
123 return class1;
124
125 /* Rule #3: If one of the classes is MEMORY, the result is MEMORY. */
126 if (class1 == X86_64_MEMORY_CLASS || class2 == X86_64_MEMORY_CLASS)
127 return X86_64_MEMORY_CLASS;
128
129 /* Rule #4: If one of the classes is INTEGER, the result is INTEGER. */
130 if ((class1 == X86_64_INTEGERSI_CLASS && class2 == X86_64_SSESF_CLASS)
131 || (class2 == X86_64_INTEGERSI_CLASS && class1 == X86_64_SSESF_CLASS))
132 return X86_64_INTEGERSI_CLASS;
133 if (class1 == X86_64_INTEGER_CLASS || class1 == X86_64_INTEGERSI_CLASS
134 || class2 == X86_64_INTEGER_CLASS || class2 == X86_64_INTEGERSI_CLASS)
135 return X86_64_INTEGER_CLASS;
136
137 /* Rule #5: If one of the classes is X87, X87UP, or COMPLEX_X87 class,
138 MEMORY is used. */
139 if (class1 == X86_64_X87_CLASS
140 || class1 == X86_64_X87UP_CLASS
141 || class1 == X86_64_COMPLEX_X87_CLASS
142 || class2 == X86_64_X87_CLASS
143 || class2 == X86_64_X87UP_CLASS
144 || class2 == X86_64_COMPLEX_X87_CLASS)
145 return X86_64_MEMORY_CLASS;
146
147 /* Rule #6: Otherwise class SSE is used. */
148 return X86_64_SSE_CLASS;
149 }
150
151 /* Classify the argument of type TYPE and mode MODE.
152 CLASSES will be filled by the register class used to pass each word
153 of the operand. The number of words is returned. In case the parameter
154 should be passed in memory, 0 is returned. As a special case for zero
155 sized containers, classes[0] will be NO_CLASS and 1 is returned.
156
157 See the x86-64 PS ABI for details.
158 */
159 static size_t
160 classify_argument (ffi_type *type, enum x86_64_reg_class classes[],
161 size_t byte_offset)
162 {
163 switch (type->type)
164 {
165 case FFI_TYPE_UINT8:
166 case FFI_TYPE_SINT8:
167 case FFI_TYPE_UINT16:
168 case FFI_TYPE_SINT16:
169 case FFI_TYPE_UINT32:
170 case FFI_TYPE_SINT32:
171 case FFI_TYPE_UINT64:
172 case FFI_TYPE_SINT64:
173 case FFI_TYPE_POINTER:
174 {
175 size_t size = byte_offset + type->size;
176
177 if (size <= 4)
178 {
179 classes[0] = X86_64_INTEGERSI_CLASS;
180 return 1;
181 }
182 else if (size <= 8)
183 {
184 classes[0] = X86_64_INTEGER_CLASS;
185 return 1;
186 }
187 else if (size <= 12)
188 {
189 classes[0] = X86_64_INTEGER_CLASS;
190 classes[1] = X86_64_INTEGERSI_CLASS;
191 return 2;
192 }
193 else if (size <= 16)
194 {
195 classes[0] = classes[1] = X86_64_INTEGER_CLASS;
196 return 2;
197 }
198 else
199 FFI_ASSERT (0);
200 }
201 case FFI_TYPE_FLOAT:
202 if (!(byte_offset % 8))
203 classes[0] = X86_64_SSESF_CLASS;
204 else
205 classes[0] = X86_64_SSE_CLASS;
206 return 1;
207 case FFI_TYPE_DOUBLE:
208 classes[0] = X86_64_SSEDF_CLASS;
209 return 1;
210 #if FFI_TYPE_LONGDOUBLE != FFI_TYPE_DOUBLE
211 case FFI_TYPE_LONGDOUBLE:
212 classes[0] = X86_64_X87_CLASS;
213 classes[1] = X86_64_X87UP_CLASS;
214 return 2;
215 #endif
216 case FFI_TYPE_STRUCT:
217 {
218 const size_t UNITS_PER_WORD = 8;
219 size_t words = (type->size + UNITS_PER_WORD - 1) / UNITS_PER_WORD;
220 ffi_type **ptr;
221 int i;
222 enum x86_64_reg_class subclasses[MAX_CLASSES];
223
224 /* If the struct is larger than 32 bytes, pass it on the stack. */
225 if (type->size > 32)
226 return 0;
227
228 for (i = 0; i < words; i++)
229 classes[i] = X86_64_NO_CLASS;
230
231 /* Zero sized arrays or structures are NO_CLASS. We return 0 to
232 signalize memory class, so handle it as special case. */
233 if (!words)
234 {
235 classes[0] = X86_64_NO_CLASS;
236 return 1;
237 }
238
239 /* Merge the fields of structure. */
240 for (ptr = type->elements; *ptr != NULL; ptr++)
241 {
242 size_t num;
243
244 byte_offset = ALIGN (byte_offset, (*ptr)->alignment);
245
246 num = classify_argument (*ptr, subclasses, byte_offset % 8);
247 if (num == 0)
248 return 0;
249 for (i = 0; i < num; i++)
250 {
251 size_t pos = byte_offset / 8;
252 classes[i + pos] =
253 merge_classes (subclasses[i], classes[i + pos]);
254 }
255
256 byte_offset += (*ptr)->size;
257 }
258
259 if (words > 2)
260 {
261 /* When size > 16 bytes, if the first one isn't
262 X86_64_SSE_CLASS or any other ones aren't
263 X86_64_SSEUP_CLASS, everything should be passed in
264 memory. */
265 if (classes[0] != X86_64_SSE_CLASS)
266 return 0;
267
268 for (i = 1; i < words; i++)
269 if (classes[i] != X86_64_SSEUP_CLASS)
270 return 0;
271 }
272
273 /* Final merger cleanup. */
274 for (i = 0; i < words; i++)
275 {
276 /* If one class is MEMORY, everything should be passed in
277 memory. */
278 if (classes[i] == X86_64_MEMORY_CLASS)
279 return 0;
280
281 /* The X86_64_SSEUP_CLASS should be always preceded by
282 X86_64_SSE_CLASS or X86_64_SSEUP_CLASS. */
283 if (classes[i] == X86_64_SSEUP_CLASS
284 && classes[i - 1] != X86_64_SSE_CLASS
285 && classes[i - 1] != X86_64_SSEUP_CLASS)
286 {
287 /* The first one should never be X86_64_SSEUP_CLASS. */
288 FFI_ASSERT (i != 0);
289 classes[i] = X86_64_SSE_CLASS;
290 }
291
292 /* If X86_64_X87UP_CLASS isn't preceded by X86_64_X87_CLASS,
293 everything should be passed in memory. */
294 if (classes[i] == X86_64_X87UP_CLASS
295 && (classes[i - 1] != X86_64_X87_CLASS))
296 {
297 /* The first one should never be X86_64_X87UP_CLASS. */
298 FFI_ASSERT (i != 0);
299 return 0;
300 }
301 }
302 return words;
303 }
304
305 default:
306 FFI_ASSERT(0);
307 }
308 return 0; /* Never reached. */
309 }
310
311 /* Examine the argument and return set number of register required in each
312 class. Return zero iff parameter should be passed in memory, otherwise
313 the number of registers. */
314
315 static size_t
316 examine_argument (ffi_type *type, enum x86_64_reg_class classes[MAX_CLASSES],
317 _Bool in_return, int *pngpr, int *pnsse)
318 {
319 size_t n;
320 int i, ngpr, nsse;
321
322 n = classify_argument (type, classes, 0);
323 if (n == 0)
324 return 0;
325
326 ngpr = nsse = 0;
327 for (i = 0; i < n; ++i)
328 switch (classes[i])
329 {
330 case X86_64_INTEGER_CLASS:
331 case X86_64_INTEGERSI_CLASS:
332 ngpr++;
333 break;
334 case X86_64_SSE_CLASS:
335 case X86_64_SSESF_CLASS:
336 case X86_64_SSEDF_CLASS:
337 nsse++;
338 break;
339 case X86_64_NO_CLASS:
340 case X86_64_SSEUP_CLASS:
341 break;
342 case X86_64_X87_CLASS:
343 case X86_64_X87UP_CLASS:
344 case X86_64_COMPLEX_X87_CLASS:
345 return in_return != 0;
346 default:
347 abort ();
348 }
349
350 *pngpr = ngpr;
351 *pnsse = nsse;
352
353 return n;
354 }
355
356 /* Perform machine dependent cif processing. */
357
358 ffi_status
359 ffi_prep_cif_machdep (ffi_cif *cif)
360 {
361 int gprcount, ssecount, i, avn, ngpr, nsse, flags;
362 enum x86_64_reg_class classes[MAX_CLASSES];
363 size_t bytes, n;
364 ffi_type *rtype;
365
366 if (cif->abi != FFI_UNIX64)
367 return FFI_BAD_ABI;
368
369 gprcount = ssecount = 0;
370
371 rtype = cif->rtype;
372 switch (rtype->type)
373 {
374 case FFI_TYPE_VOID:
375 flags = UNIX64_RET_VOID;
376 break;
377 case FFI_TYPE_UINT8:
378 flags = UNIX64_RET_UINT8;
379 break;
380 case FFI_TYPE_SINT8:
381 flags = UNIX64_RET_SINT8;
382 break;
383 case FFI_TYPE_UINT16:
384 flags = UNIX64_RET_UINT16;
385 break;
386 case FFI_TYPE_SINT16:
387 flags = UNIX64_RET_SINT16;
388 break;
389 case FFI_TYPE_UINT32:
390 flags = UNIX64_RET_UINT32;
391 break;
392 case FFI_TYPE_INT:
393 case FFI_TYPE_SINT32:
394 flags = UNIX64_RET_SINT32;
395 break;
396 case FFI_TYPE_UINT64:
397 case FFI_TYPE_SINT64:
398 flags = UNIX64_RET_INT64;
399 break;
400 case FFI_TYPE_POINTER:
401 flags = (sizeof(void *) == 4 ? UNIX64_RET_UINT32 : UNIX64_RET_INT64);
402 break;
403 case FFI_TYPE_FLOAT:
404 flags = UNIX64_RET_XMM32;
405 break;
406 case FFI_TYPE_DOUBLE:
407 flags = UNIX64_RET_XMM64;
408 break;
409 case FFI_TYPE_LONGDOUBLE:
410 flags = UNIX64_RET_X87;
411 break;
412 case FFI_TYPE_STRUCT:
413 n = examine_argument (cif->rtype, classes, 1, &ngpr, &nsse);
414 if (n == 0)
415 {
416 /* The return value is passed in memory. A pointer to that
417 memory is the first argument. Allocate a register for it. */
418 gprcount++;
419 /* We don't have to do anything in asm for the return. */
420 flags = UNIX64_RET_VOID | UNIX64_FLAG_RET_IN_MEM;
421 }
422 else
423 {
424 /* Mark which registers the result appears in. */
425 _Bool sse0 = SSE_CLASS_P (classes[0]);
426 _Bool sse1 = n == 2 && SSE_CLASS_P (classes[1]);
427 if (sse0)
428 flags = (sse1 ? UNIX64_RET_ST_XMM0_XMM1 : UNIX64_RET_ST_XMM0_RAX);
429 else
430 flags = (sse1 ? UNIX64_RET_ST_RAX_XMM0 : UNIX64_RET_ST_RAX_RDX);
431
432 /* Mark the true size of the structure. */
433 flags |= rtype->size << UNIX64_SIZE_SHIFT;
434 }
435 break;
436 default:
437 return FFI_BAD_TYPEDEF;
438 }
439
440 /* Go over all arguments and determine the way they should be passed.
441 If it's in a register and there is space for it, let that be so. If
442 not, add it's size to the stack byte count. */
443 for (bytes = 0, i = 0, avn = cif->nargs; i < avn; i++)
444 {
445 if (examine_argument (cif->arg_types[i], classes, 0, &ngpr, &nsse) == 0
446 || gprcount + ngpr > MAX_GPR_REGS
447 || ssecount + nsse > MAX_SSE_REGS)
448 {
449 long align = cif->arg_types[i]->alignment;
450
451 if (align < 8)
452 align = 8;
453
454 bytes = ALIGN (bytes, align);
455 bytes += cif->arg_types[i]->size;
456 }
457 else
458 {
459 gprcount += ngpr;
460 ssecount += nsse;
461 }
462 }
463 if (ssecount)
464 flags |= UNIX64_FLAG_XMM_ARGS;
465
466 cif->flags = flags;
467 cif->bytes = ALIGN (bytes, 8);
468
469 return FFI_OK;
470 }
471
472 static void
473 ffi_call_int (ffi_cif *cif, void (*fn)(void), void *rvalue,
474 void **avalue, void *closure)
475 {
476 enum x86_64_reg_class classes[MAX_CLASSES];
477 char *stack, *argp;
478 ffi_type **arg_types;
479 int gprcount, ssecount, ngpr, nsse, i, avn, flags;
480 struct register_args *reg_args;
481
482 /* Can't call 32-bit mode from 64-bit mode. */
483 FFI_ASSERT (cif->abi == FFI_UNIX64);
484
485 /* If the return value is a struct and we don't have a return value
486 address then we need to make one. Otherwise we can ignore it. */
487 flags = cif->flags;
488 if (rvalue == NULL)
489 {
490 if (flags & UNIX64_FLAG_RET_IN_MEM)
491 rvalue = alloca (cif->rtype->size);
492 else
493 flags = UNIX64_RET_VOID;
494 }
495
496 /* Allocate the space for the arguments, plus 4 words of temp space. */
497 stack = alloca (sizeof (struct register_args) + cif->bytes + 4*8);
498 reg_args = (struct register_args *) stack;
499 argp = stack + sizeof (struct register_args);
500
501 reg_args->r10 = (uintptr_t) closure;
502
503 gprcount = ssecount = 0;
504
505 /* If the return value is passed in memory, add the pointer as the
506 first integer argument. */
507 if (flags & UNIX64_FLAG_RET_IN_MEM)
508 reg_args->gpr[gprcount++] = (unsigned long) rvalue;
509
510 avn = cif->nargs;
511 arg_types = cif->arg_types;
512
513 for (i = 0; i < avn; ++i)
514 {
515 size_t n, size = arg_types[i]->size;
516
517 n = examine_argument (arg_types[i], classes, 0, &ngpr, &nsse);
518 if (n == 0
519 || gprcount + ngpr > MAX_GPR_REGS
520 || ssecount + nsse > MAX_SSE_REGS)
521 {
522 long align = arg_types[i]->alignment;
523
524 /* Stack arguments are *always* at least 8 byte aligned. */
525 if (align < 8)
526 align = 8;
527
528 /* Pass this argument in memory. */
529 argp = (void *) ALIGN (argp, align);
530 memcpy (argp, avalue[i], size);
531 argp += size;
532 }
533 else
534 {
535 /* The argument is passed entirely in registers. */
536 char *a = (char *) avalue[i];
537 int j;
538
539 for (j = 0; j < n; j++, a += 8, size -= 8)
540 {
541 switch (classes[j])
542 {
543 case X86_64_INTEGER_CLASS:
544 case X86_64_INTEGERSI_CLASS:
545 /* Sign-extend integer arguments passed in general
546 purpose registers, to cope with the fact that
547 LLVM incorrectly assumes that this will be done
548 (the x86-64 PS ABI does not specify this). */
549 switch (arg_types[i]->type)
550 {
551 case FFI_TYPE_SINT8:
552 reg_args->gpr[gprcount] = (SINT64) *((SINT8 *) a);
553 break;
554 case FFI_TYPE_SINT16:
555 reg_args->gpr[gprcount] = (SINT64) *((SINT16 *) a);
556 break;
557 case FFI_TYPE_SINT32:
558 reg_args->gpr[gprcount] = (SINT64) *((SINT32 *) a);
559 break;
560 default:
561 reg_args->gpr[gprcount] = 0;
562 memcpy (&reg_args->gpr[gprcount], a, size);
563 }
564 gprcount++;
565 break;
566 case X86_64_SSE_CLASS:
567 case X86_64_SSEDF_CLASS:
568 reg_args->sse[ssecount++].i64 = *(UINT64 *) a;
569 break;
570 case X86_64_SSESF_CLASS:
571 reg_args->sse[ssecount++].i32 = *(UINT32 *) a;
572 break;
573 default:
574 abort();
575 }
576 }
577 }
578 }
579 reg_args->rax = ssecount;
580
581 ffi_call_unix64 (stack, cif->bytes + sizeof (struct register_args),
582 flags, rvalue, fn);
583 }
584
585 void
586 ffi_call (ffi_cif *cif, void (*fn)(void), void *rvalue, void **avalue)
587 {
588 ffi_call_int (cif, fn, rvalue, avalue, NULL);
589 }
590
591 void
592 ffi_call_go (ffi_cif *cif, void (*fn)(void), void *rvalue,
593 void **avalue, void *closure)
594 {
595 ffi_call_int (cif, fn, rvalue, avalue, closure);
596 }
597
598 extern void ffi_closure_unix64(void) FFI_HIDDEN;
599 extern void ffi_closure_unix64_sse(void) FFI_HIDDEN;
600
601 ffi_status
602 ffi_prep_closure_loc (ffi_closure* closure,
603 ffi_cif* cif,
604 void (*fun)(ffi_cif*, void*, void**, void*),
605 void *user_data,
606 void *codeloc)
607 {
608 static const unsigned char trampoline[16] = {
609 /* leaq -0x7(%rip),%r10 # 0x0 */
610 0x4c, 0x8d, 0x15, 0xf9, 0xff, 0xff, 0xff,
611 /* jmpq *0x3(%rip) # 0x10 */
612 0xff, 0x25, 0x03, 0x00, 0x00, 0x00,
613 /* nopl (%rax) */
614 0x0f, 0x1f, 0x00
615 };
616 void (*dest)(void);
617 char *tramp = closure->tramp;
618
619 if (cif->abi != FFI_UNIX64)
620 return FFI_BAD_ABI;
621
622 if (cif->flags & UNIX64_FLAG_XMM_ARGS)
623 dest = ffi_closure_unix64_sse;
624 else
625 dest = ffi_closure_unix64;
626
627 memcpy (tramp, trampoline, sizeof(trampoline));
628 *(UINT64 *)(tramp + 16) = (uintptr_t)dest;
629
630 closure->cif = cif;
631 closure->fun = fun;
632 closure->user_data = user_data;
633
634 return FFI_OK;
635 }
636
637 int FFI_HIDDEN
638 ffi_closure_unix64_inner(ffi_cif *cif,
639 void (*fun)(ffi_cif*, void*, void**, void*),
640 void *user_data,
641 void *rvalue,
642 struct register_args *reg_args,
643 char *argp)
644 {
645 void **avalue;
646 ffi_type **arg_types;
647 long i, avn;
648 int gprcount, ssecount, ngpr, nsse;
649 int flags;
650
651 avn = cif->nargs;
652 flags = cif->flags;
653 avalue = alloca(avn * sizeof(void *));
654 gprcount = ssecount = 0;
655
656 if (flags & UNIX64_FLAG_RET_IN_MEM)
657 rvalue = (void *)(uintptr_t)reg_args->gpr[gprcount++];
658
659 arg_types = cif->arg_types;
660 for (i = 0; i < avn; ++i)
661 {
662 enum x86_64_reg_class classes[MAX_CLASSES];
663 size_t n;
664
665 n = examine_argument (arg_types[i], classes, 0, &ngpr, &nsse);
666 if (n == 0
667 || gprcount + ngpr > MAX_GPR_REGS
668 || ssecount + nsse > MAX_SSE_REGS)
669 {
670 long align = arg_types[i]->alignment;
671
672 /* Stack arguments are *always* at least 8 byte aligned. */
673 if (align < 8)
674 align = 8;
675
676 /* Pass this argument in memory. */
677 argp = (void *) ALIGN (argp, align);
678 avalue[i] = argp;
679 argp += arg_types[i]->size;
680 }
681 /* If the argument is in a single register, or two consecutive
682 integer registers, then we can use that address directly. */
683 else if (n == 1
684 || (n == 2 && !(SSE_CLASS_P (classes[0])
685 || SSE_CLASS_P (classes[1]))))
686 {
687 /* The argument is in a single register. */
688 if (SSE_CLASS_P (classes[0]))
689 {
690 avalue[i] = &reg_args->sse[ssecount];
691 ssecount += n;
692 }
693 else
694 {
695 avalue[i] = &reg_args->gpr[gprcount];
696 gprcount += n;
697 }
698 }
699 /* Otherwise, allocate space to make them consecutive. */
700 else
701 {
702 char *a = alloca (16);
703 int j;
704
705 avalue[i] = a;
706 for (j = 0; j < n; j++, a += 8)
707 {
708 if (SSE_CLASS_P (classes[j]))
709 memcpy (a, &reg_args->sse[ssecount++], 8);
710 else
711 memcpy (a, &reg_args->gpr[gprcount++], 8);
712 }
713 }
714 }
715
716 /* Invoke the closure. */
717 fun (cif, rvalue, avalue, user_data);
718
719 /* Tell assembly how to perform return type promotions. */
720 return flags;
721 }
722
723 extern void ffi_go_closure_unix64(void) FFI_HIDDEN;
724 extern void ffi_go_closure_unix64_sse(void) FFI_HIDDEN;
725
726 ffi_status
727 ffi_prep_go_closure (ffi_go_closure* closure, ffi_cif* cif,
728 void (*fun)(ffi_cif*, void*, void**, void*))
729 {
730 if (cif->abi != FFI_UNIX64)
731 return FFI_BAD_ABI;
732
733 closure->tramp = (cif->flags & UNIX64_FLAG_XMM_ARGS
734 ? ffi_go_closure_unix64_sse
735 : ffi_go_closure_unix64);
736 closure->cif = cif;
737 closure->fun = fun;
738
739 return FFI_OK;
740 }
741
742 #endif /* __x86_64__ */