add a thp-based huge page alloc fallback
[pcsx_rearmed.git] / deps / lightning / lib / jit_s390.c
CommitLineData
4a71579b 1/*
79bfeef6 2 * Copyright (C) 2013-2023 Free Software Foundation, Inc.
4a71579b
PC
3 *
4 * This file is part of GNU lightning.
5 *
6 * GNU lightning is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU Lesser General Public License as published
8 * by the Free Software Foundation; either version 3, or (at your option)
9 * any later version.
10 *
11 * GNU lightning is distributed in the hope that it will be useful, but
12 * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
13 * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public
14 * License for more details.
15 *
16 * Authors:
17 * Paulo Cesar Pereira de Andrade
18 */
79bfeef6
PC
19#define CHECK_FLOGR 0
20
21#if CHECK_FLOGR
22#include <signal.h>
23#include <setjmp.h>
24#endif
4a71579b
PC
25
26#include <lightning.h>
27#include <lightning/jit_private.h>
28
29#if __WORDSIZE == 32
30# define NUM_FLOAT_REG_ARGS 2
31#else
32# define NUM_FLOAT_REG_ARGS 4
33#endif
34#define jit_arg_reg_p(i) ((i) >= 0 && (i) < 5)
35#define jit_arg_f_reg_p(i) ((i) >= 0 && (i) < NUM_FLOAT_REG_ARGS)
36
37/*
38 * Types
39 */
40typedef struct jit_va_list {
41 /* The offsets are "1" based, as addresses are fixed in the
42 * standard stack frame format. */
43 jit_word_t gpoff;
44 jit_word_t fpoff;
45
46 /* Easier when there is an explicitly defined type...
47(gdb) ptype ap
48type = struct __va_list_tag {
49 long __gpr;
50 long __fpr;
51 void *__overflow_arg_area;
52 void *__reg_save_area;
53
54 Note that gopff (__gpr) and fpoff (__fpr) are jit_word_t equivalent
55 and, again, "1" (unit) based, so must be adjusted at va_arg time.
56 */
57 jit_pointer_t over;
58 jit_pointer_t save;
59
60 /* For variadic functions, always allocate space to save callee
61 * save fpr registers.
62 * Note that s390 has a standard stack frame format that lightning
63 * does not fully comply with, but for variadic functions it must,
64 * for those (variadic) do not use the "empty" spaces for any
65 * callee save fpr register, but save them after the va_list
66 * space; and use the standard stack frame format, as required
67 * by variadic functions (and have a compatible va_list pointer). */
68 jit_float64_t f8;
69 jit_float64_t f9;
70 jit_float64_t f10;
71 jit_float64_t f11;
72 jit_float64_t f12;
73 jit_float64_t f13;
74 jit_float64_t f14;
75 jit_float64_t f15;
76} jit_va_list_t;
77
78/*
79 * Prototypes
80 */
81#define jit_get_reg_pair() _jit_get_reg_pair(_jit)
82static jit_int32_t _jit_get_reg_pair(jit_state_t*);
83#define jit_unget_reg_pair(regno) _jit_unget_reg_pair(_jit,regno)
84static void _jit_unget_reg_pair(jit_state_t*,jit_int32_t);
85#define jit_get_reg_but_zero(flags) _jit_get_reg_but_zero(_jit,flags)
86static jit_int32_t _jit_get_reg_but_zero(jit_state_t*,jit_int32_t);
87#define jit_unget_reg_but_zero(reg) jit_unget_reg(reg)
88#define patch(instr, node) _patch(_jit, instr, node)
89static void _patch(jit_state_t*,jit_word_t,jit_node_t*);
90
91/* libgcc */
92extern void __clear_cache(void *, void *);
93
94#define PROTO 1
95# include "jit_s390-cpu.c"
96# include "jit_s390-fpu.c"
ba86ff93 97# include "jit_fallback.c"
4a71579b
PC
98#undef PROTO
99
100/*
101 * Initialization
102 */
79bfeef6 103jit_cpu_t jit_cpu;
4a71579b
PC
104jit_register_t _rvs[] = {
105 { rc(gpr) | 0x0, "%r0" },
106 { rc(gpr) | 0x1, "%r1" },
107 { rc(gpr) | rc(sav) | 0xc, "%r12" },
108 { rc(gpr) | rc(sav) | 0xb, "%r11" },
109 { rc(gpr) | rc(sav) | 0xa, "%r10" },
110 { rc(gpr) | rc(sav) | 0x9, "%r9" },
111 { rc(gpr) | rc(sav) | 0x8, "%r8" },
112 { rc(gpr) | rc(sav) | 0x7, "%r7" },
113 { rc(gpr) | rc(arg) | rc(sav) | 0x6,"%r6" },
114 { rc(gpr) | rc(arg) | 0x5, "%r5" },
115 { rc(gpr) | rc(arg) | 0x4, "%r4" },
116 { rc(gpr) | rc(arg) | 0x3, "%r3" },
117 { rc(gpr) | rc(arg) | 0x2, "%r2" },
118 { rc(sav) | 0xd, "%r13" }, /* used as JIT_FP */
119 { 0xe, "%r14" },
120 { rc(sav) | 0xf, "%r15" },
121 { rc(fpr) | 0x1, "%f1" },
122 { rc(fpr) | 0x3, "%f3" },
123 { rc(fpr) | 0x5, "%f5" },
124 { rc(fpr) | 0x7, "%f7" },
125 { rc(fpr) | rc(sav) | 0xe, "%f14" },
126 /* Do not use as temporary to simplify stack layout */
127 { 0xf, "%f15" },
128 { rc(fpr) | rc(sav) | 0x8, "%f8" },
129 { rc(fpr) | rc(sav) | 0x9, "%f9" },
130 { rc(fpr) | rc(sav) | 0xa, "%f10" },
131 { rc(fpr) | rc(sav) | 0xb, "%f11" },
132 { rc(fpr) | rc(sav) | 0xc, "%f12" },
133 { rc(fpr) | rc(sav) | 0xd, "%f13" },
134 { rc(fpr) | rc(arg) | 0x6, "%f6" },
135 { rc(fpr) | rc(arg) | 0x4, "%f4" },
136 { rc(fpr) | rc(arg) | 0x2, "%f2" },
137 { rc(fpr) | rc(arg) | 0x0, "%f0" },
138 { _NOREG, "<none>" },
139};
79bfeef6
PC
140#if CHECK_FLOGR
141static sigjmp_buf jit_env;
142#endif
4a71579b
PC
143
144/*
145 * Implementation
146 */
79bfeef6
PC
147#if CHECK_FLOGR
148static void
149sigill_handler(int signum)
150{
151 jit_cpu.flogr = 0;
152 siglongjmp(jit_env, 1);
153}
154#endif
155
4a71579b
PC
156void
157jit_get_cpu(void)
158{
79bfeef6
PC
159#if CHECK_FLOGR
160 int r12, r13;
161 struct sigaction new_action, old_action;
162 new_action.sa_handler = sigill_handler;
163 sigemptyset(&new_action.sa_mask);
164 new_action.sa_flags = 0;
165 sigaction(SIGILL, NULL, &old_action);
166 if (old_action.sa_handler != SIG_IGN) {
167 sigaction(SIGILL, &new_action, NULL);
168 if (!sigsetjmp(jit_env, 1)) {
169 jit_cpu.flogr = 1;
170 /* flogr %r12, %r12 */
171 __asm__ volatile("lgr %%r12, %0; lgr %%r13, %1;"
172 "flogr %%r12, %%r12;"
173 "lgr %1, %%r13; lgr %0, %%r12;"
174 : "=r" (r12), "=r" (r13));
175 sigaction(SIGILL, &old_action, NULL);
176 }
177 }
178#else
179 /* By default, assume it is available */
180 jit_cpu.flogr = 1;
181#endif
4a71579b
PC
182}
183
184void
185_jit_init(jit_state_t *_jit)
186{
187 _jitc->reglen = jit_size(_rvs) - 1;
188}
189
190void
191_jit_prolog(jit_state_t *_jit)
192{
193 jit_int32_t offset;
194
195 if (_jitc->function)
196 jit_epilog();
197 assert(jit_regset_cmp_ui(&_jitc->regarg, 0) == 0);
198 jit_regset_set_ui(&_jitc->regsav, 0);
199 offset = _jitc->functions.offset;
200 if (offset >= _jitc->functions.length) {
201 jit_realloc((jit_pointer_t *)&_jitc->functions.ptr,
202 _jitc->functions.length * sizeof(jit_function_t),
203 (_jitc->functions.length + 16) * sizeof(jit_function_t));
204 _jitc->functions.length += 16;
205 }
206 _jitc->function = _jitc->functions.ptr + _jitc->functions.offset++;
207 _jitc->function->self.size = stack_framesize;
208 _jitc->function->self.argi = _jitc->function->self.argf =
209 _jitc->function->self.aoff = _jitc->function->self.alen = 0;
210 /* preallocate 8 bytes if not using a constant data buffer */
211 if (_jitc->no_data)
212 _jitc->function->self.aoff = -8;
213 _jitc->function->self.call = jit_call_default;
214 jit_alloc((jit_pointer_t *)&_jitc->function->regoff,
215 _jitc->reglen * sizeof(jit_int32_t));
216
217 /* _no_link here does not mean the jit_link() call can be removed
218 * by rewriting as:
219 * _jitc->function->prolog = jit_new_node(jit_code_prolog);
220 */
221 _jitc->function->prolog = jit_new_node_no_link(jit_code_prolog);
222 jit_link(_jitc->function->prolog);
223 _jitc->function->prolog->w.w = offset;
224 _jitc->function->epilog = jit_new_node_no_link(jit_code_epilog);
225 /* u: label value
226 * v: offset in blocks vector
227 * w: offset in functions vector
228 */
229 _jitc->function->epilog->w.w = offset;
230
231 jit_regset_new(&_jitc->function->regset);
232}
233
234jit_int32_t
235_jit_allocai(jit_state_t *_jit, jit_int32_t length)
236{
237 assert(_jitc->function);
238 switch (length) {
239 case 0: case 1: break;
240 case 2: _jitc->function->self.aoff &= -2; break;
241 case 3: case 4: _jitc->function->self.aoff &= -4; break;
242 default: _jitc->function->self.aoff &= -8; break;
243 }
244 _jitc->function->self.aoff -= length;
245 if (!_jitc->realize) {
246 jit_inc_synth_ww(allocai, _jitc->function->self.aoff, length);
247 jit_dec_synth();
248 }
249 return (_jitc->function->self.aoff);
250}
251
252void
253_jit_allocar(jit_state_t *_jit, jit_int32_t u, jit_int32_t v)
254{
255 jit_int32_t reg;
256 assert(_jitc->function);
257 jit_inc_synth_ww(allocar, u, v);
258 if (!_jitc->function->allocar) {
259 _jitc->function->aoffoff = jit_allocai(sizeof(jit_int32_t));
260 _jitc->function->allocar = 1;
261 }
262 reg = jit_get_reg(jit_class_gpr);
263 jit_negr(reg, v);
264 jit_andi(reg, reg, -8);
265 jit_ldxi_i(u, JIT_FP, _jitc->function->aoffoff);
266 jit_addr(u, u, reg);
267 jit_addr(JIT_SP, JIT_SP, reg);
268 jit_stxi_i(_jitc->function->aoffoff, JIT_FP, u);
269 jit_unget_reg(reg);
270 jit_dec_synth();
271}
272
273void
274_jit_ret(jit_state_t *_jit)
275{
276 jit_node_t *instr;
277 assert(_jitc->function);
278 jit_inc_synth(ret);
279 /* jump to epilog */
280 instr = jit_jmpi();
281 jit_patch_at(instr, _jitc->function->epilog);
282 jit_dec_synth();
283}
284
285void
79bfeef6 286_jit_retr(jit_state_t *_jit, jit_int32_t u, jit_code_t code)
4a71579b 287{
79bfeef6 288 jit_code_inc_synth_w(code, u);
4a71579b
PC
289 jit_movr(JIT_RET, u);
290 jit_ret();
291 jit_dec_synth();
292}
293
294void
79bfeef6 295_jit_reti(jit_state_t *_jit, jit_word_t u, jit_code_t code)
4a71579b 296{
79bfeef6 297 jit_code_inc_synth_w(code, u);
4a71579b
PC
298 jit_movi(JIT_RET, u);
299 jit_ret();
300 jit_dec_synth();
301}
302
303void
304_jit_retr_f(jit_state_t *_jit, jit_int32_t u)
305{
306 jit_inc_synth_w(retr_f, u);
307 jit_movr_f(JIT_FRET, u);
308 jit_ret();
309 jit_dec_synth();
310}
311
312void
313_jit_reti_f(jit_state_t *_jit, jit_float32_t u)
314{
315 jit_inc_synth_f(reti_f, u);
316 jit_movi_f(JIT_FRET, u);
317 jit_ret();
318 jit_dec_synth();
319}
320
321void
322_jit_retr_d(jit_state_t *_jit, jit_int32_t u)
323{
324 jit_inc_synth_w(retr_d, u);
325 jit_movr_d(JIT_FRET, u);
326 jit_ret();
327 jit_dec_synth();
328}
329
330void
331_jit_reti_d(jit_state_t *_jit, jit_float64_t u)
332{
333 jit_inc_synth_d(reti_d, u);
334 jit_movi_d(JIT_FRET, u);
335 jit_ret();
336 jit_dec_synth();
337}
338
339void
340_jit_epilog(jit_state_t *_jit)
341{
342 assert(_jitc->function);
343 assert(_jitc->function->epilog->next == NULL);
344 jit_link(_jitc->function->epilog);
345 _jitc->function = NULL;
346}
347
348jit_bool_t
349_jit_arg_register_p(jit_state_t *_jit, jit_node_t *u)
350{
79bfeef6 351 if (u->code >= jit_code_arg_c && u->code <= jit_code_arg)
4a71579b
PC
352 return (jit_arg_reg_p(u->u.w));
353 assert(u->code == jit_code_arg_f || u->code == jit_code_arg_d);
354 return (jit_arg_f_reg_p(u->u.w));
355}
356
357void
358_jit_ellipsis(jit_state_t *_jit)
359{
360 jit_inc_synth(ellipsis);
361 if (_jitc->prepare) {
362 jit_link_prepare();
363 assert(!(_jitc->function->call.call & jit_call_varargs));
364 _jitc->function->call.call |= jit_call_varargs;
365 }
366 else {
367 jit_link_prolog();
368 assert(!(_jitc->function->self.call & jit_call_varargs));
369 _jitc->function->self.call |= jit_call_varargs;
370
371 /* Allocate va_list like object in the stack. */
372 _jitc->function->vaoff = jit_allocai(sizeof(jit_va_list_t));
373
374 /* Initialize gp offset in save area. */
375 if (jit_arg_reg_p(_jitc->function->self.argi))
376 _jitc->function->vagp = _jitc->function->self.argi;
377 else
378 _jitc->function->vagp = 5;
379
380 /* Initialize fp offset in save area. */
381 if (jit_arg_f_reg_p(_jitc->function->self.argf))
382 _jitc->function->vafp = _jitc->function->self.argf;
383 else
384 _jitc->function->vafp = NUM_FLOAT_REG_ARGS;
385 }
386 jit_dec_synth();
387}
388
389void
390_jit_va_push(jit_state_t *_jit, jit_int32_t u)
391{
392 jit_inc_synth_w(va_push, u);
393 jit_pushargr(u);
394 jit_dec_synth();
395}
396
397jit_node_t *
79bfeef6 398_jit_arg(jit_state_t *_jit, jit_code_t code)
4a71579b
PC
399{
400 jit_node_t *node;
401 jit_int32_t offset;
402 assert(_jitc->function);
79bfeef6
PC
403 assert(!(_jitc->function->self.call & jit_call_varargs));
404#if STRONG_TYPE_CHECKING
405 assert(code >= jit_code_arg_c && code <= jit_code_arg);
406#endif
4a71579b
PC
407 if (jit_arg_reg_p(_jitc->function->self.argi))
408 offset = _jitc->function->self.argi++;
409 else {
410 offset = _jitc->function->self.size;
411 _jitc->function->self.size += sizeof(jit_word_t);
412 }
79bfeef6 413 node = jit_new_node_ww(code, offset,
4a71579b
PC
414 ++_jitc->function->self.argn);
415 jit_link_prolog();
416 return (node);
417}
418
419jit_node_t *
420_jit_arg_f(jit_state_t *_jit)
421{
422 jit_node_t *node;
423 jit_int32_t offset;
424 assert(_jitc->function);
425 if (jit_arg_f_reg_p(_jitc->function->self.argf))
426 offset = _jitc->function->self.argf++;
427 else {
428 offset = _jitc->function->self.size;
429 _jitc->function->self.size += sizeof(jit_word_t);
430 }
431 node = jit_new_node_ww(jit_code_arg_f, offset,
432 ++_jitc->function->self.argn);
433 jit_link_prolog();
434 return (node);
435}
436
437jit_node_t *
438_jit_arg_d(jit_state_t *_jit)
439{
440 jit_node_t *node;
441 jit_int32_t offset;
442 assert(_jitc->function);
443 if (jit_arg_f_reg_p(_jitc->function->self.argf))
444 offset = _jitc->function->self.argf++;
445 else {
446 offset = _jitc->function->self.size;
447 _jitc->function->self.size += sizeof(jit_float64_t);
448 }
449 node = jit_new_node_ww(jit_code_arg_d, offset,
450 ++_jitc->function->self.argn);
451 jit_link_prolog();
452 return (node);
453}
454
455void
456_jit_getarg_c(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
457{
79bfeef6 458 assert_arg_type(v->code, jit_code_arg_c);
4a71579b
PC
459 jit_inc_synth_wp(getarg_c, u, v);
460 if (jit_arg_reg_p(v->u.w))
461 jit_extr_c(u, _R2 - v->u.w);
462 else
463 jit_ldxi_c(u, JIT_FP,
464 v->u.w + (__WORDSIZE >> 3) - sizeof(jit_int8_t));
465 jit_dec_synth();
466}
467
468void
469_jit_getarg_uc(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
470{
79bfeef6 471 assert_arg_type(v->code, jit_code_arg_c);
4a71579b
PC
472 jit_inc_synth_wp(getarg_uc, u, v);
473 if (jit_arg_reg_p(v->u.w))
474 jit_extr_uc(u, _R2 - v->u.w);
475 else
476 jit_ldxi_uc(u, JIT_FP,
477 v->u.w + (__WORDSIZE >> 3) - sizeof(jit_uint8_t));
478 jit_dec_synth();
479}
480
481void
482_jit_getarg_s(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
483{
79bfeef6 484 assert_arg_type(v->code, jit_code_arg_s);
4a71579b
PC
485 jit_inc_synth_wp(getarg_s, u, v);
486 if (jit_arg_reg_p(v->u.w))
487 jit_extr_s(u, _R2 - v->u.w);
488 else
489 jit_ldxi_s(u, JIT_FP,
490 v->u.w + (__WORDSIZE >> 3) - sizeof(jit_int16_t));
491 jit_dec_synth();
492}
493
494void
495_jit_getarg_us(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
496{
79bfeef6 497 assert_arg_type(v->code, jit_code_arg_s);
4a71579b
PC
498 jit_inc_synth_wp(getarg_us, u, v);
499 if (jit_arg_reg_p(v->u.w))
500 jit_extr_us(u, _R2 - v->u.w);
501 else
502 jit_ldxi_us(u, JIT_FP,
503 v->u.w + (__WORDSIZE >> 3) - sizeof(jit_uint16_t));
504 jit_dec_synth();
505}
506
507void
508_jit_getarg_i(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
509{
79bfeef6 510 assert_arg_type(v->code, jit_code_arg_i);
4a71579b
PC
511 jit_inc_synth_wp(getarg_i, u, v);
512 if (jit_arg_reg_p(v->u.w)) {
513#if __WORDSIZE == 32
514 jit_movr(u, _R2 - v->u.w);
515#else
516 jit_extr_i(u, _R2 - v->u.w);
517#endif
518 }
519 else
520 jit_ldxi_i(u, JIT_FP,
521 v->u.w + (__WORDSIZE >> 3) - sizeof(jit_int32_t));
522 jit_dec_synth();
523}
524
525#if __WORDSIZE == 64
526void
527_jit_getarg_ui(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
528{
79bfeef6 529 assert_arg_type(v->code, jit_code_arg_i);
4a71579b
PC
530 jit_inc_synth_wp(getarg_ui, u, v);
531 if (jit_arg_reg_p(v->u.w))
532 jit_extr_ui(u, _R2 - v->u.w);
533 else
534 jit_ldxi_ui(u, JIT_FP,
535 v->u.w + (__WORDSIZE >> 3) - sizeof(jit_uint32_t));
536 jit_dec_synth();
537}
538
539void
540_jit_getarg_l(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
541{
79bfeef6 542 assert_arg_type(v->code, jit_code_arg_l);
4a71579b
PC
543 jit_inc_synth_wp(getarg_l, u, v);
544 if (jit_arg_reg_p(v->u.w))
545 jit_movr(u, _R2 - v->u.w);
546 else
547 jit_ldxi_l(u, JIT_FP, v->u.w);
548 jit_dec_synth();
549}
550#endif
551
552void
79bfeef6 553_jit_putargr(jit_state_t *_jit, jit_int32_t u, jit_node_t *v, jit_code_t code)
4a71579b 554{
79bfeef6
PC
555 assert_putarg_type(code, v->code);
556 jit_code_inc_synth_wp(code, u, v);
4a71579b
PC
557 if (jit_arg_reg_p(v->u.w))
558 jit_movr(_R2 - v->u.w, u);
559 else
560 jit_stxi(v->u.w, JIT_FP, u);
561 jit_dec_synth();
562}
563
564void
79bfeef6 565_jit_putargi(jit_state_t *_jit, jit_word_t u, jit_node_t *v, jit_code_t code)
4a71579b
PC
566{
567 jit_int32_t regno;
79bfeef6
PC
568 assert_putarg_type(code, v->code);
569 jit_code_inc_synth_wp(code, u, v);
4a71579b
PC
570 if (jit_arg_reg_p(v->u.w))
571 jit_movi(_R2 - v->u.w, u);
572 else {
573 regno = jit_get_reg(jit_class_gpr);
574 jit_movi(regno, u);
575 jit_stxi(v->u.w, JIT_FP, regno);
576 jit_unget_reg(regno);
577 }
578 jit_dec_synth();
579}
580
581void
582_jit_getarg_f(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
583{
584 assert(v->code == jit_code_arg_f);
585 jit_inc_synth_wp(getarg_f, u, v);
586 if (jit_arg_f_reg_p(v->u.w))
587 jit_movr_f(u, _F0 - v->u.w);
588 else
589 jit_ldxi_f(u, JIT_FP,
590 v->u.w
591#if __WORDSIZE == 64
592 + (__WORDSIZE >> 3) - sizeof(jit_float32_t)
593#endif
594 );
595 jit_dec_synth();
596}
597
598void
599_jit_putargr_f(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
600{
601 assert(v->code == jit_code_arg_f);
602 jit_inc_synth_wp(putargr_f, u, v);
603 if (jit_arg_f_reg_p(v->u.w))
604 jit_movr_f(_F0 - v->u.w, u);
605 else
606 jit_stxi_f(v->u.w
607#if __WORDSIZE == 64
608 + (__WORDSIZE >> 3) - sizeof(jit_float32_t)
609#endif
610 , JIT_FP, u);
611 jit_dec_synth();
612}
613
614void
615_jit_putargi_f(jit_state_t *_jit, jit_float32_t u, jit_node_t *v)
616{
617 jit_int32_t regno;
618 assert(v->code == jit_code_arg_f);
619 jit_inc_synth_fp(putargi_f, u, v);
620 if (jit_arg_f_reg_p(v->u.w))
621 jit_movi_f(_F0 - v->u.w, u);
622 else {
623 regno = jit_get_reg(jit_class_fpr);
624 jit_movi_f(regno, u);
625 jit_stxi_f(v->u.w
626#if __WORDSIZE == 64
627 + (__WORDSIZE >> 3) - sizeof(jit_float32_t)
628#endif
629 , JIT_FP, regno);
630 jit_unget_reg(regno);
631 }
632 jit_dec_synth();
633}
634
635void
636_jit_getarg_d(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
637{
638 assert(v->code == jit_code_arg_d);
639 jit_inc_synth_wp(getarg_d, u, v);
640 if (jit_arg_f_reg_p(v->u.w))
641 jit_movr_d(u, _F0 - v->u.w);
642 else
643 jit_ldxi_d(u, JIT_FP, v->u.w);
644 jit_dec_synth();
645}
646
647void
648_jit_putargr_d(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
649{
650 assert(v->code == jit_code_arg_d);
651 jit_inc_synth_wp(putargr_d, u, v);
652 if (jit_arg_f_reg_p(v->u.w))
653 jit_movr_d(_F0 - v->u.w, u);
654 else
655 jit_stxi_d(v->u.w, JIT_FP, u);
656 jit_dec_synth();
657}
658
659void
660_jit_putargi_d(jit_state_t *_jit, jit_float64_t u, jit_node_t *v)
661{
662 jit_int32_t regno;
663 assert(v->code == jit_code_arg_d);
664 jit_inc_synth_dp(putargi_d, u, v);
665 if (jit_arg_f_reg_p(v->u.w))
666 jit_movi_d(_F0 - v->u.w, u);
667 else {
668 regno = jit_get_reg(jit_class_fpr);
669 jit_movi_d(regno, u);
670 jit_stxi_d(v->u.w, JIT_FP, regno);
671 jit_unget_reg(regno);
672 }
673 jit_dec_synth();
674}
675
676void
79bfeef6 677_jit_pushargr(jit_state_t *_jit, jit_int32_t u, jit_code_t code)
4a71579b
PC
678{
679 assert(_jitc->function);
79bfeef6 680 jit_code_inc_synth_w(code, u);
4a71579b
PC
681 jit_link_prepare();
682 if (jit_arg_reg_p(_jitc->function->call.argi)) {
683 jit_movr(_R2 - _jitc->function->call.argi, u);
684 ++_jitc->function->call.argi;
685 }
686 else {
687 jit_stxi(_jitc->function->call.size + stack_framesize, JIT_SP, u);
688 _jitc->function->call.size += sizeof(jit_word_t);
689 }
690 jit_dec_synth();
691}
692
693void
79bfeef6 694_jit_pushargi(jit_state_t *_jit, jit_word_t u, jit_code_t code)
4a71579b
PC
695{
696 jit_int32_t regno;
697 assert(_jitc->function);
79bfeef6 698 jit_code_inc_synth_w(code, u);
4a71579b
PC
699 jit_link_prepare();
700 if (jit_arg_reg_p(_jitc->function->call.argi)) {
701 jit_movi(_R2 - _jitc->function->call.argi, u);
702 ++_jitc->function->call.argi;
703 }
704 else {
705 regno = jit_get_reg(jit_class_gpr);
706 jit_movi(regno, u);
707 jit_stxi(_jitc->function->call.size + stack_framesize, JIT_SP, regno);
708 jit_unget_reg(regno);
709 _jitc->function->call.size += sizeof(jit_word_t);
710 }
711 jit_dec_synth();
712}
713
714void
715_jit_pushargr_f(jit_state_t *_jit, jit_int32_t u)
716{
717 assert(_jitc->function);
718 jit_inc_synth_w(pushargr_f, u);
719 jit_link_prepare();
720 if (jit_arg_f_reg_p(_jitc->function->call.argf)) {
721 jit_movr_f(_F0 - _jitc->function->call.argf, u);
722 ++_jitc->function->call.argf;
723 }
724 else {
725 jit_stxi_f(_jitc->function->call.size + stack_framesize
726#if __WORDSIZE == 64
727 + (__WORDSIZE >> 3) - sizeof(jit_float32_t)
728#endif
729 , JIT_SP, u);
730 _jitc->function->call.size += sizeof(jit_word_t);
731 }
732 jit_dec_synth();
733}
734
735void
736_jit_pushargi_f(jit_state_t *_jit, jit_float32_t u)
737{
738 jit_int32_t regno;
739 assert(_jitc->function);
740 jit_inc_synth_f(pushargi_f, u);
741 jit_link_prepare();
742 if (jit_arg_f_reg_p(_jitc->function->call.argf)) {
743 jit_movi_f(_F0 - _jitc->function->call.argf, u);
744 ++_jitc->function->call.argf;
745 }
746 else {
747 regno = jit_get_reg(jit_class_fpr);
748 jit_movi_f(regno, u);
749 jit_stxi_f(_jitc->function->call.size + stack_framesize
750#if __WORDSIZE == 64
751 + (__WORDSIZE >> 3) - sizeof(jit_float32_t)
752#endif
753 , JIT_SP, regno);
754 jit_unget_reg(regno);
755 _jitc->function->call.size += sizeof(jit_word_t);
756 }
757 jit_dec_synth();
758}
759
760void
761_jit_pushargr_d(jit_state_t *_jit, jit_int32_t u)
762{
763 assert(_jitc->function);
764 jit_inc_synth_w(pushargr_d, u);
765 jit_link_prepare();
766 if (jit_arg_f_reg_p(_jitc->function->call.argf)) {
767 jit_movr_d(_F0 - _jitc->function->call.argf, u);
768 ++_jitc->function->call.argf;
769 }
770 else {
771 jit_stxi_d(_jitc->function->call.size + stack_framesize, JIT_SP, u);
772 _jitc->function->call.size += sizeof(jit_float64_t);
773 }
774 jit_dec_synth();
775}
776
777void
778_jit_pushargi_d(jit_state_t *_jit, jit_float64_t u)
779{
780 jit_int32_t regno;
781 assert(_jitc->function);
782 jit_inc_synth_d(pushargi_d, u);
783 jit_link_prepare();
784 if (jit_arg_f_reg_p(_jitc->function->call.argf)) {
785 jit_movi_d(_F0 - _jitc->function->call.argf, u);
786 ++_jitc->function->call.argf;
787 }
788 else {
789 regno = jit_get_reg(jit_class_fpr);
790 jit_movi_d(regno, u);
791 jit_stxi_d(_jitc->function->call.size + stack_framesize, JIT_SP, regno);
792 jit_unget_reg(regno);
793 _jitc->function->call.size += sizeof(jit_float64_t);
794 }
795 jit_dec_synth();
796}
797
798jit_bool_t
799_jit_regarg_p(jit_state_t *_jit, jit_node_t *node, jit_int32_t regno)
800{
801 jit_int32_t spec;
802 spec = jit_class(_rvs[regno].spec);
803 if (spec & jit_class_arg) {
804 regno = _R2 - regno;
805 if (regno >= 0 && regno < node->v.w)
806 return (1);
807 if (spec & jit_class_fpr) {
808 regno = _F0 - regno;
809 if (regno >= 0 && regno < node->w.w)
810 return (1);
811 }
812 }
813 return (0);
814}
815
816void
817_jit_finishr(jit_state_t *_jit, jit_int32_t r0)
818{
819 jit_node_t *call;
820 assert(_jitc->function);
821 jit_inc_synth_w(finishr, r0);
822 if (_jitc->function->self.alen < _jitc->function->call.size)
823 _jitc->function->self.alen = _jitc->function->call.size;
824 call = jit_callr(r0);
825 call->v.w = _jitc->function->call.argi;
826 call->w.w = _jitc->function->call.argf;
827 _jitc->function->call.argi = _jitc->function->call.argf =
828 _jitc->function->call.size = 0;
829 _jitc->prepare = 0;
830 jit_dec_synth();
831}
832
833jit_node_t *
834_jit_finishi(jit_state_t *_jit, jit_pointer_t i0)
835{
836 jit_node_t *node;
837 assert(_jitc->function);
838 jit_inc_synth_w(finishi, (jit_word_t)i0);
839 if (_jitc->function->self.alen < _jitc->function->call.size)
840 _jitc->function->self.alen = _jitc->function->call.size;
841 node = jit_calli(i0);
842 node->v.w = _jitc->function->call.argi;
843 node->w.w = _jitc->function->call.argf;
844 _jitc->function->call.argi = _jitc->function->call.argf =
845 _jitc->function->call.size = 0;
846 _jitc->prepare = 0;
847 jit_dec_synth();
848 return (node);
849}
850
851void
852_jit_retval_c(jit_state_t *_jit, jit_int32_t r0)
853{
854 jit_inc_synth_w(retval_c, r0);
855 jit_extr_c(r0, JIT_RET);
856 jit_dec_synth();
857}
858
859void
860_jit_retval_uc(jit_state_t *_jit, jit_int32_t r0)
861{
862 jit_inc_synth_w(retval_uc, r0);
863 jit_extr_uc(r0, JIT_RET);
864 jit_dec_synth();
865}
866
867void
868_jit_retval_s(jit_state_t *_jit, jit_int32_t r0)
869{
870 jit_inc_synth_w(retval_s, r0);
871 jit_extr_s(r0, JIT_RET);
872 jit_dec_synth();
873}
874
875void
876_jit_retval_us(jit_state_t *_jit, jit_int32_t r0)
877{
878 jit_inc_synth_w(retval_us, r0);
879 jit_extr_us(r0, JIT_RET);
880 jit_dec_synth();
881}
882
883void
884_jit_retval_i(jit_state_t *_jit, jit_int32_t r0)
885{
886 jit_inc_synth_w(retval_i, r0);
887#if __WORDSIZE == 64
888 jit_extr_i(r0, JIT_RET);
889#else
890 jit_movr(r0, JIT_RET);
891#endif
892 jit_dec_synth();
893}
894
895#if __WORDSIZE == 64
896void
897_jit_retval_ui(jit_state_t *_jit, jit_int32_t r0)
898{
899 jit_inc_synth_w(retval_ui, r0);
900 jit_extr_ui(r0, JIT_RET);
901 jit_dec_synth();
902}
903
904void
905_jit_retval_l(jit_state_t *_jit, jit_int32_t r0)
906{
907 jit_inc_synth_w(retval_l, r0);
908 jit_movr(r0, JIT_RET);
909 jit_dec_synth();
910}
911#endif
912
913void
914_jit_retval_f(jit_state_t *_jit, jit_int32_t r0)
915{
916 jit_inc_synth_w(retval_f, r0);
917 jit_movr_f(r0, JIT_FRET);
918 jit_dec_synth();
919}
920
921void
922_jit_retval_d(jit_state_t *_jit, jit_int32_t r0)
923{
924 jit_inc_synth_w(retval_d, r0);
925 jit_movr_d(r0, JIT_FRET);
926 jit_dec_synth();
927}
928
929jit_pointer_t
930_emit_code(jit_state_t *_jit)
931{
932 jit_node_t *node;
933 jit_node_t *temp;
934 jit_word_t word;
935 jit_int32_t value;
936 jit_int32_t offset;
937 struct {
938 jit_node_t *node;
939 jit_word_t word;
79bfeef6 940 jit_function_t func;
4a71579b
PC
941#if DEVEL_DISASSEMBLER
942 jit_word_t prevw;
943#endif
944 jit_int32_t patch_offset;
945 } undo;
946#if DEVEL_DISASSEMBLER
947 jit_word_t prevw;
948#endif
949
950 _jitc->function = NULL;
951
952 jit_reglive_setup();
953
954 undo.word = 0;
955 undo.node = NULL;
956 undo.patch_offset = 0;
957
958#define assert_data(node) /**/
959#define case_rr(name, type) \
960 case jit_code_##name##r##type: \
961 name##r##type(rn(node->u.w), rn(node->v.w)); \
962 break
963#define case_rw(name, type) \
964 case jit_code_##name##i##type: \
965 name##i##type(rn(node->u.w), node->v.w); \
966 break
967#define case_wr(name, type) \
968 case jit_code_##name##i##type: \
969 name##i##type(node->u.w, rn(node->v.w)); \
970 break
971#define case_rrr(name, type) \
972 case jit_code_##name##r##type: \
973 name##r##type(rn(node->u.w), \
974 rn(node->v.w), rn(node->w.w)); \
975 break
976#define case_rrrr(name, type) \
977 case jit_code_##name##r##type: \
978 name##r##type(rn(node->u.q.l), rn(node->u.q.h), \
979 rn(node->v.w), rn(node->w.w)); \
980 break
ba86ff93
PC
981#define case_rqr(name, type) \
982 case jit_code_##name##r##type: \
983 name##r##type(rn(node->u.w), rn(node->v.q.l), \
984 rn(node->v.q.h), rn(node->w.w)); \
985 case jit_code_##name##i##type: \
986 break;
4a71579b
PC
987#define case_rrw(name, type) \
988 case jit_code_##name##i##type: \
989 name##i##type(rn(node->u.w), rn(node->v.w), node->w.w); \
990 break
991#define case_rrrw(name, type) \
992 case jit_code_##name##i##type: \
993 name##i##type(rn(node->u.q.l), rn(node->u.q.h), \
994 rn(node->v.w), node->w.w); \
995 break
996#define case_rrf(name) \
997 case jit_code_##name##i_f: \
998 assert_data(node); \
999 name##i_f(rn(node->u.w), rn(node->v.w), \
1000 (jit_float32_t *)node->w.n->u.w); \
1001 break
1002#define case_rrd(name) \
1003 case jit_code_##name##i_d: \
1004 assert_data(node); \
1005 name##i_d(rn(node->u.w), rn(node->v.w), \
1006 (jit_float64_t *)node->w.n->u.w); \
1007 break
1008#define case_wrr(name, type) \
1009 case jit_code_##name##i##type: \
1010 name##i##type(node->u.w, rn(node->v.w), rn(node->w.w)); \
1011 break
1012#define case_brr(name, type) \
1013 case jit_code_##name##r##type: \
1014 temp = node->u.n; \
1015 assert(temp->code == jit_code_label || \
1016 temp->code == jit_code_epilog); \
1017 if (temp->flag & jit_flag_patch) \
1018 name##r##type(temp->u.w, rn(node->v.w), \
1019 rn(node->w.w)); \
1020 else { \
1021 word = name##r##type##_p(_jit->pc.w, \
1022 rn(node->v.w), \
1023 rn(node->w.w)); \
1024 patch(word, node); \
1025 } \
1026 break
1027#define case_brw(name, type) \
1028 case jit_code_##name##i##type: \
1029 temp = node->u.n; \
1030 assert(temp->code == jit_code_label || \
1031 temp->code == jit_code_epilog); \
1032 if (temp->flag & jit_flag_patch) \
1033 name##i##type(temp->u.w, \
1034 rn(node->v.w), node->w.w); \
1035 else { \
1036 word = name##i##type##_p(_jit->pc.w, \
1037 rn(node->v.w), node->w.w); \
1038 patch(word, node); \
1039 } \
1040 break;
1041#define case_brf(name) \
1042 case jit_code_##name##i_f: \
1043 temp = node->u.n; \
1044 assert(temp->code == jit_code_label || \
1045 temp->code == jit_code_epilog); \
1046 if (temp->flag & jit_flag_patch) \
1047 name##i_f(temp->u.w, rn(node->v.w), \
1048 (jit_float32_t *)node->w.n->u.w); \
1049 else { \
1050 word = name##i_f_p(_jit->pc.w, rn(node->v.w), \
1051 (jit_float32_t *)node->w.n->u.w);\
1052 patch(word, node); \
1053 } \
1054 break
1055#define case_brd(name) \
1056 case jit_code_##name##i_d: \
1057 temp = node->u.n; \
1058 assert(temp->code == jit_code_label || \
1059 temp->code == jit_code_epilog); \
1060 if (temp->flag & jit_flag_patch) \
1061 name##i_d(temp->u.w, rn(node->v.w), \
1062 (jit_float64_t *)node->w.n->u.w); \
1063 else { \
1064 word = name##i_d_p(_jit->pc.w, rn(node->v.w), \
1065 (jit_float64_t *)node->w.n->u.w);\
1066 patch(word, node); \
1067 } \
1068 break
1069#if DEVEL_DISASSEMBLER
1070 prevw = _jit->pc.w;
1071#endif
1072 for (node = _jitc->head; node; node = node->next) {
1073 if (_jit->pc.uc >= _jitc->code.end)
1074 return (NULL);
1075
1076#if DEVEL_DISASSEMBLER
1077 node->offset = (jit_uword_t)_jit->pc.w - (jit_uword_t)prevw;
1078 prevw = _jit->pc.w;
1079#endif
1080 value = jit_classify(node->code);
1081 jit_regarg_set(node, value);
1082 switch (node->code) {
1083 case jit_code_align:
c0c16242
PC
1084 /* Must align to a power of two */
1085 assert(!(node->u.w & (node->u.w - 1)));
1086 if ((word = _jit->pc.w & (node->u.w - 1)))
1087 nop(node->u.w - word);
4a71579b 1088 break;
79bfeef6
PC
1089 case jit_code_skip:
1090 nop((node->u.w + 1) & ~1);
1091 break;
4a71579b
PC
1092 case jit_code_note: case jit_code_name:
1093 node->u.w = _jit->pc.w;
1094 break;
1095 case jit_code_label:
1096 if ((node->link || (node->flag & jit_flag_use)) &&
1097 (word = _jit->pc.w & 3))
1098 nop(4 - word);
1099 /* remember label is defined */
1100 node->flag |= jit_flag_patch;
1101 node->u.w = _jit->pc.w;
1102 break;
1103 case_rrr(add,);
1104 case_rrw(add,);
1105 case_rrr(addc,);
1106 case_rrw(addc,);
1107 case_rrr(addx,);
1108 case_rrw(addx,);
1109 case_rrr(sub,);
1110 case_rrw(sub,);
1111 case_rrr(subc,);
1112 case_rrw(subc,);
1113 case_rrr(subx,);
1114 case_rrw(subx,);
1115 case_rrw(rsb,);
1116 case_rrr(mul,);
1117 case_rrw(mul,);
ba86ff93
PC
1118 case_rrr(hmul,);
1119 case_rrw(hmul,);
1120 case_rrr(hmul, _u);
1121 case_rrw(hmul, _u);
4a71579b
PC
1122 case_rrrr(qmul,);
1123 case_rrrw(qmul,);
1124 case_rrrr(qmul, _u);
1125 case_rrrw(qmul, _u);
1126 case_rrr(div,);
1127 case_rrw(div,);
1128 case_rrr(div, _u);
1129 case_rrw(div, _u);
1130 case_rrr(rem,);
1131 case_rrw(rem,);
1132 case_rrr(rem, _u);
1133 case_rrw(rem, _u);
1134 case_rrrr(qdiv,);
1135 case_rrrw(qdiv,);
1136 case_rrrr(qdiv, _u);
1137 case_rrrw(qdiv, _u);
1138 case_rrr(lsh,);
1139 case_rrw(lsh,);
ba86ff93
PC
1140#define qlshr(r0, r1, r2, r3) fallback_qlshr(r0, r1, r2, r3)
1141#define qlshi(r0, r1, r2, i0) fallback_qlshi(r0, r1, r2, i0)
1142#define qlshr_u(r0, r1, r2, r3) fallback_qlshr_u(r0, r1, r2, r3)
1143#define qlshi_u(r0, r1, r2, i0) fallback_qlshi_u(r0, r1, r2, i0)
1144 case_rrrr(qlsh,);
1145 case_rrrw(qlsh,);
1146 case_rrrr(qlsh, _u);
1147 case_rrrw(qlsh, _u);
4a71579b
PC
1148 case_rrr(rsh,);
1149 case_rrw(rsh,);
1150 case_rrr(rsh, _u);
1151 case_rrw(rsh, _u);
ba86ff93
PC
1152#define qrshr(r0, r1, r2, r3) fallback_qrshr(r0, r1, r2, r3)
1153#define qrshi(r0, r1, r2, i0) fallback_qrshi(r0, r1, r2, i0)
1154#define qrshr_u(r0, r1, r2, r3) fallback_qrshr_u(r0, r1, r2, r3)
1155#define qrshi_u(r0, r1, r2, i0) fallback_qrshi_u(r0, r1, r2, i0)
1156 case_rrrr(qrsh,);
1157 case_rrrw(qrsh,);
1158 case_rrrr(qrsh, _u);
1159 case_rrrw(qrsh, _u);
1160 case_rrr(lrot,);
1161 case_rrw(lrot,);
1162 case_rrr(rrot,);
1163 case_rrw(rrot,);
4a71579b
PC
1164 case_rr(neg,);
1165 case_rr(com,);
79bfeef6
PC
1166 case_rr(clo,);
1167 case_rr(clz,);
1168 case_rr(cto,);
1169 case_rr(ctz,);
ba86ff93
PC
1170#define rbitr(r0, r1) fallback_rbit(r0, r1)
1171#define popcntr(r0, r1) fallback_popcnt(r0, r1)
1172 case_rr(rbit,);
1173 case_rr(popcnt,);
4a71579b
PC
1174 case_rrr(and,);
1175 case_rrw(and,);
1176 case_rrr(or,);
1177 case_rrw(or,);
1178 case_rrr(xor,);
1179 case_rrw(xor,);
1180 case_rr(trunc, _f_i);
1181 case_rr(trunc, _d_i);
1182#if __WORDSIZE == 64
1183 case_rr(trunc, _f_l);
1184 case_rr(trunc, _d_l);
1185#endif
1186 case_rr(ld, _c);
1187 case_rw(ld, _c);
1188 case_rr(ld, _uc);
1189 case_rw(ld, _uc);
1190 case_rr(ld, _s);
1191 case_rw(ld, _s);
1192 case_rr(ld, _us);
1193 case_rw(ld, _us);
1194 case_rr(ld, _i);
1195 case_rw(ld, _i);
1196#if __WORDSIZE == 64
1197 case_rr(ld, _ui);
1198 case_rw(ld, _ui);
1199 case_rr(ld, _l);
1200 case_rw(ld, _l);
1201#endif
1202 case_rrr(ldx, _c);
1203 case_rrw(ldx, _c);
1204 case_rrr(ldx, _uc);
1205 case_rrw(ldx, _uc);
1206 case_rrr(ldx, _s);
1207 case_rrw(ldx, _s);
1208 case_rrr(ldx, _us);
1209 case_rrw(ldx, _us);
1210 case_rrr(ldx, _i);
1211 case_rrw(ldx, _i);
1212#if __WORDSIZE == 64
1213 case_rrr(ldx, _ui);
1214 case_rrw(ldx, _ui);
1215 case_rrr(ldx, _l);
1216 case_rrw(ldx, _l);
1217#endif
ba86ff93
PC
1218#define unldr(r0, r1, i0) fallback_unldr(r0, r1, i0)
1219 case jit_code_unldr:
1220 unldr(rn(node->u.w), rn(node->v.w), node->w.w);
1221 break;
1222#define unldi(r0, i0, i1) fallback_unldi(r0, i0, i1)
1223 case jit_code_unldi:
1224 unldi(rn(node->u.w), node->v.w, node->w.w);
1225 break;
1226#define unldr_u(r0, r1, i0) fallback_unldr_u(r0, r1, i0)
1227 case jit_code_unldr_u:
1228 unldr_u(rn(node->u.w), rn(node->v.w), node->w.w);
1229 break;
1230#define unldi_u(r0, i0, i1) fallback_unldi_u(r0, i0, i1)
1231 case jit_code_unldi_u:
1232 unldi_u(rn(node->u.w), node->v.w, node->w.w);
1233 break;
4a71579b
PC
1234 case_rr(st, _c);
1235 case_wr(st, _c);
1236 case_rr(st, _s);
1237 case_wr(st, _s);
1238 case_rr(st, _i);
1239 case_wr(st, _i);
1240#if __WORDSIZE == 64
1241 case_rr(st, _l);
1242 case_wr(st, _l);
1243#endif
1244 case_rrr(stx, _c);
1245 case_wrr(stx, _c);
1246 case_rrr(stx, _s);
1247 case_wrr(stx, _s);
1248 case_rrr(stx, _i);
1249 case_wrr(stx, _i);
1250#if __WORDSIZE == 64
1251 case_rrr(stx, _l);
1252 case_wrr(stx, _l);
1253#endif
ba86ff93
PC
1254#define unstr(r0, r1, i0) fallback_unstr(r0, r1, i0)
1255 case jit_code_unstr:
1256 unstr(rn(node->u.w), rn(node->v.w), node->w.w);
1257 break;
1258#define unsti(i0, r0, i1) fallback_unsti(i0, r0, i1)
1259 case jit_code_unsti:
1260 unsti(node->u.w, rn(node->v.w), node->w.w);
1261 break;
4a71579b
PC
1262 case_rr(hton, _us);
1263 case_rr(hton, _ui);
1264#if __WORDSIZE == 64
1265 case_rr(hton, _ul);
40a44dcb
PC
1266#endif
1267 case_rr(bswap, _us);
1268 case_rr(bswap, _ui);
1269#if __WORDSIZE == 64
1270 case_rr(bswap, _ul);
4a71579b 1271#endif
ba86ff93
PC
1272 case jit_code_extr:
1273 extr(rn(node->u.w), rn(node->v.w), node->w.q.l, node->w.q.h);
1274 break;
1275 case jit_code_extr_u:
1276 extr_u(rn(node->u.w), rn(node->v.w), node->w.q.l, node->w.q.h);
1277 break;
1278 case jit_code_depr:
1279 depr(rn(node->u.w), rn(node->v.w), node->w.q.l, node->w.q.h);
1280 break;
1281 case jit_code_depi:
1282 depi(rn(node->u.w), node->v.w, node->w.q.l, node->w.q.h);
1283 break;
4a71579b
PC
1284 case_rr(ext, _c);
1285 case_rr(ext, _uc);
1286 case_rr(ext, _s);
1287 case_rr(ext, _us);
1288#if __WORDSIZE == 64
1289 case_rr(ext, _i);
1290 case_rr(ext, _ui);
1291#endif
ba3814c1
PC
1292 case jit_code_casr:
1293 casr(rn(node->u.w), rn(node->v.w),
1294 rn(node->w.q.l), rn(node->w.q.h));
1295 break;
1296 case jit_code_casi:
1297 casi(rn(node->u.w), node->v.w,
1298 rn(node->w.q.l), rn(node->w.q.h));
1299 break;
40a44dcb
PC
1300 case_rrr(movn,);
1301 case_rrr(movz,);
4a71579b
PC
1302 case_rr(mov,);
1303 case jit_code_movi:
1304 if (node->flag & jit_flag_node) {
1305 temp = node->v.n;
1306 if (temp->code == jit_code_data ||
1307 (temp->code == jit_code_label &&
1308 (temp->flag & jit_flag_patch)))
1309 movi(rn(node->u.w), temp->u.w);
1310 else {
1311 assert(temp->code == jit_code_label ||
1312 temp->code == jit_code_epilog);
1313 word = movi_p(rn(node->u.w), temp->u.w);
1314 patch(word, node);
1315 }
1316 }
1317 else
1318 movi(rn(node->u.w), node->v.w);
1319 break;
1320 case_rrr(lt,);
1321 case_rrw(lt,);
1322 case_rrr(lt, _u);
1323 case_rrw(lt, _u);
1324 case_rrr(le,);
1325 case_rrw(le,);
1326 case_rrr(le, _u);
1327 case_rrw(le, _u);
1328 case_rrr(eq,);
1329 case_rrw(eq,);
1330 case_rrr(ge,);
1331 case_rrw(ge,);
1332 case_rrr(ge, _u);
1333 case_rrw(ge, _u);
1334 case_rrr(gt,);
1335 case_rrw(gt,);
1336 case_rrr(gt, _u);
1337 case_rrw(gt, _u);
1338 case_rrr(ne,);
1339 case_rrw(ne,);
1340 case_brr(blt,);
1341 case_brw(blt,);
1342 case_brr(blt, _u);
1343 case_brw(blt, _u);
1344 case_brr(ble,);
1345 case_brw(ble,);
1346 case_brr(ble, _u);
1347 case_brw(ble, _u);
1348 case_brr(beq,);
1349 case_brw(beq,);
1350 case_brr(bge,);
1351 case_brw(bge,);
1352 case_brr(bge, _u);
1353 case_brw(bge, _u);
1354 case_brr(bgt,);
1355 case_brw(bgt,);
1356 case_brr(bgt, _u);
1357 case_brw(bgt, _u);
1358 case_brr(bne,);
1359 case_brw(bne,);
1360 case_brr(boadd,);
1361 case_brw(boadd,);
1362 case_brr(boadd, _u);
1363 case_brw(boadd, _u);
1364 case_brr(bxadd,);
1365 case_brw(bxadd,);
1366 case_brr(bxadd, _u);
1367 case_brw(bxadd, _u);
1368 case_brr(bosub,);
1369 case_brw(bosub,);
1370 case_brr(bosub, _u);
1371 case_brw(bosub, _u);
1372 case_brr(bxsub,);
1373 case_brw(bxsub,);
1374 case_brr(bxsub, _u);
1375 case_brw(bxsub, _u);
1376 case_brr(bms,);
1377 case_brw(bms,);
1378 case_brr(bmc,);
1379 case_brw(bmc,);
1380 case_rrr(add, _f);
1381 case_rrf(add);
1382 case_rrr(sub, _f);
1383 case_rrf(sub);
1384 case_rrf(rsb);
1385 case_rrr(mul, _f);
1386 case_rrf(mul);
1387 case_rrr(div, _f);
1388 case_rrf(div);
1389 case_rr(abs, _f);
1390 case_rr(neg, _f);
1391 case_rr(sqrt, _f);
ba86ff93
PC
1392 case_rqr(fma, _f);
1393 case_rqr(fms, _f);
1394 case_rqr(fnma, _f);
1395 case_rqr(fnms, _f);
4a71579b
PC
1396 case_rr(ext, _f);
1397 case_rr(ld, _f);
1398 case_rw(ld, _f);
1399 case_rrr(ldx, _f);
1400 case_rrw(ldx, _f);
ba86ff93
PC
1401#define unldr_x(r0, r1, i0) fallback_unldr_x(r0, r1, i0)
1402 case jit_code_unldr_x:
1403 unldr_x(rn(node->u.w), rn(node->v.w), node->w.w);
1404 break;
1405#define unldi_x(r0, i0, i1) fallback_unldi_x(r0, i0, i1)
1406 case jit_code_unldi_x:
1407 unldi_x(rn(node->u.w), node->v.w, node->w.w);
1408 break;
4a71579b
PC
1409 case_rr(st, _f);
1410 case_wr(st, _f);
1411 case_rrr(stx, _f);
1412 case_wrr(stx, _f);
ba86ff93
PC
1413#define unstr_x(r0, r1, i0) fallback_unstr_x(r0, r1, i0)
1414 case jit_code_unstr_x:
1415 unstr_x(rn(node->u.w), rn(node->v.w), node->w.w);
1416 break;
1417#define unsti_x(i0, r0, i1) fallback_unsti_x(i0, r0, i1)
1418 case jit_code_unsti_x:
1419 unsti_x(node->u.w, rn(node->v.w), node->w.w);
1420 break;
4a71579b
PC
1421 case_rr(mov, _f);
1422 case jit_code_movi_f:
1423 assert_data(node);
1424 movi_f(rn(node->u.w), (jit_float32_t *)node->v.n->u.w);
1425 break;
1426 case_rr(ext, _d_f);
1427 case_rrr(lt, _f);
1428 case_rrf(lt);
1429 case_rrr(le, _f);
1430 case_rrf(le);
1431 case_rrr(eq, _f);
1432 case_rrf(eq);
1433 case_rrr(ge, _f);
1434 case_rrf(ge);
1435 case_rrr(gt, _f);
1436 case_rrf(gt);
1437 case_rrr(ne, _f);
1438 case_rrf(ne);
1439 case_rrr(unlt, _f);
1440 case_rrf(unlt);
1441 case_rrr(unle, _f);
1442 case_rrf(unle);
1443 case_rrr(uneq, _f);
1444 case_rrf(uneq);
1445 case_rrr(unge, _f);
1446 case_rrf(unge);
1447 case_rrr(ungt, _f);
1448 case_rrf(ungt);
1449 case_rrr(ltgt, _f);
1450 case_rrf(ltgt);
1451 case_rrr(ord, _f);
1452 case_rrf(ord);
1453 case_rrr(unord, _f);
1454 case_rrf(unord);
1455 case_brr(blt, _f);
1456 case_brf(blt);
1457 case_brr(ble, _f);
1458 case_brf(ble);
1459 case_brr(beq, _f);
1460 case_brf(beq);
1461 case_brr(bge, _f);
1462 case_brf(bge);
1463 case_brr(bgt, _f);
1464 case_brf(bgt);
1465 case_brr(bne, _f);
1466 case_brf(bne);
1467 case_brr(bunlt, _f);
1468 case_brf(bunlt);
1469 case_brr(bunle, _f);
1470 case_brf(bunle);
1471 case_brr(buneq, _f);
1472 case_brf(buneq);
1473 case_brr(bunge, _f);
1474 case_brf(bunge);
1475 case_brr(bungt, _f);
1476 case_brf(bungt);
1477 case_brr(bltgt, _f);
1478 case_brf(bltgt);
1479 case_brr(bord, _f);
1480 case_brf(bord);
1481 case_brr(bunord, _f);
1482 case_brf(bunord);
1483 case_rrr(add, _d);
1484 case_rrd(add);
1485 case_rrr(sub, _d);
1486 case_rrd(sub);
1487 case_rrd(rsb);
1488 case_rrr(mul, _d);
1489 case_rrd(mul);
1490 case_rrr(div, _d);
1491 case_rrd(div);
1492 case_rr(abs, _d);
1493 case_rr(neg, _d);
1494 case_rr(sqrt, _d);
ba86ff93
PC
1495 case_rqr(fma, _d);
1496 case_rqr(fms, _d);
1497 case_rqr(fnma, _d);
1498 case_rqr(fnms, _d);
4a71579b
PC
1499 case_rr(ext, _d);
1500 case_rr(ld, _d);
1501 case_rw(ld, _d);
1502 case_rrr(ldx, _d);
1503 case_rrw(ldx, _d);
1504 case_rr(st, _d);
1505 case_wr(st, _d);
1506 case_rrr(stx, _d);
1507 case_wrr(stx, _d);
1508 case_rr(mov, _d);
1509 case jit_code_movi_d:
1510 assert_data(node);
1511 movi_d(rn(node->u.w), (jit_float64_t *)node->v.n->u.w);
1512 break;
1513 case_rr(ext, _f_d);
1514 case_rrr(lt, _d);
1515 case_rrd(lt);
1516 case_rrr(le, _d);
1517 case_rrd(le);
1518 case_rrr(eq, _d);
1519 case_rrd(eq);
1520 case_rrr(ge, _d);
1521 case_rrd(ge);
1522 case_rrr(gt, _d);
1523 case_rrd(gt);
1524 case_rrr(ne, _d);
1525 case_rrd(ne);
1526 case_rrr(unlt, _d);
1527 case_rrd(unlt);
1528 case_rrr(unle, _d);
1529 case_rrd(unle);
1530 case_rrr(uneq, _d);
1531 case_rrd(uneq);
1532 case_rrr(unge, _d);
1533 case_rrd(unge);
1534 case_rrr(ungt, _d);
1535 case_rrd(ungt);
1536 case_rrr(ltgt, _d);
1537 case_rrd(ltgt);
1538 case_rrr(ord, _d);
1539 case_rrd(ord);
1540 case_rrr(unord, _d);
1541 case_rrd(unord);
1542 case_brr(blt, _d);
1543 case_brd(blt);
1544 case_brr(ble, _d);
1545 case_brd(ble);
1546 case_brr(beq, _d);
1547 case_brd(beq);
1548 case_brr(bge, _d);
1549 case_brd(bge);
1550 case_brr(bgt, _d);
1551 case_brd(bgt);
1552 case_brr(bne, _d);
1553 case_brd(bne);
1554 case_brr(bunlt, _d);
1555 case_brd(bunlt);
1556 case_brr(bunle, _d);
1557 case_brd(bunle);
1558 case_brr(buneq, _d);
1559 case_brd(buneq);
1560 case_brr(bunge, _d);
1561 case_brd(bunge);
1562 case_brr(bungt, _d);
1563 case_brd(bungt);
1564 case_brr(bltgt, _d);
1565 case_brd(bltgt);
1566 case_brr(bord, _d);
1567 case_brd(bord);
1568 case_brr(bunord, _d);
1569 case_brd(bunord);
1570 case jit_code_jmpr:
1571 jmpr(rn(node->u.w));
1572 break;
1573 case jit_code_jmpi:
1574 if (node->flag & jit_flag_node) {
1575 temp = node->u.n;
1576 assert(temp->code == jit_code_label ||
1577 temp->code == jit_code_epilog);
1578 if (temp->flag & jit_flag_patch)
79bfeef6 1579 jmpi(temp->u.w, 1);
4a71579b 1580 else {
79bfeef6
PC
1581 word = _jit->code.length -
1582 (_jit->pc.uc - _jit->code.ptr);
1583 if (s32_p(word)) {
1584 offset = s16_p(word);
1585 word = jmpi(_jit->pc.w, offset);
1586 }
1587 else
1588 word = jmpi_p(_jit->pc.w);
4a71579b
PC
1589 patch(word, node);
1590 }
1591 }
1592 else
79bfeef6 1593 jmpi(node->u.w, 1);
4a71579b
PC
1594 break;
1595 case jit_code_callr:
1596 callr(rn(node->u.w));
1597 break;
1598 case jit_code_calli:
1599 if (node->flag & jit_flag_node) {
1600 temp = node->u.n;
1601 assert(temp->code == jit_code_label ||
1602 temp->code == jit_code_epilog);
1603 if (temp->flag & jit_flag_patch)
79bfeef6 1604 calli(temp->u.w, 1);
4a71579b 1605 else {
79bfeef6
PC
1606 word = _jit->code.length -
1607 (_jit->pc.uc - _jit->code.ptr);
1608 if (s32_p(word)) {
1609 offset =s16_p(word);
1610 word = calli(_jit->pc.w, offset);
1611 }
1612 else
1613 word = calli_p(_jit->pc.w);
4a71579b
PC
1614 patch(word, node);
1615 }
1616 }
1617 else
79bfeef6 1618 calli(node->u.w, 1);
4a71579b
PC
1619 break;
1620 case jit_code_prolog:
1621 _jitc->function = _jitc->functions.ptr + node->w.w;
1622 undo.node = node;
1623 undo.word = _jit->pc.w;
79bfeef6 1624 memcpy(&undo.func, _jitc->function, sizeof(undo.func));
4a71579b
PC
1625#if DEVEL_DISASSEMBLER
1626 undo.prevw = prevw;
1627#endif
1628 undo.patch_offset = _jitc->patches.offset;
1629 restart_function:
1630 _jitc->again = 0;
1631 prolog(node);
1632 break;
1633 case jit_code_epilog:
1634 assert(_jitc->function == _jitc->functions.ptr + node->w.w);
1635 if (_jitc->again) {
1636 for (temp = undo.node->next;
1637 temp != node; temp = temp->next) {
1638 if (temp->code == jit_code_label ||
1639 temp->code == jit_code_epilog)
1640 temp->flag &= ~jit_flag_patch;
1641 }
1642 temp->flag &= ~jit_flag_patch;
1643 node = undo.node;
1644 _jit->pc.w = undo.word;
79bfeef6
PC
1645 /* undo.func.self.aoff and undo.func.regset should not
1646 * be undone, as they will be further updated, and are
1647 * the reason of the undo. */
1648 undo.func.self.aoff = _jitc->function->frame +
1649 _jitc->function->self.aoff;
1650 jit_regset_set(&undo.func.regset, &_jitc->function->regset);
1651 /* allocar information also does not need to be undone */
1652 undo.func.aoffoff = _jitc->function->aoffoff;
1653 undo.func.allocar = _jitc->function->allocar;
ba86ff93
PC
1654 /* cvt_offset must also not be undone */
1655 undo.func.cvt_offset = _jitc->function->cvt_offset;
79bfeef6 1656 memcpy(_jitc->function, &undo.func, sizeof(undo.func));
4a71579b
PC
1657#if DEVEL_DISASSEMBLER
1658 prevw = undo.prevw;
1659#endif
1660 _jitc->patches.offset = undo.patch_offset;
1661 goto restart_function;
1662 }
1663 if (node->link && (word = _jit->pc.w & 3))
1664 nop(4 - word);
1665 /* remember label is defined */
1666 node->flag |= jit_flag_patch;
1667 node->u.w = _jit->pc.w;
1668 epilog(node);
1669 _jitc->function = NULL;
1670 break;
ba86ff93
PC
1671 case jit_code_movr_w_f:
1672 movr_w_f(rn(node->u.w), rn(node->v.w));
1673 break;
1674 case jit_code_movr_f_w:
1675 movr_f_w(rn(node->u.w), rn(node->v.w));
1676 break;
1677 case jit_code_movi_f_w:
1678 assert(node->flag & jit_flag_data);
1679 movi_f_w(rn(node->u.w), *(jit_float32_t *)node->v.n->u.w);
1680 break;
1681 case jit_code_movi_w_f:
1682 movi_w_f(rn(node->u.w), node->v.w);
1683 break;
1684#if __WORDSIZE == 32
1685 case jit_code_movr_ww_d:
1686 movr_ww_d(rn(node->u.w), rn(node->v.w), rn(node->w.w));
1687 break;
1688 case jit_code_movr_d_ww:
1689 movr_d_ww(rn(node->u.w), rn(node->v.w), rn(node->w.w));
1690 break;
1691 case jit_code_movi_d_ww:
1692 assert(node->flag & jit_flag_data);
1693 movi_d_ww(rn(node->u.w), rn(node->v.w),
1694 *(jit_float64_t *)node->w.n->u.w);
1695 break;
1696 case jit_code_movi_ww_d:
1697 movi_ww_d(rn(node->u.w), node->v.w, node->w.w);
1698 break;
1699#else
1700 case jit_code_movr_w_d:
1701 movr_w_d(rn(node->u.w), rn(node->v.w));
1702 break;
1703 case jit_code_movr_d_w:
1704 movr_d_w(rn(node->u.w), rn(node->v.w));
1705 break;
1706 case jit_code_movi_d_w:
1707 assert(node->flag & jit_flag_data);
1708 movi_d_w(rn(node->u.w), *(jit_float64_t *)node->v.n->u.w);
1709 break;
1710 case jit_code_movi_w_d:
1711 movi_w_d(rn(node->u.w), node->v.w);
1712 break;
1713#endif
4a71579b
PC
1714 case jit_code_va_start:
1715 vastart(rn(node->u.w));
1716 break;
1717 case jit_code_va_arg:
1718 vaarg(rn(node->u.w), rn(node->v.w));
1719 break;
1720 case jit_code_va_arg_d:
1721 vaarg_d(rn(node->u.w), rn(node->v.w));
1722 break;
1723 case jit_code_live: case jit_code_ellipsis:
1724 case jit_code_va_push:
1725 case jit_code_allocai: case jit_code_allocar:
79bfeef6
PC
1726 case jit_code_arg_c: case jit_code_arg_s:
1727 case jit_code_arg_i:
1728# if __WORDSIZE == 64
1729 case jit_code_arg_l:
1730# endif
4a71579b
PC
1731 case jit_code_arg_f: case jit_code_arg_d:
1732 case jit_code_va_end:
1733 case jit_code_ret:
79bfeef6
PC
1734 case jit_code_retr_c: case jit_code_reti_c:
1735 case jit_code_retr_uc: case jit_code_reti_uc:
1736 case jit_code_retr_s: case jit_code_reti_s:
1737 case jit_code_retr_us: case jit_code_reti_us:
1738 case jit_code_retr_i: case jit_code_reti_i:
1739#if __WORDSIZE == 64
1740 case jit_code_retr_ui: case jit_code_reti_ui:
1741 case jit_code_retr_l: case jit_code_reti_l:
1742#endif
4a71579b
PC
1743 case jit_code_retr_f: case jit_code_reti_f:
1744 case jit_code_retr_d: case jit_code_reti_d:
1745 case jit_code_getarg_c: case jit_code_getarg_uc:
1746 case jit_code_getarg_s: case jit_code_getarg_us:
1747 case jit_code_getarg_i:
1748#if __WORDSIZE == 64
1749 case jit_code_getarg_ui: case jit_code_getarg_l:
1750#endif
1751 case jit_code_getarg_f: case jit_code_getarg_d:
79bfeef6
PC
1752 case jit_code_putargr_c: case jit_code_putargi_c:
1753 case jit_code_putargr_uc: case jit_code_putargi_uc:
1754 case jit_code_putargr_s: case jit_code_putargi_s:
1755 case jit_code_putargr_us: case jit_code_putargi_us:
1756 case jit_code_putargr_i: case jit_code_putargi_i:
1757#if __WORDSIZE == 64
1758 case jit_code_putargr_ui: case jit_code_putargi_ui:
1759 case jit_code_putargr_l: case jit_code_putargi_l:
1760#endif
4a71579b
PC
1761 case jit_code_putargr_f: case jit_code_putargi_f:
1762 case jit_code_putargr_d: case jit_code_putargi_d:
79bfeef6
PC
1763 case jit_code_pushargr_c: case jit_code_pushargi_c:
1764 case jit_code_pushargr_uc: case jit_code_pushargi_uc:
1765 case jit_code_pushargr_s: case jit_code_pushargi_s:
1766 case jit_code_pushargr_us: case jit_code_pushargi_us:
1767 case jit_code_pushargr_i: case jit_code_pushargi_i:
1768#if __WORDSIZE == 64
1769 case jit_code_pushargr_ui: case jit_code_pushargi_ui:
1770 case jit_code_pushargr_l: case jit_code_pushargi_l:
1771#endif
4a71579b
PC
1772 case jit_code_pushargr_f: case jit_code_pushargi_f:
1773 case jit_code_pushargr_d: case jit_code_pushargi_d:
1774 case jit_code_retval_c: case jit_code_retval_uc:
1775 case jit_code_retval_s: case jit_code_retval_us:
1776 case jit_code_retval_i:
1777#if __WORDSIZE == 64
1778 case jit_code_retval_ui: case jit_code_retval_l:
1779#endif
1780 case jit_code_retval_f: case jit_code_retval_d:
1781 case jit_code_prepare:
1782 case jit_code_finishr: case jit_code_finishi:
ba86ff93
PC
1783 case jit_code_negi_f: case jit_code_absi_f:
1784 case jit_code_sqrti_f: case jit_code_negi_d:
1785 case jit_code_absi_d: case jit_code_sqrti_d:
1786 break;
1787 case jit_code_negi:
1788 negi(rn(node->u.w), node->v.w);
1789 break;
1790 case jit_code_comi:
1791 comi(rn(node->u.w), node->v.w);
1792 break;
1793 case jit_code_exti_c:
1794 exti_c(rn(node->u.w), node->v.w);
1795 break;
1796 case jit_code_exti_uc:
1797 exti_uc(rn(node->u.w), node->v.w);
1798 break;
1799 case jit_code_exti_s:
1800 exti_s(rn(node->u.w), node->v.w);
1801 break;
1802 case jit_code_exti_us:
1803 exti_us(rn(node->u.w), node->v.w);
1804 break;
1805 case jit_code_bswapi_us:
1806 bswapi_us(rn(node->u.w), node->v.w);
1807 break;
1808 case jit_code_bswapi_ui:
1809 bswapi_ui(rn(node->u.w), node->v.w);
1810 break;
1811 case jit_code_htoni_us:
1812 htoni_us(rn(node->u.w), node->v.w);
1813 break;
1814 case jit_code_htoni_ui:
1815 htoni_ui(rn(node->u.w), node->v.w);
1816 break;
1817#if __WORDSIZE == 64
1818 case jit_code_exti_i:
1819 exti_i(rn(node->u.w), node->v.w);
1820 break;
1821 case jit_code_exti_ui:
1822 exti_ui(rn(node->u.w), node->v.w);
1823 break;
1824 case jit_code_bswapi_ul:
1825 bswapi_ul(rn(node->u.w), node->v.w);
1826 break;
1827 case jit_code_htoni_ul:
1828 htoni_ul(rn(node->u.w), node->v.w);
1829 break;
1830#endif
1831 case jit_code_cloi:
1832 cloi(rn(node->u.w), node->v.w);
1833 break;
1834 case jit_code_clzi:
1835 clzi(rn(node->u.w), node->v.w);
1836 break;
1837 case jit_code_ctoi:
1838 ctoi(rn(node->u.w), node->v.w);
1839 break;
1840 case jit_code_ctzi:
1841 ctzi(rn(node->u.w), node->v.w);
1842 break;
1843 case jit_code_rbiti:
1844 rbiti(rn(node->u.w), node->v.w);
1845 break;
1846 case jit_code_popcnti:
1847 popcnti(rn(node->u.w), node->v.w);
1848 break;
1849 case jit_code_exti:
1850 exti(rn(node->u.w), node->v.w, node->w.q.l, node->w.q.h);
1851 break;
1852 case jit_code_exti_u:
1853 exti_u(rn(node->u.w), node->v.w, node->w.q.l, node->w.q.h);
4a71579b
PC
1854 break;
1855 default:
1856 abort();
1857 }
1858 jit_regarg_clr(node, value);
1859 assert(_jitc->regarg == 0 && _jitc->synth == 0);
1860 /* update register live state */
1861 jit_reglive(node);
1862 }
1863#undef case_brw
1864#undef case_brr
1865#undef case_wrr
1866#undef case_rrw
1867#undef case_rrr
1868#undef case_wr
1869#undef case_rw
1870#undef case_rr
1871
1872 for (offset = 0; offset < _jitc->patches.offset; offset++) {
1873 node = _jitc->patches.ptr[offset].node;
1874 word = node->code == jit_code_movi ? node->v.n->u.w : node->u.n->u.w;
1875 patch_at(_jitc->patches.ptr[offset].inst, word);
1876 }
1877
1878 jit_flush(_jit->code.ptr, _jit->pc.uc);
1879
1880 return (_jit->code.ptr);
1881}
1882
1883#define CODE 1
1884# include "jit_s390-cpu.c"
1885# include "jit_s390-fpu.c"
ba86ff93 1886# include "jit_fallback.c"
4a71579b
PC
1887#undef CODE
1888
1889void
1890jit_flush(void *fptr, void *tptr)
1891{
1892#if defined(__GNUC__)
1893 jit_word_t f, t, s;
1894
1895 s = sysconf(_SC_PAGE_SIZE);
1896 f = (jit_word_t)fptr & -s;
1897 t = (((jit_word_t)tptr) + s - 1) & -s;
1898 __clear_cache((void *)f, (void *)t);
1899#endif
1900}
1901
1902void
1903_emit_ldxi(jit_state_t *_jit, jit_gpr_t r0, jit_gpr_t r1, jit_word_t i0)
1904{
1905 ldxi(rn(r0), rn(r1), i0);
1906}
1907
1908void
1909_emit_stxi(jit_state_t *_jit, jit_word_t i0, jit_gpr_t r0, jit_gpr_t r1)
1910{
1911 stxi(i0, rn(r0), rn(r1));
1912}
1913
1914void
1915_emit_ldxi_d(jit_state_t *_jit, jit_fpr_t r0, jit_gpr_t r1, jit_word_t i0)
1916{
1917 ldxi_d(rn(r0), rn(r1), i0);
1918}
1919
1920void
1921_emit_stxi_d(jit_state_t *_jit, jit_word_t i0, jit_gpr_t r0, jit_fpr_t r1)
1922{
1923 stxi_d(i0, rn(r0), rn(r1));
1924}
1925
1926static jit_int32_t
1927_jit_get_reg_pair(jit_state_t *_jit)
1928{
1929 jit_int32_t r1, r2;
1930 /* Try to find a register pair for use with operations that
1931 * require a odd based register pair. Search for the best
1932 * match to avoid spills or at least a valid operation.
1933 */
1934
1935 /* Try non callee save first */
1936 if (jit_reg_free_p(_R0) && jit_reg_free_p(_R1))
1937 r1 = _R0, r2 = _R1;
1938 else if (jit_reg_free_p(_R2) && jit_reg_free_p(_R3))
1939 r1 = _R2, r2 = _R3;
1940 else if (jit_reg_free_p(_R4) && jit_reg_free_p(_R5))
1941 r1 = _R4, r2 = _R5;
1942 /* Try callee save registers */
1943 else if (jit_reg_free_p(_R10) && jit_reg_free_p(_R11))
1944 r1 = _R10, r2 = _R11;
1945 else if (jit_reg_free_p(_R8) && jit_reg_free_p(_R9))
1946 r1 = _R8, r2 = _R9;
1947 else if (jit_reg_free_p(_R6) && jit_reg_free_p(_R7))
1948 r1 = _R6, r2 = _R7;
1949
1950 /* We *must* find a register pair */
1951 else if (jit_reg_free_if_spill_p(_R0) && jit_reg_free_if_spill_p(_R1))
1952 r1 = _R0, r2 = _R1;
1953 else if (jit_reg_free_if_spill_p(_R2) && jit_reg_free_if_spill_p(_R3))
1954 r1 = _R2, r2 = _R3;
1955 else if (jit_reg_free_if_spill_p(_R4) && jit_reg_free_if_spill_p(_R5))
1956 r1 = _R4, r2 = _R5;
1957 else if (jit_reg_free_if_spill_p(_R10) && jit_reg_free_if_spill_p(_R11))
1958 r1 = _R10, r2 = _R11;
1959 else if (jit_reg_free_if_spill_p(_R8) && jit_reg_free_if_spill_p(_R9))
1960 r1 = _R8, r2 = _R9;
1961 else if (jit_reg_free_if_spill_p(_R6) && jit_reg_free_if_spill_p(_R7))
1962 r1 = _R6, r2 = _R7;
1963 else
1964 /* Do not jit_get_reg() all registers to avoid it */
1965 abort();
1966
1967 (void)jit_get_reg(jit_class_gpr|jit_class_named|r1);
1968 (void)jit_get_reg(jit_class_gpr|jit_class_named|r2);
1969
1970 return (r1);
1971}
1972
1973static void
1974_jit_unget_reg_pair(jit_state_t *_jit, jit_int32_t reg)
1975{
1976 jit_int32_t r1, r2;
1977 r1 = reg;
1978 switch (r1) {
1979 case _R0: r2 = _R1; break;
1980 case _R2: r2 = _R3; break;
1981 case _R4: r2 = _R5; break;
1982 case _R6: r2 = _R7; break;
1983 case _R8: r2 = _R9; break;
1984 case _R10: r2 = _R11; break;
1985 default: abort();
1986 }
1987 jit_unget_reg(r1);
1988 jit_unget_reg(r2);
1989}
1990
1991static jit_int32_t
1992_jit_get_reg_but_zero(jit_state_t *_jit, jit_int32_t flags)
1993{
1994 jit_int32_t reg;
1995 reg = jit_get_reg(jit_class_gpr);
1996 if (reg == _R0) {
1997 reg = jit_get_reg(jit_class_gpr|flags);
1998 jit_unget_reg(_R0);
1999 }
2000 return (reg);
2001}
2002
2003static void
2004_patch(jit_state_t *_jit, jit_word_t instr, jit_node_t *node)
2005{
2006 jit_int32_t flag;
2007
2008 assert(node->flag & jit_flag_node);
2009 if (node->code == jit_code_movi)
2010 flag = node->v.n->flag;
2011 else
2012 flag = node->u.n->flag;
2013 assert(!(flag & jit_flag_patch));
2014 if (_jitc->patches.offset >= _jitc->patches.length) {
2015 jit_realloc((jit_pointer_t *)&_jitc->patches.ptr,
2016 _jitc->patches.length * sizeof(jit_patch_t),
2017 (_jitc->patches.length + 1024) * sizeof(jit_patch_t));
2018 _jitc->patches.length += 1024;
2019 }
2020 _jitc->patches.ptr[_jitc->patches.offset].inst = instr;
2021 _jitc->patches.ptr[_jitc->patches.offset].node = node;
2022 ++_jitc->patches.offset;
2023}