notaz.gp2x.de / pcsx_rearmed.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
add a thp-based huge page alloc fallback
[pcsx_rearmed.git] / deps / lightning / lib / jit_ppc.c
CommitLineData
4a71579b 1/*
79bfeef6 2 * Copyright (C) 2012-2023 Free Software Foundation, Inc.
4a71579b
PC		 3 *
4 * This file is part of GNU lightning.
5 *
6 * GNU lightning is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU Lesser General Public License as published
8 * by the Free Software Foundation; either version 3, or (at your option)
9 * any later version.
10 *
11 * GNU lightning is distributed in the hope that it will be useful, but
12 * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
13 * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public
14 * License for more details.
15 *
16 * Authors:
17 * Paulo Cesar Pereira de Andrade
18 */
19
20#define jit_arg_reg_p(i) ((i) >= 0 && (i) < 8)
21#if !_CALL_SYSV
22# define jit_arg_f_reg_p(i) ((i) >= 0 && (i) < 13)
23#else
24# define jit_arg_f_reg_p(i) ((i) >= 0 && (i) < 8)
25# if __WORDSIZE == 32
26# define va_gp_shift 2
27# else
28# define va_gp_shift 3
29# endif
30# define va_gp_increment sizeof(jit_word_t)
31# define first_gp_argument r3
32# define first_gp_offset offsetof(jit_va_list_t, \
33 first_gp_argument)
34# define va_fp_increment sizeof(jit_float64_t)
35# define first_fp_argument f1
36# define first_fp_offset offsetof(jit_va_list_t, \
37 first_fp_argument)
38#endif
39#if __BYTE_ORDER == __LITTLE_ENDIAN
40# define C_DISP 0
41# define S_DISP 0
42# define I_DISP 0
43# define F_DISP 0
44#else
45# define C_DISP (__WORDSIZE >> 3) - sizeof(jit_int8_t)
46# define S_DISP (__WORDSIZE >> 3) - sizeof(jit_int16_t)
47# define I_DISP (__WORDSIZE >> 3) - sizeof(jit_int32_t)
48# define F_DISP (__WORDSIZE >> 3) - sizeof(jit_float32_t)
49#endif
ba86ff93
PC		 50#define CVT_OFFSET _jitc->function->cvt_offset
51#define CHECK_CVT_OFFSET() \
52 do { \
53 if (!_jitc->function->cvt_offset) { \
54 _jitc->again = 1; \
55 _jitc->function->cvt_offset = \
56 jit_allocai(sizeof(jit_float64_t)); \
57 } \
58 } while (0)
4a71579b
PC		 59
60/*
61 * Types
62 */
63#if _CALL_SYSV
64typedef struct jit_va_list {
65 jit_uint8_t ngpr;
66 jit_uint8_t nfpr;
67 jit_uint16_t _pad;
68# if __WORDSIZE == 64
69 jit_uint32_t _pad2;
70# endif
71 jit_pointer_t over;
72 jit_pointer_t save;
73# if __WORDSIZE == 32
74 jit_word_t _pad2;
75# endif
76 jit_word_t r3;
77 jit_word_t r4;
78 jit_word_t r5;
79 jit_word_t r6;
80 jit_word_t r7;
81 jit_word_t r8;
82 jit_word_t r9;
83 jit_word_t r10;
84 jit_float64_t f1;
85 jit_float64_t f2;
86 jit_float64_t f3;
87 jit_float64_t f4;
88 jit_float64_t f5;
89 jit_float64_t f6;
90 jit_float64_t f7;
91 jit_float64_t f8;
92} jit_va_list_t;
93#else
94typedef jit_pointer_t jit_va_list_t;
95#endif
96
97/*
98 * Prototypes
99 */
100#define patch(instr, node) _patch(_jit, instr, node)
101static void _patch(jit_state_t*,jit_word_t,jit_node_t*);
102
103/* libgcc */
104extern void __clear_cache(void *, void *);
105
106#define PROTO 1
107# include "jit_ppc-cpu.c"
108# include "jit_ppc-fpu.c"
ba86ff93 109# include "jit_fallback.c"
4a71579b
PC		 110#undef PROTO
111
112/*
113 * Initialization
114 */
115jit_register_t _rvs[] = {
116 { rc(sav) | 0, "r0" },
117 { rc(sav) | 11, "r11" }, /* env */
118 { rc(sav) | 12, "r12" }, /* exception */
119 { rc(sav) | 13, "r13" }, /* thread */
120 { rc(sav) | 2, "r2" }, /* toc */
121 { rc(sav) | rc(gpr) | 14, "r14" },
122 { rc(sav) | rc(gpr) | 15, "r15" },
123 { rc(sav) | rc(gpr) | 16, "r16" },
124 { rc(sav) | rc(gpr) | 17, "r17" },
125 { rc(sav) | rc(gpr) | 18, "r18" },
126 { rc(sav) | rc(gpr) | 19, "r19" },
127 { rc(sav) | rc(gpr) | 20, "r20" },
128 { rc(sav) | rc(gpr) | 21, "r21" },
129 { rc(sav) | rc(gpr) | 22, "r22" },
130 { rc(sav) | rc(gpr) | 23, "r23" },
131 { rc(sav) | rc(gpr) | 24, "r24" },
132 { rc(sav) | rc(gpr) | 25, "r25" },
133 { rc(sav) | rc(gpr) | 26, "r26" },
134 { rc(sav) | rc(gpr) | 27, "r27" },
135 { rc(sav) | rc(gpr) | 28, "r28" },
136 { rc(sav) | rc(gpr) | 29, "r29" },
137 { rc(sav) | rc(gpr) | 30, "r30" },
138 { rc(sav) | 1, "r1" },
139 { rc(sav) | 31, "r31" },
140 { rc(arg) | rc(gpr) | 10, "r10" },
141 { rc(arg) | rc(gpr) | 9, "r9" },
142 { rc(arg) | rc(gpr) | 8, "r8" },
143 { rc(arg) | rc(gpr) | 7, "r7" },
144 { rc(arg) | rc(gpr) | 6, "r6" },
145 { rc(arg) | rc(gpr) | 5, "r5" },
146 { rc(arg) | rc(gpr) | 4, "r4" },
147 { rc(arg) | rc(gpr) | 3, "r3" },
148 { rc(fpr) | 0, "f0" },
149 { rc(sav) | rc(fpr) | 14, "f14" },
150 { rc(sav) | rc(fpr) | 15, "f15" },
151 { rc(sav) | rc(fpr) | 16, "f16" },
152 { rc(sav) | rc(fpr) | 17, "f17" },
153 { rc(sav) | rc(fpr) | 18, "f18" },
154 { rc(sav) | rc(fpr) | 19, "f19" },
155 { rc(sav) | rc(fpr) | 20, "f20" },
156 { rc(sav) | rc(fpr) | 21, "f21" },
157 { rc(sav) | rc(fpr) | 22, "f22" },
158 { rc(sav) | rc(fpr) | 23, "f23" },
159 { rc(sav) | rc(fpr) | 24, "f24" },
160 { rc(sav) | rc(fpr) | 25, "f25" },
161 { rc(sav) | rc(fpr) | 26, "f26" },
162 { rc(sav) | rc(fpr) | 27, "f27" },
163 { rc(sav) | rc(fpr) | 28, "f28" },
164 { rc(sav) | rc(fpr) | 29, "f29" },
165 { rc(sav) | rc(fpr) | 30, "f30" },
166 { rc(sav) | rc(fpr) | 31, "f31" },
167#if !_CALL_SYSV
168 { rc(arg) | rc(fpr) | 13, "f13" },
169 { rc(arg) | rc(fpr) | 12, "f12" },
170 { rc(arg) | rc(fpr) | 11, "f11" },
171 { rc(arg) | rc(fpr) | 10, "f10" },
172 { rc(arg) | rc(fpr) | 9, "f9" },
173#else
174 { rc(fpr) | 13, "f13" },
175 { rc(fpr) | 12, "f12" },
176 { rc(fpr) | 11, "f11" },
177 { rc(fpr) | 10, "f10" },
178 { rc(fpr) | 9, "f9" },
179#endif
180 { rc(arg) | rc(fpr) | 8, "f8" },
181 { rc(arg) | rc(fpr) | 7, "f7" },
182 { rc(arg) | rc(fpr) | 6, "f6" },
183 { rc(arg) | rc(fpr) | 5, "f5" },
184 { rc(arg) | rc(fpr) | 4, "f4" },
185 { rc(arg) | rc(fpr) | 3, "f3" },
186 { rc(arg) | rc(fpr) | 2, "f2" },
187 { rc(arg) | rc(fpr) | 1, "f1" },
188 { _NOREG, "<none>" },
189};
190
ba86ff93
PC		 191static jit_int32_t iregs[] = {
192 _R14, _R15, _R16, _R17, _R18, _R19, _R20, _R21, _R22,
193 _R23, _R24, _R25, _R26, _R27, _R28, _R29, _R30
194};
195
196static jit_int32_t fregs[] = {
197 _F14, _F15, _F16, _F17, _F18, _F19, _F20, _F21,
198};
199
4a71579b
PC		 200/*
201 * Implementation
202 */
203void
204jit_get_cpu(void)
205{
206}
207
208void
209_jit_init(jit_state_t *_jit)
210{
211 _jitc->reglen = jit_size(_rvs) - 1;
212}
213
214void
215_jit_prolog(jit_state_t *_jit)
216{
217 jit_int32_t offset;
218
219 if (_jitc->function)
220 jit_epilog();
221 assert(jit_regset_cmp_ui(&_jitc->regarg, 0) == 0);
222 jit_regset_set_ui(&_jitc->regsav, 0);
223 offset = _jitc->functions.offset;
224 if (offset >= _jitc->functions.length) {
225 jit_realloc((jit_pointer_t *)&_jitc->functions.ptr,
226 _jitc->functions.length * sizeof(jit_function_t),
227 (_jitc->functions.length + 16) * sizeof(jit_function_t));
228 _jitc->functions.length += 16;
229 }
230 _jitc->function = _jitc->functions.ptr + _jitc->functions.offset++;
231 _jitc->function->self.size = params_offset;
232 _jitc->function->self.argi = _jitc->function->self.argf =
233 _jitc->function->self.alen = 0;
234 /* float conversion */
ba86ff93 235 _jitc->function->cvt_offset = 0;
4a71579b
PC		 236 _jitc->function->self.aoff = alloca_offset - 8;
237 _jitc->function->self.call = jit_call_default;
238 jit_alloc((jit_pointer_t *)&_jitc->function->regoff,
239 _jitc->reglen * sizeof(jit_int32_t));
240
241 /* _no_link here does not mean the jit_link() call can be removed
242 * by rewriting as:
243 * _jitc->function->prolog = jit_new_node(jit_code_prolog);
244 */
245 _jitc->function->prolog = jit_new_node_no_link(jit_code_prolog);
246 jit_link(_jitc->function->prolog);
247 _jitc->function->prolog->w.w = offset;
248 _jitc->function->epilog = jit_new_node_no_link(jit_code_epilog);
249 /* u: label value
250 * v: offset in blocks vector
251 * w: offset in functions vector
252 */
253 _jitc->function->epilog->w.w = offset;
254
255 jit_regset_new(&_jitc->function->regset);
256}
257
258jit_int32_t
259_jit_allocai(jit_state_t *_jit, jit_int32_t length)
260{
261 assert(_jitc->function);
ba86ff93 262 jit_check_frame();
4a71579b
PC		 263 switch (length) {
264 case 0: case 1: break;
265 case 2: _jitc->function->self.aoff &= -2; break;
266 case 3: case 4: _jitc->function->self.aoff &= -4; break;
267 default: _jitc->function->self.aoff &= -8; break;
268 }
269 _jitc->function->self.aoff -= length;
270 if (!_jitc->realize) {
271 jit_inc_synth_ww(allocai, _jitc->function->self.aoff, length);
272 jit_dec_synth();
273 }
274 return (_jitc->function->self.aoff);
275}
276
277void
278_jit_allocar(jit_state_t *_jit, jit_int32_t u, jit_int32_t v)
279{
280 jit_int32_t r0, r1;
281 assert(_jitc->function);
282 jit_inc_synth_ww(allocar, u, v);
283 if (!_jitc->function->allocar) {
284 _jitc->function->aoffoff = jit_allocai(sizeof(jit_int32_t));
285 _jitc->function->allocar = 1;
286 }
287 r0 = jit_get_reg(jit_class_gpr);
288 r1 = jit_get_reg(jit_class_gpr);
289 jit_ldr(r0, JIT_SP);
290 jit_negr(r1, v);
291 jit_andi(r1, r1, -16);
292 jit_ldxi_i(u, JIT_FP, _jitc->function->aoffoff);
293 jit_addr(u, u, r1);
294 jit_addr(JIT_SP, JIT_SP, r1);
295 jit_stxi_i(_jitc->function->aoffoff, JIT_FP, u);
296 jit_str(JIT_SP, r0);
297 jit_unget_reg(r1);
298 jit_unget_reg(r0);
299 jit_dec_synth();
300}
301
302void
303_jit_ret(jit_state_t *_jit)
304{
305 jit_node_t *instr;
306 assert(_jitc->function);
307 jit_inc_synth(ret);
308 /* jump to epilog */
309 instr = jit_jmpi();
310 jit_patch_at(instr, _jitc->function->epilog);
311 jit_dec_synth();
312}
313
314void
79bfeef6 315_jit_retr(jit_state_t *_jit, jit_int32_t u, jit_code_t code)
4a71579b 316{
79bfeef6
PC		 317 jit_code_inc_synth_w(code, u);
318 jit_movr(JIT_RET, u);
4a71579b
PC		 319 jit_ret();
320 jit_dec_synth();
321}
322
323void
79bfeef6 324_jit_reti(jit_state_t *_jit, jit_word_t u, jit_code_t code)
4a71579b 325{
79bfeef6 326 jit_code_inc_synth_w(code, u);
4a71579b
PC		 327 jit_movi(JIT_RET, u);
328 jit_ret();
329 jit_dec_synth();
330}
331
332void
333_jit_retr_f(jit_state_t *_jit, jit_int32_t u)
334{
335 jit_inc_synth_w(retr_f, u);
336 if (JIT_RET != u)
337 jit_movr_f(JIT_FRET, u);
338 else
339 jit_live(JIT_FRET);
340 jit_ret();
341 jit_dec_synth();
342}
343
344void
345_jit_reti_f(jit_state_t *_jit, jit_float32_t u)
346{
347 jit_inc_synth_f(reti_f, u);
348 jit_movi_f(JIT_FRET, u);
349 jit_ret();
350 jit_dec_synth();
351}
352
353void
354_jit_retr_d(jit_state_t *_jit, jit_int32_t u)
355{
356 jit_inc_synth_w(retr_d, u);
357 if (JIT_FRET != u)
358 jit_movr_d(JIT_FRET, u);
359 else
360 jit_live(JIT_FRET);
361 jit_ret();
362 jit_dec_synth();
363}
364
365void
366_jit_reti_d(jit_state_t *_jit, jit_float64_t u)
367{
368 jit_inc_synth_d(reti_d, u);
369 jit_movi_d(JIT_FRET, u);
370 jit_ret();
371 jit_dec_synth();
372}
373
374void
375_jit_epilog(jit_state_t *_jit)
376{
377 assert(_jitc->function);
378 assert(_jitc->function->epilog->next == NULL);
379 jit_link(_jitc->function->epilog);
380 _jitc->function = NULL;
381}
382
383jit_bool_t
384_jit_arg_register_p(jit_state_t *_jit, jit_node_t *u)
385{
79bfeef6 386 if (u->code >= jit_code_arg_c && u->code <= jit_code_arg)
4a71579b
PC		 387 return (jit_arg_reg_p(u->u.w));
388 assert(u->code == jit_code_arg_f || u->code == jit_code_arg_d);
389 return (jit_arg_f_reg_p(u->u.w));
390}
391
392void
393_jit_ellipsis(jit_state_t *_jit)
394{
395 jit_inc_synth(ellipsis);
396 if (_jitc->prepare) {
397 jit_link_prepare();
398 assert(!(_jitc->function->call.call & jit_call_varargs));
399 _jitc->function->call.call |= jit_call_varargs;
400 }
401 else {
402 jit_link_prolog();
403 assert(!(_jitc->function->self.call & jit_call_varargs));
404 _jitc->function->self.call |= jit_call_varargs;
405#if _CALL_SYSV
406 /* Allocate va_list like object in the stack.
407 * If applicable, with enough space to save all argument
408 * registers, and use fixed offsets for them. */
409 _jitc->function->vaoff = jit_allocai(sizeof(jit_va_list_t));
410#endif
411 _jitc->function->vagp = _jitc->function->self.argi;
412 _jitc->function->vafp = _jitc->function->self.argf;
413 }
414 jit_dec_synth();
415}
416
417void
418_jit_va_push(jit_state_t *_jit, jit_int32_t u)
419{
420 jit_inc_synth_w(va_push, u);
421 jit_pushargr(u);
422 jit_dec_synth();
423}
424
425jit_node_t *
79bfeef6 426_jit_arg(jit_state_t *_jit, jit_code_t code)
4a71579b
PC		 427{
428 jit_node_t *node;
429 jit_int32_t offset;
430 jit_bool_t incr = 1;
431 assert(_jitc->function);
79bfeef6
PC		 432 assert(!(_jitc->function->self.call & jit_call_varargs));
433#if STRONG_TYPE_CHECKING
434 assert(code >= jit_code_arg_c && code <= jit_code_arg);
435#endif
4a71579b
PC		 436 if (jit_arg_reg_p(_jitc->function->self.argi)) {
437 offset = _jitc->function->self.argi++;
438#if _CALL_SYSV
439 incr = 0;
440#endif
441 }
442 else
443 offset = _jitc->function->self.size;
444 if (incr)
445 _jitc->function->self.size += sizeof(jit_word_t);
79bfeef6 446 node = jit_new_node_ww(code, offset,
4a71579b
PC		 447 ++_jitc->function->self.argn);
448 jit_link_prolog();
449 return (node);
450}
451
452jit_node_t *
453_jit_arg_f(jit_state_t *_jit)
454{
455 jit_node_t *node;
456 jit_int32_t offset;
457 jit_bool_t incr = 1;
458 assert(_jitc->function);
459 if (jit_arg_f_reg_p(_jitc->function->self.argf)) {
460 offset = _jitc->function->self.argf++;
461#if _CALL_SYSV
462 incr = 0;
463#endif
464 }
465 else
466 offset = _jitc->function->self.size + F_DISP;
467#if !_CALL_SYSV
468 if (jit_arg_reg_p(_jitc->function->self.argi)) {
469# if __WORDSIZE == 32
470 _jitc->function->self.argi += 2;
471# else
472 _jitc->function->self.argi++;
473# endif
474 }
475#endif
476 if (incr)
477 _jitc->function->self.size += sizeof(jit_word_t);
478 node = jit_new_node_ww(jit_code_arg_f, offset,
479 ++_jitc->function->self.argn);
480 jit_link_prolog();
481 return (node);
482}
483
484jit_node_t *
485_jit_arg_d(jit_state_t *_jit)
486{
487 jit_node_t *node;
488 jit_int32_t offset;
489 jit_bool_t incr = 1;
490 assert(_jitc->function);
491 if (jit_arg_f_reg_p(_jitc->function->self.argf)) {
492 offset = _jitc->function->self.argf++;
493#if _CALL_SYSV
494 incr = 0;
495#endif
496 }
497 else {
498#if _CALL_SYSV
499 if (_jitc->function->self.size & 7)
500 _jitc->function->self.size += 4;
501#endif
502 offset = _jitc->function->self.size;
503 }
504#if !_CALL_SYSV
505 if (jit_arg_reg_p(_jitc->function->self.argi)) {
506# if __WORDSIZE == 32
507 _jitc->function->self.argi += 2;
508# else
509 _jitc->function->self.argi++;
510# endif
511 }
512#endif
513 if (incr)
514 _jitc->function->self.size += sizeof(jit_float64_t);
515 node = jit_new_node_ww(jit_code_arg_d, offset,
516 ++_jitc->function->self.argn);
517 jit_link_prolog();
518 return (node);
519}
520
521void
522_jit_getarg_c(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
523{
79bfeef6 524 assert_arg_type(v->code, jit_code_arg_c);
4a71579b
PC		 525 jit_inc_synth_wp(getarg_c, u, v);
526 if (jit_arg_reg_p(v->u.w))
527 jit_extr_c(u, JIT_RA0 - v->u.w);
ba86ff93
PC		 528 else {
529 jit_check_frame();
4a71579b 530 jit_ldxi_c(u, JIT_FP, v->u.w + C_DISP);
ba86ff93 531 }
4a71579b
PC		 532 jit_dec_synth();
533}
534
535void
536_jit_getarg_uc(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
537{
79bfeef6 538 assert_arg_type(v->code, jit_code_arg_c);
4a71579b
PC		 539 jit_inc_synth_wp(getarg_uc, u, v);
540 if (jit_arg_reg_p(v->u.w))
541 jit_extr_uc(u, JIT_RA0 - v->u.w);
ba86ff93
PC		 542 else {
543 jit_check_frame();
4a71579b 544 jit_ldxi_uc(u, JIT_FP, v->u.w + C_DISP);
ba86ff93 545 }
4a71579b
PC		 546 jit_dec_synth();
547}
548
549void
550_jit_getarg_s(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
551{
79bfeef6 552 assert_arg_type(v->code, jit_code_arg_s);
4a71579b
PC		 553 jit_inc_synth_wp(getarg_s, u, v);
554 if (jit_arg_reg_p(v->u.w))
555 jit_extr_s(u, JIT_RA0 - v->u.w);
ba86ff93
PC		 556 else {
557 jit_check_frame();
4a71579b 558 jit_ldxi_s(u, JIT_FP, v->u.w + S_DISP);
ba86ff93 559 }
4a71579b
PC		 560 jit_dec_synth();
561}
562
563void
564_jit_getarg_us(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
565{
79bfeef6 566 assert_arg_type(v->code, jit_code_arg_s);
4a71579b
PC		 567 jit_inc_synth_wp(getarg_us, u, v);
568 if (jit_arg_reg_p(v->u.w))
569 jit_extr_us(u, JIT_RA0 - v->u.w);
ba86ff93
PC		 570 else {
571 jit_check_frame();
4a71579b 572 jit_ldxi_us(u, JIT_FP, v->u.w + S_DISP);
ba86ff93 573 }
4a71579b
PC		 574 jit_dec_synth();
575}
576
577void
578_jit_getarg_i(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
579{
79bfeef6 580 assert_arg_type(v->code, jit_code_arg_i);
4a71579b
PC		 581 jit_inc_synth_wp(getarg_i, u, v);
582 if (jit_arg_reg_p(v->u.w)) {
583#if __WORDSIZE == 32
584 jit_movr(u, JIT_RA0 - v->u.w);
585#else
586 jit_extr_i(u, JIT_RA0 - v->u.w);
587#endif
588 }
ba86ff93
PC		 589 else {
590 jit_check_frame();
4a71579b 591 jit_ldxi_i(u, JIT_FP, v->u.w + I_DISP);
ba86ff93 592 }
4a71579b
PC		 593 jit_dec_synth();
594}
595
596#if __WORDSIZE == 64
597void
598_jit_getarg_ui(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
599{
79bfeef6 600 assert_arg_type(v->code, jit_code_arg_i);
4a71579b
PC		 601 jit_inc_synth_wp(getarg_ui, u, v);
602 if (jit_arg_reg_p(v->u.w))
603 jit_extr_ui(u, JIT_RA0 - v->u.w);
ba86ff93
PC		 604 else {
605 jit_check_frame();
4a71579b 606 jit_ldxi_ui(u, JIT_FP, v->u.w + I_DISP);
ba86ff93 607 }
4a71579b
PC		 608 jit_dec_synth();
609}
610
611void
612_jit_getarg_l(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
613{
79bfeef6 614 assert_arg_type(v->code, jit_code_arg_l);
4a71579b
PC		 615 jit_inc_synth_wp(getarg_l, u, v);
616 if (jit_arg_reg_p(v->u.w))
617 jit_movr(u, JIT_RA0 - v->u.w);
ba86ff93
PC		 618 else {
619 jit_check_frame();
4a71579b 620 jit_ldxi_l(u, JIT_FP, v->u.w);
ba86ff93 621 }
4a71579b
PC		 622 jit_dec_synth();
623}
624#endif
625
626void
79bfeef6 627_jit_putargr(jit_state_t *_jit, jit_int32_t u, jit_node_t *v, jit_code_t code)
4a71579b 628{
79bfeef6
PC		 629 assert_putarg_type(code, v->code);
630 jit_code_inc_synth_wp(code, u, v);
4a71579b
PC		 631 if (jit_arg_reg_p(v->u.w))
632 jit_movr(JIT_RA0 - v->u.w, u);
ba86ff93
PC		 633 else {
634 jit_check_frame();
4a71579b 635 jit_stxi(v->u.w, JIT_FP, u);
ba86ff93 636 }
4a71579b
PC		 637 jit_dec_synth();
638}
639
640void
79bfeef6 641_jit_putargi(jit_state_t *_jit, jit_word_t u, jit_node_t *v, jit_code_t code)
4a71579b
PC		 642{
643 jit_int32_t regno;
79bfeef6
PC		 644 assert_putarg_type(code, v->code);
645 jit_code_inc_synth_wp(code, u, v);
4a71579b
PC		 646 if (jit_arg_reg_p(v->u.w))
647 jit_movi(JIT_RA0 - v->u.w, u);
648 else {
ba86ff93 649 jit_check_frame();
4a71579b
PC		 650 regno = jit_get_reg(jit_class_gpr);
651 jit_movi(regno, u);
652 jit_stxi(v->u.w, JIT_FP, regno);
653 jit_unget_reg(regno);
654 }
655 jit_dec_synth();
656}
657
658void
659_jit_getarg_f(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
660{
661 assert(v->code == jit_code_arg_f);
662 jit_inc_synth_wp(getarg_f, u, v);
663 if (jit_arg_f_reg_p(v->u.w))
664 jit_movr_d(u, JIT_FA0 - v->u.w);
ba86ff93
PC		 665 else {
666 jit_check_frame();
4a71579b 667 jit_ldxi_f(u, JIT_FP, v->u.w);
ba86ff93 668 }
4a71579b
PC		 669 jit_dec_synth();
670}
671
672void
673_jit_putargr_f(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
674{
675 assert(v->code == jit_code_arg_f);
676 jit_inc_synth_wp(putargr_f, u, v);
677 if (jit_arg_f_reg_p(v->u.w))
678 jit_movr_d(JIT_FA0 - v->u.w, u);
ba86ff93
PC		 679 else {
680 jit_check_frame();
4a71579b 681 jit_stxi_f(v->u.w, JIT_FP, u);
ba86ff93 682 }
4a71579b
PC		 683 jit_dec_synth();
684}
685
686void
687_jit_putargi_f(jit_state_t *_jit, jit_float32_t u, jit_node_t *v)
688{
689 jit_int32_t regno;
690 assert(v->code == jit_code_arg_f);
691 jit_inc_synth_fp(putargi_f, u, v);
692 if (jit_arg_f_reg_p(v->u.w))
693 jit_movi_d(JIT_FA0 - v->u.w, u);
694 else {
ba86ff93 695 jit_check_frame();
4a71579b
PC		 696 regno = jit_get_reg(jit_class_fpr);
697 jit_movi_d(regno, u);
698 jit_stxi_f(v->u.w, JIT_FP, regno);
699 jit_unget_reg(regno);
700 }
701 jit_dec_synth();
702}
703
704void
705_jit_getarg_d(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
706{
707 assert(v->code == jit_code_arg_d);
708 jit_inc_synth_wp(getarg_d, u, v);
709 if (jit_arg_f_reg_p(v->u.w))
710 jit_movr_d(u, JIT_FA0 - v->u.w);
ba86ff93
PC		 711 else {
712 jit_check_frame();
4a71579b 713 jit_ldxi_d(u, JIT_FP, v->u.w);
ba86ff93 714 }
4a71579b
PC		 715 jit_dec_synth();
716}
717
718void
719_jit_putargr_d(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
720{
721 assert(v->code == jit_code_arg_d);
722 jit_inc_synth_wp(putargr_d, u, v);
723 if (jit_arg_f_reg_p(v->u.w))
724 jit_movr_d(JIT_FA0 - v->u.w, u);
ba86ff93
PC		 725 else {
726 jit_check_frame();
4a71579b 727 jit_stxi_d(v->u.w, JIT_FP, u);
ba86ff93 728 }
4a71579b
PC		 729 jit_dec_synth();
730}
731
732void
733_jit_putargi_d(jit_state_t *_jit, jit_float64_t u, jit_node_t *v)
734{
735 jit_int32_t regno;
736 assert(v->code == jit_code_arg_d);
737 jit_inc_synth_dp(putargi_d, u, v);
738 if (jit_arg_f_reg_p(v->u.w))
739 jit_movi_d(JIT_FA0 - v->u.w, u);
740 else {
ba86ff93 741 jit_check_frame();
4a71579b
PC		 742 regno = jit_get_reg(jit_class_fpr);
743 jit_movi_d(regno, u);
744 jit_stxi_d(v->u.w, JIT_FP, regno);
745 jit_unget_reg(regno);
746 }
747 jit_dec_synth();
748}
749
750void
79bfeef6 751_jit_pushargr(jit_state_t *_jit, jit_int32_t u, jit_code_t code)
4a71579b
PC		 752{
753 jit_bool_t incr = 1;
754 assert(_jitc->function);
79bfeef6 755 jit_code_inc_synth_w(code, u);
4a71579b
PC		 756 jit_link_prepare();
757 if (jit_arg_reg_p(_jitc->function->call.argi)) {
758 jit_movr(JIT_RA0 - _jitc->function->call.argi, u);
759 ++_jitc->function->call.argi;
760#if _CALL_SYSV
761 incr = 0;
762#endif
763 }
764 else
765 jit_stxi(_jitc->function->call.size + params_offset, JIT_SP, u);
766 if (incr)
767 _jitc->function->call.size += sizeof(jit_word_t);
768 jit_dec_synth();
769}
770
771void
79bfeef6 772_jit_pushargi(jit_state_t *_jit, jit_word_t u, jit_code_t code)
4a71579b
PC		 773{
774 jit_int32_t regno;
775 jit_bool_t incr = 1;
776 assert(_jitc->function);
79bfeef6 777 jit_code_inc_synth_w(code, u);
4a71579b
PC		 778 jit_link_prepare();
779 if (jit_arg_reg_p(_jitc->function->call.argi)) {
780 jit_movi(JIT_RA0 - _jitc->function->call.argi, u);
781 ++_jitc->function->call.argi;
782#if _CALL_SYSV
783 incr = 0;
784#endif
785 }
786 else {
787 regno = jit_get_reg(jit_class_gpr);
788 jit_movi(regno, u);
789 jit_stxi(_jitc->function->call.size + params_offset, JIT_SP, regno);
790 jit_unget_reg(regno);
791 }
792 if (incr)
793 _jitc->function->call.size += sizeof(jit_word_t);
794 jit_dec_synth();
795}
796
797void
798_jit_pushargr_f(jit_state_t *_jit, jit_int32_t u)
799{
800 jit_bool_t incr = 1;
801 assert(_jitc->function);
802 jit_inc_synth_w(pushargr_f, u);
803 jit_link_prepare();
804 if (jit_arg_f_reg_p(_jitc->function->call.argf)
805#if !_CALL_SYSV
806 && !(_jitc->function->call.call & jit_call_varargs)
807#endif
808 ) {
809 jit_movr_d(JIT_FA0 - _jitc->function->call.argf, u);
810 ++_jitc->function->call.argf;
811#if !_CALL_SYSV
812 /* in case of excess arguments */
813 if (jit_arg_reg_p(_jitc->function->call.argi)) {
814# if __WORDSIZE == 32
815 _jitc->function->call.argi += 2;
816 if (!jit_arg_reg_p(_jitc->function->call.argi - 1))
817 --_jitc->function->call.argi;
818# else
819 _jitc->function->call.argi++;
820# endif
821 }
822#elif _CALL_SYSV
823 incr = 0;
824#endif
825 }
826#if !_CALL_SYSV
827 else if (jit_arg_reg_p(_jitc->function->call.argi
828# if __WORDSIZE == 32
829 + 1
830# endif
831 )) {
ba86ff93
PC		 832 CHECK_CVT_OFFSET();
833 jit_check_frame();
834 jit_stxi_d(CVT_OFFSET, JIT_FP, u);
835 jit_ldxi(JIT_RA0 - _jitc->function->call.argi, JIT_FP, CVT_OFFSET);
4a71579b
PC		 836 _jitc->function->call.argi++;
837# if __WORDSIZE == 32
838 jit_ldxi(JIT_RA0 - _jitc->function->call.argi, JIT_FP,
ba86ff93 839 CVT_OFFSET + 4);
4a71579b
PC		 840 _jitc->function->call.argi++;
841# endif
842 }
843#endif
844 else
845 jit_stxi_f(_jitc->function->call.size + params_offset + F_DISP,
846 JIT_SP, u);
847 if (incr)
848 _jitc->function->call.size += sizeof(jit_word_t);
849 jit_dec_synth();
850}
851
852void
853_jit_pushargi_f(jit_state_t *_jit, jit_float32_t u)
854{
855 jit_bool_t incr = 1;
856 jit_int32_t regno;
857 assert(_jitc->function);
858 jit_inc_synth_f(pushargi_f, u);
859 jit_link_prepare();
860 if (jit_arg_f_reg_p(_jitc->function->call.argf)
861#if !_CALL_SYSV
862 && !(_jitc->function->call.call & jit_call_varargs)
863#endif
864 ) {
865 jit_movi_d(JIT_FA0 - _jitc->function->call.argf, u);
866 ++_jitc->function->call.argf;
867#if !_CALL_SYSV
868 /* in case of excess arguments */
869# if __WORDSIZE == 32
870 _jitc->function->call.argi += 2;
871 if (!jit_arg_reg_p(_jitc->function->call.argi - 1))
872 --_jitc->function->call.argi;
873# else
874 _jitc->function->call.argi++;
875# endif
876#elif _CALL_SYSV
877 incr = 0;
878#endif
879 }
880 else {
881 regno = jit_get_reg(jit_class_fpr);
882 jit_movi_f(regno, u);
883#if !_CALL_SYSV
884 if (jit_arg_reg_p(_jitc->function->call.argi
885# if __WORDSIZE == 32
886 + 1
887# endif
888 )) {
ba86ff93
PC		 889 CHECK_CVT_OFFSET();
890 jit_check_frame();
891 jit_stxi_d(CVT_OFFSET, JIT_FP, regno);
4a71579b 892 jit_ldxi(JIT_RA0 - _jitc->function->call.argi, JIT_FP,
ba86ff93 893 CVT_OFFSET);
4a71579b
PC		 894 _jitc->function->call.argi++;
895# if __WORDSIZE == 32
896 jit_ldxi(JIT_RA0 - _jitc->function->call.argi, JIT_FP,
ba86ff93 897 CVT_OFFSET + 4);
4a71579b
PC		 898 _jitc->function->call.argi++;
899# endif
900 }
901 else
902#endif
903 jit_stxi_f(_jitc->function->call.size + params_offset + F_DISP,
904 JIT_SP, regno);
905 jit_unget_reg(regno);
906 }
907 if (incr)
908 _jitc->function->call.size += sizeof(jit_word_t);
909 jit_dec_synth();
910}
911
912void
913_jit_pushargr_d(jit_state_t *_jit, jit_int32_t u)
914{
915 jit_bool_t incr = 1;
916 assert(_jitc->function);
917 jit_inc_synth_w(pushargr_d, u);
918 jit_link_prepare();
919 if (jit_arg_f_reg_p(_jitc->function->call.argf)
920#if !_CALL_SYSV
921 && !(_jitc->function->call.call & jit_call_varargs)
922#endif
923 ) {
924 jit_movr_d(JIT_FA0 - _jitc->function->call.argf, u);
925 ++_jitc->function->call.argf;
926#if !_CALL_SYSV
927 /* in case of excess arguments */
928# if __WORDSIZE == 32
929 _jitc->function->call.argi += 2;
930 if (!jit_arg_reg_p(_jitc->function->call.argi - 1))
931 --_jitc->function->call.argi;
932# else
933 _jitc->function->call.argi++;
934# endif
935#else /* _CALL_SYSV */
936 incr = 0;
937#endif
938 }
939#if !_CALL_SYSV
940 else if (jit_arg_reg_p(_jitc->function->call.argi
941# if __WORDSIZE == 32
942 + 1
943# endif
944 )) {
ba86ff93
PC		 945 CHECK_CVT_OFFSET();
946 jit_check_frame();
947 jit_stxi_d(CVT_OFFSET, JIT_FP, u);
948 jit_ldxi(JIT_RA0 - _jitc->function->call.argi, JIT_FP, CVT_OFFSET);
4a71579b
PC		 949 _jitc->function->call.argi++;
950# if __WORDSIZE == 32
ba86ff93 951 jit_ldxi(JIT_RA0 - _jitc->function->call.argi, JIT_FP, CVT_OFFSET + 4);
4a71579b
PC		 952 _jitc->function->call.argi++;
953# endif
954 }
955 else
956#endif /* !_CALL_SYSV */
957 {
958#if _CALL_SYSV
959 if (_jitc->function->call.size & 7)
960 _jitc->function->call.size += 4;
961#endif
962 jit_stxi_d(_jitc->function->call.size + params_offset, JIT_SP, u);
963#if !_CALL_SYSV && __WORDSIZE == 32
964 if (jit_arg_reg_p(_jitc->function->call.argi)) {
965 jit_ldxi(JIT_RA0 - _jitc->function->call.argi, JIT_SP,
966 _jitc->function->call.size + params_offset);
967 _jitc->function->call.argi++;
968 }
969#endif
970 }
971 if (incr)
972 _jitc->function->call.size += sizeof(jit_float64_t);
973 jit_dec_synth();
974}
975
976void
977_jit_pushargi_d(jit_state_t *_jit, jit_float64_t u)
978{
979 jit_int32_t regno;
980 jit_bool_t incr = 1;
981 assert(_jitc->function);
982 jit_inc_synth_d(pushargi_d, u);
983 jit_link_prepare();
984 if (jit_arg_f_reg_p(_jitc->function->call.argf)
985#if !_CALL_SYSV
986 && !(_jitc->function->call.call & jit_call_varargs)
987#endif
988 ) {
989 jit_movi_d(JIT_FA0 - _jitc->function->call.argf, u);
990 ++_jitc->function->call.argf;
991#if !_CALL_SYSV
992 /* in case of excess arguments */
993 if (jit_arg_reg_p(_jitc->function->call.argi)) {
994# if __WORDSIZE == 32
995 _jitc->function->call.argi += 2;
996 if (!jit_arg_reg_p(_jitc->function->call.argi - 1))
997 --_jitc->function->call.argi;
998# else
999 _jitc->function->call.argi++;
1000# endif
1001 }
1002#else /* _CALL_SYSV */
1003 incr = 0;
1004#endif
1005 }
1006 else {
1007 regno = jit_get_reg(jit_class_fpr);
1008 jit_movi_d(regno, u);
1009#if !_CALL_SYSV
1010 if (jit_arg_reg_p(_jitc->function->call.argi
1011# if __WORDSIZE == 32
1012 + 1
1013# endif
1014 )) {
ba86ff93
PC		 1015 CHECK_CVT_OFFSET();
1016 jit_check_frame();
1017 jit_stxi_d(CVT_OFFSET, JIT_FP, regno);
4a71579b 1018 jit_ldxi(JIT_RA0 - _jitc->function->call.argi, JIT_FP,
ba86ff93 1019 CVT_OFFSET);
4a71579b
PC		 1020 _jitc->function->call.argi++;
1021# if __WORDSIZE == 32
1022 jit_ldxi(JIT_RA0 - _jitc->function->call.argi, JIT_FP,
ba86ff93 1023 CVT_OFFSET + 4);
4a71579b
PC		 1024 _jitc->function->call.argi++;
1025# endif
1026 }
1027 else
1028#endif /* !_CALL_SYSV */
1029 {
1030#if _CALL_SYSV
1031 if (_jitc->function->call.size & 7)
1032 _jitc->function->call.size += 4;
1033#endif
1034 jit_stxi_d(_jitc->function->call.size + params_offset,
1035 JIT_SP, regno);
1036#if !_CALL_SYSV && __WORDSIZE == 32
1037 if (jit_arg_reg_p(_jitc->function->call.argi)) {
1038 jit_ldxi(JIT_RA0 - _jitc->function->call.argi, JIT_SP,
1039 _jitc->function->call.size + params_offset);
1040 _jitc->function->call.argi++;
1041 }
1042#endif
1043 }
1044 jit_unget_reg(regno);
1045 }
1046 if (incr)
1047 _jitc->function->call.size += sizeof(jit_float64_t);
1048 jit_dec_synth();
1049}
1050
1051jit_bool_t
1052_jit_regarg_p(jit_state_t *_jit, jit_node_t *node, jit_int32_t regno)
1053{
1054 jit_int32_t spec;
1055 spec = jit_class(_rvs[regno].spec);
1056 if (spec & jit_class_arg) {
1057 if (spec & jit_class_gpr) {
1058 regno = JIT_RA0 - regno;
1059 if (regno >= 0 && regno < node->v.w)
1060 return (1);
1061 }
1062 else if (spec & jit_class_fpr) {
1063 regno = JIT_FA0 - regno;
1064 if (regno >= 0 && regno < node->w.w)
1065 return (1);
1066 }
1067 }
1068 return (0);
1069}
1070
1071void
1072_jit_finishr(jit_state_t *_jit, jit_int32_t r0)
1073{
1074 jit_node_t *call;
1075 assert(_jitc->function);
1076 jit_inc_synth_w(finishr, r0);
1077 if (_jitc->function->self.alen < _jitc->function->call.size)
1078 _jitc->function->self.alen = _jitc->function->call.size;
1079 call = jit_callr(r0);
1080 call->v.w = _jitc->function->call.argi;
1081 call->w.w = _jitc->function->call.argf;
1082#if _CALL_SYSV
1083 /* If passing float arguments in registers */
1084 if ((_jitc->function->call.call & jit_call_varargs) && call->w.w)
1085 call->flag |= jit_flag_varargs;
1086#endif
1087 _jitc->function->call.argi = _jitc->function->call.argf = 0;
1088 _jitc->prepare = 0;
1089 jit_dec_synth();
1090}
1091
1092jit_node_t *
1093_jit_finishi(jit_state_t *_jit, jit_pointer_t i0)
1094{
1095 jit_node_t *node;
1096 assert(_jitc->function);
1097 jit_inc_synth_w(finishi, (jit_word_t)i0);
1098 if (_jitc->function->self.alen < _jitc->function->call.size)
1099 _jitc->function->self.alen = _jitc->function->call.size;
1100 node = jit_calli(i0);
1101 node->v.w = _jitc->function->call.argi;
1102 node->w.w = _jitc->function->call.argf;
1103#if _CALL_SYSV
1104 if ((_jitc->function->call.call & jit_call_varargs) && node->w.w)
1105 node->flag |= jit_flag_varargs;
1106#endif
1107 _jitc->function->call.argi = _jitc->function->call.argf = 0;
1108 _jitc->prepare = 0;
1109 jit_dec_synth();
1110 return (node);
1111}
1112
1113void
1114_jit_retval_c(jit_state_t *_jit, jit_int32_t r0)
1115{
1116 jit_inc_synth(retval_c);
1117 jit_extr_c(r0, JIT_RET);
1118 jit_dec_synth();
1119}
1120
1121void
1122_jit_retval_uc(jit_state_t *_jit, jit_int32_t r0)
1123{
1124 jit_inc_synth(retval_uc);
1125 jit_extr_uc(r0, JIT_RET);
1126 jit_dec_synth();
1127}
1128
1129void
1130_jit_retval_s(jit_state_t *_jit, jit_int32_t r0)
1131{
1132 jit_inc_synth(retval_s);
1133 jit_extr_s(r0, JIT_RET);
1134 jit_dec_synth();
1135}
1136
1137void
1138_jit_retval_us(jit_state_t *_jit, jit_int32_t r0)
1139{
1140 jit_inc_synth(retval_us);
1141 jit_extr_us(r0, JIT_RET);
1142 jit_dec_synth();
1143}
1144
1145void
1146_jit_retval_i(jit_state_t *_jit, jit_int32_t r0)
1147{
1148 jit_inc_synth(retval_i);
1149#if __WORDSIZE == 32
1150 if (r0 != JIT_RET)
1151 jit_movr(r0, JIT_RET);
1152#else
1153 jit_extr_i(r0, JIT_RET);
1154#endif
1155 jit_dec_synth();
1156}
1157
1158#if __WORDSIZE == 64
1159void
1160_jit_retval_ui(jit_state_t *_jit, jit_int32_t r0)
1161{
1162 jit_inc_synth(retval_ui);
1163 jit_extr_ui(r0, JIT_RET);
1164 jit_dec_synth();
1165}
1166
1167void
1168_jit_retval_l(jit_state_t *_jit, jit_int32_t r0)
1169{
1170 jit_inc_synth(retval_l);
1171 if (r0 != JIT_RET)
1172 jit_movr(r0, JIT_RET);
1173 jit_dec_synth();
1174}
1175#endif
1176
1177void
1178_jit_retval_f(jit_state_t *_jit, jit_int32_t r0)
1179{
1180 jit_inc_synth(retval_f);
1181 jit_retval_d(r0);
1182 jit_dec_synth();
1183}
1184
1185void
1186_jit_retval_d(jit_state_t *_jit, jit_int32_t r0)
1187{
1188 jit_inc_synth(retval_d);
1189 if (r0 != JIT_FRET)
1190 jit_movr_d(r0, JIT_FRET);
1191 jit_dec_synth();
1192}
1193
1194jit_pointer_t
1195_emit_code(jit_state_t *_jit)
1196{
1197 jit_node_t *node;
1198 jit_node_t *temp;
1199 jit_word_t word;
1200 jit_int32_t value;
1201 jit_int32_t offset;
ba3814c1
PC		 1202 jit_bool_t no_flag = 0; /* Set if previous instruction is
1203 * *not* a jump target. */
4a71579b
PC		 1204 struct {
1205 jit_node_t *node;
1206 jit_word_t word;
79bfeef6 1207 jit_function_t func;
4a71579b
PC		 1208#if DEVEL_DISASSEMBLER
1209 jit_word_t prevw;
1210#endif
1211 jit_word_t patch_offset;
1212#if _CALL_AIXDESC
1213 jit_word_t prolog_offset;
1214#endif
1215 } undo;
1216#if DEVEL_DISASSEMBLER
1217 jit_word_t prevw;
1218#endif
1219
1220 _jitc->function = NULL;
1221
1222 jit_reglive_setup();
1223
1224 undo.word = 0;
1225 undo.node = NULL;
1226 undo.patch_offset = 0;
1227
1228#if DEVEL_DISASSEMBLER
1229 prevw = _jit->pc.w;
1230#endif
1231#if _CALL_AIXDESC
1232 undo.prolog_offset = 0;
1233 for (node = _jitc->head; node; node = node->next)
1234 if (node->code != jit_code_label &&
1235 node->code != jit_code_note &&
1236 node->code != jit_code_name)
1237 break;
1238 if (node && (node->code != jit_code_prolog ||
1239 !(_jitc->functions.ptr + node->w.w)->assume_frame)) {
1240 /* code may start with a jump so add an initial function descriptor */
1241 word = _jit->pc.w + sizeof(void*) * 3;
1242 iw(word); /* addr */
1243 iw(0); /* toc */
1244 iw(0); /* env */
1245 }
1246#endif
1247
1248#define case_rr(name, type) \
1249 case jit_code_##name##r##type: \
1250 name##r##type(rn(node->u.w), rn(node->v.w)); \
1251 break
1252#define case_rw(name, type) \
1253 case jit_code_##name##i##type: \
1254 name##i##type(rn(node->u.w), node->v.w); \
1255 break
1256#define case_wr(name, type) \
1257 case jit_code_##name##i##type: \
1258 name##i##type(node->u.w, rn(node->v.w)); \
1259 break
1260#define case_rrr(name, type) \
1261 case jit_code_##name##r##type: \
1262 name##r##type(rn(node->u.w), \
1263 rn(node->v.w), rn(node->w.w)); \
1264 break
1265#define case_rrrr(name, type) \
1266 case jit_code_##name##r##type: \
1267 name##r##type(rn(node->u.q.l), rn(node->u.q.h), \
1268 rn(node->v.w), rn(node->w.w)); \
1269 break
ba86ff93
PC		 1270#define case_rqr(name, type) \
1271 case jit_code_##name##r##type: \
1272 name##r##type(rn(node->u.w), rn(node->v.q.l), \
1273 rn(node->v.q.h), rn(node->w.w)); \
1274 case jit_code_##name##i##type: \
1275 break;
4a71579b
PC		 1276#define case_rrw(name, type) \
1277 case jit_code_##name##i##type: \
1278 name##i##type(rn(node->u.w), rn(node->v.w), node->w.w); \
1279 break
1280#define case_rrrw(name, type) \
1281 case jit_code_##name##i##type: \
1282 name##i##type(rn(node->u.q.l), rn(node->u.q.h), \
1283 rn(node->v.w), node->w.w); \
1284 break
1285#define case_rrf(name, type, size) \
1286 case jit_code_##name##i##type: \
1287 assert(node->flag & jit_flag_data); \
1288 name##i##type(rn(node->u.w), rn(node->v.w), \
1289 (jit_float##size##_t *)node->w.n->u.w); \
1290 break
1291#define case_wrr(name, type) \
1292 case jit_code_##name##i##type: \
1293 name##i##type(node->u.w, rn(node->v.w), rn(node->w.w)); \
1294 break
1295#define case_brr(name, type) \
1296 case jit_code_##name##r##type: \
1297 temp = node->u.n; \
1298 assert(temp->code == jit_code_label || \
1299 temp->code == jit_code_epilog); \
1300 if (temp->flag & jit_flag_patch) \
1301 name##r##type(temp->u.w, rn(node->v.w), \
1302 rn(node->w.w)); \
1303 else { \
1304 word = name##r##type(_jit->pc.w, \
1305 rn(node->v.w), rn(node->w.w)); \
1306 patch(word, node); \
1307 } \
1308 break
1309#define case_brw(name, type) \
1310 case jit_code_##name##i##type: \
1311 temp = node->u.n; \
1312 assert(temp->code == jit_code_label || \
1313 temp->code == jit_code_epilog); \
1314 if (temp->flag & jit_flag_patch) \
1315 name##i##type(temp->u.w, \
1316 rn(node->v.w), node->w.w); \
1317 else { \
1318 word = name##i##type(_jit->pc.w, \
1319 rn(node->v.w), node->w.w); \
1320 patch(word, node); \
1321 } \
1322 break
1323#define case_brf(name, type, size) \
1324 case jit_code_##name##i##type: \
1325 temp = node->u.n; \
1326 assert(temp->code == jit_code_label || \
1327 temp->code == jit_code_epilog); \
1328 if (temp->flag & jit_flag_patch) \
1329 name##i##type(temp->u.w, rn(node->v.w), \
1330 (jit_float##size##_t *)node->w.n->u.w); \
1331 else { \
1332 word = name##i##type(_jit->pc.w, rn(node->v.w), \
1333 (jit_float##size##_t *)node->w.n->u.w); \
1334 patch(word, node); \
1335 } \
1336 break
1337 for (node = _jitc->head; node; node = node->next) {
1338 if (_jit->pc.uc >= _jitc->code.end)
1339 return (NULL);
1340
1341#if DEVEL_DISASSEMBLER
1342 node->offset = (jit_uword_t)_jit->pc.w - (jit_uword_t)prevw;
1343 prevw = _jit->pc.w;
1344#endif
1345 value = jit_classify(node->code);
1346 jit_regarg_set(node, value);
1347 switch (node->code) {
1348 case jit_code_align:
c0c16242
PC		 1349 /* Must align to a power of two */
1350 assert(!(node->u.w & (node->u.w - 1)));
1351 if ((word = _jit->pc.w & (node->u.w - 1)))
1352 nop(node->u.w - word);
4a71579b 1353 break;
79bfeef6
PC		 1354 case jit_code_skip:
1355 nop((node->u.w + 3) & ~3);
1356 break;
4a71579b
PC		 1357 case jit_code_note: case jit_code_name:
1358 node->u.w = _jit->pc.w;
1359 break;
1360 case jit_code_label:
1361 /* remember label is defined */
1362 node->flag |= jit_flag_patch;
1363 node->u.w = _jit->pc.w;
1364 break;
1365 case_rrr(add,);
1366 case_rrw(add,);
1367 case_rrr(addc,);
1368 case_rrw(addc,);
1369 case_rrr(addx,);
1370 case_rrw(addx,);
1371 case_rrr(sub,);
1372 case_rrw(sub,);
1373 case_rrr(subc,);
1374 case_rrw(subc,);
1375 case_rrr(subx,);
1376 case_rrw(subx,);
1377 case_rrw(rsb,);
1378 case_rrr(mul,);
1379 case_rrw(mul,);
ba86ff93
PC		 1380 case_rrr(hmul,);
1381 case_rrw(hmul,);
1382 case_rrr(hmul, _u);
1383 case_rrw(hmul, _u);
4a71579b
PC		 1384 case_rrrr(qmul,);
1385 case_rrrw(qmul,);
1386 case_rrrr(qmul, _u);
1387 case_rrrw(qmul, _u);
1388 case_rrr(div,);
1389 case_rrw(div,);
1390 case_rrr(div, _u);
1391 case_rrw(div, _u);
1392 case_rrrr(qdiv,);
1393 case_rrrw(qdiv,);
1394 case_rrrr(qdiv, _u);
1395 case_rrrw(qdiv, _u);
1396 case_rrr(rem,);
1397 case_rrw(rem,);
1398 case_rrr(rem, _u);
1399 case_rrw(rem, _u);
1400 case_rrr(and,);
1401 case_rrw(and,);
1402 case_rrr(or,);
1403 case_rrw(or,);
1404 case_rrr(xor,);
1405 case_rrw(xor,);
1406 case_rrr(lsh,);
1407 case_rrw(lsh,);
ba86ff93
PC		 1408 case_rrrr(qlsh,);
1409 case_rrrw(qlsh,);
1410 case_rrrr(qlsh, _u);
1411 case_rrrw(qlsh, _u);
4a71579b
PC		 1412 case_rrr(rsh,);
1413 case_rrw(rsh,);
1414 case_rrr(rsh, _u);
1415 case_rrw(rsh, _u);
ba86ff93
PC		 1416 case_rrrr(qrsh,);
1417 case_rrrw(qrsh,);
1418 case_rrrr(qrsh, _u);
1419 case_rrrw(qrsh, _u);
1420 case_rrr(lrot,);
1421 case_rrw(lrot,);
1422 case_rrr(rrot,);
1423 case_rrw(rrot,);
1424 case jit_code_extr:
1425 extr(rn(node->u.w), rn(node->v.w), node->w.q.l, node->w.q.h);
1426 break;
1427 case jit_code_extr_u:
1428 extr_u(rn(node->u.w), rn(node->v.w), node->w.q.l, node->w.q.h);
1429 break;
1430 case jit_code_depr:
1431 depr(rn(node->u.w), rn(node->v.w), node->w.q.l, node->w.q.h);
1432 break;
1433 case jit_code_depi:
1434 depi(rn(node->u.w), node->v.w, node->w.q.l, node->w.q.h);
1435 break;
4a71579b
PC		 1436 case_rr(ext, _c);
1437 case_rr(ext, _uc);
1438 case_rr(ext, _s);
1439 case_rr(ext, _us);
1440# if __WORDSIZE == 64
1441 case_rr(ext, _i);
1442 case_rr(ext, _ui);
1443# endif
1444 case_rr(hton, _us);
1445 case_rr(hton, _ui);
1446# if __WORDSIZE == 64
1447 case_rr(hton, _ul);
40a44dcb 1448# endif
ba3814c1
PC		 1449 case jit_code_bswapr_us:
1450 bswapr_us_lh(rn(node->u.w), rn(node->v.w), no_flag);
1451 break;
1452 case jit_code_bswapr_ui:
1453 bswapr_ui_lw(rn(node->u.w), rn(node->v.w), no_flag);
1454 break;
40a44dcb
PC		 1455# if __WORDSIZE == 64
1456 case_rr(bswap, _ul);
4a71579b
PC		 1457# endif
1458 case_rr(neg,);
1459 case_rr(com,);
79bfeef6
PC		 1460 case_rr(clo,);
1461 case_rr(clz,);
1462 case_rr(cto,);
1463 case_rr(ctz,);
ba86ff93
PC		 1464#define rbitr(r0, r1) fallback_rbit(r0, r1)
1465 case_rr(rbit,);
1466 case_rr(popcnt,);
ba3814c1
PC		 1467 case jit_code_casr:
1468 casr(rn(node->u.w), rn(node->v.w),
1469 rn(node->w.q.l), rn(node->w.q.h));
1470 break;
1471 case jit_code_casi:
1472 casi(rn(node->u.w), node->v.w,
1473 rn(node->w.q.l), rn(node->w.q.h));
1474 break;
e0659411
PC		 1475 case_rrr(movn,);
1476 case_rrr(movz,);
4a71579b
PC		 1477 case_rr(mov,);
1478 case jit_code_movi:
1479 if (node->flag & jit_flag_node) {
1480 temp = node->v.n;
1481 if (temp->code == jit_code_data ||
1482 (temp->code == jit_code_label &&
1483 (temp->flag & jit_flag_patch)))
1484 movi(rn(node->u.w), temp->u.w);
1485 else {
1486 assert(temp->code == jit_code_label ||
1487 temp->code == jit_code_epilog);
1488 word = movi_p(rn(node->u.w), node->v.w);
1489 patch(word, node);
1490 }
1491 }
1492 else
1493 movi(rn(node->u.w), node->v.w);
1494 break;
1495 case_rr(trunc, _f_i);
1496 case_rr(trunc, _d_i);
1497# if __WORDSIZE == 64
1498 case_rr(trunc, _f_l);
1499 case_rr(trunc, _d_l);
1500# endif
1501 case_rrr(lt,);
1502 case_rrw(lt,);
1503 case_rrr(lt, _u);
1504 case_rrw(lt, _u);
1505 case_rrr(le,);
1506 case_rrw(le,);
1507 case_rrr(le, _u);
1508 case_rrw(le, _u);
1509 case_rrr(eq,);
1510 case_rrw(eq,);
1511 case_rrr(ge,);
1512 case_rrw(ge,);
1513 case_rrr(ge, _u);
1514 case_rrw(ge, _u);
1515 case_rrr(gt,);
1516 case_rrw(gt,);
1517 case_rrr(gt, _u);
1518 case_rrw(gt, _u);
1519 case_rrr(ne,);
1520 case_rrw(ne,);
1521 case_rr(ld, _c);
1522 case_rw(ld, _c);
1523 case_brr(blt,);
1524 case_brw(blt,);
1525 case_brr(blt, _u);
1526 case_brw(blt, _u);
1527 case_brr(ble,);
1528 case_brw(ble,);
1529 case_brr(ble, _u);
1530 case_brw(ble, _u);
1531 case_brr(beq,);
1532 case_brw(beq,);
1533 case_brr(bge,);
1534 case_brw(bge,);
1535 case_brr(bge, _u);
1536 case_brw(bge, _u);
1537 case_brr(bgt,);
1538 case_brw(bgt,);
1539 case_brr(bgt, _u);
1540 case_brw(bgt, _u);
1541 case_brr(bne,);
1542 case_brw(bne,);
1543 case_brr(bms,);
1544 case_brw(bms,);
1545 case_brr(bmc,);
1546 case_brw(bmc,);
1547 case_brr(boadd,);
1548 case_brw(boadd,);
1549 case_brr(boadd, _u);
1550 case_brw(boadd, _u);
1551 case_brr(bxadd,);
1552 case_brw(bxadd,);
1553 case_brr(bxadd, _u);
1554 case_brw(bxadd, _u);
1555 case_brr(bosub,);
1556 case_brw(bosub,);
1557 case_brr(bosub, _u);
1558 case_brw(bosub, _u);
1559 case_brr(bxsub,);
1560 case_brw(bxsub,);
1561 case_brr(bxsub, _u);
1562 case_brw(bxsub, _u);
1563 case_rrr(ldx, _c);
1564 case_rrw(ldx, _c);
1565 case_rr(ld, _uc);
1566 case_rw(ld, _uc);
1567 case_rrr(ldx, _uc);
1568 case_rrw(ldx, _uc);
1569 case_rr(ld, _s);
1570 case_rw(ld, _s);
1571 case_rrr(ldx, _s);
1572 case_rrw(ldx, _s);
1573 case_rr(ld, _us);
1574 case_rw(ld, _us);
1575 case_rrr(ldx, _us);
1576 case_rrw(ldx, _us);
1577 case_rr(ld, _i);
1578 case_rw(ld, _i);
1579 case_rrr(ldx, _i);
1580 case_rrw(ldx, _i);
1581#if __WORDSIZE == 64
1582 case_rr(ld, _ui);
1583 case_rw(ld, _ui);
1584 case_rrr(ldx, _ui);
1585 case_rrw(ldx, _ui);
1586 case_rr(ld, _l);
1587 case_rw(ld, _l);
1588 case_rrr(ldx, _l);
1589 case_rrw(ldx, _l);
1590#endif
ba86ff93
PC		 1591#define unldr(r0, r1, i0) fallback_unldr(r0, r1, i0)
1592 case jit_code_unldr:
1593 unldr(rn(node->u.w), rn(node->v.w), node->w.w);
1594 break;
1595#define unldi(r0, i0, i1) fallback_unldi(r0, i0, i1)
1596 case jit_code_unldi:
1597 unldi(rn(node->u.w), node->v.w, node->w.w);
1598 break;
1599#define unldr_u(r0, r1, i0) fallback_unldr_u(r0, r1, i0)
1600 case jit_code_unldr_u:
1601 unldr_u(rn(node->u.w), rn(node->v.w), node->w.w);
1602 break;
1603#define unldi_u(r0, i0, i1) fallback_unldi_u(r0, i0, i1)
1604 case jit_code_unldi_u:
1605 unldi_u(rn(node->u.w), node->v.w, node->w.w);
1606 break;
4a71579b
PC		 1607 case_rr(st, _c);
1608 case_wr(st, _c);
1609 case_rrr(stx, _c);
1610 case_wrr(stx, _c);
1611 case_rr(st, _s);
1612 case_wr(st, _s);
1613 case_rrr(stx, _s);
1614 case_wrr(stx, _s);
1615 case_rr(st, _i);
1616 case_wr(st, _i);
1617 case_rrr(stx, _i);
1618 case_wrr(stx, _i);
1619#if __WORDSIZE == 64
1620 case_rr(st, _l);
1621 case_wr(st, _l);
1622 case_rrr(stx, _l);
1623 case_wrr(stx, _l);
1624#endif
ba86ff93
PC		 1625#define unstr(r0, r1, i0) fallback_unstr(r0, r1, i0)
1626 case jit_code_unstr:
1627 unstr(rn(node->u.w), rn(node->v.w), node->w.w);
1628 break;
1629#define unsti(i0, r0, i1) fallback_unsti(i0, r0, i1)
1630 case jit_code_unsti:
1631 unsti(node->u.w, rn(node->v.w), node->w.w);
1632 break;
4a71579b
PC		 1633 case_rr(mov, _f);
1634 case jit_code_movi_f:
1635 assert(node->flag & jit_flag_data);
1636 movi_f(rn(node->u.w), (jit_float32_t *)node->v.n->u.w);
1637 break;
1638 case_rr(ext, _f);
1639 case_rr(ext, _d_f);
1640 case_rr(abs, _f);
1641 case_rr(neg, _f);
1642 case_rr(sqrt, _f);
ba86ff93
PC		 1643 case_rqr(fma, _f);
1644 case_rqr(fms, _f);
1645 case_rqr(fnma, _f);
1646 case_rqr(fnms, _f);
4a71579b
PC		 1647 case_rrr(add, _f);
1648 case_rrf(add, _f, 32);
1649 case_rrr(sub, _f);
1650 case_rrf(sub, _f, 32);
1651 case_rrf(rsb, _f, 32);
1652 case_rrr(mul, _f);
1653 case_rrf(mul, _f, 32);
1654 case_rrr(div, _f);
1655 case_rrf(div, _f, 32);
1656 case_rrr(lt, _f);
1657 case_rrf(lt, _f, 32);
1658 case_rrr(le, _f);
1659 case_rrf(le, _f, 32);
1660 case_rrr(eq, _f);
1661 case_rrf(eq, _f, 32);
1662 case_rrr(ge, _f);
1663 case_rrf(ge, _f, 32);
1664 case_rrr(gt, _f);
1665 case_rrf(gt, _f, 32);
1666 case_rrr(ne, _f);
1667 case_rrf(ne, _f, 32);
1668 case_rrr(unlt, _f);
1669 case_rrf(unlt, _f, 32);
1670 case_rrr(unle, _f);
1671 case_rrf(unle, _f, 32);
1672 case_rrr(uneq, _f);
1673 case_rrf(uneq, _f, 32);
1674 case_rrr(unge, _f);
1675 case_rrf(unge, _f, 32);
1676 case_rrr(ungt, _f);
1677 case_rrf(ungt, _f, 32);
1678 case_rrr(ltgt, _f);
1679 case_rrf(ltgt, _f, 32);
1680 case_rrr(ord, _f);
1681 case_rrf(ord, _f, 32);
1682 case_rrr(unord, _f);
1683 case_rrf(unord, _f, 32);
1684 case_brr(blt, _f);
1685 case_brf(blt, _f, 32);
1686 case_brr(ble, _f);
1687 case_brf(ble, _f, 32);
1688 case_brr(beq, _f);
1689 case_brf(beq, _f, 32);
1690 case_brr(bge, _f);
1691 case_brf(bge, _f, 32);
1692 case_brr(bgt, _f);
1693 case_brf(bgt, _f, 32);
1694 case_brr(bne, _f);
1695 case_brf(bne, _f, 32);
1696 case_brr(bunlt, _f);
1697 case_brf(bunlt, _f, 32);
1698 case_brr(bunle, _f);
1699 case_brf(bunle, _f, 32);
1700 case_brr(buneq, _f);
1701 case_brf(buneq, _f, 32);
1702 case_brr(bunge, _f);
1703 case_brf(bunge, _f, 32);
1704 case_brr(bungt, _f);
1705 case_brf(bungt, _f, 32);
1706 case_brr(bltgt, _f);
1707 case_brf(bltgt, _f, 32);
1708 case_brr(bord, _f);
1709 case_brf(bord, _f, 32);
1710 case_brr(bunord, _f);
1711 case_brf(bunord, _f, 32);
1712 case_rr(ld, _f);
1713 case_rw(ld, _f);
1714 case_rrr(ldx, _f);
1715 case_rrw(ldx, _f);
ba86ff93
PC		 1716#define unldr_x(r0, r1, i0) fallback_unldr_x(r0, r1, i0)
1717 case jit_code_unldr_x:
1718 unldr_x(rn(node->u.w), rn(node->v.w), node->w.w);
1719 break;
1720#define unldi_x(r0, i0, i1) fallback_unldi_x(r0, i0, i1)
1721 case jit_code_unldi_x:
1722 unldi_x(rn(node->u.w), node->v.w, node->w.w);
1723 break;
4a71579b
PC		 1724 case_rr(st, _f);
1725 case_wr(st, _f);
1726 case_rrr(stx, _f);
1727 case_wrr(stx, _f);
ba86ff93
PC		 1728#define unstr_x(r0, r1, i0) fallback_unstr_x(r0, r1, i0)
1729 case jit_code_unstr_x:
1730 unstr_x(rn(node->u.w), rn(node->v.w), node->w.w);
1731 break;
1732#define unsti_x(i0, r0, i1) fallback_unsti_x(i0, r0, i1)
1733 case jit_code_unsti_x:
1734 unsti_x(node->u.w, rn(node->v.w), node->w.w);
1735 break;
4a71579b
PC		 1736 case_rr(mov, _d);
1737 case jit_code_movi_d:
1738 assert(node->flag & jit_flag_data);
1739 movi_d(rn(node->u.w), (jit_float64_t *)node->v.n->u.w);
1740 break;
1741 case_rr(ext, _d);
1742 case_rr(ext, _f_d);
1743 case_rr(abs, _d);
1744 case_rr(neg, _d);
1745 case_rr(sqrt, _d);
ba86ff93
PC		 1746 case_rqr(fma, _d);
1747 case_rqr(fms, _d);
1748 case_rqr(fnma, _d);
1749 case_rqr(fnms, _d);
4a71579b
PC		 1750 case_rrr(add, _d);
1751 case_rrf(add, _d, 64);
1752 case_rrr(sub, _d);
1753 case_rrf(sub, _d, 64);
1754 case_rrf(rsb, _d, 64);
1755 case_rrr(mul, _d);
1756 case_rrf(mul, _d, 64);
1757 case_rrr(div, _d);
1758 case_rrf(div, _d, 64);
1759 case_rrr(lt, _d);
1760 case_rrf(lt, _d, 64);
1761 case_rrr(le, _d);
1762 case_rrf(le, _d, 64);
1763 case_rrr(eq, _d);
1764 case_rrf(eq, _d, 64);
1765 case_rrr(ge, _d);
1766 case_rrf(ge, _d, 64);
1767 case_rrr(gt, _d);
1768 case_rrf(gt, _d, 64);
1769 case_rrr(ne, _d);
1770 case_rrf(ne, _d, 64);
1771 case_rrr(unlt, _d);
1772 case_rrf(unlt, _d, 64);
1773 case_rrr(unle, _d);
1774 case_rrf(unle, _d, 64);
1775 case_rrr(uneq, _d);
1776 case_rrf(uneq, _d, 64);
1777 case_rrr(unge, _d);
1778 case_rrf(unge, _d, 64);
1779 case_rrr(ungt, _d);
1780 case_rrf(ungt, _d, 64);
1781 case_rrr(ltgt, _d);
1782 case_rrf(ltgt, _d, 64);
1783 case_rrr(ord, _d);
1784 case_rrf(ord, _d, 64);
1785 case_rrr(unord, _d);
1786 case_rrf(unord, _d, 64);
1787 case_brr(blt, _d);
1788 case_brf(blt, _d, 64);
1789 case_brr(ble, _d);
1790 case_brf(ble, _d, 64);
1791 case_brr(beq, _d);
1792 case_brf(beq, _d, 64);
1793 case_brr(bge, _d);
1794 case_brf(bge, _d, 64);
1795 case_brr(bgt, _d);
1796 case_brf(bgt, _d, 64);
1797 case_brr(bne, _d);
1798 case_brf(bne, _d, 64);
1799 case_brr(bunlt, _d);
1800 case_brf(bunlt, _d, 64);
1801 case_brr(bunle, _d);
1802 case_brf(bunle, _d, 64);
1803 case_brr(buneq, _d);
1804 case_brf(buneq, _d, 64);
1805 case_brr(bunge, _d);
1806 case_brf(bunge, _d, 64);
1807 case_brr(bungt, _d);
1808 case_brf(bungt, _d, 64);
1809 case_brr(bltgt, _d);
1810 case_brf(bltgt, _d, 64);
1811 case_brr(bord, _d);
1812 case_brf(bord, _d, 64);
1813 case_brr(bunord, _d);
1814 case_brf(bunord, _d, 64);
1815 case_rr(ld, _d);
1816 case_rw(ld, _d);
1817 case_rrr(ldx, _d);
1818 case_rrw(ldx, _d);
1819 case_rr(st, _d);
1820 case_wr(st, _d);
1821 case_rrr(stx, _d);
1822 case_wrr(stx, _d);
1823 case jit_code_jmpr:
ba86ff93 1824 jit_check_frame();
4a71579b
PC		 1825 jmpr(rn(node->u.w));
1826 break;
1827 case jit_code_jmpi:
1828 if (node->flag & jit_flag_node) {
1829#if _CALL_AIXDESC
1830 if (_jit->pc.uc == _jit->code.ptr + sizeof(void*) * 3)
1831 _jitc->jump = 1;
1832#endif
1833 temp = node->u.n;
1834 assert(temp->code == jit_code_label ||
1835 temp->code == jit_code_epilog);
1836 if (temp->flag & jit_flag_patch)
1837 jmpi(temp->u.w);
1838 else {
79bfeef6
PC		 1839 word = _jit->code.length -
1840 (_jit->pc.uc - _jit->code.ptr);
1841 if (can_sign_extend_jump_p(word))
1842 word = jmpi(_jit->pc.w);
1843 else
1844 word = jmpi_p(_jit->pc.w);
4a71579b
PC		 1845 patch(word, node);
1846 }
1847 }
ba86ff93
PC		 1848 else {
1849 jit_check_frame();
ba3814c1 1850 jmpi(node->u.w);
ba86ff93 1851 }
4a71579b
PC		 1852 break;
1853 case jit_code_callr:
4a71579b 1854#if _CALL_SYSV
79bfeef6
PC		 1855# define xcallr(u, v) callr(u, v)
1856# define xcalli_p(u, v) calli_p(u, v)
1857# define xcalli(u, v) calli(u, v)
1858#else
1859# define xcallr(u, v) callr(u)
1860# define xcalli_p(u, v) calli_p(u)
1861# define xcalli(u, v) calli(u)
4a71579b 1862#endif
ba86ff93 1863 jit_check_frame();
79bfeef6 1864 xcallr(rn(node->u.w), !!(node->flag & jit_flag_varargs));
4a71579b
PC		 1865 break;
1866 case jit_code_calli:
79bfeef6 1867 value = !!(node->flag & jit_flag_varargs);
4a71579b 1868 if (node->flag & jit_flag_node) {
ba86ff93 1869 _jitc->function->need_return = 1;
4a71579b
PC		 1870 temp = node->u.n;
1871 assert(temp->code == jit_code_label ||
1872 temp->code == jit_code_epilog);
79bfeef6
PC		 1873 if (temp->flag & jit_flag_patch)
1874 xcalli(temp->u.w, value);
1875 else {
1876 word = _jit->code.length -
1877 (_jit->pc.uc - _jit->code.ptr);
4a71579b 1878#if _CALL_SYSV
79bfeef6
PC		 1879 if (can_sign_extend_jump_p(word + value * 4))
1880 word = xcalli(_jit->pc.w, value);
1881 else
4a71579b 1882#endif
79bfeef6 1883 word = xcalli_p(_jit->pc.w, value);
4a71579b 1884 patch(word, node);
79bfeef6 1885 }
4a71579b 1886 }
ba86ff93
PC		 1887 else {
1888 jit_check_frame();
79bfeef6 1889 xcalli(node->u.w, value);
ba86ff93 1890 }
4a71579b
PC		 1891 break;
1892 case jit_code_prolog:
1893 _jitc->function = _jitc->functions.ptr + node->w.w;
1894 undo.node = node;
1895 undo.word = _jit->pc.w;
79bfeef6 1896 memcpy(&undo.func, _jitc->function, sizeof(undo.func));
4a71579b
PC		 1897#if DEVEL_DISASSEMBLER
1898 undo.prevw = prevw;
1899#endif
1900 undo.patch_offset = _jitc->patches.offset;
1901#if _CALL_AIXDESC
1902 undo.prolog_offset = _jitc->prolog.offset;
1903#endif
1904 restart_function:
1905 _jitc->again = 0;
1906#if _CALL_AIXDESC
1907 if (_jitc->jump && !_jitc->function->assume_frame) {
1908 /* remember prolog to hide offset adjustment for a jump
1909 * to the start of a function, what is expected to be
1910 * a common practice as first jit instruction */
1911 if (_jitc->prolog.offset >= _jitc->prolog.length) {
1912 _jitc->prolog.length += 16;
1913 jit_realloc((jit_pointer_t *)&_jitc->prolog.ptr,
1914 (_jitc->prolog.length - 16) *
1915 sizeof(jit_word_t),
1916 _jitc->prolog.length * sizeof(jit_word_t));
1917 }
1918 _jitc->prolog.ptr[_jitc->prolog.offset++] = _jit->pc.w;
1919 /* function descriptor */
1920 word = _jit->pc.w + sizeof(void*) * 3;
1921 iw(word); /* addr */
1922 iw(0); /* toc */
1923 iw(0); /* env */
1924 }
1925#endif
1926 prolog(node);
1927 break;
1928 case jit_code_epilog:
1929 assert(_jitc->function == _jitc->functions.ptr + node->w.w);
1930 if (_jitc->again) {
1931 for (temp = undo.node->next;
1932 temp != node; temp = temp->next) {
1933 if (temp->code == jit_code_label ||
1934 temp->code == jit_code_epilog)
1935 temp->flag &= ~jit_flag_patch;
1936 }
1937 temp->flag &= ~jit_flag_patch;
1938 node = undo.node;
1939 _jit->pc.w = undo.word;
79bfeef6
PC		 1940 /* undo.func.self.aoff and undo.func.regset should not
1941 * be undone, as they will be further updated, and are
1942 * the reason of the undo. */
1943 undo.func.self.aoff = _jitc->function->frame +
1944 _jitc->function->self.aoff;
ba86ff93
PC		 1945 undo.func.need_frame = _jitc->function->need_frame;
1946 undo.func.need_stack = _jitc->function->need_stack;
1947 undo.func.need_return = _jitc->function->need_return;
79bfeef6
PC		 1948 jit_regset_set(&undo.func.regset, &_jitc->function->regset);
1949 /* allocar information also does not need to be undone */
1950 undo.func.aoffoff = _jitc->function->aoffoff;
1951 undo.func.allocar = _jitc->function->allocar;
ba86ff93
PC		 1952 /* cvt_offset must also not be undone */
1953 undo.func.cvt_offset = _jitc->function->cvt_offset;
79bfeef6 1954 memcpy(_jitc->function, &undo.func, sizeof(undo.func));
4a71579b
PC		 1955#if DEVEL_DISASSEMBLER
1956 prevw = undo.prevw;
1957#endif
1958 _jitc->patches.offset = undo.patch_offset;
1959#if _CALL_AIXDESC
1960 _jitc->prolog.offset = undo.prolog_offset;
1961#endif
1962 goto restart_function;
1963 }
1964 /* remember label is defined */
1965 node->flag |= jit_flag_patch;
1966 node->u.w = _jit->pc.w;
1967 epilog(node);
1968 _jitc->function = NULL;
1969 break;
ba86ff93
PC		 1970 case jit_code_movr_w_f:
1971 movr_w_f(rn(node->u.w), rn(node->v.w));
1972 break;
1973 case jit_code_movr_f_w:
1974 movr_f_w(rn(node->u.w), rn(node->v.w));
1975 break;
1976 case jit_code_movi_f_w:
1977 assert(node->flag & jit_flag_data);
1978 movi_f_w(rn(node->u.w), *(jit_float32_t *)node->v.n->u.w);
1979 break;
1980 case jit_code_movi_w_f:
1981 movi_w_f(rn(node->u.w), node->v.w);
1982 break;
1983#if __WORDSIZE == 64
1984 case jit_code_movr_d_w:
1985 movr_d_w(rn(node->u.w), rn(node->v.w));
1986 break;
1987 case jit_code_movi_d_w:
1988 assert(node->flag & jit_flag_data);
1989 movi_d_w(rn(node->u.w), *(jit_float64_t *)node->v.n->u.w);
1990 break;
1991 case jit_code_movr_w_d:
1992 movr_w_d(rn(node->u.w), rn(node->v.w));
1993 break;
1994 case jit_code_movi_w_d:
1995 movi_w_d(rn(node->u.w), node->v.w);
1996 break;
1997#else
1998 case jit_code_movr_ww_d:
1999 movr_ww_d(rn(node->u.w), rn(node->v.w), rn(node->w.w));
2000 break;
2001 case jit_code_movr_d_ww:
2002 movr_d_ww(rn(node->u.w), rn(node->v.w), rn(node->w.w));
2003 break;
2004 case jit_code_movi_d_ww:
2005 assert(node->flag & jit_flag_data);
2006 movi_d_ww(rn(node->u.w), rn(node->v.w),
2007 *(jit_float64_t *)node->w.n->u.w);
2008 break;
2009 case jit_code_movi_ww_d:
2010 movi_ww_d(rn(node->u.w), node->v.w, node->w.w);
2011 break;
2012#endif
4a71579b
PC		 2013 case jit_code_va_start:
2014 vastart(rn(node->u.w));
2015 break;
2016 case jit_code_va_arg:
2017 vaarg(rn(node->u.w), rn(node->v.w));
2018 break;
2019 case jit_code_va_arg_d:
2020 vaarg_d(rn(node->u.w), rn(node->v.w));
2021 break;
79bfeef6 2022 case jit_code_live: case jit_code_ellipsis:
4a71579b
PC		 2023 case jit_code_va_push:
2024 case jit_code_allocai: case jit_code_allocar:
79bfeef6
PC		 2025 case jit_code_arg_c: case jit_code_arg_s:
2026 case jit_code_arg_i:
2027# if __WORDSIZE == 64
2028 case jit_code_arg_l:
2029# endif
4a71579b
PC		 2030 case jit_code_arg_f: case jit_code_arg_d:
2031 case jit_code_va_end:
2032 case jit_code_ret:
79bfeef6
PC		 2033 case jit_code_retr_c: case jit_code_reti_c:
2034 case jit_code_retr_uc: case jit_code_reti_uc:
2035 case jit_code_retr_s: case jit_code_reti_s:
2036 case jit_code_retr_us: case jit_code_reti_us:
2037 case jit_code_retr_i: case jit_code_reti_i:
2038#if __WORDSIZE == 64
2039 case jit_code_retr_ui: case jit_code_reti_ui:
2040 case jit_code_retr_l: case jit_code_reti_l:
2041#endif
4a71579b
PC		 2042 case jit_code_retr_f: case jit_code_reti_f:
2043 case jit_code_retr_d: case jit_code_reti_d:
2044 case jit_code_getarg_c: case jit_code_getarg_uc:
2045 case jit_code_getarg_s: case jit_code_getarg_us:
2046 case jit_code_getarg_i:
2047#if __WORDSIZE == 64
2048 case jit_code_getarg_ui: case jit_code_getarg_l:
2049#endif
2050 case jit_code_getarg_f: case jit_code_getarg_d:
79bfeef6
PC		 2051 case jit_code_putargr_c: case jit_code_putargi_c:
2052 case jit_code_putargr_uc: case jit_code_putargi_uc:
2053 case jit_code_putargr_s: case jit_code_putargi_s:
2054 case jit_code_putargr_us: case jit_code_putargi_us:
2055 case jit_code_putargr_i: case jit_code_putargi_i:
2056#if __WORDSIZE == 64
2057 case jit_code_putargr_ui: case jit_code_putargi_ui:
2058 case jit_code_putargr_l: case jit_code_putargi_l:
2059#endif
4a71579b
PC		 2060 case jit_code_putargr_f: case jit_code_putargi_f:
2061 case jit_code_putargr_d: case jit_code_putargi_d:
79bfeef6
PC		 2062 case jit_code_pushargr_c: case jit_code_pushargi_c:
2063 case jit_code_pushargr_uc: case jit_code_pushargi_uc:
2064 case jit_code_pushargr_s: case jit_code_pushargi_s:
2065 case jit_code_pushargr_us: case jit_code_pushargi_us:
2066 case jit_code_pushargr_i: case jit_code_pushargi_i:
2067#if __WORDSIZE == 64
2068 case jit_code_pushargr_ui: case jit_code_pushargi_ui:
2069 case jit_code_pushargr_l: case jit_code_pushargi_l:
2070#endif
4a71579b
PC		 2071 case jit_code_pushargr_f: case jit_code_pushargi_f:
2072 case jit_code_pushargr_d: case jit_code_pushargi_d:
2073 case jit_code_retval_c: case jit_code_retval_uc:
2074 case jit_code_retval_s: case jit_code_retval_us:
2075 case jit_code_retval_i:
2076#if __WORDSIZE == 64
2077 case jit_code_retval_ui: case jit_code_retval_l:
2078#endif
2079 case jit_code_retval_f: case jit_code_retval_d:
2080 case jit_code_prepare:
2081 case jit_code_finishr: case jit_code_finishi:
ba86ff93
PC		 2082 case jit_code_negi_f: case jit_code_absi_f:
2083 case jit_code_sqrti_f: case jit_code_negi_d:
2084 case jit_code_absi_d: case jit_code_sqrti_d:
2085 break;
2086 case jit_code_negi:
2087 negi(rn(node->u.w), node->v.w);
2088 break;
2089 case jit_code_comi:
2090 comi(rn(node->u.w), node->v.w);
2091 break;
2092 case jit_code_exti_c:
2093 exti_c(rn(node->u.w), node->v.w);
2094 break;
2095 case jit_code_exti_uc:
2096 exti_uc(rn(node->u.w), node->v.w);
2097 break;
2098 case jit_code_exti_s:
2099 exti_s(rn(node->u.w), node->v.w);
2100 break;
2101 case jit_code_exti_us:
2102 exti_us(rn(node->u.w), node->v.w);
2103 break;
2104 case jit_code_bswapi_us:
2105 bswapi_us(rn(node->u.w), node->v.w);
2106 break;
2107 case jit_code_bswapi_ui:
2108 bswapi_ui(rn(node->u.w), node->v.w);
2109 break;
2110 case jit_code_htoni_us:
2111 htoni_us(rn(node->u.w), node->v.w);
2112 break;
2113 case jit_code_htoni_ui:
2114 htoni_ui(rn(node->u.w), node->v.w);
2115 break;
2116#if __WORDSIZE == 64
2117 case jit_code_exti_i:
2118 exti_i(rn(node->u.w), node->v.w);
2119 break;
2120 case jit_code_exti_ui:
2121 exti_ui(rn(node->u.w), node->v.w);
2122 break;
2123 case jit_code_bswapi_ul:
2124 bswapi_ul(rn(node->u.w), node->v.w);
2125 break;
2126 case jit_code_htoni_ul:
2127 htoni_ul(rn(node->u.w), node->v.w);
2128 break;
2129#endif
2130 case jit_code_cloi:
2131 cloi(rn(node->u.w), node->v.w);
2132 break;
2133 case jit_code_clzi:
2134 clzi(rn(node->u.w), node->v.w);
2135 break;
2136 case jit_code_ctoi:
2137 ctoi(rn(node->u.w), node->v.w);
2138 break;
2139 case jit_code_ctzi:
2140 ctzi(rn(node->u.w), node->v.w);
2141 break;
2142 case jit_code_rbiti:
2143 rbiti(rn(node->u.w), node->v.w);
2144 break;
2145 case jit_code_popcnti:
2146 popcnti(rn(node->u.w), node->v.w);
2147 break;
2148 case jit_code_exti:
2149 exti(rn(node->u.w), node->v.w, node->w.q.l, node->w.q.h);
2150 break;
2151 case jit_code_exti_u:
2152 exti_u(rn(node->u.w), node->v.w, node->w.q.l, node->w.q.h);
4a71579b
PC		 2153 break;
2154 default:
2155 abort();
2156 }
2157 jit_regarg_clr(node, value);
2158 assert(_jitc->regarg == 0 && _jitc->synth == 0);
2159 /* update register live state */
2160 jit_reglive(node);
ba3814c1
PC		 2161
2162 no_flag = !(node->flag & jit_flag_patch);
4a71579b
PC		 2163 }
2164#undef case_brf
2165#undef case_brw
2166#undef case_brr
2167#undef case_wrr
2168#undef case_rrf
2169#undef case_rrw
2170#undef case_rrr
2171#undef case_wr
2172#undef case_rw
2173#undef case_rr
2174
2175 for (offset = 0; offset < _jitc->patches.offset; offset++) {
2176 node = _jitc->patches.ptr[offset].node;
2177 word = node->code == jit_code_movi ? node->v.n->u.w : node->u.n->u.w;
2178 patch_at(_jitc->patches.ptr[offset].inst, word);
2179 }
2180
2181 jit_flush(_jit->code.ptr, _jit->pc.uc);
2182
2183 return (_jit->code.ptr);
2184}
2185
2186#define CODE 1
2187# include "jit_ppc-cpu.c"
2188# include "jit_ppc-fpu.c"
ba86ff93 2189# include "jit_fallback.c"
4a71579b
PC		 2190#undef CODE
2191
2192void
2193jit_flush(void *fptr, void *tptr)
2194{
2195#if defined(__GNUC__)
2196 jit_word_t f, t, s;
2197
2198 s = sysconf(_SC_PAGE_SIZE);
2199 f = (jit_word_t)fptr & -s;
2200 t = (((jit_word_t)tptr) + s - 1) & -s;
2201 __clear_cache((void *)f, (void *)t);
2202#endif
2203}
2204
2205void
2206_emit_ldxi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
2207{
2208#if __WORDSIZE == 32
2209 ldxi_i(rn(r0), rn(r1), i0);
2210#else
2211 ldxi_l(rn(r0), rn(r1), i0);
2212#endif
2213}
2214
2215void
2216_emit_stxi(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
2217{
2218#if __WORDSIZE == 32
2219 stxi_i(i0, rn(r0), rn(r1));
2220#else
2221 stxi_l(i0, rn(r0), rn(r1));
2222#endif
2223}
2224
2225void
2226_emit_ldxi_d(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
2227{
2228 ldxi_d(rn(r0), rn(r1), i0);
2229}
2230
2231void
2232_emit_stxi_d(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
2233{
2234 stxi_d(i0, rn(r0), rn(r1));
2235}
2236
2237static void
2238_patch(jit_state_t *_jit, jit_word_t instr, jit_node_t *node)
2239{
2240 jit_int32_t flag;
2241
2242 assert(node->flag & jit_flag_node);
2243 if (node->code == jit_code_movi)
2244 flag = node->v.n->flag;
2245 else
2246 flag = node->u.n->flag;
2247 assert(!(flag & jit_flag_patch));
2248 if (_jitc->patches.offset >= _jitc->patches.length) {
2249 jit_realloc((jit_pointer_t *)&_jitc->patches.ptr,
2250 _jitc->patches.length * sizeof(jit_patch_t),
2251 (_jitc->patches.length + 1024) * sizeof(jit_patch_t));
2252 _jitc->patches.length += 1024;
2253 }
2254 _jitc->patches.ptr[_jitc->patches.offset].inst = instr;
2255 _jitc->patches.ptr[_jitc->patches.offset].node = node;
2256 ++_jitc->patches.offset;
2257}
ARM optimized PCSX fork