notaz.gp2x.de / pcsx_rearmed.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
add a thp-based huge page alloc fallback
[pcsx_rearmed.git] / deps / lightning / lib / jit_riscv.c
CommitLineData
4a71579b 1/*
79bfeef6 2 * Copyright (C) 2019-2023 Free Software Foundation, Inc.
4a71579b
PC		 3 *
4 * This file is part of GNU lightning.
5 *
6 * GNU lightning is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU Lesser General Public License as published
8 * by the Free Software Foundation; either version 3, or (at your option)
9 * any later version.
10 *
11 * GNU lightning is distributed in the hope that it will be useful, but
12 * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
13 * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public
14 * License for more details.
15 *
16 * Authors:
17 * Paulo Cesar Pereira de Andrade
18 */
19
79bfeef6
PC		 20/* callee save + variadic arguments
21 * align16(ra+fp+s[1-9]+s10+s11+fs[0-9]+fs10+fs11)+align16(a[0-7]) */
22#define stack_framesize (208 + 64)
23
4a71579b
PC		 24#define jit_arg_reg_p(i) ((i) >= 0 && (i) < 8)
25#define jit_arg_f_reg_p(i) ((i) >= 0 && (i) < 8)
26
27/*
28 * Types
29 */
30typedef jit_pointer_t jit_va_list_t;
31
32/*
33 * Prototypes
34 */
79bfeef6
PC		 35#define compute_framesize() _compute_framesize(_jit)
36static void _compute_framesize(jit_state_t*);
c0c16242
PC		 37#if __WORDSIZE == 64
38# define load_const(r0, i0) _load_const(_jit, r0, i0)
39static void _load_const(jit_state_t*, jit_int32_t, jit_word_t);
40static jit_word_t hash_const(jit_word_t);
41# define put_const(i0) _put_const(_jit, i0)
42static void _put_const(jit_state_t*, jit_word_t);
43# define get_const(i0) _get_const(_jit, i0)
44static jit_word_t _get_const(jit_state_t*, jit_word_t);
45#endif
4a71579b
PC		 46#define patch(instr, node) _patch(_jit, instr, node)
47static void _patch(jit_state_t*,jit_word_t,jit_node_t*);
48
49#define PROTO 1
50# include "jit_riscv-cpu.c"
51# include "jit_riscv-fpu.c"
79bfeef6 52# include "jit_fallback.c"
4a71579b
PC		 53#undef PROTO
54
55/*
56 * Initialization
57 */
ba86ff93 58jit_cpu_t jit_cpu;
4a71579b
PC		 59jit_register_t _rvs[] = {
60 { 0x00, "zero" },
61 { 0x01, "ra" },
62 { 0x02, "sp" },
63 { 0x03, "gp" },
64#if 0 /* Pretend it does not exist, so _NOREG can be used in
65 * a 64 bit bitmask */
66 { 0x04, "tp" },
67#endif
68 { rc(gpr) | 0x05, "t0" },
69 { rc(gpr) | 0x06, "t1" },
70 { rc(gpr) | 0x07, "t2" },
71 { rc(gpr) | 0x1c, "t3" },
72 { rc(gpr) | 0x1d, "t4" },
73 { rc(gpr) | 0x1e, "t5" },
74 { rc(gpr) | 0x1f, "t6" },
75 { 0x08, "fp" },
76 { rc(sav) | rc(gpr) | 0x09, "s1" },
77 { rc(sav) | rc(gpr) | 0x12, "s2" },
78 { rc(sav) | rc(gpr) | 0x13, "s3" },
79 { rc(sav) | rc(gpr) | 0x14, "s4" },
80 { rc(sav) | rc(gpr) | 0x15, "s5" },
81 { rc(sav) | rc(gpr) | 0x16, "s6" },
82 { rc(sav) | rc(gpr) | 0x17, "s7" },
83 { rc(sav) | rc(gpr) | 0x18, "s8" },
84 { rc(sav) | rc(gpr) | 0x19, "s9" },
85 { rc(sav) | rc(gpr) | 0x1a, "s10" },
86 { rc(sav) | rc(gpr) | 0x1b, "s11" },
87 { rc(arg) | rc(gpr) | 0x11, "a7" },
88 { rc(arg) | rc(gpr) | 0x10, "a6" },
89 { rc(arg) | rc(gpr) | 0x0f, "a5" },
90 { rc(arg) | rc(gpr) | 0x0e, "a4" },
91 { rc(arg) | rc(gpr) | 0x0d, "a3" },
92 { rc(arg) | rc(gpr) | 0x0c, "a2" },
93 { rc(arg) | rc(gpr) | 0x0b, "a1" },
94 { rc(arg) | rc(gpr) | 0x0a, "a0" },
95 { rc(fpr) | 0x00, "ft0" },
96 { rc(fpr) | 0x01, "ft1" },
97 { rc(fpr) | 0x02, "ft2" },
98 { rc(fpr) | 0x03, "ft3" },
99 { rc(fpr) | 0x04, "ft4" },
100 { rc(fpr) | 0x05, "ft5" },
101 { rc(fpr) | 0x06, "ft6" },
102 { rc(fpr) | 0x07, "ft7" },
103 { rc(fpr) | 0x1c, "ft8" },
104 { rc(fpr) | 0x1d, "ft9" },
105 { rc(fpr) | 0x1e, "ft10" },
106 { rc(fpr) | 0x1f, "ft11" },
107 { rc(sav) | rc(fpr) | 0x08, "fs0" },
108 { rc(sav) | rc(fpr) | 0x09, "fs1" },
109 { rc(sav) | rc(fpr) | 0x12, "fs2" },
110 { rc(sav) | rc(fpr) | 0x13, "fs3" },
111 { rc(sav) | rc(fpr) | 0x14, "fs4" },
112 { rc(sav) | rc(fpr) | 0x15, "fs5" },
113 { rc(sav) | rc(fpr) | 0x16, "fs6" },
114 { rc(sav) | rc(fpr) | 0x17, "fs7" },
115 { rc(sav) | rc(fpr) | 0x18, "fs8" },
116 { rc(sav) | rc(fpr) | 0x19, "fs9" },
117 { rc(sav) | rc(fpr) | 0x1a, "fs10" },
118 { rc(sav) | rc(fpr) | 0x1b, "fs11" },
119 { rc(arg) | rc(fpr) | 0x11, "fa7" },
120 { rc(arg) | rc(fpr) | 0x10, "fa6" },
121 { rc(arg) | rc(fpr) | 0x0f, "fa5" },
122 { rc(arg) | rc(fpr) | 0x0e, "fa4" },
123 { rc(arg) | rc(fpr) | 0x0d, "fa3" },
124 { rc(arg) | rc(fpr) | 0x0c, "fa2" },
125 { rc(arg) | rc(fpr) | 0x0b, "fa1" },
126 { rc(arg) | rc(fpr) | 0x0a, "fa0" },
127 { _NOREG, "<none>" },
128};
129
79bfeef6
PC		 130static jit_int32_t iregs[] = {
131 _S1, _S2, _S3, _S4, _S5, _S6, _S7, _S8, _S9, _S10, _S11
132};
133
134static jit_int32_t fregs[] = {
135 _FS0, _FS1, _FS2, _FS3, _FS4, _FS5, _FS6, _FS7, _FS8, _FS9, _FS10, _FS11
136};
137
4a71579b
PC		 138/*
139 * Implementation
140 */
141void
142jit_get_cpu(void)
143{
ba86ff93
PC		 144 /* By default generate extra instructions for unaligned load/store. */
145 jit_cpu.unaligned = 0;
4a71579b
PC		 146}
147
148void
149_jit_init(jit_state_t *_jit)
150{
151 _jitc->reglen = jit_size(_rvs) - 1;
152 jit_carry = _NOREG;
153}
154
155void
156_jit_prolog(jit_state_t *_jit)
157{
158 jit_int32_t offset;
159
160 if (_jitc->function)
161 jit_epilog();
162 assert(jit_regset_cmp_ui(&_jitc->regarg, 0) == 0);
163 jit_regset_set_ui(&_jitc->regsav, 0);
164 offset = _jitc->functions.offset;
165 if (offset >= _jitc->functions.length) {
166 jit_realloc((jit_pointer_t *)&_jitc->functions.ptr,
167 _jitc->functions.length * sizeof(jit_function_t),
168 (_jitc->functions.length + 16) * sizeof(jit_function_t));
169 _jitc->functions.length += 16;
170 }
171 _jitc->function = _jitc->functions.ptr + _jitc->functions.offset++;
172 _jitc->function->self.size = stack_framesize;
173 _jitc->function->self.argi = _jitc->function->self.argf =
174 _jitc->function->self.alen = 0;
175 _jitc->function->self.aoff = 0;
176 _jitc->function->self.call = jit_call_default;
177 jit_alloc((jit_pointer_t *)&_jitc->function->regoff,
178 _jitc->reglen * sizeof(jit_int32_t));
179
180 /* _no_link here does not mean the jit_link() call can be removed
181 * by rewriting as:
182 * _jitc->function->prolog = jit_new_node(jit_code_prolog);
183 */
184 _jitc->function->prolog = jit_new_node_no_link(jit_code_prolog);
185 jit_link(_jitc->function->prolog);
186 _jitc->function->prolog->w.w = offset;
187 _jitc->function->epilog = jit_new_node_no_link(jit_code_epilog);
188 /* u: label value
189 * v: offset in blocks vector
190 * w: offset in functions vector
191 */
192 _jitc->function->epilog->w.w = offset;
193
194 jit_regset_new(&_jitc->function->regset);
195}
196
197jit_int32_t
198_jit_allocai(jit_state_t *_jit, jit_int32_t length)
199{
200 assert(_jitc->function);
79bfeef6 201 jit_check_frame();
4a71579b
PC		 202 switch (length) {
203 case 0: case 1: break;
204 case 2: _jitc->function->self.aoff &= -2; break;
205 case 3: case 4: _jitc->function->self.aoff &= -4; break;
206 default: _jitc->function->self.aoff &= -8; break;
207 }
208 _jitc->function->self.aoff -= length;
209 if (!_jitc->realize) {
210 jit_inc_synth_ww(allocai, _jitc->function->self.aoff, length);
211 jit_dec_synth();
212 }
213 return (_jitc->function->self.aoff);
214}
215
216void
217_jit_allocar(jit_state_t *_jit, jit_int32_t u, jit_int32_t v)
218{
219 jit_int32_t r0;
220 assert(_jitc->function);
221 jit_inc_synth_ww(allocar, u, v);
222 if (!_jitc->function->allocar) {
223 _jitc->function->aoffoff = jit_allocai(sizeof(jit_int32_t));
224 _jitc->function->allocar = 1;
225 }
226 r0 = jit_get_reg(jit_class_gpr);
227 jit_negr(r0, v);
228 jit_andi(r0, r0, -16);
229 jit_ldxi_i(u, JIT_FP, _jitc->function->aoffoff);
230 jit_addr(u, u, r0);
231 jit_addr(JIT_SP, JIT_SP, r0);
232 jit_stxi_i(_jitc->function->aoffoff, JIT_FP, u);
233 jit_unget_reg(r0);
234 jit_dec_synth();
235}
236
237void
238_jit_ret(jit_state_t *_jit)
239{
240 jit_node_t *instr;
241 assert(_jitc->function);
242 jit_inc_synth(ret);
243 /* jump to epilog */
244 instr = jit_jmpi();
245 jit_patch_at(instr, _jitc->function->epilog);
246 jit_dec_synth();
247}
248
249void
79bfeef6 250_jit_retr(jit_state_t *_jit, jit_int32_t u, jit_code_t code)
4a71579b 251{
79bfeef6
PC		 252 jit_code_inc_synth_w(code, u);
253 jit_movr(JIT_RET, u);
4a71579b
PC		 254 jit_ret();
255 jit_dec_synth();
256}
257
258void
79bfeef6 259_jit_reti(jit_state_t *_jit, jit_word_t u, jit_code_t code)
4a71579b 260{
79bfeef6 261 jit_code_inc_synth_w(code, u);
4a71579b
PC		 262 jit_movi(JIT_RET, u);
263 jit_ret();
264 jit_dec_synth();
265}
266
267void
268_jit_retr_f(jit_state_t *_jit, jit_int32_t u)
269{
270 jit_inc_synth_w(retr_f, u);
271 if (u != JIT_FRET)
272 jit_movr_f(JIT_FRET, u);
273 else
274 jit_live(JIT_FRET);
275 jit_ret();
276 jit_dec_synth();
277}
278
279void
280_jit_reti_f(jit_state_t *_jit, jit_float32_t u)
281{
282 jit_inc_synth_f(reti_f, u);
283 jit_movi_f(JIT_FRET, u);
284 jit_ret();
285 jit_dec_synth();
286}
287
288void
289_jit_retr_d(jit_state_t *_jit, jit_int32_t u)
290{
291 jit_inc_synth_w(retr_d, u);
292 if (u != JIT_FRET)
293 jit_movr_d(JIT_FRET, u);
294 else
295 jit_live(JIT_FRET);
296 jit_ret();
297 jit_dec_synth();
298}
299
300void
301_jit_reti_d(jit_state_t *_jit, jit_float64_t u)
302{
303 jit_inc_synth_d(reti_d, u);
304 jit_movi_d(JIT_FRET, u);
305 jit_ret();
306 jit_dec_synth();
307}
308
309void
310_jit_epilog(jit_state_t *_jit)
311{
312 assert(_jitc->function);
313 assert(_jitc->function->epilog->next == NULL);
314 jit_link(_jitc->function->epilog);
315 _jitc->function = NULL;
316}
317
318jit_bool_t
319_jit_arg_register_p(jit_state_t *_jit, jit_node_t *u)
320{
79bfeef6 321 if (u->code >= jit_code_arg_c && u->code <= jit_code_arg)
4a71579b
PC		 322 return (jit_arg_reg_p(u->u.w));
323 assert(u->code == jit_code_arg_f || u->code == jit_code_arg_d);
79bfeef6 324 return (jit_arg_f_reg_p(u->u.w) || jit_arg_reg_p(u->u.w - 8));
4a71579b
PC		 325}
326
327void
328_jit_ellipsis(jit_state_t *_jit)
329{
330 jit_inc_synth(ellipsis);
79bfeef6 331 jit_check_frame();
4a71579b
PC		 332 if (_jitc->prepare) {
333 jit_link_prepare();
334 assert(!(_jitc->function->call.call & jit_call_varargs));
335 _jitc->function->call.call |= jit_call_varargs;
336 }
337 else {
338 jit_link_prolog();
339 assert(!(_jitc->function->self.call & jit_call_varargs));
340 _jitc->function->self.call |= jit_call_varargs;
341 _jitc->function->vagp = _jitc->function->self.argi;
342 }
343 jit_dec_synth();
344}
345
346void
347_jit_va_push(jit_state_t *_jit, jit_int32_t u)
348{
349 jit_inc_synth_w(va_push, u);
350 jit_pushargr(u);
351 jit_dec_synth();
352}
353
354jit_node_t *
79bfeef6 355_jit_arg(jit_state_t *_jit, jit_code_t code)
4a71579b
PC		 356{
357 jit_node_t *node;
358 jit_int32_t offset;
359 assert(_jitc->function);
360 assert(!(_jitc->function->self.call & jit_call_varargs));
79bfeef6
PC		 361#if STRONG_TYPE_CHECKING
362 assert(code >= jit_code_arg_c && code <= jit_code_arg);
363#endif
4a71579b
PC		 364 if (jit_arg_reg_p(_jitc->function->self.argi))
365 offset = _jitc->function->self.argi++;
366 else {
367 offset = _jitc->function->self.size;
368 _jitc->function->self.size += sizeof(jit_word_t);
79bfeef6 369 jit_check_frame();
4a71579b 370 }
79bfeef6 371 node = jit_new_node_ww(code, offset,
4a71579b
PC		 372 ++_jitc->function->self.argn);
373 jit_link_prolog();
374 return (node);
375}
376
377jit_node_t *
378_jit_arg_f(jit_state_t *_jit)
379{
380 jit_node_t *node;
381 jit_int32_t offset;
382 assert(_jitc->function);
383 assert(!(_jitc->function->self.call & jit_call_varargs));
384 if (jit_arg_f_reg_p(_jitc->function->self.argf))
385 offset = _jitc->function->self.argf++;
386 else if (jit_arg_reg_p(_jitc->function->self.argi)) {
387 offset = _jitc->function->self.argi++;
388 offset += 8;
389 }
390 else {
391 offset = _jitc->function->self.size;
392 _jitc->function->self.size += sizeof(jit_word_t);
79bfeef6 393 jit_check_frame();
4a71579b
PC		 394 }
395 node = jit_new_node_ww(jit_code_arg_f, offset,
396 ++_jitc->function->self.argn);
397 jit_link_prolog();
398 return (node);
399}
400
401jit_node_t *
402_jit_arg_d(jit_state_t *_jit)
403{
404 jit_node_t *node;
405 jit_int32_t offset;
406 assert(_jitc->function);
407 assert(!(_jitc->function->self.call & jit_call_varargs));
408 if (jit_arg_f_reg_p(_jitc->function->self.argf))
409 offset = _jitc->function->self.argf++;
410 else if (jit_arg_reg_p(_jitc->function->self.argi)) {
411 offset = _jitc->function->self.argi++;
412 offset += 8;
413 }
414 else {
415 offset = _jitc->function->self.size;
416 _jitc->function->self.size += sizeof(jit_word_t);
79bfeef6 417 jit_check_frame();
4a71579b
PC		 418 }
419 node = jit_new_node_ww(jit_code_arg_d, offset,
420 ++_jitc->function->self.argn);
421 jit_link_prolog();
422 return (node);
423}
424
425void
426_jit_getarg_c(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
427{
79bfeef6 428 assert_arg_type(v->code, jit_code_arg_c);
4a71579b
PC		 429 jit_inc_synth_wp(getarg_c, u, v);
430 if (jit_arg_reg_p(v->u.w))
431 jit_extr_c(u, JIT_RA0 - v->u.w);
79bfeef6
PC		 432 else {
433 jit_node_t *node = jit_ldxi_c(u, JIT_FP, v->u.w);
434 jit_link_alist(node);
435 }
4a71579b
PC		 436 jit_dec_synth();
437}
438
439void
440_jit_getarg_uc(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
441{
79bfeef6 442 assert_arg_type(v->code, jit_code_arg_c);
4a71579b
PC		 443 jit_inc_synth_wp(getarg_uc, u, v);
444 if (jit_arg_reg_p(v->u.w))
445 jit_extr_uc(u, JIT_RA0 - v->u.w);
79bfeef6
PC		 446 else {
447 jit_node_t *node = jit_ldxi_uc(u, JIT_FP, v->u.w);
448 jit_link_alist(node);
449 }
4a71579b
PC		 450 jit_dec_synth();
451}
452
453void
454_jit_getarg_s(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
455{
79bfeef6 456 assert_arg_type(v->code, jit_code_arg_s);
4a71579b
PC		 457 jit_inc_synth_wp(getarg_s, u, v);
458 if (jit_arg_reg_p(v->u.w))
459 jit_extr_s(u, JIT_RA0 - v->u.w);
79bfeef6
PC		 460 else {
461 jit_node_t *node = jit_ldxi_s(u, JIT_FP, v->u.w);
462 jit_link_alist(node);
463 }
4a71579b
PC		 464 jit_dec_synth();
465}
466
467void
468_jit_getarg_us(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
469{
79bfeef6 470 assert_arg_type(v->code, jit_code_arg_s);
4a71579b
PC		 471 jit_inc_synth_wp(getarg_us, u, v);
472 if (jit_arg_reg_p(v->u.w))
473 jit_extr_us(u, JIT_RA0 - v->u.w);
79bfeef6
PC		 474 else {
475 jit_node_t *node = jit_ldxi_us(u, JIT_FP, v->u.w);
476 jit_link_alist(node);
477 }
4a71579b
PC		 478 jit_dec_synth();
479}
480
481void
482_jit_getarg_i(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
483{
79bfeef6 484 assert_arg_type(v->code, jit_code_arg_i);
4a71579b
PC		 485 jit_inc_synth_wp(getarg_i, u, v);
486 if (jit_arg_reg_p(v->u.w))
487 jit_extr_i(u, JIT_RA0 - v->u.w);
79bfeef6
PC		 488 else {
489 jit_node_t *node = jit_ldxi_i(u, JIT_FP, v->u.w);
490 jit_link_alist(node);
491 }
4a71579b
PC		 492 jit_dec_synth();
493}
494
495void
496_jit_getarg_ui(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
497{
79bfeef6 498 assert_arg_type(v->code, jit_code_arg_i);
4a71579b
PC		 499 jit_inc_synth_wp(getarg_ui, u, v);
500 if (jit_arg_reg_p(v->u.w))
501 jit_extr_ui(u, JIT_RA0 - v->u.w);
79bfeef6
PC		 502 else {
503 jit_node_t *node = jit_ldxi_ui(u, JIT_FP, v->u.w);
504 jit_link_alist(node);
505 }
4a71579b
PC		 506 jit_dec_synth();
507}
508
509void
510_jit_getarg_l(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
511{
79bfeef6 512 assert_arg_type(v->code, jit_code_arg_l);
4a71579b
PC		 513 jit_inc_synth_wp(getarg_l, u, v);
514 if (jit_arg_reg_p(v->u.w))
515 jit_movr(u, JIT_RA0 - v->u.w);
79bfeef6
PC		 516 else {
517 jit_node_t *node = jit_ldxi_l(u, JIT_FP, v->u.w);
518 jit_link_alist(node);
519 }
4a71579b
PC		 520 jit_dec_synth();
521}
522
523void
79bfeef6 524_jit_putargr(jit_state_t *_jit, jit_int32_t u, jit_node_t *v, jit_code_t code)
4a71579b 525{
79bfeef6
PC		 526 assert_putarg_type(code, v->code);
527 jit_code_inc_synth_wp(code, u, v);
4a71579b
PC		 528 if (jit_arg_reg_p(v->u.w))
529 jit_movr(JIT_RA0 - v->u.w, u);
79bfeef6
PC		 530 else {
531 jit_node_t *node = jit_stxi(v->u.w, JIT_FP, u);
532 jit_link_alist(node);
533 }
4a71579b
PC		 534 jit_dec_synth();
535}
536
537void
79bfeef6 538_jit_putargi(jit_state_t *_jit, jit_word_t u, jit_node_t *v, jit_code_t code)
4a71579b
PC		 539{
540 jit_int32_t regno;
79bfeef6
PC		 541 assert_putarg_type(code, v->code);
542 jit_code_inc_synth_wp(code, u, v);
4a71579b
PC		 543 if (jit_arg_reg_p(v->u.w))
544 jit_movi(JIT_RA0 - v->u.w, u);
545 else {
79bfeef6 546 jit_node_t *node;
4a71579b
PC		 547 regno = jit_get_reg(jit_class_gpr);
548 jit_movi(regno, u);
79bfeef6
PC		 549 node = jit_stxi(v->u.w, JIT_FP, regno);
550 jit_link_alist(node);
4a71579b
PC		 551 jit_unget_reg(regno);
552 }
553 jit_dec_synth();
554}
555
556void
557_jit_getarg_f(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
558{
559 assert(v->code == jit_code_arg_f);
560 jit_inc_synth_wp(getarg_f, u, v);
561 if (jit_arg_f_reg_p(v->u.w))
562 jit_movr_f(u, JIT_FA0 - v->u.w);
563 else if (jit_arg_reg_p(v->u.w - 8))
564 jit_movr_w_f(u, JIT_RA0 - (v->u.w - 8));
79bfeef6
PC		 565 else {
566 jit_node_t *node = jit_ldxi_f(u, JIT_FP, v->u.w);
567 jit_link_alist(node);
568 }
4a71579b
PC		 569 jit_dec_synth();
570}
571
572void
573_jit_putargr_f(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
574{
575 assert(v->code == jit_code_arg_f);
576 jit_inc_synth_wp(putargr_f, u, v);
577 if (jit_arg_f_reg_p(v->u.w))
578 jit_movr_f(JIT_FA0 - v->u.w, u);
579 else if (jit_arg_reg_p(v->u.w - 8))
580 jit_movr_f_w(JIT_RA0 - (v->u.w - 8), u);
79bfeef6
PC		 581 else {
582 jit_node_t *node = jit_stxi_f(v->u.w, JIT_FP, u);
583 jit_link_alist(node);
584 }
4a71579b
PC		 585 jit_dec_synth();
586}
587
588void
589_jit_putargi_f(jit_state_t *_jit, jit_float32_t u, jit_node_t *v)
590{
591 jit_int32_t regno;
592 assert(v->code == jit_code_arg_f);
593 jit_inc_synth_fp(putargi_f, u, v);
594 if (jit_arg_f_reg_p(v->u.w))
595 jit_movi_f(JIT_FA0 - v->u.w, u);
79bfeef6
PC		 596 else if (jit_arg_reg_p(v->u.w - 8))
597 jit_movi_f_w(JIT_RA0 - (v->u.w - 8), u);
4a71579b 598 else {
79bfeef6 599 jit_node_t *node;
4a71579b
PC		 600 regno = jit_get_reg(jit_class_fpr);
601 jit_movi_f(regno, u);
79bfeef6
PC		 602 node = jit_stxi_f(v->u.w, JIT_FP, regno);
603 jit_link_alist(node);
4a71579b
PC		 604 jit_unget_reg(regno);
605 }
606 jit_dec_synth();
607}
608
609void
610_jit_getarg_d(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
611{
612 assert(v->code == jit_code_arg_d);
613 jit_inc_synth_wp(getarg_d, u, v);
614 if (jit_arg_f_reg_p(v->u.w))
615 jit_movr_d(u, JIT_FA0 - v->u.w);
616 else if (jit_arg_reg_p(v->u.w - 8))
617 jit_movr_w_d(u, JIT_RA0 - (v->u.w - 8));
79bfeef6
PC		 618 else {
619 jit_node_t *node = jit_ldxi_d(u, JIT_FP, v->u.w);
620 jit_link_alist(node);
621 }
4a71579b
PC		 622 jit_dec_synth();
623}
624
625void
626_jit_putargr_d(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
627{
628 assert(v->code == jit_code_arg_d);
629 jit_inc_synth_wp(putargr_d, u, v);
630 if (jit_arg_reg_p(v->u.w))
631 jit_movr_d(JIT_FA0 - v->u.w, u);
632 else if (jit_arg_reg_p(v->u.w - 8))
633 jit_movr_d_w(JIT_RA0 - (v->u.w - 8), u);
79bfeef6
PC		 634 else {
635 jit_node_t *node = jit_stxi_d(v->u.w, JIT_FP, u);
636 jit_link_alist(node);
637 }
4a71579b
PC		 638 jit_dec_synth();
639}
640
641void
642_jit_putargi_d(jit_state_t *_jit, jit_float64_t u, jit_node_t *v)
643{
644 jit_int32_t regno;
645 assert(v->code == jit_code_arg_d);
646 jit_inc_synth_dp(putargi_d, u, v);
647 if (jit_arg_reg_p(v->u.w))
648 jit_movi_d(JIT_FA0 - v->u.w, u);
79bfeef6
PC		 649 else if (jit_arg_reg_p(v->u.w - 8))
650 jit_movi_d_w(JIT_RA0 - (v->u.w - 8), u);
4a71579b 651 else {
79bfeef6 652 jit_node_t *node;
4a71579b
PC		 653 regno = jit_get_reg(jit_class_fpr);
654 jit_movi_d(regno, u);
79bfeef6
PC		 655 node = jit_stxi_d(v->u.w, JIT_FP, regno);
656 jit_link_alist(node);
4a71579b
PC		 657 jit_unget_reg(regno);
658 }
659 jit_dec_synth();
660}
661
662void
79bfeef6 663_jit_pushargr(jit_state_t *_jit, jit_int32_t u, jit_code_t code)
4a71579b
PC		 664{
665 assert(_jitc->function);
79bfeef6 666 jit_code_inc_synth_w(code, u);
4a71579b
PC		 667 jit_link_prepare();
668 if (jit_arg_reg_p(_jitc->function->call.argi)) {
669 jit_movr(JIT_RA0 - _jitc->function->call.argi, u);
670 ++_jitc->function->call.argi;
671 }
672 else {
673 jit_stxi(_jitc->function->call.size, JIT_SP, u);
674 _jitc->function->call.size += sizeof(jit_word_t);
79bfeef6 675 jit_check_frame();
4a71579b
PC		 676 }
677 jit_dec_synth();
678}
679
680void
79bfeef6 681_jit_pushargi(jit_state_t *_jit, jit_word_t u, jit_code_t code)
4a71579b
PC		 682{
683 jit_int32_t regno;
684 assert(_jitc->function);
79bfeef6 685 jit_code_inc_synth_w(code, u);
4a71579b
PC		 686 jit_link_prepare();
687 if (jit_arg_reg_p(_jitc->function->call.argi)) {
688 jit_movi(JIT_RA0 - _jitc->function->call.argi, u);
689 ++_jitc->function->call.argi;
690 }
691 else {
692 regno = jit_get_reg(jit_class_gpr);
693 jit_movi(regno, u);
694 jit_stxi(_jitc->function->call.size, JIT_SP, regno);
695 jit_unget_reg(regno);
696 _jitc->function->call.size += sizeof(jit_word_t);
79bfeef6 697 jit_check_frame();
4a71579b
PC		 698 }
699 jit_dec_synth();
700}
701
702void
703_jit_pushargr_f(jit_state_t *_jit, jit_int32_t u)
704{
705 assert(_jitc->function);
706 jit_inc_synth_w(pushargr_f, u);
707 jit_link_prepare();
708 if (jit_arg_f_reg_p(_jitc->function->call.argf) &&
709 !(_jitc->function->call.call & jit_call_varargs)) {
710 jit_movr_f(JIT_FA0 - _jitc->function->call.argf, u);
711 ++_jitc->function->call.argf;
712 }
713 else if (jit_arg_reg_p(_jitc->function->call.argi)) {
714 jit_movr_f_w(JIT_RA0 - _jitc->function->call.argi, u);
715 ++_jitc->function->call.argi;
716 }
717 else {
718 jit_stxi_f(_jitc->function->call.size, JIT_SP, u);
719 _jitc->function->call.size += sizeof(jit_word_t);
79bfeef6 720 jit_check_frame();
4a71579b
PC		 721 }
722 jit_dec_synth();
723}
724
725void
726_jit_pushargi_f(jit_state_t *_jit, jit_float32_t u)
727{
728 jit_int32_t regno;
729 assert(_jitc->function);
730 jit_inc_synth_f(pushargi_f, u);
731 jit_link_prepare();
732 if (jit_arg_f_reg_p(_jitc->function->call.argf) &&
733 !(_jitc->function->call.call & jit_call_varargs)) {
734 jit_movi_f(JIT_FA0 - _jitc->function->call.argf, u);
735 ++_jitc->function->call.argf;
736 }
737 else if (jit_arg_reg_p(_jitc->function->call.argi)) {
738 jit_movi_f_w(JIT_RA0 - _jitc->function->call.argi, u);
739 ++_jitc->function->call.argi;
740 }
741 else {
742 regno = jit_get_reg(jit_class_fpr);
743 jit_movi_f(regno, u);
744 jit_stxi_f(_jitc->function->call.size, JIT_SP, regno);
745 jit_unget_reg(regno);
746 _jitc->function->call.size += sizeof(jit_word_t);
79bfeef6 747 jit_check_frame();
4a71579b
PC		 748 }
749 jit_dec_synth();
750}
751
752void
753_jit_pushargr_d(jit_state_t *_jit, jit_int32_t u)
754{
755 assert(_jitc->function);
756 jit_inc_synth_w(pushargr_d, u);
757 jit_link_prepare();
758 if (jit_arg_f_reg_p(_jitc->function->call.argf) &&
759 !(_jitc->function->call.call & jit_call_varargs)) {
760 jit_movr_d(JIT_FA0 - _jitc->function->call.argf, u);
761 ++_jitc->function->call.argf;
762 }
763 else if (jit_arg_reg_p(_jitc->function->call.argi)) {
764 jit_movr_d_w(JIT_RA0 - _jitc->function->call.argi, u);
765 ++_jitc->function->call.argi;
766 }
767 else {
768 jit_stxi_d(_jitc->function->call.size, JIT_SP, u);
769 _jitc->function->call.size += sizeof(jit_word_t);
79bfeef6 770 jit_check_frame();
4a71579b
PC		 771 }
772 jit_dec_synth();
773}
774
775void
776_jit_pushargi_d(jit_state_t *_jit, jit_float64_t u)
777{
778 jit_int32_t regno;
779 assert(_jitc->function);
780 jit_inc_synth_d(pushargi_d, u);
781 jit_link_prepare();
782 if (jit_arg_f_reg_p(_jitc->function->call.argf) &&
783 !(_jitc->function->call.call & jit_call_varargs)) {
784 jit_movi_d(JIT_FA0 - _jitc->function->call.argf, u);
785 ++_jitc->function->call.argf;
786 }
787 else if (jit_arg_reg_p(_jitc->function->call.argi)) {
788 jit_movi_d_w(JIT_RA0 - _jitc->function->call.argi, u);
789 ++_jitc->function->call.argi;
790 }
791 else {
792 regno = jit_get_reg(jit_class_fpr);
793 jit_movi_d(regno, u);
794 jit_stxi_d(_jitc->function->call.size, JIT_SP, regno);
795 jit_unget_reg(regno);
796 _jitc->function->call.size += sizeof(jit_word_t);
79bfeef6 797 jit_check_frame();
4a71579b
PC		 798 }
799 jit_dec_synth();
800}
801
802jit_bool_t
803_jit_regarg_p(jit_state_t *_jit, jit_node_t *node, jit_int32_t regno)
804{
805 jit_int32_t spec;
806 spec = jit_class(_rvs[regno].spec);
807 if (spec & jit_class_arg) {
808 regno = JIT_RA0 - regno;
809 if (regno >= 0 && regno < node->v.w)
810 return (1);
811 if (spec & jit_class_fpr) {
812 regno = JIT_FA0 - regno;
813 if (regno >= 0 && regno < node->w.w)
814 return (1);
815 }
816 }
817
818 return (0);
819}
820
821void
822_jit_finishr(jit_state_t *_jit, jit_int32_t r0)
823{
824 jit_node_t *node;
825 assert(_jitc->function);
79bfeef6 826 jit_check_frame();
4a71579b
PC		 827 jit_inc_synth_w(finishr, r0);
828 if (_jitc->function->self.alen < _jitc->function->call.size)
829 _jitc->function->self.alen = _jitc->function->call.size;
830 node = jit_callr(r0);
831 node->v.w = _jitc->function->self.argi;
832 node->w.w = _jitc->function->call.argf;
833 _jitc->function->call.argi = _jitc->function->call.argf =
834 _jitc->function->call.size = 0;
835 _jitc->prepare = 0;
836 jit_dec_synth();
837}
838
839jit_node_t *
840_jit_finishi(jit_state_t *_jit, jit_pointer_t i0)
841{
842 jit_node_t *node;
843 assert(_jitc->function);
79bfeef6 844 jit_check_frame();
4a71579b
PC		 845 jit_inc_synth_w(finishi, (jit_word_t)i0);
846 if (_jitc->function->self.alen < _jitc->function->call.size)
847 _jitc->function->self.alen = _jitc->function->call.size;
848 node = jit_calli(i0);
849 node->v.w = _jitc->function->call.argi;
850 node->w.w = _jitc->function->call.argf;
851 _jitc->function->call.argi = _jitc->function->call.argf =
852 _jitc->function->call.size = 0;
853 _jitc->prepare = 0;
854 jit_dec_synth();
855 return (node);
856}
857
858void
859_jit_retval_c(jit_state_t *_jit, jit_int32_t r0)
860{
861 jit_inc_synth_w(retval_c, r0);
862 jit_extr_c(r0, JIT_RET);
863 jit_dec_synth();
864}
865
866void
867_jit_retval_uc(jit_state_t *_jit, jit_int32_t r0)
868{
869 jit_inc_synth_w(retval_uc, r0);
870 jit_extr_uc(r0, JIT_RET);
871 jit_dec_synth();
872}
873
874void
875_jit_retval_s(jit_state_t *_jit, jit_int32_t r0)
876{
877 jit_inc_synth_w(retval_s, r0);
878 jit_extr_s(r0, JIT_RET);
879 jit_dec_synth();
880}
881
882void
883_jit_retval_us(jit_state_t *_jit, jit_int32_t r0)
884{
885 jit_inc_synth_w(retval_us, r0);
886 jit_extr_us(r0, JIT_RET);
887 jit_dec_synth();
888}
889
890void
891_jit_retval_i(jit_state_t *_jit, jit_int32_t r0)
892{
893 jit_inc_synth_w(retval_i, r0);
894 jit_extr_i(r0, JIT_RET);
895 jit_dec_synth();
896}
897
898void
899_jit_retval_ui(jit_state_t *_jit, jit_int32_t r0)
900{
901 jit_inc_synth_w(retval_ui, r0);
902 jit_extr_ui(r0, JIT_RET);
903 jit_dec_synth();
904}
905
906void
907_jit_retval_l(jit_state_t *_jit, jit_int32_t r0)
908{
909 jit_inc_synth_w(retval_l, r0);
910 if (r0 != JIT_RET)
911 jit_movr(r0, JIT_RET);
912 jit_dec_synth();
913}
914
915void
916_jit_retval_f(jit_state_t *_jit, jit_int32_t r0)
917{
918 jit_inc_synth_w(retval_f, r0);
919 if (r0 != JIT_FRET)
920 jit_movr_f(r0, JIT_FRET);
921 jit_dec_synth();
922}
923
924void
925_jit_retval_d(jit_state_t *_jit, jit_int32_t r0)
926{
927 jit_inc_synth_w(retval_d, r0);
928 if (r0 != JIT_FRET)
929 jit_movr_d(r0, JIT_FRET);
930 jit_dec_synth();
931}
932
933jit_pointer_t
934_emit_code(jit_state_t *_jit)
935{
936 jit_node_t *node;
937 jit_node_t *temp;
938 jit_word_t word;
939 jit_word_t value;
940 jit_int32_t offset;
941 struct {
942 jit_node_t *node;
943 jit_uint8_t *data;
944 jit_word_t word;
79bfeef6 945 jit_function_t func;
4a71579b
PC		 946#if DEVEL_DISASSEMBLER
947 jit_word_t prevw;
948#endif
949 jit_int32_t const_offset;
950 jit_int32_t patch_offset;
951 } undo;
952#if DEVEL_DISASSEMBLER
953 jit_word_t prevw;
954#endif
955
c0c16242
PC		 956#if __WORDSIZE == 64
957 if (!_jitc->consts.hash.table) {
958 jit_alloc((jit_pointer_t *)&_jitc->consts.hash.table,
959 16 * sizeof(jit_const_t *));
960 _jitc->consts.hash.size = 16;
961 jit_alloc((jit_pointer_t *)&_jitc->consts.pool.ptr,
962 sizeof(jit_const_t *));
963 jit_alloc((jit_pointer_t *)_jitc->consts.pool.ptr,
964 1024 * sizeof(jit_const_t));
965 _jitc->consts.pool.length = 1;
966 }
967 /* Reset table if starting over jit generation */
968 else
969 memset(_jitc->consts.hash.table, 0,
970 _jitc->consts.hash.size * sizeof(jit_word_t));
971 for (offset = 0; offset < _jitc->consts.pool.length; offset++) {
972 jit_int32_t i;
973 jit_const_t *list = _jitc->consts.pool.ptr[offset];
974 for (i = 0; i < 1023; ++i, ++list)
975 list->next = list + 1;
976 if (offset + 1 < _jitc->consts.pool.length)
977 list->next = _jitc->consts.pool.ptr[offset + 1];
978 else
979 list->next = NULL;
980 }
981 _jitc->consts.pool.list = _jitc->consts.pool.ptr[0];
982 _jitc->consts.hash.count = 0;
983 if (!_jitc->consts.vector.instrs) {
984 jit_alloc((jit_pointer_t *)&_jitc->consts.vector.instrs,
985 16 * sizeof(jit_word_t));
986 jit_alloc((jit_pointer_t *)&_jitc->consts.vector.values,
987 16 * sizeof(jit_word_t));
988 _jitc->consts.vector.length = 16;
989 }
990 _jitc->consts.vector.offset = 0;
991#endif
992
4a71579b
PC		 993 _jitc->function = NULL;
994
995 jit_reglive_setup();
996
997 undo.word = 0;
998 undo.node = NULL;
999 undo.const_offset = undo.patch_offset = 0;
1000# define assert_data(node) /**/
1001#define case_rr(name, type) \
1002 case jit_code_##name##r##type: \
1003 name##r##type(rn(node->u.w), rn(node->v.w)); \
1004 break
1005#define case_rw(name, type) \
1006 case jit_code_##name##i##type: \
1007 name##i##type(rn(node->u.w), node->v.w); \
1008 break
1009#define case_wr(name, type) \
1010 case jit_code_##name##i##type: \
1011 name##i##type(node->u.w, rn(node->v.w)); \
1012 break
1013#define case_rrr(name, type) \
1014 case jit_code_##name##r##type: \
1015 name##r##type(rn(node->u.w), \
1016 rn(node->v.w), rn(node->w.w)); \
1017 break
1018#define case_rrrr(name, type) \
1019 case jit_code_##name##r##type: \
1020 name##r##type(rn(node->u.q.l), rn(node->u.q.h), \
1021 rn(node->v.w), rn(node->w.w)); \
1022 break
ba86ff93
PC		 1023#define case_rqr(name, type) \
1024 case jit_code_##name##r##type: \
1025 name##r##type(rn(node->u.w), rn(node->v.q.l), \
1026 rn(node->v.q.h), rn(node->w.w)); \
1027 case jit_code_##name##i##type: \
1028 break;
4a71579b
PC		 1029#define case_rrw(name, type) \
1030 case jit_code_##name##i##type: \
1031 name##i##type(rn(node->u.w), rn(node->v.w), node->w.w); \
1032 break
1033#define case_rrrw(name, type) \
1034 case jit_code_##name##i##type: \
1035 name##i##type(rn(node->u.q.l), rn(node->u.q.h), \
1036 rn(node->v.w), node->w.w); \
1037 break
1038#define case_rrf(name) \
1039 case jit_code_##name##i_f: \
1040 assert_data(node); \
1041 name##i_f(rn(node->u.w), rn(node->v.w), node->w.f); \
1042 break
1043#define case_rrd(name) \
1044 case jit_code_##name##i_d: \
1045 assert_data(node); \
1046 name##i_d(rn(node->u.w), rn(node->v.w), node->w.d); \
1047 break
1048#define case_wrr(name, type) \
1049 case jit_code_##name##i##type: \
1050 name##i##type(node->u.w, rn(node->v.w), rn(node->w.w)); \
1051 break
1052#define case_brr(name, type) \
1053 case jit_code_##name##r##type: \
1054 temp = node->u.n; \
1055 assert(temp->code == jit_code_label || \
1056 temp->code == jit_code_epilog); \
1057 if (temp->flag & jit_flag_patch) \
1058 name##r##type(temp->u.w, rn(node->v.w), \
1059 rn(node->w.w)); \
1060 else { \
1061 word = name##r##type(_jit->pc.w, \
1062 rn(node->v.w), rn(node->w.w)); \
1063 patch(word, node); \
1064 } \
1065 break
1066#define case_brw(name, type) \
1067 case jit_code_##name##i##type: \
1068 temp = node->u.n; \
1069 assert(temp->code == jit_code_label || \
1070 temp->code == jit_code_epilog); \
1071 if (temp->flag & jit_flag_patch) \
1072 name##i##type(temp->u.w, \
1073 rn(node->v.w), node->w.w); \
1074 else { \
1075 word = name##i##type(_jit->pc.w, \
1076 rn(node->v.w), node->w.w); \
1077 patch(word, node); \
1078 } \
1079 break;
1080#define case_brf(name) \
1081 case jit_code_##name##i_f: \
1082 temp = node->u.n; \
1083 assert(temp->code == jit_code_label || \
1084 temp->code == jit_code_epilog); \
1085 if (temp->flag & jit_flag_patch) \
1086 name##i_f(temp->u.w, rn(node->v.w), node->w.f); \
1087 else { \
1088 word = name##i_f(_jit->pc.w, rn(node->v.w), \
1089 node->w.f); \
1090 patch(word, node); \
1091 } \
1092 break
1093#define case_brd(name) \
1094 case jit_code_##name##i_d: \
1095 temp = node->u.n; \
1096 assert(temp->code == jit_code_label || \
1097 temp->code == jit_code_epilog); \
1098 if (temp->flag & jit_flag_patch) \
1099 name##i_d(temp->u.w, rn(node->v.w), node->w.d); \
1100 else { \
1101 word = name##i_d(_jit->pc.w, rn(node->v.w), \
1102 node->w.d); \
1103 patch(word, node); \
1104 } \
1105 break
1106#if DEVEL_DISASSEMBLER
1107 prevw = _jit->pc.w;
1108#endif
1109 for (node = _jitc->head; node; node = node->next) {
1110 if (_jit->pc.uc >= _jitc->code.end)
1111 return (NULL);
1112
1113#if DEVEL_DISASSEMBLER
1114 node->offset = (jit_uword_t)_jit->pc.w - (jit_uword_t)prevw;
1115 prevw = _jit->pc.w;
1116#endif
1117 value = jit_classify(node->code);
1118 jit_regarg_set(node, value);
1119 switch (node->code) {
1120 case jit_code_align:
c0c16242
PC		 1121 /* Must align to a power of two */
1122 assert(!(node->u.w & (node->u.w - 1)));
1123 if ((word = _jit->pc.w & (node->u.w - 1)))
1124 nop(node->u.w - word);
4a71579b 1125 break;
79bfeef6
PC		 1126 case jit_code_skip:
1127 nop((node->u.w + 3) & ~3);
1128 break;
4a71579b
PC		 1129 case jit_code_note: case jit_code_name:
1130 node->u.w = _jit->pc.w;
1131 break;
1132 case jit_code_label:
1133 /* remember label is defined */
1134 node->flag |= jit_flag_patch;
1135 node->u.w = _jit->pc.w;
1136 break;
1137 case_rrr(add,);
1138 case_rrw(add,);
1139 case_rrr(addc,);
1140 case_rrw(addc,);
1141 case_rrr(addx,);
1142 case_rrw(addx,);
1143 case_rrr(sub,);
1144 case_rrw(sub,);
1145 case_rrr(subc,);
1146 case_rrw(subc,);
1147 case_rrr(subx,);
1148 case_rrw(subx,);
1149 case_rrw(rsb,);
1150 case_rrr(mul,);
1151 case_rrw(mul,);
ba86ff93
PC		 1152 case_rrr(hmul,);
1153 case_rrw(hmul,);
1154 case_rrr(hmul, _u);
1155 case_rrw(hmul, _u);
4a71579b
PC		 1156 case_rrrr(qmul,);
1157 case_rrrw(qmul,);
1158 case_rrrr(qmul, _u);
1159 case_rrrw(qmul, _u);
1160 case_rrr(div,);
1161 case_rrw(div,);
1162 case_rrr(div, _u);
1163 case_rrw(div, _u);
1164 case_rrrr(qdiv,);
1165 case_rrrw(qdiv,);
1166 case_rrrr(qdiv, _u);
1167 case_rrrw(qdiv, _u);
1168 case_rrr(rem,);
1169 case_rrw(rem,);
1170 case_rrr(rem, _u);
1171 case_rrw(rem, _u);
1172 case_rrr(lsh,);
1173 case_rrw(lsh,);
1174 case_rrr(rsh,);
1175 case_rrw(rsh,);
ba86ff93
PC		 1176#define qlshr(r0, r1, r2, r3) fallback_qlshr(r0, r1, r2, r3)
1177#define qlshi(r0, r1, r2, i0) fallback_qlshi(r0, r1, r2, i0)
1178#define qlshr_u(r0, r1, r2, r3) fallback_qlshr_u(r0, r1, r2, r3)
1179#define qlshi_u(r0, r1, r2, i0) fallback_qlshi_u(r0, r1, r2, i0)
1180 case_rrrr(qlsh,);
1181 case_rrrw(qlsh,);
1182 case_rrrr(qlsh, _u);
1183 case_rrrw(qlsh, _u);
4a71579b
PC		 1184 case_rrr(rsh, _u);
1185 case_rrw(rsh, _u);
ba86ff93
PC		 1186#define qrshr(r0, r1, r2, r3) fallback_qrshr(r0, r1, r2, r3)
1187#define qrshi(r0, r1, r2, i0) fallback_qrshi(r0, r1, r2, i0)
1188#define qrshr_u(r0, r1, r2, r3) fallback_qrshr_u(r0, r1, r2, r3)
1189#define qrshi_u(r0, r1, r2, i0) fallback_qrshi_u(r0, r1, r2, i0)
1190 case_rrrr(qrsh,);
1191 case_rrrw(qrsh,);
1192 case_rrrr(qrsh, _u);
1193 case_rrrw(qrsh, _u);
1194#define lrotr(r0,r1,r2) fallback_lrotr(r0,r1,r2)
1195#define lroti(r0,r1,i0) fallback_lroti(r0,r1,i0)
1196#define rrotr(r0,r1,r2) fallback_rrotr(r0,r1,r2)
1197#define rroti(r0,r1,i0) fallback_rroti(r0,r1,i0)
1198 case_rrr(lrot,);
1199 case_rrw(lrot,);
1200 case_rrr(rrot,);
1201 case_rrw(rrot,);
4a71579b
PC		 1202 case_rr(neg,);
1203 case_rr(com,);
79bfeef6
PC		 1204#define clor(r0, r1) fallback_clo(r0, r1)
1205#define clzr(r0, r1) fallback_clz(r0, r1)
1206#define ctor(r0, r1) fallback_cto(r0, r1)
1207#define ctzr(r0, r1) fallback_ctz(r0, r1)
ba86ff93
PC		 1208#define rbitr(r0, r1) fallback_rbit(r0, r1)
1209#define popcntr(r0, r1) fallback_popcnt(r0, r1)
79bfeef6
PC		 1210 case_rr(clo,);
1211 case_rr(clz,);
1212 case_rr(cto,);
1213 case_rr(ctz,);
ba86ff93
PC		 1214 case_rr(rbit,);
1215 case_rr(popcnt,);
4a71579b
PC		 1216 case_rrr(and,);
1217 case_rrw(and,);
1218 case_rrr(or,);
1219 case_rrw(or,);
1220 case_rrr(xor,);
1221 case_rrw(xor,);
1222 case_rr(trunc, _f_i);
1223 case_rr(trunc, _d_i);
1224 case_rr(trunc, _f_l);
1225 case_rr(trunc, _d_l);
1226 case_rr(ld, _c);
1227 case_rw(ld, _c);
1228 case_rr(ld, _uc);
1229 case_rw(ld, _uc);
1230 case_rr(ld, _s);
1231 case_rw(ld, _s);
1232 case_rr(ld, _us);
1233 case_rw(ld, _us);
1234 case_rr(ld, _i);
1235 case_rw(ld, _i);
1236 case_rr(ld, _ui);
1237 case_rw(ld, _ui);
1238 case_rr(ld, _l);
1239 case_rw(ld, _l);
1240 case_rrr(ldx, _c);
1241 case_rrw(ldx, _c);
1242 case_rrr(ldx, _uc);
1243 case_rrw(ldx, _uc);
1244 case_rrr(ldx, _s);
1245 case_rrw(ldx, _s);
1246 case_rrr(ldx, _us);
1247 case_rrw(ldx, _us);
1248 case_rrr(ldx, _i);
1249 case_rrw(ldx, _i);
1250 case_rrr(ldx, _ui);
1251 case_rrw(ldx, _ui);
1252 case_rrr(ldx, _l);
1253 case_rrw(ldx, _l);
ba86ff93
PC		 1254 case jit_code_unldr:
1255 unldr(rn(node->u.w), rn(node->v.w), node->w.w);
1256 break;
1257 case jit_code_unldi:
1258 unldi(rn(node->u.w), node->v.w, node->w.w);
1259 break;
1260 case jit_code_unldr_u:
1261 unldr_u(rn(node->u.w), rn(node->v.w), node->w.w);
1262 break;
1263 case jit_code_unldi_u:
1264 unldi_u(rn(node->u.w), node->v.w, node->w.w);
1265 break;
4a71579b
PC		 1266 case_rr(st, _c);
1267 case_wr(st, _c);
1268 case_rr(st, _s);
1269 case_wr(st, _s);
1270 case_rr(st, _i);
1271 case_wr(st, _i);
1272 case_rr(st, _l);
1273 case_wr(st, _l);
1274 case_rrr(stx, _c);
1275 case_wrr(stx, _c);
1276 case_rrr(stx, _s);
1277 case_wrr(stx, _s);
1278 case_rrr(stx, _i);
1279 case_wrr(stx, _i);
1280 case_rrr(stx, _l);
1281 case_wrr(stx, _l);
ba86ff93
PC		 1282 case jit_code_unstr:
1283 unstr(rn(node->u.w), rn(node->v.w), node->w.w);
1284 break;
1285 case jit_code_unsti:
1286 unsti(node->u.w, rn(node->v.w), node->w.w);
1287 break;
4a71579b
PC		 1288 case_rr(hton, _us);
1289 case_rr(hton, _ui);
1290 case_rr(hton, _ul);
40a44dcb
PC		 1291 case_rr(bswap, _us);
1292 case_rr(bswap, _ui);
1293 case_rr(bswap, _ul);
ba86ff93
PC		 1294#define extr(r0, r1, i0, i1) fallback_ext(r0, r1, i0, i1)
1295#define extr_u(r0, r1, i0, i1) fallback_ext_u(r0, r1, i0, i1)
1296#define depr(r0, r1, i0, i1) fallback_dep(r0, r1, i0, i1)
1297 case jit_code_extr:
1298 extr(rn(node->u.w), rn(node->v.w), node->w.q.l, node->w.q.h);
1299 break;
1300 case jit_code_extr_u:
1301 extr_u(rn(node->u.w), rn(node->v.w), node->w.q.l, node->w.q.h);
1302 break;
1303 case jit_code_depr:
1304 depr(rn(node->u.w), rn(node->v.w), node->w.q.l, node->w.q.h);
1305 break;
1306 case jit_code_depi:
1307 depi(rn(node->u.w), node->v.w, node->w.q.l, node->w.q.h);
1308 break;
4a71579b
PC		 1309 case_rr(ext, _c);
1310 case_rr(ext, _uc);
1311 case_rr(ext, _s);
1312 case_rr(ext, _us);
1313 case_rr(ext, _i);
1314 case_rr(ext, _ui);
ba3814c1
PC		 1315 case jit_code_casr:
1316 casr(rn(node->u.w), rn(node->v.w),
1317 rn(node->w.q.l), rn(node->w.q.h));
1318 break;
1319 case jit_code_casi:
1320 casi(rn(node->u.w), node->v.w,
1321 rn(node->w.q.l), rn(node->w.q.h));
1322 break;
40a44dcb
PC		 1323 case_rrr(movn,);
1324 case_rrr(movz,);
4a71579b
PC		 1325 case_rr(mov,);
1326 case jit_code_movi:
1327 if (node->flag & jit_flag_node) {
1328 temp = node->v.n;
1329 if (temp->code == jit_code_data ||
1330 (temp->code == jit_code_label &&
1331 (temp->flag & jit_flag_patch)))
1332 movi(rn(node->u.w), temp->u.w);
1333 else {
1334 assert(temp->code == jit_code_label ||
1335 temp->code == jit_code_epilog);
1336 word = movi_p(rn(node->u.w), temp->u.w);
1337 patch(word, node);
1338 }
1339 }
1340 else
1341 movi(rn(node->u.w), node->v.w);
1342 break;
1343 case_rrr(lt,);
1344 case_rrw(lt,);
1345 case_rrr(lt, _u);
1346 case_rrw(lt, _u);
1347 case_rrr(le,);
1348 case_rrw(le,);
1349 case_rrr(le, _u);
1350 case_rrw(le, _u);
1351 case_rrr(eq,);
1352 case_rrw(eq,);
1353 case_rrr(ge,);
1354 case_rrw(ge,);
1355 case_rrr(ge, _u);
1356 case_rrw(ge, _u);
1357 case_rrr(gt,);
1358 case_rrw(gt,);
1359 case_rrr(gt, _u);
1360 case_rrw(gt, _u);
1361 case_rrr(ne,);
1362 case_rrw(ne,);
1363 case_brr(blt,);
1364 case_brw(blt,);
1365 case_brr(blt, _u);
1366 case_brw(blt, _u);
1367 case_brr(ble,);
1368 case_brw(ble,);
1369 case_brr(ble, _u);
1370 case_brw(ble, _u);
1371 case_brr(beq,);
1372 case_brw(beq,);
1373 case_brr(bge,);
1374 case_brw(bge,);
1375 case_brr(bge, _u);
1376 case_brw(bge, _u);
1377 case_brr(bgt,);
1378 case_brw(bgt,);
1379 case_brr(bgt, _u);
1380 case_brw(bgt, _u);
1381 case_brr(bne,);
1382 case_brw(bne,);
1383 case_brr(boadd,);
1384 case_brw(boadd,);
1385 case_brr(boadd, _u);
1386 case_brw(boadd, _u);
1387 case_brr(bxadd,);
1388 case_brw(bxadd,);
1389 case_brr(bxadd, _u);
1390 case_brw(bxadd, _u);
1391 case_brr(bosub,);
1392 case_brw(bosub,);
1393 case_brr(bosub, _u);
1394 case_brw(bosub, _u);
1395 case_brr(bxsub,);
1396 case_brw(bxsub,);
1397 case_brr(bxsub, _u);
1398 case_brw(bxsub, _u);
1399 case_brr(bms,);
1400 case_brw(bms,);
1401 case_brr(bmc,);
1402 case_brw(bmc,);
1403 case_rrr(add, _f);
1404 case_rrf(add);
1405 case_rrr(sub, _f);
1406 case_rrf(sub);
1407 case_rrf(rsb);
1408 case_rrr(mul, _f);
1409 case_rrf(mul);
1410 case_rrr(div, _f);
1411 case_rrf(div);
1412 case_rr(abs, _f);
1413 case_rr(neg, _f);
1414 case_rr(sqrt, _f);
ba86ff93
PC		 1415 case_rqr(fma, _f);
1416 case_rqr(fms, _f);
1417 case_rqr(fnma, _f);
1418 case_rqr(fnms, _f);
4a71579b
PC		 1419 case_rr(ext, _f);
1420 case_rr(ld, _f);
1421 case_rw(ld, _f);
1422 case_rrr(ldx, _f);
1423 case_rrw(ldx, _f);
ba86ff93
PC		 1424 case jit_code_unldr_x:
1425 unldr_x(rn(node->u.w), rn(node->v.w), node->w.w);
1426 break;
1427 case jit_code_unldi_x:
1428 unldi_x(rn(node->u.w), node->v.w, node->w.w);
1429 break;
4a71579b
PC		 1430 case_rr(st, _f);
1431 case_wr(st, _f);
1432 case_rrr(stx, _f);
1433 case_wrr(stx, _f);
ba86ff93
PC		 1434 case jit_code_unstr_x:
1435 unstr_x(rn(node->u.w), rn(node->v.w), node->w.w);
1436 break;
1437 case jit_code_unsti_x:
1438 unsti_x(node->u.w, rn(node->v.w), node->w.w);
1439 break;
4a71579b
PC		 1440 case_rr(mov, _f);
1441 case jit_code_movi_f:
1442 assert_data(node);
1443 movi_f(rn(node->u.w), node->v.f);
1444 break;
1445 case_rr(ext, _d_f);
1446 case_rrr(lt, _f);
1447 case_rrf(lt);
1448 case_rrr(le, _f);
1449 case_rrf(le);
1450 case_rrr(eq, _f);
1451 case_rrf(eq);
1452 case_rrr(ge, _f);
1453 case_rrf(ge);
1454 case_rrr(gt, _f);
1455 case_rrf(gt);
1456 case_rrr(ne, _f);
1457 case_rrf(ne);
1458 case_rrr(unlt, _f);
1459 case_rrf(unlt);
1460 case_rrr(unle, _f);
1461 case_rrf(unle);
1462 case_rrr(uneq, _f);
1463 case_rrf(uneq);
1464 case_rrr(unge, _f);
1465 case_rrf(unge);
1466 case_rrr(ungt, _f);
1467 case_rrf(ungt);
1468 case_rrr(ltgt, _f);
1469 case_rrf(ltgt);
1470 case_rrr(ord, _f);
1471 case_rrf(ord);
1472 case_rrr(unord, _f);
1473 case_rrf(unord);
1474 case_brr(blt, _f);
1475 case_brf(blt);
1476 case_brr(ble, _f);
1477 case_brf(ble);
1478 case_brr(beq, _f);
1479 case_brf(beq);
1480 case_brr(bge, _f);
1481 case_brf(bge);
1482 case_brr(bgt, _f);
1483 case_brf(bgt);
1484 case_brr(bne, _f);
1485 case_brf(bne);
1486 case_brr(bunlt, _f);
1487 case_brf(bunlt);
1488 case_brr(bunle, _f);
1489 case_brf(bunle);
1490 case_brr(buneq, _f);
1491 case_brf(buneq);
1492 case_brr(bunge, _f);
1493 case_brf(bunge);
1494 case_brr(bungt, _f);
1495 case_brf(bungt);
1496 case_brr(bltgt, _f);
1497 case_brf(bltgt);
1498 case_brr(bord, _f);
1499 case_brf(bord);
1500 case_brr(bunord, _f);
1501 case_brf(bunord);
1502 case_rrr(add, _d);
1503 case_rrd(add);
1504 case_rrr(sub, _d);
1505 case_rrd(sub);
1506 case_rrd(rsb);
1507 case_rrr(mul, _d);
1508 case_rrd(mul);
1509 case_rrr(div, _d);
1510 case_rrd(div);
1511 case_rr(abs, _d);
1512 case_rr(neg, _d);
1513 case_rr(sqrt, _d);
ba86ff93
PC		 1514 case_rqr(fma, _d);
1515 case_rqr(fms, _d);
1516 case_rqr(fnma, _d);
1517 case_rqr(fnms, _d);
4a71579b
PC		 1518 case_rr(ext, _d);
1519 case_rr(ld, _d);
1520 case_rw(ld, _d);
1521 case_rrr(ldx, _d);
1522 case_rrw(ldx, _d);
1523 case_rr(st, _d);
1524 case_wr(st, _d);
1525 case_rrr(stx, _d);
1526 case_wrr(stx, _d);
1527 case_rr(mov, _d);
1528 case jit_code_movi_d:
1529 assert_data(node);
1530 movi_d(rn(node->u.w), node->v.d);
1531 break;
1532 case_rr(ext, _f_d);
1533 case_rrr(lt, _d);
1534 case_rrd(lt);
1535 case_rrr(le, _d);
1536 case_rrd(le);
1537 case_rrr(eq, _d);
1538 case_rrd(eq);
1539 case_rrr(ge, _d);
1540 case_rrd(ge);
1541 case_rrr(gt, _d);
1542 case_rrd(gt);
1543 case_rrr(ne, _d);
1544 case_rrd(ne);
1545 case_rrr(unlt, _d);
1546 case_rrd(unlt);
1547 case_rrr(unle, _d);
1548 case_rrd(unle);
1549 case_rrr(uneq, _d);
1550 case_rrd(uneq);
1551 case_rrr(unge, _d);
1552 case_rrd(unge);
1553 case_rrr(ungt, _d);
1554 case_rrd(ungt);
1555 case_rrr(ltgt, _d);
1556 case_rrd(ltgt);
1557 case_rrr(ord, _d);
1558 case_rrd(ord);
1559 case_rrr(unord, _d);
1560 case_rrd(unord);
1561 case_brr(blt, _d);
1562 case_brd(blt);
1563 case_brr(ble, _d);
1564 case_brd(ble);
1565 case_brr(beq, _d);
1566 case_brd(beq);
1567 case_brr(bge, _d);
1568 case_brd(bge);
1569 case_brr(bgt, _d);
1570 case_brd(bgt);
1571 case_brr(bne, _d);
1572 case_brd(bne);
1573 case_brr(bunlt, _d);
1574 case_brd(bunlt);
1575 case_brr(bunle, _d);
1576 case_brd(bunle);
1577 case_brr(buneq, _d);
1578 case_brd(buneq);
1579 case_brr(bunge, _d);
1580 case_brd(bunge);
1581 case_brr(bungt, _d);
1582 case_brd(bungt);
1583 case_brr(bltgt, _d);
1584 case_brd(bltgt);
1585 case_brr(bord, _d);
1586 case_brd(bord);
1587 case_brr(bunord, _d);
1588 case_brd(bunord);
1589 case jit_code_jmpr:
79bfeef6 1590 jit_check_frame();
4a71579b
PC		 1591 jmpr(rn(node->u.w));
1592 break;
1593 case jit_code_jmpi:
1594 if (node->flag & jit_flag_node) {
1595 temp = node->u.n;
1596 assert(temp->code == jit_code_label ||
1597 temp->code == jit_code_epilog);
1598 if (temp->flag & jit_flag_patch)
1599 jmpi(temp->u.w);
1600 else {
79bfeef6
PC		 1601 word = _jit->code.length -
1602 (_jit->pc.uc - _jit->code.ptr);
1603 if (simm20_p(word))
1604 word = jmpi(_jit->pc.w);
1605 else
4a71579b
PC		 1606 word = jmpi_p(_jit->pc.w);
1607 patch(word, node);
1608 }
1609 }
79bfeef6
PC		 1610 else {
1611 jit_check_frame();
4a71579b 1612 jmpi(node->u.w);
79bfeef6 1613 }
4a71579b
PC		 1614 break;
1615 case jit_code_callr:
79bfeef6 1616 jit_check_frame();
4a71579b
PC		 1617 callr(rn(node->u.w));
1618 break;
1619 case jit_code_calli:
1620 if (node->flag & jit_flag_node) {
1621 temp = node->u.n;
1622 assert(temp->code == jit_code_label ||
1623 temp->code == jit_code_epilog);
1624 if (temp->flag & jit_flag_patch)
1625 calli(temp->u.w);
1626 else {
79bfeef6
PC		 1627 word = _jit->code.length -
1628 (_jit->pc.uc - _jit->code.ptr);
1629 if (simm20_p(word))
1630 word = calli(_jit->pc.w);
1631 else
1632 word = calli_p(_jit->pc.w);
4a71579b
PC		 1633 patch(word, node);
1634 }
1635 }
79bfeef6
PC		 1636 else {
1637 jit_check_frame();
4a71579b 1638 calli(node->u.w);
79bfeef6 1639 }
4a71579b
PC		 1640 break;
1641 case jit_code_prolog:
1642 _jitc->function = _jitc->functions.ptr + node->w.w;
1643 undo.node = node;
1644 undo.word = _jit->pc.w;
79bfeef6 1645 memcpy(&undo.func, _jitc->function, sizeof(undo.func));
4a71579b
PC		 1646#if DEVEL_DISASSEMBLER
1647 undo.prevw = prevw;
1648#endif
ba86ff93 1649 undo.const_offset = _jitc->consts.vector.offset;
4a71579b
PC		 1650 undo.patch_offset = _jitc->patches.offset;
1651 restart_function:
79bfeef6
PC		 1652 compute_framesize();
1653 patch_alist(0);
4a71579b
PC		 1654 _jitc->again = 0;
1655 prolog(node);
1656 break;
1657 case jit_code_epilog:
1658 assert(_jitc->function == _jitc->functions.ptr + node->w.w);
1659 if (_jitc->again) {
1660 for (temp = undo.node->next;
1661 temp != node; temp = temp->next) {
1662 if (temp->code == jit_code_label ||
1663 temp->code == jit_code_epilog)
1664 temp->flag &= ~jit_flag_patch;
1665 }
1666 temp->flag &= ~jit_flag_patch;
1667 node = undo.node;
1668 _jit->pc.w = undo.word;
79bfeef6
PC		 1669 /* undo.func.self.aoff and undo.func.regset should not
1670 * be undone, as they will be further updated, and are
1671 * the reason of the undo. */
1672 undo.func.self.aoff = _jitc->function->frame +
1673 _jitc->function->self.aoff;
1674 undo.func.need_frame = _jitc->function->need_frame;
1675 jit_regset_set(&undo.func.regset, &_jitc->function->regset);
1676 /* allocar information also does not need to be undone */
1677 undo.func.aoffoff = _jitc->function->aoffoff;
1678 undo.func.allocar = _jitc->function->allocar;
1679 /* this will be recomputed but undo anyway to have it
1680 * better self documented.*/
1681 undo.func.need_stack = _jitc->function->need_stack;
1682 memcpy(_jitc->function, &undo.func, sizeof(undo.func));
4a71579b
PC		 1683#if DEVEL_DISASSEMBLER
1684 prevw = undo.prevw;
1685#endif
1686 _jitc->patches.offset = undo.patch_offset;
ba86ff93 1687 _jitc->consts.vector.offset = undo.const_offset;
79bfeef6 1688 patch_alist(1);
4a71579b
PC		 1689 goto restart_function;
1690 }
1691 /* remember label is defined */
1692 node->flag |= jit_flag_patch;
1693 node->u.w = _jit->pc.w;
1694 epilog(node);
1695 _jitc->function = NULL;
1696 break;
1697 case jit_code_movr_w_f:
1698 movr_w_f(rn(node->u.w), rn(node->v.w));
1699 break;
1700 case jit_code_movr_f_w:
1701 movr_f_w(rn(node->u.w), rn(node->v.w));
1702 break;
1703 case jit_code_movi_f_w:
1704 assert_data(node);
1705 movi_f_w(rn(node->u.w), node->v.f);
1706 break;
ba86ff93
PC		 1707 case jit_code_movi_w_f:
1708 movi_w_f(rn(node->u.w), node->v.w);
1709 break;
4a71579b
PC		 1710 case jit_code_movr_w_d:
1711 movr_w_d(rn(node->u.w), rn(node->v.w));
1712 break;
1713 case jit_code_movr_d_w:
1714 movr_d_w(rn(node->u.w), rn(node->v.w));
1715 break;
1716 case jit_code_movi_d_w:
1717 assert_data(node);
1718 movi_d_w(rn(node->u.w), node->v.d);
1719 break;
ba86ff93
PC		 1720 case jit_code_movi_w_d:
1721 movi_w_d(rn(node->u.w), node->v.w);
1722 break;
4a71579b
PC		 1723 case jit_code_va_start:
1724 vastart(rn(node->u.w));
1725 break;
1726 case jit_code_va_arg:
1727 vaarg(rn(node->u.w), rn(node->v.w));
1728 break;
1729 case jit_code_va_arg_d:
1730 vaarg_d(rn(node->u.w), rn(node->v.w));
1731 break;
1732 case jit_code_live: case jit_code_ellipsis:
1733 case jit_code_va_push:
1734 case jit_code_allocai: case jit_code_allocar:
79bfeef6
PC		 1735 case jit_code_arg_c: case jit_code_arg_s:
1736 case jit_code_arg_i:
1737 case jit_code_arg_l:
4a71579b
PC		 1738 case jit_code_arg_f: case jit_code_arg_d:
1739 case jit_code_va_end:
1740 case jit_code_ret:
79bfeef6
PC		 1741 case jit_code_retr_c: case jit_code_reti_c:
1742 case jit_code_retr_uc: case jit_code_reti_uc:
1743 case jit_code_retr_s: case jit_code_reti_s:
1744 case jit_code_retr_us: case jit_code_reti_us:
1745 case jit_code_retr_i: case jit_code_reti_i:
1746 case jit_code_retr_ui: case jit_code_reti_ui:
1747 case jit_code_retr_l: case jit_code_reti_l:
4a71579b
PC		 1748 case jit_code_retr_f: case jit_code_reti_f:
1749 case jit_code_retr_d: case jit_code_reti_d:
1750 case jit_code_getarg_c: case jit_code_getarg_uc:
1751 case jit_code_getarg_s: case jit_code_getarg_us:
1752 case jit_code_getarg_i: case jit_code_getarg_ui:
1753 case jit_code_getarg_l:
1754 case jit_code_getarg_f: case jit_code_getarg_d:
79bfeef6
PC		 1755 case jit_code_putargr_c: case jit_code_putargi_c:
1756 case jit_code_putargr_uc: case jit_code_putargi_uc:
1757 case jit_code_putargr_s: case jit_code_putargi_s:
1758 case jit_code_putargr_us: case jit_code_putargi_us:
1759 case jit_code_putargr_i: case jit_code_putargi_i:
1760 case jit_code_putargr_ui: case jit_code_putargi_ui:
1761 case jit_code_putargr_l: case jit_code_putargi_l:
4a71579b
PC		 1762 case jit_code_putargr_f: case jit_code_putargi_f:
1763 case jit_code_putargr_d: case jit_code_putargi_d:
79bfeef6
PC		 1764 case jit_code_pushargr_c: case jit_code_pushargi_c:
1765 case jit_code_pushargr_uc: case jit_code_pushargi_uc:
1766 case jit_code_pushargr_s: case jit_code_pushargi_s:
1767 case jit_code_pushargr_us: case jit_code_pushargi_us:
1768 case jit_code_pushargr_i: case jit_code_pushargi_i:
1769 case jit_code_pushargr_ui: case jit_code_pushargi_ui:
1770 case jit_code_pushargr_l: case jit_code_pushargi_l:
4a71579b
PC		 1771 case jit_code_pushargr_f: case jit_code_pushargi_f:
1772 case jit_code_pushargr_d: case jit_code_pushargi_d:
1773 case jit_code_retval_c: case jit_code_retval_uc:
1774 case jit_code_retval_s: case jit_code_retval_us:
1775 case jit_code_retval_i:
1776 case jit_code_retval_ui: case jit_code_retval_l:
1777 case jit_code_retval_f: case jit_code_retval_d:
1778 case jit_code_prepare:
1779 case jit_code_finishr: case jit_code_finishi:
ba86ff93
PC		 1780 case jit_code_negi_f: case jit_code_absi_f:
1781 case jit_code_sqrti_f: case jit_code_negi_d:
1782 case jit_code_absi_d: case jit_code_sqrti_d:
1783 break;
1784 case jit_code_negi:
1785 negi(rn(node->u.w), node->v.w);
1786 break;
1787 case jit_code_comi:
1788 comi(rn(node->u.w), node->v.w);
1789 break;
1790 case jit_code_exti_c:
1791 exti_c(rn(node->u.w), node->v.w);
1792 break;
1793 case jit_code_exti_uc:
1794 exti_uc(rn(node->u.w), node->v.w);
1795 break;
1796 case jit_code_exti_s:
1797 exti_s(rn(node->u.w), node->v.w);
1798 break;
1799 case jit_code_exti_us:
1800 exti_us(rn(node->u.w), node->v.w);
1801 break;
1802 case jit_code_bswapi_us:
1803 bswapi_us(rn(node->u.w), node->v.w);
1804 break;
1805 case jit_code_bswapi_ui:
1806 bswapi_ui(rn(node->u.w), node->v.w);
1807 break;
1808 case jit_code_htoni_us:
1809 htoni_us(rn(node->u.w), node->v.w);
1810 break;
1811 case jit_code_htoni_ui:
1812 htoni_ui(rn(node->u.w), node->v.w);
1813 break;
1814#if __WORDSIZE == 64
1815 case jit_code_exti_i:
1816 exti_i(rn(node->u.w), node->v.w);
1817 break;
1818 case jit_code_exti_ui:
1819 exti_ui(rn(node->u.w), node->v.w);
1820 break;
1821 case jit_code_bswapi_ul:
1822 bswapi_ul(rn(node->u.w), node->v.w);
1823 break;
1824 case jit_code_htoni_ul:
1825 htoni_ul(rn(node->u.w), node->v.w);
1826 break;
1827#endif
1828 case jit_code_cloi:
1829 cloi(rn(node->u.w), node->v.w);
1830 break;
1831 case jit_code_clzi:
1832 clzi(rn(node->u.w), node->v.w);
1833 break;
1834 case jit_code_ctoi:
1835 ctoi(rn(node->u.w), node->v.w);
1836 break;
1837 case jit_code_ctzi:
1838 ctzi(rn(node->u.w), node->v.w);
1839 break;
1840 case jit_code_rbiti:
1841 rbiti(rn(node->u.w), node->v.w);
1842 break;
1843 case jit_code_popcnti:
1844 popcnti(rn(node->u.w), node->v.w);
1845 break;
1846 case jit_code_exti:
1847 exti(rn(node->u.w), node->v.w, node->w.q.l, node->w.q.h);
1848 break;
1849 case jit_code_exti_u:
1850 exti_u(rn(node->u.w), node->v.w, node->w.q.l, node->w.q.h);
4a71579b
PC		 1851 break;
1852 default:
1853 abort();
1854 }
1855 if (jit_carry != _NOREG) {
1856 switch (node->code) {
1857 case jit_code_note:
1858 case jit_code_addcr: case jit_code_addci:
1859 case jit_code_addxr: case jit_code_addxi:
1860 case jit_code_subcr: case jit_code_subci:
1861 case jit_code_subxr: case jit_code_subxi:
1862 break;
1863 default:
1864 jit_unget_reg(jit_carry);
1865 jit_carry = _NOREG;
1866 break;
1867 }
1868 }
1869 jit_regarg_clr(node, value);
1870 assert(_jitc->regarg == 0 ||
1871 (jit_carry != _NOREG && _jitc->regarg == (1 << jit_carry)));
1872 assert(_jitc->synth == 0);
1873 /* update register live state */
1874 jit_reglive(node);
1875 }
1876#undef case_brw
1877#undef case_brr
1878#undef case_wrr
1879#undef case_rrw
1880#undef case_rrr
1881#undef case_wr
1882#undef case_rw
1883#undef case_rr
1884
c0c16242
PC		 1885#if __WORDSIZE == 64
1886 /* Record all constants to be patched */
1887 for (offset = 0; offset < _jitc->patches.offset; offset++) {
1888 node = _jitc->patches.ptr[offset].node;
1889 value = node->code == jit_code_movi ? node->v.n->u.w : node->u.n->u.w;
1890 put_const(value);
1891 }
1892 /* Record all direct constants */
1893 for (offset = 0; offset < _jitc->consts.vector.offset; offset++)
1894 put_const(_jitc->consts.vector.values[offset]);
1895 /* Now actually inject constants at the end of code buffer */
1896 if (_jitc->consts.hash.count) {
1897 jit_const_t *entry;
1898 /* Insert nop if aligned at 4 bytes */
1899 if (_jit->pc.w % sizeof(jit_word_t))
1900 nop(_jit->pc.w % sizeof(jit_word_t));
1901 for (offset = 0; offset < _jitc->consts.hash.size; offset++) {
1902 entry = _jitc->consts.hash.table[offset];
1903 for (; entry; entry = entry->next) {
1904 /* Make sure to not write out of bounds */
1905 if (_jit->pc.uc >= _jitc->code.end)
1906 return (NULL);
1907 entry->address = _jit->pc.w;
1908 *_jit->pc.ul++ = entry->value;
1909 }
1910 }
1911 }
1912#endif
1913
4a71579b
PC		 1914 for (offset = 0; offset < _jitc->patches.offset; offset++) {
1915 node = _jitc->patches.ptr[offset].node;
1916 word = _jitc->patches.ptr[offset].inst;
1917 value = node->code == jit_code_movi ? node->v.n->u.w : node->u.n->u.w;
1918 patch_at(word, value);
1919 }
1920
c0c16242
PC		 1921#if __WORDSIZE == 64
1922 /* Patch direct complex constants */
1923 if (_jitc->consts.vector.instrs) {
1924 for (offset = 0; offset < _jitc->consts.vector.offset; offset++)
1925 patch_at(_jitc->consts.vector.instrs[offset],
1926 _jitc->consts.vector.values[offset]);
1927 jit_free((jit_pointer_t *)&_jitc->consts.vector.instrs);
1928 jit_free((jit_pointer_t *)&_jitc->consts.vector.values);
1929 }
1930
1931 /* Hash table no longer need */
1932 if (_jitc->consts.hash.table) {
1933 jit_free((jit_pointer_t *)&_jitc->consts.hash.table);
1934 for (offset = 0; offset < _jitc->consts.pool.length; offset++)
1935 jit_free((jit_pointer_t *)_jitc->consts.pool.ptr + offset);
1936 jit_free((jit_pointer_t *)&_jitc->consts.pool.ptr);
1937 }
1938#endif
1939
4a71579b
PC		 1940 jit_flush(_jit->code.ptr, _jit->pc.uc);
1941
1942 return (_jit->code.ptr);
1943}
1944
1945#define CODE 1
1946# include "jit_riscv-cpu.c"
1947# include "jit_riscv-fpu.c"
79bfeef6 1948# include "jit_fallback.c"
4a71579b
PC		 1949#undef CODE
1950
c0c16242
PC		 1951static void
1952_load_const(jit_state_t *_jit, jit_int32_t reg, jit_word_t value)
1953{
1954 if (_jitc->consts.vector.offset >= _jitc->consts.vector.length) {
1955 jit_word_t new_size = _jitc->consts.vector.length *
1956 2 * sizeof(jit_word_t);
1957 jit_realloc((jit_pointer_t *)&_jitc->consts.vector.instrs,
1958 _jitc->consts.vector.length * sizeof(jit_word_t), new_size);
1959 jit_realloc((jit_pointer_t *)&_jitc->consts.vector.values,
1960 _jitc->consts.vector.length * sizeof(jit_word_t), new_size);
1961 _jitc->consts.vector.length *= 2;
1962 }
1963 _jitc->consts.vector.instrs[_jitc->consts.vector.offset] = _jit->pc.w;
1964 _jitc->consts.vector.values[_jitc->consts.vector.offset] = value;
1965 ++_jitc->consts.vector.offset;
1966 /* Resolve later the pc relative address */
1967 put_const(value);
1968 AUIPC(reg, 0);
1969 ADDI(reg, reg, 0);
1970 LD(reg, reg, 0);
1971}
1972
1973static jit_word_t
1974hash_const(jit_word_t value)
1975{
1976 const jit_uint8_t *ptr;
1977 jit_word_t i, key;
1978 for (i = key = 0, ptr = (jit_uint8_t *)&value; i < 4; ++i)
1979 key = (key << (key & 1)) ^ ptr[i];
1980 return (key);
1981
1982}
1983
1984static void
1985_put_const(jit_state_t *_jit, jit_word_t value)
1986{
1987 jit_word_t key;
1988 jit_const_t *entry;
1989
1990 /* Check if already inserted in table */
1991 key = hash_const(value) % _jitc->consts.hash.size;
1992 for (entry = _jitc->consts.hash.table[key]; entry; entry = entry->next) {
1993 if (entry->value == value)
1994 return;
1995 }
1996
1997 /* Check if need to increase pool size */
1998 if (_jitc->consts.pool.list->next == NULL) {
1999 jit_const_t *list;
2000 jit_word_t offset;
2001 jit_word_t new_size = (_jitc->consts.pool.length + 1) *
2002 sizeof(jit_const_t*);
2003 jit_realloc((jit_pointer_t *)&_jitc->consts.pool.ptr,
2004 _jitc->consts.pool.length * sizeof(jit_const_t*), new_size);
2005 jit_alloc((jit_pointer_t *)
2006 _jitc->consts.pool.ptr + _jitc->consts.pool.length,
2007 1024 * sizeof(jit_const_t));
2008 list = _jitc->consts.pool.ptr[_jitc->consts.pool.length];
2009 _jitc->consts.pool.list->next = list;
2010 for (offset = 0; offset < 1023; ++offset, ++list)
2011 list->next = list + 1;
2012 list->next = NULL;
2013 ++_jitc->consts.pool.length;
2014 }
2015
2016 /* Rehash if more than 75% used table */
2017 if (_jitc->consts.hash.count > (_jitc->consts.hash.size / 4) * 3) {
2018 jit_word_t i, k;
2019 jit_const_t *next;
2020 jit_const_t **table;
2021 jit_alloc((jit_pointer_t *)&table,
2022 _jitc->consts.hash.size * 2 * sizeof(jit_const_t *));
2023 for (i = 0; i < _jitc->consts.hash.size; ++i) {
2024 for (entry = _jitc->consts.hash.table[i]; entry; entry = next) {
2025 next = entry->next;
2026 k = hash_const(entry->value) % (_jitc->consts.hash.size * 2);
2027 entry->next = table[k];
2028 table[k] = entry;
2029 }
2030 }
2031 jit_free((jit_pointer_t *)&_jitc->consts.hash.table);
2032 _jitc->consts.hash.size *= 2;
2033 _jitc->consts.hash.table = table;
2034 }
2035
2036 /* Insert in hash */
2037 entry = _jitc->consts.pool.list;
2038 _jitc->consts.pool.list = entry->next;
2039 ++_jitc->consts.hash.count;
2040 entry->value = value;
2041 entry->next = _jitc->consts.hash.table[key];
2042 _jitc->consts.hash.table[key] = entry;
2043}
2044
2045static jit_word_t
2046_get_const(jit_state_t *_jit, jit_word_t value)
2047{
2048 jit_word_t key;
2049 jit_const_t *entry;
2050 key = hash_const(value) % _jitc->consts.hash.size;
2051 for (entry = _jitc->consts.hash.table[key]; entry; entry = entry->next) {
2052 if (entry->value == value)
2053 return (entry->address);
2054 }
2055 /* Only the final patch should call get_const() */
2056 abort();
2057}
2058
4a71579b
PC		 2059void
2060jit_flush(void *fptr, void *tptr)
2061{
2062#if defined(__GNUC__)
2063 jit_word_t f, t, s;
2064
2065 s = sysconf(_SC_PAGE_SIZE);
2066 f = (jit_word_t)fptr & -s;
2067 t = (((jit_word_t)tptr) + s - 1) & -s;
2068 __clear_cache((void *)f, (void *)t);
2069#endif
2070}
2071
2072void
2073_emit_ldxi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
2074{
2075 ldxi(rn(r0), rn(r1), i0);
2076}
2077
2078void
2079_emit_stxi(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
2080{
2081 stxi(i0, rn(r0), rn(r1));
2082}
2083
2084void
2085_emit_ldxi_d(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
2086{
2087 ldxi_d(rn(r0), rn(r1), i0);
2088}
2089
2090void
2091_emit_stxi_d(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
2092{
2093 stxi_d(i0, rn(r0), rn(r1));
2094}
2095
79bfeef6
PC		 2096#if __WORDSIZE != 64
2097# error "only 64 bit ports tested"
2098#endif
2099static void
2100_compute_framesize(jit_state_t *_jit)
2101{
2102 jit_int32_t reg;
2103 _jitc->framesize = 16; /* ra+fp */
2104 for (reg = 0; reg < jit_size(iregs); reg++)
2105 if (jit_regset_tstbit(&_jitc->function->regset, iregs[reg]))
2106 _jitc->framesize += sizeof(jit_word_t);
2107
2108 for (reg = 0; reg < jit_size(fregs); reg++)
2109 if (jit_regset_tstbit(&_jitc->function->regset, fregs[reg]))
2110 _jitc->framesize += sizeof(jit_float64_t);
2111
2112 /* Space to store variadic arguments */
2113 if (_jitc->function->self.call & jit_call_varargs)
2114 _jitc->framesize += (8 - _jitc->function->vagp) * 8;
2115
2116 /* Make sure functions called have a 16 byte aligned stack */
2117 _jitc->framesize = (_jitc->framesize + 15) & -16;
2118}
2119
4a71579b
PC		 2120static void
2121_patch(jit_state_t *_jit, jit_word_t instr, jit_node_t *node)
2122{
2123 jit_int32_t flag;
2124
2125 assert(node->flag & jit_flag_node);
2126 if (node->code == jit_code_movi)
2127 flag = node->v.n->flag;
2128 else
2129 flag = node->u.n->flag;
2130 assert(!(flag & jit_flag_patch));
2131 if (_jitc->patches.offset >= _jitc->patches.length) {
2132 jit_realloc((jit_pointer_t *)&_jitc->patches.ptr,
2133 _jitc->patches.length * sizeof(jit_patch_t),
2134 (_jitc->patches.length + 1024) * sizeof(jit_patch_t));
2135 _jitc->patches.length += 1024;
2136 }
2137 _jitc->patches.ptr[_jitc->patches.offset].inst = instr;
2138 _jitc->patches.ptr[_jitc->patches.offset].node = node;
2139 ++_jitc->patches.offset;
2140}
ARM optimized PCSX fork