notaz.gp2x.de / pcsx_rearmed.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
add a thp-based huge page alloc fallback
[pcsx_rearmed.git] / deps / lightning / lib / jit_arm.c
CommitLineData
4a71579b 1/*
79bfeef6 2 * Copyright (C) 2012-2023 Free Software Foundation, Inc.
4a71579b
PC		 3 *
4 * This file is part of GNU lightning.
5 *
6 * GNU lightning is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU Lesser General Public License as published
8 * by the Free Software Foundation; either version 3, or (at your option)
9 * any later version.
10 *
11 * GNU lightning is distributed in the hope that it will be useful, but
12 * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
13 * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public
14 * License for more details.
15 *
16 * Authors:
17 * Paulo Cesar Pereira de Andrade
18 */
19
20#if defined(__linux__)
21# include <stdio.h>
22#endif
23
79bfeef6
PC		 24#define stack_framesize 48
25
4a71579b
PC		 26#define jit_arg_reg_p(i) ((i) >= 0 && (i) < 4)
27#define jit_arg_f_reg_p(i) ((i) >= 0 && (i) < 16)
28#define jit_arg_d_reg_p(i) ((i) >= 0 && (i) < 15)
29
30#define arm_patch_node 0x80000000
31#define arm_patch_word 0x40000000
32#define arm_patch_jump 0x20000000
79bfeef6
PC		 33#define arm_patch_load 0x10000000
34#define arm_patch_call 0x08000000
4a71579b
PC		 35
36#define jit_fpr_p(rn) ((rn) > 15)
37
79bfeef6 38#define arg_base() (stack_framesize - 16)
4a71579b
PC		 39#define arg_offset(n) \
40 ((n) < 4 ? arg_base() + ((n) << 2) : (n))
41
42/* Assume functions called never match jit instruction set, that is
43 * libc, gmp, mpfr, etc functions are in thumb mode and jit is in
44 * arm mode, what may cause a crash upon return of that function
45 * if generating jit for a relative jump.
46 */
79bfeef6 47#define jit_exchange_p() jit_cpu.exchange
4a71579b
PC		 48
49/* FIXME is it really required to not touch _R10? */
50
79bfeef6
PC		 51#define CHECK_REG_ARGS() \
52 do { \
53 if (!_jitc->function->save_reg_args) \
54 _jitc->again = _jitc->function->save_reg_args = 1; \
55 } while (0)
56
57#define CHECK_SWF_OFFSET() \
58 do { \
59 if (!_jitc->function->swf_offset) { \
60 _jitc->again = _jitc->function->save_reg_args = \
61 _jitc->function->swf_offset = 1; \
62 _jitc->function->self.aoff = -64; \
63 } \
64 } while (0)
65
66#define CHECK_RETURN() \
67 do { \
68 if (!_jitc->function->need_frame && \
69 !_jitc->function->need_return) \
70 _jitc->again = _jitc->function->need_return = 1; \
71 } while (0)
72
4a71579b
PC		 73/*
74 * Types
75 */
76typedef union _jit_thumb_t {
77 jit_int32_t i;
78 jit_int16_t s[2];
79} jit_thumb_t;
80
81typedef jit_pointer_t jit_va_list;
82
83/*
84 * Prototypes
85 */
79bfeef6
PC		 86#define jit_make_arg(node,code) _jit_make_arg(_jit,node,code)
87static jit_node_t *_jit_make_arg(jit_state_t*,jit_node_t*,jit_code_t);
4a71579b
PC		 88#define jit_make_arg_f(node) _jit_make_arg_f(_jit,node)
89static jit_node_t *_jit_make_arg_f(jit_state_t*,jit_node_t*);
90#define jit_make_arg_d(node) _jit_make_arg_d(_jit,node)
91static jit_node_t *_jit_make_arg_d(jit_state_t*,jit_node_t*);
92#define jit_get_reg_pair() _jit_get_reg_pair(_jit)
93static jit_int32_t _jit_get_reg_pair(jit_state_t*);
94#define jit_unget_reg_pair(rn) _jit_unget_reg_pair(_jit,rn)
95static void _jit_unget_reg_pair(jit_state_t*,jit_int32_t);
96# define must_align_p(node) _must_align_p(_jit, node)
97static jit_bool_t _must_align_p(jit_state_t*,jit_node_t*);
98#define load_const(uniq,r0,i0) _load_const(_jit,uniq,r0,i0)
99static void _load_const(jit_state_t*,jit_bool_t,jit_int32_t,jit_word_t);
100#define flush_consts() _flush_consts(_jit)
101static void _flush_consts(jit_state_t*);
102#define invalidate_consts() _invalidate_consts(_jit)
103static void _invalidate_consts(jit_state_t*);
79bfeef6
PC		 104#define compute_framesize() _compute_framesize(_jit)
105static void _compute_framesize(jit_state_t*);
106#define patch(instr, node, kind) _patch(_jit, instr, node, kind)
107static void _patch(jit_state_t*,jit_word_t,jit_node_t*,jit_int32_t);
4a71579b
PC		 108
109#if defined(__GNUC__)
110/* libgcc */
111extern void __clear_cache(void *, void *);
112#endif
113
114#define PROTO 1
115# include "jit_rewind.c"
116# include "jit_arm-cpu.c"
117# include "jit_arm-swf.c"
118# include "jit_arm-vfp.c"
ba3814c1 119# include "jit_fallback.c"
4a71579b
PC		 120#undef PROTO
121
122/*
123 * Initialization
124 */
125jit_cpu_t jit_cpu;
126jit_register_t _rvs[] = {
127 { rc(gpr) | 0x0c, "ip" },
128 { rc(sav) | rc(gpr) | 0x04, "r4" },
129 { rc(sav) | rc(gpr) | 0x05, "r5" },
130 { rc(sav) | rc(gpr) | 0x06, "r6" },
131 { rc(sav) | rc(gpr) | 0x07, "r7" },
132 { rc(sav) | rc(gpr) | 0x08, "r8" },
133 { rc(sav) | rc(gpr) | 0x09, "r9" },
134 { rc(sav) | 0x0a, "sl" },
135 { rc(sav) | 0x0b, "fp" },
136 { rc(sav) | 0x0d, "sp" },
137 { rc(sav) | 0x0e, "lr" },
138 { 0x0f, "pc" },
139 { rc(arg) | rc(gpr) | 0x03, "r3" },
140 { rc(arg) | rc(gpr) | 0x02, "r2" },
141 { rc(arg) | rc(gpr) | 0x01, "r1" },
142 { rc(arg) | rc(gpr) | 0x00, "r0" },
143 { rc(fpr) | 0x20, "d8" },
144 { 0x21, "s17" },
145 { rc(fpr) | 0x22, "d9" },
146 { 0x23, "s19" },
147 { rc(fpr) | 0x24, "d10" },
148 { 0x25, "s21" },
149 { rc(fpr) | 0x26, "d11" },
150 { 0x27, "s23" },
151 { rc(fpr) | 0x28, "d12" },
152 { 0x29, "s25" },
153 { rc(fpr) | 0x2a, "d13" },
154 { 0x2b, "s27" },
155 { rc(fpr) | 0x2c, "d14" },
156 { 0x2d, "s29" },
157 { rc(fpr) | 0x2e, "d15" },
158 { 0x2f, "s31" },
159 { rc(arg) | 0x1f, "s15" },
160 { rc(arg)|rc(sft)|rc(fpr)|0x1e, "d7" },
161 { rc(arg) | 0x1d, "s13" },
162 { rc(arg)|rc(sft)|rc(fpr)|0x1c, "d6" },
163 { rc(arg) | 0x1b, "s11" },
164 { rc(arg)|rc(sft)|rc(fpr)|0x1a, "d5" },
165 { rc(arg) | 0x19, "s9" },
166 { rc(arg)|rc(sft)|rc(fpr)|0x18, "d4" },
167 { rc(arg) | 0x17, "s7" },
168 { rc(arg)|rc(sft)|rc(fpr)|0x16, "d3" },
169 { rc(arg) | 0x15, "s5" },
170 { rc(arg)|rc(sft)|rc(fpr)|0x14, "d2" },
171 { rc(arg) | 0x13, "s3" },
172 { rc(arg)|rc(sft)|rc(fpr)|0x12, "d1" },
173 { rc(arg) | 0x11, "s1" },
174 { rc(arg)|rc(sft)|rc(fpr)|0x10, "d0" },
175 { _NOREG, "<none>" },
176};
177
79bfeef6
PC		 178static jit_int32_t iregs[] = {
179 _R4, _R5, _R6, _R7, _R8, _R9,
180};
181
4a71579b
PC		 182/*
183 * Implementation
184 */
185void
186jit_get_cpu(void)
187{
188#if defined(__linux__)
189 FILE *fp;
190 char *ptr;
191 char buf[128];
192
193 if ((fp = fopen("/proc/cpuinfo", "r")) != NULL) {
194 while (fgets(buf, sizeof(buf), fp)) {
195 if (strncmp(buf, "CPU architecture:", 17) == 0) {
196 jit_cpu.version = strtol(buf + 17, &ptr, 10);
197 while (*ptr) {
198 if (*ptr == 'T' || *ptr == 't') {
199 ++ptr;
200 jit_cpu.thumb = 1;
201 }
202 else if (*ptr == 'E' || *ptr == 'e') {
203 jit_cpu.extend = 1;
204 ++ptr;
205 }
206 else
207 ++ptr;
208 }
209 }
210 else if (strncmp(buf, "Features\t:", 10) == 0) {
211 if ((ptr = strstr(buf + 10, "vfpv")))
212 jit_cpu.vfp = strtol(ptr + 4, NULL, 0);
213 if ((ptr = strstr(buf + 10, "neon")))
214 jit_cpu.neon = 1;
215 if ((ptr = strstr(buf + 10, "thumb")))
216 jit_cpu.thumb = 1;
217 }
218 }
219 fclose(fp);
220 }
221#endif
222#if defined(__ARM_PCS_VFP)
223 if (!jit_cpu.vfp)
224 jit_cpu.vfp = 3;
225 if (!jit_cpu.version)
226 jit_cpu.version = 7;
227 jit_cpu.abi = 1;
228#endif
229#if defined(__thumb2__)
230 jit_cpu.thumb = 1;
231#endif
232 /* armv6t2 todo (software float and thumb2) */
233 if (!jit_cpu.vfp && jit_cpu.thumb)
234 jit_cpu.thumb = 0;
79bfeef6
PC		 235 /* FIXME need test environments for the below. For the moment just
236 * be very conservative */
237 /* force generation of code assuming jit and function libraries called
238 * instruction set do not match */
239 jit_cpu.exchange = 1;
240 /* do not generate hardware integer division by default */
241 if (jit_cpu.version == 7)
242 jit_cpu.extend = 0;
ba86ff93
PC		 243
244 /* By default generate extra instructions for unaligned load/store. */
245 jit_cpu.unaligned = 1;
246 /* Linux should only not handle unaligned vfp load/store */
247 jit_cpu.vfp_unaligned = 1;
4a71579b
PC		 248}
249
250void
251_jit_init(jit_state_t *_jit)
252{
253 jit_int32_t regno;
254 static jit_bool_t first = 1;
255
256 _jitc->reglen = jit_size(_rvs) - 1;
257 if (first) {
258 /* jit_get_cpu() should have been already called, and only once */
259 if (!jit_cpu.vfp) {
260 /* cause register to never be allocated, because simple
261 * software float only allocates stack space for 8 slots */
262 for (regno = _D8; regno < _D7; regno++)
263 _rvs[regno].spec = 0;
264 }
265 if (!jit_cpu.abi) {
266 for (regno = _S15; regno <= _D0; regno++)
267 _rvs[regno].spec &= ~rc(arg);
268 }
269 first = 0;
270 }
271}
272
273void
274_jit_prolog(jit_state_t *_jit)
275{
276 jit_int32_t offset;
277
278 if (_jitc->function)
279 jit_epilog();
280 assert(jit_regset_cmp_ui(&_jitc->regarg, 0) == 0);
281 jit_regset_set_ui(&_jitc->regsav, 0);
282 offset = _jitc->functions.offset;
283 if (offset >= _jitc->functions.length) {
284 jit_realloc((jit_pointer_t *)&_jitc->functions.ptr,
285 _jitc->functions.length * sizeof(jit_function_t),
286 (_jitc->functions.length + 16) * sizeof(jit_function_t));
287 _jitc->functions.length += 16;
288 }
289 _jitc->function = _jitc->functions.ptr + _jitc->functions.offset++;
290 _jitc->function->self.size = stack_framesize;
4a71579b 291 _jitc->function->self.argi = _jitc->function->self.argf =
79bfeef6
PC		 292 _jitc->function->self.alen = _jitc->function->self.aoff = 0;
293 _jitc->function->swf_offset = _jitc->function->save_reg_args =
294 _jitc->function->need_return = 0;
4a71579b
PC		 295 _jitc->function->self.call = jit_call_default;
296 jit_alloc((jit_pointer_t *)&_jitc->function->regoff,
297 _jitc->reglen * sizeof(jit_int32_t));
298
299 /* _no_link here does not mean the jit_link() call can be removed
300 * by rewriting as:
301 * _jitc->function->prolog = jit_new_node(jit_code_prolog);
302 */
303 _jitc->function->prolog = jit_new_node_no_link(jit_code_prolog);
304 jit_link(_jitc->function->prolog);
305 _jitc->function->prolog->w.w = offset;
306 _jitc->function->epilog = jit_new_node_no_link(jit_code_epilog);
307 /* u: label value
308 * v: offset in blocks vector
309 * w: offset in functions vector
310 */
311 _jitc->function->epilog->w.w = offset;
312
313 jit_regset_new(&_jitc->function->regset);
314}
315
316jit_int32_t
317_jit_allocai(jit_state_t *_jit, jit_int32_t length)
318{
319 assert(_jitc->function);
79bfeef6
PC		 320 if (jit_swf_p())
321 CHECK_SWF_OFFSET();
322 jit_check_frame();
4a71579b
PC		 323 switch (length) {
324 case 0: case 1: break;
325 case 2: _jitc->function->self.aoff &= -2; break;
326 case 3: case 4: _jitc->function->self.aoff &= -4; break;
327 default: _jitc->function->self.aoff &= -8; break;
328 }
329 _jitc->function->self.aoff -= length;
330 if (!_jitc->realize) {
331 jit_inc_synth_ww(allocai, _jitc->function->self.aoff, length);
332 jit_dec_synth();
333 }
334 return (_jitc->function->self.aoff);
335}
336
337void
338_jit_allocar(jit_state_t *_jit, jit_int32_t u, jit_int32_t v)
339{
340 jit_int32_t reg;
341 assert(_jitc->function);
342 jit_inc_synth_ww(allocar, u, v);
343 if (!_jitc->function->allocar) {
344 _jitc->function->aoffoff = jit_allocai(sizeof(jit_int32_t));
345 _jitc->function->allocar = 1;
346 }
347 reg = jit_get_reg(jit_class_gpr);
348 jit_negr(reg, v);
349 jit_andi(reg, reg, -8);
350 jit_ldxi_i(u, JIT_FP, _jitc->function->aoffoff);
351 jit_addr(u, u, reg);
352 jit_addr(JIT_SP, JIT_SP, reg);
353 jit_stxi_i(_jitc->function->aoffoff, JIT_FP, u);
354 jit_unget_reg(reg);
355 jit_dec_synth();
356}
357
358void
359_jit_ret(jit_state_t *_jit)
360{
361 jit_node_t *instr;
362 assert(_jitc->function);
363 jit_inc_synth(ret);
364 /* jump to epilog */
365 instr = jit_jmpi();
366 jit_patch_at(instr, _jitc->function->epilog);
367 jit_dec_synth();
368}
369
370void
79bfeef6 371_jit_retr(jit_state_t *_jit, jit_int32_t u, jit_code_t code)
4a71579b 372{
79bfeef6
PC		 373 jit_code_inc_synth_w(code, u);
374 jit_movr(JIT_RET, u);
4a71579b
PC		 375 jit_ret();
376 jit_dec_synth();
377}
378
379void
79bfeef6 380_jit_reti(jit_state_t *_jit, jit_word_t u, jit_code_t code)
4a71579b 381{
79bfeef6 382 jit_code_inc_synth_w(code, u);
4a71579b
PC		 383 jit_movi(JIT_RET, u);
384 jit_ret();
385 jit_dec_synth();
386}
387
388void
389_jit_retr_f(jit_state_t *_jit, jit_int32_t u)
390{
391 jit_inc_synth_w(retr_f, u);
392 if (jit_cpu.abi) {
393 if (u != JIT_FRET)
394 jit_movr_f(JIT_FRET, u);
395 else
396 jit_live(JIT_FRET);
397 }
398 else {
399 if (u != JIT_RET)
400 jit_movr_f_w(JIT_RET, u);
401 else
402 jit_live(JIT_RET);
403 }
404 jit_ret();
405 jit_dec_synth();
406}
407
408void
409_jit_reti_f(jit_state_t *_jit, jit_float32_t u)
410{
411 jit_inc_synth_f(reti_f, u);
412 if (jit_cpu.abi)
413 jit_movi_f(JIT_FRET, u);
414 else
415 jit_movi_f_w(JIT_RET, u);
416 jit_ret();
417 jit_dec_synth();
418}
419
420void
421_jit_retr_d(jit_state_t *_jit, jit_int32_t u)
422{
423 jit_inc_synth_w(retr_d, u);
424 if (jit_cpu.abi) {
425 if (u != JIT_FRET)
426 jit_movr_d(JIT_FRET, u);
427 else
428 jit_live(JIT_FRET);
429 }
430 else {
431 if (u != JIT_RET)
432 jit_movr_d_ww(JIT_RET, _R1, u);
433 else
434 jit_live(JIT_RET);
435 }
436 jit_ret();
437 jit_dec_synth();
438}
439
440void
441_jit_reti_d(jit_state_t *_jit, jit_float64_t u)
442{
443 jit_inc_synth_d(reti_d, u);
444 if (jit_cpu.abi)
445 jit_movi_d(JIT_FRET, u);
446 else
447 jit_movi_d_ww(JIT_RET, _R1, u);
448 jit_ret();
449 jit_dec_synth();
450}
451
452void
453_jit_epilog(jit_state_t *_jit)
454{
455 assert(_jitc->function);
456 assert(_jitc->function->epilog->next == NULL);
457 jit_link(_jitc->function->epilog);
458 _jitc->function = NULL;
459}
460
461jit_bool_t
462_jit_arg_register_p(jit_state_t *_jit, jit_node_t *u)
463{
79bfeef6 464 if (!(u->code >= jit_code_arg_c && u->code <= jit_code_arg)) {
4a71579b
PC		 465 if (u->code == jit_code_arg_f) {
466 if (jit_cpu.abi)
467 return (jit_arg_f_reg_p(u->u.w));
468 }
469 else {
470 assert(u->code == jit_code_arg_d);
471 if (jit_cpu.abi)
472 return (jit_arg_d_reg_p(u->u.w));
473 }
474 }
475 return (jit_arg_reg_p(u->u.w));
476}
477
478static jit_node_t *
79bfeef6 479_jit_make_arg(jit_state_t *_jit, jit_node_t *node, jit_code_t code)
4a71579b
PC		 480{
481 jit_int32_t offset;
482 if (jit_arg_reg_p(_jitc->function->self.argi))
483 offset = _jitc->function->self.argi++;
484 else {
485 offset = _jitc->function->self.size;
486 _jitc->function->self.size += sizeof(jit_word_t);
487 }
488 if (node == (jit_node_t *)0)
79bfeef6 489 node = jit_new_node(code);
4a71579b
PC		 490 else
491 link_node(node);
492 node->u.w = offset;
493 node->v.w = ++_jitc->function->self.argn;
494 jit_link_prolog();
495 return (node);
496}
497
498jit_node_t *
499_jit_make_arg_f(jit_state_t *_jit, jit_node_t *node)
500{
501 jit_int32_t offset;
502 if (jit_cpu.abi && !(_jitc->function->self.call & jit_call_varargs)) {
503 if (jit_arg_f_reg_p(_jitc->function->self.argf)) {
504 offset = _jitc->function->self.argf++;
505 goto done;
506 }
507 }
508 else {
509 if (jit_arg_reg_p(_jitc->function->self.argi)) {
510 offset = _jitc->function->self.argi++;
511 goto done;
512 }
513 }
514 offset = _jitc->function->self.size;
515 _jitc->function->self.size += sizeof(jit_float32_t);
516done:
517 if (node == (jit_node_t *)0)
518 node = jit_new_node(jit_code_arg_f);
519 else
520 link_node(node);
521 node->u.w = offset;
522 node->v.w = ++_jitc->function->self.argn;
523 jit_link_prolog();
524 return (node);
525}
526
527jit_node_t *
528_jit_make_arg_d(jit_state_t *_jit, jit_node_t *node)
529{
530 jit_int32_t offset;
531 if (jit_cpu.abi && !(_jitc->function->self.call & jit_call_varargs)) {
532 if (jit_arg_d_reg_p(_jitc->function->self.argf)) {
533 if (_jitc->function->self.argf & 1)
534 ++_jitc->function->self.argf;
535 offset = _jitc->function->self.argf;
536 _jitc->function->self.argf += 2;
537 goto done;
538 }
539 }
540 else {
541 if (_jitc->function->self.argi & 1)
542 ++_jitc->function->self.argi;
543 if (jit_arg_reg_p(_jitc->function->self.argi)) {
544 offset = _jitc->function->self.argi;
545 _jitc->function->self.argi += 2;
546 goto done;
547 }
548 }
549 if (_jitc->function->self.size & 7)
550 _jitc->function->self.size += 4;
551 offset = _jitc->function->self.size;
552 _jitc->function->self.size += sizeof(jit_float64_t);
553done:
554 if (node == (jit_node_t *)0)
555 node = jit_new_node(jit_code_arg_d);
556 else
557 link_node(node);
558 node->u.w = offset;
559 node->v.w = ++_jitc->function->self.argn;
560 jit_link_prolog();
561 return (node);
562}
563
564void
565_jit_ellipsis(jit_state_t *_jit)
566{
567 if (_jitc->prepare) {
568 assert(!(_jitc->function->call.call & jit_call_varargs));
569 _jitc->function->call.call |= jit_call_varargs;
570 if (jit_cpu.abi && _jitc->function->call.argf)
571 rewind_prepare();
572 }
573 else {
574 assert(!(_jitc->function->self.call & jit_call_varargs));
575 _jitc->function->self.call |= jit_call_varargs;
79bfeef6 576 CHECK_REG_ARGS();
4a71579b
PC		 577 if (jit_cpu.abi && _jitc->function->self.argf)
578 rewind_prolog();
79bfeef6 579 /* First 4 stack addresses need to be spilled r0-r3 */
4a71579b
PC		 580 if (jit_arg_reg_p(_jitc->function->self.argi))
581 _jitc->function->vagp = _jitc->function->self.argi * 4;
582 else
583 _jitc->function->vagp = 16;
584 }
585 jit_inc_synth(ellipsis);
586 if (_jitc->prepare)
587 jit_link_prepare();
588 else
589 jit_link_prolog();
590 jit_dec_synth();
591}
592
593void
594_jit_va_push(jit_state_t *_jit, jit_int32_t u)
595{
596 jit_inc_synth_w(va_push, u);
597 jit_pushargr(u);
598 jit_dec_synth();
599}
600
601jit_node_t *
79bfeef6 602_jit_arg(jit_state_t *_jit, jit_code_t code)
4a71579b
PC		 603{
604 assert(_jitc->function);
79bfeef6
PC		 605 assert(!(_jitc->function->self.call & jit_call_varargs));
606#if STRONG_TYPE_CHECKING
607 assert(code >= jit_code_arg_c && code <= jit_code_arg);
608#endif
609 return (jit_make_arg((jit_node_t*)0, code));
4a71579b
PC		 610}
611
612jit_node_t *
613_jit_arg_f(jit_state_t *_jit)
614{
615 assert(_jitc->function);
79bfeef6 616 assert(!(_jitc->function->self.call & jit_call_varargs));
4a71579b
PC		 617 return (jit_make_arg_f((jit_node_t*)0));
618}
619
620jit_node_t *
621_jit_arg_d(jit_state_t *_jit)
622{
623 assert(_jitc->function);
79bfeef6 624 assert(!(_jitc->function->self.call & jit_call_varargs));
4a71579b
PC		 625 return (jit_make_arg_d((jit_node_t*)0));
626}
627
628void
629_jit_getarg_c(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
630{
79bfeef6
PC		 631 jit_node_t *node = NULL;
632 assert_arg_type(v->code, jit_code_arg_c);
4a71579b
PC		 633 jit_inc_synth_wp(getarg_c, u, v);
634 if (jit_swf_p())
79bfeef6 635 node = jit_ldxi_c(u, JIT_FP, arg_offset(v->u.w));
4a71579b
PC		 636 else if (jit_arg_reg_p(v->u.w))
637 jit_extr_c(u, JIT_RA0 - v->u.w);
638 else
79bfeef6
PC		 639 node = jit_ldxi_c(u, JIT_FP, v->u.w);
640 if (node) {
641 CHECK_REG_ARGS();
642 jit_link_alist(node);
643 jit_check_frame();
644 }
4a71579b
PC		 645 jit_dec_synth();
646}
647
648void
649_jit_getarg_uc(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
650{
79bfeef6
PC		 651 jit_node_t *node = NULL;
652 assert_arg_type(v->code, jit_code_arg_c);
4a71579b
PC		 653 jit_inc_synth_wp(getarg_uc, u, v);
654 if (jit_swf_p())
79bfeef6 655 node = jit_ldxi_uc(u, JIT_FP, arg_offset(v->u.w));
4a71579b
PC		 656 else if (jit_arg_reg_p(v->u.w))
657 jit_extr_uc(u, JIT_RA0 - v->u.w);
658 else
79bfeef6
PC		 659 node = jit_ldxi_uc(u, JIT_FP, v->u.w);
660 if (node) {
661 CHECK_REG_ARGS();
662 jit_link_alist(node);
663 jit_check_frame();
664 }
4a71579b
PC		 665 jit_dec_synth();
666}
667
668void
669_jit_getarg_s(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
670{
79bfeef6
PC		 671 jit_node_t *node = NULL;
672 assert_arg_type(v->code, jit_code_arg_s);
4a71579b
PC		 673 jit_inc_synth_wp(getarg_s, u, v);
674 if (jit_swf_p())
79bfeef6 675 node = jit_ldxi_s(u, JIT_FP, arg_offset(v->u.w));
4a71579b
PC		 676 else if (jit_arg_reg_p(v->u.w))
677 jit_extr_s(u, JIT_RA0 - v->u.w);
678 else
79bfeef6
PC		 679 node = jit_ldxi_s(u, JIT_FP, v->u.w);
680 if (node) {
681 CHECK_REG_ARGS();
682 jit_link_alist(node);
683 jit_check_frame();
684 }
4a71579b
PC		 685 jit_dec_synth();
686}
687
688void
689_jit_getarg_us(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
690{
79bfeef6
PC		 691 jit_node_t *node = NULL;
692 assert_arg_type(v->code, jit_code_arg_s);
4a71579b
PC		 693 jit_inc_synth_wp(getarg_us, u, v);
694 if (jit_swf_p())
79bfeef6 695 node = jit_ldxi_us(u, JIT_FP, arg_offset(v->u.w));
4a71579b
PC		 696 else if (jit_arg_reg_p(v->u.w))
697 jit_extr_us(u, JIT_RA0 - v->u.w);
698 else
79bfeef6
PC		 699 node = jit_ldxi_us(u, JIT_FP, v->u.w);
700 if (node) {
701 CHECK_REG_ARGS();
702 jit_link_alist(node);
703 jit_check_frame();
704 }
4a71579b
PC		 705 jit_dec_synth();
706}
707
708void
709_jit_getarg_i(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
710{
79bfeef6
PC		 711 jit_node_t *node = NULL;
712 assert_arg_type(v->code, jit_code_arg_i);
4a71579b
PC		 713 jit_inc_synth_wp(getarg_i, u, v);
714 if (jit_swf_p())
79bfeef6 715 node = jit_ldxi_i(u, JIT_FP, arg_offset(v->u.w));
4a71579b
PC		 716 else if (jit_arg_reg_p(v->u.w))
717 jit_movr(u, JIT_RA0 - v->u.w);
718 else
79bfeef6
PC		 719 node = jit_ldxi_i(u, JIT_FP, v->u.w);
720 if (node) {
721 CHECK_REG_ARGS();
722 jit_link_alist(node);
723 jit_check_frame();
724 }
4a71579b
PC		 725 jit_dec_synth();
726}
727
728void
79bfeef6 729_jit_putargr(jit_state_t *_jit, jit_int32_t u, jit_node_t *v, jit_code_t code)
4a71579b 730{
79bfeef6
PC		 731 jit_node_t *node = NULL;
732 assert_putarg_type(code, v->code);
733 jit_code_inc_synth_wp(code, u, v);
4a71579b 734 if (jit_swf_p())
79bfeef6 735 node = jit_stxi(arg_offset(v->u.w), JIT_FP, u);
4a71579b
PC		 736 else if (jit_arg_reg_p(v->u.w))
737 jit_movr(JIT_RA0 - v->u.w, u);
738 else
79bfeef6
PC		 739 node = jit_stxi(v->u.w, JIT_FP, u);
740 if (node) {
741 CHECK_REG_ARGS();
742 jit_link_alist(node);
743 jit_check_frame();
744 }
4a71579b
PC		 745 jit_dec_synth();
746}
747
748void
79bfeef6 749_jit_putargi(jit_state_t *_jit, jit_word_t u, jit_node_t *v, jit_code_t code)
4a71579b 750{
79bfeef6
PC		 751 jit_int32_t regno;
752 jit_node_t *node = NULL;
753 assert_putarg_type(code, v->code);
754 jit_code_inc_synth_wp(code, u, v);
4a71579b
PC		 755 if (jit_swf_p()) {
756 regno = jit_get_reg(jit_class_gpr);
757 jit_movi(regno, u);
79bfeef6 758 node = jit_stxi(arg_offset(v->u.w), JIT_FP, regno);
4a71579b
PC		 759 jit_unget_reg(regno);
760 }
761 else if (jit_arg_reg_p(v->u.w))
762 jit_movi(JIT_RA0 - v->u.w, u);
763 else {
764 regno = jit_get_reg(jit_class_gpr);
765 jit_movi(regno, u);
79bfeef6 766 node = jit_stxi(v->u.w, JIT_FP, regno);
4a71579b
PC		 767 jit_unget_reg(regno);
768 }
79bfeef6
PC		 769 if (node) {
770 CHECK_REG_ARGS();
771 jit_link_alist(node);
772 jit_check_frame();
773 }
4a71579b
PC		 774 jit_dec_synth();
775}
776
777void
778_jit_getarg_f(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
779{
79bfeef6 780 jit_node_t *node = NULL;
4a71579b
PC		 781 assert(v->code == jit_code_arg_f);
782 jit_inc_synth_wp(getarg_f, u, v);
783 if (jit_cpu.abi && !(_jitc->function->self.call & jit_call_varargs)) {
784 if (jit_arg_f_reg_p(v->u.w))
785 jit_movr_f(u, JIT_FA0 - v->u.w);
786 else
79bfeef6 787 node = jit_ldxi_f(u, JIT_FP, v->u.w);
4a71579b
PC		 788 }
789 else if (jit_swf_p())
79bfeef6 790 node = jit_ldxi_f(u, JIT_FP, arg_offset(v->u.w));
4a71579b
PC		 791 else {
792 if (jit_arg_reg_p(v->u.w))
793 jit_movr_w_f(u, JIT_RA0 - v->u.w);
794 else
79bfeef6
PC		 795 node = jit_ldxi_f(u, JIT_FP, v->u.w);
796 }
797 if (node) {
798 CHECK_REG_ARGS();
799 jit_link_alist(node);
800 jit_check_frame();
4a71579b
PC		 801 }
802 jit_dec_synth();
803}
804
805void
806_jit_putargr_f(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
807{
79bfeef6 808 jit_node_t *node = NULL;
4a71579b
PC		 809 assert(v->code == jit_code_arg_f);
810 jit_inc_synth_wp(putargr_f, u, v);
811 if (jit_cpu.abi) {
812 if (jit_arg_f_reg_p(v->u.w))
813 jit_movr_f(JIT_FA0 - v->u.w, u);
814 else
79bfeef6 815 node = jit_stxi_f(v->u.w, JIT_FP, u);
4a71579b
PC		 816 }
817 else if (jit_swf_p())
79bfeef6 818 node = jit_stxi_f(arg_offset(v->u.w), JIT_FP, u);
4a71579b
PC		 819 else {
820 if (jit_arg_reg_p(v->u.w))
821 jit_movr_f_w(JIT_RA0 - v->u.w, u);
822 else
79bfeef6
PC		 823 node = jit_stxi_f(v->u.w, JIT_FP, u);
824 }
825 if (node) {
826 CHECK_REG_ARGS();
827 jit_link_alist(node);
828 jit_check_frame();
4a71579b
PC		 829 }
830 jit_dec_synth();
831}
832
833void
834_jit_putargi_f(jit_state_t *_jit, jit_float32_t u, jit_node_t *v)
835{
79bfeef6
PC		 836 jit_int32_t regno;
837 jit_node_t *node = NULL;
4a71579b
PC		 838 assert(v->code == jit_code_arg_f);
839 jit_inc_synth_fp(putargi_f, u, v);
840 if (jit_cpu.abi) {
841 if (jit_arg_f_reg_p(v->u.w))
842 jit_movi_f(JIT_FA0 - v->u.w, u);
843 else {
844 regno = jit_get_reg(jit_class_fpr);
845 jit_movi_f(regno, u);
79bfeef6 846 node = jit_stxi_f(v->u.w, JIT_FP, regno);
4a71579b
PC		 847 jit_unget_reg(regno);
848 }
849 }
850 else if (jit_swf_p()) {
851 regno = jit_get_reg(jit_class_fpr);
852 jit_movi_f(regno, u);
79bfeef6 853 node = jit_stxi_f(arg_offset(v->u.w), JIT_FP, regno);
4a71579b
PC		 854 jit_unget_reg(regno);
855 }
856 else {
857 regno = jit_get_reg(jit_class_fpr);
858 jit_movi_f(regno, u);
859 if (jit_arg_reg_p(v->u.w))
860 jit_movr_f_w(JIT_RA0 - v->u.w, regno);
861 else
79bfeef6 862 node = jit_stxi_f(v->u.w, JIT_FP, regno);
4a71579b
PC		 863 jit_unget_reg(regno);
864 }
79bfeef6
PC		 865 if (node) {
866 CHECK_REG_ARGS();
867 jit_link_alist(node);
868 jit_check_frame();
869 }
4a71579b
PC		 870 jit_dec_synth();
871}
872
873void
874_jit_getarg_d(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
875{
79bfeef6 876 jit_node_t *node = NULL;
4a71579b
PC		 877 assert(v->code == jit_code_arg_d);
878 jit_inc_synth_wp(getarg_d, u, v);
879 if (jit_cpu.abi && !(_jitc->function->self.call & jit_call_varargs)) {
880 if (jit_arg_f_reg_p(v->u.w))
881 jit_movr_d(u, JIT_FA0 - v->u.w);
882 else
79bfeef6 883 node = jit_ldxi_d(u, JIT_FP, v->u.w);
4a71579b
PC		 884 }
885 else if (jit_swf_p())
79bfeef6 886 node = jit_ldxi_d(u, JIT_FP, arg_offset(v->u.w));
4a71579b
PC		 887 else {
888 if (jit_arg_reg_p(v->u.w))
889 jit_movr_ww_d(u, JIT_RA0 - v->u.w, JIT_RA0 - (v->u.w + 1));
890 else
79bfeef6
PC		 891 node = jit_ldxi_d(u, JIT_FP, v->u.w);
892 }
893 if (node) {
894 CHECK_REG_ARGS();
895 jit_link_alist(node);
896 jit_check_frame();
4a71579b
PC		 897 }
898 jit_dec_synth();
899}
900
901void
902_jit_putargr_d(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
903{
79bfeef6 904 jit_node_t *node = NULL;
4a71579b
PC		 905 assert(v->code == jit_code_arg_d);
906 jit_inc_synth_wp(putargr_d, u, v);
907 if (jit_cpu.abi) {
908 if (jit_arg_f_reg_p(v->u.w))
909 jit_movr_d(JIT_FA0 - v->u.w, u);
910 else
79bfeef6 911 node = jit_stxi_d(v->u.w, JIT_FP, u);
4a71579b
PC		 912 }
913 else if (jit_swf_p())
79bfeef6 914 node = jit_stxi_d(arg_offset(v->u.w), JIT_FP, u);
4a71579b
PC		 915 else {
916 if (jit_arg_reg_p(v->u.w))
917 jit_movr_d_ww(JIT_RA0 - v->u.w, JIT_RA0 - (v->u.w + 1), u);
918 else
79bfeef6
PC		 919 node = jit_stxi_d(v->u.w, JIT_FP, u);
920 }
921 if (node) {
922 CHECK_REG_ARGS();
923 jit_link_alist(node);
924 jit_check_frame();
4a71579b
PC		 925 }
926 jit_dec_synth();
927}
928
929void
930_jit_putargi_d(jit_state_t *_jit, jit_float64_t u, jit_node_t *v)
931{
79bfeef6
PC		 932 jit_int32_t regno;
933 jit_node_t *node = NULL;
4a71579b
PC		 934 assert(v->code == jit_code_arg_d);
935 jit_inc_synth_dp(putargi_d, u, v);
936 if (jit_cpu.abi) {
937 if (jit_arg_f_reg_p(v->u.w))
938 jit_movi_d(JIT_FA0 - v->u.w, u);
939 else {
940 regno = jit_get_reg(jit_class_fpr);
941 jit_movi_d(regno, u);
79bfeef6 942 node = jit_stxi_d(v->u.w, JIT_FP, regno);
4a71579b
PC		 943 jit_unget_reg(regno);
944 }
945 }
946 else if (jit_swf_p()) {
947 regno = jit_get_reg(jit_class_fpr);
948 jit_movi_d(regno, u);
79bfeef6 949 node = jit_stxi_d(arg_offset(v->u.w), JIT_FP, regno);
4a71579b
PC		 950 jit_unget_reg(regno);
951 }
952 else {
953 regno = jit_get_reg(jit_class_fpr);
954 jit_movi_d(regno, u);
955 if (jit_arg_reg_p(v->u.w))
956 jit_movr_d_ww(JIT_RA0 - v->u.w, JIT_RA0 - (v->u.w + 1), regno);
957 else
79bfeef6 958 node = jit_stxi_d(v->u.w, JIT_FP, regno);
4a71579b
PC		 959 jit_unget_reg(regno);
960 }
79bfeef6
PC		 961 if (node) {
962 CHECK_REG_ARGS();
963 jit_link_alist(node);
964 jit_check_frame();
965 }
4a71579b
PC		 966 jit_dec_synth();
967}
968
969void
79bfeef6 970_jit_pushargr(jit_state_t *_jit, jit_int32_t u, jit_code_t code)
4a71579b
PC		 971{
972 assert(_jitc->function);
79bfeef6 973 jit_code_inc_synth_w(code, u);
4a71579b
PC		 974 jit_link_prepare();
975 if (jit_arg_reg_p(_jitc->function->call.argi)) {
976 jit_movr(JIT_RA0 - _jitc->function->call.argi, u);
977 ++_jitc->function->call.argi;
978 }
979 else {
980 jit_stxi(_jitc->function->call.size, JIT_SP, u);
981 _jitc->function->call.size += sizeof(jit_word_t);
982 }
983 jit_dec_synth();
984}
985
986void
79bfeef6 987_jit_pushargi(jit_state_t *_jit, jit_word_t u, jit_code_t code)
4a71579b
PC		 988{
989 jit_int32_t regno;
990 assert(_jitc->function);
79bfeef6 991 jit_code_inc_synth_w(code, u);
4a71579b
PC		 992 jit_link_prepare();
993 if (jit_arg_reg_p(_jitc->function->call.argi)) {
994 jit_movi(JIT_RA0 - _jitc->function->call.argi, u);
995 ++_jitc->function->call.argi;
996 }
997 else {
998 regno = jit_get_reg(jit_class_gpr);
999 jit_movi(regno, u);
1000 jit_stxi(_jitc->function->call.size, JIT_SP, regno);
1001 jit_unget_reg(regno);
1002 _jitc->function->call.size += sizeof(jit_word_t);
1003 }
1004 jit_dec_synth();
1005}
1006
1007void
1008_jit_pushargr_f(jit_state_t *_jit, jit_int32_t u)
1009{
1010 assert(_jitc->function);
1011 jit_inc_synth_w(pushargr_f, u);
1012 jit_link_prepare();
1013 if (jit_cpu.abi && !(_jitc->function->call.call & jit_call_varargs)) {
1014 if (jit_arg_f_reg_p(_jitc->function->call.argf)) {
1015 jit_movr_f(JIT_FA0 - _jitc->function->call.argf, u);
1016 ++_jitc->function->call.argf;
1017 goto done;
1018 }
1019 }
1020 else {
1021 if (jit_arg_reg_p(_jitc->function->call.argi)) {
1022 jit_movr_f_w(JIT_RA0 - _jitc->function->call.argi, u);
1023 ++_jitc->function->call.argi;
1024 goto done;
1025 }
1026 }
1027 jit_stxi_f(_jitc->function->call.size, JIT_SP, u);
1028 _jitc->function->call.size += sizeof(jit_word_t);
1029done:
1030 jit_dec_synth();
1031}
1032
1033void
1034_jit_pushargi_f(jit_state_t *_jit, jit_float32_t u)
1035{
1036 jit_int32_t regno;
1037 assert(_jitc->function);
1038 jit_inc_synth_f(pushargi_f, u);
1039 jit_link_prepare();
1040 if (jit_cpu.abi && !(_jitc->function->call.call & jit_call_varargs)) {
1041 if (jit_arg_f_reg_p(_jitc->function->call.argf)) {
1042 /* cannot jit_movi_f in the argument register because
1043 * float arguments are packed, and that would cause
1044 * either an assertion in debug mode, or overwritting
1045 * two registers */
1046 regno = jit_get_reg(jit_class_fpr);
1047 jit_movi_f(regno, u);
1048 jit_movr_f(JIT_FA0 - _jitc->function->call.argf, regno);
1049 jit_unget_reg(regno);
1050 ++_jitc->function->call.argf;
1051 goto done;
1052 }
1053 }
1054 else {
1055 if (jit_arg_reg_p(_jitc->function->call.argi)) {
1056 jit_movi_f_w(JIT_RA0 - _jitc->function->call.argi, u);
1057 ++_jitc->function->call.argi;
1058 goto done;
1059 }
1060 }
1061 regno = jit_get_reg(jit_class_fpr);
1062 jit_movi_f(regno, u);
1063 jit_stxi_f(_jitc->function->call.size, JIT_SP, regno);
1064 jit_unget_reg(regno);
1065 _jitc->function->call.size += sizeof(jit_word_t);
1066done:
1067 jit_dec_synth();
1068}
1069
1070void
1071_jit_pushargr_d(jit_state_t *_jit, jit_int32_t u)
1072{
1073 assert(_jitc->function);
1074 jit_inc_synth_w(pushargr_d, u);
1075 jit_link_prepare();
1076 if (jit_cpu.abi && !(_jitc->function->call.call & jit_call_varargs)) {
1077 if (jit_arg_d_reg_p(_jitc->function->call.argf)) {
1078 if (_jitc->function->call.argf & 1)
1079 ++_jitc->function->call.argf;
1080 jit_movr_d(JIT_FA0 - _jitc->function->call.argf, u);
1081 _jitc->function->call.argf += 2;
1082 goto done;
1083 }
1084 }
1085 else {
1086 if (_jitc->function->call.argi & 1)
1087 ++_jitc->function->call.argi;
1088 if (jit_arg_reg_p(_jitc->function->call.argi)) {
1089 jit_movr_d_ww(JIT_RA0 - _jitc->function->call.argi,
1090 JIT_RA0 - (_jitc->function->call.argi + 1),
1091 u);
1092 _jitc->function->call.argi += 2;
1093 goto done;
1094 }
1095 }
1096 if (_jitc->function->call.size & 7)
1097 _jitc->function->call.size += 4;
1098 jit_stxi_d(_jitc->function->call.size, JIT_SP, u);
1099 _jitc->function->call.size += sizeof(jit_float64_t);
1100done:
1101 jit_dec_synth();
1102}
1103
1104void
1105_jit_pushargi_d(jit_state_t *_jit, jit_float64_t u)
1106{
1107 jit_int32_t regno;
1108 assert(_jitc->function);
1109 jit_inc_synth_d(pushargi_d, u);
1110 jit_link_prepare();
1111 if (jit_cpu.abi && !(_jitc->function->call.call & jit_call_varargs)) {
1112 if (jit_arg_d_reg_p(_jitc->function->call.argf)) {
1113 if (_jitc->function->call.argf & 1)
1114 ++_jitc->function->call.argf;
1115 jit_movi_d(JIT_FA0 - _jitc->function->call.argf, u);
1116 _jitc->function->call.argf += 2;
1117 goto done;
1118 }
1119 }
1120 else {
1121 if (_jitc->function->call.argi & 1)
1122 ++_jitc->function->call.argi;
1123 if (jit_arg_reg_p(_jitc->function->call.argi)) {
1124 jit_movi_d_ww(JIT_RA0 - _jitc->function->call.argi,
1125 JIT_RA0 - (_jitc->function->call.argi + 1),
1126 u);
1127 _jitc->function->call.argi += 2;
1128 goto done;
1129 }
1130 }
1131 if (_jitc->function->call.size & 7)
1132 _jitc->function->call.size += 4;
1133 regno = jit_get_reg(jit_class_fpr);
1134 jit_movi_d(regno, u);
1135 jit_stxi_d(_jitc->function->call.size, JIT_SP, regno);
1136 jit_unget_reg(regno);
1137 _jitc->function->call.size += sizeof(jit_float64_t);
1138done:
1139 jit_dec_synth();
1140}
1141
1142jit_bool_t
1143_jit_regarg_p(jit_state_t *_jit, jit_node_t *node, jit_int32_t regno)
1144{
1145 jit_int32_t spec;
1146 spec = jit_class(_rvs[regno].spec);
1147 if (spec & jit_class_arg) {
1148 regno = JIT_RA0 - regno;
1149 if (regno >= 0 && regno < node->v.w)
1150 return (1);
1151 if (jit_cpu.abi && spec & jit_class_fpr) {
1152 regno = JIT_FA0 - regno;
1153 if (regno >= 0 && regno < node->w.w)
1154 return (1);
1155 }
1156 }
1157
1158 return (0);
1159}
1160
1161void
1162_jit_finishr(jit_state_t *_jit, jit_int32_t r0)
1163{
1164 jit_node_t *node;
1165 assert(_jitc->function);
1166 jit_inc_synth_w(finishr, r0);
1167 if (_jitc->function->self.alen < _jitc->function->call.size)
1168 _jitc->function->self.alen = _jitc->function->call.size;
1169 node = jit_callr(r0);
1170 node->v.w = _jitc->function->self.argi;
1171 node->w.w = _jitc->function->call.argf;
1172 _jitc->function->call.argi = _jitc->function->call.argf =
1173 _jitc->function->call.size = 0;
1174 _jitc->prepare = 0;
1175 jit_dec_synth();
1176}
1177
1178jit_node_t *
1179_jit_finishi(jit_state_t *_jit, jit_pointer_t i0)
1180{
1181 jit_node_t *node;
1182 assert(_jitc->function);
1183 jit_inc_synth_w(finishi, (jit_word_t)i0);
1184 if (_jitc->function->self.alen < _jitc->function->call.size)
1185 _jitc->function->self.alen = _jitc->function->call.size;
1186 node = jit_calli(i0);
1187 node->v.w = _jitc->function->call.argi;
1188 node->w.w = _jitc->function->call.argf;
1189 _jitc->function->call.argi = _jitc->function->call.argf =
1190 _jitc->function->call.size = 0;
1191 _jitc->prepare = 0;
1192 jit_dec_synth();
1193 return (node);
1194}
1195
1196void
1197_jit_retval_c(jit_state_t *_jit, jit_int32_t r0)
1198{
1199 jit_inc_synth_w(retval_c, r0);
1200 jit_extr_c(r0, JIT_RET);
1201 jit_dec_synth();
1202}
1203
1204void
1205_jit_retval_uc(jit_state_t *_jit, jit_int32_t r0)
1206{
1207 jit_inc_synth_w(retval_uc, r0);
1208 jit_extr_uc(r0, JIT_RET);
1209 jit_dec_synth();
1210}
1211
1212void
1213_jit_retval_s(jit_state_t *_jit, jit_int32_t r0)
1214{
1215 jit_inc_synth_w(retval_s, r0);
1216 jit_extr_s(r0, JIT_RET);
1217 jit_dec_synth();
1218}
1219
1220void
1221_jit_retval_us(jit_state_t *_jit, jit_int32_t r0)
1222{
1223 jit_inc_synth_w(retval_us, r0);
1224 jit_extr_us(r0, JIT_RET);
1225 jit_dec_synth();
1226}
1227
1228void
1229_jit_retval_i(jit_state_t *_jit, jit_int32_t r0)
1230{
1231 jit_inc_synth_w(retval_i, r0);
1232 if (r0 != JIT_RET)
1233 jit_movr(r0, JIT_RET);
1234 jit_dec_synth();
1235}
1236
1237void
1238_jit_retval_f(jit_state_t *_jit, jit_int32_t r0)
1239{
1240 jit_inc_synth_w(retval_f, r0);
1241 if (jit_cpu.abi) {
1242 if (r0 != JIT_FRET)
1243 jit_movr_f(r0, JIT_FRET);
1244 }
1245 else if (r0 != JIT_RET)
1246 jit_movr_w_f(r0, JIT_RET);
1247 jit_dec_synth();
1248}
1249
1250void
1251_jit_retval_d(jit_state_t *_jit, jit_int32_t r0)
1252{
1253 jit_inc_synth_w(retval_d, r0);
1254 if (jit_cpu.abi) {
1255 if (r0 != JIT_FRET)
1256 jit_movr_d(r0, JIT_FRET);
1257 }
1258 else if (r0 != JIT_RET)
1259 jit_movr_ww_d(r0, JIT_RET, _R1);
1260 jit_dec_synth();
1261}
1262
1263jit_pointer_t
1264_emit_code(jit_state_t *_jit)
1265{
1266 jit_node_t *node;
1267 jit_node_t *temp;
1268 jit_word_t word;
1269 jit_int32_t value;
1270 jit_int32_t offset;
1271 struct {
1272 jit_node_t *node;
1273 jit_uint8_t *data;
1274 jit_word_t word;
79bfeef6 1275 jit_function_t func;
4a71579b
PC		 1276#if DEVEL_DISASSEMBLER
1277 jit_word_t prevw;
1278#endif
1279 jit_uword_t thumb;
1280#if DISASSEMBLER
1281 jit_int32_t info_offset;
1282#endif
1283 jit_int32_t const_offset;
1284 jit_int32_t patch_offset;
1285 } undo;
1286#if DEVEL_DISASSEMBLER
1287 jit_word_t prevw;
1288#endif
1289
1290 _jitc->function = NULL;
1291 _jitc->thumb = 0;
1292
1293 jit_reglive_setup();
1294
1295 _jitc->consts.data = NULL;
1296 _jitc->consts.offset = _jitc->consts.length = 0;
1297
1298 undo.word = 0;
1299 undo.node = NULL;
1300 undo.data = NULL;
1301 undo.thumb = 0;
1302#if DISASSEMBLER
1303 undo.info_offset =
1304#endif
1305 undo.const_offset = undo.patch_offset = 0;
1306# define assert_data(node) /**/
1307#define case_rr(name, type) \
1308 case jit_code_##name##r##type: \
1309 name##r##type(rn(node->u.w), rn(node->v.w)); \
1310 break
1311#define case_rw(name, type) \
1312 case jit_code_##name##i##type: \
1313 name##i##type(rn(node->u.w), node->v.w); \
1314 break
1315#define case_vv(name, type) \
1316 case jit_code_##name##r##type: \
1317 if (jit_swf_p()) \
1318 swf_##name##r##type(rn(node->u.w), rn(node->v.w)); \
1319 else \
1320 vfp_##name##r##type(rn(node->u.w), rn(node->v.w)); \
1321 break
1322#define case_vw(name, type) \
1323 case jit_code_##name##i##type: \
1324 if (jit_swf_p()) \
1325 swf_##name##i##type(rn(node->u.w), node->v.w); \
1326 else \
1327 vfp_##name##i##type(rn(node->u.w), node->v.w); \
1328 break
1329#define case_wr(name, type) \
1330 case jit_code_##name##i##type: \
1331 name##i##type(node->u.w, rn(node->v.w)); \
1332 break
1333#define case_wv(name, type) \
1334 case jit_code_##name##i##type: \
1335 if (jit_swf_p()) \
1336 swf_##name##i##type(node->u.w, rn(node->v.w)); \
1337 else \
1338 vfp_##name##i##type(node->u.w, rn(node->v.w)); \
1339 break
1340#define case_rrr(name, type) \
1341 case jit_code_##name##r##type: \
1342 name##r##type(rn(node->u.w), \
1343 rn(node->v.w), rn(node->w.w)); \
1344 break
ba86ff93
PC		 1345#define case_rqr(name, type) \
1346 case jit_code_##name##r##type: \
1347 if (jit_swf_p()) \
1348 swf_##name##r##type(rn(node->u.w), rn(node->v.q.l), \
1349 rn(node->v.q.h), rn(node->w.w));\
1350 else \
1351 vfp_##name##r##type(rn(node->u.w), rn(node->v.q.l), \
1352 rn(node->v.q.h), rn(node->w.w));\
1353 case jit_code_##name##i##type: \
1354 break
4a71579b
PC		 1355#define case_rrrr(name, type) \
1356 case jit_code_##name##r##type: \
1357 name##r##type(rn(node->u.q.l), rn(node->u.q.h), \
1358 rn(node->v.w), rn(node->w.w)); \
1359 break
1360#define case_vvv(name, type) \
1361 case jit_code_##name##r##type: \
1362 if (jit_swf_p()) \
1363 swf_##name##r##type(rn(node->u.w), \
1364 rn(node->v.w), rn(node->w.w)); \
1365 else \
1366 vfp_##name##r##type(rn(node->u.w), \
1367 rn(node->v.w), rn(node->w.w)); \
1368 break
1369#define case_rrw(name, type) \
1370 case jit_code_##name##i##type: \
1371 name##i##type(rn(node->u.w), rn(node->v.w), node->w.w); \
1372 break
1373#define case_rrrw(name, type) \
1374 case jit_code_##name##i##type: \
1375 name##i##type(rn(node->u.q.l), rn(node->u.q.h), \
1376 rn(node->v.w), node->w.w); \
1377 break
1378#define case_vvw(name, type) \
1379 case jit_code_##name##i##type: \
1380 if (jit_swf_p()) \
1381 swf_##name##i##type(rn(node->u.w), \
1382 rn(node->v.w), node->w.w); \
1383 else \
1384 vfp_##name##i##type(rn(node->u.w), \
1385 rn(node->v.w), node->w.w); \
1386 break
1387#define case_vvf(name) \
1388 case jit_code_##name##i_f: \
1389 assert_data(node); \
1390 if (jit_swf_p()) \
1391 swf_##name##i_f(rn(node->u.w), rn(node->v.w), \
1392 node->w.f); \
1393 else \
1394 vfp_##name##i_f(rn(node->u.w), rn(node->v.w), \
1395 node->w.f); \
1396 break
1397#define case_vvd(name) \
1398 case jit_code_##name##i_d: \
1399 assert_data(node); \
1400 if (jit_swf_p()) \
1401 swf_##name##i_d(rn(node->u.w), rn(node->v.w), \
1402 node->w.d); \
1403 else \
1404 vfp_##name##i_d(rn(node->u.w), rn(node->v.w), \
1405 node->w.d); \
1406 break
1407#define case_wrr(name, type) \
1408 case jit_code_##name##i##type: \
1409 name##i##type(node->u.w, rn(node->v.w), rn(node->w.w)); \
1410 break
1411#define case_wvv(name, type) \
1412 case jit_code_##name##i##type: \
1413 if (jit_swf_p()) \
1414 swf_##name##i##type(node->u.w, \
1415 rn(node->v.w), rn(node->w.w)); \
1416 else \
1417 vfp_##name##i##type(node->u.w, \
1418 rn(node->v.w), rn(node->w.w)); \
1419 break
1420#define case_brr(name, type) \
1421 case jit_code_##name##r##type: \
1422 temp = node->u.n; \
1423 assert(temp->code == jit_code_label || \
1424 temp->code == jit_code_epilog); \
1425 if (temp->flag & jit_flag_patch) \
1426 name##r##type(temp->u.w, rn(node->v.w), \
1427 rn(node->w.w)); \
1428 else { \
1429 word = name##r##type(_jit->pc.w, \
1430 rn(node->v.w), rn(node->w.w)); \
79bfeef6 1431 patch(word, node, arm_patch_jump); \
4a71579b
PC		 1432 } \
1433 break
1434#define case_bvv(name, type) \
1435 case jit_code_##name##r##type: \
1436 temp = node->u.n; \
1437 assert(temp->code == jit_code_label || \
1438 temp->code == jit_code_epilog); \
1439 if (temp->flag & jit_flag_patch) { \
1440 if (jit_swf_p()) \
1441 swf_##name##r##type(temp->u.w, rn(node->v.w), \
1442 rn(node->w.w)); \
1443 else \
1444 vfp_##name##r##type(temp->u.w, rn(node->v.w), \
1445 rn(node->w.w)); \
1446 } \
1447 else { \
1448 if (jit_swf_p()) \
1449 word = swf_##name##r##type(_jit->pc.w, \
1450 rn(node->v.w), \
1451 rn(node->w.w)); \
1452 else \
1453 word = vfp_##name##r##type(_jit->pc.w, \
1454 rn(node->v.w), \
1455 rn(node->w.w)); \
79bfeef6 1456 patch(word, node, arm_patch_jump); \
4a71579b
PC		 1457 } \
1458 break
1459#define case_brw(name, type) \
1460 case jit_code_##name##i##type: \
1461 temp = node->u.n; \
1462 assert(temp->code == jit_code_label || \
1463 temp->code == jit_code_epilog); \
1464 if (temp->flag & jit_flag_patch) \
1465 name##i##type(temp->u.w, \
1466 rn(node->v.w), node->w.w); \
1467 else { \
1468 word = name##i##type(_jit->pc.w, \
1469 rn(node->v.w), node->w.w); \
79bfeef6 1470 patch(word, node, arm_patch_jump); \
4a71579b
PC		 1471 } \
1472 break;
1473#define case_bvf(name) \
1474 case jit_code_##name##i_f: \
1475 temp = node->u.n; \
1476 assert(temp->code == jit_code_label || \
1477 temp->code == jit_code_epilog); \
1478 if (temp->flag & jit_flag_patch) { \
1479 if (jit_swf_p()) \
1480 swf_##name##i_f(temp->u.w, rn(node->v.w), \
1481 node->w.f); \
1482 else \
1483 vfp_##name##i_f(temp->u.w, rn(node->v.w), \
1484 node->w.f); \
1485 } \
1486 else { \
1487 if (jit_swf_p()) \
1488 word = swf_##name##i_f(_jit->pc.w, \
1489 rn(node->v.w), \
1490 node->w.f); \
1491 else \
1492 word = vfp_##name##i_f(_jit->pc.w, \
1493 rn(node->v.w), \
1494 node->w.f); \
79bfeef6 1495 patch(word, node, arm_patch_jump); \
4a71579b
PC		 1496 } \
1497 break
1498#define case_bvd(name) \
1499 case jit_code_##name##i_d: \
1500 temp = node->u.n; \
1501 assert(temp->code == jit_code_label || \
1502 temp->code == jit_code_epilog); \
1503 if (temp->flag & jit_flag_patch) { \
1504 if (jit_swf_p()) \
1505 swf_##name##i_d(temp->u.w, rn(node->v.w), \
1506 node->w.d); \
1507 else \
1508 vfp_##name##i_d(temp->u.w, rn(node->v.w), \
1509 node->w.d); \
1510 } \
1511 else { \
1512 if (jit_swf_p()) \
1513 word = swf_##name##i_d(_jit->pc.w, \
1514 rn(node->v.w), \
1515 node->w.d); \
1516 else \
1517 word = vfp_##name##i_d(_jit->pc.w, \
1518 rn(node->v.w), \
1519 node->w.d); \
79bfeef6 1520 patch(word, node, arm_patch_jump); \
4a71579b
PC		 1521 } \
1522 break
1523#if DEVEL_DISASSEMBLER
1524 prevw = _jit->pc.w;
1525#endif
1526 for (node = _jitc->head; node; node = node->next) {
1527 if (_jit->pc.uc >= _jitc->code.end)
1528 return (NULL);
1529
1530#if DEVEL_DISASSEMBLER
1531 node->offset = (jit_uword_t)_jit->pc.w - (jit_uword_t)prevw;
1532 prevw = _jit->pc.w;
1533#endif
1534 value = jit_classify(node->code);
1535 jit_regarg_set(node, value);
1536 switch (node->code) {
1537 case jit_code_align:
c0c16242
PC		 1538 /* Must align to a power of two */
1539 assert(!(node->u.w & (node->u.w - 1)));
1540 if ((word = _jit->pc.w & (node->u.w - 1)))
1541 nop(node->u.w - word);
4a71579b 1542 break;
79bfeef6
PC		 1543 case jit_code_skip:
1544 if (jit_thumb_p())
1545 nop((node->u.w + 1) & ~1);
1546 else
1547 nop((node->u.w + 3) & ~3);
1548 break;
4a71579b
PC		 1549 case jit_code_note: case jit_code_name:
1550 if (must_align_p(node->next))
1551 nop(2);
1552 node->u.w = _jit->pc.w;
1553 break;
1554 case jit_code_label:
1555 if (must_align_p(node->next))
1556 nop(2);
1557 /* remember label is defined */
1558 node->flag |= jit_flag_patch;
1559 node->u.w = _jit->pc.w;
1560 break;
1561 case_rrr(add,);
1562 case_rrw(add,);
1563 case_rrr(addc,);
1564 case_rrw(addc,);
1565 case_rrr(addx,);
1566 case_rrw(addx,);
1567 case_rrr(sub,);
1568 case_rrw(sub,);
1569 case_rrr(subc,);
1570 case_rrw(subc,);
1571 case_rrr(subx,);
1572 case_rrw(subx,);
1573 case_rrw(rsb,);
1574 case_rrr(mul,);
1575 case_rrw(mul,);
ba86ff93
PC		 1576 case_rrr(hmul,);
1577 case_rrw(hmul,);
1578 case_rrr(hmul, _u);
1579 case_rrw(hmul, _u);
4a71579b
PC		 1580 case_rrrr(qmul,);
1581 case_rrrw(qmul,);
1582 case_rrrr(qmul, _u);
1583 case_rrrw(qmul, _u);
1584 case_rrr(div,);
1585 case_rrw(div,);
1586 case_rrr(div, _u);
1587 case_rrw(div, _u);
1588 case_rrrr(qdiv,);
1589 case_rrrw(qdiv,);
1590 case_rrrr(qdiv, _u);
1591 case_rrrw(qdiv, _u);
1592 case_rrr(rem,);
1593 case_rrw(rem,);
1594 case_rrr(rem, _u);
1595 case_rrw(rem, _u);
1596 case_rrr(lsh,);
1597 case_rrw(lsh,);
ba86ff93
PC		 1598#define qlshr(r0, r1, r2, r3) fallback_qlshr(r0, r1, r2, r3)
1599#define qlshi(r0, r1, r2, i0) fallback_qlshi(r0, r1, r2, i0)
1600#define qlshr_u(r0, r1, r2, r3) fallback_qlshr_u(r0, r1, r2, r3)
1601#define qlshi_u(r0, r1, r2, i0) fallback_qlshi_u(r0, r1, r2, i0)
1602 case_rrrr(qlsh,);
1603 case_rrrw(qlsh,);
1604 case_rrrr(qlsh, _u);
1605 case_rrrw(qlsh, _u);
4a71579b
PC		 1606 case_rrr(rsh,);
1607 case_rrw(rsh,);
1608 case_rrr(rsh, _u);
1609 case_rrw(rsh, _u);
ba86ff93
PC		 1610#define qrshr(r0, r1, r2, r3) fallback_qrshr(r0, r1, r2, r3)
1611#define qrshi(r0, r1, r2, i0) fallback_qrshi(r0, r1, r2, i0)
1612#define qrshr_u(r0, r1, r2, r3) fallback_qrshr_u(r0, r1, r2, r3)
1613#define qrshi_u(r0, r1, r2, i0) fallback_qrshi_u(r0, r1, r2, i0)
1614 case_rrrr(qrsh,);
1615 case_rrrw(qrsh,);
1616 case_rrrr(qrsh, _u);
1617 case_rrrw(qrsh, _u);
1618 case_rrr(lrot,);
1619 case_rrw(lrot,);
1620 case_rrr(rrot,);
1621 case_rrw(rrot,);
4a71579b
PC		 1622 case_rr(neg,);
1623 case_rr(com,);
79bfeef6
PC		 1624 case_rr(clo,);
1625 case_rr(clz,);
1626 case_rr(cto,);
1627 case_rr(ctz,);
ba86ff93
PC		 1628 case_rr(rbit,);
1629 case_rr(popcnt,);
4a71579b
PC		 1630 case_rrr(and,);
1631 case_rrw(and,);
1632 case_rrr(or,);
1633 case_rrw(or,);
1634 case_rrr(xor,);
1635 case_rrw(xor,);
1636 case_vv(trunc, _f_i);
1637 case_vv(trunc, _d_i);
1638 case_rr(ld, _c);
1639 case_rw(ld, _c);
1640 case_rr(ld, _uc);
1641 case_rw(ld, _uc);
1642 case_rr(ld, _s);
1643 case_rw(ld, _s);
1644 case_rr(ld, _us);
1645 case_rw(ld, _us);
1646 case_rr(ld, _i);
1647 case_rw(ld, _i);
1648 case_rrr(ldx, _c);
1649 case_rrw(ldx, _c);
1650 case_rrr(ldx, _uc);
1651 case_rrw(ldx, _uc);
1652 case_rrr(ldx, _s);
1653 case_rrw(ldx, _s);
1654 case_rrr(ldx, _us);
1655 case_rrw(ldx, _us);
1656 case_rrr(ldx, _i);
1657 case_rrw(ldx, _i);
ba86ff93
PC		 1658 case jit_code_unldr:
1659 unldr(rn(node->u.w), rn(node->v.w), node->w.w);
1660 break;
1661 case jit_code_unldi:
1662 unldi(rn(node->u.w), node->v.w, node->w.w);
1663 break;
1664 case jit_code_unldr_u:
1665 unldr_u(rn(node->u.w), rn(node->v.w), node->w.w);
1666 break;
1667 case jit_code_unldi_u:
1668 unldi_u(rn(node->u.w), node->v.w, node->w.w);
1669 break;
4a71579b
PC		 1670 case_rr(st, _c);
1671 case_wr(st, _c);
1672 case_rr(st, _s);
1673 case_wr(st, _s);
1674 case_rr(st, _i);
1675 case_wr(st, _i);
1676 case_rrr(stx, _c);
1677 case_wrr(stx, _c);
1678 case_rrr(stx, _s);
1679 case_wrr(stx, _s);
1680 case_rrr(stx, _i);
1681 case_wrr(stx, _i);
ba86ff93
PC		 1682 case jit_code_unstr:
1683 unstr(rn(node->u.w), rn(node->v.w), node->w.w);
1684 break;
1685 case jit_code_unsti:
1686 unsti(node->u.w, rn(node->v.w), node->w.w);
1687 break;
4a71579b
PC		 1688 case_rr(hton, _us);
1689 case_rr(hton, _ui);
40a44dcb
PC		 1690 case_rr(bswap, _us);
1691 case_rr(bswap, _ui);
ba86ff93
PC		 1692 case jit_code_extr:
1693 extr(rn(node->u.w), rn(node->v.w), node->w.q.l, node->w.q.h);
1694 break;
1695 case jit_code_extr_u:
1696 extr_u(rn(node->u.w), rn(node->v.w), node->w.q.l, node->w.q.h);
1697 break;
1698 case jit_code_depr:
1699 depr(rn(node->u.w), rn(node->v.w), node->w.q.l, node->w.q.h);
1700 break;
1701 case jit_code_depi:
1702 depi(rn(node->u.w), node->v.w, node->w.q.l, node->w.q.h);
1703 break;
4a71579b
PC		 1704 case_rr(ext, _c);
1705 case_rr(ext, _uc);
1706 case_rr(ext, _s);
1707 case_rr(ext, _us);
ba3814c1
PC		 1708 case jit_code_casr:
1709 casr(rn(node->u.w), rn(node->v.w),
1710 rn(node->w.q.l), rn(node->w.q.h));
1711 break;
1712 case jit_code_casi:
1713 casi(rn(node->u.w), node->v.w,
1714 rn(node->w.q.l), rn(node->w.q.h));
1715 break;
4a71579b 1716 case_rr(mov,);
e0659411
PC		 1717 case_rrr(movn,);
1718 case_rrr(movz,);
4a71579b
PC		 1719 case jit_code_movi:
1720 if (node->flag & jit_flag_node) {
1721 temp = node->v.n;
1722 if (temp->code == jit_code_data ||
1723 (temp->code == jit_code_label &&
1724 (temp->flag & jit_flag_patch)))
1725 movi(rn(node->u.w), temp->u.w);
1726 else {
1727 assert(temp->code == jit_code_label ||
1728 temp->code == jit_code_epilog);
1729 word = movi_p(rn(node->u.w), temp->u.w);
79bfeef6 1730 patch(word, node, arm_patch_word);
4a71579b
PC		 1731 }
1732 }
1733 else
1734 movi(rn(node->u.w), node->v.w);
1735 break;
1736 case_rrr(lt,);
1737 case_rrw(lt,);
1738 case_rrr(lt, _u);
1739 case_rrw(lt, _u);
1740 case_rrr(le,);
1741 case_rrw(le,);
1742 case_rrr(le, _u);
1743 case_rrw(le, _u);
1744 case_rrr(eq,);
1745 case_rrw(eq,);
1746 case_rrr(ge,);
1747 case_rrw(ge,);
1748 case_rrr(ge, _u);
1749 case_rrw(ge, _u);
1750 case_rrr(gt,);
1751 case_rrw(gt,);
1752 case_rrr(gt, _u);
1753 case_rrw(gt, _u);
1754 case_rrr(ne,);
1755 case_rrw(ne,);
1756 case_brr(blt,);
1757 case_brw(blt,);
1758 case_brr(blt, _u);
1759 case_brw(blt, _u);
1760 case_brr(ble,);
1761 case_brw(ble,);
1762 case_brr(ble, _u);
1763 case_brw(ble, _u);
1764 case_brr(beq,);
1765 case_brw(beq,);
1766 case_brr(bge,);
1767 case_brw(bge,);
1768 case_brr(bge, _u);
1769 case_brw(bge, _u);
1770 case_brr(bgt,);
1771 case_brw(bgt,);
1772 case_brr(bgt, _u);
1773 case_brw(bgt, _u);
1774 case_brr(bne,);
1775 case_brw(bne,);
1776 case_brr(boadd,);
1777 case_brw(boadd,);
1778 case_brr(boadd, _u);
1779 case_brw(boadd, _u);
1780 case_brr(bxadd,);
1781 case_brw(bxadd,);
1782 case_brr(bxadd, _u);
1783 case_brw(bxadd, _u);
1784 case_brr(bosub,);
1785 case_brw(bosub,);
1786 case_brr(bosub, _u);
1787 case_brw(bosub, _u);
1788 case_brr(bxsub,);
1789 case_brw(bxsub,);
1790 case_brr(bxsub, _u);
1791 case_brw(bxsub, _u);
1792 case_brr(bms,);
1793 case_brw(bms,);
1794 case_brr(bmc,);
1795 case_brw(bmc,);
1796 case_vvv(add, _f);
1797 case_vvf(add);
1798 case_vvv(sub, _f);
1799 case_vvf(sub);
1800 case_vvf(rsb);
1801 case_vvv(mul, _f);
1802 case_vvf(mul);
1803 case_vvv(div, _f);
1804 case_vvf(div);
1805 case_vv(abs, _f);
1806 case_vv(neg, _f);
1807 case_vv(sqrt, _f);
ba86ff93
PC		 1808 case_rqr(fma, _f);
1809 case_rqr(fms, _f);
1810 case_rqr(fnma, _f);
1811 case_rqr(fnms, _f);
4a71579b
PC		 1812 case_vv(ext, _f);
1813 case_vv(ld, _f);
1814 case_vw(ld, _f);
1815 case_vvv(ldx, _f);
1816 case_vvw(ldx, _f);
ba86ff93
PC		 1817 case jit_code_unldr_x:
1818 if (jit_swf_p())
1819 swf_unldr_x(rn(node->u.w), rn(node->v.w), node->w.w);
1820 else
1821 vfp_unldr_x(rn(node->u.w), rn(node->v.w), node->w.w);
1822 break;
1823 case jit_code_unldi_x:
1824 if (jit_swf_p())
1825 swf_unldi_x(rn(node->u.w), node->v.w, node->w.w);
1826 else
1827 vfp_unldi_x(rn(node->u.w), node->v.w, node->w.w);
1828 break;
4a71579b
PC		 1829 case_vv(st, _f);
1830 case_wv(st, _f);
1831 case_vvv(stx, _f);
1832 case_wvv(stx, _f);
ba86ff93
PC		 1833 case jit_code_unstr_x:
1834 if (jit_swf_p())
1835 swf_unstr_x(rn(node->u.w), rn(node->v.w), node->w.w);
1836 else
1837 vfp_unstr_x(rn(node->u.w), rn(node->v.w), node->w.w);
1838 break;
1839 case jit_code_unsti_x:
1840 if (jit_swf_p())
1841 swf_unsti_x(node->u.w, rn(node->v.w), node->w.w);
1842 else
1843 vfp_unsti_x(node->u.w, rn(node->v.w), node->w.w);
1844 break;
4a71579b
PC		 1845 case_vv(mov, _f);
1846 case jit_code_movi_f:
1847 assert_data(node);
1848 if (jit_swf_p())
1849 swf_movi_f(rn(node->u.w), node->v.f);
1850 else
1851 vfp_movi_f(rn(node->u.w), node->v.f);
1852 break;
1853 case_vv(ext, _d_f);
1854 case_vvv(lt, _f);
1855 case_vvf(lt);
1856 case_vvv(le, _f);
1857 case_vvf(le);
1858 case_vvv(eq, _f);
1859 case_vvf(eq);
1860 case_vvv(ge, _f);
1861 case_vvf(ge);
1862 case_vvv(gt, _f);
1863 case_vvf(gt);
1864 case_vvv(ne, _f);
1865 case_vvf(ne);
1866 case_vvv(unlt, _f);
1867 case_vvf(unlt);
1868 case_vvv(unle, _f);
1869 case_vvf(unle);
1870 case_vvv(uneq, _f);
1871 case_vvf(uneq);
1872 case_vvv(unge, _f);
1873 case_vvf(unge);
1874 case_vvv(ungt, _f);
1875 case_vvf(ungt);
1876 case_vvv(ltgt, _f);
1877 case_vvf(ltgt);
1878 case_vvv(ord, _f);
1879 case_vvf(ord);
1880 case_vvv(unord, _f);
1881 case_vvf(unord);
1882 case_bvv(blt, _f);
1883 case_bvf(blt);
1884 case_bvv(ble, _f);
1885 case_bvf(ble);
1886 case_bvv(beq, _f);
1887 case_bvf(beq);
1888 case_bvv(bge, _f);
1889 case_bvf(bge);
1890 case_bvv(bgt, _f);
1891 case_bvf(bgt);
1892 case_bvv(bne, _f);
1893 case_bvf(bne);
1894 case_bvv(bunlt, _f);
1895 case_bvf(bunlt);
1896 case_bvv(bunle, _f);
1897 case_bvf(bunle);
1898 case_bvv(buneq, _f);
1899 case_bvf(buneq);
1900 case_bvv(bunge, _f);
1901 case_bvf(bunge);
1902 case_bvv(bungt, _f);
1903 case_bvf(bungt);
1904 case_bvv(bltgt, _f);
1905 case_bvf(bltgt);
1906 case_bvv(bord, _f);
1907 case_bvf(bord);
1908 case_bvv(bunord, _f);
1909 case_bvf(bunord);
1910 case_vvv(add, _d);
1911 case_vvd(add);
1912 case_vvv(sub, _d);
1913 case_vvd(sub);
1914 case_vvd(rsb);
1915 case_vvv(mul, _d);
1916 case_vvd(mul);
1917 case_vvv(div, _d);
1918 case_vvd(div);
1919 case_vv(abs, _d);
1920 case_vv(neg, _d);
1921 case_vv(sqrt, _d);
ba86ff93
PC		 1922 case_rqr(fma, _d);
1923 case_rqr(fms, _d);
1924 case_rqr(fnma, _d);
1925 case_rqr(fnms, _d);
4a71579b
PC		 1926 case_vv(ext, _d);
1927 case_vv(ld, _d);
1928 case_vw(ld, _d);
1929 case_vvv(ldx, _d);
1930 case_vvw(ldx, _d);
1931 case_vv(st, _d);
1932 case_wv(st, _d);
1933 case_vvv(stx, _d);
1934 case_wvv(stx, _d);
1935 case_vv(mov, _d);
1936 case jit_code_movi_d:
1937 assert_data(node);
1938 if (jit_swf_p())
1939 swf_movi_d(rn(node->u.w), node->v.d);
1940 else
1941 vfp_movi_d(rn(node->u.w), node->v.d);
1942 break;
1943 case_vv(ext, _f_d);
1944 case_vvv(lt, _d);
1945 case_vvd(lt);
1946 case_vvv(le, _d);
1947 case_vvd(le);
1948 case_vvv(eq, _d);
1949 case_vvd(eq);
1950 case_vvv(ge, _d);
1951 case_vvd(ge);
1952 case_vvv(gt, _d);
1953 case_vvd(gt);
1954 case_vvv(ne, _d);
1955 case_vvd(ne);
1956 case_vvv(unlt, _d);
1957 case_vvd(unlt);
1958 case_vvv(unle, _d);
1959 case_vvd(unle);
1960 case_vvv(uneq, _d);
1961 case_vvd(uneq);
1962 case_vvv(unge, _d);
1963 case_vvd(unge);
1964 case_vvv(ungt, _d);
1965 case_vvd(ungt);
1966 case_vvv(ltgt, _d);
1967 case_vvd(ltgt);
1968 case_vvv(ord, _d);
1969 case_vvd(ord);
1970 case_vvv(unord, _d);
1971 case_vvd(unord);
1972 case_bvv(blt, _d);
1973 case_bvd(blt);
1974 case_bvv(ble, _d);
1975 case_bvd(ble);
1976 case_bvv(beq, _d);
1977 case_bvd(beq);
1978 case_bvv(bge, _d);
1979 case_bvd(bge);
1980 case_bvv(bgt, _d);
1981 case_bvd(bgt);
1982 case_bvv(bne, _d);
1983 case_bvd(bne);
1984 case_bvv(bunlt, _d);
1985 case_bvd(bunlt);
1986 case_bvv(bunle, _d);
1987 case_bvd(bunle);
1988 case_bvv(buneq, _d);
1989 case_bvd(buneq);
1990 case_bvv(bunge, _d);
1991 case_bvd(bunge);
1992 case_bvv(bungt, _d);
1993 case_bvd(bungt);
1994 case_bvv(bltgt, _d);
1995 case_bvd(bltgt);
1996 case_bvv(bord, _d);
1997 case_bvd(bord);
1998 case_bvv(bunord, _d);
1999 case_bvd(bunord);
2000 case jit_code_jmpr:
79bfeef6 2001 jit_check_frame();
4a71579b
PC		 2002 jmpr(rn(node->u.w));
2003 flush_consts();
2004 break;
2005 case jit_code_jmpi:
2006 if (node->flag & jit_flag_node) {
2007 temp = node->u.n;
2008 assert(temp->code == jit_code_label ||
2009 temp->code == jit_code_epilog);
2010 if (temp->flag & jit_flag_patch)
2011 jmpi(temp->u.w);
2012 else {
79bfeef6
PC		 2013 word = _jit->code.length -
2014 (_jit->pc.uc - _jit->code.ptr);
2015 if (jit_thumb_p()) word >>= 1;
2016 else word >>= 2;
2017 word -= 2;
2018 value = _s24P(word);
2019 word = jmpi_p(_jit->pc.w, value);
2020 patch(word, node, value ?
2021 arm_patch_jump : arm_patch_word);
4a71579b
PC		 2022 }
2023 }
79bfeef6
PC		 2024 else {
2025 jit_check_frame();
4a71579b 2026 jmpi(node->u.w);
79bfeef6 2027 }
4a71579b
PC		 2028 flush_consts();
2029 break;
2030 case jit_code_callr:
79bfeef6 2031 jit_check_frame();
4a71579b
PC		 2032 callr(rn(node->u.w));
2033 break;
2034 case jit_code_calli:
2035 if (node->flag & jit_flag_node) {
79bfeef6 2036 CHECK_RETURN();
4a71579b
PC		 2037 temp = node->u.n;
2038 assert(temp->code == jit_code_label ||
2039 temp->code == jit_code_epilog);
2040 if (temp->flag & jit_flag_patch)
79bfeef6 2041 calli(temp->u.w, 0);
4a71579b 2042 else {
79bfeef6
PC		 2043 word = _jit->code.length -
2044 (_jit->pc.uc - _jit->code.ptr);
2045 if (jit_exchange_p())
2046 word -= 8;
2047 if (jit_thumb_p()) word >>= 1;
2048 else word >>= 2;
2049 word -= 2;
2050 value = _s24P(word);
2051 word = calli_p(_jit->pc.w, value);
2052 patch(word, node, value ?
2053 arm_patch_call : arm_patch_word);
4a71579b
PC		 2054 }
2055 }
79bfeef6
PC		 2056 else {
2057 jit_check_frame();
2058 calli(node->u.w, jit_exchange_p());
2059 }
4a71579b
PC		 2060 break;
2061 case jit_code_prolog:
2062 _jitc->function = _jitc->functions.ptr + node->w.w;
2063 undo.node = node;
2064 undo.word = _jit->pc.w;
79bfeef6 2065 memcpy(&undo.func, _jitc->function, sizeof(undo.func));
4a71579b
PC		 2066#if DEVEL_DISASSEMBLER
2067 undo.prevw = prevw;
2068#endif
2069 undo.data = _jitc->consts.data;
2070 undo.thumb = _jitc->thumb;
2071 undo.const_offset = _jitc->consts.offset;
2072 undo.patch_offset = _jitc->patches.offset;
2073#if DISASSEMBLER
2074 if (_jitc->data_info.ptr)
2075 undo.info_offset = _jitc->data_info.offset;
2076#endif
2077 restart_function:
2078 _jitc->again = 0;
79bfeef6
PC		 2079 compute_framesize();
2080 patch_alist(0);
4a71579b
PC		 2081 prolog(node);
2082 break;
2083 case jit_code_epilog:
2084 assert(_jitc->function == _jitc->functions.ptr + node->w.w);
2085 if (_jitc->again) {
2086 for (temp = undo.node->next;
2087 temp != node; temp = temp->next) {
2088 if (temp->code == jit_code_label ||
2089 temp->code == jit_code_epilog)
2090 temp->flag &= ~jit_flag_patch;
2091 }
2092 temp->flag &= ~jit_flag_patch;
2093 node = undo.node;
2094 _jit->pc.w = undo.word;
79bfeef6
PC		 2095 /* undo.func.self.aoff and undo.func.regset should not
2096 * be undone, as they will be further updated, and are
2097 * the reason of the undo. */
2098 undo.func.self.aoff = _jitc->function->frame +
2099 _jitc->function->self.aoff;
2100 undo.func.need_frame = _jitc->function->need_frame;
2101 undo.func.need_return = _jitc->function->need_return;
2102 jit_regset_set(&undo.func.regset, &_jitc->function->regset);
2103 /* allocar information also does not need to be undone */
2104 undo.func.aoffoff = _jitc->function->aoffoff;
2105 undo.func.allocar = _jitc->function->allocar;
2106 /* swf_offset and check_reg_args must also not be undone */
2107 undo.func.swf_offset = _jitc->function->swf_offset;
2108 undo.func.save_reg_args = _jitc->function->save_reg_args;
2109 memcpy(_jitc->function, &undo.func, sizeof(undo.func));
4a71579b
PC		 2110#if DEVEL_DISASSEMBLER
2111 prevw = undo.prevw;
2112#endif
2113 invalidate_consts();
2114 _jitc->consts.data = undo.data;
2115 _jitc->thumb = undo.thumb;
2116 _jitc->consts.offset = undo.const_offset;
2117 _jitc->patches.offset = undo.patch_offset;
2118#if DISASSEMBLER
2119 if (_jitc->data_info.ptr)
2120 _jitc->data_info.offset = undo.info_offset;
2121#endif
79bfeef6 2122 patch_alist(1);
4a71579b
PC		 2123 goto restart_function;
2124 }
2125 /* remember label is defined */
2126 node->flag |= jit_flag_patch;
2127 node->u.w = _jit->pc.w;
2128 epilog(node);
2129 _jitc->function = NULL;
2130 flush_consts();
2131 break;
2132 case jit_code_movr_w_f:
2133 if (jit_swf_p())
ba86ff93 2134 swf_movr_w_f(rn(node->u.w), rn(node->v.w));
4a71579b 2135 else
ba86ff93 2136 vfp_movr_w_f(rn(node->u.w), rn(node->v.w));
4a71579b
PC		 2137 break;
2138 case jit_code_movr_f_w:
2139 if (jit_swf_p())
ba86ff93 2140 swf_movr_f_w(rn(node->u.w), rn(node->v.w));
4a71579b 2141 else
ba86ff93 2142 vfp_movr_f_w(rn(node->u.w), rn(node->v.w));
4a71579b
PC		 2143 break;
2144 case jit_code_movi_f_w:
2145 assert_data(node);
ba86ff93
PC		 2146 movi_f_w(rn(node->u.w), node->v.f);
2147 break;
2148 case jit_code_movi_w_f:
4a71579b 2149 if (jit_swf_p())
ba86ff93 2150 swf_movi_w_f(rn(node->u.w), node->v.w);
4a71579b 2151 else
ba86ff93 2152 vfp_movi_w_f(rn(node->u.w), node->v.w);
4a71579b
PC		 2153 break;
2154 case jit_code_movr_ww_d:
2155 if (jit_swf_p())
ba86ff93 2156 swf_movr_ww_d(rn(node->u.w), rn(node->v.w), rn(node->w.w));
4a71579b 2157 else
ba86ff93 2158 vfp_movr_ww_d(rn(node->u.w), rn(node->v.w), rn(node->w.w));
4a71579b
PC		 2159 break;
2160 case jit_code_movr_d_ww:
2161 if (jit_swf_p())
ba86ff93 2162 swf_movr_d_ww(rn(node->u.w), rn(node->v.w), rn(node->w.w));
4a71579b 2163 else
ba86ff93 2164 vfp_movr_d_ww(rn(node->u.w), rn(node->v.w), rn(node->w.w));
4a71579b
PC		 2165 break;
2166 case jit_code_movi_d_ww:
ba86ff93
PC		 2167 movi_d_ww(rn(node->u.w), rn(node->v.w), node->w.d);
2168 break;
2169 case jit_code_movi_ww_d:
4a71579b 2170 if (jit_swf_p())
ba86ff93 2171 swf_movi_ww_d(rn(node->u.w), node->v.w, node->w.w);
4a71579b 2172 else
ba86ff93 2173 vfp_movi_ww_d(rn(node->u.w), node->v.w, node->w.w);
4a71579b
PC		 2174 break;
2175 case jit_code_va_start:
2176 vastart(rn(node->u.w));
2177 break;
2178 case jit_code_va_arg:
2179 vaarg(rn(node->u.w), rn(node->v.w));
2180 break;
2181 case jit_code_va_arg_d:
2182 if (jit_swf_p())
2183 swf_vaarg_d(rn(node->u.w), rn(node->v.w));
2184 else
2185 vfp_vaarg_d(rn(node->u.w), rn(node->v.w));
2186 break;
2187 case jit_code_live: case jit_code_ellipsis:
2188 case jit_code_va_push:
2189 case jit_code_allocai: case jit_code_allocar:
79bfeef6
PC		 2190 case jit_code_arg_c: case jit_code_arg_s:
2191 case jit_code_arg_i:
4a71579b
PC		 2192 case jit_code_arg_f: case jit_code_arg_d:
2193 case jit_code_va_end:
2194 case jit_code_ret:
79bfeef6
PC		 2195 case jit_code_retr_c: case jit_code_reti_c:
2196 case jit_code_retr_uc: case jit_code_reti_uc:
2197 case jit_code_retr_s: case jit_code_reti_s:
2198 case jit_code_retr_us: case jit_code_reti_us:
2199 case jit_code_retr_i: case jit_code_reti_i:
4a71579b
PC		 2200 case jit_code_retr_f: case jit_code_reti_f:
2201 case jit_code_retr_d: case jit_code_reti_d:
2202 case jit_code_getarg_c: case jit_code_getarg_uc:
2203 case jit_code_getarg_s: case jit_code_getarg_us:
2204 case jit_code_getarg_i:
2205 case jit_code_getarg_f: case jit_code_getarg_d:
79bfeef6
PC		 2206 case jit_code_putargr_c: case jit_code_putargi_c:
2207 case jit_code_putargr_uc: case jit_code_putargi_uc:
2208 case jit_code_putargr_s: case jit_code_putargi_s:
2209 case jit_code_putargr_us: case jit_code_putargi_us:
2210 case jit_code_putargr_i: case jit_code_putargi_i:
4a71579b
PC		 2211 case jit_code_putargr_f: case jit_code_putargi_f:
2212 case jit_code_putargr_d: case jit_code_putargi_d:
79bfeef6
PC		 2213 case jit_code_pushargr_c: case jit_code_pushargi_c:
2214 case jit_code_pushargr_uc: case jit_code_pushargi_uc:
2215 case jit_code_pushargr_s: case jit_code_pushargi_s:
2216 case jit_code_pushargr_us: case jit_code_pushargi_us:
2217 case jit_code_pushargr_i: case jit_code_pushargi_i:
4a71579b
PC		 2218 case jit_code_pushargr_f: case jit_code_pushargi_f:
2219 case jit_code_pushargr_d: case jit_code_pushargi_d:
2220 case jit_code_retval_c: case jit_code_retval_uc:
2221 case jit_code_retval_s: case jit_code_retval_us:
2222 case jit_code_retval_i:
2223 case jit_code_retval_f: case jit_code_retval_d:
2224 case jit_code_prepare:
2225 case jit_code_finishr: case jit_code_finishi:
ba86ff93
PC		 2226 case jit_code_negi_f: case jit_code_absi_f:
2227 case jit_code_sqrti_f: case jit_code_negi_d:
2228 case jit_code_absi_d: case jit_code_sqrti_d:
2229 break;
2230 case jit_code_negi:
2231 negi(rn(node->u.w), node->v.w);
2232 break;
2233 case jit_code_comi:
2234 comi(rn(node->u.w), node->v.w);
2235 break;
2236 case jit_code_exti_c:
2237 exti_c(rn(node->u.w), node->v.w);
2238 break;
2239 case jit_code_exti_uc:
2240 exti_uc(rn(node->u.w), node->v.w);
2241 break;
2242 case jit_code_exti_s:
2243 exti_s(rn(node->u.w), node->v.w);
2244 break;
2245 case jit_code_exti_us:
2246 exti_us(rn(node->u.w), node->v.w);
2247 break;
2248 case jit_code_bswapi_us:
2249 bswapi_us(rn(node->u.w), node->v.w);
2250 break;
2251 case jit_code_bswapi_ui:
2252 bswapi_ui(rn(node->u.w), node->v.w);
2253 break;
2254 case jit_code_htoni_us:
2255 htoni_us(rn(node->u.w), node->v.w);
2256 break;
2257 case jit_code_htoni_ui:
2258 htoni_ui(rn(node->u.w), node->v.w);
2259 break;
2260 case jit_code_cloi:
2261 cloi(rn(node->u.w), node->v.w);
2262 break;
2263 case jit_code_clzi:
2264 clzi(rn(node->u.w), node->v.w);
2265 break;
2266 case jit_code_ctoi:
2267 ctoi(rn(node->u.w), node->v.w);
2268 break;
2269 case jit_code_ctzi:
2270 ctzi(rn(node->u.w), node->v.w);
2271 break;
2272 case jit_code_rbiti:
2273 rbiti(rn(node->u.w), node->v.w);
2274 break;
2275 case jit_code_popcnti:
2276 popcnti(rn(node->u.w), node->v.w);
2277 break;
2278 case jit_code_exti:
2279 exti(rn(node->u.w), node->v.w, node->w.q.l, node->w.q.h);
2280 break;
2281 case jit_code_exti_u:
2282 exti_u(rn(node->u.w), node->v.w, node->w.q.l, node->w.q.h);
4a71579b
PC		 2283 break;
2284 default:
2285 abort();
2286 }
2287 jit_regarg_clr(node, value);
2288 assert(_jitc->regarg == 0 && _jitc->synth == 0);
2289 /* update register live state */
2290 jit_reglive(node);
2291
ba86ff93
PC		 2292#if defined JIT_INSTR_MAX
2293 word = 4096 - JIT_INSTR_MAX;
2294#else
2295 word = 3968;
2296#endif
2297 /* longest sequence should be 64 bytes, but preventively
2298 * do not let it go past 256 remaining bytes before a flush */
2299 if (word > 3968)
2300 word = 3968;
4a71579b 2301 if (_jitc->consts.length &&
ba86ff93 2302 (_jit->pc.uc - _jitc->consts.data >= word ||
4a71579b 2303 (jit_uword_t)_jit->pc.uc -
ba86ff93 2304 (jit_uword_t)_jitc->consts.patches[0] >= word)) {
4a71579b
PC		 2305 if (node->next &&
2306 node->next->code != jit_code_jmpi &&
2307 node->next->code != jit_code_jmpr &&
2308 node->next->code != jit_code_epilog) {
2309 /* insert a jump, flush constants and continue */
2310 word = _jit->pc.w;
2311 assert(!jit_thumb_p());
2312 B(0);
2313 flush_consts();
2314 patch_at(arm_patch_jump, word, _jit->pc.w);
2315 }
2316 }
2317 }
2318#undef case_bvd
2319#undef case_bvf
2320#undef case_brw
2321#undef case_bvv
2322#undef case_brr
2323#undef case_wvv
2324#undef case_wrr
2325#undef case_vvd
2326#undef case_vvf
2327#undef case_vvw
2328#undef case_rrw
2329#undef case_vvv
2330#undef case_rrr
2331#undef case_wv
2332#undef case_wr
2333#undef case_vw
2334#undef case_vv
2335#undef case_rw
2336#undef case_rr
2337
2338 flush_consts();
2339 for (offset = 0; offset < _jitc->patches.offset; offset++) {
2340 assert(_jitc->patches.ptr[offset].kind & arm_patch_node);
2341 node = _jitc->patches.ptr[offset].node;
2342 word = _jitc->patches.ptr[offset].inst;
2343 if (!jit_thumb_p() &&
79bfeef6
PC		 2344 (node->code == jit_code_movi ||
2345 (node->code == jit_code_calli &&
2346 (_jitc->patches.ptr[offset].kind & ~arm_patch_node) ==
2347 arm_patch_word))) {
4a71579b
PC		 2348 /* calculate where to patch word */
2349 value = *(jit_int32_t *)word;
2350 assert((value & 0x0f700000) == ARM_LDRI);
2351 /* offset may become negative (-4) if last instruction
2352 * before unconditional branch and data following
2353 * FIXME can this cause issues in the preprocessor prefetch
2354 * or something else? should not, as the constants are after
2355 * an unconditional jump */
2356 if (value & ARM_P) value = value & 0x00000fff;
2357 else value = -(value & 0x00000fff);
2358 word = word + 8 + value;
2359 }
2360 value = node->code == jit_code_movi ? node->v.n->u.w : node->u.n->u.w;
2361 patch_at(_jitc->patches.ptr[offset].kind & ~arm_patch_node, word, value);
2362 }
2363
2364 jit_flush(_jit->code.ptr, _jit->pc.uc);
2365
2366 return (_jit->code.ptr);
2367}
2368
2369#define CODE 1
2370# include "jit_rewind.c"
2371# include "jit_arm-cpu.c"
2372# include "jit_arm-swf.c"
2373# include "jit_arm-vfp.c"
ba3814c1 2374# include "jit_fallback.c"
4a71579b
PC		 2375#undef CODE
2376
2377void
2378jit_flush(void *fptr, void *tptr)
2379{
2380#if defined(__GNUC__)
2381 jit_uword_t i, f, t, s;
2382
2383 s = sysconf(_SC_PAGE_SIZE);
2384 f = (jit_uword_t)fptr & -s;
2385 t = (((jit_uword_t)tptr) + s - 1) & -s;
2386 for (i = f; i < t; i += s)
2387 __clear_cache((void *)i, (void *)(i + s));
2388#endif
2389}
2390
2391void
2392_emit_ldxi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
2393{
2394 ldxi_i(rn(r0), rn(r1), i0);
2395}
2396
2397void
2398_emit_stxi(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
2399{
2400 stxi_i(i0, rn(r0), rn(r1));
2401}
2402
2403void
2404_emit_ldxi_d(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
2405{
2406 if (jit_swf_p())
2407 swf_ldxi_d(rn(r0), rn(r1), i0);
2408 else
2409 vfp_ldxi_d(rn(r0), rn(r1), i0);
2410}
2411
2412void
2413_emit_stxi_d(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
2414{
2415 if (jit_swf_p())
2416 swf_stxi_d(i0, rn(r0), rn(r1));
2417 else
2418 vfp_stxi_d(i0, rn(r0), rn(r1));
2419}
2420
2421static jit_int32_t
2422_jit_get_reg_pair(jit_state_t *_jit)
2423{
2424 /* bypass jit_get_reg() with argument or'ed with jit_class_chk
2425 * and try to find an consecutive, even free register pair, or
2426 * return JIT_NOREG if fail, as the cost of spills is greater
2427 * than splitting a double load/store in two operations. */
2428 if (jit_reg_free_p(_R0) && jit_reg_free_p(_R1)) {
2429 jit_regset_setbit(&_jitc->regarg, _R0);
2430 jit_regset_setbit(&_jitc->regarg, _R1);
2431 return (_R0);
2432 }
2433 if (jit_reg_free_p(_R2) && jit_reg_free_p(_R3)) {
2434 jit_regset_setbit(&_jitc->regarg, _R2);
2435 jit_regset_setbit(&_jitc->regarg, _R3);
2436 return (_R2);
2437 }
2438 if (jit_reg_free_p(_R4) && jit_reg_free_p(_R5)) {
2439 jit_regset_setbit(&_jitc->regarg, _R4);
2440 jit_regset_setbit(&_jitc->regarg, _R5);
2441 return (_R4);
2442 }
2443 if (jit_reg_free_p(_R6) && jit_reg_free_p(_R7)) {
2444 jit_regset_setbit(&_jitc->regarg, _R6);
2445 jit_regset_setbit(&_jitc->regarg, _R7);
2446 return (_R6);
2447 }
2448 if (jit_reg_free_p(_R8) && jit_reg_free_p(_R9)) {
2449 jit_regset_setbit(&_jitc->regarg, _R8);
2450 jit_regset_setbit(&_jitc->regarg, _R9);
2451 return (_R8);
2452 }
2453 return (JIT_NOREG);
2454}
2455
2456static void
2457_jit_unget_reg_pair(jit_state_t *_jit, jit_int32_t reg)
2458{
2459 jit_unget_reg(reg);
2460 switch (reg) {
2461 case _R0: jit_unget_reg(_R1); break;
2462 case _R2: jit_unget_reg(_R3); break;
2463 case _R4: jit_unget_reg(_R5); break;
2464 case _R6: jit_unget_reg(_R7); break;
2465 case _R8: jit_unget_reg(_R9); break;
2466 default: abort();
2467 }
2468}
2469
2470/* A prolog must be aligned at mod 4 bytes boundary.
2471 * This condition was not being required to be tested by
2472 * accident previously, but with the jit_frame and jit_tramp
2473 * code it is required */
2474static jit_bool_t
2475_must_align_p(jit_state_t *_jit, jit_node_t *node)
2476{
2477 if (jit_thumb_p() && (_jit->pc.w & 3)) {
2478 for (; node; node = node->next) {
2479 switch (node->code) {
2480 case jit_code_note:
2481 case jit_code_name:
2482 case jit_code_label:
2483 break;
2484 case jit_code_prolog:
2485 return (1);
2486 default:
2487 return (0);
2488 }
2489 }
2490 }
2491 return (0);
2492}
2493
2494static void
2495_load_const(jit_state_t *_jit, jit_bool_t uniq, jit_int32_t r0, jit_word_t i0)
2496{
2497 jit_word_t w;
2498 jit_word_t d;
2499 jit_word_t base;
2500 jit_int32_t *data;
2501 jit_int32_t size;
2502 jit_int32_t offset;
2503
2504 assert(!jit_thumb_p());
2505 if (!uniq) {
2506 /* use zero, a valid directly encoded immediate, to avoid the
2507 * need of a bitmask to know what offsets will be patched, so
2508 * that comparison will always fail for constants that cannot
2509 * be encoded */
2510 assert(i0 != 0);
2511
2512 /* Actually, code is (currently at least) not self modifying,
2513 * so, any value reachable backwards is valid as a constant. */
2514
2515 /* FIXME a quickly updateable/mutable hash table could be
2516 * better here, but most times only a few comparisons
2517 * should be done
2518 */
2519
2520 /* search in previous constant pool */
2521 if ((data = (jit_int32_t *)_jitc->consts.data)) {
2522 w = (jit_word_t)data;
2523 /* maximum backwards offset */
2524 base = (_jit->pc.w + 8) - 4092;
2525 if (base <= w)
2526 /* can scan all possible available backward constants */
2527 base = 0;
2528 else
2529 base = (base - w) >> 2;
2530 size = _jitc->consts.size >> 2;
2531 for (offset = size - 1; offset >= base; offset--) {
2532 if (data[offset] == i0) {
2533 w = (jit_word_t)(data + offset);
2534 d = (_jit->pc.w + 8) - w;
2535 LDRIN(r0, _R15_REGNO, d);
2536 return;
2537 }
2538 }
2539 }
2540 }
2541 else
2542 assert(i0 == 0);
2543
2544 _jitc->consts.patches[_jitc->consts.offset++] = _jit->pc.w;
2545 /* (probably) positive forward offset */
2546 LDRI(r0, _R15_REGNO, 0);
2547
2548 if (!uniq) {
2549 /* search already requested values */
2550 for (offset = 0; offset < _jitc->consts.length; offset++) {
2551 if (_jitc->consts.values[offset] == i0) {
2552 _jitc->consts.patches[_jitc->consts.offset++] = offset;
2553 return;
2554 }
2555 }
2556 }
2557
2558#if DEBUG
2559 /* cannot run out of space because of limited range
2560 * but assert anyway to catch logic errors */
2561 assert(_jitc->consts.length < 1024);
2562 assert(_jitc->consts.offset < 2048);
2563#endif
2564 _jitc->consts.patches[_jitc->consts.offset++] = _jitc->consts.length;
2565 _jitc->consts.values[_jitc->consts.length++] = i0;
2566}
2567
2568static void
2569_flush_consts(jit_state_t *_jit)
2570{
2571 jit_word_t word;
2572 jit_int32_t offset;
2573
2574 /* if no forward constants */
2575 if (!_jitc->consts.length)
2576 return;
2577 assert(!jit_thumb_p());
2578 word = _jit->pc.w;
2579 _jitc->consts.data = _jit->pc.uc;
2580 _jitc->consts.size = _jitc->consts.length << 2;
2581 /* FIXME check will not overrun, otherwise, need to reallocate
2582 * code buffer and start over */
2583 jit_memcpy(_jitc->consts.data, _jitc->consts.values, _jitc->consts.size);
2584 _jit->pc.w += _jitc->consts.size;
2585
2586#if DISASSEMBLER
2587 if (_jitc->data_info.ptr) {
2588 if (_jitc->data_info.offset >= _jitc->data_info.length) {
2589 jit_realloc((jit_pointer_t *)&_jitc->data_info.ptr,
2590 _jitc->data_info.length * sizeof(jit_data_info_t),
2591 (_jitc->data_info.length + 1024) *
2592 sizeof(jit_data_info_t));
2593 _jitc->data_info.length += 1024;
2594 }
2595 _jitc->data_info.ptr[_jitc->data_info.offset].code = word;
2596 _jitc->data_info.ptr[_jitc->data_info.offset].length = _jitc->consts.size;
2597 ++_jitc->data_info.offset;
2598 }
2599#endif
2600
2601 for (offset = 0; offset < _jitc->consts.offset; offset += 2)
2602 patch_at(arm_patch_load, _jitc->consts.patches[offset],
2603 word + (_jitc->consts.patches[offset + 1] << 2));
2604 _jitc->consts.length = _jitc->consts.offset = 0;
2605}
2606
2607/* to be called if needing to start over a function */
2608static void
2609_invalidate_consts(jit_state_t *_jit)
2610{
2611 /* if no forward constants */
2612 if (_jitc->consts.length)
2613 _jitc->consts.length = _jitc->consts.offset = 0;
2614}
2615
2616static void
79bfeef6
PC		 2617_compute_framesize(jit_state_t *_jit)
2618{
2619 jit_int32_t reg;
2620 _jitc->framesize = sizeof(jit_word_t) * 2; /* lr+fp */
2621 for (reg = 0; reg < jit_size(iregs); reg++)
2622 if (jit_regset_tstbit(&_jitc->function->regset, iregs[reg]))
2623 _jitc->framesize += sizeof(jit_word_t);
2624
2625 if (_jitc->function->save_reg_args)
2626 _jitc->framesize += 16;
2627
2628 /* Make sure functions called have a 8 byte aligned stack */
2629 _jitc->framesize = (_jitc->framesize + 7) & -8;
2630}
2631
2632static void
2633_patch(jit_state_t *_jit, jit_word_t instr, jit_node_t *node, jit_int32_t kind)
4a71579b
PC		 2634{
2635 jit_int32_t flag;
4a71579b
PC		 2636
2637 assert(node->flag & jit_flag_node);
79bfeef6 2638 if (node->code == jit_code_movi)
4a71579b 2639 flag = node->v.n->flag;
79bfeef6 2640 else
4a71579b 2641 flag = node->u.n->flag;
4a71579b
PC		 2642 assert(!(flag & jit_flag_patch));
2643 kind |= arm_patch_node;
2644 if (_jitc->patches.offset >= _jitc->patches.length) {
2645 jit_realloc((jit_pointer_t *)&_jitc->patches.ptr,
2646 _jitc->patches.length * sizeof(jit_patch_t),
2647 (_jitc->patches.length + 1024) * sizeof(jit_patch_t));
2648 _jitc->patches.length += 1024;
2649 }
2650 _jitc->patches.ptr[_jitc->patches.offset].kind = kind;
2651 _jitc->patches.ptr[_jitc->patches.offset].inst = instr;
2652 _jitc->patches.ptr[_jitc->patches.offset].node = node;
2653 ++_jitc->patches.offset;
2654}
ARM optimized PCSX fork