notaz.gp2x.de / pcsx_rearmed.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
add a thp-based huge page alloc fallback
[pcsx_rearmed.git] / deps / lightning / lib / lightning.c
CommitLineData
4a71579b 1/*
79bfeef6 2 * Copyright (C) 2012-2023 Free Software Foundation, Inc.
4a71579b
PC		 3 *
4 * This file is part of GNU lightning.
5 *
6 * GNU lightning is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU Lesser General Public License as published
8 * by the Free Software Foundation; either version 3, or (at your option)
9 * any later version.
10 *
11 * GNU lightning is distributed in the hope that it will be useful, but
12 * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
13 * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public
14 * License for more details.
15 *
16 * Authors:
17 * Paulo Cesar Pereira de Andrade
18 */
19
20#include <lightning.h>
21#include <lightning/jit_private.h>
40a44dcb
PC		 22#if HAVE_MMAP
23# include <sys/mman.h>
24#endif
4a71579b
PC		 25#if defined(__sgi)
26# include <fcntl.h>
27#endif
28
29#ifndef MAP_ANON
30# define MAP_ANON MAP_ANONYMOUS
31# ifndef MAP_ANONYMOUS
32# define MAP_ANONYMOUS 0
33# endif
34#endif
35
36#define jit_regload_reload 0 /* convert to reload */
37#define jit_regload_delete 1 /* just remove node */
38#define jit_regload_isdead 2 /* delete and unset live bit */
39
40/*
41 * Prototypes
42 */
43static jit_word_t hash_data(const void*, jit_word_t);
44
45#define new_pool() _new_pool(_jit)
46static void _new_pool(jit_state_t*);
47
48#define new_node(u) _new_node(_jit, u)
49static jit_node_t *_new_node(jit_state_t*, jit_code_t);
50
51#define link_node(u) _link_node(_jit, u)
52static inline jit_node_t *_link_node(jit_state_t*, jit_node_t*);
53
54#define del_node(u, v) _del_node(_jit, u, v)
55static inline void _del_node(jit_state_t*, jit_node_t*, jit_node_t*);
56
57#define free_node(u) _free_node(_jit, u)
58static inline void _free_node(jit_state_t*, jit_node_t*);
59
60#define del_label(u, v) _del_label(_jit, u, v)
61static void _del_label(jit_state_t*, jit_node_t*, jit_node_t*);
62
63#define jit_dataset() _jit_dataset(_jit)
64static void
65_jit_dataset(jit_state_t *_jit);
66
c0c16242
PC		 67#define block_update_set(block, target) _block_update_set(_jit, block, target)
68static jit_bool_t _block_update_set(jit_state_t*, jit_block_t*, jit_block_t*);
69
24d91c0d
PC		 70#define propagate_backward(block) _propagate_backward(_jit, block)
71static void _propagate_backward(jit_state_t*, jit_block_t*);
72
c0c16242
PC		 73#define check_block_again() _check_block_again(_jit)
74static jit_bool_t _check_block_again(jit_state_t*);
75
76#define do_setup() _do_setup(_jit)
77static void _do_setup(jit_state_t*);
78
4a71579b
PC		 79#define jit_setup(block) _jit_setup(_jit, block)
80static void
81_jit_setup(jit_state_t *_jit, jit_block_t *block);
82
c0c16242
PC		 83#define do_follow(always) _do_follow(_jit, always)
84static void _do_follow(jit_state_t*, jit_bool_t);
85
86#define jit_follow(block) _jit_follow(_jit, block)
4a71579b 87static void
c0c16242 88_jit_follow(jit_state_t *_jit, jit_block_t *block);
4a71579b
PC		 89
90#define jit_update(node, live, mask) _jit_update(_jit, node, live, mask)
91static void
92_jit_update(jit_state_t *_jit, jit_node_t *node,
93 jit_regset_t *live, jit_regset_t *mask);
94
95#define thread_jumps() _thread_jumps(_jit)
96static void
97_thread_jumps(jit_state_t *_jit);
98
99#define sequential_labels() _sequential_labels(_jit)
100static void
101_sequential_labels(jit_state_t *_jit);
102
103#define split_branches() _split_branches(_jit)
104static void
105_split_branches(jit_state_t *_jit);
106
107#define shortcut_jump(prev, node) _shortcut_jump(_jit, prev, node)
108static jit_bool_t
109_shortcut_jump(jit_state_t *_jit, jit_node_t *prev, jit_node_t *node);
110
111#define redundant_jump(prev, node) _redundant_jump(_jit, prev, node)
112static jit_bool_t
113_redundant_jump(jit_state_t *_jit, jit_node_t *prev, jit_node_t *node);
114
115static jit_code_t
116reverse_jump_code(jit_code_t code);
117
118#define reverse_jump(prev, node) _reverse_jump(_jit, prev, node)
119static jit_bool_t
120_reverse_jump(jit_state_t *_jit, jit_node_t *prev, jit_node_t *node);
121
122#define redundant_store(node, jump) _redundant_store(_jit, node, jump)
ba3814c1 123static jit_bool_t
4a71579b
PC		 124_redundant_store(jit_state_t *_jit, jit_node_t *node, jit_bool_t jump);
125
126#define simplify_movr(p, n, k, s) _simplify_movr(_jit, p, n, k, s)
127static jit_bool_t
128_simplify_movr(jit_state_t *_jit, jit_node_t *prev, jit_node_t *node,
129 jit_int32_t kind, jit_int32_t size);
130
131#define simplify_movi(p, n, k, s) _simplify_movi(_jit, p, n, k, s)
132static jit_bool_t
133_simplify_movi(jit_state_t *_jit, jit_node_t *prev, jit_node_t *node,
134 jit_int32_t kind, jit_int32_t size);
135
136#define simplify_ldxi(prev, node) _simplify_ldxi(_jit, prev, node)
137static jit_bool_t
138_simplify_ldxi(jit_state_t *_jit, jit_node_t *prev, jit_node_t *node);
139
140#define simplify_stxi(prev, node) _simplify_stxi(_jit, prev, node)
141static jit_bool_t
142_simplify_stxi(jit_state_t *_jit, jit_node_t *prev, jit_node_t *node);
143
144#define simplify_spill(node, regno) _simplify_spill(_jit, node, regno)
145static void
146_simplify_spill(jit_state_t *_jit, jit_node_t *node, jit_int32_t regno);
147
148#define simplify() _simplify(_jit)
ba3814c1 149static jit_bool_t
4a71579b
PC		 150_simplify(jit_state_t *_jit);
151
152#define jit_reg_undef -1
153#define jit_reg_static 0
154#define jit_reg_change 1
155#define register_change_p(n, l, r) _register_change_p(_jit, n, l, r)
156static jit_int32_t
157_register_change_p(jit_state_t *_jit, jit_node_t *node, jit_node_t *link,
158 jit_int32_t regno);
159
160#define spill_reglive_p(node, regno) _spill_reglive_p(_jit, node, regno)
161static jit_bool_t
162_spill_reglive_p(jit_state_t *_jit, jit_node_t *node, jit_int32_t regno);
163
164#define patch_registers() _patch_registers(_jit)
165static void
166_patch_registers(jit_state_t *_jit);
167
168#define patch_register(n,l,r,p) _patch_register(_jit,n,l,r,p)
169static void
170_patch_register(jit_state_t *jit, jit_node_t *node, jit_node_t *link,
171 jit_int32_t regno, jit_int32_t patch);
172
173/*
174 * Initialization
175 */
176#if !defined(__sgi)
177#define mmap_fd -1
178#endif
179
180/*
181 * Implementation
182 */
183void
184init_jit(const char *progname)
185{
186 jit_get_cpu();
187 jit_init_debug(progname);
188 jit_init_size();
189}
190
191void
192finish_jit(void)
193{
194 jit_finish_debug();
195 jit_finish_size();
196}
197
198jit_int32_t
199_jit_get_reg(jit_state_t *_jit, jit_int32_t regspec)
200{
201 jit_int32_t spec;
202 jit_int32_t regno;
203
204 spec = regspec & ~(jit_class_chk|jit_class_nospill);
205 if (spec & jit_class_named) {
206 regno = jit_regno(spec);
207 if (jit_regset_tstbit(&_jitc->regsav, regno))
208 /* fail if register is spilled */
209 goto fail;
210 if (jit_regset_tstbit(&_jitc->regarg, regno))
211 /* fail if register is an argument to current instruction */
212 goto fail;
213 if (jit_regset_tstbit(&_jitc->reglive, regno)) {
214 if (regspec & jit_class_nospill)
215 /* fail if register is live and should not spill/reload */
216 goto fail;
217 goto spill;
218 }
219 jit_regset_setbit(&_jitc->regarg, regno);
220 return (regno);
221 }
222 else
223 assert(jit_class(spec) != 0);
224
225 if (_jitc->emit) {
226 /* search for a free register matching spec */
227 for (regno = 0; regno < _jitc->reglen; regno++) {
228 if ((jit_class(_rvs[regno].spec) & spec) == spec &&
229 !jit_regset_tstbit(&_jitc->regarg, regno) &&
79bfeef6
PC		 230 !jit_regset_tstbit(&_jitc->reglive, regno)) {
231 if (jit_regset_tstbit(&_jitc->regmask, regno)) {
232 /* search further, attempting to find a truly known
233 * free register, not just one in unknown state. */
234 jit_int32_t regfree;
235
236 for (regfree = regno + 1;
237 regfree < _jitc->reglen; regfree++) {
238 if ((jit_class(_rvs[regfree].spec) & spec) == spec &&
239 !jit_regset_tstbit(&_jitc->regarg, regfree) &&
240 !jit_regset_tstbit(&_jitc->reglive, regfree) &&
241 !jit_regset_tstbit(&_jitc->regmask, regfree)) {
242 regno = regfree;
243 break;
244 }
245 }
246 }
4a71579b 247 goto regarg;
79bfeef6 248 }
4a71579b
PC		 249 }
250
251 /* search for a register matching spec that is not an argument
252 * for the current instruction */
253 for (regno = 0; regno < _jitc->reglen; regno++) {
254 if ((jit_class(_rvs[regno].spec) & spec) == spec &&
255 !jit_regset_tstbit(&_jitc->regsav, regno) &&
256 !jit_regset_tstbit(&_jitc->regarg, regno) &&
257 !(regspec & jit_class_nospill)) {
258 spill:
259 assert(_jitc->function != NULL);
260 if (spec & jit_class_gpr) {
261 if (!_jitc->function->regoff[regno]) {
262 _jitc->function->regoff[regno] =
263 jit_allocai(sizeof(jit_word_t));
264 _jitc->again = 1;
265 }
266#if DEBUG
267 /* emit_stxi must not need temporary registers */
268 assert(!_jitc->getreg);
269 _jitc->getreg = 1;
270#endif
271 emit_stxi(_jitc->function->regoff[regno], JIT_FP, regno);
272#if DEBUG
273 _jitc->getreg = 0;
274#endif
275 }
276 else {
277 if (!_jitc->function->regoff[regno]) {
278 _jitc->function->regoff[regno] =
279 jit_allocai(sizeof(jit_float64_t));
280 _jitc->again = 1;
281 }
282#if DEBUG
283 /* emit_stxi must not need temporary registers */
284 assert(!_jitc->getreg);
285 _jitc->getreg = 1;
286#endif
287 emit_stxi_d(_jitc->function->regoff[regno], JIT_FP, regno);
288#if DEBUG
289 _jitc->getreg = 0;
290#endif
291 }
292 jit_regset_setbit(&_jitc->regsav, regno);
293 regarg:
294 jit_regset_setbit(&_jitc->regarg, regno);
295 if (jit_class(_rvs[regno].spec) & jit_class_sav) {
296 /* if will modify callee save registers without a
297 * function prolog, better patch this assertion */
298 assert(_jitc->function != NULL);
299 if (!jit_regset_tstbit(&_jitc->function->regset, regno)) {
300 jit_regset_setbit(&_jitc->function->regset, regno);
301 _jitc->again = 1;
302 }
303 }
304 return (regno);
305 }
306 }
307 }
308 else {
309 /* nospill hint only valid during emit" */
310 assert(!(regspec & jit_class_nospill));
311 for (regno = 0; regno < _jitc->reglen; regno++) {
312 if ((jit_class(_rvs[regno].spec) & spec) == spec &&
313 !jit_regset_tstbit(&_jitc->regsav, regno) &&
314 !jit_regset_tstbit(&_jitc->regarg, regno)) {
315 jit_regset_setbit(&_jitc->regarg, regno);
316 jit_regset_setbit(&_jitc->regsav, regno);
317 jit_save(regno);
318 return (jit_regno_patch|regno);
319 }
320 }
321 }
322
323 /* Out of hardware registers */
324fail:
325 assert(regspec & jit_class_chk);
326 return (JIT_NOREG);
327}
328
329void
330_jit_unget_reg(jit_state_t *_jit, jit_int32_t regno)
331{
332 regno = jit_regno(regno);
333 if (jit_regset_tstbit(&_jitc->regsav, regno)) {
334 if (_jitc->emit) {
335#if DEBUG
336 /* emit_ldxi must not need a temporary register */
337 assert(!_jitc->getreg);
338 _jitc->getreg = 1;
339#endif
340 if (jit_class(_rvs[regno].spec) & jit_class_gpr)
341 emit_ldxi(regno, JIT_FP, _jitc->function->regoff[regno]);
342 else
343 emit_ldxi_d(regno, JIT_FP, _jitc->function->regoff[regno]);
344#if DEBUG
345 /* emit_ldxi must not need a temporary register */
346 _jitc->getreg = 0;
347#endif
348 }
349 else
350 jit_load(regno);
351 jit_regset_clrbit(&_jitc->regsav, regno);
352 }
353#if defined(jit_carry)
354 assert((regno == jit_carry /*&& _NOREG != jit_carry*/) ||
355 jit_regset_tstbit(&_jitc->regarg, regno) != 0);
356#else
357 assert(jit_regset_tstbit(&_jitc->regarg, regno) != 0);
358#endif
359 jit_regset_clrbit(&_jitc->regarg, regno);
360}
361
362jit_bool_t
363_jit_callee_save_p(jit_state_t *_jit, jit_int32_t regno)
364{
365 assert(regno >= 0 && regno < JIT_NOREG);
366 return (!!(_rvs[regno].spec & jit_class_sav));
367}
368
369extern jit_bool_t
370_jit_pointer_p(jit_state_t *_jit, jit_pointer_t address)
371{
372 return ((jit_uint8_t *)address >= _jit->code.ptr &&
373 (jit_word_t)address < _jit->pc.w);
374}
375
376#if __ia64__
377void
378jit_regset_com(jit_regset_t *u, jit_regset_t *v)
379{
380 u->rl = ~v->rl; u->rh = ~v->rh;
381 u->fl = ~v->fl; u->fh = ~v->fh;
382}
383
384void
385jit_regset_and(jit_regset_t *u, jit_regset_t *v, jit_regset_t *w)
386{
387 u->rl = v->rl & w->rl; u->rh = v->rh & w->rh;
388 u->fl = v->fl & w->fl; u->fh = v->fh & w->fh;
389}
390
391void
392jit_regset_ior(jit_regset_t *u, jit_regset_t *v, jit_regset_t *w)
393{
394 u->rl = v->rl | w->rl; u->rh = v->rh | w->rh;
395 u->fl = v->fl | w->fl; u->fh = v->fh | w->fh;
396}
397
398void
399jit_regset_xor(jit_regset_t *u, jit_regset_t *v, jit_regset_t *w)
400{
401 u->rl = v->rl ^ w->rl; u->rh = v->rh ^ w->rh;
402 u->fl = v->fl ^ w->fl; u->fh = v->fh ^ w->fh;
403}
404
405void
406jit_regset_set(jit_regset_t *u, jit_regset_t *v)
407{
408 u->rl = v->rl; u->rh = v->rh;
409 u->fl = v->fl; u->fh = v->fh;
410}
411
412void
413jit_regset_set_mask(jit_regset_t *u, jit_int32_t v)
414{
415 jit_bool_t w = !!(v & (v - 1));
416
417 assert(v >= 0 && v <= 256);
418 if (v == 0)
419 u->rl = u->rh = u->fl = u->fh = -1LL;
420 else if (v <= 64) {
421 u->rl = w ? (1LL << v) - 1 : -1LL;
422 u->rh = u->fl = u->fh = 0;
423 }
424 else if (v <= 128) {
425 u->rl = -1LL;
426 u->rh = w ? (1LL << (v - 64)) - 1 : -1LL;
427 u->fl = u->fh = 0;
428 }
429 else if (v <= 192) {
430 u->rl = u->rh = -1LL;
431 u->fl = w ? (1LL << (v - 128)) - 1 : -1LL;
432 u->fh = 0;
433 }
434 else {
435 u->rl = u->rh = u->fl = -1LL;
436 u->fh = w ? (1LL << (v - 128)) - 1 : -1LL;
437 }
438}
439
440jit_bool_t
441jit_regset_cmp_ui(jit_regset_t *u, jit_word_t v)
442{
443 return !((u->rl == v && u->rh == 0 && u->fl == 0 && u->fh == 0));
444}
445
446void
447jit_regset_set_ui(jit_regset_t *u, jit_word_t v)
448{
449 u->rl = v;
450 u->rh = u->fl = u->fh = 0;
451}
452
453jit_bool_t
454jit_regset_set_p(jit_regset_t *u)
455{
456 return (u->rl || u->rh || u->fl || u->fh);
457}
458
459void
460jit_regset_clrbit(jit_regset_t *set, jit_int32_t bit)
461{
462 assert(bit >= 0 && bit <= 255);
463 if (bit < 64)
464 set->rl &= ~(1LL << bit);
465 else if (bit < 128)
466 set->rh &= ~(1LL << (bit - 64));
467 else if (bit < 192)
468 set->fl &= ~(1LL << (bit - 128));
469 else
470 set->fh &= ~(1LL << (bit - 192));
471}
472
473void
474jit_regset_setbit(jit_regset_t *set, jit_int32_t bit)
475{
476 assert(bit >= 0 && bit <= 255);
477 if (bit < 64)
478 set->rl |= 1LL << bit;
479 else if (bit < 128)
480 set->rh |= 1LL << (bit - 64);
481 else if (bit < 192)
482 set->fl |= 1LL << (bit - 128);
483 else
484 set->fh |= 1LL << (bit - 192);
485}
486
487jit_bool_t
488jit_regset_tstbit(jit_regset_t *set, jit_int32_t bit)
489{
490 assert(bit >= 0 && bit <= 255);
491 if (bit < 64)
492 return (!!(set->rl & (1LL << bit)));
493 else if (bit < 128)
494 return (!!(set->rh & (1LL << (bit - 64))));
495 else if (bit < 192)
496 return (!!(set->fl & (1LL << (bit - 128))));
497 return (!!(set->fh & (1LL << (bit - 192))));
498}
499
500unsigned long
501jit_regset_scan1(jit_regset_t *set, jit_int32_t offset)
502{
503 assert(offset >= 0 && offset <= 255);
504 for (; offset < 64; offset++) {
505 if (set->rl & (1LL << offset))
506 return (offset);
507 }
508 for (; offset < 128; offset++) {
509 if (set->rh & (1LL << (offset - 64)))
510 return (offset);
511 }
512 for (; offset < 192; offset++) {
513 if (set->fl & (1LL << (offset - 128)))
514 return (offset);
515 }
516 for (; offset < 256; offset++) {
517 if (set->fh & (1LL << (offset - 192)))
518 return (offset);
519 }
520 return (ULONG_MAX);
521}
522
523#elif __sparc__ && __WORDSIZE == 64
524void
525jit_regset_com(jit_regset_t *u, jit_regset_t *v)
526{
527 u->rl = ~v->rl; u->rh = ~v->rh;
528}
529
530void
531jit_regset_and(jit_regset_t *u, jit_regset_t *v, jit_regset_t *w)
532{
533 u->rl = v->rl & w->rl; u->rh = v->rh & w->rh;
534}
535
536void
537jit_regset_ior(jit_regset_t *u, jit_regset_t *v, jit_regset_t *w)
538{
539 u->rl = v->rl | w->rl; u->rh = v->rh | w->rh;
540}
541
542void
543jit_regset_xor(jit_regset_t *u, jit_regset_t *v, jit_regset_t *w)
544{
545 u->rl = v->rl ^ w->rl; u->rh = v->rh ^ w->rh;
546}
547
548void
549jit_regset_set(jit_regset_t *u, jit_regset_t *v)
550{
551 u->rl = v->rl; u->rh = v->rh;
552}
553
554void
555jit_regset_set_mask(jit_regset_t *u, jit_int32_t v)
556{
557 jit_bool_t w = !!(v & (v - 1));
558
559 assert(v >= 0 && v <= 128);
560 if (v == 0)
561 u->rl = u->rh = -1LL;
562 else if (v <= 64) {
563 u->rl = w ? (1LL << v) - 1 : -1LL;
564 u->rh = 0;
565 }
566 else {
567 u->rl = -1LL;
568 u->rh = w ? (1LL << (v - 64)) - 1 : -1LL;
569 }
570}
571
572jit_bool_t
573jit_regset_cmp_ui(jit_regset_t *u, jit_word_t v)
574{
575 return !((u->rl == v && u->rh == 0));
576}
577
578void
579jit_regset_set_ui(jit_regset_t *u, jit_word_t v)
580{
581 u->rl = v;
582 u->rh = 0;
583}
584
585jit_bool_t
586jit_regset_set_p(jit_regset_t *u)
587{
588 return (u->rl || u->rh);
589}
590
591void
592jit_regset_clrbit(jit_regset_t *set, jit_int32_t bit)
593{
594 assert(bit >= 0 && bit <= 128);
595 if (bit < 64)
596 set->rl &= ~(1LL << bit);
597 else
598 set->rh &= ~(1LL << (bit - 64));
599}
600
601void
602jit_regset_setbit(jit_regset_t *set, jit_int32_t bit)
603{
604 assert(bit >= 0 && bit <= 127);
605 if (bit < 64)
606 set->rl |= 1LL << bit;
607 else
608 set->rh |= 1LL << (bit - 64);
609}
610
611jit_bool_t
612jit_regset_tstbit(jit_regset_t *set, jit_int32_t bit)
613{
614 assert(bit >= 0 && bit <= 127);
615 if (bit < 64)
616 return (!!(set->rl & (1LL << bit)));
617 else
618 return (!!(set->rh & (1LL << (bit - 64))));
619}
620
621unsigned long
622jit_regset_scan1(jit_regset_t *set, jit_int32_t offset)
623{
624 assert(offset >= 0 && offset <= 127);
625 for (; offset < 64; offset++) {
626 if (set->rl & (1LL << offset))
627 return (offset);
628 }
629 for (; offset < 128; offset++) {
630 if (set->rh & (1LL << (offset - 64)))
631 return (offset);
632 }
633 return (ULONG_MAX);
634}
635
636#else
637unsigned long
638jit_regset_scan1(jit_regset_t *set, jit_int32_t offset)
639{
640 jit_regset_t mask;
641 assert(offset >= 0 && offset <= 63);
642 if ((mask = *set >> offset)) {
643 for (;;) {
644 if (mask & 1)
645 return (offset);
646 mask >>= 1;
647 ++offset;
648 }
649 }
650 return (ULONG_MAX);
651}
652#endif
653
654void
655_jit_save(jit_state_t *_jit, jit_int32_t reg)
656{
657 reg = jit_regno(reg);
658 assert(!_jitc->realize);
659 _jitc->spill[reg] = jit_new_node_w(jit_code_save, reg);
660}
661
662void
663_jit_load(jit_state_t *_jit, jit_int32_t reg)
664{
665 jit_node_t *node;
666
667 reg = jit_regno(reg);
668 assert(!_jitc->realize);
669 assert(_jitc->spill[reg] != NULL);
670 node = jit_new_node_w(jit_code_load, reg);
671 /* create a path to flag the save/load is not required */
672 node->link = _jitc->spill[reg];
673 node->link->link = node;
674 _jitc->spill[reg] = NULL;
675}
676
677static jit_word_t
678hash_data(const void *data, jit_word_t length)
679{
680 const jit_uint8_t *ptr;
681 jit_word_t i, key;
682 for (i = key = 0, ptr = data; i < length; i++)
683 key = (key << (key & 1)) ^ ptr[i];
684 return (key);
685}
686
687jit_pointer_t
688_jit_address(jit_state_t *_jit, jit_node_t *node)
689{
690 assert(_jitc->done);
691 assert(node != NULL &&
692 /* If a node type that is documented to be a fixed marker */
693 (node->code == jit_code_note || node->code == jit_code_name ||
694 /* If another special fixed marker, returned by jit_indirect() */
695 (node->code == jit_code_label && (node->flag & jit_flag_use) != 0)));
696 return ((jit_pointer_t)node->u.w);
697}
698
699jit_node_t *
700_jit_data(jit_state_t *_jit, const void *data,
701 jit_word_t length, jit_int32_t align)
702{
703 jit_word_t key;
704 jit_node_t *node;
705
706 assert(!_jitc->realize);
707
708 /* Ensure there is space even if asking for a duplicate */
709 if (((_jitc->data.offset + 7) & -8) + length > _jit->data.length) {
710 jit_word_t size;
711
712 size = (_jit->data.length + length + 4096) & - 4095;
713 assert(size >= _jit->data.length);
714 if (_jitc->data.ptr == NULL)
715 jit_alloc((jit_pointer_t *)&_jitc->data.ptr, size);
716 else
717 jit_realloc((jit_pointer_t *)&_jitc->data.ptr,
718 _jit->data.length, size);
719 _jit->data.length = size;
720 }
721 if (_jitc->data.table == NULL)
722 jit_alloc((jit_pointer_t *)&_jitc->data.table,
723 (_jitc->data.size = 16) * sizeof(jit_node_t*));
724
725 key = hash_data(data, length) & (_jitc->data.size - 1);
726 node = _jitc->data.table[key];
727 for (; node; node = node->next) {
728 if (node->v.w == length &&
729 memcmp(_jitc->data.ptr + node->u.w, data, length) == 0)
730 break;
731 }
732
733 if (!node) {
734 node = jit_new_node_no_link(jit_code_data);
735 if (!align)
736 align = length;
737 switch (align) {
738 case 0: case 1:
739 break;
740 case 2:
741 _jitc->data.offset = (_jitc->data.offset + 1) & -2;
742 break;
743 case 3: case 4:
744 _jitc->data.offset = (_jitc->data.offset + 3) & -4;
745 break;
746 default:
747 _jitc->data.offset = (_jitc->data.offset + 7) & -8;
748 break;
749 }
750 node->u.w = _jitc->data.offset;
751 node->v.w = length;
752 jit_memcpy(_jitc->data.ptr + _jitc->data.offset, data, length);
753 _jitc->data.offset += length;
754
755 node->next = _jitc->data.table[key];
756 _jitc->data.table[key] = node;
757 ++_jitc->data.count;
758
759 /* Rehash if more than 75% used table */
760 if (_jitc->data.count >
761 (_jitc->data.size >> 1) + (_jitc->data.size >> 2) &&
762 (_jitc->data.size << 1) > _jitc->data.size) {
763 jit_word_t i;
764 jit_node_t **hash;
765 jit_node_t *next;
766 jit_node_t *temp;
767
768 jit_alloc((jit_pointer_t *)&hash,
769 (_jitc->data.size << 1) * sizeof(jit_node_t*));
770 for (i = 0; i < _jitc->data.size; i++) {
771 temp = _jitc->data.table[i];
772 for (; temp; temp = next) {
773 next = temp->next;
774 key = hash_data(_jitc->data.ptr + temp->u.w, temp->v.w) &
775 ((_jitc->data.size << 1) - 1);
776 temp->next = hash[key];
777 hash[key] = temp;
778 }
779 }
780 jit_free((jit_pointer_t *)&_jitc->data.table);
781 _jitc->data.table = hash;
782 _jitc->data.size <<= 1;
783 }
784 }
785
786 return (node);
787}
788
789static void
790_new_pool(jit_state_t *_jit)
791{
792 jit_node_t *list;
793 jit_int32_t offset;
794
795 if (_jitc->pool.offset >= _jitc->pool.length) {
796 jit_int32_t length;
797
798 length = _jitc->pool.length + 16;
799 jit_realloc((jit_pointer_t *)&_jitc->pool.ptr,
800 _jitc->pool.length * sizeof(jit_node_t *),
801 length * sizeof(jit_node_t *));
802 _jitc->pool.length = length;
803 }
804 jit_alloc((jit_pointer_t *)(_jitc->pool.ptr + _jitc->pool.offset),
805 sizeof(jit_node_t) * 1024);
806 list = _jitc->pool.ptr[_jitc->pool.offset];
807 for (offset = 1; offset < 1024; offset++, list++)
808 list->next = list + 1;
809 list->next = _jitc->list;
810 _jitc->list = _jitc->pool.ptr[_jitc->pool.offset];
811 ++_jitc->pool.offset;
812}
813
814static jit_node_t *
815_new_node(jit_state_t *_jit, jit_code_t code)
816{
817 jit_node_t *node;
818
819 if (_jitc->list == NULL)
820 new_pool();
821 node = _jitc->list;
822 _jitc->list = node->next;
823 if (_jitc->synth)
824 node->flag |= jit_flag_synth;
825 node->next = NULL;
826 node->code = code;
827
828 return (node);
829}
830
831static inline jit_node_t *
832_link_node(jit_state_t *_jit, jit_node_t *node)
833{
834 if (_jitc->tail)
835 _jitc->tail->next = node;
836 else
837 _jitc->head = node;
838 return (_jitc->tail = node);
839}
840
841static inline void
842_del_node(jit_state_t *_jit, jit_node_t *prev, jit_node_t *node)
843{
844 if (prev == node) {
845 assert(prev == _jitc->head);
846 _jitc->head = node->next;
847 }
848 else
849 prev->next = node->next;
850 memset(node, 0, sizeof(jit_node_t));
851 node->next = _jitc->list;
852 _jitc->list = node;
853}
854
855static inline void
856_free_node(jit_state_t *_jit, jit_node_t *node)
857{
858 memset(node, 0, sizeof(jit_node_t));
859 node->next = _jitc->list;
860 _jitc->list = node;
861}
862
863static void
864_del_label(jit_state_t *_jit, jit_node_t *prev, jit_node_t *node)
865{
866 jit_block_t *block;
867
868 /* only allow call to del_label on linked labels */
869 block = _jitc->blocks.ptr + node->v.w;
870 assert(block->label == node);
871
872 /* del_label() should only be called when optimizing.
873 * This will leave an empty block index */
874 jit_regset_del(&block->reglive);
875 jit_regset_del(&block->regmask);
876 block->label = NULL;
877
878 /* redundant, should be already true */
879 assert(node->link == NULL);
880 del_node(prev, node);
881}
882
883jit_state_t *
884jit_new_state(void)
885{
886 jit_state_t *_jit;
887
888 jit_alloc((jit_pointer_t *)&_jit, sizeof(jit_state_t));
889 jit_alloc((jit_pointer_t *)&_jitc, sizeof(jit_compiler_t));
890 jit_regset_new(&_jitc->regarg);
891 jit_regset_new(&_jitc->regsav);
892 jit_regset_new(&_jitc->reglive);
893 jit_regset_new(&_jitc->regmask);
79bfeef6 894 jit_regset_new(&_jitc->explive);
4a71579b
PC		 895
896 jit_init();
897
898 jit_alloc((jit_pointer_t *)&_jitc->spill,
899 _jitc->reglen * sizeof(jit_node_t*));
900 jit_alloc((jit_pointer_t *)&_jitc->gen,
901 _jitc->reglen * sizeof(jit_int32_t));
902 jit_alloc((jit_pointer_t *)&_jitc->values,
903 _jitc->reglen * sizeof(jit_value_t));
904
905 jit_alloc((jit_pointer_t *)&_jitc->patches.ptr,
906 (_jitc->patches.length = 1024) * sizeof(jit_patch_t));
907 jit_alloc((jit_pointer_t *)&_jitc->functions.ptr,
908 (_jitc->functions.length = 16) * sizeof(jit_function_t));
909 jit_alloc((jit_pointer_t *)&_jitc->pool.ptr,
910 (_jitc->pool.length = 16) * sizeof(jit_node_t*));
911 jit_alloc((jit_pointer_t *)&_jitc->blocks.ptr,
912 (_jitc->blocks.length = 16) * sizeof(jit_block_t));
913#if __arm__ && DISASSEMBLER
914 jit_alloc((jit_pointer_t *)&_jitc->data_info.ptr,
915 (_jitc->data_info.length = 1024) * sizeof(jit_data_info_t));
916#endif
917
918 /* allocate at most one extra note in case jit_name() is
919 * never called, or called after adding at least one note */
920 _jit->note.length = 1;
921 _jitc->note.size = sizeof(jit_note_t);
922
923 return (_jit);
924}
925
926void
927_jit_clear_state(jit_state_t *_jit)
928{
929#if DEVEL_DISASSEMBLER
930# define jit_really_clear_state() _jit_really_clear_state(_jit)
931}
932
933void _jit_really_clear_state(jit_state_t *_jit)
934{
935#endif
936 jit_word_t offset;
937 jit_function_t *function;
938
939 /* release memory not required at jit execution time and set
940 * pointers to NULL to explicitly know they are released */
941 _jitc->head = _jitc->tail = NULL;
942
943 jit_free((jit_pointer_t *)&_jitc->data.table);
944 _jitc->data.size = _jitc->data.count = 0;
945
946 jit_free((jit_pointer_t *)&_jitc->spill);
947 jit_free((jit_pointer_t *)&_jitc->gen);
948 jit_free((jit_pointer_t *)&_jitc->values);
949
950 jit_free((jit_pointer_t *)&_jitc->blocks.ptr);
951
952 jit_free((jit_pointer_t *)&_jitc->patches.ptr);
953 _jitc->patches.offset = _jitc->patches.length = 0;
954
955 for (offset = 0; offset < _jitc->functions.offset; offset++) {
956 function = _jitc->functions.ptr + offset;
957 jit_free((jit_pointer_t *)&function->regoff);
958 }
959 jit_free((jit_pointer_t *)&_jitc->functions.ptr);
960 _jitc->functions.offset = _jitc->functions.length = 0;
961 _jitc->function = NULL;
962
963 for (offset = 0; offset < _jitc->pool.offset; offset++)
964 jit_free((jit_pointer_t *)(_jitc->pool.ptr + offset));
965 jit_free((jit_pointer_t *)&_jitc->pool.ptr);
966 _jitc->pool.offset = _jitc->pool.length = 0;
967 _jitc->list = NULL;
968
969 _jitc->note.head = _jitc->note.tail =
970 _jitc->note.name = _jitc->note.note = NULL;
971 _jitc->note.base = NULL;
972
973#if __arm__ && DISASSEMBLER
974 jit_free((jit_pointer_t *)&_jitc->data_info.ptr);
975#endif
976
977#if (__powerpc__ && _CALL_AIXDESC) || __ia64__
978 jit_free((jit_pointer_t *)&_jitc->prolog.ptr);
979#endif
980
981#if __ia64__
982 jit_regset_del(&_jitc->regs);
983#endif
984
985 jit_free((jit_pointer_t *)&_jitc);
986}
987
988void
989_jit_destroy_state(jit_state_t *_jit)
990{
991#if DEVEL_DISASSEMBLER
992 jit_really_clear_state();
993#endif
40a44dcb 994#if HAVE_MMAP
4a71579b
PC		 995 if (!_jit->user_code)
996 munmap(_jit->code.ptr, _jit->code.length);
997 if (!_jit->user_data)
998 munmap(_jit->data.ptr, _jit->data.length);
40a44dcb 999#endif
4a71579b
PC		 1000 jit_free((jit_pointer_t *)&_jit);
1001}
1002
1003void
1004_jit_synth_inc(jit_state_t *_jit)
1005{
1006 assert(_jitc->synth < 8);
1007 ++_jitc->synth;
1008}
1009
1010jit_node_t *
1011_jit_new_node(jit_state_t *_jit, jit_code_t code)
1012{
1013 assert(!_jitc->realize);
1014 return (link_node(new_node(code)));
1015}
1016
1017jit_node_t *
1018_jit_new_node_no_link(jit_state_t *_jit, jit_code_t code)
1019{
1020 assert(!_jitc->realize);
1021 return (new_node(code));
1022}
1023
1024void
1025_jit_link_node(jit_state_t *_jit, jit_node_t *node)
1026{
1027 assert(!_jitc->realize);
1028 link_node(node);
1029}
1030
1031void
1032_jit_synth_dec(jit_state_t *_jit)
1033{
1034 assert(_jitc->synth > 0);
1035 --_jitc->synth;
1036}
1037
1038jit_node_t *
1039_jit_new_node_w(jit_state_t *_jit, jit_code_t code,
1040 jit_word_t u)
1041{
1042 jit_node_t *node = new_node(code);
1043 assert(!_jitc->realize);
1044 node->u.w = u;
1045 return (link_node(node));
1046}
1047
1048jit_node_t *
1049_jit_new_node_f(jit_state_t *_jit, jit_code_t code,
1050 jit_float32_t u)
1051{
1052 jit_node_t *node = new_node(code);
1053 assert(!_jitc->realize);
1054 node->u.f = u;
1055 return (link_node(node));
1056}
1057
1058jit_node_t *
1059_jit_new_node_d(jit_state_t *_jit, jit_code_t code,
1060 jit_float64_t u)
1061{
1062 jit_node_t *node = new_node(code);
1063 assert(!_jitc->realize);
1064 node->u.d = u;
1065 return (link_node(node));
1066}
1067
1068jit_node_t *
1069_jit_new_node_p(jit_state_t *_jit, jit_code_t code,
1070 jit_pointer_t u)
1071{
1072 jit_node_t *node = new_node(code);
1073 assert(!_jitc->realize);
1074 node->u.p = u;
1075 return (link_node(node));
1076}
1077
1078jit_node_t *
1079_jit_new_node_ww(jit_state_t *_jit, jit_code_t code,
1080 jit_word_t u, jit_word_t v)
1081{
1082 jit_node_t *node = new_node(code);
1083 assert(!_jitc->realize);
1084 node->u.w = u;
1085 node->v.w = v;
1086 return (link_node(node));
1087}
1088
1089jit_node_t *
1090_jit_new_node_wp(jit_state_t *_jit, jit_code_t code,
1091 jit_word_t u, jit_pointer_t v)
1092{
1093 return (jit_new_node_ww(code, u, (jit_word_t)v));
1094}
1095
1096jit_node_t *
1097_jit_new_node_fp(jit_state_t *_jit, jit_code_t code,
1098 jit_float32_t u, jit_pointer_t v)
1099{
1100 jit_node_t *node = new_node(code);
1101 assert(!_jitc->realize);
1102 node->u.f = u;
1103 node->v.w = (jit_word_t)v;
1104 return (link_node(node));
1105}
1106
1107jit_node_t *
1108_jit_new_node_dp(jit_state_t *_jit, jit_code_t code,
1109 jit_float64_t u, jit_pointer_t v)
1110{
1111 jit_node_t *node = new_node(code);
1112 assert(!_jitc->realize);
1113 node->u.d = u;
1114 node->v.w = (jit_word_t)v;
1115 return (link_node(node));
1116}
1117
1118jit_node_t *
1119_jit_new_node_pw(jit_state_t *_jit, jit_code_t code,
1120 jit_pointer_t u, jit_word_t v)
1121{
1122 return (jit_new_node_ww(code, (jit_word_t)u, v));
1123}
1124
1125jit_node_t *
1126_jit_new_node_wf(jit_state_t *_jit, jit_code_t code,
1127 jit_word_t u, jit_float32_t v)
1128{
1129 jit_node_t *node = new_node(code);
1130 assert(!_jitc->realize);
1131 node->u.w = u;
1132 node->v.f = v;
1133 return (link_node(node));
1134}
1135
1136jit_node_t *
1137_jit_new_node_wd(jit_state_t *_jit, jit_code_t code,
1138 jit_word_t u, jit_float64_t v)
1139{
1140 jit_node_t *node = new_node(code);
1141 assert(!_jitc->realize);
1142 node->u.w = u;
1143 node->v.d = v;
1144 return (link_node(node));
1145}
1146
1147jit_node_t *
1148_jit_new_node_www(jit_state_t *_jit, jit_code_t code,
1149 jit_word_t u, jit_word_t v, jit_word_t w)
1150{
1151 jit_node_t *node = new_node(code);
1152 assert(!_jitc->realize);
1153 node->u.w = u;
1154 node->v.w = v;
1155 node->w.w = w;
1156 return (link_node(node));
1157}
1158
1159jit_node_t *
1160_jit_new_node_qww(jit_state_t *_jit, jit_code_t code,
1161 jit_int32_t l, jit_int32_t h,
1162 jit_word_t v, jit_word_t w)
1163{
1164 jit_node_t *node = new_node(code);
1165 assert(!_jitc->realize);
1166 assert(l != h);
1167 node->u.q.l = l;
1168 node->u.q.h = h;
1169 node->v.w = v;
1170 node->w.w = w;
1171 return (link_node(node));
1172}
1173
ba86ff93
PC		 1174jit_node_t *
1175_jit_new_node_wqw(jit_state_t *_jit, jit_code_t code,
1176 jit_word_t u, jit_int32_t l,
1177 jit_int32_t h, jit_word_t w)
1178{
1179 jit_node_t *node = new_node(code);
1180 assert(!_jitc->realize);
1181 assert(l != h);
1182 node->u.w = u;
1183 node->v.q.l = l;
1184 node->v.q.h = h;
1185 node->w.w = w;
1186 return (link_node(node));
1187}
1188
ba3814c1
PC		 1189jit_node_t *
1190_jit_new_node_wwq(jit_state_t *_jit, jit_code_t code,
1191 jit_word_t u, jit_word_t v,
1192 jit_int32_t l, jit_int32_t h)
1193{
1194 jit_node_t *node = new_node(code);
1195 assert(!_jitc->realize);
1196 node->u.w = u;
1197 node->v.w = v;
1198 node->w.q.l = l;
1199 node->w.q.h = h;
1200 return (link_node(node));
1201}
1202
4a71579b
PC		 1203jit_node_t *
1204_jit_new_node_wwf(jit_state_t *_jit, jit_code_t code,
1205 jit_word_t u, jit_word_t v, jit_float32_t w)
1206{
1207 jit_node_t *node = new_node(code);
1208 assert(!_jitc->realize);
1209 node->u.w = u;
1210 node->v.w = v;
1211 node->w.f = w;
1212 return (link_node(node));
1213}
1214
ba86ff93
PC		 1215jit_node_t *
1216_jit_new_node_wqf(jit_state_t *_jit, jit_code_t code,
1217 jit_word_t u, jit_int32_t l,
1218 jit_int32_t h, jit_float32_t w)
1219{
1220 jit_node_t *node = new_node(code);
1221 assert(!_jitc->realize);
1222 assert(l != h);
1223 node->u.w = u;
1224 node->v.q.l = l;
1225 node->v.q.h = h;
1226 node->w.f = w;
1227 return (link_node(node));
1228}
1229
4a71579b
PC		 1230jit_node_t *
1231_jit_new_node_wwd(jit_state_t *_jit, jit_code_t code,
1232 jit_word_t u, jit_word_t v, jit_float64_t w)
1233{
1234 jit_node_t *node = new_node(code);
1235 assert(!_jitc->realize);
1236 node->u.w = u;
1237 node->v.w = v;
1238 node->w.d = w;
1239 return (link_node(node));
1240}
1241
ba86ff93
PC		 1242jit_node_t *
1243_jit_new_node_wqd(jit_state_t *_jit, jit_code_t code,
1244 jit_word_t u, jit_int32_t l,
1245 jit_int32_t h, jit_float64_t w)
1246{
1247 jit_node_t *node = new_node(code);
1248 assert(!_jitc->realize);
1249 assert(l != h);
1250 node->u.w = u;
1251 node->v.q.l = l;
1252 node->v.q.h = h;
1253 node->w.d = w;
1254 return (link_node(node));
1255}
1256
4a71579b
PC		 1257jit_node_t *
1258_jit_new_node_pww(jit_state_t *_jit, jit_code_t code,
1259 jit_pointer_t u, jit_word_t v, jit_word_t w)
1260{
1261 jit_node_t *node = new_node(code);
1262 assert(!_jitc->realize);
1263 node->u.p = u;
1264 node->v.w = v;
1265 node->w.w = w;
1266 return (link_node(node));
1267}
1268
1269jit_node_t *
1270_jit_new_node_pwf(jit_state_t *_jit, jit_code_t code,
1271 jit_pointer_t u, jit_word_t v, jit_float32_t w)
1272{
1273 jit_node_t *node = new_node(code);
1274 assert(!_jitc->realize);
1275 node->u.p = u;
1276 node->v.w = v;
1277 node->w.f = w;
1278 return (link_node(node));
1279}
1280
1281jit_node_t *
1282_jit_new_node_pwd(jit_state_t *_jit, jit_code_t code,
1283 jit_pointer_t u, jit_word_t v, jit_float64_t w)
1284{
1285 jit_node_t *node = new_node(code);
1286 assert(!_jitc->realize);
1287 node->u.p = u;
1288 node->v.w = v;
1289 node->w.d = w;
1290 return (link_node(node));
1291}
1292
1293jit_node_t *
1294_jit_label(jit_state_t *_jit)
1295{
1296 jit_node_t *node;
1297
1298 if (!(node = _jitc->tail) || node->code != jit_code_label) {
1299 node = jit_forward();
1300 jit_link(node);
1301 }
1302
1303 return (node);
1304}
1305
1306jit_node_t *
1307_jit_forward(jit_state_t *_jit)
1308{
1309 return (jit_new_node_no_link(jit_code_label));
1310}
1311
1312jit_node_t *
1313_jit_indirect(jit_state_t *_jit)
1314{
1315 jit_node_t *node;
1316
1317 node = jit_label();
1318 node->flag |= jit_flag_use;
1319
1320 return (node);
1321}
1322
1323void
1324_jit_link(jit_state_t *_jit, jit_node_t *node)
1325{
1326 jit_block_t *block;
1327
1328 assert((node->code == jit_code_label ||
1329 node->code == jit_code_prolog ||
1330 node->code == jit_code_epilog) && !node->next);
1331 jit_link_node(node);
1332 if (_jitc->blocks.offset >= _jitc->blocks.length) {
1333 jit_word_t length;
1334
1335 length = _jitc->blocks.length + 16;
1336 jit_realloc((jit_pointer_t *)&_jitc->blocks.ptr,
1337 _jitc->blocks.length * sizeof(jit_block_t),
1338 length * sizeof(jit_block_t));
1339 _jitc->blocks.length = length;
1340 }
1341 block = _jitc->blocks.ptr + _jitc->blocks.offset;
1342 block->label = node;
1343 node->v.w = _jitc->blocks.offset;
1344 jit_regset_new(&block->reglive);
1345 jit_regset_new(&block->regmask);
1346 ++_jitc->blocks.offset;
1347}
1348
1349jit_bool_t
1350_jit_forward_p(jit_state_t *_jit, jit_node_t *node)
1351{
1352 return (node->code == jit_code_label && !node->next && node != _jitc->tail);
1353}
1354
1355jit_bool_t
1356_jit_indirect_p(jit_state_t *_jit, jit_node_t *node)
1357{
1358 return (node->code == jit_code_label && !!(node->flag & jit_flag_use));
1359}
1360
1361jit_bool_t
1362_jit_target_p(jit_state_t *_jit, jit_node_t *node)
1363{
1364 return (node->code == jit_code_label && !!node->link);
1365}
1366
1367void
1368_jit_prepare(jit_state_t *_jit)
1369{
1370 assert(_jitc->function != NULL);
1371 _jitc->function->call.call = jit_call_default;
1372 _jitc->function->call.argi =
1373 _jitc->function->call.argf =
1374 _jitc->function->call.size = 0;
1375 _jitc->prepare = jit_new_node(jit_code_prepare);
1376}
1377
1378void
1379_jit_patch(jit_state_t* _jit, jit_node_t *instr)
1380{
1381 jit_node_t *label;
1382
1383 if (!(label = _jitc->tail) || label->code != jit_code_label)
1384 label = jit_label();
1385 jit_patch_at(instr, label);
1386}
1387
1388jit_int32_t
1389_jit_classify(jit_state_t *_jit, jit_code_t code)
1390{
1391 jit_int32_t mask;
1392
1393 switch (code) {
1394 case jit_code_data: case jit_code_save: case jit_code_load:
1395 case jit_code_name: case jit_code_label: case jit_code_note:
1396 case jit_code_prolog: case jit_code_ellipsis: case jit_code_va_push:
1397 case jit_code_epilog: case jit_code_ret: case jit_code_prepare:
1398 mask = 0;
1399 break;
1400 case jit_code_live: case jit_code_va_end:
79bfeef6
PC		 1401 case jit_code_retr_c: case jit_code_retr_uc:
1402 case jit_code_retr_s: case jit_code_retr_us:
1403 case jit_code_retr_i: case jit_code_retr_ui:
1404 case jit_code_retr_l:
1405 case jit_code_retr_f: case jit_code_retr_d:
1406 case jit_code_pushargr_c:
1407 case jit_code_pushargr_uc:
1408 case jit_code_pushargr_s:
1409 case jit_code_pushargr_us:
1410 case jit_code_pushargr_i:
1411 case jit_code_pushargr_ui:
1412 case jit_code_pushargr_l:
1413 case jit_code_pushargr_f:
4a71579b
PC		 1414 case jit_code_pushargr_d:
1415 case jit_code_finishr: /* synthesized will set jit_cc_a0_jmp */
1416 mask = jit_cc_a0_reg;
1417 break;
79bfeef6
PC		 1418 case jit_code_align: case jit_code_skip:
1419 case jit_code_reti_c: case jit_code_reti_uc:
1420 case jit_code_reti_s: case jit_code_reti_us:
1421 case jit_code_reti_i: case jit_code_reti_ui:
1422 case jit_code_reti_l:
1423 case jit_code_pushargi_c:
1424 case jit_code_pushargi_uc:
1425 case jit_code_pushargi_s:
1426 case jit_code_pushargi_us:
1427 case jit_code_pushargi_i:
1428 case jit_code_pushargi_ui:
1429 case jit_code_pushargi_l:
1430 case jit_code_finishi: /* synthesized will set jit_cc_a0_jmp */
4a71579b
PC		 1431 mask = jit_cc_a0_int;
1432 break;
1433 case jit_code_reti_f: case jit_code_pushargi_f:
1434 mask = jit_cc_a0_flt;
1435 break;
1436 case jit_code_reti_d: case jit_code_pushargi_d:
1437 mask = jit_cc_a0_dbl;
1438 break;
1439 case jit_code_allocai:
1440 mask = jit_cc_a0_int|jit_cc_a1_int;
1441 break;
79bfeef6
PC		 1442 case jit_code_arg_c: case jit_code_arg_s:
1443 case jit_code_arg_i: case jit_code_arg_l:
1444 case jit_code_arg_f: case jit_code_arg_d:
4a71579b
PC		 1445 mask = jit_cc_a0_int|jit_cc_a0_arg;
1446 break;
1447 case jit_code_calli: case jit_code_jmpi:
1448 mask = jit_cc_a0_jmp;
1449 break;
1450 case jit_code_callr: case jit_code_jmpr:
1451 mask = jit_cc_a0_reg|jit_cc_a0_jmp;
1452 break;
1453 case jit_code_retval_c: case jit_code_retval_uc:
1454 case jit_code_retval_s: case jit_code_retval_us:
1455 case jit_code_retval_i: case jit_code_retval_ui:
1456 case jit_code_retval_l:
1457 case jit_code_retval_f: case jit_code_retval_d:
1458 case jit_code_va_start:
1459 mask = jit_cc_a0_reg|jit_cc_a0_chg;
1460 break;
1461 case jit_code_getarg_c: case jit_code_getarg_uc:
1462 case jit_code_getarg_s: case jit_code_getarg_us:
1463 case jit_code_getarg_i: case jit_code_getarg_ui:
1464 case jit_code_getarg_l:
1465 case jit_code_getarg_f: case jit_code_getarg_d:
1466 mask = jit_cc_a0_reg|jit_cc_a0_chg|jit_cc_a1_arg;
1467 break;
79bfeef6
PC		 1468 case jit_code_putargr_c:case jit_code_putargr_uc:
1469 case jit_code_putargr_s:case jit_code_putargr_us:
1470 case jit_code_putargr_i:case jit_code_putargr_ui:
1471 case jit_code_putargr_l:
1472 case jit_code_putargr_f:case jit_code_putargr_d:
4a71579b
PC		 1473 mask = jit_cc_a0_reg|jit_cc_a1_arg;
1474 break;
79bfeef6
PC		 1475 case jit_code_putargi_c:case jit_code_putargi_uc:
1476 case jit_code_putargi_s:case jit_code_putargi_us:
1477 case jit_code_putargi_i:case jit_code_putargi_ui:
1478 case jit_code_putargi_l:
4a71579b
PC		 1479 mask = jit_cc_a0_int|jit_cc_a1_arg;
1480 break;
1481 case jit_code_putargi_f:
1482 mask = jit_cc_a0_flt|jit_cc_a1_arg;
1483 break;
1484 case jit_code_putargi_d:
1485 mask = jit_cc_a0_dbl|jit_cc_a1_arg;
1486 break;
1487 case jit_code_movi: case jit_code_ldi_c: case jit_code_ldi_uc:
1488 case jit_code_ldi_s: case jit_code_ldi_us: case jit_code_ldi_i:
1489 case jit_code_ldi_ui: case jit_code_ldi_l: case jit_code_ldi_f:
1490 case jit_code_ldi_d:
ba86ff93
PC		 1491 case jit_code_negi: case jit_code_comi:
1492 case jit_code_exti_c: case jit_code_exti_uc:
1493 case jit_code_exti_s: case jit_code_exti_us:
1494 case jit_code_exti_i: case jit_code_exti_ui:
1495 case jit_code_bswapi_us:case jit_code_bswapi_ui:
1496 case jit_code_bswapi_ul:
1497 case jit_code_htoni_us: case jit_code_htoni_ui:
1498 case jit_code_htoni_ul:
1499 case jit_code_cloi: case jit_code_clzi:
1500 case jit_code_ctoi: case jit_code_ctzi:
1501 case jit_code_rbiti: case jit_code_popcnti:
1502 case jit_code_movi_w_f: case jit_code_movi_w_d:
4a71579b
PC		 1503 mask = jit_cc_a0_reg|jit_cc_a0_chg|jit_cc_a1_int;
1504 break;
ba86ff93
PC		 1505 case jit_code_unldi: case jit_code_unldi_u: case jit_code_unldi_x:
1506 mask = jit_cc_a0_reg|jit_cc_a0_chg|jit_cc_a1_int|jit_cc_a2_int;
1507 break;
1508 case jit_code_movi_ww_d:
1509 mask = jit_cc_a0_reg|jit_cc_a0_chg|jit_cc_a1_int|jit_cc_a2_int;
1510 break;
4a71579b 1511 case jit_code_movi_f: case jit_code_movi_f_w:
ba86ff93
PC		 1512 case jit_code_negi_f: case jit_code_absi_f:
1513 case jit_code_sqrti_f:
4a71579b
PC		 1514 mask = jit_cc_a0_reg|jit_cc_a0_chg|jit_cc_a1_flt;
1515 break;
1516 case jit_code_movi_d: case jit_code_movi_d_w:
ba86ff93
PC		 1517 case jit_code_negi_d: case jit_code_absi_d:
1518 case jit_code_sqrti_d:
4a71579b
PC		 1519 mask = jit_cc_a0_reg|jit_cc_a0_chg|jit_cc_a1_dbl;
1520 break;
1521 case jit_code_movi_d_ww:
1522 mask = jit_cc_a0_reg|jit_cc_a0_chg|jit_cc_a1_reg|jit_cc_a1_chg|
1523 jit_cc_a2_dbl;
1524 break;
1525 case jit_code_negr: case jit_code_comr: case jit_code_movr:
1526 case jit_code_extr_c: case jit_code_extr_uc: case jit_code_extr_s:
1527 case jit_code_extr_us: case jit_code_extr_i: case jit_code_extr_ui:
1528 case jit_code_truncr_f_i: case jit_code_truncr_f_l:
1529 case jit_code_truncr_d_i: case jit_code_truncr_d_l:
1530 case jit_code_htonr_us: case jit_code_htonr_ui: case jit_code_htonr_ul:
40a44dcb 1531 case jit_code_bswapr_us: case jit_code_bswapr_ui: case jit_code_bswapr_ul:
4a71579b
PC		 1532 case jit_code_ldr_c: case jit_code_ldr_uc:
1533 case jit_code_ldr_s: case jit_code_ldr_us: case jit_code_ldr_i:
1534 case jit_code_ldr_ui: case jit_code_ldr_l: case jit_code_negr_f:
1535 case jit_code_absr_f: case jit_code_sqrtr_f: case jit_code_movr_f:
1536 case jit_code_extr_f: case jit_code_extr_d_f: case jit_code_ldr_f:
1537 case jit_code_negr_d: case jit_code_absr_d: case jit_code_sqrtr_d:
1538 case jit_code_movr_d: case jit_code_extr_d: case jit_code_extr_f_d:
ba86ff93 1539 case jit_code_ldr_d: case jit_code_rbitr: case jit_code_popcntr:
79bfeef6
PC		 1540 case jit_code_clor: case jit_code_clzr:
1541 case jit_code_ctor: case jit_code_ctzr:
4a71579b
PC		 1542 case jit_code_movr_w_f: case jit_code_movr_f_w:
1543 case jit_code_movr_w_d: case jit_code_movr_d_w:
1544 case jit_code_va_arg: case jit_code_va_arg_d:
1545 mask = jit_cc_a0_reg|jit_cc_a0_chg|jit_cc_a1_reg;
1546 break;
1547 case jit_code_movr_d_ww:
1548 mask = jit_cc_a0_reg|jit_cc_a0_chg|jit_cc_a1_reg|jit_cc_a1_chg|
1549 jit_cc_a2_reg;
1550 break;
1551 case jit_code_addi: case jit_code_addxi: case jit_code_addci:
1552 case jit_code_subi: case jit_code_subxi: case jit_code_subci:
1553 case jit_code_rsbi:
ba86ff93
PC		 1554 case jit_code_muli: case jit_code_hmuli: case jit_code_hmuli_u:
1555 case jit_code_divi: case jit_code_divi_u:
4a71579b
PC		 1556 case jit_code_remi: case jit_code_remi_u: case jit_code_andi:
1557 case jit_code_ori: case jit_code_xori: case jit_code_lshi:
ba86ff93
PC		 1558 case jit_code_rshi: case jit_code_rshi_u: case jit_code_lroti:
1559 case jit_code_rroti: case jit_code_lti:
4a71579b
PC		 1560 case jit_code_lti_u: case jit_code_lei: case jit_code_lei_u:
1561 case jit_code_eqi: case jit_code_gei: case jit_code_gei_u:
1562 case jit_code_gti: case jit_code_gti_u: case jit_code_nei:
1563 case jit_code_ldxi_c: case jit_code_ldxi_uc: case jit_code_ldxi_s:
1564 case jit_code_ldxi_us: case jit_code_ldxi_i: case jit_code_ldxi_ui:
1565 case jit_code_ldxi_l: case jit_code_ldxi_f: case jit_code_ldxi_d:
ba86ff93 1566 case jit_code_unldr: case jit_code_unldr_u: case jit_code_unldr_x:
4a71579b
PC		 1567 mask = jit_cc_a0_reg|jit_cc_a0_chg|jit_cc_a1_reg|jit_cc_a2_int;
1568 break;
1569 case jit_code_qmuli: case jit_code_qmuli_u:
1570 case jit_code_qdivi: case jit_code_qdivi_u:
ba86ff93
PC		 1571 case jit_code_qlshi: case jit_code_qlshi_u:
1572 case jit_code_qrshi: case jit_code_qrshi_u:
4a71579b
PC		 1573 mask = jit_cc_a0_reg|jit_cc_a0_rlh|jit_cc_a0_chg|
1574 jit_cc_a1_reg|jit_cc_a2_int;
1575 break;
1576 case jit_code_addi_f: case jit_code_subi_f: case jit_code_rsbi_f:
1577 case jit_code_muli_f: case jit_code_divi_f: case jit_code_lti_f:
1578 case jit_code_lei_f: case jit_code_eqi_f: case jit_code_gei_f:
1579 case jit_code_gti_f: case jit_code_nei_f: case jit_code_unlti_f:
1580 case jit_code_unlei_f: case jit_code_uneqi_f: case jit_code_ungei_f:
1581 case jit_code_ungti_f: case jit_code_ltgti_f: case jit_code_ordi_f:
1582 case jit_code_unordi_f:
1583 mask = jit_cc_a0_reg|jit_cc_a0_chg|jit_cc_a1_reg|jit_cc_a2_flt;
1584 break;
1585 case jit_code_addi_d: case jit_code_subi_d: case jit_code_rsbi_d:
1586 case jit_code_muli_d: case jit_code_divi_d: case jit_code_lti_d:
1587 case jit_code_lei_d: case jit_code_eqi_d: case jit_code_gei_d:
1588 case jit_code_gti_d: case jit_code_nei_d: case jit_code_unlti_d:
1589 case jit_code_unlei_d: case jit_code_uneqi_d: case jit_code_ungei_d:
1590 case jit_code_ungti_d: case jit_code_ltgti_d: case jit_code_ordi_d:
1591 case jit_code_unordi_d:
1592 mask = jit_cc_a0_reg|jit_cc_a0_chg|jit_cc_a1_reg|jit_cc_a2_dbl;
1593 break;
1594 case jit_code_addr: case jit_code_addxr: case jit_code_addcr:
1595 case jit_code_subr: case jit_code_subxr: case jit_code_subcr:
ba86ff93
PC		 1596 case jit_code_mulr: case jit_code_hmulr: case jit_code_hmulr_u:
1597 case jit_code_divr: case jit_code_divr_u:
4a71579b
PC		 1598 case jit_code_remr: case jit_code_remr_u: case jit_code_andr:
1599 case jit_code_orr: case jit_code_xorr: case jit_code_lshr:
ba86ff93
PC		 1600 case jit_code_rshr: case jit_code_rshr_u: case jit_code_lrotr:
1601 case jit_code_rrotr: case jit_code_ltr:
4a71579b
PC		 1602 case jit_code_ltr_u: case jit_code_ler: case jit_code_ler_u:
1603 case jit_code_eqr: case jit_code_ger: case jit_code_ger_u:
1604 case jit_code_gtr: case jit_code_gtr_u: case jit_code_ner:
1605 case jit_code_ldxr_c: case jit_code_ldxr_uc: case jit_code_ldxr_s:
1606 case jit_code_ldxr_us: case jit_code_ldxr_i: case jit_code_ldxr_ui:
1607 case jit_code_ldxr_l: case jit_code_addr_f: case jit_code_subr_f:
1608 case jit_code_mulr_f: case jit_code_divr_f: case jit_code_ltr_f:
1609 case jit_code_ler_f: case jit_code_eqr_f: case jit_code_ger_f:
1610 case jit_code_gtr_f: case jit_code_ner_f: case jit_code_unltr_f:
1611 case jit_code_unler_f: case jit_code_uneqr_f: case jit_code_unger_f:
1612 case jit_code_ungtr_f: case jit_code_ltgtr_f: case jit_code_ordr_f:
1613 case jit_code_unordr_f: case jit_code_ldxr_f: case jit_code_addr_d:
1614 case jit_code_subr_d: case jit_code_mulr_d: case jit_code_divr_d:
1615 case jit_code_ltr_d: case jit_code_ler_d: case jit_code_eqr_d:
1616 case jit_code_ger_d: case jit_code_gtr_d: case jit_code_ner_d:
1617 case jit_code_unltr_d: case jit_code_unler_d: case jit_code_uneqr_d:
1618 case jit_code_unger_d: case jit_code_ungtr_d: case jit_code_ltgtr_d:
1619 case jit_code_ordr_d: case jit_code_unordr_d: case jit_code_ldxr_d:
1620 case jit_code_movr_ww_d:
1621 mask = jit_cc_a0_reg|jit_cc_a0_chg|jit_cc_a1_reg|jit_cc_a2_reg;
1622 break;
1623 case jit_code_qmulr: case jit_code_qmulr_u:
1624 case jit_code_qdivr: case jit_code_qdivr_u:
ba86ff93
PC		 1625 case jit_code_qlshr: case jit_code_qlshr_u:
1626 case jit_code_qrshr: case jit_code_qrshr_u:
4a71579b
PC		 1627 mask = jit_cc_a0_reg|jit_cc_a0_rlh|jit_cc_a0_chg|
1628 jit_cc_a1_reg|jit_cc_a2_reg;
1629 break;
1630 case jit_code_sti_c: case jit_code_sti_s: case jit_code_sti_i:
1631 case jit_code_sti_l: case jit_code_sti_f: case jit_code_sti_d:
1632 mask = jit_cc_a0_int|jit_cc_a1_reg;
1633 break;
ba86ff93
PC		 1634 case jit_code_unsti: case jit_code_unsti_x:
1635 mask = jit_cc_a0_int|jit_cc_a1_reg|jit_cc_a2_int;
1636 break;
4a71579b
PC		 1637 case jit_code_blti: case jit_code_blti_u: case jit_code_blei:
1638 case jit_code_blei_u: case jit_code_beqi: case jit_code_bgei:
1639 case jit_code_bgei_u: case jit_code_bgti: case jit_code_bgti_u:
1640 case jit_code_bnei: case jit_code_bmsi: case jit_code_bmci:
1641 mask = jit_cc_a0_jmp|jit_cc_a1_reg|jit_cc_a2_int;
1642 break;
1643 case jit_code_blti_f: case jit_code_blei_f: case jit_code_beqi_f:
1644 case jit_code_bgei_f: case jit_code_bgti_f: case jit_code_bnei_f:
1645 case jit_code_bunlti_f: case jit_code_bunlei_f: case jit_code_buneqi_f:
1646 case jit_code_bungei_f: case jit_code_bungti_f: case jit_code_bltgti_f:
1647 case jit_code_bordi_f: case jit_code_bunordi_f:
1648 mask = jit_cc_a0_jmp|jit_cc_a1_reg|jit_cc_a2_flt;
1649 break;
1650 case jit_code_blti_d: case jit_code_blei_d: case jit_code_beqi_d:
1651 case jit_code_bgei_d: case jit_code_bgti_d: case jit_code_bnei_d:
1652 case jit_code_bunlti_d: case jit_code_bunlei_d: case jit_code_buneqi_d:
1653 case jit_code_bungei_d: case jit_code_bungti_d: case jit_code_bltgti_d:
1654 case jit_code_bordi_d: case jit_code_bunordi_d:
1655 mask = jit_cc_a0_jmp|jit_cc_a1_reg|jit_cc_a2_dbl;
1656 break;
1657 case jit_code_allocar: /* synthesized instructions make it
1658 * equivalent to jit_cc_a0_chg */
1659 case jit_code_str_c: case jit_code_str_s: case jit_code_str_i:
1660 case jit_code_str_l: case jit_code_str_f: case jit_code_str_d:
1661 mask = jit_cc_a0_reg|jit_cc_a1_reg;
1662 break;
ba86ff93
PC		 1663 case jit_code_unstr: case jit_code_unstr_x:
1664 mask = jit_cc_a0_reg|jit_cc_a1_reg|jit_cc_a2_int;
1665 break;
4a71579b
PC		 1666 case jit_code_stxi_c: case jit_code_stxi_s: case jit_code_stxi_i:
1667 case jit_code_stxi_l: case jit_code_stxi_f: case jit_code_stxi_d:
1668 mask = jit_cc_a0_int|jit_cc_a1_reg|jit_cc_a2_reg;
1669 break;
1670 case jit_code_bltr: case jit_code_bltr_u: case jit_code_bler:
1671 case jit_code_bler_u: case jit_code_beqr: case jit_code_bger:
1672 case jit_code_bger_u: case jit_code_bgtr: case jit_code_bgtr_u:
1673 case jit_code_bner: case jit_code_bmsr: case jit_code_bmcr:
1674 case jit_code_bltr_f: case jit_code_bler_f: case jit_code_beqr_f:
1675 case jit_code_bger_f: case jit_code_bgtr_f: case jit_code_bner_f:
1676 case jit_code_bunltr_f: case jit_code_bunler_f: case jit_code_buneqr_f:
1677 case jit_code_bunger_f: case jit_code_bungtr_f: case jit_code_bltgtr_f:
1678 case jit_code_bordr_f: case jit_code_bunordr_f:case jit_code_bltr_d:
1679 case jit_code_bler_d: case jit_code_beqr_d: case jit_code_bger_d:
1680 case jit_code_bgtr_d: case jit_code_bner_d: case jit_code_bunltr_d:
1681 case jit_code_bunler_d: case jit_code_buneqr_d: case jit_code_bunger_d:
1682 case jit_code_bungtr_d: case jit_code_bltgtr_d: case jit_code_bordr_d:
1683 case jit_code_bunordr_d:
1684 mask = jit_cc_a0_jmp|jit_cc_a1_reg|jit_cc_a2_reg;
1685 break;
1686 case jit_code_boaddi: case jit_code_boaddi_u: case jit_code_bxaddi:
1687 case jit_code_bxaddi_u: case jit_code_bosubi: case jit_code_bosubi_u:
1688 case jit_code_bxsubi: case jit_code_bxsubi_u:
1689 mask = jit_cc_a0_jmp|jit_cc_a1_reg|jit_cc_a1_chg|jit_cc_a2_int;
1690 break;
1691 case jit_code_stxr_c: case jit_code_stxr_s: case jit_code_stxr_i:
1692 case jit_code_stxr_l: case jit_code_stxr_f: case jit_code_stxr_d:
1693 mask = jit_cc_a0_reg|jit_cc_a1_reg|jit_cc_a2_reg;
1694 break;
1695 case jit_code_boaddr: case jit_code_boaddr_u: case jit_code_bxaddr:
1696 case jit_code_bxaddr_u: case jit_code_bosubr: case jit_code_bosubr_u:
1697 case jit_code_bxsubr: case jit_code_bxsubr_u:
1698 mask = jit_cc_a0_jmp|jit_cc_a1_reg|jit_cc_a1_chg|jit_cc_a2_reg;
1699 break;
40a44dcb
PC		 1700 case jit_code_movnr: case jit_code_movzr:
1701 mask = jit_cc_a0_reg|jit_cc_a0_cnd|jit_cc_a1_reg|jit_cc_a2_reg;
1702 break;
ba3814c1
PC		 1703 case jit_code_casr:
1704 mask = jit_cc_a0_reg|jit_cc_a0_chg|jit_cc_a1_reg|
1705 jit_cc_a2_reg|jit_cc_a2_rlh;
1706 break;
1707 case jit_code_casi:
1708 mask = jit_cc_a0_reg|jit_cc_a0_chg|jit_cc_a1_int|
1709 jit_cc_a2_reg|jit_cc_a2_rlh;
1710 break;
ba86ff93
PC		 1711 case jit_code_extr: case jit_code_extr_u: case jit_code_depr:
1712 mask = jit_cc_a0_reg|jit_cc_a0_chg|jit_cc_a1_reg|jit_cc_a2_rlh;
1713 break;
1714 case jit_code_exti: case jit_code_exti_u: case jit_code_depi:
1715 mask = jit_cc_a0_reg|jit_cc_a0_chg|jit_cc_a1_int|jit_cc_a2_rlh;
1716 break;
1717 case jit_code_fmar_f: case jit_code_fmar_d:
1718 case jit_code_fmsr_f: case jit_code_fmsr_d:
1719 case jit_code_fnmar_f: case jit_code_fnmar_d:
1720 case jit_code_fnmsr_f: case jit_code_fnmsr_d:
1721 mask = jit_cc_a0_reg|jit_cc_a0_chg|
1722 jit_cc_a1_reg|jit_cc_a1_rlh|jit_cc_a2_reg;
1723 break;
1724 case jit_code_fmai_f: case jit_code_fmsi_f:
1725 case jit_code_fnmai_f: case jit_code_fnmsi_f:
1726 mask = jit_cc_a0_reg|jit_cc_a0_chg|
1727 jit_cc_a1_reg|jit_cc_a1_rlh|jit_cc_a2_flt;
1728 break;
1729 case jit_code_fmai_d: case jit_code_fmsi_d:
1730 case jit_code_fnmai_d: case jit_code_fnmsi_d:
1731 mask = jit_cc_a0_reg|jit_cc_a0_chg|
1732 jit_cc_a1_reg|jit_cc_a1_rlh|jit_cc_a2_dbl;
1733 break;
4a71579b
PC		 1734 default:
1735 abort();
1736 }
1737
1738 return (mask);
1739}
1740
1741void
1742_jit_patch_abs(jit_state_t *_jit, jit_node_t *instr, jit_pointer_t address)
1743{
1744 jit_int32_t mask;
1745
1746 switch (instr->code) {
1747 case jit_code_movi: case jit_code_ldi_c: case jit_code_ldi_uc:
1748 case jit_code_ldi_s: case jit_code_ldi_us: case jit_code_ldi_i:
1749 case jit_code_ldi_ui: case jit_code_ldi_l: case jit_code_ldi_f:
1750 case jit_code_ldi_d:
1751 instr->v.p = address;
1752 break;
1753 case jit_code_sti_c: case jit_code_sti_s: case jit_code_sti_i:
1754 case jit_code_sti_l: case jit_code_sti_f: case jit_code_sti_d:
1755 instr->u.p = address;
1756 break;
1757 default:
1758 mask = jit_classify(instr->code);
1759 assert((mask & (jit_cc_a0_reg|jit_cc_a0_jmp)) == jit_cc_a0_jmp);
1760 instr->u.p = address;
1761 }
1762}
1763
1764void
1765_jit_patch_at(jit_state_t *_jit, jit_node_t *instr, jit_node_t *label)
1766{
1767 jit_int32_t mask;
1768
1769 assert(!(instr->flag & jit_flag_node));
1770 instr->flag |= jit_flag_node;
1771 switch (instr->code) {
1772 case jit_code_movi:
1773 assert(label->code == jit_code_label ||
1774 label->code == jit_code_data);
1775 instr->v.n = label;
1776 if (label->code == jit_code_data)
1777 instr->flag |= jit_flag_data;
1778 break;
1779 case jit_code_jmpi:
1780 assert(label->code == jit_code_label ||
1781 label->code == jit_code_epilog);
1782 instr->u.n = label;
1783 break;
1784 default:
1785 mask = jit_classify(instr->code);
1786 assert((mask & (jit_cc_a0_reg|jit_cc_a0_jmp)) == jit_cc_a0_jmp);
1787 assert(label->code == jit_code_label);
1788 instr->u.n = label;
1789 break;
1790 }
1791 /* link field is used as list of nodes associated with a given label */
1792 instr->link = label->link;
1793 label->link = instr;
1794}
1795
c0c16242
PC		 1796static void
1797_do_setup(jit_state_t *_jit)
4a71579b 1798{
4a71579b
PC		 1799 jit_block_t *block;
1800 jit_word_t offset;
1801
4a71579b
PC		 1802 /* create initial mapping of live register values
1803 * at the start of a basic block */
1804 for (offset = 0; offset < _jitc->blocks.offset; offset++) {
1805 block = _jitc->blocks.ptr + offset;
79bfeef6 1806 if (!block->label)
4a71579b 1807 continue;
79bfeef6
PC		 1808 if (block->label->code == jit_code_epilog) {
1809 jit_regset_setbit(&block->reglive, JIT_RET);
1810 jit_regset_setbit(&block->reglive, JIT_FRET);
1811 jit_regset_com(&block->regmask, &block->reglive);
1812 continue;
1813 }
c0c16242 1814 jit_setup(block);
4a71579b 1815 }
c0c16242 1816}
4a71579b 1817
c0c16242
PC		 1818static jit_bool_t
1819_block_update_set(jit_state_t *_jit,
1820 jit_block_t *block, jit_block_t *target)
1821{
1822 jit_regset_t regmask;
1823
1824 jit_regset_ior(&regmask, &block->reglive, &target->reglive);
1825 jit_regset_and(&regmask, &regmask, &block->regmask);
1826 if (jit_regset_set_p(&regmask)) {
1827 jit_regset_ior(&block->reglive, &block->reglive, &regmask);
1828 jit_regset_and(&regmask, &block->reglive, &block->regmask);
1829 jit_regset_com(&regmask, &regmask);
1830 jit_regset_and(&block->regmask, &block->regmask, &regmask);
1831 block->again = 1;
1832 return (1);
1833 }
1834 return (0);
1835}
4a71579b 1836
24d91c0d
PC		 1837static void
1838_propagate_backward(jit_state_t *_jit, jit_block_t *block)
1839{
1840 jit_block_t *prev;
1841 jit_word_t offset;
1842
1843 for (offset = block->label->v.w - 1;
1844 offset >= 0; --offset) {
1845 prev = _jitc->blocks.ptr + offset;
1846 if (!block_update_set(prev, block) ||
1847 !(prev->label->flag & jit_flag_head))
1848 break;
1849 }
1850}
1851
c0c16242
PC		 1852static jit_bool_t
1853_check_block_again(jit_state_t *_jit)
1854{
1855 jit_int32_t todo;
1856 jit_word_t offset;
1857 jit_node_t *node, *label;
1858 jit_block_t *block, *target;
4a71579b 1859
c0c16242
PC		 1860 todo = 0;
1861 for (offset = 0; offset < _jitc->blocks.offset; offset++) {
1862 block = _jitc->blocks.ptr + offset;
1863 if (block->again) {
ba3814c1 1864 todo = 1;
c0c16242 1865 break;
ba3814c1 1866 }
c0c16242
PC		 1867 }
1868 /* If no block changed state */
1869 if (!todo)
1870 return (0);
ba3814c1 1871
c0c16242
PC		 1872 do {
1873 todo = 0;
1874 block = NULL;
ba3814c1 1875 for (node = _jitc->head; node; node = node->next) {
c0c16242
PC		 1876 /* Special jumps that match jit_cc_a0_jmp */
1877 if (node->code == jit_code_calli || node->code == jit_code_callr)
1878 continue;
1879
1880 /* Remember current label */
1881 if (node->code == jit_code_label ||
1882 node->code == jit_code_prolog ||
1883 node->code == jit_code_epilog) {
1884
1885 /* If previous block does not pass through */
1886 if (!(node->flag & jit_flag_head))
1887 block = NULL;
1888
1889 target = _jitc->blocks.ptr + node->v.w;
24d91c0d
PC		 1890 if (block && target->again && block_update_set(block, target)) {
1891 propagate_backward(block);
ba3814c1 1892 todo = 1;
24d91c0d 1893 }
c0c16242 1894 block = target;
ba3814c1 1895 }
c0c16242
PC		 1896 /* If not the first jmpi */
1897 else if (block) {
1898 /* If a jump to dynamic address or if a jump to raw address */
1899 if (!(jit_classify(node->code) & jit_cc_a0_jmp) ||
1900 !(node->flag & jit_flag_node))
1901 continue;
1902 label = node->u.n;
1903 /* Mark predecessor needs updating due to target change */
1904 target = _jitc->blocks.ptr + label->v.w;
24d91c0d
PC		 1905 if (target->again && block_update_set(block, target)) {
1906 propagate_backward(block);
ba3814c1 1907 todo = 1;
24d91c0d 1908 }
ba3814c1
PC		 1909 }
1910 }
c0c16242
PC		 1911 }
1912 while (todo);
1913
79bfeef6 1914 return (todo);
c0c16242 1915}
ba3814c1 1916
c0c16242
PC		 1917static void
1918_do_follow(jit_state_t *_jit, jit_bool_t always)
1919{
1920 jit_block_t *block;
1921 jit_word_t offset;
1922
1923 /* set live state of registers not referenced in a block, but
1924 * referenced in a jump target or normal flow */
1925 for (offset = 0; offset < _jitc->blocks.offset; offset++) {
1926 block = _jitc->blocks.ptr + offset;
1927 if (!block->label || block->label->code == jit_code_epilog)
1928 continue;
1929 if (always || block->again) {
1930 block->again = 0;
1931 jit_follow(block);
1932 }
1933 }
1934}
1935
1936void
1937_jit_optimize(jit_state_t *_jit)
1938{
1939 jit_bool_t jump;
1940 jit_bool_t todo;
1941 jit_int32_t mask;
1942 jit_node_t *node;
1943 jit_block_t *block;
1944 jit_word_t offset;
79bfeef6 1945 jit_regset_t regmask;
c0c16242
PC		 1946
1947 todo = 0;
1948 _jitc->function = NULL;
1949
1950 thread_jumps();
1951 sequential_labels();
1952 split_branches();
1953 do_setup();
1954 do_follow(1);
1955
79bfeef6
PC		 1956 jit_regset_set_ui(&regmask, 0);
1957 for (offset = 0; offset < _jitc->reglen; offset++) {
1958 if ((jit_class(_rvs[offset].spec) & (jit_class_gpr|jit_class_fpr)) &&
1959 (jit_class(_rvs[offset].spec) & jit_class_sav) == jit_class_sav)
1960 jit_regset_setbit(&regmask, offset);
1961 }
1962
1963 /* Figure out labels that are only reached with a jump */
c0c16242
PC		 1964 jump = 1;
1965 for (node = _jitc->head; node; node = node->next) {
1966 switch (node->code) {
1967 case jit_code_label:
79bfeef6 1968 if (!jump) {
c0c16242 1969 node->flag |= jit_flag_head;
79bfeef6
PC		 1970 if (!node->link) {
1971 /* Block is dead code or only reachable with an
1972 * indirect jumps. In such condition, must assume
1973 * all callee save registers are live. */
1974 block = _jitc->blocks.ptr + node->v.w;
1975 jit_regset_ior(&block->reglive,
1976 &block->reglive, &regmask);
1977 /* Cleanup regmask */
1978 block_update_set(block, block);
1979 }
1980 }
c0c16242
PC		 1981 break;
1982 case jit_code_jmpi: case jit_code_jmpr:
1983 case jit_code_epilog:
1984 jump = 1;
1985 break;
1986 case jit_code_data: case jit_code_note:
1987 break;
1988 default:
1989 jump = 0;
1990 break;
1991 }
1992 }
1993
ba86ff93
PC		 1994 patch_registers();
1995 if (simplify())
1996 todo = 1;
1997
c0c16242
PC		 1998 for (node = _jitc->head; node; node = node->next) {
1999 mask = jit_classify(node->code);
2000 if (mask & jit_cc_a0_reg)
2001 node->u.w &= ~jit_regno_patch;
2002 if (mask & jit_cc_a1_reg)
2003 node->v.w &= ~jit_regno_patch;
2004 if (mask & jit_cc_a2_reg)
2005 node->w.w &= ~jit_regno_patch;
2006 if (node->code == jit_code_beqi) {
2007 if (redundant_store(node, 1)) {
2008 block = _jitc->blocks.ptr + ((jit_node_t *)node->u.n)->v.w;
2009 block->again = 1;
2010 todo = 1;
2011 }
2012 }
2013 else if (node->code == jit_code_bnei) {
2014 if (redundant_store(node, 0)) {
2015 block = _jitc->blocks.ptr + ((jit_node_t *)node->u.n)->v.w;
2016 block->again = 1;
2017 todo = 1;
2018 }
4a71579b
PC		 2019 }
2020 }
2021
c0c16242
PC		 2022 if (!todo)
2023 todo = check_block_again();
2024
2025 /* If instructions were removed or first pass did modify the entry
2026 * state of any block */
2027 if (todo) {
2028 do_setup();
2029 todo = 0;
2030 do {
2031 do_follow(0);
2032 /* If any block again has the entry state modified. */
2033 todo = check_block_again();
2034 } while (todo);
2035 }
2036
4a71579b
PC		 2037 for (node = _jitc->head; node; node = node->next) {
2038 mask = jit_classify(node->code);
2039 if (mask & jit_cc_a0_reg)
2040 node->u.w &= ~jit_regno_patch;
2041 if (mask & jit_cc_a1_reg)
2042 node->v.w &= ~jit_regno_patch;
2043 if (mask & jit_cc_a2_reg)
2044 node->w.w &= ~jit_regno_patch;
ba3814c1
PC		 2045 if (node->code == jit_code_prolog)
2046 _jitc->function = _jitc->functions.ptr + node->w.w;
2047 else if(node->code == jit_code_epilog)
2048 _jitc->function = NULL;
2049 else {
4a71579b 2050#if JIT_HASH_CONSTS
ba3814c1
PC		 2051 if (mask & jit_cc_a0_flt) {
2052 node->u.p = jit_data(&node->u.f, sizeof(jit_float32_t), 4);
2053 node->flag |= jit_flag_node | jit_flag_data;
2054 }
2055 else if (mask & jit_cc_a0_dbl) {
2056 node->u.p = jit_data(&node->u.d, sizeof(jit_float64_t), 8);
2057 node->flag |= jit_flag_node | jit_flag_data;
2058 }
2059 else if (mask & jit_cc_a1_flt) {
2060 node->v.p = jit_data(&node->v.f, sizeof(jit_float32_t), 4);
2061 node->flag |= jit_flag_node | jit_flag_data;
2062 }
2063 else if (mask & jit_cc_a1_dbl) {
2064 node->v.p = jit_data(&node->v.d, sizeof(jit_float64_t), 8);
2065 node->flag |= jit_flag_node | jit_flag_data;
2066 }
2067 else if (mask & jit_cc_a2_flt) {
2068 node->w.p = jit_data(&node->w.f, sizeof(jit_float32_t), 4);
2069 node->flag |= jit_flag_node | jit_flag_data;
2070 }
2071 else if (mask & jit_cc_a2_dbl) {
2072 node->w.p = jit_data(&node->w.d, sizeof(jit_float64_t), 8);
2073 node->flag |= jit_flag_node | jit_flag_data;
2074 }
4a71579b 2075#endif
ba3814c1
PC		 2076 if (_jitc->function) {
2077 if ((mask & (jit_cc_a0_reg|jit_cc_a0_chg)) ==
2078 (jit_cc_a0_reg|jit_cc_a0_chg)) {
2079 if (mask & jit_cc_a0_rlh) {
2080 jit_regset_setbit(&_jitc->function->regset,
2081 jit_regno(node->u.q.l));
4a71579b 2082 jit_regset_setbit(&_jitc->function->regset,
ba3814c1
PC		 2083 jit_regno(node->u.q.h));
2084 }
2085 else
4a71579b 2086 jit_regset_setbit(&_jitc->function->regset,
ba3814c1 2087 jit_regno(node->u.w));
4a71579b 2088 }
ba3814c1
PC		 2089 if ((mask & (jit_cc_a1_reg|jit_cc_a1_chg)) ==
2090 (jit_cc_a1_reg|jit_cc_a1_chg))
2091 jit_regset_setbit(&_jitc->function->regset,
2092 jit_regno(node->v.w));
2093 if ((mask & (jit_cc_a2_reg|jit_cc_a2_chg)) ==
2094 (jit_cc_a2_reg|jit_cc_a2_chg))
2095 jit_regset_setbit(&_jitc->function->regset,
2096 jit_regno(node->w.w));
2097 }
4a71579b
PC		 2098 }
2099 }
2100}
2101
2102void
2103_jit_reglive(jit_state_t *_jit, jit_node_t *node)
2104{
2105 jit_int32_t spec;
2106 jit_int32_t value;
2107 jit_block_t *block;
2108
2109 switch (node->code) {
2110 case jit_code_label: case jit_code_prolog: case jit_code_epilog:
2111 block = _jitc->blocks.ptr + node->v.w;
2112 jit_regset_set(&_jitc->reglive, &block->reglive);
79bfeef6
PC		 2113 jit_regset_set_ui(&_jitc->explive, 0);
2114 break;
2115 case jit_code_live:
2116 jit_regset_setbit(&_jitc->explive, node->u.w);
4a71579b
PC		 2117 break;
2118 case jit_code_callr:
2119 value = jit_regno(node->u.w);
2120 if (!(node->u.w & jit_regno_patch)) {
2121 jit_regset_setbit(&_jitc->reglive, value);
2122 }
2123 case jit_code_calli:
2124 for (value = 0; value < _jitc->reglen; value++) {
2125 spec = jit_class(_rvs[value].spec);
2126 if ((spec & jit_class_arg) && jit_regarg_p(node, value))
2127 jit_regset_setbit(&_jitc->reglive, value);
2128 else if (!(spec & jit_class_sav))
2129 jit_regset_clrbit(&_jitc->reglive, value);
2130 }
2131 break;
2132 default:
2133 value = jit_classify(node->code);
2134 if (value & jit_cc_a0_reg) {
2135 if (value & jit_cc_a0_rlh) {
2136 if (!(node->u.q.l & jit_regno_patch)) {
2137 if (value & jit_cc_a0_chg) {
2138 jit_regset_clrbit(&_jitc->reglive, node->u.q.l);
2139 jit_regset_setbit(&_jitc->regmask, node->u.q.l);
2140 }
2141 else
2142 jit_regset_setbit(&_jitc->reglive, node->u.q.l);
2143 }
2144 if (!(node->u.q.h & jit_regno_patch)) {
2145 if (value & jit_cc_a0_chg) {
2146 jit_regset_clrbit(&_jitc->reglive, node->u.q.h);
2147 jit_regset_setbit(&_jitc->regmask, node->u.q.h);
2148 }
2149 else
2150 jit_regset_setbit(&_jitc->reglive, node->u.q.h);
2151 }
2152 }
2153 else {
2154 if (!(node->u.w & jit_regno_patch)) {
2155 if (value & jit_cc_a0_chg) {
2156 jit_regset_clrbit(&_jitc->reglive, node->u.w);
2157 jit_regset_setbit(&_jitc->regmask, node->u.w);
2158 }
2159 else
2160 jit_regset_setbit(&_jitc->reglive, node->u.w);
2161 }
2162 }
2163 }
ba86ff93
PC		 2164 if (value & jit_cc_a1_reg) {
2165 if (value & jit_cc_a1_rlh) {
2166 /* Assume registers are not changed */
2167 if (!(node->v.q.l & jit_regno_patch))
2168 jit_regset_setbit(&_jitc->reglive, node->v.q.l);
2169 if (!(node->v.q.h & jit_regno_patch))
2170 jit_regset_setbit(&_jitc->reglive, node->v.q.h);
2171 }
2172 else if (!(node->v.w & jit_regno_patch)) {
2173 if (value & jit_cc_a1_chg) {
2174 jit_regset_clrbit(&_jitc->reglive, node->v.w);
2175 jit_regset_setbit(&_jitc->regmask, node->v.w);
2176 }
2177 else
2178 jit_regset_setbit(&_jitc->reglive, node->v.w);
4a71579b 2179 }
4a71579b 2180 }
ba3814c1
PC		 2181 if (value & jit_cc_a2_reg) {
2182 if (value & jit_cc_a2_rlh) {
2183 /* Assume registers are not changed */
2184 if (!(node->w.q.l & jit_regno_patch))
2185 jit_regset_setbit(&_jitc->reglive, node->w.q.l);
2186 if (!(node->w.q.h & jit_regno_patch))
2187 jit_regset_setbit(&_jitc->reglive, node->w.q.h);
2188 }
2189 else {
2190 if (!(node->w.w & jit_regno_patch)) {
2191 if (value & jit_cc_a2_chg) {
2192 jit_regset_clrbit(&_jitc->reglive, node->w.w);
2193 jit_regset_setbit(&_jitc->regmask, node->w.w);
2194 }
2195 else
2196 jit_regset_setbit(&_jitc->reglive, node->w.w);
2197 }
4a71579b 2198 }
4a71579b
PC		 2199 }
2200 if (jit_regset_set_p(&_jitc->regmask)) {
2201 jit_update(node->next, &_jitc->reglive, &_jitc->regmask);
2202 if (jit_regset_set_p(&_jitc->regmask)) {
2203 /* any unresolved live state is considered as live */
2204 jit_regset_ior(&_jitc->reglive,
2205 &_jitc->reglive, &_jitc->regmask);
2206 jit_regset_set_ui(&_jitc->regmask, 0);
2207 }
2208 }
2209 break;
2210 }
2211}
2212
2213void
2214_jit_regarg_set(jit_state_t *_jit, jit_node_t *node, jit_int32_t value)
2215{
2216#if GET_JIT_SIZE
2217 jit_size_prepare();
2218#endif
2219 if (value & jit_cc_a0_reg) {
2220 if (value & jit_cc_a0_rlh) {
2221 jit_regset_setbit(&_jitc->regarg, jit_regno(node->u.q.l));
2222 jit_regset_setbit(&_jitc->regarg, jit_regno(node->u.q.h));
2223 }
2224 else
2225 jit_regset_setbit(&_jitc->regarg, jit_regno(node->u.w));
2226 }
ba86ff93
PC		 2227 if (value & jit_cc_a1_reg) {
2228 if (value & jit_cc_a1_rlh) {
2229 jit_regset_setbit(&_jitc->regarg, jit_regno(node->v.q.l));
2230 jit_regset_setbit(&_jitc->regarg, jit_regno(node->v.q.h));
2231 }
2232 else
2233 jit_regset_setbit(&_jitc->regarg, jit_regno(node->v.w));
2234 }
ba3814c1
PC		 2235 if (value & jit_cc_a2_reg) {
2236 if (value & jit_cc_a2_rlh) {
2237 jit_regset_setbit(&_jitc->regarg, jit_regno(node->w.q.l));
2238 jit_regset_setbit(&_jitc->regarg, jit_regno(node->w.q.h));
2239 }
2240 else
2241 jit_regset_setbit(&_jitc->regarg, jit_regno(node->w.w));
2242 }
79bfeef6
PC		 2243 /* Prevent incorrect detection of running out of registers
2244 * if will need to patch jump, and all registers have been
2245 * used in the current block. */
2246 if (node->code == jit_code_jmpi && (node->flag & jit_flag_node)) {
2247 jit_node_t *label = node->u.n;
2248 jit_block_t *block = _jitc->blocks.ptr + label->v.w;
2249 jit_regset_set(&_jitc->reglive, &block->reglive);
2250 jit_regset_set(&_jitc->regmask, &block->regmask);
2251 if (jit_regset_set_p(&_jitc->explive)) {
2252 jit_regset_ior(&_jitc->reglive, &block->reglive, &_jitc->explive);
2253 jit_regset_xor(&_jitc->regmask, &_jitc->regmask, &_jitc->explive);
2254 }
2255 }
4a71579b
PC		 2256}
2257
2258void
2259_jit_regarg_clr(jit_state_t *_jit, jit_node_t *node, jit_int32_t value)
2260{
2261#if GET_JIT_SIZE
2262 jit_size_collect(node);
2263#endif
2264 if (value & jit_cc_a0_reg) {
2265 if (value & jit_cc_a0_rlh) {
2266 jit_regset_clrbit(&_jitc->regarg, jit_regno(node->u.q.l));
2267 jit_regset_clrbit(&_jitc->regarg, jit_regno(node->u.q.h));
2268 }
2269 else
2270 jit_regset_clrbit(&_jitc->regarg, jit_regno(node->u.w));
2271 }
ba86ff93
PC		 2272 if (value & jit_cc_a1_reg) {
2273 if (value & jit_cc_a1_rlh) {
2274 jit_regset_clrbit(&_jitc->regarg, jit_regno(node->v.q.l));
2275 jit_regset_clrbit(&_jitc->regarg, jit_regno(node->v.q.h));
2276 }
2277 else
2278 jit_regset_clrbit(&_jitc->regarg, jit_regno(node->v.w));
2279 }
ba3814c1
PC		 2280 if (value & jit_cc_a2_reg) {
2281 if (value & jit_cc_a2_rlh) {
2282 jit_regset_clrbit(&_jitc->regarg, jit_regno(node->w.q.l));
2283 jit_regset_clrbit(&_jitc->regarg, jit_regno(node->w.q.h));
2284 }
2285 else
2286 jit_regset_clrbit(&_jitc->regarg, jit_regno(node->w.w));
2287 }
4a71579b
PC		 2288}
2289
2290void
2291_jit_realize(jit_state_t *_jit)
2292{
2293 assert(!_jitc->realize);
2294 if (_jitc->function)
2295 jit_epilog();
2296 jit_optimize();
2297 _jitc->realize = 1;
2298
2299 /* ensure it is aligned */
2300 _jitc->data.offset = (_jitc->data.offset + 7) & -8;
2301
2302#if GET_JIT_SIZE
2303 /* Heuristic to guess code buffer size */
2304 _jitc->mult = 4;
2305 _jit->code.length = _jitc->pool.length * 1024 * _jitc->mult;
2306#else
2307 _jit->code.length = jit_get_size();
2308#endif
2309}
2310
2311void
2312_jit_dataset(jit_state_t *_jit)
2313{
2314 jit_uint8_t *ptr;
2315 jit_node_t *node;
2316 jit_word_t offset;
2317#if defined(__sgi)
2318 int mmap_fd;
2319#endif
2320
2321 assert(!_jitc->dataset);
40a44dcb
PC		 2322#if !HAVE_MMAP
2323 assert(_jit->user_data);
2324#else
4a71579b
PC		 2325 if (!_jit->user_data) {
2326
2327 /* create read only data buffer */
2328 _jit->data.length = (_jitc->data.offset +
2329 /* reserve space for annotations */
2330 _jitc->note.size + 4095) & -4096;
2331#if defined(__sgi)
2332 mmap_fd = open("/dev/zero", O_RDWR);
2333#endif
2334 _jit->data.ptr = mmap(NULL, _jit->data.length,
2335 PROT_READ | PROT_WRITE,
2336 MAP_PRIVATE | MAP_ANON, mmap_fd, 0);
2337 assert(_jit->data.ptr != MAP_FAILED);
2338#if defined(__sgi)
2339 close(mmap_fd);
2340#endif
2341 }
40a44dcb 2342#endif /* !HAVE_MMAP */
4a71579b
PC		 2343
2344 if (!_jitc->no_data)
2345 jit_memcpy(_jit->data.ptr, _jitc->data.ptr, _jitc->data.offset);
2346
2347 if (_jitc->no_note) {
2348 /* Space for one note is always allocated, so revert it here
2349 * if after jit_new_state was called, it is also requested to
2350 * not generate annotation information */
2351 _jit->note.length = 0;
2352 _jitc->note.size = 0;
2353 }
2354 else {
2355 _jitc->note.base = _jit->data.ptr;
2356 if (!_jitc->no_data)
2357 _jitc->note.base += _jitc->data.offset;
2358 memset(_jitc->note.base, 0, _jitc->note.size);
2359 }
2360
2361 if (_jit->user_data)
2362 /* Need the temporary hashed data until jit_emit is finished */
2363 ptr = _jitc->no_data ? _jitc->data.ptr : _jit->data.ptr;
2364 else {
2365 ptr = _jit->data.ptr;
2366 /* Temporary hashed data no longer required */
2367 jit_free((jit_pointer_t *)&_jitc->data.ptr);
2368 }
2369
2370 for (offset = 0; offset < _jitc->data.size; offset++) {
2371 for (node = _jitc->data.table[offset]; node; node = node->next) {
2372 node->flag |= jit_flag_patch;
2373 node->u.w = (jit_word_t)(ptr + node->u.w);
2374 }
2375 }
2376
2377 _jitc->dataset = 1;
2378}
2379
2380jit_pointer_t
2381_jit_get_code(jit_state_t *_jit, jit_word_t *length)
2382{
2383 assert(_jitc->realize);
2384 if (length) {
2385 if (_jitc->done)
2386 /* If code already generated, return exact size of code */
2387 *length = _jit->pc.uc - _jit->code.ptr;
2388 else
2389 /* Else return current size of the code buffer */
2390 *length = _jit->code.length;
2391 }
2392
2393 return (_jit->code.ptr);
2394}
2395
2396void
2397_jit_set_code(jit_state_t *_jit, jit_pointer_t ptr, jit_word_t length)
2398{
2399 assert(_jitc->realize);
2400 _jit->code.ptr = ptr;
2401 _jit->code.length = length;
2402 _jit->user_code = 1;
2403}
2404
2405jit_pointer_t
2406_jit_get_data(jit_state_t *_jit, jit_word_t *data_size, jit_word_t *note_size)
2407{
2408 assert(_jitc->realize);
2409 if (data_size)
2410 *data_size = _jitc->data.offset;
2411 if (note_size)
2412 *note_size = _jitc->note.size;
2413 return (_jit->data.ptr);
2414}
2415
2416void
2417_jit_set_data(jit_state_t *_jit, jit_pointer_t ptr,
2418 jit_word_t length, jit_word_t flags)
2419{
2420 assert(_jitc->realize);
2421 if (flags & JIT_DISABLE_DATA)
2422 _jitc->no_data = 1;
2423 else
2424 assert(length >= _jitc->data.offset);
2425 if (flags & JIT_DISABLE_NOTE)
2426 _jitc->no_note = 1;
2427 else {
2428 if (flags & JIT_DISABLE_DATA)
2429 assert(length >= _jitc->note.size);
2430 else
2431 assert(length >= _jitc->data.offset + _jitc->note.size);
2432 }
2433 _jit->data.ptr = ptr;
2434 _jit->data.length = length;
2435 _jit->user_data = 1;
2436}
2437
2438jit_pointer_t
2439_jit_emit(jit_state_t *_jit)
2440{
2441 jit_pointer_t code;
2442 jit_node_t *node;
2443 size_t length;
2444 int result;
2445#if defined(__sgi)
2446 int mmap_fd;
2447#endif
c0c16242 2448 int mmap_prot, mmap_flags;
4a71579b
PC		 2449
2450 if (!_jitc->realize)
2451 jit_realize();
2452
2453 if (!_jitc->dataset)
2454 jit_dataset();
2455
2456 _jitc->emit = 1;
2457
40a44dcb
PC		 2458#if !HAVE_MMAP
2459 assert(_jit->user_code);
2460#else
4a71579b 2461 if (!_jit->user_code) {
c0c16242 2462 mmap_prot = PROT_READ | PROT_WRITE;
79bfeef6 2463#if !(__OpenBSD__ || __APPLE__)
c0c16242
PC		 2464 mmap_prot |= PROT_EXEC;
2465#endif
2466#if __NetBSD__
2467 mmap_prot = PROT_MPROTECT(mmap_prot);
2468 mmap_flags = 0;
2469#else
2470 mmap_flags = MAP_PRIVATE;
2471#endif
2472 mmap_flags |= MAP_ANON;
4a71579b
PC		 2473#if defined(__sgi)
2474 mmap_fd = open("/dev/zero", O_RDWR);
2475#endif
2476 _jit->code.ptr = mmap(NULL, _jit->code.length,
c0c16242 2477 mmap_prot, mmap_flags, mmap_fd, 0);
4a71579b
PC		 2478 assert(_jit->code.ptr != MAP_FAILED);
2479 }
40a44dcb 2480#endif /* !HAVE_MMAP */
4a71579b
PC		 2481 _jitc->code.end = _jit->code.ptr + _jit->code.length -
2482 jit_get_max_instr();
2483 _jit->pc.uc = _jit->code.ptr;
2484
2485 for (;;) {
c0c16242
PC		 2486#if __NetBSD__
2487 result = mprotect(_jit->code.ptr, _jit->code.length,
2488 PROT_READ | PROT_WRITE);
2489 assert(result == 0);
2490#endif
4a71579b
PC		 2491 if ((code = emit_code()) == NULL) {
2492 _jitc->patches.offset = 0;
2493 for (node = _jitc->head; node; node = node->next) {
2494 if (node->link &&
2495 (node->code == jit_code_label ||
2496 node->code == jit_code_epilog))
2497 node->flag &= ~jit_flag_patch;
2498 }
40a44dcb
PC		 2499#if !HAVE_MMAP
2500 assert(_jit->user_code);
2501#else
4a71579b
PC		 2502 if (_jit->user_code)
2503 goto fail;
2504#if GET_JIT_SIZE
2505 ++_jitc->mult;
2506 length = _jitc->pool.length * 1024 * _jitc->mult;
2507#else
2508 /* Should only happen on very special cases */
2509 length = _jit->code.length + 4096;
2510#endif
2511
2512#if !HAVE_MREMAP
2513 munmap(_jit->code.ptr, _jit->code.length);
2514#endif
2515
2516#if HAVE_MREMAP
2517# if __NetBSD__
2518 _jit->code.ptr = mremap(_jit->code.ptr, _jit->code.length,
2519 _jit->code.ptr, length, 0);
2520# else
2521 _jit->code.ptr = mremap(_jit->code.ptr, _jit->code.length,
2522 length, MREMAP_MAYMOVE, NULL);
2523# endif
2524#else
2525 _jit->code.ptr = mmap(NULL, length,
79bfeef6 2526 mmap_prot, mmap_flags, mmap_fd, 0);
4a71579b
PC		 2527#endif
2528
2529 assert(_jit->code.ptr != MAP_FAILED);
2530 _jit->code.length = length;
2531 _jitc->code.end = _jit->code.ptr + _jit->code.length -
2532 jit_get_max_instr();
2533 _jit->pc.uc = _jit->code.ptr;
40a44dcb 2534#endif /* !HAVE_MMAP */
4a71579b
PC		 2535 }
2536 else
2537 break;
2538 }
2539
2540#if defined(__sgi)
2541 if (!_jit->user_code)
2542 close(mmap_fd);
2543#endif
2544
2545 _jitc->done = 1;
2546 if (!_jitc->no_note)
2547 jit_annotate();
2548
2549 if (_jit->user_data)
2550 jit_free((jit_pointer_t *)&_jitc->data.ptr);
40a44dcb 2551#if HAVE_MMAP
4a71579b 2552 else {
c0c16242
PC		 2553 result = mprotect(_jit->data.ptr,
2554 _jit->data.length, PROT_READ);
4a71579b
PC		 2555 assert(result == 0);
2556 }
2557 if (!_jit->user_code) {
ba86ff93 2558 _jit->code.protect = _jit->pc.uc - _jit->code.ptr;
c0c16242
PC		 2559# if __riscv && __WORDSIZE == 64
2560 /* FIXME should start adding consts at a page boundary */
ba86ff93 2561 _jit->code.protect -= _jitc->consts.hash.count * sizeof(jit_word_t);
c0c16242 2562# endif
ba86ff93 2563 result = mprotect(_jit->code.ptr, _jit->code.protect, PROT_READ | PROT_EXEC);
4a71579b
PC		 2564 assert(result == 0);
2565 }
40a44dcb 2566#endif /* HAVE_MMAP */
4a71579b
PC		 2567
2568 return (_jit->code.ptr);
2569fail:
2570 return (NULL);
2571}
2572
79bfeef6
PC		 2573void
2574_jit_protect(jit_state_t *_jit)
2575{
2576#if !HAVE_MMAP
2577 assert (_jit->user_code);
2578#else
2579 int result;
2580 if (_jit->user_code) return;
ba86ff93 2581 result = mprotect (_jit->code.ptr, _jit->code.protect, PROT_READ | PROT_EXEC);
79bfeef6
PC		 2582 assert (result == 0);
2583#endif
2584}
2585
2586void
2587_jit_unprotect(jit_state_t *_jit)
2588{
2589#if !HAVE_MMAP
2590 assert (_jit->user_code);
2591#else
2592 int result;
2593 if (_jit->user_code) return;
ba86ff93 2594 result = mprotect (_jit->code.ptr, _jit->code.protect, PROT_READ | PROT_WRITE);
79bfeef6
PC		 2595 assert (result == 0);
2596#endif
2597}
2598
4a71579b
PC		 2599void
2600_jit_frame(jit_state_t *_jit, jit_int32_t frame)
2601{
2602 jit_trampoline(frame, 1);
2603}
2604
2605void
2606_jit_tramp(jit_state_t *_jit, jit_int32_t frame)
2607{
2608 jit_trampoline(frame, 0);
2609}
2610
2611void
2612_jit_trampoline(jit_state_t *_jit, jit_int32_t frame, jit_bool_t prolog)
2613{
2614 jit_int32_t regno;
2615
2616 /* Must be called after prolog, actually, just to simplify
2617 * tests and know there is a current function and that
2618 * _jitc->function->self.aoff is at the before any alloca value */
2619 assert(_jitc->tail && _jitc->tail->code == jit_code_prolog);
2620
2621 /* + 24 for 3 possible spilled temporaries (that could be a double) */
2622 frame += 24;
2623#if defined(__hppa__)
2624 frame += _jitc->function->self.aoff;
2625#else
2626 frame -= _jitc->function->self.aoff;
2627#endif
2628 _jitc->function->frame = frame;
2629 if (prolog)
2630 _jitc->function->define_frame = 1;
2631 else
2632 _jitc->function->assume_frame = 1;
2633 for (regno = 0; regno < _jitc->reglen; regno++)
2634 if (jit_class(_rvs[regno].spec) & jit_class_sav)
2635 jit_regset_setbit(&_jitc->function->regset, regno);
2636}
2637
2638/* Compute initial reglive and regmask set values of a basic block.
2639 * reglive is the set of known live registers
2640 * regmask is the set of registers not referenced in the block
2641 * Registers in regmask might be live.
2642 */
2643static void
2644_jit_setup(jit_state_t *_jit, jit_block_t *block)
2645{
2646 jit_node_t *node;
2647 jit_bool_t live;
2648 unsigned long value;
2649
2650 jit_regset_set_mask(&block->regmask, _jitc->reglen);
2651 for (value = 0; value < _jitc->reglen; ++value)
2652 if (!(jit_class(_rvs[value].spec) & (jit_class_gpr|jit_class_fpr)))
2653 jit_regset_clrbit(&block->regmask, value);
2654
2655 for (node = block->label->next; node; node = node->next) {
2656 switch (node->code) {
2657 case jit_code_label: case jit_code_prolog:
2658 case jit_code_epilog:
2659 return;
2660 default:
2661 /* Check argument registers in reverse order to properly
2662 * handle registers that are both, argument and result */
2663 value = jit_classify(node->code);
ba86ff93 2664 if (value & jit_cc_a2_reg) {
4a71579b 2665 live = !(value & jit_cc_a2_chg);
ba86ff93
PC		 2666 if (value & jit_cc_a2_rlh) {
2667 /* Assume will not modify a pair in second argument */
2668 assert(live);
2669 if (!(node->w.q.l & jit_regno_patch) &&
2670 jit_regset_tstbit(&block->regmask, node->w.q.l)) {
2671 jit_regset_clrbit(&block->regmask, node->w.q.l);
2672 }
2673 if (!(node->w.q.h & jit_regno_patch) &&
2674 jit_regset_tstbit(&block->regmask, node->w.q.h)) {
2675 jit_regset_clrbit(&block->regmask, node->w.q.h);
2676 }
2677 }
2678 else {
2679 if (!(node->w.w & jit_regno_patch) &&
2680 jit_regset_tstbit(&block->regmask, node->w.w)) {
2681 jit_regset_clrbit(&block->regmask, node->w.w);
2682 if (live)
2683 jit_regset_setbit(&block->reglive, node->w.w);
2684 }
2685 }
4a71579b 2686 }
ba86ff93 2687 if (value & jit_cc_a1_reg) {
4a71579b 2688 live = !(value & jit_cc_a1_chg);
ba86ff93
PC		 2689 if (value & jit_cc_a1_rlh) {
2690 /* Assume will not modify a pair in second argument */
2691 assert(live);
2692 if (!(node->v.q.l & jit_regno_patch) &&
2693 jit_regset_tstbit(&block->regmask, node->v.q.l)) {
2694 jit_regset_clrbit(&block->regmask, node->v.q.l);
2695 }
2696 if (!(node->v.q.h & jit_regno_patch) &&
2697 jit_regset_tstbit(&block->regmask, node->v.q.h)) {
2698 jit_regset_clrbit(&block->regmask, node->v.q.h);
2699 }
2700 }
2701 else {
2702 if (!(node->v.w & jit_regno_patch) &&
2703 jit_regset_tstbit(&block->regmask, node->v.w)) {
2704 jit_regset_clrbit(&block->regmask, node->v.w);
2705 if (live)
2706 jit_regset_setbit(&block->reglive, node->v.w);
2707 }
2708 }
4a71579b
PC		 2709 }
2710 if (value & jit_cc_a0_reg) {
2711 live = !(value & jit_cc_a0_chg);
2712 if (value & jit_cc_a0_rlh) {
2713 if (!(node->u.q.l & jit_regno_patch) &&
2714 jit_regset_tstbit(&block->regmask, node->u.q.l)) {
2715 jit_regset_clrbit(&block->regmask, node->u.q.l);
2716 if (live)
2717 jit_regset_setbit(&block->reglive, node->u.q.l);
2718 }
2719 if (!(node->u.q.h & jit_regno_patch) &&
2720 jit_regset_tstbit(&block->regmask, node->u.q.h)) {
2721 jit_regset_clrbit(&block->regmask, node->u.q.h);
2722 if (live)
2723 jit_regset_setbit(&block->reglive, node->u.q.h);
2724 }
2725 }
2726 else {
2727 if (!(node->u.w & jit_regno_patch) &&
2728 jit_regset_tstbit(&block->regmask, node->u.w)) {
2729 jit_regset_clrbit(&block->regmask, node->u.w);
2730 if (live)
2731 jit_regset_setbit(&block->reglive, node->u.w);
2732 }
2733 }
2734 }
2735 break;
2736 }
2737 }
2738}
2739
2740/* Update regmask and reglive of blocks at entry point of branch targets
2741 * or normal flow that have a live register not used in this block.
2742 */
2743static void
c0c16242 2744_jit_follow(jit_state_t *_jit, jit_block_t *block)
4a71579b
PC		 2745{
2746 jit_node_t *node;
2747 jit_block_t *next;
2748 jit_int32_t spec;
2749 jit_int32_t regno;
2750 unsigned long value;
2751 jit_node_t *label;
2752 jit_regset_t reglive;
2753 jit_regset_t regmask;
2754 jit_regset_t regtemp;
2755
2756 jit_regset_set(&reglive, &block->reglive);
2757 jit_regset_set(&regmask, &block->regmask);
2758 for (node = block->label->next; node; node = node->next) {
2759 switch (node->code) {
2760 case jit_code_label:
2761 /* Do not consider jmpi and jmpr cannot jump to the
2762 * next instruction. */
2763 next = _jitc->blocks.ptr + node->v.w;
2764 /* Set of live registers in next block that are at unknown
2765 * state in this block. */
2766 jit_regset_and(&regtemp, &regmask, &next->reglive);
2767 if (jit_regset_set_p(&regtemp)) {
2768 /* Add live state of next block to current block. */
2769 jit_regset_ior(&block->reglive, &block->reglive, &regtemp);
2770 /* Remove from unknown state bitmask. */
2771 jit_regset_com(&regtemp, &regtemp);
2772 jit_regset_and(&block->regmask, &block->regmask, &regtemp);
c0c16242 2773 block->again = 1;
4a71579b
PC		 2774 }
2775 case jit_code_prolog:
2776 case jit_code_epilog:
2777 return;
2778 case jit_code_callr:
2779 value = jit_regno(node->u.w);
2780 if (!(node->u.w & jit_regno_patch)) {
2781 if (jit_regset_tstbit(&regmask, value)) {
2782 jit_regset_clrbit(&regmask, value);
2783 jit_regset_setbit(&reglive, value);
2784 }
2785 }
2786 case jit_code_calli:
2787 for (value = 0; value < _jitc->reglen; ++value) {
2788 value = jit_regset_scan1(&regmask, value);
2789 if (value >= _jitc->reglen)
2790 break;
2791 spec = jit_class(_rvs[value].spec);
2792 if (!(spec & jit_class_sav))
2793 jit_regset_clrbit(&regmask, value);
2794 if ((spec & jit_class_arg) && jit_regarg_p(node, value))
2795 jit_regset_setbit(&reglive, value);
2796 }
2797 break;
2798 default:
2799 value = jit_classify(node->code);
2800 if (value & jit_cc_a2_reg) {
ba3814c1
PC		 2801 if (value & jit_cc_a2_rlh) {
2802 if (!(node->w.q.l & jit_regno_patch)) {
2803 /* Assume register is not changed */
2804 if (jit_regset_tstbit(&regmask, node->w.q.l))
2805 jit_regset_clrbit(&regmask, node->w.q.l);
2806 }
2807 if (!(node->w.q.h & jit_regno_patch)) {
2808 if (jit_regset_tstbit(&regmask, node->w.q.h))
2809 jit_regset_clrbit(&regmask, node->w.q.h);
2810 }
2811 }
2812 else {
ba86ff93
PC		 2813 if (!(node->w.w & jit_regno_patch)) {
2814 if (jit_regset_tstbit(&regmask, node->w.w)) {
2815 jit_regset_clrbit(&regmask, node->w.w);
2816 if (!(value & jit_cc_a2_chg))
2817 jit_regset_setbit(&reglive, node->w.w);
ba3814c1 2818 }
4a71579b
PC		 2819 }
2820 }
2821 }
2822 if (value & jit_cc_a1_reg) {
ba86ff93
PC		 2823 if (value & jit_cc_a1_rlh) {
2824 if (!(node->v.q.l & jit_regno_patch)) {
2825 /* Assume register is not changed */
2826 if (jit_regset_tstbit(&regmask, node->v.q.l))
2827 jit_regset_clrbit(&regmask, node->v.q.l);
2828 }
2829 if (!(node->v.q.h & jit_regno_patch)) {
2830 if (jit_regset_tstbit(&regmask, node->v.q.h))
2831 jit_regset_clrbit(&regmask, node->v.q.h);
2832 }
2833 }
2834 else {
2835 if (!(node->v.w & jit_regno_patch)) {
2836 if (jit_regset_tstbit(&regmask, node->v.w)) {
2837 jit_regset_clrbit(&regmask, node->v.w);
2838 if (!(value & jit_cc_a1_chg))
2839 jit_regset_setbit(&reglive, node->v.w);
2840 }
4a71579b
PC		 2841 }
2842 }
2843 }
2844 if (value & jit_cc_a0_reg) {
2845 if (value & jit_cc_a0_rlh) {
2846 if (!(node->u.q.l & jit_regno_patch)) {
2847 if (jit_regset_tstbit(&regmask, node->u.q.l)) {
2848 jit_regset_clrbit(&regmask, node->u.q.l);
2849 if (!(value & jit_cc_a0_chg))
2850 jit_regset_setbit(&reglive, node->u.q.l);
2851 }
2852 }
2853 if (!(node->u.q.h & jit_regno_patch)) {
2854 if (jit_regset_tstbit(&regmask, node->u.q.h)) {
2855 jit_regset_clrbit(&regmask, node->u.q.h);
2856 if (!(value & jit_cc_a0_chg))
2857 jit_regset_setbit(&reglive, node->u.q.h);
2858 }
2859 }
2860 }
2861 else {
2862 if (!(node->u.w & jit_regno_patch)) {
2863 if (jit_regset_tstbit(&regmask, node->u.w)) {
2864 jit_regset_clrbit(&regmask, node->u.w);
2865 if (!(value & jit_cc_a0_chg))
2866 jit_regset_setbit(&reglive, node->u.w);
2867 }
2868 }
2869 }
2870 }
2871 if (value & jit_cc_a0_jmp) {
2872 if (node->flag & jit_flag_node) {
2873 label = node->u.n;
2874 /* Do not consider jmpi and jmpr cannot jump to the
2875 * next instruction. */
2876 next = _jitc->blocks.ptr + label->v.w;
2877 jit_regset_and(&regtemp, &regmask, &next->reglive);
2878 if (jit_regset_set_p(&regtemp)) {
2879 /* Add live state. */
2880 jit_regset_ior(&block->reglive,
2881 &block->reglive, &regtemp);
2882 /* Remove from unknown state bitmask. */
2883 jit_regset_com(&regtemp, &regtemp);
2884 jit_regset_and(&block->regmask,
2885 &block->regmask, &regtemp);
c0c16242 2886 block->again = 1;
4a71579b
PC		 2887 }
2888 }
2889 else {
2890 /* Jump to unknown location.
2891 * This is a pitfall of the implementation.
2892 * Only jmpi to not a jit code should reach here,
2893 * or a jmpr of a computed address.
2894 * Because the implementation needs jit_class_nospill
2895 * registers, must treat jmpr as a function call. This
2896 * means that only JIT_Vn registers can be trusted on
2897 * arrival of jmpr.
2898 */
ba3814c1 2899 jit_regset_set_ui(&regmask, 0);
4a71579b
PC		 2900 for (regno = 0; regno < _jitc->reglen; regno++) {
2901 spec = jit_class(_rvs[regno].spec);
ba3814c1
PC		 2902 if ((spec & (jit_class_gpr|jit_class_fpr)) &&
2903 (spec & jit_class_sav))
2904 jit_regset_setbit(&regmask, regno);
4a71579b
PC		 2905 }
2906 /* Assume non callee save registers are live due
2907 * to jump to unknown location. */
2908 /* Treat all callee save as live. */
ba3814c1 2909 jit_regset_ior(&block->reglive, &reglive, &regmask);
4a71579b 2910 /* Treat anything else as dead. */
ba3814c1 2911 return;
4a71579b
PC		 2912 }
2913 }
2914 break;
2915 }
2916 }
2917}
2918
2919/* Follow code generation up to finding a label or end of code.
2920 * When finding a label, update the set of live registers.
2921 * On branches, update based on taken branch or normal flow.
2922 */
2923static void
2924_jit_update(jit_state_t *_jit, jit_node_t *node,
2925 jit_regset_t *live, jit_regset_t *mask)
2926{
2927 jit_int32_t spec;
2928 jit_int32_t regno;
2929 unsigned long value;
2930 jit_block_t *block;
2931 jit_node_t *label;
2932 jit_regset_t regtemp;
2933
2934 for (; node; node = node->next) {
2935 if (jit_regset_set_p(mask) == 0)
2936 break;
2937 switch (node->code) {
2938 case jit_code_label:
2939 block = _jitc->blocks.ptr + node->v.w;
2940 jit_regset_and(&regtemp, mask, &block->reglive);
2941 if (jit_regset_set_p(&regtemp)) {
2942 /* Add live state. */
2943 jit_regset_ior(live, live, &regtemp);
2944 /* Remove from unknown state bitmask. */
2945 jit_regset_com(&regtemp, &regtemp);
2946 jit_regset_and(mask, mask, &regtemp);
2947 }
2948 return;
2949 case jit_code_prolog:
2950 jit_regset_set_ui(mask, 0);
2951 return;
2952 case jit_code_epilog:
2953 jit_regset_set_ui(mask, 0);
2954 return;
2955 case jit_code_callr:
2956 value = jit_regno(node->u.w);
2957 if (!(node->u.w & jit_regno_patch)) {
2958 if (jit_regset_tstbit(mask, value)) {
2959 jit_regset_clrbit(mask, value);
2960 jit_regset_setbit(live, value);
2961 }
2962 }
2963 case jit_code_calli:
2964 for (value = 0; value < _jitc->reglen; ++value) {
2965 value = jit_regset_scan1(mask, value);
2966 if (value >= _jitc->reglen)
2967 break;
2968 spec = jit_class(_rvs[value].spec);
2969 if (!(spec & jit_class_sav))
2970 jit_regset_clrbit(mask, value);
2971 if ((spec & jit_class_arg) && jit_regarg_p(node, value))
2972 jit_regset_setbit(live, value);
2973 }
2974 break;
2975 default:
2976 value = jit_classify(node->code);
2977 if (value & jit_cc_a2_reg) {
ba3814c1
PC		 2978 if (value & jit_cc_a2_rlh) {
2979 if (!(node->w.q.l & jit_regno_patch)) {
2980 /* Assume register is not changed */
2981 if (jit_regset_tstbit(mask, node->w.q.l))
2982 jit_regset_clrbit(mask, node->w.q.l);
2983 }
2984 if (!(node->w.q.h & jit_regno_patch)) {
2985 if (jit_regset_tstbit(mask, node->w.q.h))
2986 jit_regset_clrbit(mask, node->w.q.h);
2987 }
2988 }
2989 else {
2990 if (!(node->w.w & jit_regno_patch)) {
2991 if (jit_regset_tstbit(mask, node->w.w)) {
2992 jit_regset_clrbit(mask, node->w.w);
2993 if (!(value & jit_cc_a2_chg))
2994 jit_regset_setbit(live, node->w.w);
2995 }
4a71579b
PC		 2996 }
2997 }
2998 }
2999 if (value & jit_cc_a1_reg) {
ba86ff93
PC		 3000 if (value & jit_cc_a1_rlh) {
3001 if (!(node->v.q.l & jit_regno_patch)) {
3002 /* Assume register is not changed */
3003 if (jit_regset_tstbit(mask, node->v.q.l))
3004 jit_regset_clrbit(mask, node->v.q.l);
3005 }
3006 if (!(node->v.q.h & jit_regno_patch)) {
3007 if (jit_regset_tstbit(mask, node->v.q.h))
3008 jit_regset_clrbit(mask, node->v.q.h);
3009 }
3010 }
3011 else {
3012 if (!(node->v.w & jit_regno_patch)) {
3013 if (jit_regset_tstbit(mask, node->v.w)) {
3014 jit_regset_clrbit(mask, node->v.w);
3015 if (!(value & jit_cc_a1_chg))
3016 jit_regset_setbit(live, node->v.w);
3017 }
4a71579b
PC		 3018 }
3019 }
3020 }
3021 if (value & jit_cc_a0_reg) {
3022 if (value & jit_cc_a0_rlh) {
3023 if (!(node->u.q.l & jit_regno_patch)) {
3024 if (jit_regset_tstbit(mask, node->u.q.l)) {
3025 jit_regset_clrbit(mask, node->u.q.l);
3026 if (!(value & jit_cc_a0_chg))
3027 jit_regset_setbit(live, node->u.q.l);
3028 }
3029 }
3030 if (!(node->u.q.h & jit_regno_patch)) {
3031 if (jit_regset_tstbit(mask, node->u.q.h)) {
3032 jit_regset_clrbit(mask, node->u.q.h);
3033 if (!(value & jit_cc_a0_chg))
3034 jit_regset_setbit(live, node->u.q.h);
3035 }
3036 }
3037 }
3038 else {
3039 if (!(node->u.w & jit_regno_patch)) {
3040 if (jit_regset_tstbit(mask, node->u.w)) {
3041 jit_regset_clrbit(mask, node->u.w);
3042 if (!(value & jit_cc_a0_chg))
3043 jit_regset_setbit(live, node->u.w);
3044 }
3045 }
3046 }
3047 }
3048 if (value & jit_cc_a0_jmp) {
3049 if (node->flag & jit_flag_node) {
3050 label = node->u.n;
3051 /* Do not consider jmpi and jmpr cannot jump to the
3052 * next instruction. */
3053 block = _jitc->blocks.ptr + label->v.w;
3054 jit_regset_and(&regtemp, mask, &block->reglive);
3055 if (jit_regset_set_p(&regtemp)) {
3056 /* Add live state. */
3057 jit_regset_ior(live, live, &regtemp);
3058 /* Remove from unknown state bitmask. */
3059 jit_regset_com(&regtemp, &regtemp);
3060 jit_regset_and(mask, mask, &regtemp);
3061 }
3062 }
3063 else {
3064 /* Jump to unknown location.
3065 * This is a pitfall of the implementation.
3066 * Only jmpi to not a jit code should reach here,
3067 * or a jmpr of a computed address.
3068 * Because the implementation needs jit_class_nospill
3069 * registers, must treat jmpr as a function call. This
3070 * means that only JIT_Vn registers can be trusted on
3071 * arrival of jmpr.
3072 */
ba3814c1 3073 jit_regset_set_ui(mask, 0);
4a71579b
PC		 3074 for (regno = 0; regno < _jitc->reglen; regno++) {
3075 spec = jit_class(_rvs[regno].spec);
ba3814c1
PC		 3076 if ((spec & (jit_class_gpr|jit_class_fpr)) &&
3077 (spec & jit_class_sav))
3078 jit_regset_setbit(mask, regno);
4a71579b
PC		 3079 }
3080 /* Assume non callee save registers are live due
3081 * to jump to unknown location. */
3082 /* Treat all callee save as live. */
3083 jit_regset_ior(live, live, mask);
79bfeef6
PC		 3084 /* Prevent explicitly set as live registers to
3085 * be used as a temporary for the jmpi. */
3086 jit_regset_ior(live, live, &_jitc->explive);
4a71579b 3087 /* Treat anything else as dead. */
ba3814c1 3088 return;
4a71579b
PC		 3089 }
3090 }
3091 break;
3092 }
3093 }
3094}
3095
3096static void
3097_thread_jumps(jit_state_t *_jit)
3098{
3099 jit_node_t *prev;
3100 jit_node_t *node;
3101 jit_node_t *next;
3102 jit_int32_t mask;
3103
3104 for (prev = node = _jitc->head; node;) {
3105 next = node->next;
3106 switch (node->code) {
3107 case jit_code_jmpi:
3108 if (redundant_jump(prev, node)) {
3109 node = prev;
3110 continue;
3111 }
3112 if (shortcut_jump(prev, node))
3113 continue;
3114 break;
3115 case jit_code_jmpr:
3116 case jit_code_callr: case jit_code_calli:
3117 /* non optimizable jump like code */
3118 break;
3119 default:
3120 mask = jit_classify(node->code);
3121 if (mask & jit_cc_a0_jmp) {
3122 if (reverse_jump(prev, node) ||
3123 shortcut_jump(prev, node))
3124 continue;
3125 }
3126 break;
3127 }
3128 prev = node;
3129 node = next;
3130 }
3131}
3132
3133static void
3134_sequential_labels(jit_state_t *_jit)
3135{
3136 jit_node_t *jump;
3137 jit_node_t *link;
3138 jit_node_t *prev;
3139 jit_node_t *next;
3140 jit_node_t *node;
3141
3142 for (prev = node = _jitc->head; node; node = next) {
3143 next = node->next;
3144 if (node->code == jit_code_label) {
3145 if (!node->flag) {
3146 if (!node->link) {
3147 del_label(prev, node);
3148 continue;
3149 }
3150 if (prev != node && prev->code == jit_code_label) {
3151 if ((jump = node->link)) {
3152 for (; jump; jump = link) {
3153 link = jump->link;
79bfeef6
PC		 3154 if (jump->code == jit_code_movi)
3155 jump->v.n = prev;
3156 else
3157 jump->u.n = prev;
4a71579b
PC		 3158 jump->link = prev->link;
3159 prev->link = jump;
3160 }
3161 node->link = NULL;
3162 }
3163 del_label(prev, node);
3164 continue;
3165 }
3166 }
3167 if (next && next->code == jit_code_label && !next->flag) {
3168 if ((jump = next->link)) {
3169 for (; jump; jump = link) {
3170 link = jump->link;
79bfeef6
PC		 3171 if (jump->code == jit_code_movi)
3172 jump->v.n = node;
3173 else
3174 jump->u.n = node;
4a71579b
PC		 3175 jump->link = node->link;
3176 node->link = jump;
3177 }
3178 next->link = NULL;
3179 }
3180 del_label(node, next);
3181 next = node->next;
3182 continue;
3183 }
3184 }
3185 prev = node;
3186 }
3187}
3188
3189static void
3190_split_branches(jit_state_t *_jit)
3191{
3192 jit_node_t *node;
3193 jit_node_t *next;
3194 jit_node_t *label;
3195 jit_block_t *block;
24d91c0d
PC		 3196 jit_block_t *blocks;
3197 jit_word_t offset;
3198 jit_word_t length;
4a71579b 3199
24d91c0d
PC		 3200 length = _jitc->blocks.length;
3201 jit_alloc((jit_pointer_t *)&blocks, length * sizeof(jit_block_t));
3202 if ((node = _jitc->head) &&
3203 (node->code == jit_code_label || node->code == jit_code_prolog)) {
3204 block = _jitc->blocks.ptr + node->v.w;
3205 memcpy(blocks, block, sizeof(jit_block_t));
3206 node->v.w = 0;
3207 offset = 1;
3208 }
3209 else
3210 offset = 0;
4a71579b
PC		 3211 for (node = _jitc->head; node; node = next) {
3212 if ((next = node->next)) {
3213 if (next->code == jit_code_label ||
3214 next->code == jit_code_prolog ||
24d91c0d
PC		 3215 next->code == jit_code_epilog) {
3216 if (offset >= length) {
3217 jit_realloc((jit_pointer_t *)&blocks,
3218 length * sizeof(jit_block_t),
3219 (length + 16) * sizeof(jit_block_t));
3220 length += 16;
3221 }
3222 block = _jitc->blocks.ptr + next->v.w;
3223 memcpy(blocks + offset, block, sizeof(jit_block_t));
3224 next->v.w = offset++;
3225 }
4a71579b 3226 /* split block on branches */
24d91c0d 3227 else if (jit_classify(node->code) & jit_cc_a0_jmp) {
4a71579b
PC		 3228 label = new_node(jit_code_label);
3229 label->next = next;
3230 node->next = label;
24d91c0d
PC		 3231 if (offset >= length) {
3232 jit_realloc((jit_pointer_t *)&blocks,
3233 length * sizeof(jit_block_t),
3234 (length + 16) * sizeof(jit_block_t));
3235 length += 16;
4a71579b 3236 }
24d91c0d 3237 block = blocks + offset;
4a71579b 3238 block->label = label;
24d91c0d 3239 label->v.w = offset++;
4a71579b
PC		 3240 jit_regset_new(&block->reglive);
3241 jit_regset_new(&block->regmask);
4a71579b
PC		 3242 }
3243 }
3244 }
24d91c0d
PC		 3245 jit_free((jit_pointer_t *)&_jitc->blocks.ptr);
3246 _jitc->blocks.ptr = blocks;
3247 _jitc->blocks.offset = offset;
3248 _jitc->blocks.length = length;
4a71579b
PC		 3249}
3250
3251static jit_bool_t
3252_shortcut_jump(jit_state_t *_jit, jit_node_t *prev, jit_node_t *node)
3253{
3254 jit_bool_t cond;
3255 jit_node_t *jump;
3256 jit_node_t *next;
3257 jit_node_t *temp;
3258
3259 if (!(node->flag & jit_flag_node))
3260 return (0);
3261 assert(node->code != jit_code_jmpr);
3262 cond = node->code != jit_code_jmpi;
3263 jump = node->u.n;
3264 for (next = jump->next; next; next = next->next) {
3265 switch (next->code) {
3266 case jit_code_jmpi:
3267 if (!(next->flag & jit_flag_node))
3268 return (0);
3269 if (jump->link == node)
3270 jump->link = node->link;
3271 else {
3272 for (temp = jump->link;
3273 temp->link != node;
3274 temp = temp->link)
3275 assert(temp != NULL);
3276 temp->link = node->link;
3277 }
3278 jump = next->u.n;
3279 node->u.n = jump;
3280 node->link = jump->link;
3281 jump->link = node;
3282 return (1);
3283 case jit_code_jmpr:
3284 if (cond)
3285 return (0);
3286 node->code = jit_code_jmpr;
3287 node->u.w = next->u.w;
3288 node->link = NULL;
3289 node->flag &= ~jit_flag_node;
3290 return (1);
3291 case jit_code_note: case jit_code_label:
3292 break;
3293 default:
3294 return (0);
3295 }
3296 }
3297 return (0);
3298}
3299
3300static jit_bool_t
3301_redundant_jump(jit_state_t *_jit, jit_node_t *prev, jit_node_t *node)
3302{
3303 jit_node_t *local_prev;
3304 jit_node_t *local_next;
3305
3306 if (!(node->flag & jit_flag_node))
3307 return (0);
3308 for (local_prev = node, local_next = node->next;
3309 local_next;
3310 local_prev = local_next, local_next = local_next->next) {
3311
3312 switch (local_next->code) {
3313 case jit_code_label: case jit_code_epilog:
3314 if (node->u.n == local_next) {
3315 if (local_next->link == node)
3316 local_next->link = node->link;
3317 else {
3318 for (local_prev = local_next->link;
3319 local_prev->link != node;
3320 local_prev = local_prev->link)
3321 assert(local_prev != NULL);
3322 local_prev->link = node->link;
3323 }
3324 del_node(prev, node);
3325 return (1);
3326 }
3327 break;
3328 case jit_code_name: case jit_code_note:
4a71579b
PC		 3329 break;
3330 default:
3331 return (0);
3332 }
3333 }
3334 return (0);
3335}
3336
3337static jit_code_t
3338reverse_jump_code(jit_code_t code)
3339{
3340 switch (code) {
3341 case jit_code_bltr: return (jit_code_bger);
3342 case jit_code_blti: return (jit_code_bgei);
3343 case jit_code_bltr_u: return (jit_code_bger_u);
3344 case jit_code_blti_u: return (jit_code_bgei_u);
3345 case jit_code_bler: return (jit_code_bgtr);
3346 case jit_code_blei: return (jit_code_bgti);
3347 case jit_code_bler_u: return (jit_code_bgtr_u);
3348 case jit_code_blei_u: return (jit_code_bgti_u);
3349 case jit_code_beqr: return (jit_code_bner);
3350 case jit_code_beqi: return (jit_code_bnei);
3351 case jit_code_bger: return (jit_code_bltr);
3352 case jit_code_bgei: return (jit_code_blti);
3353 case jit_code_bger_u: return (jit_code_bltr_u);
3354 case jit_code_bgei_u: return (jit_code_blti_u);
3355 case jit_code_bgtr: return (jit_code_bler);
3356 case jit_code_bgti: return (jit_code_blei);
3357 case jit_code_bgtr_u: return (jit_code_bler_u);
3358 case jit_code_bgti_u: return (jit_code_blei_u);
3359 case jit_code_bner: return (jit_code_beqr);
3360 case jit_code_bnei: return (jit_code_beqi);
3361 case jit_code_bmsr: return (jit_code_bmcr);
3362 case jit_code_bmsi: return (jit_code_bmci);
3363 case jit_code_bmcr: return (jit_code_bmsr);
3364 case jit_code_bmci: return (jit_code_bmsi);
3365 case jit_code_bltr_f: return (jit_code_bunger_f);
3366 case jit_code_blti_f: return (jit_code_bungei_f);
3367 case jit_code_bler_f: return (jit_code_bungtr_f);
3368 case jit_code_blei_f: return (jit_code_bungti_f);
3369
3370 case jit_code_beqr_f: return (jit_code_bner_f);
3371 case jit_code_beqi_f: return (jit_code_bnei_f);
3372
3373 case jit_code_bger_f: return (jit_code_bunltr_f);
3374 case jit_code_bgei_f: return (jit_code_bunlti_f);
3375 case jit_code_bgtr_f: return (jit_code_bunler_f);
3376 case jit_code_bgti_f: return (jit_code_bunlei_f);
3377
3378 case jit_code_bner_f: return (jit_code_beqr_f);
79bfeef6 3379 case jit_code_bnei_f: return (jit_code_beqi_f);
4a71579b
PC		 3380
3381 case jit_code_bunltr_f: return (jit_code_bger_f);
3382 case jit_code_bunlti_f: return (jit_code_bgei_f);
3383 case jit_code_bunler_f: return (jit_code_bgtr_f);
3384 case jit_code_bunlei_f: return (jit_code_bgti_f);
3385
3386 case jit_code_buneqr_f: return (jit_code_bltgtr_f);
3387 case jit_code_buneqi_f: return (jit_code_bltgti_f);
3388
3389 case jit_code_bunger_f: return (jit_code_bltr_f);
3390 case jit_code_bungei_f: return (jit_code_blti_f);
3391 case jit_code_bungtr_f: return (jit_code_bler_f);
3392 case jit_code_bungti_f: return (jit_code_blei_f);
3393
3394 case jit_code_bltgtr_f: return (jit_code_buneqr_f);
3395 case jit_code_bltgti_f: return (jit_code_buneqi_f);
3396
3397 case jit_code_bordr_f: return (jit_code_bunordr_f);
3398 case jit_code_bordi_f: return (jit_code_bunordi_f);
3399 case jit_code_bunordr_f:return (jit_code_bordr_f);
3400 case jit_code_bunordi_f:return (jit_code_bordi_f);
3401 case jit_code_bltr_d: return (jit_code_bunger_d);
3402 case jit_code_blti_d: return (jit_code_bungei_d);
3403 case jit_code_bler_d: return (jit_code_bungtr_d);
3404 case jit_code_blei_d: return (jit_code_bungti_d);
3405
3406 case jit_code_beqr_d: return (jit_code_bner_d);
3407 case jit_code_beqi_d: return (jit_code_bnei_d);
3408
3409 case jit_code_bger_d: return (jit_code_bunltr_d);
3410 case jit_code_bgei_d: return (jit_code_bunlti_d);
3411 case jit_code_bgtr_d: return (jit_code_bunler_d);
3412 case jit_code_bgti_d: return (jit_code_bunlei_d);
3413
3414 case jit_code_bner_d: return (jit_code_beqr_d);
3415 case jit_code_bnei_d: return (jit_code_beqi_d);
3416
3417 case jit_code_bunltr_d: return (jit_code_bger_d);
3418 case jit_code_bunlti_d: return (jit_code_bgei_d);
3419 case jit_code_bunler_d: return (jit_code_bgtr_d);
3420 case jit_code_bunlei_d: return (jit_code_bgti_d);
3421
3422 case jit_code_buneqr_d: return (jit_code_bltgtr_d);
3423 case jit_code_buneqi_d: return (jit_code_bltgti_d);
3424
3425 case jit_code_bunger_d: return (jit_code_bltr_d);
3426 case jit_code_bungei_d: return (jit_code_blti_d);
3427 case jit_code_bungtr_d: return (jit_code_bler_d);
3428 case jit_code_bungti_d: return (jit_code_blei_d);
3429
3430 case jit_code_bltgtr_d: return (jit_code_buneqr_d);
3431 case jit_code_bltgti_d: return (jit_code_buneqi_d);
3432
3433 case jit_code_bordr_d: return (jit_code_bunordr_d);
3434 case jit_code_bordi_d: return (jit_code_bunordi_d);
3435 case jit_code_bunordr_d:return (jit_code_bordr_d);
3436 case jit_code_bunordi_d:return (jit_code_bordi_d);
3437 case jit_code_boaddr: return (jit_code_bxaddr);
3438 case jit_code_boaddi: return (jit_code_bxaddi);
3439 case jit_code_boaddr_u: return (jit_code_bxaddr_u);
3440 case jit_code_boaddi_u: return (jit_code_bxaddi_u);
3441 case jit_code_bxaddr: return (jit_code_boaddr);
3442 case jit_code_bxaddi: return (jit_code_boaddi);
3443 case jit_code_bxaddr_u: return (jit_code_boaddr_u);
3444 case jit_code_bxaddi_u: return (jit_code_boaddi_u);
3445 case jit_code_bosubr: return (jit_code_bxsubr);
3446 case jit_code_bosubi: return (jit_code_bxsubi);
3447 case jit_code_bosubr_u: return (jit_code_bxsubr_u);
3448 case jit_code_bosubi_u: return (jit_code_bxsubi_u);
3449 case jit_code_bxsubr: return (jit_code_bosubr);
3450 case jit_code_bxsubi: return (jit_code_bosubi);
3451 case jit_code_bxsubr_u: return (jit_code_bosubr_u);
3452 case jit_code_bxsubi_u: return (jit_code_bosubi_u);
3453 default: abort(); /* invalid jump code */
3454 }
3455}
3456
3457/*
3458 * change common pattern:
3459 * <cond_jump L0> <jump L1> <label L0>
3460 * into
3461 * <reverse_cond_jump L1>
3462 */
3463static jit_bool_t
3464_reverse_jump(jit_state_t *_jit, jit_node_t *prev, jit_node_t *node)
3465{
3466 jit_node_t *local_prev;
3467 jit_node_t *local_next;
3468 jit_node_t *local_jump;
3469
3470 if (!(node->flag & jit_flag_node))
3471 return (0);
3472 /* =><cond_jump L0> <jump L1> <label L0> */
3473 local_next = node->next;
3474 if (local_next->code != jit_code_jmpi ||
3475 !(local_next->flag & jit_flag_node))
3476 return (0);
3477 /* <cond_jump L0> =><jump L1> <label L0> */
3478
3479 local_jump = local_next->u.n;
3480 for (local_prev = local_next, local_next = local_next->next;
3481 local_next;
3482 local_prev = local_next, local_next = local_next->next) {
3483 switch (local_next->code) {
3484 case jit_code_label: case jit_code_epilog:
3485 if (node->u.n == local_next) {
3486 if (local_next->link == node)
3487 local_next->link = node->link;
3488 else {
3489 for (local_prev = local_next->link;
3490 local_prev->link != node;
3491 local_prev = local_prev->link)
3492 assert(local_prev != NULL);
3493 local_prev->link = node->link;
3494 }
3495 del_node(node, node->next);
3496 node->code = reverse_jump_code(node->code);
3497 node->u.n = local_jump;
3498 node->link = local_jump->link;
3499 local_jump->link = node;
3500 return (1);
3501 }
3502 break;
3503 case jit_code_note:
3504 break;
3505 default:
3506 return (0);
3507 }
3508 }
3509 return (0);
3510}
3511
ba3814c1 3512static jit_bool_t
4a71579b
PC		 3513_redundant_store(jit_state_t *_jit, jit_node_t *node, jit_bool_t jump)
3514{
3515 jit_node_t *iter;
3516 jit_node_t *prev;
3517 jit_word_t word;
3518 jit_int32_t spec;
3519 jit_int32_t regno;
ba3814c1 3520 jit_bool_t result;
4a71579b
PC		 3521
3522 if (jump) {
3523 prev = node->u.n;
3524 if (prev->code == jit_code_epilog)
ba3814c1 3525 return (0);
4a71579b
PC		 3526 assert(prev->code == jit_code_label);
3527 if ((prev->flag & jit_flag_head) || node->link || prev->link != node)
3528 /* multiple sources */
ba3814c1 3529 return (0);
4a71579b
PC		 3530 /* if there are sequential labels it will return below */
3531 }
3532 else
3533 prev = node;
ba3814c1 3534 result = 0;
4a71579b
PC		 3535 word = node->w.w;
3536 regno = jit_regno(node->v.w);
3537 for (iter = prev->next; iter; prev = iter, iter = iter->next) {
3538 switch (iter->code) {
3539 case jit_code_label: case jit_code_prolog:
3540 case jit_code_epilog:
ba3814c1 3541 return (result);
4a71579b
PC		 3542 case jit_code_movi:
3543 if (regno == jit_regno(iter->u.w)) {
3544 if (iter->flag || iter->v.w != word)
ba3814c1
PC		 3545 return (result);
3546 result = 1;
4a71579b
PC		 3547 del_node(prev, iter);
3548 iter = prev;
3549 }
3550 break;
3551 default:
3552 spec = jit_classify(iter->code);
3553 if (spec & jit_cc_a0_jmp)
ba3814c1 3554 return (result);
4a71579b
PC		 3555 if ((spec & (jit_cc_a0_reg|jit_cc_a0_chg)) ==
3556 (jit_cc_a0_reg|jit_cc_a0_chg)) {
3557 if (spec & jit_cc_a0_rlh) {
3558 if (regno == jit_regno(iter->u.q.l) ||
3559 regno == jit_regno(iter->u.q.h))
ba3814c1 3560 return (result);
4a71579b
PC		 3561 }
3562 else {
3563 if (regno == jit_regno(iter->u.w))
ba3814c1 3564 return (result);
4a71579b
PC		 3565 }
3566 }
3567 if ((spec & (jit_cc_a1_reg|jit_cc_a1_chg)) ==
3568 (jit_cc_a1_reg|jit_cc_a1_chg)) {
3569 if (regno == jit_regno(iter->v.w))
ba3814c1 3570 return (result);
4a71579b
PC		 3571 }
3572 if ((spec & (jit_cc_a2_reg|jit_cc_a2_chg)) ==
3573 (jit_cc_a2_reg|jit_cc_a2_chg)) {
3574 if (regno == jit_regno(iter->w.w))
ba3814c1 3575 return (result);
4a71579b
PC		 3576 }
3577 break;
3578 }
3579 }
c0c16242
PC		 3580
3581 return (result);
4a71579b
PC		 3582}
3583
3584static jit_bool_t
3585_simplify_movr(jit_state_t *_jit, jit_node_t *prev, jit_node_t *node,
3586 jit_int32_t kind, jit_int32_t size)
3587{
3588 jit_int32_t regno;
3589 jit_int32_t right;
ba86ff93 3590 jit_int32_t spec;
4a71579b
PC		 3591 jit_value_t *value;
3592
3593 regno = jit_regno(node->u.w);
3594 right = jit_regno(node->v.w);
3595 value = _jitc->values + regno;
ba86ff93
PC		 3596
3597 spec = jit_class(_rvs[regno].spec);
3598 if (!(spec & (jit_class_gpr | jit_class_fpr))) {
3599 /* reserved register */
3600 value->kind = 0;
3601 ++_jitc->gen[regno];
3602 return (0);
3603 }
3604
4a71579b
PC		 3605 if ((value->kind == jit_kind_register &&
3606 jit_regno(value->base.q.l) == right &&
3607 value->base.q.h == _jitc->gen[right]) ||
3608 (value->kind == kind && _jitc->values[right].kind == kind &&
3609 memcmp(&value->base.w, &_jitc->values[right].base.w, size) == 0)) {
3610 del_node(prev, node);
3611 return (1);
3612 }
3613 if (_jitc->values[right].kind == jit_kind_word)
3614 jit_memcpy(value, _jitc->values + right, sizeof(jit_value_t));
3615 else {
3616 value->kind = jit_kind_register;
3617 value->base.q.l = right;
3618 value->base.q.h = _jitc->gen[right];
3619 }
3620 ++_jitc->gen[regno];
3621
3622 return (0);
3623}
3624
3625static jit_bool_t
3626_simplify_movi(jit_state_t *_jit, jit_node_t *prev, jit_node_t *node,
3627 jit_int32_t kind, jit_int32_t size)
3628{
3629 jit_value_t *value;
3630 jit_int32_t spec;
3631 jit_int32_t regno;
3632 jit_int32_t offset;
3633
3634 regno = jit_regno(node->u.w);
3635 value = _jitc->values + regno;
3636 if (node->flag & jit_flag_node) {
3637 /* set to undefined if value will be patched */
3638 value->kind = 0;
3639 ++_jitc->gen[regno];
3640 return (0);
3641 }
ba86ff93
PC		 3642 spec = jit_class(_rvs[regno].spec);
3643 if (!(spec & (jit_class_gpr | jit_class_fpr))) {
3644 /* reserved register */
3645 value->kind = 0;
3646 ++_jitc->gen[regno];
3647 return (0);
3648 }
4a71579b
PC		 3649 if (value->kind == kind) {
3650 if (memcmp(&node->v.w, &value->base.w, size) == 0) {
3651 del_node(prev, node);
3652 return (1);
3653 }
4a71579b
PC		 3654 if (kind == jit_kind_word)
3655 spec &= jit_class_gpr;
3656 else
3657 spec &= (jit_class_xpr | jit_class_fpr);
3658 for (offset = 0; offset < _jitc->reglen; offset++) {
3659 if (_jitc->values[offset].kind == kind &&
3660 memcmp(&node->v.w, &_jitc->values[offset].base.w, size) == 0 &&
3661 (jit_class(_rvs[offset].spec) & spec) == spec) {
3662 if (kind == jit_kind_word)
3663 node->code = jit_code_movr;
3664 else if (kind == jit_kind_float32)
3665 node->code = jit_code_movr_f;
3666 else
3667 node->code = jit_code_movr_d;
3668 node->v.w = offset;
3669 jit_memcpy(value, _jitc->values + offset, sizeof(jit_value_t));
3670 ++_jitc->gen[regno];
3671 return (0);
3672 }
3673 }
3674 }
3675 value->kind = kind;
3676 jit_memcpy(&value->base.w, &node->v.w, size);
3677 ++_jitc->gen[regno];
3678
3679 return (0);
3680}
3681
3682/* simple/safe redundandy test not checking if another register
3683 * holds the same value
3684 */
3685static jit_bool_t
3686_simplify_ldxi(jit_state_t *_jit, jit_node_t *prev, jit_node_t *node)
3687{
3688 jit_value_t *value;
3689 jit_int32_t regno;
3690 jit_int32_t right;
3691
3692 regno = jit_regno(node->u.w);
3693 right = jit_regno(node->v.w);
3694 value = _jitc->values + regno;
3695 if (regno != right &&
3696 value->kind == jit_kind_code && value->code == node->code &&
3697 value->base.q.l == right && value->base.q.h == _jitc->gen[right] &&
3698 node->w.w == value->disp.w) {
3699 del_node(prev, node);
3700 return (1);
3701 }
3702 value->kind = jit_kind_code;
3703 value->code = node->code;
3704 value->base.q.l = right;
3705 value->base.q.h = _jitc->gen[right];
3706 value->disp.w = node->w.w;
3707 ++_jitc->gen[regno];
3708
3709 return (0);
3710}
3711
3712static jit_bool_t
3713_simplify_stxi(jit_state_t *_jit, jit_node_t *prev, jit_node_t *node)
3714{
3715 jit_value_t *value;
3716 jit_int32_t regno;
3717 jit_int32_t right;
3718 jit_int32_t offset;
3719
3720 regno = jit_regno(node->w.w);
3721 right = jit_regno(node->v.w);
3722 value = _jitc->values + regno;
3723
3724 /* check for redundant store after load */
3725 if (regno != right &&
3726 value->kind == jit_kind_code && value->code == node->code &&
3727 value->base.q.l == right && value->base.q.h == _jitc->gen[right] &&
3728 node->u.w == value->disp.w) {
3729 del_node(prev, node);
3730 return (1);
3731 }
3732
3733 /* assume anything can alias, and invalidate tracked values */
3734 for (offset = 0; offset < _jitc->reglen; offset++) {
3735 if (_jitc->values[offset].kind == jit_kind_code) {
3736 _jitc->values[offset].kind = 0;
3737 ++_jitc->gen[offset];
3738 }
3739 }
3740
3741 /* no multiple information, so, if set to a constant,
3742 * prefer to keep that information */
3743 if (value->kind == 0) {
4a71579b
PC		 3744 switch (node->code) {
3745 /* no information about signed/unsigned either */
3746 case jit_code_stxi_c: value->code = jit_code_ldxi_c; break;
3747 case jit_code_stxi_s: value->code = jit_code_ldxi_s; break;
3748 case jit_code_stxi_i: value->code = jit_code_ldxi_i; break;
3749 case jit_code_stxi_l: value->code = jit_code_ldxi_l; break;
3750 case jit_code_stxi_f: value->code = jit_code_ldxi_f; break;
3751 case jit_code_stxi_d: value->code = jit_code_ldxi_d; break;
3752 default: abort();
3753 }
3754 value->kind = jit_kind_code;
3755 value->base.q.l = right;
3756 value->base.q.h = _jitc->gen[right];
3757 value->disp.w = node->u.w;
3758 }
3759
3760 return (0);
3761}
3762
3763/* usually there should be only one store in the
3764 * jit_get_reg/jit_unget_reg, but properly handle
3765 * multiple ones by moving the save node */
3766static void
3767_simplify_spill(jit_state_t *_jit, jit_node_t *node, jit_int32_t regno)
3768{
3769 jit_node_t *save;
3770 jit_node_t *temp;
3771
3772 if ((temp = _jitc->spill[regno]) && (save = temp->next) != node) {
3773 temp->next = save->next;
3774 save->next = node->next;
3775 node->next = save;
3776 _jitc->spill[regno] = node;
3777 }
3778}
3779
3780/* checks for simple cases where a register is set more than
3781 * once to the same value, and is a common pattern of calls
3782 * to jit_pushargi and jit_pushargr
3783 */
ba3814c1 3784static jit_bool_t
4a71579b
PC		 3785_simplify(jit_state_t *_jit)
3786{
3787 jit_node_t *prev;
3788 jit_node_t *node;
3789 jit_node_t *next;
3790 jit_int32_t info;
3791 jit_int32_t regno;
ba3814c1 3792 jit_bool_t result;
4a71579b 3793
ba3814c1 3794 result = 0;
4a71579b
PC		 3795 for (prev = NULL, node = _jitc->head; node; prev = node, node = next) {
3796 next = node->next;
3797 switch (node->code) {
3798 case jit_code_label: case jit_code_prolog:
3799 case jit_code_callr: case jit_code_calli:
3800 reset:
3801 memset(_jitc->gen, 0, sizeof(jit_int32_t) * _jitc->reglen);
3802 memset(_jitc->values, 0, sizeof(jit_value_t) * _jitc->reglen);
3803 break;
3804 case jit_code_save:
3805 _jitc->spill[jit_regno(node->u.w)] = prev;
3806 break;
3807 case jit_code_load:
3808 regno = jit_regno(node->u.w);
3809 if (register_change_p(node->link->next, node, regno) !=
3810 jit_reg_change) {
3811 /* spill not required due to optimizing common
3812 * redundancy case of calling jit_get_reg/jit_unget_reg
3813 * and then setting the register to the value it is
3814 * already holding */
3815 patch_register(node->link->next, node,
3816 jit_regno_patch|regno, regno);
ba3814c1 3817 result = 1;
4a71579b
PC		 3818 del_node(_jitc->spill[regno], node->link);
3819 del_node(prev, node);
3820 node = prev;
3821 }
3822 _jitc->spill[regno] = NULL;
3823 break;
3824 case jit_code_movr:
3825 regno = jit_regno(node->u.w);
3826 if (simplify_movr(prev, node,
ba3814c1
PC		 3827 jit_kind_word, sizeof(jit_word_t))) {
3828 result = 1;
4a71579b 3829 simplify_spill(node = prev, regno);
ba3814c1 3830 }
4a71579b
PC		 3831 break;
3832 case jit_code_movi:
3833 regno = jit_regno(node->u.w);
3834 if (simplify_movi(prev, node,
ba3814c1
PC		 3835 jit_kind_word, sizeof(jit_word_t))) {
3836 result = 1;
4a71579b 3837 simplify_spill(node = prev, regno);
ba3814c1 3838 }
4a71579b
PC		 3839 break;
3840 case jit_code_movr_f:
3841 regno = jit_regno(node->u.w);
3842 if (simplify_movr(prev, node,
ba3814c1
PC		 3843 jit_kind_float32, sizeof(jit_float32_t))) {
3844 result = 1;
4a71579b 3845 simplify_spill(node = prev, regno);
ba3814c1 3846 }
4a71579b
PC		 3847 break;
3848 case jit_code_movi_f:
3849 regno = jit_regno(node->u.w);
3850 if (simplify_movi(prev, node,
ba3814c1
PC		 3851 jit_kind_float32, sizeof(jit_float32_t))) {
3852 result = 1;
4a71579b 3853 simplify_spill(node = prev, regno);
ba3814c1 3854 }
4a71579b
PC		 3855 break;
3856 case jit_code_movr_d:
3857 regno = jit_regno(node->u.w);
3858 if (simplify_movr(prev, node,
ba3814c1
PC		 3859 jit_kind_float64, sizeof(jit_float64_t))) {
3860 result = 1;
4a71579b 3861 simplify_spill(node = prev, regno);
ba3814c1 3862 }
4a71579b
PC		 3863 break;
3864 case jit_code_movi_d:
3865 regno = jit_regno(node->u.w);
3866 if (simplify_movi(prev, node,
ba3814c1
PC		 3867 jit_kind_float64, sizeof(jit_float64_t))) {
3868 result = 1;
4a71579b 3869 simplify_spill(node = prev, regno);
ba3814c1 3870 }
4a71579b
PC		 3871 break;
3872 case jit_code_ldxi_c: case jit_code_ldxi_uc:
3873 case jit_code_ldxi_s: case jit_code_ldxi_us:
3874 case jit_code_ldxi_i: case jit_code_ldxi_ui:
3875 case jit_code_ldxi_l:
3876 case jit_code_ldxi_f: case jit_code_ldxi_d:
3877 regno = jit_regno(node->u.w);
ba3814c1
PC		 3878 if (simplify_ldxi(prev, node)) {
3879 result = 1;
4a71579b 3880 simplify_spill(node = prev, regno);
ba3814c1 3881 }
4a71579b
PC		 3882 break;
3883 case jit_code_stxi_c: case jit_code_stxi_s:
3884 case jit_code_stxi_i: case jit_code_stxi_l:
3885 case jit_code_stxi_f: case jit_code_stxi_d:
3886 regno = jit_regno(node->u.w);
ba3814c1
PC		 3887 if (simplify_stxi(prev, node)) {
3888 result = 1;
4a71579b 3889 simplify_spill(node = prev, regno);
ba3814c1 3890 }
4a71579b
PC		 3891 break;
3892 default:
3893 info = jit_classify(node->code);
3894 if (info & jit_cc_a0_jmp)
3895 /* labels are not implicitly added when not taking
3896 * a conditional branch */
3897 goto reset;
3898 if (info & jit_cc_a0_chg) {
3899 if (info & jit_cc_a0_rlh) {
3900 regno = jit_regno(node->u.q.l);
3901 _jitc->values[regno].kind = 0;
3902 ++_jitc->gen[regno];
3903 regno = jit_regno(node->u.q.h);
3904 _jitc->values[regno].kind = 0;
3905 ++_jitc->gen[regno];
3906 }
3907 else {
3908 regno = jit_regno(node->u.w);
3909 _jitc->values[regno].kind = 0;
3910 ++_jitc->gen[regno];
3911 }
3912 }
3913 if (info & jit_cc_a1_chg) {
ba86ff93
PC		 3914#if 0
3915 /* Assume registers are not changed */
3916 if (info & jit_cc_a1_rlh) {
3917 regno = jit_regno(node->v.q.l);
3918 _jitc->values[regno].kind = 0;
3919 ++_jitc->gen[regno];
3920 regno = jit_regno(node->v.q.h);
3921 _jitc->values[regno].kind = 0;
3922 ++_jitc->gen[regno];
3923 }
3924 else {
3925#endif
3926 regno = jit_regno(node->v.w);
3927 _jitc->values[regno].kind = 0;
3928 ++_jitc->gen[regno];
3929#if 0
3930 }
3931#endif
4a71579b
PC		 3932 }
3933 if (info & jit_cc_a2_chg) {
ba3814c1
PC		 3934#if 0
3935 /* Assume registers are not changed */
3936 if (info & jit_cc_a2_rlh) {
3937 regno = jit_regno(node->w.q.l);
3938 _jitc->values[regno].kind = 0;
3939 ++_jitc->gen[regno];
3940 regno = jit_regno(node->w.q.h);
3941 _jitc->values[regno].kind = 0;
3942 ++_jitc->gen[regno];
3943 }
3944 else {
3945#endif
3946 regno = jit_regno(node->w.w);
3947 _jitc->values[regno].kind = 0;
3948 ++_jitc->gen[regno];
3949#if 0
3950 }
3951#endif
4a71579b
PC		 3952 }
3953 break;
3954 }
3955 }
ba3814c1 3956 return (result);
4a71579b
PC		 3957}
3958
3959static jit_int32_t
3960_register_change_p(jit_state_t *_jit, jit_node_t *node, jit_node_t *link,
3961 jit_int32_t regno)
3962{
3963 jit_int32_t value;
ba86ff93 3964 jit_bool_t use, change;
4a71579b
PC		 3965
3966 for (; node != link; node = node->next) {
3967 switch (node->code) {
3968 case jit_code_label: case jit_code_prolog:
ba86ff93
PC		 3969 /* lack of extra information so assume it is live */
3970 return (jit_reg_static);
4a71579b
PC		 3971 case jit_code_callr: case jit_code_calli:
3972 if (!(jit_class(_rvs[regno].spec) & jit_class_sav))
3973 return (jit_reg_undef);
3974 break;
3975 default:
3976 value = jit_classify(node->code);
ba86ff93
PC		 3977 use = change = 0;
3978 if (value & jit_cc_a0_rlh) {
3979 if (node->u.q.l == regno || node->u.q.h == regno) {
3980 if (value & jit_cc_a0_chg)
3981 change = 1;
3982 use = !change;
3983 }
3984 }
3985 else if ((value & jit_cc_a0_reg) && node->u.w == regno) {
3986 if (value & jit_cc_a0_chg)
3987 change = 1;
3988 use = !change;
3989 }
3990 if (!use &&
3991 (value & jit_cc_a1_reg) && node->v.w == regno) {
3992 if (value & jit_cc_a1_chg)
3993 change = 1;
3994 use = !change;
3995 }
3996 if (!use &&
3997 (value & jit_cc_a2_reg) && node->w.w == regno) {
3998 if (value & jit_cc_a2_chg)
3999 change = 1;
4000 use = !change;
4001 }
4a71579b 4002 /* lack of extra information */
ba86ff93
PC		 4003 if (!use && (value & (jit_cc_a0_jmp|jit_cc_a0_cnd)) &&
4004 /* In case of indirect branches, always consider
4005 * callee save registers as live. */
4006 !(jit_class(_rvs[regno].spec) & jit_class_sav))
4007 change = 1;
4008 if (!use && change)
4a71579b
PC		 4009 return (jit_reg_change);
4010 }
4011 }
4012
4013 return (jit_reg_static);
4014}
4015
4016/* most of this could be done at the same time as generating jit, but
4017 * avoid complications on different cpu backends and patch spill/loads
4018 * here, by simulating jit generation */
4019static jit_bool_t
4020_spill_reglive_p(jit_state_t *_jit, jit_node_t *node, jit_int32_t regno)
4021{
4022 if (!jit_regset_tstbit(&_jitc->reglive, regno)) {
4023 jit_regset_setbit(&_jitc->regmask, regno);
4024 jit_update(node->next, &_jitc->reglive, &_jitc->regmask);
4025 if (!jit_regset_tstbit(&_jitc->reglive, regno) &&
4026 register_change_p(node->next, node->link, regno) != jit_reg_change)
4027 return (0);
4028 }
4029
4030 return (1);
4031}
4032
4033static void
4034_patch_registers(jit_state_t *_jit)
4035{
4036 jit_node_t *prev;
4037 jit_node_t *node;
4038 jit_node_t *next;
4039 jit_int32_t info;
4040 jit_int32_t spec;
4041 jit_int32_t regno;
4042 jit_int32_t value;
4043
4044 _jitc->function = NULL;
4045
4046 jit_reglive_setup();
4047 for (prev = NULL, node = _jitc->head; node; node = next) {
4048 next = node->next;
4049
4050 info = jit_classify(node->code);
4051 jit_regarg_set(node, info);
4052
4053 switch (node->code) {
4054 case jit_code_save:
4055 regno = jit_regno(node->u.w);
4056 if (!spill_reglive_p(node, regno)) {
4057 /* register is not live, just remove spill/reload */
4058 jit_regarg_clr(node, info);
4059 node->link->v.w = jit_regload_delete;
4060 del_node(prev, node);
4061 continue;
4062 }
4063 else {
4064 /* try to find a free register of the same class */
4065 spec = jit_class(_rvs[regno].spec) & ~jit_class_arg;
4066 for (value = 0; value < _jitc->reglen; value++) {
4067 if (value != regno &&
4068 ((jit_class(_rvs[value].spec) & spec) &
4069 ~jit_class_arg) == spec &&
4070 !jit_regset_tstbit(&_jitc->regarg, value) &&
4071 !spill_reglive_p(node, value))
4072 break;
4073 }
4074 if (value < _jitc->reglen) {
4075 jit_regarg_clr(node, info);
4076 patch_register(node->next, node->link,
4077 jit_regno_patch|node->u.w,
4078 jit_regno_patch|value);
4079 /* mark as live just in case there are nested
4080 * register patches, so that next patch will
4081 * not want to use the same register */
4082 jit_regset_setbit(&_jitc->reglive, value);
4083 /* register is not live, just remove spill/reload */
4084 node->link->v.w = jit_regload_isdead;
4085 del_node(prev, node);
4086 continue;
4087 }
4088 else {
4089 /* failed to find a free register */
4090 if (spec & jit_class_gpr) {
4091 if (!_jitc->function->regoff[regno])
4092 _jitc->function->regoff[regno] =
4093 jit_allocai(sizeof(jit_word_t));
4094#if __WORDSIZE == 32
4095 node->code = jit_code_stxi_i;
4096#else
4097 node->code = jit_code_stxi_l;
4098#endif
4099 }
4100 else {
4101 node->code = jit_code_stxi_d;
4102 if (!_jitc->function->regoff[regno])
4103 _jitc->function->regoff[regno] =
4104 jit_allocai(sizeof(jit_float64_t));
4105 }
4106 node->u.w = _jitc->function->regoff[regno];
4107 node->v.w = JIT_FP;
4108 node->w.w = regno;
4109 node->link = NULL;
4110 }
4111 }
4112 break;
4113 case jit_code_load:
4114 regno = jit_regno(node->u.w);
4115 if (node->v.w) {
4116 if (node->v.w == jit_regload_isdead)
4117 jit_regset_clrbit(&_jitc->reglive, regno);
4118 del_node(prev, node);
4119 continue;
4120 }
4121 spec = jit_class(_rvs[regno].spec);
4122 if (spec & jit_class_gpr) {
4123#if __WORDSIZE == 32
4124 node->code = jit_code_ldxi_i;
4125#else
4126 node->code = jit_code_ldxi_l;
4127#endif
4128 }
4129 else
4130 node->code = jit_code_ldxi_d;
4131 node->v.w = regno;
4132 node->v.w = JIT_FP;
4133 node->w.w = _jitc->function->regoff[regno];
4134 node->link = NULL;
4135 break;
4136 case jit_code_prolog:
4137 _jitc->function = _jitc->functions.ptr + node->w.w;
4138 break;
4139 case jit_code_epilog:
4140 _jitc->function = NULL;
4141 break;
4142 default:
4143 break;
4144 }
4145
4146 jit_regarg_clr(node, info);
4147 /* update register live state */
4148 jit_reglive(node);
4149 prev = node;
4150 }
4151}
4152
4153static void
4154_patch_register(jit_state_t *_jit, jit_node_t *node, jit_node_t *link,
4155 jit_int32_t regno, jit_int32_t patch)
4156{
4157 jit_int32_t value;
4158
4159 for (; node != link; node = node->next) {
4160 value = jit_classify(node->code);
4161 if (value & jit_cc_a0_reg) {
4162 if (value & jit_cc_a0_rlh) {
4163 if (node->u.q.l == regno)
4164 node->u.q.l = patch;
4165 if (node->u.q.h == regno)
4166 node->u.q.h = patch;
4167 }
4168 else {
4169 if (node->u.w == regno)
4170 node->u.w = patch;
4171 }
4172 }
ba86ff93
PC		 4173 if (value & jit_cc_a1_reg) {
4174 if (value & jit_cc_a1_rlh) {
4175 if (node->v.q.l == regno)
4176 node->v.q.l = patch;
4177 if (node->v.q.h == regno)
4178 node->v.q.h = patch;
4179 }
4180 else {
4181 if (node->v.w == regno)
4182 node->v.w = patch;
4183 }
4184 }
ba3814c1
PC		 4185 if (value & jit_cc_a2_reg) {
4186 if (value & jit_cc_a2_rlh) {
4187 if (node->w.q.l == regno)
4188 node->w.q.l = patch;
4189 if (node->w.q.h == regno)
4190 node->w.q.h = patch;
4191 }
4192 else {
4193 if (node->w.w == regno)
4194 node->w.w = patch;
4195 }
4196 }
4a71579b
PC		 4197 }
4198}
4199
40a44dcb
PC		 4200#if __BYTE_ORDER == __LITTLE_ENDIAN
4201# define htonr_us(r0,r1) bswapr_us(r0,r1)
4202# define htonr_ui(r0,r1) bswapr_ui(r0,r1)
4203# if __WORDSIZE == 64
4204# define htonr_ul(r0,r1) bswapr_ul(r0,r1)
4205# endif
4206#else
4207# define htonr_us(r0,r1) extr_us(r0,r1)
4208# if __WORDSIZE == 32
4209# define htonr_ui(r0,r1) movr(r0,r1)
4210# else
4211# define htonr_ui(r0,r1) extr_ui(r0,r1)
4212# define htonr_ul(r0,r1) movr(r0,r1)
4213# endif
4214#endif
4215
4216static maybe_unused void
4217generic_bswapr_us(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1);
4218static maybe_unused void
4219generic_bswapr_ui(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1);
4220#if __WORDSIZE == 64
4221static maybe_unused void
4222generic_bswapr_ul(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1);
4223#endif
4224
ba86ff93
PC		 4225#define depi(r0, i0, i1, i2) _depi(_jit, r0, i0, i1, i2)
4226static void _depi(jit_state_t*,jit_int32_t, jit_word_t, jit_word_t, jit_word_t);
4227#define negi(r0, i0) _negi(_jit, r0, i0)
4228static void _negi(jit_state_t*, jit_int32_t, jit_word_t);
4229#define comi(r0, i0) _comi(_jit, r0, i0)
4230static void _comi(jit_state_t*, jit_int32_t, jit_word_t);
4231#define exti_c(r0, i0) _exti_c(_jit, r0, i0)
4232static void _exti_c(jit_state_t*, jit_int32_t, jit_word_t);
4233#define exti_uc(r0, i0) _exti_uc(_jit, r0, i0)
4234static void _exti_uc(jit_state_t*, jit_int32_t, jit_word_t);
4235#define exti_s(r0, i0) _exti_s(_jit, r0, i0)
4236static void _exti_s(jit_state_t*, jit_int32_t, jit_word_t);
4237#define exti_us(r0, i0) _exti_us(_jit, r0, i0)
4238static void _exti_us(jit_state_t*, jit_int32_t, jit_word_t);
4239#if __WORDSIZE == 64
4240#define exti_i(r0, i0) _exti_i(_jit, r0, i0)
4241static void _exti_i(jit_state_t*, jit_int32_t, jit_word_t);
4242#define exti_ui(r0, i0) _exti_ui(_jit, r0, i0)
4243static void _exti_ui(jit_state_t*, jit_int32_t, jit_word_t);
4244#endif
4245#define bswapi_us(r0, i0) _bswapi_us(_jit, r0, i0)
4246static void _bswapi_us(jit_state_t*, jit_int32_t, jit_word_t);
4247#define bswapi_ui(r0, i0) _bswapi_ui(_jit, r0, i0)
4248static void _bswapi_ui(jit_state_t*, jit_int32_t, jit_word_t);
4249#if __WORDSIZE == 64
4250# define bswapi_ul(r0, i0) _bswapi_ul(_jit, r0, i0)
4251static void _bswapi_ul(jit_state_t*, jit_int32_t, jit_word_t);
4252#endif
4253#define htoni_us(r0, i0) _htoni_us(_jit, r0, i0)
4254static void _htoni_us(jit_state_t*, jit_int32_t, jit_word_t);
4255#define htoni_ui(r0, i0) _htoni_ui(_jit, r0, i0)
4256static void _htoni_ui(jit_state_t*, jit_int32_t, jit_word_t);
4257#if __WORDSIZE == 64
4258# define htoni_ul(r0, i0) _htoni_ul(_jit, r0, i0)
4259static void _htoni_ul(jit_state_t*, jit_int32_t, jit_word_t);
4260#endif
4261# define movi_f_w(r0, i0) _movi_f_w(_jit, r0, i0)
4262static void _movi_f_w(jit_state_t*, jit_int32_t, jit_float32_t);
4263#if __WORDSIZE == 32
4264# define movi_d_ww(r0, r1, i0) _movi_d_ww(_jit, r0, r1, i0)
4265static void _movi_d_ww(jit_state_t*, jit_int32_t, jit_int32_t, jit_float64_t);
4266#else
4267# define movi_d_w(r0, i0) _movi_d_w(_jit, r0, i0)
4268static void _movi_d_w(jit_state_t*, jit_int32_t, jit_float64_t);
4269#endif
4270#define cloi(r0, i0) _cloi(_jit, r0, i0)
4271static void _cloi(jit_state_t*, jit_int32_t, jit_word_t);
4272#define clzi(r0, i0) _clzi(_jit, r0, i0)
4273static void _clzi(jit_state_t*, jit_int32_t, jit_word_t);
4274#define ctoi(r0, i0) _ctoi(_jit, r0, i0)
4275static void _ctoi(jit_state_t*, jit_int32_t, jit_word_t);
4276#define ctzi(r0, i0) _ctzi(_jit, r0, i0)
4277static void _ctzi(jit_state_t*, jit_int32_t, jit_word_t);
4278#define rbiti(r0, i0) _rbiti(_jit, r0, i0)
4279static void _rbiti(jit_state_t*, jit_int32_t, jit_word_t);
4280#define popcnti(r0, i0) _popcnti(_jit, r0, i0)
4281static void _popcnti(jit_state_t*, jit_int32_t, jit_word_t);
4282#define exti(r0, i0, i1, i2) _exti(_jit, r0, i0, i1, i2)
4283static void _exti(jit_state_t*,jit_int32_t, jit_word_t, jit_word_t, jit_word_t);
4284#define exti_u(r0, i0, i1, i2) _exti_u(_jit, r0, i0, i1, i2)
4285static void _exti_u(jit_state_t*,
4286 jit_int32_t, jit_word_t, jit_word_t, jit_word_t);
4287#define generic_unldr(r0, r1, i0) _generic_unldr(_jit, r0, r1, i0)
4288static void _generic_unldr(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
4289#define generic_unldi(r0, i0, i1) _generic_unldi(_jit, r0, i0, i1)
4290static void _generic_unldi(jit_state_t*, jit_int32_t, jit_word_t, jit_word_t);
4291#define generic_unldr_u(r0, r1, i0) _generic_unldr_u(_jit, r0, r1, i0)
4292static void _generic_unldr_u(jit_state_t*,
4293 jit_int32_t, jit_int32_t, jit_word_t);
4294#define generic_unldi_u(r0, i0, i1) _generic_unldi_u(_jit, r0, i0, i1)
4295static void _generic_unldi_u(jit_state_t*, jit_int32_t, jit_word_t, jit_word_t);
4296#define generic_unstr(r0, r1, i0) _generic_unstr(_jit, r0, r1, i0)
4297static void _generic_unstr(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
4298#define generic_unsti(i0, r0, i1) _generic_unsti(_jit, i0, r0, i1)
4299static void _generic_unsti(jit_state_t*, jit_word_t, jit_int32_t, jit_word_t);
4300#if !defined(__i386__) && !defined(__x86_64__)
4301# define generic_unldr_x(r0, r1, i0) _generic_unldr_x(_jit, r0, r1, i0)
4302static void _generic_unldr_x(jit_state_t*,
4303 jit_int32_t, jit_int32_t, jit_word_t);
4304# define generic_unldi_x(r0, i0, i1) _generic_unldi_x(_jit, r0, i0, i1)
4305static void _generic_unldi_x(jit_state_t*, jit_int32_t, jit_word_t, jit_word_t);
4306# define generic_unstr_x(r0, r1, i0) _generic_unstr_x(_jit, r0, r1, i0)
4307static void _generic_unstr_x(jit_state_t*,
4308 jit_int32_t, jit_int32_t, jit_word_t);
4309# define generic_unsti_x(i0, r0, i1) _generic_unsti_x(_jit, i0, r0, i1)
4310static void _generic_unsti_x(jit_state_t*, jit_word_t, jit_int32_t, jit_word_t);
4311#endif
79bfeef6
PC		 4312#define patch_alist(revert) _patch_alist(_jit, revert)
4313static maybe_unused void _patch_alist(jit_state_t *_jit, jit_bool_t revert);
4314
4a71579b
PC		 4315#if defined(__i386__) || defined(__x86_64__)
4316# include "jit_x86.c"
4317#elif defined(__mips__)
4318# include "jit_mips.c"
4319#elif defined(__arm__)
4320# include "jit_arm.c"
4321#elif defined(__powerpc__)
4322# include "jit_ppc.c"
4323#elif defined(__sparc__)
4324# include "jit_sparc.c"
4325#elif defined(__ia64__)
4326# include "jit_ia64.c"
4327#elif defined(__hppa__)
4328# include "jit_hppa.c"
4329#elif defined(__aarch64__)
4330# include "jit_aarch64.c"
4331#elif defined(__s390__) || defined(__s390x__)
4332# include "jit_s390.c"
4333#elif defined(__alpha__)
4334# include "jit_alpha.c"
4335#elif defined(__riscv)
4336# include "jit_riscv.c"
24d91c0d
PC		 4337#elif defined(__loongarch__)
4338# include "jit_loongarch.c"
4a71579b 4339#endif
40a44dcb
PC		 4340
4341static maybe_unused void
4342generic_bswapr_us(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
4343{
4344 jit_int32_t reg = jit_get_reg(jit_class_gpr);
4345
4346 rshi(rn(reg), r1, 8);
4347 andi(r0, r1, 0xff);
4348 andi(rn(reg), rn(reg), 0xff);
4349 lshi(r0, r0, 8);
4350 orr(r0, r0, rn(reg));
4351
4352 jit_unget_reg(reg);
4353}
4354
4355static maybe_unused void
4356generic_bswapr_ui(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
4357{
4358 jit_int32_t reg = jit_get_reg(jit_class_gpr);
4359
4360 rshi(rn(reg), r1, 16);
4361 bswapr_us(r0, r1);
4362 bswapr_us(rn(reg), rn(reg));
4363 lshi(r0, r0, 16);
4364 orr(r0, r0, rn(reg));
4365
4366 jit_unget_reg(reg);
4367}
4368
4369#if __WORDSIZE == 64
4370static maybe_unused void
4371generic_bswapr_ul(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
4372{
4373 jit_int32_t reg = jit_get_reg(jit_class_gpr);
4374
4375 rshi_u(rn(reg), r1, 32);
4376 bswapr_ui(r0, r1);
4377 bswapr_ui(rn(reg), rn(reg));
4378 lshi(r0, r0, 32);
4379 orr(r0, r0, rn(reg));
4380
4381 jit_unget_reg(reg);
4382}
4383#endif
79bfeef6 4384
ba86ff93
PC		 4385static void
4386_depi(jit_state_t *_jit,
4387 jit_int32_t r0, jit_word_t i0, jit_word_t i1, jit_word_t i2)
4388{
4389 jit_int32_t reg;
4390 reg = jit_get_reg(jit_class_gpr);
4391 movi(rn(reg), i0);
4392 depr(r0, rn(reg), i1, i2);
4393 jit_unget_reg(reg);
4394}
4395
4396static void
4397_negi(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
4398{
4399 movi(r0, -i0);
4400}
4401
4402static void
4403_comi(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
4404{
4405 movi(r0, ~i0);
4406}
4407
4408static void
4409_exti_c(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
4410{
4411 movi(r0, (jit_int8_t)i0);
4412}
4413
4414static void
4415_exti_uc(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
4416{
4417 movi(r0, (jit_uint8_t)i0);
4418}
4419
4420static void
4421_exti_s(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
4422{
4423 movi(r0, (jit_int16_t)i0);
4424}
4425
4426static void
4427_exti_us(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
4428{
4429 movi(r0, (jit_uint16_t)i0);
4430}
4431
4432#if __WORDSIZE == 64
4433static void
4434_exti_i(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
4435{
4436 movi(r0, (jit_int32_t)i0);
4437}
4438
4439static void
4440_exti_ui(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
4441{
4442 movi(r0, (jit_uint32_t)i0);
4443}
4444#endif
4445
4446static void
4447_bswapi_us(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
4448{
4449 union {
4450 jit_uint16_t us;
4451 jit_uint8_t v[2];
4452 } l, h;
4453 l.us = i0;
4454 h.v[0] = l.v[1];
4455 h.v[1] = l.v[0];
4456 movi(r0, h.us);
4457}
4458
4459static void
4460_bswapi_ui(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
4461{
4462 union {
4463 jit_uint32_t ui;
4464 jit_uint8_t v[4];
4465 } l, h;
4466 l.ui = i0;
4467 h.v[0] = l.v[3];
4468 h.v[1] = l.v[2];
4469 h.v[2] = l.v[1];
4470 h.v[3] = l.v[0];
4471 movi(r0, h.ui);
4472}
4473
4474#if __WORDSIZE == 64
4475static void
4476_bswapi_ul(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
4477{
4478 union {
4479 jit_uint64_t ul;
4480 jit_uint8_t v[8];
4481 } l, h;
4482 l.ul = i0;
4483 h.v[0] = l.v[7];
4484 h.v[1] = l.v[6];
4485 h.v[2] = l.v[5];
4486 h.v[3] = l.v[4];
4487 h.v[4] = l.v[3];
4488 h.v[5] = l.v[2];
4489 h.v[6] = l.v[1];
4490 h.v[7] = l.v[0];
4491 movi(r0, h.ul);
4492}
4493#endif
4494
4495static void
4496_htoni_us(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
4497{
4498#if __BYTE_ORDER == __LITTLE_ENDIAN
4499 bswapi_us(r0, i0);
4500#else
4501 exti_us(r0, i0);
4502#endif
4503}
4504
4505static void
4506_htoni_ui(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
4507{
4508#if __BYTE_ORDER == __LITTLE_ENDIAN
4509 bswapi_ui(r0, i0);
4510#else
4511# if __WORDSIZE == 32
4512 movi(r0, i0);
4513# else
4514 exti_ui(r0, i0);
4515# endif
4516#endif
4517}
4518
4519#if __WORDSIZE == 64
4520static void
4521_htoni_ul(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
4522{
4523# if __BYTE_ORDER == __LITTLE_ENDIAN
4524 bswapi_ul(r0, i0);
4525# else
4526 movi(r0, i0);
4527# endif
4528}
4529#endif
4530
4531static void
4532_movi_f_w(jit_state_t *_jit, jit_int32_t r0, jit_float32_t i0)
4533{
4534 union {
4535 jit_int32_t i;
4536 jit_float32_t f;
4537 } data;
4538 data.f = i0;
4539#if defined(__ia64__)
4540 /* Should be used only in this case (with out0 == 120) */
4541 if (r0 >= 120)
4542 r0 = _jitc->rout + (r0 - 120);
4543#endif
4544 movi(r0, data.i);
4545}
4546
4547#if __WORDSIZE == 32
4548static void
4549_movi_d_ww(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_float64_t i0)
4550{
4551 union {
4552 jit_int32_t i[2];
4553 jit_float64_t d;
4554 } data;
4555 data.d = i0;
4556 /* Mips does not change byte order of double values */
4557# if __BYTE_ORDER == __LITTLE_ENDIAN || defined(__mips__)
4558 movi(r0, data.i[0]);
4559 movi(r1, data.i[1]);
4560# else
4561 movi(r1, data.i[0]);
4562 movi(r0, data.i[1]);
4563# endif
4564}
4565
4566#else
4567static void
4568_movi_d_w(jit_state_t *_jit, jit_int32_t r0, jit_float64_t i0)
4569{
4570 union {
4571 jit_int64_t l;
4572 jit_float64_t d;
4573 } data;
4574 data.d = i0;
4575# if defined(__ia64__)
4576 /* Should be used only in this case (with out0 == 120) */
4577 if (r0 >= 120)
4578 r0 = _jitc->rout + (r0 - 120);
4579# endif
4580 movi(r0, data.l);
4581}
4582#endif
4583
4584 void
4585_jit_negi_f(jit_state_t *_jit, jit_fpr_t u, jit_float32_t v)
4586{
4587 jit_inc_synth_wf(negi_f, u, v);
4588 jit_movi_f(u, v);
4589 jit_negr_f(u, u);
4590 jit_dec_synth();
4591}
4592
4593void
4594_jit_absi_f(jit_state_t *_jit, jit_fpr_t u, jit_float32_t v)
4595{
4596 jit_inc_synth_wf(absi_f, u, v);
4597 jit_movi_f(u, v);
4598 jit_absr_f(u, u);
4599 jit_dec_synth();
4600}
4601
4602void
4603_jit_sqrti_f(jit_state_t *_jit, jit_fpr_t u, jit_float32_t v)
4604{
4605 jit_inc_synth_wf(sqrti_f, u, v);
4606 jit_movi_f(u, v);
4607 jit_sqrtr_f(u, u);
4608 jit_dec_synth();
4609}
4610
4611void
4612_jit_fmai_f(jit_state_t *_jit,
4613 jit_fpr_t u, jit_fpr_t v, jit_fpr_t w, jit_float32_t x)
4614{
4615 jit_int32_t y;
4616 jit_inc_synth_wqf(fmai_f, u, v, w, x);
4617 if (u != v && u != w) {
4618 jit_movi_f(u, x);
4619 jit_fmar_f(u, v, w, u);
4620 }
4621 else {
4622 y = jit_get_reg(jit_class_fpr);
4623 jit_movi_f(y, x);
4624 jit_fmar_f(u, v, w, y);
4625 jit_unget_reg(y);
4626 }
4627 jit_dec_synth();
4628}
4629
4630void
4631_jit_fmsi_f(jit_state_t *_jit,
4632 jit_fpr_t u, jit_fpr_t v, jit_fpr_t w, jit_float32_t x)
4633{
4634 jit_int32_t y;
4635 jit_inc_synth_wqf(fmai_f, u, v, w, x);
4636 if (u != v && u != w) {
4637 jit_movi_f(u, x);
4638 jit_fmsr_f(u, v, w, u);
4639 }
4640 else {
4641 y = jit_get_reg(jit_class_fpr);
4642 jit_movi_f(y, x);
4643 jit_fmsr_f(u, v, w, y);
4644 jit_unget_reg(y);
4645 }
4646 jit_dec_synth();
4647}
4648
4649void
4650_jit_fnmai_f(jit_state_t *_jit,
4651 jit_fpr_t u, jit_fpr_t v, jit_fpr_t w, jit_float32_t x)
4652{
4653 jit_int32_t y;
4654 jit_inc_synth_wqf(fmai_f, u, v, w, x);
4655 if (u != v && u != w) {
4656 jit_movi_f(u, x);
4657 jit_fnmar_f(u, v, w, u);
4658 }
4659 else {
4660 y = jit_get_reg(jit_class_fpr);
4661 jit_movi_f(y, x);
4662 jit_fnmar_f(u, v, w, y);
4663 jit_unget_reg(y);
4664 }
4665 jit_dec_synth();
4666}
4667
4668void
4669_jit_fnmsi_f(jit_state_t *_jit,
4670 jit_fpr_t u, jit_fpr_t v, jit_fpr_t w, jit_float32_t x)
4671{
4672 jit_int32_t y;
4673 jit_inc_synth_wqf(fmai_f, u, v, w, x);
4674 if (u != v && u != w) {
4675 jit_movi_f(u, x);
4676 jit_fnmsr_f(u, v, w, u);
4677 }
4678 else {
4679 y = jit_get_reg(jit_class_fpr);
4680 jit_movi_f(y, x);
4681 jit_fnmsr_f(u, v, w, y);
4682 jit_unget_reg(y);
4683 }
4684 jit_dec_synth();
4685}
4686
4687void
4688_jit_negi_d(jit_state_t *_jit, jit_fpr_t u, jit_float64_t v)
4689{
4690 jit_inc_synth_wd(negi_d, u, v);
4691 jit_movi_d(u, v);
4692 jit_negr_d(u, u);
4693 jit_dec_synth();
4694}
4695
4696void
4697_jit_absi_d(jit_state_t *_jit, jit_fpr_t u, jit_float64_t v)
4698{
4699 jit_inc_synth_wd(absi_d, u, v);
4700 jit_movi_d(u, v);
4701 jit_absr_d(u, u);
4702 jit_dec_synth();
4703}
4704
4705void
4706_jit_sqrti_d(jit_state_t *_jit, jit_fpr_t u, jit_float64_t v)
4707{
4708 jit_inc_synth_wd(sqrti_d, u, v);
4709 jit_movi_d(u, v);
4710 jit_sqrtr_d(u, u);
4711 jit_dec_synth();
4712}
4713
4714void
4715_jit_fmai_d(jit_state_t *_jit,
4716 jit_fpr_t u, jit_fpr_t v, jit_fpr_t w, jit_float64_t x)
4717{
4718 jit_int32_t y;
4719 jit_inc_synth_wqd(fmai_d, u, v, w, x);
4720 if (u != v && u != w) {
4721 jit_movi_d(u, x);
4722 jit_fmar_d(u, v, w, u);
4723 }
4724 else {
4725 y = jit_get_reg(jit_class_fpr);
4726 jit_movi_d(y, x);
4727 jit_fmar_d(u, v, w, y);
4728 jit_unget_reg(y);
4729 }
4730 jit_dec_synth();
4731}
4732
4733void
4734_jit_fmsi_d(jit_state_t *_jit,
4735 jit_fpr_t u, jit_fpr_t v, jit_fpr_t w, jit_float64_t x)
4736{
4737 jit_int32_t y;
4738 jit_inc_synth_wqd(fmai_d, u, v, w, x);
4739 if (u != v && u != w) {
4740 jit_movi_d(u, x);
4741 jit_fmsr_d(u, v, w, u);
4742 }
4743 else {
4744 y = jit_get_reg(jit_class_fpr);
4745 jit_movi_d(y, x);
4746 jit_fmsr_d(u, v, w, y);
4747 jit_unget_reg(y);
4748 }
4749 jit_dec_synth();
4750}
4751
4752void
4753_jit_fnmai_d(jit_state_t *_jit,
4754 jit_fpr_t u, jit_fpr_t v, jit_fpr_t w, jit_float64_t x)
4755{
4756 jit_int32_t y;
4757 jit_inc_synth_wqd(fmai_d, u, v, w, x);
4758 if (u != v && u != w) {
4759 jit_movi_d(u, x);
4760 jit_fnmar_d(u, v, w, u);
4761 }
4762 else {
4763 y = jit_get_reg(jit_class_fpr);
4764 jit_movi_d(y, x);
4765 jit_fnmar_d(u, v, w, y);
4766 jit_unget_reg(y);
4767 }
4768 jit_dec_synth();
4769}
4770
4771void
4772_jit_fnmsi_d(jit_state_t *_jit,
4773 jit_fpr_t u, jit_fpr_t v, jit_fpr_t w, jit_float64_t x)
4774{
4775 jit_int32_t y;
4776 jit_inc_synth_wqd(fmai_d, u, v, w, x);
4777 if (u != v && u != w) {
4778 jit_movi_d(u, x);
4779 jit_fnmsr_d(u, v, w, u);
4780 }
4781 else {
4782 y = jit_get_reg(jit_class_fpr);
4783 jit_movi_d(y, x);
4784 jit_fnmsr_d(u, v, w, y);
4785 jit_unget_reg(y);
4786 }
4787 jit_dec_synth();
4788}
4789
4790static void
4791_cloi(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
4792{
4793 clzi(r0, ~i0);
4794}
4795
4796static void
4797_clzi(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
4798{
4799#if __WORDSIZE == 64 && _WIN32
4800 movi(r0, (i0) ? __builtin_clzll(i0) : __WORDSIZE);
4801#else
4802 movi(r0, (i0) ? __builtin_clzl(i0) : __WORDSIZE);
4803#endif
4804}
4805
4806static void
4807_ctoi(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
4808{
4809 ctzi(r0, ~i0);
4810}
4811
4812static void
4813_ctzi(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
4814{
4815#if __WORDSIZE == 64 && _WIN32
4816 movi(r0, (i0) ? __builtin_ctzll(i0) : __WORDSIZE);
4817#else
4818 movi(r0, (i0) ? __builtin_ctzl(i0) : __WORDSIZE);
4819#endif
4820}
4821
4822static void
4823_rbiti(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
4824{
4825 jit_int32_t i;
4826 union {
4827 jit_uword_t w;
4828 jit_uint8_t v[__WORDSIZE >> 3];
4829 } u, v;
4830 static const unsigned char swap_tab[256] = {
4831 0, 128, 64, 192, 32, 160, 96, 224,
4832 16, 144, 80, 208, 48, 176, 112, 240,
4833 8, 136, 72, 200, 40, 168, 104, 232,
4834 24, 152, 88, 216 ,56, 184, 120, 248,
4835 4, 132, 68, 196, 36, 164, 100, 228,
4836 20, 148, 84, 212, 52, 180, 116, 244,
4837 12, 140, 76, 204, 44, 172, 108, 236,
4838 28, 156, 92, 220, 60, 188, 124, 252,
4839 2, 130, 66, 194, 34, 162, 98, 226,
4840 18, 146, 82, 210, 50, 178, 114, 242,
4841 10, 138, 74, 202, 42, 170, 106, 234,
4842 26, 154, 90, 218, 58, 186, 122, 250,
4843 6, 134, 70, 198, 38, 166, 102, 230,
4844 22, 150, 86, 214, 54, 182, 118, 246,
4845 14, 142, 78, 206, 46, 174, 110, 238,
4846 30, 158, 94, 222, 62, 190, 126, 254,
4847 1, 129, 65, 193, 33, 161, 97, 225,
4848 17, 145, 81, 209, 49, 177, 113, 241,
4849 9, 137, 73, 201, 41, 169, 105, 233,
4850 25, 153, 89, 217, 57, 185, 121, 249,
4851 5, 133, 69, 197, 37, 165, 101, 229,
4852 21, 149, 85, 213, 53, 181, 117, 245,
4853 13, 141, 77, 205, 45, 173, 109, 237,
4854 29, 157, 93, 221, 61, 189, 125, 253,
4855 3, 131, 67, 195, 35, 163, 99, 227,
4856 19, 147, 83, 211, 51, 179, 115, 243,
4857 11, 139, 75, 203, 43, 171, 107, 235,
4858 27, 155, 91, 219, 59, 187, 123, 251,
4859 7, 135, 71, 199, 39, 167, 103, 231,
4860 23, 151, 87, 215, 55, 183, 119, 247,
4861 15, 143, 79, 207, 47, 175, 111, 239,
4862 31, 159, 95, 223, 63, 191, 127, 255
4863 };
4864 u.w = i0;
4865 for (i = 0; i < sizeof(jit_word_t); ++i)
4866 v.v[i] = swap_tab[u.v[sizeof(jit_word_t) - i - 1]];
4867 movi(r0, v.w);
4868}
4869
4870static void
4871_popcnti(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
4872{
4873#if __WORDSIZE == 64 && _WIN32
4874 movi(r0, (i0) ? __builtin_popcountll(i0) : __WORDSIZE);
4875#else
4876 movi(r0, (i0) ? __builtin_popcountl(i0) : __WORDSIZE);
4877#endif
4878}
4879
4880static void _exti(jit_state_t *_jit,
4881 jit_int32_t r0, jit_word_t i0, jit_word_t i1, jit_word_t i2)
4882{
4883#if __BYTE_ORDER == __BIG_ENDIAN
4884 i1 = __WORDSIZE - (i1 + i2);
4885#endif
4886 i0 <<= __WORDSIZE - (i1 + i2);
4887 i0 >>= __WORDSIZE - i2;
4888 movi(r0, i0);
4889}
4890
4891static void _exti_u(jit_state_t *_jit,
4892 jit_int32_t r0, jit_word_t i0, jit_word_t i1, jit_word_t i2)
4893{
4894 jit_word_t t;
4895#if __BYTE_ORDER == __BIG_ENDIAN
4896 i1 = __WORDSIZE - (i1 + i2);
4897#endif
4898 if (i1)
4899 i0 >>= __WORDSIZE - i2;
4900#if __WORDSIZE == 64 && !_WIN32
4901 i0 &= (1L << i2) - 1;
4902#else
4903 i0 &= (1LL << i2) - 1;
4904#endif
4905 movi(r0, i0);
4906}
4907
4908static void
4909_generic_unldr(jit_state_t *_jit,
4910 jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
4911{
4912 jit_int32_t t0, r2;
4913 assert(i0 >= 1 && i0 <= sizeof(jit_word_t));
4914 if (i0 & (i0 - 1)) {
4915 t0 = jit_get_reg(jit_class_gpr);
4916 r2 = rn(t0);
4917 movr(r2, r1);
4918 }
4919 switch (i0) {
4920 case 1:
4921 ldr_c(r0, r1);
4922 break;
4923 case 2:
4924 ldr_s(r0, r1);
4925 break;
4926 case 3:
4927#if __BYTE_ORDER == __LITTLE_ENDIAN
4928 ldr_us(r0, r2);
4929 ldxi_c(r2, r2, 2);
4930 lshi(r2, r2, 16);
4931#else
4932 ldr_c(r0, r2);
4933 lshi(r0, r0, 16);
4934 ldxi_us(r2, r2, 1);
4935#endif
4936 break;
4937#if __WORDSIZE == 32
4938 default:
4939 ldr_i(r0, r1);
4940 break;
4941#else
4942 case 4:
4943 ldr_i(r0, r1);
4944 break;
4945 case 5:
4946# if __BYTE_ORDER == __LITTLE_ENDIAN
4947 ldr_ui(r0, r2);
4948 ldxi_c(r2, r2, 4);
4949 lshi(r2, r2, 32);
4950# else
4951 ldr_i(r0, r2);
4952 lshi(r0, r0, 8);
4953 ldxi_uc(r2, r2, 4);
4954# endif
4955 break;
4956 case 6:
4957# if __BYTE_ORDER == __LITTLE_ENDIAN
4958 ldr_ui(r0, r2);
4959 ldxi_s(r2, r2, 4);
4960 lshi(r2, r2, 32);
4961# else
4962 ldr_i(r0, r2);
4963 lshi(r0, r0, 16);
4964 ldxi_us(r2, r2, 4);
4965# endif
4966 break;
4967 case 7:
4968# if __BYTE_ORDER == __LITTLE_ENDIAN
4969 ldr_ui(r0, r2);
4970 ldxi_i(r2, r2, 4);
4971 lshi(r2, r2, 40);
4972 rshi(r2, r2, 8);
4973# else
4974 ldr_i(r0, r2);
4975 lshi(r0, r0, 24);
4976 ldxi_ui(r2, r2, 4);
4977 rshi(r2, r2, 8);
4978# endif
4979 break;
4980 default:
4981 ldr_l(r0, r1);
4982 break;
4983#endif
4984 }
4985 if (i0 & (i0 - 1)) {
4986 orr(r0, r0, r2);
4987 jit_unget_reg(t0);
4988 }
4989}
4990
4991static void
4992_generic_unldi(jit_state_t *_jit,
4993 jit_int32_t r0, jit_word_t i0, jit_word_t i1)
4994{
4995 jit_int32_t t0, r2;
4996 assert(i1 >= 1 && i1 <= sizeof(jit_word_t));
4997 if (i1 & (i1 - 1)) {
4998 t0 = jit_get_reg(jit_class_gpr);
4999 r2 = rn(t0);
5000 }
5001 switch (i1) {
5002 case 1:
5003 ldi_c(r0, i0);
5004 break;
5005 case 2:
5006 ldi_s(r0, i0);
5007 break;
5008 case 3:
5009# if __BYTE_ORDER == __LITTLE_ENDIAN
5010 ldi_us(r0, i0);
5011 ldi_c(r2, i0 + 2);
5012 lshi(r2, r2, 16);
5013# else
5014 ldi_c(r0, i0);
5015 lshi(r0, r0, 16);
5016 ldi_us(r2, i0 + 1);
5017# endif
5018 break;
5019# if __WORDSIZE == 32
5020 default:
5021 ldi_i(r0, i0);
5022 break;
5023# else
5024 case 4:
5025 ldi_i(r0, i0);
5026 break;
5027 case 5:
5028# if __BYTE_ORDER == __LITTLE_ENDIAN
5029 ldi_ui(r0, i0);
5030 ldi_c(r2, i0 + 4);
5031 lshi(r2, r2, 32);
5032# else
5033 ldi_i(r0, i0);
5034 lshi(r0, r0, 8);
5035 ldi_uc(r2, i0 + 4);
5036# endif
5037 break;
5038 case 6:
5039# if __BYTE_ORDER == __LITTLE_ENDIAN
5040 ldi_ui(r0, i0);
5041 ldi_s(r2, i0 + 4);
5042 lshi(r2, r2, 32);
5043# else
5044 ldi_i(r0, i0);
5045 lshi(r0, r0, 16);
5046 ldi_us(r2, i0 + 4);
5047# endif
5048 break;
5049 case 7:
5050# if __BYTE_ORDER == __LITTLE_ENDIAN
5051 ldi_ui(r0, i0);
5052 ldi_i(r2, i0 + 4);
5053 lshi(r2, r2, 40);
5054 rshi(r2, r2, 8);
5055# else
5056 ldi_i(r0, i0);
5057 lshi(r0, r0, 24);
5058 ldi_ui(r2, i0 + 4);
5059 rshi(r2, r2, 8);
5060# endif
5061 break;
5062 default:
5063 ldi_l(r0, i0);
5064 break;
5065# endif
5066 }
5067 if (i1 & (i1 - 1)) {
5068 orr(r0, r0, r2);
5069 jit_unget_reg(t0);
5070 }
5071}
5072
5073static void
5074_generic_unldr_u(jit_state_t *_jit,
5075 jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
5076{
5077 jit_int32_t t0, r2;
5078 assert(i0 >= 1 && i0 <= sizeof(jit_word_t));
5079 if (i0 & (i0 - 1)) {
5080 t0 = jit_get_reg(jit_class_gpr);
5081 r2 = rn(t0);
5082 movr(r2, r1);
5083 }
5084 switch (i0) {
5085 case 1:
5086 ldr_uc(r0, r1);
5087 break;
5088 case 2:
5089 ldr_us(r0, r1);
5090 break;
5091 case 3:
5092# if __BYTE_ORDER == __LITTLE_ENDIAN
5093 ldr_us(r0, r2);
5094 ldxi_uc(r2, r2, 2);
5095 lshi(r2, r2, 16);
5096# else
5097 ldr_uc(r0, r2);
5098 lshi(r0, r0, 16);
5099 ldxi_us(r2, r2, 1);
5100# endif
5101 break;
5102# if __WORDSIZE == 32
5103 default:
5104 ldr_i(r0, r1);
5105 break;
5106# else
5107 case 4:
5108 ldr_ui(r0, r1);
5109 break;
5110 case 5:
5111# if __BYTE_ORDER == __LITTLE_ENDIAN
5112 ldr_ui(r0, r2);
5113 ldxi_uc(r2, r2, 4);
5114 lshi(r2, r2, 32);
5115# else
5116 ldr_ui(r0, r2);
5117 lshi(r0, r0, 8);
5118 ldxi_uc(r2, r2, 4);
5119# endif
5120 break;
5121 case 6:
5122# if __BYTE_ORDER == __LITTLE_ENDIAN
5123 ldr_ui(r0, r2);
5124 ldxi_us(r2, r2, 4);
5125 lshi(r2, r2, 32);
5126# else
5127 ldr_ui(r0, r2);
5128 lshi(r0, r0, 16);
5129 ldxi_us(r2, r2, 4);
5130# endif
5131 break;
5132 case 7:
5133# if __BYTE_ORDER == __LITTLE_ENDIAN
5134 ldr_ui(r0, r2);
5135 ldxi_ui(r2, r2, 4);
5136 lshi(r2, r2, 40);
5137 rshi_u(r2, r2, 8);
5138# else
5139 ldr_ui(r0, r2);
5140 lshi(r0, r0, 24);
5141 ldxi_ui(r2, r2, 4);
5142 rshi(r2, r2, 8);
5143# endif
5144 break;
5145 default:
5146 ldr_l(r0, r1);
5147 break;
5148# endif
5149 }
5150 if (i0 & (i0 - 1)) {
5151 orr(r0, r0, r2);
5152 jit_unget_reg(t0);
5153 }
5154}
5155
5156static void
5157_generic_unldi_u(jit_state_t *_jit,
5158 jit_int32_t r0, jit_word_t i0, jit_word_t i1)
5159{
5160 jit_int32_t t0, r2;
5161 assert(i1 >= 1 && i1 <= sizeof(jit_word_t));
5162 if (i1 & (i1 - 1)) {
5163 t0 = jit_get_reg(jit_class_gpr);
5164 r2 = rn(t0);
5165 }
5166 switch (i1) {
5167 case 1:
5168 ldi_uc(r0, i0);
5169 break;
5170 case 2:
5171 ldi_us(r0, i0);
5172 break;
5173 case 3:
5174# if __BYTE_ORDER == __LITTLE_ENDIAN
5175 ldi_us(r0, i0);
5176 ldi_uc(r2, i0 + 2);
5177 lshi(r2, r2, 16);
5178# else
5179 ldi_uc(r0, i0);
5180 lshi(r0, r0, 16);
5181 ldi_us(r2, i0 + 1);
5182# endif
5183 break;
5184# if __WORDSIZE == 32
5185 default:
5186 ldi_i(r0, i0);
5187 break;
5188# else
5189 case 4:
5190 ldi_ui(r0, i0);
5191 break;
5192 case 5:
5193# if __BYTE_ORDER == __LITTLE_ENDIAN
5194 ldi_ui(r0, i0);
5195 ldi_uc(r2, i0 + 4);
5196 lshi(r2, r2, 32);
5197# else
5198 ldi_ui(r0, i0);
5199 lshi(r0, r0, 8);
5200 ldi_uc(r2, i0 + 4);
5201# endif
5202 break;
5203 case 6:
5204# if __BYTE_ORDER == __LITTLE_ENDIAN
5205 ldi_ui(r0, i0);
5206 ldi_us(r2, i0 + 4);
5207 lshi(r2, r2, 32);
5208# else
5209 ldi_ui(r0, i0);
5210 lshi(r0, r0, 16);
5211 ldi_us(r2, i0 + 4);
5212# endif
5213 break;
5214 case 7:
5215# if __BYTE_ORDER == __LITTLE_ENDIAN
5216 ldi_ui(r0, i0);
5217 ldi_ui(r2, i0 + 4);
5218 lshi(r2, r2, 40);
5219 rshi_u(r2, r2, 8);
5220# else
5221 ldi_ui(r0, i0);
5222 lshi(r0, r0, 24);
5223 ldi_ui(r2, i0 + 4);
5224 rshi(r2, r2, 8);
5225# endif
5226 break;
5227 default:
5228 ldi_l(r0, i0);
5229 break;
5230# endif
5231 }
5232 if (i1 & (i1 - 1)) {
5233 orr(r0, r0, r2);
5234 jit_unget_reg(t0);
5235 }
5236}
5237
5238static void
5239_generic_unstr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
5240{
5241 jit_word_t t0, r2;
5242 assert(i0 > 0 && i0 <= sizeof(jit_word_t));
5243 if (i0 & (i0 - 1)) {
5244 t0 = jit_get_reg(jit_class_gpr);
5245 r2 = rn(t0);
5246 }
5247 switch (i0) {
5248 case 1:
5249 str_c(r0, r1);
5250 break;
5251 case 2:
5252 str_s(r0, r1);
5253 break;
5254 case 3:
5255# if __BYTE_ORDER == __LITTLE_ENDIAN
5256 str_s(r0, r1);
5257 rshi(r2, r1, 16);
5258 stxi_c(2, r0, r2);
5259# else
5260 stxi_c(2, r0, r1);
5261 rshi(r2, r1, 8);
5262 str_s(r0, r2);
5263# endif
5264 break;
5265# if __WORDSIZE == 32
5266 default:
5267 str_i(r0, r1);
5268 break;
5269# else
5270 case 4:
5271 str_i(r0, r1);
5272 break;
5273 case 5:
5274# if __BYTE_ORDER == __LITTLE_ENDIAN
5275 str_i(r0, r1);
5276 rshi(r2, r1, 32);
5277 stxi_c(4, r0, r2);
5278# else
5279 stxi_c(4, r0, r1);
5280 rshi(r2, r1, 8);
5281 str_i(r0, r2);
5282# endif
5283 break;
5284 case 6:
5285# if __BYTE_ORDER == __LITTLE_ENDIAN
5286 str_i(r0, r1);
5287 rshi(r2, r1, 32);
5288 stxi_s(4, r0, r2);
5289# else
5290 stxi_s(4, r0, r1);
5291 rshi(r2, r1, 16);
5292 str_i(r0, r2);
5293# endif
5294 break;
5295 case 7:
5296# if __BYTE_ORDER == __LITTLE_ENDIAN
5297 str_i(r0, r1);
5298 rshi(r2, r1, 32);
5299 stxi_s(4, r0, r2);
5300 rshi(r2, r2, 16);
5301 stxi_c(6, r0, r2);
5302# else
5303 stxi_c(6, r0, r1);
5304 rshi(r2, r1, 8);
5305 stxi_s(4, r0, r2);
5306 rshi(r2, r2, 16);
5307 str_i(r0, r2);
5308# endif
5309 break;
5310 default:
5311 str_l(r0, r1);
5312 break;
5313# endif
5314 }
5315 if (i0 & (i0 - 1))
5316 jit_unget_reg(t0);
5317}
5318
5319static void
5320_generic_unsti(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
5321{
5322 jit_word_t t0, r2;
5323 assert(i1 > 0 && i1 <= sizeof(jit_word_t));
5324 if (i1 & (i1 - 1)) {
5325 t0 = jit_get_reg(jit_class_gpr);
5326 r2 = rn(t0);
5327 }
5328 switch (i1) {
5329 case 1:
5330 sti_c(i0, r0);
5331 break;
5332 case 2:
5333 sti_s(i0, r0);
5334 break;
5335 case 3:
5336# if __BYTE_ORDER == __LITTLE_ENDIAN
5337 sti_s(i0, r0);
5338 rshi(r2, r0, 16);
5339 sti_c(2 + i0, r2);
5340# else
5341 sti_c(2 + i0, r0);
5342 rshi(r2, r0, 8);
5343 sti_s(i0, r2);
5344# endif
5345 break;
5346# if __WORDSIZE == 32
5347 default:
5348 sti_i(i0, r0);
5349 break;
5350# else
5351 case 4:
5352 sti_i(i0, r0);
5353 break;
5354 case 5:
5355# if __BYTE_ORDER == __LITTLE_ENDIAN
5356 sti_i(i0, r0);
5357 rshi(r2, r0, 32);
5358 sti_c(4 + i0, r2);
5359# else
5360 stxi_c(4, i0, r0);
5361 rshi(r2, r0, 8);
5362 sti_i(i0, r2);
5363# endif
5364 break;
5365 case 6:
5366# if __BYTE_ORDER == __LITTLE_ENDIAN
5367 sti_i(i0, r0);
5368 rshi(r2, r0, 32);
5369 sti_s(4 + i0, r2);
5370# else
5371 sti_s(4 + i0, r0);
5372 rshi(r2, r0, 16);
5373 sti_i(i0, r2);
5374# endif
5375 break;
5376 case 7:
5377# if __BYTE_ORDER == __LITTLE_ENDIAN
5378 sti_i(i0, r0);
5379 rshi(r2, r0, 32);
5380 sti_s(4 + i0, r2);
5381 rshi(r2, r2, 16);
5382 sti_c(6 + i0, r2);
5383# else
5384 sti_c(6 + i0, r0);
5385 rshi(r2, r0, 8);
5386 sti_s(4 + i0, r2);
5387 rshi(r2, r2, 16);
5388 sti_i(i0, r2);
5389# endif
5390 break;
5391 default:
5392 sti_l(i0, r0);
5393 break;
5394# endif
5395 }
5396 if (i1 & (i1 - 1))
5397 jit_unget_reg(t0);
5398}
5399
5400#if !defined(__i386__) && !defined(__x86_64__) && !defined(__arm__)
5401static void
5402_generic_unldr_x(jit_state_t *_jit,
5403 jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
5404{
5405 assert(i0 == 4 || i0 == 8);
5406 if (i0 == 4)
5407 ldr_f(r0, r1);
5408 else
5409 ldr_d(r0, r1);
5410}
5411
5412static void
5413_generic_unldi_x(jit_state_t *_jit,
5414 jit_int32_t r0, jit_word_t i0, jit_word_t i1)
5415{
5416 assert(i1 == 4 || i1 == 8);
5417 if (i1 == 4)
5418 ldi_f(r0, i0);
5419 else
5420 ldi_d(r0, i0);
5421}
5422
5423static void
5424_generic_unstr_x(jit_state_t *_jit,
5425 jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
5426{
5427 assert(i0 == 4 || i0 == 8);
5428 if (i0 == 4)
5429 str_f(r0, r1);
5430 else
5431 str_d(r0, r1);
5432}
5433
5434static void
5435_generic_unsti_x(jit_state_t *_jit,
5436 jit_word_t i0, jit_int32_t r0, jit_word_t i1)
5437{
5438 assert(i1 == 4 || i1 == 8);
5439 if (i1 == 4)
5440 sti_f(i0, r0);
5441 else
5442 sti_d(i0, r0);
5443}
5444#endif
5445
79bfeef6
PC		 5446#if defined(stack_framesize)
5447static maybe_unused void
5448_patch_alist(jit_state_t *_jit, jit_bool_t revert)
5449{
5450 jit_int32_t diff;
5451 jit_node_t *node;
5452 diff = jit_diffsize();
5453 if (diff) {
5454 if (revert)
5455 diff = -diff;
5456 for (node = _jitc->function->alist; node; node = node->link) {
5457 switch (node->code) {
5458 case jit_code_ldxi_c: case jit_code_ldxi_uc:
5459 case jit_code_ldxi_s: case jit_code_ldxi_us:
5460 case jit_code_ldxi_i:
5461#if __WORDSIZE == 64
5462 case jit_code_ldxi_ui: case jit_code_ldxi_l:
5463#endif
5464 case jit_code_ldxi_f: case jit_code_ldxi_d:
5465 node->w.w -= diff;
5466 break;
5467 case jit_code_stxi_c: case jit_code_stxi_s:
5468 case jit_code_stxi_i:
5469#if __WORDSIZE == 64
5470 case jit_code_stxi_l:
5471#endif
5472 case jit_code_stxi_f: case jit_code_stxi_d:
5473 node->u.w -= diff;
5474 break;
5475 default:
5476 abort();
5477 }
5478 }
5479 }
5480}
5481#endif
ARM optimized PCSX fork