notaz.gp2x.de / pcsx_rearmed.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
Update lightrec 20220910 (#686)
[pcsx_rearmed.git] / deps / lightning / lib / lightning.c
CommitLineData
4a71579b
PC		 1/*
2 * Copyright (C) 2012-2019 Free Software Foundation, Inc.
3 *
4 * This file is part of GNU lightning.
5 *
6 * GNU lightning is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU Lesser General Public License as published
8 * by the Free Software Foundation; either version 3, or (at your option)
9 * any later version.
10 *
11 * GNU lightning is distributed in the hope that it will be useful, but
12 * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
13 * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public
14 * License for more details.
15 *
16 * Authors:
17 * Paulo Cesar Pereira de Andrade
18 */
19
20#include <lightning.h>
21#include <lightning/jit_private.h>
40a44dcb
PC		 22#if HAVE_MMAP
23# include <sys/mman.h>
24#endif
4a71579b
PC		 25#if defined(__sgi)
26# include <fcntl.h>
27#endif
28
29#ifndef MAP_ANON
30# define MAP_ANON MAP_ANONYMOUS
31# ifndef MAP_ANONYMOUS
32# define MAP_ANONYMOUS 0
33# endif
34#endif
35
36#define jit_regload_reload 0 /* convert to reload */
37#define jit_regload_delete 1 /* just remove node */
38#define jit_regload_isdead 2 /* delete and unset live bit */
39
40/*
41 * Prototypes
42 */
43static jit_word_t hash_data(const void*, jit_word_t);
44
45#define new_pool() _new_pool(_jit)
46static void _new_pool(jit_state_t*);
47
48#define new_node(u) _new_node(_jit, u)
49static jit_node_t *_new_node(jit_state_t*, jit_code_t);
50
51#define link_node(u) _link_node(_jit, u)
52static inline jit_node_t *_link_node(jit_state_t*, jit_node_t*);
53
54#define del_node(u, v) _del_node(_jit, u, v)
55static inline void _del_node(jit_state_t*, jit_node_t*, jit_node_t*);
56
57#define free_node(u) _free_node(_jit, u)
58static inline void _free_node(jit_state_t*, jit_node_t*);
59
60#define del_label(u, v) _del_label(_jit, u, v)
61static void _del_label(jit_state_t*, jit_node_t*, jit_node_t*);
62
63#define jit_dataset() _jit_dataset(_jit)
64static void
65_jit_dataset(jit_state_t *_jit);
66
67#define jit_setup(block) _jit_setup(_jit, block)
68static void
69_jit_setup(jit_state_t *_jit, jit_block_t *block);
70
71#define jit_follow(block, todo) _jit_follow(_jit, block, todo)
72static void
73_jit_follow(jit_state_t *_jit, jit_block_t *block, jit_bool_t *todo);
74
75#define jit_update(node, live, mask) _jit_update(_jit, node, live, mask)
76static void
77_jit_update(jit_state_t *_jit, jit_node_t *node,
78 jit_regset_t *live, jit_regset_t *mask);
79
80#define thread_jumps() _thread_jumps(_jit)
81static void
82_thread_jumps(jit_state_t *_jit);
83
84#define sequential_labels() _sequential_labels(_jit)
85static void
86_sequential_labels(jit_state_t *_jit);
87
88#define split_branches() _split_branches(_jit)
89static void
90_split_branches(jit_state_t *_jit);
91
92#define shortcut_jump(prev, node) _shortcut_jump(_jit, prev, node)
93static jit_bool_t
94_shortcut_jump(jit_state_t *_jit, jit_node_t *prev, jit_node_t *node);
95
96#define redundant_jump(prev, node) _redundant_jump(_jit, prev, node)
97static jit_bool_t
98_redundant_jump(jit_state_t *_jit, jit_node_t *prev, jit_node_t *node);
99
100static jit_code_t
101reverse_jump_code(jit_code_t code);
102
103#define reverse_jump(prev, node) _reverse_jump(_jit, prev, node)
104static jit_bool_t
105_reverse_jump(jit_state_t *_jit, jit_node_t *prev, jit_node_t *node);
106
107#define redundant_store(node, jump) _redundant_store(_jit, node, jump)
ba3814c1 108static jit_bool_t
4a71579b
PC		 109_redundant_store(jit_state_t *_jit, jit_node_t *node, jit_bool_t jump);
110
111#define simplify_movr(p, n, k, s) _simplify_movr(_jit, p, n, k, s)
112static jit_bool_t
113_simplify_movr(jit_state_t *_jit, jit_node_t *prev, jit_node_t *node,
114 jit_int32_t kind, jit_int32_t size);
115
116#define simplify_movi(p, n, k, s) _simplify_movi(_jit, p, n, k, s)
117static jit_bool_t
118_simplify_movi(jit_state_t *_jit, jit_node_t *prev, jit_node_t *node,
119 jit_int32_t kind, jit_int32_t size);
120
121#define simplify_ldxi(prev, node) _simplify_ldxi(_jit, prev, node)
122static jit_bool_t
123_simplify_ldxi(jit_state_t *_jit, jit_node_t *prev, jit_node_t *node);
124
125#define simplify_stxi(prev, node) _simplify_stxi(_jit, prev, node)
126static jit_bool_t
127_simplify_stxi(jit_state_t *_jit, jit_node_t *prev, jit_node_t *node);
128
129#define simplify_spill(node, regno) _simplify_spill(_jit, node, regno)
130static void
131_simplify_spill(jit_state_t *_jit, jit_node_t *node, jit_int32_t regno);
132
133#define simplify() _simplify(_jit)
ba3814c1 134static jit_bool_t
4a71579b
PC		 135_simplify(jit_state_t *_jit);
136
137#define jit_reg_undef -1
138#define jit_reg_static 0
139#define jit_reg_change 1
140#define register_change_p(n, l, r) _register_change_p(_jit, n, l, r)
141static jit_int32_t
142_register_change_p(jit_state_t *_jit, jit_node_t *node, jit_node_t *link,
143 jit_int32_t regno);
144
145#define spill_reglive_p(node, regno) _spill_reglive_p(_jit, node, regno)
146static jit_bool_t
147_spill_reglive_p(jit_state_t *_jit, jit_node_t *node, jit_int32_t regno);
148
149#define patch_registers() _patch_registers(_jit)
150static void
151_patch_registers(jit_state_t *_jit);
152
153#define patch_register(n,l,r,p) _patch_register(_jit,n,l,r,p)
154static void
155_patch_register(jit_state_t *jit, jit_node_t *node, jit_node_t *link,
156 jit_int32_t regno, jit_int32_t patch);
157
158/*
159 * Initialization
160 */
161#if !defined(__sgi)
162#define mmap_fd -1
163#endif
164
165/*
166 * Implementation
167 */
168void
169init_jit(const char *progname)
170{
171 jit_get_cpu();
172 jit_init_debug(progname);
173 jit_init_size();
174}
175
176void
177finish_jit(void)
178{
179 jit_finish_debug();
180 jit_finish_size();
181}
182
183jit_int32_t
184_jit_get_reg(jit_state_t *_jit, jit_int32_t regspec)
185{
186 jit_int32_t spec;
187 jit_int32_t regno;
188
189 spec = regspec & ~(jit_class_chk|jit_class_nospill);
190 if (spec & jit_class_named) {
191 regno = jit_regno(spec);
192 if (jit_regset_tstbit(&_jitc->regsav, regno))
193 /* fail if register is spilled */
194 goto fail;
195 if (jit_regset_tstbit(&_jitc->regarg, regno))
196 /* fail if register is an argument to current instruction */
197 goto fail;
198 if (jit_regset_tstbit(&_jitc->reglive, regno)) {
199 if (regspec & jit_class_nospill)
200 /* fail if register is live and should not spill/reload */
201 goto fail;
202 goto spill;
203 }
204 jit_regset_setbit(&_jitc->regarg, regno);
205 return (regno);
206 }
207 else
208 assert(jit_class(spec) != 0);
209
210 if (_jitc->emit) {
211 /* search for a free register matching spec */
212 for (regno = 0; regno < _jitc->reglen; regno++) {
213 if ((jit_class(_rvs[regno].spec) & spec) == spec &&
214 !jit_regset_tstbit(&_jitc->regarg, regno) &&
215 !jit_regset_tstbit(&_jitc->reglive, regno))
216 goto regarg;
217 }
218
219 /* search for a register matching spec that is not an argument
220 * for the current instruction */
221 for (regno = 0; regno < _jitc->reglen; regno++) {
222 if ((jit_class(_rvs[regno].spec) & spec) == spec &&
223 !jit_regset_tstbit(&_jitc->regsav, regno) &&
224 !jit_regset_tstbit(&_jitc->regarg, regno) &&
225 !(regspec & jit_class_nospill)) {
226 spill:
227 assert(_jitc->function != NULL);
228 if (spec & jit_class_gpr) {
229 if (!_jitc->function->regoff[regno]) {
230 _jitc->function->regoff[regno] =
231 jit_allocai(sizeof(jit_word_t));
232 _jitc->again = 1;
233 }
234#if DEBUG
235 /* emit_stxi must not need temporary registers */
236 assert(!_jitc->getreg);
237 _jitc->getreg = 1;
238#endif
239 emit_stxi(_jitc->function->regoff[regno], JIT_FP, regno);
240#if DEBUG
241 _jitc->getreg = 0;
242#endif
243 }
244 else {
245 if (!_jitc->function->regoff[regno]) {
246 _jitc->function->regoff[regno] =
247 jit_allocai(sizeof(jit_float64_t));
248 _jitc->again = 1;
249 }
250#if DEBUG
251 /* emit_stxi must not need temporary registers */
252 assert(!_jitc->getreg);
253 _jitc->getreg = 1;
254#endif
255 emit_stxi_d(_jitc->function->regoff[regno], JIT_FP, regno);
256#if DEBUG
257 _jitc->getreg = 0;
258#endif
259 }
260 jit_regset_setbit(&_jitc->regsav, regno);
261 regarg:
262 jit_regset_setbit(&_jitc->regarg, regno);
263 if (jit_class(_rvs[regno].spec) & jit_class_sav) {
264 /* if will modify callee save registers without a
265 * function prolog, better patch this assertion */
266 assert(_jitc->function != NULL);
267 if (!jit_regset_tstbit(&_jitc->function->regset, regno)) {
268 jit_regset_setbit(&_jitc->function->regset, regno);
269 _jitc->again = 1;
270 }
271 }
272 return (regno);
273 }
274 }
275 }
276 else {
277 /* nospill hint only valid during emit" */
278 assert(!(regspec & jit_class_nospill));
279 for (regno = 0; regno < _jitc->reglen; regno++) {
280 if ((jit_class(_rvs[regno].spec) & spec) == spec &&
281 !jit_regset_tstbit(&_jitc->regsav, regno) &&
282 !jit_regset_tstbit(&_jitc->regarg, regno)) {
283 jit_regset_setbit(&_jitc->regarg, regno);
284 jit_regset_setbit(&_jitc->regsav, regno);
285 jit_save(regno);
286 return (jit_regno_patch|regno);
287 }
288 }
289 }
290
291 /* Out of hardware registers */
292fail:
293 assert(regspec & jit_class_chk);
294 return (JIT_NOREG);
295}
296
297void
298_jit_unget_reg(jit_state_t *_jit, jit_int32_t regno)
299{
300 regno = jit_regno(regno);
301 if (jit_regset_tstbit(&_jitc->regsav, regno)) {
302 if (_jitc->emit) {
303#if DEBUG
304 /* emit_ldxi must not need a temporary register */
305 assert(!_jitc->getreg);
306 _jitc->getreg = 1;
307#endif
308 if (jit_class(_rvs[regno].spec) & jit_class_gpr)
309 emit_ldxi(regno, JIT_FP, _jitc->function->regoff[regno]);
310 else
311 emit_ldxi_d(regno, JIT_FP, _jitc->function->regoff[regno]);
312#if DEBUG
313 /* emit_ldxi must not need a temporary register */
314 _jitc->getreg = 0;
315#endif
316 }
317 else
318 jit_load(regno);
319 jit_regset_clrbit(&_jitc->regsav, regno);
320 }
321#if defined(jit_carry)
322 assert((regno == jit_carry /*&& _NOREG != jit_carry*/) ||
323 jit_regset_tstbit(&_jitc->regarg, regno) != 0);
324#else
325 assert(jit_regset_tstbit(&_jitc->regarg, regno) != 0);
326#endif
327 jit_regset_clrbit(&_jitc->regarg, regno);
328}
329
330jit_bool_t
331_jit_callee_save_p(jit_state_t *_jit, jit_int32_t regno)
332{
333 assert(regno >= 0 && regno < JIT_NOREG);
334 return (!!(_rvs[regno].spec & jit_class_sav));
335}
336
337extern jit_bool_t
338_jit_pointer_p(jit_state_t *_jit, jit_pointer_t address)
339{
340 return ((jit_uint8_t *)address >= _jit->code.ptr &&
341 (jit_word_t)address < _jit->pc.w);
342}
343
344#if __ia64__
345void
346jit_regset_com(jit_regset_t *u, jit_regset_t *v)
347{
348 u->rl = ~v->rl; u->rh = ~v->rh;
349 u->fl = ~v->fl; u->fh = ~v->fh;
350}
351
352void
353jit_regset_and(jit_regset_t *u, jit_regset_t *v, jit_regset_t *w)
354{
355 u->rl = v->rl & w->rl; u->rh = v->rh & w->rh;
356 u->fl = v->fl & w->fl; u->fh = v->fh & w->fh;
357}
358
359void
360jit_regset_ior(jit_regset_t *u, jit_regset_t *v, jit_regset_t *w)
361{
362 u->rl = v->rl | w->rl; u->rh = v->rh | w->rh;
363 u->fl = v->fl | w->fl; u->fh = v->fh | w->fh;
364}
365
366void
367jit_regset_xor(jit_regset_t *u, jit_regset_t *v, jit_regset_t *w)
368{
369 u->rl = v->rl ^ w->rl; u->rh = v->rh ^ w->rh;
370 u->fl = v->fl ^ w->fl; u->fh = v->fh ^ w->fh;
371}
372
373void
374jit_regset_set(jit_regset_t *u, jit_regset_t *v)
375{
376 u->rl = v->rl; u->rh = v->rh;
377 u->fl = v->fl; u->fh = v->fh;
378}
379
380void
381jit_regset_set_mask(jit_regset_t *u, jit_int32_t v)
382{
383 jit_bool_t w = !!(v & (v - 1));
384
385 assert(v >= 0 && v <= 256);
386 if (v == 0)
387 u->rl = u->rh = u->fl = u->fh = -1LL;
388 else if (v <= 64) {
389 u->rl = w ? (1LL << v) - 1 : -1LL;
390 u->rh = u->fl = u->fh = 0;
391 }
392 else if (v <= 128) {
393 u->rl = -1LL;
394 u->rh = w ? (1LL << (v - 64)) - 1 : -1LL;
395 u->fl = u->fh = 0;
396 }
397 else if (v <= 192) {
398 u->rl = u->rh = -1LL;
399 u->fl = w ? (1LL << (v - 128)) - 1 : -1LL;
400 u->fh = 0;
401 }
402 else {
403 u->rl = u->rh = u->fl = -1LL;
404 u->fh = w ? (1LL << (v - 128)) - 1 : -1LL;
405 }
406}
407
408jit_bool_t
409jit_regset_cmp_ui(jit_regset_t *u, jit_word_t v)
410{
411 return !((u->rl == v && u->rh == 0 && u->fl == 0 && u->fh == 0));
412}
413
414void
415jit_regset_set_ui(jit_regset_t *u, jit_word_t v)
416{
417 u->rl = v;
418 u->rh = u->fl = u->fh = 0;
419}
420
421jit_bool_t
422jit_regset_set_p(jit_regset_t *u)
423{
424 return (u->rl || u->rh || u->fl || u->fh);
425}
426
427void
428jit_regset_clrbit(jit_regset_t *set, jit_int32_t bit)
429{
430 assert(bit >= 0 && bit <= 255);
431 if (bit < 64)
432 set->rl &= ~(1LL << bit);
433 else if (bit < 128)
434 set->rh &= ~(1LL << (bit - 64));
435 else if (bit < 192)
436 set->fl &= ~(1LL << (bit - 128));
437 else
438 set->fh &= ~(1LL << (bit - 192));
439}
440
441void
442jit_regset_setbit(jit_regset_t *set, jit_int32_t bit)
443{
444 assert(bit >= 0 && bit <= 255);
445 if (bit < 64)
446 set->rl |= 1LL << bit;
447 else if (bit < 128)
448 set->rh |= 1LL << (bit - 64);
449 else if (bit < 192)
450 set->fl |= 1LL << (bit - 128);
451 else
452 set->fh |= 1LL << (bit - 192);
453}
454
455jit_bool_t
456jit_regset_tstbit(jit_regset_t *set, jit_int32_t bit)
457{
458 assert(bit >= 0 && bit <= 255);
459 if (bit < 64)
460 return (!!(set->rl & (1LL << bit)));
461 else if (bit < 128)
462 return (!!(set->rh & (1LL << (bit - 64))));
463 else if (bit < 192)
464 return (!!(set->fl & (1LL << (bit - 128))));
465 return (!!(set->fh & (1LL << (bit - 192))));
466}
467
468unsigned long
469jit_regset_scan1(jit_regset_t *set, jit_int32_t offset)
470{
471 assert(offset >= 0 && offset <= 255);
472 for (; offset < 64; offset++) {
473 if (set->rl & (1LL << offset))
474 return (offset);
475 }
476 for (; offset < 128; offset++) {
477 if (set->rh & (1LL << (offset - 64)))
478 return (offset);
479 }
480 for (; offset < 192; offset++) {
481 if (set->fl & (1LL << (offset - 128)))
482 return (offset);
483 }
484 for (; offset < 256; offset++) {
485 if (set->fh & (1LL << (offset - 192)))
486 return (offset);
487 }
488 return (ULONG_MAX);
489}
490
491#elif __sparc__ && __WORDSIZE == 64
492void
493jit_regset_com(jit_regset_t *u, jit_regset_t *v)
494{
495 u->rl = ~v->rl; u->rh = ~v->rh;
496}
497
498void
499jit_regset_and(jit_regset_t *u, jit_regset_t *v, jit_regset_t *w)
500{
501 u->rl = v->rl & w->rl; u->rh = v->rh & w->rh;
502}
503
504void
505jit_regset_ior(jit_regset_t *u, jit_regset_t *v, jit_regset_t *w)
506{
507 u->rl = v->rl | w->rl; u->rh = v->rh | w->rh;
508}
509
510void
511jit_regset_xor(jit_regset_t *u, jit_regset_t *v, jit_regset_t *w)
512{
513 u->rl = v->rl ^ w->rl; u->rh = v->rh ^ w->rh;
514}
515
516void
517jit_regset_set(jit_regset_t *u, jit_regset_t *v)
518{
519 u->rl = v->rl; u->rh = v->rh;
520}
521
522void
523jit_regset_set_mask(jit_regset_t *u, jit_int32_t v)
524{
525 jit_bool_t w = !!(v & (v - 1));
526
527 assert(v >= 0 && v <= 128);
528 if (v == 0)
529 u->rl = u->rh = -1LL;
530 else if (v <= 64) {
531 u->rl = w ? (1LL << v) - 1 : -1LL;
532 u->rh = 0;
533 }
534 else {
535 u->rl = -1LL;
536 u->rh = w ? (1LL << (v - 64)) - 1 : -1LL;
537 }
538}
539
540jit_bool_t
541jit_regset_cmp_ui(jit_regset_t *u, jit_word_t v)
542{
543 return !((u->rl == v && u->rh == 0));
544}
545
546void
547jit_regset_set_ui(jit_regset_t *u, jit_word_t v)
548{
549 u->rl = v;
550 u->rh = 0;
551}
552
553jit_bool_t
554jit_regset_set_p(jit_regset_t *u)
555{
556 return (u->rl || u->rh);
557}
558
559void
560jit_regset_clrbit(jit_regset_t *set, jit_int32_t bit)
561{
562 assert(bit >= 0 && bit <= 128);
563 if (bit < 64)
564 set->rl &= ~(1LL << bit);
565 else
566 set->rh &= ~(1LL << (bit - 64));
567}
568
569void
570jit_regset_setbit(jit_regset_t *set, jit_int32_t bit)
571{
572 assert(bit >= 0 && bit <= 127);
573 if (bit < 64)
574 set->rl |= 1LL << bit;
575 else
576 set->rh |= 1LL << (bit - 64);
577}
578
579jit_bool_t
580jit_regset_tstbit(jit_regset_t *set, jit_int32_t bit)
581{
582 assert(bit >= 0 && bit <= 127);
583 if (bit < 64)
584 return (!!(set->rl & (1LL << bit)));
585 else
586 return (!!(set->rh & (1LL << (bit - 64))));
587}
588
589unsigned long
590jit_regset_scan1(jit_regset_t *set, jit_int32_t offset)
591{
592 assert(offset >= 0 && offset <= 127);
593 for (; offset < 64; offset++) {
594 if (set->rl & (1LL << offset))
595 return (offset);
596 }
597 for (; offset < 128; offset++) {
598 if (set->rh & (1LL << (offset - 64)))
599 return (offset);
600 }
601 return (ULONG_MAX);
602}
603
604#else
605unsigned long
606jit_regset_scan1(jit_regset_t *set, jit_int32_t offset)
607{
608 jit_regset_t mask;
609 assert(offset >= 0 && offset <= 63);
610 if ((mask = *set >> offset)) {
611 for (;;) {
612 if (mask & 1)
613 return (offset);
614 mask >>= 1;
615 ++offset;
616 }
617 }
618 return (ULONG_MAX);
619}
620#endif
621
622void
623_jit_save(jit_state_t *_jit, jit_int32_t reg)
624{
625 reg = jit_regno(reg);
626 assert(!_jitc->realize);
627 _jitc->spill[reg] = jit_new_node_w(jit_code_save, reg);
628}
629
630void
631_jit_load(jit_state_t *_jit, jit_int32_t reg)
632{
633 jit_node_t *node;
634
635 reg = jit_regno(reg);
636 assert(!_jitc->realize);
637 assert(_jitc->spill[reg] != NULL);
638 node = jit_new_node_w(jit_code_load, reg);
639 /* create a path to flag the save/load is not required */
640 node->link = _jitc->spill[reg];
641 node->link->link = node;
642 _jitc->spill[reg] = NULL;
643}
644
645static jit_word_t
646hash_data(const void *data, jit_word_t length)
647{
648 const jit_uint8_t *ptr;
649 jit_word_t i, key;
650 for (i = key = 0, ptr = data; i < length; i++)
651 key = (key << (key & 1)) ^ ptr[i];
652 return (key);
653}
654
655jit_pointer_t
656_jit_address(jit_state_t *_jit, jit_node_t *node)
657{
658 assert(_jitc->done);
659 assert(node != NULL &&
660 /* If a node type that is documented to be a fixed marker */
661 (node->code == jit_code_note || node->code == jit_code_name ||
662 /* If another special fixed marker, returned by jit_indirect() */
663 (node->code == jit_code_label && (node->flag & jit_flag_use) != 0)));
664 return ((jit_pointer_t)node->u.w);
665}
666
667jit_node_t *
668_jit_data(jit_state_t *_jit, const void *data,
669 jit_word_t length, jit_int32_t align)
670{
671 jit_word_t key;
672 jit_node_t *node;
673
674 assert(!_jitc->realize);
675
676 /* Ensure there is space even if asking for a duplicate */
677 if (((_jitc->data.offset + 7) & -8) + length > _jit->data.length) {
678 jit_word_t size;
679
680 size = (_jit->data.length + length + 4096) & - 4095;
681 assert(size >= _jit->data.length);
682 if (_jitc->data.ptr == NULL)
683 jit_alloc((jit_pointer_t *)&_jitc->data.ptr, size);
684 else
685 jit_realloc((jit_pointer_t *)&_jitc->data.ptr,
686 _jit->data.length, size);
687 _jit->data.length = size;
688 }
689 if (_jitc->data.table == NULL)
690 jit_alloc((jit_pointer_t *)&_jitc->data.table,
691 (_jitc->data.size = 16) * sizeof(jit_node_t*));
692
693 key = hash_data(data, length) & (_jitc->data.size - 1);
694 node = _jitc->data.table[key];
695 for (; node; node = node->next) {
696 if (node->v.w == length &&
697 memcmp(_jitc->data.ptr + node->u.w, data, length) == 0)
698 break;
699 }
700
701 if (!node) {
702 node = jit_new_node_no_link(jit_code_data);
703 if (!align)
704 align = length;
705 switch (align) {
706 case 0: case 1:
707 break;
708 case 2:
709 _jitc->data.offset = (_jitc->data.offset + 1) & -2;
710 break;
711 case 3: case 4:
712 _jitc->data.offset = (_jitc->data.offset + 3) & -4;
713 break;
714 default:
715 _jitc->data.offset = (_jitc->data.offset + 7) & -8;
716 break;
717 }
718 node->u.w = _jitc->data.offset;
719 node->v.w = length;
720 jit_memcpy(_jitc->data.ptr + _jitc->data.offset, data, length);
721 _jitc->data.offset += length;
722
723 node->next = _jitc->data.table[key];
724 _jitc->data.table[key] = node;
725 ++_jitc->data.count;
726
727 /* Rehash if more than 75% used table */
728 if (_jitc->data.count >
729 (_jitc->data.size >> 1) + (_jitc->data.size >> 2) &&
730 (_jitc->data.size << 1) > _jitc->data.size) {
731 jit_word_t i;
732 jit_node_t **hash;
733 jit_node_t *next;
734 jit_node_t *temp;
735
736 jit_alloc((jit_pointer_t *)&hash,
737 (_jitc->data.size << 1) * sizeof(jit_node_t*));
738 for (i = 0; i < _jitc->data.size; i++) {
739 temp = _jitc->data.table[i];
740 for (; temp; temp = next) {
741 next = temp->next;
742 key = hash_data(_jitc->data.ptr + temp->u.w, temp->v.w) &
743 ((_jitc->data.size << 1) - 1);
744 temp->next = hash[key];
745 hash[key] = temp;
746 }
747 }
748 jit_free((jit_pointer_t *)&_jitc->data.table);
749 _jitc->data.table = hash;
750 _jitc->data.size <<= 1;
751 }
752 }
753
754 return (node);
755}
756
757static void
758_new_pool(jit_state_t *_jit)
759{
760 jit_node_t *list;
761 jit_int32_t offset;
762
763 if (_jitc->pool.offset >= _jitc->pool.length) {
764 jit_int32_t length;
765
766 length = _jitc->pool.length + 16;
767 jit_realloc((jit_pointer_t *)&_jitc->pool.ptr,
768 _jitc->pool.length * sizeof(jit_node_t *),
769 length * sizeof(jit_node_t *));
770 _jitc->pool.length = length;
771 }
772 jit_alloc((jit_pointer_t *)(_jitc->pool.ptr + _jitc->pool.offset),
773 sizeof(jit_node_t) * 1024);
774 list = _jitc->pool.ptr[_jitc->pool.offset];
775 for (offset = 1; offset < 1024; offset++, list++)
776 list->next = list + 1;
777 list->next = _jitc->list;
778 _jitc->list = _jitc->pool.ptr[_jitc->pool.offset];
779 ++_jitc->pool.offset;
780}
781
782static jit_node_t *
783_new_node(jit_state_t *_jit, jit_code_t code)
784{
785 jit_node_t *node;
786
787 if (_jitc->list == NULL)
788 new_pool();
789 node = _jitc->list;
790 _jitc->list = node->next;
791 if (_jitc->synth)
792 node->flag |= jit_flag_synth;
793 node->next = NULL;
794 node->code = code;
795
796 return (node);
797}
798
799static inline jit_node_t *
800_link_node(jit_state_t *_jit, jit_node_t *node)
801{
802 if (_jitc->tail)
803 _jitc->tail->next = node;
804 else
805 _jitc->head = node;
806 return (_jitc->tail = node);
807}
808
809static inline void
810_del_node(jit_state_t *_jit, jit_node_t *prev, jit_node_t *node)
811{
812 if (prev == node) {
813 assert(prev == _jitc->head);
814 _jitc->head = node->next;
815 }
816 else
817 prev->next = node->next;
818 memset(node, 0, sizeof(jit_node_t));
819 node->next = _jitc->list;
820 _jitc->list = node;
821}
822
823static inline void
824_free_node(jit_state_t *_jit, jit_node_t *node)
825{
826 memset(node, 0, sizeof(jit_node_t));
827 node->next = _jitc->list;
828 _jitc->list = node;
829}
830
831static void
832_del_label(jit_state_t *_jit, jit_node_t *prev, jit_node_t *node)
833{
834 jit_block_t *block;
835
836 /* only allow call to del_label on linked labels */
837 block = _jitc->blocks.ptr + node->v.w;
838 assert(block->label == node);
839
840 /* del_label() should only be called when optimizing.
841 * This will leave an empty block index */
842 jit_regset_del(&block->reglive);
843 jit_regset_del(&block->regmask);
844 block->label = NULL;
845
846 /* redundant, should be already true */
847 assert(node->link == NULL);
848 del_node(prev, node);
849}
850
851jit_state_t *
852jit_new_state(void)
853{
854 jit_state_t *_jit;
855
856 jit_alloc((jit_pointer_t *)&_jit, sizeof(jit_state_t));
857 jit_alloc((jit_pointer_t *)&_jitc, sizeof(jit_compiler_t));
858 jit_regset_new(&_jitc->regarg);
859 jit_regset_new(&_jitc->regsav);
860 jit_regset_new(&_jitc->reglive);
861 jit_regset_new(&_jitc->regmask);
862
863 jit_init();
864
865 jit_alloc((jit_pointer_t *)&_jitc->spill,
866 _jitc->reglen * sizeof(jit_node_t*));
867 jit_alloc((jit_pointer_t *)&_jitc->gen,
868 _jitc->reglen * sizeof(jit_int32_t));
869 jit_alloc((jit_pointer_t *)&_jitc->values,
870 _jitc->reglen * sizeof(jit_value_t));
871
872 jit_alloc((jit_pointer_t *)&_jitc->patches.ptr,
873 (_jitc->patches.length = 1024) * sizeof(jit_patch_t));
874 jit_alloc((jit_pointer_t *)&_jitc->functions.ptr,
875 (_jitc->functions.length = 16) * sizeof(jit_function_t));
876 jit_alloc((jit_pointer_t *)&_jitc->pool.ptr,
877 (_jitc->pool.length = 16) * sizeof(jit_node_t*));
878 jit_alloc((jit_pointer_t *)&_jitc->blocks.ptr,
879 (_jitc->blocks.length = 16) * sizeof(jit_block_t));
880#if __arm__ && DISASSEMBLER
881 jit_alloc((jit_pointer_t *)&_jitc->data_info.ptr,
882 (_jitc->data_info.length = 1024) * sizeof(jit_data_info_t));
883#endif
884
885 /* allocate at most one extra note in case jit_name() is
886 * never called, or called after adding at least one note */
887 _jit->note.length = 1;
888 _jitc->note.size = sizeof(jit_note_t);
889
890 return (_jit);
891}
892
893void
894_jit_clear_state(jit_state_t *_jit)
895{
896#if DEVEL_DISASSEMBLER
897# define jit_really_clear_state() _jit_really_clear_state(_jit)
898}
899
900void _jit_really_clear_state(jit_state_t *_jit)
901{
902#endif
903 jit_word_t offset;
904 jit_function_t *function;
905
906 /* release memory not required at jit execution time and set
907 * pointers to NULL to explicitly know they are released */
908 _jitc->head = _jitc->tail = NULL;
909
910 jit_free((jit_pointer_t *)&_jitc->data.table);
911 _jitc->data.size = _jitc->data.count = 0;
912
913 jit_free((jit_pointer_t *)&_jitc->spill);
914 jit_free((jit_pointer_t *)&_jitc->gen);
915 jit_free((jit_pointer_t *)&_jitc->values);
916
917 jit_free((jit_pointer_t *)&_jitc->blocks.ptr);
918
919 jit_free((jit_pointer_t *)&_jitc->patches.ptr);
920 _jitc->patches.offset = _jitc->patches.length = 0;
921
922 for (offset = 0; offset < _jitc->functions.offset; offset++) {
923 function = _jitc->functions.ptr + offset;
924 jit_free((jit_pointer_t *)&function->regoff);
925 }
926 jit_free((jit_pointer_t *)&_jitc->functions.ptr);
927 _jitc->functions.offset = _jitc->functions.length = 0;
928 _jitc->function = NULL;
929
930 for (offset = 0; offset < _jitc->pool.offset; offset++)
931 jit_free((jit_pointer_t *)(_jitc->pool.ptr + offset));
932 jit_free((jit_pointer_t *)&_jitc->pool.ptr);
933 _jitc->pool.offset = _jitc->pool.length = 0;
934 _jitc->list = NULL;
935
936 _jitc->note.head = _jitc->note.tail =
937 _jitc->note.name = _jitc->note.note = NULL;
938 _jitc->note.base = NULL;
939
940#if __arm__ && DISASSEMBLER
941 jit_free((jit_pointer_t *)&_jitc->data_info.ptr);
942#endif
943
944#if (__powerpc__ && _CALL_AIXDESC) || __ia64__
945 jit_free((jit_pointer_t *)&_jitc->prolog.ptr);
946#endif
947
948#if __ia64__
949 jit_regset_del(&_jitc->regs);
950#endif
951
952 jit_free((jit_pointer_t *)&_jitc);
953}
954
955void
956_jit_destroy_state(jit_state_t *_jit)
957{
958#if DEVEL_DISASSEMBLER
959 jit_really_clear_state();
960#endif
40a44dcb 961#if HAVE_MMAP
4a71579b
PC		 962 if (!_jit->user_code)
963 munmap(_jit->code.ptr, _jit->code.length);
964 if (!_jit->user_data)
965 munmap(_jit->data.ptr, _jit->data.length);
40a44dcb 966#endif
4a71579b
PC		 967 jit_free((jit_pointer_t *)&_jit);
968}
969
970void
971_jit_synth_inc(jit_state_t *_jit)
972{
973 assert(_jitc->synth < 8);
974 ++_jitc->synth;
975}
976
977jit_node_t *
978_jit_new_node(jit_state_t *_jit, jit_code_t code)
979{
980 assert(!_jitc->realize);
981 return (link_node(new_node(code)));
982}
983
984jit_node_t *
985_jit_new_node_no_link(jit_state_t *_jit, jit_code_t code)
986{
987 assert(!_jitc->realize);
988 return (new_node(code));
989}
990
991void
992_jit_link_node(jit_state_t *_jit, jit_node_t *node)
993{
994 assert(!_jitc->realize);
995 link_node(node);
996}
997
998void
999_jit_synth_dec(jit_state_t *_jit)
1000{
1001 assert(_jitc->synth > 0);
1002 --_jitc->synth;
1003}
1004
1005jit_node_t *
1006_jit_new_node_w(jit_state_t *_jit, jit_code_t code,
1007 jit_word_t u)
1008{
1009 jit_node_t *node = new_node(code);
1010 assert(!_jitc->realize);
1011 node->u.w = u;
1012 return (link_node(node));
1013}
1014
1015jit_node_t *
1016_jit_new_node_f(jit_state_t *_jit, jit_code_t code,
1017 jit_float32_t u)
1018{
1019 jit_node_t *node = new_node(code);
1020 assert(!_jitc->realize);
1021 node->u.f = u;
1022 return (link_node(node));
1023}
1024
1025jit_node_t *
1026_jit_new_node_d(jit_state_t *_jit, jit_code_t code,
1027 jit_float64_t u)
1028{
1029 jit_node_t *node = new_node(code);
1030 assert(!_jitc->realize);
1031 node->u.d = u;
1032 return (link_node(node));
1033}
1034
1035jit_node_t *
1036_jit_new_node_p(jit_state_t *_jit, jit_code_t code,
1037 jit_pointer_t u)
1038{
1039 jit_node_t *node = new_node(code);
1040 assert(!_jitc->realize);
1041 node->u.p = u;
1042 return (link_node(node));
1043}
1044
1045jit_node_t *
1046_jit_new_node_ww(jit_state_t *_jit, jit_code_t code,
1047 jit_word_t u, jit_word_t v)
1048{
1049 jit_node_t *node = new_node(code);
1050 assert(!_jitc->realize);
1051 node->u.w = u;
1052 node->v.w = v;
1053 return (link_node(node));
1054}
1055
1056jit_node_t *
1057_jit_new_node_wp(jit_state_t *_jit, jit_code_t code,
1058 jit_word_t u, jit_pointer_t v)
1059{
1060 return (jit_new_node_ww(code, u, (jit_word_t)v));
1061}
1062
1063jit_node_t *
1064_jit_new_node_fp(jit_state_t *_jit, jit_code_t code,
1065 jit_float32_t u, jit_pointer_t v)
1066{
1067 jit_node_t *node = new_node(code);
1068 assert(!_jitc->realize);
1069 node->u.f = u;
1070 node->v.w = (jit_word_t)v;
1071 return (link_node(node));
1072}
1073
1074jit_node_t *
1075_jit_new_node_dp(jit_state_t *_jit, jit_code_t code,
1076 jit_float64_t u, jit_pointer_t v)
1077{
1078 jit_node_t *node = new_node(code);
1079 assert(!_jitc->realize);
1080 node->u.d = u;
1081 node->v.w = (jit_word_t)v;
1082 return (link_node(node));
1083}
1084
1085jit_node_t *
1086_jit_new_node_pw(jit_state_t *_jit, jit_code_t code,
1087 jit_pointer_t u, jit_word_t v)
1088{
1089 return (jit_new_node_ww(code, (jit_word_t)u, v));
1090}
1091
1092jit_node_t *
1093_jit_new_node_wf(jit_state_t *_jit, jit_code_t code,
1094 jit_word_t u, jit_float32_t v)
1095{
1096 jit_node_t *node = new_node(code);
1097 assert(!_jitc->realize);
1098 node->u.w = u;
1099 node->v.f = v;
1100 return (link_node(node));
1101}
1102
1103jit_node_t *
1104_jit_new_node_wd(jit_state_t *_jit, jit_code_t code,
1105 jit_word_t u, jit_float64_t v)
1106{
1107 jit_node_t *node = new_node(code);
1108 assert(!_jitc->realize);
1109 node->u.w = u;
1110 node->v.d = v;
1111 return (link_node(node));
1112}
1113
1114jit_node_t *
1115_jit_new_node_www(jit_state_t *_jit, jit_code_t code,
1116 jit_word_t u, jit_word_t v, jit_word_t w)
1117{
1118 jit_node_t *node = new_node(code);
1119 assert(!_jitc->realize);
1120 node->u.w = u;
1121 node->v.w = v;
1122 node->w.w = w;
1123 return (link_node(node));
1124}
1125
1126jit_node_t *
1127_jit_new_node_qww(jit_state_t *_jit, jit_code_t code,
1128 jit_int32_t l, jit_int32_t h,
1129 jit_word_t v, jit_word_t w)
1130{
1131 jit_node_t *node = new_node(code);
1132 assert(!_jitc->realize);
1133 assert(l != h);
1134 node->u.q.l = l;
1135 node->u.q.h = h;
1136 node->v.w = v;
1137 node->w.w = w;
1138 return (link_node(node));
1139}
1140
ba3814c1
PC		 1141jit_node_t *
1142_jit_new_node_wwq(jit_state_t *_jit, jit_code_t code,
1143 jit_word_t u, jit_word_t v,
1144 jit_int32_t l, jit_int32_t h)
1145{
1146 jit_node_t *node = new_node(code);
1147 assert(!_jitc->realize);
1148 node->u.w = u;
1149 node->v.w = v;
1150 node->w.q.l = l;
1151 node->w.q.h = h;
1152 return (link_node(node));
1153}
1154
4a71579b
PC		 1155jit_node_t *
1156_jit_new_node_wwf(jit_state_t *_jit, jit_code_t code,
1157 jit_word_t u, jit_word_t v, jit_float32_t w)
1158{
1159 jit_node_t *node = new_node(code);
1160 assert(!_jitc->realize);
1161 node->u.w = u;
1162 node->v.w = v;
1163 node->w.f = w;
1164 return (link_node(node));
1165}
1166
1167jit_node_t *
1168_jit_new_node_wwd(jit_state_t *_jit, jit_code_t code,
1169 jit_word_t u, jit_word_t v, jit_float64_t w)
1170{
1171 jit_node_t *node = new_node(code);
1172 assert(!_jitc->realize);
1173 node->u.w = u;
1174 node->v.w = v;
1175 node->w.d = w;
1176 return (link_node(node));
1177}
1178
1179jit_node_t *
1180_jit_new_node_pww(jit_state_t *_jit, jit_code_t code,
1181 jit_pointer_t u, jit_word_t v, jit_word_t w)
1182{
1183 jit_node_t *node = new_node(code);
1184 assert(!_jitc->realize);
1185 node->u.p = u;
1186 node->v.w = v;
1187 node->w.w = w;
1188 return (link_node(node));
1189}
1190
1191jit_node_t *
1192_jit_new_node_pwf(jit_state_t *_jit, jit_code_t code,
1193 jit_pointer_t u, jit_word_t v, jit_float32_t w)
1194{
1195 jit_node_t *node = new_node(code);
1196 assert(!_jitc->realize);
1197 node->u.p = u;
1198 node->v.w = v;
1199 node->w.f = w;
1200 return (link_node(node));
1201}
1202
1203jit_node_t *
1204_jit_new_node_pwd(jit_state_t *_jit, jit_code_t code,
1205 jit_pointer_t u, jit_word_t v, jit_float64_t w)
1206{
1207 jit_node_t *node = new_node(code);
1208 assert(!_jitc->realize);
1209 node->u.p = u;
1210 node->v.w = v;
1211 node->w.d = w;
1212 return (link_node(node));
1213}
1214
1215jit_node_t *
1216_jit_label(jit_state_t *_jit)
1217{
1218 jit_node_t *node;
1219
1220 if (!(node = _jitc->tail) || node->code != jit_code_label) {
1221 node = jit_forward();
1222 jit_link(node);
1223 }
1224
1225 return (node);
1226}
1227
1228jit_node_t *
1229_jit_forward(jit_state_t *_jit)
1230{
1231 return (jit_new_node_no_link(jit_code_label));
1232}
1233
1234jit_node_t *
1235_jit_indirect(jit_state_t *_jit)
1236{
1237 jit_node_t *node;
1238
1239 node = jit_label();
1240 node->flag |= jit_flag_use;
1241
1242 return (node);
1243}
1244
1245void
1246_jit_link(jit_state_t *_jit, jit_node_t *node)
1247{
1248 jit_block_t *block;
1249
1250 assert((node->code == jit_code_label ||
1251 node->code == jit_code_prolog ||
1252 node->code == jit_code_epilog) && !node->next);
1253 jit_link_node(node);
1254 if (_jitc->blocks.offset >= _jitc->blocks.length) {
1255 jit_word_t length;
1256
1257 length = _jitc->blocks.length + 16;
1258 jit_realloc((jit_pointer_t *)&_jitc->blocks.ptr,
1259 _jitc->blocks.length * sizeof(jit_block_t),
1260 length * sizeof(jit_block_t));
1261 _jitc->blocks.length = length;
1262 }
1263 block = _jitc->blocks.ptr + _jitc->blocks.offset;
1264 block->label = node;
1265 node->v.w = _jitc->blocks.offset;
1266 jit_regset_new(&block->reglive);
1267 jit_regset_new(&block->regmask);
1268 ++_jitc->blocks.offset;
1269}
1270
1271jit_bool_t
1272_jit_forward_p(jit_state_t *_jit, jit_node_t *node)
1273{
1274 return (node->code == jit_code_label && !node->next && node != _jitc->tail);
1275}
1276
1277jit_bool_t
1278_jit_indirect_p(jit_state_t *_jit, jit_node_t *node)
1279{
1280 return (node->code == jit_code_label && !!(node->flag & jit_flag_use));
1281}
1282
1283jit_bool_t
1284_jit_target_p(jit_state_t *_jit, jit_node_t *node)
1285{
1286 return (node->code == jit_code_label && !!node->link);
1287}
1288
1289void
1290_jit_prepare(jit_state_t *_jit)
1291{
1292 assert(_jitc->function != NULL);
1293 _jitc->function->call.call = jit_call_default;
1294 _jitc->function->call.argi =
1295 _jitc->function->call.argf =
1296 _jitc->function->call.size = 0;
1297 _jitc->prepare = jit_new_node(jit_code_prepare);
1298}
1299
1300void
1301_jit_patch(jit_state_t* _jit, jit_node_t *instr)
1302{
1303 jit_node_t *label;
1304
1305 if (!(label = _jitc->tail) || label->code != jit_code_label)
1306 label = jit_label();
1307 jit_patch_at(instr, label);
1308}
1309
1310jit_int32_t
1311_jit_classify(jit_state_t *_jit, jit_code_t code)
1312{
1313 jit_int32_t mask;
1314
1315 switch (code) {
1316 case jit_code_data: case jit_code_save: case jit_code_load:
1317 case jit_code_name: case jit_code_label: case jit_code_note:
1318 case jit_code_prolog: case jit_code_ellipsis: case jit_code_va_push:
1319 case jit_code_epilog: case jit_code_ret: case jit_code_prepare:
1320 mask = 0;
1321 break;
1322 case jit_code_live: case jit_code_va_end:
1323 case jit_code_retr: case jit_code_retr_f: case jit_code_retr_d:
1324 case jit_code_pushargr: case jit_code_pushargr_f:
1325 case jit_code_pushargr_d:
1326 case jit_code_finishr: /* synthesized will set jit_cc_a0_jmp */
1327 mask = jit_cc_a0_reg;
1328 break;
1329 case jit_code_align: case jit_code_reti: case jit_code_pushargi:
1330 case jit_code_finishi: /* synthesized will set jit_cc_a0_jmp */
1331 mask = jit_cc_a0_int;
1332 break;
1333 case jit_code_reti_f: case jit_code_pushargi_f:
1334 mask = jit_cc_a0_flt;
1335 break;
1336 case jit_code_reti_d: case jit_code_pushargi_d:
1337 mask = jit_cc_a0_dbl;
1338 break;
1339 case jit_code_allocai:
1340 mask = jit_cc_a0_int|jit_cc_a1_int;
1341 break;
1342 case jit_code_arg: case jit_code_arg_f: case jit_code_arg_d:
1343 mask = jit_cc_a0_int|jit_cc_a0_arg;
1344 break;
1345 case jit_code_calli: case jit_code_jmpi:
1346 mask = jit_cc_a0_jmp;
1347 break;
1348 case jit_code_callr: case jit_code_jmpr:
1349 mask = jit_cc_a0_reg|jit_cc_a0_jmp;
1350 break;
1351 case jit_code_retval_c: case jit_code_retval_uc:
1352 case jit_code_retval_s: case jit_code_retval_us:
1353 case jit_code_retval_i: case jit_code_retval_ui:
1354 case jit_code_retval_l:
1355 case jit_code_retval_f: case jit_code_retval_d:
1356 case jit_code_va_start:
1357 mask = jit_cc_a0_reg|jit_cc_a0_chg;
1358 break;
1359 case jit_code_getarg_c: case jit_code_getarg_uc:
1360 case jit_code_getarg_s: case jit_code_getarg_us:
1361 case jit_code_getarg_i: case jit_code_getarg_ui:
1362 case jit_code_getarg_l:
1363 case jit_code_getarg_f: case jit_code_getarg_d:
1364 mask = jit_cc_a0_reg|jit_cc_a0_chg|jit_cc_a1_arg;
1365 break;
1366 case jit_code_putargr: case jit_code_putargr_f:
1367 case jit_code_putargr_d:
1368 mask = jit_cc_a0_reg|jit_cc_a1_arg;
1369 break;
1370 case jit_code_putargi:
1371 mask = jit_cc_a0_int|jit_cc_a1_arg;
1372 break;
1373 case jit_code_putargi_f:
1374 mask = jit_cc_a0_flt|jit_cc_a1_arg;
1375 break;
1376 case jit_code_putargi_d:
1377 mask = jit_cc_a0_dbl|jit_cc_a1_arg;
1378 break;
1379 case jit_code_movi: case jit_code_ldi_c: case jit_code_ldi_uc:
1380 case jit_code_ldi_s: case jit_code_ldi_us: case jit_code_ldi_i:
1381 case jit_code_ldi_ui: case jit_code_ldi_l: case jit_code_ldi_f:
1382 case jit_code_ldi_d:
1383 mask = jit_cc_a0_reg|jit_cc_a0_chg|jit_cc_a1_int;
1384 break;
1385 case jit_code_movi_f: case jit_code_movi_f_w:
1386 mask = jit_cc_a0_reg|jit_cc_a0_chg|jit_cc_a1_flt;
1387 break;
1388 case jit_code_movi_d: case jit_code_movi_d_w:
1389 mask = jit_cc_a0_reg|jit_cc_a0_chg|jit_cc_a1_dbl;
1390 break;
1391 case jit_code_movi_d_ww:
1392 mask = jit_cc_a0_reg|jit_cc_a0_chg|jit_cc_a1_reg|jit_cc_a1_chg|
1393 jit_cc_a2_dbl;
1394 break;
1395 case jit_code_negr: case jit_code_comr: case jit_code_movr:
1396 case jit_code_extr_c: case jit_code_extr_uc: case jit_code_extr_s:
1397 case jit_code_extr_us: case jit_code_extr_i: case jit_code_extr_ui:
1398 case jit_code_truncr_f_i: case jit_code_truncr_f_l:
1399 case jit_code_truncr_d_i: case jit_code_truncr_d_l:
1400 case jit_code_htonr_us: case jit_code_htonr_ui: case jit_code_htonr_ul:
40a44dcb 1401 case jit_code_bswapr_us: case jit_code_bswapr_ui: case jit_code_bswapr_ul:
4a71579b
PC		 1402 case jit_code_ldr_c: case jit_code_ldr_uc:
1403 case jit_code_ldr_s: case jit_code_ldr_us: case jit_code_ldr_i:
1404 case jit_code_ldr_ui: case jit_code_ldr_l: case jit_code_negr_f:
1405 case jit_code_absr_f: case jit_code_sqrtr_f: case jit_code_movr_f:
1406 case jit_code_extr_f: case jit_code_extr_d_f: case jit_code_ldr_f:
1407 case jit_code_negr_d: case jit_code_absr_d: case jit_code_sqrtr_d:
1408 case jit_code_movr_d: case jit_code_extr_d: case jit_code_extr_f_d:
1409 case jit_code_ldr_d:
1410 case jit_code_movr_w_f: case jit_code_movr_f_w:
1411 case jit_code_movr_w_d: case jit_code_movr_d_w:
1412 case jit_code_va_arg: case jit_code_va_arg_d:
1413 mask = jit_cc_a0_reg|jit_cc_a0_chg|jit_cc_a1_reg;
1414 break;
1415 case jit_code_movr_d_ww:
1416 mask = jit_cc_a0_reg|jit_cc_a0_chg|jit_cc_a1_reg|jit_cc_a1_chg|
1417 jit_cc_a2_reg;
1418 break;
1419 case jit_code_addi: case jit_code_addxi: case jit_code_addci:
1420 case jit_code_subi: case jit_code_subxi: case jit_code_subci:
1421 case jit_code_rsbi:
1422 case jit_code_muli: case jit_code_divi: case jit_code_divi_u:
1423 case jit_code_remi: case jit_code_remi_u: case jit_code_andi:
1424 case jit_code_ori: case jit_code_xori: case jit_code_lshi:
1425 case jit_code_rshi: case jit_code_rshi_u: case jit_code_lti:
1426 case jit_code_lti_u: case jit_code_lei: case jit_code_lei_u:
1427 case jit_code_eqi: case jit_code_gei: case jit_code_gei_u:
1428 case jit_code_gti: case jit_code_gti_u: case jit_code_nei:
1429 case jit_code_ldxi_c: case jit_code_ldxi_uc: case jit_code_ldxi_s:
1430 case jit_code_ldxi_us: case jit_code_ldxi_i: case jit_code_ldxi_ui:
1431 case jit_code_ldxi_l: case jit_code_ldxi_f: case jit_code_ldxi_d:
1432 mask = jit_cc_a0_reg|jit_cc_a0_chg|jit_cc_a1_reg|jit_cc_a2_int;
1433 break;
1434 case jit_code_qmuli: case jit_code_qmuli_u:
1435 case jit_code_qdivi: case jit_code_qdivi_u:
1436 mask = jit_cc_a0_reg|jit_cc_a0_rlh|jit_cc_a0_chg|
1437 jit_cc_a1_reg|jit_cc_a2_int;
1438 break;
1439 case jit_code_addi_f: case jit_code_subi_f: case jit_code_rsbi_f:
1440 case jit_code_muli_f: case jit_code_divi_f: case jit_code_lti_f:
1441 case jit_code_lei_f: case jit_code_eqi_f: case jit_code_gei_f:
1442 case jit_code_gti_f: case jit_code_nei_f: case jit_code_unlti_f:
1443 case jit_code_unlei_f: case jit_code_uneqi_f: case jit_code_ungei_f:
1444 case jit_code_ungti_f: case jit_code_ltgti_f: case jit_code_ordi_f:
1445 case jit_code_unordi_f:
1446 mask = jit_cc_a0_reg|jit_cc_a0_chg|jit_cc_a1_reg|jit_cc_a2_flt;
1447 break;
1448 case jit_code_addi_d: case jit_code_subi_d: case jit_code_rsbi_d:
1449 case jit_code_muli_d: case jit_code_divi_d: case jit_code_lti_d:
1450 case jit_code_lei_d: case jit_code_eqi_d: case jit_code_gei_d:
1451 case jit_code_gti_d: case jit_code_nei_d: case jit_code_unlti_d:
1452 case jit_code_unlei_d: case jit_code_uneqi_d: case jit_code_ungei_d:
1453 case jit_code_ungti_d: case jit_code_ltgti_d: case jit_code_ordi_d:
1454 case jit_code_unordi_d:
1455 mask = jit_cc_a0_reg|jit_cc_a0_chg|jit_cc_a1_reg|jit_cc_a2_dbl;
1456 break;
1457 case jit_code_addr: case jit_code_addxr: case jit_code_addcr:
1458 case jit_code_subr: case jit_code_subxr: case jit_code_subcr:
1459 case jit_code_mulr: case jit_code_divr: case jit_code_divr_u:
1460 case jit_code_remr: case jit_code_remr_u: case jit_code_andr:
1461 case jit_code_orr: case jit_code_xorr: case jit_code_lshr:
1462 case jit_code_rshr: case jit_code_rshr_u: case jit_code_ltr:
1463 case jit_code_ltr_u: case jit_code_ler: case jit_code_ler_u:
1464 case jit_code_eqr: case jit_code_ger: case jit_code_ger_u:
1465 case jit_code_gtr: case jit_code_gtr_u: case jit_code_ner:
1466 case jit_code_ldxr_c: case jit_code_ldxr_uc: case jit_code_ldxr_s:
1467 case jit_code_ldxr_us: case jit_code_ldxr_i: case jit_code_ldxr_ui:
1468 case jit_code_ldxr_l: case jit_code_addr_f: case jit_code_subr_f:
1469 case jit_code_mulr_f: case jit_code_divr_f: case jit_code_ltr_f:
1470 case jit_code_ler_f: case jit_code_eqr_f: case jit_code_ger_f:
1471 case jit_code_gtr_f: case jit_code_ner_f: case jit_code_unltr_f:
1472 case jit_code_unler_f: case jit_code_uneqr_f: case jit_code_unger_f:
1473 case jit_code_ungtr_f: case jit_code_ltgtr_f: case jit_code_ordr_f:
1474 case jit_code_unordr_f: case jit_code_ldxr_f: case jit_code_addr_d:
1475 case jit_code_subr_d: case jit_code_mulr_d: case jit_code_divr_d:
1476 case jit_code_ltr_d: case jit_code_ler_d: case jit_code_eqr_d:
1477 case jit_code_ger_d: case jit_code_gtr_d: case jit_code_ner_d:
1478 case jit_code_unltr_d: case jit_code_unler_d: case jit_code_uneqr_d:
1479 case jit_code_unger_d: case jit_code_ungtr_d: case jit_code_ltgtr_d:
1480 case jit_code_ordr_d: case jit_code_unordr_d: case jit_code_ldxr_d:
1481 case jit_code_movr_ww_d:
1482 mask = jit_cc_a0_reg|jit_cc_a0_chg|jit_cc_a1_reg|jit_cc_a2_reg;
1483 break;
1484 case jit_code_qmulr: case jit_code_qmulr_u:
1485 case jit_code_qdivr: case jit_code_qdivr_u:
1486 mask = jit_cc_a0_reg|jit_cc_a0_rlh|jit_cc_a0_chg|
1487 jit_cc_a1_reg|jit_cc_a2_reg;
1488 break;
1489 case jit_code_sti_c: case jit_code_sti_s: case jit_code_sti_i:
1490 case jit_code_sti_l: case jit_code_sti_f: case jit_code_sti_d:
1491 mask = jit_cc_a0_int|jit_cc_a1_reg;
1492 break;
1493 case jit_code_blti: case jit_code_blti_u: case jit_code_blei:
1494 case jit_code_blei_u: case jit_code_beqi: case jit_code_bgei:
1495 case jit_code_bgei_u: case jit_code_bgti: case jit_code_bgti_u:
1496 case jit_code_bnei: case jit_code_bmsi: case jit_code_bmci:
1497 mask = jit_cc_a0_jmp|jit_cc_a1_reg|jit_cc_a2_int;
1498 break;
1499 case jit_code_blti_f: case jit_code_blei_f: case jit_code_beqi_f:
1500 case jit_code_bgei_f: case jit_code_bgti_f: case jit_code_bnei_f:
1501 case jit_code_bunlti_f: case jit_code_bunlei_f: case jit_code_buneqi_f:
1502 case jit_code_bungei_f: case jit_code_bungti_f: case jit_code_bltgti_f:
1503 case jit_code_bordi_f: case jit_code_bunordi_f:
1504 mask = jit_cc_a0_jmp|jit_cc_a1_reg|jit_cc_a2_flt;
1505 break;
1506 case jit_code_blti_d: case jit_code_blei_d: case jit_code_beqi_d:
1507 case jit_code_bgei_d: case jit_code_bgti_d: case jit_code_bnei_d:
1508 case jit_code_bunlti_d: case jit_code_bunlei_d: case jit_code_buneqi_d:
1509 case jit_code_bungei_d: case jit_code_bungti_d: case jit_code_bltgti_d:
1510 case jit_code_bordi_d: case jit_code_bunordi_d:
1511 mask = jit_cc_a0_jmp|jit_cc_a1_reg|jit_cc_a2_dbl;
1512 break;
1513 case jit_code_allocar: /* synthesized instructions make it
1514 * equivalent to jit_cc_a0_chg */
1515 case jit_code_str_c: case jit_code_str_s: case jit_code_str_i:
1516 case jit_code_str_l: case jit_code_str_f: case jit_code_str_d:
1517 mask = jit_cc_a0_reg|jit_cc_a1_reg;
1518 break;
1519 case jit_code_stxi_c: case jit_code_stxi_s: case jit_code_stxi_i:
1520 case jit_code_stxi_l: case jit_code_stxi_f: case jit_code_stxi_d:
1521 mask = jit_cc_a0_int|jit_cc_a1_reg|jit_cc_a2_reg;
1522 break;
1523 case jit_code_bltr: case jit_code_bltr_u: case jit_code_bler:
1524 case jit_code_bler_u: case jit_code_beqr: case jit_code_bger:
1525 case jit_code_bger_u: case jit_code_bgtr: case jit_code_bgtr_u:
1526 case jit_code_bner: case jit_code_bmsr: case jit_code_bmcr:
1527 case jit_code_bltr_f: case jit_code_bler_f: case jit_code_beqr_f:
1528 case jit_code_bger_f: case jit_code_bgtr_f: case jit_code_bner_f:
1529 case jit_code_bunltr_f: case jit_code_bunler_f: case jit_code_buneqr_f:
1530 case jit_code_bunger_f: case jit_code_bungtr_f: case jit_code_bltgtr_f:
1531 case jit_code_bordr_f: case jit_code_bunordr_f:case jit_code_bltr_d:
1532 case jit_code_bler_d: case jit_code_beqr_d: case jit_code_bger_d:
1533 case jit_code_bgtr_d: case jit_code_bner_d: case jit_code_bunltr_d:
1534 case jit_code_bunler_d: case jit_code_buneqr_d: case jit_code_bunger_d:
1535 case jit_code_bungtr_d: case jit_code_bltgtr_d: case jit_code_bordr_d:
1536 case jit_code_bunordr_d:
1537 mask = jit_cc_a0_jmp|jit_cc_a1_reg|jit_cc_a2_reg;
1538 break;
1539 case jit_code_boaddi: case jit_code_boaddi_u: case jit_code_bxaddi:
1540 case jit_code_bxaddi_u: case jit_code_bosubi: case jit_code_bosubi_u:
1541 case jit_code_bxsubi: case jit_code_bxsubi_u:
1542 mask = jit_cc_a0_jmp|jit_cc_a1_reg|jit_cc_a1_chg|jit_cc_a2_int;
1543 break;
1544 case jit_code_stxr_c: case jit_code_stxr_s: case jit_code_stxr_i:
1545 case jit_code_stxr_l: case jit_code_stxr_f: case jit_code_stxr_d:
1546 mask = jit_cc_a0_reg|jit_cc_a1_reg|jit_cc_a2_reg;
1547 break;
1548 case jit_code_boaddr: case jit_code_boaddr_u: case jit_code_bxaddr:
1549 case jit_code_bxaddr_u: case jit_code_bosubr: case jit_code_bosubr_u:
1550 case jit_code_bxsubr: case jit_code_bxsubr_u:
1551 mask = jit_cc_a0_jmp|jit_cc_a1_reg|jit_cc_a1_chg|jit_cc_a2_reg;
1552 break;
40a44dcb
PC		 1553 case jit_code_movnr: case jit_code_movzr:
1554 mask = jit_cc_a0_reg|jit_cc_a0_cnd|jit_cc_a1_reg|jit_cc_a2_reg;
1555 break;
ba3814c1
PC		 1556 case jit_code_casr:
1557 mask = jit_cc_a0_reg|jit_cc_a0_chg|jit_cc_a1_reg|
1558 jit_cc_a2_reg|jit_cc_a2_rlh;
1559 break;
1560 case jit_code_casi:
1561 mask = jit_cc_a0_reg|jit_cc_a0_chg|jit_cc_a1_int|
1562 jit_cc_a2_reg|jit_cc_a2_rlh;
1563 break;
4a71579b
PC		 1564 default:
1565 abort();
1566 }
1567
1568 return (mask);
1569}
1570
1571void
1572_jit_patch_abs(jit_state_t *_jit, jit_node_t *instr, jit_pointer_t address)
1573{
1574 jit_int32_t mask;
1575
1576 switch (instr->code) {
1577 case jit_code_movi: case jit_code_ldi_c: case jit_code_ldi_uc:
1578 case jit_code_ldi_s: case jit_code_ldi_us: case jit_code_ldi_i:
1579 case jit_code_ldi_ui: case jit_code_ldi_l: case jit_code_ldi_f:
1580 case jit_code_ldi_d:
1581 instr->v.p = address;
1582 break;
1583 case jit_code_sti_c: case jit_code_sti_s: case jit_code_sti_i:
1584 case jit_code_sti_l: case jit_code_sti_f: case jit_code_sti_d:
1585 instr->u.p = address;
1586 break;
1587 default:
1588 mask = jit_classify(instr->code);
1589 assert((mask & (jit_cc_a0_reg|jit_cc_a0_jmp)) == jit_cc_a0_jmp);
1590 instr->u.p = address;
1591 }
1592}
1593
1594void
1595_jit_patch_at(jit_state_t *_jit, jit_node_t *instr, jit_node_t *label)
1596{
1597 jit_int32_t mask;
1598
1599 assert(!(instr->flag & jit_flag_node));
1600 instr->flag |= jit_flag_node;
1601 switch (instr->code) {
1602 case jit_code_movi:
1603 assert(label->code == jit_code_label ||
1604 label->code == jit_code_data);
1605 instr->v.n = label;
1606 if (label->code == jit_code_data)
1607 instr->flag |= jit_flag_data;
1608 break;
1609 case jit_code_jmpi:
1610 assert(label->code == jit_code_label ||
1611 label->code == jit_code_epilog);
1612 instr->u.n = label;
1613 break;
1614 default:
1615 mask = jit_classify(instr->code);
1616 assert((mask & (jit_cc_a0_reg|jit_cc_a0_jmp)) == jit_cc_a0_jmp);
1617 assert(label->code == jit_code_label);
1618 instr->u.n = label;
1619 break;
1620 }
1621 /* link field is used as list of nodes associated with a given label */
1622 instr->link = label->link;
1623 label->link = instr;
1624}
1625
1626void
1627_jit_optimize(jit_state_t *_jit)
1628{
ba3814c1 1629 jit_int32_t pass;
4a71579b
PC		 1630 jit_bool_t jump;
1631 jit_bool_t todo;
1632 jit_int32_t mask;
1633 jit_node_t *node;
1634 jit_block_t *block;
1635 jit_word_t offset;
1636
1637 _jitc->function = NULL;
1638
1639 thread_jumps();
1640 sequential_labels();
1641 split_branches();
1642
ba3814c1
PC		 1643 pass = 0;
1644
1645second_pass:
4a71579b
PC		 1646 /* create initial mapping of live register values
1647 * at the start of a basic block */
1648 for (offset = 0; offset < _jitc->blocks.offset; offset++) {
1649 block = _jitc->blocks.ptr + offset;
1650 if (!block->label)
1651 continue;
1652 if (block->label->code != jit_code_epilog)
1653 jit_setup(block);
1654 }
1655
1656 /* set live state of registers not referenced in a block, but
1657 * referenced in a jump target or normal flow */
1658 do {
1659 todo = 0;
1660 for (offset = 0; offset < _jitc->blocks.offset; offset++) {
1661 block = _jitc->blocks.ptr + offset;
1662 if (!block->label)
1663 continue;
1664 if (block->label->code != jit_code_epilog)
1665 jit_follow(block, &todo);
1666 }
1667 } while (todo);
1668
ba3814c1
PC		 1669 if (pass == 0) {
1670 todo = 0;
4a71579b 1671
ba3814c1
PC		 1672 patch_registers();
1673 if (simplify())
1674 todo = 1;
1675
1676 /* figure out labels that are only reached with a jump
1677 * and is required to do a simple redundant_store removal
1678 * on jit_beqi below */
1679 jump = 1;
1680 for (node = _jitc->head; node; node = node->next) {
1681 switch (node->code) {
1682 case jit_code_label:
1683 if (!jump)
1684 node->flag |= jit_flag_head;
1685 break;
1686 case jit_code_jmpi: case jit_code_jmpr:
1687 case jit_code_epilog:
1688 jump = 1;
1689 break;
1690 case jit_code_data: case jit_code_note:
1691 break;
1692 default:
1693 jump = 0;
1694 break;
1695 }
1696 }
1697
1698 for (node = _jitc->head; node; node = node->next) {
1699 mask = jit_classify(node->code);
1700 if (mask & jit_cc_a0_reg)
1701 node->u.w &= ~jit_regno_patch;
1702 if (mask & jit_cc_a1_reg)
1703 node->v.w &= ~jit_regno_patch;
1704 if (mask & jit_cc_a2_reg)
1705 node->w.w &= ~jit_regno_patch;
1706 if (node->code == jit_code_beqi) {
1707 if (redundant_store(node, 1))
1708 todo = 1;
1709 }
1710 else if (node->code == jit_code_bnei) {
1711 if (redundant_store(node, 0))
1712 todo = 1;
1713 }
1714 }
1715
1716 /* If instructions were removed, must recompute state at
1717 * start of blocks. */
1718 if (todo) {
1719 pass = 1;
1720 goto second_pass;
4a71579b
PC		 1721 }
1722 }
1723
1724 for (node = _jitc->head; node; node = node->next) {
1725 mask = jit_classify(node->code);
1726 if (mask & jit_cc_a0_reg)
1727 node->u.w &= ~jit_regno_patch;
1728 if (mask & jit_cc_a1_reg)
1729 node->v.w &= ~jit_regno_patch;
1730 if (mask & jit_cc_a2_reg)
1731 node->w.w &= ~jit_regno_patch;
ba3814c1
PC		 1732 if (node->code == jit_code_prolog)
1733 _jitc->function = _jitc->functions.ptr + node->w.w;
1734 else if(node->code == jit_code_epilog)
1735 _jitc->function = NULL;
1736 else {
4a71579b 1737#if JIT_HASH_CONSTS
ba3814c1
PC		 1738 if (mask & jit_cc_a0_flt) {
1739 node->u.p = jit_data(&node->u.f, sizeof(jit_float32_t), 4);
1740 node->flag |= jit_flag_node | jit_flag_data;
1741 }
1742 else if (mask & jit_cc_a0_dbl) {
1743 node->u.p = jit_data(&node->u.d, sizeof(jit_float64_t), 8);
1744 node->flag |= jit_flag_node | jit_flag_data;
1745 }
1746 else if (mask & jit_cc_a1_flt) {
1747 node->v.p = jit_data(&node->v.f, sizeof(jit_float32_t), 4);
1748 node->flag |= jit_flag_node | jit_flag_data;
1749 }
1750 else if (mask & jit_cc_a1_dbl) {
1751 node->v.p = jit_data(&node->v.d, sizeof(jit_float64_t), 8);
1752 node->flag |= jit_flag_node | jit_flag_data;
1753 }
1754 else if (mask & jit_cc_a2_flt) {
1755 node->w.p = jit_data(&node->w.f, sizeof(jit_float32_t), 4);
1756 node->flag |= jit_flag_node | jit_flag_data;
1757 }
1758 else if (mask & jit_cc_a2_dbl) {
1759 node->w.p = jit_data(&node->w.d, sizeof(jit_float64_t), 8);
1760 node->flag |= jit_flag_node | jit_flag_data;
1761 }
4a71579b 1762#endif
ba3814c1
PC		 1763 if (_jitc->function) {
1764 if ((mask & (jit_cc_a0_reg|jit_cc_a0_chg)) ==
1765 (jit_cc_a0_reg|jit_cc_a0_chg)) {
1766 if (mask & jit_cc_a0_rlh) {
1767 jit_regset_setbit(&_jitc->function->regset,
1768 jit_regno(node->u.q.l));
4a71579b 1769 jit_regset_setbit(&_jitc->function->regset,
ba3814c1
PC		 1770 jit_regno(node->u.q.h));
1771 }
1772 else
4a71579b 1773 jit_regset_setbit(&_jitc->function->regset,
ba3814c1 1774 jit_regno(node->u.w));
4a71579b 1775 }
ba3814c1
PC		 1776 if ((mask & (jit_cc_a1_reg|jit_cc_a1_chg)) ==
1777 (jit_cc_a1_reg|jit_cc_a1_chg))
1778 jit_regset_setbit(&_jitc->function->regset,
1779 jit_regno(node->v.w));
1780 if ((mask & (jit_cc_a2_reg|jit_cc_a2_chg)) ==
1781 (jit_cc_a2_reg|jit_cc_a2_chg))
1782 jit_regset_setbit(&_jitc->function->regset,
1783 jit_regno(node->w.w));
1784 }
4a71579b
PC		 1785 }
1786 }
1787}
1788
1789void
1790_jit_reglive(jit_state_t *_jit, jit_node_t *node)
1791{
1792 jit_int32_t spec;
1793 jit_int32_t value;
1794 jit_block_t *block;
1795
1796 switch (node->code) {
1797 case jit_code_label: case jit_code_prolog: case jit_code_epilog:
1798 block = _jitc->blocks.ptr + node->v.w;
1799 jit_regset_set(&_jitc->reglive, &block->reglive);
1800 break;
1801 case jit_code_callr:
1802 value = jit_regno(node->u.w);
1803 if (!(node->u.w & jit_regno_patch)) {
1804 jit_regset_setbit(&_jitc->reglive, value);
1805 }
1806 case jit_code_calli:
1807 for (value = 0; value < _jitc->reglen; value++) {
1808 spec = jit_class(_rvs[value].spec);
1809 if ((spec & jit_class_arg) && jit_regarg_p(node, value))
1810 jit_regset_setbit(&_jitc->reglive, value);
1811 else if (!(spec & jit_class_sav))
1812 jit_regset_clrbit(&_jitc->reglive, value);
1813 }
1814 break;
1815 default:
1816 value = jit_classify(node->code);
1817 if (value & jit_cc_a0_reg) {
1818 if (value & jit_cc_a0_rlh) {
1819 if (!(node->u.q.l & jit_regno_patch)) {
1820 if (value & jit_cc_a0_chg) {
1821 jit_regset_clrbit(&_jitc->reglive, node->u.q.l);
1822 jit_regset_setbit(&_jitc->regmask, node->u.q.l);
1823 }
1824 else
1825 jit_regset_setbit(&_jitc->reglive, node->u.q.l);
1826 }
1827 if (!(node->u.q.h & jit_regno_patch)) {
1828 if (value & jit_cc_a0_chg) {
1829 jit_regset_clrbit(&_jitc->reglive, node->u.q.h);
1830 jit_regset_setbit(&_jitc->regmask, node->u.q.h);
1831 }
1832 else
1833 jit_regset_setbit(&_jitc->reglive, node->u.q.h);
1834 }
1835 }
1836 else {
1837 if (!(node->u.w & jit_regno_patch)) {
1838 if (value & jit_cc_a0_chg) {
1839 jit_regset_clrbit(&_jitc->reglive, node->u.w);
1840 jit_regset_setbit(&_jitc->regmask, node->u.w);
1841 }
1842 else
1843 jit_regset_setbit(&_jitc->reglive, node->u.w);
1844 }
1845 }
1846 }
1847 if ((value & jit_cc_a1_reg) && !(node->v.w & jit_regno_patch)) {
1848 if (value & jit_cc_a1_chg) {
1849 jit_regset_clrbit(&_jitc->reglive, node->v.w);
1850 jit_regset_setbit(&_jitc->regmask, node->v.w);
1851 }
1852 else
1853 jit_regset_setbit(&_jitc->reglive, node->v.w);
1854 }
ba3814c1
PC		 1855 if (value & jit_cc_a2_reg) {
1856 if (value & jit_cc_a2_rlh) {
1857 /* Assume registers are not changed */
1858 if (!(node->w.q.l & jit_regno_patch))
1859 jit_regset_setbit(&_jitc->reglive, node->w.q.l);
1860 if (!(node->w.q.h & jit_regno_patch))
1861 jit_regset_setbit(&_jitc->reglive, node->w.q.h);
1862 }
1863 else {
1864 if (!(node->w.w & jit_regno_patch)) {
1865 if (value & jit_cc_a2_chg) {
1866 jit_regset_clrbit(&_jitc->reglive, node->w.w);
1867 jit_regset_setbit(&_jitc->regmask, node->w.w);
1868 }
1869 else
1870 jit_regset_setbit(&_jitc->reglive, node->w.w);
1871 }
4a71579b 1872 }
4a71579b
PC		 1873 }
1874 if (jit_regset_set_p(&_jitc->regmask)) {
1875 jit_update(node->next, &_jitc->reglive, &_jitc->regmask);
1876 if (jit_regset_set_p(&_jitc->regmask)) {
1877 /* any unresolved live state is considered as live */
1878 jit_regset_ior(&_jitc->reglive,
1879 &_jitc->reglive, &_jitc->regmask);
1880 jit_regset_set_ui(&_jitc->regmask, 0);
1881 }
1882 }
1883 break;
1884 }
1885}
1886
1887void
1888_jit_regarg_set(jit_state_t *_jit, jit_node_t *node, jit_int32_t value)
1889{
1890#if GET_JIT_SIZE
1891 jit_size_prepare();
1892#endif
1893 if (value & jit_cc_a0_reg) {
1894 if (value & jit_cc_a0_rlh) {
1895 jit_regset_setbit(&_jitc->regarg, jit_regno(node->u.q.l));
1896 jit_regset_setbit(&_jitc->regarg, jit_regno(node->u.q.h));
1897 }
1898 else
1899 jit_regset_setbit(&_jitc->regarg, jit_regno(node->u.w));
1900 }
1901 if (value & jit_cc_a1_reg)
1902 jit_regset_setbit(&_jitc->regarg, jit_regno(node->v.w));
ba3814c1
PC		 1903 if (value & jit_cc_a2_reg) {
1904 if (value & jit_cc_a2_rlh) {
1905 jit_regset_setbit(&_jitc->regarg, jit_regno(node->w.q.l));
1906 jit_regset_setbit(&_jitc->regarg, jit_regno(node->w.q.h));
1907 }
1908 else
1909 jit_regset_setbit(&_jitc->regarg, jit_regno(node->w.w));
1910 }
4a71579b
PC		 1911}
1912
1913void
1914_jit_regarg_clr(jit_state_t *_jit, jit_node_t *node, jit_int32_t value)
1915{
1916#if GET_JIT_SIZE
1917 jit_size_collect(node);
1918#endif
1919 if (value & jit_cc_a0_reg) {
1920 if (value & jit_cc_a0_rlh) {
1921 jit_regset_clrbit(&_jitc->regarg, jit_regno(node->u.q.l));
1922 jit_regset_clrbit(&_jitc->regarg, jit_regno(node->u.q.h));
1923 }
1924 else
1925 jit_regset_clrbit(&_jitc->regarg, jit_regno(node->u.w));
1926 }
1927 if (value & jit_cc_a1_reg)
1928 jit_regset_clrbit(&_jitc->regarg, jit_regno(node->v.w));
ba3814c1
PC		 1929 if (value & jit_cc_a2_reg) {
1930 if (value & jit_cc_a2_rlh) {
1931 jit_regset_clrbit(&_jitc->regarg, jit_regno(node->w.q.l));
1932 jit_regset_clrbit(&_jitc->regarg, jit_regno(node->w.q.h));
1933 }
1934 else
1935 jit_regset_clrbit(&_jitc->regarg, jit_regno(node->w.w));
1936 }
4a71579b
PC		 1937}
1938
1939void
1940_jit_realize(jit_state_t *_jit)
1941{
1942 assert(!_jitc->realize);
1943 if (_jitc->function)
1944 jit_epilog();
1945 jit_optimize();
1946 _jitc->realize = 1;
1947
1948 /* ensure it is aligned */
1949 _jitc->data.offset = (_jitc->data.offset + 7) & -8;
1950
1951#if GET_JIT_SIZE
1952 /* Heuristic to guess code buffer size */
1953 _jitc->mult = 4;
1954 _jit->code.length = _jitc->pool.length * 1024 * _jitc->mult;
1955#else
1956 _jit->code.length = jit_get_size();
1957#endif
1958}
1959
1960void
1961_jit_dataset(jit_state_t *_jit)
1962{
1963 jit_uint8_t *ptr;
1964 jit_node_t *node;
1965 jit_word_t offset;
1966#if defined(__sgi)
1967 int mmap_fd;
1968#endif
1969
1970 assert(!_jitc->dataset);
40a44dcb
PC		 1971#if !HAVE_MMAP
1972 assert(_jit->user_data);
1973#else
4a71579b
PC		 1974 if (!_jit->user_data) {
1975
1976 /* create read only data buffer */
1977 _jit->data.length = (_jitc->data.offset +
1978 /* reserve space for annotations */
1979 _jitc->note.size + 4095) & -4096;
1980#if defined(__sgi)
1981 mmap_fd = open("/dev/zero", O_RDWR);
1982#endif
1983 _jit->data.ptr = mmap(NULL, _jit->data.length,
1984 PROT_READ | PROT_WRITE,
1985 MAP_PRIVATE | MAP_ANON, mmap_fd, 0);
1986 assert(_jit->data.ptr != MAP_FAILED);
1987#if defined(__sgi)
1988 close(mmap_fd);
1989#endif
1990 }
40a44dcb 1991#endif /* !HAVE_MMAP */
4a71579b
PC		 1992
1993 if (!_jitc->no_data)
1994 jit_memcpy(_jit->data.ptr, _jitc->data.ptr, _jitc->data.offset);
1995
1996 if (_jitc->no_note) {
1997 /* Space for one note is always allocated, so revert it here
1998 * if after jit_new_state was called, it is also requested to
1999 * not generate annotation information */
2000 _jit->note.length = 0;
2001 _jitc->note.size = 0;
2002 }
2003 else {
2004 _jitc->note.base = _jit->data.ptr;
2005 if (!_jitc->no_data)
2006 _jitc->note.base += _jitc->data.offset;
2007 memset(_jitc->note.base, 0, _jitc->note.size);
2008 }
2009
2010 if (_jit->user_data)
2011 /* Need the temporary hashed data until jit_emit is finished */
2012 ptr = _jitc->no_data ? _jitc->data.ptr : _jit->data.ptr;
2013 else {
2014 ptr = _jit->data.ptr;
2015 /* Temporary hashed data no longer required */
2016 jit_free((jit_pointer_t *)&_jitc->data.ptr);
2017 }
2018
2019 for (offset = 0; offset < _jitc->data.size; offset++) {
2020 for (node = _jitc->data.table[offset]; node; node = node->next) {
2021 node->flag |= jit_flag_patch;
2022 node->u.w = (jit_word_t)(ptr + node->u.w);
2023 }
2024 }
2025
2026 _jitc->dataset = 1;
2027}
2028
2029jit_pointer_t
2030_jit_get_code(jit_state_t *_jit, jit_word_t *length)
2031{
2032 assert(_jitc->realize);
2033 if (length) {
2034 if (_jitc->done)
2035 /* If code already generated, return exact size of code */
2036 *length = _jit->pc.uc - _jit->code.ptr;
2037 else
2038 /* Else return current size of the code buffer */
2039 *length = _jit->code.length;
2040 }
2041
2042 return (_jit->code.ptr);
2043}
2044
2045void
2046_jit_set_code(jit_state_t *_jit, jit_pointer_t ptr, jit_word_t length)
2047{
2048 assert(_jitc->realize);
2049 _jit->code.ptr = ptr;
2050 _jit->code.length = length;
2051 _jit->user_code = 1;
2052}
2053
2054jit_pointer_t
2055_jit_get_data(jit_state_t *_jit, jit_word_t *data_size, jit_word_t *note_size)
2056{
2057 assert(_jitc->realize);
2058 if (data_size)
2059 *data_size = _jitc->data.offset;
2060 if (note_size)
2061 *note_size = _jitc->note.size;
2062 return (_jit->data.ptr);
2063}
2064
2065void
2066_jit_set_data(jit_state_t *_jit, jit_pointer_t ptr,
2067 jit_word_t length, jit_word_t flags)
2068{
2069 assert(_jitc->realize);
2070 if (flags & JIT_DISABLE_DATA)
2071 _jitc->no_data = 1;
2072 else
2073 assert(length >= _jitc->data.offset);
2074 if (flags & JIT_DISABLE_NOTE)
2075 _jitc->no_note = 1;
2076 else {
2077 if (flags & JIT_DISABLE_DATA)
2078 assert(length >= _jitc->note.size);
2079 else
2080 assert(length >= _jitc->data.offset + _jitc->note.size);
2081 }
2082 _jit->data.ptr = ptr;
2083 _jit->data.length = length;
2084 _jit->user_data = 1;
2085}
2086
2087jit_pointer_t
2088_jit_emit(jit_state_t *_jit)
2089{
2090 jit_pointer_t code;
2091 jit_node_t *node;
2092 size_t length;
2093 int result;
2094#if defined(__sgi)
2095 int mmap_fd;
2096#endif
2097
2098 if (!_jitc->realize)
2099 jit_realize();
2100
2101 if (!_jitc->dataset)
2102 jit_dataset();
2103
2104 _jitc->emit = 1;
2105
40a44dcb
PC		 2106#if !HAVE_MMAP
2107 assert(_jit->user_code);
2108#else
4a71579b
PC		 2109 if (!_jit->user_code) {
2110#if defined(__sgi)
2111 mmap_fd = open("/dev/zero", O_RDWR);
2112#endif
2113 _jit->code.ptr = mmap(NULL, _jit->code.length,
2114 PROT_EXEC | PROT_READ | PROT_WRITE,
2115 MAP_PRIVATE | MAP_ANON, mmap_fd, 0);
2116 assert(_jit->code.ptr != MAP_FAILED);
2117 }
40a44dcb 2118#endif /* !HAVE_MMAP */
4a71579b
PC		 2119 _jitc->code.end = _jit->code.ptr + _jit->code.length -
2120 jit_get_max_instr();
2121 _jit->pc.uc = _jit->code.ptr;
2122
2123 for (;;) {
2124 if ((code = emit_code()) == NULL) {
2125 _jitc->patches.offset = 0;
2126 for (node = _jitc->head; node; node = node->next) {
2127 if (node->link &&
2128 (node->code == jit_code_label ||
2129 node->code == jit_code_epilog))
2130 node->flag &= ~jit_flag_patch;
2131 }
40a44dcb
PC		 2132#if !HAVE_MMAP
2133 assert(_jit->user_code);
2134#else
4a71579b
PC		 2135 if (_jit->user_code)
2136 goto fail;
2137#if GET_JIT_SIZE
2138 ++_jitc->mult;
2139 length = _jitc->pool.length * 1024 * _jitc->mult;
2140#else
2141 /* Should only happen on very special cases */
2142 length = _jit->code.length + 4096;
2143#endif
2144
2145#if !HAVE_MREMAP
2146 munmap(_jit->code.ptr, _jit->code.length);
2147#endif
2148
2149#if HAVE_MREMAP
2150# if __NetBSD__
2151 _jit->code.ptr = mremap(_jit->code.ptr, _jit->code.length,
2152 _jit->code.ptr, length, 0);
2153# else
2154 _jit->code.ptr = mremap(_jit->code.ptr, _jit->code.length,
2155 length, MREMAP_MAYMOVE, NULL);
2156# endif
2157#else
2158 _jit->code.ptr = mmap(NULL, length,
2159 PROT_EXEC | PROT_READ | PROT_WRITE,
2160 MAP_PRIVATE | MAP_ANON, mmap_fd, 0);
2161#endif
2162
2163 assert(_jit->code.ptr != MAP_FAILED);
2164 _jit->code.length = length;
2165 _jitc->code.end = _jit->code.ptr + _jit->code.length -
2166 jit_get_max_instr();
2167 _jit->pc.uc = _jit->code.ptr;
40a44dcb 2168#endif /* !HAVE_MMAP */
4a71579b
PC		 2169 }
2170 else
2171 break;
2172 }
2173
2174#if defined(__sgi)
2175 if (!_jit->user_code)
2176 close(mmap_fd);
2177#endif
2178
2179 _jitc->done = 1;
2180 if (!_jitc->no_note)
2181 jit_annotate();
2182
2183 if (_jit->user_data)
2184 jit_free((jit_pointer_t *)&_jitc->data.ptr);
40a44dcb 2185#if HAVE_MMAP
4a71579b
PC		 2186 else {
2187 result = mprotect(_jit->data.ptr, _jit->data.length, PROT_READ);
2188 assert(result == 0);
2189 }
2190 if (!_jit->user_code) {
2191 result = mprotect(_jit->code.ptr, _jit->code.length,
2192 PROT_READ | PROT_EXEC);
2193 assert(result == 0);
2194 }
40a44dcb 2195#endif /* HAVE_MMAP */
4a71579b
PC		 2196
2197 return (_jit->code.ptr);
2198fail:
2199 return (NULL);
2200}
2201
2202void
2203_jit_frame(jit_state_t *_jit, jit_int32_t frame)
2204{
2205 jit_trampoline(frame, 1);
2206}
2207
2208void
2209_jit_tramp(jit_state_t *_jit, jit_int32_t frame)
2210{
2211 jit_trampoline(frame, 0);
2212}
2213
2214void
2215_jit_trampoline(jit_state_t *_jit, jit_int32_t frame, jit_bool_t prolog)
2216{
2217 jit_int32_t regno;
2218
2219 /* Must be called after prolog, actually, just to simplify
2220 * tests and know there is a current function and that
2221 * _jitc->function->self.aoff is at the before any alloca value */
2222 assert(_jitc->tail && _jitc->tail->code == jit_code_prolog);
2223
2224 /* + 24 for 3 possible spilled temporaries (that could be a double) */
2225 frame += 24;
2226#if defined(__hppa__)
2227 frame += _jitc->function->self.aoff;
2228#else
2229 frame -= _jitc->function->self.aoff;
2230#endif
2231 _jitc->function->frame = frame;
2232 if (prolog)
2233 _jitc->function->define_frame = 1;
2234 else
2235 _jitc->function->assume_frame = 1;
2236 for (regno = 0; regno < _jitc->reglen; regno++)
2237 if (jit_class(_rvs[regno].spec) & jit_class_sav)
2238 jit_regset_setbit(&_jitc->function->regset, regno);
2239}
2240
2241/* Compute initial reglive and regmask set values of a basic block.
2242 * reglive is the set of known live registers
2243 * regmask is the set of registers not referenced in the block
2244 * Registers in regmask might be live.
2245 */
2246static void
2247_jit_setup(jit_state_t *_jit, jit_block_t *block)
2248{
2249 jit_node_t *node;
2250 jit_bool_t live;
2251 unsigned long value;
2252
2253 jit_regset_set_mask(&block->regmask, _jitc->reglen);
2254 for (value = 0; value < _jitc->reglen; ++value)
2255 if (!(jit_class(_rvs[value].spec) & (jit_class_gpr|jit_class_fpr)))
2256 jit_regset_clrbit(&block->regmask, value);
2257
2258 for (node = block->label->next; node; node = node->next) {
2259 switch (node->code) {
2260 case jit_code_label: case jit_code_prolog:
2261 case jit_code_epilog:
2262 return;
2263 default:
2264 /* Check argument registers in reverse order to properly
2265 * handle registers that are both, argument and result */
2266 value = jit_classify(node->code);
2267 if ((value & jit_cc_a2_reg) &&
2268 !(node->w.w & jit_regno_patch) &&
2269 jit_regset_tstbit(&block->regmask, node->w.w)) {
2270 live = !(value & jit_cc_a2_chg);
2271 jit_regset_clrbit(&block->regmask, node->w.w);
2272 if (live)
2273 jit_regset_setbit(&block->reglive, node->w.w);
2274 }
2275 if ((value & jit_cc_a1_reg) &&
2276 !(node->v.w & jit_regno_patch) &&
2277 jit_regset_tstbit(&block->regmask, node->v.w)) {
2278 live = !(value & jit_cc_a1_chg);
2279 jit_regset_clrbit(&block->regmask, node->v.w);
2280 if (live)
2281 jit_regset_setbit(&block->reglive, node->v.w);
2282 }
2283 if (value & jit_cc_a0_reg) {
2284 live = !(value & jit_cc_a0_chg);
2285 if (value & jit_cc_a0_rlh) {
2286 if (!(node->u.q.l & jit_regno_patch) &&
2287 jit_regset_tstbit(&block->regmask, node->u.q.l)) {
2288 jit_regset_clrbit(&block->regmask, node->u.q.l);
2289 if (live)
2290 jit_regset_setbit(&block->reglive, node->u.q.l);
2291 }
2292 if (!(node->u.q.h & jit_regno_patch) &&
2293 jit_regset_tstbit(&block->regmask, node->u.q.h)) {
2294 jit_regset_clrbit(&block->regmask, node->u.q.h);
2295 if (live)
2296 jit_regset_setbit(&block->reglive, node->u.q.h);
2297 }
2298 }
2299 else {
2300 if (!(node->u.w & jit_regno_patch) &&
2301 jit_regset_tstbit(&block->regmask, node->u.w)) {
2302 jit_regset_clrbit(&block->regmask, node->u.w);
2303 if (live)
2304 jit_regset_setbit(&block->reglive, node->u.w);
2305 }
2306 }
2307 }
2308 break;
2309 }
2310 }
2311}
2312
2313/* Update regmask and reglive of blocks at entry point of branch targets
2314 * or normal flow that have a live register not used in this block.
2315 */
2316static void
2317_jit_follow(jit_state_t *_jit, jit_block_t *block, jit_bool_t *todo)
2318{
2319 jit_node_t *node;
2320 jit_block_t *next;
2321 jit_int32_t spec;
2322 jit_int32_t regno;
2323 unsigned long value;
2324 jit_node_t *label;
2325 jit_regset_t reglive;
2326 jit_regset_t regmask;
2327 jit_regset_t regtemp;
2328
2329 jit_regset_set(&reglive, &block->reglive);
2330 jit_regset_set(&regmask, &block->regmask);
2331 for (node = block->label->next; node; node = node->next) {
2332 switch (node->code) {
2333 case jit_code_label:
2334 /* Do not consider jmpi and jmpr cannot jump to the
2335 * next instruction. */
2336 next = _jitc->blocks.ptr + node->v.w;
2337 /* Set of live registers in next block that are at unknown
2338 * state in this block. */
2339 jit_regset_and(&regtemp, &regmask, &next->reglive);
2340 if (jit_regset_set_p(&regtemp)) {
2341 /* Add live state of next block to current block. */
2342 jit_regset_ior(&block->reglive, &block->reglive, &regtemp);
2343 /* Remove from unknown state bitmask. */
2344 jit_regset_com(&regtemp, &regtemp);
2345 jit_regset_and(&block->regmask, &block->regmask, &regtemp);
2346 *todo = 1;
2347 }
2348 case jit_code_prolog:
2349 case jit_code_epilog:
2350 return;
2351 case jit_code_callr:
2352 value = jit_regno(node->u.w);
2353 if (!(node->u.w & jit_regno_patch)) {
2354 if (jit_regset_tstbit(&regmask, value)) {
2355 jit_regset_clrbit(&regmask, value);
2356 jit_regset_setbit(&reglive, value);
2357 }
2358 }
2359 case jit_code_calli:
2360 for (value = 0; value < _jitc->reglen; ++value) {
2361 value = jit_regset_scan1(&regmask, value);
2362 if (value >= _jitc->reglen)
2363 break;
2364 spec = jit_class(_rvs[value].spec);
2365 if (!(spec & jit_class_sav))
2366 jit_regset_clrbit(&regmask, value);
2367 if ((spec & jit_class_arg) && jit_regarg_p(node, value))
2368 jit_regset_setbit(&reglive, value);
2369 }
2370 break;
2371 default:
2372 value = jit_classify(node->code);
2373 if (value & jit_cc_a2_reg) {
ba3814c1
PC		 2374 if (value & jit_cc_a2_rlh) {
2375 if (!(node->w.q.l & jit_regno_patch)) {
2376 /* Assume register is not changed */
2377 if (jit_regset_tstbit(&regmask, node->w.q.l))
2378 jit_regset_clrbit(&regmask, node->w.q.l);
2379 }
2380 if (!(node->w.q.h & jit_regno_patch)) {
2381 if (jit_regset_tstbit(&regmask, node->w.q.h))
2382 jit_regset_clrbit(&regmask, node->w.q.h);
2383 }
2384 }
2385 else {
2386 if (value & jit_cc_a2_reg) {
2387 if (!(node->w.w & jit_regno_patch)) {
2388 if (jit_regset_tstbit(&regmask, node->w.w)) {
2389 jit_regset_clrbit(&regmask, node->w.w);
2390 if (!(value & jit_cc_a2_chg))
2391 jit_regset_setbit(&reglive, node->w.w);
2392 }
2393 }
4a71579b
PC		 2394 }
2395 }
2396 }
2397 if (value & jit_cc_a1_reg) {
2398 if (!(node->v.w & jit_regno_patch)) {
2399 if (jit_regset_tstbit(&regmask, node->v.w)) {
2400 jit_regset_clrbit(&regmask, node->v.w);
2401 if (!(value & jit_cc_a1_chg))
2402 jit_regset_setbit(&reglive, node->v.w);
2403 }
2404 }
2405 }
2406 if (value & jit_cc_a0_reg) {
2407 if (value & jit_cc_a0_rlh) {
2408 if (!(node->u.q.l & jit_regno_patch)) {
2409 if (jit_regset_tstbit(&regmask, node->u.q.l)) {
2410 jit_regset_clrbit(&regmask, node->u.q.l);
2411 if (!(value & jit_cc_a0_chg))
2412 jit_regset_setbit(&reglive, node->u.q.l);
2413 }
2414 }
2415 if (!(node->u.q.h & jit_regno_patch)) {
2416 if (jit_regset_tstbit(&regmask, node->u.q.h)) {
2417 jit_regset_clrbit(&regmask, node->u.q.h);
2418 if (!(value & jit_cc_a0_chg))
2419 jit_regset_setbit(&reglive, node->u.q.h);
2420 }
2421 }
2422 }
2423 else {
2424 if (!(node->u.w & jit_regno_patch)) {
2425 if (jit_regset_tstbit(&regmask, node->u.w)) {
2426 jit_regset_clrbit(&regmask, node->u.w);
2427 if (!(value & jit_cc_a0_chg))
2428 jit_regset_setbit(&reglive, node->u.w);
2429 }
2430 }
2431 }
2432 }
2433 if (value & jit_cc_a0_jmp) {
2434 if (node->flag & jit_flag_node) {
2435 label = node->u.n;
2436 /* Do not consider jmpi and jmpr cannot jump to the
2437 * next instruction. */
2438 next = _jitc->blocks.ptr + label->v.w;
2439 jit_regset_and(&regtemp, &regmask, &next->reglive);
2440 if (jit_regset_set_p(&regtemp)) {
2441 /* Add live state. */
2442 jit_regset_ior(&block->reglive,
2443 &block->reglive, &regtemp);
2444 /* Remove from unknown state bitmask. */
2445 jit_regset_com(&regtemp, &regtemp);
2446 jit_regset_and(&block->regmask,
2447 &block->regmask, &regtemp);
2448 *todo = 1;
2449 }
2450 }
2451 else {
2452 /* Jump to unknown location.
2453 * This is a pitfall of the implementation.
2454 * Only jmpi to not a jit code should reach here,
2455 * or a jmpr of a computed address.
2456 * Because the implementation needs jit_class_nospill
2457 * registers, must treat jmpr as a function call. This
2458 * means that only JIT_Vn registers can be trusted on
2459 * arrival of jmpr.
2460 */
ba3814c1 2461 jit_regset_set_ui(&regmask, 0);
4a71579b
PC		 2462 for (regno = 0; regno < _jitc->reglen; regno++) {
2463 spec = jit_class(_rvs[regno].spec);
ba3814c1
PC		 2464 if ((spec & (jit_class_gpr|jit_class_fpr)) &&
2465 (spec & jit_class_sav))
2466 jit_regset_setbit(&regmask, regno);
4a71579b
PC		 2467 }
2468 /* Assume non callee save registers are live due
2469 * to jump to unknown location. */
2470 /* Treat all callee save as live. */
ba3814c1 2471 jit_regset_ior(&block->reglive, &reglive, &regmask);
4a71579b 2472 /* Treat anything else as dead. */
ba3814c1 2473 return;
4a71579b
PC		 2474 }
2475 }
2476 break;
2477 }
2478 }
2479}
2480
2481/* Follow code generation up to finding a label or end of code.
2482 * When finding a label, update the set of live registers.
2483 * On branches, update based on taken branch or normal flow.
2484 */
2485static void
2486_jit_update(jit_state_t *_jit, jit_node_t *node,
2487 jit_regset_t *live, jit_regset_t *mask)
2488{
2489 jit_int32_t spec;
2490 jit_int32_t regno;
2491 unsigned long value;
2492 jit_block_t *block;
2493 jit_node_t *label;
2494 jit_regset_t regtemp;
2495
2496 for (; node; node = node->next) {
2497 if (jit_regset_set_p(mask) == 0)
2498 break;
2499 switch (node->code) {
2500 case jit_code_label:
2501 block = _jitc->blocks.ptr + node->v.w;
2502 jit_regset_and(&regtemp, mask, &block->reglive);
2503 if (jit_regset_set_p(&regtemp)) {
2504 /* Add live state. */
2505 jit_regset_ior(live, live, &regtemp);
2506 /* Remove from unknown state bitmask. */
2507 jit_regset_com(&regtemp, &regtemp);
2508 jit_regset_and(mask, mask, &regtemp);
2509 }
2510 return;
2511 case jit_code_prolog:
2512 jit_regset_set_ui(mask, 0);
2513 return;
2514 case jit_code_epilog:
2515 jit_regset_set_ui(mask, 0);
2516 return;
2517 case jit_code_callr:
2518 value = jit_regno(node->u.w);
2519 if (!(node->u.w & jit_regno_patch)) {
2520 if (jit_regset_tstbit(mask, value)) {
2521 jit_regset_clrbit(mask, value);
2522 jit_regset_setbit(live, value);
2523 }
2524 }
2525 case jit_code_calli:
2526 for (value = 0; value < _jitc->reglen; ++value) {
2527 value = jit_regset_scan1(mask, value);
2528 if (value >= _jitc->reglen)
2529 break;
2530 spec = jit_class(_rvs[value].spec);
2531 if (!(spec & jit_class_sav))
2532 jit_regset_clrbit(mask, value);
2533 if ((spec & jit_class_arg) && jit_regarg_p(node, value))
2534 jit_regset_setbit(live, value);
2535 }
2536 break;
2537 default:
2538 value = jit_classify(node->code);
2539 if (value & jit_cc_a2_reg) {
ba3814c1
PC		 2540 if (value & jit_cc_a2_rlh) {
2541 if (!(node->w.q.l & jit_regno_patch)) {
2542 /* Assume register is not changed */
2543 if (jit_regset_tstbit(mask, node->w.q.l))
2544 jit_regset_clrbit(mask, node->w.q.l);
2545 }
2546 if (!(node->w.q.h & jit_regno_patch)) {
2547 if (jit_regset_tstbit(mask, node->w.q.h))
2548 jit_regset_clrbit(mask, node->w.q.h);
2549 }
2550 }
2551 else {
2552 if (!(node->w.w & jit_regno_patch)) {
2553 if (jit_regset_tstbit(mask, node->w.w)) {
2554 jit_regset_clrbit(mask, node->w.w);
2555 if (!(value & jit_cc_a2_chg))
2556 jit_regset_setbit(live, node->w.w);
2557 }
4a71579b
PC		 2558 }
2559 }
2560 }
2561 if (value & jit_cc_a1_reg) {
2562 if (!(node->v.w & jit_regno_patch)) {
2563 if (jit_regset_tstbit(mask, node->v.w)) {
2564 jit_regset_clrbit(mask, node->v.w);
2565 if (!(value & jit_cc_a1_chg))
2566 jit_regset_setbit(live, node->v.w);
2567 }
2568 }
2569 }
2570 if (value & jit_cc_a0_reg) {
2571 if (value & jit_cc_a0_rlh) {
2572 if (!(node->u.q.l & jit_regno_patch)) {
2573 if (jit_regset_tstbit(mask, node->u.q.l)) {
2574 jit_regset_clrbit(mask, node->u.q.l);
2575 if (!(value & jit_cc_a0_chg))
2576 jit_regset_setbit(live, node->u.q.l);
2577 }
2578 }
2579 if (!(node->u.q.h & jit_regno_patch)) {
2580 if (jit_regset_tstbit(mask, node->u.q.h)) {
2581 jit_regset_clrbit(mask, node->u.q.h);
2582 if (!(value & jit_cc_a0_chg))
2583 jit_regset_setbit(live, node->u.q.h);
2584 }
2585 }
2586 }
2587 else {
2588 if (!(node->u.w & jit_regno_patch)) {
2589 if (jit_regset_tstbit(mask, node->u.w)) {
2590 jit_regset_clrbit(mask, node->u.w);
2591 if (!(value & jit_cc_a0_chg))
2592 jit_regset_setbit(live, node->u.w);
2593 }
2594 }
2595 }
2596 }
2597 if (value & jit_cc_a0_jmp) {
2598 if (node->flag & jit_flag_node) {
2599 label = node->u.n;
2600 /* Do not consider jmpi and jmpr cannot jump to the
2601 * next instruction. */
2602 block = _jitc->blocks.ptr + label->v.w;
2603 jit_regset_and(&regtemp, mask, &block->reglive);
2604 if (jit_regset_set_p(&regtemp)) {
2605 /* Add live state. */
2606 jit_regset_ior(live, live, &regtemp);
2607 /* Remove from unknown state bitmask. */
2608 jit_regset_com(&regtemp, &regtemp);
2609 jit_regset_and(mask, mask, &regtemp);
2610 }
2611 }
2612 else {
2613 /* Jump to unknown location.
2614 * This is a pitfall of the implementation.
2615 * Only jmpi to not a jit code should reach here,
2616 * or a jmpr of a computed address.
2617 * Because the implementation needs jit_class_nospill
2618 * registers, must treat jmpr as a function call. This
2619 * means that only JIT_Vn registers can be trusted on
2620 * arrival of jmpr.
2621 */
ba3814c1 2622 jit_regset_set_ui(mask, 0);
4a71579b
PC		 2623 for (regno = 0; regno < _jitc->reglen; regno++) {
2624 spec = jit_class(_rvs[regno].spec);
ba3814c1
PC		 2625 if ((spec & (jit_class_gpr|jit_class_fpr)) &&
2626 (spec & jit_class_sav))
2627 jit_regset_setbit(mask, regno);
4a71579b
PC		 2628 }
2629 /* Assume non callee save registers are live due
2630 * to jump to unknown location. */
2631 /* Treat all callee save as live. */
2632 jit_regset_ior(live, live, mask);
2633 /* Treat anything else as dead. */
ba3814c1 2634 return;
4a71579b
PC		 2635 }
2636 }
2637 break;
2638 }
2639 }
2640}
2641
2642static void
2643_thread_jumps(jit_state_t *_jit)
2644{
2645 jit_node_t *prev;
2646 jit_node_t *node;
2647 jit_node_t *next;
2648 jit_int32_t mask;
2649
2650 for (prev = node = _jitc->head; node;) {
2651 next = node->next;
2652 switch (node->code) {
2653 case jit_code_jmpi:
2654 if (redundant_jump(prev, node)) {
2655 node = prev;
2656 continue;
2657 }
2658 if (shortcut_jump(prev, node))
2659 continue;
2660 break;
2661 case jit_code_jmpr:
2662 case jit_code_callr: case jit_code_calli:
2663 /* non optimizable jump like code */
2664 break;
2665 default:
2666 mask = jit_classify(node->code);
2667 if (mask & jit_cc_a0_jmp) {
2668 if (reverse_jump(prev, node) ||
2669 shortcut_jump(prev, node))
2670 continue;
2671 }
2672 break;
2673 }
2674 prev = node;
2675 node = next;
2676 }
2677}
2678
2679static void
2680_sequential_labels(jit_state_t *_jit)
2681{
2682 jit_node_t *jump;
2683 jit_node_t *link;
2684 jit_node_t *prev;
2685 jit_node_t *next;
2686 jit_node_t *node;
2687
2688 for (prev = node = _jitc->head; node; node = next) {
2689 next = node->next;
2690 if (node->code == jit_code_label) {
2691 if (!node->flag) {
2692 if (!node->link) {
2693 del_label(prev, node);
2694 continue;
2695 }
2696 if (prev != node && prev->code == jit_code_label) {
2697 if ((jump = node->link)) {
2698 for (; jump; jump = link) {
2699 link = jump->link;
2700 jump->u.n = prev;
2701 jump->link = prev->link;
2702 prev->link = jump;
2703 }
2704 node->link = NULL;
2705 }
2706 del_label(prev, node);
2707 continue;
2708 }
2709 }
2710 if (next && next->code == jit_code_label && !next->flag) {
2711 if ((jump = next->link)) {
2712 for (; jump; jump = link) {
2713 link = jump->link;
2714 jump->u.n = node;
2715 jump->link = node->link;
2716 node->link = jump;
2717 }
2718 next->link = NULL;
2719 }
2720 del_label(node, next);
2721 next = node->next;
2722 continue;
2723 }
2724 }
2725 prev = node;
2726 }
2727}
2728
2729static void
2730_split_branches(jit_state_t *_jit)
2731{
2732 jit_node_t *node;
2733 jit_node_t *next;
2734 jit_node_t *label;
2735 jit_block_t *block;
2736
2737 for (node = _jitc->head; node; node = next) {
2738 if ((next = node->next)) {
2739 if (next->code == jit_code_label ||
2740 next->code == jit_code_prolog ||
2741 next->code == jit_code_epilog)
2742 continue;
2743 /* split block on branches */
2744 if (jit_classify(node->code) & jit_cc_a0_jmp) {
2745 label = new_node(jit_code_label);
2746 label->next = next;
2747 node->next = label;
2748 if (_jitc->blocks.offset >= _jitc->blocks.length) {
2749 jit_word_t length;
2750
2751 length = _jitc->blocks.length + 16;
2752 jit_realloc((jit_pointer_t *)&_jitc->blocks.ptr,
2753 _jitc->blocks.length * sizeof(jit_block_t),
2754 length * sizeof(jit_block_t));
2755 _jitc->blocks.length = length;
2756 }
2757 block = _jitc->blocks.ptr + _jitc->blocks.offset;
2758 block->label = label;
2759 label->v.w = _jitc->blocks.offset;
2760 jit_regset_new(&block->reglive);
2761 jit_regset_new(&block->regmask);
2762 ++_jitc->blocks.offset;
2763 }
2764 }
2765 }
2766}
2767
2768static jit_bool_t
2769_shortcut_jump(jit_state_t *_jit, jit_node_t *prev, jit_node_t *node)
2770{
2771 jit_bool_t cond;
2772 jit_node_t *jump;
2773 jit_node_t *next;
2774 jit_node_t *temp;
2775
2776 if (!(node->flag & jit_flag_node))
2777 return (0);
2778 assert(node->code != jit_code_jmpr);
2779 cond = node->code != jit_code_jmpi;
2780 jump = node->u.n;
2781 for (next = jump->next; next; next = next->next) {
2782 switch (next->code) {
2783 case jit_code_jmpi:
2784 if (!(next->flag & jit_flag_node))
2785 return (0);
2786 if (jump->link == node)
2787 jump->link = node->link;
2788 else {
2789 for (temp = jump->link;
2790 temp->link != node;
2791 temp = temp->link)
2792 assert(temp != NULL);
2793 temp->link = node->link;
2794 }
2795 jump = next->u.n;
2796 node->u.n = jump;
2797 node->link = jump->link;
2798 jump->link = node;
2799 return (1);
2800 case jit_code_jmpr:
2801 if (cond)
2802 return (0);
2803 node->code = jit_code_jmpr;
2804 node->u.w = next->u.w;
2805 node->link = NULL;
2806 node->flag &= ~jit_flag_node;
2807 return (1);
2808 case jit_code_note: case jit_code_label:
2809 break;
2810 default:
2811 return (0);
2812 }
2813 }
2814 return (0);
2815}
2816
2817static jit_bool_t
2818_redundant_jump(jit_state_t *_jit, jit_node_t *prev, jit_node_t *node)
2819{
2820 jit_node_t *local_prev;
2821 jit_node_t *local_next;
2822
2823 if (!(node->flag & jit_flag_node))
2824 return (0);
2825 for (local_prev = node, local_next = node->next;
2826 local_next;
2827 local_prev = local_next, local_next = local_next->next) {
2828
2829 switch (local_next->code) {
2830 case jit_code_label: case jit_code_epilog:
2831 if (node->u.n == local_next) {
2832 if (local_next->link == node)
2833 local_next->link = node->link;
2834 else {
2835 for (local_prev = local_next->link;
2836 local_prev->link != node;
2837 local_prev = local_prev->link)
2838 assert(local_prev != NULL);
2839 local_prev->link = node->link;
2840 }
2841 del_node(prev, node);
2842 return (1);
2843 }
2844 break;
2845 case jit_code_name: case jit_code_note:
2846 case jit_code_align:
2847 break;
2848 default:
2849 return (0);
2850 }
2851 }
2852 return (0);
2853}
2854
2855static jit_code_t
2856reverse_jump_code(jit_code_t code)
2857{
2858 switch (code) {
2859 case jit_code_bltr: return (jit_code_bger);
2860 case jit_code_blti: return (jit_code_bgei);
2861 case jit_code_bltr_u: return (jit_code_bger_u);
2862 case jit_code_blti_u: return (jit_code_bgei_u);
2863 case jit_code_bler: return (jit_code_bgtr);
2864 case jit_code_blei: return (jit_code_bgti);
2865 case jit_code_bler_u: return (jit_code_bgtr_u);
2866 case jit_code_blei_u: return (jit_code_bgti_u);
2867 case jit_code_beqr: return (jit_code_bner);
2868 case jit_code_beqi: return (jit_code_bnei);
2869 case jit_code_bger: return (jit_code_bltr);
2870 case jit_code_bgei: return (jit_code_blti);
2871 case jit_code_bger_u: return (jit_code_bltr_u);
2872 case jit_code_bgei_u: return (jit_code_blti_u);
2873 case jit_code_bgtr: return (jit_code_bler);
2874 case jit_code_bgti: return (jit_code_blei);
2875 case jit_code_bgtr_u: return (jit_code_bler_u);
2876 case jit_code_bgti_u: return (jit_code_blei_u);
2877 case jit_code_bner: return (jit_code_beqr);
2878 case jit_code_bnei: return (jit_code_beqi);
2879 case jit_code_bmsr: return (jit_code_bmcr);
2880 case jit_code_bmsi: return (jit_code_bmci);
2881 case jit_code_bmcr: return (jit_code_bmsr);
2882 case jit_code_bmci: return (jit_code_bmsi);
2883 case jit_code_bltr_f: return (jit_code_bunger_f);
2884 case jit_code_blti_f: return (jit_code_bungei_f);
2885 case jit_code_bler_f: return (jit_code_bungtr_f);
2886 case jit_code_blei_f: return (jit_code_bungti_f);
2887
2888 case jit_code_beqr_f: return (jit_code_bner_f);
2889 case jit_code_beqi_f: return (jit_code_bnei_f);
2890
2891 case jit_code_bger_f: return (jit_code_bunltr_f);
2892 case jit_code_bgei_f: return (jit_code_bunlti_f);
2893 case jit_code_bgtr_f: return (jit_code_bunler_f);
2894 case jit_code_bgti_f: return (jit_code_bunlei_f);
2895
2896 case jit_code_bner_f: return (jit_code_beqr_f);
2897 case jit_code_bnei_f: return (jit_code_beqr_f);
2898
2899 case jit_code_bunltr_f: return (jit_code_bger_f);
2900 case jit_code_bunlti_f: return (jit_code_bgei_f);
2901 case jit_code_bunler_f: return (jit_code_bgtr_f);
2902 case jit_code_bunlei_f: return (jit_code_bgti_f);
2903
2904 case jit_code_buneqr_f: return (jit_code_bltgtr_f);
2905 case jit_code_buneqi_f: return (jit_code_bltgti_f);
2906
2907 case jit_code_bunger_f: return (jit_code_bltr_f);
2908 case jit_code_bungei_f: return (jit_code_blti_f);
2909 case jit_code_bungtr_f: return (jit_code_bler_f);
2910 case jit_code_bungti_f: return (jit_code_blei_f);
2911
2912 case jit_code_bltgtr_f: return (jit_code_buneqr_f);
2913 case jit_code_bltgti_f: return (jit_code_buneqi_f);
2914
2915 case jit_code_bordr_f: return (jit_code_bunordr_f);
2916 case jit_code_bordi_f: return (jit_code_bunordi_f);
2917 case jit_code_bunordr_f:return (jit_code_bordr_f);
2918 case jit_code_bunordi_f:return (jit_code_bordi_f);
2919 case jit_code_bltr_d: return (jit_code_bunger_d);
2920 case jit_code_blti_d: return (jit_code_bungei_d);
2921 case jit_code_bler_d: return (jit_code_bungtr_d);
2922 case jit_code_blei_d: return (jit_code_bungti_d);
2923
2924 case jit_code_beqr_d: return (jit_code_bner_d);
2925 case jit_code_beqi_d: return (jit_code_bnei_d);
2926
2927 case jit_code_bger_d: return (jit_code_bunltr_d);
2928 case jit_code_bgei_d: return (jit_code_bunlti_d);
2929 case jit_code_bgtr_d: return (jit_code_bunler_d);
2930 case jit_code_bgti_d: return (jit_code_bunlei_d);
2931
2932 case jit_code_bner_d: return (jit_code_beqr_d);
2933 case jit_code_bnei_d: return (jit_code_beqi_d);
2934
2935 case jit_code_bunltr_d: return (jit_code_bger_d);
2936 case jit_code_bunlti_d: return (jit_code_bgei_d);
2937 case jit_code_bunler_d: return (jit_code_bgtr_d);
2938 case jit_code_bunlei_d: return (jit_code_bgti_d);
2939
2940 case jit_code_buneqr_d: return (jit_code_bltgtr_d);
2941 case jit_code_buneqi_d: return (jit_code_bltgti_d);
2942
2943 case jit_code_bunger_d: return (jit_code_bltr_d);
2944 case jit_code_bungei_d: return (jit_code_blti_d);
2945 case jit_code_bungtr_d: return (jit_code_bler_d);
2946 case jit_code_bungti_d: return (jit_code_blei_d);
2947
2948 case jit_code_bltgtr_d: return (jit_code_buneqr_d);
2949 case jit_code_bltgti_d: return (jit_code_buneqi_d);
2950
2951 case jit_code_bordr_d: return (jit_code_bunordr_d);
2952 case jit_code_bordi_d: return (jit_code_bunordi_d);
2953 case jit_code_bunordr_d:return (jit_code_bordr_d);
2954 case jit_code_bunordi_d:return (jit_code_bordi_d);
2955 case jit_code_boaddr: return (jit_code_bxaddr);
2956 case jit_code_boaddi: return (jit_code_bxaddi);
2957 case jit_code_boaddr_u: return (jit_code_bxaddr_u);
2958 case jit_code_boaddi_u: return (jit_code_bxaddi_u);
2959 case jit_code_bxaddr: return (jit_code_boaddr);
2960 case jit_code_bxaddi: return (jit_code_boaddi);
2961 case jit_code_bxaddr_u: return (jit_code_boaddr_u);
2962 case jit_code_bxaddi_u: return (jit_code_boaddi_u);
2963 case jit_code_bosubr: return (jit_code_bxsubr);
2964 case jit_code_bosubi: return (jit_code_bxsubi);
2965 case jit_code_bosubr_u: return (jit_code_bxsubr_u);
2966 case jit_code_bosubi_u: return (jit_code_bxsubi_u);
2967 case jit_code_bxsubr: return (jit_code_bosubr);
2968 case jit_code_bxsubi: return (jit_code_bosubi);
2969 case jit_code_bxsubr_u: return (jit_code_bosubr_u);
2970 case jit_code_bxsubi_u: return (jit_code_bosubi_u);
2971 default: abort(); /* invalid jump code */
2972 }
2973}
2974
2975/*
2976 * change common pattern:
2977 * <cond_jump L0> <jump L1> <label L0>
2978 * into
2979 * <reverse_cond_jump L1>
2980 */
2981static jit_bool_t
2982_reverse_jump(jit_state_t *_jit, jit_node_t *prev, jit_node_t *node)
2983{
2984 jit_node_t *local_prev;
2985 jit_node_t *local_next;
2986 jit_node_t *local_jump;
2987
2988 if (!(node->flag & jit_flag_node))
2989 return (0);
2990 /* =><cond_jump L0> <jump L1> <label L0> */
2991 local_next = node->next;
2992 if (local_next->code != jit_code_jmpi ||
2993 !(local_next->flag & jit_flag_node))
2994 return (0);
2995 /* <cond_jump L0> =><jump L1> <label L0> */
2996
2997 local_jump = local_next->u.n;
2998 for (local_prev = local_next, local_next = local_next->next;
2999 local_next;
3000 local_prev = local_next, local_next = local_next->next) {
3001 switch (local_next->code) {
3002 case jit_code_label: case jit_code_epilog:
3003 if (node->u.n == local_next) {
3004 if (local_next->link == node)
3005 local_next->link = node->link;
3006 else {
3007 for (local_prev = local_next->link;
3008 local_prev->link != node;
3009 local_prev = local_prev->link)
3010 assert(local_prev != NULL);
3011 local_prev->link = node->link;
3012 }
3013 del_node(node, node->next);
3014 node->code = reverse_jump_code(node->code);
3015 node->u.n = local_jump;
3016 node->link = local_jump->link;
3017 local_jump->link = node;
3018 return (1);
3019 }
3020 break;
3021 case jit_code_note:
3022 break;
3023 default:
3024 return (0);
3025 }
3026 }
3027 return (0);
3028}
3029
ba3814c1 3030static jit_bool_t
4a71579b
PC		 3031_redundant_store(jit_state_t *_jit, jit_node_t *node, jit_bool_t jump)
3032{
3033 jit_node_t *iter;
3034 jit_node_t *prev;
3035 jit_word_t word;
3036 jit_int32_t spec;
3037 jit_int32_t regno;
ba3814c1 3038 jit_bool_t result;
4a71579b
PC		 3039
3040 if (jump) {
3041 prev = node->u.n;
3042 if (prev->code == jit_code_epilog)
ba3814c1 3043 return (0);
4a71579b
PC		 3044 assert(prev->code == jit_code_label);
3045 if ((prev->flag & jit_flag_head) || node->link || prev->link != node)
3046 /* multiple sources */
ba3814c1 3047 return (0);
4a71579b
PC		 3048 /* if there are sequential labels it will return below */
3049 }
3050 else
3051 prev = node;
ba3814c1 3052 result = 0;
4a71579b
PC		 3053 word = node->w.w;
3054 regno = jit_regno(node->v.w);
3055 for (iter = prev->next; iter; prev = iter, iter = iter->next) {
3056 switch (iter->code) {
3057 case jit_code_label: case jit_code_prolog:
3058 case jit_code_epilog:
ba3814c1 3059 return (result);
4a71579b
PC		 3060 case jit_code_movi:
3061 if (regno == jit_regno(iter->u.w)) {
3062 if (iter->flag || iter->v.w != word)
ba3814c1
PC		 3063 return (result);
3064 result = 1;
4a71579b
PC		 3065 del_node(prev, iter);
3066 iter = prev;
3067 }
3068 break;
3069 default:
3070 spec = jit_classify(iter->code);
3071 if (spec & jit_cc_a0_jmp)
ba3814c1 3072 return (result);
4a71579b
PC		 3073 if ((spec & (jit_cc_a0_reg|jit_cc_a0_chg)) ==
3074 (jit_cc_a0_reg|jit_cc_a0_chg)) {
3075 if (spec & jit_cc_a0_rlh) {
3076 if (regno == jit_regno(iter->u.q.l) ||
3077 regno == jit_regno(iter->u.q.h))
ba3814c1 3078 return (result);
4a71579b
PC		 3079 }
3080 else {
3081 if (regno == jit_regno(iter->u.w))
ba3814c1 3082 return (result);
4a71579b
PC		 3083 }
3084 }
3085 if ((spec & (jit_cc_a1_reg|jit_cc_a1_chg)) ==
3086 (jit_cc_a1_reg|jit_cc_a1_chg)) {
3087 if (regno == jit_regno(iter->v.w))
ba3814c1 3088 return (result);
4a71579b
PC		 3089 }
3090 if ((spec & (jit_cc_a2_reg|jit_cc_a2_chg)) ==
3091 (jit_cc_a2_reg|jit_cc_a2_chg)) {
3092 if (regno == jit_regno(iter->w.w))
ba3814c1 3093 return (result);
4a71579b
PC		 3094 }
3095 break;
3096 }
3097 }
3098}
3099
3100static jit_bool_t
3101_simplify_movr(jit_state_t *_jit, jit_node_t *prev, jit_node_t *node,
3102 jit_int32_t kind, jit_int32_t size)
3103{
3104 jit_int32_t regno;
3105 jit_int32_t right;
3106 jit_value_t *value;
3107
3108 regno = jit_regno(node->u.w);
3109 right = jit_regno(node->v.w);
3110 value = _jitc->values + regno;
3111 if ((value->kind == jit_kind_register &&
3112 jit_regno(value->base.q.l) == right &&
3113 value->base.q.h == _jitc->gen[right]) ||
3114 (value->kind == kind && _jitc->values[right].kind == kind &&
3115 memcmp(&value->base.w, &_jitc->values[right].base.w, size) == 0)) {
3116 del_node(prev, node);
3117 return (1);
3118 }
3119 if (_jitc->values[right].kind == jit_kind_word)
3120 jit_memcpy(value, _jitc->values + right, sizeof(jit_value_t));
3121 else {
3122 value->kind = jit_kind_register;
3123 value->base.q.l = right;
3124 value->base.q.h = _jitc->gen[right];
3125 }
3126 ++_jitc->gen[regno];
3127
3128 return (0);
3129}
3130
3131static jit_bool_t
3132_simplify_movi(jit_state_t *_jit, jit_node_t *prev, jit_node_t *node,
3133 jit_int32_t kind, jit_int32_t size)
3134{
3135 jit_value_t *value;
3136 jit_int32_t spec;
3137 jit_int32_t regno;
3138 jit_int32_t offset;
3139
3140 regno = jit_regno(node->u.w);
3141 value = _jitc->values + regno;
3142 if (node->flag & jit_flag_node) {
3143 /* set to undefined if value will be patched */
3144 value->kind = 0;
3145 ++_jitc->gen[regno];
3146 return (0);
3147 }
3148 if (value->kind == kind) {
3149 if (memcmp(&node->v.w, &value->base.w, size) == 0) {
3150 del_node(prev, node);
3151 return (1);
3152 }
3153 spec = jit_class(_rvs[regno].spec);
3154 if (kind == jit_kind_word)
3155 spec &= jit_class_gpr;
3156 else
3157 spec &= (jit_class_xpr | jit_class_fpr);
3158 for (offset = 0; offset < _jitc->reglen; offset++) {
3159 if (_jitc->values[offset].kind == kind &&
3160 memcmp(&node->v.w, &_jitc->values[offset].base.w, size) == 0 &&
3161 (jit_class(_rvs[offset].spec) & spec) == spec) {
3162 if (kind == jit_kind_word)
3163 node->code = jit_code_movr;
3164 else if (kind == jit_kind_float32)
3165 node->code = jit_code_movr_f;
3166 else
3167 node->code = jit_code_movr_d;
3168 node->v.w = offset;
3169 jit_memcpy(value, _jitc->values + offset, sizeof(jit_value_t));
3170 ++_jitc->gen[regno];
3171 return (0);
3172 }
3173 }
3174 }
3175 value->kind = kind;
3176 jit_memcpy(&value->base.w, &node->v.w, size);
3177 ++_jitc->gen[regno];
3178
3179 return (0);
3180}
3181
3182/* simple/safe redundandy test not checking if another register
3183 * holds the same value
3184 */
3185static jit_bool_t
3186_simplify_ldxi(jit_state_t *_jit, jit_node_t *prev, jit_node_t *node)
3187{
3188 jit_value_t *value;
3189 jit_int32_t regno;
3190 jit_int32_t right;
3191
3192 regno = jit_regno(node->u.w);
3193 right = jit_regno(node->v.w);
3194 value = _jitc->values + regno;
3195 if (regno != right &&
3196 value->kind == jit_kind_code && value->code == node->code &&
3197 value->base.q.l == right && value->base.q.h == _jitc->gen[right] &&
3198 node->w.w == value->disp.w) {
3199 del_node(prev, node);
3200 return (1);
3201 }
3202 value->kind = jit_kind_code;
3203 value->code = node->code;
3204 value->base.q.l = right;
3205 value->base.q.h = _jitc->gen[right];
3206 value->disp.w = node->w.w;
3207 ++_jitc->gen[regno];
3208
3209 return (0);
3210}
3211
3212static jit_bool_t
3213_simplify_stxi(jit_state_t *_jit, jit_node_t *prev, jit_node_t *node)
3214{
3215 jit_value_t *value;
3216 jit_int32_t regno;
3217 jit_int32_t right;
3218 jit_int32_t offset;
3219
3220 regno = jit_regno(node->w.w);
3221 right = jit_regno(node->v.w);
3222 value = _jitc->values + regno;
3223
3224 /* check for redundant store after load */
3225 if (regno != right &&
3226 value->kind == jit_kind_code && value->code == node->code &&
3227 value->base.q.l == right && value->base.q.h == _jitc->gen[right] &&
3228 node->u.w == value->disp.w) {
3229 del_node(prev, node);
3230 return (1);
3231 }
3232
3233 /* assume anything can alias, and invalidate tracked values */
3234 for (offset = 0; offset < _jitc->reglen; offset++) {
3235 if (_jitc->values[offset].kind == jit_kind_code) {
3236 _jitc->values[offset].kind = 0;
3237 ++_jitc->gen[offset];
3238 }
3239 }
3240
3241 /* no multiple information, so, if set to a constant,
3242 * prefer to keep that information */
3243 if (value->kind == 0) {
3244 value->kind = jit_kind_code;
3245 switch (node->code) {
3246 /* no information about signed/unsigned either */
3247 case jit_code_stxi_c: value->code = jit_code_ldxi_c; break;
3248 case jit_code_stxi_s: value->code = jit_code_ldxi_s; break;
3249 case jit_code_stxi_i: value->code = jit_code_ldxi_i; break;
3250 case jit_code_stxi_l: value->code = jit_code_ldxi_l; break;
3251 case jit_code_stxi_f: value->code = jit_code_ldxi_f; break;
3252 case jit_code_stxi_d: value->code = jit_code_ldxi_d; break;
3253 default: abort();
3254 }
3255 value->kind = jit_kind_code;
3256 value->base.q.l = right;
3257 value->base.q.h = _jitc->gen[right];
3258 value->disp.w = node->u.w;
3259 }
3260
3261 return (0);
3262}
3263
3264/* usually there should be only one store in the
3265 * jit_get_reg/jit_unget_reg, but properly handle
3266 * multiple ones by moving the save node */
3267static void
3268_simplify_spill(jit_state_t *_jit, jit_node_t *node, jit_int32_t regno)
3269{
3270 jit_node_t *save;
3271 jit_node_t *temp;
3272
3273 if ((temp = _jitc->spill[regno]) && (save = temp->next) != node) {
3274 temp->next = save->next;
3275 save->next = node->next;
3276 node->next = save;
3277 _jitc->spill[regno] = node;
3278 }
3279}
3280
3281/* checks for simple cases where a register is set more than
3282 * once to the same value, and is a common pattern of calls
3283 * to jit_pushargi and jit_pushargr
3284 */
ba3814c1 3285static jit_bool_t
4a71579b
PC		 3286_simplify(jit_state_t *_jit)
3287{
3288 jit_node_t *prev;
3289 jit_node_t *node;
3290 jit_node_t *next;
3291 jit_int32_t info;
3292 jit_int32_t regno;
ba3814c1 3293 jit_bool_t result;
4a71579b 3294
ba3814c1 3295 result = 0;
4a71579b
PC		 3296 for (prev = NULL, node = _jitc->head; node; prev = node, node = next) {
3297 next = node->next;
3298 switch (node->code) {
3299 case jit_code_label: case jit_code_prolog:
3300 case jit_code_callr: case jit_code_calli:
3301 reset:
3302 memset(_jitc->gen, 0, sizeof(jit_int32_t) * _jitc->reglen);
3303 memset(_jitc->values, 0, sizeof(jit_value_t) * _jitc->reglen);
3304 break;
3305 case jit_code_save:
3306 _jitc->spill[jit_regno(node->u.w)] = prev;
3307 break;
3308 case jit_code_load:
3309 regno = jit_regno(node->u.w);
3310 if (register_change_p(node->link->next, node, regno) !=
3311 jit_reg_change) {
3312 /* spill not required due to optimizing common
3313 * redundancy case of calling jit_get_reg/jit_unget_reg
3314 * and then setting the register to the value it is
3315 * already holding */
3316 patch_register(node->link->next, node,
3317 jit_regno_patch|regno, regno);
ba3814c1 3318 result = 1;
4a71579b
PC		 3319 del_node(_jitc->spill[regno], node->link);
3320 del_node(prev, node);
3321 node = prev;
3322 }
3323 _jitc->spill[regno] = NULL;
3324 break;
3325 case jit_code_movr:
3326 regno = jit_regno(node->u.w);
3327 if (simplify_movr(prev, node,
ba3814c1
PC		 3328 jit_kind_word, sizeof(jit_word_t))) {
3329 result = 1;
4a71579b 3330 simplify_spill(node = prev, regno);
ba3814c1 3331 }
4a71579b
PC		 3332 break;
3333 case jit_code_movi:
3334 regno = jit_regno(node->u.w);
3335 if (simplify_movi(prev, node,
ba3814c1
PC		 3336 jit_kind_word, sizeof(jit_word_t))) {
3337 result = 1;
4a71579b 3338 simplify_spill(node = prev, regno);
ba3814c1 3339 }
4a71579b
PC		 3340 break;
3341 case jit_code_movr_f:
3342 regno = jit_regno(node->u.w);
3343 if (simplify_movr(prev, node,
ba3814c1
PC		 3344 jit_kind_float32, sizeof(jit_float32_t))) {
3345 result = 1;
4a71579b 3346 simplify_spill(node = prev, regno);
ba3814c1 3347 }
4a71579b
PC		 3348 break;
3349 case jit_code_movi_f:
3350 regno = jit_regno(node->u.w);
3351 if (simplify_movi(prev, node,
ba3814c1
PC		 3352 jit_kind_float32, sizeof(jit_float32_t))) {
3353 result = 1;
4a71579b 3354 simplify_spill(node = prev, regno);
ba3814c1 3355 }
4a71579b
PC		 3356 break;
3357 case jit_code_movr_d:
3358 regno = jit_regno(node->u.w);
3359 if (simplify_movr(prev, node,
ba3814c1
PC		 3360 jit_kind_float64, sizeof(jit_float64_t))) {
3361 result = 1;
4a71579b 3362 simplify_spill(node = prev, regno);
ba3814c1 3363 }
4a71579b
PC		 3364 break;
3365 case jit_code_movi_d:
3366 regno = jit_regno(node->u.w);
3367 if (simplify_movi(prev, node,
ba3814c1
PC		 3368 jit_kind_float64, sizeof(jit_float64_t))) {
3369 result = 1;
4a71579b 3370 simplify_spill(node = prev, regno);
ba3814c1 3371 }
4a71579b
PC		 3372 break;
3373 case jit_code_ldxi_c: case jit_code_ldxi_uc:
3374 case jit_code_ldxi_s: case jit_code_ldxi_us:
3375 case jit_code_ldxi_i: case jit_code_ldxi_ui:
3376 case jit_code_ldxi_l:
3377 case jit_code_ldxi_f: case jit_code_ldxi_d:
3378 regno = jit_regno(node->u.w);
ba3814c1
PC		 3379 if (simplify_ldxi(prev, node)) {
3380 result = 1;
4a71579b 3381 simplify_spill(node = prev, regno);
ba3814c1 3382 }
4a71579b
PC		 3383 break;
3384 case jit_code_stxi_c: case jit_code_stxi_s:
3385 case jit_code_stxi_i: case jit_code_stxi_l:
3386 case jit_code_stxi_f: case jit_code_stxi_d:
3387 regno = jit_regno(node->u.w);
ba3814c1
PC		 3388 if (simplify_stxi(prev, node)) {
3389 result = 1;
4a71579b 3390 simplify_spill(node = prev, regno);
ba3814c1 3391 }
4a71579b
PC		 3392 break;
3393 default:
3394 info = jit_classify(node->code);
3395 if (info & jit_cc_a0_jmp)
3396 /* labels are not implicitly added when not taking
3397 * a conditional branch */
3398 goto reset;
3399 if (info & jit_cc_a0_chg) {
3400 if (info & jit_cc_a0_rlh) {
3401 regno = jit_regno(node->u.q.l);
3402 _jitc->values[regno].kind = 0;
3403 ++_jitc->gen[regno];
3404 regno = jit_regno(node->u.q.h);
3405 _jitc->values[regno].kind = 0;
3406 ++_jitc->gen[regno];
3407 }
3408 else {
3409 regno = jit_regno(node->u.w);
3410 _jitc->values[regno].kind = 0;
3411 ++_jitc->gen[regno];
3412 }
3413 }
3414 if (info & jit_cc_a1_chg) {
3415 regno = jit_regno(node->v.w);
3416 _jitc->values[regno].kind = 0;
3417 ++_jitc->gen[regno];
3418 }
3419 if (info & jit_cc_a2_chg) {
ba3814c1
PC		 3420#if 0
3421 /* Assume registers are not changed */
3422 if (info & jit_cc_a2_rlh) {
3423 regno = jit_regno(node->w.q.l);
3424 _jitc->values[regno].kind = 0;
3425 ++_jitc->gen[regno];
3426 regno = jit_regno(node->w.q.h);
3427 _jitc->values[regno].kind = 0;
3428 ++_jitc->gen[regno];
3429 }
3430 else {
3431#endif
3432 regno = jit_regno(node->w.w);
3433 _jitc->values[regno].kind = 0;
3434 ++_jitc->gen[regno];
3435#if 0
3436 }
3437#endif
4a71579b
PC		 3438 }
3439 break;
3440 }
3441 }
ba3814c1 3442 return (result);
4a71579b
PC		 3443}
3444
3445static jit_int32_t
3446_register_change_p(jit_state_t *_jit, jit_node_t *node, jit_node_t *link,
3447 jit_int32_t regno)
3448{
3449 jit_int32_t value;
3450
3451 for (; node != link; node = node->next) {
3452 switch (node->code) {
3453 case jit_code_label: case jit_code_prolog:
3454 /* lack of extra information so cannot say it is undefined */
3455 return (jit_reg_change);
3456 case jit_code_callr: case jit_code_calli:
3457 if (!(jit_class(_rvs[regno].spec) & jit_class_sav))
3458 return (jit_reg_undef);
3459 break;
3460 default:
3461 value = jit_classify(node->code);
3462 /* lack of extra information */
40a44dcb 3463 if (value & (jit_cc_a0_jmp|jit_cc_a0_cnd))
4a71579b
PC		 3464 return (jit_reg_change);
3465 else if ((value & (jit_cc_a0_reg|jit_cc_a0_chg)) ==
3466 (jit_cc_a0_reg|jit_cc_a0_chg) &&
3467 (((value & jit_cc_a0_rlh) &&
3468 (node->u.q.l == regno || node->u.q.h == regno)) ||
3469 (!(value & jit_cc_a0_rlh) &&
3470 node->u.w == regno)))
3471 return (jit_reg_change);
3472 else if ((value & jit_cc_a1_reg) && node->v.w == regno &&
3473 (value & jit_cc_a1_chg))
3474 return (jit_reg_change);
3475 else if ((value & jit_cc_a2_reg) && node->w.w == regno &&
3476 (value & jit_cc_a2_chg))
3477 return (jit_reg_change);
3478 }
3479 }
3480
3481 return (jit_reg_static);
3482}
3483
3484/* most of this could be done at the same time as generating jit, but
3485 * avoid complications on different cpu backends and patch spill/loads
3486 * here, by simulating jit generation */
3487static jit_bool_t
3488_spill_reglive_p(jit_state_t *_jit, jit_node_t *node, jit_int32_t regno)
3489{
3490 if (!jit_regset_tstbit(&_jitc->reglive, regno)) {
3491 jit_regset_setbit(&_jitc->regmask, regno);
3492 jit_update(node->next, &_jitc->reglive, &_jitc->regmask);
3493 if (!jit_regset_tstbit(&_jitc->reglive, regno) &&
3494 register_change_p(node->next, node->link, regno) != jit_reg_change)
3495 return (0);
3496 }
3497
3498 return (1);
3499}
3500
3501static void
3502_patch_registers(jit_state_t *_jit)
3503{
3504 jit_node_t *prev;
3505 jit_node_t *node;
3506 jit_node_t *next;
3507 jit_int32_t info;
3508 jit_int32_t spec;
3509 jit_int32_t regno;
3510 jit_int32_t value;
3511
3512 _jitc->function = NULL;
3513
3514 jit_reglive_setup();
3515 for (prev = NULL, node = _jitc->head; node; node = next) {
3516 next = node->next;
3517
3518 info = jit_classify(node->code);
3519 jit_regarg_set(node, info);
3520
3521 switch (node->code) {
3522 case jit_code_save:
3523 regno = jit_regno(node->u.w);
3524 if (!spill_reglive_p(node, regno)) {
3525 /* register is not live, just remove spill/reload */
3526 jit_regarg_clr(node, info);
3527 node->link->v.w = jit_regload_delete;
3528 del_node(prev, node);
3529 continue;
3530 }
3531 else {
3532 /* try to find a free register of the same class */
3533 spec = jit_class(_rvs[regno].spec) & ~jit_class_arg;
3534 for (value = 0; value < _jitc->reglen; value++) {
3535 if (value != regno &&
3536 ((jit_class(_rvs[value].spec) & spec) &
3537 ~jit_class_arg) == spec &&
3538 !jit_regset_tstbit(&_jitc->regarg, value) &&
3539 !spill_reglive_p(node, value))
3540 break;
3541 }
3542 if (value < _jitc->reglen) {
3543 jit_regarg_clr(node, info);
3544 patch_register(node->next, node->link,
3545 jit_regno_patch|node->u.w,
3546 jit_regno_patch|value);
3547 /* mark as live just in case there are nested
3548 * register patches, so that next patch will
3549 * not want to use the same register */
3550 jit_regset_setbit(&_jitc->reglive, value);
3551 /* register is not live, just remove spill/reload */
3552 node->link->v.w = jit_regload_isdead;
3553 del_node(prev, node);
3554 continue;
3555 }
3556 else {
3557 /* failed to find a free register */
3558 if (spec & jit_class_gpr) {
3559 if (!_jitc->function->regoff[regno])
3560 _jitc->function->regoff[regno] =
3561 jit_allocai(sizeof(jit_word_t));
3562#if __WORDSIZE == 32
3563 node->code = jit_code_stxi_i;
3564#else
3565 node->code = jit_code_stxi_l;
3566#endif
3567 }
3568 else {
3569 node->code = jit_code_stxi_d;
3570 if (!_jitc->function->regoff[regno])
3571 _jitc->function->regoff[regno] =
3572 jit_allocai(sizeof(jit_float64_t));
3573 }
3574 node->u.w = _jitc->function->regoff[regno];
3575 node->v.w = JIT_FP;
3576 node->w.w = regno;
3577 node->link = NULL;
3578 }
3579 }
3580 break;
3581 case jit_code_load:
3582 regno = jit_regno(node->u.w);
3583 if (node->v.w) {
3584 if (node->v.w == jit_regload_isdead)
3585 jit_regset_clrbit(&_jitc->reglive, regno);
3586 del_node(prev, node);
3587 continue;
3588 }
3589 spec = jit_class(_rvs[regno].spec);
3590 if (spec & jit_class_gpr) {
3591#if __WORDSIZE == 32
3592 node->code = jit_code_ldxi_i;
3593#else
3594 node->code = jit_code_ldxi_l;
3595#endif
3596 }
3597 else
3598 node->code = jit_code_ldxi_d;
3599 node->v.w = regno;
3600 node->v.w = JIT_FP;
3601 node->w.w = _jitc->function->regoff[regno];
3602 node->link = NULL;
3603 break;
3604 case jit_code_prolog:
3605 _jitc->function = _jitc->functions.ptr + node->w.w;
3606 break;
3607 case jit_code_epilog:
3608 _jitc->function = NULL;
3609 break;
3610 default:
3611 break;
3612 }
3613
3614 jit_regarg_clr(node, info);
3615 /* update register live state */
3616 jit_reglive(node);
3617 prev = node;
3618 }
3619}
3620
3621static void
3622_patch_register(jit_state_t *_jit, jit_node_t *node, jit_node_t *link,
3623 jit_int32_t regno, jit_int32_t patch)
3624{
3625 jit_int32_t value;
3626
3627 for (; node != link; node = node->next) {
3628 value = jit_classify(node->code);
3629 if (value & jit_cc_a0_reg) {
3630 if (value & jit_cc_a0_rlh) {
3631 if (node->u.q.l == regno)
3632 node->u.q.l = patch;
3633 if (node->u.q.h == regno)
3634 node->u.q.h = patch;
3635 }
3636 else {
3637 if (node->u.w == regno)
3638 node->u.w = patch;
3639 }
3640 }
3641 if ((value & jit_cc_a1_reg) && node->v.w == regno)
3642 node->v.w = patch;
ba3814c1
PC		 3643 if (value & jit_cc_a2_reg) {
3644 if (value & jit_cc_a2_rlh) {
3645 if (node->w.q.l == regno)
3646 node->w.q.l = patch;
3647 if (node->w.q.h == regno)
3648 node->w.q.h = patch;
3649 }
3650 else {
3651 if (node->w.w == regno)
3652 node->w.w = patch;
3653 }
3654 }
4a71579b
PC		 3655 }
3656}
3657
40a44dcb
PC		 3658#if __BYTE_ORDER == __LITTLE_ENDIAN
3659# define htonr_us(r0,r1) bswapr_us(r0,r1)
3660# define htonr_ui(r0,r1) bswapr_ui(r0,r1)
3661# if __WORDSIZE == 64
3662# define htonr_ul(r0,r1) bswapr_ul(r0,r1)
3663# endif
3664#else
3665# define htonr_us(r0,r1) extr_us(r0,r1)
3666# if __WORDSIZE == 32
3667# define htonr_ui(r0,r1) movr(r0,r1)
3668# else
3669# define htonr_ui(r0,r1) extr_ui(r0,r1)
3670# define htonr_ul(r0,r1) movr(r0,r1)
3671# endif
3672#endif
3673
3674static maybe_unused void
3675generic_bswapr_us(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1);
3676static maybe_unused void
3677generic_bswapr_ui(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1);
3678#if __WORDSIZE == 64
3679static maybe_unused void
3680generic_bswapr_ul(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1);
3681#endif
3682
4a71579b
PC		 3683#if defined(__i386__) || defined(__x86_64__)
3684# include "jit_x86.c"
3685#elif defined(__mips__)
3686# include "jit_mips.c"
3687#elif defined(__arm__)
3688# include "jit_arm.c"
3689#elif defined(__powerpc__)
3690# include "jit_ppc.c"
3691#elif defined(__sparc__)
3692# include "jit_sparc.c"
3693#elif defined(__ia64__)
3694# include "jit_ia64.c"
3695#elif defined(__hppa__)
3696# include "jit_hppa.c"
3697#elif defined(__aarch64__)
3698# include "jit_aarch64.c"
3699#elif defined(__s390__) || defined(__s390x__)
3700# include "jit_s390.c"
3701#elif defined(__alpha__)
3702# include "jit_alpha.c"
3703#elif defined(__riscv)
3704# include "jit_riscv.c"
3705#endif
40a44dcb
PC		 3706
3707static maybe_unused void
3708generic_bswapr_us(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
3709{
3710 jit_int32_t reg = jit_get_reg(jit_class_gpr);
3711
3712 rshi(rn(reg), r1, 8);
3713 andi(r0, r1, 0xff);
3714 andi(rn(reg), rn(reg), 0xff);
3715 lshi(r0, r0, 8);
3716 orr(r0, r0, rn(reg));
3717
3718 jit_unget_reg(reg);
3719}
3720
3721static maybe_unused void
3722generic_bswapr_ui(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
3723{
3724 jit_int32_t reg = jit_get_reg(jit_class_gpr);
3725
3726 rshi(rn(reg), r1, 16);
3727 bswapr_us(r0, r1);
3728 bswapr_us(rn(reg), rn(reg));
3729 lshi(r0, r0, 16);
3730 orr(r0, r0, rn(reg));
3731
3732 jit_unget_reg(reg);
3733}
3734
3735#if __WORDSIZE == 64
3736static maybe_unused void
3737generic_bswapr_ul(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
3738{
3739 jit_int32_t reg = jit_get_reg(jit_class_gpr);
3740
3741 rshi_u(rn(reg), r1, 32);
3742 bswapr_ui(r0, r1);
3743 bswapr_ui(rn(reg), rn(reg));
3744 lshi(r0, r0, 32);
3745 orr(r0, r0, rn(reg));
3746
3747 jit_unget_reg(reg);
3748}
3749#endif
ARM optimized PCSX fork