notaz.gp2x.de / pcsx_rearmed.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
git subrepo pull (merge) --force deps/lightning
[pcsx_rearmed.git] / deps / lightning / lib / lightning.c
CommitLineData
4a71579b 1/*
79bfeef6 2 * Copyright (C) 2012-2023 Free Software Foundation, Inc.
4a71579b
PC		 3 *
4 * This file is part of GNU lightning.
5 *
6 * GNU lightning is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU Lesser General Public License as published
8 * by the Free Software Foundation; either version 3, or (at your option)
9 * any later version.
10 *
11 * GNU lightning is distributed in the hope that it will be useful, but
12 * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
13 * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public
14 * License for more details.
15 *
16 * Authors:
17 * Paulo Cesar Pereira de Andrade
18 */
19
20#include <lightning.h>
21#include <lightning/jit_private.h>
40a44dcb
PC		 22#if HAVE_MMAP
23# include <sys/mman.h>
24#endif
4a71579b
PC		 25#if defined(__sgi)
26# include <fcntl.h>
27#endif
28
29#ifndef MAP_ANON
30# define MAP_ANON MAP_ANONYMOUS
31# ifndef MAP_ANONYMOUS
32# define MAP_ANONYMOUS 0
33# endif
34#endif
35
36#define jit_regload_reload 0 /* convert to reload */
37#define jit_regload_delete 1 /* just remove node */
38#define jit_regload_isdead 2 /* delete and unset live bit */
39
40/*
41 * Prototypes
42 */
43static jit_word_t hash_data(const void*, jit_word_t);
44
45#define new_pool() _new_pool(_jit)
46static void _new_pool(jit_state_t*);
47
48#define new_node(u) _new_node(_jit, u)
49static jit_node_t *_new_node(jit_state_t*, jit_code_t);
50
51#define link_node(u) _link_node(_jit, u)
52static inline jit_node_t *_link_node(jit_state_t*, jit_node_t*);
53
54#define del_node(u, v) _del_node(_jit, u, v)
55static inline void _del_node(jit_state_t*, jit_node_t*, jit_node_t*);
56
57#define free_node(u) _free_node(_jit, u)
58static inline void _free_node(jit_state_t*, jit_node_t*);
59
60#define del_label(u, v) _del_label(_jit, u, v)
61static void _del_label(jit_state_t*, jit_node_t*, jit_node_t*);
62
63#define jit_dataset() _jit_dataset(_jit)
64static void
65_jit_dataset(jit_state_t *_jit);
66
c0c16242
PC		 67#define block_update_set(block, target) _block_update_set(_jit, block, target)
68static jit_bool_t _block_update_set(jit_state_t*, jit_block_t*, jit_block_t*);
69
24d91c0d
PC		 70#define propagate_backward(block) _propagate_backward(_jit, block)
71static void _propagate_backward(jit_state_t*, jit_block_t*);
72
c0c16242
PC		 73#define check_block_again() _check_block_again(_jit)
74static jit_bool_t _check_block_again(jit_state_t*);
75
76#define do_setup() _do_setup(_jit)
77static void _do_setup(jit_state_t*);
78
4a71579b
PC		 79#define jit_setup(block) _jit_setup(_jit, block)
80static void
81_jit_setup(jit_state_t *_jit, jit_block_t *block);
82
c0c16242
PC		 83#define do_follow(always) _do_follow(_jit, always)
84static void _do_follow(jit_state_t*, jit_bool_t);
85
86#define jit_follow(block) _jit_follow(_jit, block)
4a71579b 87static void
c0c16242 88_jit_follow(jit_state_t *_jit, jit_block_t *block);
4a71579b
PC		 89
90#define jit_update(node, live, mask) _jit_update(_jit, node, live, mask)
91static void
92_jit_update(jit_state_t *_jit, jit_node_t *node,
93 jit_regset_t *live, jit_regset_t *mask);
94
95#define thread_jumps() _thread_jumps(_jit)
96static void
97_thread_jumps(jit_state_t *_jit);
98
99#define sequential_labels() _sequential_labels(_jit)
100static void
101_sequential_labels(jit_state_t *_jit);
102
103#define split_branches() _split_branches(_jit)
104static void
105_split_branches(jit_state_t *_jit);
106
107#define shortcut_jump(prev, node) _shortcut_jump(_jit, prev, node)
108static jit_bool_t
109_shortcut_jump(jit_state_t *_jit, jit_node_t *prev, jit_node_t *node);
110
111#define redundant_jump(prev, node) _redundant_jump(_jit, prev, node)
112static jit_bool_t
113_redundant_jump(jit_state_t *_jit, jit_node_t *prev, jit_node_t *node);
114
115static jit_code_t
116reverse_jump_code(jit_code_t code);
117
118#define reverse_jump(prev, node) _reverse_jump(_jit, prev, node)
119static jit_bool_t
120_reverse_jump(jit_state_t *_jit, jit_node_t *prev, jit_node_t *node);
121
122#define redundant_store(node, jump) _redundant_store(_jit, node, jump)
ba3814c1 123static jit_bool_t
4a71579b
PC		 124_redundant_store(jit_state_t *_jit, jit_node_t *node, jit_bool_t jump);
125
126#define simplify_movr(p, n, k, s) _simplify_movr(_jit, p, n, k, s)
127static jit_bool_t
128_simplify_movr(jit_state_t *_jit, jit_node_t *prev, jit_node_t *node,
129 jit_int32_t kind, jit_int32_t size);
130
131#define simplify_movi(p, n, k, s) _simplify_movi(_jit, p, n, k, s)
132static jit_bool_t
133_simplify_movi(jit_state_t *_jit, jit_node_t *prev, jit_node_t *node,
134 jit_int32_t kind, jit_int32_t size);
135
136#define simplify_ldxi(prev, node) _simplify_ldxi(_jit, prev, node)
137static jit_bool_t
138_simplify_ldxi(jit_state_t *_jit, jit_node_t *prev, jit_node_t *node);
139
140#define simplify_stxi(prev, node) _simplify_stxi(_jit, prev, node)
141static jit_bool_t
142_simplify_stxi(jit_state_t *_jit, jit_node_t *prev, jit_node_t *node);
143
144#define simplify_spill(node, regno) _simplify_spill(_jit, node, regno)
145static void
146_simplify_spill(jit_state_t *_jit, jit_node_t *node, jit_int32_t regno);
147
148#define simplify() _simplify(_jit)
ba3814c1 149static jit_bool_t
4a71579b
PC		 150_simplify(jit_state_t *_jit);
151
152#define jit_reg_undef -1
153#define jit_reg_static 0
154#define jit_reg_change 1
155#define register_change_p(n, l, r) _register_change_p(_jit, n, l, r)
156static jit_int32_t
157_register_change_p(jit_state_t *_jit, jit_node_t *node, jit_node_t *link,
158 jit_int32_t regno);
159
160#define spill_reglive_p(node, regno) _spill_reglive_p(_jit, node, regno)
161static jit_bool_t
162_spill_reglive_p(jit_state_t *_jit, jit_node_t *node, jit_int32_t regno);
163
164#define patch_registers() _patch_registers(_jit)
165static void
166_patch_registers(jit_state_t *_jit);
167
168#define patch_register(n,l,r,p) _patch_register(_jit,n,l,r,p)
169static void
170_patch_register(jit_state_t *jit, jit_node_t *node, jit_node_t *link,
171 jit_int32_t regno, jit_int32_t patch);
172
173/*
174 * Initialization
175 */
176#if !defined(__sgi)
177#define mmap_fd -1
178#endif
179
180/*
181 * Implementation
182 */
183void
184init_jit(const char *progname)
185{
186 jit_get_cpu();
187 jit_init_debug(progname);
188 jit_init_size();
189}
190
191void
192finish_jit(void)
193{
194 jit_finish_debug();
195 jit_finish_size();
196}
197
198jit_int32_t
199_jit_get_reg(jit_state_t *_jit, jit_int32_t regspec)
200{
201 jit_int32_t spec;
202 jit_int32_t regno;
203
204 spec = regspec & ~(jit_class_chk|jit_class_nospill);
205 if (spec & jit_class_named) {
206 regno = jit_regno(spec);
207 if (jit_regset_tstbit(&_jitc->regsav, regno))
208 /* fail if register is spilled */
209 goto fail;
210 if (jit_regset_tstbit(&_jitc->regarg, regno))
211 /* fail if register is an argument to current instruction */
212 goto fail;
213 if (jit_regset_tstbit(&_jitc->reglive, regno)) {
214 if (regspec & jit_class_nospill)
215 /* fail if register is live and should not spill/reload */
216 goto fail;
217 goto spill;
218 }
219 jit_regset_setbit(&_jitc->regarg, regno);
220 return (regno);
221 }
222 else
223 assert(jit_class(spec) != 0);
224
225 if (_jitc->emit) {
226 /* search for a free register matching spec */
227 for (regno = 0; regno < _jitc->reglen; regno++) {
228 if ((jit_class(_rvs[regno].spec) & spec) == spec &&
229 !jit_regset_tstbit(&_jitc->regarg, regno) &&
79bfeef6
PC		 230 !jit_regset_tstbit(&_jitc->reglive, regno)) {
231 if (jit_regset_tstbit(&_jitc->regmask, regno)) {
232 /* search further, attempting to find a truly known
233 * free register, not just one in unknown state. */
234 jit_int32_t regfree;
235
236 for (regfree = regno + 1;
237 regfree < _jitc->reglen; regfree++) {
238 if ((jit_class(_rvs[regfree].spec) & spec) == spec &&
239 !jit_regset_tstbit(&_jitc->regarg, regfree) &&
240 !jit_regset_tstbit(&_jitc->reglive, regfree) &&
241 !jit_regset_tstbit(&_jitc->regmask, regfree)) {
242 regno = regfree;
243 break;
244 }
245 }
246 }
4a71579b 247 goto regarg;
79bfeef6 248 }
4a71579b
PC		 249 }
250
251 /* search for a register matching spec that is not an argument
252 * for the current instruction */
253 for (regno = 0; regno < _jitc->reglen; regno++) {
254 if ((jit_class(_rvs[regno].spec) & spec) == spec &&
255 !jit_regset_tstbit(&_jitc->regsav, regno) &&
256 !jit_regset_tstbit(&_jitc->regarg, regno) &&
257 !(regspec & jit_class_nospill)) {
258 spill:
259 assert(_jitc->function != NULL);
260 if (spec & jit_class_gpr) {
261 if (!_jitc->function->regoff[regno]) {
262 _jitc->function->regoff[regno] =
263 jit_allocai(sizeof(jit_word_t));
264 _jitc->again = 1;
265 }
266#if DEBUG
267 /* emit_stxi must not need temporary registers */
268 assert(!_jitc->getreg);
269 _jitc->getreg = 1;
270#endif
271 emit_stxi(_jitc->function->regoff[regno], JIT_FP, regno);
272#if DEBUG
273 _jitc->getreg = 0;
274#endif
275 }
276 else {
277 if (!_jitc->function->regoff[regno]) {
278 _jitc->function->regoff[regno] =
279 jit_allocai(sizeof(jit_float64_t));
280 _jitc->again = 1;
281 }
282#if DEBUG
283 /* emit_stxi must not need temporary registers */
284 assert(!_jitc->getreg);
285 _jitc->getreg = 1;
286#endif
287 emit_stxi_d(_jitc->function->regoff[regno], JIT_FP, regno);
288#if DEBUG
289 _jitc->getreg = 0;
290#endif
291 }
292 jit_regset_setbit(&_jitc->regsav, regno);
293 regarg:
294 jit_regset_setbit(&_jitc->regarg, regno);
295 if (jit_class(_rvs[regno].spec) & jit_class_sav) {
296 /* if will modify callee save registers without a
297 * function prolog, better patch this assertion */
298 assert(_jitc->function != NULL);
299 if (!jit_regset_tstbit(&_jitc->function->regset, regno)) {
300 jit_regset_setbit(&_jitc->function->regset, regno);
301 _jitc->again = 1;
302 }
303 }
304 return (regno);
305 }
306 }
307 }
308 else {
309 /* nospill hint only valid during emit" */
310 assert(!(regspec & jit_class_nospill));
311 for (regno = 0; regno < _jitc->reglen; regno++) {
312 if ((jit_class(_rvs[regno].spec) & spec) == spec &&
313 !jit_regset_tstbit(&_jitc->regsav, regno) &&
314 !jit_regset_tstbit(&_jitc->regarg, regno)) {
315 jit_regset_setbit(&_jitc->regarg, regno);
316 jit_regset_setbit(&_jitc->regsav, regno);
317 jit_save(regno);
318 return (jit_regno_patch|regno);
319 }
320 }
321 }
322
323 /* Out of hardware registers */
324fail:
325 assert(regspec & jit_class_chk);
326 return (JIT_NOREG);
327}
328
329void
330_jit_unget_reg(jit_state_t *_jit, jit_int32_t regno)
331{
332 regno = jit_regno(regno);
333 if (jit_regset_tstbit(&_jitc->regsav, regno)) {
334 if (_jitc->emit) {
335#if DEBUG
336 /* emit_ldxi must not need a temporary register */
337 assert(!_jitc->getreg);
338 _jitc->getreg = 1;
339#endif
340 if (jit_class(_rvs[regno].spec) & jit_class_gpr)
341 emit_ldxi(regno, JIT_FP, _jitc->function->regoff[regno]);
342 else
343 emit_ldxi_d(regno, JIT_FP, _jitc->function->regoff[regno]);
344#if DEBUG
345 /* emit_ldxi must not need a temporary register */
346 _jitc->getreg = 0;
347#endif
348 }
349 else
350 jit_load(regno);
351 jit_regset_clrbit(&_jitc->regsav, regno);
352 }
353#if defined(jit_carry)
354 assert((regno == jit_carry /*&& _NOREG != jit_carry*/) ||
355 jit_regset_tstbit(&_jitc->regarg, regno) != 0);
356#else
357 assert(jit_regset_tstbit(&_jitc->regarg, regno) != 0);
358#endif
359 jit_regset_clrbit(&_jitc->regarg, regno);
360}
361
362jit_bool_t
363_jit_callee_save_p(jit_state_t *_jit, jit_int32_t regno)
364{
365 assert(regno >= 0 && regno < JIT_NOREG);
366 return (!!(_rvs[regno].spec & jit_class_sav));
367}
368
369extern jit_bool_t
370_jit_pointer_p(jit_state_t *_jit, jit_pointer_t address)
371{
372 return ((jit_uint8_t *)address >= _jit->code.ptr &&
373 (jit_word_t)address < _jit->pc.w);
374}
375
376#if __ia64__
377void
378jit_regset_com(jit_regset_t *u, jit_regset_t *v)
379{
380 u->rl = ~v->rl; u->rh = ~v->rh;
381 u->fl = ~v->fl; u->fh = ~v->fh;
382}
383
384void
385jit_regset_and(jit_regset_t *u, jit_regset_t *v, jit_regset_t *w)
386{
387 u->rl = v->rl & w->rl; u->rh = v->rh & w->rh;
388 u->fl = v->fl & w->fl; u->fh = v->fh & w->fh;
389}
390
391void
392jit_regset_ior(jit_regset_t *u, jit_regset_t *v, jit_regset_t *w)
393{
394 u->rl = v->rl | w->rl; u->rh = v->rh | w->rh;
395 u->fl = v->fl | w->fl; u->fh = v->fh | w->fh;
396}
397
398void
399jit_regset_xor(jit_regset_t *u, jit_regset_t *v, jit_regset_t *w)
400{
401 u->rl = v->rl ^ w->rl; u->rh = v->rh ^ w->rh;
402 u->fl = v->fl ^ w->fl; u->fh = v->fh ^ w->fh;
403}
404
405void
406jit_regset_set(jit_regset_t *u, jit_regset_t *v)
407{
408 u->rl = v->rl; u->rh = v->rh;
409 u->fl = v->fl; u->fh = v->fh;
410}
411
412void
413jit_regset_set_mask(jit_regset_t *u, jit_int32_t v)
414{
415 jit_bool_t w = !!(v & (v - 1));
416
417 assert(v >= 0 && v <= 256);
418 if (v == 0)
419 u->rl = u->rh = u->fl = u->fh = -1LL;
420 else if (v <= 64) {
421 u->rl = w ? (1LL << v) - 1 : -1LL;
422 u->rh = u->fl = u->fh = 0;
423 }
424 else if (v <= 128) {
425 u->rl = -1LL;
426 u->rh = w ? (1LL << (v - 64)) - 1 : -1LL;
427 u->fl = u->fh = 0;
428 }
429 else if (v <= 192) {
430 u->rl = u->rh = -1LL;
431 u->fl = w ? (1LL << (v - 128)) - 1 : -1LL;
432 u->fh = 0;
433 }
434 else {
435 u->rl = u->rh = u->fl = -1LL;
436 u->fh = w ? (1LL << (v - 128)) - 1 : -1LL;
437 }
438}
439
440jit_bool_t
441jit_regset_cmp_ui(jit_regset_t *u, jit_word_t v)
442{
443 return !((u->rl == v && u->rh == 0 && u->fl == 0 && u->fh == 0));
444}
445
446void
447jit_regset_set_ui(jit_regset_t *u, jit_word_t v)
448{
449 u->rl = v;
450 u->rh = u->fl = u->fh = 0;
451}
452
453jit_bool_t
454jit_regset_set_p(jit_regset_t *u)
455{
456 return (u->rl || u->rh || u->fl || u->fh);
457}
458
459void
460jit_regset_clrbit(jit_regset_t *set, jit_int32_t bit)
461{
462 assert(bit >= 0 && bit <= 255);
463 if (bit < 64)
464 set->rl &= ~(1LL << bit);
465 else if (bit < 128)
466 set->rh &= ~(1LL << (bit - 64));
467 else if (bit < 192)
468 set->fl &= ~(1LL << (bit - 128));
469 else
470 set->fh &= ~(1LL << (bit - 192));
471}
472
473void
474jit_regset_setbit(jit_regset_t *set, jit_int32_t bit)
475{
476 assert(bit >= 0 && bit <= 255);
477 if (bit < 64)
478 set->rl |= 1LL << bit;
479 else if (bit < 128)
480 set->rh |= 1LL << (bit - 64);
481 else if (bit < 192)
482 set->fl |= 1LL << (bit - 128);
483 else
484 set->fh |= 1LL << (bit - 192);
485}
486
487jit_bool_t
488jit_regset_tstbit(jit_regset_t *set, jit_int32_t bit)
489{
490 assert(bit >= 0 && bit <= 255);
491 if (bit < 64)
492 return (!!(set->rl & (1LL << bit)));
493 else if (bit < 128)
494 return (!!(set->rh & (1LL << (bit - 64))));
495 else if (bit < 192)
496 return (!!(set->fl & (1LL << (bit - 128))));
497 return (!!(set->fh & (1LL << (bit - 192))));
498}
499
500unsigned long
501jit_regset_scan1(jit_regset_t *set, jit_int32_t offset)
502{
503 assert(offset >= 0 && offset <= 255);
504 for (; offset < 64; offset++) {
505 if (set->rl & (1LL << offset))
506 return (offset);
507 }
508 for (; offset < 128; offset++) {
509 if (set->rh & (1LL << (offset - 64)))
510 return (offset);
511 }
512 for (; offset < 192; offset++) {
513 if (set->fl & (1LL << (offset - 128)))
514 return (offset);
515 }
516 for (; offset < 256; offset++) {
517 if (set->fh & (1LL << (offset - 192)))
518 return (offset);
519 }
520 return (ULONG_MAX);
521}
522
523#elif __sparc__ && __WORDSIZE == 64
524void
525jit_regset_com(jit_regset_t *u, jit_regset_t *v)
526{
527 u->rl = ~v->rl; u->rh = ~v->rh;
528}
529
530void
531jit_regset_and(jit_regset_t *u, jit_regset_t *v, jit_regset_t *w)
532{
533 u->rl = v->rl & w->rl; u->rh = v->rh & w->rh;
534}
535
536void
537jit_regset_ior(jit_regset_t *u, jit_regset_t *v, jit_regset_t *w)
538{
539 u->rl = v->rl | w->rl; u->rh = v->rh | w->rh;
540}
541
542void
543jit_regset_xor(jit_regset_t *u, jit_regset_t *v, jit_regset_t *w)
544{
545 u->rl = v->rl ^ w->rl; u->rh = v->rh ^ w->rh;
546}
547
548void
549jit_regset_set(jit_regset_t *u, jit_regset_t *v)
550{
551 u->rl = v->rl; u->rh = v->rh;
552}
553
554void
555jit_regset_set_mask(jit_regset_t *u, jit_int32_t v)
556{
557 jit_bool_t w = !!(v & (v - 1));
558
559 assert(v >= 0 && v <= 128);
560 if (v == 0)
561 u->rl = u->rh = -1LL;
562 else if (v <= 64) {
563 u->rl = w ? (1LL << v) - 1 : -1LL;
564 u->rh = 0;
565 }
566 else {
567 u->rl = -1LL;
568 u->rh = w ? (1LL << (v - 64)) - 1 : -1LL;
569 }
570}
571
572jit_bool_t
573jit_regset_cmp_ui(jit_regset_t *u, jit_word_t v)
574{
575 return !((u->rl == v && u->rh == 0));
576}
577
578void
579jit_regset_set_ui(jit_regset_t *u, jit_word_t v)
580{
581 u->rl = v;
582 u->rh = 0;
583}
584
585jit_bool_t
586jit_regset_set_p(jit_regset_t *u)
587{
588 return (u->rl || u->rh);
589}
590
591void
592jit_regset_clrbit(jit_regset_t *set, jit_int32_t bit)
593{
594 assert(bit >= 0 && bit <= 128);
595 if (bit < 64)
596 set->rl &= ~(1LL << bit);
597 else
598 set->rh &= ~(1LL << (bit - 64));
599}
600
601void
602jit_regset_setbit(jit_regset_t *set, jit_int32_t bit)
603{
604 assert(bit >= 0 && bit <= 127);
605 if (bit < 64)
606 set->rl |= 1LL << bit;
607 else
608 set->rh |= 1LL << (bit - 64);
609}
610
611jit_bool_t
612jit_regset_tstbit(jit_regset_t *set, jit_int32_t bit)
613{
614 assert(bit >= 0 && bit <= 127);
615 if (bit < 64)
616 return (!!(set->rl & (1LL << bit)));
617 else
618 return (!!(set->rh & (1LL << (bit - 64))));
619}
620
621unsigned long
622jit_regset_scan1(jit_regset_t *set, jit_int32_t offset)
623{
624 assert(offset >= 0 && offset <= 127);
625 for (; offset < 64; offset++) {
626 if (set->rl & (1LL << offset))
627 return (offset);
628 }
629 for (; offset < 128; offset++) {
630 if (set->rh & (1LL << (offset - 64)))
631 return (offset);
632 }
633 return (ULONG_MAX);
634}
635
636#else
637unsigned long
638jit_regset_scan1(jit_regset_t *set, jit_int32_t offset)
639{
640 jit_regset_t mask;
641 assert(offset >= 0 && offset <= 63);
642 if ((mask = *set >> offset)) {
643 for (;;) {
644 if (mask & 1)
645 return (offset);
646 mask >>= 1;
647 ++offset;
648 }
649 }
650 return (ULONG_MAX);
651}
652#endif
653
654void
655_jit_save(jit_state_t *_jit, jit_int32_t reg)
656{
657 reg = jit_regno(reg);
658 assert(!_jitc->realize);
659 _jitc->spill[reg] = jit_new_node_w(jit_code_save, reg);
660}
661
662void
663_jit_load(jit_state_t *_jit, jit_int32_t reg)
664{
665 jit_node_t *node;
666
667 reg = jit_regno(reg);
668 assert(!_jitc->realize);
669 assert(_jitc->spill[reg] != NULL);
670 node = jit_new_node_w(jit_code_load, reg);
671 /* create a path to flag the save/load is not required */
672 node->link = _jitc->spill[reg];
673 node->link->link = node;
674 _jitc->spill[reg] = NULL;
675}
676
677static jit_word_t
678hash_data(const void *data, jit_word_t length)
679{
680 const jit_uint8_t *ptr;
681 jit_word_t i, key;
682 for (i = key = 0, ptr = data; i < length; i++)
683 key = (key << (key & 1)) ^ ptr[i];
684 return (key);
685}
686
687jit_pointer_t
688_jit_address(jit_state_t *_jit, jit_node_t *node)
689{
690 assert(_jitc->done);
691 assert(node != NULL &&
692 /* If a node type that is documented to be a fixed marker */
693 (node->code == jit_code_note || node->code == jit_code_name ||
694 /* If another special fixed marker, returned by jit_indirect() */
695 (node->code == jit_code_label && (node->flag & jit_flag_use) != 0)));
696 return ((jit_pointer_t)node->u.w);
697}
698
699jit_node_t *
700_jit_data(jit_state_t *_jit, const void *data,
701 jit_word_t length, jit_int32_t align)
702{
703 jit_word_t key;
704 jit_node_t *node;
705
706 assert(!_jitc->realize);
707
708 /* Ensure there is space even if asking for a duplicate */
709 if (((_jitc->data.offset + 7) & -8) + length > _jit->data.length) {
710 jit_word_t size;
711
712 size = (_jit->data.length + length + 4096) & - 4095;
713 assert(size >= _jit->data.length);
714 if (_jitc->data.ptr == NULL)
715 jit_alloc((jit_pointer_t *)&_jitc->data.ptr, size);
716 else
717 jit_realloc((jit_pointer_t *)&_jitc->data.ptr,
718 _jit->data.length, size);
719 _jit->data.length = size;
720 }
721 if (_jitc->data.table == NULL)
722 jit_alloc((jit_pointer_t *)&_jitc->data.table,
723 (_jitc->data.size = 16) * sizeof(jit_node_t*));
724
725 key = hash_data(data, length) & (_jitc->data.size - 1);
726 node = _jitc->data.table[key];
727 for (; node; node = node->next) {
728 if (node->v.w == length &&
729 memcmp(_jitc->data.ptr + node->u.w, data, length) == 0)
730 break;
731 }
732
733 if (!node) {
734 node = jit_new_node_no_link(jit_code_data);
735 if (!align)
736 align = length;
737 switch (align) {
738 case 0: case 1:
739 break;
740 case 2:
741 _jitc->data.offset = (_jitc->data.offset + 1) & -2;
742 break;
743 case 3: case 4:
744 _jitc->data.offset = (_jitc->data.offset + 3) & -4;
745 break;
746 default:
747 _jitc->data.offset = (_jitc->data.offset + 7) & -8;
748 break;
749 }
750 node->u.w = _jitc->data.offset;
751 node->v.w = length;
752 jit_memcpy(_jitc->data.ptr + _jitc->data.offset, data, length);
753 _jitc->data.offset += length;
754
755 node->next = _jitc->data.table[key];
756 _jitc->data.table[key] = node;
757 ++_jitc->data.count;
758
759 /* Rehash if more than 75% used table */
760 if (_jitc->data.count >
761 (_jitc->data.size >> 1) + (_jitc->data.size >> 2) &&
762 (_jitc->data.size << 1) > _jitc->data.size) {
763 jit_word_t i;
764 jit_node_t **hash;
765 jit_node_t *next;
766 jit_node_t *temp;
767
768 jit_alloc((jit_pointer_t *)&hash,
769 (_jitc->data.size << 1) * sizeof(jit_node_t*));
770 for (i = 0; i < _jitc->data.size; i++) {
771 temp = _jitc->data.table[i];
772 for (; temp; temp = next) {
773 next = temp->next;
774 key = hash_data(_jitc->data.ptr + temp->u.w, temp->v.w) &
775 ((_jitc->data.size << 1) - 1);
776 temp->next = hash[key];
777 hash[key] = temp;
778 }
779 }
780 jit_free((jit_pointer_t *)&_jitc->data.table);
781 _jitc->data.table = hash;
782 _jitc->data.size <<= 1;
783 }
784 }
785
786 return (node);
787}
788
789static void
790_new_pool(jit_state_t *_jit)
791{
792 jit_node_t *list;
793 jit_int32_t offset;
794
795 if (_jitc->pool.offset >= _jitc->pool.length) {
796 jit_int32_t length;
797
798 length = _jitc->pool.length + 16;
799 jit_realloc((jit_pointer_t *)&_jitc->pool.ptr,
800 _jitc->pool.length * sizeof(jit_node_t *),
801 length * sizeof(jit_node_t *));
802 _jitc->pool.length = length;
803 }
804 jit_alloc((jit_pointer_t *)(_jitc->pool.ptr + _jitc->pool.offset),
805 sizeof(jit_node_t) * 1024);
806 list = _jitc->pool.ptr[_jitc->pool.offset];
807 for (offset = 1; offset < 1024; offset++, list++)
808 list->next = list + 1;
809 list->next = _jitc->list;
810 _jitc->list = _jitc->pool.ptr[_jitc->pool.offset];
811 ++_jitc->pool.offset;
812}
813
814static jit_node_t *
815_new_node(jit_state_t *_jit, jit_code_t code)
816{
817 jit_node_t *node;
818
819 if (_jitc->list == NULL)
820 new_pool();
821 node = _jitc->list;
822 _jitc->list = node->next;
823 if (_jitc->synth)
824 node->flag |= jit_flag_synth;
825 node->next = NULL;
826 node->code = code;
827
828 return (node);
829}
830
831static inline jit_node_t *
832_link_node(jit_state_t *_jit, jit_node_t *node)
833{
834 if (_jitc->tail)
835 _jitc->tail->next = node;
836 else
837 _jitc->head = node;
838 return (_jitc->tail = node);
839}
840
841static inline void
842_del_node(jit_state_t *_jit, jit_node_t *prev, jit_node_t *node)
843{
844 if (prev == node) {
845 assert(prev == _jitc->head);
846 _jitc->head = node->next;
847 }
848 else
849 prev->next = node->next;
850 memset(node, 0, sizeof(jit_node_t));
851 node->next = _jitc->list;
852 _jitc->list = node;
853}
854
855static inline void
856_free_node(jit_state_t *_jit, jit_node_t *node)
857{
858 memset(node, 0, sizeof(jit_node_t));
859 node->next = _jitc->list;
860 _jitc->list = node;
861}
862
863static void
864_del_label(jit_state_t *_jit, jit_node_t *prev, jit_node_t *node)
865{
866 jit_block_t *block;
867
868 /* only allow call to del_label on linked labels */
869 block = _jitc->blocks.ptr + node->v.w;
870 assert(block->label == node);
871
872 /* del_label() should only be called when optimizing.
873 * This will leave an empty block index */
874 jit_regset_del(&block->reglive);
875 jit_regset_del(&block->regmask);
876 block->label = NULL;
877
878 /* redundant, should be already true */
879 assert(node->link == NULL);
880 del_node(prev, node);
881}
882
883jit_state_t *
884jit_new_state(void)
885{
886 jit_state_t *_jit;
887
888 jit_alloc((jit_pointer_t *)&_jit, sizeof(jit_state_t));
889 jit_alloc((jit_pointer_t *)&_jitc, sizeof(jit_compiler_t));
890 jit_regset_new(&_jitc->regarg);
891 jit_regset_new(&_jitc->regsav);
892 jit_regset_new(&_jitc->reglive);
893 jit_regset_new(&_jitc->regmask);
79bfeef6 894 jit_regset_new(&_jitc->explive);
4a71579b
PC		 895
896 jit_init();
897
898 jit_alloc((jit_pointer_t *)&_jitc->spill,
899 _jitc->reglen * sizeof(jit_node_t*));
900 jit_alloc((jit_pointer_t *)&_jitc->gen,
901 _jitc->reglen * sizeof(jit_int32_t));
902 jit_alloc((jit_pointer_t *)&_jitc->values,
903 _jitc->reglen * sizeof(jit_value_t));
904
905 jit_alloc((jit_pointer_t *)&_jitc->patches.ptr,
906 (_jitc->patches.length = 1024) * sizeof(jit_patch_t));
907 jit_alloc((jit_pointer_t *)&_jitc->functions.ptr,
908 (_jitc->functions.length = 16) * sizeof(jit_function_t));
909 jit_alloc((jit_pointer_t *)&_jitc->pool.ptr,
910 (_jitc->pool.length = 16) * sizeof(jit_node_t*));
911 jit_alloc((jit_pointer_t *)&_jitc->blocks.ptr,
912 (_jitc->blocks.length = 16) * sizeof(jit_block_t));
913#if __arm__ && DISASSEMBLER
914 jit_alloc((jit_pointer_t *)&_jitc->data_info.ptr,
915 (_jitc->data_info.length = 1024) * sizeof(jit_data_info_t));
916#endif
917
918 /* allocate at most one extra note in case jit_name() is
919 * never called, or called after adding at least one note */
920 _jit->note.length = 1;
921 _jitc->note.size = sizeof(jit_note_t);
922
923 return (_jit);
924}
925
926void
927_jit_clear_state(jit_state_t *_jit)
928{
929#if DEVEL_DISASSEMBLER
930# define jit_really_clear_state() _jit_really_clear_state(_jit)
931}
932
933void _jit_really_clear_state(jit_state_t *_jit)
934{
935#endif
936 jit_word_t offset;
937 jit_function_t *function;
938
939 /* release memory not required at jit execution time and set
940 * pointers to NULL to explicitly know they are released */
941 _jitc->head = _jitc->tail = NULL;
942
943 jit_free((jit_pointer_t *)&_jitc->data.table);
944 _jitc->data.size = _jitc->data.count = 0;
945
946 jit_free((jit_pointer_t *)&_jitc->spill);
947 jit_free((jit_pointer_t *)&_jitc->gen);
948 jit_free((jit_pointer_t *)&_jitc->values);
949
950 jit_free((jit_pointer_t *)&_jitc->blocks.ptr);
951
952 jit_free((jit_pointer_t *)&_jitc->patches.ptr);
953 _jitc->patches.offset = _jitc->patches.length = 0;
954
955 for (offset = 0; offset < _jitc->functions.offset; offset++) {
956 function = _jitc->functions.ptr + offset;
957 jit_free((jit_pointer_t *)&function->regoff);
958 }
959 jit_free((jit_pointer_t *)&_jitc->functions.ptr);
960 _jitc->functions.offset = _jitc->functions.length = 0;
961 _jitc->function = NULL;
962
963 for (offset = 0; offset < _jitc->pool.offset; offset++)
964 jit_free((jit_pointer_t *)(_jitc->pool.ptr + offset));
965 jit_free((jit_pointer_t *)&_jitc->pool.ptr);
966 _jitc->pool.offset = _jitc->pool.length = 0;
967 _jitc->list = NULL;
968
969 _jitc->note.head = _jitc->note.tail =
970 _jitc->note.name = _jitc->note.note = NULL;
971 _jitc->note.base = NULL;
972
973#if __arm__ && DISASSEMBLER
974 jit_free((jit_pointer_t *)&_jitc->data_info.ptr);
975#endif
976
977#if (__powerpc__ && _CALL_AIXDESC) || __ia64__
978 jit_free((jit_pointer_t *)&_jitc->prolog.ptr);
979#endif
980
981#if __ia64__
982 jit_regset_del(&_jitc->regs);
983#endif
984
985 jit_free((jit_pointer_t *)&_jitc);
986}
987
988void
989_jit_destroy_state(jit_state_t *_jit)
990{
991#if DEVEL_DISASSEMBLER
992 jit_really_clear_state();
993#endif
40a44dcb 994#if HAVE_MMAP
4a71579b
PC		 995 if (!_jit->user_code)
996 munmap(_jit->code.ptr, _jit->code.length);
997 if (!_jit->user_data)
998 munmap(_jit->data.ptr, _jit->data.length);
40a44dcb 999#endif
4a71579b
PC		 1000 jit_free((jit_pointer_t *)&_jit);
1001}
1002
1003void
1004_jit_synth_inc(jit_state_t *_jit)
1005{
1006 assert(_jitc->synth < 8);
1007 ++_jitc->synth;
1008}
1009
1010jit_node_t *
1011_jit_new_node(jit_state_t *_jit, jit_code_t code)
1012{
1013 assert(!_jitc->realize);
1014 return (link_node(new_node(code)));
1015}
1016
1017jit_node_t *
1018_jit_new_node_no_link(jit_state_t *_jit, jit_code_t code)
1019{
1020 assert(!_jitc->realize);
1021 return (new_node(code));
1022}
1023
1024void
1025_jit_link_node(jit_state_t *_jit, jit_node_t *node)
1026{
1027 assert(!_jitc->realize);
1028 link_node(node);
1029}
1030
1031void
1032_jit_synth_dec(jit_state_t *_jit)
1033{
1034 assert(_jitc->synth > 0);
1035 --_jitc->synth;
1036}
1037
1038jit_node_t *
1039_jit_new_node_w(jit_state_t *_jit, jit_code_t code,
1040 jit_word_t u)
1041{
1042 jit_node_t *node = new_node(code);
1043 assert(!_jitc->realize);
1044 node->u.w = u;
1045 return (link_node(node));
1046}
1047
1048jit_node_t *
1049_jit_new_node_f(jit_state_t *_jit, jit_code_t code,
1050 jit_float32_t u)
1051{
1052 jit_node_t *node = new_node(code);
1053 assert(!_jitc->realize);
1054 node->u.f = u;
1055 return (link_node(node));
1056}
1057
1058jit_node_t *
1059_jit_new_node_d(jit_state_t *_jit, jit_code_t code,
1060 jit_float64_t u)
1061{
1062 jit_node_t *node = new_node(code);
1063 assert(!_jitc->realize);
1064 node->u.d = u;
1065 return (link_node(node));
1066}
1067
1068jit_node_t *
1069_jit_new_node_p(jit_state_t *_jit, jit_code_t code,
1070 jit_pointer_t u)
1071{
1072 jit_node_t *node = new_node(code);
1073 assert(!_jitc->realize);
1074 node->u.p = u;
1075 return (link_node(node));
1076}
1077
1078jit_node_t *
1079_jit_new_node_ww(jit_state_t *_jit, jit_code_t code,
1080 jit_word_t u, jit_word_t v)
1081{
1082 jit_node_t *node = new_node(code);
1083 assert(!_jitc->realize);
1084 node->u.w = u;
1085 node->v.w = v;
1086 return (link_node(node));
1087}
1088
1089jit_node_t *
1090_jit_new_node_wp(jit_state_t *_jit, jit_code_t code,
1091 jit_word_t u, jit_pointer_t v)
1092{
1093 return (jit_new_node_ww(code, u, (jit_word_t)v));
1094}
1095
1096jit_node_t *
1097_jit_new_node_fp(jit_state_t *_jit, jit_code_t code,
1098 jit_float32_t u, jit_pointer_t v)
1099{
1100 jit_node_t *node = new_node(code);
1101 assert(!_jitc->realize);
1102 node->u.f = u;
1103 node->v.w = (jit_word_t)v;
1104 return (link_node(node));
1105}
1106
1107jit_node_t *
1108_jit_new_node_dp(jit_state_t *_jit, jit_code_t code,
1109 jit_float64_t u, jit_pointer_t v)
1110{
1111 jit_node_t *node = new_node(code);
1112 assert(!_jitc->realize);
1113 node->u.d = u;
1114 node->v.w = (jit_word_t)v;
1115 return (link_node(node));
1116}
1117
1118jit_node_t *
1119_jit_new_node_pw(jit_state_t *_jit, jit_code_t code,
1120 jit_pointer_t u, jit_word_t v)
1121{
1122 return (jit_new_node_ww(code, (jit_word_t)u, v));
1123}
1124
1125jit_node_t *
1126_jit_new_node_wf(jit_state_t *_jit, jit_code_t code,
1127 jit_word_t u, jit_float32_t v)
1128{
1129 jit_node_t *node = new_node(code);
1130 assert(!_jitc->realize);
1131 node->u.w = u;
1132 node->v.f = v;
1133 return (link_node(node));
1134}
1135
1136jit_node_t *
1137_jit_new_node_wd(jit_state_t *_jit, jit_code_t code,
1138 jit_word_t u, jit_float64_t v)
1139{
1140 jit_node_t *node = new_node(code);
1141 assert(!_jitc->realize);
1142 node->u.w = u;
1143 node->v.d = v;
1144 return (link_node(node));
1145}
1146
1147jit_node_t *
1148_jit_new_node_www(jit_state_t *_jit, jit_code_t code,
1149 jit_word_t u, jit_word_t v, jit_word_t w)
1150{
1151 jit_node_t *node = new_node(code);
1152 assert(!_jitc->realize);
1153 node->u.w = u;
1154 node->v.w = v;
1155 node->w.w = w;
1156 return (link_node(node));
1157}
1158
1159jit_node_t *
1160_jit_new_node_qww(jit_state_t *_jit, jit_code_t code,
1161 jit_int32_t l, jit_int32_t h,
1162 jit_word_t v, jit_word_t w)
1163{
1164 jit_node_t *node = new_node(code);
1165 assert(!_jitc->realize);
1166 assert(l != h);
1167 node->u.q.l = l;
1168 node->u.q.h = h;
1169 node->v.w = v;
1170 node->w.w = w;
1171 return (link_node(node));
1172}
1173
ba86ff93
PC		 1174jit_node_t *
1175_jit_new_node_wqw(jit_state_t *_jit, jit_code_t code,
1176 jit_word_t u, jit_int32_t l,
1177 jit_int32_t h, jit_word_t w)
1178{
1179 jit_node_t *node = new_node(code);
1180 assert(!_jitc->realize);
1181 assert(l != h);
1182 node->u.w = u;
1183 node->v.q.l = l;
1184 node->v.q.h = h;
1185 node->w.w = w;
1186 return (link_node(node));
1187}
1188
ba3814c1
PC		 1189jit_node_t *
1190_jit_new_node_wwq(jit_state_t *_jit, jit_code_t code,
1191 jit_word_t u, jit_word_t v,
1192 jit_int32_t l, jit_int32_t h)
1193{
1194 jit_node_t *node = new_node(code);
1195 assert(!_jitc->realize);
1196 node->u.w = u;
1197 node->v.w = v;
1198 node->w.q.l = l;
1199 node->w.q.h = h;
1200 return (link_node(node));
1201}
1202
4a71579b
PC		 1203jit_node_t *
1204_jit_new_node_wwf(jit_state_t *_jit, jit_code_t code,
1205 jit_word_t u, jit_word_t v, jit_float32_t w)
1206{
1207 jit_node_t *node = new_node(code);
1208 assert(!_jitc->realize);
1209 node->u.w = u;
1210 node->v.w = v;
1211 node->w.f = w;
1212 return (link_node(node));
1213}
1214
ba86ff93
PC		 1215jit_node_t *
1216_jit_new_node_wqf(jit_state_t *_jit, jit_code_t code,
1217 jit_word_t u, jit_int32_t l,
1218 jit_int32_t h, jit_float32_t w)
1219{
1220 jit_node_t *node = new_node(code);
1221 assert(!_jitc->realize);
1222 assert(l != h);
1223 node->u.w = u;
1224 node->v.q.l = l;
1225 node->v.q.h = h;
1226 node->w.f = w;
1227 return (link_node(node));
1228}
1229
4a71579b
PC		 1230jit_node_t *
1231_jit_new_node_wwd(jit_state_t *_jit, jit_code_t code,
1232 jit_word_t u, jit_word_t v, jit_float64_t w)
1233{
1234 jit_node_t *node = new_node(code);
1235 assert(!_jitc->realize);
1236 node->u.w = u;
1237 node->v.w = v;
1238 node->w.d = w;
1239 return (link_node(node));
1240}
1241
ba86ff93
PC		 1242jit_node_t *
1243_jit_new_node_wqd(jit_state_t *_jit, jit_code_t code,
1244 jit_word_t u, jit_int32_t l,
1245 jit_int32_t h, jit_float64_t w)
1246{
1247 jit_node_t *node = new_node(code);
1248 assert(!_jitc->realize);
1249 assert(l != h);
1250 node->u.w = u;
1251 node->v.q.l = l;
1252 node->v.q.h = h;
1253 node->w.d = w;
1254 return (link_node(node));
1255}
1256
4a71579b
PC		 1257jit_node_t *
1258_jit_new_node_pww(jit_state_t *_jit, jit_code_t code,
1259 jit_pointer_t u, jit_word_t v, jit_word_t w)
1260{
1261 jit_node_t *node = new_node(code);
1262 assert(!_jitc->realize);
1263 node->u.p = u;
1264 node->v.w = v;
1265 node->w.w = w;
1266 return (link_node(node));
1267}
1268
1269jit_node_t *
1270_jit_new_node_pwf(jit_state_t *_jit, jit_code_t code,
1271 jit_pointer_t u, jit_word_t v, jit_float32_t w)
1272{
1273 jit_node_t *node = new_node(code);
1274 assert(!_jitc->realize);
1275 node->u.p = u;
1276 node->v.w = v;
1277 node->w.f = w;
1278 return (link_node(node));
1279}
1280
1281jit_node_t *
1282_jit_new_node_pwd(jit_state_t *_jit, jit_code_t code,
1283 jit_pointer_t u, jit_word_t v, jit_float64_t w)
1284{
1285 jit_node_t *node = new_node(code);
1286 assert(!_jitc->realize);
1287 node->u.p = u;
1288 node->v.w = v;
1289 node->w.d = w;
1290 return (link_node(node));
1291}
1292
1293jit_node_t *
1294_jit_label(jit_state_t *_jit)
1295{
1296 jit_node_t *node;
1297
1298 if (!(node = _jitc->tail) || node->code != jit_code_label) {
1299 node = jit_forward();
1300 jit_link(node);
1301 }
1302
1303 return (node);
1304}
1305
1306jit_node_t *
1307_jit_forward(jit_state_t *_jit)
1308{
1309 return (jit_new_node_no_link(jit_code_label));
1310}
1311
1312jit_node_t *
1313_jit_indirect(jit_state_t *_jit)
1314{
1315 jit_node_t *node;
1316
1317 node = jit_label();
1318 node->flag |= jit_flag_use;
1319
1320 return (node);
1321}
1322
1323void
1324_jit_link(jit_state_t *_jit, jit_node_t *node)
1325{
1326 jit_block_t *block;
1327
1328 assert((node->code == jit_code_label ||
1329 node->code == jit_code_prolog ||
1330 node->code == jit_code_epilog) && !node->next);
1331 jit_link_node(node);
1332 if (_jitc->blocks.offset >= _jitc->blocks.length) {
1333 jit_word_t length;
1334
1335 length = _jitc->blocks.length + 16;
1336 jit_realloc((jit_pointer_t *)&_jitc->blocks.ptr,
1337 _jitc->blocks.length * sizeof(jit_block_t),
1338 length * sizeof(jit_block_t));
1339 _jitc->blocks.length = length;
1340 }
1341 block = _jitc->blocks.ptr + _jitc->blocks.offset;
1342 block->label = node;
1343 node->v.w = _jitc->blocks.offset;
1344 jit_regset_new(&block->reglive);
1345 jit_regset_new(&block->regmask);
1346 ++_jitc->blocks.offset;
1347}
1348
1349jit_bool_t
1350_jit_forward_p(jit_state_t *_jit, jit_node_t *node)
1351{
1352 return (node->code == jit_code_label && !node->next && node != _jitc->tail);
1353}
1354
1355jit_bool_t
1356_jit_indirect_p(jit_state_t *_jit, jit_node_t *node)
1357{
1358 return (node->code == jit_code_label && !!(node->flag & jit_flag_use));
1359}
1360
1361jit_bool_t
1362_jit_target_p(jit_state_t *_jit, jit_node_t *node)
1363{
1364 return (node->code == jit_code_label && !!node->link);
1365}
1366
1367void
1368_jit_prepare(jit_state_t *_jit)
1369{
1370 assert(_jitc->function != NULL);
1371 _jitc->function->call.call = jit_call_default;
1372 _jitc->function->call.argi =
1373 _jitc->function->call.argf =
1374 _jitc->function->call.size = 0;
1375 _jitc->prepare = jit_new_node(jit_code_prepare);
1376}
1377
1378void
1379_jit_patch(jit_state_t* _jit, jit_node_t *instr)
1380{
1381 jit_node_t *label;
1382
1383 if (!(label = _jitc->tail) || label->code != jit_code_label)
1384 label = jit_label();
1385 jit_patch_at(instr, label);
1386}
1387
1388jit_int32_t
1389_jit_classify(jit_state_t *_jit, jit_code_t code)
1390{
1391 jit_int32_t mask;
1392
1393 switch (code) {
1394 case jit_code_data: case jit_code_save: case jit_code_load:
1395 case jit_code_name: case jit_code_label: case jit_code_note:
1396 case jit_code_prolog: case jit_code_ellipsis: case jit_code_va_push:
1397 case jit_code_epilog: case jit_code_ret: case jit_code_prepare:
1398 mask = 0;
1399 break;
1400 case jit_code_live: case jit_code_va_end:
79bfeef6
PC		 1401 case jit_code_retr_c: case jit_code_retr_uc:
1402 case jit_code_retr_s: case jit_code_retr_us:
1403 case jit_code_retr_i: case jit_code_retr_ui:
1404 case jit_code_retr_l:
1405 case jit_code_retr_f: case jit_code_retr_d:
1406 case jit_code_pushargr_c:
1407 case jit_code_pushargr_uc:
1408 case jit_code_pushargr_s:
1409 case jit_code_pushargr_us:
1410 case jit_code_pushargr_i:
1411 case jit_code_pushargr_ui:
1412 case jit_code_pushargr_l:
1413 case jit_code_pushargr_f:
4a71579b
PC		 1414 case jit_code_pushargr_d:
1415 case jit_code_finishr: /* synthesized will set jit_cc_a0_jmp */
1416 mask = jit_cc_a0_reg;
1417 break;
79bfeef6
PC		 1418 case jit_code_align: case jit_code_skip:
1419 case jit_code_reti_c: case jit_code_reti_uc:
1420 case jit_code_reti_s: case jit_code_reti_us:
1421 case jit_code_reti_i: case jit_code_reti_ui:
1422 case jit_code_reti_l:
1423 case jit_code_pushargi_c:
1424 case jit_code_pushargi_uc:
1425 case jit_code_pushargi_s:
1426 case jit_code_pushargi_us:
1427 case jit_code_pushargi_i:
1428 case jit_code_pushargi_ui:
1429 case jit_code_pushargi_l:
1430 case jit_code_finishi: /* synthesized will set jit_cc_a0_jmp */
4a71579b
PC		 1431 mask = jit_cc_a0_int;
1432 break;
1433 case jit_code_reti_f: case jit_code_pushargi_f:
1434 mask = jit_cc_a0_flt;
1435 break;
1436 case jit_code_reti_d: case jit_code_pushargi_d:
1437 mask = jit_cc_a0_dbl;
1438 break;
1439 case jit_code_allocai:
1440 mask = jit_cc_a0_int|jit_cc_a1_int;
1441 break;
79bfeef6
PC		 1442 case jit_code_arg_c: case jit_code_arg_s:
1443 case jit_code_arg_i: case jit_code_arg_l:
1444 case jit_code_arg_f: case jit_code_arg_d:
4a71579b
PC		 1445 mask = jit_cc_a0_int|jit_cc_a0_arg;
1446 break;
1447 case jit_code_calli: case jit_code_jmpi:
1448 mask = jit_cc_a0_jmp;
1449 break;
1450 case jit_code_callr: case jit_code_jmpr:
1451 mask = jit_cc_a0_reg|jit_cc_a0_jmp;
1452 break;
1453 case jit_code_retval_c: case jit_code_retval_uc:
1454 case jit_code_retval_s: case jit_code_retval_us:
1455 case jit_code_retval_i: case jit_code_retval_ui:
1456 case jit_code_retval_l:
1457 case jit_code_retval_f: case jit_code_retval_d:
1458 case jit_code_va_start:
1459 mask = jit_cc_a0_reg|jit_cc_a0_chg;
1460 break;
1461 case jit_code_getarg_c: case jit_code_getarg_uc:
1462 case jit_code_getarg_s: case jit_code_getarg_us:
1463 case jit_code_getarg_i: case jit_code_getarg_ui:
1464 case jit_code_getarg_l:
1465 case jit_code_getarg_f: case jit_code_getarg_d:
1466 mask = jit_cc_a0_reg|jit_cc_a0_chg|jit_cc_a1_arg;
1467 break;
79bfeef6
PC		 1468 case jit_code_putargr_c:case jit_code_putargr_uc:
1469 case jit_code_putargr_s:case jit_code_putargr_us:
1470 case jit_code_putargr_i:case jit_code_putargr_ui:
1471 case jit_code_putargr_l:
1472 case jit_code_putargr_f:case jit_code_putargr_d:
4a71579b
PC		 1473 mask = jit_cc_a0_reg|jit_cc_a1_arg;
1474 break;
79bfeef6
PC		 1475 case jit_code_putargi_c:case jit_code_putargi_uc:
1476 case jit_code_putargi_s:case jit_code_putargi_us:
1477 case jit_code_putargi_i:case jit_code_putargi_ui:
1478 case jit_code_putargi_l:
4a71579b
PC		 1479 mask = jit_cc_a0_int|jit_cc_a1_arg;
1480 break;
1481 case jit_code_putargi_f:
1482 mask = jit_cc_a0_flt|jit_cc_a1_arg;
1483 break;
1484 case jit_code_putargi_d:
1485 mask = jit_cc_a0_dbl|jit_cc_a1_arg;
1486 break;
1487 case jit_code_movi: case jit_code_ldi_c: case jit_code_ldi_uc:
1488 case jit_code_ldi_s: case jit_code_ldi_us: case jit_code_ldi_i:
1489 case jit_code_ldi_ui: case jit_code_ldi_l: case jit_code_ldi_f:
1490 case jit_code_ldi_d:
ba86ff93
PC		 1491 case jit_code_negi: case jit_code_comi:
1492 case jit_code_exti_c: case jit_code_exti_uc:
1493 case jit_code_exti_s: case jit_code_exti_us:
1494 case jit_code_exti_i: case jit_code_exti_ui:
1495 case jit_code_bswapi_us:case jit_code_bswapi_ui:
1496 case jit_code_bswapi_ul:
1497 case jit_code_htoni_us: case jit_code_htoni_ui:
1498 case jit_code_htoni_ul:
1499 case jit_code_cloi: case jit_code_clzi:
1500 case jit_code_ctoi: case jit_code_ctzi:
1501 case jit_code_rbiti: case jit_code_popcnti:
1502 case jit_code_movi_w_f: case jit_code_movi_w_d:
4a71579b
PC		 1503 mask = jit_cc_a0_reg|jit_cc_a0_chg|jit_cc_a1_int;
1504 break;
ba86ff93
PC		 1505 case jit_code_unldi: case jit_code_unldi_u: case jit_code_unldi_x:
1506 mask = jit_cc_a0_reg|jit_cc_a0_chg|jit_cc_a1_int|jit_cc_a2_int;
1507 break;
1508 case jit_code_movi_ww_d:
1509 mask = jit_cc_a0_reg|jit_cc_a0_chg|jit_cc_a1_int|jit_cc_a2_int;
1510 break;
4a71579b 1511 case jit_code_movi_f: case jit_code_movi_f_w:
ba86ff93
PC		 1512 case jit_code_negi_f: case jit_code_absi_f:
1513 case jit_code_sqrti_f:
4a71579b
PC		 1514 mask = jit_cc_a0_reg|jit_cc_a0_chg|jit_cc_a1_flt;
1515 break;
1516 case jit_code_movi_d: case jit_code_movi_d_w:
ba86ff93
PC		 1517 case jit_code_negi_d: case jit_code_absi_d:
1518 case jit_code_sqrti_d:
4a71579b
PC		 1519 mask = jit_cc_a0_reg|jit_cc_a0_chg|jit_cc_a1_dbl;
1520 break;
1521 case jit_code_movi_d_ww:
1522 mask = jit_cc_a0_reg|jit_cc_a0_chg|jit_cc_a1_reg|jit_cc_a1_chg|
1523 jit_cc_a2_dbl;
1524 break;
1525 case jit_code_negr: case jit_code_comr: case jit_code_movr:
1526 case jit_code_extr_c: case jit_code_extr_uc: case jit_code_extr_s:
1527 case jit_code_extr_us: case jit_code_extr_i: case jit_code_extr_ui:
1528 case jit_code_truncr_f_i: case jit_code_truncr_f_l:
1529 case jit_code_truncr_d_i: case jit_code_truncr_d_l:
1530 case jit_code_htonr_us: case jit_code_htonr_ui: case jit_code_htonr_ul:
40a44dcb 1531 case jit_code_bswapr_us: case jit_code_bswapr_ui: case jit_code_bswapr_ul:
4a71579b
PC		 1532 case jit_code_ldr_c: case jit_code_ldr_uc:
1533 case jit_code_ldr_s: case jit_code_ldr_us: case jit_code_ldr_i:
1534 case jit_code_ldr_ui: case jit_code_ldr_l: case jit_code_negr_f:
1535 case jit_code_absr_f: case jit_code_sqrtr_f: case jit_code_movr_f:
1536 case jit_code_extr_f: case jit_code_extr_d_f: case jit_code_ldr_f:
1537 case jit_code_negr_d: case jit_code_absr_d: case jit_code_sqrtr_d:
1538 case jit_code_movr_d: case jit_code_extr_d: case jit_code_extr_f_d:
ba86ff93 1539 case jit_code_ldr_d: case jit_code_rbitr: case jit_code_popcntr:
79bfeef6
PC		 1540 case jit_code_clor: case jit_code_clzr:
1541 case jit_code_ctor: case jit_code_ctzr:
4a71579b
PC		 1542 case jit_code_movr_w_f: case jit_code_movr_f_w:
1543 case jit_code_movr_w_d: case jit_code_movr_d_w:
1544 case jit_code_va_arg: case jit_code_va_arg_d:
1545 mask = jit_cc_a0_reg|jit_cc_a0_chg|jit_cc_a1_reg;
1546 break;
1547 case jit_code_movr_d_ww:
1548 mask = jit_cc_a0_reg|jit_cc_a0_chg|jit_cc_a1_reg|jit_cc_a1_chg|
1549 jit_cc_a2_reg;
1550 break;
1551 case jit_code_addi: case jit_code_addxi: case jit_code_addci:
1552 case jit_code_subi: case jit_code_subxi: case jit_code_subci:
1553 case jit_code_rsbi:
ba86ff93
PC		 1554 case jit_code_muli: case jit_code_hmuli: case jit_code_hmuli_u:
1555 case jit_code_divi: case jit_code_divi_u:
4a71579b
PC		 1556 case jit_code_remi: case jit_code_remi_u: case jit_code_andi:
1557 case jit_code_ori: case jit_code_xori: case jit_code_lshi:
ba86ff93
PC		 1558 case jit_code_rshi: case jit_code_rshi_u: case jit_code_lroti:
1559 case jit_code_rroti: case jit_code_lti:
4a71579b
PC		 1560 case jit_code_lti_u: case jit_code_lei: case jit_code_lei_u:
1561 case jit_code_eqi: case jit_code_gei: case jit_code_gei_u:
1562 case jit_code_gti: case jit_code_gti_u: case jit_code_nei:
1563 case jit_code_ldxi_c: case jit_code_ldxi_uc: case jit_code_ldxi_s:
1564 case jit_code_ldxi_us: case jit_code_ldxi_i: case jit_code_ldxi_ui:
1565 case jit_code_ldxi_l: case jit_code_ldxi_f: case jit_code_ldxi_d:
ba86ff93 1566 case jit_code_unldr: case jit_code_unldr_u: case jit_code_unldr_x:
4a71579b
PC		 1567 mask = jit_cc_a0_reg|jit_cc_a0_chg|jit_cc_a1_reg|jit_cc_a2_int;
1568 break;
1569 case jit_code_qmuli: case jit_code_qmuli_u:
1570 case jit_code_qdivi: case jit_code_qdivi_u:
ba86ff93
PC		 1571 case jit_code_qlshi: case jit_code_qlshi_u:
1572 case jit_code_qrshi: case jit_code_qrshi_u:
4a71579b
PC		 1573 mask = jit_cc_a0_reg|jit_cc_a0_rlh|jit_cc_a0_chg|
1574 jit_cc_a1_reg|jit_cc_a2_int;
1575 break;
1576 case jit_code_addi_f: case jit_code_subi_f: case jit_code_rsbi_f:
1577 case jit_code_muli_f: case jit_code_divi_f: case jit_code_lti_f:
1578 case jit_code_lei_f: case jit_code_eqi_f: case jit_code_gei_f:
1579 case jit_code_gti_f: case jit_code_nei_f: case jit_code_unlti_f:
1580 case jit_code_unlei_f: case jit_code_uneqi_f: case jit_code_ungei_f:
1581 case jit_code_ungti_f: case jit_code_ltgti_f: case jit_code_ordi_f:
1582 case jit_code_unordi_f:
1583 mask = jit_cc_a0_reg|jit_cc_a0_chg|jit_cc_a1_reg|jit_cc_a2_flt;
1584 break;
1585 case jit_code_addi_d: case jit_code_subi_d: case jit_code_rsbi_d:
1586 case jit_code_muli_d: case jit_code_divi_d: case jit_code_lti_d:
1587 case jit_code_lei_d: case jit_code_eqi_d: case jit_code_gei_d:
1588 case jit_code_gti_d: case jit_code_nei_d: case jit_code_unlti_d:
1589 case jit_code_unlei_d: case jit_code_uneqi_d: case jit_code_ungei_d:
1590 case jit_code_ungti_d: case jit_code_ltgti_d: case jit_code_ordi_d:
1591 case jit_code_unordi_d:
1592 mask = jit_cc_a0_reg|jit_cc_a0_chg|jit_cc_a1_reg|jit_cc_a2_dbl;
1593 break;
1594 case jit_code_addr: case jit_code_addxr: case jit_code_addcr:
1595 case jit_code_subr: case jit_code_subxr: case jit_code_subcr:
ba86ff93
PC		 1596 case jit_code_mulr: case jit_code_hmulr: case jit_code_hmulr_u:
1597 case jit_code_divr: case jit_code_divr_u:
4a71579b
PC		 1598 case jit_code_remr: case jit_code_remr_u: case jit_code_andr:
1599 case jit_code_orr: case jit_code_xorr: case jit_code_lshr:
ba86ff93
PC		 1600 case jit_code_rshr: case jit_code_rshr_u: case jit_code_lrotr:
1601 case jit_code_rrotr: case jit_code_ltr:
4a71579b
PC		 1602 case jit_code_ltr_u: case jit_code_ler: case jit_code_ler_u:
1603 case jit_code_eqr: case jit_code_ger: case jit_code_ger_u:
1604 case jit_code_gtr: case jit_code_gtr_u: case jit_code_ner:
1605 case jit_code_ldxr_c: case jit_code_ldxr_uc: case jit_code_ldxr_s:
1606 case jit_code_ldxr_us: case jit_code_ldxr_i: case jit_code_ldxr_ui:
1607 case jit_code_ldxr_l: case jit_code_addr_f: case jit_code_subr_f:
1608 case jit_code_mulr_f: case jit_code_divr_f: case jit_code_ltr_f:
1609 case jit_code_ler_f: case jit_code_eqr_f: case jit_code_ger_f:
1610 case jit_code_gtr_f: case jit_code_ner_f: case jit_code_unltr_f:
1611 case jit_code_unler_f: case jit_code_uneqr_f: case jit_code_unger_f:
1612 case jit_code_ungtr_f: case jit_code_ltgtr_f: case jit_code_ordr_f:
1613 case jit_code_unordr_f: case jit_code_ldxr_f: case jit_code_addr_d:
1614 case jit_code_subr_d: case jit_code_mulr_d: case jit_code_divr_d:
1615 case jit_code_ltr_d: case jit_code_ler_d: case jit_code_eqr_d:
1616 case jit_code_ger_d: case jit_code_gtr_d: case jit_code_ner_d:
1617 case jit_code_unltr_d: case jit_code_unler_d: case jit_code_uneqr_d:
1618 case jit_code_unger_d: case jit_code_ungtr_d: case jit_code_ltgtr_d:
1619 case jit_code_ordr_d: case jit_code_unordr_d: case jit_code_ldxr_d:
1620 case jit_code_movr_ww_d:
1621 mask = jit_cc_a0_reg|jit_cc_a0_chg|jit_cc_a1_reg|jit_cc_a2_reg;
1622 break;
1623 case jit_code_qmulr: case jit_code_qmulr_u:
1624 case jit_code_qdivr: case jit_code_qdivr_u:
ba86ff93
PC		 1625 case jit_code_qlshr: case jit_code_qlshr_u:
1626 case jit_code_qrshr: case jit_code_qrshr_u:
4a71579b
PC		 1627 mask = jit_cc_a0_reg|jit_cc_a0_rlh|jit_cc_a0_chg|
1628 jit_cc_a1_reg|jit_cc_a2_reg;
1629 break;
1630 case jit_code_sti_c: case jit_code_sti_s: case jit_code_sti_i:
1631 case jit_code_sti_l: case jit_code_sti_f: case jit_code_sti_d:
1632 mask = jit_cc_a0_int|jit_cc_a1_reg;
1633 break;
ba86ff93
PC		 1634 case jit_code_unsti: case jit_code_unsti_x:
1635 mask = jit_cc_a0_int|jit_cc_a1_reg|jit_cc_a2_int;
1636 break;
4a71579b
PC		 1637 case jit_code_blti: case jit_code_blti_u: case jit_code_blei:
1638 case jit_code_blei_u: case jit_code_beqi: case jit_code_bgei:
1639 case jit_code_bgei_u: case jit_code_bgti: case jit_code_bgti_u:
1640 case jit_code_bnei: case jit_code_bmsi: case jit_code_bmci:
1641 mask = jit_cc_a0_jmp|jit_cc_a1_reg|jit_cc_a2_int;
1642 break;
1643 case jit_code_blti_f: case jit_code_blei_f: case jit_code_beqi_f:
1644 case jit_code_bgei_f: case jit_code_bgti_f: case jit_code_bnei_f:
1645 case jit_code_bunlti_f: case jit_code_bunlei_f: case jit_code_buneqi_f:
1646 case jit_code_bungei_f: case jit_code_bungti_f: case jit_code_bltgti_f:
1647 case jit_code_bordi_f: case jit_code_bunordi_f:
1648 mask = jit_cc_a0_jmp|jit_cc_a1_reg|jit_cc_a2_flt;
1649 break;
1650 case jit_code_blti_d: case jit_code_blei_d: case jit_code_beqi_d:
1651 case jit_code_bgei_d: case jit_code_bgti_d: case jit_code_bnei_d:
1652 case jit_code_bunlti_d: case jit_code_bunlei_d: case jit_code_buneqi_d:
1653 case jit_code_bungei_d: case jit_code_bungti_d: case jit_code_bltgti_d:
1654 case jit_code_bordi_d: case jit_code_bunordi_d:
1655 mask = jit_cc_a0_jmp|jit_cc_a1_reg|jit_cc_a2_dbl;
1656 break;
1657 case jit_code_allocar: /* synthesized instructions make it
1658 * equivalent to jit_cc_a0_chg */
1659 case jit_code_str_c: case jit_code_str_s: case jit_code_str_i:
1660 case jit_code_str_l: case jit_code_str_f: case jit_code_str_d:
1661 mask = jit_cc_a0_reg|jit_cc_a1_reg;
1662 break;
ba86ff93
PC		 1663 case jit_code_unstr: case jit_code_unstr_x:
1664 mask = jit_cc_a0_reg|jit_cc_a1_reg|jit_cc_a2_int;
1665 break;
4a71579b
PC		 1666 case jit_code_stxi_c: case jit_code_stxi_s: case jit_code_stxi_i:
1667 case jit_code_stxi_l: case jit_code_stxi_f: case jit_code_stxi_d:
1668 mask = jit_cc_a0_int|jit_cc_a1_reg|jit_cc_a2_reg;
1669 break;
1670 case jit_code_bltr: case jit_code_bltr_u: case jit_code_bler:
1671 case jit_code_bler_u: case jit_code_beqr: case jit_code_bger:
1672 case jit_code_bger_u: case jit_code_bgtr: case jit_code_bgtr_u:
1673 case jit_code_bner: case jit_code_bmsr: case jit_code_bmcr:
1674 case jit_code_bltr_f: case jit_code_bler_f: case jit_code_beqr_f:
1675 case jit_code_bger_f: case jit_code_bgtr_f: case jit_code_bner_f:
1676 case jit_code_bunltr_f: case jit_code_bunler_f: case jit_code_buneqr_f:
1677 case jit_code_bunger_f: case jit_code_bungtr_f: case jit_code_bltgtr_f:
1678 case jit_code_bordr_f: case jit_code_bunordr_f:case jit_code_bltr_d:
1679 case jit_code_bler_d: case jit_code_beqr_d: case jit_code_bger_d:
1680 case jit_code_bgtr_d: case jit_code_bner_d: case jit_code_bunltr_d:
1681 case jit_code_bunler_d: case jit_code_buneqr_d: case jit_code_bunger_d:
1682 case jit_code_bungtr_d: case jit_code_bltgtr_d: case jit_code_bordr_d:
1683 case jit_code_bunordr_d:
1684 mask = jit_cc_a0_jmp|jit_cc_a1_reg|jit_cc_a2_reg;
1685 break;
1686 case jit_code_boaddi: case jit_code_boaddi_u: case jit_code_bxaddi:
1687 case jit_code_bxaddi_u: case jit_code_bosubi: case jit_code_bosubi_u:
1688 case jit_code_bxsubi: case jit_code_bxsubi_u:
1689 mask = jit_cc_a0_jmp|jit_cc_a1_reg|jit_cc_a1_chg|jit_cc_a2_int;
1690 break;
1691 case jit_code_stxr_c: case jit_code_stxr_s: case jit_code_stxr_i:
1692 case jit_code_stxr_l: case jit_code_stxr_f: case jit_code_stxr_d:
1693 mask = jit_cc_a0_reg|jit_cc_a1_reg|jit_cc_a2_reg;
1694 break;
1695 case jit_code_boaddr: case jit_code_boaddr_u: case jit_code_bxaddr:
1696 case jit_code_bxaddr_u: case jit_code_bosubr: case jit_code_bosubr_u:
1697 case jit_code_bxsubr: case jit_code_bxsubr_u:
1698 mask = jit_cc_a0_jmp|jit_cc_a1_reg|jit_cc_a1_chg|jit_cc_a2_reg;
1699 break;
40a44dcb
PC		 1700 case jit_code_movnr: case jit_code_movzr:
1701 mask = jit_cc_a0_reg|jit_cc_a0_cnd|jit_cc_a1_reg|jit_cc_a2_reg;
1702 break;
ba3814c1
PC		 1703 case jit_code_casr:
1704 mask = jit_cc_a0_reg|jit_cc_a0_chg|jit_cc_a1_reg|
1705 jit_cc_a2_reg|jit_cc_a2_rlh;
1706 break;
1707 case jit_code_casi:
1708 mask = jit_cc_a0_reg|jit_cc_a0_chg|jit_cc_a1_int|
1709 jit_cc_a2_reg|jit_cc_a2_rlh;
1710 break;
ba86ff93
PC		 1711 case jit_code_extr: case jit_code_extr_u: case jit_code_depr:
1712 mask = jit_cc_a0_reg|jit_cc_a0_chg|jit_cc_a1_reg|jit_cc_a2_rlh;
1713 break;
1714 case jit_code_exti: case jit_code_exti_u: case jit_code_depi:
1715 mask = jit_cc_a0_reg|jit_cc_a0_chg|jit_cc_a1_int|jit_cc_a2_rlh;
1716 break;
1717 case jit_code_fmar_f: case jit_code_fmar_d:
1718 case jit_code_fmsr_f: case jit_code_fmsr_d:
1719 case jit_code_fnmar_f: case jit_code_fnmar_d:
1720 case jit_code_fnmsr_f: case jit_code_fnmsr_d:
1721 mask = jit_cc_a0_reg|jit_cc_a0_chg|
1722 jit_cc_a1_reg|jit_cc_a1_rlh|jit_cc_a2_reg;
1723 break;
1724 case jit_code_fmai_f: case jit_code_fmsi_f:
1725 case jit_code_fnmai_f: case jit_code_fnmsi_f:
1726 mask = jit_cc_a0_reg|jit_cc_a0_chg|
1727 jit_cc_a1_reg|jit_cc_a1_rlh|jit_cc_a2_flt;
1728 break;
1729 case jit_code_fmai_d: case jit_code_fmsi_d:
1730 case jit_code_fnmai_d: case jit_code_fnmsi_d:
1731 mask = jit_cc_a0_reg|jit_cc_a0_chg|
1732 jit_cc_a1_reg|jit_cc_a1_rlh|jit_cc_a2_dbl;
1733 break;
4a71579b
PC		 1734 default:
1735 abort();
1736 }
1737
1738 return (mask);
1739}
1740
1741void
1742_jit_patch_abs(jit_state_t *_jit, jit_node_t *instr, jit_pointer_t address)
1743{
d481fb64 1744#ifndef NDEBUG
4a71579b 1745 jit_int32_t mask;
d481fb64 1746#endif
4a71579b
PC		 1747 switch (instr->code) {
1748 case jit_code_movi: case jit_code_ldi_c: case jit_code_ldi_uc:
1749 case jit_code_ldi_s: case jit_code_ldi_us: case jit_code_ldi_i:
1750 case jit_code_ldi_ui: case jit_code_ldi_l: case jit_code_ldi_f:
1751 case jit_code_ldi_d:
1752 instr->v.p = address;
1753 break;
1754 case jit_code_sti_c: case jit_code_sti_s: case jit_code_sti_i:
1755 case jit_code_sti_l: case jit_code_sti_f: case jit_code_sti_d:
1756 instr->u.p = address;
1757 break;
1758 default:
d481fb64 1759#ifndef NDEBUG
4a71579b 1760 mask = jit_classify(instr->code);
d481fb64 1761#endif
4a71579b
PC		 1762 assert((mask & (jit_cc_a0_reg|jit_cc_a0_jmp)) == jit_cc_a0_jmp);
1763 instr->u.p = address;
1764 }
1765}
1766
1767void
1768_jit_patch_at(jit_state_t *_jit, jit_node_t *instr, jit_node_t *label)
1769{
d481fb64 1770#ifndef NDEBUG
4a71579b 1771 jit_int32_t mask;
d481fb64 1772#endif
4a71579b
PC		 1773 assert(!(instr->flag & jit_flag_node));
1774 instr->flag |= jit_flag_node;
1775 switch (instr->code) {
1776 case jit_code_movi:
1777 assert(label->code == jit_code_label ||
1778 label->code == jit_code_data);
1779 instr->v.n = label;
1780 if (label->code == jit_code_data)
1781 instr->flag |= jit_flag_data;
1782 break;
1783 case jit_code_jmpi:
1784 assert(label->code == jit_code_label ||
1785 label->code == jit_code_epilog);
1786 instr->u.n = label;
1787 break;
1788 default:
d481fb64 1789#ifndef NDEBUG
4a71579b 1790 mask = jit_classify(instr->code);
d481fb64 1791#endif
4a71579b
PC		 1792 assert((mask & (jit_cc_a0_reg|jit_cc_a0_jmp)) == jit_cc_a0_jmp);
1793 assert(label->code == jit_code_label);
1794 instr->u.n = label;
1795 break;
1796 }
1797 /* link field is used as list of nodes associated with a given label */
1798 instr->link = label->link;
1799 label->link = instr;
1800}
1801
c0c16242
PC		 1802static void
1803_do_setup(jit_state_t *_jit)
4a71579b 1804{
4a71579b
PC		 1805 jit_block_t *block;
1806 jit_word_t offset;
1807
4a71579b
PC		 1808 /* create initial mapping of live register values
1809 * at the start of a basic block */
1810 for (offset = 0; offset < _jitc->blocks.offset; offset++) {
1811 block = _jitc->blocks.ptr + offset;
79bfeef6 1812 if (!block->label)
4a71579b 1813 continue;
79bfeef6
PC		 1814 if (block->label->code == jit_code_epilog) {
1815 jit_regset_setbit(&block->reglive, JIT_RET);
1816 jit_regset_setbit(&block->reglive, JIT_FRET);
1817 jit_regset_com(&block->regmask, &block->reglive);
1818 continue;
1819 }
c0c16242 1820 jit_setup(block);
4a71579b 1821 }
c0c16242 1822}
4a71579b 1823
c0c16242
PC		 1824static jit_bool_t
1825_block_update_set(jit_state_t *_jit,
1826 jit_block_t *block, jit_block_t *target)
1827{
1828 jit_regset_t regmask;
1829
1830 jit_regset_ior(&regmask, &block->reglive, &target->reglive);
1831 jit_regset_and(&regmask, &regmask, &block->regmask);
1832 if (jit_regset_set_p(&regmask)) {
1833 jit_regset_ior(&block->reglive, &block->reglive, &regmask);
1834 jit_regset_and(&regmask, &block->reglive, &block->regmask);
1835 jit_regset_com(&regmask, &regmask);
1836 jit_regset_and(&block->regmask, &block->regmask, &regmask);
1837 block->again = 1;
1838 return (1);
1839 }
1840 return (0);
1841}
4a71579b 1842
24d91c0d
PC		 1843static void
1844_propagate_backward(jit_state_t *_jit, jit_block_t *block)
1845{
1846 jit_block_t *prev;
1847 jit_word_t offset;
1848
1849 for (offset = block->label->v.w - 1;
1850 offset >= 0; --offset) {
1851 prev = _jitc->blocks.ptr + offset;
1852 if (!block_update_set(prev, block) ||
1853 !(prev->label->flag & jit_flag_head))
1854 break;
1855 }
1856}
1857
c0c16242
PC		 1858static jit_bool_t
1859_check_block_again(jit_state_t *_jit)
1860{
1861 jit_int32_t todo;
1862 jit_word_t offset;
1863 jit_node_t *node, *label;
1864 jit_block_t *block, *target;
4a71579b 1865
c0c16242
PC		 1866 todo = 0;
1867 for (offset = 0; offset < _jitc->blocks.offset; offset++) {
1868 block = _jitc->blocks.ptr + offset;
1869 if (block->again) {
ba3814c1 1870 todo = 1;
c0c16242 1871 break;
ba3814c1 1872 }
c0c16242
PC		 1873 }
1874 /* If no block changed state */
1875 if (!todo)
1876 return (0);
ba3814c1 1877
c0c16242
PC		 1878 do {
1879 todo = 0;
1880 block = NULL;
ba3814c1 1881 for (node = _jitc->head; node; node = node->next) {
c0c16242
PC		 1882 /* Special jumps that match jit_cc_a0_jmp */
1883 if (node->code == jit_code_calli || node->code == jit_code_callr)
1884 continue;
1885
1886 /* Remember current label */
1887 if (node->code == jit_code_label ||
1888 node->code == jit_code_prolog ||
1889 node->code == jit_code_epilog) {
1890
1891 /* If previous block does not pass through */
1892 if (!(node->flag & jit_flag_head))
1893 block = NULL;
1894
1895 target = _jitc->blocks.ptr + node->v.w;
24d91c0d
PC		 1896 if (block && target->again && block_update_set(block, target)) {
1897 propagate_backward(block);
ba3814c1 1898 todo = 1;
24d91c0d 1899 }
c0c16242 1900 block = target;
ba3814c1 1901 }
c0c16242
PC		 1902 /* If not the first jmpi */
1903 else if (block) {
1904 /* If a jump to dynamic address or if a jump to raw address */
1905 if (!(jit_classify(node->code) & jit_cc_a0_jmp) ||
1906 !(node->flag & jit_flag_node))
1907 continue;
1908 label = node->u.n;
1909 /* Mark predecessor needs updating due to target change */
1910 target = _jitc->blocks.ptr + label->v.w;
24d91c0d
PC		 1911 if (target->again && block_update_set(block, target)) {
1912 propagate_backward(block);
ba3814c1 1913 todo = 1;
24d91c0d 1914 }
ba3814c1
PC		 1915 }
1916 }
c0c16242
PC		 1917 }
1918 while (todo);
1919
79bfeef6 1920 return (todo);
c0c16242 1921}
ba3814c1 1922
c0c16242
PC		 1923static void
1924_do_follow(jit_state_t *_jit, jit_bool_t always)
1925{
1926 jit_block_t *block;
1927 jit_word_t offset;
1928
1929 /* set live state of registers not referenced in a block, but
1930 * referenced in a jump target or normal flow */
1931 for (offset = 0; offset < _jitc->blocks.offset; offset++) {
1932 block = _jitc->blocks.ptr + offset;
1933 if (!block->label || block->label->code == jit_code_epilog)
1934 continue;
1935 if (always || block->again) {
1936 block->again = 0;
1937 jit_follow(block);
1938 }
1939 }
1940}
1941
1942void
1943_jit_optimize(jit_state_t *_jit)
1944{
1945 jit_bool_t jump;
1946 jit_bool_t todo;
1947 jit_int32_t mask;
1948 jit_node_t *node;
1949 jit_block_t *block;
1950 jit_word_t offset;
79bfeef6 1951 jit_regset_t regmask;
c0c16242
PC		 1952
1953 todo = 0;
1954 _jitc->function = NULL;
1955
1956 thread_jumps();
1957 sequential_labels();
1958 split_branches();
1959 do_setup();
1960 do_follow(1);
1961
79bfeef6
PC		 1962 jit_regset_set_ui(&regmask, 0);
1963 for (offset = 0; offset < _jitc->reglen; offset++) {
1964 if ((jit_class(_rvs[offset].spec) & (jit_class_gpr|jit_class_fpr)) &&
1965 (jit_class(_rvs[offset].spec) & jit_class_sav) == jit_class_sav)
1966 jit_regset_setbit(&regmask, offset);
1967 }
1968
1969 /* Figure out labels that are only reached with a jump */
c0c16242
PC		 1970 jump = 1;
1971 for (node = _jitc->head; node; node = node->next) {
1972 switch (node->code) {
1973 case jit_code_label:
79bfeef6 1974 if (!jump) {
c0c16242 1975 node->flag |= jit_flag_head;
79bfeef6
PC		 1976 if (!node->link) {
1977 /* Block is dead code or only reachable with an
1978 * indirect jumps. In such condition, must assume
1979 * all callee save registers are live. */
1980 block = _jitc->blocks.ptr + node->v.w;
1981 jit_regset_ior(&block->reglive,
1982 &block->reglive, &regmask);
1983 /* Cleanup regmask */
1984 block_update_set(block, block);
1985 }
1986 }
c0c16242
PC		 1987 break;
1988 case jit_code_jmpi: case jit_code_jmpr:
1989 case jit_code_epilog:
1990 jump = 1;
1991 break;
1992 case jit_code_data: case jit_code_note:
1993 break;
1994 default:
1995 jump = 0;
1996 break;
1997 }
1998 }
1999
ba86ff93
PC		 2000 patch_registers();
2001 if (simplify())
2002 todo = 1;
2003
c0c16242
PC		 2004 for (node = _jitc->head; node; node = node->next) {
2005 mask = jit_classify(node->code);
2006 if (mask & jit_cc_a0_reg)
2007 node->u.w &= ~jit_regno_patch;
2008 if (mask & jit_cc_a1_reg)
2009 node->v.w &= ~jit_regno_patch;
2010 if (mask & jit_cc_a2_reg)
2011 node->w.w &= ~jit_regno_patch;
2012 if (node->code == jit_code_beqi) {
2013 if (redundant_store(node, 1)) {
2014 block = _jitc->blocks.ptr + ((jit_node_t *)node->u.n)->v.w;
2015 block->again = 1;
2016 todo = 1;
2017 }
2018 }
2019 else if (node->code == jit_code_bnei) {
2020 if (redundant_store(node, 0)) {
2021 block = _jitc->blocks.ptr + ((jit_node_t *)node->u.n)->v.w;
2022 block->again = 1;
2023 todo = 1;
2024 }
4a71579b
PC		 2025 }
2026 }
2027
c0c16242
PC		 2028 if (!todo)
2029 todo = check_block_again();
2030
2031 /* If instructions were removed or first pass did modify the entry
2032 * state of any block */
2033 if (todo) {
2034 do_setup();
2035 todo = 0;
2036 do {
2037 do_follow(0);
2038 /* If any block again has the entry state modified. */
2039 todo = check_block_again();
2040 } while (todo);
2041 }
2042
4a71579b
PC		 2043 for (node = _jitc->head; node; node = node->next) {
2044 mask = jit_classify(node->code);
2045 if (mask & jit_cc_a0_reg)
2046 node->u.w &= ~jit_regno_patch;
2047 if (mask & jit_cc_a1_reg)
2048 node->v.w &= ~jit_regno_patch;
2049 if (mask & jit_cc_a2_reg)
2050 node->w.w &= ~jit_regno_patch;
ba3814c1
PC		 2051 if (node->code == jit_code_prolog)
2052 _jitc->function = _jitc->functions.ptr + node->w.w;
2053 else if(node->code == jit_code_epilog)
2054 _jitc->function = NULL;
2055 else {
4a71579b 2056#if JIT_HASH_CONSTS
ba3814c1
PC		 2057 if (mask & jit_cc_a0_flt) {
2058 node->u.p = jit_data(&node->u.f, sizeof(jit_float32_t), 4);
2059 node->flag |= jit_flag_node | jit_flag_data;
2060 }
2061 else if (mask & jit_cc_a0_dbl) {
2062 node->u.p = jit_data(&node->u.d, sizeof(jit_float64_t), 8);
2063 node->flag |= jit_flag_node | jit_flag_data;
2064 }
2065 else if (mask & jit_cc_a1_flt) {
2066 node->v.p = jit_data(&node->v.f, sizeof(jit_float32_t), 4);
2067 node->flag |= jit_flag_node | jit_flag_data;
2068 }
2069 else if (mask & jit_cc_a1_dbl) {
2070 node->v.p = jit_data(&node->v.d, sizeof(jit_float64_t), 8);
2071 node->flag |= jit_flag_node | jit_flag_data;
2072 }
2073 else if (mask & jit_cc_a2_flt) {
2074 node->w.p = jit_data(&node->w.f, sizeof(jit_float32_t), 4);
2075 node->flag |= jit_flag_node | jit_flag_data;
2076 }
2077 else if (mask & jit_cc_a2_dbl) {
2078 node->w.p = jit_data(&node->w.d, sizeof(jit_float64_t), 8);
2079 node->flag |= jit_flag_node | jit_flag_data;
2080 }
4a71579b 2081#endif
ba3814c1
PC		 2082 if (_jitc->function) {
2083 if ((mask & (jit_cc_a0_reg|jit_cc_a0_chg)) ==
2084 (jit_cc_a0_reg|jit_cc_a0_chg)) {
2085 if (mask & jit_cc_a0_rlh) {
2086 jit_regset_setbit(&_jitc->function->regset,
2087 jit_regno(node->u.q.l));
4a71579b 2088 jit_regset_setbit(&_jitc->function->regset,
ba3814c1
PC		 2089 jit_regno(node->u.q.h));
2090 }
2091 else
4a71579b 2092 jit_regset_setbit(&_jitc->function->regset,
ba3814c1 2093 jit_regno(node->u.w));
4a71579b 2094 }
ba3814c1
PC		 2095 if ((mask & (jit_cc_a1_reg|jit_cc_a1_chg)) ==
2096 (jit_cc_a1_reg|jit_cc_a1_chg))
2097 jit_regset_setbit(&_jitc->function->regset,
2098 jit_regno(node->v.w));
2099 if ((mask & (jit_cc_a2_reg|jit_cc_a2_chg)) ==
2100 (jit_cc_a2_reg|jit_cc_a2_chg))
2101 jit_regset_setbit(&_jitc->function->regset,
2102 jit_regno(node->w.w));
2103 }
4a71579b
PC		 2104 }
2105 }
2106}
2107
2108void
2109_jit_reglive(jit_state_t *_jit, jit_node_t *node)
2110{
2111 jit_int32_t spec;
2112 jit_int32_t value;
2113 jit_block_t *block;
2114
2115 switch (node->code) {
2116 case jit_code_label: case jit_code_prolog: case jit_code_epilog:
2117 block = _jitc->blocks.ptr + node->v.w;
2118 jit_regset_set(&_jitc->reglive, &block->reglive);
79bfeef6
PC		 2119 jit_regset_set_ui(&_jitc->explive, 0);
2120 break;
2121 case jit_code_live:
2122 jit_regset_setbit(&_jitc->explive, node->u.w);
4a71579b
PC		 2123 break;
2124 case jit_code_callr:
2125 value = jit_regno(node->u.w);
2126 if (!(node->u.w & jit_regno_patch)) {
2127 jit_regset_setbit(&_jitc->reglive, value);
2128 }
2129 case jit_code_calli:
2130 for (value = 0; value < _jitc->reglen; value++) {
2131 spec = jit_class(_rvs[value].spec);
2132 if ((spec & jit_class_arg) && jit_regarg_p(node, value))
2133 jit_regset_setbit(&_jitc->reglive, value);
2134 else if (!(spec & jit_class_sav))
2135 jit_regset_clrbit(&_jitc->reglive, value);
2136 }
2137 break;
2138 default:
2139 value = jit_classify(node->code);
2140 if (value & jit_cc_a0_reg) {
2141 if (value & jit_cc_a0_rlh) {
2142 if (!(node->u.q.l & jit_regno_patch)) {
2143 if (value & jit_cc_a0_chg) {
2144 jit_regset_clrbit(&_jitc->reglive, node->u.q.l);
2145 jit_regset_setbit(&_jitc->regmask, node->u.q.l);
2146 }
2147 else
2148 jit_regset_setbit(&_jitc->reglive, node->u.q.l);
2149 }
2150 if (!(node->u.q.h & jit_regno_patch)) {
2151 if (value & jit_cc_a0_chg) {
2152 jit_regset_clrbit(&_jitc->reglive, node->u.q.h);
2153 jit_regset_setbit(&_jitc->regmask, node->u.q.h);
2154 }
2155 else
2156 jit_regset_setbit(&_jitc->reglive, node->u.q.h);
2157 }
2158 }
2159 else {
2160 if (!(node->u.w & jit_regno_patch)) {
2161 if (value & jit_cc_a0_chg) {
2162 jit_regset_clrbit(&_jitc->reglive, node->u.w);
2163 jit_regset_setbit(&_jitc->regmask, node->u.w);
2164 }
2165 else
2166 jit_regset_setbit(&_jitc->reglive, node->u.w);
2167 }
2168 }
2169 }
ba86ff93
PC		 2170 if (value & jit_cc_a1_reg) {
2171 if (value & jit_cc_a1_rlh) {
2172 /* Assume registers are not changed */
2173 if (!(node->v.q.l & jit_regno_patch))
2174 jit_regset_setbit(&_jitc->reglive, node->v.q.l);
2175 if (!(node->v.q.h & jit_regno_patch))
2176 jit_regset_setbit(&_jitc->reglive, node->v.q.h);
2177 }
2178 else if (!(node->v.w & jit_regno_patch)) {
2179 if (value & jit_cc_a1_chg) {
2180 jit_regset_clrbit(&_jitc->reglive, node->v.w);
2181 jit_regset_setbit(&_jitc->regmask, node->v.w);
2182 }
2183 else
2184 jit_regset_setbit(&_jitc->reglive, node->v.w);
4a71579b 2185 }
4a71579b 2186 }
ba3814c1
PC		 2187 if (value & jit_cc_a2_reg) {
2188 if (value & jit_cc_a2_rlh) {
2189 /* Assume registers are not changed */
2190 if (!(node->w.q.l & jit_regno_patch))
2191 jit_regset_setbit(&_jitc->reglive, node->w.q.l);
2192 if (!(node->w.q.h & jit_regno_patch))
2193 jit_regset_setbit(&_jitc->reglive, node->w.q.h);
2194 }
2195 else {
2196 if (!(node->w.w & jit_regno_patch)) {
2197 if (value & jit_cc_a2_chg) {
2198 jit_regset_clrbit(&_jitc->reglive, node->w.w);
2199 jit_regset_setbit(&_jitc->regmask, node->w.w);
2200 }
2201 else
2202 jit_regset_setbit(&_jitc->reglive, node->w.w);
2203 }
4a71579b 2204 }
4a71579b
PC		 2205 }
2206 if (jit_regset_set_p(&_jitc->regmask)) {
2207 jit_update(node->next, &_jitc->reglive, &_jitc->regmask);
2208 if (jit_regset_set_p(&_jitc->regmask)) {
2209 /* any unresolved live state is considered as live */
2210 jit_regset_ior(&_jitc->reglive,
2211 &_jitc->reglive, &_jitc->regmask);
2212 jit_regset_set_ui(&_jitc->regmask, 0);
2213 }
2214 }
2215 break;
2216 }
2217}
2218
2219void
2220_jit_regarg_set(jit_state_t *_jit, jit_node_t *node, jit_int32_t value)
2221{
2222#if GET_JIT_SIZE
2223 jit_size_prepare();
2224#endif
2225 if (value & jit_cc_a0_reg) {
2226 if (value & jit_cc_a0_rlh) {
2227 jit_regset_setbit(&_jitc->regarg, jit_regno(node->u.q.l));
2228 jit_regset_setbit(&_jitc->regarg, jit_regno(node->u.q.h));
2229 }
2230 else
2231 jit_regset_setbit(&_jitc->regarg, jit_regno(node->u.w));
2232 }
ba86ff93
PC		 2233 if (value & jit_cc_a1_reg) {
2234 if (value & jit_cc_a1_rlh) {
2235 jit_regset_setbit(&_jitc->regarg, jit_regno(node->v.q.l));
2236 jit_regset_setbit(&_jitc->regarg, jit_regno(node->v.q.h));
2237 }
2238 else
2239 jit_regset_setbit(&_jitc->regarg, jit_regno(node->v.w));
2240 }
ba3814c1
PC		 2241 if (value & jit_cc_a2_reg) {
2242 if (value & jit_cc_a2_rlh) {
2243 jit_regset_setbit(&_jitc->regarg, jit_regno(node->w.q.l));
2244 jit_regset_setbit(&_jitc->regarg, jit_regno(node->w.q.h));
2245 }
2246 else
2247 jit_regset_setbit(&_jitc->regarg, jit_regno(node->w.w));
2248 }
79bfeef6
PC		 2249 /* Prevent incorrect detection of running out of registers
2250 * if will need to patch jump, and all registers have been
2251 * used in the current block. */
2252 if (node->code == jit_code_jmpi && (node->flag & jit_flag_node)) {
2253 jit_node_t *label = node->u.n;
2254 jit_block_t *block = _jitc->blocks.ptr + label->v.w;
2255 jit_regset_set(&_jitc->reglive, &block->reglive);
2256 jit_regset_set(&_jitc->regmask, &block->regmask);
2257 if (jit_regset_set_p(&_jitc->explive)) {
2258 jit_regset_ior(&_jitc->reglive, &block->reglive, &_jitc->explive);
2259 jit_regset_xor(&_jitc->regmask, &_jitc->regmask, &_jitc->explive);
2260 }
2261 }
4a71579b
PC		 2262}
2263
2264void
2265_jit_regarg_clr(jit_state_t *_jit, jit_node_t *node, jit_int32_t value)
2266{
2267#if GET_JIT_SIZE
2268 jit_size_collect(node);
2269#endif
2270 if (value & jit_cc_a0_reg) {
2271 if (value & jit_cc_a0_rlh) {
2272 jit_regset_clrbit(&_jitc->regarg, jit_regno(node->u.q.l));
2273 jit_regset_clrbit(&_jitc->regarg, jit_regno(node->u.q.h));
2274 }
2275 else
2276 jit_regset_clrbit(&_jitc->regarg, jit_regno(node->u.w));
2277 }
ba86ff93
PC		 2278 if (value & jit_cc_a1_reg) {
2279 if (value & jit_cc_a1_rlh) {
2280 jit_regset_clrbit(&_jitc->regarg, jit_regno(node->v.q.l));
2281 jit_regset_clrbit(&_jitc->regarg, jit_regno(node->v.q.h));
2282 }
2283 else
2284 jit_regset_clrbit(&_jitc->regarg, jit_regno(node->v.w));
2285 }
ba3814c1
PC		 2286 if (value & jit_cc_a2_reg) {
2287 if (value & jit_cc_a2_rlh) {
2288 jit_regset_clrbit(&_jitc->regarg, jit_regno(node->w.q.l));
2289 jit_regset_clrbit(&_jitc->regarg, jit_regno(node->w.q.h));
2290 }
2291 else
2292 jit_regset_clrbit(&_jitc->regarg, jit_regno(node->w.w));
2293 }
4a71579b
PC		 2294}
2295
2296void
2297_jit_realize(jit_state_t *_jit)
2298{
2299 assert(!_jitc->realize);
2300 if (_jitc->function)
2301 jit_epilog();
2302 jit_optimize();
2303 _jitc->realize = 1;
2304
2305 /* ensure it is aligned */
2306 _jitc->data.offset = (_jitc->data.offset + 7) & -8;
2307
2308#if GET_JIT_SIZE
2309 /* Heuristic to guess code buffer size */
2310 _jitc->mult = 4;
2311 _jit->code.length = _jitc->pool.length * 1024 * _jitc->mult;
2312#else
2313 _jit->code.length = jit_get_size();
2314#endif
2315}
2316
2317void
2318_jit_dataset(jit_state_t *_jit)
2319{
2320 jit_uint8_t *ptr;
2321 jit_node_t *node;
2322 jit_word_t offset;
2323#if defined(__sgi)
2324 int mmap_fd;
2325#endif
2326
2327 assert(!_jitc->dataset);
40a44dcb
PC		 2328#if !HAVE_MMAP
2329 assert(_jit->user_data);
2330#else
4a71579b
PC		 2331 if (!_jit->user_data) {
2332
2333 /* create read only data buffer */
2334 _jit->data.length = (_jitc->data.offset +
2335 /* reserve space for annotations */
2336 _jitc->note.size + 4095) & -4096;
2337#if defined(__sgi)
2338 mmap_fd = open("/dev/zero", O_RDWR);
2339#endif
2340 _jit->data.ptr = mmap(NULL, _jit->data.length,
2341 PROT_READ | PROT_WRITE,
2342 MAP_PRIVATE | MAP_ANON, mmap_fd, 0);
2343 assert(_jit->data.ptr != MAP_FAILED);
2344#if defined(__sgi)
2345 close(mmap_fd);
2346#endif
2347 }
40a44dcb 2348#endif /* !HAVE_MMAP */
4a71579b
PC		 2349
2350 if (!_jitc->no_data)
2351 jit_memcpy(_jit->data.ptr, _jitc->data.ptr, _jitc->data.offset);
2352
2353 if (_jitc->no_note) {
2354 /* Space for one note is always allocated, so revert it here
2355 * if after jit_new_state was called, it is also requested to
2356 * not generate annotation information */
2357 _jit->note.length = 0;
2358 _jitc->note.size = 0;
2359 }
2360 else {
2361 _jitc->note.base = _jit->data.ptr;
2362 if (!_jitc->no_data)
2363 _jitc->note.base += _jitc->data.offset;
2364 memset(_jitc->note.base, 0, _jitc->note.size);
2365 }
2366
2367 if (_jit->user_data)
2368 /* Need the temporary hashed data until jit_emit is finished */
2369 ptr = _jitc->no_data ? _jitc->data.ptr : _jit->data.ptr;
2370 else {
2371 ptr = _jit->data.ptr;
2372 /* Temporary hashed data no longer required */
2373 jit_free((jit_pointer_t *)&_jitc->data.ptr);
2374 }
2375
2376 for (offset = 0; offset < _jitc->data.size; offset++) {
2377 for (node = _jitc->data.table[offset]; node; node = node->next) {
2378 node->flag |= jit_flag_patch;
2379 node->u.w = (jit_word_t)(ptr + node->u.w);
2380 }
2381 }
2382
2383 _jitc->dataset = 1;
2384}
2385
2386jit_pointer_t
2387_jit_get_code(jit_state_t *_jit, jit_word_t *length)
2388{
2389 assert(_jitc->realize);
2390 if (length) {
2391 if (_jitc->done)
2392 /* If code already generated, return exact size of code */
2393 *length = _jit->pc.uc - _jit->code.ptr;
2394 else
2395 /* Else return current size of the code buffer */
2396 *length = _jit->code.length;
2397 }
2398
2399 return (_jit->code.ptr);
2400}
2401
2402void
2403_jit_set_code(jit_state_t *_jit, jit_pointer_t ptr, jit_word_t length)
2404{
2405 assert(_jitc->realize);
2406 _jit->code.ptr = ptr;
2407 _jit->code.length = length;
2408 _jit->user_code = 1;
2409}
2410
2411jit_pointer_t
2412_jit_get_data(jit_state_t *_jit, jit_word_t *data_size, jit_word_t *note_size)
2413{
2414 assert(_jitc->realize);
2415 if (data_size)
2416 *data_size = _jitc->data.offset;
2417 if (note_size)
2418 *note_size = _jitc->note.size;
2419 return (_jit->data.ptr);
2420}
2421
2422void
2423_jit_set_data(jit_state_t *_jit, jit_pointer_t ptr,
2424 jit_word_t length, jit_word_t flags)
2425{
2426 assert(_jitc->realize);
2427 if (flags & JIT_DISABLE_DATA)
2428 _jitc->no_data = 1;
2429 else
2430 assert(length >= _jitc->data.offset);
2431 if (flags & JIT_DISABLE_NOTE)
2432 _jitc->no_note = 1;
2433 else {
2434 if (flags & JIT_DISABLE_DATA)
2435 assert(length >= _jitc->note.size);
2436 else
2437 assert(length >= _jitc->data.offset + _jitc->note.size);
2438 }
2439 _jit->data.ptr = ptr;
2440 _jit->data.length = length;
2441 _jit->user_data = 1;
2442}
2443
2444jit_pointer_t
2445_jit_emit(jit_state_t *_jit)
2446{
2447 jit_pointer_t code;
2448 jit_node_t *node;
2449 size_t length;
d481fb64 2450#ifndef NDEBUG
4a71579b 2451 int result;
d481fb64 2452#endif
4a71579b
PC		 2453#if defined(__sgi)
2454 int mmap_fd;
2455#endif
c0c16242 2456 int mmap_prot, mmap_flags;
4a71579b
PC		 2457
2458 if (!_jitc->realize)
2459 jit_realize();
2460
2461 if (!_jitc->dataset)
2462 jit_dataset();
2463
2464 _jitc->emit = 1;
2465
40a44dcb
PC		 2466#if !HAVE_MMAP
2467 assert(_jit->user_code);
2468#else
4a71579b 2469 if (!_jit->user_code) {
c0c16242 2470 mmap_prot = PROT_READ | PROT_WRITE;
79bfeef6 2471#if !(__OpenBSD__ || __APPLE__)
c0c16242
PC		 2472 mmap_prot |= PROT_EXEC;
2473#endif
2474#if __NetBSD__
2475 mmap_prot = PROT_MPROTECT(mmap_prot);
2476 mmap_flags = 0;
2477#else
2478 mmap_flags = MAP_PRIVATE;
2479#endif
2480 mmap_flags |= MAP_ANON;
4a71579b
PC		 2481#if defined(__sgi)
2482 mmap_fd = open("/dev/zero", O_RDWR);
2483#endif
2484 _jit->code.ptr = mmap(NULL, _jit->code.length,
c0c16242 2485 mmap_prot, mmap_flags, mmap_fd, 0);
4a71579b
PC		 2486 assert(_jit->code.ptr != MAP_FAILED);
2487 }
40a44dcb 2488#endif /* !HAVE_MMAP */
4a71579b
PC		 2489 _jitc->code.end = _jit->code.ptr + _jit->code.length -
2490 jit_get_max_instr();
2491 _jit->pc.uc = _jit->code.ptr;
2492
2493 for (;;) {
c0c16242 2494#if __NetBSD__
d481fb64
PC		 2495#ifndef NDEBUG
2496 result =
2497#endif
2498 mprotect(_jit->code.ptr, _jit->code.length, PROT_READ | PROT_WRITE);
c0c16242
PC		 2499 assert(result == 0);
2500#endif
4a71579b
PC		 2501 if ((code = emit_code()) == NULL) {
2502 _jitc->patches.offset = 0;
2503 for (node = _jitc->head; node; node = node->next) {
2504 if (node->link &&
2505 (node->code == jit_code_label ||
2506 node->code == jit_code_epilog))
2507 node->flag &= ~jit_flag_patch;
2508 }
40a44dcb
PC		 2509#if !HAVE_MMAP
2510 assert(_jit->user_code);
2511#else
4a71579b
PC		 2512 if (_jit->user_code)
2513 goto fail;
2514#if GET_JIT_SIZE
2515 ++_jitc->mult;
2516 length = _jitc->pool.length * 1024 * _jitc->mult;
2517#else
2518 /* Should only happen on very special cases */
2519 length = _jit->code.length + 4096;
2520#endif
2521
2522#if !HAVE_MREMAP
2523 munmap(_jit->code.ptr, _jit->code.length);
2524#endif
2525
2526#if HAVE_MREMAP
2527# if __NetBSD__
2528 _jit->code.ptr = mremap(_jit->code.ptr, _jit->code.length,
2529 _jit->code.ptr, length, 0);
2530# else
2531 _jit->code.ptr = mremap(_jit->code.ptr, _jit->code.length,
2532 length, MREMAP_MAYMOVE, NULL);
2533# endif
2534#else
2535 _jit->code.ptr = mmap(NULL, length,
79bfeef6 2536 mmap_prot, mmap_flags, mmap_fd, 0);
4a71579b
PC		 2537#endif
2538
2539 assert(_jit->code.ptr != MAP_FAILED);
2540 _jit->code.length = length;
2541 _jitc->code.end = _jit->code.ptr + _jit->code.length -
2542 jit_get_max_instr();
2543 _jit->pc.uc = _jit->code.ptr;
40a44dcb 2544#endif /* !HAVE_MMAP */
4a71579b
PC		 2545 }
2546 else
2547 break;
2548 }
2549
2550#if defined(__sgi)
2551 if (!_jit->user_code)
2552 close(mmap_fd);
2553#endif
2554
2555 _jitc->done = 1;
2556 if (!_jitc->no_note)
2557 jit_annotate();
2558
2559 if (_jit->user_data)
2560 jit_free((jit_pointer_t *)&_jitc->data.ptr);
40a44dcb 2561#if HAVE_MMAP
4a71579b 2562 else {
d481fb64
PC		 2563# ifndef NDEBUG
2564 result =
2565# endif
2566 mprotect(_jit->data.ptr, _jit->data.length, PROT_READ);
4a71579b
PC		 2567 assert(result == 0);
2568 }
2569 if (!_jit->user_code) {
ba86ff93 2570 _jit->code.protect = _jit->pc.uc - _jit->code.ptr;
c0c16242
PC		 2571# if __riscv && __WORDSIZE == 64
2572 /* FIXME should start adding consts at a page boundary */
ba86ff93 2573 _jit->code.protect -= _jitc->consts.hash.count * sizeof(jit_word_t);
c0c16242 2574# endif
d481fb64
PC		 2575# ifndef NDEBUG
2576 result =
2577 mprotect(_jit->code.ptr, _jit->code.protect, PROT_READ | PROT_EXEC);
2578# endif
4a71579b
PC		 2579 assert(result == 0);
2580 }
40a44dcb 2581#endif /* HAVE_MMAP */
4a71579b
PC		 2582
2583 return (_jit->code.ptr);
2584fail:
2585 return (NULL);
2586}
2587
79bfeef6
PC		 2588void
2589_jit_protect(jit_state_t *_jit)
2590{
2591#if !HAVE_MMAP
2592 assert (_jit->user_code);
2593#else
d481fb64 2594# ifndef NDEBUG
79bfeef6 2595 int result;
d481fb64 2596# endif
79bfeef6 2597 if (_jit->user_code) return;
d481fb64
PC		 2598# ifndef NDEBUG
2599 result =
2600#endif
2601 mprotect (_jit->code.ptr, _jit->code.protect, PROT_READ | PROT_EXEC);
79bfeef6
PC		 2602 assert (result == 0);
2603#endif
2604}
2605
2606void
2607_jit_unprotect(jit_state_t *_jit)
2608{
2609#if !HAVE_MMAP
2610 assert (_jit->user_code);
2611#else
d481fb64 2612# ifndef NDEBUG
79bfeef6 2613 int result;
d481fb64 2614# endif
79bfeef6 2615 if (_jit->user_code) return;
d481fb64
PC		 2616# ifndef NDEBUG
2617 result =
2618# endif
2619 mprotect (_jit->code.ptr, _jit->code.protect, PROT_READ | PROT_WRITE);
79bfeef6
PC		 2620 assert (result == 0);
2621#endif
2622}
2623
4a71579b
PC		 2624void
2625_jit_frame(jit_state_t *_jit, jit_int32_t frame)
2626{
2627 jit_trampoline(frame, 1);
2628}
2629
2630void
2631_jit_tramp(jit_state_t *_jit, jit_int32_t frame)
2632{
2633 jit_trampoline(frame, 0);
2634}
2635
2636void
2637_jit_trampoline(jit_state_t *_jit, jit_int32_t frame, jit_bool_t prolog)
2638{
2639 jit_int32_t regno;
2640
2641 /* Must be called after prolog, actually, just to simplify
2642 * tests and know there is a current function and that
2643 * _jitc->function->self.aoff is at the before any alloca value */
2644 assert(_jitc->tail && _jitc->tail->code == jit_code_prolog);
2645
2646 /* + 24 for 3 possible spilled temporaries (that could be a double) */
2647 frame += 24;
2648#if defined(__hppa__)
2649 frame += _jitc->function->self.aoff;
2650#else
2651 frame -= _jitc->function->self.aoff;
2652#endif
2653 _jitc->function->frame = frame;
2654 if (prolog)
2655 _jitc->function->define_frame = 1;
2656 else
2657 _jitc->function->assume_frame = 1;
2658 for (regno = 0; regno < _jitc->reglen; regno++)
2659 if (jit_class(_rvs[regno].spec) & jit_class_sav)
2660 jit_regset_setbit(&_jitc->function->regset, regno);
2661}
2662
2663/* Compute initial reglive and regmask set values of a basic block.
2664 * reglive is the set of known live registers
2665 * regmask is the set of registers not referenced in the block
2666 * Registers in regmask might be live.
2667 */
2668static void
2669_jit_setup(jit_state_t *_jit, jit_block_t *block)
2670{
2671 jit_node_t *node;
2672 jit_bool_t live;
2673 unsigned long value;
2674
2675 jit_regset_set_mask(&block->regmask, _jitc->reglen);
2676 for (value = 0; value < _jitc->reglen; ++value)
2677 if (!(jit_class(_rvs[value].spec) & (jit_class_gpr|jit_class_fpr)))
2678 jit_regset_clrbit(&block->regmask, value);
2679
2680 for (node = block->label->next; node; node = node->next) {
2681 switch (node->code) {
2682 case jit_code_label: case jit_code_prolog:
2683 case jit_code_epilog:
2684 return;
2685 default:
2686 /* Check argument registers in reverse order to properly
2687 * handle registers that are both, argument and result */
2688 value = jit_classify(node->code);
ba86ff93 2689 if (value & jit_cc_a2_reg) {
4a71579b 2690 live = !(value & jit_cc_a2_chg);
ba86ff93
PC		 2691 if (value & jit_cc_a2_rlh) {
2692 /* Assume will not modify a pair in second argument */
2693 assert(live);
2694 if (!(node->w.q.l & jit_regno_patch) &&
2695 jit_regset_tstbit(&block->regmask, node->w.q.l)) {
2696 jit_regset_clrbit(&block->regmask, node->w.q.l);
2697 }
2698 if (!(node->w.q.h & jit_regno_patch) &&
2699 jit_regset_tstbit(&block->regmask, node->w.q.h)) {
2700 jit_regset_clrbit(&block->regmask, node->w.q.h);
2701 }
2702 }
2703 else {
2704 if (!(node->w.w & jit_regno_patch) &&
2705 jit_regset_tstbit(&block->regmask, node->w.w)) {
2706 jit_regset_clrbit(&block->regmask, node->w.w);
2707 if (live)
2708 jit_regset_setbit(&block->reglive, node->w.w);
2709 }
2710 }
4a71579b 2711 }
ba86ff93 2712 if (value & jit_cc_a1_reg) {
4a71579b 2713 live = !(value & jit_cc_a1_chg);
ba86ff93
PC		 2714 if (value & jit_cc_a1_rlh) {
2715 /* Assume will not modify a pair in second argument */
2716 assert(live);
2717 if (!(node->v.q.l & jit_regno_patch) &&
2718 jit_regset_tstbit(&block->regmask, node->v.q.l)) {
2719 jit_regset_clrbit(&block->regmask, node->v.q.l);
2720 }
2721 if (!(node->v.q.h & jit_regno_patch) &&
2722 jit_regset_tstbit(&block->regmask, node->v.q.h)) {
2723 jit_regset_clrbit(&block->regmask, node->v.q.h);
2724 }
2725 }
2726 else {
2727 if (!(node->v.w & jit_regno_patch) &&
2728 jit_regset_tstbit(&block->regmask, node->v.w)) {
2729 jit_regset_clrbit(&block->regmask, node->v.w);
2730 if (live)
2731 jit_regset_setbit(&block->reglive, node->v.w);
2732 }
2733 }
4a71579b
PC		 2734 }
2735 if (value & jit_cc_a0_reg) {
2736 live = !(value & jit_cc_a0_chg);
2737 if (value & jit_cc_a0_rlh) {
2738 if (!(node->u.q.l & jit_regno_patch) &&
2739 jit_regset_tstbit(&block->regmask, node->u.q.l)) {
2740 jit_regset_clrbit(&block->regmask, node->u.q.l);
2741 if (live)
2742 jit_regset_setbit(&block->reglive, node->u.q.l);
2743 }
2744 if (!(node->u.q.h & jit_regno_patch) &&
2745 jit_regset_tstbit(&block->regmask, node->u.q.h)) {
2746 jit_regset_clrbit(&block->regmask, node->u.q.h);
2747 if (live)
2748 jit_regset_setbit(&block->reglive, node->u.q.h);
2749 }
2750 }
2751 else {
2752 if (!(node->u.w & jit_regno_patch) &&
2753 jit_regset_tstbit(&block->regmask, node->u.w)) {
2754 jit_regset_clrbit(&block->regmask, node->u.w);
2755 if (live)
2756 jit_regset_setbit(&block->reglive, node->u.w);
2757 }
2758 }
2759 }
2760 break;
2761 }
2762 }
2763}
2764
2765/* Update regmask and reglive of blocks at entry point of branch targets
2766 * or normal flow that have a live register not used in this block.
2767 */
2768static void
c0c16242 2769_jit_follow(jit_state_t *_jit, jit_block_t *block)
4a71579b
PC		 2770{
2771 jit_node_t *node;
2772 jit_block_t *next;
2773 jit_int32_t spec;
2774 jit_int32_t regno;
2775 unsigned long value;
2776 jit_node_t *label;
2777 jit_regset_t reglive;
2778 jit_regset_t regmask;
2779 jit_regset_t regtemp;
2780
2781 jit_regset_set(&reglive, &block->reglive);
2782 jit_regset_set(&regmask, &block->regmask);
2783 for (node = block->label->next; node; node = node->next) {
2784 switch (node->code) {
2785 case jit_code_label:
2786 /* Do not consider jmpi and jmpr cannot jump to the
2787 * next instruction. */
2788 next = _jitc->blocks.ptr + node->v.w;
2789 /* Set of live registers in next block that are at unknown
2790 * state in this block. */
2791 jit_regset_and(&regtemp, &regmask, &next->reglive);
2792 if (jit_regset_set_p(&regtemp)) {
2793 /* Add live state of next block to current block. */
2794 jit_regset_ior(&block->reglive, &block->reglive, &regtemp);
2795 /* Remove from unknown state bitmask. */
2796 jit_regset_com(&regtemp, &regtemp);
2797 jit_regset_and(&block->regmask, &block->regmask, &regtemp);
c0c16242 2798 block->again = 1;
4a71579b
PC		 2799 }
2800 case jit_code_prolog:
2801 case jit_code_epilog:
2802 return;
2803 case jit_code_callr:
2804 value = jit_regno(node->u.w);
2805 if (!(node->u.w & jit_regno_patch)) {
2806 if (jit_regset_tstbit(&regmask, value)) {
2807 jit_regset_clrbit(&regmask, value);
2808 jit_regset_setbit(&reglive, value);
2809 }
2810 }
2811 case jit_code_calli:
2812 for (value = 0; value < _jitc->reglen; ++value) {
2813 value = jit_regset_scan1(&regmask, value);
2814 if (value >= _jitc->reglen)
2815 break;
2816 spec = jit_class(_rvs[value].spec);
2817 if (!(spec & jit_class_sav))
2818 jit_regset_clrbit(&regmask, value);
2819 if ((spec & jit_class_arg) && jit_regarg_p(node, value))
2820 jit_regset_setbit(&reglive, value);
2821 }
2822 break;
2823 default:
2824 value = jit_classify(node->code);
2825 if (value & jit_cc_a2_reg) {
ba3814c1
PC		 2826 if (value & jit_cc_a2_rlh) {
2827 if (!(node->w.q.l & jit_regno_patch)) {
2828 /* Assume register is not changed */
2829 if (jit_regset_tstbit(&regmask, node->w.q.l))
2830 jit_regset_clrbit(&regmask, node->w.q.l);
2831 }
2832 if (!(node->w.q.h & jit_regno_patch)) {
2833 if (jit_regset_tstbit(&regmask, node->w.q.h))
2834 jit_regset_clrbit(&regmask, node->w.q.h);
2835 }
2836 }
2837 else {
ba86ff93
PC		 2838 if (!(node->w.w & jit_regno_patch)) {
2839 if (jit_regset_tstbit(&regmask, node->w.w)) {
2840 jit_regset_clrbit(&regmask, node->w.w);
2841 if (!(value & jit_cc_a2_chg))
2842 jit_regset_setbit(&reglive, node->w.w);
ba3814c1 2843 }
4a71579b
PC		 2844 }
2845 }
2846 }
2847 if (value & jit_cc_a1_reg) {
ba86ff93
PC		 2848 if (value & jit_cc_a1_rlh) {
2849 if (!(node->v.q.l & jit_regno_patch)) {
2850 /* Assume register is not changed */
2851 if (jit_regset_tstbit(&regmask, node->v.q.l))
2852 jit_regset_clrbit(&regmask, node->v.q.l);
2853 }
2854 if (!(node->v.q.h & jit_regno_patch)) {
2855 if (jit_regset_tstbit(&regmask, node->v.q.h))
2856 jit_regset_clrbit(&regmask, node->v.q.h);
2857 }
2858 }
2859 else {
2860 if (!(node->v.w & jit_regno_patch)) {
2861 if (jit_regset_tstbit(&regmask, node->v.w)) {
2862 jit_regset_clrbit(&regmask, node->v.w);
2863 if (!(value & jit_cc_a1_chg))
2864 jit_regset_setbit(&reglive, node->v.w);
2865 }
4a71579b
PC		 2866 }
2867 }
2868 }
2869 if (value & jit_cc_a0_reg) {
2870 if (value & jit_cc_a0_rlh) {
2871 if (!(node->u.q.l & jit_regno_patch)) {
2872 if (jit_regset_tstbit(&regmask, node->u.q.l)) {
2873 jit_regset_clrbit(&regmask, node->u.q.l);
2874 if (!(value & jit_cc_a0_chg))
2875 jit_regset_setbit(&reglive, node->u.q.l);
2876 }
2877 }
2878 if (!(node->u.q.h & jit_regno_patch)) {
2879 if (jit_regset_tstbit(&regmask, node->u.q.h)) {
2880 jit_regset_clrbit(&regmask, node->u.q.h);
2881 if (!(value & jit_cc_a0_chg))
2882 jit_regset_setbit(&reglive, node->u.q.h);
2883 }
2884 }
2885 }
2886 else {
2887 if (!(node->u.w & jit_regno_patch)) {
2888 if (jit_regset_tstbit(&regmask, node->u.w)) {
2889 jit_regset_clrbit(&regmask, node->u.w);
2890 if (!(value & jit_cc_a0_chg))
2891 jit_regset_setbit(&reglive, node->u.w);
2892 }
2893 }
2894 }
2895 }
2896 if (value & jit_cc_a0_jmp) {
2897 if (node->flag & jit_flag_node) {
2898 label = node->u.n;
2899 /* Do not consider jmpi and jmpr cannot jump to the
2900 * next instruction. */
2901 next = _jitc->blocks.ptr + label->v.w;
2902 jit_regset_and(&regtemp, &regmask, &next->reglive);
2903 if (jit_regset_set_p(&regtemp)) {
2904 /* Add live state. */
2905 jit_regset_ior(&block->reglive,
2906 &block->reglive, &regtemp);
2907 /* Remove from unknown state bitmask. */
2908 jit_regset_com(&regtemp, &regtemp);
2909 jit_regset_and(&block->regmask,
2910 &block->regmask, &regtemp);
c0c16242 2911 block->again = 1;
4a71579b
PC		 2912 }
2913 }
2914 else {
2915 /* Jump to unknown location.
2916 * This is a pitfall of the implementation.
2917 * Only jmpi to not a jit code should reach here,
2918 * or a jmpr of a computed address.
2919 * Because the implementation needs jit_class_nospill
2920 * registers, must treat jmpr as a function call. This
2921 * means that only JIT_Vn registers can be trusted on
2922 * arrival of jmpr.
2923 */
ba3814c1 2924 jit_regset_set_ui(&regmask, 0);
4a71579b
PC		 2925 for (regno = 0; regno < _jitc->reglen; regno++) {
2926 spec = jit_class(_rvs[regno].spec);
ba3814c1
PC		 2927 if ((spec & (jit_class_gpr|jit_class_fpr)) &&
2928 (spec & jit_class_sav))
2929 jit_regset_setbit(&regmask, regno);
4a71579b
PC		 2930 }
2931 /* Assume non callee save registers are live due
2932 * to jump to unknown location. */
2933 /* Treat all callee save as live. */
ba3814c1 2934 jit_regset_ior(&block->reglive, &reglive, &regmask);
4a71579b 2935 /* Treat anything else as dead. */
ba3814c1 2936 return;
4a71579b
PC		 2937 }
2938 }
2939 break;
2940 }
2941 }
2942}
2943
2944/* Follow code generation up to finding a label or end of code.
2945 * When finding a label, update the set of live registers.
2946 * On branches, update based on taken branch or normal flow.
2947 */
2948static void
2949_jit_update(jit_state_t *_jit, jit_node_t *node,
2950 jit_regset_t *live, jit_regset_t *mask)
2951{
2952 jit_int32_t spec;
2953 jit_int32_t regno;
2954 unsigned long value;
2955 jit_block_t *block;
2956 jit_node_t *label;
2957 jit_regset_t regtemp;
2958
2959 for (; node; node = node->next) {
2960 if (jit_regset_set_p(mask) == 0)
2961 break;
2962 switch (node->code) {
2963 case jit_code_label:
2964 block = _jitc->blocks.ptr + node->v.w;
2965 jit_regset_and(&regtemp, mask, &block->reglive);
2966 if (jit_regset_set_p(&regtemp)) {
2967 /* Add live state. */
2968 jit_regset_ior(live, live, &regtemp);
2969 /* Remove from unknown state bitmask. */
2970 jit_regset_com(&regtemp, &regtemp);
2971 jit_regset_and(mask, mask, &regtemp);
2972 }
2973 return;
2974 case jit_code_prolog:
2975 jit_regset_set_ui(mask, 0);
2976 return;
2977 case jit_code_epilog:
2978 jit_regset_set_ui(mask, 0);
2979 return;
2980 case jit_code_callr:
2981 value = jit_regno(node->u.w);
2982 if (!(node->u.w & jit_regno_patch)) {
2983 if (jit_regset_tstbit(mask, value)) {
2984 jit_regset_clrbit(mask, value);
2985 jit_regset_setbit(live, value);
2986 }
2987 }
2988 case jit_code_calli:
2989 for (value = 0; value < _jitc->reglen; ++value) {
2990 value = jit_regset_scan1(mask, value);
2991 if (value >= _jitc->reglen)
2992 break;
2993 spec = jit_class(_rvs[value].spec);
2994 if (!(spec & jit_class_sav))
2995 jit_regset_clrbit(mask, value);
2996 if ((spec & jit_class_arg) && jit_regarg_p(node, value))
2997 jit_regset_setbit(live, value);
2998 }
2999 break;
3000 default:
3001 value = jit_classify(node->code);
3002 if (value & jit_cc_a2_reg) {
ba3814c1
PC		 3003 if (value & jit_cc_a2_rlh) {
3004 if (!(node->w.q.l & jit_regno_patch)) {
3005 /* Assume register is not changed */
3006 if (jit_regset_tstbit(mask, node->w.q.l))
3007 jit_regset_clrbit(mask, node->w.q.l);
3008 }
3009 if (!(node->w.q.h & jit_regno_patch)) {
3010 if (jit_regset_tstbit(mask, node->w.q.h))
3011 jit_regset_clrbit(mask, node->w.q.h);
3012 }
3013 }
3014 else {
3015 if (!(node->w.w & jit_regno_patch)) {
3016 if (jit_regset_tstbit(mask, node->w.w)) {
3017 jit_regset_clrbit(mask, node->w.w);
3018 if (!(value & jit_cc_a2_chg))
3019 jit_regset_setbit(live, node->w.w);
3020 }
4a71579b
PC		 3021 }
3022 }
3023 }
3024 if (value & jit_cc_a1_reg) {
ba86ff93
PC		 3025 if (value & jit_cc_a1_rlh) {
3026 if (!(node->v.q.l & jit_regno_patch)) {
3027 /* Assume register is not changed */
3028 if (jit_regset_tstbit(mask, node->v.q.l))
3029 jit_regset_clrbit(mask, node->v.q.l);
3030 }
3031 if (!(node->v.q.h & jit_regno_patch)) {
3032 if (jit_regset_tstbit(mask, node->v.q.h))
3033 jit_regset_clrbit(mask, node->v.q.h);
3034 }
3035 }
3036 else {
3037 if (!(node->v.w & jit_regno_patch)) {
3038 if (jit_regset_tstbit(mask, node->v.w)) {
3039 jit_regset_clrbit(mask, node->v.w);
3040 if (!(value & jit_cc_a1_chg))
3041 jit_regset_setbit(live, node->v.w);
3042 }
4a71579b
PC		 3043 }
3044 }
3045 }
3046 if (value & jit_cc_a0_reg) {
3047 if (value & jit_cc_a0_rlh) {
3048 if (!(node->u.q.l & jit_regno_patch)) {
3049 if (jit_regset_tstbit(mask, node->u.q.l)) {
3050 jit_regset_clrbit(mask, node->u.q.l);
3051 if (!(value & jit_cc_a0_chg))
3052 jit_regset_setbit(live, node->u.q.l);
3053 }
3054 }
3055 if (!(node->u.q.h & jit_regno_patch)) {
3056 if (jit_regset_tstbit(mask, node->u.q.h)) {
3057 jit_regset_clrbit(mask, node->u.q.h);
3058 if (!(value & jit_cc_a0_chg))
3059 jit_regset_setbit(live, node->u.q.h);
3060 }
3061 }
3062 }
3063 else {
3064 if (!(node->u.w & jit_regno_patch)) {
3065 if (jit_regset_tstbit(mask, node->u.w)) {
3066 jit_regset_clrbit(mask, node->u.w);
3067 if (!(value & jit_cc_a0_chg))
3068 jit_regset_setbit(live, node->u.w);
3069 }
3070 }
3071 }
3072 }
3073 if (value & jit_cc_a0_jmp) {
3074 if (node->flag & jit_flag_node) {
3075 label = node->u.n;
3076 /* Do not consider jmpi and jmpr cannot jump to the
3077 * next instruction. */
3078 block = _jitc->blocks.ptr + label->v.w;
3079 jit_regset_and(&regtemp, mask, &block->reglive);
3080 if (jit_regset_set_p(&regtemp)) {
3081 /* Add live state. */
3082 jit_regset_ior(live, live, &regtemp);
3083 /* Remove from unknown state bitmask. */
3084 jit_regset_com(&regtemp, &regtemp);
3085 jit_regset_and(mask, mask, &regtemp);
3086 }
3087 }
3088 else {
3089 /* Jump to unknown location.
3090 * This is a pitfall of the implementation.
3091 * Only jmpi to not a jit code should reach here,
3092 * or a jmpr of a computed address.
3093 * Because the implementation needs jit_class_nospill
3094 * registers, must treat jmpr as a function call. This
3095 * means that only JIT_Vn registers can be trusted on
3096 * arrival of jmpr.
3097 */
ba3814c1 3098 jit_regset_set_ui(mask, 0);
4a71579b
PC		 3099 for (regno = 0; regno < _jitc->reglen; regno++) {
3100 spec = jit_class(_rvs[regno].spec);
ba3814c1
PC		 3101 if ((spec & (jit_class_gpr|jit_class_fpr)) &&
3102 (spec & jit_class_sav))
3103 jit_regset_setbit(mask, regno);
4a71579b
PC		 3104 }
3105 /* Assume non callee save registers are live due
3106 * to jump to unknown location. */
3107 /* Treat all callee save as live. */
3108 jit_regset_ior(live, live, mask);
79bfeef6
PC		 3109 /* Prevent explicitly set as live registers to
3110 * be used as a temporary for the jmpi. */
3111 jit_regset_ior(live, live, &_jitc->explive);
4a71579b 3112 /* Treat anything else as dead. */
ba3814c1 3113 return;
4a71579b
PC		 3114 }
3115 }
3116 break;
3117 }
3118 }
3119}
3120
3121static void
3122_thread_jumps(jit_state_t *_jit)
3123{
3124 jit_node_t *prev;
3125 jit_node_t *node;
3126 jit_node_t *next;
3127 jit_int32_t mask;
3128
3129 for (prev = node = _jitc->head; node;) {
3130 next = node->next;
3131 switch (node->code) {
3132 case jit_code_jmpi:
3133 if (redundant_jump(prev, node)) {
3134 node = prev;
3135 continue;
3136 }
3137 if (shortcut_jump(prev, node))
3138 continue;
3139 break;
3140 case jit_code_jmpr:
3141 case jit_code_callr: case jit_code_calli:
3142 /* non optimizable jump like code */
3143 break;
3144 default:
3145 mask = jit_classify(node->code);
3146 if (mask & jit_cc_a0_jmp) {
3147 if (reverse_jump(prev, node) ||
3148 shortcut_jump(prev, node))
3149 continue;
3150 }
3151 break;
3152 }
3153 prev = node;
3154 node = next;
3155 }
3156}
3157
3158static void
3159_sequential_labels(jit_state_t *_jit)
3160{
3161 jit_node_t *jump;
3162 jit_node_t *link;
3163 jit_node_t *prev;
3164 jit_node_t *next;
3165 jit_node_t *node;
3166
3167 for (prev = node = _jitc->head; node; node = next) {
3168 next = node->next;
3169 if (node->code == jit_code_label) {
3170 if (!node->flag) {
3171 if (!node->link) {
3172 del_label(prev, node);
3173 continue;
3174 }
3175 if (prev != node && prev->code == jit_code_label) {
3176 if ((jump = node->link)) {
3177 for (; jump; jump = link) {
3178 link = jump->link;
79bfeef6
PC		 3179 if (jump->code == jit_code_movi)
3180 jump->v.n = prev;
3181 else
3182 jump->u.n = prev;
4a71579b
PC		 3183 jump->link = prev->link;
3184 prev->link = jump;
3185 }
3186 node->link = NULL;
3187 }
3188 del_label(prev, node);
3189 continue;
3190 }
3191 }
3192 if (next && next->code == jit_code_label && !next->flag) {
3193 if ((jump = next->link)) {
3194 for (; jump; jump = link) {
3195 link = jump->link;
79bfeef6
PC		 3196 if (jump->code == jit_code_movi)
3197 jump->v.n = node;
3198 else
3199 jump->u.n = node;
4a71579b
PC		 3200 jump->link = node->link;
3201 node->link = jump;
3202 }
3203 next->link = NULL;
3204 }
3205 del_label(node, next);
3206 next = node->next;
3207 continue;
3208 }
3209 }
3210 prev = node;
3211 }
3212}
3213
3214static void
3215_split_branches(jit_state_t *_jit)
3216{
3217 jit_node_t *node;
3218 jit_node_t *next;
3219 jit_node_t *label;
3220 jit_block_t *block;
24d91c0d
PC		 3221 jit_block_t *blocks;
3222 jit_word_t offset;
3223 jit_word_t length;
4a71579b 3224
24d91c0d
PC		 3225 length = _jitc->blocks.length;
3226 jit_alloc((jit_pointer_t *)&blocks, length * sizeof(jit_block_t));
3227 if ((node = _jitc->head) &&
3228 (node->code == jit_code_label || node->code == jit_code_prolog)) {
3229 block = _jitc->blocks.ptr + node->v.w;
3230 memcpy(blocks, block, sizeof(jit_block_t));
3231 node->v.w = 0;
3232 offset = 1;
3233 }
3234 else
3235 offset = 0;
4a71579b
PC		 3236 for (node = _jitc->head; node; node = next) {
3237 if ((next = node->next)) {
3238 if (next->code == jit_code_label ||
3239 next->code == jit_code_prolog ||
24d91c0d
PC		 3240 next->code == jit_code_epilog) {
3241 if (offset >= length) {
3242 jit_realloc((jit_pointer_t *)&blocks,
3243 length * sizeof(jit_block_t),
3244 (length + 16) * sizeof(jit_block_t));
3245 length += 16;
3246 }
3247 block = _jitc->blocks.ptr + next->v.w;
3248 memcpy(blocks + offset, block, sizeof(jit_block_t));
3249 next->v.w = offset++;
3250 }
4a71579b 3251 /* split block on branches */
24d91c0d 3252 else if (jit_classify(node->code) & jit_cc_a0_jmp) {
4a71579b
PC		 3253 label = new_node(jit_code_label);
3254 label->next = next;
3255 node->next = label;
24d91c0d
PC		 3256 if (offset >= length) {
3257 jit_realloc((jit_pointer_t *)&blocks,
3258 length * sizeof(jit_block_t),
3259 (length + 16) * sizeof(jit_block_t));
3260 length += 16;
4a71579b 3261 }
24d91c0d 3262 block = blocks + offset;
4a71579b 3263 block->label = label;
24d91c0d 3264 label->v.w = offset++;
4a71579b
PC		 3265 jit_regset_new(&block->reglive);
3266 jit_regset_new(&block->regmask);
4a71579b
PC		 3267 }
3268 }
3269 }
24d91c0d
PC		 3270 jit_free((jit_pointer_t *)&_jitc->blocks.ptr);
3271 _jitc->blocks.ptr = blocks;
3272 _jitc->blocks.offset = offset;
3273 _jitc->blocks.length = length;
4a71579b
PC		 3274}
3275
3276static jit_bool_t
3277_shortcut_jump(jit_state_t *_jit, jit_node_t *prev, jit_node_t *node)
3278{
3279 jit_bool_t cond;
3280 jit_node_t *jump;
3281 jit_node_t *next;
3282 jit_node_t *temp;
3283
3284 if (!(node->flag & jit_flag_node))
3285 return (0);
3286 assert(node->code != jit_code_jmpr);
3287 cond = node->code != jit_code_jmpi;
3288 jump = node->u.n;
3289 for (next = jump->next; next; next = next->next) {
3290 switch (next->code) {
3291 case jit_code_jmpi:
3292 if (!(next->flag & jit_flag_node))
3293 return (0);
3294 if (jump->link == node)
3295 jump->link = node->link;
3296 else {
3297 for (temp = jump->link;
3298 temp->link != node;
3299 temp = temp->link)
3300 assert(temp != NULL);
3301 temp->link = node->link;
3302 }
3303 jump = next->u.n;
3304 node->u.n = jump;
3305 node->link = jump->link;
3306 jump->link = node;
3307 return (1);
3308 case jit_code_jmpr:
3309 if (cond)
3310 return (0);
3311 node->code = jit_code_jmpr;
3312 node->u.w = next->u.w;
3313 node->link = NULL;
3314 node->flag &= ~jit_flag_node;
3315 return (1);
3316 case jit_code_note: case jit_code_label:
3317 break;
3318 default:
3319 return (0);
3320 }
3321 }
3322 return (0);
3323}
3324
3325static jit_bool_t
3326_redundant_jump(jit_state_t *_jit, jit_node_t *prev, jit_node_t *node)
3327{
3328 jit_node_t *local_prev;
3329 jit_node_t *local_next;
3330
3331 if (!(node->flag & jit_flag_node))
3332 return (0);
3333 for (local_prev = node, local_next = node->next;
3334 local_next;
3335 local_prev = local_next, local_next = local_next->next) {
3336
3337 switch (local_next->code) {
3338 case jit_code_label: case jit_code_epilog:
3339 if (node->u.n == local_next) {
3340 if (local_next->link == node)
3341 local_next->link = node->link;
3342 else {
3343 for (local_prev = local_next->link;
3344 local_prev->link != node;
3345 local_prev = local_prev->link)
3346 assert(local_prev != NULL);
3347 local_prev->link = node->link;
3348 }
3349 del_node(prev, node);
3350 return (1);
3351 }
3352 break;
3353 case jit_code_name: case jit_code_note:
4a71579b
PC		 3354 break;
3355 default:
3356 return (0);
3357 }
3358 }
3359 return (0);
3360}
3361
3362static jit_code_t
3363reverse_jump_code(jit_code_t code)
3364{
3365 switch (code) {
3366 case jit_code_bltr: return (jit_code_bger);
3367 case jit_code_blti: return (jit_code_bgei);
3368 case jit_code_bltr_u: return (jit_code_bger_u);
3369 case jit_code_blti_u: return (jit_code_bgei_u);
3370 case jit_code_bler: return (jit_code_bgtr);
3371 case jit_code_blei: return (jit_code_bgti);
3372 case jit_code_bler_u: return (jit_code_bgtr_u);
3373 case jit_code_blei_u: return (jit_code_bgti_u);
3374 case jit_code_beqr: return (jit_code_bner);
3375 case jit_code_beqi: return (jit_code_bnei);
3376 case jit_code_bger: return (jit_code_bltr);
3377 case jit_code_bgei: return (jit_code_blti);
3378 case jit_code_bger_u: return (jit_code_bltr_u);
3379 case jit_code_bgei_u: return (jit_code_blti_u);
3380 case jit_code_bgtr: return (jit_code_bler);
3381 case jit_code_bgti: return (jit_code_blei);
3382 case jit_code_bgtr_u: return (jit_code_bler_u);
3383 case jit_code_bgti_u: return (jit_code_blei_u);
3384 case jit_code_bner: return (jit_code_beqr);
3385 case jit_code_bnei: return (jit_code_beqi);
3386 case jit_code_bmsr: return (jit_code_bmcr);
3387 case jit_code_bmsi: return (jit_code_bmci);
3388 case jit_code_bmcr: return (jit_code_bmsr);
3389 case jit_code_bmci: return (jit_code_bmsi);
3390 case jit_code_bltr_f: return (jit_code_bunger_f);
3391 case jit_code_blti_f: return (jit_code_bungei_f);
3392 case jit_code_bler_f: return (jit_code_bungtr_f);
3393 case jit_code_blei_f: return (jit_code_bungti_f);
3394
3395 case jit_code_beqr_f: return (jit_code_bner_f);
3396 case jit_code_beqi_f: return (jit_code_bnei_f);
3397
3398 case jit_code_bger_f: return (jit_code_bunltr_f);
3399 case jit_code_bgei_f: return (jit_code_bunlti_f);
3400 case jit_code_bgtr_f: return (jit_code_bunler_f);
3401 case jit_code_bgti_f: return (jit_code_bunlei_f);
3402
3403 case jit_code_bner_f: return (jit_code_beqr_f);
79bfeef6 3404 case jit_code_bnei_f: return (jit_code_beqi_f);
4a71579b
PC		 3405
3406 case jit_code_bunltr_f: return (jit_code_bger_f);
3407 case jit_code_bunlti_f: return (jit_code_bgei_f);
3408 case jit_code_bunler_f: return (jit_code_bgtr_f);
3409 case jit_code_bunlei_f: return (jit_code_bgti_f);
3410
3411 case jit_code_buneqr_f: return (jit_code_bltgtr_f);
3412 case jit_code_buneqi_f: return (jit_code_bltgti_f);
3413
3414 case jit_code_bunger_f: return (jit_code_bltr_f);
3415 case jit_code_bungei_f: return (jit_code_blti_f);
3416 case jit_code_bungtr_f: return (jit_code_bler_f);
3417 case jit_code_bungti_f: return (jit_code_blei_f);
3418
3419 case jit_code_bltgtr_f: return (jit_code_buneqr_f);
3420 case jit_code_bltgti_f: return (jit_code_buneqi_f);
3421
3422 case jit_code_bordr_f: return (jit_code_bunordr_f);
3423 case jit_code_bordi_f: return (jit_code_bunordi_f);
3424 case jit_code_bunordr_f:return (jit_code_bordr_f);
3425 case jit_code_bunordi_f:return (jit_code_bordi_f);
3426 case jit_code_bltr_d: return (jit_code_bunger_d);
3427 case jit_code_blti_d: return (jit_code_bungei_d);
3428 case jit_code_bler_d: return (jit_code_bungtr_d);
3429 case jit_code_blei_d: return (jit_code_bungti_d);
3430
3431 case jit_code_beqr_d: return (jit_code_bner_d);
3432 case jit_code_beqi_d: return (jit_code_bnei_d);
3433
3434 case jit_code_bger_d: return (jit_code_bunltr_d);
3435 case jit_code_bgei_d: return (jit_code_bunlti_d);
3436 case jit_code_bgtr_d: return (jit_code_bunler_d);
3437 case jit_code_bgti_d: return (jit_code_bunlei_d);
3438
3439 case jit_code_bner_d: return (jit_code_beqr_d);
3440 case jit_code_bnei_d: return (jit_code_beqi_d);
3441
3442 case jit_code_bunltr_d: return (jit_code_bger_d);
3443 case jit_code_bunlti_d: return (jit_code_bgei_d);
3444 case jit_code_bunler_d: return (jit_code_bgtr_d);
3445 case jit_code_bunlei_d: return (jit_code_bgti_d);
3446
3447 case jit_code_buneqr_d: return (jit_code_bltgtr_d);
3448 case jit_code_buneqi_d: return (jit_code_bltgti_d);
3449
3450 case jit_code_bunger_d: return (jit_code_bltr_d);
3451 case jit_code_bungei_d: return (jit_code_blti_d);
3452 case jit_code_bungtr_d: return (jit_code_bler_d);
3453 case jit_code_bungti_d: return (jit_code_blei_d);
3454
3455 case jit_code_bltgtr_d: return (jit_code_buneqr_d);
3456 case jit_code_bltgti_d: return (jit_code_buneqi_d);
3457
3458 case jit_code_bordr_d: return (jit_code_bunordr_d);
3459 case jit_code_bordi_d: return (jit_code_bunordi_d);
3460 case jit_code_bunordr_d:return (jit_code_bordr_d);
3461 case jit_code_bunordi_d:return (jit_code_bordi_d);
3462 case jit_code_boaddr: return (jit_code_bxaddr);
3463 case jit_code_boaddi: return (jit_code_bxaddi);
3464 case jit_code_boaddr_u: return (jit_code_bxaddr_u);
3465 case jit_code_boaddi_u: return (jit_code_bxaddi_u);
3466 case jit_code_bxaddr: return (jit_code_boaddr);
3467 case jit_code_bxaddi: return (jit_code_boaddi);
3468 case jit_code_bxaddr_u: return (jit_code_boaddr_u);
3469 case jit_code_bxaddi_u: return (jit_code_boaddi_u);
3470 case jit_code_bosubr: return (jit_code_bxsubr);
3471 case jit_code_bosubi: return (jit_code_bxsubi);
3472 case jit_code_bosubr_u: return (jit_code_bxsubr_u);
3473 case jit_code_bosubi_u: return (jit_code_bxsubi_u);
3474 case jit_code_bxsubr: return (jit_code_bosubr);
3475 case jit_code_bxsubi: return (jit_code_bosubi);
3476 case jit_code_bxsubr_u: return (jit_code_bosubr_u);
3477 case jit_code_bxsubi_u: return (jit_code_bosubi_u);
3478 default: abort(); /* invalid jump code */
3479 }
3480}
3481
3482/*
3483 * change common pattern:
3484 * <cond_jump L0> <jump L1> <label L0>
3485 * into
3486 * <reverse_cond_jump L1>
3487 */
3488static jit_bool_t
3489_reverse_jump(jit_state_t *_jit, jit_node_t *prev, jit_node_t *node)
3490{
3491 jit_node_t *local_prev;
3492 jit_node_t *local_next;
3493 jit_node_t *local_jump;
3494
3495 if (!(node->flag & jit_flag_node))
3496 return (0);
3497 /* =><cond_jump L0> <jump L1> <label L0> */
3498 local_next = node->next;
3499 if (local_next->code != jit_code_jmpi ||
3500 !(local_next->flag & jit_flag_node))
3501 return (0);
3502 /* <cond_jump L0> =><jump L1> <label L0> */
3503
3504 local_jump = local_next->u.n;
3505 for (local_prev = local_next, local_next = local_next->next;
3506 local_next;
3507 local_prev = local_next, local_next = local_next->next) {
3508 switch (local_next->code) {
3509 case jit_code_label: case jit_code_epilog:
3510 if (node->u.n == local_next) {
3511 if (local_next->link == node)
3512 local_next->link = node->link;
3513 else {
3514 for (local_prev = local_next->link;
3515 local_prev->link != node;
3516 local_prev = local_prev->link)
3517 assert(local_prev != NULL);
3518 local_prev->link = node->link;
3519 }
3520 del_node(node, node->next);
3521 node->code = reverse_jump_code(node->code);
3522 node->u.n = local_jump;
3523 node->link = local_jump->link;
3524 local_jump->link = node;
3525 return (1);
3526 }
3527 break;
3528 case jit_code_note:
3529 break;
3530 default:
3531 return (0);
3532 }
3533 }
3534 return (0);
3535}
3536
ba3814c1 3537static jit_bool_t
4a71579b
PC		 3538_redundant_store(jit_state_t *_jit, jit_node_t *node, jit_bool_t jump)
3539{
3540 jit_node_t *iter;
3541 jit_node_t *prev;
3542 jit_word_t word;
3543 jit_int32_t spec;
3544 jit_int32_t regno;
ba3814c1 3545 jit_bool_t result;
4a71579b
PC		 3546
3547 if (jump) {
3548 prev = node->u.n;
3549 if (prev->code == jit_code_epilog)
ba3814c1 3550 return (0);
4a71579b
PC		 3551 assert(prev->code == jit_code_label);
3552 if ((prev->flag & jit_flag_head) || node->link || prev->link != node)
3553 /* multiple sources */
ba3814c1 3554 return (0);
4a71579b
PC		 3555 /* if there are sequential labels it will return below */
3556 }
3557 else
3558 prev = node;
ba3814c1 3559 result = 0;
4a71579b
PC		 3560 word = node->w.w;
3561 regno = jit_regno(node->v.w);
3562 for (iter = prev->next; iter; prev = iter, iter = iter->next) {
3563 switch (iter->code) {
3564 case jit_code_label: case jit_code_prolog:
3565 case jit_code_epilog:
ba3814c1 3566 return (result);
4a71579b
PC		 3567 case jit_code_movi:
3568 if (regno == jit_regno(iter->u.w)) {
3569 if (iter->flag || iter->v.w != word)
ba3814c1
PC		 3570 return (result);
3571 result = 1;
4a71579b
PC		 3572 del_node(prev, iter);
3573 iter = prev;
3574 }
3575 break;
3576 default:
3577 spec = jit_classify(iter->code);
3578 if (spec & jit_cc_a0_jmp)
ba3814c1 3579 return (result);
4a71579b
PC		 3580 if ((spec & (jit_cc_a0_reg|jit_cc_a0_chg)) ==
3581 (jit_cc_a0_reg|jit_cc_a0_chg)) {
3582 if (spec & jit_cc_a0_rlh) {
3583 if (regno == jit_regno(iter->u.q.l) ||
3584 regno == jit_regno(iter->u.q.h))
ba3814c1 3585 return (result);
4a71579b
PC		 3586 }
3587 else {
3588 if (regno == jit_regno(iter->u.w))
ba3814c1 3589 return (result);
4a71579b
PC		 3590 }
3591 }
3592 if ((spec & (jit_cc_a1_reg|jit_cc_a1_chg)) ==
3593 (jit_cc_a1_reg|jit_cc_a1_chg)) {
3594 if (regno == jit_regno(iter->v.w))
ba3814c1 3595 return (result);
4a71579b
PC		 3596 }
3597 if ((spec & (jit_cc_a2_reg|jit_cc_a2_chg)) ==
3598 (jit_cc_a2_reg|jit_cc_a2_chg)) {
3599 if (regno == jit_regno(iter->w.w))
ba3814c1 3600 return (result);
4a71579b
PC		 3601 }
3602 break;
3603 }
3604 }
c0c16242
PC		 3605
3606 return (result);
4a71579b
PC		 3607}
3608
3609static jit_bool_t
3610_simplify_movr(jit_state_t *_jit, jit_node_t *prev, jit_node_t *node,
3611 jit_int32_t kind, jit_int32_t size)
3612{
3613 jit_int32_t regno;
3614 jit_int32_t right;
ba86ff93 3615 jit_int32_t spec;
4a71579b
PC		 3616 jit_value_t *value;
3617
3618 regno = jit_regno(node->u.w);
3619 right = jit_regno(node->v.w);
3620 value = _jitc->values + regno;
ba86ff93
PC		 3621
3622 spec = jit_class(_rvs[regno].spec);
3623 if (!(spec & (jit_class_gpr | jit_class_fpr))) {
3624 /* reserved register */
3625 value->kind = 0;
3626 ++_jitc->gen[regno];
3627 return (0);
3628 }
3629
4a71579b
PC		 3630 if ((value->kind == jit_kind_register &&
3631 jit_regno(value->base.q.l) == right &&
3632 value->base.q.h == _jitc->gen[right]) ||
3633 (value->kind == kind && _jitc->values[right].kind == kind &&
3634 memcmp(&value->base.w, &_jitc->values[right].base.w, size) == 0)) {
3635 del_node(prev, node);
3636 return (1);
3637 }
3638 if (_jitc->values[right].kind == jit_kind_word)
3639 jit_memcpy(value, _jitc->values + right, sizeof(jit_value_t));
3640 else {
3641 value->kind = jit_kind_register;
3642 value->base.q.l = right;
3643 value->base.q.h = _jitc->gen[right];
3644 }
3645 ++_jitc->gen[regno];
3646
3647 return (0);
3648}
3649
3650static jit_bool_t
3651_simplify_movi(jit_state_t *_jit, jit_node_t *prev, jit_node_t *node,
3652 jit_int32_t kind, jit_int32_t size)
3653{
3654 jit_value_t *value;
3655 jit_int32_t spec;
3656 jit_int32_t regno;
3657 jit_int32_t offset;
3658
3659 regno = jit_regno(node->u.w);
3660 value = _jitc->values + regno;
3661 if (node->flag & jit_flag_node) {
3662 /* set to undefined if value will be patched */
3663 value->kind = 0;
3664 ++_jitc->gen[regno];
3665 return (0);
3666 }
ba86ff93
PC		 3667 spec = jit_class(_rvs[regno].spec);
3668 if (!(spec & (jit_class_gpr | jit_class_fpr))) {
3669 /* reserved register */
3670 value->kind = 0;
3671 ++_jitc->gen[regno];
3672 return (0);
3673 }
4a71579b
PC		 3674 if (value->kind == kind) {
3675 if (memcmp(&node->v.w, &value->base.w, size) == 0) {
3676 del_node(prev, node);
3677 return (1);
3678 }
4a71579b
PC		 3679 if (kind == jit_kind_word)
3680 spec &= jit_class_gpr;
3681 else
3682 spec &= (jit_class_xpr | jit_class_fpr);
3683 for (offset = 0; offset < _jitc->reglen; offset++) {
3684 if (_jitc->values[offset].kind == kind &&
3685 memcmp(&node->v.w, &_jitc->values[offset].base.w, size) == 0 &&
3686 (jit_class(_rvs[offset].spec) & spec) == spec) {
3687 if (kind == jit_kind_word)
3688 node->code = jit_code_movr;
3689 else if (kind == jit_kind_float32)
3690 node->code = jit_code_movr_f;
3691 else
3692 node->code = jit_code_movr_d;
3693 node->v.w = offset;
3694 jit_memcpy(value, _jitc->values + offset, sizeof(jit_value_t));
3695 ++_jitc->gen[regno];
3696 return (0);
3697 }
3698 }
3699 }
3700 value->kind = kind;
3701 jit_memcpy(&value->base.w, &node->v.w, size);
3702 ++_jitc->gen[regno];
3703
3704 return (0);
3705}
3706
3707/* simple/safe redundandy test not checking if another register
3708 * holds the same value
3709 */
3710static jit_bool_t
3711_simplify_ldxi(jit_state_t *_jit, jit_node_t *prev, jit_node_t *node)
3712{
3713 jit_value_t *value;
3714 jit_int32_t regno;
3715 jit_int32_t right;
3716
3717 regno = jit_regno(node->u.w);
3718 right = jit_regno(node->v.w);
3719 value = _jitc->values + regno;
3720 if (regno != right &&
3721 value->kind == jit_kind_code && value->code == node->code &&
3722 value->base.q.l == right && value->base.q.h == _jitc->gen[right] &&
3723 node->w.w == value->disp.w) {
3724 del_node(prev, node);
3725 return (1);
3726 }
3727 value->kind = jit_kind_code;
3728 value->code = node->code;
3729 value->base.q.l = right;
3730 value->base.q.h = _jitc->gen[right];
3731 value->disp.w = node->w.w;
3732 ++_jitc->gen[regno];
3733
3734 return (0);
3735}
3736
3737static jit_bool_t
3738_simplify_stxi(jit_state_t *_jit, jit_node_t *prev, jit_node_t *node)
3739{
3740 jit_value_t *value;
3741 jit_int32_t regno;
3742 jit_int32_t right;
3743 jit_int32_t offset;
3744
3745 regno = jit_regno(node->w.w);
3746 right = jit_regno(node->v.w);
3747 value = _jitc->values + regno;
3748
3749 /* check for redundant store after load */
3750 if (regno != right &&
3751 value->kind == jit_kind_code && value->code == node->code &&
3752 value->base.q.l == right && value->base.q.h == _jitc->gen[right] &&
3753 node->u.w == value->disp.w) {
3754 del_node(prev, node);
3755 return (1);
3756 }
3757
3758 /* assume anything can alias, and invalidate tracked values */
3759 for (offset = 0; offset < _jitc->reglen; offset++) {
3760 if (_jitc->values[offset].kind == jit_kind_code) {
3761 _jitc->values[offset].kind = 0;
3762 ++_jitc->gen[offset];
3763 }
3764 }
3765
3766 /* no multiple information, so, if set to a constant,
3767 * prefer to keep that information */
3768 if (value->kind == 0) {
4a71579b
PC		 3769 switch (node->code) {
3770 /* no information about signed/unsigned either */
3771 case jit_code_stxi_c: value->code = jit_code_ldxi_c; break;
3772 case jit_code_stxi_s: value->code = jit_code_ldxi_s; break;
3773 case jit_code_stxi_i: value->code = jit_code_ldxi_i; break;
3774 case jit_code_stxi_l: value->code = jit_code_ldxi_l; break;
3775 case jit_code_stxi_f: value->code = jit_code_ldxi_f; break;
3776 case jit_code_stxi_d: value->code = jit_code_ldxi_d; break;
3777 default: abort();
3778 }
3779 value->kind = jit_kind_code;
3780 value->base.q.l = right;
3781 value->base.q.h = _jitc->gen[right];
3782 value->disp.w = node->u.w;
3783 }
3784
3785 return (0);
3786}
3787
3788/* usually there should be only one store in the
3789 * jit_get_reg/jit_unget_reg, but properly handle
3790 * multiple ones by moving the save node */
3791static void
3792_simplify_spill(jit_state_t *_jit, jit_node_t *node, jit_int32_t regno)
3793{
3794 jit_node_t *save;
3795 jit_node_t *temp;
3796
3797 if ((temp = _jitc->spill[regno]) && (save = temp->next) != node) {
3798 temp->next = save->next;
3799 save->next = node->next;
3800 node->next = save;
3801 _jitc->spill[regno] = node;
3802 }
3803}
3804
3805/* checks for simple cases where a register is set more than
3806 * once to the same value, and is a common pattern of calls
3807 * to jit_pushargi and jit_pushargr
3808 */
ba3814c1 3809static jit_bool_t
4a71579b
PC		 3810_simplify(jit_state_t *_jit)
3811{
3812 jit_node_t *prev;
3813 jit_node_t *node;
3814 jit_node_t *next;
3815 jit_int32_t info;
3816 jit_int32_t regno;
ba3814c1 3817 jit_bool_t result;
4a71579b 3818
ba3814c1 3819 result = 0;
4a71579b
PC		 3820 for (prev = NULL, node = _jitc->head; node; prev = node, node = next) {
3821 next = node->next;
3822 switch (node->code) {
3823 case jit_code_label: case jit_code_prolog:
3824 case jit_code_callr: case jit_code_calli:
3825 reset:
3826 memset(_jitc->gen, 0, sizeof(jit_int32_t) * _jitc->reglen);
3827 memset(_jitc->values, 0, sizeof(jit_value_t) * _jitc->reglen);
3828 break;
3829 case jit_code_save:
3830 _jitc->spill[jit_regno(node->u.w)] = prev;
3831 break;
3832 case jit_code_load:
3833 regno = jit_regno(node->u.w);
3834 if (register_change_p(node->link->next, node, regno) !=
3835 jit_reg_change) {
3836 /* spill not required due to optimizing common
3837 * redundancy case of calling jit_get_reg/jit_unget_reg
3838 * and then setting the register to the value it is
3839 * already holding */
3840 patch_register(node->link->next, node,
3841 jit_regno_patch|regno, regno);
ba3814c1 3842 result = 1;
4a71579b
PC		 3843 del_node(_jitc->spill[regno], node->link);
3844 del_node(prev, node);
3845 node = prev;
3846 }
3847 _jitc->spill[regno] = NULL;
3848 break;
3849 case jit_code_movr:
3850 regno = jit_regno(node->u.w);
3851 if (simplify_movr(prev, node,
ba3814c1
PC		 3852 jit_kind_word, sizeof(jit_word_t))) {
3853 result = 1;
4a71579b 3854 simplify_spill(node = prev, regno);
ba3814c1 3855 }
4a71579b
PC		 3856 break;
3857 case jit_code_movi:
3858 regno = jit_regno(node->u.w);
3859 if (simplify_movi(prev, node,
ba3814c1
PC		 3860 jit_kind_word, sizeof(jit_word_t))) {
3861 result = 1;
4a71579b 3862 simplify_spill(node = prev, regno);
ba3814c1 3863 }
4a71579b
PC		 3864 break;
3865 case jit_code_movr_f:
3866 regno = jit_regno(node->u.w);
3867 if (simplify_movr(prev, node,
ba3814c1
PC		 3868 jit_kind_float32, sizeof(jit_float32_t))) {
3869 result = 1;
4a71579b 3870 simplify_spill(node = prev, regno);
ba3814c1 3871 }
4a71579b
PC		 3872 break;
3873 case jit_code_movi_f:
3874 regno = jit_regno(node->u.w);
3875 if (simplify_movi(prev, node,
ba3814c1
PC		 3876 jit_kind_float32, sizeof(jit_float32_t))) {
3877 result = 1;
4a71579b 3878 simplify_spill(node = prev, regno);
ba3814c1 3879 }
4a71579b
PC		 3880 break;
3881 case jit_code_movr_d:
3882 regno = jit_regno(node->u.w);
3883 if (simplify_movr(prev, node,
ba3814c1
PC		 3884 jit_kind_float64, sizeof(jit_float64_t))) {
3885 result = 1;
4a71579b 3886 simplify_spill(node = prev, regno);
ba3814c1 3887 }
4a71579b
PC		 3888 break;
3889 case jit_code_movi_d:
3890 regno = jit_regno(node->u.w);
3891 if (simplify_movi(prev, node,
ba3814c1
PC		 3892 jit_kind_float64, sizeof(jit_float64_t))) {
3893 result = 1;
4a71579b 3894 simplify_spill(node = prev, regno);
ba3814c1 3895 }
4a71579b
PC		 3896 break;
3897 case jit_code_ldxi_c: case jit_code_ldxi_uc:
3898 case jit_code_ldxi_s: case jit_code_ldxi_us:
3899 case jit_code_ldxi_i: case jit_code_ldxi_ui:
3900 case jit_code_ldxi_l:
3901 case jit_code_ldxi_f: case jit_code_ldxi_d:
3902 regno = jit_regno(node->u.w);
ba3814c1
PC		 3903 if (simplify_ldxi(prev, node)) {
3904 result = 1;
4a71579b 3905 simplify_spill(node = prev, regno);
ba3814c1 3906 }
4a71579b
PC		 3907 break;
3908 case jit_code_stxi_c: case jit_code_stxi_s:
3909 case jit_code_stxi_i: case jit_code_stxi_l:
3910 case jit_code_stxi_f: case jit_code_stxi_d:
3911 regno = jit_regno(node->u.w);
ba3814c1
PC		 3912 if (simplify_stxi(prev, node)) {
3913 result = 1;
4a71579b 3914 simplify_spill(node = prev, regno);
ba3814c1 3915 }
4a71579b
PC		 3916 break;
3917 default:
3918 info = jit_classify(node->code);
3919 if (info & jit_cc_a0_jmp)
3920 /* labels are not implicitly added when not taking
3921 * a conditional branch */
3922 goto reset;
3923 if (info & jit_cc_a0_chg) {
3924 if (info & jit_cc_a0_rlh) {
3925 regno = jit_regno(node->u.q.l);
3926 _jitc->values[regno].kind = 0;
3927 ++_jitc->gen[regno];
3928 regno = jit_regno(node->u.q.h);
3929 _jitc->values[regno].kind = 0;
3930 ++_jitc->gen[regno];
3931 }
3932 else {
3933 regno = jit_regno(node->u.w);
3934 _jitc->values[regno].kind = 0;
3935 ++_jitc->gen[regno];
3936 }
3937 }
3938 if (info & jit_cc_a1_chg) {
ba86ff93
PC		 3939#if 0
3940 /* Assume registers are not changed */
3941 if (info & jit_cc_a1_rlh) {
3942 regno = jit_regno(node->v.q.l);
3943 _jitc->values[regno].kind = 0;
3944 ++_jitc->gen[regno];
3945 regno = jit_regno(node->v.q.h);
3946 _jitc->values[regno].kind = 0;
3947 ++_jitc->gen[regno];
3948 }
3949 else {
3950#endif
3951 regno = jit_regno(node->v.w);
3952 _jitc->values[regno].kind = 0;
3953 ++_jitc->gen[regno];
3954#if 0
3955 }
3956#endif
4a71579b
PC		 3957 }
3958 if (info & jit_cc_a2_chg) {
ba3814c1
PC		 3959#if 0
3960 /* Assume registers are not changed */
3961 if (info & jit_cc_a2_rlh) {
3962 regno = jit_regno(node->w.q.l);
3963 _jitc->values[regno].kind = 0;
3964 ++_jitc->gen[regno];
3965 regno = jit_regno(node->w.q.h);
3966 _jitc->values[regno].kind = 0;
3967 ++_jitc->gen[regno];
3968 }
3969 else {
3970#endif
3971 regno = jit_regno(node->w.w);
3972 _jitc->values[regno].kind = 0;
3973 ++_jitc->gen[regno];
3974#if 0
3975 }
3976#endif
4a71579b
PC		 3977 }
3978 break;
3979 }
3980 }
ba3814c1 3981 return (result);
4a71579b
PC		 3982}
3983
3984static jit_int32_t
3985_register_change_p(jit_state_t *_jit, jit_node_t *node, jit_node_t *link,
3986 jit_int32_t regno)
3987{
3988 jit_int32_t value;
ba86ff93 3989 jit_bool_t use, change;
4a71579b
PC		 3990
3991 for (; node != link; node = node->next) {
3992 switch (node->code) {
3993 case jit_code_label: case jit_code_prolog:
ba86ff93
PC		 3994 /* lack of extra information so assume it is live */
3995 return (jit_reg_static);
4a71579b
PC		 3996 case jit_code_callr: case jit_code_calli:
3997 if (!(jit_class(_rvs[regno].spec) & jit_class_sav))
3998 return (jit_reg_undef);
3999 break;
4000 default:
4001 value = jit_classify(node->code);
ba86ff93
PC		 4002 use = change = 0;
4003 if (value & jit_cc_a0_rlh) {
4004 if (node->u.q.l == regno || node->u.q.h == regno) {
4005 if (value & jit_cc_a0_chg)
4006 change = 1;
4007 use = !change;
4008 }
4009 }
4010 else if ((value & jit_cc_a0_reg) && node->u.w == regno) {
4011 if (value & jit_cc_a0_chg)
4012 change = 1;
4013 use = !change;
4014 }
4015 if (!use &&
4016 (value & jit_cc_a1_reg) && node->v.w == regno) {
4017 if (value & jit_cc_a1_chg)
4018 change = 1;
4019 use = !change;
4020 }
4021 if (!use &&
4022 (value & jit_cc_a2_reg) && node->w.w == regno) {
4023 if (value & jit_cc_a2_chg)
4024 change = 1;
4025 use = !change;
4026 }
4a71579b 4027 /* lack of extra information */
ba86ff93
PC		 4028 if (!use && (value & (jit_cc_a0_jmp|jit_cc_a0_cnd)) &&
4029 /* In case of indirect branches, always consider
4030 * callee save registers as live. */
4031 !(jit_class(_rvs[regno].spec) & jit_class_sav))
4032 change = 1;
4033 if (!use && change)
4a71579b
PC		 4034 return (jit_reg_change);
4035 }
4036 }
4037
4038 return (jit_reg_static);
4039}
4040
4041/* most of this could be done at the same time as generating jit, but
4042 * avoid complications on different cpu backends and patch spill/loads
4043 * here, by simulating jit generation */
4044static jit_bool_t
4045_spill_reglive_p(jit_state_t *_jit, jit_node_t *node, jit_int32_t regno)
4046{
4047 if (!jit_regset_tstbit(&_jitc->reglive, regno)) {
4048 jit_regset_setbit(&_jitc->regmask, regno);
4049 jit_update(node->next, &_jitc->reglive, &_jitc->regmask);
4050 if (!jit_regset_tstbit(&_jitc->reglive, regno) &&
4051 register_change_p(node->next, node->link, regno) != jit_reg_change)
4052 return (0);
4053 }
4054
4055 return (1);
4056}
4057
4058static void
4059_patch_registers(jit_state_t *_jit)
4060{
4061 jit_node_t *prev;
4062 jit_node_t *node;
4063 jit_node_t *next;
4064 jit_int32_t info;
4065 jit_int32_t spec;
4066 jit_int32_t regno;
4067 jit_int32_t value;
4068
4069 _jitc->function = NULL;
4070
4071 jit_reglive_setup();
4072 for (prev = NULL, node = _jitc->head; node; node = next) {
4073 next = node->next;
4074
4075 info = jit_classify(node->code);
4076 jit_regarg_set(node, info);
4077
4078 switch (node->code) {
4079 case jit_code_save:
4080 regno = jit_regno(node->u.w);
4081 if (!spill_reglive_p(node, regno)) {
4082 /* register is not live, just remove spill/reload */
4083 jit_regarg_clr(node, info);
4084 node->link->v.w = jit_regload_delete;
4085 del_node(prev, node);
4086 continue;
4087 }
4088 else {
4089 /* try to find a free register of the same class */
4090 spec = jit_class(_rvs[regno].spec) & ~jit_class_arg;
4091 for (value = 0; value < _jitc->reglen; value++) {
4092 if (value != regno &&
4093 ((jit_class(_rvs[value].spec) & spec) &
4094 ~jit_class_arg) == spec &&
4095 !jit_regset_tstbit(&_jitc->regarg, value) &&
4096 !spill_reglive_p(node, value))
4097 break;
4098 }
4099 if (value < _jitc->reglen) {
4100 jit_regarg_clr(node, info);
4101 patch_register(node->next, node->link,
4102 jit_regno_patch|node->u.w,
4103 jit_regno_patch|value);
4104 /* mark as live just in case there are nested
4105 * register patches, so that next patch will
4106 * not want to use the same register */
4107 jit_regset_setbit(&_jitc->reglive, value);
4108 /* register is not live, just remove spill/reload */
4109 node->link->v.w = jit_regload_isdead;
4110 del_node(prev, node);
4111 continue;
4112 }
4113 else {
4114 /* failed to find a free register */
4115 if (spec & jit_class_gpr) {
4116 if (!_jitc->function->regoff[regno])
4117 _jitc->function->regoff[regno] =
4118 jit_allocai(sizeof(jit_word_t));
4119#if __WORDSIZE == 32
4120 node->code = jit_code_stxi_i;
4121#else
4122 node->code = jit_code_stxi_l;
4123#endif
4124 }
4125 else {
4126 node->code = jit_code_stxi_d;
4127 if (!_jitc->function->regoff[regno])
4128 _jitc->function->regoff[regno] =
4129 jit_allocai(sizeof(jit_float64_t));
4130 }
4131 node->u.w = _jitc->function->regoff[regno];
4132 node->v.w = JIT_FP;
4133 node->w.w = regno;
4134 node->link = NULL;
4135 }
4136 }
4137 break;
4138 case jit_code_load:
4139 regno = jit_regno(node->u.w);
4140 if (node->v.w) {
4141 if (node->v.w == jit_regload_isdead)
4142 jit_regset_clrbit(&_jitc->reglive, regno);
4143 del_node(prev, node);
4144 continue;
4145 }
4146 spec = jit_class(_rvs[regno].spec);
4147 if (spec & jit_class_gpr) {
4148#if __WORDSIZE == 32
4149 node->code = jit_code_ldxi_i;
4150#else
4151 node->code = jit_code_ldxi_l;
4152#endif
4153 }
4154 else
4155 node->code = jit_code_ldxi_d;
4156 node->v.w = regno;
4157 node->v.w = JIT_FP;
4158 node->w.w = _jitc->function->regoff[regno];
4159 node->link = NULL;
4160 break;
4161 case jit_code_prolog:
4162 _jitc->function = _jitc->functions.ptr + node->w.w;
4163 break;
4164 case jit_code_epilog:
4165 _jitc->function = NULL;
4166 break;
4167 default:
4168 break;
4169 }
4170
4171 jit_regarg_clr(node, info);
4172 /* update register live state */
4173 jit_reglive(node);
4174 prev = node;
4175 }
4176}
4177
4178static void
4179_patch_register(jit_state_t *_jit, jit_node_t *node, jit_node_t *link,
4180 jit_int32_t regno, jit_int32_t patch)
4181{
4182 jit_int32_t value;
4183
4184 for (; node != link; node = node->next) {
4185 value = jit_classify(node->code);
4186 if (value & jit_cc_a0_reg) {
4187 if (value & jit_cc_a0_rlh) {
4188 if (node->u.q.l == regno)
4189 node->u.q.l = patch;
4190 if (node->u.q.h == regno)
4191 node->u.q.h = patch;
4192 }
4193 else {
4194 if (node->u.w == regno)
4195 node->u.w = patch;
4196 }
4197 }
ba86ff93
PC		 4198 if (value & jit_cc_a1_reg) {
4199 if (value & jit_cc_a1_rlh) {
4200 if (node->v.q.l == regno)
4201 node->v.q.l = patch;
4202 if (node->v.q.h == regno)
4203 node->v.q.h = patch;
4204 }
4205 else {
4206 if (node->v.w == regno)
4207 node->v.w = patch;
4208 }
4209 }
ba3814c1
PC		 4210 if (value & jit_cc_a2_reg) {
4211 if (value & jit_cc_a2_rlh) {
4212 if (node->w.q.l == regno)
4213 node->w.q.l = patch;
4214 if (node->w.q.h == regno)
4215 node->w.q.h = patch;
4216 }
4217 else {
4218 if (node->w.w == regno)
4219 node->w.w = patch;
4220 }
4221 }
4a71579b
PC		 4222 }
4223}
4224
40a44dcb
PC		 4225#if __BYTE_ORDER == __LITTLE_ENDIAN
4226# define htonr_us(r0,r1) bswapr_us(r0,r1)
4227# define htonr_ui(r0,r1) bswapr_ui(r0,r1)
4228# if __WORDSIZE == 64
4229# define htonr_ul(r0,r1) bswapr_ul(r0,r1)
4230# endif
4231#else
4232# define htonr_us(r0,r1) extr_us(r0,r1)
4233# if __WORDSIZE == 32
4234# define htonr_ui(r0,r1) movr(r0,r1)
4235# else
4236# define htonr_ui(r0,r1) extr_ui(r0,r1)
4237# define htonr_ul(r0,r1) movr(r0,r1)
4238# endif
4239#endif
4240
4241static maybe_unused void
4242generic_bswapr_us(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1);
4243static maybe_unused void
4244generic_bswapr_ui(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1);
4245#if __WORDSIZE == 64
4246static maybe_unused void
4247generic_bswapr_ul(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1);
4248#endif
4249
ba86ff93
PC		 4250#define depi(r0, i0, i1, i2) _depi(_jit, r0, i0, i1, i2)
4251static void _depi(jit_state_t*,jit_int32_t, jit_word_t, jit_word_t, jit_word_t);
4252#define negi(r0, i0) _negi(_jit, r0, i0)
4253static void _negi(jit_state_t*, jit_int32_t, jit_word_t);
4254#define comi(r0, i0) _comi(_jit, r0, i0)
4255static void _comi(jit_state_t*, jit_int32_t, jit_word_t);
4256#define exti_c(r0, i0) _exti_c(_jit, r0, i0)
4257static void _exti_c(jit_state_t*, jit_int32_t, jit_word_t);
4258#define exti_uc(r0, i0) _exti_uc(_jit, r0, i0)
4259static void _exti_uc(jit_state_t*, jit_int32_t, jit_word_t);
4260#define exti_s(r0, i0) _exti_s(_jit, r0, i0)
4261static void _exti_s(jit_state_t*, jit_int32_t, jit_word_t);
4262#define exti_us(r0, i0) _exti_us(_jit, r0, i0)
4263static void _exti_us(jit_state_t*, jit_int32_t, jit_word_t);
4264#if __WORDSIZE == 64
4265#define exti_i(r0, i0) _exti_i(_jit, r0, i0)
4266static void _exti_i(jit_state_t*, jit_int32_t, jit_word_t);
4267#define exti_ui(r0, i0) _exti_ui(_jit, r0, i0)
4268static void _exti_ui(jit_state_t*, jit_int32_t, jit_word_t);
4269#endif
4270#define bswapi_us(r0, i0) _bswapi_us(_jit, r0, i0)
4271static void _bswapi_us(jit_state_t*, jit_int32_t, jit_word_t);
4272#define bswapi_ui(r0, i0) _bswapi_ui(_jit, r0, i0)
4273static void _bswapi_ui(jit_state_t*, jit_int32_t, jit_word_t);
4274#if __WORDSIZE == 64
4275# define bswapi_ul(r0, i0) _bswapi_ul(_jit, r0, i0)
4276static void _bswapi_ul(jit_state_t*, jit_int32_t, jit_word_t);
4277#endif
4278#define htoni_us(r0, i0) _htoni_us(_jit, r0, i0)
4279static void _htoni_us(jit_state_t*, jit_int32_t, jit_word_t);
4280#define htoni_ui(r0, i0) _htoni_ui(_jit, r0, i0)
4281static void _htoni_ui(jit_state_t*, jit_int32_t, jit_word_t);
4282#if __WORDSIZE == 64
4283# define htoni_ul(r0, i0) _htoni_ul(_jit, r0, i0)
4284static void _htoni_ul(jit_state_t*, jit_int32_t, jit_word_t);
4285#endif
4286# define movi_f_w(r0, i0) _movi_f_w(_jit, r0, i0)
4287static void _movi_f_w(jit_state_t*, jit_int32_t, jit_float32_t);
d481fb64 4288#if __WORDSIZE == 32 && !(defined(__mips__) && NEW_ABI)
ba86ff93
PC		 4289# define movi_d_ww(r0, r1, i0) _movi_d_ww(_jit, r0, r1, i0)
4290static void _movi_d_ww(jit_state_t*, jit_int32_t, jit_int32_t, jit_float64_t);
4291#else
4292# define movi_d_w(r0, i0) _movi_d_w(_jit, r0, i0)
4293static void _movi_d_w(jit_state_t*, jit_int32_t, jit_float64_t);
4294#endif
4295#define cloi(r0, i0) _cloi(_jit, r0, i0)
4296static void _cloi(jit_state_t*, jit_int32_t, jit_word_t);
4297#define clzi(r0, i0) _clzi(_jit, r0, i0)
4298static void _clzi(jit_state_t*, jit_int32_t, jit_word_t);
4299#define ctoi(r0, i0) _ctoi(_jit, r0, i0)
4300static void _ctoi(jit_state_t*, jit_int32_t, jit_word_t);
4301#define ctzi(r0, i0) _ctzi(_jit, r0, i0)
4302static void _ctzi(jit_state_t*, jit_int32_t, jit_word_t);
4303#define rbiti(r0, i0) _rbiti(_jit, r0, i0)
4304static void _rbiti(jit_state_t*, jit_int32_t, jit_word_t);
4305#define popcnti(r0, i0) _popcnti(_jit, r0, i0)
4306static void _popcnti(jit_state_t*, jit_int32_t, jit_word_t);
4307#define exti(r0, i0, i1, i2) _exti(_jit, r0, i0, i1, i2)
4308static void _exti(jit_state_t*,jit_int32_t, jit_word_t, jit_word_t, jit_word_t);
4309#define exti_u(r0, i0, i1, i2) _exti_u(_jit, r0, i0, i1, i2)
4310static void _exti_u(jit_state_t*,
4311 jit_int32_t, jit_word_t, jit_word_t, jit_word_t);
4312#define generic_unldr(r0, r1, i0) _generic_unldr(_jit, r0, r1, i0)
4313static void _generic_unldr(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
4314#define generic_unldi(r0, i0, i1) _generic_unldi(_jit, r0, i0, i1)
4315static void _generic_unldi(jit_state_t*, jit_int32_t, jit_word_t, jit_word_t);
4316#define generic_unldr_u(r0, r1, i0) _generic_unldr_u(_jit, r0, r1, i0)
4317static void _generic_unldr_u(jit_state_t*,
4318 jit_int32_t, jit_int32_t, jit_word_t);
4319#define generic_unldi_u(r0, i0, i1) _generic_unldi_u(_jit, r0, i0, i1)
4320static void _generic_unldi_u(jit_state_t*, jit_int32_t, jit_word_t, jit_word_t);
4321#define generic_unstr(r0, r1, i0) _generic_unstr(_jit, r0, r1, i0)
4322static void _generic_unstr(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
4323#define generic_unsti(i0, r0, i1) _generic_unsti(_jit, i0, r0, i1)
4324static void _generic_unsti(jit_state_t*, jit_word_t, jit_int32_t, jit_word_t);
4325#if !defined(__i386__) && !defined(__x86_64__)
4326# define generic_unldr_x(r0, r1, i0) _generic_unldr_x(_jit, r0, r1, i0)
4327static void _generic_unldr_x(jit_state_t*,
4328 jit_int32_t, jit_int32_t, jit_word_t);
4329# define generic_unldi_x(r0, i0, i1) _generic_unldi_x(_jit, r0, i0, i1)
4330static void _generic_unldi_x(jit_state_t*, jit_int32_t, jit_word_t, jit_word_t);
4331# define generic_unstr_x(r0, r1, i0) _generic_unstr_x(_jit, r0, r1, i0)
4332static void _generic_unstr_x(jit_state_t*,
4333 jit_int32_t, jit_int32_t, jit_word_t);
4334# define generic_unsti_x(i0, r0, i1) _generic_unsti_x(_jit, i0, r0, i1)
4335static void _generic_unsti_x(jit_state_t*, jit_word_t, jit_int32_t, jit_word_t);
4336#endif
79bfeef6
PC		 4337#define patch_alist(revert) _patch_alist(_jit, revert)
4338static maybe_unused void _patch_alist(jit_state_t *_jit, jit_bool_t revert);
4339
4a71579b
PC		 4340#if defined(__i386__) || defined(__x86_64__)
4341# include "jit_x86.c"
4342#elif defined(__mips__)
4343# include "jit_mips.c"
4344#elif defined(__arm__)
4345# include "jit_arm.c"
4346#elif defined(__powerpc__)
4347# include "jit_ppc.c"
4348#elif defined(__sparc__)
4349# include "jit_sparc.c"
4350#elif defined(__ia64__)
4351# include "jit_ia64.c"
4352#elif defined(__hppa__)
4353# include "jit_hppa.c"
4354#elif defined(__aarch64__)
4355# include "jit_aarch64.c"
4356#elif defined(__s390__) || defined(__s390x__)
4357# include "jit_s390.c"
4358#elif defined(__alpha__)
4359# include "jit_alpha.c"
4360#elif defined(__riscv)
4361# include "jit_riscv.c"
24d91c0d
PC		 4362#elif defined(__loongarch__)
4363# include "jit_loongarch.c"
4a71579b 4364#endif
40a44dcb
PC		 4365
4366static maybe_unused void
4367generic_bswapr_us(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
4368{
4369 jit_int32_t reg = jit_get_reg(jit_class_gpr);
4370
4371 rshi(rn(reg), r1, 8);
4372 andi(r0, r1, 0xff);
4373 andi(rn(reg), rn(reg), 0xff);
4374 lshi(r0, r0, 8);
4375 orr(r0, r0, rn(reg));
4376
4377 jit_unget_reg(reg);
4378}
4379
4380static maybe_unused void
4381generic_bswapr_ui(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
4382{
4383 jit_int32_t reg = jit_get_reg(jit_class_gpr);
4384
4385 rshi(rn(reg), r1, 16);
4386 bswapr_us(r0, r1);
4387 bswapr_us(rn(reg), rn(reg));
4388 lshi(r0, r0, 16);
4389 orr(r0, r0, rn(reg));
4390
4391 jit_unget_reg(reg);
4392}
4393
4394#if __WORDSIZE == 64
4395static maybe_unused void
4396generic_bswapr_ul(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
4397{
4398 jit_int32_t reg = jit_get_reg(jit_class_gpr);
4399
4400 rshi_u(rn(reg), r1, 32);
4401 bswapr_ui(r0, r1);
4402 bswapr_ui(rn(reg), rn(reg));
4403 lshi(r0, r0, 32);
4404 orr(r0, r0, rn(reg));
4405
4406 jit_unget_reg(reg);
4407}
4408#endif
79bfeef6 4409
ba86ff93
PC		 4410static void
4411_depi(jit_state_t *_jit,
4412 jit_int32_t r0, jit_word_t i0, jit_word_t i1, jit_word_t i2)
4413{
4414 jit_int32_t reg;
4415 reg = jit_get_reg(jit_class_gpr);
4416 movi(rn(reg), i0);
4417 depr(r0, rn(reg), i1, i2);
4418 jit_unget_reg(reg);
4419}
4420
4421static void
4422_negi(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
4423{
4424 movi(r0, -i0);
4425}
4426
4427static void
4428_comi(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
4429{
4430 movi(r0, ~i0);
4431}
4432
4433static void
4434_exti_c(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
4435{
4436 movi(r0, (jit_int8_t)i0);
4437}
4438
4439static void
4440_exti_uc(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
4441{
4442 movi(r0, (jit_uint8_t)i0);
4443}
4444
4445static void
4446_exti_s(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
4447{
4448 movi(r0, (jit_int16_t)i0);
4449}
4450
4451static void
4452_exti_us(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
4453{
4454 movi(r0, (jit_uint16_t)i0);
4455}
4456
4457#if __WORDSIZE == 64
4458static void
4459_exti_i(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
4460{
4461 movi(r0, (jit_int32_t)i0);
4462}
4463
4464static void
4465_exti_ui(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
4466{
4467 movi(r0, (jit_uint32_t)i0);
4468}
4469#endif
4470
4471static void
4472_bswapi_us(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
4473{
4474 union {
4475 jit_uint16_t us;
4476 jit_uint8_t v[2];
4477 } l, h;
4478 l.us = i0;
4479 h.v[0] = l.v[1];
4480 h.v[1] = l.v[0];
4481 movi(r0, h.us);
4482}
4483
4484static void
4485_bswapi_ui(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
4486{
4487 union {
4488 jit_uint32_t ui;
4489 jit_uint8_t v[4];
4490 } l, h;
4491 l.ui = i0;
4492 h.v[0] = l.v[3];
4493 h.v[1] = l.v[2];
4494 h.v[2] = l.v[1];
4495 h.v[3] = l.v[0];
4496 movi(r0, h.ui);
4497}
4498
4499#if __WORDSIZE == 64
4500static void
4501_bswapi_ul(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
4502{
4503 union {
4504 jit_uint64_t ul;
4505 jit_uint8_t v[8];
4506 } l, h;
4507 l.ul = i0;
4508 h.v[0] = l.v[7];
4509 h.v[1] = l.v[6];
4510 h.v[2] = l.v[5];
4511 h.v[3] = l.v[4];
4512 h.v[4] = l.v[3];
4513 h.v[5] = l.v[2];
4514 h.v[6] = l.v[1];
4515 h.v[7] = l.v[0];
4516 movi(r0, h.ul);
4517}
4518#endif
4519
4520static void
4521_htoni_us(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
4522{
4523#if __BYTE_ORDER == __LITTLE_ENDIAN
4524 bswapi_us(r0, i0);
4525#else
4526 exti_us(r0, i0);
4527#endif
4528}
4529
4530static void
4531_htoni_ui(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
4532{
4533#if __BYTE_ORDER == __LITTLE_ENDIAN
4534 bswapi_ui(r0, i0);
4535#else
4536# if __WORDSIZE == 32
4537 movi(r0, i0);
4538# else
4539 exti_ui(r0, i0);
4540# endif
4541#endif
4542}
4543
4544#if __WORDSIZE == 64
4545static void
4546_htoni_ul(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
4547{
4548# if __BYTE_ORDER == __LITTLE_ENDIAN
4549 bswapi_ul(r0, i0);
4550# else
4551 movi(r0, i0);
4552# endif
4553}
4554#endif
4555
4556static void
4557_movi_f_w(jit_state_t *_jit, jit_int32_t r0, jit_float32_t i0)
4558{
4559 union {
4560 jit_int32_t i;
4561 jit_float32_t f;
4562 } data;
4563 data.f = i0;
4564#if defined(__ia64__)
4565 /* Should be used only in this case (with out0 == 120) */
4566 if (r0 >= 120)
4567 r0 = _jitc->rout + (r0 - 120);
4568#endif
4569 movi(r0, data.i);
4570}
4571
d481fb64 4572#if __WORDSIZE == 32 && !(defined(__mips__) && NEW_ABI)
ba86ff93
PC		 4573static void
4574_movi_d_ww(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_float64_t i0)
4575{
4576 union {
4577 jit_int32_t i[2];
4578 jit_float64_t d;
4579 } data;
4580 data.d = i0;
4581 /* Mips does not change byte order of double values */
4582# if __BYTE_ORDER == __LITTLE_ENDIAN || defined(__mips__)
4583 movi(r0, data.i[0]);
4584 movi(r1, data.i[1]);
4585# else
4586 movi(r1, data.i[0]);
4587 movi(r0, data.i[1]);
4588# endif
4589}
4590
4591#else
4592static void
4593_movi_d_w(jit_state_t *_jit, jit_int32_t r0, jit_float64_t i0)
4594{
4595 union {
4596 jit_int64_t l;
4597 jit_float64_t d;
4598 } data;
4599 data.d = i0;
4600# if defined(__ia64__)
4601 /* Should be used only in this case (with out0 == 120) */
4602 if (r0 >= 120)
4603 r0 = _jitc->rout + (r0 - 120);
4604# endif
4605 movi(r0, data.l);
4606}
4607#endif
4608
4609 void
4610_jit_negi_f(jit_state_t *_jit, jit_fpr_t u, jit_float32_t v)
4611{
4612 jit_inc_synth_wf(negi_f, u, v);
4613 jit_movi_f(u, v);
4614 jit_negr_f(u, u);
4615 jit_dec_synth();
4616}
4617
4618void
4619_jit_absi_f(jit_state_t *_jit, jit_fpr_t u, jit_float32_t v)
4620{
4621 jit_inc_synth_wf(absi_f, u, v);
4622 jit_movi_f(u, v);
4623 jit_absr_f(u, u);
4624 jit_dec_synth();
4625}
4626
4627void
4628_jit_sqrti_f(jit_state_t *_jit, jit_fpr_t u, jit_float32_t v)
4629{
4630 jit_inc_synth_wf(sqrti_f, u, v);
4631 jit_movi_f(u, v);
4632 jit_sqrtr_f(u, u);
4633 jit_dec_synth();
4634}
4635
4636void
4637_jit_fmai_f(jit_state_t *_jit,
4638 jit_fpr_t u, jit_fpr_t v, jit_fpr_t w, jit_float32_t x)
4639{
4640 jit_int32_t y;
4641 jit_inc_synth_wqf(fmai_f, u, v, w, x);
4642 if (u != v && u != w) {
4643 jit_movi_f(u, x);
4644 jit_fmar_f(u, v, w, u);
4645 }
4646 else {
4647 y = jit_get_reg(jit_class_fpr);
4648 jit_movi_f(y, x);
4649 jit_fmar_f(u, v, w, y);
4650 jit_unget_reg(y);
4651 }
4652 jit_dec_synth();
4653}
4654
4655void
4656_jit_fmsi_f(jit_state_t *_jit,
4657 jit_fpr_t u, jit_fpr_t v, jit_fpr_t w, jit_float32_t x)
4658{
4659 jit_int32_t y;
4660 jit_inc_synth_wqf(fmai_f, u, v, w, x);
4661 if (u != v && u != w) {
4662 jit_movi_f(u, x);
4663 jit_fmsr_f(u, v, w, u);
4664 }
4665 else {
4666 y = jit_get_reg(jit_class_fpr);
4667 jit_movi_f(y, x);
4668 jit_fmsr_f(u, v, w, y);
4669 jit_unget_reg(y);
4670 }
4671 jit_dec_synth();
4672}
4673
4674void
4675_jit_fnmai_f(jit_state_t *_jit,
4676 jit_fpr_t u, jit_fpr_t v, jit_fpr_t w, jit_float32_t x)
4677{
4678 jit_int32_t y;
4679 jit_inc_synth_wqf(fmai_f, u, v, w, x);
4680 if (u != v && u != w) {
4681 jit_movi_f(u, x);
4682 jit_fnmar_f(u, v, w, u);
4683 }
4684 else {
4685 y = jit_get_reg(jit_class_fpr);
4686 jit_movi_f(y, x);
4687 jit_fnmar_f(u, v, w, y);
4688 jit_unget_reg(y);
4689 }
4690 jit_dec_synth();
4691}
4692
4693void
4694_jit_fnmsi_f(jit_state_t *_jit,
4695 jit_fpr_t u, jit_fpr_t v, jit_fpr_t w, jit_float32_t x)
4696{
4697 jit_int32_t y;
4698 jit_inc_synth_wqf(fmai_f, u, v, w, x);
4699 if (u != v && u != w) {
4700 jit_movi_f(u, x);
4701 jit_fnmsr_f(u, v, w, u);
4702 }
4703 else {
4704 y = jit_get_reg(jit_class_fpr);
4705 jit_movi_f(y, x);
4706 jit_fnmsr_f(u, v, w, y);
4707 jit_unget_reg(y);
4708 }
4709 jit_dec_synth();
4710}
4711
4712void
4713_jit_negi_d(jit_state_t *_jit, jit_fpr_t u, jit_float64_t v)
4714{
4715 jit_inc_synth_wd(negi_d, u, v);
4716 jit_movi_d(u, v);
4717 jit_negr_d(u, u);
4718 jit_dec_synth();
4719}
4720
4721void
4722_jit_absi_d(jit_state_t *_jit, jit_fpr_t u, jit_float64_t v)
4723{
4724 jit_inc_synth_wd(absi_d, u, v);
4725 jit_movi_d(u, v);
4726 jit_absr_d(u, u);
4727 jit_dec_synth();
4728}
4729
4730void
4731_jit_sqrti_d(jit_state_t *_jit, jit_fpr_t u, jit_float64_t v)
4732{
4733 jit_inc_synth_wd(sqrti_d, u, v);
4734 jit_movi_d(u, v);
4735 jit_sqrtr_d(u, u);
4736 jit_dec_synth();
4737}
4738
4739void
4740_jit_fmai_d(jit_state_t *_jit,
4741 jit_fpr_t u, jit_fpr_t v, jit_fpr_t w, jit_float64_t x)
4742{
4743 jit_int32_t y;
4744 jit_inc_synth_wqd(fmai_d, u, v, w, x);
4745 if (u != v && u != w) {
4746 jit_movi_d(u, x);
4747 jit_fmar_d(u, v, w, u);
4748 }
4749 else {
4750 y = jit_get_reg(jit_class_fpr);
4751 jit_movi_d(y, x);
4752 jit_fmar_d(u, v, w, y);
4753 jit_unget_reg(y);
4754 }
4755 jit_dec_synth();
4756}
4757
4758void
4759_jit_fmsi_d(jit_state_t *_jit,
4760 jit_fpr_t u, jit_fpr_t v, jit_fpr_t w, jit_float64_t x)
4761{
4762 jit_int32_t y;
4763 jit_inc_synth_wqd(fmai_d, u, v, w, x);
4764 if (u != v && u != w) {
4765 jit_movi_d(u, x);
4766 jit_fmsr_d(u, v, w, u);
4767 }
4768 else {
4769 y = jit_get_reg(jit_class_fpr);
4770 jit_movi_d(y, x);
4771 jit_fmsr_d(u, v, w, y);
4772 jit_unget_reg(y);
4773 }
4774 jit_dec_synth();
4775}
4776
4777void
4778_jit_fnmai_d(jit_state_t *_jit,
4779 jit_fpr_t u, jit_fpr_t v, jit_fpr_t w, jit_float64_t x)
4780{
4781 jit_int32_t y;
4782 jit_inc_synth_wqd(fmai_d, u, v, w, x);
4783 if (u != v && u != w) {
4784 jit_movi_d(u, x);
4785 jit_fnmar_d(u, v, w, u);
4786 }
4787 else {
4788 y = jit_get_reg(jit_class_fpr);
4789 jit_movi_d(y, x);
4790 jit_fnmar_d(u, v, w, y);
4791 jit_unget_reg(y);
4792 }
4793 jit_dec_synth();
4794}
4795
4796void
4797_jit_fnmsi_d(jit_state_t *_jit,
4798 jit_fpr_t u, jit_fpr_t v, jit_fpr_t w, jit_float64_t x)
4799{
4800 jit_int32_t y;
4801 jit_inc_synth_wqd(fmai_d, u, v, w, x);
4802 if (u != v && u != w) {
4803 jit_movi_d(u, x);
4804 jit_fnmsr_d(u, v, w, u);
4805 }
4806 else {
4807 y = jit_get_reg(jit_class_fpr);
4808 jit_movi_d(y, x);
4809 jit_fnmsr_d(u, v, w, y);
4810 jit_unget_reg(y);
4811 }
4812 jit_dec_synth();
4813}
4814
4815static void
4816_cloi(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
4817{
4818 clzi(r0, ~i0);
4819}
4820
4821static void
4822_clzi(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
4823{
4824#if __WORDSIZE == 64 && _WIN32
4825 movi(r0, (i0) ? __builtin_clzll(i0) : __WORDSIZE);
4826#else
4827 movi(r0, (i0) ? __builtin_clzl(i0) : __WORDSIZE);
4828#endif
4829}
4830
4831static void
4832_ctoi(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
4833{
4834 ctzi(r0, ~i0);
4835}
4836
4837static void
4838_ctzi(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
4839{
4840#if __WORDSIZE == 64 && _WIN32
4841 movi(r0, (i0) ? __builtin_ctzll(i0) : __WORDSIZE);
4842#else
4843 movi(r0, (i0) ? __builtin_ctzl(i0) : __WORDSIZE);
4844#endif
4845}
4846
4847static void
4848_rbiti(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
4849{
4850 jit_int32_t i;
4851 union {
4852 jit_uword_t w;
4853 jit_uint8_t v[__WORDSIZE >> 3];
4854 } u, v;
4855 static const unsigned char swap_tab[256] = {
4856 0, 128, 64, 192, 32, 160, 96, 224,
4857 16, 144, 80, 208, 48, 176, 112, 240,
4858 8, 136, 72, 200, 40, 168, 104, 232,
4859 24, 152, 88, 216 ,56, 184, 120, 248,
4860 4, 132, 68, 196, 36, 164, 100, 228,
4861 20, 148, 84, 212, 52, 180, 116, 244,
4862 12, 140, 76, 204, 44, 172, 108, 236,
4863 28, 156, 92, 220, 60, 188, 124, 252,
4864 2, 130, 66, 194, 34, 162, 98, 226,
4865 18, 146, 82, 210, 50, 178, 114, 242,
4866 10, 138, 74, 202, 42, 170, 106, 234,
4867 26, 154, 90, 218, 58, 186, 122, 250,
4868 6, 134, 70, 198, 38, 166, 102, 230,
4869 22, 150, 86, 214, 54, 182, 118, 246,
4870 14, 142, 78, 206, 46, 174, 110, 238,
4871 30, 158, 94, 222, 62, 190, 126, 254,
4872 1, 129, 65, 193, 33, 161, 97, 225,
4873 17, 145, 81, 209, 49, 177, 113, 241,
4874 9, 137, 73, 201, 41, 169, 105, 233,
4875 25, 153, 89, 217, 57, 185, 121, 249,
4876 5, 133, 69, 197, 37, 165, 101, 229,
4877 21, 149, 85, 213, 53, 181, 117, 245,
4878 13, 141, 77, 205, 45, 173, 109, 237,
4879 29, 157, 93, 221, 61, 189, 125, 253,
4880 3, 131, 67, 195, 35, 163, 99, 227,
4881 19, 147, 83, 211, 51, 179, 115, 243,
4882 11, 139, 75, 203, 43, 171, 107, 235,
4883 27, 155, 91, 219, 59, 187, 123, 251,
4884 7, 135, 71, 199, 39, 167, 103, 231,
4885 23, 151, 87, 215, 55, 183, 119, 247,
4886 15, 143, 79, 207, 47, 175, 111, 239,
4887 31, 159, 95, 223, 63, 191, 127, 255
4888 };
4889 u.w = i0;
4890 for (i = 0; i < sizeof(jit_word_t); ++i)
4891 v.v[i] = swap_tab[u.v[sizeof(jit_word_t) - i - 1]];
4892 movi(r0, v.w);
4893}
4894
4895static void
4896_popcnti(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
4897{
4898#if __WORDSIZE == 64 && _WIN32
4899 movi(r0, (i0) ? __builtin_popcountll(i0) : __WORDSIZE);
4900#else
4901 movi(r0, (i0) ? __builtin_popcountl(i0) : __WORDSIZE);
4902#endif
4903}
4904
4905static void _exti(jit_state_t *_jit,
4906 jit_int32_t r0, jit_word_t i0, jit_word_t i1, jit_word_t i2)
4907{
4908#if __BYTE_ORDER == __BIG_ENDIAN
4909 i1 = __WORDSIZE - (i1 + i2);
4910#endif
4911 i0 <<= __WORDSIZE - (i1 + i2);
4912 i0 >>= __WORDSIZE - i2;
4913 movi(r0, i0);
4914}
4915
4916static void _exti_u(jit_state_t *_jit,
4917 jit_int32_t r0, jit_word_t i0, jit_word_t i1, jit_word_t i2)
4918{
4919 jit_word_t t;
4920#if __BYTE_ORDER == __BIG_ENDIAN
4921 i1 = __WORDSIZE - (i1 + i2);
4922#endif
4923 if (i1)
4924 i0 >>= __WORDSIZE - i2;
4925#if __WORDSIZE == 64 && !_WIN32
4926 i0 &= (1L << i2) - 1;
4927#else
4928 i0 &= (1LL << i2) - 1;
4929#endif
4930 movi(r0, i0);
4931}
4932
4933static void
4934_generic_unldr(jit_state_t *_jit,
4935 jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
4936{
4937 jit_int32_t t0, r2;
4938 assert(i0 >= 1 && i0 <= sizeof(jit_word_t));
4939 if (i0 & (i0 - 1)) {
4940 t0 = jit_get_reg(jit_class_gpr);
4941 r2 = rn(t0);
4942 movr(r2, r1);
4943 }
4944 switch (i0) {
4945 case 1:
4946 ldr_c(r0, r1);
4947 break;
4948 case 2:
4949 ldr_s(r0, r1);
4950 break;
4951 case 3:
4952#if __BYTE_ORDER == __LITTLE_ENDIAN
4953 ldr_us(r0, r2);
4954 ldxi_c(r2, r2, 2);
4955 lshi(r2, r2, 16);
4956#else
4957 ldr_c(r0, r2);
4958 lshi(r0, r0, 16);
4959 ldxi_us(r2, r2, 1);
4960#endif
4961 break;
4962#if __WORDSIZE == 32
4963 default:
4964 ldr_i(r0, r1);
4965 break;
4966#else
4967 case 4:
4968 ldr_i(r0, r1);
4969 break;
4970 case 5:
4971# if __BYTE_ORDER == __LITTLE_ENDIAN
4972 ldr_ui(r0, r2);
4973 ldxi_c(r2, r2, 4);
4974 lshi(r2, r2, 32);
4975# else
4976 ldr_i(r0, r2);
4977 lshi(r0, r0, 8);
4978 ldxi_uc(r2, r2, 4);
4979# endif
4980 break;
4981 case 6:
4982# if __BYTE_ORDER == __LITTLE_ENDIAN
4983 ldr_ui(r0, r2);
4984 ldxi_s(r2, r2, 4);
4985 lshi(r2, r2, 32);
4986# else
4987 ldr_i(r0, r2);
4988 lshi(r0, r0, 16);
4989 ldxi_us(r2, r2, 4);
4990# endif
4991 break;
4992 case 7:
4993# if __BYTE_ORDER == __LITTLE_ENDIAN
4994 ldr_ui(r0, r2);
4995 ldxi_i(r2, r2, 4);
4996 lshi(r2, r2, 40);
4997 rshi(r2, r2, 8);
4998# else
4999 ldr_i(r0, r2);
5000 lshi(r0, r0, 24);
5001 ldxi_ui(r2, r2, 4);
5002 rshi(r2, r2, 8);
5003# endif
5004 break;
5005 default:
5006 ldr_l(r0, r1);
5007 break;
5008#endif
5009 }
5010 if (i0 & (i0 - 1)) {
5011 orr(r0, r0, r2);
5012 jit_unget_reg(t0);
5013 }
5014}
5015
5016static void
5017_generic_unldi(jit_state_t *_jit,
5018 jit_int32_t r0, jit_word_t i0, jit_word_t i1)
5019{
5020 jit_int32_t t0, r2;
5021 assert(i1 >= 1 && i1 <= sizeof(jit_word_t));
5022 if (i1 & (i1 - 1)) {
5023 t0 = jit_get_reg(jit_class_gpr);
5024 r2 = rn(t0);
5025 }
5026 switch (i1) {
5027 case 1:
5028 ldi_c(r0, i0);
5029 break;
5030 case 2:
5031 ldi_s(r0, i0);
5032 break;
5033 case 3:
5034# if __BYTE_ORDER == __LITTLE_ENDIAN
5035 ldi_us(r0, i0);
5036 ldi_c(r2, i0 + 2);
5037 lshi(r2, r2, 16);
5038# else
5039 ldi_c(r0, i0);
5040 lshi(r0, r0, 16);
5041 ldi_us(r2, i0 + 1);
5042# endif
5043 break;
5044# if __WORDSIZE == 32
5045 default:
5046 ldi_i(r0, i0);
5047 break;
5048# else
5049 case 4:
5050 ldi_i(r0, i0);
5051 break;
5052 case 5:
5053# if __BYTE_ORDER == __LITTLE_ENDIAN
5054 ldi_ui(r0, i0);
5055 ldi_c(r2, i0 + 4);
5056 lshi(r2, r2, 32);
5057# else
5058 ldi_i(r0, i0);
5059 lshi(r0, r0, 8);
5060 ldi_uc(r2, i0 + 4);
5061# endif
5062 break;
5063 case 6:
5064# if __BYTE_ORDER == __LITTLE_ENDIAN
5065 ldi_ui(r0, i0);
5066 ldi_s(r2, i0 + 4);
5067 lshi(r2, r2, 32);
5068# else
5069 ldi_i(r0, i0);
5070 lshi(r0, r0, 16);
5071 ldi_us(r2, i0 + 4);
5072# endif
5073 break;
5074 case 7:
5075# if __BYTE_ORDER == __LITTLE_ENDIAN
5076 ldi_ui(r0, i0);
5077 ldi_i(r2, i0 + 4);
5078 lshi(r2, r2, 40);
5079 rshi(r2, r2, 8);
5080# else
5081 ldi_i(r0, i0);
5082 lshi(r0, r0, 24);
5083 ldi_ui(r2, i0 + 4);
5084 rshi(r2, r2, 8);
5085# endif
5086 break;
5087 default:
5088 ldi_l(r0, i0);
5089 break;
5090# endif
5091 }
5092 if (i1 & (i1 - 1)) {
5093 orr(r0, r0, r2);
5094 jit_unget_reg(t0);
5095 }
5096}
5097
5098static void
5099_generic_unldr_u(jit_state_t *_jit,
5100 jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
5101{
5102 jit_int32_t t0, r2;
5103 assert(i0 >= 1 && i0 <= sizeof(jit_word_t));
5104 if (i0 & (i0 - 1)) {
5105 t0 = jit_get_reg(jit_class_gpr);
5106 r2 = rn(t0);
5107 movr(r2, r1);
5108 }
5109 switch (i0) {
5110 case 1:
5111 ldr_uc(r0, r1);
5112 break;
5113 case 2:
5114 ldr_us(r0, r1);
5115 break;
5116 case 3:
5117# if __BYTE_ORDER == __LITTLE_ENDIAN
5118 ldr_us(r0, r2);
5119 ldxi_uc(r2, r2, 2);
5120 lshi(r2, r2, 16);
5121# else
5122 ldr_uc(r0, r2);
5123 lshi(r0, r0, 16);
5124 ldxi_us(r2, r2, 1);
5125# endif
5126 break;
5127# if __WORDSIZE == 32
5128 default:
5129 ldr_i(r0, r1);
5130 break;
5131# else
5132 case 4:
5133 ldr_ui(r0, r1);
5134 break;
5135 case 5:
5136# if __BYTE_ORDER == __LITTLE_ENDIAN
5137 ldr_ui(r0, r2);
5138 ldxi_uc(r2, r2, 4);
5139 lshi(r2, r2, 32);
5140# else
5141 ldr_ui(r0, r2);
5142 lshi(r0, r0, 8);
5143 ldxi_uc(r2, r2, 4);
5144# endif
5145 break;
5146 case 6:
5147# if __BYTE_ORDER == __LITTLE_ENDIAN
5148 ldr_ui(r0, r2);
5149 ldxi_us(r2, r2, 4);
5150 lshi(r2, r2, 32);
5151# else
5152 ldr_ui(r0, r2);
5153 lshi(r0, r0, 16);
5154 ldxi_us(r2, r2, 4);
5155# endif
5156 break;
5157 case 7:
5158# if __BYTE_ORDER == __LITTLE_ENDIAN
5159 ldr_ui(r0, r2);
5160 ldxi_ui(r2, r2, 4);
5161 lshi(r2, r2, 40);
5162 rshi_u(r2, r2, 8);
5163# else
5164 ldr_ui(r0, r2);
5165 lshi(r0, r0, 24);
5166 ldxi_ui(r2, r2, 4);
5167 rshi(r2, r2, 8);
5168# endif
5169 break;
5170 default:
5171 ldr_l(r0, r1);
5172 break;
5173# endif
5174 }
5175 if (i0 & (i0 - 1)) {
5176 orr(r0, r0, r2);
5177 jit_unget_reg(t0);
5178 }
5179}
5180
5181static void
5182_generic_unldi_u(jit_state_t *_jit,
5183 jit_int32_t r0, jit_word_t i0, jit_word_t i1)
5184{
5185 jit_int32_t t0, r2;
5186 assert(i1 >= 1 && i1 <= sizeof(jit_word_t));
5187 if (i1 & (i1 - 1)) {
5188 t0 = jit_get_reg(jit_class_gpr);
5189 r2 = rn(t0);
5190 }
5191 switch (i1) {
5192 case 1:
5193 ldi_uc(r0, i0);
5194 break;
5195 case 2:
5196 ldi_us(r0, i0);
5197 break;
5198 case 3:
5199# if __BYTE_ORDER == __LITTLE_ENDIAN
5200 ldi_us(r0, i0);
5201 ldi_uc(r2, i0 + 2);
5202 lshi(r2, r2, 16);
5203# else
5204 ldi_uc(r0, i0);
5205 lshi(r0, r0, 16);
5206 ldi_us(r2, i0 + 1);
5207# endif
5208 break;
5209# if __WORDSIZE == 32
5210 default:
5211 ldi_i(r0, i0);
5212 break;
5213# else
5214 case 4:
5215 ldi_ui(r0, i0);
5216 break;
5217 case 5:
5218# if __BYTE_ORDER == __LITTLE_ENDIAN
5219 ldi_ui(r0, i0);
5220 ldi_uc(r2, i0 + 4);
5221 lshi(r2, r2, 32);
5222# else
5223 ldi_ui(r0, i0);
5224 lshi(r0, r0, 8);
5225 ldi_uc(r2, i0 + 4);
5226# endif
5227 break;
5228 case 6:
5229# if __BYTE_ORDER == __LITTLE_ENDIAN
5230 ldi_ui(r0, i0);
5231 ldi_us(r2, i0 + 4);
5232 lshi(r2, r2, 32);
5233# else
5234 ldi_ui(r0, i0);
5235 lshi(r0, r0, 16);
5236 ldi_us(r2, i0 + 4);
5237# endif
5238 break;
5239 case 7:
5240# if __BYTE_ORDER == __LITTLE_ENDIAN
5241 ldi_ui(r0, i0);
5242 ldi_ui(r2, i0 + 4);
5243 lshi(r2, r2, 40);
5244 rshi_u(r2, r2, 8);
5245# else
5246 ldi_ui(r0, i0);
5247 lshi(r0, r0, 24);
5248 ldi_ui(r2, i0 + 4);
5249 rshi(r2, r2, 8);
5250# endif
5251 break;
5252 default:
5253 ldi_l(r0, i0);
5254 break;
5255# endif
5256 }
5257 if (i1 & (i1 - 1)) {
5258 orr(r0, r0, r2);
5259 jit_unget_reg(t0);
5260 }
5261}
5262
5263static void
5264_generic_unstr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
5265{
5266 jit_word_t t0, r2;
5267 assert(i0 > 0 && i0 <= sizeof(jit_word_t));
5268 if (i0 & (i0 - 1)) {
5269 t0 = jit_get_reg(jit_class_gpr);
5270 r2 = rn(t0);
5271 }
5272 switch (i0) {
5273 case 1:
5274 str_c(r0, r1);
5275 break;
5276 case 2:
5277 str_s(r0, r1);
5278 break;
5279 case 3:
5280# if __BYTE_ORDER == __LITTLE_ENDIAN
5281 str_s(r0, r1);
5282 rshi(r2, r1, 16);
5283 stxi_c(2, r0, r2);
5284# else
5285 stxi_c(2, r0, r1);
5286 rshi(r2, r1, 8);
5287 str_s(r0, r2);
5288# endif
5289 break;
5290# if __WORDSIZE == 32
5291 default:
5292 str_i(r0, r1);
5293 break;
5294# else
5295 case 4:
5296 str_i(r0, r1);
5297 break;
5298 case 5:
5299# if __BYTE_ORDER == __LITTLE_ENDIAN
5300 str_i(r0, r1);
5301 rshi(r2, r1, 32);
5302 stxi_c(4, r0, r2);
5303# else
5304 stxi_c(4, r0, r1);
5305 rshi(r2, r1, 8);
5306 str_i(r0, r2);
5307# endif
5308 break;
5309 case 6:
5310# if __BYTE_ORDER == __LITTLE_ENDIAN
5311 str_i(r0, r1);
5312 rshi(r2, r1, 32);
5313 stxi_s(4, r0, r2);
5314# else
5315 stxi_s(4, r0, r1);
5316 rshi(r2, r1, 16);
5317 str_i(r0, r2);
5318# endif
5319 break;
5320 case 7:
5321# if __BYTE_ORDER == __LITTLE_ENDIAN
5322 str_i(r0, r1);
5323 rshi(r2, r1, 32);
5324 stxi_s(4, r0, r2);
5325 rshi(r2, r2, 16);
5326 stxi_c(6, r0, r2);
5327# else
5328 stxi_c(6, r0, r1);
5329 rshi(r2, r1, 8);
5330 stxi_s(4, r0, r2);
5331 rshi(r2, r2, 16);
5332 str_i(r0, r2);
5333# endif
5334 break;
5335 default:
5336 str_l(r0, r1);
5337 break;
5338# endif
5339 }
5340 if (i0 & (i0 - 1))
5341 jit_unget_reg(t0);
5342}
5343
5344static void
5345_generic_unsti(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
5346{
5347 jit_word_t t0, r2;
5348 assert(i1 > 0 && i1 <= sizeof(jit_word_t));
5349 if (i1 & (i1 - 1)) {
5350 t0 = jit_get_reg(jit_class_gpr);
5351 r2 = rn(t0);
5352 }
5353 switch (i1) {
5354 case 1:
5355 sti_c(i0, r0);
5356 break;
5357 case 2:
5358 sti_s(i0, r0);
5359 break;
5360 case 3:
5361# if __BYTE_ORDER == __LITTLE_ENDIAN
5362 sti_s(i0, r0);
5363 rshi(r2, r0, 16);
5364 sti_c(2 + i0, r2);
5365# else
5366 sti_c(2 + i0, r0);
5367 rshi(r2, r0, 8);
5368 sti_s(i0, r2);
5369# endif
5370 break;
5371# if __WORDSIZE == 32
5372 default:
5373 sti_i(i0, r0);
5374 break;
5375# else
5376 case 4:
5377 sti_i(i0, r0);
5378 break;
5379 case 5:
5380# if __BYTE_ORDER == __LITTLE_ENDIAN
5381 sti_i(i0, r0);
5382 rshi(r2, r0, 32);
5383 sti_c(4 + i0, r2);
5384# else
5385 stxi_c(4, i0, r0);
5386 rshi(r2, r0, 8);
5387 sti_i(i0, r2);
5388# endif
5389 break;
5390 case 6:
5391# if __BYTE_ORDER == __LITTLE_ENDIAN
5392 sti_i(i0, r0);
5393 rshi(r2, r0, 32);
5394 sti_s(4 + i0, r2);
5395# else
5396 sti_s(4 + i0, r0);
5397 rshi(r2, r0, 16);
5398 sti_i(i0, r2);
5399# endif
5400 break;
5401 case 7:
5402# if __BYTE_ORDER == __LITTLE_ENDIAN
5403 sti_i(i0, r0);
5404 rshi(r2, r0, 32);
5405 sti_s(4 + i0, r2);
5406 rshi(r2, r2, 16);
5407 sti_c(6 + i0, r2);
5408# else
5409 sti_c(6 + i0, r0);
5410 rshi(r2, r0, 8);
5411 sti_s(4 + i0, r2);
5412 rshi(r2, r2, 16);
5413 sti_i(i0, r2);
5414# endif
5415 break;
5416 default:
5417 sti_l(i0, r0);
5418 break;
5419# endif
5420 }
5421 if (i1 & (i1 - 1))
5422 jit_unget_reg(t0);
5423}
5424
5425#if !defined(__i386__) && !defined(__x86_64__) && !defined(__arm__)
5426static void
5427_generic_unldr_x(jit_state_t *_jit,
5428 jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
5429{
5430 assert(i0 == 4 || i0 == 8);
5431 if (i0 == 4)
5432 ldr_f(r0, r1);
5433 else
5434 ldr_d(r0, r1);
5435}
5436
5437static void
5438_generic_unldi_x(jit_state_t *_jit,
5439 jit_int32_t r0, jit_word_t i0, jit_word_t i1)
5440{
5441 assert(i1 == 4 || i1 == 8);
5442 if (i1 == 4)
5443 ldi_f(r0, i0);
5444 else
5445 ldi_d(r0, i0);
5446}
5447
5448static void
5449_generic_unstr_x(jit_state_t *_jit,
5450 jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
5451{
5452 assert(i0 == 4 || i0 == 8);
5453 if (i0 == 4)
5454 str_f(r0, r1);
5455 else
5456 str_d(r0, r1);
5457}
5458
5459static void
5460_generic_unsti_x(jit_state_t *_jit,
5461 jit_word_t i0, jit_int32_t r0, jit_word_t i1)
5462{
5463 assert(i1 == 4 || i1 == 8);
5464 if (i1 == 4)
5465 sti_f(i0, r0);
5466 else
5467 sti_d(i0, r0);
5468}
5469#endif
5470
79bfeef6
PC		 5471#if defined(stack_framesize)
5472static maybe_unused void
5473_patch_alist(jit_state_t *_jit, jit_bool_t revert)
5474{
5475 jit_int32_t diff;
5476 jit_node_t *node;
5477 diff = jit_diffsize();
5478 if (diff) {
5479 if (revert)
5480 diff = -diff;
5481 for (node = _jitc->function->alist; node; node = node->link) {
5482 switch (node->code) {
5483 case jit_code_ldxi_c: case jit_code_ldxi_uc:
5484 case jit_code_ldxi_s: case jit_code_ldxi_us:
5485 case jit_code_ldxi_i:
5486#if __WORDSIZE == 64
5487 case jit_code_ldxi_ui: case jit_code_ldxi_l:
5488#endif
5489 case jit_code_ldxi_f: case jit_code_ldxi_d:
5490 node->w.w -= diff;
5491 break;
5492 case jit_code_stxi_c: case jit_code_stxi_s:
5493 case jit_code_stxi_i:
5494#if __WORDSIZE == 64
5495 case jit_code_stxi_l:
5496#endif
5497 case jit_code_stxi_f: case jit_code_stxi_d:
5498 node->u.w -= diff;
5499 break;
5500 default:
5501 abort();
5502 }
5503 }
5504 }
5505}
5506#endif
ARM optimized PCSX fork