notaz.gp2x.de / pcsx_rearmed.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
git subrepo pull (merge) --force deps/lightning
[pcsx_rearmed.git] / deps / lightning / lib / lightning.c
CommitLineData
4a71579b 1/*
79bfeef6 2 * Copyright (C) 2012-2023 Free Software Foundation, Inc.
4a71579b
PC		 3 *
4 * This file is part of GNU lightning.
5 *
6 * GNU lightning is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU Lesser General Public License as published
8 * by the Free Software Foundation; either version 3, or (at your option)
9 * any later version.
10 *
11 * GNU lightning is distributed in the hope that it will be useful, but
12 * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
13 * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public
14 * License for more details.
15 *
16 * Authors:
17 * Paulo Cesar Pereira de Andrade
18 */
19
20#include <lightning.h>
21#include <lightning/jit_private.h>
40a44dcb
PC		 22#if HAVE_MMAP
23# include <sys/mman.h>
24#endif
4a71579b
PC		 25#if defined(__sgi)
26# include <fcntl.h>
27#endif
28
29#ifndef MAP_ANON
30# define MAP_ANON MAP_ANONYMOUS
31# ifndef MAP_ANONYMOUS
32# define MAP_ANONYMOUS 0
33# endif
34#endif
35
36#define jit_regload_reload 0 /* convert to reload */
37#define jit_regload_delete 1 /* just remove node */
38#define jit_regload_isdead 2 /* delete and unset live bit */
39
40/*
41 * Prototypes
42 */
43static jit_word_t hash_data(const void*, jit_word_t);
44
45#define new_pool() _new_pool(_jit)
46static void _new_pool(jit_state_t*);
47
48#define new_node(u) _new_node(_jit, u)
49static jit_node_t *_new_node(jit_state_t*, jit_code_t);
50
51#define link_node(u) _link_node(_jit, u)
52static inline jit_node_t *_link_node(jit_state_t*, jit_node_t*);
53
54#define del_node(u, v) _del_node(_jit, u, v)
55static inline void _del_node(jit_state_t*, jit_node_t*, jit_node_t*);
56
57#define free_node(u) _free_node(_jit, u)
58static inline void _free_node(jit_state_t*, jit_node_t*);
59
60#define del_label(u, v) _del_label(_jit, u, v)
61static void _del_label(jit_state_t*, jit_node_t*, jit_node_t*);
62
63#define jit_dataset() _jit_dataset(_jit)
64static void
65_jit_dataset(jit_state_t *_jit);
66
c0c16242
PC		 67#define block_update_set(block, target) _block_update_set(_jit, block, target)
68static jit_bool_t _block_update_set(jit_state_t*, jit_block_t*, jit_block_t*);
69
24d91c0d
PC		 70#define propagate_backward(block) _propagate_backward(_jit, block)
71static void _propagate_backward(jit_state_t*, jit_block_t*);
72
c0c16242
PC		 73#define check_block_again() _check_block_again(_jit)
74static jit_bool_t _check_block_again(jit_state_t*);
75
76#define do_setup() _do_setup(_jit)
77static void _do_setup(jit_state_t*);
78
4a71579b
PC		 79#define jit_setup(block) _jit_setup(_jit, block)
80static void
81_jit_setup(jit_state_t *_jit, jit_block_t *block);
82
c0c16242
PC		 83#define do_follow(always) _do_follow(_jit, always)
84static void _do_follow(jit_state_t*, jit_bool_t);
85
86#define jit_follow(block) _jit_follow(_jit, block)
4a71579b 87static void
c0c16242 88_jit_follow(jit_state_t *_jit, jit_block_t *block);
4a71579b
PC		 89
90#define jit_update(node, live, mask) _jit_update(_jit, node, live, mask)
91static void
92_jit_update(jit_state_t *_jit, jit_node_t *node,
93 jit_regset_t *live, jit_regset_t *mask);
94
95#define thread_jumps() _thread_jumps(_jit)
96static void
97_thread_jumps(jit_state_t *_jit);
98
99#define sequential_labels() _sequential_labels(_jit)
100static void
101_sequential_labels(jit_state_t *_jit);
102
103#define split_branches() _split_branches(_jit)
104static void
105_split_branches(jit_state_t *_jit);
106
107#define shortcut_jump(prev, node) _shortcut_jump(_jit, prev, node)
108static jit_bool_t
109_shortcut_jump(jit_state_t *_jit, jit_node_t *prev, jit_node_t *node);
110
111#define redundant_jump(prev, node) _redundant_jump(_jit, prev, node)
112static jit_bool_t
113_redundant_jump(jit_state_t *_jit, jit_node_t *prev, jit_node_t *node);
114
115static jit_code_t
116reverse_jump_code(jit_code_t code);
117
118#define reverse_jump(prev, node) _reverse_jump(_jit, prev, node)
119static jit_bool_t
120_reverse_jump(jit_state_t *_jit, jit_node_t *prev, jit_node_t *node);
121
122#define redundant_store(node, jump) _redundant_store(_jit, node, jump)
ba3814c1 123static jit_bool_t
4a71579b
PC		 124_redundant_store(jit_state_t *_jit, jit_node_t *node, jit_bool_t jump);
125
126#define simplify_movr(p, n, k, s) _simplify_movr(_jit, p, n, k, s)
127static jit_bool_t
128_simplify_movr(jit_state_t *_jit, jit_node_t *prev, jit_node_t *node,
129 jit_int32_t kind, jit_int32_t size);
130
131#define simplify_movi(p, n, k, s) _simplify_movi(_jit, p, n, k, s)
132static jit_bool_t
133_simplify_movi(jit_state_t *_jit, jit_node_t *prev, jit_node_t *node,
134 jit_int32_t kind, jit_int32_t size);
135
136#define simplify_ldxi(prev, node) _simplify_ldxi(_jit, prev, node)
137static jit_bool_t
138_simplify_ldxi(jit_state_t *_jit, jit_node_t *prev, jit_node_t *node);
139
140#define simplify_stxi(prev, node) _simplify_stxi(_jit, prev, node)
141static jit_bool_t
142_simplify_stxi(jit_state_t *_jit, jit_node_t *prev, jit_node_t *node);
143
144#define simplify_spill(node, regno) _simplify_spill(_jit, node, regno)
145static void
146_simplify_spill(jit_state_t *_jit, jit_node_t *node, jit_int32_t regno);
147
148#define simplify() _simplify(_jit)
ba3814c1 149static jit_bool_t
4a71579b
PC		 150_simplify(jit_state_t *_jit);
151
152#define jit_reg_undef -1
153#define jit_reg_static 0
154#define jit_reg_change 1
155#define register_change_p(n, l, r) _register_change_p(_jit, n, l, r)
156static jit_int32_t
157_register_change_p(jit_state_t *_jit, jit_node_t *node, jit_node_t *link,
158 jit_int32_t regno);
159
160#define spill_reglive_p(node, regno) _spill_reglive_p(_jit, node, regno)
161static jit_bool_t
162_spill_reglive_p(jit_state_t *_jit, jit_node_t *node, jit_int32_t regno);
163
164#define patch_registers() _patch_registers(_jit)
165static void
166_patch_registers(jit_state_t *_jit);
167
168#define patch_register(n,l,r,p) _patch_register(_jit,n,l,r,p)
169static void
170_patch_register(jit_state_t *jit, jit_node_t *node, jit_node_t *link,
171 jit_int32_t regno, jit_int32_t patch);
172
173/*
174 * Initialization
175 */
176#if !defined(__sgi)
177#define mmap_fd -1
178#endif
179
180/*
181 * Implementation
182 */
183void
b68d544b 184init_jit_with_debug(const char *progname, FILE *dbg_out)
4a71579b
PC		 185{
186 jit_get_cpu();
b68d544b 187 jit_init_debug(progname, dbg_out);
4a71579b
PC		 188 jit_init_size();
189}
190
b68d544b
PC		 191void
192init_jit(const char *progname)
193{
194 init_jit_with_debug(progname, stderr);
195}
196
4a71579b
PC		 197void
198finish_jit(void)
199{
200 jit_finish_debug();
201 jit_finish_size();
202}
203
204jit_int32_t
205_jit_get_reg(jit_state_t *_jit, jit_int32_t regspec)
206{
207 jit_int32_t spec;
208 jit_int32_t regno;
209
210 spec = regspec & ~(jit_class_chk|jit_class_nospill);
211 if (spec & jit_class_named) {
212 regno = jit_regno(spec);
213 if (jit_regset_tstbit(&_jitc->regsav, regno))
214 /* fail if register is spilled */
215 goto fail;
216 if (jit_regset_tstbit(&_jitc->regarg, regno))
217 /* fail if register is an argument to current instruction */
218 goto fail;
219 if (jit_regset_tstbit(&_jitc->reglive, regno)) {
220 if (regspec & jit_class_nospill)
221 /* fail if register is live and should not spill/reload */
222 goto fail;
223 goto spill;
224 }
225 jit_regset_setbit(&_jitc->regarg, regno);
226 return (regno);
227 }
228 else
229 assert(jit_class(spec) != 0);
230
231 if (_jitc->emit) {
232 /* search for a free register matching spec */
233 for (regno = 0; regno < _jitc->reglen; regno++) {
234 if ((jit_class(_rvs[regno].spec) & spec) == spec &&
235 !jit_regset_tstbit(&_jitc->regarg, regno) &&
79bfeef6
PC		 236 !jit_regset_tstbit(&_jitc->reglive, regno)) {
237 if (jit_regset_tstbit(&_jitc->regmask, regno)) {
238 /* search further, attempting to find a truly known
239 * free register, not just one in unknown state. */
240 jit_int32_t regfree;
241
242 for (regfree = regno + 1;
243 regfree < _jitc->reglen; regfree++) {
244 if ((jit_class(_rvs[regfree].spec) & spec) == spec &&
245 !jit_regset_tstbit(&_jitc->regarg, regfree) &&
246 !jit_regset_tstbit(&_jitc->reglive, regfree) &&
247 !jit_regset_tstbit(&_jitc->regmask, regfree)) {
248 regno = regfree;
249 break;
250 }
251 }
252 }
4a71579b 253 goto regarg;
79bfeef6 254 }
4a71579b
PC		 255 }
256
257 /* search for a register matching spec that is not an argument
258 * for the current instruction */
259 for (regno = 0; regno < _jitc->reglen; regno++) {
260 if ((jit_class(_rvs[regno].spec) & spec) == spec &&
261 !jit_regset_tstbit(&_jitc->regsav, regno) &&
262 !jit_regset_tstbit(&_jitc->regarg, regno) &&
263 !(regspec & jit_class_nospill)) {
264 spill:
265 assert(_jitc->function != NULL);
266 if (spec & jit_class_gpr) {
267 if (!_jitc->function->regoff[regno]) {
268 _jitc->function->regoff[regno] =
269 jit_allocai(sizeof(jit_word_t));
270 _jitc->again = 1;
271 }
272#if DEBUG
273 /* emit_stxi must not need temporary registers */
274 assert(!_jitc->getreg);
275 _jitc->getreg = 1;
276#endif
277 emit_stxi(_jitc->function->regoff[regno], JIT_FP, regno);
278#if DEBUG
279 _jitc->getreg = 0;
280#endif
281 }
282 else {
283 if (!_jitc->function->regoff[regno]) {
284 _jitc->function->regoff[regno] =
285 jit_allocai(sizeof(jit_float64_t));
286 _jitc->again = 1;
287 }
288#if DEBUG
289 /* emit_stxi must not need temporary registers */
290 assert(!_jitc->getreg);
291 _jitc->getreg = 1;
292#endif
293 emit_stxi_d(_jitc->function->regoff[regno], JIT_FP, regno);
294#if DEBUG
295 _jitc->getreg = 0;
296#endif
297 }
298 jit_regset_setbit(&_jitc->regsav, regno);
299 regarg:
300 jit_regset_setbit(&_jitc->regarg, regno);
301 if (jit_class(_rvs[regno].spec) & jit_class_sav) {
302 /* if will modify callee save registers without a
303 * function prolog, better patch this assertion */
304 assert(_jitc->function != NULL);
305 if (!jit_regset_tstbit(&_jitc->function->regset, regno)) {
306 jit_regset_setbit(&_jitc->function->regset, regno);
307 _jitc->again = 1;
308 }
309 }
310 return (regno);
311 }
312 }
313 }
314 else {
315 /* nospill hint only valid during emit" */
316 assert(!(regspec & jit_class_nospill));
317 for (regno = 0; regno < _jitc->reglen; regno++) {
318 if ((jit_class(_rvs[regno].spec) & spec) == spec &&
319 !jit_regset_tstbit(&_jitc->regsav, regno) &&
320 !jit_regset_tstbit(&_jitc->regarg, regno)) {
321 jit_regset_setbit(&_jitc->regarg, regno);
322 jit_regset_setbit(&_jitc->regsav, regno);
323 jit_save(regno);
324 return (jit_regno_patch|regno);
325 }
326 }
327 }
328
329 /* Out of hardware registers */
330fail:
331 assert(regspec & jit_class_chk);
332 return (JIT_NOREG);
333}
334
335void
336_jit_unget_reg(jit_state_t *_jit, jit_int32_t regno)
337{
338 regno = jit_regno(regno);
339 if (jit_regset_tstbit(&_jitc->regsav, regno)) {
340 if (_jitc->emit) {
341#if DEBUG
342 /* emit_ldxi must not need a temporary register */
343 assert(!_jitc->getreg);
344 _jitc->getreg = 1;
345#endif
346 if (jit_class(_rvs[regno].spec) & jit_class_gpr)
347 emit_ldxi(regno, JIT_FP, _jitc->function->regoff[regno]);
348 else
349 emit_ldxi_d(regno, JIT_FP, _jitc->function->regoff[regno]);
350#if DEBUG
351 /* emit_ldxi must not need a temporary register */
352 _jitc->getreg = 0;
353#endif
354 }
355 else
356 jit_load(regno);
357 jit_regset_clrbit(&_jitc->regsav, regno);
358 }
359#if defined(jit_carry)
360 assert((regno == jit_carry /*&& _NOREG != jit_carry*/) ||
361 jit_regset_tstbit(&_jitc->regarg, regno) != 0);
362#else
363 assert(jit_regset_tstbit(&_jitc->regarg, regno) != 0);
364#endif
365 jit_regset_clrbit(&_jitc->regarg, regno);
366}
367
368jit_bool_t
369_jit_callee_save_p(jit_state_t *_jit, jit_int32_t regno)
370{
371 assert(regno >= 0 && regno < JIT_NOREG);
372 return (!!(_rvs[regno].spec & jit_class_sav));
373}
374
375extern jit_bool_t
376_jit_pointer_p(jit_state_t *_jit, jit_pointer_t address)
377{
378 return ((jit_uint8_t *)address >= _jit->code.ptr &&
379 (jit_word_t)address < _jit->pc.w);
380}
381
382#if __ia64__
383void
384jit_regset_com(jit_regset_t *u, jit_regset_t *v)
385{
386 u->rl = ~v->rl; u->rh = ~v->rh;
387 u->fl = ~v->fl; u->fh = ~v->fh;
388}
389
390void
391jit_regset_and(jit_regset_t *u, jit_regset_t *v, jit_regset_t *w)
392{
393 u->rl = v->rl & w->rl; u->rh = v->rh & w->rh;
394 u->fl = v->fl & w->fl; u->fh = v->fh & w->fh;
395}
396
397void
398jit_regset_ior(jit_regset_t *u, jit_regset_t *v, jit_regset_t *w)
399{
400 u->rl = v->rl | w->rl; u->rh = v->rh | w->rh;
401 u->fl = v->fl | w->fl; u->fh = v->fh | w->fh;
402}
403
404void
405jit_regset_xor(jit_regset_t *u, jit_regset_t *v, jit_regset_t *w)
406{
407 u->rl = v->rl ^ w->rl; u->rh = v->rh ^ w->rh;
408 u->fl = v->fl ^ w->fl; u->fh = v->fh ^ w->fh;
409}
410
411void
412jit_regset_set(jit_regset_t *u, jit_regset_t *v)
413{
414 u->rl = v->rl; u->rh = v->rh;
415 u->fl = v->fl; u->fh = v->fh;
416}
417
418void
419jit_regset_set_mask(jit_regset_t *u, jit_int32_t v)
420{
421 jit_bool_t w = !!(v & (v - 1));
422
423 assert(v >= 0 && v <= 256);
424 if (v == 0)
425 u->rl = u->rh = u->fl = u->fh = -1LL;
426 else if (v <= 64) {
427 u->rl = w ? (1LL << v) - 1 : -1LL;
428 u->rh = u->fl = u->fh = 0;
429 }
430 else if (v <= 128) {
431 u->rl = -1LL;
432 u->rh = w ? (1LL << (v - 64)) - 1 : -1LL;
433 u->fl = u->fh = 0;
434 }
435 else if (v <= 192) {
436 u->rl = u->rh = -1LL;
437 u->fl = w ? (1LL << (v - 128)) - 1 : -1LL;
438 u->fh = 0;
439 }
440 else {
441 u->rl = u->rh = u->fl = -1LL;
442 u->fh = w ? (1LL << (v - 128)) - 1 : -1LL;
443 }
444}
445
446jit_bool_t
447jit_regset_cmp_ui(jit_regset_t *u, jit_word_t v)
448{
449 return !((u->rl == v && u->rh == 0 && u->fl == 0 && u->fh == 0));
450}
451
452void
453jit_regset_set_ui(jit_regset_t *u, jit_word_t v)
454{
455 u->rl = v;
456 u->rh = u->fl = u->fh = 0;
457}
458
459jit_bool_t
460jit_regset_set_p(jit_regset_t *u)
461{
462 return (u->rl || u->rh || u->fl || u->fh);
463}
464
465void
466jit_regset_clrbit(jit_regset_t *set, jit_int32_t bit)
467{
468 assert(bit >= 0 && bit <= 255);
469 if (bit < 64)
470 set->rl &= ~(1LL << bit);
471 else if (bit < 128)
472 set->rh &= ~(1LL << (bit - 64));
473 else if (bit < 192)
474 set->fl &= ~(1LL << (bit - 128));
475 else
476 set->fh &= ~(1LL << (bit - 192));
477}
478
479void
480jit_regset_setbit(jit_regset_t *set, jit_int32_t bit)
481{
482 assert(bit >= 0 && bit <= 255);
483 if (bit < 64)
484 set->rl |= 1LL << bit;
485 else if (bit < 128)
486 set->rh |= 1LL << (bit - 64);
487 else if (bit < 192)
488 set->fl |= 1LL << (bit - 128);
489 else
490 set->fh |= 1LL << (bit - 192);
491}
492
493jit_bool_t
494jit_regset_tstbit(jit_regset_t *set, jit_int32_t bit)
495{
496 assert(bit >= 0 && bit <= 255);
497 if (bit < 64)
498 return (!!(set->rl & (1LL << bit)));
499 else if (bit < 128)
500 return (!!(set->rh & (1LL << (bit - 64))));
501 else if (bit < 192)
502 return (!!(set->fl & (1LL << (bit - 128))));
503 return (!!(set->fh & (1LL << (bit - 192))));
504}
505
506unsigned long
507jit_regset_scan1(jit_regset_t *set, jit_int32_t offset)
508{
509 assert(offset >= 0 && offset <= 255);
510 for (; offset < 64; offset++) {
511 if (set->rl & (1LL << offset))
512 return (offset);
513 }
514 for (; offset < 128; offset++) {
515 if (set->rh & (1LL << (offset - 64)))
516 return (offset);
517 }
518 for (; offset < 192; offset++) {
519 if (set->fl & (1LL << (offset - 128)))
520 return (offset);
521 }
522 for (; offset < 256; offset++) {
523 if (set->fh & (1LL << (offset - 192)))
524 return (offset);
525 }
526 return (ULONG_MAX);
527}
528
529#elif __sparc__ && __WORDSIZE == 64
530void
531jit_regset_com(jit_regset_t *u, jit_regset_t *v)
532{
533 u->rl = ~v->rl; u->rh = ~v->rh;
534}
535
536void
537jit_regset_and(jit_regset_t *u, jit_regset_t *v, jit_regset_t *w)
538{
539 u->rl = v->rl & w->rl; u->rh = v->rh & w->rh;
540}
541
542void
543jit_regset_ior(jit_regset_t *u, jit_regset_t *v, jit_regset_t *w)
544{
545 u->rl = v->rl | w->rl; u->rh = v->rh | w->rh;
546}
547
548void
549jit_regset_xor(jit_regset_t *u, jit_regset_t *v, jit_regset_t *w)
550{
551 u->rl = v->rl ^ w->rl; u->rh = v->rh ^ w->rh;
552}
553
554void
555jit_regset_set(jit_regset_t *u, jit_regset_t *v)
556{
557 u->rl = v->rl; u->rh = v->rh;
558}
559
560void
561jit_regset_set_mask(jit_regset_t *u, jit_int32_t v)
562{
563 jit_bool_t w = !!(v & (v - 1));
564
565 assert(v >= 0 && v <= 128);
566 if (v == 0)
567 u->rl = u->rh = -1LL;
568 else if (v <= 64) {
569 u->rl = w ? (1LL << v) - 1 : -1LL;
570 u->rh = 0;
571 }
572 else {
573 u->rl = -1LL;
574 u->rh = w ? (1LL << (v - 64)) - 1 : -1LL;
575 }
576}
577
578jit_bool_t
579jit_regset_cmp_ui(jit_regset_t *u, jit_word_t v)
580{
581 return !((u->rl == v && u->rh == 0));
582}
583
584void
585jit_regset_set_ui(jit_regset_t *u, jit_word_t v)
586{
587 u->rl = v;
588 u->rh = 0;
589}
590
591jit_bool_t
592jit_regset_set_p(jit_regset_t *u)
593{
594 return (u->rl || u->rh);
595}
596
597void
598jit_regset_clrbit(jit_regset_t *set, jit_int32_t bit)
599{
600 assert(bit >= 0 && bit <= 128);
601 if (bit < 64)
602 set->rl &= ~(1LL << bit);
603 else
604 set->rh &= ~(1LL << (bit - 64));
605}
606
607void
608jit_regset_setbit(jit_regset_t *set, jit_int32_t bit)
609{
610 assert(bit >= 0 && bit <= 127);
611 if (bit < 64)
612 set->rl |= 1LL << bit;
613 else
614 set->rh |= 1LL << (bit - 64);
615}
616
617jit_bool_t
618jit_regset_tstbit(jit_regset_t *set, jit_int32_t bit)
619{
620 assert(bit >= 0 && bit <= 127);
621 if (bit < 64)
622 return (!!(set->rl & (1LL << bit)));
623 else
624 return (!!(set->rh & (1LL << (bit - 64))));
625}
626
627unsigned long
628jit_regset_scan1(jit_regset_t *set, jit_int32_t offset)
629{
630 assert(offset >= 0 && offset <= 127);
631 for (; offset < 64; offset++) {
632 if (set->rl & (1LL << offset))
633 return (offset);
634 }
635 for (; offset < 128; offset++) {
636 if (set->rh & (1LL << (offset - 64)))
637 return (offset);
638 }
639 return (ULONG_MAX);
640}
641
642#else
643unsigned long
644jit_regset_scan1(jit_regset_t *set, jit_int32_t offset)
645{
646 jit_regset_t mask;
647 assert(offset >= 0 && offset <= 63);
648 if ((mask = *set >> offset)) {
649 for (;;) {
650 if (mask & 1)
651 return (offset);
652 mask >>= 1;
653 ++offset;
654 }
655 }
656 return (ULONG_MAX);
657}
658#endif
659
660void
661_jit_save(jit_state_t *_jit, jit_int32_t reg)
662{
663 reg = jit_regno(reg);
664 assert(!_jitc->realize);
665 _jitc->spill[reg] = jit_new_node_w(jit_code_save, reg);
666}
667
668void
669_jit_load(jit_state_t *_jit, jit_int32_t reg)
670{
671 jit_node_t *node;
672
673 reg = jit_regno(reg);
674 assert(!_jitc->realize);
675 assert(_jitc->spill[reg] != NULL);
676 node = jit_new_node_w(jit_code_load, reg);
677 /* create a path to flag the save/load is not required */
678 node->link = _jitc->spill[reg];
679 node->link->link = node;
680 _jitc->spill[reg] = NULL;
681}
682
683static jit_word_t
684hash_data(const void *data, jit_word_t length)
685{
686 const jit_uint8_t *ptr;
687 jit_word_t i, key;
688 for (i = key = 0, ptr = data; i < length; i++)
689 key = (key << (key & 1)) ^ ptr[i];
690 return (key);
691}
692
693jit_pointer_t
694_jit_address(jit_state_t *_jit, jit_node_t *node)
695{
696 assert(_jitc->done);
697 assert(node != NULL &&
698 /* If a node type that is documented to be a fixed marker */
699 (node->code == jit_code_note || node->code == jit_code_name ||
700 /* If another special fixed marker, returned by jit_indirect() */
701 (node->code == jit_code_label && (node->flag & jit_flag_use) != 0)));
702 return ((jit_pointer_t)node->u.w);
703}
704
705jit_node_t *
706_jit_data(jit_state_t *_jit, const void *data,
707 jit_word_t length, jit_int32_t align)
708{
709 jit_word_t key;
710 jit_node_t *node;
711
712 assert(!_jitc->realize);
713
714 /* Ensure there is space even if asking for a duplicate */
715 if (((_jitc->data.offset + 7) & -8) + length > _jit->data.length) {
716 jit_word_t size;
717
718 size = (_jit->data.length + length + 4096) & - 4095;
719 assert(size >= _jit->data.length);
720 if (_jitc->data.ptr == NULL)
721 jit_alloc((jit_pointer_t *)&_jitc->data.ptr, size);
722 else
723 jit_realloc((jit_pointer_t *)&_jitc->data.ptr,
724 _jit->data.length, size);
725 _jit->data.length = size;
726 }
727 if (_jitc->data.table == NULL)
728 jit_alloc((jit_pointer_t *)&_jitc->data.table,
729 (_jitc->data.size = 16) * sizeof(jit_node_t*));
730
731 key = hash_data(data, length) & (_jitc->data.size - 1);
732 node = _jitc->data.table[key];
733 for (; node; node = node->next) {
734 if (node->v.w == length &&
735 memcmp(_jitc->data.ptr + node->u.w, data, length) == 0)
736 break;
737 }
738
739 if (!node) {
740 node = jit_new_node_no_link(jit_code_data);
741 if (!align)
742 align = length;
743 switch (align) {
744 case 0: case 1:
745 break;
746 case 2:
747 _jitc->data.offset = (_jitc->data.offset + 1) & -2;
748 break;
749 case 3: case 4:
750 _jitc->data.offset = (_jitc->data.offset + 3) & -4;
751 break;
752 default:
753 _jitc->data.offset = (_jitc->data.offset + 7) & -8;
754 break;
755 }
756 node->u.w = _jitc->data.offset;
757 node->v.w = length;
758 jit_memcpy(_jitc->data.ptr + _jitc->data.offset, data, length);
759 _jitc->data.offset += length;
760
761 node->next = _jitc->data.table[key];
762 _jitc->data.table[key] = node;
763 ++_jitc->data.count;
764
765 /* Rehash if more than 75% used table */
766 if (_jitc->data.count >
767 (_jitc->data.size >> 1) + (_jitc->data.size >> 2) &&
768 (_jitc->data.size << 1) > _jitc->data.size) {
769 jit_word_t i;
770 jit_node_t **hash;
771 jit_node_t *next;
772 jit_node_t *temp;
773
774 jit_alloc((jit_pointer_t *)&hash,
775 (_jitc->data.size << 1) * sizeof(jit_node_t*));
776 for (i = 0; i < _jitc->data.size; i++) {
777 temp = _jitc->data.table[i];
778 for (; temp; temp = next) {
779 next = temp->next;
780 key = hash_data(_jitc->data.ptr + temp->u.w, temp->v.w) &
781 ((_jitc->data.size << 1) - 1);
782 temp->next = hash[key];
783 hash[key] = temp;
784 }
785 }
786 jit_free((jit_pointer_t *)&_jitc->data.table);
787 _jitc->data.table = hash;
788 _jitc->data.size <<= 1;
789 }
790 }
791
792 return (node);
793}
794
795static void
796_new_pool(jit_state_t *_jit)
797{
798 jit_node_t *list;
799 jit_int32_t offset;
800
801 if (_jitc->pool.offset >= _jitc->pool.length) {
802 jit_int32_t length;
803
804 length = _jitc->pool.length + 16;
805 jit_realloc((jit_pointer_t *)&_jitc->pool.ptr,
806 _jitc->pool.length * sizeof(jit_node_t *),
807 length * sizeof(jit_node_t *));
808 _jitc->pool.length = length;
809 }
810 jit_alloc((jit_pointer_t *)(_jitc->pool.ptr + _jitc->pool.offset),
811 sizeof(jit_node_t) * 1024);
812 list = _jitc->pool.ptr[_jitc->pool.offset];
813 for (offset = 1; offset < 1024; offset++, list++)
814 list->next = list + 1;
815 list->next = _jitc->list;
816 _jitc->list = _jitc->pool.ptr[_jitc->pool.offset];
817 ++_jitc->pool.offset;
818}
819
820static jit_node_t *
821_new_node(jit_state_t *_jit, jit_code_t code)
822{
823 jit_node_t *node;
824
825 if (_jitc->list == NULL)
826 new_pool();
827 node = _jitc->list;
828 _jitc->list = node->next;
829 if (_jitc->synth)
830 node->flag |= jit_flag_synth;
831 node->next = NULL;
832 node->code = code;
833
834 return (node);
835}
836
837static inline jit_node_t *
838_link_node(jit_state_t *_jit, jit_node_t *node)
839{
840 if (_jitc->tail)
841 _jitc->tail->next = node;
842 else
843 _jitc->head = node;
844 return (_jitc->tail = node);
845}
846
847static inline void
848_del_node(jit_state_t *_jit, jit_node_t *prev, jit_node_t *node)
849{
850 if (prev == node) {
851 assert(prev == _jitc->head);
852 _jitc->head = node->next;
853 }
854 else
855 prev->next = node->next;
856 memset(node, 0, sizeof(jit_node_t));
857 node->next = _jitc->list;
858 _jitc->list = node;
859}
860
861static inline void
862_free_node(jit_state_t *_jit, jit_node_t *node)
863{
864 memset(node, 0, sizeof(jit_node_t));
865 node->next = _jitc->list;
866 _jitc->list = node;
867}
868
869static void
870_del_label(jit_state_t *_jit, jit_node_t *prev, jit_node_t *node)
871{
872 jit_block_t *block;
873
874 /* only allow call to del_label on linked labels */
875 block = _jitc->blocks.ptr + node->v.w;
876 assert(block->label == node);
877
878 /* del_label() should only be called when optimizing.
879 * This will leave an empty block index */
880 jit_regset_del(&block->reglive);
881 jit_regset_del(&block->regmask);
882 block->label = NULL;
883
884 /* redundant, should be already true */
885 assert(node->link == NULL);
886 del_node(prev, node);
887}
888
889jit_state_t *
890jit_new_state(void)
891{
892 jit_state_t *_jit;
893
894 jit_alloc((jit_pointer_t *)&_jit, sizeof(jit_state_t));
895 jit_alloc((jit_pointer_t *)&_jitc, sizeof(jit_compiler_t));
896 jit_regset_new(&_jitc->regarg);
897 jit_regset_new(&_jitc->regsav);
898 jit_regset_new(&_jitc->reglive);
899 jit_regset_new(&_jitc->regmask);
79bfeef6 900 jit_regset_new(&_jitc->explive);
4a71579b
PC		 901
902 jit_init();
903
904 jit_alloc((jit_pointer_t *)&_jitc->spill,
905 _jitc->reglen * sizeof(jit_node_t*));
906 jit_alloc((jit_pointer_t *)&_jitc->gen,
907 _jitc->reglen * sizeof(jit_int32_t));
908 jit_alloc((jit_pointer_t *)&_jitc->values,
909 _jitc->reglen * sizeof(jit_value_t));
910
911 jit_alloc((jit_pointer_t *)&_jitc->patches.ptr,
912 (_jitc->patches.length = 1024) * sizeof(jit_patch_t));
913 jit_alloc((jit_pointer_t *)&_jitc->functions.ptr,
914 (_jitc->functions.length = 16) * sizeof(jit_function_t));
915 jit_alloc((jit_pointer_t *)&_jitc->pool.ptr,
916 (_jitc->pool.length = 16) * sizeof(jit_node_t*));
917 jit_alloc((jit_pointer_t *)&_jitc->blocks.ptr,
918 (_jitc->blocks.length = 16) * sizeof(jit_block_t));
919#if __arm__ && DISASSEMBLER
920 jit_alloc((jit_pointer_t *)&_jitc->data_info.ptr,
921 (_jitc->data_info.length = 1024) * sizeof(jit_data_info_t));
922#endif
923
924 /* allocate at most one extra note in case jit_name() is
925 * never called, or called after adding at least one note */
926 _jit->note.length = 1;
927 _jitc->note.size = sizeof(jit_note_t);
928
929 return (_jit);
930}
931
932void
933_jit_clear_state(jit_state_t *_jit)
934{
935#if DEVEL_DISASSEMBLER
936# define jit_really_clear_state() _jit_really_clear_state(_jit)
937}
938
939void _jit_really_clear_state(jit_state_t *_jit)
940{
941#endif
942 jit_word_t offset;
943 jit_function_t *function;
944
945 /* release memory not required at jit execution time and set
946 * pointers to NULL to explicitly know they are released */
947 _jitc->head = _jitc->tail = NULL;
948
949 jit_free((jit_pointer_t *)&_jitc->data.table);
950 _jitc->data.size = _jitc->data.count = 0;
951
952 jit_free((jit_pointer_t *)&_jitc->spill);
953 jit_free((jit_pointer_t *)&_jitc->gen);
954 jit_free((jit_pointer_t *)&_jitc->values);
955
956 jit_free((jit_pointer_t *)&_jitc->blocks.ptr);
957
958 jit_free((jit_pointer_t *)&_jitc->patches.ptr);
959 _jitc->patches.offset = _jitc->patches.length = 0;
960
961 for (offset = 0; offset < _jitc->functions.offset; offset++) {
962 function = _jitc->functions.ptr + offset;
963 jit_free((jit_pointer_t *)&function->regoff);
964 }
965 jit_free((jit_pointer_t *)&_jitc->functions.ptr);
966 _jitc->functions.offset = _jitc->functions.length = 0;
967 _jitc->function = NULL;
968
969 for (offset = 0; offset < _jitc->pool.offset; offset++)
970 jit_free((jit_pointer_t *)(_jitc->pool.ptr + offset));
971 jit_free((jit_pointer_t *)&_jitc->pool.ptr);
972 _jitc->pool.offset = _jitc->pool.length = 0;
973 _jitc->list = NULL;
974
975 _jitc->note.head = _jitc->note.tail =
976 _jitc->note.name = _jitc->note.note = NULL;
977 _jitc->note.base = NULL;
978
979#if __arm__ && DISASSEMBLER
980 jit_free((jit_pointer_t *)&_jitc->data_info.ptr);
981#endif
982
983#if (__powerpc__ && _CALL_AIXDESC) || __ia64__
984 jit_free((jit_pointer_t *)&_jitc->prolog.ptr);
985#endif
986
987#if __ia64__
988 jit_regset_del(&_jitc->regs);
989#endif
990
991 jit_free((jit_pointer_t *)&_jitc);
992}
993
994void
995_jit_destroy_state(jit_state_t *_jit)
996{
997#if DEVEL_DISASSEMBLER
998 jit_really_clear_state();
999#endif
40a44dcb 1000#if HAVE_MMAP
4a71579b
PC		 1001 if (!_jit->user_code)
1002 munmap(_jit->code.ptr, _jit->code.length);
1003 if (!_jit->user_data)
1004 munmap(_jit->data.ptr, _jit->data.length);
40a44dcb 1005#endif
4a71579b
PC		 1006 jit_free((jit_pointer_t *)&_jit);
1007}
1008
1009void
1010_jit_synth_inc(jit_state_t *_jit)
1011{
1012 assert(_jitc->synth < 8);
1013 ++_jitc->synth;
1014}
1015
1016jit_node_t *
1017_jit_new_node(jit_state_t *_jit, jit_code_t code)
1018{
1019 assert(!_jitc->realize);
1020 return (link_node(new_node(code)));
1021}
1022
1023jit_node_t *
1024_jit_new_node_no_link(jit_state_t *_jit, jit_code_t code)
1025{
1026 assert(!_jitc->realize);
1027 return (new_node(code));
1028}
1029
1030void
1031_jit_link_node(jit_state_t *_jit, jit_node_t *node)
1032{
1033 assert(!_jitc->realize);
1034 link_node(node);
1035}
1036
1037void
1038_jit_synth_dec(jit_state_t *_jit)
1039{
1040 assert(_jitc->synth > 0);
1041 --_jitc->synth;
1042}
1043
1044jit_node_t *
1045_jit_new_node_w(jit_state_t *_jit, jit_code_t code,
1046 jit_word_t u)
1047{
1048 jit_node_t *node = new_node(code);
1049 assert(!_jitc->realize);
1050 node->u.w = u;
1051 return (link_node(node));
1052}
1053
1054jit_node_t *
1055_jit_new_node_f(jit_state_t *_jit, jit_code_t code,
1056 jit_float32_t u)
1057{
1058 jit_node_t *node = new_node(code);
1059 assert(!_jitc->realize);
1060 node->u.f = u;
1061 return (link_node(node));
1062}
1063
1064jit_node_t *
1065_jit_new_node_d(jit_state_t *_jit, jit_code_t code,
1066 jit_float64_t u)
1067{
1068 jit_node_t *node = new_node(code);
1069 assert(!_jitc->realize);
1070 node->u.d = u;
1071 return (link_node(node));
1072}
1073
1074jit_node_t *
1075_jit_new_node_p(jit_state_t *_jit, jit_code_t code,
1076 jit_pointer_t u)
1077{
1078 jit_node_t *node = new_node(code);
1079 assert(!_jitc->realize);
1080 node->u.p = u;
1081 return (link_node(node));
1082}
1083
1084jit_node_t *
1085_jit_new_node_ww(jit_state_t *_jit, jit_code_t code,
1086 jit_word_t u, jit_word_t v)
1087{
1088 jit_node_t *node = new_node(code);
1089 assert(!_jitc->realize);
1090 node->u.w = u;
1091 node->v.w = v;
1092 return (link_node(node));
1093}
1094
1095jit_node_t *
1096_jit_new_node_wp(jit_state_t *_jit, jit_code_t code,
1097 jit_word_t u, jit_pointer_t v)
1098{
1099 return (jit_new_node_ww(code, u, (jit_word_t)v));
1100}
1101
1102jit_node_t *
1103_jit_new_node_fp(jit_state_t *_jit, jit_code_t code,
1104 jit_float32_t u, jit_pointer_t v)
1105{
1106 jit_node_t *node = new_node(code);
1107 assert(!_jitc->realize);
1108 node->u.f = u;
1109 node->v.w = (jit_word_t)v;
1110 return (link_node(node));
1111}
1112
1113jit_node_t *
1114_jit_new_node_dp(jit_state_t *_jit, jit_code_t code,
1115 jit_float64_t u, jit_pointer_t v)
1116{
1117 jit_node_t *node = new_node(code);
1118 assert(!_jitc->realize);
1119 node->u.d = u;
1120 node->v.w = (jit_word_t)v;
1121 return (link_node(node));
1122}
1123
1124jit_node_t *
1125_jit_new_node_pw(jit_state_t *_jit, jit_code_t code,
1126 jit_pointer_t u, jit_word_t v)
1127{
1128 return (jit_new_node_ww(code, (jit_word_t)u, v));
1129}
1130
1131jit_node_t *
1132_jit_new_node_wf(jit_state_t *_jit, jit_code_t code,
1133 jit_word_t u, jit_float32_t v)
1134{
1135 jit_node_t *node = new_node(code);
1136 assert(!_jitc->realize);
1137 node->u.w = u;
1138 node->v.f = v;
1139 return (link_node(node));
1140}
1141
1142jit_node_t *
1143_jit_new_node_wd(jit_state_t *_jit, jit_code_t code,
1144 jit_word_t u, jit_float64_t v)
1145{
1146 jit_node_t *node = new_node(code);
1147 assert(!_jitc->realize);
1148 node->u.w = u;
1149 node->v.d = v;
1150 return (link_node(node));
1151}
1152
1153jit_node_t *
1154_jit_new_node_www(jit_state_t *_jit, jit_code_t code,
1155 jit_word_t u, jit_word_t v, jit_word_t w)
1156{
1157 jit_node_t *node = new_node(code);
1158 assert(!_jitc->realize);
1159 node->u.w = u;
1160 node->v.w = v;
1161 node->w.w = w;
1162 return (link_node(node));
1163}
1164
1165jit_node_t *
1166_jit_new_node_qww(jit_state_t *_jit, jit_code_t code,
1167 jit_int32_t l, jit_int32_t h,
1168 jit_word_t v, jit_word_t w)
1169{
1170 jit_node_t *node = new_node(code);
1171 assert(!_jitc->realize);
1172 assert(l != h);
1173 node->u.q.l = l;
1174 node->u.q.h = h;
1175 node->v.w = v;
1176 node->w.w = w;
1177 return (link_node(node));
1178}
1179
ba86ff93
PC		 1180jit_node_t *
1181_jit_new_node_wqw(jit_state_t *_jit, jit_code_t code,
1182 jit_word_t u, jit_int32_t l,
1183 jit_int32_t h, jit_word_t w)
1184{
1185 jit_node_t *node = new_node(code);
1186 assert(!_jitc->realize);
1187 assert(l != h);
1188 node->u.w = u;
1189 node->v.q.l = l;
1190 node->v.q.h = h;
1191 node->w.w = w;
1192 return (link_node(node));
1193}
1194
ba3814c1
PC		 1195jit_node_t *
1196_jit_new_node_wwq(jit_state_t *_jit, jit_code_t code,
1197 jit_word_t u, jit_word_t v,
1198 jit_int32_t l, jit_int32_t h)
1199{
1200 jit_node_t *node = new_node(code);
1201 assert(!_jitc->realize);
1202 node->u.w = u;
1203 node->v.w = v;
1204 node->w.q.l = l;
1205 node->w.q.h = h;
1206 return (link_node(node));
1207}
1208
4a71579b
PC		 1209jit_node_t *
1210_jit_new_node_wwf(jit_state_t *_jit, jit_code_t code,
1211 jit_word_t u, jit_word_t v, jit_float32_t w)
1212{
1213 jit_node_t *node = new_node(code);
1214 assert(!_jitc->realize);
1215 node->u.w = u;
1216 node->v.w = v;
1217 node->w.f = w;
1218 return (link_node(node));
1219}
1220
ba86ff93
PC		 1221jit_node_t *
1222_jit_new_node_wqf(jit_state_t *_jit, jit_code_t code,
1223 jit_word_t u, jit_int32_t l,
1224 jit_int32_t h, jit_float32_t w)
1225{
1226 jit_node_t *node = new_node(code);
1227 assert(!_jitc->realize);
1228 assert(l != h);
1229 node->u.w = u;
1230 node->v.q.l = l;
1231 node->v.q.h = h;
1232 node->w.f = w;
1233 return (link_node(node));
1234}
1235
4a71579b
PC		 1236jit_node_t *
1237_jit_new_node_wwd(jit_state_t *_jit, jit_code_t code,
1238 jit_word_t u, jit_word_t v, jit_float64_t w)
1239{
1240 jit_node_t *node = new_node(code);
1241 assert(!_jitc->realize);
1242 node->u.w = u;
1243 node->v.w = v;
1244 node->w.d = w;
1245 return (link_node(node));
1246}
1247
ba86ff93
PC		 1248jit_node_t *
1249_jit_new_node_wqd(jit_state_t *_jit, jit_code_t code,
1250 jit_word_t u, jit_int32_t l,
1251 jit_int32_t h, jit_float64_t w)
1252{
1253 jit_node_t *node = new_node(code);
1254 assert(!_jitc->realize);
1255 assert(l != h);
1256 node->u.w = u;
1257 node->v.q.l = l;
1258 node->v.q.h = h;
1259 node->w.d = w;
1260 return (link_node(node));
1261}
1262
4a71579b
PC		 1263jit_node_t *
1264_jit_new_node_pww(jit_state_t *_jit, jit_code_t code,
1265 jit_pointer_t u, jit_word_t v, jit_word_t w)
1266{
1267 jit_node_t *node = new_node(code);
1268 assert(!_jitc->realize);
1269 node->u.p = u;
1270 node->v.w = v;
1271 node->w.w = w;
1272 return (link_node(node));
1273}
1274
1275jit_node_t *
1276_jit_new_node_pwf(jit_state_t *_jit, jit_code_t code,
1277 jit_pointer_t u, jit_word_t v, jit_float32_t w)
1278{
1279 jit_node_t *node = new_node(code);
1280 assert(!_jitc->realize);
1281 node->u.p = u;
1282 node->v.w = v;
1283 node->w.f = w;
1284 return (link_node(node));
1285}
1286
1287jit_node_t *
1288_jit_new_node_pwd(jit_state_t *_jit, jit_code_t code,
1289 jit_pointer_t u, jit_word_t v, jit_float64_t w)
1290{
1291 jit_node_t *node = new_node(code);
1292 assert(!_jitc->realize);
1293 node->u.p = u;
1294 node->v.w = v;
1295 node->w.d = w;
1296 return (link_node(node));
1297}
1298
1299jit_node_t *
1300_jit_label(jit_state_t *_jit)
1301{
1302 jit_node_t *node;
1303
1304 if (!(node = _jitc->tail) || node->code != jit_code_label) {
1305 node = jit_forward();
1306 jit_link(node);
1307 }
1308
1309 return (node);
1310}
1311
1312jit_node_t *
1313_jit_forward(jit_state_t *_jit)
1314{
1315 return (jit_new_node_no_link(jit_code_label));
1316}
1317
1318jit_node_t *
1319_jit_indirect(jit_state_t *_jit)
1320{
1321 jit_node_t *node;
1322
1323 node = jit_label();
1324 node->flag |= jit_flag_use;
1325
1326 return (node);
1327}
1328
1329void
1330_jit_link(jit_state_t *_jit, jit_node_t *node)
1331{
1332 jit_block_t *block;
1333
1334 assert((node->code == jit_code_label ||
1335 node->code == jit_code_prolog ||
1336 node->code == jit_code_epilog) && !node->next);
1337 jit_link_node(node);
1338 if (_jitc->blocks.offset >= _jitc->blocks.length) {
1339 jit_word_t length;
1340
1341 length = _jitc->blocks.length + 16;
1342 jit_realloc((jit_pointer_t *)&_jitc->blocks.ptr,
1343 _jitc->blocks.length * sizeof(jit_block_t),
1344 length * sizeof(jit_block_t));
1345 _jitc->blocks.length = length;
1346 }
1347 block = _jitc->blocks.ptr + _jitc->blocks.offset;
1348 block->label = node;
1349 node->v.w = _jitc->blocks.offset;
1350 jit_regset_new(&block->reglive);
1351 jit_regset_new(&block->regmask);
1352 ++_jitc->blocks.offset;
1353}
1354
1355jit_bool_t
1356_jit_forward_p(jit_state_t *_jit, jit_node_t *node)
1357{
1358 return (node->code == jit_code_label && !node->next && node != _jitc->tail);
1359}
1360
1361jit_bool_t
1362_jit_indirect_p(jit_state_t *_jit, jit_node_t *node)
1363{
1364 return (node->code == jit_code_label && !!(node->flag & jit_flag_use));
1365}
1366
1367jit_bool_t
1368_jit_target_p(jit_state_t *_jit, jit_node_t *node)
1369{
1370 return (node->code == jit_code_label && !!node->link);
1371}
1372
1373void
1374_jit_prepare(jit_state_t *_jit)
1375{
1376 assert(_jitc->function != NULL);
1377 _jitc->function->call.call = jit_call_default;
1378 _jitc->function->call.argi =
1379 _jitc->function->call.argf =
1380 _jitc->function->call.size = 0;
1381 _jitc->prepare = jit_new_node(jit_code_prepare);
1382}
1383
1384void
1385_jit_patch(jit_state_t* _jit, jit_node_t *instr)
1386{
1387 jit_node_t *label;
1388
1389 if (!(label = _jitc->tail) || label->code != jit_code_label)
1390 label = jit_label();
1391 jit_patch_at(instr, label);
1392}
1393
1394jit_int32_t
1395_jit_classify(jit_state_t *_jit, jit_code_t code)
1396{
1397 jit_int32_t mask;
1398
1399 switch (code) {
1400 case jit_code_data: case jit_code_save: case jit_code_load:
1401 case jit_code_name: case jit_code_label: case jit_code_note:
1402 case jit_code_prolog: case jit_code_ellipsis: case jit_code_va_push:
1403 case jit_code_epilog: case jit_code_ret: case jit_code_prepare:
1404 mask = 0;
1405 break;
1406 case jit_code_live: case jit_code_va_end:
79bfeef6
PC		 1407 case jit_code_retr_c: case jit_code_retr_uc:
1408 case jit_code_retr_s: case jit_code_retr_us:
1409 case jit_code_retr_i: case jit_code_retr_ui:
1410 case jit_code_retr_l:
1411 case jit_code_retr_f: case jit_code_retr_d:
1412 case jit_code_pushargr_c:
1413 case jit_code_pushargr_uc:
1414 case jit_code_pushargr_s:
1415 case jit_code_pushargr_us:
1416 case jit_code_pushargr_i:
1417 case jit_code_pushargr_ui:
1418 case jit_code_pushargr_l:
1419 case jit_code_pushargr_f:
4a71579b
PC		 1420 case jit_code_pushargr_d:
1421 case jit_code_finishr: /* synthesized will set jit_cc_a0_jmp */
1422 mask = jit_cc_a0_reg;
1423 break;
79bfeef6
PC		 1424 case jit_code_align: case jit_code_skip:
1425 case jit_code_reti_c: case jit_code_reti_uc:
1426 case jit_code_reti_s: case jit_code_reti_us:
1427 case jit_code_reti_i: case jit_code_reti_ui:
1428 case jit_code_reti_l:
1429 case jit_code_pushargi_c:
1430 case jit_code_pushargi_uc:
1431 case jit_code_pushargi_s:
1432 case jit_code_pushargi_us:
1433 case jit_code_pushargi_i:
1434 case jit_code_pushargi_ui:
1435 case jit_code_pushargi_l:
1436 case jit_code_finishi: /* synthesized will set jit_cc_a0_jmp */
4a71579b
PC		 1437 mask = jit_cc_a0_int;
1438 break;
1439 case jit_code_reti_f: case jit_code_pushargi_f:
1440 mask = jit_cc_a0_flt;
1441 break;
1442 case jit_code_reti_d: case jit_code_pushargi_d:
1443 mask = jit_cc_a0_dbl;
1444 break;
1445 case jit_code_allocai:
1446 mask = jit_cc_a0_int|jit_cc_a1_int;
1447 break;
79bfeef6
PC		 1448 case jit_code_arg_c: case jit_code_arg_s:
1449 case jit_code_arg_i: case jit_code_arg_l:
1450 case jit_code_arg_f: case jit_code_arg_d:
4a71579b
PC		 1451 mask = jit_cc_a0_int|jit_cc_a0_arg;
1452 break;
1453 case jit_code_calli: case jit_code_jmpi:
1454 mask = jit_cc_a0_jmp;
1455 break;
1456 case jit_code_callr: case jit_code_jmpr:
1457 mask = jit_cc_a0_reg|jit_cc_a0_jmp;
1458 break;
1459 case jit_code_retval_c: case jit_code_retval_uc:
1460 case jit_code_retval_s: case jit_code_retval_us:
1461 case jit_code_retval_i: case jit_code_retval_ui:
1462 case jit_code_retval_l:
1463 case jit_code_retval_f: case jit_code_retval_d:
1464 case jit_code_va_start:
1465 mask = jit_cc_a0_reg|jit_cc_a0_chg;
1466 break;
1467 case jit_code_getarg_c: case jit_code_getarg_uc:
1468 case jit_code_getarg_s: case jit_code_getarg_us:
1469 case jit_code_getarg_i: case jit_code_getarg_ui:
1470 case jit_code_getarg_l:
1471 case jit_code_getarg_f: case jit_code_getarg_d:
1472 mask = jit_cc_a0_reg|jit_cc_a0_chg|jit_cc_a1_arg;
1473 break;
79bfeef6
PC		 1474 case jit_code_putargr_c:case jit_code_putargr_uc:
1475 case jit_code_putargr_s:case jit_code_putargr_us:
1476 case jit_code_putargr_i:case jit_code_putargr_ui:
1477 case jit_code_putargr_l:
1478 case jit_code_putargr_f:case jit_code_putargr_d:
4a71579b
PC		 1479 mask = jit_cc_a0_reg|jit_cc_a1_arg;
1480 break;
79bfeef6
PC		 1481 case jit_code_putargi_c:case jit_code_putargi_uc:
1482 case jit_code_putargi_s:case jit_code_putargi_us:
1483 case jit_code_putargi_i:case jit_code_putargi_ui:
1484 case jit_code_putargi_l:
4a71579b
PC		 1485 mask = jit_cc_a0_int|jit_cc_a1_arg;
1486 break;
1487 case jit_code_putargi_f:
1488 mask = jit_cc_a0_flt|jit_cc_a1_arg;
1489 break;
1490 case jit_code_putargi_d:
1491 mask = jit_cc_a0_dbl|jit_cc_a1_arg;
1492 break;
1493 case jit_code_movi: case jit_code_ldi_c: case jit_code_ldi_uc:
1494 case jit_code_ldi_s: case jit_code_ldi_us: case jit_code_ldi_i:
1495 case jit_code_ldi_ui: case jit_code_ldi_l: case jit_code_ldi_f:
1496 case jit_code_ldi_d:
ba86ff93
PC		 1497 case jit_code_negi: case jit_code_comi:
1498 case jit_code_exti_c: case jit_code_exti_uc:
1499 case jit_code_exti_s: case jit_code_exti_us:
1500 case jit_code_exti_i: case jit_code_exti_ui:
1501 case jit_code_bswapi_us:case jit_code_bswapi_ui:
1502 case jit_code_bswapi_ul:
1503 case jit_code_htoni_us: case jit_code_htoni_ui:
1504 case jit_code_htoni_ul:
1505 case jit_code_cloi: case jit_code_clzi:
1506 case jit_code_ctoi: case jit_code_ctzi:
1507 case jit_code_rbiti: case jit_code_popcnti:
1508 case jit_code_movi_w_f: case jit_code_movi_w_d:
4a71579b
PC		 1509 mask = jit_cc_a0_reg|jit_cc_a0_chg|jit_cc_a1_int;
1510 break;
ba86ff93
PC		 1511 case jit_code_unldi: case jit_code_unldi_u: case jit_code_unldi_x:
1512 mask = jit_cc_a0_reg|jit_cc_a0_chg|jit_cc_a1_int|jit_cc_a2_int;
1513 break;
1514 case jit_code_movi_ww_d:
1515 mask = jit_cc_a0_reg|jit_cc_a0_chg|jit_cc_a1_int|jit_cc_a2_int;
1516 break;
4a71579b 1517 case jit_code_movi_f: case jit_code_movi_f_w:
ba86ff93
PC		 1518 case jit_code_negi_f: case jit_code_absi_f:
1519 case jit_code_sqrti_f:
4a71579b
PC		 1520 mask = jit_cc_a0_reg|jit_cc_a0_chg|jit_cc_a1_flt;
1521 break;
1522 case jit_code_movi_d: case jit_code_movi_d_w:
ba86ff93
PC		 1523 case jit_code_negi_d: case jit_code_absi_d:
1524 case jit_code_sqrti_d:
4a71579b
PC		 1525 mask = jit_cc_a0_reg|jit_cc_a0_chg|jit_cc_a1_dbl;
1526 break;
1527 case jit_code_movi_d_ww:
1528 mask = jit_cc_a0_reg|jit_cc_a0_chg|jit_cc_a1_reg|jit_cc_a1_chg|
1529 jit_cc_a2_dbl;
1530 break;
1531 case jit_code_negr: case jit_code_comr: case jit_code_movr:
1532 case jit_code_extr_c: case jit_code_extr_uc: case jit_code_extr_s:
1533 case jit_code_extr_us: case jit_code_extr_i: case jit_code_extr_ui:
1534 case jit_code_truncr_f_i: case jit_code_truncr_f_l:
1535 case jit_code_truncr_d_i: case jit_code_truncr_d_l:
1536 case jit_code_htonr_us: case jit_code_htonr_ui: case jit_code_htonr_ul:
40a44dcb 1537 case jit_code_bswapr_us: case jit_code_bswapr_ui: case jit_code_bswapr_ul:
4a71579b
PC		 1538 case jit_code_ldr_c: case jit_code_ldr_uc:
1539 case jit_code_ldr_s: case jit_code_ldr_us: case jit_code_ldr_i:
1540 case jit_code_ldr_ui: case jit_code_ldr_l: case jit_code_negr_f:
1541 case jit_code_absr_f: case jit_code_sqrtr_f: case jit_code_movr_f:
1542 case jit_code_extr_f: case jit_code_extr_d_f: case jit_code_ldr_f:
1543 case jit_code_negr_d: case jit_code_absr_d: case jit_code_sqrtr_d:
1544 case jit_code_movr_d: case jit_code_extr_d: case jit_code_extr_f_d:
ba86ff93 1545 case jit_code_ldr_d: case jit_code_rbitr: case jit_code_popcntr:
79bfeef6
PC		 1546 case jit_code_clor: case jit_code_clzr:
1547 case jit_code_ctor: case jit_code_ctzr:
4a71579b
PC		 1548 case jit_code_movr_w_f: case jit_code_movr_f_w:
1549 case jit_code_movr_w_d: case jit_code_movr_d_w:
1550 case jit_code_va_arg: case jit_code_va_arg_d:
1551 mask = jit_cc_a0_reg|jit_cc_a0_chg|jit_cc_a1_reg;
1552 break;
1553 case jit_code_movr_d_ww:
1554 mask = jit_cc_a0_reg|jit_cc_a0_chg|jit_cc_a1_reg|jit_cc_a1_chg|
1555 jit_cc_a2_reg;
1556 break;
1557 case jit_code_addi: case jit_code_addxi: case jit_code_addci:
1558 case jit_code_subi: case jit_code_subxi: case jit_code_subci:
1559 case jit_code_rsbi:
ba86ff93
PC		 1560 case jit_code_muli: case jit_code_hmuli: case jit_code_hmuli_u:
1561 case jit_code_divi: case jit_code_divi_u:
4a71579b
PC		 1562 case jit_code_remi: case jit_code_remi_u: case jit_code_andi:
1563 case jit_code_ori: case jit_code_xori: case jit_code_lshi:
ba86ff93
PC		 1564 case jit_code_rshi: case jit_code_rshi_u: case jit_code_lroti:
1565 case jit_code_rroti: case jit_code_lti:
4a71579b
PC		 1566 case jit_code_lti_u: case jit_code_lei: case jit_code_lei_u:
1567 case jit_code_eqi: case jit_code_gei: case jit_code_gei_u:
1568 case jit_code_gti: case jit_code_gti_u: case jit_code_nei:
1569 case jit_code_ldxi_c: case jit_code_ldxi_uc: case jit_code_ldxi_s:
1570 case jit_code_ldxi_us: case jit_code_ldxi_i: case jit_code_ldxi_ui:
1571 case jit_code_ldxi_l: case jit_code_ldxi_f: case jit_code_ldxi_d:
ba86ff93 1572 case jit_code_unldr: case jit_code_unldr_u: case jit_code_unldr_x:
4a71579b
PC		 1573 mask = jit_cc_a0_reg|jit_cc_a0_chg|jit_cc_a1_reg|jit_cc_a2_int;
1574 break;
1575 case jit_code_qmuli: case jit_code_qmuli_u:
1576 case jit_code_qdivi: case jit_code_qdivi_u:
ba86ff93
PC		 1577 case jit_code_qlshi: case jit_code_qlshi_u:
1578 case jit_code_qrshi: case jit_code_qrshi_u:
4a71579b
PC		 1579 mask = jit_cc_a0_reg|jit_cc_a0_rlh|jit_cc_a0_chg|
1580 jit_cc_a1_reg|jit_cc_a2_int;
1581 break;
1582 case jit_code_addi_f: case jit_code_subi_f: case jit_code_rsbi_f:
1583 case jit_code_muli_f: case jit_code_divi_f: case jit_code_lti_f:
1584 case jit_code_lei_f: case jit_code_eqi_f: case jit_code_gei_f:
1585 case jit_code_gti_f: case jit_code_nei_f: case jit_code_unlti_f:
1586 case jit_code_unlei_f: case jit_code_uneqi_f: case jit_code_ungei_f:
1587 case jit_code_ungti_f: case jit_code_ltgti_f: case jit_code_ordi_f:
1588 case jit_code_unordi_f:
1589 mask = jit_cc_a0_reg|jit_cc_a0_chg|jit_cc_a1_reg|jit_cc_a2_flt;
1590 break;
1591 case jit_code_addi_d: case jit_code_subi_d: case jit_code_rsbi_d:
1592 case jit_code_muli_d: case jit_code_divi_d: case jit_code_lti_d:
1593 case jit_code_lei_d: case jit_code_eqi_d: case jit_code_gei_d:
1594 case jit_code_gti_d: case jit_code_nei_d: case jit_code_unlti_d:
1595 case jit_code_unlei_d: case jit_code_uneqi_d: case jit_code_ungei_d:
1596 case jit_code_ungti_d: case jit_code_ltgti_d: case jit_code_ordi_d:
1597 case jit_code_unordi_d:
1598 mask = jit_cc_a0_reg|jit_cc_a0_chg|jit_cc_a1_reg|jit_cc_a2_dbl;
1599 break;
1600 case jit_code_addr: case jit_code_addxr: case jit_code_addcr:
1601 case jit_code_subr: case jit_code_subxr: case jit_code_subcr:
ba86ff93
PC		 1602 case jit_code_mulr: case jit_code_hmulr: case jit_code_hmulr_u:
1603 case jit_code_divr: case jit_code_divr_u:
4a71579b
PC		 1604 case jit_code_remr: case jit_code_remr_u: case jit_code_andr:
1605 case jit_code_orr: case jit_code_xorr: case jit_code_lshr:
ba86ff93
PC		 1606 case jit_code_rshr: case jit_code_rshr_u: case jit_code_lrotr:
1607 case jit_code_rrotr: case jit_code_ltr:
4a71579b
PC		 1608 case jit_code_ltr_u: case jit_code_ler: case jit_code_ler_u:
1609 case jit_code_eqr: case jit_code_ger: case jit_code_ger_u:
1610 case jit_code_gtr: case jit_code_gtr_u: case jit_code_ner:
1611 case jit_code_ldxr_c: case jit_code_ldxr_uc: case jit_code_ldxr_s:
1612 case jit_code_ldxr_us: case jit_code_ldxr_i: case jit_code_ldxr_ui:
1613 case jit_code_ldxr_l: case jit_code_addr_f: case jit_code_subr_f:
1614 case jit_code_mulr_f: case jit_code_divr_f: case jit_code_ltr_f:
1615 case jit_code_ler_f: case jit_code_eqr_f: case jit_code_ger_f:
1616 case jit_code_gtr_f: case jit_code_ner_f: case jit_code_unltr_f:
1617 case jit_code_unler_f: case jit_code_uneqr_f: case jit_code_unger_f:
1618 case jit_code_ungtr_f: case jit_code_ltgtr_f: case jit_code_ordr_f:
1619 case jit_code_unordr_f: case jit_code_ldxr_f: case jit_code_addr_d:
1620 case jit_code_subr_d: case jit_code_mulr_d: case jit_code_divr_d:
1621 case jit_code_ltr_d: case jit_code_ler_d: case jit_code_eqr_d:
1622 case jit_code_ger_d: case jit_code_gtr_d: case jit_code_ner_d:
1623 case jit_code_unltr_d: case jit_code_unler_d: case jit_code_uneqr_d:
1624 case jit_code_unger_d: case jit_code_ungtr_d: case jit_code_ltgtr_d:
1625 case jit_code_ordr_d: case jit_code_unordr_d: case jit_code_ldxr_d:
1626 case jit_code_movr_ww_d:
1627 mask = jit_cc_a0_reg|jit_cc_a0_chg|jit_cc_a1_reg|jit_cc_a2_reg;
1628 break;
1629 case jit_code_qmulr: case jit_code_qmulr_u:
1630 case jit_code_qdivr: case jit_code_qdivr_u:
ba86ff93
PC		 1631 case jit_code_qlshr: case jit_code_qlshr_u:
1632 case jit_code_qrshr: case jit_code_qrshr_u:
4a71579b
PC		 1633 mask = jit_cc_a0_reg|jit_cc_a0_rlh|jit_cc_a0_chg|
1634 jit_cc_a1_reg|jit_cc_a2_reg;
1635 break;
1636 case jit_code_sti_c: case jit_code_sti_s: case jit_code_sti_i:
1637 case jit_code_sti_l: case jit_code_sti_f: case jit_code_sti_d:
1638 mask = jit_cc_a0_int|jit_cc_a1_reg;
1639 break;
ba86ff93
PC		 1640 case jit_code_unsti: case jit_code_unsti_x:
1641 mask = jit_cc_a0_int|jit_cc_a1_reg|jit_cc_a2_int;
1642 break;
4a71579b
PC		 1643 case jit_code_blti: case jit_code_blti_u: case jit_code_blei:
1644 case jit_code_blei_u: case jit_code_beqi: case jit_code_bgei:
1645 case jit_code_bgei_u: case jit_code_bgti: case jit_code_bgti_u:
1646 case jit_code_bnei: case jit_code_bmsi: case jit_code_bmci:
1647 mask = jit_cc_a0_jmp|jit_cc_a1_reg|jit_cc_a2_int;
1648 break;
1649 case jit_code_blti_f: case jit_code_blei_f: case jit_code_beqi_f:
1650 case jit_code_bgei_f: case jit_code_bgti_f: case jit_code_bnei_f:
1651 case jit_code_bunlti_f: case jit_code_bunlei_f: case jit_code_buneqi_f:
1652 case jit_code_bungei_f: case jit_code_bungti_f: case jit_code_bltgti_f:
1653 case jit_code_bordi_f: case jit_code_bunordi_f:
1654 mask = jit_cc_a0_jmp|jit_cc_a1_reg|jit_cc_a2_flt;
1655 break;
1656 case jit_code_blti_d: case jit_code_blei_d: case jit_code_beqi_d:
1657 case jit_code_bgei_d: case jit_code_bgti_d: case jit_code_bnei_d:
1658 case jit_code_bunlti_d: case jit_code_bunlei_d: case jit_code_buneqi_d:
1659 case jit_code_bungei_d: case jit_code_bungti_d: case jit_code_bltgti_d:
1660 case jit_code_bordi_d: case jit_code_bunordi_d:
1661 mask = jit_cc_a0_jmp|jit_cc_a1_reg|jit_cc_a2_dbl;
1662 break;
1663 case jit_code_allocar: /* synthesized instructions make it
1664 * equivalent to jit_cc_a0_chg */
1665 case jit_code_str_c: case jit_code_str_s: case jit_code_str_i:
1666 case jit_code_str_l: case jit_code_str_f: case jit_code_str_d:
1667 mask = jit_cc_a0_reg|jit_cc_a1_reg;
1668 break;
ba86ff93
PC		 1669 case jit_code_unstr: case jit_code_unstr_x:
1670 mask = jit_cc_a0_reg|jit_cc_a1_reg|jit_cc_a2_int;
1671 break;
4a71579b
PC		 1672 case jit_code_stxi_c: case jit_code_stxi_s: case jit_code_stxi_i:
1673 case jit_code_stxi_l: case jit_code_stxi_f: case jit_code_stxi_d:
1674 mask = jit_cc_a0_int|jit_cc_a1_reg|jit_cc_a2_reg;
1675 break;
1676 case jit_code_bltr: case jit_code_bltr_u: case jit_code_bler:
1677 case jit_code_bler_u: case jit_code_beqr: case jit_code_bger:
1678 case jit_code_bger_u: case jit_code_bgtr: case jit_code_bgtr_u:
1679 case jit_code_bner: case jit_code_bmsr: case jit_code_bmcr:
1680 case jit_code_bltr_f: case jit_code_bler_f: case jit_code_beqr_f:
1681 case jit_code_bger_f: case jit_code_bgtr_f: case jit_code_bner_f:
1682 case jit_code_bunltr_f: case jit_code_bunler_f: case jit_code_buneqr_f:
1683 case jit_code_bunger_f: case jit_code_bungtr_f: case jit_code_bltgtr_f:
1684 case jit_code_bordr_f: case jit_code_bunordr_f:case jit_code_bltr_d:
1685 case jit_code_bler_d: case jit_code_beqr_d: case jit_code_bger_d:
1686 case jit_code_bgtr_d: case jit_code_bner_d: case jit_code_bunltr_d:
1687 case jit_code_bunler_d: case jit_code_buneqr_d: case jit_code_bunger_d:
1688 case jit_code_bungtr_d: case jit_code_bltgtr_d: case jit_code_bordr_d:
1689 case jit_code_bunordr_d:
1690 mask = jit_cc_a0_jmp|jit_cc_a1_reg|jit_cc_a2_reg;
1691 break;
1692 case jit_code_boaddi: case jit_code_boaddi_u: case jit_code_bxaddi:
1693 case jit_code_bxaddi_u: case jit_code_bosubi: case jit_code_bosubi_u:
1694 case jit_code_bxsubi: case jit_code_bxsubi_u:
1695 mask = jit_cc_a0_jmp|jit_cc_a1_reg|jit_cc_a1_chg|jit_cc_a2_int;
1696 break;
1697 case jit_code_stxr_c: case jit_code_stxr_s: case jit_code_stxr_i:
1698 case jit_code_stxr_l: case jit_code_stxr_f: case jit_code_stxr_d:
1699 mask = jit_cc_a0_reg|jit_cc_a1_reg|jit_cc_a2_reg;
1700 break;
1701 case jit_code_boaddr: case jit_code_boaddr_u: case jit_code_bxaddr:
1702 case jit_code_bxaddr_u: case jit_code_bosubr: case jit_code_bosubr_u:
1703 case jit_code_bxsubr: case jit_code_bxsubr_u:
1704 mask = jit_cc_a0_jmp|jit_cc_a1_reg|jit_cc_a1_chg|jit_cc_a2_reg;
1705 break;
40a44dcb
PC		 1706 case jit_code_movnr: case jit_code_movzr:
1707 mask = jit_cc_a0_reg|jit_cc_a0_cnd|jit_cc_a1_reg|jit_cc_a2_reg;
1708 break;
ba3814c1
PC		 1709 case jit_code_casr:
1710 mask = jit_cc_a0_reg|jit_cc_a0_chg|jit_cc_a1_reg|
1711 jit_cc_a2_reg|jit_cc_a2_rlh;
1712 break;
1713 case jit_code_casi:
1714 mask = jit_cc_a0_reg|jit_cc_a0_chg|jit_cc_a1_int|
1715 jit_cc_a2_reg|jit_cc_a2_rlh;
1716 break;
ba86ff93
PC		 1717 case jit_code_extr: case jit_code_extr_u: case jit_code_depr:
1718 mask = jit_cc_a0_reg|jit_cc_a0_chg|jit_cc_a1_reg|jit_cc_a2_rlh;
1719 break;
1720 case jit_code_exti: case jit_code_exti_u: case jit_code_depi:
1721 mask = jit_cc_a0_reg|jit_cc_a0_chg|jit_cc_a1_int|jit_cc_a2_rlh;
1722 break;
1723 case jit_code_fmar_f: case jit_code_fmar_d:
1724 case jit_code_fmsr_f: case jit_code_fmsr_d:
1725 case jit_code_fnmar_f: case jit_code_fnmar_d:
1726 case jit_code_fnmsr_f: case jit_code_fnmsr_d:
1727 mask = jit_cc_a0_reg|jit_cc_a0_chg|
1728 jit_cc_a1_reg|jit_cc_a1_rlh|jit_cc_a2_reg;
1729 break;
1730 case jit_code_fmai_f: case jit_code_fmsi_f:
1731 case jit_code_fnmai_f: case jit_code_fnmsi_f:
1732 mask = jit_cc_a0_reg|jit_cc_a0_chg|
1733 jit_cc_a1_reg|jit_cc_a1_rlh|jit_cc_a2_flt;
1734 break;
1735 case jit_code_fmai_d: case jit_code_fmsi_d:
1736 case jit_code_fnmai_d: case jit_code_fnmsi_d:
1737 mask = jit_cc_a0_reg|jit_cc_a0_chg|
1738 jit_cc_a1_reg|jit_cc_a1_rlh|jit_cc_a2_dbl;
1739 break;
4a71579b
PC		 1740 default:
1741 abort();
1742 }
1743
1744 return (mask);
1745}
1746
1747void
1748_jit_patch_abs(jit_state_t *_jit, jit_node_t *instr, jit_pointer_t address)
1749{
d481fb64 1750#ifndef NDEBUG
4a71579b 1751 jit_int32_t mask;
d481fb64 1752#endif
4a71579b
PC		 1753 switch (instr->code) {
1754 case jit_code_movi: case jit_code_ldi_c: case jit_code_ldi_uc:
1755 case jit_code_ldi_s: case jit_code_ldi_us: case jit_code_ldi_i:
1756 case jit_code_ldi_ui: case jit_code_ldi_l: case jit_code_ldi_f:
1757 case jit_code_ldi_d:
1758 instr->v.p = address;
1759 break;
1760 case jit_code_sti_c: case jit_code_sti_s: case jit_code_sti_i:
1761 case jit_code_sti_l: case jit_code_sti_f: case jit_code_sti_d:
1762 instr->u.p = address;
1763 break;
1764 default:
d481fb64 1765#ifndef NDEBUG
4a71579b 1766 mask = jit_classify(instr->code);
d481fb64 1767#endif
4a71579b
PC		 1768 assert((mask & (jit_cc_a0_reg|jit_cc_a0_jmp)) == jit_cc_a0_jmp);
1769 instr->u.p = address;
1770 }
1771}
1772
1773void
1774_jit_patch_at(jit_state_t *_jit, jit_node_t *instr, jit_node_t *label)
1775{
d481fb64 1776#ifndef NDEBUG
4a71579b 1777 jit_int32_t mask;
d481fb64 1778#endif
4a71579b
PC		 1779 assert(!(instr->flag & jit_flag_node));
1780 instr->flag |= jit_flag_node;
1781 switch (instr->code) {
1782 case jit_code_movi:
1783 assert(label->code == jit_code_label ||
1784 label->code == jit_code_data);
1785 instr->v.n = label;
1786 if (label->code == jit_code_data)
1787 instr->flag |= jit_flag_data;
1788 break;
1789 case jit_code_jmpi:
1790 assert(label->code == jit_code_label ||
1791 label->code == jit_code_epilog);
1792 instr->u.n = label;
1793 break;
1794 default:
d481fb64 1795#ifndef NDEBUG
4a71579b 1796 mask = jit_classify(instr->code);
d481fb64 1797#endif
4a71579b
PC		 1798 assert((mask & (jit_cc_a0_reg|jit_cc_a0_jmp)) == jit_cc_a0_jmp);
1799 assert(label->code == jit_code_label);
1800 instr->u.n = label;
1801 break;
1802 }
1803 /* link field is used as list of nodes associated with a given label */
1804 instr->link = label->link;
1805 label->link = instr;
1806}
1807
c0c16242
PC		 1808static void
1809_do_setup(jit_state_t *_jit)
4a71579b 1810{
4a71579b
PC		 1811 jit_block_t *block;
1812 jit_word_t offset;
1813
4a71579b
PC		 1814 /* create initial mapping of live register values
1815 * at the start of a basic block */
1816 for (offset = 0; offset < _jitc->blocks.offset; offset++) {
1817 block = _jitc->blocks.ptr + offset;
79bfeef6 1818 if (!block->label)
4a71579b 1819 continue;
79bfeef6
PC		 1820 if (block->label->code == jit_code_epilog) {
1821 jit_regset_setbit(&block->reglive, JIT_RET);
1822 jit_regset_setbit(&block->reglive, JIT_FRET);
1823 jit_regset_com(&block->regmask, &block->reglive);
1824 continue;
1825 }
c0c16242 1826 jit_setup(block);
4a71579b 1827 }
c0c16242 1828}
4a71579b 1829
c0c16242
PC		 1830static jit_bool_t
1831_block_update_set(jit_state_t *_jit,
1832 jit_block_t *block, jit_block_t *target)
1833{
1834 jit_regset_t regmask;
1835
1836 jit_regset_ior(&regmask, &block->reglive, &target->reglive);
1837 jit_regset_and(&regmask, &regmask, &block->regmask);
1838 if (jit_regset_set_p(&regmask)) {
1839 jit_regset_ior(&block->reglive, &block->reglive, &regmask);
1840 jit_regset_and(&regmask, &block->reglive, &block->regmask);
1841 jit_regset_com(&regmask, &regmask);
1842 jit_regset_and(&block->regmask, &block->regmask, &regmask);
1843 block->again = 1;
1844 return (1);
1845 }
1846 return (0);
1847}
4a71579b 1848
24d91c0d
PC		 1849static void
1850_propagate_backward(jit_state_t *_jit, jit_block_t *block)
1851{
1852 jit_block_t *prev;
1853 jit_word_t offset;
1854
1855 for (offset = block->label->v.w - 1;
1856 offset >= 0; --offset) {
1857 prev = _jitc->blocks.ptr + offset;
1858 if (!block_update_set(prev, block) ||
1859 !(prev->label->flag & jit_flag_head))
1860 break;
1861 }
1862}
1863
c0c16242
PC		 1864static jit_bool_t
1865_check_block_again(jit_state_t *_jit)
1866{
1867 jit_int32_t todo;
1868 jit_word_t offset;
1869 jit_node_t *node, *label;
1870 jit_block_t *block, *target;
4a71579b 1871
c0c16242
PC		 1872 todo = 0;
1873 for (offset = 0; offset < _jitc->blocks.offset; offset++) {
1874 block = _jitc->blocks.ptr + offset;
1875 if (block->again) {
ba3814c1 1876 todo = 1;
c0c16242 1877 break;
ba3814c1 1878 }
c0c16242
PC		 1879 }
1880 /* If no block changed state */
1881 if (!todo)
1882 return (0);
ba3814c1 1883
c0c16242
PC		 1884 do {
1885 todo = 0;
1886 block = NULL;
ba3814c1 1887 for (node = _jitc->head; node; node = node->next) {
c0c16242
PC		 1888 /* Special jumps that match jit_cc_a0_jmp */
1889 if (node->code == jit_code_calli || node->code == jit_code_callr)
1890 continue;
1891
1892 /* Remember current label */
1893 if (node->code == jit_code_label ||
1894 node->code == jit_code_prolog ||
1895 node->code == jit_code_epilog) {
1896
1897 /* If previous block does not pass through */
1898 if (!(node->flag & jit_flag_head))
1899 block = NULL;
1900
1901 target = _jitc->blocks.ptr + node->v.w;
24d91c0d
PC		 1902 if (block && target->again && block_update_set(block, target)) {
1903 propagate_backward(block);
ba3814c1 1904 todo = 1;
24d91c0d 1905 }
c0c16242 1906 block = target;
ba3814c1 1907 }
c0c16242
PC		 1908 /* If not the first jmpi */
1909 else if (block) {
1910 /* If a jump to dynamic address or if a jump to raw address */
1911 if (!(jit_classify(node->code) & jit_cc_a0_jmp) ||
1912 !(node->flag & jit_flag_node))
1913 continue;
1914 label = node->u.n;
1915 /* Mark predecessor needs updating due to target change */
1916 target = _jitc->blocks.ptr + label->v.w;
24d91c0d
PC		 1917 if (target->again && block_update_set(block, target)) {
1918 propagate_backward(block);
ba3814c1 1919 todo = 1;
24d91c0d 1920 }
ba3814c1
PC		 1921 }
1922 }
c0c16242
PC		 1923 }
1924 while (todo);
1925
79bfeef6 1926 return (todo);
c0c16242 1927}
ba3814c1 1928
c0c16242
PC		 1929static void
1930_do_follow(jit_state_t *_jit, jit_bool_t always)
1931{
1932 jit_block_t *block;
1933 jit_word_t offset;
1934
1935 /* set live state of registers not referenced in a block, but
1936 * referenced in a jump target or normal flow */
1937 for (offset = 0; offset < _jitc->blocks.offset; offset++) {
1938 block = _jitc->blocks.ptr + offset;
1939 if (!block->label || block->label->code == jit_code_epilog)
1940 continue;
1941 if (always || block->again) {
1942 block->again = 0;
1943 jit_follow(block);
1944 }
1945 }
1946}
1947
1948void
1949_jit_optimize(jit_state_t *_jit)
1950{
1951 jit_bool_t jump;
1952 jit_bool_t todo;
1953 jit_int32_t mask;
1954 jit_node_t *node;
1955 jit_block_t *block;
1956 jit_word_t offset;
79bfeef6 1957 jit_regset_t regmask;
c0c16242
PC		 1958
1959 todo = 0;
1960 _jitc->function = NULL;
1961
1962 thread_jumps();
1963 sequential_labels();
1964 split_branches();
1965 do_setup();
1966 do_follow(1);
1967
79bfeef6
PC		 1968 jit_regset_set_ui(&regmask, 0);
1969 for (offset = 0; offset < _jitc->reglen; offset++) {
1970 if ((jit_class(_rvs[offset].spec) & (jit_class_gpr|jit_class_fpr)) &&
1971 (jit_class(_rvs[offset].spec) & jit_class_sav) == jit_class_sav)
1972 jit_regset_setbit(&regmask, offset);
1973 }
1974
1975 /* Figure out labels that are only reached with a jump */
c0c16242
PC		 1976 jump = 1;
1977 for (node = _jitc->head; node; node = node->next) {
1978 switch (node->code) {
1979 case jit_code_label:
79bfeef6 1980 if (!jump) {
c0c16242 1981 node->flag |= jit_flag_head;
79bfeef6
PC		 1982 if (!node->link) {
1983 /* Block is dead code or only reachable with an
1984 * indirect jumps. In such condition, must assume
1985 * all callee save registers are live. */
1986 block = _jitc->blocks.ptr + node->v.w;
1987 jit_regset_ior(&block->reglive,
1988 &block->reglive, &regmask);
1989 /* Cleanup regmask */
1990 block_update_set(block, block);
1991 }
1992 }
c0c16242
PC		 1993 break;
1994 case jit_code_jmpi: case jit_code_jmpr:
1995 case jit_code_epilog:
1996 jump = 1;
1997 break;
1998 case jit_code_data: case jit_code_note:
1999 break;
2000 default:
2001 jump = 0;
2002 break;
2003 }
2004 }
2005
ba86ff93
PC		 2006 patch_registers();
2007 if (simplify())
2008 todo = 1;
2009
c0c16242
PC		 2010 for (node = _jitc->head; node; node = node->next) {
2011 mask = jit_classify(node->code);
2012 if (mask & jit_cc_a0_reg)
2013 node->u.w &= ~jit_regno_patch;
2014 if (mask & jit_cc_a1_reg)
2015 node->v.w &= ~jit_regno_patch;
2016 if (mask & jit_cc_a2_reg)
2017 node->w.w &= ~jit_regno_patch;
2018 if (node->code == jit_code_beqi) {
2019 if (redundant_store(node, 1)) {
2020 block = _jitc->blocks.ptr + ((jit_node_t *)node->u.n)->v.w;
2021 block->again = 1;
2022 todo = 1;
2023 }
2024 }
2025 else if (node->code == jit_code_bnei) {
2026 if (redundant_store(node, 0)) {
2027 block = _jitc->blocks.ptr + ((jit_node_t *)node->u.n)->v.w;
2028 block->again = 1;
2029 todo = 1;
2030 }
4a71579b
PC		 2031 }
2032 }
2033
c0c16242
PC		 2034 if (!todo)
2035 todo = check_block_again();
2036
2037 /* If instructions were removed or first pass did modify the entry
2038 * state of any block */
2039 if (todo) {
2040 do_setup();
2041 todo = 0;
2042 do {
2043 do_follow(0);
2044 /* If any block again has the entry state modified. */
2045 todo = check_block_again();
2046 } while (todo);
2047 }
2048
4a71579b
PC		 2049 for (node = _jitc->head; node; node = node->next) {
2050 mask = jit_classify(node->code);
2051 if (mask & jit_cc_a0_reg)
2052 node->u.w &= ~jit_regno_patch;
2053 if (mask & jit_cc_a1_reg)
2054 node->v.w &= ~jit_regno_patch;
2055 if (mask & jit_cc_a2_reg)
2056 node->w.w &= ~jit_regno_patch;
ba3814c1
PC		 2057 if (node->code == jit_code_prolog)
2058 _jitc->function = _jitc->functions.ptr + node->w.w;
2059 else if(node->code == jit_code_epilog)
2060 _jitc->function = NULL;
2061 else {
4a71579b 2062#if JIT_HASH_CONSTS
ba3814c1
PC		 2063 if (mask & jit_cc_a0_flt) {
2064 node->u.p = jit_data(&node->u.f, sizeof(jit_float32_t), 4);
2065 node->flag |= jit_flag_node | jit_flag_data;
2066 }
2067 else if (mask & jit_cc_a0_dbl) {
2068 node->u.p = jit_data(&node->u.d, sizeof(jit_float64_t), 8);
2069 node->flag |= jit_flag_node | jit_flag_data;
2070 }
2071 else if (mask & jit_cc_a1_flt) {
2072 node->v.p = jit_data(&node->v.f, sizeof(jit_float32_t), 4);
2073 node->flag |= jit_flag_node | jit_flag_data;
2074 }
2075 else if (mask & jit_cc_a1_dbl) {
2076 node->v.p = jit_data(&node->v.d, sizeof(jit_float64_t), 8);
2077 node->flag |= jit_flag_node | jit_flag_data;
2078 }
2079 else if (mask & jit_cc_a2_flt) {
2080 node->w.p = jit_data(&node->w.f, sizeof(jit_float32_t), 4);
2081 node->flag |= jit_flag_node | jit_flag_data;
2082 }
2083 else if (mask & jit_cc_a2_dbl) {
2084 node->w.p = jit_data(&node->w.d, sizeof(jit_float64_t), 8);
2085 node->flag |= jit_flag_node | jit_flag_data;
2086 }
4a71579b 2087#endif
ba3814c1
PC		 2088 if (_jitc->function) {
2089 if ((mask & (jit_cc_a0_reg|jit_cc_a0_chg)) ==
2090 (jit_cc_a0_reg|jit_cc_a0_chg)) {
2091 if (mask & jit_cc_a0_rlh) {
2092 jit_regset_setbit(&_jitc->function->regset,
2093 jit_regno(node->u.q.l));
4a71579b 2094 jit_regset_setbit(&_jitc->function->regset,
ba3814c1
PC		 2095 jit_regno(node->u.q.h));
2096 }
2097 else
4a71579b 2098 jit_regset_setbit(&_jitc->function->regset,
ba3814c1 2099 jit_regno(node->u.w));
4a71579b 2100 }
ba3814c1
PC		 2101 if ((mask & (jit_cc_a1_reg|jit_cc_a1_chg)) ==
2102 (jit_cc_a1_reg|jit_cc_a1_chg))
2103 jit_regset_setbit(&_jitc->function->regset,
2104 jit_regno(node->v.w));
2105 if ((mask & (jit_cc_a2_reg|jit_cc_a2_chg)) ==
2106 (jit_cc_a2_reg|jit_cc_a2_chg))
2107 jit_regset_setbit(&_jitc->function->regset,
2108 jit_regno(node->w.w));
2109 }
4a71579b
PC		 2110 }
2111 }
2112}
2113
2114void
2115_jit_reglive(jit_state_t *_jit, jit_node_t *node)
2116{
2117 jit_int32_t spec;
2118 jit_int32_t value;
2119 jit_block_t *block;
2120
2121 switch (node->code) {
2122 case jit_code_label: case jit_code_prolog: case jit_code_epilog:
2123 block = _jitc->blocks.ptr + node->v.w;
2124 jit_regset_set(&_jitc->reglive, &block->reglive);
79bfeef6
PC		 2125 jit_regset_set_ui(&_jitc->explive, 0);
2126 break;
2127 case jit_code_live:
2128 jit_regset_setbit(&_jitc->explive, node->u.w);
4a71579b
PC		 2129 break;
2130 case jit_code_callr:
2131 value = jit_regno(node->u.w);
2132 if (!(node->u.w & jit_regno_patch)) {
2133 jit_regset_setbit(&_jitc->reglive, value);
2134 }
2135 case jit_code_calli:
2136 for (value = 0; value < _jitc->reglen; value++) {
2137 spec = jit_class(_rvs[value].spec);
2138 if ((spec & jit_class_arg) && jit_regarg_p(node, value))
2139 jit_regset_setbit(&_jitc->reglive, value);
2140 else if (!(spec & jit_class_sav))
2141 jit_regset_clrbit(&_jitc->reglive, value);
2142 }
2143 break;
2144 default:
2145 value = jit_classify(node->code);
2146 if (value & jit_cc_a0_reg) {
2147 if (value & jit_cc_a0_rlh) {
2148 if (!(node->u.q.l & jit_regno_patch)) {
2149 if (value & jit_cc_a0_chg) {
2150 jit_regset_clrbit(&_jitc->reglive, node->u.q.l);
2151 jit_regset_setbit(&_jitc->regmask, node->u.q.l);
2152 }
2153 else
2154 jit_regset_setbit(&_jitc->reglive, node->u.q.l);
2155 }
2156 if (!(node->u.q.h & jit_regno_patch)) {
2157 if (value & jit_cc_a0_chg) {
2158 jit_regset_clrbit(&_jitc->reglive, node->u.q.h);
2159 jit_regset_setbit(&_jitc->regmask, node->u.q.h);
2160 }
2161 else
2162 jit_regset_setbit(&_jitc->reglive, node->u.q.h);
2163 }
2164 }
2165 else {
2166 if (!(node->u.w & jit_regno_patch)) {
2167 if (value & jit_cc_a0_chg) {
2168 jit_regset_clrbit(&_jitc->reglive, node->u.w);
2169 jit_regset_setbit(&_jitc->regmask, node->u.w);
2170 }
2171 else
2172 jit_regset_setbit(&_jitc->reglive, node->u.w);
2173 }
2174 }
2175 }
ba86ff93
PC		 2176 if (value & jit_cc_a1_reg) {
2177 if (value & jit_cc_a1_rlh) {
2178 /* Assume registers are not changed */
2179 if (!(node->v.q.l & jit_regno_patch))
2180 jit_regset_setbit(&_jitc->reglive, node->v.q.l);
2181 if (!(node->v.q.h & jit_regno_patch))
2182 jit_regset_setbit(&_jitc->reglive, node->v.q.h);
2183 }
2184 else if (!(node->v.w & jit_regno_patch)) {
2185 if (value & jit_cc_a1_chg) {
2186 jit_regset_clrbit(&_jitc->reglive, node->v.w);
2187 jit_regset_setbit(&_jitc->regmask, node->v.w);
2188 }
2189 else
2190 jit_regset_setbit(&_jitc->reglive, node->v.w);
4a71579b 2191 }
4a71579b 2192 }
ba3814c1
PC		 2193 if (value & jit_cc_a2_reg) {
2194 if (value & jit_cc_a2_rlh) {
2195 /* Assume registers are not changed */
2196 if (!(node->w.q.l & jit_regno_patch))
2197 jit_regset_setbit(&_jitc->reglive, node->w.q.l);
2198 if (!(node->w.q.h & jit_regno_patch))
2199 jit_regset_setbit(&_jitc->reglive, node->w.q.h);
2200 }
2201 else {
2202 if (!(node->w.w & jit_regno_patch)) {
2203 if (value & jit_cc_a2_chg) {
2204 jit_regset_clrbit(&_jitc->reglive, node->w.w);
2205 jit_regset_setbit(&_jitc->regmask, node->w.w);
2206 }
2207 else
2208 jit_regset_setbit(&_jitc->reglive, node->w.w);
2209 }
4a71579b 2210 }
4a71579b
PC		 2211 }
2212 if (jit_regset_set_p(&_jitc->regmask)) {
2213 jit_update(node->next, &_jitc->reglive, &_jitc->regmask);
2214 if (jit_regset_set_p(&_jitc->regmask)) {
2215 /* any unresolved live state is considered as live */
2216 jit_regset_ior(&_jitc->reglive,
2217 &_jitc->reglive, &_jitc->regmask);
2218 jit_regset_set_ui(&_jitc->regmask, 0);
2219 }
2220 }
2221 break;
2222 }
2223}
2224
2225void
2226_jit_regarg_set(jit_state_t *_jit, jit_node_t *node, jit_int32_t value)
2227{
2228#if GET_JIT_SIZE
2229 jit_size_prepare();
2230#endif
2231 if (value & jit_cc_a0_reg) {
2232 if (value & jit_cc_a0_rlh) {
2233 jit_regset_setbit(&_jitc->regarg, jit_regno(node->u.q.l));
2234 jit_regset_setbit(&_jitc->regarg, jit_regno(node->u.q.h));
2235 }
2236 else
2237 jit_regset_setbit(&_jitc->regarg, jit_regno(node->u.w));
2238 }
ba86ff93
PC		 2239 if (value & jit_cc_a1_reg) {
2240 if (value & jit_cc_a1_rlh) {
2241 jit_regset_setbit(&_jitc->regarg, jit_regno(node->v.q.l));
2242 jit_regset_setbit(&_jitc->regarg, jit_regno(node->v.q.h));
2243 }
2244 else
2245 jit_regset_setbit(&_jitc->regarg, jit_regno(node->v.w));
2246 }
ba3814c1
PC		 2247 if (value & jit_cc_a2_reg) {
2248 if (value & jit_cc_a2_rlh) {
2249 jit_regset_setbit(&_jitc->regarg, jit_regno(node->w.q.l));
2250 jit_regset_setbit(&_jitc->regarg, jit_regno(node->w.q.h));
2251 }
2252 else
2253 jit_regset_setbit(&_jitc->regarg, jit_regno(node->w.w));
2254 }
79bfeef6
PC		 2255 /* Prevent incorrect detection of running out of registers
2256 * if will need to patch jump, and all registers have been
2257 * used in the current block. */
2258 if (node->code == jit_code_jmpi && (node->flag & jit_flag_node)) {
2259 jit_node_t *label = node->u.n;
2260 jit_block_t *block = _jitc->blocks.ptr + label->v.w;
2261 jit_regset_set(&_jitc->reglive, &block->reglive);
2262 jit_regset_set(&_jitc->regmask, &block->regmask);
2263 if (jit_regset_set_p(&_jitc->explive)) {
2264 jit_regset_ior(&_jitc->reglive, &block->reglive, &_jitc->explive);
2265 jit_regset_xor(&_jitc->regmask, &_jitc->regmask, &_jitc->explive);
2266 }
2267 }
4a71579b
PC		 2268}
2269
2270void
2271_jit_regarg_clr(jit_state_t *_jit, jit_node_t *node, jit_int32_t value)
2272{
2273#if GET_JIT_SIZE
2274 jit_size_collect(node);
2275#endif
2276 if (value & jit_cc_a0_reg) {
2277 if (value & jit_cc_a0_rlh) {
2278 jit_regset_clrbit(&_jitc->regarg, jit_regno(node->u.q.l));
2279 jit_regset_clrbit(&_jitc->regarg, jit_regno(node->u.q.h));
2280 }
2281 else
2282 jit_regset_clrbit(&_jitc->regarg, jit_regno(node->u.w));
2283 }
ba86ff93
PC		 2284 if (value & jit_cc_a1_reg) {
2285 if (value & jit_cc_a1_rlh) {
2286 jit_regset_clrbit(&_jitc->regarg, jit_regno(node->v.q.l));
2287 jit_regset_clrbit(&_jitc->regarg, jit_regno(node->v.q.h));
2288 }
2289 else
2290 jit_regset_clrbit(&_jitc->regarg, jit_regno(node->v.w));
2291 }
ba3814c1
PC		 2292 if (value & jit_cc_a2_reg) {
2293 if (value & jit_cc_a2_rlh) {
2294 jit_regset_clrbit(&_jitc->regarg, jit_regno(node->w.q.l));
2295 jit_regset_clrbit(&_jitc->regarg, jit_regno(node->w.q.h));
2296 }
2297 else
2298 jit_regset_clrbit(&_jitc->regarg, jit_regno(node->w.w));
2299 }
4a71579b
PC		 2300}
2301
2302void
2303_jit_realize(jit_state_t *_jit)
2304{
2305 assert(!_jitc->realize);
2306 if (_jitc->function)
2307 jit_epilog();
2308 jit_optimize();
2309 _jitc->realize = 1;
2310
2311 /* ensure it is aligned */
2312 _jitc->data.offset = (_jitc->data.offset + 7) & -8;
2313
2314#if GET_JIT_SIZE
2315 /* Heuristic to guess code buffer size */
2316 _jitc->mult = 4;
2317 _jit->code.length = _jitc->pool.length * 1024 * _jitc->mult;
2318#else
2319 _jit->code.length = jit_get_size();
2320#endif
2321}
2322
2323void
2324_jit_dataset(jit_state_t *_jit)
2325{
2326 jit_uint8_t *ptr;
2327 jit_node_t *node;
2328 jit_word_t offset;
2329#if defined(__sgi)
2330 int mmap_fd;
2331#endif
2332
2333 assert(!_jitc->dataset);
40a44dcb
PC		 2334#if !HAVE_MMAP
2335 assert(_jit->user_data);
2336#else
4a71579b
PC		 2337 if (!_jit->user_data) {
2338
2339 /* create read only data buffer */
2340 _jit->data.length = (_jitc->data.offset +
2341 /* reserve space for annotations */
2342 _jitc->note.size + 4095) & -4096;
2343#if defined(__sgi)
2344 mmap_fd = open("/dev/zero", O_RDWR);
2345#endif
2346 _jit->data.ptr = mmap(NULL, _jit->data.length,
2347 PROT_READ | PROT_WRITE,
2348 MAP_PRIVATE | MAP_ANON, mmap_fd, 0);
2349 assert(_jit->data.ptr != MAP_FAILED);
2350#if defined(__sgi)
2351 close(mmap_fd);
2352#endif
2353 }
40a44dcb 2354#endif /* !HAVE_MMAP */
4a71579b
PC		 2355
2356 if (!_jitc->no_data)
2357 jit_memcpy(_jit->data.ptr, _jitc->data.ptr, _jitc->data.offset);
2358
2359 if (_jitc->no_note) {
2360 /* Space for one note is always allocated, so revert it here
2361 * if after jit_new_state was called, it is also requested to
2362 * not generate annotation information */
2363 _jit->note.length = 0;
2364 _jitc->note.size = 0;
2365 }
2366 else {
2367 _jitc->note.base = _jit->data.ptr;
2368 if (!_jitc->no_data)
2369 _jitc->note.base += _jitc->data.offset;
2370 memset(_jitc->note.base, 0, _jitc->note.size);
2371 }
2372
2373 if (_jit->user_data)
2374 /* Need the temporary hashed data until jit_emit is finished */
2375 ptr = _jitc->no_data ? _jitc->data.ptr : _jit->data.ptr;
2376 else {
2377 ptr = _jit->data.ptr;
2378 /* Temporary hashed data no longer required */
2379 jit_free((jit_pointer_t *)&_jitc->data.ptr);
2380 }
2381
2382 for (offset = 0; offset < _jitc->data.size; offset++) {
2383 for (node = _jitc->data.table[offset]; node; node = node->next) {
2384 node->flag |= jit_flag_patch;
2385 node->u.w = (jit_word_t)(ptr + node->u.w);
2386 }
2387 }
2388
2389 _jitc->dataset = 1;
2390}
2391
2392jit_pointer_t
2393_jit_get_code(jit_state_t *_jit, jit_word_t *length)
2394{
2395 assert(_jitc->realize);
2396 if (length) {
2397 if (_jitc->done)
2398 /* If code already generated, return exact size of code */
2399 *length = _jit->pc.uc - _jit->code.ptr;
2400 else
2401 /* Else return current size of the code buffer */
2402 *length = _jit->code.length;
2403 }
2404
2405 return (_jit->code.ptr);
2406}
2407
2408void
2409_jit_set_code(jit_state_t *_jit, jit_pointer_t ptr, jit_word_t length)
2410{
2411 assert(_jitc->realize);
2412 _jit->code.ptr = ptr;
2413 _jit->code.length = length;
2414 _jit->user_code = 1;
2415}
2416
2417jit_pointer_t
2418_jit_get_data(jit_state_t *_jit, jit_word_t *data_size, jit_word_t *note_size)
2419{
2420 assert(_jitc->realize);
2421 if (data_size)
2422 *data_size = _jitc->data.offset;
2423 if (note_size)
2424 *note_size = _jitc->note.size;
2425 return (_jit->data.ptr);
2426}
2427
2428void
2429_jit_set_data(jit_state_t *_jit, jit_pointer_t ptr,
2430 jit_word_t length, jit_word_t flags)
2431{
2432 assert(_jitc->realize);
2433 if (flags & JIT_DISABLE_DATA)
2434 _jitc->no_data = 1;
2435 else
2436 assert(length >= _jitc->data.offset);
2437 if (flags & JIT_DISABLE_NOTE)
2438 _jitc->no_note = 1;
2439 else {
2440 if (flags & JIT_DISABLE_DATA)
2441 assert(length >= _jitc->note.size);
2442 else
2443 assert(length >= _jitc->data.offset + _jitc->note.size);
2444 }
2445 _jit->data.ptr = ptr;
2446 _jit->data.length = length;
2447 _jit->user_data = 1;
2448}
2449
2450jit_pointer_t
2451_jit_emit(jit_state_t *_jit)
2452{
2453 jit_pointer_t code;
2454 jit_node_t *node;
2455 size_t length;
d481fb64 2456#ifndef NDEBUG
4a71579b 2457 int result;
d481fb64 2458#endif
4a71579b
PC		 2459#if defined(__sgi)
2460 int mmap_fd;
2461#endif
c0c16242 2462 int mmap_prot, mmap_flags;
4a71579b
PC		 2463
2464 if (!_jitc->realize)
2465 jit_realize();
2466
2467 if (!_jitc->dataset)
2468 jit_dataset();
2469
2470 _jitc->emit = 1;
2471
40a44dcb
PC		 2472#if !HAVE_MMAP
2473 assert(_jit->user_code);
2474#else
4a71579b 2475 if (!_jit->user_code) {
c0c16242 2476 mmap_prot = PROT_READ | PROT_WRITE;
79bfeef6 2477#if !(__OpenBSD__ || __APPLE__)
c0c16242
PC		 2478 mmap_prot |= PROT_EXEC;
2479#endif
2480#if __NetBSD__
2481 mmap_prot = PROT_MPROTECT(mmap_prot);
2482 mmap_flags = 0;
2483#else
2484 mmap_flags = MAP_PRIVATE;
2485#endif
2486 mmap_flags |= MAP_ANON;
4a71579b
PC		 2487#if defined(__sgi)
2488 mmap_fd = open("/dev/zero", O_RDWR);
2489#endif
2490 _jit->code.ptr = mmap(NULL, _jit->code.length,
c0c16242 2491 mmap_prot, mmap_flags, mmap_fd, 0);
4a71579b
PC		 2492 assert(_jit->code.ptr != MAP_FAILED);
2493 }
40a44dcb 2494#endif /* !HAVE_MMAP */
4a71579b
PC		 2495 _jitc->code.end = _jit->code.ptr + _jit->code.length -
2496 jit_get_max_instr();
2497 _jit->pc.uc = _jit->code.ptr;
2498
2499 for (;;) {
c0c16242 2500#if __NetBSD__
d481fb64
PC		 2501#ifndef NDEBUG
2502 result =
2503#endif
2504 mprotect(_jit->code.ptr, _jit->code.length, PROT_READ | PROT_WRITE);
c0c16242
PC		 2505 assert(result == 0);
2506#endif
4a71579b
PC		 2507 if ((code = emit_code()) == NULL) {
2508 _jitc->patches.offset = 0;
2509 for (node = _jitc->head; node; node = node->next) {
2510 if (node->link &&
2511 (node->code == jit_code_label ||
2512 node->code == jit_code_epilog))
2513 node->flag &= ~jit_flag_patch;
2514 }
40a44dcb
PC		 2515#if !HAVE_MMAP
2516 assert(_jit->user_code);
2517#else
4a71579b
PC		 2518 if (_jit->user_code)
2519 goto fail;
2520#if GET_JIT_SIZE
2521 ++_jitc->mult;
2522 length = _jitc->pool.length * 1024 * _jitc->mult;
2523#else
2524 /* Should only happen on very special cases */
2525 length = _jit->code.length + 4096;
2526#endif
2527
2528#if !HAVE_MREMAP
2529 munmap(_jit->code.ptr, _jit->code.length);
2530#endif
2531
2532#if HAVE_MREMAP
2533# if __NetBSD__
2534 _jit->code.ptr = mremap(_jit->code.ptr, _jit->code.length,
2535 _jit->code.ptr, length, 0);
2536# else
2537 _jit->code.ptr = mremap(_jit->code.ptr, _jit->code.length,
2538 length, MREMAP_MAYMOVE, NULL);
2539# endif
2540#else
2541 _jit->code.ptr = mmap(NULL, length,
79bfeef6 2542 mmap_prot, mmap_flags, mmap_fd, 0);
4a71579b
PC		 2543#endif
2544
2545 assert(_jit->code.ptr != MAP_FAILED);
2546 _jit->code.length = length;
2547 _jitc->code.end = _jit->code.ptr + _jit->code.length -
2548 jit_get_max_instr();
2549 _jit->pc.uc = _jit->code.ptr;
40a44dcb 2550#endif /* !HAVE_MMAP */
4a71579b
PC		 2551 }
2552 else
2553 break;
2554 }
2555
2556#if defined(__sgi)
2557 if (!_jit->user_code)
2558 close(mmap_fd);
2559#endif
2560
2561 _jitc->done = 1;
2562 if (!_jitc->no_note)
2563 jit_annotate();
2564
2565 if (_jit->user_data)
2566 jit_free((jit_pointer_t *)&_jitc->data.ptr);
40a44dcb 2567#if HAVE_MMAP
4a71579b 2568 else {
d481fb64
PC		 2569# ifndef NDEBUG
2570 result =
2571# endif
2572 mprotect(_jit->data.ptr, _jit->data.length, PROT_READ);
4a71579b
PC		 2573 assert(result == 0);
2574 }
2575 if (!_jit->user_code) {
ba86ff93 2576 _jit->code.protect = _jit->pc.uc - _jit->code.ptr;
c0c16242
PC		 2577# if __riscv && __WORDSIZE == 64
2578 /* FIXME should start adding consts at a page boundary */
ba86ff93 2579 _jit->code.protect -= _jitc->consts.hash.count * sizeof(jit_word_t);
c0c16242 2580# endif
d481fb64
PC		 2581# ifndef NDEBUG
2582 result =
2583 mprotect(_jit->code.ptr, _jit->code.protect, PROT_READ | PROT_EXEC);
2584# endif
4a71579b
PC		 2585 assert(result == 0);
2586 }
40a44dcb 2587#endif /* HAVE_MMAP */
4a71579b
PC		 2588
2589 return (_jit->code.ptr);
2590fail:
2591 return (NULL);
2592}
2593
79bfeef6
PC		 2594void
2595_jit_protect(jit_state_t *_jit)
2596{
2597#if !HAVE_MMAP
2598 assert (_jit->user_code);
2599#else
d481fb64 2600# ifndef NDEBUG
79bfeef6 2601 int result;
d481fb64 2602# endif
79bfeef6 2603 if (_jit->user_code) return;
d481fb64
PC		 2604# ifndef NDEBUG
2605 result =
2606#endif
2607 mprotect (_jit->code.ptr, _jit->code.protect, PROT_READ | PROT_EXEC);
79bfeef6
PC		 2608 assert (result == 0);
2609#endif
2610}
2611
2612void
2613_jit_unprotect(jit_state_t *_jit)
2614{
2615#if !HAVE_MMAP
2616 assert (_jit->user_code);
2617#else
d481fb64 2618# ifndef NDEBUG
79bfeef6 2619 int result;
d481fb64 2620# endif
79bfeef6 2621 if (_jit->user_code) return;
d481fb64
PC		 2622# ifndef NDEBUG
2623 result =
2624# endif
2625 mprotect (_jit->code.ptr, _jit->code.protect, PROT_READ | PROT_WRITE);
79bfeef6
PC		 2626 assert (result == 0);
2627#endif
2628}
2629
4a71579b
PC		 2630void
2631_jit_frame(jit_state_t *_jit, jit_int32_t frame)
2632{
2633 jit_trampoline(frame, 1);
2634}
2635
2636void
2637_jit_tramp(jit_state_t *_jit, jit_int32_t frame)
2638{
2639 jit_trampoline(frame, 0);
2640}
2641
2642void
2643_jit_trampoline(jit_state_t *_jit, jit_int32_t frame, jit_bool_t prolog)
2644{
2645 jit_int32_t regno;
2646
2647 /* Must be called after prolog, actually, just to simplify
2648 * tests and know there is a current function and that
2649 * _jitc->function->self.aoff is at the before any alloca value */
2650 assert(_jitc->tail && _jitc->tail->code == jit_code_prolog);
2651
2652 /* + 24 for 3 possible spilled temporaries (that could be a double) */
2653 frame += 24;
2654#if defined(__hppa__)
2655 frame += _jitc->function->self.aoff;
2656#else
2657 frame -= _jitc->function->self.aoff;
2658#endif
2659 _jitc->function->frame = frame;
2660 if (prolog)
2661 _jitc->function->define_frame = 1;
2662 else
2663 _jitc->function->assume_frame = 1;
2664 for (regno = 0; regno < _jitc->reglen; regno++)
2665 if (jit_class(_rvs[regno].spec) & jit_class_sav)
2666 jit_regset_setbit(&_jitc->function->regset, regno);
2667}
2668
2669/* Compute initial reglive and regmask set values of a basic block.
2670 * reglive is the set of known live registers
2671 * regmask is the set of registers not referenced in the block
2672 * Registers in regmask might be live.
2673 */
2674static void
2675_jit_setup(jit_state_t *_jit, jit_block_t *block)
2676{
2677 jit_node_t *node;
2678 jit_bool_t live;
2679 unsigned long value;
2680
2681 jit_regset_set_mask(&block->regmask, _jitc->reglen);
2682 for (value = 0; value < _jitc->reglen; ++value)
2683 if (!(jit_class(_rvs[value].spec) & (jit_class_gpr|jit_class_fpr)))
2684 jit_regset_clrbit(&block->regmask, value);
2685
2686 for (node = block->label->next; node; node = node->next) {
2687 switch (node->code) {
2688 case jit_code_label: case jit_code_prolog:
2689 case jit_code_epilog:
2690 return;
2691 default:
2692 /* Check argument registers in reverse order to properly
2693 * handle registers that are both, argument and result */
2694 value = jit_classify(node->code);
ba86ff93 2695 if (value & jit_cc_a2_reg) {
4a71579b 2696 live = !(value & jit_cc_a2_chg);
ba86ff93
PC		 2697 if (value & jit_cc_a2_rlh) {
2698 /* Assume will not modify a pair in second argument */
2699 assert(live);
2700 if (!(node->w.q.l & jit_regno_patch) &&
2701 jit_regset_tstbit(&block->regmask, node->w.q.l)) {
2702 jit_regset_clrbit(&block->regmask, node->w.q.l);
2703 }
2704 if (!(node->w.q.h & jit_regno_patch) &&
2705 jit_regset_tstbit(&block->regmask, node->w.q.h)) {
2706 jit_regset_clrbit(&block->regmask, node->w.q.h);
2707 }
2708 }
2709 else {
2710 if (!(node->w.w & jit_regno_patch) &&
2711 jit_regset_tstbit(&block->regmask, node->w.w)) {
2712 jit_regset_clrbit(&block->regmask, node->w.w);
2713 if (live)
2714 jit_regset_setbit(&block->reglive, node->w.w);
2715 }
2716 }
4a71579b 2717 }
ba86ff93 2718 if (value & jit_cc_a1_reg) {
4a71579b 2719 live = !(value & jit_cc_a1_chg);
ba86ff93
PC		 2720 if (value & jit_cc_a1_rlh) {
2721 /* Assume will not modify a pair in second argument */
2722 assert(live);
2723 if (!(node->v.q.l & jit_regno_patch) &&
2724 jit_regset_tstbit(&block->regmask, node->v.q.l)) {
2725 jit_regset_clrbit(&block->regmask, node->v.q.l);
2726 }
2727 if (!(node->v.q.h & jit_regno_patch) &&
2728 jit_regset_tstbit(&block->regmask, node->v.q.h)) {
2729 jit_regset_clrbit(&block->regmask, node->v.q.h);
2730 }
2731 }
2732 else {
2733 if (!(node->v.w & jit_regno_patch) &&
2734 jit_regset_tstbit(&block->regmask, node->v.w)) {
2735 jit_regset_clrbit(&block->regmask, node->v.w);
2736 if (live)
2737 jit_regset_setbit(&block->reglive, node->v.w);
2738 }
2739 }
4a71579b
PC		 2740 }
2741 if (value & jit_cc_a0_reg) {
2742 live = !(value & jit_cc_a0_chg);
2743 if (value & jit_cc_a0_rlh) {
2744 if (!(node->u.q.l & jit_regno_patch) &&
2745 jit_regset_tstbit(&block->regmask, node->u.q.l)) {
2746 jit_regset_clrbit(&block->regmask, node->u.q.l);
2747 if (live)
2748 jit_regset_setbit(&block->reglive, node->u.q.l);
2749 }
2750 if (!(node->u.q.h & jit_regno_patch) &&
2751 jit_regset_tstbit(&block->regmask, node->u.q.h)) {
2752 jit_regset_clrbit(&block->regmask, node->u.q.h);
2753 if (live)
2754 jit_regset_setbit(&block->reglive, node->u.q.h);
2755 }
2756 }
2757 else {
2758 if (!(node->u.w & jit_regno_patch) &&
2759 jit_regset_tstbit(&block->regmask, node->u.w)) {
2760 jit_regset_clrbit(&block->regmask, node->u.w);
2761 if (live)
2762 jit_regset_setbit(&block->reglive, node->u.w);
2763 }
2764 }
2765 }
2766 break;
2767 }
2768 }
2769}
2770
2771/* Update regmask and reglive of blocks at entry point of branch targets
2772 * or normal flow that have a live register not used in this block.
2773 */
2774static void
c0c16242 2775_jit_follow(jit_state_t *_jit, jit_block_t *block)
4a71579b
PC		 2776{
2777 jit_node_t *node;
2778 jit_block_t *next;
2779 jit_int32_t spec;
2780 jit_int32_t regno;
2781 unsigned long value;
2782 jit_node_t *label;
2783 jit_regset_t reglive;
2784 jit_regset_t regmask;
2785 jit_regset_t regtemp;
2786
2787 jit_regset_set(&reglive, &block->reglive);
2788 jit_regset_set(&regmask, &block->regmask);
2789 for (node = block->label->next; node; node = node->next) {
2790 switch (node->code) {
2791 case jit_code_label:
2792 /* Do not consider jmpi and jmpr cannot jump to the
2793 * next instruction. */
2794 next = _jitc->blocks.ptr + node->v.w;
2795 /* Set of live registers in next block that are at unknown
2796 * state in this block. */
2797 jit_regset_and(&regtemp, &regmask, &next->reglive);
2798 if (jit_regset_set_p(&regtemp)) {
2799 /* Add live state of next block to current block. */
2800 jit_regset_ior(&block->reglive, &block->reglive, &regtemp);
2801 /* Remove from unknown state bitmask. */
2802 jit_regset_com(&regtemp, &regtemp);
2803 jit_regset_and(&block->regmask, &block->regmask, &regtemp);
c0c16242 2804 block->again = 1;
4a71579b
PC		 2805 }
2806 case jit_code_prolog:
2807 case jit_code_epilog:
2808 return;
2809 case jit_code_callr:
2810 value = jit_regno(node->u.w);
2811 if (!(node->u.w & jit_regno_patch)) {
2812 if (jit_regset_tstbit(&regmask, value)) {
2813 jit_regset_clrbit(&regmask, value);
2814 jit_regset_setbit(&reglive, value);
2815 }
2816 }
2817 case jit_code_calli:
2818 for (value = 0; value < _jitc->reglen; ++value) {
2819 value = jit_regset_scan1(&regmask, value);
2820 if (value >= _jitc->reglen)
2821 break;
2822 spec = jit_class(_rvs[value].spec);
2823 if (!(spec & jit_class_sav))
2824 jit_regset_clrbit(&regmask, value);
2825 if ((spec & jit_class_arg) && jit_regarg_p(node, value))
2826 jit_regset_setbit(&reglive, value);
2827 }
2828 break;
2829 default:
2830 value = jit_classify(node->code);
2831 if (value & jit_cc_a2_reg) {
ba3814c1
PC		 2832 if (value & jit_cc_a2_rlh) {
2833 if (!(node->w.q.l & jit_regno_patch)) {
2834 /* Assume register is not changed */
2835 if (jit_regset_tstbit(&regmask, node->w.q.l))
2836 jit_regset_clrbit(&regmask, node->w.q.l);
2837 }
2838 if (!(node->w.q.h & jit_regno_patch)) {
2839 if (jit_regset_tstbit(&regmask, node->w.q.h))
2840 jit_regset_clrbit(&regmask, node->w.q.h);
2841 }
2842 }
2843 else {
ba86ff93
PC		 2844 if (!(node->w.w & jit_regno_patch)) {
2845 if (jit_regset_tstbit(&regmask, node->w.w)) {
2846 jit_regset_clrbit(&regmask, node->w.w);
2847 if (!(value & jit_cc_a2_chg))
2848 jit_regset_setbit(&reglive, node->w.w);
ba3814c1 2849 }
4a71579b
PC		 2850 }
2851 }
2852 }
2853 if (value & jit_cc_a1_reg) {
ba86ff93
PC		 2854 if (value & jit_cc_a1_rlh) {
2855 if (!(node->v.q.l & jit_regno_patch)) {
2856 /* Assume register is not changed */
2857 if (jit_regset_tstbit(&regmask, node->v.q.l))
2858 jit_regset_clrbit(&regmask, node->v.q.l);
2859 }
2860 if (!(node->v.q.h & jit_regno_patch)) {
2861 if (jit_regset_tstbit(&regmask, node->v.q.h))
2862 jit_regset_clrbit(&regmask, node->v.q.h);
2863 }
2864 }
2865 else {
2866 if (!(node->v.w & jit_regno_patch)) {
2867 if (jit_regset_tstbit(&regmask, node->v.w)) {
2868 jit_regset_clrbit(&regmask, node->v.w);
2869 if (!(value & jit_cc_a1_chg))
2870 jit_regset_setbit(&reglive, node->v.w);
2871 }
4a71579b
PC		 2872 }
2873 }
2874 }
2875 if (value & jit_cc_a0_reg) {
2876 if (value & jit_cc_a0_rlh) {
2877 if (!(node->u.q.l & jit_regno_patch)) {
2878 if (jit_regset_tstbit(&regmask, node->u.q.l)) {
2879 jit_regset_clrbit(&regmask, node->u.q.l);
2880 if (!(value & jit_cc_a0_chg))
2881 jit_regset_setbit(&reglive, node->u.q.l);
2882 }
2883 }
2884 if (!(node->u.q.h & jit_regno_patch)) {
2885 if (jit_regset_tstbit(&regmask, node->u.q.h)) {
2886 jit_regset_clrbit(&regmask, node->u.q.h);
2887 if (!(value & jit_cc_a0_chg))
2888 jit_regset_setbit(&reglive, node->u.q.h);
2889 }
2890 }
2891 }
2892 else {
2893 if (!(node->u.w & jit_regno_patch)) {
2894 if (jit_regset_tstbit(&regmask, node->u.w)) {
2895 jit_regset_clrbit(&regmask, node->u.w);
2896 if (!(value & jit_cc_a0_chg))
2897 jit_regset_setbit(&reglive, node->u.w);
2898 }
2899 }
2900 }
2901 }
2902 if (value & jit_cc_a0_jmp) {
2903 if (node->flag & jit_flag_node) {
2904 label = node->u.n;
2905 /* Do not consider jmpi and jmpr cannot jump to the
2906 * next instruction. */
2907 next = _jitc->blocks.ptr + label->v.w;
2908 jit_regset_and(&regtemp, &regmask, &next->reglive);
2909 if (jit_regset_set_p(&regtemp)) {
2910 /* Add live state. */
2911 jit_regset_ior(&block->reglive,
2912 &block->reglive, &regtemp);
2913 /* Remove from unknown state bitmask. */
2914 jit_regset_com(&regtemp, &regtemp);
2915 jit_regset_and(&block->regmask,
2916 &block->regmask, &regtemp);
c0c16242 2917 block->again = 1;
4a71579b
PC		 2918 }
2919 }
2920 else {
2921 /* Jump to unknown location.
2922 * This is a pitfall of the implementation.
2923 * Only jmpi to not a jit code should reach here,
2924 * or a jmpr of a computed address.
2925 * Because the implementation needs jit_class_nospill
2926 * registers, must treat jmpr as a function call. This
2927 * means that only JIT_Vn registers can be trusted on
2928 * arrival of jmpr.
2929 */
ba3814c1 2930 jit_regset_set_ui(&regmask, 0);
4a71579b
PC		 2931 for (regno = 0; regno < _jitc->reglen; regno++) {
2932 spec = jit_class(_rvs[regno].spec);
ba3814c1
PC		 2933 if ((spec & (jit_class_gpr|jit_class_fpr)) &&
2934 (spec & jit_class_sav))
2935 jit_regset_setbit(&regmask, regno);
4a71579b
PC		 2936 }
2937 /* Assume non callee save registers are live due
2938 * to jump to unknown location. */
2939 /* Treat all callee save as live. */
ba3814c1 2940 jit_regset_ior(&block->reglive, &reglive, &regmask);
4a71579b 2941 /* Treat anything else as dead. */
ba3814c1 2942 return;
4a71579b
PC		 2943 }
2944 }
2945 break;
2946 }
2947 }
2948}
2949
2950/* Follow code generation up to finding a label or end of code.
2951 * When finding a label, update the set of live registers.
2952 * On branches, update based on taken branch or normal flow.
2953 */
2954static void
2955_jit_update(jit_state_t *_jit, jit_node_t *node,
2956 jit_regset_t *live, jit_regset_t *mask)
2957{
2958 jit_int32_t spec;
2959 jit_int32_t regno;
2960 unsigned long value;
2961 jit_block_t *block;
2962 jit_node_t *label;
2963 jit_regset_t regtemp;
2964
2965 for (; node; node = node->next) {
2966 if (jit_regset_set_p(mask) == 0)
2967 break;
2968 switch (node->code) {
2969 case jit_code_label:
2970 block = _jitc->blocks.ptr + node->v.w;
2971 jit_regset_and(&regtemp, mask, &block->reglive);
2972 if (jit_regset_set_p(&regtemp)) {
2973 /* Add live state. */
2974 jit_regset_ior(live, live, &regtemp);
2975 /* Remove from unknown state bitmask. */
2976 jit_regset_com(&regtemp, &regtemp);
2977 jit_regset_and(mask, mask, &regtemp);
2978 }
2979 return;
2980 case jit_code_prolog:
2981 jit_regset_set_ui(mask, 0);
2982 return;
2983 case jit_code_epilog:
2984 jit_regset_set_ui(mask, 0);
2985 return;
2986 case jit_code_callr:
2987 value = jit_regno(node->u.w);
2988 if (!(node->u.w & jit_regno_patch)) {
2989 if (jit_regset_tstbit(mask, value)) {
2990 jit_regset_clrbit(mask, value);
2991 jit_regset_setbit(live, value);
2992 }
2993 }
2994 case jit_code_calli:
2995 for (value = 0; value < _jitc->reglen; ++value) {
2996 value = jit_regset_scan1(mask, value);
2997 if (value >= _jitc->reglen)
2998 break;
2999 spec = jit_class(_rvs[value].spec);
3000 if (!(spec & jit_class_sav))
3001 jit_regset_clrbit(mask, value);
3002 if ((spec & jit_class_arg) && jit_regarg_p(node, value))
3003 jit_regset_setbit(live, value);
3004 }
3005 break;
3006 default:
3007 value = jit_classify(node->code);
3008 if (value & jit_cc_a2_reg) {
ba3814c1
PC		 3009 if (value & jit_cc_a2_rlh) {
3010 if (!(node->w.q.l & jit_regno_patch)) {
3011 /* Assume register is not changed */
3012 if (jit_regset_tstbit(mask, node->w.q.l))
3013 jit_regset_clrbit(mask, node->w.q.l);
3014 }
3015 if (!(node->w.q.h & jit_regno_patch)) {
3016 if (jit_regset_tstbit(mask, node->w.q.h))
3017 jit_regset_clrbit(mask, node->w.q.h);
3018 }
3019 }
3020 else {
3021 if (!(node->w.w & jit_regno_patch)) {
3022 if (jit_regset_tstbit(mask, node->w.w)) {
3023 jit_regset_clrbit(mask, node->w.w);
3024 if (!(value & jit_cc_a2_chg))
3025 jit_regset_setbit(live, node->w.w);
3026 }
4a71579b
PC		 3027 }
3028 }
3029 }
3030 if (value & jit_cc_a1_reg) {
ba86ff93
PC		 3031 if (value & jit_cc_a1_rlh) {
3032 if (!(node->v.q.l & jit_regno_patch)) {
3033 /* Assume register is not changed */
3034 if (jit_regset_tstbit(mask, node->v.q.l))
3035 jit_regset_clrbit(mask, node->v.q.l);
3036 }
3037 if (!(node->v.q.h & jit_regno_patch)) {
3038 if (jit_regset_tstbit(mask, node->v.q.h))
3039 jit_regset_clrbit(mask, node->v.q.h);
3040 }
3041 }
3042 else {
3043 if (!(node->v.w & jit_regno_patch)) {
3044 if (jit_regset_tstbit(mask, node->v.w)) {
3045 jit_regset_clrbit(mask, node->v.w);
3046 if (!(value & jit_cc_a1_chg))
3047 jit_regset_setbit(live, node->v.w);
3048 }
4a71579b
PC		 3049 }
3050 }
3051 }
3052 if (value & jit_cc_a0_reg) {
3053 if (value & jit_cc_a0_rlh) {
3054 if (!(node->u.q.l & jit_regno_patch)) {
3055 if (jit_regset_tstbit(mask, node->u.q.l)) {
3056 jit_regset_clrbit(mask, node->u.q.l);
3057 if (!(value & jit_cc_a0_chg))
3058 jit_regset_setbit(live, node->u.q.l);
3059 }
3060 }
3061 if (!(node->u.q.h & jit_regno_patch)) {
3062 if (jit_regset_tstbit(mask, node->u.q.h)) {
3063 jit_regset_clrbit(mask, node->u.q.h);
3064 if (!(value & jit_cc_a0_chg))
3065 jit_regset_setbit(live, node->u.q.h);
3066 }
3067 }
3068 }
3069 else {
3070 if (!(node->u.w & jit_regno_patch)) {
3071 if (jit_regset_tstbit(mask, node->u.w)) {
3072 jit_regset_clrbit(mask, node->u.w);
3073 if (!(value & jit_cc_a0_chg))
3074 jit_regset_setbit(live, node->u.w);
3075 }
3076 }
3077 }
3078 }
3079 if (value & jit_cc_a0_jmp) {
3080 if (node->flag & jit_flag_node) {
3081 label = node->u.n;
3082 /* Do not consider jmpi and jmpr cannot jump to the
3083 * next instruction. */
3084 block = _jitc->blocks.ptr + label->v.w;
3085 jit_regset_and(&regtemp, mask, &block->reglive);
3086 if (jit_regset_set_p(&regtemp)) {
3087 /* Add live state. */
3088 jit_regset_ior(live, live, &regtemp);
3089 /* Remove from unknown state bitmask. */
3090 jit_regset_com(&regtemp, &regtemp);
3091 jit_regset_and(mask, mask, &regtemp);
3092 }
3093 }
3094 else {
3095 /* Jump to unknown location.
3096 * This is a pitfall of the implementation.
3097 * Only jmpi to not a jit code should reach here,
3098 * or a jmpr of a computed address.
3099 * Because the implementation needs jit_class_nospill
3100 * registers, must treat jmpr as a function call. This
3101 * means that only JIT_Vn registers can be trusted on
3102 * arrival of jmpr.
3103 */
ba3814c1 3104 jit_regset_set_ui(mask, 0);
4a71579b
PC		 3105 for (regno = 0; regno < _jitc->reglen; regno++) {
3106 spec = jit_class(_rvs[regno].spec);
ba3814c1
PC		 3107 if ((spec & (jit_class_gpr|jit_class_fpr)) &&
3108 (spec & jit_class_sav))
3109 jit_regset_setbit(mask, regno);
4a71579b
PC		 3110 }
3111 /* Assume non callee save registers are live due
3112 * to jump to unknown location. */
3113 /* Treat all callee save as live. */
3114 jit_regset_ior(live, live, mask);
79bfeef6
PC		 3115 /* Prevent explicitly set as live registers to
3116 * be used as a temporary for the jmpi. */
3117 jit_regset_ior(live, live, &_jitc->explive);
4a71579b 3118 /* Treat anything else as dead. */
ba3814c1 3119 return;
4a71579b
PC		 3120 }
3121 }
3122 break;
3123 }
3124 }
3125}
3126
3127static void
3128_thread_jumps(jit_state_t *_jit)
3129{
3130 jit_node_t *prev;
3131 jit_node_t *node;
3132 jit_node_t *next;
3133 jit_int32_t mask;
3134
3135 for (prev = node = _jitc->head; node;) {
3136 next = node->next;
3137 switch (node->code) {
3138 case jit_code_jmpi:
3139 if (redundant_jump(prev, node)) {
3140 node = prev;
3141 continue;
3142 }
3143 if (shortcut_jump(prev, node))
3144 continue;
3145 break;
3146 case jit_code_jmpr:
3147 case jit_code_callr: case jit_code_calli:
3148 /* non optimizable jump like code */
3149 break;
3150 default:
3151 mask = jit_classify(node->code);
3152 if (mask & jit_cc_a0_jmp) {
3153 if (reverse_jump(prev, node) ||
3154 shortcut_jump(prev, node))
3155 continue;
3156 }
3157 break;
3158 }
3159 prev = node;
3160 node = next;
3161 }
3162}
3163
3164static void
3165_sequential_labels(jit_state_t *_jit)
3166{
3167 jit_node_t *jump;
3168 jit_node_t *link;
3169 jit_node_t *prev;
3170 jit_node_t *next;
3171 jit_node_t *node;
3172
3173 for (prev = node = _jitc->head; node; node = next) {
3174 next = node->next;
3175 if (node->code == jit_code_label) {
3176 if (!node->flag) {
3177 if (!node->link) {
3178 del_label(prev, node);
3179 continue;
3180 }
3181 if (prev != node && prev->code == jit_code_label) {
3182 if ((jump = node->link)) {
3183 for (; jump; jump = link) {
3184 link = jump->link;
79bfeef6
PC		 3185 if (jump->code == jit_code_movi)
3186 jump->v.n = prev;
3187 else
3188 jump->u.n = prev;
4a71579b
PC		 3189 jump->link = prev->link;
3190 prev->link = jump;
3191 }
3192 node->link = NULL;
3193 }
3194 del_label(prev, node);
3195 continue;
3196 }
3197 }
3198 if (next && next->code == jit_code_label && !next->flag) {
3199 if ((jump = next->link)) {
3200 for (; jump; jump = link) {
3201 link = jump->link;
79bfeef6
PC		 3202 if (jump->code == jit_code_movi)
3203 jump->v.n = node;
3204 else
3205 jump->u.n = node;
4a71579b
PC		 3206 jump->link = node->link;
3207 node->link = jump;
3208 }
3209 next->link = NULL;
3210 }
3211 del_label(node, next);
3212 next = node->next;
3213 continue;
3214 }
3215 }
3216 prev = node;
3217 }
3218}
3219
3220static void
3221_split_branches(jit_state_t *_jit)
3222{
3223 jit_node_t *node;
3224 jit_node_t *next;
3225 jit_node_t *label;
3226 jit_block_t *block;
24d91c0d
PC		 3227 jit_block_t *blocks;
3228 jit_word_t offset;
3229 jit_word_t length;
4a71579b 3230
24d91c0d
PC		 3231 length = _jitc->blocks.length;
3232 jit_alloc((jit_pointer_t *)&blocks, length * sizeof(jit_block_t));
3233 if ((node = _jitc->head) &&
3234 (node->code == jit_code_label || node->code == jit_code_prolog)) {
3235 block = _jitc->blocks.ptr + node->v.w;
3236 memcpy(blocks, block, sizeof(jit_block_t));
3237 node->v.w = 0;
3238 offset = 1;
3239 }
3240 else
3241 offset = 0;
4a71579b
PC		 3242 for (node = _jitc->head; node; node = next) {
3243 if ((next = node->next)) {
3244 if (next->code == jit_code_label ||
3245 next->code == jit_code_prolog ||
24d91c0d
PC		 3246 next->code == jit_code_epilog) {
3247 if (offset >= length) {
3248 jit_realloc((jit_pointer_t *)&blocks,
3249 length * sizeof(jit_block_t),
3250 (length + 16) * sizeof(jit_block_t));
3251 length += 16;
3252 }
3253 block = _jitc->blocks.ptr + next->v.w;
3254 memcpy(blocks + offset, block, sizeof(jit_block_t));
3255 next->v.w = offset++;
3256 }
4a71579b 3257 /* split block on branches */
24d91c0d 3258 else if (jit_classify(node->code) & jit_cc_a0_jmp) {
4a71579b
PC		 3259 label = new_node(jit_code_label);
3260 label->next = next;
3261 node->next = label;
24d91c0d
PC		 3262 if (offset >= length) {
3263 jit_realloc((jit_pointer_t *)&blocks,
3264 length * sizeof(jit_block_t),
3265 (length + 16) * sizeof(jit_block_t));
3266 length += 16;
4a71579b 3267 }
24d91c0d 3268 block = blocks + offset;
4a71579b 3269 block->label = label;
24d91c0d 3270 label->v.w = offset++;
4a71579b
PC		 3271 jit_regset_new(&block->reglive);
3272 jit_regset_new(&block->regmask);
4a71579b
PC		 3273 }
3274 }
3275 }
24d91c0d
PC		 3276 jit_free((jit_pointer_t *)&_jitc->blocks.ptr);
3277 _jitc->blocks.ptr = blocks;
3278 _jitc->blocks.offset = offset;
3279 _jitc->blocks.length = length;
4a71579b
PC		 3280}
3281
3282static jit_bool_t
3283_shortcut_jump(jit_state_t *_jit, jit_node_t *prev, jit_node_t *node)
3284{
3285 jit_bool_t cond;
3286 jit_node_t *jump;
3287 jit_node_t *next;
3288 jit_node_t *temp;
3289
3290 if (!(node->flag & jit_flag_node))
3291 return (0);
3292 assert(node->code != jit_code_jmpr);
3293 cond = node->code != jit_code_jmpi;
3294 jump = node->u.n;
3295 for (next = jump->next; next; next = next->next) {
3296 switch (next->code) {
3297 case jit_code_jmpi:
3298 if (!(next->flag & jit_flag_node))
3299 return (0);
3300 if (jump->link == node)
3301 jump->link = node->link;
3302 else {
3303 for (temp = jump->link;
3304 temp->link != node;
3305 temp = temp->link)
3306 assert(temp != NULL);
3307 temp->link = node->link;
3308 }
3309 jump = next->u.n;
3310 node->u.n = jump;
3311 node->link = jump->link;
3312 jump->link = node;
3313 return (1);
3314 case jit_code_jmpr:
3315 if (cond)
3316 return (0);
3317 node->code = jit_code_jmpr;
3318 node->u.w = next->u.w;
3319 node->link = NULL;
3320 node->flag &= ~jit_flag_node;
3321 return (1);
3322 case jit_code_note: case jit_code_label:
3323 break;
3324 default:
3325 return (0);
3326 }
3327 }
3328 return (0);
3329}
3330
3331static jit_bool_t
3332_redundant_jump(jit_state_t *_jit, jit_node_t *prev, jit_node_t *node)
3333{
3334 jit_node_t *local_prev;
3335 jit_node_t *local_next;
3336
3337 if (!(node->flag & jit_flag_node))
3338 return (0);
3339 for (local_prev = node, local_next = node->next;
3340 local_next;
3341 local_prev = local_next, local_next = local_next->next) {
3342
3343 switch (local_next->code) {
3344 case jit_code_label: case jit_code_epilog:
3345 if (node->u.n == local_next) {
3346 if (local_next->link == node)
3347 local_next->link = node->link;
3348 else {
3349 for (local_prev = local_next->link;
3350 local_prev->link != node;
3351 local_prev = local_prev->link)
3352 assert(local_prev != NULL);
3353 local_prev->link = node->link;
3354 }
3355 del_node(prev, node);
3356 return (1);
3357 }
3358 break;
3359 case jit_code_name: case jit_code_note:
4a71579b
PC		 3360 break;
3361 default:
3362 return (0);
3363 }
3364 }
3365 return (0);
3366}
3367
3368static jit_code_t
3369reverse_jump_code(jit_code_t code)
3370{
3371 switch (code) {
3372 case jit_code_bltr: return (jit_code_bger);
3373 case jit_code_blti: return (jit_code_bgei);
3374 case jit_code_bltr_u: return (jit_code_bger_u);
3375 case jit_code_blti_u: return (jit_code_bgei_u);
3376 case jit_code_bler: return (jit_code_bgtr);
3377 case jit_code_blei: return (jit_code_bgti);
3378 case jit_code_bler_u: return (jit_code_bgtr_u);
3379 case jit_code_blei_u: return (jit_code_bgti_u);
3380 case jit_code_beqr: return (jit_code_bner);
3381 case jit_code_beqi: return (jit_code_bnei);
3382 case jit_code_bger: return (jit_code_bltr);
3383 case jit_code_bgei: return (jit_code_blti);
3384 case jit_code_bger_u: return (jit_code_bltr_u);
3385 case jit_code_bgei_u: return (jit_code_blti_u);
3386 case jit_code_bgtr: return (jit_code_bler);
3387 case jit_code_bgti: return (jit_code_blei);
3388 case jit_code_bgtr_u: return (jit_code_bler_u);
3389 case jit_code_bgti_u: return (jit_code_blei_u);
3390 case jit_code_bner: return (jit_code_beqr);
3391 case jit_code_bnei: return (jit_code_beqi);
3392 case jit_code_bmsr: return (jit_code_bmcr);
3393 case jit_code_bmsi: return (jit_code_bmci);
3394 case jit_code_bmcr: return (jit_code_bmsr);
3395 case jit_code_bmci: return (jit_code_bmsi);
3396 case jit_code_bltr_f: return (jit_code_bunger_f);
3397 case jit_code_blti_f: return (jit_code_bungei_f);
3398 case jit_code_bler_f: return (jit_code_bungtr_f);
3399 case jit_code_blei_f: return (jit_code_bungti_f);
3400
3401 case jit_code_beqr_f: return (jit_code_bner_f);
3402 case jit_code_beqi_f: return (jit_code_bnei_f);
3403
3404 case jit_code_bger_f: return (jit_code_bunltr_f);
3405 case jit_code_bgei_f: return (jit_code_bunlti_f);
3406 case jit_code_bgtr_f: return (jit_code_bunler_f);
3407 case jit_code_bgti_f: return (jit_code_bunlei_f);
3408
3409 case jit_code_bner_f: return (jit_code_beqr_f);
79bfeef6 3410 case jit_code_bnei_f: return (jit_code_beqi_f);
4a71579b
PC		 3411
3412 case jit_code_bunltr_f: return (jit_code_bger_f);
3413 case jit_code_bunlti_f: return (jit_code_bgei_f);
3414 case jit_code_bunler_f: return (jit_code_bgtr_f);
3415 case jit_code_bunlei_f: return (jit_code_bgti_f);
3416
3417 case jit_code_buneqr_f: return (jit_code_bltgtr_f);
3418 case jit_code_buneqi_f: return (jit_code_bltgti_f);
3419
3420 case jit_code_bunger_f: return (jit_code_bltr_f);
3421 case jit_code_bungei_f: return (jit_code_blti_f);
3422 case jit_code_bungtr_f: return (jit_code_bler_f);
3423 case jit_code_bungti_f: return (jit_code_blei_f);
3424
3425 case jit_code_bltgtr_f: return (jit_code_buneqr_f);
3426 case jit_code_bltgti_f: return (jit_code_buneqi_f);
3427
3428 case jit_code_bordr_f: return (jit_code_bunordr_f);
3429 case jit_code_bordi_f: return (jit_code_bunordi_f);
3430 case jit_code_bunordr_f:return (jit_code_bordr_f);
3431 case jit_code_bunordi_f:return (jit_code_bordi_f);
3432 case jit_code_bltr_d: return (jit_code_bunger_d);
3433 case jit_code_blti_d: return (jit_code_bungei_d);
3434 case jit_code_bler_d: return (jit_code_bungtr_d);
3435 case jit_code_blei_d: return (jit_code_bungti_d);
3436
3437 case jit_code_beqr_d: return (jit_code_bner_d);
3438 case jit_code_beqi_d: return (jit_code_bnei_d);
3439
3440 case jit_code_bger_d: return (jit_code_bunltr_d);
3441 case jit_code_bgei_d: return (jit_code_bunlti_d);
3442 case jit_code_bgtr_d: return (jit_code_bunler_d);
3443 case jit_code_bgti_d: return (jit_code_bunlei_d);
3444
3445 case jit_code_bner_d: return (jit_code_beqr_d);
3446 case jit_code_bnei_d: return (jit_code_beqi_d);
3447
3448 case jit_code_bunltr_d: return (jit_code_bger_d);
3449 case jit_code_bunlti_d: return (jit_code_bgei_d);
3450 case jit_code_bunler_d: return (jit_code_bgtr_d);
3451 case jit_code_bunlei_d: return (jit_code_bgti_d);
3452
3453 case jit_code_buneqr_d: return (jit_code_bltgtr_d);
3454 case jit_code_buneqi_d: return (jit_code_bltgti_d);
3455
3456 case jit_code_bunger_d: return (jit_code_bltr_d);
3457 case jit_code_bungei_d: return (jit_code_blti_d);
3458 case jit_code_bungtr_d: return (jit_code_bler_d);
3459 case jit_code_bungti_d: return (jit_code_blei_d);
3460
3461 case jit_code_bltgtr_d: return (jit_code_buneqr_d);
3462 case jit_code_bltgti_d: return (jit_code_buneqi_d);
3463
3464 case jit_code_bordr_d: return (jit_code_bunordr_d);
3465 case jit_code_bordi_d: return (jit_code_bunordi_d);
3466 case jit_code_bunordr_d:return (jit_code_bordr_d);
3467 case jit_code_bunordi_d:return (jit_code_bordi_d);
3468 case jit_code_boaddr: return (jit_code_bxaddr);
3469 case jit_code_boaddi: return (jit_code_bxaddi);
3470 case jit_code_boaddr_u: return (jit_code_bxaddr_u);
3471 case jit_code_boaddi_u: return (jit_code_bxaddi_u);
3472 case jit_code_bxaddr: return (jit_code_boaddr);
3473 case jit_code_bxaddi: return (jit_code_boaddi);
3474 case jit_code_bxaddr_u: return (jit_code_boaddr_u);
3475 case jit_code_bxaddi_u: return (jit_code_boaddi_u);
3476 case jit_code_bosubr: return (jit_code_bxsubr);
3477 case jit_code_bosubi: return (jit_code_bxsubi);
3478 case jit_code_bosubr_u: return (jit_code_bxsubr_u);
3479 case jit_code_bosubi_u: return (jit_code_bxsubi_u);
3480 case jit_code_bxsubr: return (jit_code_bosubr);
3481 case jit_code_bxsubi: return (jit_code_bosubi);
3482 case jit_code_bxsubr_u: return (jit_code_bosubr_u);
3483 case jit_code_bxsubi_u: return (jit_code_bosubi_u);
3484 default: abort(); /* invalid jump code */
3485 }
3486}
3487
3488/*
3489 * change common pattern:
3490 * <cond_jump L0> <jump L1> <label L0>
3491 * into
3492 * <reverse_cond_jump L1>
3493 */
3494static jit_bool_t
3495_reverse_jump(jit_state_t *_jit, jit_node_t *prev, jit_node_t *node)
3496{
3497 jit_node_t *local_prev;
3498 jit_node_t *local_next;
3499 jit_node_t *local_jump;
3500
3501 if (!(node->flag & jit_flag_node))
3502 return (0);
3503 /* =><cond_jump L0> <jump L1> <label L0> */
3504 local_next = node->next;
3505 if (local_next->code != jit_code_jmpi ||
3506 !(local_next->flag & jit_flag_node))
3507 return (0);
3508 /* <cond_jump L0> =><jump L1> <label L0> */
3509
3510 local_jump = local_next->u.n;
3511 for (local_prev = local_next, local_next = local_next->next;
3512 local_next;
3513 local_prev = local_next, local_next = local_next->next) {
3514 switch (local_next->code) {
3515 case jit_code_label: case jit_code_epilog:
3516 if (node->u.n == local_next) {
3517 if (local_next->link == node)
3518 local_next->link = node->link;
3519 else {
3520 for (local_prev = local_next->link;
3521 local_prev->link != node;
3522 local_prev = local_prev->link)
3523 assert(local_prev != NULL);
3524 local_prev->link = node->link;
3525 }
3526 del_node(node, node->next);
3527 node->code = reverse_jump_code(node->code);
3528 node->u.n = local_jump;
3529 node->link = local_jump->link;
3530 local_jump->link = node;
3531 return (1);
3532 }
3533 break;
3534 case jit_code_note:
3535 break;
3536 default:
3537 return (0);
3538 }
3539 }
3540 return (0);
3541}
3542
ba3814c1 3543static jit_bool_t
4a71579b
PC		 3544_redundant_store(jit_state_t *_jit, jit_node_t *node, jit_bool_t jump)
3545{
3546 jit_node_t *iter;
3547 jit_node_t *prev;
3548 jit_word_t word;
3549 jit_int32_t spec;
3550 jit_int32_t regno;
ba3814c1 3551 jit_bool_t result;
4a71579b
PC		 3552
3553 if (jump) {
3554 prev = node->u.n;
3555 if (prev->code == jit_code_epilog)
ba3814c1 3556 return (0);
4a71579b
PC		 3557 assert(prev->code == jit_code_label);
3558 if ((prev->flag & jit_flag_head) || node->link || prev->link != node)
3559 /* multiple sources */
ba3814c1 3560 return (0);
4a71579b
PC		 3561 /* if there are sequential labels it will return below */
3562 }
3563 else
3564 prev = node;
ba3814c1 3565 result = 0;
4a71579b
PC		 3566 word = node->w.w;
3567 regno = jit_regno(node->v.w);
3568 for (iter = prev->next; iter; prev = iter, iter = iter->next) {
3569 switch (iter->code) {
3570 case jit_code_label: case jit_code_prolog:
3571 case jit_code_epilog:
ba3814c1 3572 return (result);
4a71579b
PC		 3573 case jit_code_movi:
3574 if (regno == jit_regno(iter->u.w)) {
3575 if (iter->flag || iter->v.w != word)
ba3814c1
PC		 3576 return (result);
3577 result = 1;
4a71579b
PC		 3578 del_node(prev, iter);
3579 iter = prev;
3580 }
3581 break;
3582 default:
3583 spec = jit_classify(iter->code);
3584 if (spec & jit_cc_a0_jmp)
ba3814c1 3585 return (result);
4a71579b
PC		 3586 if ((spec & (jit_cc_a0_reg|jit_cc_a0_chg)) ==
3587 (jit_cc_a0_reg|jit_cc_a0_chg)) {
3588 if (spec & jit_cc_a0_rlh) {
3589 if (regno == jit_regno(iter->u.q.l) ||
3590 regno == jit_regno(iter->u.q.h))
ba3814c1 3591 return (result);
4a71579b
PC		 3592 }
3593 else {
3594 if (regno == jit_regno(iter->u.w))
ba3814c1 3595 return (result);
4a71579b
PC		 3596 }
3597 }
3598 if ((spec & (jit_cc_a1_reg|jit_cc_a1_chg)) ==
3599 (jit_cc_a1_reg|jit_cc_a1_chg)) {
3600 if (regno == jit_regno(iter->v.w))
ba3814c1 3601 return (result);
4a71579b
PC		 3602 }
3603 if ((spec & (jit_cc_a2_reg|jit_cc_a2_chg)) ==
3604 (jit_cc_a2_reg|jit_cc_a2_chg)) {
3605 if (regno == jit_regno(iter->w.w))
ba3814c1 3606 return (result);
4a71579b
PC		 3607 }
3608 break;
3609 }
3610 }
c0c16242
PC		 3611
3612 return (result);
4a71579b
PC		 3613}
3614
3615static jit_bool_t
3616_simplify_movr(jit_state_t *_jit, jit_node_t *prev, jit_node_t *node,
3617 jit_int32_t kind, jit_int32_t size)
3618{
3619 jit_int32_t regno;
3620 jit_int32_t right;
ba86ff93 3621 jit_int32_t spec;
4a71579b
PC		 3622 jit_value_t *value;
3623
3624 regno = jit_regno(node->u.w);
3625 right = jit_regno(node->v.w);
3626 value = _jitc->values + regno;
ba86ff93
PC		 3627
3628 spec = jit_class(_rvs[regno].spec);
3629 if (!(spec & (jit_class_gpr | jit_class_fpr))) {
3630 /* reserved register */
3631 value->kind = 0;
3632 ++_jitc->gen[regno];
3633 return (0);
3634 }
3635
4a71579b
PC		 3636 if ((value->kind == jit_kind_register &&
3637 jit_regno(value->base.q.l) == right &&
3638 value->base.q.h == _jitc->gen[right]) ||
3639 (value->kind == kind && _jitc->values[right].kind == kind &&
3640 memcmp(&value->base.w, &_jitc->values[right].base.w, size) == 0)) {
3641 del_node(prev, node);
3642 return (1);
3643 }
3644 if (_jitc->values[right].kind == jit_kind_word)
3645 jit_memcpy(value, _jitc->values + right, sizeof(jit_value_t));
3646 else {
3647 value->kind = jit_kind_register;
3648 value->base.q.l = right;
3649 value->base.q.h = _jitc->gen[right];
3650 }
3651 ++_jitc->gen[regno];
3652
3653 return (0);
3654}
3655
3656static jit_bool_t
3657_simplify_movi(jit_state_t *_jit, jit_node_t *prev, jit_node_t *node,
3658 jit_int32_t kind, jit_int32_t size)
3659{
3660 jit_value_t *value;
3661 jit_int32_t spec;
3662 jit_int32_t regno;
3663 jit_int32_t offset;
3664
3665 regno = jit_regno(node->u.w);
3666 value = _jitc->values + regno;
3667 if (node->flag & jit_flag_node) {
3668 /* set to undefined if value will be patched */
3669 value->kind = 0;
3670 ++_jitc->gen[regno];
3671 return (0);
3672 }
ba86ff93
PC		 3673 spec = jit_class(_rvs[regno].spec);
3674 if (!(spec & (jit_class_gpr | jit_class_fpr))) {
3675 /* reserved register */
3676 value->kind = 0;
3677 ++_jitc->gen[regno];
3678 return (0);
3679 }
4a71579b
PC		 3680 if (value->kind == kind) {
3681 if (memcmp(&node->v.w, &value->base.w, size) == 0) {
3682 del_node(prev, node);
3683 return (1);
3684 }
4a71579b
PC		 3685 if (kind == jit_kind_word)
3686 spec &= jit_class_gpr;
3687 else
3688 spec &= (jit_class_xpr | jit_class_fpr);
3689 for (offset = 0; offset < _jitc->reglen; offset++) {
3690 if (_jitc->values[offset].kind == kind &&
3691 memcmp(&node->v.w, &_jitc->values[offset].base.w, size) == 0 &&
3692 (jit_class(_rvs[offset].spec) & spec) == spec) {
3693 if (kind == jit_kind_word)
3694 node->code = jit_code_movr;
3695 else if (kind == jit_kind_float32)
3696 node->code = jit_code_movr_f;
3697 else
3698 node->code = jit_code_movr_d;
3699 node->v.w = offset;
3700 jit_memcpy(value, _jitc->values + offset, sizeof(jit_value_t));
3701 ++_jitc->gen[regno];
3702 return (0);
3703 }
3704 }
3705 }
3706 value->kind = kind;
3707 jit_memcpy(&value->base.w, &node->v.w, size);
3708 ++_jitc->gen[regno];
3709
3710 return (0);
3711}
3712
3713/* simple/safe redundandy test not checking if another register
3714 * holds the same value
3715 */
3716static jit_bool_t
3717_simplify_ldxi(jit_state_t *_jit, jit_node_t *prev, jit_node_t *node)
3718{
3719 jit_value_t *value;
3720 jit_int32_t regno;
3721 jit_int32_t right;
3722
3723 regno = jit_regno(node->u.w);
3724 right = jit_regno(node->v.w);
3725 value = _jitc->values + regno;
3726 if (regno != right &&
3727 value->kind == jit_kind_code && value->code == node->code &&
3728 value->base.q.l == right && value->base.q.h == _jitc->gen[right] &&
3729 node->w.w == value->disp.w) {
3730 del_node(prev, node);
3731 return (1);
3732 }
3733 value->kind = jit_kind_code;
3734 value->code = node->code;
3735 value->base.q.l = right;
3736 value->base.q.h = _jitc->gen[right];
3737 value->disp.w = node->w.w;
3738 ++_jitc->gen[regno];
3739
3740 return (0);
3741}
3742
3743static jit_bool_t
3744_simplify_stxi(jit_state_t *_jit, jit_node_t *prev, jit_node_t *node)
3745{
3746 jit_value_t *value;
3747 jit_int32_t regno;
3748 jit_int32_t right;
3749 jit_int32_t offset;
3750
3751 regno = jit_regno(node->w.w);
3752 right = jit_regno(node->v.w);
3753 value = _jitc->values + regno;
3754
3755 /* check for redundant store after load */
3756 if (regno != right &&
3757 value->kind == jit_kind_code && value->code == node->code &&
3758 value->base.q.l == right && value->base.q.h == _jitc->gen[right] &&
3759 node->u.w == value->disp.w) {
3760 del_node(prev, node);
3761 return (1);
3762 }
3763
3764 /* assume anything can alias, and invalidate tracked values */
3765 for (offset = 0; offset < _jitc->reglen; offset++) {
3766 if (_jitc->values[offset].kind == jit_kind_code) {
3767 _jitc->values[offset].kind = 0;
3768 ++_jitc->gen[offset];
3769 }
3770 }
3771
3772 /* no multiple information, so, if set to a constant,
3773 * prefer to keep that information */
3774 if (value->kind == 0) {
4a71579b
PC		 3775 switch (node->code) {
3776 /* no information about signed/unsigned either */
3777 case jit_code_stxi_c: value->code = jit_code_ldxi_c; break;
3778 case jit_code_stxi_s: value->code = jit_code_ldxi_s; break;
3779 case jit_code_stxi_i: value->code = jit_code_ldxi_i; break;
3780 case jit_code_stxi_l: value->code = jit_code_ldxi_l; break;
3781 case jit_code_stxi_f: value->code = jit_code_ldxi_f; break;
3782 case jit_code_stxi_d: value->code = jit_code_ldxi_d; break;
3783 default: abort();
3784 }
3785 value->kind = jit_kind_code;
3786 value->base.q.l = right;
3787 value->base.q.h = _jitc->gen[right];
3788 value->disp.w = node->u.w;
3789 }
3790
3791 return (0);
3792}
3793
3794/* usually there should be only one store in the
3795 * jit_get_reg/jit_unget_reg, but properly handle
3796 * multiple ones by moving the save node */
3797static void
3798_simplify_spill(jit_state_t *_jit, jit_node_t *node, jit_int32_t regno)
3799{
3800 jit_node_t *save;
3801 jit_node_t *temp;
3802
3803 if ((temp = _jitc->spill[regno]) && (save = temp->next) != node) {
3804 temp->next = save->next;
3805 save->next = node->next;
3806 node->next = save;
3807 _jitc->spill[regno] = node;
3808 }
3809}
3810
3811/* checks for simple cases where a register is set more than
3812 * once to the same value, and is a common pattern of calls
3813 * to jit_pushargi and jit_pushargr
3814 */
ba3814c1 3815static jit_bool_t
4a71579b
PC		 3816_simplify(jit_state_t *_jit)
3817{
3818 jit_node_t *prev;
3819 jit_node_t *node;
3820 jit_node_t *next;
3821 jit_int32_t info;
3822 jit_int32_t regno;
ba3814c1 3823 jit_bool_t result;
4a71579b 3824
ba3814c1 3825 result = 0;
4a71579b
PC		 3826 for (prev = NULL, node = _jitc->head; node; prev = node, node = next) {
3827 next = node->next;
3828 switch (node->code) {
3829 case jit_code_label: case jit_code_prolog:
3830 case jit_code_callr: case jit_code_calli:
3831 reset:
3832 memset(_jitc->gen, 0, sizeof(jit_int32_t) * _jitc->reglen);
3833 memset(_jitc->values, 0, sizeof(jit_value_t) * _jitc->reglen);
3834 break;
3835 case jit_code_save:
3836 _jitc->spill[jit_regno(node->u.w)] = prev;
3837 break;
3838 case jit_code_load:
3839 regno = jit_regno(node->u.w);
3840 if (register_change_p(node->link->next, node, regno) !=
3841 jit_reg_change) {
3842 /* spill not required due to optimizing common
3843 * redundancy case of calling jit_get_reg/jit_unget_reg
3844 * and then setting the register to the value it is
3845 * already holding */
3846 patch_register(node->link->next, node,
3847 jit_regno_patch|regno, regno);
ba3814c1 3848 result = 1;
4a71579b
PC		 3849 del_node(_jitc->spill[regno], node->link);
3850 del_node(prev, node);
3851 node = prev;
3852 }
3853 _jitc->spill[regno] = NULL;
3854 break;
3855 case jit_code_movr:
3856 regno = jit_regno(node->u.w);
3857 if (simplify_movr(prev, node,
ba3814c1
PC		 3858 jit_kind_word, sizeof(jit_word_t))) {
3859 result = 1;
4a71579b 3860 simplify_spill(node = prev, regno);
ba3814c1 3861 }
4a71579b
PC		 3862 break;
3863 case jit_code_movi:
3864 regno = jit_regno(node->u.w);
3865 if (simplify_movi(prev, node,
ba3814c1
PC		 3866 jit_kind_word, sizeof(jit_word_t))) {
3867 result = 1;
4a71579b 3868 simplify_spill(node = prev, regno);
ba3814c1 3869 }
4a71579b
PC		 3870 break;
3871 case jit_code_movr_f:
3872 regno = jit_regno(node->u.w);
3873 if (simplify_movr(prev, node,
ba3814c1
PC		 3874 jit_kind_float32, sizeof(jit_float32_t))) {
3875 result = 1;
4a71579b 3876 simplify_spill(node = prev, regno);
ba3814c1 3877 }
4a71579b
PC		 3878 break;
3879 case jit_code_movi_f:
3880 regno = jit_regno(node->u.w);
3881 if (simplify_movi(prev, node,
ba3814c1
PC		 3882 jit_kind_float32, sizeof(jit_float32_t))) {
3883 result = 1;
4a71579b 3884 simplify_spill(node = prev, regno);
ba3814c1 3885 }
4a71579b
PC		 3886 break;
3887 case jit_code_movr_d:
3888 regno = jit_regno(node->u.w);
3889 if (simplify_movr(prev, node,
ba3814c1
PC		 3890 jit_kind_float64, sizeof(jit_float64_t))) {
3891 result = 1;
4a71579b 3892 simplify_spill(node = prev, regno);
ba3814c1 3893 }
4a71579b
PC		 3894 break;
3895 case jit_code_movi_d:
3896 regno = jit_regno(node->u.w);
3897 if (simplify_movi(prev, node,
ba3814c1
PC		 3898 jit_kind_float64, sizeof(jit_float64_t))) {
3899 result = 1;
4a71579b 3900 simplify_spill(node = prev, regno);
ba3814c1 3901 }
4a71579b
PC		 3902 break;
3903 case jit_code_ldxi_c: case jit_code_ldxi_uc:
3904 case jit_code_ldxi_s: case jit_code_ldxi_us:
3905 case jit_code_ldxi_i: case jit_code_ldxi_ui:
3906 case jit_code_ldxi_l:
3907 case jit_code_ldxi_f: case jit_code_ldxi_d:
3908 regno = jit_regno(node->u.w);
ba3814c1
PC		 3909 if (simplify_ldxi(prev, node)) {
3910 result = 1;
4a71579b 3911 simplify_spill(node = prev, regno);
ba3814c1 3912 }
4a71579b
PC		 3913 break;
3914 case jit_code_stxi_c: case jit_code_stxi_s:
3915 case jit_code_stxi_i: case jit_code_stxi_l:
3916 case jit_code_stxi_f: case jit_code_stxi_d:
3917 regno = jit_regno(node->u.w);
ba3814c1
PC		 3918 if (simplify_stxi(prev, node)) {
3919 result = 1;
4a71579b 3920 simplify_spill(node = prev, regno);
ba3814c1 3921 }
4a71579b
PC		 3922 break;
3923 default:
3924 info = jit_classify(node->code);
3925 if (info & jit_cc_a0_jmp)
3926 /* labels are not implicitly added when not taking
3927 * a conditional branch */
3928 goto reset;
3929 if (info & jit_cc_a0_chg) {
3930 if (info & jit_cc_a0_rlh) {
3931 regno = jit_regno(node->u.q.l);
3932 _jitc->values[regno].kind = 0;
3933 ++_jitc->gen[regno];
3934 regno = jit_regno(node->u.q.h);
3935 _jitc->values[regno].kind = 0;
3936 ++_jitc->gen[regno];
3937 }
3938 else {
3939 regno = jit_regno(node->u.w);
3940 _jitc->values[regno].kind = 0;
3941 ++_jitc->gen[regno];
3942 }
3943 }
3944 if (info & jit_cc_a1_chg) {
ba86ff93
PC		 3945#if 0
3946 /* Assume registers are not changed */
3947 if (info & jit_cc_a1_rlh) {
3948 regno = jit_regno(node->v.q.l);
3949 _jitc->values[regno].kind = 0;
3950 ++_jitc->gen[regno];
3951 regno = jit_regno(node->v.q.h);
3952 _jitc->values[regno].kind = 0;
3953 ++_jitc->gen[regno];
3954 }
3955 else {
3956#endif
3957 regno = jit_regno(node->v.w);
3958 _jitc->values[regno].kind = 0;
3959 ++_jitc->gen[regno];
3960#if 0
3961 }
3962#endif
4a71579b
PC		 3963 }
3964 if (info & jit_cc_a2_chg) {
ba3814c1
PC		 3965#if 0
3966 /* Assume registers are not changed */
3967 if (info & jit_cc_a2_rlh) {
3968 regno = jit_regno(node->w.q.l);
3969 _jitc->values[regno].kind = 0;
3970 ++_jitc->gen[regno];
3971 regno = jit_regno(node->w.q.h);
3972 _jitc->values[regno].kind = 0;
3973 ++_jitc->gen[regno];
3974 }
3975 else {
3976#endif
3977 regno = jit_regno(node->w.w);
3978 _jitc->values[regno].kind = 0;
3979 ++_jitc->gen[regno];
3980#if 0
3981 }
3982#endif
4a71579b
PC		 3983 }
3984 break;
3985 }
3986 }
ba3814c1 3987 return (result);
4a71579b
PC		 3988}
3989
3990static jit_int32_t
3991_register_change_p(jit_state_t *_jit, jit_node_t *node, jit_node_t *link,
3992 jit_int32_t regno)
3993{
3994 jit_int32_t value;
ba86ff93 3995 jit_bool_t use, change;
4a71579b
PC		 3996
3997 for (; node != link; node = node->next) {
3998 switch (node->code) {
3999 case jit_code_label: case jit_code_prolog:
ba86ff93
PC		 4000 /* lack of extra information so assume it is live */
4001 return (jit_reg_static);
4a71579b
PC		 4002 case jit_code_callr: case jit_code_calli:
4003 if (!(jit_class(_rvs[regno].spec) & jit_class_sav))
4004 return (jit_reg_undef);
4005 break;
4006 default:
4007 value = jit_classify(node->code);
ba86ff93
PC		 4008 use = change = 0;
4009 if (value & jit_cc_a0_rlh) {
4010 if (node->u.q.l == regno || node->u.q.h == regno) {
4011 if (value & jit_cc_a0_chg)
4012 change = 1;
4013 use = !change;
4014 }
4015 }
4016 else if ((value & jit_cc_a0_reg) && node->u.w == regno) {
4017 if (value & jit_cc_a0_chg)
4018 change = 1;
4019 use = !change;
4020 }
4021 if (!use &&
4022 (value & jit_cc_a1_reg) && node->v.w == regno) {
4023 if (value & jit_cc_a1_chg)
4024 change = 1;
4025 use = !change;
4026 }
4027 if (!use &&
4028 (value & jit_cc_a2_reg) && node->w.w == regno) {
4029 if (value & jit_cc_a2_chg)
4030 change = 1;
4031 use = !change;
4032 }
4a71579b 4033 /* lack of extra information */
ba86ff93
PC		 4034 if (!use && (value & (jit_cc_a0_jmp|jit_cc_a0_cnd)) &&
4035 /* In case of indirect branches, always consider
4036 * callee save registers as live. */
4037 !(jit_class(_rvs[regno].spec) & jit_class_sav))
4038 change = 1;
4039 if (!use && change)
4a71579b
PC		 4040 return (jit_reg_change);
4041 }
4042 }
4043
4044 return (jit_reg_static);
4045}
4046
4047/* most of this could be done at the same time as generating jit, but
4048 * avoid complications on different cpu backends and patch spill/loads
4049 * here, by simulating jit generation */
4050static jit_bool_t
4051_spill_reglive_p(jit_state_t *_jit, jit_node_t *node, jit_int32_t regno)
4052{
4053 if (!jit_regset_tstbit(&_jitc->reglive, regno)) {
4054 jit_regset_setbit(&_jitc->regmask, regno);
4055 jit_update(node->next, &_jitc->reglive, &_jitc->regmask);
4056 if (!jit_regset_tstbit(&_jitc->reglive, regno) &&
4057 register_change_p(node->next, node->link, regno) != jit_reg_change)
4058 return (0);
4059 }
4060
4061 return (1);
4062}
4063
4064static void
4065_patch_registers(jit_state_t *_jit)
4066{
4067 jit_node_t *prev;
4068 jit_node_t *node;
4069 jit_node_t *next;
4070 jit_int32_t info;
4071 jit_int32_t spec;
4072 jit_int32_t regno;
4073 jit_int32_t value;
4074
4075 _jitc->function = NULL;
4076
4077 jit_reglive_setup();
4078 for (prev = NULL, node = _jitc->head; node; node = next) {
4079 next = node->next;
4080
4081 info = jit_classify(node->code);
4082 jit_regarg_set(node, info);
4083
4084 switch (node->code) {
4085 case jit_code_save:
4086 regno = jit_regno(node->u.w);
4087 if (!spill_reglive_p(node, regno)) {
4088 /* register is not live, just remove spill/reload */
4089 jit_regarg_clr(node, info);
4090 node->link->v.w = jit_regload_delete;
4091 del_node(prev, node);
4092 continue;
4093 }
4094 else {
4095 /* try to find a free register of the same class */
4096 spec = jit_class(_rvs[regno].spec) & ~jit_class_arg;
4097 for (value = 0; value < _jitc->reglen; value++) {
4098 if (value != regno &&
4099 ((jit_class(_rvs[value].spec) & spec) &
4100 ~jit_class_arg) == spec &&
4101 !jit_regset_tstbit(&_jitc->regarg, value) &&
4102 !spill_reglive_p(node, value))
4103 break;
4104 }
4105 if (value < _jitc->reglen) {
4106 jit_regarg_clr(node, info);
4107 patch_register(node->next, node->link,
4108 jit_regno_patch|node->u.w,
4109 jit_regno_patch|value);
4110 /* mark as live just in case there are nested
4111 * register patches, so that next patch will
4112 * not want to use the same register */
4113 jit_regset_setbit(&_jitc->reglive, value);
4114 /* register is not live, just remove spill/reload */
4115 node->link->v.w = jit_regload_isdead;
4116 del_node(prev, node);
4117 continue;
4118 }
4119 else {
4120 /* failed to find a free register */
4121 if (spec & jit_class_gpr) {
4122 if (!_jitc->function->regoff[regno])
4123 _jitc->function->regoff[regno] =
4124 jit_allocai(sizeof(jit_word_t));
4125#if __WORDSIZE == 32
4126 node->code = jit_code_stxi_i;
4127#else
4128 node->code = jit_code_stxi_l;
4129#endif
4130 }
4131 else {
4132 node->code = jit_code_stxi_d;
4133 if (!_jitc->function->regoff[regno])
4134 _jitc->function->regoff[regno] =
4135 jit_allocai(sizeof(jit_float64_t));
4136 }
4137 node->u.w = _jitc->function->regoff[regno];
4138 node->v.w = JIT_FP;
4139 node->w.w = regno;
4140 node->link = NULL;
4141 }
4142 }
4143 break;
4144 case jit_code_load:
4145 regno = jit_regno(node->u.w);
4146 if (node->v.w) {
4147 if (node->v.w == jit_regload_isdead)
4148 jit_regset_clrbit(&_jitc->reglive, regno);
4149 del_node(prev, node);
4150 continue;
4151 }
4152 spec = jit_class(_rvs[regno].spec);
4153 if (spec & jit_class_gpr) {
4154#if __WORDSIZE == 32
4155 node->code = jit_code_ldxi_i;
4156#else
4157 node->code = jit_code_ldxi_l;
4158#endif
4159 }
4160 else
4161 node->code = jit_code_ldxi_d;
4162 node->v.w = regno;
4163 node->v.w = JIT_FP;
4164 node->w.w = _jitc->function->regoff[regno];
4165 node->link = NULL;
4166 break;
4167 case jit_code_prolog:
4168 _jitc->function = _jitc->functions.ptr + node->w.w;
4169 break;
4170 case jit_code_epilog:
4171 _jitc->function = NULL;
4172 break;
4173 default:
4174 break;
4175 }
4176
4177 jit_regarg_clr(node, info);
4178 /* update register live state */
4179 jit_reglive(node);
4180 prev = node;
4181 }
4182}
4183
4184static void
4185_patch_register(jit_state_t *_jit, jit_node_t *node, jit_node_t *link,
4186 jit_int32_t regno, jit_int32_t patch)
4187{
4188 jit_int32_t value;
4189
4190 for (; node != link; node = node->next) {
4191 value = jit_classify(node->code);
4192 if (value & jit_cc_a0_reg) {
4193 if (value & jit_cc_a0_rlh) {
4194 if (node->u.q.l == regno)
4195 node->u.q.l = patch;
4196 if (node->u.q.h == regno)
4197 node->u.q.h = patch;
4198 }
4199 else {
4200 if (node->u.w == regno)
4201 node->u.w = patch;
4202 }
4203 }
ba86ff93
PC		 4204 if (value & jit_cc_a1_reg) {
4205 if (value & jit_cc_a1_rlh) {
4206 if (node->v.q.l == regno)
4207 node->v.q.l = patch;
4208 if (node->v.q.h == regno)
4209 node->v.q.h = patch;
4210 }
4211 else {
4212 if (node->v.w == regno)
4213 node->v.w = patch;
4214 }
4215 }
ba3814c1
PC		 4216 if (value & jit_cc_a2_reg) {
4217 if (value & jit_cc_a2_rlh) {
4218 if (node->w.q.l == regno)
4219 node->w.q.l = patch;
4220 if (node->w.q.h == regno)
4221 node->w.q.h = patch;
4222 }
4223 else {
4224 if (node->w.w == regno)
4225 node->w.w = patch;
4226 }
4227 }
4a71579b
PC		 4228 }
4229}
4230
40a44dcb
PC		 4231#if __BYTE_ORDER == __LITTLE_ENDIAN
4232# define htonr_us(r0,r1) bswapr_us(r0,r1)
4233# define htonr_ui(r0,r1) bswapr_ui(r0,r1)
4234# if __WORDSIZE == 64
4235# define htonr_ul(r0,r1) bswapr_ul(r0,r1)
4236# endif
4237#else
4238# define htonr_us(r0,r1) extr_us(r0,r1)
4239# if __WORDSIZE == 32
4240# define htonr_ui(r0,r1) movr(r0,r1)
4241# else
4242# define htonr_ui(r0,r1) extr_ui(r0,r1)
4243# define htonr_ul(r0,r1) movr(r0,r1)
4244# endif
4245#endif
4246
4247static maybe_unused void
4248generic_bswapr_us(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1);
4249static maybe_unused void
4250generic_bswapr_ui(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1);
4251#if __WORDSIZE == 64
4252static maybe_unused void
4253generic_bswapr_ul(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1);
4254#endif
4255
ba86ff93
PC		 4256#define depi(r0, i0, i1, i2) _depi(_jit, r0, i0, i1, i2)
4257static void _depi(jit_state_t*,jit_int32_t, jit_word_t, jit_word_t, jit_word_t);
4258#define negi(r0, i0) _negi(_jit, r0, i0)
4259static void _negi(jit_state_t*, jit_int32_t, jit_word_t);
4260#define comi(r0, i0) _comi(_jit, r0, i0)
4261static void _comi(jit_state_t*, jit_int32_t, jit_word_t);
4262#define exti_c(r0, i0) _exti_c(_jit, r0, i0)
4263static void _exti_c(jit_state_t*, jit_int32_t, jit_word_t);
4264#define exti_uc(r0, i0) _exti_uc(_jit, r0, i0)
4265static void _exti_uc(jit_state_t*, jit_int32_t, jit_word_t);
4266#define exti_s(r0, i0) _exti_s(_jit, r0, i0)
4267static void _exti_s(jit_state_t*, jit_int32_t, jit_word_t);
4268#define exti_us(r0, i0) _exti_us(_jit, r0, i0)
4269static void _exti_us(jit_state_t*, jit_int32_t, jit_word_t);
4270#if __WORDSIZE == 64
4271#define exti_i(r0, i0) _exti_i(_jit, r0, i0)
4272static void _exti_i(jit_state_t*, jit_int32_t, jit_word_t);
4273#define exti_ui(r0, i0) _exti_ui(_jit, r0, i0)
4274static void _exti_ui(jit_state_t*, jit_int32_t, jit_word_t);
4275#endif
4276#define bswapi_us(r0, i0) _bswapi_us(_jit, r0, i0)
4277static void _bswapi_us(jit_state_t*, jit_int32_t, jit_word_t);
4278#define bswapi_ui(r0, i0) _bswapi_ui(_jit, r0, i0)
4279static void _bswapi_ui(jit_state_t*, jit_int32_t, jit_word_t);
4280#if __WORDSIZE == 64
4281# define bswapi_ul(r0, i0) _bswapi_ul(_jit, r0, i0)
4282static void _bswapi_ul(jit_state_t*, jit_int32_t, jit_word_t);
4283#endif
4284#define htoni_us(r0, i0) _htoni_us(_jit, r0, i0)
4285static void _htoni_us(jit_state_t*, jit_int32_t, jit_word_t);
4286#define htoni_ui(r0, i0) _htoni_ui(_jit, r0, i0)
4287static void _htoni_ui(jit_state_t*, jit_int32_t, jit_word_t);
4288#if __WORDSIZE == 64
4289# define htoni_ul(r0, i0) _htoni_ul(_jit, r0, i0)
4290static void _htoni_ul(jit_state_t*, jit_int32_t, jit_word_t);
4291#endif
4292# define movi_f_w(r0, i0) _movi_f_w(_jit, r0, i0)
4293static void _movi_f_w(jit_state_t*, jit_int32_t, jit_float32_t);
d481fb64 4294#if __WORDSIZE == 32 && !(defined(__mips__) && NEW_ABI)
ba86ff93
PC		 4295# define movi_d_ww(r0, r1, i0) _movi_d_ww(_jit, r0, r1, i0)
4296static void _movi_d_ww(jit_state_t*, jit_int32_t, jit_int32_t, jit_float64_t);
4297#else
4298# define movi_d_w(r0, i0) _movi_d_w(_jit, r0, i0)
4299static void _movi_d_w(jit_state_t*, jit_int32_t, jit_float64_t);
4300#endif
4301#define cloi(r0, i0) _cloi(_jit, r0, i0)
4302static void _cloi(jit_state_t*, jit_int32_t, jit_word_t);
4303#define clzi(r0, i0) _clzi(_jit, r0, i0)
4304static void _clzi(jit_state_t*, jit_int32_t, jit_word_t);
4305#define ctoi(r0, i0) _ctoi(_jit, r0, i0)
4306static void _ctoi(jit_state_t*, jit_int32_t, jit_word_t);
4307#define ctzi(r0, i0) _ctzi(_jit, r0, i0)
4308static void _ctzi(jit_state_t*, jit_int32_t, jit_word_t);
4309#define rbiti(r0, i0) _rbiti(_jit, r0, i0)
4310static void _rbiti(jit_state_t*, jit_int32_t, jit_word_t);
4311#define popcnti(r0, i0) _popcnti(_jit, r0, i0)
4312static void _popcnti(jit_state_t*, jit_int32_t, jit_word_t);
4313#define exti(r0, i0, i1, i2) _exti(_jit, r0, i0, i1, i2)
4314static void _exti(jit_state_t*,jit_int32_t, jit_word_t, jit_word_t, jit_word_t);
4315#define exti_u(r0, i0, i1, i2) _exti_u(_jit, r0, i0, i1, i2)
4316static void _exti_u(jit_state_t*,
4317 jit_int32_t, jit_word_t, jit_word_t, jit_word_t);
4318#define generic_unldr(r0, r1, i0) _generic_unldr(_jit, r0, r1, i0)
4319static void _generic_unldr(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
4320#define generic_unldi(r0, i0, i1) _generic_unldi(_jit, r0, i0, i1)
4321static void _generic_unldi(jit_state_t*, jit_int32_t, jit_word_t, jit_word_t);
4322#define generic_unldr_u(r0, r1, i0) _generic_unldr_u(_jit, r0, r1, i0)
4323static void _generic_unldr_u(jit_state_t*,
4324 jit_int32_t, jit_int32_t, jit_word_t);
4325#define generic_unldi_u(r0, i0, i1) _generic_unldi_u(_jit, r0, i0, i1)
4326static void _generic_unldi_u(jit_state_t*, jit_int32_t, jit_word_t, jit_word_t);
4327#define generic_unstr(r0, r1, i0) _generic_unstr(_jit, r0, r1, i0)
4328static void _generic_unstr(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
4329#define generic_unsti(i0, r0, i1) _generic_unsti(_jit, i0, r0, i1)
4330static void _generic_unsti(jit_state_t*, jit_word_t, jit_int32_t, jit_word_t);
4331#if !defined(__i386__) && !defined(__x86_64__)
4332# define generic_unldr_x(r0, r1, i0) _generic_unldr_x(_jit, r0, r1, i0)
4333static void _generic_unldr_x(jit_state_t*,
4334 jit_int32_t, jit_int32_t, jit_word_t);
4335# define generic_unldi_x(r0, i0, i1) _generic_unldi_x(_jit, r0, i0, i1)
4336static void _generic_unldi_x(jit_state_t*, jit_int32_t, jit_word_t, jit_word_t);
4337# define generic_unstr_x(r0, r1, i0) _generic_unstr_x(_jit, r0, r1, i0)
4338static void _generic_unstr_x(jit_state_t*,
4339 jit_int32_t, jit_int32_t, jit_word_t);
4340# define generic_unsti_x(i0, r0, i1) _generic_unsti_x(_jit, i0, r0, i1)
4341static void _generic_unsti_x(jit_state_t*, jit_word_t, jit_int32_t, jit_word_t);
4342#endif
79bfeef6
PC		 4343#define patch_alist(revert) _patch_alist(_jit, revert)
4344static maybe_unused void _patch_alist(jit_state_t *_jit, jit_bool_t revert);
4345
4a71579b
PC		 4346#if defined(__i386__) || defined(__x86_64__)
4347# include "jit_x86.c"
4348#elif defined(__mips__)
4349# include "jit_mips.c"
4350#elif defined(__arm__)
4351# include "jit_arm.c"
4352#elif defined(__powerpc__)
4353# include "jit_ppc.c"
4354#elif defined(__sparc__)
4355# include "jit_sparc.c"
4356#elif defined(__ia64__)
4357# include "jit_ia64.c"
4358#elif defined(__hppa__)
4359# include "jit_hppa.c"
4360#elif defined(__aarch64__)
4361# include "jit_aarch64.c"
4362#elif defined(__s390__) || defined(__s390x__)
4363# include "jit_s390.c"
4364#elif defined(__alpha__)
4365# include "jit_alpha.c"
4366#elif defined(__riscv)
4367# include "jit_riscv.c"
24d91c0d
PC		 4368#elif defined(__loongarch__)
4369# include "jit_loongarch.c"
4a71579b 4370#endif
40a44dcb
PC		 4371
4372static maybe_unused void
4373generic_bswapr_us(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
4374{
4375 jit_int32_t reg = jit_get_reg(jit_class_gpr);
4376
4377 rshi(rn(reg), r1, 8);
4378 andi(r0, r1, 0xff);
4379 andi(rn(reg), rn(reg), 0xff);
4380 lshi(r0, r0, 8);
4381 orr(r0, r0, rn(reg));
4382
4383 jit_unget_reg(reg);
4384}
4385
4386static maybe_unused void
4387generic_bswapr_ui(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
4388{
4389 jit_int32_t reg = jit_get_reg(jit_class_gpr);
4390
4391 rshi(rn(reg), r1, 16);
4392 bswapr_us(r0, r1);
4393 bswapr_us(rn(reg), rn(reg));
4394 lshi(r0, r0, 16);
4395 orr(r0, r0, rn(reg));
4396
4397 jit_unget_reg(reg);
4398}
4399
4400#if __WORDSIZE == 64
4401static maybe_unused void
4402generic_bswapr_ul(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
4403{
4404 jit_int32_t reg = jit_get_reg(jit_class_gpr);
4405
4406 rshi_u(rn(reg), r1, 32);
4407 bswapr_ui(r0, r1);
4408 bswapr_ui(rn(reg), rn(reg));
4409 lshi(r0, r0, 32);
4410 orr(r0, r0, rn(reg));
4411
4412 jit_unget_reg(reg);
4413}
4414#endif
79bfeef6 4415
ba86ff93
PC		 4416static void
4417_depi(jit_state_t *_jit,
4418 jit_int32_t r0, jit_word_t i0, jit_word_t i1, jit_word_t i2)
4419{
4420 jit_int32_t reg;
4421 reg = jit_get_reg(jit_class_gpr);
4422 movi(rn(reg), i0);
4423 depr(r0, rn(reg), i1, i2);
4424 jit_unget_reg(reg);
4425}
4426
4427static void
4428_negi(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
4429{
4430 movi(r0, -i0);
4431}
4432
4433static void
4434_comi(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
4435{
4436 movi(r0, ~i0);
4437}
4438
4439static void
4440_exti_c(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
4441{
4442 movi(r0, (jit_int8_t)i0);
4443}
4444
4445static void
4446_exti_uc(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
4447{
4448 movi(r0, (jit_uint8_t)i0);
4449}
4450
4451static void
4452_exti_s(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
4453{
4454 movi(r0, (jit_int16_t)i0);
4455}
4456
4457static void
4458_exti_us(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
4459{
4460 movi(r0, (jit_uint16_t)i0);
4461}
4462
4463#if __WORDSIZE == 64
4464static void
4465_exti_i(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
4466{
4467 movi(r0, (jit_int32_t)i0);
4468}
4469
4470static void
4471_exti_ui(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
4472{
4473 movi(r0, (jit_uint32_t)i0);
4474}
4475#endif
4476
4477static void
4478_bswapi_us(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
4479{
4480 union {
4481 jit_uint16_t us;
4482 jit_uint8_t v[2];
4483 } l, h;
4484 l.us = i0;
4485 h.v[0] = l.v[1];
4486 h.v[1] = l.v[0];
4487 movi(r0, h.us);
4488}
4489
4490static void
4491_bswapi_ui(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
4492{
4493 union {
4494 jit_uint32_t ui;
4495 jit_uint8_t v[4];
4496 } l, h;
4497 l.ui = i0;
4498 h.v[0] = l.v[3];
4499 h.v[1] = l.v[2];
4500 h.v[2] = l.v[1];
4501 h.v[3] = l.v[0];
4502 movi(r0, h.ui);
4503}
4504
4505#if __WORDSIZE == 64
4506static void
4507_bswapi_ul(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
4508{
4509 union {
4510 jit_uint64_t ul;
4511 jit_uint8_t v[8];
4512 } l, h;
4513 l.ul = i0;
4514 h.v[0] = l.v[7];
4515 h.v[1] = l.v[6];
4516 h.v[2] = l.v[5];
4517 h.v[3] = l.v[4];
4518 h.v[4] = l.v[3];
4519 h.v[5] = l.v[2];
4520 h.v[6] = l.v[1];
4521 h.v[7] = l.v[0];
4522 movi(r0, h.ul);
4523}
4524#endif
4525
4526static void
4527_htoni_us(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
4528{
4529#if __BYTE_ORDER == __LITTLE_ENDIAN
4530 bswapi_us(r0, i0);
4531#else
4532 exti_us(r0, i0);
4533#endif
4534}
4535
4536static void
4537_htoni_ui(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
4538{
4539#if __BYTE_ORDER == __LITTLE_ENDIAN
4540 bswapi_ui(r0, i0);
4541#else
4542# if __WORDSIZE == 32
4543 movi(r0, i0);
4544# else
4545 exti_ui(r0, i0);
4546# endif
4547#endif
4548}
4549
4550#if __WORDSIZE == 64
4551static void
4552_htoni_ul(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
4553{
4554# if __BYTE_ORDER == __LITTLE_ENDIAN
4555 bswapi_ul(r0, i0);
4556# else
4557 movi(r0, i0);
4558# endif
4559}
4560#endif
4561
4562static void
4563_movi_f_w(jit_state_t *_jit, jit_int32_t r0, jit_float32_t i0)
4564{
4565 union {
4566 jit_int32_t i;
4567 jit_float32_t f;
4568 } data;
4569 data.f = i0;
4570#if defined(__ia64__)
4571 /* Should be used only in this case (with out0 == 120) */
4572 if (r0 >= 120)
4573 r0 = _jitc->rout + (r0 - 120);
4574#endif
4575 movi(r0, data.i);
4576}
4577
d481fb64 4578#if __WORDSIZE == 32 && !(defined(__mips__) && NEW_ABI)
ba86ff93
PC		 4579static void
4580_movi_d_ww(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_float64_t i0)
4581{
4582 union {
4583 jit_int32_t i[2];
4584 jit_float64_t d;
4585 } data;
4586 data.d = i0;
4587 /* Mips does not change byte order of double values */
4588# if __BYTE_ORDER == __LITTLE_ENDIAN || defined(__mips__)
4589 movi(r0, data.i[0]);
4590 movi(r1, data.i[1]);
4591# else
4592 movi(r1, data.i[0]);
4593 movi(r0, data.i[1]);
4594# endif
4595}
4596
4597#else
4598static void
4599_movi_d_w(jit_state_t *_jit, jit_int32_t r0, jit_float64_t i0)
4600{
4601 union {
4602 jit_int64_t l;
4603 jit_float64_t d;
4604 } data;
4605 data.d = i0;
4606# if defined(__ia64__)
4607 /* Should be used only in this case (with out0 == 120) */
4608 if (r0 >= 120)
4609 r0 = _jitc->rout + (r0 - 120);
4610# endif
4611 movi(r0, data.l);
4612}
4613#endif
4614
4615 void
4616_jit_negi_f(jit_state_t *_jit, jit_fpr_t u, jit_float32_t v)
4617{
4618 jit_inc_synth_wf(negi_f, u, v);
4619 jit_movi_f(u, v);
4620 jit_negr_f(u, u);
4621 jit_dec_synth();
4622}
4623
4624void
4625_jit_absi_f(jit_state_t *_jit, jit_fpr_t u, jit_float32_t v)
4626{
4627 jit_inc_synth_wf(absi_f, u, v);
4628 jit_movi_f(u, v);
4629 jit_absr_f(u, u);
4630 jit_dec_synth();
4631}
4632
4633void
4634_jit_sqrti_f(jit_state_t *_jit, jit_fpr_t u, jit_float32_t v)
4635{
4636 jit_inc_synth_wf(sqrti_f, u, v);
4637 jit_movi_f(u, v);
4638 jit_sqrtr_f(u, u);
4639 jit_dec_synth();
4640}
4641
4642void
4643_jit_fmai_f(jit_state_t *_jit,
4644 jit_fpr_t u, jit_fpr_t v, jit_fpr_t w, jit_float32_t x)
4645{
4646 jit_int32_t y;
4647 jit_inc_synth_wqf(fmai_f, u, v, w, x);
4648 if (u != v && u != w) {
4649 jit_movi_f(u, x);
4650 jit_fmar_f(u, v, w, u);
4651 }
4652 else {
4653 y = jit_get_reg(jit_class_fpr);
4654 jit_movi_f(y, x);
4655 jit_fmar_f(u, v, w, y);
4656 jit_unget_reg(y);
4657 }
4658 jit_dec_synth();
4659}
4660
4661void
4662_jit_fmsi_f(jit_state_t *_jit,
4663 jit_fpr_t u, jit_fpr_t v, jit_fpr_t w, jit_float32_t x)
4664{
4665 jit_int32_t y;
4666 jit_inc_synth_wqf(fmai_f, u, v, w, x);
4667 if (u != v && u != w) {
4668 jit_movi_f(u, x);
4669 jit_fmsr_f(u, v, w, u);
4670 }
4671 else {
4672 y = jit_get_reg(jit_class_fpr);
4673 jit_movi_f(y, x);
4674 jit_fmsr_f(u, v, w, y);
4675 jit_unget_reg(y);
4676 }
4677 jit_dec_synth();
4678}
4679
4680void
4681_jit_fnmai_f(jit_state_t *_jit,
4682 jit_fpr_t u, jit_fpr_t v, jit_fpr_t w, jit_float32_t x)
4683{
4684 jit_int32_t y;
4685 jit_inc_synth_wqf(fmai_f, u, v, w, x);
4686 if (u != v && u != w) {
4687 jit_movi_f(u, x);
4688 jit_fnmar_f(u, v, w, u);
4689 }
4690 else {
4691 y = jit_get_reg(jit_class_fpr);
4692 jit_movi_f(y, x);
4693 jit_fnmar_f(u, v, w, y);
4694 jit_unget_reg(y);
4695 }
4696 jit_dec_synth();
4697}
4698
4699void
4700_jit_fnmsi_f(jit_state_t *_jit,
4701 jit_fpr_t u, jit_fpr_t v, jit_fpr_t w, jit_float32_t x)
4702{
4703 jit_int32_t y;
4704 jit_inc_synth_wqf(fmai_f, u, v, w, x);
4705 if (u != v && u != w) {
4706 jit_movi_f(u, x);
4707 jit_fnmsr_f(u, v, w, u);
4708 }
4709 else {
4710 y = jit_get_reg(jit_class_fpr);
4711 jit_movi_f(y, x);
4712 jit_fnmsr_f(u, v, w, y);
4713 jit_unget_reg(y);
4714 }
4715 jit_dec_synth();
4716}
4717
4718void
4719_jit_negi_d(jit_state_t *_jit, jit_fpr_t u, jit_float64_t v)
4720{
4721 jit_inc_synth_wd(negi_d, u, v);
4722 jit_movi_d(u, v);
4723 jit_negr_d(u, u);
4724 jit_dec_synth();
4725}
4726
4727void
4728_jit_absi_d(jit_state_t *_jit, jit_fpr_t u, jit_float64_t v)
4729{
4730 jit_inc_synth_wd(absi_d, u, v);
4731 jit_movi_d(u, v);
4732 jit_absr_d(u, u);
4733 jit_dec_synth();
4734}
4735
4736void
4737_jit_sqrti_d(jit_state_t *_jit, jit_fpr_t u, jit_float64_t v)
4738{
4739 jit_inc_synth_wd(sqrti_d, u, v);
4740 jit_movi_d(u, v);
4741 jit_sqrtr_d(u, u);
4742 jit_dec_synth();
4743}
4744
4745void
4746_jit_fmai_d(jit_state_t *_jit,
4747 jit_fpr_t u, jit_fpr_t v, jit_fpr_t w, jit_float64_t x)
4748{
4749 jit_int32_t y;
4750 jit_inc_synth_wqd(fmai_d, u, v, w, x);
4751 if (u != v && u != w) {
4752 jit_movi_d(u, x);
4753 jit_fmar_d(u, v, w, u);
4754 }
4755 else {
4756 y = jit_get_reg(jit_class_fpr);
4757 jit_movi_d(y, x);
4758 jit_fmar_d(u, v, w, y);
4759 jit_unget_reg(y);
4760 }
4761 jit_dec_synth();
4762}
4763
4764void
4765_jit_fmsi_d(jit_state_t *_jit,
4766 jit_fpr_t u, jit_fpr_t v, jit_fpr_t w, jit_float64_t x)
4767{
4768 jit_int32_t y;
4769 jit_inc_synth_wqd(fmai_d, u, v, w, x);
4770 if (u != v && u != w) {
4771 jit_movi_d(u, x);
4772 jit_fmsr_d(u, v, w, u);
4773 }
4774 else {
4775 y = jit_get_reg(jit_class_fpr);
4776 jit_movi_d(y, x);
4777 jit_fmsr_d(u, v, w, y);
4778 jit_unget_reg(y);
4779 }
4780 jit_dec_synth();
4781}
4782
4783void
4784_jit_fnmai_d(jit_state_t *_jit,
4785 jit_fpr_t u, jit_fpr_t v, jit_fpr_t w, jit_float64_t x)
4786{
4787 jit_int32_t y;
4788 jit_inc_synth_wqd(fmai_d, u, v, w, x);
4789 if (u != v && u != w) {
4790 jit_movi_d(u, x);
4791 jit_fnmar_d(u, v, w, u);
4792 }
4793 else {
4794 y = jit_get_reg(jit_class_fpr);
4795 jit_movi_d(y, x);
4796 jit_fnmar_d(u, v, w, y);
4797 jit_unget_reg(y);
4798 }
4799 jit_dec_synth();
4800}
4801
4802void
4803_jit_fnmsi_d(jit_state_t *_jit,
4804 jit_fpr_t u, jit_fpr_t v, jit_fpr_t w, jit_float64_t x)
4805{
4806 jit_int32_t y;
4807 jit_inc_synth_wqd(fmai_d, u, v, w, x);
4808 if (u != v && u != w) {
4809 jit_movi_d(u, x);
4810 jit_fnmsr_d(u, v, w, u);
4811 }
4812 else {
4813 y = jit_get_reg(jit_class_fpr);
4814 jit_movi_d(y, x);
4815 jit_fnmsr_d(u, v, w, y);
4816 jit_unget_reg(y);
4817 }
4818 jit_dec_synth();
4819}
4820
4821static void
4822_cloi(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
4823{
4824 clzi(r0, ~i0);
4825}
4826
4827static void
4828_clzi(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
4829{
4830#if __WORDSIZE == 64 && _WIN32
4831 movi(r0, (i0) ? __builtin_clzll(i0) : __WORDSIZE);
4832#else
4833 movi(r0, (i0) ? __builtin_clzl(i0) : __WORDSIZE);
4834#endif
4835}
4836
4837static void
4838_ctoi(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
4839{
4840 ctzi(r0, ~i0);
4841}
4842
4843static void
4844_ctzi(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
4845{
4846#if __WORDSIZE == 64 && _WIN32
4847 movi(r0, (i0) ? __builtin_ctzll(i0) : __WORDSIZE);
4848#else
4849 movi(r0, (i0) ? __builtin_ctzl(i0) : __WORDSIZE);
4850#endif
4851}
4852
4853static void
4854_rbiti(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
4855{
4856 jit_int32_t i;
4857 union {
4858 jit_uword_t w;
4859 jit_uint8_t v[__WORDSIZE >> 3];
4860 } u, v;
4861 static const unsigned char swap_tab[256] = {
4862 0, 128, 64, 192, 32, 160, 96, 224,
4863 16, 144, 80, 208, 48, 176, 112, 240,
4864 8, 136, 72, 200, 40, 168, 104, 232,
4865 24, 152, 88, 216 ,56, 184, 120, 248,
4866 4, 132, 68, 196, 36, 164, 100, 228,
4867 20, 148, 84, 212, 52, 180, 116, 244,
4868 12, 140, 76, 204, 44, 172, 108, 236,
4869 28, 156, 92, 220, 60, 188, 124, 252,
4870 2, 130, 66, 194, 34, 162, 98, 226,
4871 18, 146, 82, 210, 50, 178, 114, 242,
4872 10, 138, 74, 202, 42, 170, 106, 234,
4873 26, 154, 90, 218, 58, 186, 122, 250,
4874 6, 134, 70, 198, 38, 166, 102, 230,
4875 22, 150, 86, 214, 54, 182, 118, 246,
4876 14, 142, 78, 206, 46, 174, 110, 238,
4877 30, 158, 94, 222, 62, 190, 126, 254,
4878 1, 129, 65, 193, 33, 161, 97, 225,
4879 17, 145, 81, 209, 49, 177, 113, 241,
4880 9, 137, 73, 201, 41, 169, 105, 233,
4881 25, 153, 89, 217, 57, 185, 121, 249,
4882 5, 133, 69, 197, 37, 165, 101, 229,
4883 21, 149, 85, 213, 53, 181, 117, 245,
4884 13, 141, 77, 205, 45, 173, 109, 237,
4885 29, 157, 93, 221, 61, 189, 125, 253,
4886 3, 131, 67, 195, 35, 163, 99, 227,
4887 19, 147, 83, 211, 51, 179, 115, 243,
4888 11, 139, 75, 203, 43, 171, 107, 235,
4889 27, 155, 91, 219, 59, 187, 123, 251,
4890 7, 135, 71, 199, 39, 167, 103, 231,
4891 23, 151, 87, 215, 55, 183, 119, 247,
4892 15, 143, 79, 207, 47, 175, 111, 239,
4893 31, 159, 95, 223, 63, 191, 127, 255
4894 };
4895 u.w = i0;
4896 for (i = 0; i < sizeof(jit_word_t); ++i)
4897 v.v[i] = swap_tab[u.v[sizeof(jit_word_t) - i - 1]];
4898 movi(r0, v.w);
4899}
4900
4901static void
4902_popcnti(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
4903{
4904#if __WORDSIZE == 64 && _WIN32
4905 movi(r0, (i0) ? __builtin_popcountll(i0) : __WORDSIZE);
4906#else
4907 movi(r0, (i0) ? __builtin_popcountl(i0) : __WORDSIZE);
4908#endif
4909}
4910
4911static void _exti(jit_state_t *_jit,
4912 jit_int32_t r0, jit_word_t i0, jit_word_t i1, jit_word_t i2)
4913{
4914#if __BYTE_ORDER == __BIG_ENDIAN
4915 i1 = __WORDSIZE - (i1 + i2);
4916#endif
4917 i0 <<= __WORDSIZE - (i1 + i2);
4918 i0 >>= __WORDSIZE - i2;
4919 movi(r0, i0);
4920}
4921
4922static void _exti_u(jit_state_t *_jit,
4923 jit_int32_t r0, jit_word_t i0, jit_word_t i1, jit_word_t i2)
4924{
4925 jit_word_t t;
4926#if __BYTE_ORDER == __BIG_ENDIAN
4927 i1 = __WORDSIZE - (i1 + i2);
4928#endif
4929 if (i1)
4930 i0 >>= __WORDSIZE - i2;
4931#if __WORDSIZE == 64 && !_WIN32
4932 i0 &= (1L << i2) - 1;
4933#else
4934 i0 &= (1LL << i2) - 1;
4935#endif
4936 movi(r0, i0);
4937}
4938
4939static void
4940_generic_unldr(jit_state_t *_jit,
4941 jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
4942{
4943 jit_int32_t t0, r2;
4944 assert(i0 >= 1 && i0 <= sizeof(jit_word_t));
4945 if (i0 & (i0 - 1)) {
4946 t0 = jit_get_reg(jit_class_gpr);
4947 r2 = rn(t0);
4948 movr(r2, r1);
4949 }
4950 switch (i0) {
4951 case 1:
4952 ldr_c(r0, r1);
4953 break;
4954 case 2:
4955 ldr_s(r0, r1);
4956 break;
4957 case 3:
4958#if __BYTE_ORDER == __LITTLE_ENDIAN
4959 ldr_us(r0, r2);
4960 ldxi_c(r2, r2, 2);
4961 lshi(r2, r2, 16);
4962#else
4963 ldr_c(r0, r2);
4964 lshi(r0, r0, 16);
4965 ldxi_us(r2, r2, 1);
4966#endif
4967 break;
4968#if __WORDSIZE == 32
4969 default:
4970 ldr_i(r0, r1);
4971 break;
4972#else
4973 case 4:
4974 ldr_i(r0, r1);
4975 break;
4976 case 5:
4977# if __BYTE_ORDER == __LITTLE_ENDIAN
4978 ldr_ui(r0, r2);
4979 ldxi_c(r2, r2, 4);
4980 lshi(r2, r2, 32);
4981# else
4982 ldr_i(r0, r2);
4983 lshi(r0, r0, 8);
4984 ldxi_uc(r2, r2, 4);
4985# endif
4986 break;
4987 case 6:
4988# if __BYTE_ORDER == __LITTLE_ENDIAN
4989 ldr_ui(r0, r2);
4990 ldxi_s(r2, r2, 4);
4991 lshi(r2, r2, 32);
4992# else
4993 ldr_i(r0, r2);
4994 lshi(r0, r0, 16);
4995 ldxi_us(r2, r2, 4);
4996# endif
4997 break;
4998 case 7:
4999# if __BYTE_ORDER == __LITTLE_ENDIAN
5000 ldr_ui(r0, r2);
5001 ldxi_i(r2, r2, 4);
5002 lshi(r2, r2, 40);
5003 rshi(r2, r2, 8);
5004# else
5005 ldr_i(r0, r2);
5006 lshi(r0, r0, 24);
5007 ldxi_ui(r2, r2, 4);
5008 rshi(r2, r2, 8);
5009# endif
5010 break;
5011 default:
5012 ldr_l(r0, r1);
5013 break;
5014#endif
5015 }
5016 if (i0 & (i0 - 1)) {
5017 orr(r0, r0, r2);
5018 jit_unget_reg(t0);
5019 }
5020}
5021
5022static void
5023_generic_unldi(jit_state_t *_jit,
5024 jit_int32_t r0, jit_word_t i0, jit_word_t i1)
5025{
5026 jit_int32_t t0, r2;
5027 assert(i1 >= 1 && i1 <= sizeof(jit_word_t));
5028 if (i1 & (i1 - 1)) {
5029 t0 = jit_get_reg(jit_class_gpr);
5030 r2 = rn(t0);
5031 }
5032 switch (i1) {
5033 case 1:
5034 ldi_c(r0, i0);
5035 break;
5036 case 2:
5037 ldi_s(r0, i0);
5038 break;
5039 case 3:
5040# if __BYTE_ORDER == __LITTLE_ENDIAN
5041 ldi_us(r0, i0);
5042 ldi_c(r2, i0 + 2);
5043 lshi(r2, r2, 16);
5044# else
5045 ldi_c(r0, i0);
5046 lshi(r0, r0, 16);
5047 ldi_us(r2, i0 + 1);
5048# endif
5049 break;
5050# if __WORDSIZE == 32
5051 default:
5052 ldi_i(r0, i0);
5053 break;
5054# else
5055 case 4:
5056 ldi_i(r0, i0);
5057 break;
5058 case 5:
5059# if __BYTE_ORDER == __LITTLE_ENDIAN
5060 ldi_ui(r0, i0);
5061 ldi_c(r2, i0 + 4);
5062 lshi(r2, r2, 32);
5063# else
5064 ldi_i(r0, i0);
5065 lshi(r0, r0, 8);
5066 ldi_uc(r2, i0 + 4);
5067# endif
5068 break;
5069 case 6:
5070# if __BYTE_ORDER == __LITTLE_ENDIAN
5071 ldi_ui(r0, i0);
5072 ldi_s(r2, i0 + 4);
5073 lshi(r2, r2, 32);
5074# else
5075 ldi_i(r0, i0);
5076 lshi(r0, r0, 16);
5077 ldi_us(r2, i0 + 4);
5078# endif
5079 break;
5080 case 7:
5081# if __BYTE_ORDER == __LITTLE_ENDIAN
5082 ldi_ui(r0, i0);
5083 ldi_i(r2, i0 + 4);
5084 lshi(r2, r2, 40);
5085 rshi(r2, r2, 8);
5086# else
5087 ldi_i(r0, i0);
5088 lshi(r0, r0, 24);
5089 ldi_ui(r2, i0 + 4);
5090 rshi(r2, r2, 8);
5091# endif
5092 break;
5093 default:
5094 ldi_l(r0, i0);
5095 break;
5096# endif
5097 }
5098 if (i1 & (i1 - 1)) {
5099 orr(r0, r0, r2);
5100 jit_unget_reg(t0);
5101 }
5102}
5103
5104static void
5105_generic_unldr_u(jit_state_t *_jit,
5106 jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
5107{
5108 jit_int32_t t0, r2;
5109 assert(i0 >= 1 && i0 <= sizeof(jit_word_t));
5110 if (i0 & (i0 - 1)) {
5111 t0 = jit_get_reg(jit_class_gpr);
5112 r2 = rn(t0);
5113 movr(r2, r1);
5114 }
5115 switch (i0) {
5116 case 1:
5117 ldr_uc(r0, r1);
5118 break;
5119 case 2:
5120 ldr_us(r0, r1);
5121 break;
5122 case 3:
5123# if __BYTE_ORDER == __LITTLE_ENDIAN
5124 ldr_us(r0, r2);
5125 ldxi_uc(r2, r2, 2);
5126 lshi(r2, r2, 16);
5127# else
5128 ldr_uc(r0, r2);
5129 lshi(r0, r0, 16);
5130 ldxi_us(r2, r2, 1);
5131# endif
5132 break;
5133# if __WORDSIZE == 32
5134 default:
5135 ldr_i(r0, r1);
5136 break;
5137# else
5138 case 4:
5139 ldr_ui(r0, r1);
5140 break;
5141 case 5:
5142# if __BYTE_ORDER == __LITTLE_ENDIAN
5143 ldr_ui(r0, r2);
5144 ldxi_uc(r2, r2, 4);
5145 lshi(r2, r2, 32);
5146# else
5147 ldr_ui(r0, r2);
5148 lshi(r0, r0, 8);
5149 ldxi_uc(r2, r2, 4);
5150# endif
5151 break;
5152 case 6:
5153# if __BYTE_ORDER == __LITTLE_ENDIAN
5154 ldr_ui(r0, r2);
5155 ldxi_us(r2, r2, 4);
5156 lshi(r2, r2, 32);
5157# else
5158 ldr_ui(r0, r2);
5159 lshi(r0, r0, 16);
5160 ldxi_us(r2, r2, 4);
5161# endif
5162 break;
5163 case 7:
5164# if __BYTE_ORDER == __LITTLE_ENDIAN
5165 ldr_ui(r0, r2);
5166 ldxi_ui(r2, r2, 4);
5167 lshi(r2, r2, 40);
5168 rshi_u(r2, r2, 8);
5169# else
5170 ldr_ui(r0, r2);
5171 lshi(r0, r0, 24);
5172 ldxi_ui(r2, r2, 4);
5173 rshi(r2, r2, 8);
5174# endif
5175 break;
5176 default:
5177 ldr_l(r0, r1);
5178 break;
5179# endif
5180 }
5181 if (i0 & (i0 - 1)) {
5182 orr(r0, r0, r2);
5183 jit_unget_reg(t0);
5184 }
5185}
5186
5187static void
5188_generic_unldi_u(jit_state_t *_jit,
5189 jit_int32_t r0, jit_word_t i0, jit_word_t i1)
5190{
5191 jit_int32_t t0, r2;
5192 assert(i1 >= 1 && i1 <= sizeof(jit_word_t));
5193 if (i1 & (i1 - 1)) {
5194 t0 = jit_get_reg(jit_class_gpr);
5195 r2 = rn(t0);
5196 }
5197 switch (i1) {
5198 case 1:
5199 ldi_uc(r0, i0);
5200 break;
5201 case 2:
5202 ldi_us(r0, i0);
5203 break;
5204 case 3:
5205# if __BYTE_ORDER == __LITTLE_ENDIAN
5206 ldi_us(r0, i0);
5207 ldi_uc(r2, i0 + 2);
5208 lshi(r2, r2, 16);
5209# else
5210 ldi_uc(r0, i0);
5211 lshi(r0, r0, 16);
5212 ldi_us(r2, i0 + 1);
5213# endif
5214 break;
5215# if __WORDSIZE == 32
5216 default:
5217 ldi_i(r0, i0);
5218 break;
5219# else
5220 case 4:
5221 ldi_ui(r0, i0);
5222 break;
5223 case 5:
5224# if __BYTE_ORDER == __LITTLE_ENDIAN
5225 ldi_ui(r0, i0);
5226 ldi_uc(r2, i0 + 4);
5227 lshi(r2, r2, 32);
5228# else
5229 ldi_ui(r0, i0);
5230 lshi(r0, r0, 8);
5231 ldi_uc(r2, i0 + 4);
5232# endif
5233 break;
5234 case 6:
5235# if __BYTE_ORDER == __LITTLE_ENDIAN
5236 ldi_ui(r0, i0);
5237 ldi_us(r2, i0 + 4);
5238 lshi(r2, r2, 32);
5239# else
5240 ldi_ui(r0, i0);
5241 lshi(r0, r0, 16);
5242 ldi_us(r2, i0 + 4);
5243# endif
5244 break;
5245 case 7:
5246# if __BYTE_ORDER == __LITTLE_ENDIAN
5247 ldi_ui(r0, i0);
5248 ldi_ui(r2, i0 + 4);
5249 lshi(r2, r2, 40);
5250 rshi_u(r2, r2, 8);
5251# else
5252 ldi_ui(r0, i0);
5253 lshi(r0, r0, 24);
5254 ldi_ui(r2, i0 + 4);
5255 rshi(r2, r2, 8);
5256# endif
5257 break;
5258 default:
5259 ldi_l(r0, i0);
5260 break;
5261# endif
5262 }
5263 if (i1 & (i1 - 1)) {
5264 orr(r0, r0, r2);
5265 jit_unget_reg(t0);
5266 }
5267}
5268
5269static void
5270_generic_unstr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
5271{
5272 jit_word_t t0, r2;
5273 assert(i0 > 0 && i0 <= sizeof(jit_word_t));
5274 if (i0 & (i0 - 1)) {
5275 t0 = jit_get_reg(jit_class_gpr);
5276 r2 = rn(t0);
5277 }
5278 switch (i0) {
5279 case 1:
5280 str_c(r0, r1);
5281 break;
5282 case 2:
5283 str_s(r0, r1);
5284 break;
5285 case 3:
5286# if __BYTE_ORDER == __LITTLE_ENDIAN
5287 str_s(r0, r1);
5288 rshi(r2, r1, 16);
5289 stxi_c(2, r0, r2);
5290# else
5291 stxi_c(2, r0, r1);
5292 rshi(r2, r1, 8);
5293 str_s(r0, r2);
5294# endif
5295 break;
5296# if __WORDSIZE == 32
5297 default:
5298 str_i(r0, r1);
5299 break;
5300# else
5301 case 4:
5302 str_i(r0, r1);
5303 break;
5304 case 5:
5305# if __BYTE_ORDER == __LITTLE_ENDIAN
5306 str_i(r0, r1);
5307 rshi(r2, r1, 32);
5308 stxi_c(4, r0, r2);
5309# else
5310 stxi_c(4, r0, r1);
5311 rshi(r2, r1, 8);
5312 str_i(r0, r2);
5313# endif
5314 break;
5315 case 6:
5316# if __BYTE_ORDER == __LITTLE_ENDIAN
5317 str_i(r0, r1);
5318 rshi(r2, r1, 32);
5319 stxi_s(4, r0, r2);
5320# else
5321 stxi_s(4, r0, r1);
5322 rshi(r2, r1, 16);
5323 str_i(r0, r2);
5324# endif
5325 break;
5326 case 7:
5327# if __BYTE_ORDER == __LITTLE_ENDIAN
5328 str_i(r0, r1);
5329 rshi(r2, r1, 32);
5330 stxi_s(4, r0, r2);
5331 rshi(r2, r2, 16);
5332 stxi_c(6, r0, r2);
5333# else
5334 stxi_c(6, r0, r1);
5335 rshi(r2, r1, 8);
5336 stxi_s(4, r0, r2);
5337 rshi(r2, r2, 16);
5338 str_i(r0, r2);
5339# endif
5340 break;
5341 default:
5342 str_l(r0, r1);
5343 break;
5344# endif
5345 }
5346 if (i0 & (i0 - 1))
5347 jit_unget_reg(t0);
5348}
5349
5350static void
5351_generic_unsti(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
5352{
5353 jit_word_t t0, r2;
5354 assert(i1 > 0 && i1 <= sizeof(jit_word_t));
5355 if (i1 & (i1 - 1)) {
5356 t0 = jit_get_reg(jit_class_gpr);
5357 r2 = rn(t0);
5358 }
5359 switch (i1) {
5360 case 1:
5361 sti_c(i0, r0);
5362 break;
5363 case 2:
5364 sti_s(i0, r0);
5365 break;
5366 case 3:
5367# if __BYTE_ORDER == __LITTLE_ENDIAN
5368 sti_s(i0, r0);
5369 rshi(r2, r0, 16);
5370 sti_c(2 + i0, r2);
5371# else
5372 sti_c(2 + i0, r0);
5373 rshi(r2, r0, 8);
5374 sti_s(i0, r2);
5375# endif
5376 break;
5377# if __WORDSIZE == 32
5378 default:
5379 sti_i(i0, r0);
5380 break;
5381# else
5382 case 4:
5383 sti_i(i0, r0);
5384 break;
5385 case 5:
5386# if __BYTE_ORDER == __LITTLE_ENDIAN
5387 sti_i(i0, r0);
5388 rshi(r2, r0, 32);
5389 sti_c(4 + i0, r2);
5390# else
5391 stxi_c(4, i0, r0);
5392 rshi(r2, r0, 8);
5393 sti_i(i0, r2);
5394# endif
5395 break;
5396 case 6:
5397# if __BYTE_ORDER == __LITTLE_ENDIAN
5398 sti_i(i0, r0);
5399 rshi(r2, r0, 32);
5400 sti_s(4 + i0, r2);
5401# else
5402 sti_s(4 + i0, r0);
5403 rshi(r2, r0, 16);
5404 sti_i(i0, r2);
5405# endif
5406 break;
5407 case 7:
5408# if __BYTE_ORDER == __LITTLE_ENDIAN
5409 sti_i(i0, r0);
5410 rshi(r2, r0, 32);
5411 sti_s(4 + i0, r2);
5412 rshi(r2, r2, 16);
5413 sti_c(6 + i0, r2);
5414# else
5415 sti_c(6 + i0, r0);
5416 rshi(r2, r0, 8);
5417 sti_s(4 + i0, r2);
5418 rshi(r2, r2, 16);
5419 sti_i(i0, r2);
5420# endif
5421 break;
5422 default:
5423 sti_l(i0, r0);
5424 break;
5425# endif
5426 }
5427 if (i1 & (i1 - 1))
5428 jit_unget_reg(t0);
5429}
5430
5431#if !defined(__i386__) && !defined(__x86_64__) && !defined(__arm__)
5432static void
5433_generic_unldr_x(jit_state_t *_jit,
5434 jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
5435{
5436 assert(i0 == 4 || i0 == 8);
5437 if (i0 == 4)
5438 ldr_f(r0, r1);
5439 else
5440 ldr_d(r0, r1);
5441}
5442
5443static void
5444_generic_unldi_x(jit_state_t *_jit,
5445 jit_int32_t r0, jit_word_t i0, jit_word_t i1)
5446{
5447 assert(i1 == 4 || i1 == 8);
5448 if (i1 == 4)
5449 ldi_f(r0, i0);
5450 else
5451 ldi_d(r0, i0);
5452}
5453
5454static void
5455_generic_unstr_x(jit_state_t *_jit,
5456 jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
5457{
5458 assert(i0 == 4 || i0 == 8);
5459 if (i0 == 4)
5460 str_f(r0, r1);
5461 else
5462 str_d(r0, r1);
5463}
5464
5465static void
5466_generic_unsti_x(jit_state_t *_jit,
5467 jit_word_t i0, jit_int32_t r0, jit_word_t i1)
5468{
5469 assert(i1 == 4 || i1 == 8);
5470 if (i1 == 4)
5471 sti_f(i0, r0);
5472 else
5473 sti_d(i0, r0);
5474}
5475#endif
5476
79bfeef6
PC		 5477#if defined(stack_framesize)
5478static maybe_unused void
5479_patch_alist(jit_state_t *_jit, jit_bool_t revert)
5480{
5481 jit_int32_t diff;
5482 jit_node_t *node;
5483 diff = jit_diffsize();
5484 if (diff) {
5485 if (revert)
5486 diff = -diff;
5487 for (node = _jitc->function->alist; node; node = node->link) {
5488 switch (node->code) {
5489 case jit_code_ldxi_c: case jit_code_ldxi_uc:
5490 case jit_code_ldxi_s: case jit_code_ldxi_us:
5491 case jit_code_ldxi_i:
5492#if __WORDSIZE == 64
5493 case jit_code_ldxi_ui: case jit_code_ldxi_l:
5494#endif
5495 case jit_code_ldxi_f: case jit_code_ldxi_d:
5496 node->w.w -= diff;
5497 break;
5498 case jit_code_stxi_c: case jit_code_stxi_s:
5499 case jit_code_stxi_i:
5500#if __WORDSIZE == 64
5501 case jit_code_stxi_l:
5502#endif
5503 case jit_code_stxi_f: case jit_code_stxi_d:
5504 node->u.w -= diff;
5505 break;
5506 default:
5507 abort();
5508 }
5509 }
5510 }
5511}
5512#endif
ARM optimized PCSX fork