deps/lightrec/emitter.c

   1 // SPDX-License-Identifier: LGPL-2.1-or-later
   2 /*
   3  * Copyright (C) 2014-2021 Paul Cercueil <paul@crapouillou.net>
   4  */
   5
   6 #include "blockcache.h"
   7 #include "debug.h"
   8 #include "disassembler.h"
   9 #include "emitter.h"
  10 #include "lightning-wrapper.h"
  11 #include "optimizer.h"
  12 #include "regcache.h"
  13
  14 #include <stdbool.h>
  15 #include <stddef.h>
  16
  17 typedef void (*lightrec_rec_func_t)(struct lightrec_cstate *, const struct block *, u16);
  18
  19 /* Forward declarations */
  20 static void rec_SPECIAL(struct lightrec_cstate *state, const struct block *block, u16 offset);
  21 static void rec_REGIMM(struct lightrec_cstate *state, const struct block *block, u16 offset);
  22 static void rec_CP0(struct lightrec_cstate *state, const struct block *block, u16 offset);
  23 static void rec_CP2(struct lightrec_cstate *state, const struct block *block, u16 offset);
  24
  25 static void unknown_opcode(struct lightrec_cstate *state, const struct block *block, u16 offset)
  26 {
  27         pr_warn("Unknown opcode: 0x%08x at PC 0x%08x\n",
  28                 block->opcode_list[offset].c.opcode,
  29                 block->pc + (offset << 2));
  30 }
  31
  32 static void
  33 lightrec_jump_to_eob(struct lightrec_cstate *state, jit_state_t *_jit)
  34 {
  35         /* Prevent jit_jmpi() from using our cycles register as a temporary */
  36         jit_live(LIGHTREC_REG_CYCLE);
  37
  38         jit_patch_abs(jit_jmpi(), state->state->eob_wrapper_func);
  39 }
  40
  41 static void lightrec_emit_end_of_block(struct lightrec_cstate *state,
  42                                        const struct block *block, u16 offset,
  43                                        s8 reg_new_pc, u32 imm, u8 ra_reg,
  44                                        u32 link, bool update_cycles)
  45 {
  46         struct regcache *reg_cache = state->reg_cache;
  47         jit_state_t *_jit = block->_jit;
  48         const struct opcode *op = &block->opcode_list[offset],
  49                             *next = &block->opcode_list[offset + 1];
  50         u32 cycles = state->cycles + lightrec_cycles_of_opcode(op->c);
  51
  52         jit_note(__FILE__, __LINE__);
  53
  54         if (link) {
  55                 /* Update the $ra register */
  56                 u8 link_reg = lightrec_alloc_reg_out(reg_cache, _jit, ra_reg, 0);
  57                 jit_movi(link_reg, link);
  58                 lightrec_free_reg(reg_cache, link_reg);
  59         }
  60
  61         if (reg_new_pc < 0) {
  62                 reg_new_pc = lightrec_alloc_reg(reg_cache, _jit, JIT_V0);
  63                 lightrec_lock_reg(reg_cache, _jit, reg_new_pc);
  64
  65                 jit_movi(reg_new_pc, imm);
  66         }
  67
  68         if (has_delay_slot(op->c) &&
  69             !op_flag_no_ds(op->flags) && !op_flag_local_branch(op->flags)) {
  70                 cycles += lightrec_cycles_of_opcode(next->c);
  71
  72                 /* Recompile the delay slot */
  73                 if (next->c.opcode)
  74                         lightrec_rec_opcode(state, block, offset + 1);
  75         }
  76
  77         /* Clean the remaining registers */
  78         lightrec_clean_regs(reg_cache, _jit);
  79
  80         jit_movr(JIT_V0, reg_new_pc);
  81
  82         if (cycles && update_cycles) {
  83                 jit_subi(LIGHTREC_REG_CYCLE, LIGHTREC_REG_CYCLE, cycles);
  84                 pr_debug("EOB: %u cycles\n", cycles);
  85         }
  86
  87         lightrec_jump_to_eob(state, _jit);
  88 }
  89
  90 void lightrec_emit_eob(struct lightrec_cstate *state, const struct block *block,
  91                        u16 offset, bool after_op)
  92 {
  93         struct regcache *reg_cache = state->reg_cache;
  94         jit_state_t *_jit = block->_jit;
  95         union code c = block->opcode_list[offset].c;
  96         u32 cycles = state->cycles;
  97
  98         if (after_op)
  99                 cycles += lightrec_cycles_of_opcode(c);
 100
 101         lightrec_clean_regs(reg_cache, _jit);
 102
 103         jit_movi(JIT_V0, block->pc + (offset << 2));
 104         jit_subi(LIGHTREC_REG_CYCLE, LIGHTREC_REG_CYCLE, cycles);
 105
 106         lightrec_jump_to_eob(state, _jit);
 107 }
 108
 109 static u8 get_jr_jalr_reg(struct lightrec_cstate *state, const struct block *block, u16 offset)
 110 {
 111         struct regcache *reg_cache = state->reg_cache;
 112         jit_state_t *_jit = block->_jit;
 113         const struct opcode *op = &block->opcode_list[offset];
 114         u8 rs;
 115
 116         rs = lightrec_request_reg_in(reg_cache, _jit, op->r.rs, JIT_V0);
 117         lightrec_lock_reg(reg_cache, _jit, rs);
 118
 119         return rs;
 120 }
 121
 122 static void rec_special_JR(struct lightrec_cstate *state, const struct block *block, u16 offset)
 123 {
 124         u8 rs = get_jr_jalr_reg(state, block, offset);
 125
 126         _jit_name(block->_jit, __func__);
 127         lightrec_emit_end_of_block(state, block, offset, rs, 0, 31, 0, true);
 128 }
 129
 130 static void rec_special_JALR(struct lightrec_cstate *state, const struct block *block, u16 offset)
 131 {
 132         u8 rs = get_jr_jalr_reg(state, block, offset);
 133         union code c = block->opcode_list[offset].c;
 134
 135         _jit_name(block->_jit, __func__);
 136         lightrec_emit_end_of_block(state, block, offset, rs, 0, c.r.rd,
 137                                    get_branch_pc(block, offset, 2), true);
 138 }
 139
 140 static void rec_J(struct lightrec_cstate *state, const struct block *block, u16 offset)
 141 {
 142         union code c = block->opcode_list[offset].c;
 143
 144         _jit_name(block->_jit, __func__);
 145         lightrec_emit_end_of_block(state, block, offset, -1,
 146                                    (block->pc & 0xf0000000) | (c.j.imm << 2),
 147                                    31, 0, true);
 148 }
 149
 150 static void rec_JAL(struct lightrec_cstate *state, const struct block *block, u16 offset)
 151 {
 152         union code c = block->opcode_list[offset].c;
 153
 154         _jit_name(block->_jit, __func__);
 155         lightrec_emit_end_of_block(state, block, offset, -1,
 156                                    (block->pc & 0xf0000000) | (c.j.imm << 2),
 157                                    31, get_branch_pc(block, offset, 2), true);
 158 }
 159
 160 static void lightrec_do_early_unload(struct lightrec_cstate *state,
 161                                      const struct block *block, u16 offset)
 162 {
 163         struct regcache *reg_cache = state->reg_cache;
 164         const struct opcode *op = &block->opcode_list[offset];
 165         jit_state_t *_jit = block->_jit;
 166         unsigned int i;
 167         u8 reg;
 168         struct {
 169                 u8 reg, op;
 170         } reg_ops[3] = {
 171                 { op->r.rd, LIGHTREC_FLAGS_GET_RD(op->flags), },
 172                 { op->i.rt, LIGHTREC_FLAGS_GET_RT(op->flags), },
 173                 { op->i.rs, LIGHTREC_FLAGS_GET_RS(op->flags), },
 174         };
 175
 176         for (i = 0; i < ARRAY_SIZE(reg_ops); i++) {
 177                 reg = reg_ops[i].reg;
 178
 179                 switch (reg_ops[i].op) {
 180                 case LIGHTREC_REG_UNLOAD:
 181                         lightrec_clean_reg_if_loaded(reg_cache, _jit, reg, true);
 182                         break;
 183
 184                 case LIGHTREC_REG_DISCARD:
 185                         lightrec_discard_reg_if_loaded(reg_cache, reg);
 186                         break;
 187
 188                 case LIGHTREC_REG_CLEAN:
 189                         lightrec_clean_reg_if_loaded(reg_cache, _jit, reg, false);
 190                         break;
 191                 default:
 192                         break;
 193                 };
 194         }
 195 }
 196
 197 static void rec_b(struct lightrec_cstate *state, const struct block *block, u16 offset,
 198                   jit_code_t code, jit_code_t code2, u32 link, bool unconditional, bool bz)
 199 {
 200         struct regcache *reg_cache = state->reg_cache;
 201         struct native_register *regs_backup;
 202         jit_state_t *_jit = block->_jit;
 203         struct lightrec_branch *branch;
 204         const struct opcode *op = &block->opcode_list[offset],
 205                             *next = &block->opcode_list[offset + 1];
 206         jit_node_t *addr;
 207         u8 link_reg, rs, rt;
 208         bool is_forward = (s16)op->i.imm >= -1;
 209         int op_cycles = lightrec_cycles_of_opcode(op->c);
 210         u32 target_offset, cycles = state->cycles + op_cycles;
 211         bool no_indirection = false;
 212         u32 next_pc;
 213
 214         jit_note(__FILE__, __LINE__);
 215
 216         if (!op_flag_no_ds(op->flags))
 217                 cycles += lightrec_cycles_of_opcode(next->c);
 218
 219         state->cycles = -op_cycles;
 220
 221         if (!unconditional) {
 222                 rs = lightrec_alloc_reg_in(reg_cache, _jit, op->i.rs, REG_EXT);
 223                 rt = bz ? 0 : lightrec_alloc_reg_in(reg_cache,
 224                                                     _jit, op->i.rt, REG_EXT);
 225
 226                 /* Unload dead registers before evaluating the branch */
 227                 if (OPT_EARLY_UNLOAD)
 228                         lightrec_do_early_unload(state, block, offset);
 229
 230                 if (op_flag_local_branch(op->flags) &&
 231                     (op_flag_no_ds(op->flags) || !next->opcode) &&
 232                     is_forward && !lightrec_has_dirty_regs(reg_cache))
 233                         no_indirection = true;
 234
 235                 if (no_indirection)
 236                         pr_debug("Using no indirection for branch at offset 0x%hx\n", offset << 2);
 237         }
 238
 239         if (cycles)
 240                 jit_subi(LIGHTREC_REG_CYCLE, LIGHTREC_REG_CYCLE, cycles);
 241
 242         if (!unconditional) {
 243                 /* Generate the branch opcode */
 244                 if (!no_indirection)
 245                         addr = jit_new_node_pww(code, NULL, rs, rt);
 246
 247                 lightrec_free_regs(reg_cache);
 248                 regs_backup = lightrec_regcache_enter_branch(reg_cache);
 249         }
 250
 251         if (op_flag_local_branch(op->flags)) {
 252                 /* Recompile the delay slot */
 253                 if (next && next->opcode && !op_flag_no_ds(op->flags))
 254                         lightrec_rec_opcode(state, block, offset + 1);
 255
 256                 if (link) {
 257                         /* Update the $ra register */
 258                         link_reg = lightrec_alloc_reg_out(reg_cache, _jit, 31, 0);
 259                         jit_movi(link_reg, link);
 260                         lightrec_free_reg(reg_cache, link_reg);
 261                 }
 262
 263                 /* Clean remaining registers */
 264                 lightrec_clean_regs(reg_cache, _jit);
 265
 266                 target_offset = offset + 1 + (s16)op->i.imm
 267                         - !!op_flag_no_ds(op->flags);
 268                 pr_debug("Adding local branch to offset 0x%x\n",
 269                          target_offset << 2);
 270                 branch = &state->local_branches[
 271                         state->nb_local_branches++];
 272
 273                 branch->target = target_offset;
 274
 275                 if (no_indirection)
 276                         branch->branch = jit_new_node_pww(code2, NULL, rs, rt);
 277                 else if (is_forward)
 278                         branch->branch = jit_b();
 279                 else
 280                         branch->branch = jit_bgti(LIGHTREC_REG_CYCLE, 0);
 281         }
 282
 283         if (!op_flag_local_branch(op->flags) || !is_forward) {
 284                 next_pc = get_branch_pc(block, offset, 1 + (s16)op->i.imm);
 285                 lightrec_emit_end_of_block(state, block, offset, -1, next_pc,
 286                                            31, link, false);
 287         }
 288
 289         if (!unconditional) {
 290                 if (!no_indirection)
 291                         jit_patch(addr);
 292
 293                 lightrec_regcache_leave_branch(reg_cache, regs_backup);
 294
 295                 if (bz && link) {
 296                         /* Update the $ra register */
 297                         link_reg = lightrec_alloc_reg_out(reg_cache, _jit,
 298                                                           31, REG_EXT);
 299                         jit_movi(link_reg, (s32)link);
 300                         lightrec_free_reg(reg_cache, link_reg);
 301                 }
 302
 303                 if (!op_flag_no_ds(op->flags) && next->opcode)
 304                         lightrec_rec_opcode(state, block, offset + 1);
 305         }
 306 }
 307
 308 static void rec_BNE(struct lightrec_cstate *state,
 309                     const struct block *block, u16 offset)
 310 {
 311         union code c = block->opcode_list[offset].c;
 312
 313         _jit_name(block->_jit, __func__);
 314
 315         if (c.i.rt == 0)
 316                 rec_b(state, block, offset, jit_code_beqi, jit_code_bnei, 0, false, true);
 317         else
 318                 rec_b(state, block, offset, jit_code_beqr, jit_code_bner, 0, false, false);
 319 }
 320
 321 static void rec_BEQ(struct lightrec_cstate *state,
 322                     const struct block *block, u16 offset)
 323 {
 324         union code c = block->opcode_list[offset].c;
 325
 326         _jit_name(block->_jit, __func__);
 327
 328         if (c.i.rt == 0)
 329                 rec_b(state, block, offset, jit_code_bnei, jit_code_beqi, 0, c.i.rs == 0, true);
 330         else
 331                 rec_b(state, block, offset, jit_code_bner, jit_code_beqr, 0, c.i.rs == c.i.rt, false);
 332 }
 333
 334 static void rec_BLEZ(struct lightrec_cstate *state,
 335                      const struct block *block, u16 offset)
 336 {
 337         union code c = block->opcode_list[offset].c;
 338
 339         _jit_name(block->_jit, __func__);
 340         rec_b(state, block, offset, jit_code_bgti, jit_code_blei, 0, c.i.rs == 0, true);
 341 }
 342
 343 static void rec_BGTZ(struct lightrec_cstate *state,
 344                      const struct block *block, u16 offset)
 345 {
 346         _jit_name(block->_jit, __func__);
 347         rec_b(state, block, offset, jit_code_blei, jit_code_bgti, 0, false, true);
 348 }
 349
 350 static void rec_regimm_BLTZ(struct lightrec_cstate *state,
 351                             const struct block *block, u16 offset)
 352 {
 353         _jit_name(block->_jit, __func__);
 354         rec_b(state, block, offset, jit_code_bgei, jit_code_blti, 0, false, true);
 355 }
 356
 357 static void rec_regimm_BLTZAL(struct lightrec_cstate *state,
 358                               const struct block *block, u16 offset)
 359 {
 360         _jit_name(block->_jit, __func__);
 361         rec_b(state, block, offset, jit_code_bgei, jit_code_blti,
 362               get_branch_pc(block, offset, 2), false, true);
 363 }
 364
 365 static void rec_regimm_BGEZ(struct lightrec_cstate *state,
 366                             const struct block *block, u16 offset)
 367 {
 368         union code c = block->opcode_list[offset].c;
 369
 370         _jit_name(block->_jit, __func__);
 371         rec_b(state, block, offset, jit_code_blti, jit_code_bgei, 0, !c.i.rs, true);
 372 }
 373
 374 static void rec_regimm_BGEZAL(struct lightrec_cstate *state,
 375                               const struct block *block, u16 offset)
 376 {
 377         const struct opcode *op = &block->opcode_list[offset];
 378         _jit_name(block->_jit, __func__);
 379         rec_b(state, block, offset, jit_code_blti, jit_code_bgei,
 380               get_branch_pc(block, offset, 2),
 381               !op->i.rs, true);
 382 }
 383
 384 static void rec_alu_imm(struct lightrec_cstate *state, const struct block *block,
 385                         u16 offset, jit_code_t code, bool slti)
 386 {
 387         struct regcache *reg_cache = state->reg_cache;
 388         union code c = block->opcode_list[offset].c;
 389         jit_state_t *_jit = block->_jit;
 390         u8 rs, rt, out_flags = REG_EXT;
 391
 392         if (slti)
 393                 out_flags |= REG_ZEXT;
 394
 395         jit_note(__FILE__, __LINE__);
 396         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rs, REG_EXT);
 397         rt = lightrec_alloc_reg_out(reg_cache, _jit, c.i.rt, out_flags);
 398
 399         jit_new_node_www(code, rt, rs, (s32)(s16) c.i.imm);
 400
 401         lightrec_free_reg(reg_cache, rs);
 402         lightrec_free_reg(reg_cache, rt);
 403 }
 404
 405 static void rec_alu_special(struct lightrec_cstate *state, const struct block *block,
 406                             u16 offset, jit_code_t code, bool out_ext)
 407 {
 408         struct regcache *reg_cache = state->reg_cache;
 409         union code c = block->opcode_list[offset].c;
 410         jit_state_t *_jit = block->_jit;
 411         u8 rd, rt, rs;
 412
 413         jit_note(__FILE__, __LINE__);
 414         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rs, REG_EXT);
 415         rt = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rt, REG_EXT);
 416         rd = lightrec_alloc_reg_out(reg_cache, _jit, c.r.rd,
 417                                     out_ext ? REG_EXT | REG_ZEXT : 0);
 418
 419         jit_new_node_www(code, rd, rs, rt);
 420
 421         lightrec_free_reg(reg_cache, rs);
 422         lightrec_free_reg(reg_cache, rt);
 423         lightrec_free_reg(reg_cache, rd);
 424 }
 425
 426 static void rec_alu_shiftv(struct lightrec_cstate *state, const struct block *block,
 427                            u16 offset, jit_code_t code)
 428 {
 429         struct regcache *reg_cache = state->reg_cache;
 430         union code c = block->opcode_list[offset].c;
 431         jit_state_t *_jit = block->_jit;
 432         u8 rd, rt, rs, temp, flags = 0;
 433
 434         jit_note(__FILE__, __LINE__);
 435         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rs, 0);
 436
 437         if (code == jit_code_rshr)
 438                 flags = REG_EXT;
 439         else if (code == jit_code_rshr_u)
 440                 flags = REG_ZEXT;
 441
 442         rt = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rt, flags);
 443         rd = lightrec_alloc_reg_out(reg_cache, _jit, c.r.rd, flags);
 444
 445         if (rs != rd && rt != rd) {
 446                 jit_andi(rd, rs, 0x1f);
 447                 jit_new_node_www(code, rd, rt, rd);
 448         } else {
 449                 temp = lightrec_alloc_reg_temp(reg_cache, _jit);
 450                 jit_andi(temp, rs, 0x1f);
 451                 jit_new_node_www(code, rd, rt, temp);
 452                 lightrec_free_reg(reg_cache, temp);
 453         }
 454
 455         lightrec_free_reg(reg_cache, rs);
 456         lightrec_free_reg(reg_cache, rt);
 457         lightrec_free_reg(reg_cache, rd);
 458 }
 459
 460 static void rec_movi(struct lightrec_cstate *state,
 461                      const struct block *block, u16 offset)
 462 {
 463         struct regcache *reg_cache = state->reg_cache;
 464         union code c = block->opcode_list[offset].c;
 465         jit_state_t *_jit = block->_jit;
 466         u16 flags = REG_EXT;
 467         u8 rt;
 468
 469         if (!(c.i.imm & 0x8000))
 470                 flags |= REG_ZEXT;
 471
 472         rt = lightrec_alloc_reg_out(reg_cache, _jit, c.i.rt, flags);
 473
 474         jit_movi(rt, (s32)(s16) c.i.imm);
 475
 476         lightrec_free_reg(reg_cache, rt);
 477 }
 478
 479 static void rec_ADDIU(struct lightrec_cstate *state,
 480                       const struct block *block, u16 offset)
 481 {
 482         _jit_name(block->_jit, __func__);
 483
 484         if (block->opcode_list[offset].c.i.rs)
 485                 rec_alu_imm(state, block, offset, jit_code_addi, false);
 486         else
 487                 rec_movi(state, block, offset);
 488 }
 489
 490 static void rec_ADDI(struct lightrec_cstate *state,
 491                      const struct block *block, u16 offset)
 492 {
 493         /* TODO: Handle the exception? */
 494         _jit_name(block->_jit, __func__);
 495         rec_ADDIU(state, block, offset);
 496 }
 497
 498 static void rec_SLTIU(struct lightrec_cstate *state,
 499                       const struct block *block, u16 offset)
 500 {
 501         _jit_name(block->_jit, __func__);
 502         rec_alu_imm(state, block, offset, jit_code_lti_u, true);
 503 }
 504
 505 static void rec_SLTI(struct lightrec_cstate *state,
 506                      const struct block *block, u16 offset)
 507 {
 508         _jit_name(block->_jit, __func__);
 509         rec_alu_imm(state, block, offset, jit_code_lti, true);
 510 }
 511
 512 static void rec_ANDI(struct lightrec_cstate *state,
 513                      const struct block *block, u16 offset)
 514 {
 515         struct regcache *reg_cache = state->reg_cache;
 516         union code c = block->opcode_list[offset].c;
 517         jit_state_t *_jit = block->_jit;
 518         u8 rs, rt;
 519
 520         _jit_name(block->_jit, __func__);
 521         jit_note(__FILE__, __LINE__);
 522         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rs, 0);
 523         rt = lightrec_alloc_reg_out(reg_cache, _jit, c.i.rt,
 524                                     REG_EXT | REG_ZEXT);
 525
 526         /* PSX code uses ANDI 0xff / ANDI 0xffff a lot, which are basically
 527          * casts to uint8_t / uint16_t. */
 528         if (c.i.imm == 0xff)
 529                 jit_extr_uc(rt, rs);
 530         else if (c.i.imm == 0xffff)
 531                 jit_extr_us(rt, rs);
 532         else
 533                 jit_andi(rt, rs, (u32)(u16) c.i.imm);
 534
 535         lightrec_free_reg(reg_cache, rs);
 536         lightrec_free_reg(reg_cache, rt);
 537 }
 538
 539 static void rec_alu_or_xor(struct lightrec_cstate *state, const struct block *block,
 540                            u16 offset, jit_code_t code)
 541 {
 542         struct regcache *reg_cache = state->reg_cache;
 543         union code c = block->opcode_list[offset].c;
 544         jit_state_t *_jit = block->_jit;
 545         u8 rs, rt, flags;
 546
 547         jit_note(__FILE__, __LINE__);
 548         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rs, 0);
 549         rt = lightrec_alloc_reg_out(reg_cache, _jit, c.i.rt, 0);
 550
 551         flags = lightrec_get_reg_in_flags(reg_cache, rs);
 552         lightrec_set_reg_out_flags(reg_cache, rt, flags);
 553
 554         jit_new_node_www(code, rt, rs, (u32)(u16) c.i.imm);
 555
 556         lightrec_free_reg(reg_cache, rs);
 557         lightrec_free_reg(reg_cache, rt);
 558 }
 559
 560
 561 static void rec_ORI(struct lightrec_cstate *state,
 562                     const struct block *block, u16 offset)
 563 {
 564         _jit_name(block->_jit, __func__);
 565         rec_alu_or_xor(state, block, offset, jit_code_ori);
 566 }
 567
 568 static void rec_XORI(struct lightrec_cstate *state,
 569                      const struct block *block, u16 offset)
 570 {
 571         _jit_name(block->_jit, __func__);
 572         rec_alu_or_xor(state, block, offset, jit_code_xori);
 573 }
 574
 575 static void rec_LUI(struct lightrec_cstate *state,
 576                     const struct block *block, u16 offset)
 577 {
 578         struct regcache *reg_cache = state->reg_cache;
 579         union code c = block->opcode_list[offset].c;
 580         jit_state_t *_jit = block->_jit;
 581         u8 rt, flags = REG_EXT;
 582
 583         jit_name(__func__);
 584         jit_note(__FILE__, __LINE__);
 585
 586         if (!(c.i.imm & BIT(15)))
 587                 flags |= REG_ZEXT;
 588
 589         rt = lightrec_alloc_reg_out(reg_cache, _jit, c.i.rt, flags);
 590
 591         jit_movi(rt, (s32)(c.i.imm << 16));
 592
 593         lightrec_free_reg(reg_cache, rt);
 594 }
 595
 596 static void rec_special_ADDU(struct lightrec_cstate *state,
 597                              const struct block *block, u16 offset)
 598 {
 599         _jit_name(block->_jit, __func__);
 600         rec_alu_special(state, block, offset, jit_code_addr, false);
 601 }
 602
 603 static void rec_special_ADD(struct lightrec_cstate *state,
 604                             const struct block *block, u16 offset)
 605 {
 606         /* TODO: Handle the exception? */
 607         _jit_name(block->_jit, __func__);
 608         rec_alu_special(state, block, offset, jit_code_addr, false);
 609 }
 610
 611 static void rec_special_SUBU(struct lightrec_cstate *state,
 612                              const struct block *block, u16 offset)
 613 {
 614         _jit_name(block->_jit, __func__);
 615         rec_alu_special(state, block, offset, jit_code_subr, false);
 616 }
 617
 618 static void rec_special_SUB(struct lightrec_cstate *state,
 619                             const struct block *block, u16 offset)
 620 {
 621         /* TODO: Handle the exception? */
 622         _jit_name(block->_jit, __func__);
 623         rec_alu_special(state, block, offset, jit_code_subr, false);
 624 }
 625
 626 static void rec_special_AND(struct lightrec_cstate *state,
 627                             const struct block *block, u16 offset)
 628 {
 629         struct regcache *reg_cache = state->reg_cache;
 630         union code c = block->opcode_list[offset].c;
 631         jit_state_t *_jit = block->_jit;
 632         u8 rd, rt, rs, flags_rs, flags_rt, flags_rd;
 633
 634         _jit_name(block->_jit, __func__);
 635         jit_note(__FILE__, __LINE__);
 636         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rs, 0);
 637         rt = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rt, 0);
 638         rd = lightrec_alloc_reg_out(reg_cache, _jit, c.r.rd, 0);
 639
 640         flags_rs = lightrec_get_reg_in_flags(reg_cache, rs);
 641         flags_rt = lightrec_get_reg_in_flags(reg_cache, rt);
 642
 643         /* Z(rd) = Z(rs) | Z(rt) */
 644         flags_rd = REG_ZEXT & (flags_rs | flags_rt);
 645
 646         /* E(rd) = (E(rt) & Z(rt)) | (E(rs) & Z(rs)) | (E(rs) & E(rt)) */
 647         if (((flags_rs & REG_EXT) && (flags_rt & REG_ZEXT)) ||
 648             ((flags_rt & REG_EXT) && (flags_rs & REG_ZEXT)) ||
 649             (REG_EXT & flags_rs & flags_rt))
 650                 flags_rd |= REG_EXT;
 651
 652         lightrec_set_reg_out_flags(reg_cache, rd, flags_rd);
 653
 654         jit_andr(rd, rs, rt);
 655
 656         lightrec_free_reg(reg_cache, rs);
 657         lightrec_free_reg(reg_cache, rt);
 658         lightrec_free_reg(reg_cache, rd);
 659 }
 660
 661 static void rec_special_or_nor(struct lightrec_cstate *state,
 662                                const struct block *block, u16 offset, bool nor)
 663 {
 664         struct regcache *reg_cache = state->reg_cache;
 665         union code c = block->opcode_list[offset].c;
 666         jit_state_t *_jit = block->_jit;
 667         u8 rd, rt, rs, flags_rs, flags_rt, flags_rd = 0;
 668
 669         jit_note(__FILE__, __LINE__);
 670         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rs, 0);
 671         rt = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rt, 0);
 672         rd = lightrec_alloc_reg_out(reg_cache, _jit, c.r.rd, 0);
 673
 674         flags_rs = lightrec_get_reg_in_flags(reg_cache, rs);
 675         flags_rt = lightrec_get_reg_in_flags(reg_cache, rt);
 676
 677         /* or: Z(rd) = Z(rs) & Z(rt)
 678          * nor: Z(rd) = 0 */
 679         if (!nor)
 680                 flags_rd = REG_ZEXT & flags_rs & flags_rt;
 681
 682         /* E(rd) = (E(rs) & E(rt)) | (E(rt) & !Z(rt)) | (E(rs) & !Z(rs)) */
 683         if ((REG_EXT & flags_rs & flags_rt) ||
 684             ((flags_rt & (REG_EXT | REG_ZEXT)) == REG_EXT) ||
 685             ((flags_rs & (REG_EXT | REG_ZEXT)) == REG_EXT))
 686                 flags_rd |= REG_EXT;
 687
 688         lightrec_set_reg_out_flags(reg_cache, rd, flags_rd);
 689
 690         jit_orr(rd, rs, rt);
 691
 692         if (nor)
 693                 jit_comr(rd, rd);
 694
 695         lightrec_free_reg(reg_cache, rs);
 696         lightrec_free_reg(reg_cache, rt);
 697         lightrec_free_reg(reg_cache, rd);
 698 }
 699
 700 static void rec_special_OR(struct lightrec_cstate *state,
 701                            const struct block *block, u16 offset)
 702 {
 703         _jit_name(block->_jit, __func__);
 704         rec_special_or_nor(state, block, offset, false);
 705 }
 706
 707 static void rec_special_NOR(struct lightrec_cstate *state,
 708                             const struct block *block, u16 offset)
 709 {
 710         _jit_name(block->_jit, __func__);
 711         rec_special_or_nor(state, block, offset, true);
 712 }
 713
 714 static void rec_special_XOR(struct lightrec_cstate *state,
 715                             const struct block *block, u16 offset)
 716 {
 717         struct regcache *reg_cache = state->reg_cache;
 718         union code c = block->opcode_list[offset].c;
 719         jit_state_t *_jit = block->_jit;
 720         u8 rd, rt, rs, flags_rs, flags_rt, flags_rd;
 721
 722         _jit_name(block->_jit, __func__);
 723
 724         jit_note(__FILE__, __LINE__);
 725         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rs, 0);
 726         rt = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rt, 0);
 727         rd = lightrec_alloc_reg_out(reg_cache, _jit, c.r.rd, 0);
 728
 729         flags_rs = lightrec_get_reg_in_flags(reg_cache, rs);
 730         flags_rt = lightrec_get_reg_in_flags(reg_cache, rt);
 731
 732         /* Z(rd) = Z(rs) & Z(rt) */
 733         flags_rd = REG_ZEXT & flags_rs & flags_rt;
 734
 735         /* E(rd) = E(rs) & E(rt) */
 736         flags_rd |= REG_EXT & flags_rs & flags_rt;
 737
 738         lightrec_set_reg_out_flags(reg_cache, rd, flags_rd);
 739
 740         jit_xorr(rd, rs, rt);
 741
 742         lightrec_free_reg(reg_cache, rs);
 743         lightrec_free_reg(reg_cache, rt);
 744         lightrec_free_reg(reg_cache, rd);
 745 }
 746
 747 static void rec_special_SLTU(struct lightrec_cstate *state,
 748                              const struct block *block, u16 offset)
 749 {
 750         _jit_name(block->_jit, __func__);
 751         rec_alu_special(state, block, offset, jit_code_ltr_u, true);
 752 }
 753
 754 static void rec_special_SLT(struct lightrec_cstate *state,
 755                             const struct block *block, u16 offset)
 756 {
 757         _jit_name(block->_jit, __func__);
 758         rec_alu_special(state, block, offset, jit_code_ltr, true);
 759 }
 760
 761 static void rec_special_SLLV(struct lightrec_cstate *state,
 762                              const struct block *block, u16 offset)
 763 {
 764         _jit_name(block->_jit, __func__);
 765         rec_alu_shiftv(state, block, offset, jit_code_lshr);
 766 }
 767
 768 static void rec_special_SRLV(struct lightrec_cstate *state,
 769                              const struct block *block, u16 offset)
 770 {
 771         _jit_name(block->_jit, __func__);
 772         rec_alu_shiftv(state, block, offset, jit_code_rshr_u);
 773 }
 774
 775 static void rec_special_SRAV(struct lightrec_cstate *state,
 776                              const struct block *block, u16 offset)
 777 {
 778         _jit_name(block->_jit, __func__);
 779         rec_alu_shiftv(state, block, offset, jit_code_rshr);
 780 }
 781
 782 static void rec_alu_shift(struct lightrec_cstate *state, const struct block *block,
 783                           u16 offset, jit_code_t code)
 784 {
 785         struct regcache *reg_cache = state->reg_cache;
 786         union code c = block->opcode_list[offset].c;
 787         jit_state_t *_jit = block->_jit;
 788         u8 rd, rt, flags = 0;
 789
 790         jit_note(__FILE__, __LINE__);
 791
 792         if (code == jit_code_rshi)
 793                 flags = REG_EXT;
 794         else if (code == jit_code_rshi_u)
 795                 flags = REG_ZEXT;
 796
 797         rt = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rt, flags);
 798
 799         /* Input reg is zero-extended, if we SRL at least by one bit, we know
 800          * the output reg will be both zero-extended and sign-extended. */
 801         if (code == jit_code_rshi_u && c.r.imm)
 802                 flags |= REG_EXT;
 803         rd = lightrec_alloc_reg_out(reg_cache, _jit, c.r.rd, flags);
 804
 805         jit_new_node_www(code, rd, rt, c.r.imm);
 806
 807         lightrec_free_reg(reg_cache, rt);
 808         lightrec_free_reg(reg_cache, rd);
 809 }
 810
 811 static void rec_special_SLL(struct lightrec_cstate *state,
 812                             const struct block *block, u16 offset)
 813 {
 814         _jit_name(block->_jit, __func__);
 815         rec_alu_shift(state, block, offset, jit_code_lshi);
 816 }
 817
 818 static void rec_special_SRL(struct lightrec_cstate *state,
 819                             const struct block *block, u16 offset)
 820 {
 821         _jit_name(block->_jit, __func__);
 822         rec_alu_shift(state, block, offset, jit_code_rshi_u);
 823 }
 824
 825 static void rec_special_SRA(struct lightrec_cstate *state,
 826                             const struct block *block, u16 offset)
 827 {
 828         _jit_name(block->_jit, __func__);
 829         rec_alu_shift(state, block, offset, jit_code_rshi);
 830 }
 831
 832 static void rec_alu_mult(struct lightrec_cstate *state,
 833                          const struct block *block, u16 offset, bool is_signed)
 834 {
 835         struct regcache *reg_cache = state->reg_cache;
 836         union code c = block->opcode_list[offset].c;
 837         u32 flags = block->opcode_list[offset].flags;
 838         u8 reg_lo = get_mult_div_lo(c);
 839         u8 reg_hi = get_mult_div_hi(c);
 840         jit_state_t *_jit = block->_jit;
 841         u8 lo, hi, rs, rt, rflags = 0;
 842
 843         jit_note(__FILE__, __LINE__);
 844
 845         if (is_signed)
 846                 rflags = REG_EXT;
 847         else
 848                 rflags = REG_ZEXT;
 849
 850         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rs, rflags);
 851         rt = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rt, rflags);
 852
 853         if (!op_flag_no_lo(flags))
 854                 lo = lightrec_alloc_reg_out(reg_cache, _jit, reg_lo, 0);
 855         else if (__WORDSIZE == 32)
 856                 lo = lightrec_alloc_reg_temp(reg_cache, _jit);
 857
 858         if (!op_flag_no_hi(flags))
 859                 hi = lightrec_alloc_reg_out(reg_cache, _jit, reg_hi, REG_EXT);
 860
 861         if (__WORDSIZE == 32) {
 862                 /* On 32-bit systems, do a 32*32->64 bit operation, or a 32*32->32 bit
 863                  * operation if the MULT was detected a 32-bit only. */
 864                 if (!op_flag_no_hi(flags)) {
 865                         if (is_signed)
 866                                 jit_qmulr(lo, hi, rs, rt);
 867                         else
 868                                 jit_qmulr_u(lo, hi, rs, rt);
 869                 } else {
 870                         jit_mulr(lo, rs, rt);
 871                 }
 872         } else {
 873                 /* On 64-bit systems, do a 64*64->64 bit operation. */
 874                 if (op_flag_no_lo(flags)) {
 875                         jit_mulr(hi, rs, rt);
 876                         jit_rshi(hi, hi, 32);
 877                 } else {
 878                         jit_mulr(lo, rs, rt);
 879
 880                         /* The 64-bit output value is in $lo, store the upper 32 bits in $hi */
 881                         if (!op_flag_no_hi(flags))
 882                                 jit_rshi(hi, lo, 32);
 883                 }
 884         }
 885
 886         lightrec_free_reg(reg_cache, rs);
 887         lightrec_free_reg(reg_cache, rt);
 888         if (!op_flag_no_lo(flags) || __WORDSIZE == 32)
 889                 lightrec_free_reg(reg_cache, lo);
 890         if (!op_flag_no_hi(flags))
 891                 lightrec_free_reg(reg_cache, hi);
 892 }
 893
 894 static void rec_alu_div(struct lightrec_cstate *state,
 895                         const struct block *block, u16 offset, bool is_signed)
 896 {
 897         struct regcache *reg_cache = state->reg_cache;
 898         union code c = block->opcode_list[offset].c;
 899         u32 flags = block->opcode_list[offset].flags;
 900         bool no_check = op_flag_no_div_check(flags);
 901         u8 reg_lo = get_mult_div_lo(c);
 902         u8 reg_hi = get_mult_div_hi(c);
 903         jit_state_t *_jit = block->_jit;
 904         jit_node_t *branch, *to_end;
 905         u8 lo = 0, hi = 0, rs, rt, rflags = 0;
 906
 907         jit_note(__FILE__, __LINE__);
 908
 909         if (is_signed)
 910                 rflags = REG_EXT;
 911         else
 912                 rflags = REG_ZEXT;
 913
 914         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rs, rflags);
 915         rt = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rt, rflags);
 916
 917         if (!op_flag_no_lo(flags))
 918                 lo = lightrec_alloc_reg_out(reg_cache, _jit, reg_lo, 0);
 919
 920         if (!op_flag_no_hi(flags))
 921                 hi = lightrec_alloc_reg_out(reg_cache, _jit, reg_hi, 0);
 922
 923         /* Jump to special handler if dividing by zero  */
 924         if (!no_check)
 925                 branch = jit_beqi(rt, 0);
 926
 927         if (op_flag_no_lo(flags)) {
 928                 if (is_signed)
 929                         jit_remr(hi, rs, rt);
 930                 else
 931                         jit_remr_u(hi, rs, rt);
 932         } else if (op_flag_no_hi(flags)) {
 933                 if (is_signed)
 934                         jit_divr(lo, rs, rt);
 935                 else
 936                         jit_divr_u(lo, rs, rt);
 937         } else {
 938                 if (is_signed)
 939                         jit_qdivr(lo, hi, rs, rt);
 940                 else
 941                         jit_qdivr_u(lo, hi, rs, rt);
 942         }
 943
 944         if (!no_check) {
 945                 /* Jump above the div-by-zero handler */
 946                 to_end = jit_b();
 947
 948                 jit_patch(branch);
 949
 950                 if (!op_flag_no_lo(flags)) {
 951                         if (is_signed) {
 952                                 jit_lti(lo, rs, 0);
 953                                 jit_lshi(lo, lo, 1);
 954                                 jit_subi(lo, lo, 1);
 955                         } else {
 956                                 jit_movi(lo, 0xffffffff);
 957                         }
 958                 }
 959
 960                 if (!op_flag_no_hi(flags))
 961                         jit_movr(hi, rs);
 962
 963                 jit_patch(to_end);
 964         }
 965
 966         lightrec_free_reg(reg_cache, rs);
 967         lightrec_free_reg(reg_cache, rt);
 968
 969         if (!op_flag_no_lo(flags))
 970                 lightrec_free_reg(reg_cache, lo);
 971
 972         if (!op_flag_no_hi(flags))
 973                 lightrec_free_reg(reg_cache, hi);
 974 }
 975
 976 static void rec_special_MULT(struct lightrec_cstate *state,
 977                              const struct block *block, u16 offset)
 978 {
 979         _jit_name(block->_jit, __func__);
 980         rec_alu_mult(state, block, offset, true);
 981 }
 982
 983 static void rec_special_MULTU(struct lightrec_cstate *state,
 984                               const struct block *block, u16 offset)
 985 {
 986         _jit_name(block->_jit, __func__);
 987         rec_alu_mult(state, block, offset, false);
 988 }
 989
 990 static void rec_special_DIV(struct lightrec_cstate *state,
 991                             const struct block *block, u16 offset)
 992 {
 993         _jit_name(block->_jit, __func__);
 994         rec_alu_div(state, block, offset, true);
 995 }
 996
 997 static void rec_special_DIVU(struct lightrec_cstate *state,
 998                              const struct block *block, u16 offset)
 999 {
1000         _jit_name(block->_jit, __func__);
1001         rec_alu_div(state, block, offset, false);
1002 }
1003
1004 static void rec_alu_mv_lo_hi(struct lightrec_cstate *state,
1005                              const struct block *block, u8 dst, u8 src)
1006 {
1007         struct regcache *reg_cache = state->reg_cache;
1008         jit_state_t *_jit = block->_jit;
1009
1010         jit_note(__FILE__, __LINE__);
1011         src = lightrec_alloc_reg_in(reg_cache, _jit, src, 0);
1012         dst = lightrec_alloc_reg_out(reg_cache, _jit, dst, REG_EXT);
1013
1014         jit_extr_i(dst, src);
1015
1016         lightrec_free_reg(reg_cache, src);
1017         lightrec_free_reg(reg_cache, dst);
1018 }
1019
1020 static void rec_special_MFHI(struct lightrec_cstate *state,
1021                              const struct block *block, u16 offset)
1022 {
1023         union code c = block->opcode_list[offset].c;
1024
1025         _jit_name(block->_jit, __func__);
1026         rec_alu_mv_lo_hi(state, block, c.r.rd, REG_HI);
1027 }
1028
1029 static void rec_special_MTHI(struct lightrec_cstate *state,
1030                              const struct block *block, u16 offset)
1031 {
1032         union code c = block->opcode_list[offset].c;
1033
1034         _jit_name(block->_jit, __func__);
1035         rec_alu_mv_lo_hi(state, block, REG_HI, c.r.rs);
1036 }
1037
1038 static void rec_special_MFLO(struct lightrec_cstate *state,
1039                              const struct block *block, u16 offset)
1040 {
1041         union code c = block->opcode_list[offset].c;
1042
1043         _jit_name(block->_jit, __func__);
1044         rec_alu_mv_lo_hi(state, block, c.r.rd, REG_LO);
1045 }
1046
1047 static void rec_special_MTLO(struct lightrec_cstate *state,
1048                              const struct block *block, u16 offset)
1049 {
1050         union code c = block->opcode_list[offset].c;
1051
1052         _jit_name(block->_jit, __func__);
1053         rec_alu_mv_lo_hi(state, block, REG_LO, c.r.rs);
1054 }
1055
1056 static void call_to_c_wrapper(struct lightrec_cstate *state,
1057                               const struct block *block, u32 arg,
1058                               enum c_wrappers wrapper)
1059 {
1060         struct regcache *reg_cache = state->reg_cache;
1061         jit_state_t *_jit = block->_jit;
1062         s8 tmp, tmp2;
1063
1064         /* Make sure JIT_R1 is not mapped; it will be used in the C wrapper. */
1065         tmp2 = lightrec_alloc_reg(reg_cache, _jit, JIT_R1);
1066
1067         tmp = lightrec_get_reg_with_value(reg_cache,
1068                                           (intptr_t) state->state->wrappers_eps[wrapper]);
1069         if (tmp < 0) {
1070                 tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
1071                 jit_ldxi(tmp, LIGHTREC_REG_STATE,
1072                          offsetof(struct lightrec_state, wrappers_eps[wrapper]));
1073
1074                 lightrec_temp_set_value(reg_cache, tmp,
1075                                         (intptr_t) state->state->wrappers_eps[wrapper]);
1076         }
1077
1078         lightrec_free_reg(reg_cache, tmp2);
1079
1080 #ifdef __mips__
1081         /* On MIPS, register t9 is always used as the target register for JALR.
1082          * Therefore if it does not contain the target address we must
1083          * invalidate it. */
1084         if (tmp != _T9)
1085                 lightrec_unload_reg(reg_cache, _jit, _T9);
1086 #endif
1087
1088         jit_prepare();
1089         jit_pushargi(arg);
1090
1091         lightrec_regcache_mark_live(reg_cache, _jit);
1092         jit_callr(tmp);
1093
1094         lightrec_free_reg(reg_cache, tmp);
1095         lightrec_regcache_mark_live(reg_cache, _jit);
1096 }
1097
1098 static void rec_io(struct lightrec_cstate *state,
1099                    const struct block *block, u16 offset,
1100                    bool load_rt, bool read_rt)
1101 {
1102         struct regcache *reg_cache = state->reg_cache;
1103         jit_state_t *_jit = block->_jit;
1104         union code c = block->opcode_list[offset].c;
1105         u32 flags = block->opcode_list[offset].flags;
1106         bool is_tagged = LIGHTREC_FLAGS_GET_IO_MODE(flags);
1107         u32 lut_entry;
1108
1109         jit_note(__FILE__, __LINE__);
1110
1111         lightrec_clean_reg_if_loaded(reg_cache, _jit, c.i.rs, false);
1112
1113         if (read_rt && likely(c.i.rt))
1114                 lightrec_clean_reg_if_loaded(reg_cache, _jit, c.i.rt, true);
1115         else if (load_rt)
1116                 lightrec_clean_reg_if_loaded(reg_cache, _jit, c.i.rt, false);
1117
1118         if (is_tagged) {
1119                 call_to_c_wrapper(state, block, c.opcode, C_WRAPPER_RW);
1120         } else {
1121                 lut_entry = lightrec_get_lut_entry(block);
1122                 call_to_c_wrapper(state, block, (lut_entry << 16) | offset,
1123                                   C_WRAPPER_RW_GENERIC);
1124         }
1125 }
1126
1127 static u32 rec_ram_mask(struct lightrec_state *state)
1128 {
1129         return (RAM_SIZE << (state->mirrors_mapped * 2)) - 1;
1130 }
1131
1132 static void rec_store_memory(struct lightrec_cstate *cstate,
1133                              const struct block *block,
1134                              u16 offset, jit_code_t code,
1135                              jit_code_t swap_code,
1136                              uintptr_t addr_offset, u32 addr_mask,
1137                              bool invalidate)
1138 {
1139         const struct lightrec_state *state = cstate->state;
1140         struct regcache *reg_cache = cstate->reg_cache;
1141         struct opcode *op = &block->opcode_list[offset];
1142         jit_state_t *_jit = block->_jit;
1143         union code c = op->c;
1144         u8 rs, rt, tmp, tmp2, tmp3, addr_reg, addr_reg2;
1145         s16 imm = (s16)c.i.imm;
1146         s32 simm = (s32)imm << (1 - lut_is_32bit(state));
1147         s32 lut_offt = offsetof(struct lightrec_state, code_lut);
1148         bool no_mask = op_flag_no_mask(op->flags);
1149         bool add_imm = c.i.imm &&
1150                 ((!state->mirrors_mapped && !no_mask) || (invalidate &&
1151                 ((imm & 0x3) || simm + lut_offt != (s16)(simm + lut_offt))));
1152         bool need_tmp = !no_mask || addr_offset || add_imm || invalidate;
1153
1154         rt = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rt, 0);
1155         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rs, 0);
1156         if (need_tmp)
1157                 tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
1158
1159         addr_reg = rs;
1160
1161         if (add_imm) {
1162                 jit_addi(tmp, addr_reg, (s16)c.i.imm);
1163                 addr_reg = tmp;
1164                 imm = 0;
1165         } else if (simm) {
1166                 lut_offt += simm;
1167         }
1168
1169         if (!no_mask) {
1170                 jit_andi(tmp, addr_reg, addr_mask);
1171                 addr_reg = tmp;
1172         }
1173
1174         if (addr_offset) {
1175                 tmp2 = lightrec_alloc_reg_temp(reg_cache, _jit);
1176                 jit_addi(tmp2, addr_reg, addr_offset);
1177                 addr_reg2 = tmp2;
1178         } else {
1179                 addr_reg2 = addr_reg;
1180         }
1181
1182         if (is_big_endian() && swap_code && c.i.rt) {
1183                 tmp3 = lightrec_alloc_reg_temp(reg_cache, _jit);
1184
1185                 jit_new_node_ww(swap_code, tmp3, rt);
1186                 jit_new_node_www(code, imm, addr_reg2, tmp3);
1187
1188                 lightrec_free_reg(reg_cache, tmp3);
1189         } else {
1190                 jit_new_node_www(code, imm, addr_reg2, rt);
1191         }
1192
1193         lightrec_free_reg(reg_cache, rt);
1194
1195         if (invalidate) {
1196                 tmp3 = lightrec_alloc_reg_in(reg_cache, _jit, 0, 0);
1197
1198                 if (c.i.op != OP_SW) {
1199                         jit_andi(tmp, addr_reg, ~3);
1200                         addr_reg = tmp;
1201                 }
1202
1203                 if (!lut_is_32bit(state)) {
1204                         jit_lshi(tmp, addr_reg, 1);
1205                         addr_reg = tmp;
1206                 }
1207
1208                 if (addr_reg == rs && c.i.rs == 0) {
1209                         addr_reg = LIGHTREC_REG_STATE;
1210                 } else {
1211                         jit_addr(tmp, addr_reg, LIGHTREC_REG_STATE);
1212                         addr_reg = tmp;
1213                 }
1214
1215                 if (lut_is_32bit(state))
1216                         jit_stxi_i(lut_offt, addr_reg, tmp3);
1217                 else
1218                         jit_stxi(lut_offt, addr_reg, tmp3);
1219
1220                 lightrec_free_reg(reg_cache, tmp3);
1221         }
1222
1223         if (addr_offset)
1224                 lightrec_free_reg(reg_cache, tmp2);
1225         if (need_tmp)
1226                 lightrec_free_reg(reg_cache, tmp);
1227         lightrec_free_reg(reg_cache, rs);
1228 }
1229
1230 static void rec_store_ram(struct lightrec_cstate *cstate,
1231                           const struct block *block,
1232                           u16 offset, jit_code_t code,
1233                           jit_code_t swap_code, bool invalidate)
1234 {
1235         struct lightrec_state *state = cstate->state;
1236
1237         _jit_note(block->_jit, __FILE__, __LINE__);
1238
1239         return rec_store_memory(cstate, block, offset, code, swap_code,
1240                                 state->offset_ram, rec_ram_mask(state),
1241                                 invalidate);
1242 }
1243
1244 static void rec_store_scratch(struct lightrec_cstate *cstate,
1245                               const struct block *block, u16 offset,
1246                               jit_code_t code, jit_code_t swap_code)
1247 {
1248         _jit_note(block->_jit, __FILE__, __LINE__);
1249
1250         return rec_store_memory(cstate, block, offset, code, swap_code,
1251                                 cstate->state->offset_scratch,
1252                                 0x1fffffff, false);
1253 }
1254
1255 static void rec_store_io(struct lightrec_cstate *cstate,
1256                          const struct block *block, u16 offset,
1257                          jit_code_t code, jit_code_t swap_code)
1258 {
1259         _jit_note(block->_jit, __FILE__, __LINE__);
1260
1261         return rec_store_memory(cstate, block, offset, code, swap_code,
1262                                 cstate->state->offset_io,
1263                                 0x1fffffff, false);
1264 }
1265
1266 static void rec_store_direct_no_invalidate(struct lightrec_cstate *cstate,
1267                                            const struct block *block,
1268                                            u16 offset, jit_code_t code,
1269                                            jit_code_t swap_code)
1270 {
1271         struct lightrec_state *state = cstate->state;
1272         struct regcache *reg_cache = cstate->reg_cache;
1273         union code c = block->opcode_list[offset].c;
1274         jit_state_t *_jit = block->_jit;
1275         jit_node_t *to_not_ram, *to_end;
1276         u8 tmp, tmp2, rs, rt;
1277         s16 imm;
1278
1279         jit_note(__FILE__, __LINE__);
1280         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rs, 0);
1281         tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
1282
1283         if (state->offset_ram || state->offset_scratch)
1284                 tmp2 = lightrec_alloc_reg_temp(reg_cache, _jit);
1285
1286         /* Convert to KUNSEG and avoid RAM mirrors */
1287         if (state->mirrors_mapped) {
1288                 imm = (s16)c.i.imm;
1289                 jit_andi(tmp, rs, 0x1f800000 | (4 * RAM_SIZE - 1));
1290         } else if (c.i.imm) {
1291                 imm = 0;
1292                 jit_addi(tmp, rs, (s16)c.i.imm);
1293                 jit_andi(tmp, tmp, 0x1f800000 | (RAM_SIZE - 1));
1294         } else {
1295                 imm = 0;
1296                 jit_andi(tmp, rs, 0x1f800000 | (RAM_SIZE - 1));
1297         }
1298
1299         lightrec_free_reg(reg_cache, rs);
1300
1301         if (state->offset_ram != state->offset_scratch) {
1302                 to_not_ram = jit_bmsi(tmp, BIT(28));
1303
1304                 jit_movi(tmp2, state->offset_ram);
1305
1306                 to_end = jit_b();
1307                 jit_patch(to_not_ram);
1308
1309                 jit_movi(tmp2, state->offset_scratch);
1310                 jit_patch(to_end);
1311         } else if (state->offset_ram) {
1312                 jit_movi(tmp2, state->offset_ram);
1313         }
1314
1315         if (state->offset_ram || state->offset_scratch) {
1316                 jit_addr(tmp, tmp, tmp2);
1317                 lightrec_free_reg(reg_cache, tmp2);
1318         }
1319
1320         rt = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rt, 0);
1321
1322         if (is_big_endian() && swap_code && c.i.rt) {
1323                 tmp2 = lightrec_alloc_reg_temp(reg_cache, _jit);
1324
1325                 jit_new_node_ww(swap_code, tmp2, rt);
1326                 jit_new_node_www(code, imm, tmp, tmp2);
1327
1328                 lightrec_free_reg(reg_cache, tmp2);
1329         } else {
1330                 jit_new_node_www(code, imm, tmp, rt);
1331         }
1332
1333         lightrec_free_reg(reg_cache, rt);
1334         lightrec_free_reg(reg_cache, tmp);
1335 }
1336
1337 static void rec_store_direct(struct lightrec_cstate *cstate, const struct block *block,
1338                              u16 offset, jit_code_t code, jit_code_t swap_code)
1339 {
1340         struct lightrec_state *state = cstate->state;
1341         u32 ram_size = state->mirrors_mapped ? RAM_SIZE * 4 : RAM_SIZE;
1342         struct regcache *reg_cache = cstate->reg_cache;
1343         union code c = block->opcode_list[offset].c;
1344         jit_state_t *_jit = block->_jit;
1345         jit_node_t *to_not_ram, *to_end;
1346         u8 tmp, tmp2, tmp3, rs, rt;
1347
1348         jit_note(__FILE__, __LINE__);
1349
1350         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rs, 0);
1351         tmp2 = lightrec_alloc_reg_temp(reg_cache, _jit);
1352         tmp3 = lightrec_alloc_reg_in(reg_cache, _jit, 0, 0);
1353
1354         /* Convert to KUNSEG and avoid RAM mirrors */
1355         if (c.i.imm) {
1356                 jit_addi(tmp2, rs, (s16)c.i.imm);
1357                 jit_andi(tmp2, tmp2, 0x1f800000 | (ram_size - 1));
1358         } else {
1359                 jit_andi(tmp2, rs, 0x1f800000 | (ram_size - 1));
1360         }
1361
1362         lightrec_free_reg(reg_cache, rs);
1363         tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
1364
1365         to_not_ram = jit_bgti(tmp2, ram_size);
1366
1367         /* Compute the offset to the code LUT */
1368         jit_andi(tmp, tmp2, (RAM_SIZE - 1) & ~3);
1369         if (!lut_is_32bit(state))
1370                 jit_lshi(tmp, tmp, 1);
1371         jit_addr(tmp, LIGHTREC_REG_STATE, tmp);
1372
1373         /* Write NULL to the code LUT to invalidate any block that's there */
1374         if (lut_is_32bit(state))
1375                 jit_stxi_i(offsetof(struct lightrec_state, code_lut), tmp, tmp3);
1376         else
1377                 jit_stxi(offsetof(struct lightrec_state, code_lut), tmp, tmp3);
1378
1379         if (state->offset_ram != state->offset_scratch) {
1380                 jit_movi(tmp, state->offset_ram);
1381
1382                 to_end = jit_b();
1383         }
1384
1385         jit_patch(to_not_ram);
1386
1387         if (state->offset_ram || state->offset_scratch)
1388                 jit_movi(tmp, state->offset_scratch);
1389
1390         if (state->offset_ram != state->offset_scratch)
1391                 jit_patch(to_end);
1392
1393         if (state->offset_ram || state->offset_scratch)
1394                 jit_addr(tmp2, tmp2, tmp);
1395
1396         lightrec_free_reg(reg_cache, tmp);
1397         lightrec_free_reg(reg_cache, tmp3);
1398
1399         rt = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rt, 0);
1400
1401         if (is_big_endian() && swap_code && c.i.rt) {
1402                 tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
1403
1404                 jit_new_node_ww(swap_code, tmp, rt);
1405                 jit_new_node_www(code, 0, tmp2, tmp);
1406
1407                 lightrec_free_reg(reg_cache, tmp);
1408         } else {
1409                 jit_new_node_www(code, 0, tmp2, rt);
1410         }
1411
1412         lightrec_free_reg(reg_cache, rt);
1413         lightrec_free_reg(reg_cache, tmp2);
1414 }
1415
1416 static void rec_store(struct lightrec_cstate *state,
1417                       const struct block *block, u16 offset,
1418                       jit_code_t code, jit_code_t swap_code)
1419 {
1420         u32 flags = block->opcode_list[offset].flags;
1421         bool no_invalidate = op_flag_no_invalidate(flags) ||
1422                 state->state->invalidate_from_dma_only;
1423
1424         switch (LIGHTREC_FLAGS_GET_IO_MODE(flags)) {
1425         case LIGHTREC_IO_RAM:
1426                 rec_store_ram(state, block, offset, code,
1427                               swap_code, !no_invalidate);
1428                 break;
1429         case LIGHTREC_IO_SCRATCH:
1430                 rec_store_scratch(state, block, offset, code, swap_code);
1431                 break;
1432         case LIGHTREC_IO_DIRECT:
1433                 if (no_invalidate) {
1434                         rec_store_direct_no_invalidate(state, block, offset,
1435                                                        code, swap_code);
1436                 } else {
1437                         rec_store_direct(state, block, offset, code, swap_code);
1438                 }
1439                 break;
1440         case LIGHTREC_IO_DIRECT_HW:
1441                 rec_store_io(state, block, offset, code, swap_code);
1442                 break;
1443         default:
1444                 rec_io(state, block, offset, true, false);
1445                 break;
1446         }
1447 }
1448
1449 static void rec_SB(struct lightrec_cstate *state,
1450                    const struct block *block, u16 offset)
1451 {
1452         _jit_name(block->_jit, __func__);
1453         rec_store(state, block, offset, jit_code_stxi_c, 0);
1454 }
1455
1456 static void rec_SH(struct lightrec_cstate *state,
1457                    const struct block *block, u16 offset)
1458 {
1459         _jit_name(block->_jit, __func__);
1460         rec_store(state, block, offset,
1461                   jit_code_stxi_s, jit_code_bswapr_us);
1462 }
1463
1464 static void rec_SW(struct lightrec_cstate *state,
1465                    const struct block *block, u16 offset)
1466
1467 {
1468         _jit_name(block->_jit, __func__);
1469         rec_store(state, block, offset,
1470                   jit_code_stxi_i, jit_code_bswapr_ui);
1471 }
1472
1473 static void rec_SWL(struct lightrec_cstate *state,
1474                     const struct block *block, u16 offset)
1475 {
1476         _jit_name(block->_jit, __func__);
1477         rec_io(state, block, offset, true, false);
1478 }
1479
1480 static void rec_SWR(struct lightrec_cstate *state,
1481                     const struct block *block, u16 offset)
1482 {
1483         _jit_name(block->_jit, __func__);
1484         rec_io(state, block, offset, true, false);
1485 }
1486
1487 static void rec_SWC2(struct lightrec_cstate *state,
1488                      const struct block *block, u16 offset)
1489 {
1490         _jit_name(block->_jit, __func__);
1491         rec_io(state, block, offset, false, false);
1492 }
1493
1494 static void rec_load_memory(struct lightrec_cstate *cstate,
1495                             const struct block *block, u16 offset,
1496                             jit_code_t code, jit_code_t swap_code, bool is_unsigned,
1497                             uintptr_t addr_offset, u32 addr_mask)
1498 {
1499         struct regcache *reg_cache = cstate->reg_cache;
1500         struct opcode *op = &block->opcode_list[offset];
1501         jit_state_t *_jit = block->_jit;
1502         u8 rs, rt, addr_reg, flags = REG_EXT;
1503         bool no_mask = op_flag_no_mask(op->flags);
1504         union code c = op->c;
1505         s16 imm;
1506
1507         if (!c.i.rt)
1508                 return;
1509
1510         if (is_unsigned)
1511                 flags |= REG_ZEXT;
1512
1513         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rs, 0);
1514         rt = lightrec_alloc_reg_out(reg_cache, _jit, c.i.rt, flags);
1515
1516         if (!cstate->state->mirrors_mapped && c.i.imm && !no_mask) {
1517                 jit_addi(rt, rs, (s16)c.i.imm);
1518                 addr_reg = rt;
1519                 imm = 0;
1520         } else {
1521                 addr_reg = rs;
1522                 imm = (s16)c.i.imm;
1523         }
1524
1525         if (!no_mask) {
1526                 jit_andi(rt, addr_reg, addr_mask);
1527                 addr_reg = rt;
1528         }
1529
1530         if (addr_offset) {
1531                 jit_addi(rt, addr_reg, addr_offset);
1532                 addr_reg = rt;
1533         }
1534
1535         jit_new_node_www(code, rt, addr_reg, imm);
1536
1537         if (is_big_endian() && swap_code) {
1538                 jit_new_node_ww(swap_code, rt, rt);
1539
1540                 if (c.i.op == OP_LH)
1541                         jit_extr_s(rt, rt);
1542                 else if (c.i.op == OP_LW && __WORDSIZE == 64)
1543                         jit_extr_i(rt, rt);
1544         }
1545
1546         lightrec_free_reg(reg_cache, rs);
1547         lightrec_free_reg(reg_cache, rt);
1548 }
1549
1550 static void rec_load_ram(struct lightrec_cstate *cstate,
1551                          const struct block *block, u16 offset,
1552                          jit_code_t code, jit_code_t swap_code, bool is_unsigned)
1553 {
1554         _jit_note(block->_jit, __FILE__, __LINE__);
1555
1556         rec_load_memory(cstate, block, offset, code, swap_code, is_unsigned,
1557                         cstate->state->offset_ram, rec_ram_mask(cstate->state));
1558 }
1559
1560 static void rec_load_bios(struct lightrec_cstate *cstate,
1561                           const struct block *block, u16 offset,
1562                           jit_code_t code, jit_code_t swap_code, bool is_unsigned)
1563 {
1564         _jit_note(block->_jit, __FILE__, __LINE__);
1565
1566         rec_load_memory(cstate, block, offset, code, swap_code, is_unsigned,
1567                         cstate->state->offset_bios, 0x1fffffff);
1568 }
1569
1570 static void rec_load_scratch(struct lightrec_cstate *cstate,
1571                              const struct block *block, u16 offset,
1572                              jit_code_t code, jit_code_t swap_code, bool is_unsigned)
1573 {
1574         _jit_note(block->_jit, __FILE__, __LINE__);
1575
1576         rec_load_memory(cstate, block, offset, code, swap_code, is_unsigned,
1577                         cstate->state->offset_scratch, 0x1fffffff);
1578 }
1579
1580 static void rec_load_io(struct lightrec_cstate *cstate,
1581                         const struct block *block, u16 offset,
1582                         jit_code_t code, jit_code_t swap_code, bool is_unsigned)
1583 {
1584         _jit_note(block->_jit, __FILE__, __LINE__);
1585
1586         rec_load_memory(cstate, block, offset, code, swap_code, is_unsigned,
1587                         cstate->state->offset_io, 0x1fffffff);
1588 }
1589
1590 static void rec_load_direct(struct lightrec_cstate *cstate,
1591                             const struct block *block, u16 offset,
1592                             jit_code_t code, jit_code_t swap_code,
1593                             bool is_unsigned)
1594 {
1595         struct lightrec_state *state = cstate->state;
1596         struct regcache *reg_cache = cstate->reg_cache;
1597         union code c = block->opcode_list[offset].c;
1598         jit_state_t *_jit = block->_jit;
1599         jit_node_t *to_not_ram, *to_not_bios, *to_end, *to_end2;
1600         u8 tmp, rs, rt, addr_reg, flags = REG_EXT;
1601         s16 imm;
1602
1603         if (!c.i.rt)
1604                 return;
1605
1606         if (is_unsigned)
1607                 flags |= REG_ZEXT;
1608
1609         jit_note(__FILE__, __LINE__);
1610         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rs, 0);
1611         rt = lightrec_alloc_reg_out(reg_cache, _jit, c.i.rt, flags);
1612
1613         if ((state->offset_ram == state->offset_bios &&
1614             state->offset_ram == state->offset_scratch &&
1615             state->mirrors_mapped) || !c.i.imm) {
1616                 addr_reg = rs;
1617                 imm = (s16)c.i.imm;
1618         } else {
1619                 jit_addi(rt, rs, (s16)c.i.imm);
1620                 addr_reg = rt;
1621                 imm = 0;
1622
1623                 if (c.i.rs != c.i.rt)
1624                         lightrec_free_reg(reg_cache, rs);
1625         }
1626
1627         tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
1628
1629         if (state->offset_ram == state->offset_bios &&
1630             state->offset_ram == state->offset_scratch) {
1631                 if (!state->mirrors_mapped) {
1632                         jit_andi(tmp, addr_reg, BIT(28));
1633                         jit_rshi_u(tmp, tmp, 28 - 22);
1634                         jit_ori(tmp, tmp, 0x1f800000 | (RAM_SIZE - 1));
1635                         jit_andr(rt, addr_reg, tmp);
1636                 } else {
1637                         jit_andi(rt, addr_reg, 0x1fffffff);
1638                 }
1639
1640                 if (state->offset_ram)
1641                         jit_movi(tmp, state->offset_ram);
1642         } else {
1643                 to_not_ram = jit_bmsi(addr_reg, BIT(28));
1644
1645                 /* Convert to KUNSEG and avoid RAM mirrors */
1646                 jit_andi(rt, addr_reg, RAM_SIZE - 1);
1647
1648                 if (state->offset_ram)
1649                         jit_movi(tmp, state->offset_ram);
1650
1651                 to_end = jit_b();
1652
1653                 jit_patch(to_not_ram);
1654
1655                 if (state->offset_bios != state->offset_scratch)
1656                         to_not_bios = jit_bmci(addr_reg, BIT(22));
1657
1658                 /* Convert to KUNSEG */
1659                 jit_andi(rt, addr_reg, 0x1fc00000 | (BIOS_SIZE - 1));
1660
1661                 jit_movi(tmp, state->offset_bios);
1662
1663                 if (state->offset_bios != state->offset_scratch) {
1664                         to_end2 = jit_b();
1665
1666                         jit_patch(to_not_bios);
1667
1668                         /* Convert to KUNSEG */
1669                         jit_andi(rt, addr_reg, 0x1f800fff);
1670
1671                         if (state->offset_scratch)
1672                                 jit_movi(tmp, state->offset_scratch);
1673
1674                         jit_patch(to_end2);
1675                 }
1676
1677                 jit_patch(to_end);
1678         }
1679
1680         if (state->offset_ram || state->offset_bios || state->offset_scratch)
1681                 jit_addr(rt, rt, tmp);
1682
1683         jit_new_node_www(code, rt, rt, imm);
1684
1685         if (is_big_endian() && swap_code) {
1686                 jit_new_node_ww(swap_code, rt, rt);
1687
1688                 if (c.i.op == OP_LH)
1689                         jit_extr_s(rt, rt);
1690                 else if (c.i.op == OP_LW && __WORDSIZE == 64)
1691                         jit_extr_i(rt, rt);
1692         }
1693
1694         lightrec_free_reg(reg_cache, addr_reg);
1695         lightrec_free_reg(reg_cache, rt);
1696         lightrec_free_reg(reg_cache, tmp);
1697 }
1698
1699 static void rec_load(struct lightrec_cstate *state, const struct block *block,
1700                      u16 offset, jit_code_t code, jit_code_t swap_code,
1701                      bool is_unsigned)
1702 {
1703         u32 flags = block->opcode_list[offset].flags;
1704
1705         switch (LIGHTREC_FLAGS_GET_IO_MODE(flags)) {
1706         case LIGHTREC_IO_RAM:
1707                 rec_load_ram(state, block, offset, code, swap_code, is_unsigned);
1708                 break;
1709         case LIGHTREC_IO_BIOS:
1710                 rec_load_bios(state, block, offset, code, swap_code, is_unsigned);
1711                 break;
1712         case LIGHTREC_IO_SCRATCH:
1713                 rec_load_scratch(state, block, offset, code, swap_code, is_unsigned);
1714                 break;
1715         case LIGHTREC_IO_DIRECT_HW:
1716                 rec_load_io(state, block, offset, code, swap_code, is_unsigned);
1717                 break;
1718         case LIGHTREC_IO_DIRECT:
1719                 rec_load_direct(state, block, offset, code, swap_code, is_unsigned);
1720                 break;
1721         default:
1722                 rec_io(state, block, offset, false, true);
1723                 break;
1724         }
1725 }
1726
1727 static void rec_LB(struct lightrec_cstate *state, const struct block *block, u16 offset)
1728 {
1729         _jit_name(block->_jit, __func__);
1730         rec_load(state, block, offset, jit_code_ldxi_c, 0, false);
1731 }
1732
1733 static void rec_LBU(struct lightrec_cstate *state, const struct block *block, u16 offset)
1734 {
1735         _jit_name(block->_jit, __func__);
1736         rec_load(state, block, offset, jit_code_ldxi_uc, 0, true);
1737 }
1738
1739 static void rec_LH(struct lightrec_cstate *state, const struct block *block, u16 offset)
1740 {
1741         jit_code_t code = is_big_endian() ? jit_code_ldxi_us : jit_code_ldxi_s;
1742
1743         _jit_name(block->_jit, __func__);
1744         rec_load(state, block, offset, code, jit_code_bswapr_us, false);
1745 }
1746
1747 static void rec_LHU(struct lightrec_cstate *state, const struct block *block, u16 offset)
1748 {
1749         _jit_name(block->_jit, __func__);
1750         rec_load(state, block, offset, jit_code_ldxi_us, jit_code_bswapr_us, true);
1751 }
1752
1753 static void rec_LWL(struct lightrec_cstate *state, const struct block *block, u16 offset)
1754 {
1755         _jit_name(block->_jit, __func__);
1756         rec_io(state, block, offset, true, true);
1757 }
1758
1759 static void rec_LWR(struct lightrec_cstate *state, const struct block *block, u16 offset)
1760 {
1761         _jit_name(block->_jit, __func__);
1762         rec_io(state, block, offset, true, true);
1763 }
1764
1765 static void rec_LW(struct lightrec_cstate *state, const struct block *block, u16 offset)
1766 {
1767         jit_code_t code;
1768
1769         if (is_big_endian() && __WORDSIZE == 64)
1770                 code = jit_code_ldxi_ui;
1771         else
1772                 code = jit_code_ldxi_i;
1773
1774         _jit_name(block->_jit, __func__);
1775         rec_load(state, block, offset, code, jit_code_bswapr_ui, false);
1776 }
1777
1778 static void rec_LWC2(struct lightrec_cstate *state, const struct block *block, u16 offset)
1779 {
1780         _jit_name(block->_jit, __func__);
1781         rec_io(state, block, offset, false, false);
1782 }
1783
1784 static void rec_break_syscall(struct lightrec_cstate *state,
1785                               const struct block *block, u16 offset,
1786                               u32 exit_code)
1787 {
1788         struct regcache *reg_cache = state->reg_cache;
1789         jit_state_t *_jit = block->_jit;
1790         u8 tmp;
1791
1792         _jit_note(block->_jit, __FILE__, __LINE__);
1793
1794         tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
1795
1796         jit_movi(tmp, exit_code);
1797         jit_stxi_i(offsetof(struct lightrec_state, exit_flags),
1798                    LIGHTREC_REG_STATE, tmp);
1799
1800         lightrec_free_reg(reg_cache, tmp);
1801
1802         /* TODO: the return address should be "pc - 4" if we're a delay slot */
1803         lightrec_emit_end_of_block(state, block, offset, -1,
1804                                    get_ds_pc(block, offset, 0),
1805                                    31, 0, true);
1806 }
1807
1808 static void rec_special_SYSCALL(struct lightrec_cstate *state,
1809                                 const struct block *block, u16 offset)
1810 {
1811         _jit_name(block->_jit, __func__);
1812         rec_break_syscall(state, block, offset, LIGHTREC_EXIT_SYSCALL);
1813 }
1814
1815 static void rec_special_BREAK(struct lightrec_cstate *state,
1816                               const struct block *block, u16 offset)
1817 {
1818         _jit_name(block->_jit, __func__);
1819         rec_break_syscall(state, block, offset, LIGHTREC_EXIT_BREAK);
1820 }
1821
1822 static void rec_mtc(struct lightrec_cstate *state, const struct block *block, u16 offset)
1823 {
1824         struct regcache *reg_cache = state->reg_cache;
1825         union code c = block->opcode_list[offset].c;
1826         jit_state_t *_jit = block->_jit;
1827
1828         jit_note(__FILE__, __LINE__);
1829         lightrec_clean_reg_if_loaded(reg_cache, _jit, c.i.rs, false);
1830         lightrec_clean_reg_if_loaded(reg_cache, _jit, c.i.rt, false);
1831
1832         call_to_c_wrapper(state, block, c.opcode, C_WRAPPER_MTC);
1833
1834         if (c.i.op == OP_CP0 &&
1835             !op_flag_no_ds(block->opcode_list[offset].flags) &&
1836             (c.r.rd == 12 || c.r.rd == 13))
1837                 lightrec_emit_end_of_block(state, block, offset, -1,
1838                                            get_ds_pc(block, offset, 1),
1839                                            0, 0, true);
1840 }
1841
1842 static void
1843 rec_mfc0(struct lightrec_cstate *state, const struct block *block, u16 offset)
1844 {
1845         struct regcache *reg_cache = state->reg_cache;
1846         union code c = block->opcode_list[offset].c;
1847         jit_state_t *_jit = block->_jit;
1848         u8 rt;
1849
1850         jit_note(__FILE__, __LINE__);
1851
1852         rt = lightrec_alloc_reg_out(reg_cache, _jit, c.i.rt, REG_EXT);
1853
1854         jit_ldxi_i(rt, LIGHTREC_REG_STATE,
1855                    offsetof(struct lightrec_state, regs.cp0[c.r.rd]));
1856
1857         lightrec_free_reg(reg_cache, rt);
1858 }
1859
1860 static bool block_in_bios(const struct lightrec_cstate *state,
1861                           const struct block *block)
1862 {
1863         const struct lightrec_mem_map *bios = &state->state->maps[PSX_MAP_BIOS];
1864         u32 pc = kunseg(block->pc);
1865
1866         return pc >= bios->pc && pc < bios->pc + bios->length;
1867 }
1868
1869 static void
1870 rec_mtc0(struct lightrec_cstate *state, const struct block *block, u16 offset)
1871 {
1872         struct regcache *reg_cache = state->reg_cache;
1873         const union code c = block->opcode_list[offset].c;
1874         jit_state_t *_jit = block->_jit;
1875         u8 rt, tmp = 0, tmp2, status;
1876
1877         jit_note(__FILE__, __LINE__);
1878
1879         switch(c.r.rd) {
1880         case 1:
1881         case 4:
1882         case 8:
1883         case 14:
1884         case 15:
1885                 /* Those registers are read-only */
1886                 return;
1887         default:
1888                 break;
1889         }
1890
1891         if (block_in_bios(state, block) && c.r.rd == 12) {
1892                 /* If we are running code from the BIOS, handle writes to the
1893                  * Status register in C. BIOS code may toggle bit 16 which will
1894                  * map/unmap the RAM, while game code cannot do that. */
1895                 rec_mtc(state, block, offset);
1896                 return;
1897         }
1898
1899         rt = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rt, 0);
1900
1901         if (c.r.rd != 13) {
1902                 jit_stxi_i(offsetof(struct lightrec_state, regs.cp0[c.r.rd]),
1903                            LIGHTREC_REG_STATE, rt);
1904         }
1905
1906         if (c.r.rd == 12 || c.r.rd == 13) {
1907                 tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
1908                 jit_ldxi_i(tmp, LIGHTREC_REG_STATE,
1909                            offsetof(struct lightrec_state, regs.cp0[13]));
1910
1911                 tmp2 = lightrec_alloc_reg_temp(reg_cache, _jit);
1912         }
1913
1914         if (c.r.rd == 12) {
1915                 status = rt;
1916         } else if (c.r.rd == 13) {
1917                 /* Cause = (Cause & ~0x0300) | (value & 0x0300) */
1918                 jit_andi(tmp2, rt, 0x0300);
1919                 jit_ori(tmp, tmp, 0x0300);
1920                 jit_xori(tmp, tmp, 0x0300);
1921                 jit_orr(tmp, tmp, tmp2);
1922                 jit_ldxi_i(tmp2, LIGHTREC_REG_STATE,
1923                            offsetof(struct lightrec_state, regs.cp0[12]));
1924                 jit_stxi_i(offsetof(struct lightrec_state, regs.cp0[13]),
1925                            LIGHTREC_REG_STATE, tmp);
1926                 status = tmp2;
1927         }
1928
1929         if (c.r.rd == 12 || c.r.rd == 13) {
1930                 /* Exit dynarec in case there's a software interrupt.
1931                  * exit_flags = !!(status & tmp & 0x0300) & status; */
1932                 jit_andr(tmp, tmp, status);
1933                 jit_andi(tmp, tmp, 0x0300);
1934                 jit_nei(tmp, tmp, 0);
1935                 jit_andr(tmp, tmp, status);
1936         }
1937
1938         if (c.r.rd == 12) {
1939                 /* Exit dynarec in case we unmask a hardware interrupt.
1940                  * exit_flags = !(~status & 0x401) */
1941
1942                 jit_comr(tmp2, status);
1943                 jit_andi(tmp2, tmp2, 0x401);
1944                 jit_eqi(tmp2, tmp2, 0);
1945                 jit_orr(tmp, tmp, tmp2);
1946         }
1947
1948         if (c.r.rd == 12 || c.r.rd == 13) {
1949                 jit_stxi_i(offsetof(struct lightrec_state, exit_flags),
1950                            LIGHTREC_REG_STATE, tmp);
1951
1952                 lightrec_free_reg(reg_cache, tmp);
1953                 lightrec_free_reg(reg_cache, tmp2);
1954         }
1955
1956         lightrec_free_reg(reg_cache, rt);
1957
1958         if (!op_flag_no_ds(block->opcode_list[offset].flags) &&
1959             (c.r.rd == 12 || c.r.rd == 13))
1960                 lightrec_emit_eob(state, block, offset + 1, true);
1961 }
1962
1963 static void rec_cp0_MFC0(struct lightrec_cstate *state,
1964                          const struct block *block, u16 offset)
1965 {
1966         _jit_name(block->_jit, __func__);
1967         rec_mfc0(state, block, offset);
1968 }
1969
1970 static void rec_cp0_CFC0(struct lightrec_cstate *state,
1971                          const struct block *block, u16 offset)
1972 {
1973         _jit_name(block->_jit, __func__);
1974         rec_mfc0(state, block, offset);
1975 }
1976
1977 static void rec_cp0_MTC0(struct lightrec_cstate *state,
1978                          const struct block *block, u16 offset)
1979 {
1980         _jit_name(block->_jit, __func__);
1981         rec_mtc0(state, block, offset);
1982 }
1983
1984 static void rec_cp0_CTC0(struct lightrec_cstate *state,
1985                          const struct block *block, u16 offset)
1986 {
1987         _jit_name(block->_jit, __func__);
1988         rec_mtc0(state, block, offset);
1989 }
1990
1991 static unsigned int cp2d_i_offset(u8 reg)
1992 {
1993         return offsetof(struct lightrec_state, regs.cp2d[reg]);
1994 }
1995
1996 static unsigned int cp2d_s_offset(u8 reg)
1997 {
1998         return cp2d_i_offset(reg) + is_big_endian() * 2;
1999 }
2000
2001 static unsigned int cp2c_i_offset(u8 reg)
2002 {
2003         return offsetof(struct lightrec_state, regs.cp2c[reg]);
2004 }
2005
2006 static unsigned int cp2c_s_offset(u8 reg)
2007 {
2008         return cp2c_i_offset(reg) + is_big_endian() * 2;
2009 }
2010
2011 static void rec_cp2_basic_MFC2(struct lightrec_cstate *state,
2012                                const struct block *block, u16 offset)
2013 {
2014         struct regcache *reg_cache = state->reg_cache;
2015         const union code c = block->opcode_list[offset].c;
2016         jit_state_t *_jit = block->_jit;
2017         const u32 zext_regs = 0x300f0080;
2018         u8 rt, tmp, tmp2, tmp3, out, flags;
2019         u8 reg = c.r.rd == 15 ? 14 : c.r.rd;
2020         unsigned int i;
2021
2022         _jit_name(block->_jit, __func__);
2023
2024         flags = (zext_regs & BIT(reg)) ? REG_ZEXT : REG_EXT;
2025         rt = lightrec_alloc_reg_out(reg_cache, _jit, c.r.rt, flags);
2026
2027         switch (reg) {
2028         case 1:
2029         case 3:
2030         case 5:
2031         case 8:
2032         case 9:
2033         case 10:
2034         case 11:
2035                 jit_ldxi_s(rt, LIGHTREC_REG_STATE, cp2d_s_offset(reg));
2036                 break;
2037         case 7:
2038         case 16:
2039         case 17:
2040         case 18:
2041         case 19:
2042                 jit_ldxi_us(rt, LIGHTREC_REG_STATE, cp2d_s_offset(reg));
2043                 break;
2044         case 28:
2045         case 29:
2046                 tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
2047                 tmp2 = lightrec_alloc_reg_temp(reg_cache, _jit);
2048                 tmp3 = lightrec_alloc_reg_temp(reg_cache, _jit);
2049
2050                 for (i = 0; i < 3; i++) {
2051                         out = i == 0 ? rt : tmp;
2052
2053                         jit_ldxi_s(tmp, LIGHTREC_REG_STATE, cp2d_s_offset(9 + i));
2054                         jit_movi(tmp2, 0x1f);
2055                         jit_rshi(out, tmp, 7);
2056
2057                         jit_ltr(tmp3, tmp2, out);
2058                         jit_movnr(out, tmp2, tmp3);
2059
2060                         jit_gei(tmp2, out, 0);
2061                         jit_movzr(out, tmp2, tmp2);
2062
2063                         if (i > 0) {
2064                                 jit_lshi(tmp, tmp, 5 * i);
2065                                 jit_orr(rt, rt, tmp);
2066                         }
2067                 }
2068
2069
2070                 lightrec_free_reg(reg_cache, tmp);
2071                 lightrec_free_reg(reg_cache, tmp2);
2072                 lightrec_free_reg(reg_cache, tmp3);
2073                 break;
2074         default:
2075                 jit_ldxi_i(rt, LIGHTREC_REG_STATE, cp2d_i_offset(reg));
2076                 break;
2077         }
2078
2079         lightrec_free_reg(reg_cache, rt);
2080 }
2081
2082 static void rec_cp2_basic_CFC2(struct lightrec_cstate *state,
2083                                const struct block *block, u16 offset)
2084 {
2085         struct regcache *reg_cache = state->reg_cache;
2086         const union code c = block->opcode_list[offset].c;
2087         jit_state_t *_jit = block->_jit;
2088         u8 rt;
2089
2090         _jit_name(block->_jit, __func__);
2091
2092         switch (c.r.rd) {
2093         case 4:
2094         case 12:
2095         case 20:
2096         case 26:
2097         case 27:
2098         case 29:
2099         case 30:
2100                 rt = lightrec_alloc_reg_out(reg_cache, _jit, c.r.rt, REG_EXT);
2101                 jit_ldxi_s(rt, LIGHTREC_REG_STATE, cp2c_s_offset(c.r.rd));
2102                 break;
2103         default:
2104                 rt = lightrec_alloc_reg_out(reg_cache, _jit, c.r.rt, REG_ZEXT);
2105                 jit_ldxi_i(rt, LIGHTREC_REG_STATE, cp2c_i_offset(c.r.rd));
2106                 break;
2107         }
2108
2109         lightrec_free_reg(reg_cache, rt);
2110 }
2111
2112 static void rec_cp2_basic_MTC2(struct lightrec_cstate *state,
2113                                const struct block *block, u16 offset)
2114 {
2115         struct regcache *reg_cache = state->reg_cache;
2116         const union code c = block->opcode_list[offset].c;
2117         jit_state_t *_jit = block->_jit;
2118         jit_node_t *loop, *to_loop;
2119         u8 rt, tmp, tmp2, flags = 0;
2120
2121         _jit_name(block->_jit, __func__);
2122
2123         if (c.r.rd == 31)
2124                 return;
2125
2126         if (c.r.rd == 30)
2127                 flags |= REG_EXT;
2128
2129         rt = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rt, flags);
2130
2131         switch (c.r.rd) {
2132         case 15:
2133                 tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
2134                 jit_ldxi_i(tmp, LIGHTREC_REG_STATE, cp2d_i_offset(13));
2135
2136                 tmp2 = lightrec_alloc_reg_temp(reg_cache, _jit);
2137                 jit_ldxi_i(tmp2, LIGHTREC_REG_STATE, cp2d_i_offset(14));
2138
2139                 jit_stxi_i(cp2d_i_offset(12), LIGHTREC_REG_STATE, tmp);
2140                 jit_stxi_i(cp2d_i_offset(13), LIGHTREC_REG_STATE, tmp2);
2141                 jit_stxi_i(cp2d_i_offset(14), LIGHTREC_REG_STATE, rt);
2142
2143                 lightrec_free_reg(reg_cache, tmp);
2144                 lightrec_free_reg(reg_cache, tmp2);
2145                 break;
2146         case 28:
2147                 tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
2148
2149                 jit_lshi(tmp, rt, 7);
2150                 jit_andi(tmp, tmp, 0xf80);
2151                 jit_stxi_s(cp2d_s_offset(9), LIGHTREC_REG_STATE, tmp);
2152
2153                 jit_lshi(tmp, rt, 2);
2154                 jit_andi(tmp, tmp, 0xf80);
2155                 jit_stxi_s(cp2d_s_offset(10), LIGHTREC_REG_STATE, tmp);
2156
2157                 jit_rshi(tmp, rt, 3);
2158                 jit_andi(tmp, tmp, 0xf80);
2159                 jit_stxi_s(cp2d_s_offset(11), LIGHTREC_REG_STATE, tmp);
2160
2161                 lightrec_free_reg(reg_cache, tmp);
2162                 break;
2163         case 30:
2164                 tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
2165                 tmp2 = lightrec_alloc_reg_temp(reg_cache, _jit);
2166
2167                 /* if (rt < 0) rt = ~rt; */
2168                 jit_rshi(tmp, rt, 31);
2169                 jit_xorr(tmp, rt, tmp);
2170
2171                 /* We know the sign bit is 0. Left-shift by 1 to start the algorithm */
2172                 jit_lshi(tmp, tmp, 1);
2173                 jit_movi(tmp2, 33);
2174
2175                 /* Decrement tmp2 and right-shift the value by 1 until it equals zero */
2176                 loop = jit_label();
2177                 jit_subi(tmp2, tmp2, 1);
2178                 jit_rshi_u(tmp, tmp, 1);
2179                 to_loop = jit_bnei(tmp, 0);
2180
2181                 jit_patch_at(to_loop, loop);
2182
2183                 jit_stxi_i(cp2d_i_offset(31), LIGHTREC_REG_STATE, tmp2);
2184                 jit_stxi_i(cp2d_i_offset(30), LIGHTREC_REG_STATE, rt);
2185
2186                 lightrec_free_reg(reg_cache, tmp);
2187                 lightrec_free_reg(reg_cache, tmp2);
2188                 break;
2189         default:
2190                 jit_stxi_i(cp2d_i_offset(c.r.rd), LIGHTREC_REG_STATE, rt);
2191                 break;
2192         }
2193
2194         lightrec_free_reg(reg_cache, rt);
2195 }
2196
2197 static void rec_cp2_basic_CTC2(struct lightrec_cstate *state,
2198                                const struct block *block, u16 offset)
2199 {
2200         struct regcache *reg_cache = state->reg_cache;
2201         const union code c = block->opcode_list[offset].c;
2202         jit_state_t *_jit = block->_jit;
2203         u8 rt, tmp, tmp2;
2204
2205         _jit_name(block->_jit, __func__);
2206
2207         rt = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rt, 0);
2208
2209         switch (c.r.rd) {
2210         case 4:
2211         case 12:
2212         case 20:
2213         case 26:
2214         case 27:
2215         case 29:
2216         case 30:
2217                 jit_stxi_s(cp2c_s_offset(c.r.rd), LIGHTREC_REG_STATE, rt);
2218                 break;
2219         case 31:
2220                 tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
2221                 tmp2 = lightrec_alloc_reg_temp(reg_cache, _jit);
2222
2223                 jit_andi(tmp, rt, 0x7f87e000);
2224                 jit_nei(tmp, tmp, 0);
2225                 jit_lshi(tmp, tmp, 31);
2226
2227                 jit_andi(tmp2, rt, 0x7ffff000);
2228                 jit_orr(tmp, tmp2, tmp);
2229
2230                 jit_stxi_i(cp2c_i_offset(31), LIGHTREC_REG_STATE, tmp);
2231
2232                 lightrec_free_reg(reg_cache, tmp);
2233                 lightrec_free_reg(reg_cache, tmp2);
2234                 break;
2235
2236         default:
2237                 jit_stxi_i(cp2c_i_offset(c.r.rd), LIGHTREC_REG_STATE, rt);
2238         }
2239
2240         lightrec_free_reg(reg_cache, rt);
2241 }
2242
2243 static void rec_cp0_RFE(struct lightrec_cstate *state,
2244                         const struct block *block, u16 offset)
2245 {
2246         struct regcache *reg_cache = state->reg_cache;
2247         jit_state_t *_jit = block->_jit;
2248         u8 status, tmp;
2249
2250         jit_name(__func__);
2251         jit_note(__FILE__, __LINE__);
2252
2253         status = lightrec_alloc_reg_temp(reg_cache, _jit);
2254         jit_ldxi_i(status, LIGHTREC_REG_STATE,
2255                    offsetof(struct lightrec_state, regs.cp0[12]));
2256
2257         tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
2258
2259         /* status = ((status >> 2) & 0xf) | status & ~0xf; */
2260         jit_rshi(tmp, status, 2);
2261         jit_andi(tmp, tmp, 0xf);
2262         jit_andi(status, status, ~0xful);
2263         jit_orr(status, status, tmp);
2264
2265         jit_ldxi_i(tmp, LIGHTREC_REG_STATE,
2266                    offsetof(struct lightrec_state, regs.cp0[13]));
2267         jit_stxi_i(offsetof(struct lightrec_state, regs.cp0[12]),
2268                    LIGHTREC_REG_STATE, status);
2269
2270         /* Exit dynarec in case there's a software interrupt.
2271          * exit_flags = !!(status & cause & 0x0300) & status; */
2272         jit_andr(tmp, tmp, status);
2273         jit_andi(tmp, tmp, 0x0300);
2274         jit_nei(tmp, tmp, 0);
2275         jit_andr(tmp, tmp, status);
2276         jit_stxi_i(offsetof(struct lightrec_state, exit_flags),
2277                    LIGHTREC_REG_STATE, tmp);
2278
2279         lightrec_free_reg(reg_cache, status);
2280         lightrec_free_reg(reg_cache, tmp);
2281 }
2282
2283 static void rec_CP(struct lightrec_cstate *state,
2284                    const struct block *block, u16 offset)
2285 {
2286         union code c = block->opcode_list[offset].c;
2287         jit_state_t *_jit = block->_jit;
2288
2289         jit_name(__func__);
2290         jit_note(__FILE__, __LINE__);
2291
2292         call_to_c_wrapper(state, block, c.opcode, C_WRAPPER_CP);
2293 }
2294
2295 static void rec_meta_MOV(struct lightrec_cstate *state,
2296                          const struct block *block, u16 offset)
2297 {
2298         struct regcache *reg_cache = state->reg_cache;
2299         union code c = block->opcode_list[offset].c;
2300         jit_state_t *_jit = block->_jit;
2301         u8 rs, rd;
2302
2303         _jit_name(block->_jit, __func__);
2304         jit_note(__FILE__, __LINE__);
2305         if (c.r.rs)
2306                 rs = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rs, 0);
2307         rd = lightrec_alloc_reg_out(reg_cache, _jit, c.r.rd, REG_EXT);
2308
2309         if (c.r.rs == 0)
2310                 jit_movi(rd, 0);
2311         else
2312                 jit_extr_i(rd, rs);
2313
2314         if (c.r.rs)
2315                 lightrec_free_reg(reg_cache, rs);
2316         lightrec_free_reg(reg_cache, rd);
2317 }
2318
2319 static void rec_meta_EXTC_EXTS(struct lightrec_cstate *state,
2320                                const struct block *block,
2321                                u16 offset)
2322 {
2323         struct regcache *reg_cache = state->reg_cache;
2324         union code c = block->opcode_list[offset].c;
2325         jit_state_t *_jit = block->_jit;
2326         u8 rs, rt;
2327
2328         _jit_name(block->_jit, __func__);
2329         jit_note(__FILE__, __LINE__);
2330
2331         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rs, 0);
2332         rt = lightrec_alloc_reg_out(reg_cache, _jit, c.i.rt, REG_EXT);
2333
2334         if (c.i.op == OP_META_EXTC)
2335                 jit_extr_c(rt, rs);
2336         else
2337                 jit_extr_s(rt, rs);
2338
2339         lightrec_free_reg(reg_cache, rs);
2340         lightrec_free_reg(reg_cache, rt);
2341 }
2342
2343 static void rec_meta_MULT2(struct lightrec_cstate *state,
2344                            const struct block *block,
2345                            u16 offset)
2346 {
2347         struct regcache *reg_cache = state->reg_cache;
2348         union code c = block->opcode_list[offset].c;
2349         jit_state_t *_jit = block->_jit;
2350         u8 reg_lo = get_mult_div_lo(c);
2351         u8 reg_hi = get_mult_div_hi(c);
2352         u32 flags = block->opcode_list[offset].flags;
2353         bool is_signed = c.i.op == OP_META_MULT2;
2354         u8 rs, lo, hi, rflags = 0, hiflags = 0;
2355
2356         if (!op_flag_no_hi(flags) && c.r.op < 32) {
2357                 rflags = is_signed ? REG_EXT : REG_ZEXT;
2358                 hiflags = is_signed ? REG_EXT : (REG_EXT | REG_ZEXT);
2359         }
2360
2361         _jit_name(block->_jit, __func__);
2362         jit_note(__FILE__, __LINE__);
2363
2364         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rs, rflags);
2365
2366         if (!op_flag_no_lo(flags)) {
2367                 lo = lightrec_alloc_reg_out(reg_cache, _jit, reg_lo, 0);
2368
2369                 if (c.r.op < 32)
2370                         jit_lshi(lo, rs, c.r.op);
2371                 else
2372                         jit_movi(lo, 0);
2373
2374                 lightrec_free_reg(reg_cache, lo);
2375         }
2376
2377         if (!op_flag_no_hi(flags)) {
2378                 hi = lightrec_alloc_reg_out(reg_cache, _jit, reg_hi, hiflags);
2379
2380                 if (c.r.op >= 32)
2381                         jit_lshi(hi, rs, c.r.op - 32);
2382                 else if (is_signed)
2383                         jit_rshi(hi, rs, 32 - c.r.op);
2384                 else
2385                         jit_rshi_u(hi, rs, 32 - c.r.op);
2386
2387                 lightrec_free_reg(reg_cache, hi);
2388         }
2389
2390         lightrec_free_reg(reg_cache, rs);
2391
2392         _jit_name(block->_jit, __func__);
2393         jit_note(__FILE__, __LINE__);
2394 }
2395
2396 static const lightrec_rec_func_t rec_standard[64] = {
2397         SET_DEFAULT_ELM(rec_standard, unknown_opcode),
2398         [OP_SPECIAL]            = rec_SPECIAL,
2399         [OP_REGIMM]             = rec_REGIMM,
2400         [OP_J]                  = rec_J,
2401         [OP_JAL]                = rec_JAL,
2402         [OP_BEQ]                = rec_BEQ,
2403         [OP_BNE]                = rec_BNE,
2404         [OP_BLEZ]               = rec_BLEZ,
2405         [OP_BGTZ]               = rec_BGTZ,
2406         [OP_ADDI]               = rec_ADDI,
2407         [OP_ADDIU]              = rec_ADDIU,
2408         [OP_SLTI]               = rec_SLTI,
2409         [OP_SLTIU]              = rec_SLTIU,
2410         [OP_ANDI]               = rec_ANDI,
2411         [OP_ORI]                = rec_ORI,
2412         [OP_XORI]               = rec_XORI,
2413         [OP_LUI]                = rec_LUI,
2414         [OP_CP0]                = rec_CP0,
2415         [OP_CP2]                = rec_CP2,
2416         [OP_LB]                 = rec_LB,
2417         [OP_LH]                 = rec_LH,
2418         [OP_LWL]                = rec_LWL,
2419         [OP_LW]                 = rec_LW,
2420         [OP_LBU]                = rec_LBU,
2421         [OP_LHU]                = rec_LHU,
2422         [OP_LWR]                = rec_LWR,
2423         [OP_SB]                 = rec_SB,
2424         [OP_SH]                 = rec_SH,
2425         [OP_SWL]                = rec_SWL,
2426         [OP_SW]                 = rec_SW,
2427         [OP_SWR]                = rec_SWR,
2428         [OP_LWC2]               = rec_LWC2,
2429         [OP_SWC2]               = rec_SWC2,
2430
2431         [OP_META_MOV]           = rec_meta_MOV,
2432         [OP_META_EXTC]          = rec_meta_EXTC_EXTS,
2433         [OP_META_EXTS]          = rec_meta_EXTC_EXTS,
2434         [OP_META_MULT2]         = rec_meta_MULT2,
2435         [OP_META_MULTU2]        = rec_meta_MULT2,
2436 };
2437
2438 static const lightrec_rec_func_t rec_special[64] = {
2439         SET_DEFAULT_ELM(rec_special, unknown_opcode),
2440         [OP_SPECIAL_SLL]        = rec_special_SLL,
2441         [OP_SPECIAL_SRL]        = rec_special_SRL,
2442         [OP_SPECIAL_SRA]        = rec_special_SRA,
2443         [OP_SPECIAL_SLLV]       = rec_special_SLLV,
2444         [OP_SPECIAL_SRLV]       = rec_special_SRLV,
2445         [OP_SPECIAL_SRAV]       = rec_special_SRAV,
2446         [OP_SPECIAL_JR]         = rec_special_JR,
2447         [OP_SPECIAL_JALR]       = rec_special_JALR,
2448         [OP_SPECIAL_SYSCALL]    = rec_special_SYSCALL,
2449         [OP_SPECIAL_BREAK]      = rec_special_BREAK,
2450         [OP_SPECIAL_MFHI]       = rec_special_MFHI,
2451         [OP_SPECIAL_MTHI]       = rec_special_MTHI,
2452         [OP_SPECIAL_MFLO]       = rec_special_MFLO,
2453         [OP_SPECIAL_MTLO]       = rec_special_MTLO,
2454         [OP_SPECIAL_MULT]       = rec_special_MULT,
2455         [OP_SPECIAL_MULTU]      = rec_special_MULTU,
2456         [OP_SPECIAL_DIV]        = rec_special_DIV,
2457         [OP_SPECIAL_DIVU]       = rec_special_DIVU,
2458         [OP_SPECIAL_ADD]        = rec_special_ADD,
2459         [OP_SPECIAL_ADDU]       = rec_special_ADDU,
2460         [OP_SPECIAL_SUB]        = rec_special_SUB,
2461         [OP_SPECIAL_SUBU]       = rec_special_SUBU,
2462         [OP_SPECIAL_AND]        = rec_special_AND,
2463         [OP_SPECIAL_OR]         = rec_special_OR,
2464         [OP_SPECIAL_XOR]        = rec_special_XOR,
2465         [OP_SPECIAL_NOR]        = rec_special_NOR,
2466         [OP_SPECIAL_SLT]        = rec_special_SLT,
2467         [OP_SPECIAL_SLTU]       = rec_special_SLTU,
2468 };
2469
2470 static const lightrec_rec_func_t rec_regimm[64] = {
2471         SET_DEFAULT_ELM(rec_regimm, unknown_opcode),
2472         [OP_REGIMM_BLTZ]        = rec_regimm_BLTZ,
2473         [OP_REGIMM_BGEZ]        = rec_regimm_BGEZ,
2474         [OP_REGIMM_BLTZAL]      = rec_regimm_BLTZAL,
2475         [OP_REGIMM_BGEZAL]      = rec_regimm_BGEZAL,
2476 };
2477
2478 static const lightrec_rec_func_t rec_cp0[64] = {
2479         SET_DEFAULT_ELM(rec_cp0, rec_CP),
2480         [OP_CP0_MFC0]           = rec_cp0_MFC0,
2481         [OP_CP0_CFC0]           = rec_cp0_CFC0,
2482         [OP_CP0_MTC0]           = rec_cp0_MTC0,
2483         [OP_CP0_CTC0]           = rec_cp0_CTC0,
2484         [OP_CP0_RFE]            = rec_cp0_RFE,
2485 };
2486
2487 static const lightrec_rec_func_t rec_cp2_basic[64] = {
2488         SET_DEFAULT_ELM(rec_cp2_basic, rec_CP),
2489         [OP_CP2_BASIC_MFC2]     = rec_cp2_basic_MFC2,
2490         [OP_CP2_BASIC_CFC2]     = rec_cp2_basic_CFC2,
2491         [OP_CP2_BASIC_MTC2]     = rec_cp2_basic_MTC2,
2492         [OP_CP2_BASIC_CTC2]     = rec_cp2_basic_CTC2,
2493 };
2494
2495 static void rec_SPECIAL(struct lightrec_cstate *state,
2496                         const struct block *block, u16 offset)
2497 {
2498         union code c = block->opcode_list[offset].c;
2499         lightrec_rec_func_t f = rec_special[c.r.op];
2500
2501         if (!HAS_DEFAULT_ELM && unlikely(!f))
2502                 unknown_opcode(state, block, offset);
2503         else
2504                 (*f)(state, block, offset);
2505 }
2506
2507 static void rec_REGIMM(struct lightrec_cstate *state,
2508                        const struct block *block, u16 offset)
2509 {
2510         union code c = block->opcode_list[offset].c;
2511         lightrec_rec_func_t f = rec_regimm[c.r.rt];
2512
2513         if (!HAS_DEFAULT_ELM && unlikely(!f))
2514                 unknown_opcode(state, block, offset);
2515         else
2516                 (*f)(state, block, offset);
2517 }
2518
2519 static void rec_CP0(struct lightrec_cstate *state,
2520                     const struct block *block, u16 offset)
2521 {
2522         union code c = block->opcode_list[offset].c;
2523         lightrec_rec_func_t f = rec_cp0[c.r.rs];
2524
2525         if (!HAS_DEFAULT_ELM && unlikely(!f))
2526                 rec_CP(state, block, offset);
2527         else
2528                 (*f)(state, block, offset);
2529 }
2530
2531 static void rec_CP2(struct lightrec_cstate *state,
2532                     const struct block *block, u16 offset)
2533 {
2534         union code c = block->opcode_list[offset].c;
2535
2536         if (c.r.op == OP_CP2_BASIC) {
2537                 lightrec_rec_func_t f = rec_cp2_basic[c.r.rs];
2538
2539                 if (HAS_DEFAULT_ELM || likely(f)) {
2540                         (*f)(state, block, offset);
2541                         return;
2542                 }
2543         }
2544
2545         rec_CP(state, block, offset);
2546 }
2547
2548 void lightrec_rec_opcode(struct lightrec_cstate *state,
2549                          const struct block *block, u16 offset)
2550 {
2551         struct regcache *reg_cache = state->reg_cache;
2552         struct lightrec_branch_target *target;
2553         const struct opcode *op = &block->opcode_list[offset];
2554         jit_state_t *_jit = block->_jit;
2555         lightrec_rec_func_t f;
2556         u16 unload_offset;
2557
2558         if (op_flag_sync(op->flags)) {
2559                 if (state->cycles)
2560                         jit_subi(LIGHTREC_REG_CYCLE, LIGHTREC_REG_CYCLE, state->cycles);
2561                 state->cycles = 0;
2562
2563                 lightrec_storeback_regs(reg_cache, _jit);
2564                 lightrec_regcache_reset(reg_cache);
2565
2566                 pr_debug("Adding branch target at offset 0x%x\n", offset << 2);
2567                 target = &state->targets[state->nb_targets++];
2568                 target->offset = offset;
2569                 target->label = jit_indirect();
2570         }
2571
2572         if (likely(op->opcode)) {
2573                 f = rec_standard[op->i.op];
2574
2575                 if (!HAS_DEFAULT_ELM && unlikely(!f))
2576                         unknown_opcode(state, block, offset);
2577                 else
2578                         (*f)(state, block, offset);
2579         }
2580
2581         if (OPT_EARLY_UNLOAD) {
2582                 unload_offset = offset +
2583                         (has_delay_slot(op->c) && !op_flag_no_ds(op->flags));
2584
2585                 lightrec_do_early_unload(state, block, unload_offset);
2586         }
2587 }