deps/lightrec/emitter.c

   1 // SPDX-License-Identifier: LGPL-2.1-or-later
   2 /*
   3  * Copyright (C) 2014-2021 Paul Cercueil <paul@crapouillou.net>
   4  */
   5
   6 #include "blockcache.h"
   7 #include "debug.h"
   8 #include "disassembler.h"
   9 #include "emitter.h"
  10 #include "lightning-wrapper.h"
  11 #include "optimizer.h"
  12 #include "regcache.h"
  13
  14 #include <stdbool.h>
  15 #include <stddef.h>
  16
  17 typedef void (*lightrec_rec_func_t)(struct lightrec_cstate *, const struct block *, u16);
  18
  19 /* Forward declarations */
  20 static void rec_SPECIAL(struct lightrec_cstate *state, const struct block *block, u16 offset);
  21 static void rec_REGIMM(struct lightrec_cstate *state, const struct block *block, u16 offset);
  22 static void rec_CP0(struct lightrec_cstate *state, const struct block *block, u16 offset);
  23 static void rec_CP2(struct lightrec_cstate *state, const struct block *block, u16 offset);
  24
  25 static void unknown_opcode(struct lightrec_cstate *state, const struct block *block, u16 offset)
  26 {
  27         pr_warn("Unknown opcode: 0x%08x at PC 0x%08x\n",
  28                 block->opcode_list[offset].c.opcode,
  29                 block->pc + (offset << 2));
  30 }
  31
  32 static void lightrec_emit_end_of_block(struct lightrec_cstate *state,
  33                                        const struct block *block, u16 offset,
  34                                        s8 reg_new_pc, u32 imm, u8 ra_reg,
  35                                        u32 link, bool update_cycles)
  36 {
  37         struct regcache *reg_cache = state->reg_cache;
  38         u32 cycles = state->cycles;
  39         jit_state_t *_jit = block->_jit;
  40         const struct opcode *op = &block->opcode_list[offset],
  41                             *next = &block->opcode_list[offset + 1];
  42
  43         jit_note(__FILE__, __LINE__);
  44
  45         if (link) {
  46                 /* Update the $ra register */
  47                 u8 link_reg = lightrec_alloc_reg_out(reg_cache, _jit, ra_reg, 0);
  48                 jit_movi(link_reg, link);
  49                 lightrec_free_reg(reg_cache, link_reg);
  50         }
  51
  52         if (reg_new_pc < 0) {
  53                 reg_new_pc = lightrec_alloc_reg(reg_cache, _jit, JIT_V0);
  54                 lightrec_lock_reg(reg_cache, _jit, reg_new_pc);
  55
  56                 jit_movi(reg_new_pc, imm);
  57         }
  58
  59         if (has_delay_slot(op->c) &&
  60             !(op->flags & (LIGHTREC_NO_DS | LIGHTREC_LOCAL_BRANCH))) {
  61                 cycles += lightrec_cycles_of_opcode(next->c);
  62
  63                 /* Recompile the delay slot */
  64                 if (next->c.opcode)
  65                         lightrec_rec_opcode(state, block, offset + 1);
  66         }
  67
  68         /* Store back remaining registers */
  69         lightrec_storeback_regs(reg_cache, _jit);
  70
  71         jit_movr(JIT_V0, reg_new_pc);
  72
  73         if (cycles && update_cycles) {
  74                 jit_subi(LIGHTREC_REG_CYCLE, LIGHTREC_REG_CYCLE, cycles);
  75                 pr_debug("EOB: %u cycles\n", cycles);
  76         }
  77
  78         if (offset + !!(op->flags & LIGHTREC_NO_DS) < block->nb_ops - 1)
  79                 state->branches[state->nb_branches++] = jit_jmpi();
  80 }
  81
  82 void lightrec_emit_eob(struct lightrec_cstate *state, const struct block *block,
  83                        u16 offset, bool after_op)
  84 {
  85         struct regcache *reg_cache = state->reg_cache;
  86         jit_state_t *_jit = block->_jit;
  87         union code c = block->opcode_list[offset].c;
  88         u32 cycles = state->cycles;
  89
  90         if (!after_op)
  91                 cycles -= lightrec_cycles_of_opcode(c);
  92
  93         lightrec_storeback_regs(reg_cache, _jit);
  94
  95         jit_movi(JIT_V0, block->pc + (offset << 2));
  96         jit_subi(LIGHTREC_REG_CYCLE, LIGHTREC_REG_CYCLE, cycles);
  97
  98         state->branches[state->nb_branches++] = jit_jmpi();
  99 }
 100
 101 static u8 get_jr_jalr_reg(struct lightrec_cstate *state, const struct block *block, u16 offset)
 102 {
 103         struct regcache *reg_cache = state->reg_cache;
 104         jit_state_t *_jit = block->_jit;
 105         const struct opcode *op = &block->opcode_list[offset],
 106                             *next = &block->opcode_list[offset + 1];
 107         u8 rs = lightrec_request_reg_in(reg_cache, _jit, op->r.rs, JIT_V0);
 108
 109         /* If the source register is already mapped to JIT_R0 or JIT_R1, and the
 110          * delay slot is a I/O operation, unload the register, since JIT_R0 and
 111          * JIT_R1 are explicitely used by the I/O opcode generators. */
 112         if ((rs == JIT_R0 || rs == JIT_R1) &&
 113             !(op->flags & LIGHTREC_NO_DS) &&
 114             opcode_is_io(next->c) &&
 115             !(next->flags & (LIGHTREC_NO_INVALIDATE | LIGHTREC_DIRECT_IO))) {
 116                 lightrec_unload_reg(reg_cache, _jit, rs);
 117                 lightrec_free_reg(reg_cache, rs);
 118
 119                 rs = lightrec_request_reg_in(reg_cache, _jit, op->r.rs, JIT_V0);
 120         }
 121
 122         lightrec_lock_reg(reg_cache, _jit, rs);
 123
 124         return rs;
 125 }
 126
 127 static void rec_special_JR(struct lightrec_cstate *state, const struct block *block, u16 offset)
 128 {
 129         u8 rs = get_jr_jalr_reg(state, block, offset);
 130
 131         _jit_name(block->_jit, __func__);
 132         lightrec_emit_end_of_block(state, block, offset, rs, 0, 31, 0, true);
 133 }
 134
 135 static void rec_special_JALR(struct lightrec_cstate *state, const struct block *block, u16 offset)
 136 {
 137         u8 rs = get_jr_jalr_reg(state, block, offset);
 138         union code c = block->opcode_list[offset].c;
 139
 140         _jit_name(block->_jit, __func__);
 141         lightrec_emit_end_of_block(state, block, offset, rs, 0, c.r.rd,
 142                                    get_branch_pc(block, offset, 2), true);
 143 }
 144
 145 static void rec_J(struct lightrec_cstate *state, const struct block *block, u16 offset)
 146 {
 147         union code c = block->opcode_list[offset].c;
 148
 149         _jit_name(block->_jit, __func__);
 150         lightrec_emit_end_of_block(state, block, offset, -1,
 151                                    (block->pc & 0xf0000000) | (c.j.imm << 2),
 152                                    31, 0, true);
 153 }
 154
 155 static void rec_JAL(struct lightrec_cstate *state, const struct block *block, u16 offset)
 156 {
 157         union code c = block->opcode_list[offset].c;
 158
 159         _jit_name(block->_jit, __func__);
 160         lightrec_emit_end_of_block(state, block, offset, -1,
 161                                    (block->pc & 0xf0000000) | (c.j.imm << 2),
 162                                    31, get_branch_pc(block, offset, 2), true);
 163 }
 164
 165 static void rec_b(struct lightrec_cstate *state, const struct block *block, u16 offset,
 166                   jit_code_t code, u32 link, bool unconditional, bool bz)
 167 {
 168         struct regcache *reg_cache = state->reg_cache;
 169         struct native_register *regs_backup;
 170         jit_state_t *_jit = block->_jit;
 171         struct lightrec_branch *branch;
 172         const struct opcode *op = &block->opcode_list[offset],
 173                             *next = &block->opcode_list[offset + 1];
 174         jit_node_t *addr;
 175         u8 link_reg;
 176         u32 target_offset, cycles = state->cycles;
 177         bool is_forward = (s16)op->i.imm >= -1;
 178         u32 next_pc;
 179
 180         jit_note(__FILE__, __LINE__);
 181
 182         if (!(op->flags & LIGHTREC_NO_DS))
 183                 cycles += lightrec_cycles_of_opcode(next->c);
 184
 185         state->cycles = 0;
 186
 187         if (cycles)
 188                 jit_subi(LIGHTREC_REG_CYCLE, LIGHTREC_REG_CYCLE, cycles);
 189
 190         if (!unconditional) {
 191                 u8 rs = lightrec_alloc_reg_in(reg_cache, _jit, op->i.rs, REG_EXT),
 192                    rt = bz ? 0 : lightrec_alloc_reg_in(reg_cache,
 193                                                        _jit, op->i.rt, REG_EXT);
 194
 195                 /* Generate the branch opcode */
 196                 addr = jit_new_node_pww(code, NULL, rs, rt);
 197
 198                 lightrec_free_regs(reg_cache);
 199                 regs_backup = lightrec_regcache_enter_branch(reg_cache);
 200         }
 201
 202         if (op->flags & LIGHTREC_LOCAL_BRANCH) {
 203                 if (next && !(op->flags & LIGHTREC_NO_DS)) {
 204                         /* Recompile the delay slot */
 205                         if (next->opcode)
 206                                 lightrec_rec_opcode(state, block, offset + 1);
 207                 }
 208
 209                 if (link) {
 210                         /* Update the $ra register */
 211                         link_reg = lightrec_alloc_reg_out(reg_cache, _jit, 31, 0);
 212                         jit_movi(link_reg, link);
 213                         lightrec_free_reg(reg_cache, link_reg);
 214                 }
 215
 216                 /* Store back remaining registers */
 217                 lightrec_storeback_regs(reg_cache, _jit);
 218
 219                 target_offset = offset + 1 + (s16)op->i.imm
 220                         - !!(OPT_SWITCH_DELAY_SLOTS && (op->flags & LIGHTREC_NO_DS));
 221                 pr_debug("Adding local branch to offset 0x%x\n",
 222                          target_offset << 2);
 223                 branch = &state->local_branches[
 224                         state->nb_local_branches++];
 225
 226                 branch->target = target_offset;
 227                 if (is_forward)
 228                         branch->branch = jit_jmpi();
 229                 else
 230                         branch->branch = jit_bgti(LIGHTREC_REG_CYCLE, 0);
 231         }
 232
 233         if (!(op->flags & LIGHTREC_LOCAL_BRANCH) || !is_forward) {
 234                 next_pc = get_branch_pc(block, offset, 1 + (s16)op->i.imm);
 235                 lightrec_emit_end_of_block(state, block, offset, -1, next_pc,
 236                                            31, link, false);
 237         }
 238
 239         if (!unconditional) {
 240                 jit_patch(addr);
 241                 lightrec_regcache_leave_branch(reg_cache, regs_backup);
 242
 243                 if (bz && link) {
 244                         /* Update the $ra register */
 245                         link_reg = lightrec_alloc_reg_out(reg_cache, _jit,
 246                                                           31, REG_EXT);
 247                         jit_movi(link_reg, (s32)link);
 248                         lightrec_free_reg(reg_cache, link_reg);
 249                 }
 250
 251                 if (!(op->flags & LIGHTREC_NO_DS) && next->opcode)
 252                         lightrec_rec_opcode(state, block, offset + 1);
 253         }
 254 }
 255
 256 static void rec_BNE(struct lightrec_cstate *state,
 257                     const struct block *block, u16 offset)
 258 {
 259         union code c = block->opcode_list[offset].c;
 260
 261         _jit_name(block->_jit, __func__);
 262
 263         if (c.i.rt == 0)
 264                 rec_b(state, block, offset, jit_code_beqi, 0, false, true);
 265         else
 266                 rec_b(state, block, offset, jit_code_beqr, 0, false, false);
 267 }
 268
 269 static void rec_BEQ(struct lightrec_cstate *state,
 270                     const struct block *block, u16 offset)
 271 {
 272         union code c = block->opcode_list[offset].c;
 273
 274         _jit_name(block->_jit, __func__);
 275
 276         if (c.i.rt == 0)
 277                 rec_b(state, block, offset, jit_code_bnei, 0, c.i.rs == 0, true);
 278         else
 279                 rec_b(state, block, offset, jit_code_bner, 0, c.i.rs == c.i.rt, false);
 280 }
 281
 282 static void rec_BLEZ(struct lightrec_cstate *state,
 283                      const struct block *block, u16 offset)
 284 {
 285         union code c = block->opcode_list[offset].c;
 286
 287         _jit_name(block->_jit, __func__);
 288         rec_b(state, block, offset, jit_code_bgti, 0, c.i.rs == 0, true);
 289 }
 290
 291 static void rec_BGTZ(struct lightrec_cstate *state,
 292                      const struct block *block, u16 offset)
 293 {
 294         _jit_name(block->_jit, __func__);
 295         rec_b(state, block, offset, jit_code_blei, 0, false, true);
 296 }
 297
 298 static void rec_regimm_BLTZ(struct lightrec_cstate *state,
 299                             const struct block *block, u16 offset)
 300 {
 301         _jit_name(block->_jit, __func__);
 302         rec_b(state, block, offset, jit_code_bgei, 0, false, true);
 303 }
 304
 305 static void rec_regimm_BLTZAL(struct lightrec_cstate *state,
 306                               const struct block *block, u16 offset)
 307 {
 308         _jit_name(block->_jit, __func__);
 309         rec_b(state, block, offset, jit_code_bgei,
 310               get_branch_pc(block, offset, 2), false, true);
 311 }
 312
 313 static void rec_regimm_BGEZ(struct lightrec_cstate *state,
 314                             const struct block *block, u16 offset)
 315 {
 316         union code c = block->opcode_list[offset].c;
 317
 318         _jit_name(block->_jit, __func__);
 319         rec_b(state, block, offset, jit_code_blti, 0, !c.i.rs, true);
 320 }
 321
 322 static void rec_regimm_BGEZAL(struct lightrec_cstate *state,
 323                               const struct block *block, u16 offset)
 324 {
 325         const struct opcode *op = &block->opcode_list[offset];
 326         _jit_name(block->_jit, __func__);
 327         rec_b(state, block, offset, jit_code_blti,
 328               get_branch_pc(block, offset, 2),
 329               !op->i.rs, true);
 330 }
 331
 332 static void rec_alu_imm(struct lightrec_cstate *state, const struct block *block,
 333                         u16 offset, jit_code_t code, bool slti)
 334 {
 335         struct regcache *reg_cache = state->reg_cache;
 336         union code c = block->opcode_list[offset].c;
 337         jit_state_t *_jit = block->_jit;
 338         u8 rs, rt, out_flags = REG_EXT;
 339
 340         if (slti)
 341                 out_flags |= REG_ZEXT;
 342
 343         jit_note(__FILE__, __LINE__);
 344         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rs, REG_EXT);
 345         rt = lightrec_alloc_reg_out(reg_cache, _jit, c.i.rt, out_flags);
 346
 347         jit_new_node_www(code, rt, rs, (s32)(s16) c.i.imm);
 348
 349         lightrec_free_reg(reg_cache, rs);
 350         lightrec_free_reg(reg_cache, rt);
 351 }
 352
 353 static void rec_alu_special(struct lightrec_cstate *state, const struct block *block,
 354                             u16 offset, jit_code_t code, bool out_ext)
 355 {
 356         struct regcache *reg_cache = state->reg_cache;
 357         union code c = block->opcode_list[offset].c;
 358         jit_state_t *_jit = block->_jit;
 359         u8 rd, rt, rs;
 360
 361         jit_note(__FILE__, __LINE__);
 362         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rs, REG_EXT);
 363         rt = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rt, REG_EXT);
 364         rd = lightrec_alloc_reg_out(reg_cache, _jit, c.r.rd,
 365                                     out_ext ? REG_EXT | REG_ZEXT : 0);
 366
 367         jit_new_node_www(code, rd, rs, rt);
 368
 369         lightrec_free_reg(reg_cache, rs);
 370         lightrec_free_reg(reg_cache, rt);
 371         lightrec_free_reg(reg_cache, rd);
 372 }
 373
 374 static void rec_alu_shiftv(struct lightrec_cstate *state, const struct block *block,
 375                            u16 offset, jit_code_t code)
 376 {
 377         struct regcache *reg_cache = state->reg_cache;
 378         union code c = block->opcode_list[offset].c;
 379         jit_state_t *_jit = block->_jit;
 380         u8 rd, rt, rs, temp, flags = 0;
 381
 382         jit_note(__FILE__, __LINE__);
 383         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rs, 0);
 384
 385         if (code == jit_code_rshr)
 386                 flags = REG_EXT;
 387         else if (code == jit_code_rshr_u)
 388                 flags = REG_ZEXT;
 389
 390         rt = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rt, flags);
 391         rd = lightrec_alloc_reg_out(reg_cache, _jit, c.r.rd, flags);
 392
 393         if (rs != rd && rt != rd) {
 394                 jit_andi(rd, rs, 0x1f);
 395                 jit_new_node_www(code, rd, rt, rd);
 396         } else {
 397                 temp = lightrec_alloc_reg_temp(reg_cache, _jit);
 398                 jit_andi(temp, rs, 0x1f);
 399                 jit_new_node_www(code, rd, rt, temp);
 400                 lightrec_free_reg(reg_cache, temp);
 401         }
 402
 403         lightrec_free_reg(reg_cache, rs);
 404         lightrec_free_reg(reg_cache, rt);
 405         lightrec_free_reg(reg_cache, rd);
 406 }
 407
 408 static void rec_ADDIU(struct lightrec_cstate *state,
 409                       const struct block *block, u16 offset)
 410 {
 411         _jit_name(block->_jit, __func__);
 412         rec_alu_imm(state, block, offset, jit_code_addi, false);
 413 }
 414
 415 static void rec_ADDI(struct lightrec_cstate *state,
 416                      const struct block *block, u16 offset)
 417 {
 418         /* TODO: Handle the exception? */
 419         _jit_name(block->_jit, __func__);
 420         rec_alu_imm(state, block, offset, jit_code_addi, false);
 421 }
 422
 423 static void rec_SLTIU(struct lightrec_cstate *state,
 424                       const struct block *block, u16 offset)
 425 {
 426         _jit_name(block->_jit, __func__);
 427         rec_alu_imm(state, block, offset, jit_code_lti_u, true);
 428 }
 429
 430 static void rec_SLTI(struct lightrec_cstate *state,
 431                      const struct block *block, u16 offset)
 432 {
 433         _jit_name(block->_jit, __func__);
 434         rec_alu_imm(state, block, offset, jit_code_lti, true);
 435 }
 436
 437 static void rec_ANDI(struct lightrec_cstate *state,
 438                      const struct block *block, u16 offset)
 439 {
 440         struct regcache *reg_cache = state->reg_cache;
 441         union code c = block->opcode_list[offset].c;
 442         jit_state_t *_jit = block->_jit;
 443         u8 rs, rt;
 444
 445         _jit_name(block->_jit, __func__);
 446         jit_note(__FILE__, __LINE__);
 447         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rs, 0);
 448         rt = lightrec_alloc_reg_out(reg_cache, _jit, c.i.rt,
 449                                     REG_EXT | REG_ZEXT);
 450
 451         /* PSX code uses ANDI 0xff / ANDI 0xffff a lot, which are basically
 452          * casts to uint8_t / uint16_t. */
 453         if (c.i.imm == 0xff)
 454                 jit_extr_uc(rt, rs);
 455         else if (c.i.imm == 0xffff)
 456                 jit_extr_us(rt, rs);
 457         else
 458                 jit_andi(rt, rs, (u32)(u16) c.i.imm);
 459
 460         lightrec_free_reg(reg_cache, rs);
 461         lightrec_free_reg(reg_cache, rt);
 462 }
 463
 464 static void rec_alu_or_xor(struct lightrec_cstate *state, const struct block *block,
 465                            u16 offset, jit_code_t code)
 466 {
 467         struct regcache *reg_cache = state->reg_cache;
 468         union code c = block->opcode_list[offset].c;
 469         jit_state_t *_jit = block->_jit;
 470         u8 rs, rt, flags;
 471
 472         jit_note(__FILE__, __LINE__);
 473         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rs, 0);
 474         rt = lightrec_alloc_reg_out(reg_cache, _jit, c.i.rt, 0);
 475
 476         flags = lightrec_get_reg_in_flags(reg_cache, rs);
 477         lightrec_set_reg_out_flags(reg_cache, rt, flags);
 478
 479         jit_new_node_www(code, rt, rs, (u32)(u16) c.i.imm);
 480
 481         lightrec_free_reg(reg_cache, rs);
 482         lightrec_free_reg(reg_cache, rt);
 483 }
 484
 485
 486 static void rec_ORI(struct lightrec_cstate *state,
 487                     const struct block *block, u16 offset)
 488 {
 489         _jit_name(block->_jit, __func__);
 490         rec_alu_or_xor(state, block, offset, jit_code_ori);
 491 }
 492
 493 static void rec_XORI(struct lightrec_cstate *state,
 494                      const struct block *block, u16 offset)
 495 {
 496         _jit_name(block->_jit, __func__);
 497         rec_alu_or_xor(state, block, offset, jit_code_xori);
 498 }
 499
 500 static void rec_LUI(struct lightrec_cstate *state,
 501                     const struct block *block, u16 offset)
 502 {
 503         struct regcache *reg_cache = state->reg_cache;
 504         union code c = block->opcode_list[offset].c;
 505         jit_state_t *_jit = block->_jit;
 506         u8 rt, flags = REG_EXT;
 507
 508         jit_name(__func__);
 509         jit_note(__FILE__, __LINE__);
 510
 511         if (!(c.i.imm & BIT(15)))
 512                 flags |= REG_ZEXT;
 513
 514         rt = lightrec_alloc_reg_out(reg_cache, _jit, c.i.rt, flags);
 515
 516         jit_movi(rt, (s32)(c.i.imm << 16));
 517
 518         lightrec_free_reg(reg_cache, rt);
 519 }
 520
 521 static void rec_special_ADDU(struct lightrec_cstate *state,
 522                              const struct block *block, u16 offset)
 523 {
 524         _jit_name(block->_jit, __func__);
 525         rec_alu_special(state, block, offset, jit_code_addr, false);
 526 }
 527
 528 static void rec_special_ADD(struct lightrec_cstate *state,
 529                             const struct block *block, u16 offset)
 530 {
 531         /* TODO: Handle the exception? */
 532         _jit_name(block->_jit, __func__);
 533         rec_alu_special(state, block, offset, jit_code_addr, false);
 534 }
 535
 536 static void rec_special_SUBU(struct lightrec_cstate *state,
 537                              const struct block *block, u16 offset)
 538 {
 539         _jit_name(block->_jit, __func__);
 540         rec_alu_special(state, block, offset, jit_code_subr, false);
 541 }
 542
 543 static void rec_special_SUB(struct lightrec_cstate *state,
 544                             const struct block *block, u16 offset)
 545 {
 546         /* TODO: Handle the exception? */
 547         _jit_name(block->_jit, __func__);
 548         rec_alu_special(state, block, offset, jit_code_subr, false);
 549 }
 550
 551 static void rec_special_AND(struct lightrec_cstate *state,
 552                             const struct block *block, u16 offset)
 553 {
 554         struct regcache *reg_cache = state->reg_cache;
 555         union code c = block->opcode_list[offset].c;
 556         jit_state_t *_jit = block->_jit;
 557         u8 rd, rt, rs, flags_rs, flags_rt, flags_rd;
 558
 559         _jit_name(block->_jit, __func__);
 560         jit_note(__FILE__, __LINE__);
 561         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rs, 0);
 562         rt = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rt, 0);
 563         rd = lightrec_alloc_reg_out(reg_cache, _jit, c.r.rd, 0);
 564
 565         flags_rs = lightrec_get_reg_in_flags(reg_cache, rs);
 566         flags_rt = lightrec_get_reg_in_flags(reg_cache, rt);
 567
 568         /* Z(rd) = Z(rs) | Z(rt) */
 569         flags_rd = REG_ZEXT & (flags_rs | flags_rt);
 570
 571         /* E(rd) = (E(rt) & Z(rt)) | (E(rs) & Z(rs)) | (E(rs) & E(rt)) */
 572         if (((flags_rs & REG_EXT) && (flags_rt & REG_ZEXT)) ||
 573             ((flags_rt & REG_EXT) && (flags_rs & REG_ZEXT)) ||
 574             (REG_EXT & flags_rs & flags_rt))
 575                 flags_rd |= REG_EXT;
 576
 577         lightrec_set_reg_out_flags(reg_cache, rd, flags_rd);
 578
 579         jit_andr(rd, rs, rt);
 580
 581         lightrec_free_reg(reg_cache, rs);
 582         lightrec_free_reg(reg_cache, rt);
 583         lightrec_free_reg(reg_cache, rd);
 584 }
 585
 586 static void rec_special_or_nor(struct lightrec_cstate *state,
 587                                const struct block *block, u16 offset, bool nor)
 588 {
 589         struct regcache *reg_cache = state->reg_cache;
 590         union code c = block->opcode_list[offset].c;
 591         jit_state_t *_jit = block->_jit;
 592         u8 rd, rt, rs, flags_rs, flags_rt, flags_rd = 0;
 593
 594         jit_note(__FILE__, __LINE__);
 595         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rs, 0);
 596         rt = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rt, 0);
 597         rd = lightrec_alloc_reg_out(reg_cache, _jit, c.r.rd, 0);
 598
 599         flags_rs = lightrec_get_reg_in_flags(reg_cache, rs);
 600         flags_rt = lightrec_get_reg_in_flags(reg_cache, rt);
 601
 602         /* or: Z(rd) = Z(rs) & Z(rt)
 603          * nor: Z(rd) = 0 */
 604         if (!nor)
 605                 flags_rd = REG_ZEXT & flags_rs & flags_rt;
 606
 607         /* E(rd) = (E(rs) & E(rt)) | (E(rt) & !Z(rt)) | (E(rs) & !Z(rs)) */
 608         if ((REG_EXT & flags_rs & flags_rt) ||
 609             (flags_rt & (REG_EXT | REG_ZEXT) == REG_EXT) ||
 610             (flags_rs & (REG_EXT | REG_ZEXT) == REG_EXT))
 611                 flags_rd |= REG_EXT;
 612
 613         lightrec_set_reg_out_flags(reg_cache, rd, flags_rd);
 614
 615         jit_orr(rd, rs, rt);
 616
 617         if (nor)
 618                 jit_comr(rd, rd);
 619
 620         lightrec_free_reg(reg_cache, rs);
 621         lightrec_free_reg(reg_cache, rt);
 622         lightrec_free_reg(reg_cache, rd);
 623 }
 624
 625 static void rec_special_OR(struct lightrec_cstate *state,
 626                            const struct block *block, u16 offset)
 627 {
 628         _jit_name(block->_jit, __func__);
 629         rec_special_or_nor(state, block, offset, false);
 630 }
 631
 632 static void rec_special_NOR(struct lightrec_cstate *state,
 633                             const struct block *block, u16 offset)
 634 {
 635         _jit_name(block->_jit, __func__);
 636         rec_special_or_nor(state, block, offset, true);
 637 }
 638
 639 static void rec_special_XOR(struct lightrec_cstate *state,
 640                             const struct block *block, u16 offset)
 641 {
 642         struct regcache *reg_cache = state->reg_cache;
 643         union code c = block->opcode_list[offset].c;
 644         jit_state_t *_jit = block->_jit;
 645         u8 rd, rt, rs, flags_rs, flags_rt, flags_rd;
 646
 647         _jit_name(block->_jit, __func__);
 648
 649         jit_note(__FILE__, __LINE__);
 650         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rs, 0);
 651         rt = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rt, 0);
 652         rd = lightrec_alloc_reg_out(reg_cache, _jit, c.r.rd, 0);
 653
 654         flags_rs = lightrec_get_reg_in_flags(reg_cache, rs);
 655         flags_rt = lightrec_get_reg_in_flags(reg_cache, rt);
 656
 657         /* Z(rd) = Z(rs) & Z(rt) */
 658         flags_rd = REG_ZEXT & flags_rs & flags_rt;
 659
 660         /* E(rd) = E(rs) & E(rt) */
 661         flags_rd |= REG_EXT & flags_rs & flags_rt;
 662
 663         lightrec_set_reg_out_flags(reg_cache, rd, flags_rd);
 664
 665         jit_xorr(rd, rs, rt);
 666
 667         lightrec_free_reg(reg_cache, rs);
 668         lightrec_free_reg(reg_cache, rt);
 669         lightrec_free_reg(reg_cache, rd);
 670 }
 671
 672 static void rec_special_SLTU(struct lightrec_cstate *state,
 673                              const struct block *block, u16 offset)
 674 {
 675         _jit_name(block->_jit, __func__);
 676         rec_alu_special(state, block, offset, jit_code_ltr_u, true);
 677 }
 678
 679 static void rec_special_SLT(struct lightrec_cstate *state,
 680                             const struct block *block, u16 offset)
 681 {
 682         _jit_name(block->_jit, __func__);
 683         rec_alu_special(state, block, offset, jit_code_ltr, true);
 684 }
 685
 686 static void rec_special_SLLV(struct lightrec_cstate *state,
 687                              const struct block *block, u16 offset)
 688 {
 689         _jit_name(block->_jit, __func__);
 690         rec_alu_shiftv(state, block, offset, jit_code_lshr);
 691 }
 692
 693 static void rec_special_SRLV(struct lightrec_cstate *state,
 694                              const struct block *block, u16 offset)
 695 {
 696         _jit_name(block->_jit, __func__);
 697         rec_alu_shiftv(state, block, offset, jit_code_rshr_u);
 698 }
 699
 700 static void rec_special_SRAV(struct lightrec_cstate *state,
 701                              const struct block *block, u16 offset)
 702 {
 703         _jit_name(block->_jit, __func__);
 704         rec_alu_shiftv(state, block, offset, jit_code_rshr);
 705 }
 706
 707 static void rec_alu_shift(struct lightrec_cstate *state, const struct block *block,
 708                           u16 offset, jit_code_t code)
 709 {
 710         struct regcache *reg_cache = state->reg_cache;
 711         union code c = block->opcode_list[offset].c;
 712         jit_state_t *_jit = block->_jit;
 713         u8 rd, rt, flags = 0;
 714
 715         jit_note(__FILE__, __LINE__);
 716
 717         if (code == jit_code_rshi)
 718                 flags = REG_EXT;
 719         else if (code == jit_code_rshi_u)
 720                 flags = REG_ZEXT;
 721
 722         rt = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rt, flags);
 723
 724         /* Input reg is zero-extended, if we SRL at least by one bit, we know
 725          * the output reg will be both zero-extended and sign-extended. */
 726         if (code == jit_code_rshi_u && c.r.imm)
 727                 flags |= REG_EXT;
 728         rd = lightrec_alloc_reg_out(reg_cache, _jit, c.r.rd, flags);
 729
 730         jit_new_node_www(code, rd, rt, c.r.imm);
 731
 732         lightrec_free_reg(reg_cache, rt);
 733         lightrec_free_reg(reg_cache, rd);
 734 }
 735
 736 static void rec_special_SLL(struct lightrec_cstate *state,
 737                             const struct block *block, u16 offset)
 738 {
 739         _jit_name(block->_jit, __func__);
 740         rec_alu_shift(state, block, offset, jit_code_lshi);
 741 }
 742
 743 static void rec_special_SRL(struct lightrec_cstate *state,
 744                             const struct block *block, u16 offset)
 745 {
 746         _jit_name(block->_jit, __func__);
 747         rec_alu_shift(state, block, offset, jit_code_rshi_u);
 748 }
 749
 750 static void rec_special_SRA(struct lightrec_cstate *state,
 751                             const struct block *block, u16 offset)
 752 {
 753         _jit_name(block->_jit, __func__);
 754         rec_alu_shift(state, block, offset, jit_code_rshi);
 755 }
 756
 757 static void rec_alu_mult(struct lightrec_cstate *state,
 758                          const struct block *block, u16 offset, bool is_signed)
 759 {
 760         struct regcache *reg_cache = state->reg_cache;
 761         union code c = block->opcode_list[offset].c;
 762         u16 flags = block->opcode_list[offset].flags;
 763         u8 reg_lo = get_mult_div_lo(c);
 764         u8 reg_hi = get_mult_div_hi(c);
 765         jit_state_t *_jit = block->_jit;
 766         u8 lo, hi, rs, rt, rflags = 0;
 767
 768         jit_note(__FILE__, __LINE__);
 769
 770         if (is_signed)
 771                 rflags = REG_EXT;
 772         else
 773                 rflags = REG_ZEXT;
 774
 775         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rs, rflags);
 776         rt = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rt, rflags);
 777
 778         if (!(flags & LIGHTREC_NO_LO))
 779                 lo = lightrec_alloc_reg_out(reg_cache, _jit, reg_lo, 0);
 780         else if (__WORDSIZE == 32)
 781                 lo = lightrec_alloc_reg_temp(reg_cache, _jit);
 782
 783         if (!(flags & LIGHTREC_NO_HI))
 784                 hi = lightrec_alloc_reg_out(reg_cache, _jit, reg_hi, REG_EXT);
 785
 786         if (__WORDSIZE == 32) {
 787                 /* On 32-bit systems, do a 32*32->64 bit operation, or a 32*32->32 bit
 788                  * operation if the MULT was detected a 32-bit only. */
 789                 if (!(flags & LIGHTREC_NO_HI)) {
 790                         if (is_signed)
 791                                 jit_qmulr(lo, hi, rs, rt);
 792                         else
 793                                 jit_qmulr_u(lo, hi, rs, rt);
 794                 } else {
 795                         jit_mulr(lo, rs, rt);
 796                 }
 797         } else {
 798                 /* On 64-bit systems, do a 64*64->64 bit operation. */
 799                 if (flags & LIGHTREC_NO_LO) {
 800                         jit_mulr(hi, rs, rt);
 801                         jit_rshi(hi, hi, 32);
 802                 } else {
 803                         jit_mulr(lo, rs, rt);
 804
 805                         /* The 64-bit output value is in $lo, store the upper 32 bits in $hi */
 806                         if (!(flags & LIGHTREC_NO_HI))
 807                                 jit_rshi(hi, lo, 32);
 808                 }
 809         }
 810
 811         lightrec_free_reg(reg_cache, rs);
 812         lightrec_free_reg(reg_cache, rt);
 813         if (!(flags & LIGHTREC_NO_LO) || __WORDSIZE == 32)
 814                 lightrec_free_reg(reg_cache, lo);
 815         if (!(flags & LIGHTREC_NO_HI))
 816                 lightrec_free_reg(reg_cache, hi);
 817 }
 818
 819 static void rec_alu_div(struct lightrec_cstate *state,
 820                         const struct block *block, u16 offset, bool is_signed)
 821 {
 822         struct regcache *reg_cache = state->reg_cache;
 823         union code c = block->opcode_list[offset].c;
 824         u16 flags = block->opcode_list[offset].flags;
 825         bool no_check = flags & LIGHTREC_NO_DIV_CHECK;
 826         u8 reg_lo = get_mult_div_lo(c);
 827         u8 reg_hi = get_mult_div_hi(c);
 828         jit_state_t *_jit = block->_jit;
 829         jit_node_t *branch, *to_end;
 830         u8 lo = 0, hi = 0, rs, rt, rflags = 0;
 831
 832         jit_note(__FILE__, __LINE__);
 833
 834         if (is_signed)
 835                 rflags = REG_EXT;
 836         else
 837                 rflags = REG_ZEXT;
 838
 839         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rs, rflags);
 840         rt = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rt, rflags);
 841
 842         if (!(flags & LIGHTREC_NO_LO))
 843                 lo = lightrec_alloc_reg_out(reg_cache, _jit, reg_lo, 0);
 844
 845         if (!(flags & LIGHTREC_NO_HI))
 846                 hi = lightrec_alloc_reg_out(reg_cache, _jit, reg_hi, 0);
 847
 848         /* Jump to special handler if dividing by zero  */
 849         if (!no_check)
 850                 branch = jit_beqi(rt, 0);
 851
 852         if (flags & LIGHTREC_NO_LO) {
 853                 if (is_signed)
 854                         jit_remr(hi, rs, rt);
 855                 else
 856                         jit_remr_u(hi, rs, rt);
 857         } else if (flags & LIGHTREC_NO_HI) {
 858                 if (is_signed)
 859                         jit_divr(lo, rs, rt);
 860                 else
 861                         jit_divr_u(lo, rs, rt);
 862         } else {
 863                 if (is_signed)
 864                         jit_qdivr(lo, hi, rs, rt);
 865                 else
 866                         jit_qdivr_u(lo, hi, rs, rt);
 867         }
 868
 869         if (!no_check) {
 870                 lightrec_regcache_mark_live(reg_cache, _jit);
 871
 872                 /* Jump above the div-by-zero handler */
 873                 to_end = jit_jmpi();
 874
 875                 jit_patch(branch);
 876
 877                 if (!(flags & LIGHTREC_NO_LO)) {
 878                         if (is_signed) {
 879                                 jit_lti(lo, rs, 0);
 880                                 jit_lshi(lo, lo, 1);
 881                                 jit_subi(lo, lo, 1);
 882                         } else {
 883                                 jit_movi(lo, 0xffffffff);
 884                         }
 885                 }
 886
 887                 if (!(flags & LIGHTREC_NO_HI))
 888                         jit_movr(hi, rs);
 889
 890                 jit_patch(to_end);
 891         }
 892
 893         lightrec_free_reg(reg_cache, rs);
 894         lightrec_free_reg(reg_cache, rt);
 895
 896         if (!(flags & LIGHTREC_NO_LO))
 897                 lightrec_free_reg(reg_cache, lo);
 898
 899         if (!(flags & LIGHTREC_NO_HI))
 900                 lightrec_free_reg(reg_cache, hi);
 901 }
 902
 903 static void rec_special_MULT(struct lightrec_cstate *state,
 904                              const struct block *block, u16 offset)
 905 {
 906         _jit_name(block->_jit, __func__);
 907         rec_alu_mult(state, block, offset, true);
 908 }
 909
 910 static void rec_special_MULTU(struct lightrec_cstate *state,
 911                               const struct block *block, u16 offset)
 912 {
 913         _jit_name(block->_jit, __func__);
 914         rec_alu_mult(state, block, offset, false);
 915 }
 916
 917 static void rec_special_DIV(struct lightrec_cstate *state,
 918                             const struct block *block, u16 offset)
 919 {
 920         _jit_name(block->_jit, __func__);
 921         rec_alu_div(state, block, offset, true);
 922 }
 923
 924 static void rec_special_DIVU(struct lightrec_cstate *state,
 925                              const struct block *block, u16 offset)
 926 {
 927         _jit_name(block->_jit, __func__);
 928         rec_alu_div(state, block, offset, false);
 929 }
 930
 931 static void rec_alu_mv_lo_hi(struct lightrec_cstate *state,
 932                              const struct block *block, u8 dst, u8 src)
 933 {
 934         struct regcache *reg_cache = state->reg_cache;
 935         jit_state_t *_jit = block->_jit;
 936
 937         jit_note(__FILE__, __LINE__);
 938         src = lightrec_alloc_reg_in(reg_cache, _jit, src, 0);
 939         dst = lightrec_alloc_reg_out(reg_cache, _jit, dst, REG_EXT);
 940
 941         jit_extr_i(dst, src);
 942
 943         lightrec_free_reg(reg_cache, src);
 944         lightrec_free_reg(reg_cache, dst);
 945 }
 946
 947 static void rec_special_MFHI(struct lightrec_cstate *state,
 948                              const struct block *block, u16 offset)
 949 {
 950         union code c = block->opcode_list[offset].c;
 951
 952         _jit_name(block->_jit, __func__);
 953         rec_alu_mv_lo_hi(state, block, c.r.rd, REG_HI);
 954 }
 955
 956 static void rec_special_MTHI(struct lightrec_cstate *state,
 957                              const struct block *block, u16 offset)
 958 {
 959         union code c = block->opcode_list[offset].c;
 960
 961         _jit_name(block->_jit, __func__);
 962         rec_alu_mv_lo_hi(state, block, REG_HI, c.r.rs);
 963 }
 964
 965 static void rec_special_MFLO(struct lightrec_cstate *state,
 966                              const struct block *block, u16 offset)
 967 {
 968         union code c = block->opcode_list[offset].c;
 969
 970         _jit_name(block->_jit, __func__);
 971         rec_alu_mv_lo_hi(state, block, c.r.rd, REG_LO);
 972 }
 973
 974 static void rec_special_MTLO(struct lightrec_cstate *state,
 975                              const struct block *block, u16 offset)
 976 {
 977         union code c = block->opcode_list[offset].c;
 978
 979         _jit_name(block->_jit, __func__);
 980         rec_alu_mv_lo_hi(state, block, REG_LO, c.r.rs);
 981 }
 982
 983 static void call_to_c_wrapper(struct lightrec_cstate *state, const struct block *block,
 984                               u32 arg, bool with_arg, enum c_wrappers wrapper)
 985 {
 986         struct regcache *reg_cache = state->reg_cache;
 987         jit_state_t *_jit = block->_jit;
 988         u8 tmp, tmp3;
 989
 990         if (with_arg)
 991                 tmp3 = lightrec_alloc_reg(reg_cache, _jit, JIT_R1);
 992         tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
 993
 994         jit_ldxi(tmp, LIGHTREC_REG_STATE,
 995                  offsetof(struct lightrec_state, wrappers_eps[wrapper]));
 996         if (with_arg)
 997                 jit_movi(tmp3, arg);
 998
 999         jit_callr(tmp);
1000
1001         lightrec_free_reg(reg_cache, tmp);
1002         if (with_arg)
1003                 lightrec_free_reg(reg_cache, tmp3);
1004         lightrec_regcache_mark_live(reg_cache, _jit);
1005 }
1006
1007 static void rec_io(struct lightrec_cstate *state,
1008                    const struct block *block, u16 offset,
1009                    bool load_rt, bool read_rt)
1010 {
1011         struct regcache *reg_cache = state->reg_cache;
1012         jit_state_t *_jit = block->_jit;
1013         union code c = block->opcode_list[offset].c;
1014         u16 flags = block->opcode_list[offset].flags;
1015         bool is_tagged = flags & (LIGHTREC_HW_IO | LIGHTREC_DIRECT_IO);
1016         u32 lut_entry;
1017
1018         jit_note(__FILE__, __LINE__);
1019
1020         lightrec_clean_reg_if_loaded(reg_cache, _jit, c.i.rs, false);
1021
1022         if (read_rt && likely(c.i.rt))
1023                 lightrec_clean_reg_if_loaded(reg_cache, _jit, c.i.rt, true);
1024         else if (load_rt)
1025                 lightrec_clean_reg_if_loaded(reg_cache, _jit, c.i.rt, false);
1026
1027         if (is_tagged) {
1028                 call_to_c_wrapper(state, block, c.opcode, true, C_WRAPPER_RW);
1029         } else {
1030                 lut_entry = lightrec_get_lut_entry(block);
1031                 call_to_c_wrapper(state, block, (lut_entry << 16) | offset,
1032                                   true, C_WRAPPER_RW_GENERIC);
1033         }
1034 }
1035
1036 static void rec_store_direct_no_invalidate(struct lightrec_cstate *cstate,
1037                                            const struct block *block,
1038                                            u16 offset, jit_code_t code)
1039 {
1040         struct lightrec_state *state = cstate->state;
1041         struct regcache *reg_cache = cstate->reg_cache;
1042         union code c = block->opcode_list[offset].c;
1043         jit_state_t *_jit = block->_jit;
1044         jit_node_t *to_not_ram, *to_end;
1045         u8 tmp, tmp2, rs, rt;
1046         s16 imm;
1047
1048         jit_note(__FILE__, __LINE__);
1049         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rs, 0);
1050         tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
1051
1052         if (state->offset_ram || state->offset_scratch)
1053                 tmp2 = lightrec_alloc_reg_temp(reg_cache, _jit);
1054
1055         /* Convert to KUNSEG and avoid RAM mirrors */
1056         if (state->mirrors_mapped) {
1057                 imm = (s16)c.i.imm;
1058                 jit_andi(tmp, rs, 0x1f800000 | (4 * RAM_SIZE - 1));
1059         } else if (c.i.imm) {
1060                 imm = 0;
1061                 jit_addi(tmp, rs, (s16)c.i.imm);
1062                 jit_andi(tmp, tmp, 0x1f800000 | (RAM_SIZE - 1));
1063         } else {
1064                 imm = 0;
1065                 jit_andi(tmp, rs, 0x1f800000 | (RAM_SIZE - 1));
1066         }
1067
1068         lightrec_free_reg(reg_cache, rs);
1069
1070         if (state->offset_ram != state->offset_scratch) {
1071                 to_not_ram = jit_bmsi(tmp, BIT(28));
1072
1073                 lightrec_regcache_mark_live(reg_cache, _jit);
1074
1075                 jit_movi(tmp2, state->offset_ram);
1076
1077                 to_end = jit_jmpi();
1078                 jit_patch(to_not_ram);
1079
1080                 jit_movi(tmp2, state->offset_scratch);
1081                 jit_patch(to_end);
1082         } else if (state->offset_ram) {
1083                 jit_movi(tmp2, state->offset_ram);
1084         }
1085
1086         if (state->offset_ram || state->offset_scratch) {
1087                 jit_addr(tmp, tmp, tmp2);
1088                 lightrec_free_reg(reg_cache, tmp2);
1089         }
1090
1091         rt = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rt, 0);
1092         jit_new_node_www(code, imm, tmp, rt);
1093
1094         lightrec_free_reg(reg_cache, rt);
1095         lightrec_free_reg(reg_cache, tmp);
1096 }
1097
1098 static void rec_store_direct(struct lightrec_cstate *cstate, const struct block *block,
1099                              u16 offset, jit_code_t code)
1100 {
1101         struct lightrec_state *state = cstate->state;
1102         u32 ram_size = state->mirrors_mapped ? RAM_SIZE * 4 : RAM_SIZE;
1103         struct regcache *reg_cache = cstate->reg_cache;
1104         union code c = block->opcode_list[offset].c;
1105         jit_state_t *_jit = block->_jit;
1106         jit_node_t *to_not_ram, *to_end;
1107         u8 tmp, tmp2, tmp3, rs, rt;
1108
1109         jit_note(__FILE__, __LINE__);
1110
1111         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rs, 0);
1112         tmp2 = lightrec_alloc_reg_temp(reg_cache, _jit);
1113         tmp3 = lightrec_alloc_reg_in(reg_cache, _jit, 0, 0);
1114
1115         /* Convert to KUNSEG and avoid RAM mirrors */
1116         if (c.i.imm) {
1117                 jit_addi(tmp2, rs, (s16)c.i.imm);
1118                 jit_andi(tmp2, tmp2, 0x1f800000 | (ram_size - 1));
1119         } else {
1120                 jit_andi(tmp2, rs, 0x1f800000 | (ram_size - 1));
1121         }
1122
1123         lightrec_free_reg(reg_cache, rs);
1124         tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
1125
1126         to_not_ram = jit_bgti(tmp2, ram_size);
1127
1128         lightrec_regcache_mark_live(reg_cache, _jit);
1129
1130         /* Compute the offset to the code LUT */
1131         jit_andi(tmp, tmp2, (RAM_SIZE - 1) & ~3);
1132         if (__WORDSIZE == 64)
1133                 jit_lshi(tmp, tmp, 1);
1134         jit_addr(tmp, LIGHTREC_REG_STATE, tmp);
1135
1136         /* Write NULL to the code LUT to invalidate any block that's there */
1137         jit_stxi(offsetof(struct lightrec_state, code_lut), tmp, tmp3);
1138
1139         if (state->offset_ram != state->offset_scratch) {
1140                 jit_movi(tmp, state->offset_ram);
1141
1142                 to_end = jit_jmpi();
1143         }
1144
1145         jit_patch(to_not_ram);
1146
1147         if (state->offset_ram || state->offset_scratch)
1148                 jit_movi(tmp, state->offset_scratch);
1149
1150         if (state->offset_ram != state->offset_scratch)
1151                 jit_patch(to_end);
1152
1153         if (state->offset_ram || state->offset_scratch)
1154                 jit_addr(tmp2, tmp2, tmp);
1155
1156         lightrec_free_reg(reg_cache, tmp);
1157         lightrec_free_reg(reg_cache, tmp3);
1158
1159         rt = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rt, 0);
1160         jit_new_node_www(code, 0, tmp2, rt);
1161
1162         lightrec_free_reg(reg_cache, rt);
1163         lightrec_free_reg(reg_cache, tmp2);
1164 }
1165
1166 static void rec_store(struct lightrec_cstate *state,
1167                       const struct block *block, u16 offset, jit_code_t code)
1168 {
1169         u16 flags = block->opcode_list[offset].flags;
1170
1171         if (flags & LIGHTREC_NO_INVALIDATE) {
1172                 rec_store_direct_no_invalidate(state, block, offset, code);
1173         } else if (flags & LIGHTREC_DIRECT_IO) {
1174                 if (state->state->invalidate_from_dma_only)
1175                         rec_store_direct_no_invalidate(state, block, offset, code);
1176                 else
1177                         rec_store_direct(state, block, offset, code);
1178         } else {
1179                 rec_io(state, block, offset, true, false);
1180         }
1181 }
1182
1183 static void rec_SB(struct lightrec_cstate *state,
1184                    const struct block *block, u16 offset)
1185 {
1186         _jit_name(block->_jit, __func__);
1187         rec_store(state, block, offset, jit_code_stxi_c);
1188 }
1189
1190 static void rec_SH(struct lightrec_cstate *state,
1191                    const struct block *block, u16 offset)
1192 {
1193         _jit_name(block->_jit, __func__);
1194         rec_store(state, block, offset, jit_code_stxi_s);
1195 }
1196
1197 static void rec_SW(struct lightrec_cstate *state,
1198                    const struct block *block, u16 offset)
1199
1200 {
1201         _jit_name(block->_jit, __func__);
1202         rec_store(state, block, offset, jit_code_stxi_i);
1203 }
1204
1205 static void rec_SWL(struct lightrec_cstate *state,
1206                     const struct block *block, u16 offset)
1207 {
1208         _jit_name(block->_jit, __func__);
1209         rec_io(state, block, offset, true, false);
1210 }
1211
1212 static void rec_SWR(struct lightrec_cstate *state,
1213                     const struct block *block, u16 offset)
1214 {
1215         _jit_name(block->_jit, __func__);
1216         rec_io(state, block, offset, true, false);
1217 }
1218
1219 static void rec_SWC2(struct lightrec_cstate *state,
1220                      const struct block *block, u16 offset)
1221 {
1222         _jit_name(block->_jit, __func__);
1223         rec_io(state, block, offset, false, false);
1224 }
1225
1226 static void rec_load_direct(struct lightrec_cstate *cstate, const struct block *block,
1227                             u16 offset, jit_code_t code, bool is_unsigned)
1228 {
1229         struct lightrec_state *state = cstate->state;
1230         struct regcache *reg_cache = cstate->reg_cache;
1231         union code c = block->opcode_list[offset].c;
1232         jit_state_t *_jit = block->_jit;
1233         jit_node_t *to_not_ram, *to_not_bios, *to_end, *to_end2;
1234         u8 tmp, rs, rt, addr_reg, flags = REG_EXT;
1235         s16 imm;
1236
1237         if (!c.i.rt)
1238                 return;
1239
1240         if (is_unsigned)
1241                 flags |= REG_ZEXT;
1242
1243         jit_note(__FILE__, __LINE__);
1244         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rs, 0);
1245         rt = lightrec_alloc_reg_out(reg_cache, _jit, c.i.rt, flags);
1246
1247         if ((state->offset_ram == state->offset_bios &&
1248             state->offset_ram == state->offset_scratch &&
1249             state->mirrors_mapped) || !c.i.imm) {
1250                 addr_reg = rs;
1251                 imm = (s16)c.i.imm;
1252         } else {
1253                 jit_addi(rt, rs, (s16)c.i.imm);
1254                 addr_reg = rt;
1255                 imm = 0;
1256
1257                 if (c.i.rs != c.i.rt)
1258                         lightrec_free_reg(reg_cache, rs);
1259         }
1260
1261         tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
1262
1263         if (state->offset_ram == state->offset_bios &&
1264             state->offset_ram == state->offset_scratch) {
1265                 if (!state->mirrors_mapped) {
1266                         jit_andi(tmp, addr_reg, BIT(28));
1267                         jit_rshi_u(tmp, tmp, 28 - 22);
1268                         jit_ori(tmp, tmp, 0x1f800000 | (RAM_SIZE - 1));
1269                         jit_andr(rt, addr_reg, tmp);
1270                 } else {
1271                         jit_andi(rt, addr_reg, 0x1fffffff);
1272                 }
1273
1274                 if (state->offset_ram)
1275                         jit_movi(tmp, state->offset_ram);
1276         } else {
1277                 to_not_ram = jit_bmsi(addr_reg, BIT(28));
1278
1279                 lightrec_regcache_mark_live(reg_cache, _jit);
1280
1281                 /* Convert to KUNSEG and avoid RAM mirrors */
1282                 jit_andi(rt, addr_reg, RAM_SIZE - 1);
1283
1284                 if (state->offset_ram)
1285                         jit_movi(tmp, state->offset_ram);
1286
1287                 to_end = jit_jmpi();
1288
1289                 jit_patch(to_not_ram);
1290
1291                 if (state->offset_bios != state->offset_scratch)
1292                         to_not_bios = jit_bmci(addr_reg, BIT(22));
1293
1294                 /* Convert to KUNSEG */
1295                 jit_andi(rt, addr_reg, 0x1fc00000 | (BIOS_SIZE - 1));
1296
1297                 jit_movi(tmp, state->offset_bios);
1298
1299                 if (state->offset_bios != state->offset_scratch) {
1300                         to_end2 = jit_jmpi();
1301
1302                         jit_patch(to_not_bios);
1303
1304                         /* Convert to KUNSEG */
1305                         jit_andi(rt, addr_reg, 0x1f800fff);
1306
1307                         if (state->offset_scratch)
1308                                 jit_movi(tmp, state->offset_scratch);
1309
1310                         jit_patch(to_end2);
1311                 }
1312
1313                 jit_patch(to_end);
1314         }
1315
1316         if (state->offset_ram || state->offset_bios || state->offset_scratch)
1317                 jit_addr(rt, rt, tmp);
1318
1319         jit_new_node_www(code, rt, rt, imm);
1320
1321         lightrec_free_reg(reg_cache, addr_reg);
1322         lightrec_free_reg(reg_cache, rt);
1323         lightrec_free_reg(reg_cache, tmp);
1324 }
1325
1326 static void rec_load(struct lightrec_cstate *state, const struct block *block,
1327                      u16 offset, jit_code_t code, bool is_unsigned)
1328 {
1329         u16 flags = block->opcode_list[offset].flags;
1330
1331         if (flags & LIGHTREC_DIRECT_IO)
1332                 rec_load_direct(state, block, offset, code, is_unsigned);
1333         else
1334                 rec_io(state, block, offset, false, true);
1335 }
1336
1337 static void rec_LB(struct lightrec_cstate *state, const struct block *block, u16 offset)
1338 {
1339         _jit_name(block->_jit, __func__);
1340         rec_load(state, block, offset, jit_code_ldxi_c, false);
1341 }
1342
1343 static void rec_LBU(struct lightrec_cstate *state, const struct block *block, u16 offset)
1344 {
1345         _jit_name(block->_jit, __func__);
1346         rec_load(state, block, offset, jit_code_ldxi_uc, true);
1347 }
1348
1349 static void rec_LH(struct lightrec_cstate *state, const struct block *block, u16 offset)
1350 {
1351         _jit_name(block->_jit, __func__);
1352         rec_load(state, block, offset, jit_code_ldxi_s, false);
1353 }
1354
1355 static void rec_LHU(struct lightrec_cstate *state, const struct block *block, u16 offset)
1356 {
1357         _jit_name(block->_jit, __func__);
1358         rec_load(state, block, offset, jit_code_ldxi_us, true);
1359 }
1360
1361 static void rec_LWL(struct lightrec_cstate *state, const struct block *block, u16 offset)
1362 {
1363         _jit_name(block->_jit, __func__);
1364         rec_io(state, block, offset, true, true);
1365 }
1366
1367 static void rec_LWR(struct lightrec_cstate *state, const struct block *block, u16 offset)
1368 {
1369         _jit_name(block->_jit, __func__);
1370         rec_io(state, block, offset, true, true);
1371 }
1372
1373 static void rec_LW(struct lightrec_cstate *state, const struct block *block, u16 offset)
1374 {
1375         _jit_name(block->_jit, __func__);
1376         rec_load(state, block, offset, jit_code_ldxi_i, false);
1377 }
1378
1379 static void rec_LWC2(struct lightrec_cstate *state, const struct block *block, u16 offset)
1380 {
1381         _jit_name(block->_jit, __func__);
1382         rec_io(state, block, offset, false, false);
1383 }
1384
1385 static void rec_break_syscall(struct lightrec_cstate *state,
1386                               const struct block *block, u16 offset, bool is_break)
1387 {
1388         _jit_note(block->_jit, __FILE__, __LINE__);
1389
1390         if (is_break)
1391                 call_to_c_wrapper(state, block, 0, false, C_WRAPPER_BREAK);
1392         else
1393                 call_to_c_wrapper(state, block, 0, false, C_WRAPPER_SYSCALL);
1394
1395         /* TODO: the return address should be "pc - 4" if we're a delay slot */
1396         lightrec_emit_end_of_block(state, block, offset, -1,
1397                                    get_ds_pc(block, offset, 0),
1398                                    31, 0, true);
1399 }
1400
1401 static void rec_special_SYSCALL(struct lightrec_cstate *state,
1402                                 const struct block *block, u16 offset)
1403 {
1404         _jit_name(block->_jit, __func__);
1405         rec_break_syscall(state, block, offset, false);
1406 }
1407
1408 static void rec_special_BREAK(struct lightrec_cstate *state,
1409                               const struct block *block, u16 offset)
1410 {
1411         _jit_name(block->_jit, __func__);
1412         rec_break_syscall(state, block, offset, true);
1413 }
1414
1415 static void rec_mtc(struct lightrec_cstate *state, const struct block *block, u16 offset)
1416 {
1417         struct regcache *reg_cache = state->reg_cache;
1418         union code c = block->opcode_list[offset].c;
1419         jit_state_t *_jit = block->_jit;
1420
1421         jit_note(__FILE__, __LINE__);
1422         lightrec_clean_reg_if_loaded(reg_cache, _jit, c.i.rs, false);
1423         lightrec_clean_reg_if_loaded(reg_cache, _jit, c.i.rt, false);
1424
1425         call_to_c_wrapper(state, block, c.opcode, true, C_WRAPPER_MTC);
1426
1427         if (c.i.op == OP_CP0 &&
1428             !(block->opcode_list[offset].flags & LIGHTREC_NO_DS) &&
1429             (c.r.rd == 12 || c.r.rd == 13))
1430                 lightrec_emit_end_of_block(state, block, offset, -1,
1431                                            get_ds_pc(block, offset, 1),
1432                                            0, 0, true);
1433 }
1434
1435 static void
1436 rec_mfc0(struct lightrec_cstate *state, const struct block *block, u16 offset)
1437 {
1438         struct regcache *reg_cache = state->reg_cache;
1439         union code c = block->opcode_list[offset].c;
1440         jit_state_t *_jit = block->_jit;
1441         u8 rt;
1442
1443         jit_note(__FILE__, __LINE__);
1444
1445         rt = lightrec_alloc_reg_out(reg_cache, _jit, c.i.rt, REG_EXT);
1446
1447         jit_ldxi_i(rt, LIGHTREC_REG_STATE,
1448                    offsetof(struct lightrec_state, regs.cp0[c.r.rd]));
1449
1450         lightrec_free_reg(reg_cache, rt);
1451 }
1452
1453 static bool block_in_bios(const struct lightrec_cstate *state,
1454                           const struct block *block)
1455 {
1456         const struct lightrec_mem_map *bios = &state->state->maps[PSX_MAP_BIOS];
1457         u32 pc = kunseg(block->pc);
1458
1459         return pc >= bios->pc && pc < bios->pc + bios->length;
1460 }
1461
1462 static void
1463 rec_mtc0(struct lightrec_cstate *state, const struct block *block, u16 offset)
1464 {
1465         struct regcache *reg_cache = state->reg_cache;
1466         const union code c = block->opcode_list[offset].c;
1467         jit_state_t *_jit = block->_jit;
1468         u8 rt, tmp = 0, tmp2, status;
1469
1470         jit_note(__FILE__, __LINE__);
1471
1472         switch(c.r.rd) {
1473         case 1:
1474         case 4:
1475         case 8:
1476         case 14:
1477         case 15:
1478                 /* Those registers are read-only */
1479                 return;
1480         default:
1481                 break;
1482         }
1483
1484         if (block_in_bios(state, block) && c.r.rd == 12) {
1485                 /* If we are running code from the BIOS, handle writes to the
1486                  * Status register in C. BIOS code may toggle bit 16 which will
1487                  * map/unmap the RAM, while game code cannot do that. */
1488                 rec_mtc(state, block, offset);
1489                 return;
1490         }
1491
1492         rt = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rt, 0);
1493
1494         if (c.r.rd != 13) {
1495                 jit_stxi_i(offsetof(struct lightrec_state, regs.cp0[c.r.rd]),
1496                            LIGHTREC_REG_STATE, rt);
1497         }
1498
1499         if (c.r.rd == 12 || c.r.rd == 13) {
1500                 tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
1501                 jit_ldxi_i(tmp, LIGHTREC_REG_STATE,
1502                            offsetof(struct lightrec_state, regs.cp0[13]));
1503
1504                 tmp2 = lightrec_alloc_reg_temp(reg_cache, _jit);
1505         }
1506
1507         if (c.r.rd == 12) {
1508                 status = rt;
1509         } else if (c.r.rd == 13) {
1510                 /* Cause = (Cause & ~0x0300) | (value & 0x0300) */
1511                 jit_andi(tmp2, rt, 0x0300);
1512                 jit_ori(tmp, tmp, 0x0300);
1513                 jit_xori(tmp, tmp, 0x0300);
1514                 jit_orr(tmp, tmp, tmp2);
1515                 jit_ldxi_i(tmp2, LIGHTREC_REG_STATE,
1516                            offsetof(struct lightrec_state, regs.cp0[12]));
1517                 jit_stxi_i(offsetof(struct lightrec_state, regs.cp0[13]),
1518                            LIGHTREC_REG_STATE, tmp);
1519                 status = tmp2;
1520         }
1521
1522         if (c.r.rd == 12 || c.r.rd == 13) {
1523                 /* Exit dynarec in case there's a software interrupt.
1524                  * exit_flags = !!(status & tmp & 0x0300) & status; */
1525                 jit_andr(tmp, tmp, status);
1526                 jit_andi(tmp, tmp, 0x0300);
1527                 jit_nei(tmp, tmp, 0);
1528                 jit_andr(tmp, tmp, status);
1529         }
1530
1531         if (c.r.rd == 12) {
1532                 /* Exit dynarec in case we unmask a hardware interrupt.
1533                  * exit_flags = !(~status & 0x401) */
1534
1535                 jit_comr(tmp2, status);
1536                 jit_andi(tmp2, tmp2, 0x401);
1537                 jit_eqi(tmp2, tmp2, 0);
1538                 jit_orr(tmp, tmp, tmp2);
1539         }
1540
1541         if (c.r.rd == 12 || c.r.rd == 13) {
1542                 jit_stxi_i(offsetof(struct lightrec_state, exit_flags),
1543                            LIGHTREC_REG_STATE, tmp);
1544
1545                 lightrec_free_reg(reg_cache, tmp);
1546                 lightrec_free_reg(reg_cache, tmp2);
1547         }
1548
1549         lightrec_free_reg(reg_cache, rt);
1550
1551         if (!(block->opcode_list[offset].flags & LIGHTREC_NO_DS) &&
1552             (c.r.rd == 12 || c.r.rd == 13))
1553                 lightrec_emit_eob(state, block, offset + 1, true);
1554 }
1555
1556 static void rec_cp0_MFC0(struct lightrec_cstate *state,
1557                          const struct block *block, u16 offset)
1558 {
1559         _jit_name(block->_jit, __func__);
1560         rec_mfc0(state, block, offset);
1561 }
1562
1563 static void rec_cp0_CFC0(struct lightrec_cstate *state,
1564                          const struct block *block, u16 offset)
1565 {
1566         _jit_name(block->_jit, __func__);
1567         rec_mfc0(state, block, offset);
1568 }
1569
1570 static void rec_cp0_MTC0(struct lightrec_cstate *state,
1571                          const struct block *block, u16 offset)
1572 {
1573         _jit_name(block->_jit, __func__);
1574         rec_mtc0(state, block, offset);
1575 }
1576
1577 static void rec_cp0_CTC0(struct lightrec_cstate *state,
1578                          const struct block *block, u16 offset)
1579 {
1580         _jit_name(block->_jit, __func__);
1581         rec_mtc0(state, block, offset);
1582 }
1583
1584 static void rec_cp2_basic_MFC2(struct lightrec_cstate *state,
1585                                const struct block *block, u16 offset)
1586 {
1587         struct regcache *reg_cache = state->reg_cache;
1588         const union code c = block->opcode_list[offset].c;
1589         jit_state_t *_jit = block->_jit;
1590         const u32 zext_regs = 0x300f0080;
1591         u8 rt, tmp, tmp2, tmp3, out, flags;
1592         u8 reg = c.r.rd == 15 ? 14 : c.r.rd;
1593         unsigned int i;
1594
1595         _jit_name(block->_jit, __func__);
1596
1597         flags = (zext_regs & BIT(reg)) ? REG_ZEXT : REG_EXT;
1598         rt = lightrec_alloc_reg_out(reg_cache, _jit, c.r.rt, flags);
1599
1600         switch (reg) {
1601         case 1:
1602         case 3:
1603         case 5:
1604         case 8:
1605         case 9:
1606         case 10:
1607         case 11:
1608                 jit_ldxi_s(rt, LIGHTREC_REG_STATE,
1609                            offsetof(struct lightrec_state, regs.cp2d[reg]));
1610                 break;
1611         case 7:
1612         case 16:
1613         case 17:
1614         case 18:
1615         case 19:
1616                 jit_ldxi_us(rt, LIGHTREC_REG_STATE,
1617                            offsetof(struct lightrec_state, regs.cp2d[reg]));
1618                 break;
1619         case 28:
1620         case 29:
1621                 tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
1622                 tmp2 = lightrec_alloc_reg_temp(reg_cache, _jit);
1623                 tmp3 = lightrec_alloc_reg_temp(reg_cache, _jit);
1624
1625                 for (i = 0; i < 3; i++) {
1626                         out = i == 0 ? rt : tmp;
1627
1628                         jit_ldxi_s(tmp, LIGHTREC_REG_STATE,
1629                                    offsetof(struct lightrec_state, regs.cp2d[9 + i]));
1630                         jit_movi(tmp2, 0x1f);
1631                         jit_rshi(out, tmp, 7);
1632
1633                         jit_ltr(tmp3, tmp2, out);
1634                         jit_movnr(out, tmp2, tmp3);
1635
1636                         jit_gei(tmp2, out, 0);
1637                         jit_movzr(out, tmp2, tmp2);
1638
1639                         if (i > 0) {
1640                                 jit_lshi(tmp, tmp, 5 * i);
1641                                 jit_orr(rt, rt, tmp);
1642                         }
1643                 }
1644
1645
1646                 lightrec_free_reg(reg_cache, tmp);
1647                 lightrec_free_reg(reg_cache, tmp2);
1648                 lightrec_free_reg(reg_cache, tmp3);
1649                 break;
1650         default:
1651                 jit_ldxi_i(rt, LIGHTREC_REG_STATE,
1652                            offsetof(struct lightrec_state, regs.cp2d[reg]));
1653                 break;
1654         }
1655
1656         lightrec_free_reg(reg_cache, rt);
1657 }
1658
1659 static void rec_cp2_basic_CFC2(struct lightrec_cstate *state,
1660                                const struct block *block, u16 offset)
1661 {
1662         struct regcache *reg_cache = state->reg_cache;
1663         const union code c = block->opcode_list[offset].c;
1664         jit_state_t *_jit = block->_jit;
1665         u8 rt;
1666
1667         _jit_name(block->_jit, __func__);
1668
1669         switch (c.r.rd) {
1670         case 4:
1671         case 12:
1672         case 20:
1673         case 26:
1674         case 27:
1675         case 29:
1676         case 30:
1677                 rt = lightrec_alloc_reg_out(reg_cache, _jit, c.r.rt, REG_EXT);
1678                 jit_ldxi_s(rt, LIGHTREC_REG_STATE,
1679                            offsetof(struct lightrec_state, regs.cp2c[c.r.rd]));
1680                 break;
1681         default:
1682                 rt = lightrec_alloc_reg_out(reg_cache, _jit, c.r.rt, REG_ZEXT);
1683                 jit_ldxi_i(rt, LIGHTREC_REG_STATE,
1684                            offsetof(struct lightrec_state, regs.cp2c[c.r.rd]));
1685                 break;
1686         }
1687
1688         lightrec_free_reg(reg_cache, rt);
1689 }
1690
1691 static void rec_cp2_basic_MTC2(struct lightrec_cstate *state,
1692                                const struct block *block, u16 offset)
1693 {
1694         struct regcache *reg_cache = state->reg_cache;
1695         const union code c = block->opcode_list[offset].c;
1696         jit_state_t *_jit = block->_jit;
1697         jit_node_t *loop, *to_loop;
1698         u8 rt, tmp, tmp2, flags = 0;
1699
1700         _jit_name(block->_jit, __func__);
1701
1702         if (c.r.rd == 31)
1703                 return;
1704
1705         if (c.r.rd == 30)
1706                 flags |= REG_EXT;
1707
1708         rt = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rt, flags);
1709
1710         switch (c.r.rd) {
1711         case 15:
1712                 tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
1713                 jit_ldxi_i(tmp, LIGHTREC_REG_STATE,
1714                            offsetof(struct lightrec_state, regs.cp2d[13]));
1715
1716                 tmp2 = lightrec_alloc_reg_temp(reg_cache, _jit);
1717                 jit_ldxi_i(tmp2, LIGHTREC_REG_STATE,
1718                            offsetof(struct lightrec_state, regs.cp2d[14]));
1719
1720                 jit_stxi_i(offsetof(struct lightrec_state, regs.cp2d[12]),
1721                            LIGHTREC_REG_STATE, tmp);
1722                 jit_stxi_i(offsetof(struct lightrec_state, regs.cp2d[13]),
1723                            LIGHTREC_REG_STATE, tmp2);
1724                 jit_stxi_i(offsetof(struct lightrec_state, regs.cp2d[14]),
1725                            LIGHTREC_REG_STATE, rt);
1726
1727                 lightrec_free_reg(reg_cache, tmp);
1728                 lightrec_free_reg(reg_cache, tmp2);
1729                 break;
1730         case 28:
1731                 tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
1732
1733                 jit_lshi(tmp, rt, 7);
1734                 jit_andi(tmp, tmp, 0xf80);
1735                 jit_stxi_s(offsetof(struct lightrec_state, regs.cp2d[9]),
1736                            LIGHTREC_REG_STATE, tmp);
1737
1738                 jit_lshi(tmp, rt, 2);
1739                 jit_andi(tmp, tmp, 0xf80);
1740                 jit_stxi_s(offsetof(struct lightrec_state, regs.cp2d[10]),
1741                            LIGHTREC_REG_STATE, tmp);
1742
1743                 jit_rshi(tmp, rt, 3);
1744                 jit_andi(tmp, tmp, 0xf80);
1745                 jit_stxi_s(offsetof(struct lightrec_state, regs.cp2d[11]),
1746                            LIGHTREC_REG_STATE, tmp);
1747
1748                 lightrec_free_reg(reg_cache, tmp);
1749                 break;
1750         case 30:
1751                 tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
1752                 tmp2 = lightrec_alloc_reg_temp(reg_cache, _jit);
1753
1754                 /* if (rt < 0) rt = ~rt; */
1755                 jit_rshi(tmp, rt, 31);
1756                 jit_xorr(tmp, rt, tmp);
1757
1758                 /* We know the sign bit is 0. Left-shift by 1 to start the algorithm */
1759                 jit_lshi(tmp, tmp, 1);
1760                 jit_movi(tmp2, 33);
1761
1762                 /* Decrement tmp2 and right-shift the value by 1 until it equals zero */
1763                 loop = jit_label();
1764                 jit_subi(tmp2, tmp2, 1);
1765                 jit_rshi_u(tmp, tmp, 1);
1766                 to_loop = jit_bnei(tmp, 0);
1767
1768                 jit_patch_at(to_loop, loop);
1769
1770                 jit_stxi_i(offsetof(struct lightrec_state, regs.cp2d[31]),
1771                            LIGHTREC_REG_STATE, tmp2);
1772                 jit_stxi_i(offsetof(struct lightrec_state, regs.cp2d[30]),
1773                            LIGHTREC_REG_STATE, rt);
1774
1775                 lightrec_free_reg(reg_cache, tmp);
1776                 lightrec_free_reg(reg_cache, tmp2);
1777                 break;
1778         default:
1779                 jit_stxi_i(offsetof(struct lightrec_state, regs.cp2d[c.r.rd]),
1780                            LIGHTREC_REG_STATE, rt);
1781                 break;
1782         }
1783
1784         lightrec_free_reg(reg_cache, rt);
1785 }
1786
1787 static void rec_cp2_basic_CTC2(struct lightrec_cstate *state,
1788                                const struct block *block, u16 offset)
1789 {
1790         struct regcache *reg_cache = state->reg_cache;
1791         const union code c = block->opcode_list[offset].c;
1792         jit_state_t *_jit = block->_jit;
1793         u8 rt, tmp, tmp2;
1794
1795         _jit_name(block->_jit, __func__);
1796
1797         rt = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rt, 0);
1798
1799         switch (c.r.rd) {
1800         case 4:
1801         case 12:
1802         case 20:
1803         case 26:
1804         case 27:
1805         case 29:
1806         case 30:
1807                 jit_stxi_s(offsetof(struct lightrec_state, regs.cp2c[c.r.rd]),
1808                            LIGHTREC_REG_STATE, rt);
1809                 break;
1810         case 31:
1811                 tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
1812                 tmp2 = lightrec_alloc_reg_temp(reg_cache, _jit);
1813
1814                 jit_andi(tmp, rt, 0x7f87e000);
1815                 jit_nei(tmp, tmp, 0);
1816                 jit_lshi(tmp, tmp, 31);
1817
1818                 jit_andi(tmp2, rt, 0x7ffff000);
1819                 jit_orr(tmp, tmp2, tmp);
1820
1821                 jit_stxi_i(offsetof(struct lightrec_state, regs.cp2c[31]),
1822                            LIGHTREC_REG_STATE, tmp);
1823
1824                 lightrec_free_reg(reg_cache, tmp);
1825                 lightrec_free_reg(reg_cache, tmp2);
1826                 break;
1827
1828         default:
1829                 jit_stxi_i(offsetof(struct lightrec_state, regs.cp2c[c.r.rd]),
1830                            LIGHTREC_REG_STATE, rt);
1831         }
1832
1833         lightrec_free_reg(reg_cache, rt);
1834 }
1835
1836 static void rec_cp0_RFE(struct lightrec_cstate *state,
1837                         const struct block *block, u16 offset)
1838 {
1839         struct regcache *reg_cache = state->reg_cache;
1840         jit_state_t *_jit = block->_jit;
1841         u8 status, tmp;
1842
1843         jit_name(__func__);
1844         jit_note(__FILE__, __LINE__);
1845
1846         status = lightrec_alloc_reg_temp(reg_cache, _jit);
1847         jit_ldxi_i(status, LIGHTREC_REG_STATE,
1848                    offsetof(struct lightrec_state, regs.cp0[12]));
1849
1850         tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
1851
1852         /* status = ((status >> 2) & 0xf) | status & ~0xf; */
1853         jit_rshi(tmp, status, 2);
1854         jit_andi(tmp, tmp, 0xf);
1855         jit_andi(status, status, ~0xful);
1856         jit_orr(status, status, tmp);
1857
1858         jit_ldxi_i(tmp, LIGHTREC_REG_STATE,
1859                    offsetof(struct lightrec_state, regs.cp0[13]));
1860         jit_stxi_i(offsetof(struct lightrec_state, regs.cp0[12]),
1861                    LIGHTREC_REG_STATE, status);
1862
1863         /* Exit dynarec in case there's a software interrupt.
1864          * exit_flags = !!(status & cause & 0x0300) & status; */
1865         jit_andr(tmp, tmp, status);
1866         jit_andi(tmp, tmp, 0x0300);
1867         jit_nei(tmp, tmp, 0);
1868         jit_andr(tmp, tmp, status);
1869         jit_stxi_i(offsetof(struct lightrec_state, exit_flags),
1870                    LIGHTREC_REG_STATE, tmp);
1871
1872         lightrec_free_reg(reg_cache, status);
1873         lightrec_free_reg(reg_cache, tmp);
1874 }
1875
1876 static void rec_CP(struct lightrec_cstate *state,
1877                    const struct block *block, u16 offset)
1878 {
1879         union code c = block->opcode_list[offset].c;
1880         jit_state_t *_jit = block->_jit;
1881
1882         jit_name(__func__);
1883         jit_note(__FILE__, __LINE__);
1884
1885         call_to_c_wrapper(state, block, c.opcode, true, C_WRAPPER_CP);
1886 }
1887
1888 static void rec_meta_MOV(struct lightrec_cstate *state,
1889                          const struct block *block, u16 offset)
1890 {
1891         struct regcache *reg_cache = state->reg_cache;
1892         union code c = block->opcode_list[offset].c;
1893         jit_state_t *_jit = block->_jit;
1894         u8 rs, rd;
1895
1896         _jit_name(block->_jit, __func__);
1897         jit_note(__FILE__, __LINE__);
1898         if (c.r.rs)
1899                 rs = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rs, 0);
1900         rd = lightrec_alloc_reg_out(reg_cache, _jit, c.r.rd, REG_EXT);
1901
1902         if (c.r.rs == 0)
1903                 jit_movi(rd, 0);
1904         else
1905                 jit_extr_i(rd, rs);
1906
1907         if (c.r.rs)
1908                 lightrec_free_reg(reg_cache, rs);
1909         lightrec_free_reg(reg_cache, rd);
1910 }
1911
1912 static void rec_meta_EXTC_EXTS(struct lightrec_cstate *state,
1913                                const struct block *block,
1914                                u16 offset)
1915 {
1916         struct regcache *reg_cache = state->reg_cache;
1917         union code c = block->opcode_list[offset].c;
1918         jit_state_t *_jit = block->_jit;
1919         u8 rs, rt;
1920
1921         _jit_name(block->_jit, __func__);
1922         jit_note(__FILE__, __LINE__);
1923
1924         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rs, 0);
1925         rt = lightrec_alloc_reg_out(reg_cache, _jit, c.i.rt, REG_EXT);
1926
1927         if (c.i.op == OP_META_EXTC)
1928                 jit_extr_c(rt, rs);
1929         else
1930                 jit_extr_s(rt, rs);
1931
1932         lightrec_free_reg(reg_cache, rs);
1933         lightrec_free_reg(reg_cache, rt);
1934 }
1935
1936 static const lightrec_rec_func_t rec_standard[64] = {
1937         SET_DEFAULT_ELM(rec_standard, unknown_opcode),
1938         [OP_SPECIAL]            = rec_SPECIAL,
1939         [OP_REGIMM]             = rec_REGIMM,
1940         [OP_J]                  = rec_J,
1941         [OP_JAL]                = rec_JAL,
1942         [OP_BEQ]                = rec_BEQ,
1943         [OP_BNE]                = rec_BNE,
1944         [OP_BLEZ]               = rec_BLEZ,
1945         [OP_BGTZ]               = rec_BGTZ,
1946         [OP_ADDI]               = rec_ADDI,
1947         [OP_ADDIU]              = rec_ADDIU,
1948         [OP_SLTI]               = rec_SLTI,
1949         [OP_SLTIU]              = rec_SLTIU,
1950         [OP_ANDI]               = rec_ANDI,
1951         [OP_ORI]                = rec_ORI,
1952         [OP_XORI]               = rec_XORI,
1953         [OP_LUI]                = rec_LUI,
1954         [OP_CP0]                = rec_CP0,
1955         [OP_CP2]                = rec_CP2,
1956         [OP_LB]                 = rec_LB,
1957         [OP_LH]                 = rec_LH,
1958         [OP_LWL]                = rec_LWL,
1959         [OP_LW]                 = rec_LW,
1960         [OP_LBU]                = rec_LBU,
1961         [OP_LHU]                = rec_LHU,
1962         [OP_LWR]                = rec_LWR,
1963         [OP_SB]                 = rec_SB,
1964         [OP_SH]                 = rec_SH,
1965         [OP_SWL]                = rec_SWL,
1966         [OP_SW]                 = rec_SW,
1967         [OP_SWR]                = rec_SWR,
1968         [OP_LWC2]               = rec_LWC2,
1969         [OP_SWC2]               = rec_SWC2,
1970
1971         [OP_META_MOV]           = rec_meta_MOV,
1972         [OP_META_EXTC]          = rec_meta_EXTC_EXTS,
1973         [OP_META_EXTS]          = rec_meta_EXTC_EXTS,
1974 };
1975
1976 static const lightrec_rec_func_t rec_special[64] = {
1977         SET_DEFAULT_ELM(rec_special, unknown_opcode),
1978         [OP_SPECIAL_SLL]        = rec_special_SLL,
1979         [OP_SPECIAL_SRL]        = rec_special_SRL,
1980         [OP_SPECIAL_SRA]        = rec_special_SRA,
1981         [OP_SPECIAL_SLLV]       = rec_special_SLLV,
1982         [OP_SPECIAL_SRLV]       = rec_special_SRLV,
1983         [OP_SPECIAL_SRAV]       = rec_special_SRAV,
1984         [OP_SPECIAL_JR]         = rec_special_JR,
1985         [OP_SPECIAL_JALR]       = rec_special_JALR,
1986         [OP_SPECIAL_SYSCALL]    = rec_special_SYSCALL,
1987         [OP_SPECIAL_BREAK]      = rec_special_BREAK,
1988         [OP_SPECIAL_MFHI]       = rec_special_MFHI,
1989         [OP_SPECIAL_MTHI]       = rec_special_MTHI,
1990         [OP_SPECIAL_MFLO]       = rec_special_MFLO,
1991         [OP_SPECIAL_MTLO]       = rec_special_MTLO,
1992         [OP_SPECIAL_MULT]       = rec_special_MULT,
1993         [OP_SPECIAL_MULTU]      = rec_special_MULTU,
1994         [OP_SPECIAL_DIV]        = rec_special_DIV,
1995         [OP_SPECIAL_DIVU]       = rec_special_DIVU,
1996         [OP_SPECIAL_ADD]        = rec_special_ADD,
1997         [OP_SPECIAL_ADDU]       = rec_special_ADDU,
1998         [OP_SPECIAL_SUB]        = rec_special_SUB,
1999         [OP_SPECIAL_SUBU]       = rec_special_SUBU,
2000         [OP_SPECIAL_AND]        = rec_special_AND,
2001         [OP_SPECIAL_OR]         = rec_special_OR,
2002         [OP_SPECIAL_XOR]        = rec_special_XOR,
2003         [OP_SPECIAL_NOR]        = rec_special_NOR,
2004         [OP_SPECIAL_SLT]        = rec_special_SLT,
2005         [OP_SPECIAL_SLTU]       = rec_special_SLTU,
2006 };
2007
2008 static const lightrec_rec_func_t rec_regimm[64] = {
2009         SET_DEFAULT_ELM(rec_regimm, unknown_opcode),
2010         [OP_REGIMM_BLTZ]        = rec_regimm_BLTZ,
2011         [OP_REGIMM_BGEZ]        = rec_regimm_BGEZ,
2012         [OP_REGIMM_BLTZAL]      = rec_regimm_BLTZAL,
2013         [OP_REGIMM_BGEZAL]      = rec_regimm_BGEZAL,
2014 };
2015
2016 static const lightrec_rec_func_t rec_cp0[64] = {
2017         SET_DEFAULT_ELM(rec_cp0, rec_CP),
2018         [OP_CP0_MFC0]           = rec_cp0_MFC0,
2019         [OP_CP0_CFC0]           = rec_cp0_CFC0,
2020         [OP_CP0_MTC0]           = rec_cp0_MTC0,
2021         [OP_CP0_CTC0]           = rec_cp0_CTC0,
2022         [OP_CP0_RFE]            = rec_cp0_RFE,
2023 };
2024
2025 static const lightrec_rec_func_t rec_cp2_basic[64] = {
2026         SET_DEFAULT_ELM(rec_cp2_basic, rec_CP),
2027         [OP_CP2_BASIC_MFC2]     = rec_cp2_basic_MFC2,
2028         [OP_CP2_BASIC_CFC2]     = rec_cp2_basic_CFC2,
2029         [OP_CP2_BASIC_MTC2]     = rec_cp2_basic_MTC2,
2030         [OP_CP2_BASIC_CTC2]     = rec_cp2_basic_CTC2,
2031 };
2032
2033 static void rec_SPECIAL(struct lightrec_cstate *state,
2034                         const struct block *block, u16 offset)
2035 {
2036         union code c = block->opcode_list[offset].c;
2037         lightrec_rec_func_t f = rec_special[c.r.op];
2038
2039         if (!HAS_DEFAULT_ELM && unlikely(!f))
2040                 unknown_opcode(state, block, offset);
2041         else
2042                 (*f)(state, block, offset);
2043 }
2044
2045 static void rec_REGIMM(struct lightrec_cstate *state,
2046                        const struct block *block, u16 offset)
2047 {
2048         union code c = block->opcode_list[offset].c;
2049         lightrec_rec_func_t f = rec_regimm[c.r.rt];
2050
2051         if (!HAS_DEFAULT_ELM && unlikely(!f))
2052                 unknown_opcode(state, block, offset);
2053         else
2054                 (*f)(state, block, offset);
2055 }
2056
2057 static void rec_CP0(struct lightrec_cstate *state,
2058                     const struct block *block, u16 offset)
2059 {
2060         union code c = block->opcode_list[offset].c;
2061         lightrec_rec_func_t f = rec_cp0[c.r.rs];
2062
2063         if (!HAS_DEFAULT_ELM && unlikely(!f))
2064                 rec_CP(state, block, offset);
2065         else
2066                 (*f)(state, block, offset);
2067 }
2068
2069 static void rec_CP2(struct lightrec_cstate *state,
2070                     const struct block *block, u16 offset)
2071 {
2072         union code c = block->opcode_list[offset].c;
2073
2074         if (c.r.op == OP_CP2_BASIC) {
2075                 lightrec_rec_func_t f = rec_cp2_basic[c.r.rs];
2076
2077                 if (HAS_DEFAULT_ELM || likely(f)) {
2078                         (*f)(state, block, offset);
2079                         return;
2080                 }
2081         }
2082
2083         rec_CP(state, block, offset);
2084 }
2085
2086 void lightrec_rec_opcode(struct lightrec_cstate *state,
2087                          const struct block *block, u16 offset)
2088 {
2089         struct regcache *reg_cache = state->reg_cache;
2090         struct lightrec_branch_target *target;
2091         const struct opcode *op = &block->opcode_list[offset];
2092         jit_state_t *_jit = block->_jit;
2093         lightrec_rec_func_t f;
2094
2095         if (op->flags & LIGHTREC_SYNC) {
2096                 jit_subi(LIGHTREC_REG_CYCLE, LIGHTREC_REG_CYCLE, state->cycles);
2097                 state->cycles = 0;
2098
2099                 lightrec_storeback_regs(reg_cache, _jit);
2100                 lightrec_regcache_reset(reg_cache);
2101
2102                 pr_debug("Adding branch target at offset 0x%x\n", offset << 2);
2103                 target = &state->targets[state->nb_targets++];
2104                 target->offset = offset;
2105                 target->label = jit_indirect();
2106         }
2107
2108         if (likely(op->opcode)) {
2109                 f = rec_standard[op->i.op];
2110
2111                 if (!HAS_DEFAULT_ELM && unlikely(!f))
2112                         unknown_opcode(state, block, offset);
2113                 else
2114                         (*f)(state, block, offset);
2115         }
2116
2117         if (unlikely(op->flags & LIGHTREC_UNLOAD_RD)) {
2118                 lightrec_clean_reg_if_loaded(reg_cache, _jit, op->r.rd, true);
2119                 pr_debug("Cleaning RD reg %s\n", lightrec_reg_name(op->r.rd));
2120         }
2121         if (unlikely(op->flags & LIGHTREC_UNLOAD_RS)) {
2122                 lightrec_clean_reg_if_loaded(reg_cache, _jit, op->i.rs, true);
2123                 pr_debug("Cleaning RS reg %s\n", lightrec_reg_name(op->i.rt));
2124         }
2125         if (unlikely(op->flags & LIGHTREC_UNLOAD_RT)) {
2126                 lightrec_clean_reg_if_loaded(reg_cache, _jit, op->i.rt, true);
2127                 pr_debug("Cleaning RT reg %s\n", lightrec_reg_name(op->i.rt));
2128         }
2129 }