#ifdef _3DS
#include <3ds_utils.h>
#endif
+#ifdef HAVE_LIBNX
+#include <switch.h>
+static Jit g_jit;
+#endif
#include "new_dynarec_config.h"
#include "../psxhle.h"
static struct ndrc_mem ndrc_ __attribute__((aligned(4096)));
static struct ndrc_mem *ndrc = &ndrc_;
#endif
+#ifdef NDRC_WRITE_OFFSET
+# ifdef __GLIBC__
+# include <sys/types.h>
+# include <sys/stat.h>
+# include <fcntl.h>
+# include <unistd.h>
+# endif
+static long ndrc_write_ofs;
+#else
+#define ndrc_write_ofs 0
+#endif
// stubs
enum stub_type {
u_int waswritten; // MIPS regs that were used as store base before
};
-// note: asm depends on this layout
-struct ll_entry
-{
- u_int vaddr;
- void *addr;
- struct ll_entry *next;
-};
-
struct ht_entry
{
u_int vaddr[2];
u_int tc_offs;
//u_int tc_len;
u_int reg_sv_flags;
- u_short is_dirty;
+ u_char is_dirty;
+ u_char inv_near_misses;
u_short jump_in_cnt;
struct {
u_int vaddr;
} jump_in[0];
};
+struct jump_info
+{
+ int alloc;
+ int count;
+ struct {
+ u_int target_vaddr;
+ void *stub;
+ } e[0];
+};
+
static struct decoded_insn
{
u_char itype;
static u_char *out;
static struct ht_entry hash_table[65536];
static struct block_info *blocks[PAGE_COUNT];
- static struct ll_entry *jump_out[PAGE_COUNT];
+ static struct jump_info *jumps[PAGE_COUNT];
static u_int start;
static u_int *source;
static uint64_t gte_rs[MAXBLOCK]; // gte: 32 data and 32 ctl regs
sceKernelCloseVMDomain();
else
sceKernelOpenVMDomain();
+ #elif defined(HAVE_LIBNX)
+ Result rc;
+ if (is_x)
+ rc = jitTransitionToExecutable(&g_jit);
+ else
+ rc = jitTransitionToWritable(&g_jit);
+ if (R_FAILED(rc))
+ SysPrintf("jitTransition %d %08x\n", is_x, rc);
+ #elif defined(NDRC_WRITE_OFFSET)
+ // separated rx and rw areas are always available
#else
u_long mstart = (u_long)start & ~4095ul;
u_long mend = (u_long)end;
#endif
}
-static void start_tcache_write(void *start, void *end)
+static void *start_tcache_write(void *start, void *end)
{
mprotect_w_x(start, end, 0);
+ return (char *)start + ndrc_write_ofs;
}
static void end_tcache_write(void *start, void *end)
sceKernelSyncVMDomain(sceBlock, start, len);
#elif defined(_3DS)
ctr_flush_invalidate_cache();
+ #elif defined(HAVE_LIBNX)
+ // handled in mprotect_w_x()
#elif defined(__aarch64__)
// as of 2021, __clear_cache() is still broken on arm64
// so here is a custom one :(
end_tcache_write(start, out);
}
+#ifdef NDRC_CACHE_FLUSH_ALL
+
+static int needs_clear_cache;
+
+static void mark_clear_cache(void *target)
+{
+ if (!needs_clear_cache) {
+ start_tcache_write(ndrc, ndrc + 1);
+ needs_clear_cache = 1;
+ }
+}
+
+static void do_clear_cache(void)
+{
+ if (needs_clear_cache) {
+ end_tcache_write(ndrc, ndrc + 1);
+ needs_clear_cache = 0;
+ }
+}
+
+#else
+
// also takes care of w^x mappings when patching code
static u_int needs_clear_cache[1<<(TARGET_SIZE_2-17)];
}
}
-//#define DEBUG_CYCLE_COUNT 1
+#endif // NDRC_CACHE_FLUSH_ALL
#define NO_CYCLE_PENALTY_THR 12
static void mark_invalid_code(u_int vaddr, u_int len, char invalid)
{
+ u_int vaddr_m = vaddr & 0x1fffffff;
u_int i, j;
- vaddr &= 0x1fffffff;
- for (i = vaddr & ~0xfff; i < vaddr + len; i += 0x1000) {
+ for (i = vaddr_m & ~0xfff; i < vaddr_m + len; i += 0x1000) {
// ram mirrors, but should not hurt bios
for (j = 0; j < 0x800000; j += 0x200000) {
invalid_code[(i|j) >> 12] =
if (memcmp(block->source, block->copy, block->len))
continue;
- block->is_dirty = 0;
+ block->is_dirty = block->inv_near_misses = 0;
found_clean = block->jump_in[i].addr;
hash_table_add(vaddr, found_clean);
mark_invalid_code(block->start, block->len, 0);
abort();
}
if (ndrc->tramp.f[i] == NULL) {
- start_tcache_write(&ndrc->tramp.f[i], &ndrc->tramp.f[i + 1]);
- ndrc->tramp.f[i] = f;
+ const void **d = start_tcache_write(&ndrc->tramp.f[i], &ndrc->tramp.f[i + 1]);
+ *d = f;
end_tcache_write(&ndrc->tramp.f[i], &ndrc->tramp.f[i + 1]);
}
return &ndrc->tramp.ops[i];
emit_call(f);
}
-// Add virtual address mapping to linked list
-static void ll_add(struct ll_entry **head,int vaddr,void *addr)
-{
- struct ll_entry *new_entry;
- new_entry=malloc(sizeof(struct ll_entry));
- assert(new_entry!=NULL);
- new_entry->vaddr=vaddr;
- new_entry->addr=addr;
- new_entry->next=*head;
- *head=new_entry;
-}
-
// Check if an address is already compiled
// but don't return addresses which are about to expire from the cache
static void *check_addr(u_int vaddr)
return NULL;
}
-static void ll_remove_matching_addrs(struct ll_entry **head, u_int base_offs, int shift)
-{
- struct ll_entry *next;
- while (*head) {
- u_int tc_offs = (u_char *)((*head)->addr) - ndrc->translation_cache;
- if (((tc_offs ^ base_offs) >> shift) == 0) {
- inv_debug("EXP: rm link from tc_offs %x)\n", tc_offs);
- next = (*head)->next;
- free(*head);
- *head = next;
- }
- else
- {
- head = &((*head)->next);
- }
- }
-}
-
-// Remove all entries from linked list
-static void ll_clear(struct ll_entry **head)
-{
- struct ll_entry *cur;
- struct ll_entry *next;
- if((cur=*head)) {
- *head=0;
- while(cur) {
- next=cur->next;
- free(cur);
- cur=next;
- }
- }
-}
-
static void blocks_clear(struct block_info **head)
{
struct block_info *cur, *next;
}
// This is called when we write to a compiled block (see do_invstub)
-static void unlink_jumps_range(u_int start, u_int end)
+static void unlink_jumps_vaddr_range(u_int start, u_int end)
{
u_int page, start_page = get_page(start), end_page = get_page(end - 1);
- struct ll_entry **head, *next;
+ int i;
for (page = start_page; page <= end_page; page++) {
- for (head = &jump_out[page]; *head; ) {
- if ((*head)->vaddr < start || (*head)->vaddr >= end) {
- head = &((*head)->next);
+ struct jump_info *ji = jumps[page];
+ if (ji == NULL)
+ continue;
+ for (i = 0; i < ji->count; ) {
+ if (ji->e[i].target_vaddr < start || ji->e[i].target_vaddr >= end) {
+ i++;
continue;
}
- inv_debug("INV: rm link to %08x (tc_offs %zx)\n",
- (*head)->vaddr, (u_char *)((*head)->addr) - ndrc->translation_cache);
- void *host_addr = find_extjump_insn((*head)->addr);
+
+ inv_debug("INV: rm link to %08x (tc_offs %zx)\n", ji->e[i].target_vaddr,
+ (u_char *)ji->e[i].stub - ndrc->translation_cache);
+ void *host_addr = find_extjump_insn(ji->e[i].stub);
mark_clear_cache(host_addr);
- set_jump_target(host_addr, (*head)->addr); // point back to dyna_linker stub
+ set_jump_target(host_addr, ji->e[i].stub); // point back to dyna_linker stub
- next = (*head)->next;
- free(*head);
- *head = next;
stat_dec(stat_links);
+ ji->count--;
+ if (i < ji->count) {
+ ji->e[i] = ji->e[ji->count];
+ continue;
+ }
+ i++;
+ }
+ }
+}
+
+static void unlink_jumps_tc_range(struct jump_info *ji, u_int base_offs, int shift)
+{
+ int i;
+ if (ji == NULL)
+ return;
+ for (i = 0; i < ji->count; ) {
+ u_int tc_offs = (u_char *)ji->e[i].stub - ndrc->translation_cache;
+ if (((tc_offs ^ base_offs) >> shift) != 0) {
+ i++;
+ continue;
+ }
+
+ inv_debug("EXP: rm link to %08x (tc_offs %zx)\n", ji->e[i].target_vaddr, tc_offs);
+ stat_dec(stat_links);
+ ji->count--;
+ if (i < ji->count) {
+ ji->e[i] = ji->e[ji->count];
+ continue;
}
+ i++;
}
}
u_int i;
block->is_dirty = 1;
- unlink_jumps_range(block->start, block->start + block->len);
+ unlink_jumps_vaddr_range(block->start, block->start + block->len);
for (i = 0; i < block->jump_in_cnt; i++)
hash_table_remove(block->jump_in[i].vaddr);
}
static int invalidate_range(u_int start, u_int end,
u32 *inv_start_ret, u32 *inv_end_ret)
{
+ struct block_info *last_block = NULL;
u_int start_page = get_page_prev(start);
u_int end_page = get_page(end - 1);
u_int start_m = pmmask(start);
- u_int end_m = pmmask(end);
+ u_int end_m = pmmask(end - 1);
u_int inv_start, inv_end;
u_int blk_start_m, blk_end_m;
u_int page;
for (block = blocks[page]; block != NULL; block = block->next) {
if (block->is_dirty)
continue;
+ last_block = block;
blk_end_m = pmmask(block->start + block->len);
if (blk_end_m <= start_m) {
inv_start = max(inv_start, blk_end_m);
}
}
+ if (!hit && last_block && last_block->source) {
+ // could be some leftover unused block, uselessly trapping writes
+ last_block->inv_near_misses++;
+ if (last_block->inv_near_misses > 128) {
+ invalidate_block(last_block);
+ stat_inc(stat_inv_hits);
+ hit++;
+ }
+ }
if (hit) {
do_clear_cache();
#ifdef USE_MINI_HT
memset(mini_ht, -1, sizeof(mini_ht));
#endif
}
+
if (inv_start <= (start_m & ~0xfff) && inv_end >= (start_m | 0xfff))
// the whole page is empty now
mark_invalid_code(start, 1, 1);
// Add an entry to jump_out after making a link
// src should point to code by emit_extjump()
-void ndrc_add_jump_out(u_int vaddr,void *src)
+void ndrc_add_jump_out(u_int vaddr, void *src)
{
- u_int page=get_page(vaddr);
- inv_debug("ndrc_add_jump_out: %p -> %x (%d)\n",src,vaddr,page);
- check_extjump2(src);
- ll_add(jump_out+page,vaddr,src);
- //inv_debug("ndrc_add_jump_out: to %p\n",get_pointer(src));
+ inv_debug("ndrc_add_jump_out: %p -> %x\n", src, vaddr);
+ u_int page = get_page(vaddr);
+ struct jump_info *ji;
+
stat_inc(stat_links);
+ check_extjump2(src);
+ ji = jumps[page];
+ if (ji == NULL) {
+ ji = malloc(sizeof(*ji) + sizeof(ji->e[0]) * 16);
+ ji->alloc = 16;
+ ji->count = 0;
+ }
+ else if (ji->count >= ji->alloc) {
+ ji->alloc += 16;
+ ji = realloc(ji, sizeof(*ji) + sizeof(ji->e[0]) * ji->alloc);
+ }
+ jumps[page] = ji;
+ ji->e[ji->count].target_vaddr = vaddr;
+ ji->e[ji->count].stub = src;
+ ji->count++;
}
/* Register allocation */
}
else
{
- // TLBR/TLBWI/TLBWR/TLBP/ERET
+ // RFE
assert(dops[i].opcode2==0x10);
alloc_all(current,i);
}
//assert(adj==0);
emit_addimm_and_set_flags(ccadj[i] + CLOCK_ADJUST(2), HOST_CCREG);
add_stub(CC_STUB,out,NULL,0,i,-1,TAKEN,rs);
- if(dops[i+1].itype==COP0&&(source[i+1]&0x3f)==0x10)
+ if(dops[i+1].itype==COP0 && dops[i+1].opcode2==0x10)
// special case for RFE
emit_jmp(0);
else
}
SysPrintf("testing if we can run recompiled code @%p...\n", out);
- ((volatile u_int *)out)[0]++; // make cache dirty
+ ((volatile u_int *)(out + ndrc_write_ofs))[0]++; // make the cache dirty
for (i = 0; i < ARRAY_SIZE(ret); i++) {
out = ndrc->translation_cache;
f1_hack=0;
for (n = 0; n < ARRAY_SIZE(blocks); n++)
blocks_clear(&blocks[n]);
- for (n = 0; n < ARRAY_SIZE(jump_out); n++)
- ll_clear(&jump_out[n]);
+ for (n = 0; n < ARRAY_SIZE(jumps); n++) {
+ free(jumps[n]);
+ jumps[n] = NULL;
+ }
stat_clear(stat_blocks);
stat_clear(stat_links);
#elif defined(_MSC_VER)
ndrc = VirtualAlloc(NULL, sizeof(*ndrc), MEM_COMMIT | MEM_RESERVE,
PAGE_EXECUTE_READWRITE);
+ #elif defined(HAVE_LIBNX)
+ Result rc = jitCreate(&g_jit, sizeof(*ndrc));
+ if (R_FAILED(rc))
+ SysPrintf("jitCreate failed: %08x\n", rc);
+ SysPrintf("jitCreate: RX: %p RW: %p type: %d\n", g_jit.rx_addr, g_jit.rw_addr, g_jit.type);
+ ndrc = g_jit.rx_addr;
+ ndrc_write_ofs = (char *)g_jit.rw_addr - (char *)ndrc;
#else
uintptr_t desired_addr = 0;
+ int prot = PROT_READ | PROT_WRITE | PROT_EXEC;
+ int flags = MAP_PRIVATE | MAP_ANONYMOUS;
+ int fd = -1;
#ifdef __ELF__
extern char _end;
desired_addr = ((uintptr_t)&_end + 0xffffff) & ~0xffffffl;
#endif
- ndrc = mmap((void *)desired_addr, sizeof(*ndrc),
- PROT_READ | PROT_WRITE | PROT_EXEC,
- MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
+ #ifdef NDRC_WRITE_OFFSET
+ // mostly for testing
+ fd = open("/dev/shm/pcsxr", O_CREAT | O_RDWR, 0600);
+ ftruncate(fd, sizeof(*ndrc));
+ void *mw = mmap(NULL, sizeof(*ndrc), PROT_READ | PROT_WRITE,
+ (flags = MAP_SHARED), fd, 0);
+ assert(mw != MAP_FAILED);
+ prot = PROT_READ | PROT_EXEC;
+ #endif
+ ndrc = mmap((void *)desired_addr, sizeof(*ndrc), prot, flags, fd, 0);
if (ndrc == MAP_FAILED) {
SysPrintf("mmap() failed: %s\n", strerror(errno));
abort();
}
+ #ifdef NDRC_WRITE_OFFSET
+ ndrc_write_ofs = (char *)mw - (char *)ndrc;
+ #endif
#endif
#else
#ifndef NO_WRITE_EXEC
// sceBlock is managed by retroarch's bootstrap code
//sceKernelFreeMemBlock(sceBlock);
//sceBlock = -1;
+ #elif defined(HAVE_LIBNX)
+ jitClose(&g_jit);
+ ndrc = NULL;
#else
if (munmap(ndrc, sizeof(*ndrc)) < 0)
SysPrintf("munmap() failed\n");
+ ndrc = NULL;
#endif
#endif
for (n = 0; n < ARRAY_SIZE(blocks); n++)
blocks_clear(&blocks[n]);
- for (n = 0; n < ARRAY_SIZE(jump_out); n++)
- ll_clear(&jump_out[n]);
+ for (n = 0; n < ARRAY_SIZE(jumps); n++) {
+ free(jumps[n]);
+ jumps[n] = NULL;
+ }
stat_clear(stat_blocks);
stat_clear(stat_links);
#ifdef ROM_COPY
// SYSCALL instruction (software interrupt)
u=1;
}
- else if(dops[i].itype==COP0 && (source[i]&0x3f)==0x18)
+ else if(dops[i].itype==COP0 && dops[i].opcode2==0x10)
{
- // ERET instruction (return from interrupt)
+ // RFE
u=1;
}
//u=1; // DEBUG
u_int phase = (expirep >> (base_shift - 1)) & 1u;
if (!(expirep & (MAX_OUTPUT_BLOCK_SIZE / 2 - 1))) {
inv_debug("EXP: base_offs %x/%x phase %u\n", base_offs,
- out - ndrc->translation_cache phase);
+ out - ndrc->translation_cache, phase);
}
if (!phase) {
}
}
else
- ll_remove_matching_addrs(&jump_out[block_i], base_offs, base_shift);
+ unlink_jumps_tc_range(jumps[block_i], base_offs, base_shift);
}
}
block->tc_offs = beginning - ndrc->translation_cache;
//block->tc_len = out - beginning;
block->is_dirty = 0;
+ block->inv_near_misses = 0;
block->jump_in_cnt = jump_in_count;
// insert sorted by start mirror-unmasked vaddr
notaz.gp2x.de / pcsx_rearmed.git / blobdiff