drc: update cache flushing

as of now the arm64 __clear_cache workaround is still needed

diff --git a/libpcsxcore/new_dynarec/assem_arm.c b/libpcsxcore/new_dynarec/assem_arm.c
index 62038a2..f9333f2 100644 (file)
--- a/libpcsxcore/new_dynarec/assem_arm.c
+++ b/libpcsxcore/new_dynarec/assem_arm.c
@@ -2348,49 +2348,6 @@ static void do_miniht_insert(u_int return_address,int rt,int temp) {
   #endif
 }
 
   #endif
 }
 

-static void mark_clear_cache(void *target)
-{
-  u_long offset = (u_char *)target - translation_cache;
-  u_int mask = 1u << ((offset >> 12) & 31);
-  if (!(needs_clear_cache[offset >> 17] & mask)) {
-    char *start = (char *)((u_long)target & ~4095ul);
-    start_tcache_write(start, start + 4096);
-    needs_clear_cache[offset >> 17] |= mask;
-  }
-}
-
-// Clearing the cache is rather slow on ARM Linux, so mark the areas
-// that need to be cleared, and then only clear these areas once.
-static void do_clear_cache()
-{
-  int i,j;
-  for (i=0;i<(1<<(TARGET_SIZE_2-17));i++)
-  {
-    u_int bitmap=needs_clear_cache[i];
-    if(bitmap) {
-      u_char *start, *end;
-      for(j=0;j<32;j++)
-      {
-        if(bitmap&(1<<j)) {
-          start=translation_cache+i*131072+j*4096;
-          end=start+4095;
-          j++;
-          while(j<32) {
-            if(bitmap&(1<<j)) {
-              end+=4096;
-              j++;
-            }else{
-              end_tcache_write(start, end);
-              break;
-            }
-          }
-        }
-      }
-      needs_clear_cache[i]=0;
-    }
-  }
-}
-

 // CPU-architecture-specific initialization
 static void arch_init(void)
 {
 // CPU-architecture-specific initialization
 static void arch_init(void)
 {

diff --git a/libpcsxcore/new_dynarec/assem_arm64.c b/libpcsxcore/new_dynarec/assem_arm64.c
index 5483da1..ef87b29 100644 (file)
--- a/libpcsxcore/new_dynarec/assem_arm64.c
+++ b/libpcsxcore/new_dynarec/assem_arm64.c
@@ -2015,47 +2015,45 @@ static void do_miniht_insert(u_int return_address,u_int rt,int temp) {
   emit_writeword(rt,&mini_ht[(return_address&0xFF)>>3][0]);
 }
 
   emit_writeword(rt,&mini_ht[(return_address&0xFF)>>3][0]);
 }
 

-static void mark_clear_cache(void *target)
+static void clear_cache_arm64(char *start, char *end)

 {
 {

-  u_long offset = (u_char *)target - translation_cache;
-  u_int mask = 1u << ((offset >> 12) & 31);
-  if (!(needs_clear_cache[offset >> 17] & mask)) {
-    char *start = (char *)((u_long)target & ~4095ul);
-    start_tcache_write(start, start + 4096);
-    needs_clear_cache[offset >> 17] |= mask;
+  // Don't rely on GCC's __clear_cache implementation, as it caches
+  // icache/dcache cache line sizes, that can vary between cores on
+  // big.LITTLE architectures.
+  uint64_t addr, ctr_el0;
+  static size_t icache_line_size = 0xffff, dcache_line_size = 0xffff;
+  size_t isize, dsize;
+
+  __asm__ volatile("mrs %0, ctr_el0" : "=r"(ctr_el0));
+  isize = 4 << ((ctr_el0 >> 0) & 0xf);
+  dsize = 4 << ((ctr_el0 >> 16) & 0xf);
+
+  // use the global minimum cache line size
+  icache_line_size = isize = icache_line_size < isize ? icache_line_size : isize;
+  dcache_line_size = dsize = dcache_line_size < dsize ? dcache_line_size : dsize;
+
+  /* If CTR_EL0.IDC is enabled, Data cache clean to the Point of Unification is
+     not required for instruction to data coherence.  */
+  if ((ctr_el0 & (1 << 28)) == 0x0) {
+    addr = (uint64_t)start & ~(uint64_t)(dsize - 1);
+    for (; addr < (uint64_t)end; addr += dsize)
+      // use "civac" instead of "cvau", as this is the suggested workaround for
+      // Cortex-A53 errata 819472, 826319, 827319 and 824069.
+      __asm__ volatile("dc civac, %0" : : "r"(addr) : "memory");

   }
   }

-}
+  __asm__ volatile("dsb ish" : : : "memory");

 
 

-// Clearing the cache is rather slow on ARM Linux, so mark the areas
-// that need to be cleared, and then only clear these areas once.
-static void do_clear_cache()
-{
-  int i,j;
-  for (i=0;i<(1<<(TARGET_SIZE_2-17));i++)
-  {
-    u_int bitmap=needs_clear_cache[i];
-    if(bitmap) {
-      u_char *start, *end;
-      for(j=0;j<32;j++)
-      {
-        if(bitmap&(1<<j)) {
-          start=translation_cache+i*131072+j*4096;
-          end=start+4095;
-          j++;
-          while(j<32) {
-            if(bitmap&(1<<j)) {
-              end+=4096;
-              j++;
-            }else{
-              end_tcache_write(start, end);
-              break;
-            }
-          }
-        }
-      }
-      needs_clear_cache[i]=0;
-    }
+  /* If CTR_EL0.DIC is enabled, Instruction cache cleaning to the Point of
+     Unification is not required for instruction to data coherence.  */
+  if ((ctr_el0 & (1 << 29)) == 0x0) {
+    addr = (uint64_t)start & ~(uint64_t)(isize - 1);
+    for (; addr < (uint64_t)end; addr += isize)
+      __asm__ volatile("ic ivau, %0" : : "r"(addr) : "memory");
+
+    __asm__ volatile("dsb ish" : : : "memory");

   }
   }

+
+  __asm__ volatile("isb" : : : "memory");

 }
 
 // CPU-architecture-specific initialization
 }
 
 // CPU-architecture-specific initialization

diff --git a/libpcsxcore/new_dynarec/new_dynarec.c b/libpcsxcore/new_dynarec/new_dynarec.c
index cfeddc2..ef9bec7 100644 (file)
--- a/libpcsxcore/new_dynarec/new_dynarec.c
+++ b/libpcsxcore/new_dynarec/new_dynarec.c
@@ -355,7 +355,7 @@ static void start_tcache_write(void *start, void *end)
 
 static void end_tcache_write(void *start, void *end)
 {
 
 static void end_tcache_write(void *start, void *end)
 {

-#ifdef __arm__
+#if defined(__arm__) || defined(__aarch64__)

   size_t len = (char *)end - (char *)start;
   #if   defined(__BLACKBERRY_QNX__)
   msync(start, len, MS_SYNC | MS_CACHE_ONLY | MS_INVALIDATE_ICACHE);
   size_t len = (char *)end - (char *)start;
   #if   defined(__BLACKBERRY_QNX__)
   msync(start, len, MS_SYNC | MS_CACHE_ONLY | MS_INVALIDATE_ICACHE);


@@ -365,12 +365,14 @@ static void end_tcache_write(void *start, void *end)
   sceKernelSyncVMDomain(sceBlock, start, len);
   #elif defined(_3DS)
   ctr_flush_invalidate_cache();
   sceKernelSyncVMDomain(sceBlock, start, len);
   #elif defined(_3DS)
   ctr_flush_invalidate_cache();

+  #elif defined(__aarch64__)
+  // as of 2021, __clear_cache() is still broken on arm64
+  // so here is a custom one :(
+  clear_cache_arm64(start, end);

   #else
   __clear_cache(start, end);
   #endif
   (void)len;
   #else
   __clear_cache(start, end);
   #endif
   (void)len;

-#else
-  __clear_cache(start, end);

 #endif
 
   mprotect_w_x(start, end, 1);
 #endif
 
   mprotect_w_x(start, end, 1);


@@ -390,6 +392,49 @@ static void end_block(void *start)
   end_tcache_write(start, out);
 }
 
   end_tcache_write(start, out);
 }
 

+// also takes care of w^x mappings when patching code
+static u_int needs_clear_cache[1<<(TARGET_SIZE_2-17)];
+
+static void mark_clear_cache(void *target)
+{
+  uintptr_t offset = (u_char *)target - ndrc->translation_cache;
+  u_int mask = 1u << ((offset >> 12) & 31);
+  if (!(needs_clear_cache[offset >> 17] & mask)) {
+    char *start = (char *)((uintptr_t)target & ~4095l);
+    start_tcache_write(start, start + 4095);
+    needs_clear_cache[offset >> 17] |= mask;
+  }
+}
+
+// Clearing the cache is rather slow on ARM Linux, so mark the areas
+// that need to be cleared, and then only clear these areas once.
+static void do_clear_cache(void)
+{
+  int i, j;
+  for (i = 0; i < (1<<(TARGET_SIZE_2-17)); i++)
+  {
+    u_int bitmap = needs_clear_cache[i];
+    if (!bitmap)
+      continue;
+    for (j = 0; j < 32; j++)
+    {
+      u_char *start, *end;
+      if (!(bitmap & (1<<j)))
+        continue;
+
+      start = ndrc->translation_cache + i*131072 + j*4096;
+      end = start + 4095;
+      for (j++; j < 32; j++) {
+        if (!(bitmap & (1<<j)))
+          break;
+        end += 4096;
+      }
+      end_tcache_write(start, end);
+    }
+    needs_clear_cache[i] = 0;
+  }
+}
+

 //#define DEBUG_CYCLE_COUNT 1
 
 #define NO_CYCLE_PENALTY_THR 12
 //#define DEBUG_CYCLE_COUNT 1
 
 #define NO_CYCLE_PENALTY_THR 12


@@ -1054,9 +1099,7 @@ static void ll_kill_pointers(struct ll_entry *head,uintptr_t addr,int shift)
     {
       inv_debug("EXP: Kill pointer at %p (%x)\n",head->addr,head->vaddr);
       void *host_addr=find_extjump_insn(head->addr);
     {
       inv_debug("EXP: Kill pointer at %p (%x)\n",head->addr,head->vaddr);
       void *host_addr=find_extjump_insn(head->addr);

-      #if defined(__arm__) || defined(__aarch64__)
-        mark_clear_cache(host_addr);
-      #endif
+      mark_clear_cache(host_addr);

       set_jump_target(host_addr, head->addr);
     }
     head=head->next;
       set_jump_target(host_addr, head->addr);
     }
     head=head->next;


@@ -1082,9 +1125,7 @@ static void invalidate_page(u_int page)
   while(head!=NULL) {
     inv_debug("INVALIDATE: kill pointer to %x (%p)\n",head->vaddr,head->addr);
     void *host_addr=find_extjump_insn(head->addr);
   while(head!=NULL) {
     inv_debug("INVALIDATE: kill pointer to %x (%p)\n",head->vaddr,head->addr);
     void *host_addr=find_extjump_insn(head->addr);

-    #if defined(__arm__) || defined(__aarch64__)
-      mark_clear_cache(host_addr);
-    #endif
+    mark_clear_cache(host_addr);

     set_jump_target(host_addr, head->addr);
     next=head->next;
     free(head);
     set_jump_target(host_addr, head->addr);
     next=head->next;
     free(head);


@@ -1107,9 +1148,7 @@ static void invalidate_block_range(u_int block, u_int first, u_int last)
   for(first=page+1;first<last;first++) {
     invalidate_page(first);
   }
   for(first=page+1;first<last;first++) {
     invalidate_page(first);
   }

-  #if defined(__arm__) || defined(__aarch64__)
-    do_clear_cache();
-  #endif
+  do_clear_cache();

 
   // Don't trap writes
   invalid_code[block]=1;
 
   // Don't trap writes
   invalid_code[block]=1;


@@ -1206,7 +1245,7 @@ void invalidate_addr(u_int addr)
 
 // This is called when loading a save state.
 // Anything could have changed, so invalidate everything.
 
 // This is called when loading a save state.
 // Anything could have changed, so invalidate everything.

-void invalidate_all_pages()
+void invalidate_all_pages(void)

 {
   u_int page;
   for(page=0;page<4096;page++)
 {
   u_int page;
   for(page=0;page<4096;page++)


@@ -1219,6 +1258,7 @@ void invalidate_all_pages()
   #ifdef USE_MINI_HT
   memset(mini_ht,-1,sizeof(mini_ht));
   #endif
   #ifdef USE_MINI_HT
   memset(mini_ht,-1,sizeof(mini_ht));
   #endif

+  do_clear_cache();

 }
 
 static void do_invstub(int n)
 }
 
 static void do_invstub(int n)


@@ -6476,7 +6516,7 @@ static void new_dynarec_test(void)
 
 // clear the state completely, instead of just marking
 // things invalid like invalidate_all_pages() does
 
 // clear the state completely, instead of just marking
 // things invalid like invalidate_all_pages() does

-void new_dynarec_clear_full()
+void new_dynarec_clear_full(void)

 {
   int n;
   out = ndrc->translation_cache;
 {
   int n;
   out = ndrc->translation_cache;


@@ -6497,7 +6537,7 @@ void new_dynarec_clear_full()
   for(n=0;n<4096;n++) ll_clear(jump_dirty+n);
 }
 
   for(n=0;n<4096;n++) ll_clear(jump_dirty+n);
 }
 

-void new_dynarec_init()
+void new_dynarec_init(void)

 {
   SysPrintf("Init new dynarec\n");
 
 {
   SysPrintf("Init new dynarec\n");
 


@@ -6547,7 +6587,7 @@ void new_dynarec_init()
     SysPrintf("warning: RAM is not directly mapped, performance will suffer\n");
 }
 
     SysPrintf("warning: RAM is not directly mapped, performance will suffer\n");
 }
 

-void new_dynarec_cleanup()
+void new_dynarec_cleanup(void)

 {
   int n;
 #ifdef BASE_ADDR_DYNAMIC
 {
   int n;
 #ifdef BASE_ADDR_DYNAMIC


@@ -9082,10 +9122,8 @@ int new_recompile_block(u_int addr)
         break;
       case 3:
         // Clear jump_out
         break;
       case 3:
         // Clear jump_out

-        #if defined(__arm__) || defined(__aarch64__)

         if((expirep&2047)==0)
           do_clear_cache();
         if((expirep&2047)==0)
           do_clear_cache();

-        #endif

         ll_remove_matching_addrs(jump_out+(expirep&2047),base,shift);
         ll_remove_matching_addrs(jump_out+2048+(expirep&2047),base,shift);
         break;
         ll_remove_matching_addrs(jump_out+(expirep&2047),base,shift);
         ll_remove_matching_addrs(jump_out+2048+(expirep&2047),base,shift);
         break;

diff --git a/libpcsxcore/new_dynarec/new_dynarec.h b/libpcsxcore/new_dynarec/new_dynarec.h
index 1bec5e1..a19bff0 100644 (file)
--- a/libpcsxcore/new_dynarec/new_dynarec.h
+++ b/libpcsxcore/new_dynarec/new_dynarec.h
@@ -11,12 +11,12 @@ extern int cycle_multiplier; // 100 for 1.0
 #define NDHACK_GTE_NO_FLAGS    (1<<2)
 extern int new_dynarec_hacks;
 
 #define NDHACK_GTE_NO_FLAGS    (1<<2)
 extern int new_dynarec_hacks;
 

-void new_dynarec_init();
-void new_dynarec_cleanup();
-void new_dynarec_clear_full();
+void new_dynarec_init(void);
+void new_dynarec_cleanup(void);
+void new_dynarec_clear_full(void);

 void new_dyna_start(void *context);
 int  new_dynarec_save_blocks(void *save, int size);
 void new_dynarec_load_blocks(const void *save, int size);
 
 void new_dyna_start(void *context);
 int  new_dynarec_save_blocks(void *save, int size);
 void new_dynarec_load_blocks(const void *save, int size);
 

-void invalidate_all_pages();
+void invalidate_all_pages(void);

 void invalidate_block(unsigned int block);
 void invalidate_block(unsigned int block);