gte: Fix gteH division and sign extension (from PCSX4ALL)

gteH register is u16, not s16. DIVIDE macro/func assumed it was
s16 for some reason. Behavior now matches Mednafen.

Co-authored-by: senquack <dansilsby@gmail.com>

diff --git a/libpcsxcore/gte.c b/libpcsxcore/gte.c
index 83bb4b6..3ae216f 100644 (file)
--- a/libpcsxcore/gte.c
+++ b/libpcsxcore/gte.c
@@ -149,7 +149,13 @@
 #define gteBFC (((s32 *)regs->CP2C.r)[23])
 #define gteOFX (((s32 *)regs->CP2C.r)[24])
 #define gteOFY (((s32 *)regs->CP2C.r)[25])
-#define gteH   (regs->CP2C.p[26].sw.l)
+// senquack - gteH register is u16, not s16, and used in GTE that way.
+//  HOWEVER when read back by CPU using CFC2, it will be incorrectly
+//  sign-extended by bug in original hardware, according to Nocash docs
+//  GTE section 'Screen Offset and Distance'. The emulator does this
+//  sign extension when it is loaded to GTE by CTC2.
+//#define gteH   (regs->CP2C.p[26].sw.l)
+#define gteH   (regs->CP2C.p[26].w.l)
 #define gteDQA (regs->CP2C.p[27].sw.l)
 #define gteDQB (((s32 *)regs->CP2C.r)[28])
 #define gteZSF3 (regs->CP2C.p[29].sw.l)
@@ -254,7 +260,17 @@ static inline u32 limE_(psxCP2Regs *regs, u32 result) {
 #define A3U(x) (x)
 #endif
 
+//senquack - n param should be unsigned (will be 'gteH' reg which is u16)
+#ifdef GTE_USE_NATIVE_DIVIDE
+INLINE u32 DIVIDE(u16 n, u16 d) {
+       if (n < d * 2) {
+               return ((u32)n << 16) / d;
+       }
+       return 0xffffffff;
+}
+#else
 #include "gte_divider.h"
+#endif // GTE_USE_NATIVE_DIVIDE
 
 #ifndef FLAGLESS
 

diff --git a/libpcsxcore/gte_divider.h b/libpcsxcore/gte_divider.h
index a407213..765b971 100644 (file)
--- a/libpcsxcore/gte_divider.h
+++ b/libpcsxcore/gte_divider.h
@@ -18,6 +18,6 @@
 #ifndef __GTE_DIVIDER_H__
 #define __GTE_DIVIDER_H__
 
-u32 DIVIDE(s16 n, u16 d);
+u32 DIVIDE(u16 n, u16 d);
 
 #endif /* __GTE_DIVIDER_H__ */