translate: fixes for pop scan

[ia32rtools.git] / tools / translate.c

diff --git a/tools/translate.c b/tools/translate.c
index fb77c7a..cca300f 100644 (file)
--- a/tools/translate.c
+++ b/tools/translate.c
@@ -2708,8 +2708,9 @@ static const char *op_to_c(struct parsed_op *po)
 
 // note: this skips over calls and rm'd stuff assuming they're handled
 // so it's intended to use at one of final passes
+// exception: doesn't skip OPF_RSAVE stuff
 static int scan_for_pop(int i, int opcnt, int magic, int reg,
-  int depth, int seen_noreturn, int flags_set)
+  int depth, int seen_noreturn, int save_level, int flags_set)
 {
   struct parsed_op *po;
   int relevant;
@@ -2723,18 +2724,28 @@ static int scan_for_pop(int i, int opcnt, int magic, int reg,
     po->cc_scratch = magic;
 
     if (po->flags & OPF_TAIL) {
-      if (po->op == OP_CALL) {
-        if (po->pp != NULL && po->pp->is_noreturn)
-          seen_noreturn = 1;
-        else
+      if (po->op == OP_CALL && po->pp != NULL && po->pp->is_noreturn) {
+        // msvc sometimes generates stack cleanup code after
+        // noreturn, set a flag and continue
+        seen_noreturn = 1;
+
+        // ... but stop if there is another path to next insn -
+        // if msvc skipped something stack tracking may mess up
+        if (i + 1 < opcnt && g_labels[i + 1] != NULL)
           goto out;
       }
       else
         goto out;
     }
 
-    if (po->flags & (OPF_RMD|OPF_DONE|OPF_FARG))
+    if (po->flags & OPF_FARG)
       continue;
+    if (po->flags & (OPF_RMD|OPF_DONE)) {
+      if (!(po->flags & OPF_RSAVE))
+        continue;
+      // reprocess, there might be another push in some "parallel"
+      // path that took a pop what we should also take
+    }
 
     if ((po->flags & OPF_JMP) && po->op != OP_CALL) {
       if (po->btj != NULL) {
@@ -2742,7 +2753,7 @@ static int scan_for_pop(int i, int opcnt, int magic, int reg,
         for (j = 0; j < po->btj->count; j++) {
           check_i(po, po->btj->d[j].bt_i);
           ret |= scan_for_pop(po->btj->d[j].bt_i, opcnt, magic, reg,
-                   depth, seen_noreturn, flags_set);
+                   depth, seen_noreturn, save_level, flags_set);
           if (ret < 0)
             return ret; // dead end
         }
@@ -2752,7 +2763,7 @@ static int scan_for_pop(int i, int opcnt, int magic, int reg,
       check_i(po, po->bt_i);
       if (po->flags & OPF_CJMP) {
         ret |= scan_for_pop(po->bt_i, opcnt, magic, reg,
-                 depth, seen_noreturn, flags_set);
+                 depth, seen_noreturn, save_level, flags_set);
         if (ret < 0)
           return ret; // dead end
       }
@@ -2774,6 +2785,13 @@ static int scan_for_pop(int i, int opcnt, int magic, int reg,
     }
     else if (po->op == OP_POP) {
       if (relevant && depth == 0) {
+        if (flags_set == 0 && save_level > 0) {
+          ret = scan_for_pop(i + 1, opcnt, magic, reg,
+                  depth, seen_noreturn, save_level - 1, flags_set);
+          if (ret != 1)
+            // no pop for other levels, current one must be false
+            return -1;
+        }
         po->flags |= flags_set;
         return 1;
       }
@@ -4232,7 +4250,7 @@ static void check_simple_sequence(int opcnt, int *fsz)
   }
 
   // unlike pushes after sub esp,
-  // IDA treats pushed like this as part of var area
+  // IDA treats pushes like this as part of var area
   *fsz += seq_len * 4;
 }
 
@@ -4255,7 +4273,11 @@ static int scan_prologue(int i, int opcnt, int *ecx_push, int *esp_sub)
   for (; i < opcnt; i++) {
     if (i > 0 && g_labels[i] != NULL)
       break;
-    if (ops[i].op == OP_PUSH || (ops[i].flags & (OPF_JMP|OPF_TAIL)))
+    if (ops[i].flags & (OPF_JMP|OPF_TAIL))
+      break;
+    if (ops[i].flags & OPF_DONE)
+      continue;
+    if (ops[i].op == OP_PUSH)
       break;
     if (ops[i].op == OP_SUB && ops[i].operand[0].reg == xSP
       && ops[i].operand[1].type == OPT_CONST)
@@ -4293,8 +4315,8 @@ static int scan_prologue(int i, int opcnt, int *ecx_push, int *esp_sub)
         ops[j].flags |= OPF_RMD | OPF_DONE | OPF_NOREGS;
         i = j + 1;
         *esp_sub = 1;
+        break;
       }
-      break;
     }
   }
 
@@ -4488,10 +4510,12 @@ static void scan_prologue_epilogue(int opcnt, int *stack_align)
         for (; j >= 0; j--) {
           if (ops[j].op != OP_MOV)
             break;
-          if (ops[j].operand[0].type != OPT_REGMEM)
-            break;
-          if (strstr(ops[j].operand[0].name, "arg_") == NULL)
-            break;
+          if (ops[j].operand[0].type == OPT_REGMEM
+              && strstr(ops[j].operand[0].name, "arg_") != NULL)
+            continue;
+          if (ops[j].operand[0].type == OPT_REG)
+            continue; // assume arg-reg mov
+          break;
         }
       }
 
@@ -5655,6 +5679,14 @@ static int collect_call_args(struct parsed_op *po, int i,
   if (ret < 0)
     return ret;
 
+  if (pp->is_unresolved) {
+    pp->argc += ret;
+    pp->argc_stack += ret;
+    for (a = 0; a < pp->argc; a++)
+      if (pp->arg[a].type.name == NULL)
+        pp->arg[a].type.name = strdup("int");
+  }
+
   if (arg_grp != 0) {
     // propagate arg_grp
     for (a = 0; a < pp->argc; a++) {
@@ -5669,14 +5701,6 @@ static int collect_call_args(struct parsed_op *po, int i,
     }
   }
 
-  if (pp->is_unresolved) {
-    pp->argc += ret;
-    pp->argc_stack += ret;
-    for (a = 0; a < pp->argc; a++)
-      if (pp->arg[a].type.name == NULL)
-        pp->arg[a].type.name = strdup("int");
-  }
-
   return ret;
 }
 
@@ -5727,6 +5751,8 @@ static void reg_use_pass(int i, int opcnt, unsigned char *cbits,
       && !g_func_pp->is_userstack
       && po->operand[0].type == OPT_REG)
     {
+      int save_level = 0;
+
       reg = po->operand[0].reg;
       ferr_assert(po, reg >= 0);
 
@@ -5735,12 +5761,14 @@ static void reg_use_pass(int i, int opcnt, unsigned char *cbits,
       if (regmask_now & (1 << reg)) {
         already_saved = regmask_save_now & (1 << reg);
         flags_set = OPF_RSAVE | OPF_DONE;
+        save_level++;
       }
 
-      ret = scan_for_pop(i + 1, opcnt, i + opcnt * 3, reg, 0, 0, 0);
+      ret = scan_for_pop(i + 1, opcnt, i + opcnt * 3,
+              reg, 0, 0, save_level, 0);
       if (ret == 1) {
         scan_for_pop(i + 1, opcnt, i + opcnt * 4,
-          reg, 0, 0, flags_set);
+          reg, 0, 0, save_level, flags_set);
       }
       else {
         ret = scan_for_pop_ret(i + 1, opcnt, po->operand[0].reg, 0);
@@ -6329,11 +6357,16 @@ static void gen_func(FILE *fout, FILE *fhdr, const char *funcn, int opcnt)
             regmask_stack |= 1 << tmp_op->operand[0].reg;
         }
 
-        if (!((regmask_stack & (1 << xCX))
-          && (regmask_stack & (1 << xDX))))
+        // quick dumb check for potential reg-args
+        for (j = i - 1; j >= 0 && ops[j].op == OP_MOV; j--)
+          if (ops[j].operand[0].type == OPT_REG)
+            regmask_stack &= ~(1 << ops[j].operand[0].reg);
+
+        if ((regmask_stack & (mxCX|mxDX)) != (mxCX|mxDX)
+            && ((regmask | regmask_arg) & (mxCX|mxDX)))
         {
           if (pp->argc_stack != 0
-           || ((regmask | regmask_arg) & ((1 << xCX)|(1 << xDX))))
+              || ((regmask | regmask_arg) & (mxCX|mxDX)))
           {
             pp_insert_reg_arg(pp, "ecx");
             pp->is_fastcall = 1;
@@ -6341,7 +6374,7 @@ static void gen_func(FILE *fout, FILE *fhdr, const char *funcn, int opcnt)
             regmask |= 1 << xCX;
           }
           if (pp->argc_stack != 0
-           || ((regmask | regmask_arg) & (1 << xDX)))
+              || ((regmask | regmask_arg) & mxDX))
           {
             pp_insert_reg_arg(pp, "edx");
             regmask_init |= 1 << xDX;
@@ -6462,8 +6495,15 @@ static void gen_func(FILE *fout, FILE *fhdr, const char *funcn, int opcnt)
     default:
       break;
     }
+  }
+
+  // pass8: final adjustments
+  for (i = 0; i < opcnt; i++)
+  {
+    po = &ops[i];
+    if (po->flags & (OPF_RMD|OPF_DONE))
+      continue;
 
-    // this might need it's own pass...
     if (po->op != OP_FST && po->p_argnum > 0)
       save_arg_vars[po->p_arggrp] |= 1 << (po->p_argnum - 1);
 
@@ -8508,11 +8548,13 @@ static void gen_hdr_dep_pass(int i, int opcnt, unsigned char *cbits,
       if (po->flags & OPF_DONE)
         continue;
 
-      ret = scan_for_pop(i + 1, opcnt, i + opcnt * 2, reg, 0, 0, 0);
+      ret = scan_for_pop(i + 1, opcnt, i + opcnt * 2,
+              reg, 0, 0, 0, 0);
       if (ret == 1) {
         regmask_save |= 1 << reg;
         po->flags |= OPF_RMD;
-        scan_for_pop(i + 1, opcnt, i + opcnt * 3, reg, 0, 0, OPF_RMD);
+        scan_for_pop(i + 1, opcnt, i + opcnt * 3,
+          reg, 0, 0, 0, OPF_RMD);
         continue;
       }
     }