user3137783 user3137783 - 1 month ago 10
C Question

Reverse Engineering a recursive function

I have as a homework to reverse a function

The assembler output is this:

0x00000000004010c4 <+0>: sub $0x18,%rsp
0x00000000004010c8 <+4>: lea 0xc(%rsp),%rcx
0x00000000004010cd <+9>: lea 0x8(%rsp),%rdx
0x00000000004010d2 <+14>: mov $0x402995,%esi
0x00000000004010d7 <+19>: mov $0x0,%eax
0x00000000004010dc <+24>: callq 0x400cb0 <__isoc99_sscanf@plt>
0x00000000004010e1 <+29>: cmp $0x2,%eax
0x00000000004010e4 <+32>: jne 0x4010ed <phase_4+41>
0x00000000004010e6 <+34>: cmpl $0xe,0x8(%rsp)
0x00000000004010eb <+39>: jbe 0x4010f2 <phase_4+46>
0x00000000004010ed <+41>: callq 0x401671 <explode_bomb>
0x00000000004010f2 <+46>: mov $0xe,%edx
0x00000000004010f7 <+51>: mov $0x0,%esi
0x00000000004010fc <+56>: mov 0x8(%rsp),%edi
0x0000000000401100 <+60>: callq 0x401086 <func4>
0x0000000000401105 <+65>: cmp $0x3,%eax
0x0000000000401108 <+68>: jne 0x401111 <phase_4+77>
0x000000000040110a <+70>: cmpl $0x3,0xc(%rsp)
0x000000000040110f <+75>: je 0x401116 <phase_4+82>
0x0000000000401111 <+77>: callq 0x401671 <explode_bomb>
0x0000000000401116 <+82>: add $0x18,%rsp
0x000000000040111a <+86>: retq


My solution to this function looks like this:
I think func4 should return 3

int phase4(const char* read ) {
int var1, var2;
if ((sscanf(read, "%d %d", &var1, &var2) != 2) || (var1 < 0xe))
explode_bomb();

if (func4(var1, 0, 0xe /*14*/) != 3)
explode_bomb();

if (var2 != 3)
explode_bomb();
return 3;
}


func4 looks like this:

0x0000000000401086 <+0>: sub $0x8,%rsp
0x000000000040108a <+4>: mov %edx,%eax
0x000000000040108c <+6>: sub %esi,%eax
0x000000000040108e <+8>: mov %eax,%ecx
0x0000000000401090 <+10>: shr $0x1f,%ecx
0x0000000000401093 <+13>: add %ecx,%eax
0x0000000000401095 <+15>: sar %eax
0x0000000000401097 <+17>: lea (%rax,%rsi,1),%ecx
0x000000000040109a <+20>: cmp %edi,%ecx
0x000000000040109c <+22>: jle 0x4010aa <func4+36>
0x000000000040109e <+24>: lea -0x1(%rcx),%edx
0x00000000004010a1 <+27>: callq 0x401086 <func4>
0x00000000004010a6 <+32>: add %eax,%eax
0x00000000004010a8 <+34>: jmp 0x4010bf <func4+57>
0x00000000004010aa <+36>: mov $0x0,%eax
0x00000000004010af <+41>: cmp %edi,%ecx
0x00000000004010b1 <+43>: jge 0x4010bf <func4+57>
0x00000000004010b3 <+45>: lea 0x1(%rcx),%esi
0x00000000004010b6 <+48>: callq 0x401086 <func4>
0x00000000004010bb <+53>: lea 0x1(%rax,%rax,1),%eax
0x00000000004010bf <+57>: add $0x8,%rsp
0x00000000004010c3 <+61>: retq


My c code looks like this:

int func4(unsigned rsi, unsigned rdi, unsigned rdx) {
unsigned rax = rdx;
rax -= rsi;
unsigned rcx = rax;
rcx >>= (unsigned)0x1f;
rax += rcx;
rax >>= (signed)1;
rcx = rax + rsi;
if (rcx <= rdi) {
rax = 0;
if (rcx >= rdi)
return rax;
else {
rax = func4(rdi, rsi + 1, rdx);
rax = rax + rax + 1;
}
} else {
rdx = rcx - 1;
rax = func4(rdi, rsi, rdx);
rax = rax + rax;
}
return rax;
}


But when I try the values from -512 to 512 I never get 3 as a result; what am I doing wrong?

EDIT:

I found the solution it looks like this:

int func4(int32_t di, int32_t si, int32_t dx) {
int32_t ax = dx;
ax = ax - si;
int32_t cx = ax;
cx = (uint32_t)cx >> (uint32_t)0x1f;
ax = ax + cx;
ax = (int32_t)ax >> (int32_t)1;
cx = ax + si;

if (cx <= di)
goto first;

dx = cx - 1;
ax = func4(di, si, dx);
ax = ax + ax;
goto fin;

first:
ax = 0;
if (cx >= di)
goto fin;

si = cx + 1;
ax = func4(di, si, dx);
ax = ax + ax + 1;

fin:
return ax;
}

Answer

From a quick glance the problem might be here:

rax >>= (signed)1;           // sar    %eax

This is equivalent to:

rax = rax >> (signed)1;

Which does an unsigned shift (because the signedness of the shift operator is determined by the first operand, not the second). So instead you should write:

rax = (unsigned)((signed)rax >> 1);

EDIT: similarly, you translated jle and jge incorrectly. These instructions do a signed comparison, whereas your corresponding C code does an unsigned comparison. Fix that too:

if ((signed)rcx <= (signed)rdi) {
    rax = 0;
    if ((signed)rcx >= (signed)rdi)
...
Comments