Dhruv Dhruv - 15 days ago 5
Linux Question

Thread vs process: icmp packet creation fails in thread

I want to generate an

ICMP
echo request through C program, in a posix thread in linux.

As a tryout, I wrote a sample code in
main()
.
ICMP
echo and its reply worked as expected. The packet length was 28 (20 bytes
IP
header + 8 bytes
ICMP
header).

Than I shifted the code to a thread. Now the
main()
creates the thread and a wait for it to exit.

But, in the thread,
sendto()
returns 28, while when observed in
tcpdump
, this packet is shown with length 48, and a line below as IP bad-hlen 0, which signifies that the
ECHO
request was not proper. The total length field in
IP
header shows 0x30 (48 bytes) instead of 0x1c (28 bytes). Following are the tcpdump snapshots.

Successful tcpdump, using process code

06:30:58.139476 IP (tos 0x0, ttl 64, id 19213, offset 0, flags [none], proto ICMP (1), length 28)
192.168.11.34 > 192.168.11.32: ICMP echo request, id 0, seq 0, length 8
0x0000: 4500 001c 4b0d 0000 4001 9841 c0a8 0b22 E...K...@..A..."
0x0010: c0a8 0b20 0800 f7ff 0000 0000 ............
06:30:58.139819 IP (tos 0x0, ttl 64, id 6830, offset 0, flags [none], proto ICMP (1), length 28)
192.168.11.32 > 192.168.11.34: ICMP echo reply, id 0, seq 0, length 8
0x0000: 4500 001c 1aae 0000 4001 c8a0 c0a8 0b20 E.......@.......
0x0010: c0a8 0b22 0000 ffff 0000 0000 0000 0000 ..."............
0x0020: 0000 0000 0000 0000 0000 0000 0000 ..............


Packet with incorrect header length/data/some ghost.

06:33:14.513597 IP (tos 0x0, ttl 64, id 22998, offset 0, flags [DF], proto ICMP (1), length 48)
192.168.11.34 > 192.168.11.32: ICMP type-#69, length 28
IP bad-hlen 0
0x0000: 4500 0030 59d6 4000 4001 4964 c0a8 0b22 E..0Y.@.@.Id..."
0x0010: c0a8 0b20 4500 1c00 4b0d 0000 4001 7c5d ....E...K...@.|]
0x0020: c0a8 0b22 c0a8 0b20 0800 f7ff 0000 0000 ..."............


This results in failure in
recv()
.

As part of troubleshooting, dumped the buffer used for sending to a file and verified through
hexdump
. Both code generates same packet. Verified by printing hex values too. Same result. Tried forking, instead of creating thread. It worked.

The only difference in two codes is thread and process. Running out of probable problems.

The distros tried are CentOS 7.1 (Kernel 3.10) and Fedora 13 (kernel 2.6.39).

Here is the process code.

#include <stdio.h>
#include <signal.h>
#include <stdlib.h>
#include <unistd.h>
#include <netdb.h>
#include <linux/ip.h>
#include <linux/icmp.h>
#include <sys/socket.h>
#include <sys/stat.h>
#include <netinet/in.h>
#include <arpa/inet.h>
#include <sys/ioctl.h>
#include <string.h>
#include <fcntl.h>
#include <net/if.h>
#include <pthread.h>

unsigned short in_cksum(unsigned short *addr, int len)
{
register int sum = 0;
u_short answer = 0;
register u_short *w = addr;
register int nleft = len;
/*
* * Our algorithm is simple, using a 32 bit accumulator (sum), we add
* * sequential 16 bit words to it, and at the end, fold back all the
* * carry bits from the top 16 bits into the lower 16 bits.
* */
while (nleft > 1)
{
sum += *w++;
nleft -= 2;
}
/* mop up an odd byte, if necessary */
if (nleft == 1)
{
*(u_char *) (&answer) = *(u_char *) w;
sum += answer;
}
/* add back carry outs from top 16 bits to low 16 bits */
sum = (sum >> 16) + (sum & 0xffff); /* add hi 16 to low 16 */
sum += (sum >> 16); /* add carry */
answer = ~sum; /* truncate to 16 bits */
return (answer);
}

int main()
{
struct iphdr *ip, *ip_reply;
struct icmphdr *icmp, *icmp_reply;
struct sockaddr_in connection;
char *dst_addr="192.168.11.32";
unsigned char *packet, *buffer;
int sockfd, optval, ret=-1;
socklen_t addrlen;

/* open ICMP socket */
if ((sockfd = socket(AF_INET, SOCK_RAW, IPPROTO_ICMP)) == -1) {
perror("socket");
exit(EXIT_FAILURE);
}

printf("Socket is %d\n", sockfd) ;

packet = (unsigned char*)malloc(sizeof(struct iphdr) + sizeof(struct icmphdr));
buffer = (unsigned char*)malloc(sizeof(struct iphdr) + sizeof(struct icmphdr));

if(packet == NULL || buffer == NULL)
{
perror("Error in malloc") ;
}

memset(packet, 0, sizeof(struct iphdr) + sizeof(struct icmphdr));
memset(buffer, 0, sizeof(struct iphdr) + sizeof(struct icmphdr));

ip = (struct iphdr*) packet;
icmp = (struct icmphdr*) ((char*)packet + sizeof(struct iphdr));

ip->ihl = 5;
ip->version = 4;
ip->tot_len = sizeof(struct iphdr) + sizeof(struct icmphdr);
//ip->tot_len = 48;
ip->id = random()%5985;
ip->protocol = IPPROTO_ICMP;
ip->saddr = inet_addr("192.168.11.34");
ip->daddr = inet_addr(dst_addr);
// ip->daddr = inet_addr("8.8.8.8");
ip->ttl = 64;
ip->check = in_cksum((unsigned short *)ip, sizeof(struct iphdr));

icmp->type = ICMP_ECHO;
icmp->code = 0;
icmp->un.echo.id = 0;
icmp->un.echo.sequence = 0;
icmp->checksum = 0;
icmp->checksum = in_cksum((unsigned short *)icmp, sizeof(struct icmphdr));

//Dumping headers to a file, to be viewed using hexdump
int ip_file = open("working_header",O_CREAT|O_RDWR);

if(ip_file == -1)
{
perror("Error in file opening");
}

ret = write(ip_file, packet, sizeof(struct iphdr) + sizeof(struct icmphdr));

if(ret == -1)
{
perror("Error in write");
}
else
{
printf("Wrote %d bytes\n", ret) ;
}

close(ip_file);

//binding to a specific interface
struct ifreq ifr;
memset(&ifr, 0, sizeof (ifr));
snprintf (ifr.ifr_name, sizeof (ifr.ifr_name), "enp1s0");
if (ioctl (sockfd, SIOCGIFINDEX, &ifr) < 0)
{
//Failed to find interface on device
printf("Failed to find interface on device\n");
return -1;
}

if (setsockopt (sockfd, SOL_SOCKET, SO_BINDTODEVICE, &ifr, sizeof (ifr)) < 0)
{
//Failed to bind to interface enp2s0
printf("Failed to bind to interface %s\n",ifr.ifr_name);
return -1;
}
struct timeval tv;
tv.tv_sec = 3;
tv.tv_usec = 0;
if (setsockopt(sockfd, SOL_SOCKET, SO_RCVTIMEO, &tv, sizeof(tv)) < 0)
{
printf("Unable to set timeout\n");
return -1;
}

/* IP_HDRINCL must be set on the socket so that the kernel does not attempt
* * to automatically add a default ip header to the packet*/
ret = setsockopt(sockfd, IPPROTO_IP, IP_HDRINCL, &optval, sizeof(int));

if(ret == -1)
{
perror("Error in setsockopt");
}
connection.sin_family = AF_INET;
connection.sin_addr.s_addr = ip->daddr;

printf("Packet length is: %d\n",ip->tot_len);

//printing packet, byte by byte, in hex, before sending
unsigned char ch = 0;

while ( ch<28)
{
//printf("%x ",packet[ch]);
printf("0x%02x ", packet[ch]);
ch++;
}
printf("\n");
ret = sendto(sockfd, (void*)packet, ip->tot_len, 0, (struct sockaddr *)&connection, sizeof(struct sockaddr));

printf("Sent %d byte packet to %s ret = %d\n", ip->tot_len, dst_addr, ret);
// }
addrlen = sizeof(connection);
if (recvfrom(sockfd, buffer, sizeof(struct iphdr) + sizeof(struct icmphdr), 0, (struct sockaddr *)&connection, &addrlen) < 0)
{
perror("recv");
}
else
{
ip_reply = (struct iphdr*) buffer;
icmp_reply = (struct icmphdr*) (buffer + sizeof(struct iphdr));
printf("Received type %d\n", icmp_reply->type);
printf("icmp code %d\n", icmp_reply->code);
printf("TTL: %d\n", ip_reply->ttl);
printf("CheckSum: %d,%d\n", ip_reply->check,icmp_reply->checksum);
}
free(packet);
free(buffer);
close(sockfd);

return 0 ;
}


Below is the thread code.

#include <stdio.h>
#include <signal.h>
#include <stdlib.h>
#include <unistd.h>
#include <netdb.h>
#include <linux/ip.h>
#include <linux/icmp.h>
#include <sys/socket.h>
#include <sys/stat.h>
#include <netinet/in.h>
#include <arpa/inet.h>
#include <sys/ioctl.h>
#include <string.h>
#include <fcntl.h>
#include <net/if.h>
#include <pthread.h>

unsigned short in_cksum(unsigned short *addr, int len)
{
register int sum = 0;
u_short answer = 0;
register u_short *w = addr;
register int nleft = len;
/*
* * Our algorithm is simple, using a 32 bit accumulator (sum), we add
* * sequential 16 bit words to it, and at the end, fold back all the
* * carry bits from the top 16 bits into the lower 16 bits.
* */
while (nleft > 1)
{
sum += *w++;
nleft -= 2;
}
/* mop up an odd byte, if necessary */
if (nleft == 1)
{
*(u_char *) (&answer) = *(u_char *) w;
sum += answer;
}
/* add back carry outs from top 16 bits to low 16 bits */
sum = (sum >> 16) + (sum & 0xffff); /* add hi 16 to low 16 */
sum += (sum >> 16); /* add carry */
answer = ~sum; /* truncate to 16 bits */
return (answer);
}

void* thread_for_icmp(void* arg)
{
struct iphdr *ip, *ip_reply;
struct icmphdr *icmp, *icmp_reply;
struct sockaddr_in connection;
char *dst_addr="192.168.11.32";
unsigned char *packet, *buffer;
int sockfd, optval, ret=-1;
socklen_t addrlen;

arg = arg;

/* open ICMP socket */
if ((sockfd = socket(AF_INET, SOCK_RAW, IPPROTO_ICMP)) == -1) {
perror("socket");
exit(EXIT_FAILURE);
}

printf("Socket is %d in thread\n", sockfd) ;

packet = (unsigned char*)malloc(sizeof(struct iphdr) + sizeof(struct icmphdr));
buffer = (unsigned char*)malloc(sizeof(struct iphdr) + sizeof(struct icmphdr));

if(packet == NULL || buffer == NULL)
{
perror("Error in malloc") ;
}

memset(packet, 0, sizeof(struct iphdr) + sizeof(struct icmphdr));
memset(buffer, 0, sizeof(struct iphdr) + sizeof(struct icmphdr));

ip = (struct iphdr*) packet;
icmp = (struct icmphdr*) ((char*)packet + sizeof(struct iphdr));

ip->ihl = 5;
ip->version = 4;
ip->tot_len = sizeof(struct iphdr) + sizeof(struct icmphdr);
//ip->tot_len = 48;
ip->id = random()%5985;
ip->protocol = IPPROTO_ICMP;
ip->saddr = inet_addr("192.168.11.34");
ip->daddr = inet_addr(dst_addr);
// ip->daddr = inet_addr("8.8.8.8");
ip->ttl = 64;
ip->check = in_cksum((unsigned short *)ip, sizeof(struct iphdr));

icmp->type = ICMP_ECHO;
icmp->code = 0;
icmp->un.echo.id = 0;
icmp->un.echo.sequence = 0;
icmp->checksum = 0;
icmp->checksum = in_cksum((unsigned short *)icmp, sizeof(struct icmphdr));

//Dumping headers to a file, to be viewed using hexdump
int ip_file = open("header",O_CREAT|O_RDWR);

if(ip_file == -1)
{
perror("Error in file opening");
}

ret = write(ip_file, packet, sizeof(struct iphdr) + sizeof(struct icmphdr));

if(ret == -1)
{
perror("Error in write");
}
else
{
printf("Wrote %d bytes\n", ret) ;
}

close(ip_file);

//binding to a specific interface
struct ifreq ifr;
memset(&ifr, 0, sizeof (ifr));
snprintf (ifr.ifr_name, sizeof (ifr.ifr_name), "enp1s0");
if (ioctl (sockfd, SIOCGIFINDEX, &ifr) < 0)
{
//Failed to find interface on device
printf("Failed to find interface on device\n");
return NULL;
}

if (setsockopt (sockfd, SOL_SOCKET, SO_BINDTODEVICE, &ifr, sizeof (ifr)) < 0)
{
//Failed to bind to interface enp2s0
printf("Failed to bind to interface %s\n",ifr.ifr_name);
return NULL;
}
struct timeval tv;
tv.tv_sec = 3;
tv.tv_usec = 0;
if (setsockopt(sockfd, SOL_SOCKET, SO_RCVTIMEO, &tv, sizeof(tv)) < 0)
{
printf("Unable to set timeout\n");
return NULL;
}

/* IP_HDRINCL must be set on the socket so that the kernel does not attempt
* * to automatically add a default ip header to the packet*/
ret = setsockopt(sockfd, IPPROTO_IP, IP_HDRINCL, &optval, sizeof(int));

if(ret == -1)
{
perror("Error in setsockopt");
}
connection.sin_family = AF_INET;
connection.sin_addr.s_addr = ip->daddr;

printf("Packet length is: %d\n",ip->tot_len);

//printing packet, byte by byte, in hex, before sending
unsigned char ch = 0;

while ( ch<28)
{
//printf("%x ",packet[ch]);
printf("0x%02x ", packet[ch]);
ch++;
}
printf("\n");
ret = sendto(sockfd, (void*)packet, ip->tot_len, 0, (struct sockaddr *)&connection, sizeof(struct sockaddr));

printf("Sent %d byte packet to %s ret = %d\n", ip->tot_len, dst_addr, ret);
// }
addrlen = sizeof(connection);
if (recvfrom(sockfd, buffer, sizeof(struct iphdr) + sizeof(struct icmphdr), 0, (struct sockaddr *)&connection, &addrlen) < 0)
{
perror("recv");
}
else
{
ip_reply = (struct iphdr*) buffer;
icmp_reply = (struct icmphdr*) (buffer + sizeof(struct iphdr));
printf("Received type %d\n", icmp_reply->type);
printf("icmp code %d\n", icmp_reply->code);
printf("TTL: %d\n", ip_reply->ttl);
printf("CheckSum: %d,%d\n", ip_reply->check,icmp_reply->checksum);
}
free(packet);
free(buffer);
close(sockfd);

pthread_exit(NULL);
}

int main()
{
pthread_t thread;
int ret;

ret = pthread_create(&thread, NULL, thread_for_icmp, NULL);

if(ret == -1)
{
perror("Error in thread create");
}

ret = pthread_join(thread,NULL);

if(ret == -1)
{
perror("Error in thread join");
}
else
{
printf("Thread exited succesfully\n") ;
}

return 0;
}

Answer

The optval value in the setsockopt IP_HDRINCL call is not being initialized. So, I suspect that the process version is getting some non-zero leftover value from pre-main code, while the thread version is getting a zero value from a pristine stack.

If you set optval = 1; just before calling the setsockopt, it should work.

Also, note that multi-byte fields in the IP and ICMP headers should be constructed in network byte order. You are fortunate in this case that tot_len is filled in by the kernel even when IP_HDRINCL is in use (see raw(7) for reference).