defying original 4.3BSD sockets constraint

Gleb Smirnoff <[email protected]>
Newsgroups gmane.os.freebsd.devel.net
Message-ID <[email protected]>
  Hi,

I dared to question a constraint that we have been following for 40 years:

https://reviews.freebsd.org/D56489

I've been thinking of this for almost a week already and I don't see any cons.
Pros are listed in the review - clean way to improve parallelism of connect(2).
Maybe I am missing something?  Decided to post for a wider audience than just
the reviews.f.o.  Please speak up if you think I am missing something!

Digging through the history I have found this write up:

https://bugs.freebsd.org/bugzilla/show_bug.cgi?id=65928#c2

and associated fix 4658dc8325e03419be1f1fa5cbda31289116a0d4.

If I read the above correct, the fix is actually avoiding the constraint for a
specific case of TCP.

Attached a test program that proves that the constraint can be avoided with
SO_REUSEADDR, thus there's no security implications.  Also proves that traffic
would always end on the correct socket.  Program requires root, as it uses a
raw socket for injection and also tweaks net.inet.ip.portrange.randomized.
The program uses UDP, but our inpcb database is 99% agnostic of that.  There is
only one spot left where it looks at so inp->inp_socket->so_type !=
SOCK_STREAM, differentiating between TCP and UDP.  It originates exactly from
the mentioned above 4658dc8325e03!

-- 
Gleb Smirnoff
port-steal.c (text/x-csrc, 5.7 KB)
#include <sys/socket.h>
#include <sys/sysctl.h>
#include <netinet/in.h>
#include <netinet/ip.h>
#include <netinet/udp.h>
#include <arpa/inet.h>
#include <assert.h>
#include <errno.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <stdbool.h>
#include <unistd.h>

#define	PSIZE	100

#define	RANDOMIZED_SYSCTL	"net.inet.ip.portrange.randomized"
static int randomized_orig;
static void
randomized_restore(void)
{
	assert(sysctlbyname(RANDOMIZED_SYSCTL, NULL, NULL,
	    &randomized_orig, sizeof(int)) == 0);
}

static void
randomized_off(void)
{
	size_t len = sizeof(int);

	assert(sysctlbyname(RANDOMIZED_SYSCTL, &randomized_orig, &len,
	    &(int){0}, sizeof(int)) == 0);
	assert(atexit(randomized_restore) == 0);
}

static int
rawsender(void)
{
	int s;

	assert((s = socket(AF_INET, SOCK_RAW, 0)) > 0);
	assert(setsockopt(s, IPPROTO_IP, IP_HDRINCL, &(int){1},
	    sizeof(int)) == 0);
        assert(connect(s,
	    (struct sockaddr *)&(struct sockaddr_in){
		.sin_family = AF_INET,
		.sin_len = sizeof(struct sockaddr_in),
		.sin_addr = { htonl(INADDR_ANY) },
	    }, sizeof(struct sockaddr_in)) == 0);

	return (s);
}

static bool
recvcheck(int raw, int s, void *pkt, size_t plen)
{
	char buf[PSIZE];
	size_t ss;

	assert(send(raw, pkt, plen, 0) == plen);
	ss = recv(s, buf, sizeof(buf), 0);

	return (ss == sizeof(buf) &&
	    memcmp(pkt + sizeof(struct ip) + sizeof(struct udphdr), buf,
	    sizeof(buf)) == 0);
}

/*
 * The test uses UDP sockets for simplicity.  For TCP it shall be the same.
 * Our inpcb database is finally 99% protocol agnostic.  The only remaining
 * protocol aware predicate is in in_pcbbind_avail(), find there
 * `inp->inp_socket->so_type != SOCK_STREAM`.  This piece of code is suggested
 * for deletion in my proposal.
 */
int
main(int argc, char *argv[])
{
	struct pkt {
		struct ip ip;
		struct udphdr udp;
		char payload[PSIZE];
	} __packed pkt = {
		.ip.ip_v = IPVERSION,
		.ip.ip_hl = sizeof(struct ip) >> 2,
		.ip.ip_len = htons(sizeof(struct pkt)),
		.ip.ip_ttl = 16,
		.ip.ip_p = IPPROTO_UDP,
		.udp.uh_ulen = htons(sizeof(struct pkt) - sizeof(struct ip)),
	};
	struct sockaddr_in sin = {
		.sin_family = AF_INET,
	};
	socklen_t slen = sizeof(struct sockaddr_in);
	int raw, wild, spec;
	int error;
	in_addr_t addr;
	uint16_t port;
	bool old_code;

	raw = rawsender();
	arc4random_buf(&pkt.payload, sizeof(pkt.payload));

	/*
	 * Test 1. Try to bind to 0.0.0.0:port in presence of IP:port.
	 */
	assert((spec = socket(AF_INET, SOCK_DGRAM, 0)) > 0);
	/* select local IP by connect(2) to somewhere */
	assert(connect(spec, 
	    (struct sockaddr *)&(struct sockaddr_in){
		.sin_family = AF_INET,
		.sin_len = sizeof(struct sockaddr_in),
		.sin_addr = { htonl(0xc0000201) },
		.sin_port = htons(6666),
	    }, sizeof(struct sockaddr_in)) == 0);
	assert(getsockname(spec, (struct sockaddr *)&sin, &slen) == 0);
	addr = sin.sin_addr.s_addr;
	close(spec);
	assert((spec = socket(AF_INET, SOCK_DGRAM, 0)) > 0);
	error = bind(spec, (struct sockaddr *)&sin, sizeof(sin));
	printf("Test #1: specific bound to %s:%u\n", inet_ntoa(sin.sin_addr),
	    ntohs(sin.sin_port));
	/* try to bind(2) to 0.0.0.0:port */
	sin.sin_addr.s_addr = htonl(INADDR_ANY);
	assert((wild = socket(AF_INET, SOCK_DGRAM, 0)) > 0);
retry:
	error = bind(wild, (struct sockaddr *)&sin, sizeof(sin));
	if (error != 0) {
		/*
		 * in_pcbbind_setup_locked(no SO_REUSE* set) calls
		 * in_pcbbind_avail(INPLOOKUP_WILDCARD) calls
		 * in_pcblookup_local(INPLOOKUP_WILDCARD) that goes
		 * through port hash to find "conflict".
		 */
		old_code = true;
		printf("Test #1: \"steal\" bind(2) %s:%u failed: %s\n",
		    inet_ntoa(sin.sin_addr), ntohs(sin.sin_port),
		    strerror(errno));
		assert(setsockopt(wild, SOL_SOCKET, SO_REUSEADDR, &(int){1},
		    sizeof(int)) == 0);
		printf("Test #1: retrying with SO_REUSEADDR\n");
		goto retry;
	} else {
		old_code = false;
		printf("Test #1: bind(2) %s:%u successful\n",
		    inet_ntoa(sin.sin_addr), ntohs(sin.sin_port));
		pkt.ip.ip_src.s_addr = htonl(0xc0000201),
		pkt.ip.ip_dst.s_addr = addr,
		pkt.udp.uh_sport = htons(6666);
		pkt.udp.uh_dport = sin.sin_port;
		assert(recvcheck(raw, spec, &pkt, sizeof(pkt)));
		printf("Test #1: specific received. There's no steal\n");
	}
	close(wild);
	close(spec);

	/*
 	 * Test 2. Create coexisting IPv4:port and 0.0.0.0:port with
	 * help of connect(2) and "lucky" port selection.
	 */
	randomized_off();
	assert((wild = socket(AF_INET, SOCK_DGRAM, 0)) > 0);
	assert((spec = socket(AF_INET, SOCK_DGRAM, 0)) > 0);
	assert(connect(spec, 
	    (struct sockaddr *)&(struct sockaddr_in){
		.sin_family = AF_INET,
		.sin_len = sizeof(struct sockaddr_in),
		.sin_addr = { htonl(0xc0000201) },
		.sin_port = htons(6666),
	    }, sizeof(struct sockaddr_in)) == 0);
	assert(getsockname(spec, (struct sockaddr *)&sin, &slen) == 0);
	printf("Test #2: auxiliary probe socket: %s:%u\n",
	    inet_ntoa(sin.sin_addr), ntohs(sin.sin_port));
	sin.sin_addr.s_addr = INADDR_ANY;
	sin.sin_port = htons(ntohs(sin.sin_port) + 1);
	assert(bind(wild, (struct sockaddr *)&sin, sizeof(sin)) == 0);
	assert(getsockname(wild, (struct sockaddr *)&sin, &slen) == 0);
	printf("Test #2: bound socket: %s:%u\n",
	    inet_ntoa(sin.sin_addr), ntohs(sin.sin_port));
	port = sin.sin_port;
	close(spec);
	assert((spec = socket(AF_INET, SOCK_DGRAM, 0)) > 0);
	assert(connect(spec, 
	    (struct sockaddr *)&(struct sockaddr_in){
		.sin_family = AF_INET,
		.sin_len = sizeof(struct sockaddr_in),
		.sin_addr = { htonl(0xc0000202) },
		.sin_port = htons(6666),
	    }, sizeof(struct sockaddr_in)) == 0);
	assert(getsockname(spec, (struct sockaddr *)&sin, &slen) == 0);
	assert(ntohs(sin.sin_port) != port);
	printf("Test #2: connect(2) avoided \"lucky\" port: %s:%u\n",
	    inet_ntoa(sin.sin_addr), ntohs(sin.sin_port));
 
	return (0);
}
lmpx.com only provides a reader for public news (NNTP) servers. It is not affiliated with the servers or forums shown here and is not responsible for the content of articles, which is written by their respective authors.