Ticket History #542: krb524d does not deal with multihomed hosts correctly

From kenh@cmf.nrl.navy.mil Sun Feb 1 15:45:32 1998
Received: from MIT.EDU (SOUTH-STATION-ANNEX.MIT.EDU [18.72.1.2]) by rt-11.MIT.EDU (8.7.5/8.7.3) with SMTP id PAA03292 for <bugs@RT-11.MIT.EDU>; Sun, 1 Feb 1998 15:45:32 -0500
Received: from ginger.cmf.nrl.navy.mil by MIT.EDU with SMTP
id AA01156; Sun, 1 Feb 98 15:45:30 EST
Received: from nexus.cmf.nrl.navy.mil (kenh@nexus.cmf.nrl.navy.mil [134.207.8.9])
by ginger.cmf.nrl.navy.mil (8.8.5/8.8.5) with ESMTP id PAA13461
for <krb5-bugs@mit.edu>; Sun, 1 Feb 1998 15:45:28 -0500 (EST)
Received: (from kenh@localhost)
by nexus.cmf.nrl.navy.mil (8.8.5/8.8.5) id PAA10287;
Sun, 1 Feb 1998 15:45:27 -0500 (EST)
Message-Id: <199802012045.PAA10287@nexus.cmf.nrl.navy.mil>
Date: Sun, 1 Feb 1998 15:45:27 -0500 (EST)
From: Ken Hornstein <kenh@cmf.nrl.navy.mil>
Reply-To: kenh@cmf.nrl.navy.mil
To: krb5-bugs@MIT.EDU
Subject: kdc + krb524d don't work correctly on multihomed machines
X-Send-Pr-Version: 3.99

Navel Research Laboatory


System: SunOS nexus 4.1.4 1 sun4m
Architecture: sun4


The krb5kdc and krb525d do not work correctly when running on multihomed
hosts. More exactly:

- If you have a mulihomed KDC and a client sends a packet to one interface,
the KDC may (depending on the destination interface and the routing
between the client and the KDC) send a reply with a source IP address
that does _not_ match the destination address used by the client.
- This is especially bad in that Kerberos clients call connect() to get
error messages back, and connect() using the destination address;
in this case, replies from a different address are silently ignored.

Note that RFC1122 says:

(1) If the datagram is sent in response to a received
datagram, the source address for the response SHOULD be
the specific-destination address of the request. See
Sections 4.1.3.5 and 4.2.3.7 and the "General Issues"
section of [INTRO:1] for more specific requirements on
higher layers.


Configure your client to send requests to only _one_ address of a multihomed
KDC, and observe that replies are ignored.

The only way to fix this is to have the servers bind to each interface
and use the socket it received the packet on as the socket to send the
reply. This is what BIND and the Heimdal KDCs both do. The following
patch implements this for both krb5kdc and krb524d.

Index: krb5/kdc/network.c
diff -c krb5/kdc/network.c:1.1.1.1 krb5/kdc/network.c:1.3
*** krb5/kdc/network.c:1.1.1.1 Mon Jun 2 17:54:09 1997
--- krb5/kdc/network.c Thu Jan 22 20:36:26 1998
***************
*** 44,49 ****
--- 44,50 ----

static int *udp_port_fds = (int *) NULL;
static u_short *udp_port_nums = (u_short *) NULL;
+ static struct in_addr *udp_port_addrs = (struct in_addr *) NULL;
static int n_udp_ports = 0;
static int max_udp_ports = 0;
static fd_set select_fds;
***************
*** 51,66 ****

#define safe_realloc(p,n) ((p)?(realloc(p,n)):(malloc(n)))

! static krb5_error_code add_port(port)
u_short port;
{
int i;
int *new_fds;
u_short *new_ports;
int new_max;

for (i=0; i < n_udp_ports; i++) {
! if (udp_port_nums[i] == port)
return 0;
}

--- 52,70 ----

#define safe_realloc(p,n) ((p)?(realloc(p,n)):(malloc(n)))

! static krb5_error_code add_port(port, interface)
u_short port;
+ struct in_addr *interface;
{
int i;
int *new_fds;
u_short *new_ports;
+ struct in_addr *new_addrs;
int new_max;

for (i=0; i < n_udp_ports; i++) {
! if (udp_port_nums[i] == port &&
! memcmp(&udp_port_addrs[i], interface, sizeof(struct in_addr)) == 0)
return 0;
}

***************
*** 76,84 ****
return ENOMEM;
udp_port_nums = new_ports;

max_udp_ports = new_max;
}
!
udp_port_nums[n_udp_ports++] = port;
return 0;
}
--- 80,95 ----
return ENOMEM;
udp_port_nums = new_ports;

+ new_addrs = safe_realloc(udp_port_addrs, new_max *
+ sizeof(struct in_addr));
+ if (new_addrs == 0)
+ return ENOMEM;
+ udp_port_addrs = new_addrs;
+
max_udp_ports = new_max;
}
!
! memcpy(&udp_port_addrs[n_udp_ports], interface, sizeof(struct in_addr));
udp_port_nums[n_udp_ports++] = port;
return 0;
}
***************
*** 89,102 ****
const char *prog;
{
struct sockaddr_in sin;
krb5_error_code retval;
u_short port;
char *cp;
! int i;

FD_ZERO(&select_fds);
select_nfds = 0;
memset((char *)&sin, 0, sizeof(sin));

/* Handle each realm's ports */
for (i=0; i<kdc_numrealms; i++) {
--- 100,132 ----
const char *prog;
{
struct sockaddr_in sin;
+ struct in_addr saddr;
krb5_error_code retval;
u_short port;
char *cp;
! int i, j, numaddrs;
! krb5_address **localaddrs;
! const int on = 1;

FD_ZERO(&select_fds);
select_nfds = 0;
memset((char *)&sin, 0, sizeof(sin));
+ saddr.s_addr = INADDR_ANY;
+
+ /*
+ * Sigh. We need to bind to all of the interfaces individually
+ * in addition to the wildcard address, since that's the only
+ * way to deal with multihomed hosts and asymmetric routing.
+ * So first, get a list of all local interfaces.
+ */
+
+ retval = krb5_os_localaddr(kdc_context, &localaddrs);
+ if (retval) {
+ com_err(prog, 0, "Cannot find local addresses");
+ return(retval);
+ }
+
+ for (numaddrs = 0; localaddrs[numaddrs] != NULL; numaddrs++) ;

/* Handle each realm's ports */
for (i=0; i<kdc_numrealms; i++) {
***************
*** 109,117 ****
port = strtol(cp, &cp, 10);
if (cp == 0)
break;
! retval = add_port(port);
if (retval)
return retval;
}
}

--- 139,163 ----
port = strtol(cp, &cp, 10);
if (cp == 0)
break;
! /*
! * We first add an interface for the wildcard address.
! */
! retval = add_port(port, &saddr);
if (retval)
return retval;
+ /*
+ * Next, add ports for all of the network interfaces
+ */
+ for (j = 0; j < numaddrs; j++) {
+ struct in_addr tsaddr;
+ if (localaddrs[j]->addrtype == ADDRTYPE_INET) {
+ memcpy((char *)&tsaddr, (char *) localaddrs[j]->contents,
+ localaddrs[j]->length);
+ retval = add_port(port, &tsaddr);
+ if (retval)
+ return retval;
+ }
+ }
}
}

***************
*** 122,127 ****
--- 168,191 ----
udp_port_nums[i]);
return(retval);
}
+
+ /*
+ * Turn on SO_REUSEADDR on this socket (since we're now creating
+ * more than one socket per port).
+ */
+
+ if (setsockopt(udp_port_fds[i], SOL_SOCKET, SO_REUSEADDR, (char *)&on,
+ sizeof(on)) != 0) {
+ com_err(prog, errno, "setsockopt(udp, reuseaddr)");
+ /* This isn't _that_ serious ... */
+ }
+
+ /*
+ * Copy over the appropriate local address to bind to
+ */
+ memcpy((char *)&sin.sin_addr, (char *) &udp_port_addrs[i],
+ sizeof(struct in_addr));
+
sin.sin_port = htons(udp_port_nums[i]);
if (bind(udp_port_fds[i], (struct sockaddr *) &sin,
sizeof(sin)) == -1) {
***************
*** 134,139 ****
--- 198,205 ----
if (udp_port_fds[i]+1 > select_nfds)
select_nfds = udp_port_fds[i]+1;
}
+
+ krb5_free_addresses(kdc_context, localaddrs);

return 0;
}
cvs rdiff: Diffing krb5/krb524
Index: krb5/krb524/krb524d.c
diff -c krb5/krb524/krb524d.c:1.1.1.1 krb5/krb524/krb524d.c:1.2
*** krb5/krb524/krb524d.c:1.1.1.1 Mon Jun 2 17:55:41 1997
--- krb5/krb524/krb524d.c Thu Jan 22 20:37:22 1998
***************
*** 88,97 ****
struct servent *serv;
struct sockaddr_in saddr;
struct timeval timeout;
! int ret, s;
! fd_set rfds;
krb5_context context;
krb5_error_code retval;

retval = krb5_init_context(&context);
if (retval) {
--- 88,100 ----
struct servent *serv;
struct sockaddr_in saddr;
struct timeval timeout;
! int ret, s, i, numfds, maxfd;
! fd_set rfds, select_fds;
krb5_context context;
krb5_error_code retval;
+ int *addr_fds = NULL;
+ krb5_address **localaddrs;
+ const int on = 1;

retval = krb5_init_context(&context);
if (retval) {
***************
*** 128,133 ****
--- 131,142 ----
/* someday maybe there will be some config param options */
init_master(context, NULL);

+ /*
+ * We need to bind to all of interface addresses, in addition to
+ * wildcard address, so we can reply to messages using the correct
+ * source address
+ */
+
memset((char *) &saddr, 0, sizeof(struct sockaddr_in));
saddr.sin_family = AF_INET;
saddr.sin_addr.s_addr = INADDR_ANY;
***************
*** 142,160 ****
com_err(whoami, errno, "creating main socket");
cleanup_and_exit(1, context);
}
if ((ret = bind(s, (struct sockaddr *) &saddr,
sizeof(struct sockaddr_in))) < 0) {
com_err(whoami, errno, "binding main socket");
cleanup_and_exit(1, context);
}
!
while (1) {
! FD_ZERO(&rfds);
! FD_SET(s, &rfds);
timeout.tv_sec = TIMEOUT;
timeout.tv_usec = 0;

! ret = select(s+1, &rfds, NULL, NULL, &timeout);
if (signalled)
cleanup_and_exit(0, context);
else if (ret == 0) {
--- 151,227 ----
com_err(whoami, errno, "creating main socket");
cleanup_and_exit(1, context);
}
+
+ /*
+ * Set SO_REUSEADDR so that we can bind to the non-wildcard addresses
+ * later
+ */
+
+ if ((ret = setsockopt(s, SOL_SOCKET, SO_REUSEADDR, (char *) &on,
+ sizeof(on))) < 0) {
+ com_err(whoami, errno, "setting SO_REUSEADDR on main socket");
+ }
+
if ((ret = bind(s, (struct sockaddr *) &saddr,
sizeof(struct sockaddr_in))) < 0) {
com_err(whoami, errno, "binding main socket");
cleanup_and_exit(1, context);
}
!
! numfds = 1;
! addr_fds = (int *) malloc(sizeof(int));
! addr_fds[0] = s;
! maxfd = s + 1;
! FD_ZERO(&select_fds);
! FD_SET(s, &select_fds);
!
! /*
! * Now lets go through and bind a socket to each interface
! */
!
! if ((ret = krb5_os_localaddr(context, &localaddrs)) != 0) {
! com_err(whoami, ret, "getting local addresses");
! cleanup_and_exit(1, context);
! }
!
! for (i = 0; localaddrs[i] != NULL; i++) {
! if (localaddrs[i]->addrtype == ADDRTYPE_INET) {
! memcpy((char *) &saddr.sin_addr, localaddrs[i]->contents,
! localaddrs[i]->length);
! if ((s = socket(AF_INET, SOCK_DGRAM, 0)) < 0) {
! com_err(whoami, errno, "creating interface socket");
! cleanup_and_exit(1, context);
! }
! if ((ret = setsockopt(s, SOL_SOCKET, SO_REUSEADDR, (char *) &on,
! sizeof(on))) < 0) {
! com_err(whoami, errno, "setting SO_REUSEADDR on interface socket");
! }
! if ((ret = bind(s, (struct sockaddr *) &saddr,
! sizeof(struct sockaddr))) < 0) {
! com_err(whoami, errno, "binding on interface socket");
! cleanup_and_exit(1, context);
! }
! numfds++;
! addr_fds = (int *) realloc(addr_fds, numfds * sizeof(int));
! if (addr_fds == NULL) {
! com_err(whoami, ENOMEM, "allocating descriptor memory");
! cleanup_and_exit(1, context);
! }
! addr_fds[numfds - 1] = s;
! FD_SET(s, &select_fds);
! if (s + 1 > maxfd)
! maxfd = s + 1;
! }
! }
!
! krb5_free_addresses(context, localaddrs);
!
while (1) {
! rfds = select_fds;
timeout.tv_sec = TIMEOUT;
timeout.tv_usec = 0;

! ret = select(maxfd, &rfds, NULL, NULL, &timeout);
if (signalled)
cleanup_and_exit(0, context);
else if (ret == 0) {
***************
*** 168,181 ****
} else if (ret < 0 && errno != EINTR) {
com_err(whoami, errno, "in select");
cleanup_and_exit(1, context);
! } else if (FD_ISSET(s, &rfds)) {
! if (debug)
! printf("received packet\n");
! if ((ret = do_connection(s, context))) {
! com_err(whoami, ret, "handling packet");
}
! } else
! com_err(whoami, 0, "impossible situation occurred!");
}

cleanup_and_exit(0, context);
--- 235,251 ----
} else if (ret < 0 && errno != EINTR) {
com_err(whoami, errno, "in select");
cleanup_and_exit(1, context);
! } else if (ret > 0) {
! for (i = 0; i < numfds; i++) {
! if (FD_ISSET(addr_fds[i], &rfds)) {
! if (debug)
! printf("received packet\n");
! if ((ret = do_connection(addr_fds[i], context))) {
! com_err(whoami, ret, "handling packet");
! }
! }
}
! }
}

cleanup_and_exit(0, context);