Changeset 706:4ffbc9f1e922 in freeDiameter for libfdcore

Timestamp:

Feb 9, 2011, 3:26:58 PM (13 years ago)

Author:

Sebastien Decugis <sdecugis@nict.go.jp>

Branch:

default

Phase:

public

Message:

Large UNTESTED commit with the following changes:

• Improved DiameterIdentity? handling (esp. interationalization issues), and improve efficiency of some string operations in peers, sessions, and dictionary modules (closes #7)
• Cleanup in the session module to free only unreferenced sessions (#16)
• Removed fd_cpu_flush_cache(), replaced by more robust alternatives.
• Improved peer state machine algorithm to counter SCTP multistream race condition.

Location:

libfdcore

Files:

• cnxctx.c (modified) (40 diffs)
• cnxctx.h (modified) (4 diffs)
• config.c (modified) (5 diffs)
• core.c (modified) (2 diffs)
• extensions.c (modified) (2 diffs)
• fdcore-internal.h (modified) (6 diffs)
• fdd.y (modified) (1 diff)
• messages.c (modified) (9 diffs)
• p_ce.c (modified) (34 diffs)
• p_cnx.c (modified) (7 diffs)
• p_dp.c (modified) (6 diffs)
• p_dw.c (modified) (5 diffs)
• p_expiry.c (modified) (4 diffs)
• p_out.c (modified) (2 diffs)
• p_psm.c (modified) (38 diffs)
• p_sr.c (modified) (5 diffs)
• peers.c (modified) (20 diffs)
• routing_dispatch.c (modified) (39 diffs)
• sctp.c (modified) (8 diffs)
• sctps.c (modified) (7 diffs)
• server.c (modified) (9 diffs)

libfdcore/cnxctx.c

@@ -51 +51 @@
  */
 
-/* Note: this file could be moved to libfreeDiameter instead, but since it uses gnuTLS we prefer to keep it in the daemon */
-
 /* Lifetime of a cnxctx object:
  * 1) Creation
@@ -158 +156 @@
         TRACE_DEBUG(INFO, "This function should never been called when SCTP is disabled...");
         ASSERT(0);
-        CHECK_FCT_DO( ENOTSUP, );
-        return NULL;
+        CHECK_FCT_DO( ENOTSUP, return NULL);
 #else /* DISABLE_SCTP */
         struct cnxctx * cnx = NULL;
@@ -249 +246 @@
                 int  rc;
                 
-                /* Numeric values for debug */
                 rc = getnameinfo((sSA *)&ss, sSAlen(&ss), addrbuf, sizeof(addrbuf), portbuf, sizeof(portbuf), NI_NUMERICHOST | NI_NUMERICSERV);
                 if (rc) {
@@ -256 +252 @@
                 }
                 
-                snprintf(cli->cc_id, sizeof(cli->cc_id), "{%s} (%d) <- [%s]:%s (%d)", 
-                                IPPROTO_NAME(cli->cc_proto), serv->cc_socket, 
-                                addrbuf, portbuf, cli->cc_socket);
-                
-                /* Name for log messages */
+                /* Numeric values for debug... */
+                snprintf(cli->cc_id, sizeof(cli->cc_id), "%s from [%s]:%s (%d<-%d)", 
+                                IPPROTO_NAME(cli->cc_proto), addrbuf, portbuf, serv->cc_socket, cli->cc_socket);
+                
+                
+                /* ...Name for log messages */
                 rc = getnameinfo((sSA *)&ss, sSAlen(&ss), cli->cc_remid, sizeof(cli->cc_remid), NULL, 0, 0);
                 if (rc)
@@ -334 +331 @@
                 int  rc;
                 
-                /* Numeric values for debug */
+                /* Numeric values for debug... */
                 rc = getnameinfo(sa, addrlen, addrbuf, sizeof(addrbuf), portbuf, sizeof(portbuf), NI_NUMERICHOST | NI_NUMERICSERV);
                 if (rc) {
@@ -341 +338 @@
                 }
                 
-                snprintf(cnx->cc_id, sizeof(cnx->cc_id), "{TCP} -> [%s]:%s (%d)", addrbuf, portbuf, cnx->cc_socket);
-                
-                /* Name for log messages */
+                snprintf(cnx->cc_id, sizeof(cnx->cc_id), "TCP to [%s]:%s (%d)", addrbuf, portbuf, cnx->cc_socket);
+                
+                /* ...Name for log messages */
                 rc = getnameinfo(sa, addrlen, cnx->cc_remid, sizeof(cnx->cc_remid), NULL, 0, 0);
                 if (rc)
@@ -356 +353 @@
 {
 #ifdef DISABLE_SCTP
-        TRACE_DEBUG(INFO, "This function should never been called when SCTP is disabled...");
+        TRACE_DEBUG(INFO, "This function should never be called when SCTP is disabled...");
         ASSERT(0);
-        CHECK_FCT_DO( ENOTSUP, );
-        return NULL;
+        CHECK_FCT_DO( ENOTSUP, return NULL);
 #else /* DISABLE_SCTP */
         int sock = 0;
@@ -416 +412 @@
                 int  rc;
                 
-                /* Numeric values for debug */
+                /* Numeric values for debug... */
                 rc = getnameinfo((sSA *)&primary, sSAlen(&primary), addrbuf, sizeof(addrbuf), portbuf, sizeof(portbuf), NI_NUMERICHOST | NI_NUMERICSERV);
                 if (rc) {
@@ -423 +419 @@
                 }
                 
-                snprintf(cnx->cc_id, sizeof(cnx->cc_id), "{SCTP} -> [%s]:%s (%d)", addrbuf, portbuf, cnx->cc_socket);
-                
-                /* Name for log messages */
+                snprintf(cnx->cc_id, sizeof(cnx->cc_id), "SCTP to [%s]:%s (%d)", addrbuf, portbuf, cnx->cc_socket);
+                
+                /* ...Name for log messages */
                 rc = getnameinfo((sSA *)&primary, sSAlen(&primary), cnx->cc_remid, sizeof(cnx->cc_remid), NULL, 0, 0);
                 if (rc)
@@ -454 +450 @@
 
 /* Set the hostname to check during handshake */
-void fd_cnx_sethostname(struct cnxctx * conn, char * hn)
+void fd_cnx_sethostname(struct cnxctx * conn, DiamId_t hn)
 {
         CHECK_PARAMS_DO( conn, return );
@@ -460 +456 @@
 }
 
+/* We share a lock with many threads but we hold it only very short time so it is OK */
+static pthread_mutex_t state_lock = PTHREAD_MUTEX_INITIALIZER;
+uint32_t fd_cnx_getstate(struct cnxctx * conn)
+{
+        uint32_t st;
+        CHECK_POSIX_DO( pthread_mutex_lock(&state_lock), { ASSERT(0); } );
+        st = conn->cc_state;
+        CHECK_POSIX_DO( pthread_mutex_unlock(&state_lock), { ASSERT(0); } );
+        return st;
+}
+int  fd_cnx_teststate(struct cnxctx * conn, uint32_t flag)
+{
+        uint32_t st;
+        CHECK_POSIX_DO( pthread_mutex_lock(&state_lock), { ASSERT(0); } );
+        st = conn->cc_state;
+        CHECK_POSIX_DO( pthread_mutex_unlock(&state_lock), { ASSERT(0); } );
+        return st & flag;
+}
+void fd_cnx_addstate(struct cnxctx * conn, uint32_t orstate)
+{
+        CHECK_POSIX_DO( pthread_mutex_lock(&state_lock), { ASSERT(0); } );
+        conn->cc_state |= orstate;
+        CHECK_POSIX_DO( pthread_mutex_unlock(&state_lock), { ASSERT(0); } );
+}
+void fd_cnx_setstate(struct cnxctx * conn, uint32_t abstate)
+{
+        CHECK_POSIX_DO( pthread_mutex_lock(&state_lock), { ASSERT(0); } );
+        conn->cc_state = abstate;
+        CHECK_POSIX_DO( pthread_mutex_unlock(&state_lock), { ASSERT(0); } );
+}
+
+
 /* Return the TLS state of a connection */
 int fd_cnx_getTLS(struct cnxctx * conn)
 {
         CHECK_PARAMS_DO( conn, return 0 );
-        fd_cpu_flush_cache();
-        return conn->cc_status & CC_STATUS_TLS;
-}
+        return fd_cnx_teststate(conn, CC_STATUS_TLS);
+}
+
+/* Return true if the connection supports unordered delivery of messages */
+int fd_cnx_isMultichan(struct cnxctx * conn)
+{
+        CHECK_PARAMS_DO( conn, return 0 );
+        #ifdef DISABLE_SCTP
+        if (conn->cc_proto == IPPROTO_SCTP)
+                return (conn->cc_sctp_para.str_in > 1) || (conn->cc_sctp_para.str_out > 1);
+        #endif /* DISABLE_SCTP */
+        return 0;
+}
+
 
 /* Get the list of endpoints (IP addresses) of the local and remote peers on this connection */
@@ -508 +547 @@
 }
 
-/* Retrieve a list of all IP addresses of the local system from the kernel, using  */
+/* Retrieve a list of all IP addresses of the local system from the kernel, using getifaddrs */
 int fd_cnx_get_local_eps(struct fd_list * list)
 {
         struct ifaddrs *iflist, *cur;
+        
         CHECK_SYS(getifaddrs(&iflist));
         
@@ -543 +583 @@
         CHECK_PARAMS_DO( conn, goto fatal );
         
-        TRACE_DEBUG(FULL, "Error flag set for socket %d (%s / %s)", conn->cc_socket, conn->cc_remid, conn->cc_id);
+        TRACE_DEBUG(FULL, "Error flag set for socket %d (%s, %s)", conn->cc_socket, conn->cc_id, conn->cc_remid);
         
         /* Mark the error */
-        fd_cpu_flush_cache();
-        conn->cc_status |= CC_STATUS_ERROR;
+        fd_cnx_addstate(conn, CC_STATUS_ERROR);
         
         /* Report the error if not reported yet, and not closing */
-        if ((!(conn->cc_status & CC_STATUS_CLOSING )) && (!(conn->cc_status & CC_STATUS_SIGNALED )))  {
+        if (!fd_cnx_teststate(conn, CC_STATUS_CLOSING | CC_STATUS_SIGNALED ))  {
                 TRACE_DEBUG(FULL, "Sending FDEVP_CNX_ERROR event");
-                CHECK_FCT_DO( fd_event_send( Target_Queue(conn), FDEVP_CNX_ERROR, 0, NULL), goto fatal);
-                conn->cc_status |= CC_STATUS_SIGNALED;
-        }
-        fd_cpu_flush_cache();
+                CHECK_FCT_DO( fd_event_send( fd_cnx_target_queue(conn), FDEVP_CNX_ERROR, 0, NULL), goto fatal);
+                fd_cnx_addstate(conn, CC_STATUS_SIGNALED);
+        }
         return;
 fatal:
         /* An unrecoverable error occurred, stop the daemon */
+        ASSERT(0);
         CHECK_FCT_DO(fd_event_send(fd_g_config->cnf_main_ev, FDEV_TERMINATE, 0, NULL), );       
 }
@@ -583 +622 @@
         /* Handle special case of timeout */
         if ((ret < 0) && (errno == EAGAIN)) {
-                fd_cpu_flush_cache();
-                if (! (conn->cc_status & CC_STATUS_CLOSING))
+                if (! fd_cnx_teststate(conn, CC_STATUS_CLOSING ))
                         goto again; /* don't care, just ignore */
                 if (!timedout) {
@@ -592 +630 @@
         }
         
-        CHECK_SYS_DO(ret, /* continue */);
-        
         /* Mark the error */
-        if (ret <= 0)
+        if (ret <= 0) {
+                CHECK_SYS_DO(ret, /* continue, this is only used to log the error here */);
                 fd_cnx_markerror(conn);
+        }
         
         return ret;
@@ -610 +648 @@
         /* Handle special case of timeout */
         if ((ret < 0) && (errno == EAGAIN)) {
-                fd_cpu_flush_cache();
-                if (! (conn->cc_status & CC_STATUS_CLOSING))
+                if (! fd_cnx_teststate(conn, CC_STATUS_CLOSING ))
                         goto again; /* don't care, just ignore */
                 if (!timedout) {
@@ -643 +680 @@
         
         ASSERT( conn->cc_proto == IPPROTO_TCP );
-        ASSERT( ! (conn->cc_status & CC_STATUS_TLS) );
-        ASSERT( Target_Queue(conn) );
+        ASSERT( ! fd_cnx_teststate(conn, CC_STATUS_TLS ) );
+        ASSERT( fd_cnx_target_queue(conn) );
         
         /* Receive from a TCP connection: we have to rebuild the message boundaries */
@@ -666 +703 @@
 
                 /* Check the received word is a valid begining of a Diameter message */
-                if ((header[0] != DIAMETER_VERSION)     /* defined in <libfreeDiameter.h> */
+                if ((header[0] != DIAMETER_VERSION)     /* defined in <libfdproto.h> */
                    || (length > DIAMETER_MSG_SIZE_MAX)) { /* to avoid too big mallocs */
                         /* The message is suspect */
@@ -691 +728 @@
                 
                 /* We have received a complete message, pass it to the daemon */
-                fd_cpu_flush_cache();
-                CHECK_FCT_DO( fd_event_send( Target_Queue(conn), FDEVP_CNX_MSG_RECV, length, newmsg), /* continue or destroy everything? */);
+                CHECK_FCT_DO( fd_event_send( fd_cnx_target_queue(conn), FDEVP_CNX_MSG_RECV, length, newmsg), /* continue or destroy everything? */);
                 
         } while (conn->cc_loop);
@@ -726 +762 @@
         
         ASSERT( conn->cc_proto == IPPROTO_SCTP );
-        ASSERT( ! (conn->cc_status & CC_STATUS_TLS) );
-        ASSERT( Target_Queue(conn) );
+        ASSERT( ! fd_cnx_teststate(conn, CC_STATUS_TLS ) );
+        ASSERT( fd_cnx_target_queue(conn) );
         
         do {
-                fd_cpu_flush_cache();
-                CHECK_FCT_DO( fd_sctp_recvmeta(conn->cc_socket, NULL, &buf, &bufsz, &event, &conn->cc_status), goto fatal );
+                CHECK_FCT_DO( fd_sctp_recvmeta(conn, NULL, &buf, &bufsz, &event), goto fatal );
                 if (event == FDEVP_CNX_ERROR) {
                         fd_cnx_markerror(conn);
@@ -742 +777 @@
                 }
                 
-                fd_cpu_flush_cache();
-                CHECK_FCT_DO( fd_event_send( Target_Queue(conn), event, bufsz, buf), goto fatal );
+                CHECK_FCT_DO( fd_event_send( fd_cnx_target_queue(conn), event, bufsz, buf), goto fatal );
                 
         } while (conn->cc_loop || (event != FDEVP_CNX_MSG_RECV));
@@ -763 +797 @@
         TRACE_ENTRY("%p %i", conn, loop);
         
-        CHECK_PARAMS( conn && Target_Queue(conn) && (!(conn->cc_status & CC_STATUS_TLS)) && (!conn->cc_loop));
+        CHECK_PARAMS( conn && fd_cnx_target_queue(conn) && (!fd_cnx_teststate(conn, CC_STATUS_TLS)) && (!conn->cc_loop));
         
         /* Release resources in case of a previous call was already made */
@@ -803 +837 @@
                         switch (ret) {
                                 case GNUTLS_E_REHANDSHAKE: 
-                                        fd_cpu_flush_cache();
-                                        if (!(conn->cc_status & CC_STATUS_CLOSING))
+                                        if (!fd_cnx_teststate(conn, CC_STATUS_CLOSING))
                                                 CHECK_GNUTLS_DO( ret = gnutls_handshake(session),
                                                         {
@@ -815 +848 @@
                                 case GNUTLS_E_AGAIN:
                                 case GNUTLS_E_INTERRUPTED:
-                                        fd_cpu_flush_cache();
-                                        if (!(conn->cc_status & CC_STATUS_CLOSING))
+                                        if (!fd_cnx_teststate(conn, CC_STATUS_CLOSING))
                                                 goto again;
                                         TRACE_DEBUG(FULL, "Connection is closing, so abord gnutls_record_recv now.");
@@ -849 +881 @@
                         switch (ret) {
                                 case GNUTLS_E_REHANDSHAKE: 
-                                        fd_cpu_flush_cache();
-                                        if (!(conn->cc_status & CC_STATUS_CLOSING))
+                                        if (!fd_cnx_teststate(conn, CC_STATUS_CLOSING))
                                                 CHECK_GNUTLS_DO( ret = gnutls_handshake(session),
                                                         {
@@ -861 +892 @@
                                 case GNUTLS_E_AGAIN:
                                 case GNUTLS_E_INTERRUPTED:
-                                        fd_cpu_flush_cache();
-                                        if (!(conn->cc_status & CC_STATUS_CLOSING))
+                                        if (!fd_cnx_teststate(conn, CC_STATUS_CLOSING))
                                                 goto again;
                                         TRACE_DEBUG(INFO, "Connection is closing, so abord gnutls_record_send now.");
@@ -927 +957 @@
                 
                 /* We have received a complete message, pass it to the daemon */
-                fd_cpu_flush_cache();
-                CHECK_FCT_DO( ret = fd_event_send( Target_Queue(conn), FDEVP_CNX_MSG_RECV, length, newmsg), 
+                CHECK_FCT_DO( ret = fd_event_send( fd_cnx_target_queue(conn), FDEVP_CNX_MSG_RECV, length, newmsg), 
                         { 
                                 free(newmsg); 
@@ -956 +985 @@
         }
         
-        ASSERT( conn->cc_status & CC_STATUS_TLS );
-        ASSERT( Target_Queue(conn) );
+        ASSERT( fd_cnx_teststate(conn, CC_STATUS_TLS) );
+        ASSERT( fd_cnx_target_queue(conn) );
 
         /* The next function only returns when there is an error on the socket */       
@@ -1215 +1244 @@
 {
         TRACE_ENTRY( "%p %d %p %p", conn, mode, priority, alt_creds);
-        CHECK_PARAMS( conn && (!(conn->cc_status & CC_STATUS_TLS)) && ( (mode == GNUTLS_CLIENT) || (mode == GNUTLS_SERVER) ) && (!conn->cc_loop) );
+        CHECK_PARAMS( conn && (!fd_cnx_teststate(conn, CC_STATUS_TLS)) && ( (mode == GNUTLS_CLIENT) || (mode == GNUTLS_SERVER) ) && (!conn->cc_loop) );
 
         /* Save the mode */
@@ -1248 +1277 @@
 
         /* Mark the connection as protected from here, so that the gnutls credentials will be freed */
-        fd_cpu_flush_cache();
-        conn->cc_status |= CC_STATUS_TLS;
-
+        fd_cnx_addstate(conn, CC_STATUS_TLS);
+        
         /* Handshake master session */
         {
@@ -1299 +1327 @@
 {
         TRACE_ENTRY("%p %p %p", conn, cert_list, cert_list_size);
-        CHECK_PARAMS( conn && (conn->cc_status & CC_STATUS_TLS) && cert_list && cert_list_size );
+        CHECK_PARAMS( conn && fd_cnx_teststate(conn, CC_STATUS_TLS) && cert_list && cert_list_size );
         
         /* This function only works for X.509 certificates. */
@@ -1360 +1388 @@
 }
 
+/* Where the events are sent */
+struct fifo * fd_cnx_target_queue(struct cnxctx * conn)
+{
+        struct fifo *q;
+        CHECK_POSIX_DO( pthread_mutex_lock(&state_lock), { ASSERT(0); } );
+        q = conn->cc_alt ?: conn->cc_incoming;
+        CHECK_POSIX_DO( pthread_mutex_unlock(&state_lock), { ASSERT(0); } );
+        return q;
+}
+
 /* Set an alternate FIFO list to send FDEVP_CNX_* events to */
 int fd_cnx_recv_setaltfifo(struct cnxctx * conn, struct fifo * alt_fifo)
 {
+        int ret;
         TRACE_ENTRY( "%p %p", conn, alt_fifo );
         CHECK_PARAMS( conn && alt_fifo && conn->cc_incoming );
         
         /* The magic function does it all */
-        CHECK_FCT( fd_fifo_move( conn->cc_incoming, alt_fifo, &conn->cc_alt ) );
-        
-        return 0;
+        CHECK_POSIX_DO( pthread_mutex_lock(&state_lock), { ASSERT(0); } );
+        CHECK_FCT_DO( ret = fd_fifo_move( conn->cc_incoming, alt_fifo, &conn->cc_alt ), );
+        CHECK_POSIX_DO( pthread_mutex_unlock(&state_lock), { ASSERT(0); } );
+        
+        return ret;
 }
 
@@ -1379 +1420 @@
         TRACE_ENTRY("%p %p %zd", conn, buf, len);
         do {
-                fd_cpu_flush_cache();
-                if (conn->cc_status & CC_STATUS_TLS) {
+                if (fd_cnx_teststate(conn, CC_STATUS_TLS)) {
                         CHECK_GNUTLS_DO( ret = fd_tls_send_handle_error(conn, conn->cc_tls_para.session, buf + sent, len - sent),  );
                 } else {
@@ -1394 +1434 @@
 }
 
-/* Send a message -- this is synchronous -- and we assume it's never called by several threads at the same time, so we don't protect. */
+/* Send a message -- this is synchronous -- and we assume it's never called by several threads at the same time (on the same conn), so we don't protect. */
 int fd_cnx_send(struct cnxctx * conn, unsigned char * buf, size_t len, uint32_t flags)
 {
         TRACE_ENTRY("%p %p %zd %x", conn, buf, len, flags);
         
-        CHECK_PARAMS(conn && (conn->cc_socket > 0) && (! (conn->cc_status & CC_STATUS_ERROR)) && buf && len);
-
-        TRACE_DEBUG(FULL, "Sending %zdb %sdata on connection %s", len, (conn->cc_status & CC_STATUS_TLS) ? "TLS-protected ":"", conn->cc_id);
+        CHECK_PARAMS(conn && (conn->cc_socket > 0) && (! fd_cnx_teststate(conn, CC_STATUS_ERROR)) && buf && len);
+
+        TRACE_DEBUG(FULL, "Sending %zdb %sdata on connection %s", len, fd_cnx_teststate(conn, CC_STATUS_TLS) ? "TLS-protected ":"", conn->cc_id);
         
         switch (conn->cc_proto) {
@@ -1410 +1450 @@
 #ifndef DISABLE_SCTP
                 case IPPROTO_SCTP: {
-                        if (flags & FD_CNX_BROADCAST) {
-                                /* Send the buffer over all other streams */
-                                uint16_t str;
-                                fd_cpu_flush_cache();
-                                if (conn->cc_status & CC_STATUS_TLS) {
-                                        for ( str=1; str < conn->cc_sctp_para.pairs; str++) {
-                                                ssize_t ret;
-                                                size_t sent = 0;
-                                                do {
-                                                        CHECK_GNUTLS_DO( ret = fd_tls_send_handle_error(conn, conn->cc_sctps_data.array[str].session, buf + sent, len - sent), );
-                                                        if (ret <= 0)
-                                                                return ENOTCONN;
-
-                                                        sent += ret;
-                                                } while ( sent < len );
-                                        }
-                                } else {
-                                        for ( str=1; str < conn->cc_sctp_para.str_out; str++) {
-                                                CHECK_FCT_DO( fd_sctp_sendstr(conn->cc_socket, str, buf, len, &conn->cc_status), { fd_cnx_markerror(conn); return ENOTCONN; } );
-                                        }
-                                }
-                                
-                                /* Set the ORDERED flag also so that it is sent over stream 0 as well */
-                                flags &= FD_CNX_ORDERED;
-                        }
-                        
                         if (flags & FD_CNX_ORDERED) {
                                 /* We send over stream #0 */
@@ -1444 +1458 @@
                                 int another_str = 0; /* do we send over stream #0 ? */
                                 
-                                if ((conn->cc_sctp_para.str_out > 1) && ((! (conn->cc_status & CC_STATUS_TLS)) || (conn->cc_sctp_para.pairs > 1)))  {
+                                if ((conn->cc_sctp_para.str_out > 1) && ((!fd_cnx_teststate(conn, CC_STATUS_TLS)) || (conn->cc_sctp_para.pairs > 1)))  {
                                         /* Update the id of the stream we will send this message over */
                                         conn->cc_sctp_para.next += 1;
-                                        conn->cc_sctp_para.next %= ((conn->cc_status & CC_STATUS_TLS) ? conn->cc_sctp_para.pairs : conn->cc_sctp_para.str_out);
+                                        conn->cc_sctp_para.next %= (fd_cnx_teststate(conn, CC_STATUS_TLS) ? conn->cc_sctp_para.pairs : conn->cc_sctp_para.str_out);
                                         another_str = (conn->cc_sctp_para.next ? 1 : 0);
                                 }
@@ -1454 +1468 @@
                                         CHECK_FCT( send_simple(conn, buf, len) );
                                 } else {
-                                        if (!(conn->cc_status & CC_STATUS_TLS)) {
-                                                CHECK_FCT_DO( fd_sctp_sendstr(conn->cc_socket, conn->cc_sctp_para.next, buf, len, &conn->cc_status), { fd_cnx_markerror(conn); return ENOTCONN; } );
+                                        if (!fd_cnx_teststate(conn, CC_STATUS_TLS)) {
+                                                CHECK_FCT_DO( fd_sctp_sendstr(conn, conn->cc_sctp_para.next, buf, len), { fd_cnx_markerror(conn); return ENOTCONN; } );
                                         } else {
                                                 /* push the record to the appropriate session */
@@ -1496 +1510 @@
         CHECK_PARAMS_DO(conn, return);
         
-        fd_cpu_flush_cache();
-        conn->cc_status |= CC_STATUS_CLOSING;
+        fd_cnx_addstate(conn, CC_STATUS_CLOSING);
         
         /* Initiate shutdown of the TLS session(s): call gnutls_bye(WR), then read until error */
-        if (conn->cc_status & CC_STATUS_TLS) {
+        if (fd_cnx_teststate(conn, CC_STATUS_TLS)) {
 #ifndef DISABLE_SCTP
                 if (conn->cc_sctp_para.pairs > 1) {
-                        if (! (conn->cc_status & CC_STATUS_ERROR )) {
+                        if (! fd_cnx_teststate(conn, CC_STATUS_ERROR )) {
                                 /* Bye on master session */
                                 CHECK_GNUTLS_DO( gnutls_bye(conn->cc_tls_para.session, GNUTLS_SHUT_WR), fd_cnx_markerror(conn) );
                         }
 
-                        if (! (conn->cc_status & CC_STATUS_ERROR ) ) {
+                        if (! fd_cnx_teststate(conn, CC_STATUS_ERROR ) ) {
                                 /* and other stream pairs */
                                 fd_sctps_bye(conn);
                         }
 
-                        if (! (conn->cc_status & CC_STATUS_ERROR ) ) {
+                        if (! fd_cnx_teststate(conn, CC_STATUS_ERROR ) ) {
                                 /* Now wait for all decipher threads to terminate */
                                 fd_sctps_waitthreadsterm(conn);
@@ -1533 +1546 @@
                 } else {
 #endif /* DISABLE_SCTP */
-                /* We are not using the sctps wrapper layer */
-                        if (! (conn->cc_status & CC_STATUS_ERROR ) ) {
+                /* We are TLS, but not using the sctps wrapper layer */
+                        if (! fd_cnx_teststate(conn, CC_STATUS_ERROR ) ) {
                                 /* Master session */
                                 CHECK_GNUTLS_DO( gnutls_bye(conn->cc_tls_para.session, GNUTLS_SHUT_WR), fd_cnx_markerror(conn) );
                         }
 
-                        if (! (conn->cc_status & CC_STATUS_ERROR ) ) {
+                        if (! fd_cnx_teststate(conn, CC_STATUS_ERROR ) ) {
                                 /* In this case, just wait for thread rcvthr_tls_single to terminate */
                                 if (conn->cc_rcvthr != (pthread_t)NULL) {
@@ -1555 +1568 @@
                                 conn->cc_tls_para.session = NULL;
                         }
-                
 #ifndef DISABLE_SCTP
                 }

libfdcore/cnxctx.h

@@ -48 +48 @@
         int             cc_family;      /* AF_INET or AF_INET6 (mixed) */
         int             cc_proto;       /* IPPROTO_TCP or IPPROTO_SCTP */
-        uint32_t        cc_status;      /* True if the object is being destroyed: we don't send events anymore */
+        
+        uint32_t        cc_state;       /* True if the object is being destroyed: we don't send events anymore. access with fd_cnx_getstate() */
         #define         CC_STATUS_CLOSING       1
         #define         CC_STATUS_ERROR         2
@@ -59 +60 @@
         struct fifo *   cc_incoming;    /* FIFO queue of events received on the connection, FDEVP_CNX_* */
         struct fifo *   cc_alt;         /* alternate fifo to send FDEVP_CNX_* events to. */
-        #define Target_Queue(cnx)       ((cnx)->cc_alt ?: (cnx)->cc_incoming)
 
         /* If cc_tls == true */
         struct {
-                char                            *cn;            /* If not NULL, remote certif will be checked to match this Common Name */
+                DiamId_t                         cn;            /* If not NULL, remote certif will be checked to match this Common Name */
                 int                              mode;          /* GNUTLS_CLIENT / GNUTLS_SERVER */
                 gnutls_session_t                 session;       /* Session object (stream #0 in case of SCTP) */
@@ -84 +84 @@
 
 void fd_cnx_markerror(struct cnxctx * conn);
+uint32_t fd_cnx_getstate(struct cnxctx * conn);
+int  fd_cnx_teststate(struct cnxctx * conn, uint32_t flag);
+void fd_cnx_addstate(struct cnxctx * conn, uint32_t orstate);
+void fd_cnx_setstate(struct cnxctx * conn, uint32_t abstate);
+struct fifo * fd_cnx_target_queue(struct cnxctx * conn);
+
 
 /* Socket */
@@ -109 +115 @@
 int fd_sctp_get_remote_ep(int sock, struct fd_list * list);
 int fd_sctp_get_str_info( int sock, uint16_t *in, uint16_t *out, sSS *primary );
-int fd_sctp_sendstr(int sock, uint16_t strid, uint8_t * buf, size_t len, uint32_t * cc_closing);
-int fd_sctp_recvmeta(int sock, uint16_t * strid, uint8_t ** buf, size_t * len, int *event, uint32_t * cc_closing);
+int fd_sctp_sendstr(struct cnxctx * conn, uint16_t strid, uint8_t * buf, size_t len);
+int fd_sctp_recvmeta(struct cnxctx * conn, uint16_t * strid, uint8_t ** buf, size_t * len, int *event);
 
 /* TLS over SCTP (multi-stream) */

libfdcore/config.c

@@ -146 +146 @@
 {
         extern FILE * fddin;
+        char * orig = NULL;
         
         /* Attempt to find the configuration file */
@@ -154 +155 @@
         if ((fddin == NULL) && (*fd_g_config->cnf_file != '/')) {
                 /* We got a relative path, attempt to add the default directory prefix */
-                char * bkp = fd_g_config->cnf_file;
-                CHECK_MALLOC( fd_g_config->cnf_file = malloc(strlen(bkp) + strlen(DEFAULT_CONF_PATH) + 2) ); /* we will not free it, but not important */
-                sprintf( fd_g_config->cnf_file, DEFAULT_CONF_PATH "/%s", bkp );
+                orig = fd_g_config->cnf_file;
+                CHECK_MALLOC( fd_g_config->cnf_file = malloc(strlen(orig) + strlen(DEFAULT_CONF_PATH) + 2) ); /* we will not free it, but not important */
+                sprintf( fd_g_config->cnf_file, DEFAULT_CONF_PATH "/%s", orig );
                 fddin = fopen(fd_g_config->cnf_file, "r");
         }
         if (fddin == NULL) {
                 int ret = errno;
-                fprintf(stderr, "Unable to open configuration file %s for reading: %s\n", fd_g_config->cnf_file, strerror(ret));
+                if (orig) {
+                        fprintf(stderr, "Unable to open configuration file for reading\n"
+                                        "Tried the following locations:\n"
+                                        " - %s\n"
+                                        " - %s\n"
+                                        "Error: %s\n", orig, fd_g_config->cnf_file, strerror(ret));
+                } else {
+                        fprintf(stderr, "Unable to open '%s' for reading: %s\n", fd_g_config->cnf_file, strerror(ret));
+                }
                 return ret;
         }
@@ -178 +187 @@
         }
         
+        /* If the CA is not provided, let's use the same file (assuming self-signed certificate) */
+        if (! fd_g_config->cnf_sec_data.ca_file) {
+                CHECK_MALLOC( fd_g_config->cnf_sec_data.ca_file = strdup(fd_g_config->cnf_sec_data.cert_file) );
+                CHECK_GNUTLS_DO( fd_g_config->cnf_sec_data.ca_file_nr += gnutls_certificate_set_x509_trust_file( 
+                                        fd_g_config->cnf_sec_data.credentials,
+                                        fd_g_config->cnf_sec_data.ca_file,
+                                        GNUTLS_X509_FMT_PEM),
+                                { 
+                                        TRACE_DEBUG(INFO, "Unable to use the local certificate as trusted security anchor (CA), please provide a valid TLS_CA='...' directive.");
+                                        return EINVAL;
+                                } );
+        }
+        
+        
         /* Resolve hostname if not provided */
         if (fd_g_config->cnf_diamid == NULL) {
-#ifndef HOST_NAME_MAX
-#define HOST_NAME_MAX 1024
-#endif /* HOST_NAME_MAX */
                 char buf[HOST_NAME_MAX + 1];
                 struct addrinfo hints, *info;
@@ -202 +222 @@
                         return EINVAL;
                 }
-                CHECK_MALLOC( fd_g_config->cnf_diamid = strdup(info->ai_canonname) );
+                fd_g_config->cnf_diamid = info->ai_canonname;
+                CHECK_FCT( fd_os_validate_DiameterIdentity(&fd_g_config->cnf_diamid, &fd_g_config->cnf_diamid_len, 1) );
                 freeaddrinfo(info);
-        }
-        
-        /* cache the length of the diameter id for the session module */
-        fd_g_config->cnf_diamid_len = strlen(fd_g_config->cnf_diamid);
+        } else {
+                CHECK_FCT( fd_os_validate_DiameterIdentity(&fd_g_config->cnf_diamid, &fd_g_config->cnf_diamid_len, 0) );
+        }
         
         /* Handle the realm part */
@@ -220 +240 @@
                                         fd_g_config->cnf_diamid);
                         return EINVAL;
-                }               
-                
-                CHECK_MALLOC( fd_g_config->cnf_diamrlm = strdup( start + 1 )  ); 
-        }
-        fd_g_config->cnf_diamrlm_len = strlen(fd_g_config->cnf_diamrlm);
+                }
+                
+                fd_g_config->cnf_diamrlm = start + 1;
+                CHECK_FCT( fd_os_validate_DiameterIdentity(&fd_g_config->cnf_diamrlm, &fd_g_config->cnf_diamrlm_len, 1) );
+        } else {
+                CHECK_FCT( fd_os_validate_DiameterIdentity(&fd_g_config->cnf_diamrlm, &fd_g_config->cnf_diamrlm_len, 0) );
+        }
         
         /* Validate some flags */

libfdcore/core.c

@@ -45 +45 @@
 GCRY_THREAD_OPTION_PTHREAD_IMPL;
 
-/* Signal extensions when the framework is completly initialized */
+/* Signal extensions when the framework is completly initialized (they are waiting in fd_core_waitstartcomplete()) */
 static int             is_ready = 0;
 static pthread_mutex_t is_ready_mtx = PTHREAD_MUTEX_INITIALIZER;
@@ -62 +62 @@
 /* Thread that process incoming events on the main queue -- and terminates the framework when requested */
 static pthread_t core_runner = (pthread_t)NULL;
+
+/* How the thread is terminated */
 enum core_mode {
         CORE_MODE_EVENTS,

libfdcore/extensions.c

@@ -57 +57 @@
         struct fd_ext_info * new;
         
-        TRACE_ENTRY("%p(%s) %p(%s)", filename, filename?filename:"", conffile, conffile?conffile:"");
+        TRACE_ENTRY("%p %p", filename, conffile);
         
         /* Check the filename is valid */
@@ -106 +106 @@
                 ext->handler = dlopen(ext->filename, RTLD_LAZY | RTLD_GLOBAL);
 #else /* DEBUG */
-                /* We resolve immediatly so it's easier to find problems in ABI */
+                /* We resolve symbols immediatly so it's easier to find problems in ABI */
                 ext->handler = dlopen(ext->filename, RTLD_NOW | RTLD_GLOBAL);
 #endif /* DEBUG */

libfdcore/fdcore-internal.h

@@ -138 +138 @@
         char            *p_dbgorig;
         
+        /* State of the peer, and its lock */
+        enum peer_state  p_state;
+        pthread_mutex_t  p_state_mtx;
+        
         /* Chaining in peers sublists */
         struct fd_list   p_actives;     /* list of peers in the STATE_OPEN state -- used by routing */
@@ -147 +151 @@
                 unsigned pf_responder   : 1;    /* The peer has been created to handle incoming connection */
                 unsigned pf_delete      : 1;    /* Destroy the peer when the connection is terminated */
+                unsigned pf_localterm   : 1;    /* If the latest DPR/DPA was initiated from this side */
                 
                 unsigned pf_dw_pending  : 1;    /* A DWR message was sent and not answered yet */
@@ -192 +197 @@
 #define CHECK_PEER( _p ) \
         (((_p) != NULL) && (((struct fd_peer *)(_p))->p_eyec == EYEC_PEER))
+
+#define fd_peer_getstate(peer)  fd_peer_get_state((struct peer_hdr *)(peer))
+
 
 /* Events codespace for struct fd_peer->p_events */
@@ -308 +316 @@
 int fd_p_dp_handle(struct msg ** msg, int req, struct fd_peer * peer);
 int fd_p_dp_initiate(struct fd_peer * peer, char * reason);
+int fd_p_dp_newdelay(struct fd_peer * peer);
 
 /* Active peers -- routing process should only ever take the read lock, the write lock is managed by PSMs */
@@ -327 +336 @@
 struct cnxctx * fd_cnx_cli_connect_sctp(int no_ip6, uint16_t port, struct fd_list * list);
 int             fd_cnx_start_clear(struct cnxctx * conn, int loop);
-void            fd_cnx_sethostname(struct cnxctx * conn, char * hn);
+void            fd_cnx_sethostname(struct cnxctx * conn, DiamId_t hn);
 int             fd_cnx_handshake(struct cnxctx * conn, int mode, char * priority, void * alt_creds);
 char *          fd_cnx_getid(struct cnxctx * conn);
 int             fd_cnx_getproto(struct cnxctx * conn);
 int             fd_cnx_getTLS(struct cnxctx * conn);
+int             fd_cnx_isMultichan(struct cnxctx * conn);
 int             fd_cnx_getcred(struct cnxctx * conn, const gnutls_datum_t **cert_list, unsigned int *cert_list_size);
 int             fd_cnx_get_local_eps(struct fd_list * list);
@@ -343 +353 @@
 /* Flags for the fd_cnx_send function : */
 #define FD_CNX_ORDERED          (1 << 0)        /* All messages sent with this flag set will be delivered in the same order. No guarantee on other messages */
-#define FD_CNX_BROADCAST        (1 << 1)        /* The message is sent over all stream pairs, in case of SCTP. No effect on TCP */
 
 #endif /* _FDCORE_INTERNAL_H */

libfdcore/fdd.y

@@ -236 +236 @@
 appservthreads:         APPSERVTHREADS '=' INTEGER ';'
                         {
-                                CHECK_PARAMS_DO( ($3 > 0) && ($3 < 1024),
+                                CHECK_PARAMS_DO( ($3 > 0) && ($3 < 256),
                                         { yyerror (&yylloc, conf, "Invalid value"); YYERROR; } );
                                 conf->cnf_dispthr = (uint16_t)$3;

libfdcore/messages.c

@@ -36 +36 @@
 #include "fdcore-internal.h"
 
+static struct dict_object * dict_avp_SI  = NULL; /* Session-Id */
 static struct dict_object * dict_avp_OH  = NULL; /* Origin-Host */
 static struct dict_object * dict_avp_OR  = NULL; /* Origin-Realm */
@@ -54 +55 @@
         
         /* Initialize the dictionary objects that we may use frequently */
+        CHECK_FCT(  fd_dict_search( fd_g_config->cnf_dict, DICT_AVP, AVP_BY_NAME, "Session-Id",         &dict_avp_SI , ENOENT)  );
         CHECK_FCT(  fd_dict_search( fd_g_config->cnf_dict, DICT_AVP, AVP_BY_NAME, "Origin-Host",        &dict_avp_OH  , ENOENT)  );
         CHECK_FCT(  fd_dict_search( fd_g_config->cnf_dict, DICT_AVP, AVP_BY_NAME, "Origin-Realm",       &dict_avp_OR  , ENOENT)  );
@@ -89 +91 @@
         /* Set its value */
         memset(&val, 0, sizeof(val));
-        val.os.data = (unsigned char *)fd_g_config->cnf_diamid;
+        val.os.data = (os0_t)fd_g_config->cnf_diamid;
         val.os.len  = fd_g_config->cnf_diamid_len;
         CHECK_FCT( fd_msg_avp_setvalue( avp_OH, &val ) );
@@ -102 +104 @@
         /* Set its value */
         memset(&val, 0, sizeof(val));
-        val.os.data = (unsigned char *)fd_g_config->cnf_diamrlm;
+        val.os.data = (os0_t)fd_g_config->cnf_diamrlm;
         val.os.len  = fd_g_config->cnf_diamrlm_len;
         CHECK_FCT( fd_msg_avp_setvalue( avp_OR, &val ) );
@@ -124 +126 @@
         return 0;
 }
+
+/* Create a new Session-Id and add at the beginning of the message. */
+int fd_msg_new_session( struct msg * msg, os0_t opt, size_t optlen )
+{
+        union avp_value val;
+        struct avp * avp  = NULL;
+        struct session * sess = NULL;
+        os0_t sid;
+        size_t sidlen;
+        
+        TRACE_ENTRY("%p %p %zd", msg, opt, optlen);
+        CHECK_PARAMS(  msg  );
+        
+        /* Check there is not already a session in the message */
+        CHECK_FCT( fd_msg_sess_get(fd_g_config->cnf_dict, msg, &sess, NULL) );
+        CHECK_PARAMS( sess == NULL );
+        
+        /* Ok, now create the session */
+        CHECK_FCT( fd_sess_new ( &sess, fd_g_config->cnf_diamid, fd_g_config->cnf_diamid_len, opt, optlen ) );
+        CHECK_FCT( fd_sess_getsid( sess, &sid, &sidlen) );
+        
+        /* Create an AVP to hold it */
+        CHECK_FCT( fd_msg_avp_new( dict_avp_SI, 0, &avp ) );
+        
+        /* Set its value */
+        memset(&val, 0, sizeof(val));
+        val.os.data = sid;
+        val.os.len  = sidlen;
+        CHECK_FCT( fd_msg_avp_setvalue( avp, &val ) );
+        
+        /* Add it to the message */
+        CHECK_FCT( fd_msg_avp_add( msg, MSG_BRW_FIRST_CHILD, avp ) );
+        
+        /* Done! */
+        return 0;
+}
+
 
 /* Add Result-Code and eventually Failed-AVP, Error-Message and Error-Reporting-Host AVPs */
@@ -147 +186 @@
                 memset(&req, 0, sizeof(struct dict_enumval_request));
                 
-                /* First, get the enumerated type of the Result-Code AVP */
+                /* First, get the enumerated type of the Result-Code AVP (this is fast, no need to cache the object) */
                 CHECK_FCT(  fd_dict_search( fd_g_config->cnf_dict, DICT_TYPE, TYPE_OF_AVP, dict_avp_RC, &(req.type_obj), ENOENT  )  );
                 
@@ -184 +223 @@
                 /* Set its value */
                 memset(&val, 0, sizeof(val));
-                val.os.data = (unsigned char *)fd_g_config->cnf_diamid;
+                val.os.data = (uint8_t *)fd_g_config->cnf_diamid;
                 val.os.len  = fd_g_config->cnf_diamid_len;
                 CHECK_FCT( fd_msg_avp_setvalue( avp_ERH, &val ) );
@@ -246 +285 @@
                 
                 if (errormsg) {
-                        val.os.data = (unsigned char *)errormsg;
+                        val.os.data = (uint8_t *)errormsg;
                         val.os.len  = strlen(errormsg);
                 } else {
-                        val.os.data = (unsigned char *)rescode;
+                        val.os.data = (uint8_t *)rescode;
                         val.os.len  = strlen(rescode);
                 }
@@ -311 +350 @@
         if      ((ret != EBADMSG)       /* Parsing grouped AVP failed / Conflicting rule found */
                 && (ret != ENOTSUP))    /* Command is not supported / Mandatory AVP is not supported */
-                return ret;
+                return ret; /* 0 or another error */
         
         TRACE_DEBUG(INFO, "A message does not comply to the dictionary and/or rules (%s)", pei.pei_errcode);

libfdcore/p_ce.c

@@ -38 +38 @@
 /* This file contains code to handle Capabilities Exchange messages (CER and CEA) and election process */
 
-/* Compilation option:
- USE_CEA_BROADCAST
-        Define this to enable sending multiple copies of the CEA in case of SCTP connection.
-        This avoids a race condition when sending an application message over a different stream
-        than the CEA, it might be delivered first and thus ignored.
-*/
-
 /* Save a connection as peer's principal */
 static int set_peer_cnx(struct fd_peer * peer, struct cnxctx **cnx)
@@ -88 +81 @@
 }
 
-/* Election: compare the Diameter Ids, return true if the election is won */
+/* Election: compare the Diameter Ids by lexical order, return true if the election is won */
 static __inline__ int election_result(struct fd_peer * peer)
 {
@@ -245 +238 @@
 static void cleanup_remote_CE_info(struct fd_peer * peer)
 {
+        /* free linked information */
         free(peer->p_hdr.info.runtime.pir_realm);
-        peer->p_hdr.info.runtime.pir_realm = NULL;
-        peer->p_hdr.info.runtime.pir_vendorid = 0;
-        peer->p_hdr.info.runtime.pir_orstate = 0;
         free(peer->p_hdr.info.runtime.pir_prodname);
-        peer->p_hdr.info.runtime.pir_prodname = NULL;
-        peer->p_hdr.info.runtime.pir_firmrev = 0;
-        peer->p_hdr.info.runtime.pir_relay = 0;
-        peer->p_hdr.info.runtime.pir_isi = 0;
         while (!FD_IS_LIST_EMPTY(&peer->p_hdr.info.runtime.pir_apps)) {
                 struct fd_list * li = peer->p_hdr.info.runtime.pir_apps.next;
@@ -259 +246 @@
                 free(li);
         }
-        
-        fd_ep_clearflags( &peer->p_hdr.info.pi_endpoints, EP_FL_ADV /* Remove previously advertised endpoints */ );
+        /* note: pir_cert_list needs not be freed (belongs to gnutls) */
+        
+        /* cleanup the area */
+        memset(&peer->p_hdr.info.runtime, 0, sizeof(peer->p_hdr.info.runtime));
+        
+        /* reinit the list */
+        fd_list_init(&peer->p_hdr.info.runtime.pir_apps, peer);
+
+        /* Remove previously advertised endpoints */
+        fd_ep_clearflags( &peer->p_hdr.info.pi_endpoints, EP_FL_ADV );
 }
 
 /* Extract information sent by the remote peer and save it in our peer structure */
-static int save_remote_CE_info(struct msg * msg, struct fd_peer * peer, char ** error_code, uint32_t *rc)
+static int save_remote_CE_info(struct msg * msg, struct fd_peer * peer, struct fd_pei * error, uint32_t *rc)
 {
         struct avp * avp = NULL;
@@ -309 +304 @@
                                 
                                 /* We check that the value matches what we know, otherwise disconnect the peer */
-                                /* here also, using strcasecmp on (supposed) UTF8 data might be bad idea... to be improved */
-                                if (strncasecmp((char *)hdr->avp_value->os.data, peer->p_hdr.info.pi_diamid, hdr->avp_value->os.len)) {
+                                if (fd_os_almostcasecmp(hdr->avp_value->os.data, hdr->avp_value->os.len, 
+                                                        peer->p_hdr.info.pi_diamid, peer->p_hdr.info.pi_diamidlen)) {
                                         TRACE_DEBUG(INFO, "Received a message with Origin-Host set to '%.*s' while expecting '%s'\n", 
                                                         hdr->avp_value->os.len, hdr->avp_value->os.data, peer->p_hdr.info.pi_diamid);
-                                        *error_code = "DIAMETER_UNKNOWN_PEER";
+                                        error->pei_errcode = "ER_DIAMETER_AVP_NOT_ALLOWED";
+                                        error->pei_message = "Your Origin-Host value does not match my configuration.";
+                                        error->pei_avp = avp;
                                         return EINVAL;
                                 }
@@ -330 +327 @@
                                 /* In case of multiple AVPs */
                                 if (peer->p_hdr.info.runtime.pir_realm) {
-                                        TRACE_DEBUG(INFO, "Ignored multiple instances of the Origin-Realm AVP");
-                                        goto next;
-                                }
-                                
-                                /* Save the value -- we don't change the case to avoid risking breaking UTF-8 with poor tolower() impls. */
-                                CHECK_MALLOC(  peer->p_hdr.info.runtime.pir_realm = calloc( hdr->avp_value->os.len + 1, 1 )  );
-                                memcpy(peer->p_hdr.info.runtime.pir_realm, hdr->avp_value->os.data, hdr->avp_value->os.len);
+                                        TRACE_DEBUG(INFO, "Multiple instances of the Origin-Realm AVP");
+                                        error->pei_errcode = "ER_DIAMETER_AVP_OCCURS_TOO_MANY_TIMES";
+                                        error->pei_message = "I found several Origin-Realm AVPs";
+                                        error->pei_avp = avp;
+                                        return EINVAL;
+                                }
+                                
+                                /* If the octet string contains a \0 */
+                                if (!fd_os_is_valid_DiameterIdentity(hdr->avp_value->os.data, hdr->avp_value->os.len)) {
+                                        error->pei_errcode = "ER_DIAMETER_INVALID_AVP_VALUE";
+                                        error->pei_message = "Your Origin-Realm contains invalid characters.";
+                                        error->pei_avp = avp;
+                                        return EINVAL;
+                                }
+                                
+                                /* Save the value */
+                                CHECK_MALLOC(  peer->p_hdr.info.runtime.pir_realm = os0dup( hdr->avp_value->os.data, hdr->avp_value->os.len )  );
+                                peer->p_hdr.info.runtime.pir_realmlen = hdr->avp_value->os.len;
                                 break;
 
@@ -352 +360 @@
                                         /* Get the sockaddr value */
                                         memset(&ss, 0, sizeof(ss));
-                                        CHECK_FCT( fd_msg_avp_value_interpret( avp, &ss) );
+                                        CHECK_FCT_DO( fd_msg_avp_value_interpret( avp, &ss),
+                                                {
+                                                        /* in case of error, assume the AVP value was wrong */
+                                                        error->pei_errcode = "ER_DIAMETER_INVALID_AVP_VALUE";
+                                                        error->pei_avp = avp;
+                                                        return EINVAL;
+                                                } );
 
                                         /* Save this endpoint in the list as advertized */
@@ -370 +384 @@
                                 /* In case of multiple AVPs */
                                 if (peer->p_hdr.info.runtime.pir_vendorid) {
-                                        TRACE_DEBUG(INFO, "Ignored multiple instances of the Vendor-Id AVP");
-                                        goto next;
+                                        TRACE_DEBUG(INFO, "Multiple instances of the Vendor-Id AVP");
+                                        error->pei_errcode = "ER_DIAMETER_AVP_OCCURS_TOO_MANY_TIMES";
+                                        error->pei_message = "I found several Vendor-Id AVPs";
+                                        error->pei_avp = avp;
+                                        return EINVAL;
                                 }
                                 
@@ -388 +405 @@
                                 /* In case of multiple AVPs */
                                 if (peer->p_hdr.info.runtime.pir_prodname) {
-                                        TRACE_DEBUG(INFO, "Ignored multiple instances of the Product-Name AVP");
-                                        goto next;
+                                        TRACE_DEBUG(INFO, "Multiple instances of the Product-Name AVP");
+                                        error->pei_errcode = "ER_DIAMETER_AVP_OCCURS_TOO_MANY_TIMES";
+                                        error->pei_message = "I found several Product-Name AVPs";
+                                        error->pei_avp = avp;
+                                        return EINVAL;
                                 }
 
@@ -407 +427 @@
                                 /* In case of multiple AVPs */
                                 if (peer->p_hdr.info.runtime.pir_orstate) {
-                                        TRACE_DEBUG(INFO, "Ignored multiple instances of the Origin-State-Id AVP");
-                                        goto next;
+                                        TRACE_DEBUG(INFO, "Multiple instances of the Origin-State-Id AVP");
+                                        error->pei_errcode = "ER_DIAMETER_AVP_OCCURS_TOO_MANY_TIMES";
+                                        error->pei_message = "I found several Origin-State-Id AVPs";
+                                        error->pei_avp = avp;
+                                        return EINVAL;
                                 }
                                 
@@ -423 +446 @@
                                 }
                                 
-                                TRACE_DEBUG(FULL, "'%s' supports a subset of vendor %d features.", peer->p_hdr.info.pi_diamid, hdr->avp_value->u32);
+                                TRACE_DEBUG(FULL, "'%s' claims support for a subset of vendor %d features.", peer->p_hdr.info.pi_diamid, hdr->avp_value->u32);
+                                /* not that it makes a difference for us... 
+                                 -- if an application actually needs this info, we could save it somewhere.
+                                */
                                 break;
 
@@ -477 +503 @@
                                                 TRACE_DEBUG(FULL, "Invalid Vendor-Specific-Application-Id AVP received, ignored");
                                                 fd_msg_dump_one(FULL, avp);
+                                                error->pei_errcode = "ER_DIAMETER_INVALID_AVP_VALUE";
+                                                error->pei_avp = avp;
+                                                return EINVAL;
                                         } else {
                                                 /* Add an entry in the list */
@@ -510 +539 @@
                                 
                                 if (hdr->avp_value->u32 == AI_RELAY) {
+                                        /* Not clear if the relay application can be inside this AVP... */
                                         peer->p_hdr.info.runtime.pir_relay = 1;
                                 } else {
-                                        /* Not clear if the relay application can be inside this AVP... */
                                         CHECK_FCT( fd_app_merge(&peer->p_hdr.info.runtime.pir_apps, hdr->avp_value->u32, 0, 0, 1) );
                                 }
@@ -537 +566 @@
                                         goto next;
                                 }
-                                ASSERT( hdr->avp_value->u32 < 32 ); /* if false, we have to change the code bellow */
+                                if (hdr->avp_value->u32 >= 32 ) {
+                                        error->pei_errcode = "ER_DIAMETER_INVALID_AVP_VALUE";
+                                        error->pei_message = "I don't support this Inband-Security-Id value (yet).";
+                                        error->pei_avp = avp;
+                                        return EINVAL;
+                                }
                                 peer->p_hdr.info.runtime.pir_isi |= (1 << hdr->avp_value->u32);
                                 break;
@@ -559 +593 @@
         CHECK_FCT( fd_msg_new ( fd_dict_cmd_CER, MSGFL_ALLOC_ETEID, cer ) );
         
-        /* Do we need Inband-Security-Id AVPs ? */
+        /* Do we need Inband-Security-Id AVPs ? If we're already using TLS, we don't... */
         if (!fd_cnx_getTLS(cnx)) {
-                isi_none = peer->p_hdr.info.config.pic_flags.sec & PI_SEC_NONE; /* we add it event if the peer does not use the old mechanism */
+                isi_none = peer->p_hdr.info.config.pic_flags.sec & PI_SEC_NONE; /* we add it even if the peer does not use the old mechanism, it is impossible to distinguish */
                 isi_tls  = peer->p_hdr.info.config.pic_flags.sec & PI_SEC_TLS_OLD;
         }
@@ -587 +621 @@
 
 /* Reject an incoming connection attempt */
-static void receiver_reject(struct cnxctx * recv_cnx, struct msg ** cer, char * rescode, char * errormsg)
+static void receiver_reject(struct cnxctx ** recv_cnx, struct msg ** cer, struct fd_pei * error)
 {
         /* Create and send the CEA with appropriate error code */
         CHECK_FCT_DO( fd_msg_new_answer_from_req ( fd_g_config->cnf_dict, cer, MSGFL_ANSW_ERROR ), goto destroy );
-        CHECK_FCT_DO( fd_msg_rescode_set(*cer, rescode, errormsg, NULL, 1 ), goto destroy );
-        CHECK_FCT_DO( fd_out_send(cer, recv_cnx, NULL, FD_CNX_ORDERED), goto destroy );
+        CHECK_FCT_DO( fd_msg_rescode_set(*cer, error->pei_errcode, error->pei_message, error->pei_avp, 1 ), goto destroy );
+        CHECK_FCT_DO( fd_out_send(cer, *recv_cnx, NULL, FD_CNX_ORDERED), goto destroy );
         
         /* And now destroy this connection */
 destroy:
-        fd_cnx_destroy(recv_cnx);
+        fd_cnx_destroy(*recv_cnx);
+        *recv_cnx = NULL;
         if (*cer) {
                 fd_msg_log(FD_MSG_LOG_DROPPED, *cer, "An error occurred while rejecting a CER.");
@@ -614 +649 @@
         
         /* Are we doing an election ? */
-        fd_cpu_flush_cache();
-        if (peer->p_hdr.info.runtime.pir_state == STATE_WAITCNXACK_ELEC) {
+        if (fd_peer_getstate(peer) == STATE_WAITCNXACK_ELEC) {
                 if (election_result(peer)) {
                         /* Close initiator connection */
@@ -624 +658 @@
 
                 } else {
+                        struct fd_pei pei;
+                        memset(&pei, 0, sizeof(pei));
+                        pei.pei_errcode = "ELECTION_LOST";
 
                         /* Answer an ELECTION LOST to the receiver side */
-                        receiver_reject(peer->p_receiver, &peer->p_cer, "ELECTION_LOST", NULL);
-                        peer->p_receiver = NULL;
+                        receiver_reject(&peer->p_receiver, &peer->p_cer, &pei);
                         CHECK_FCT( to_waitcea(peer, initiator) );
                 }
@@ -641 +677 @@
 int fd_p_ce_msgrcv(struct msg ** msg, int req, struct fd_peer * peer)
 {
-        char * ec;
         uint32_t rc = 0;
+        int st;
+        struct fd_pei pei;
+        
         TRACE_ENTRY("%p %p", msg, peer);
         CHECK_PARAMS( msg && *msg && CHECK_PEER(peer) );
@@ -656 +694 @@
                 
                 /* Set the error code */
-                CHECK_FCT( fd_msg_rescode_set(*msg, "DIAMETER_COMMAND_UNSUPPORTED", "No CER allowed in current state", NULL, 1 ) );
+                CHECK_FCT( fd_msg_rescode_set(*msg, "ER_DIAMETER_UNABLE_TO_COMPLY", "No CER allowed in current state", NULL, 1 ) );
 
                 /* msg now contains an answer message to send back */
@@ -663 +701 @@
         
         /* If the state is not WAITCEA, just discard the message */
-        fd_cpu_flush_cache();
-        if (req || (peer->p_hdr.info.runtime.pir_state != STATE_WAITCEA)) {
+        if (req || ((st = fd_peer_getstate(peer)) != STATE_WAITCEA)) {
                 if (*msg) {
-                        fd_msg_log( FD_MSG_LOG_DROPPED, *msg, "Received CER/CEA while in '%s' state.\n", STATE_STR(peer->p_hdr.info.runtime.pir_state));
+                        fd_msg_log( FD_MSG_LOG_DROPPED, *msg, "Received CER/CEA while in '%s' state.\n", STATE_STR(st));
                         CHECK_FCT_DO( fd_msg_free(*msg), /* continue */);
                         *msg = NULL;
@@ -674 +711 @@
         }
         
+        memset(&pei, 0, sizeof(pei));
+        
         /* Save info from the CEA into the peer */
-        CHECK_FCT_DO( save_remote_CE_info(*msg, peer, &ec, &rc), goto cleanup );
+        CHECK_FCT_DO( save_remote_CE_info(*msg, peer, &pei, &rc), goto cleanup );
         
         /* Dispose of the message, we don't need it anymore */
@@ -701 +740 @@
                 default:
                         /* In any other case, we abort all attempts to connect to this peer */
-                        TRACE_DEBUG(INFO, "Peer %s replied a CEA with Result-Code AVP %d, aborting connection attempts.", peer->p_hdr.info.pi_diamid, rc);
+                        TRACE_DEBUG(INFO, "Peer %s replied a CEA with Result-Code %d, aborting connection attempts.", peer->p_hdr.info.pi_diamid, rc);
                         return EINVAL;
         }
@@ -751 +790 @@
 }
 
-/* Handle the receiver side to go to OPEN state (any election is resolved) */
+/* Handle the receiver side to go to OPEN or OPEN_NEW state (any election is resolved) */
 int fd_p_ce_process_receiver(struct fd_peer * peer)
 {
-        char * ec = NULL;
+        struct fd_pei pei;
         struct msg * msg = NULL;
         int isi = 0;
         int fatal = 0;
+        int tls_sync=0;
         
         TRACE_ENTRY("%p", peer);
@@ -765 +805 @@
         peer->p_cer = NULL;
         
+        memset(&pei, 0, sizeof(pei));
+        
         /* Parse the content of the received CER */
-        CHECK_FCT_DO( save_remote_CE_info(msg, peer, &ec, NULL), goto error_abort );
+        CHECK_FCT_DO( save_remote_CE_info(msg, peer, &pei, NULL), goto error_abort );
+        
+        /* Validate the realm if needed */
+        if (peer->p_hdr.info.config.pic_realm) {
+                size_t len = strlen(peer->p_hdr.info.config.pic_realm);
+                if (fd_os_almostcasecmp(peer->p_hdr.info.config.pic_realm, len, peer->p_hdr.info.runtime.pir_realm, peer->p_hdr.info.runtime.pir_realmlen)) {
+                        TRACE_DEBUG(INFO, "Rejected CER from peer '%s', realm mismatch with configured value (returning DIAMETER_UNKNOWN_PEER).\n", peer->p_hdr.info.pi_diamid);
+                        pei.pei_errcode = "DIAMETER_UNKNOWN_PEER"; /* maybe AVP_NOT_ALLOWED would be better fit? */
+                        goto error_abort;
+                }
+        }
         
         /* Validate the peer if needed */
@@ -773 +825 @@
                 if (res < 0) {
                         TRACE_DEBUG(INFO, "Rejected CER from peer '%s', validation failed (returning DIAMETER_UNKNOWN_PEER).\n", peer->p_hdr.info.pi_diamid);
-                        ec = "DIAMETER_UNKNOWN_PEER";
+                        pei.pei_errcode = "DIAMETER_UNKNOWN_PEER";
                         goto error_abort;
                 }
@@ -785 +837 @@
                 if (!got_common) {
                         TRACE_DEBUG(INFO, "No common application with peer '%s', sending DIAMETER_NO_COMMON_APPLICATION", peer->p_hdr.info.pi_diamid);
-                        ec = "DIAMETER_NO_COMMON_APPLICATION";
+                        pei.pei_errcode = "DIAMETER_NO_COMMON_APPLICATION";
                         fatal = 1;
                         goto error_abort;
@@ -835 +887 @@
                 if (!isi) {
                         TRACE_DEBUG(INFO, "No common security mechanism with '%s', sending DIAMETER_NO_COMMON_SECURITY", peer->p_hdr.info.pi_diamid);
-                        ec = "DIAMETER_NO_COMMON_SECURITY";
+                        pei.pei_errcode = "DIAMETER_NO_COMMON_SECURITY";
                         fatal = 1;
                         goto error_abort;
@@ -849 +901 @@
         CHECK_FCT( fd_msg_rescode_set(msg, "DIAMETER_SUCCESS", NULL, NULL, 0 ) );
         CHECK_FCT( add_CE_info(msg, peer->p_cnxctx, isi & PI_SEC_TLS_OLD, isi & PI_SEC_NONE) );
-#ifdef USE_CEA_BROADCAST
-        CHECK_FCT( fd_out_send(&msg, peer->p_cnxctx, peer, (isi & PI_SEC_TLS_OLD) ? FD_CNX_ORDERED : FD_CNX_BROADCAST) ); /* Broadcast in order to avoid further messages sent over a different stream be delivered first... */
-#else /* USE_CEA_BROADCAST */
         CHECK_FCT( fd_out_send(&msg, peer->p_cnxctx, peer, FD_CNX_ORDERED ) );
-#endif /* USE_CEA_BROADCAST */
         
         /* Handshake if needed */
@@ -878 +926 @@
                                 }  );
                 }
-                
+                tls_sync = 1;
         } else {
                 if ( ! fd_cnx_getTLS(peer->p_cnxctx) ) {
@@ -891 +939 @@
                 CHECK_FCT( fd_p_dw_reopen(peer) );
         } else {
-                fd_psm_change_state(peer, STATE_OPEN );
-                fd_psm_next_timeout(peer, 1, peer->p_hdr.info.config.pic_twtimer ?: fd_g_config->cnf_timer_tw);
+                if ((!tls_sync) && (fd_cnx_isMultichan(peer->p_cnxctx))) {
+                        fd_psm_change_state(peer, STATE_OPEN_NEW );
+                        /* send DWR */
+                        CHECK_FCT( fd_p_dw_timeout(peer) );
+                } else {
+
+                        fd_psm_change_state(peer, STATE_OPEN );
+                        fd_psm_next_timeout(peer, 1, peer->p_hdr.info.config.pic_twtimer ?: fd_g_config->cnf_timer_tw);
+                }
         }
         
@@ -898 +953 @@
 
 error_abort:
-        if (ec) {
-                /* Create the error message */
-                CHECK_FCT( fd_msg_new_answer_from_req ( fd_g_config->cnf_dict, &msg, MSGFL_ANSW_ERROR ) );
-
-                /* Set the error code */
-                CHECK_FCT( fd_msg_rescode_set(msg, ec, NULL, NULL, 1 ) );
-
-                /* msg now contains an answer message to send back */
-                CHECK_FCT_DO( fd_out_send(&msg, peer->p_cnxctx, peer, FD_CNX_ORDERED), /* In case of error the message has already been dumped */ );
+        if (pei.pei_errcode) {
+                /* Send the error */
+                receiver_reject(&peer->p_cnxctx, &msg, &pei);
         }
         
@@ -925 +974 @@
 int fd_p_ce_handle_newCER(struct msg ** msg, struct fd_peer * peer, struct cnxctx ** cnx, int valid)
 {
-        fd_cpu_flush_cache();
-        switch (peer->p_hdr.info.runtime.pir_state) {
+        struct fd_pei pei;
+        int cur_state = fd_peer_getstate(peer);
+        memset(&pei, 0, sizeof(pei));
+        
+        switch (cur_state) {
                 case STATE_CLOSED:
                         peer->p_receiver = *cnx;
@@ -960 +1012 @@
 
                                 /* Answer an ELECTION LOST to the receiver side and continue */
-                                receiver_reject(*cnx, msg, "ELECTION_LOST", "Please answer my CER instead, you won the election.");
-                                *cnx = NULL;
+                                pei.pei_errcode = "ELECTION_LOST";
+                                pei.pei_message = "Please answer my CER instead, you won the election.";
+                                receiver_reject(cnx, msg, &pei);
                         }
                         break;
 
                 default:
-                        receiver_reject(*cnx, msg, "DIAMETER_UNABLE_TO_COMPLY", "Invalid state to receive a new connection attempt");
-                        *cnx = NULL;
+                        pei.pei_errcode = "DIAMETER_UNABLE_TO_COMPLY"; /* INVALID COMMAND? in case of Capabilities-Updates? */
+                        pei.pei_message = "Invalid state to receive a new connection attempt.";
+                        receiver_reject(cnx, msg, &pei);
         }
                                 

libfdcore/p_cnx.c

@@ -36 +36 @@
 #include "fdcore-internal.h"
 
+
+/* TODO: change the behavior to handle properly forced ordering at beginning & end of OPEN state */
+
 /* This file contains code used by a peer state machine to initiate a connection to remote peer */
 
@@ -87 +90 @@
                 ret = getaddrinfo(peer->p_hdr.info.pi_diamid, NULL, &hints, &ai);
                 if (ret) {
-                        fd_log_debug("Unable to resolve address for peer '%s' (%s), aborting\n", peer->p_hdr.info.pi_diamid, gai_strerror(ret));
+                        TRACE_DEBUG(INFO, "Unable to resolve address for peer '%s' (%s), aborting\n", peer->p_hdr.info.pi_diamid, gai_strerror(ret));
                         if (ret != EAI_AGAIN)
                                 fd_psm_terminate( peer, NULL );
@@ -123 +126 @@
         /* Now check we have at least one address to attempt */
         if (FD_IS_LIST_EMPTY(&peer->p_hdr.info.pi_endpoints)) {
-                fd_log_debug("No address %savailable to connect to peer '%s', aborting\n", peer->p_hdr.info.config.pic_flags.pro3 ? "in the configured family " : "", peer->p_hdr.info.pi_diamid);
+                TRACE_DEBUG(INFO, "No address %savailable to connect to peer '%s', aborting\n", 
+                                        peer->p_hdr.info.config.pic_flags.pro3 ? "in the configured family " : "", peer->p_hdr.info.pi_diamid);
                 fd_psm_terminate( peer, NULL );
                 return 0;
@@ -219 +223 @@
         {
                 char buf[48];
-                sprintf(buf, "ConnTo:%.*s", (int)(sizeof(buf)) - 8, peer->p_hdr.info.pi_diamid);
+                snprintf(buf, sizeof(buf), "ConnTo:%s", peer->p_hdr.info.pi_diamid);
                 fd_log_threadname ( buf );
         }
@@ -247 +251 @@
 #ifndef DISABLE_SCTP                    
                         case IPPROTO_SCTP:
-                                cnx = fd_cnx_cli_connect_sctp((peer->p_hdr.info.config.pic_flags.pro3 == PI_P3_IP) ?: fd_g_config->cnf_flags.no_ip6, nc->port, &peer->p_hdr.info.pi_endpoints);
+                                cnx = fd_cnx_cli_connect_sctp((peer->p_hdr.info.config.pic_flags.pro3 == PI_P3_IP) ?: fd_g_config->cnf_flags.no_ip6, 
+                                                        nc->port, &peer->p_hdr.info.pi_endpoints);
                                 break;
 #endif /* DISABLE_SCTP */
@@ -260 +265 @@
                 pthread_testcancel();
                 
-        } while (!cnx); /* and until cancellation */
+        } while (!cnx); /* and until cancellation or all addresses attempted without success */
         
         /* Now, we have an established connection in cnx */
@@ -274 +279 @@
                         {
                                 /* Handshake failed ...  */
-                                fd_log_debug("TLS Handshake failed with peer '%s', resetting the connection\n", peer->p_hdr.info.pi_diamid);
+                                TRACE_DEBUG(INFO, "TLS Handshake failed with peer '%s', resetting the connection\n", peer->p_hdr.info.pi_diamid);
                                 fd_cnx_destroy(cnx);
                                 empty_connection_list(peer);

libfdcore/p_dp.c

@@ -38 +38 @@
 /* This file contains code to handle Disconnect Peer messages (DPR and DPA) */
 
+/* Delay to use before next reconnect attempt */
+int fd_p_dp_newdelay(struct fd_peer * peer) 
+{
+        int delay = peer->p_hdr.info.config.pic_tctimer ?: fd_g_config->cnf_timer_tc;
+        
+        switch (peer->p_hdr.info.runtime.pir_lastDC) {
+                case ACV_DC_REBOOTING:
+                default:
+                        /* We use TcTimer to attempt reconnection */
+                        break;
+                case ACV_DC_BUSY:
+                        /* No need to hammer the overloaded peer */
+                        delay *= 10;
+                        break;
+                case ACV_DC_NOT_FRIEND:
+                        /* He does not want to speak to us... let's retry a *lot* later maybe */
+                        delay *= 200;
+                        break;
+        }
+        return delay;
+}
+
 /* Handle a received message */
 int fd_p_dp_handle(struct msg ** msg, int req, struct fd_peer * peer)
@@ -44 +66 @@
         
         if (req) {
-                /* We received a DPR, save the Disconnect-Cause and terminate the connection */
+                /* We received a DPR, save the Disconnect-Cause and go to CLOSING_GRACE or terminate the connection */
                 struct avp * dc;
-                int delay = peer->p_hdr.info.config.pic_tctimer ?: fd_g_config->cnf_timer_tc;
                 
                 CHECK_FCT( fd_msg_search_avp ( *msg, fd_dict_avp_DC, &dc ));
                 if (dc) {
-                        /* Check the value is consistent with the saved one */
                         struct avp_hdr * hdr;
                         CHECK_FCT(  fd_msg_avp_hdr( dc, &hdr )  );
@@ -60 +80 @@
                         }
 
+                        /* save the cause */
                         peer->p_hdr.info.runtime.pir_lastDC = hdr->avp_value->u32;
-                        
-                        switch (hdr->avp_value->u32) {
-                                case ACV_DC_REBOOTING:
-                                default:
-                                        /* We use TcTimer to attempt reconnection */
-                                        break;
-                                case ACV_DC_BUSY:
-                                        /* No need to hammer the overloaded peer */
-                                        delay *= 10;
-                                        break;
-                                case ACV_DC_NOT_FRIEND:
-                                        /* He does not want to speak to us... let's retry a lot later maybe */
-                                        delay *= 200;
-                                        break;
-                        }
                 }
                 if (TRACE_BOOL(INFO)) {
                         if (dc) {
-                                struct dict_object * dictobj = NULL;
+                                struct dict_object * dictobj;
                                 struct dict_enumval_request er;
                                 memset(&er, 0, sizeof(er));
+                                
+                                /* prepare the request */
                                 CHECK_FCT( fd_dict_search( fd_g_config->cnf_dict, DICT_TYPE, TYPE_OF_AVP, fd_dict_avp_DC, &er.type_obj, ENOENT )  );
                                 er.search.enum_value.u32 = peer->p_hdr.info.runtime.pir_lastDC;
+                                
+                                /* Search the enum value */
                                 CHECK_FCT( fd_dict_search( fd_g_config->cnf_dict, DICT_ENUMVAL, ENUMVAL_BY_STRUCT, &er, &dictobj, 0 )  );
                                 if (dictobj) {
@@ -100 +110 @@
                 CHECK_FCT( fd_msg_rescode_set( *msg, "DIAMETER_SUCCESS", NULL, NULL, 1 ) );
                 
-                /* Move to CLOSING state to failover outgoing messages (and avoid failing the DPA...) */
+                /* Move to CLOSING state to failover outgoing messages (and avoid failing over the DPA...) */
                 CHECK_FCT( fd_psm_change_state(peer, STATE_CLOSING) );
                 
@@ -106 +116 @@
                 CHECK_FCT( fd_out_send( msg, NULL, peer, FD_CNX_ORDERED) );
                 
-                /* Move to CLOSED state */
-                fd_psm_cleanup(peer, 0);
-                
-                /* Reset the timer for next connection attempt -- we'll retry sooner or later depending on the disconnection cause */
-                fd_psm_next_timeout(peer, 1, delay);
-                
+                if (fd_cnx_isMultichan(peer->p_cnxctx)) {
+                        /* There is a possibililty that messages are still in the pipe coming here, so let's grace for 1 second */
+                        CHECK_FCT( fd_psm_change_state(peer, STATE_CLOSING_GRACE) );
+                        fd_psm_next_timeout(peer, 0, 1);
+                        
+                } else {
+                        /* Move to CLOSED state */
+                        fd_psm_cleanup(peer, 0);
+
+                        /* Reset the timer for next connection attempt -- we'll retry sooner or later depending on the disconnection cause */
+                        fd_psm_next_timeout(peer, 1, fd_p_dp_newdelay(peer));
+                }
         } else {
                 /* We received a DPA */
-                fd_cpu_flush_cache();
-                if (peer->p_hdr.info.runtime.pir_state != STATE_CLOSING) {
-                        TRACE_DEBUG(INFO, "Ignoring DPA received in state %s", STATE_STR(peer->p_hdr.info.runtime.pir_state));
+                int curstate = fd_peer_getstate(peer);
+                if (curstate != STATE_CLOSING) {
+                        TRACE_DEBUG(INFO, "Ignoring DPA received in state %s", STATE_STR(curstate));
                 }
                         
                 /* In theory, we should control the Result-Code AVP. But since we will not go back to OPEN state here anyway, let's skip it */
+                
+                /* TODO("Control Result-Code in the DPA") */
                 CHECK_FCT_DO( fd_msg_free( *msg ), /* continue */ );
                 *msg = NULL;
                 
-                /* The calling function handles cleaning the PSM and terminating the peer since we return in CLOSING state */
+                if (fd_cnx_isMultichan(peer->p_cnxctx)) {
+                        CHECK_FCT( fd_psm_change_state(peer, STATE_CLOSING_GRACE) );
+                        fd_psm_next_timeout(peer, 0, 1);
+                        peer->p_flags.pf_localterm = 1;
+                }
+                /* otherwise, return in CLOSING state, the psm will handle it */
         }
         
@@ -153 +176 @@
         CHECK_FCT( fd_dict_search( fd_g_config->cnf_dict, DICT_TYPE, TYPE_OF_AVP, fd_dict_avp_DC, &er.type_obj, ENOENT )  );
         er.search.enum_name = reason ?: "REBOOTING";
-        CHECK_FCT( fd_dict_search( fd_g_config->cnf_dict, DICT_ENUMVAL, ENUMVAL_BY_STRUCT, &er, &dictobj, ENOENT )  );
+        CHECK_FCT_DO( fd_dict_search( fd_g_config->cnf_dict, DICT_ENUMVAL, ENUMVAL_BY_STRUCT, &er, &dictobj, ENOENT ), { ASSERT(0); /* internal error: unknown reason */ }  );
         CHECK_FCT( fd_dict_getval( dictobj, &er.search ) );
         

libfdcore/p_dw.c

@@ -38 +38 @@
 /* This file contains code to handle Device Watchdog messages (DWR and DWA) */
 
-/* Check the value of Origin-State-Id is consistent in a DWR or  DWA -- we just log if it is not the case */
-static void check_state_id(struct msg * msg, struct fd_peer * peer)
+/* Check the value of Origin-State-Id is consistent in a DWR or DWA -- we return an error otherwise */
+static int check_state_id(struct msg * msg, struct fd_peer * peer)
 {
         struct avp * osi;
+        
         /* Check if the request contains the Origin-State-Id */
-        CHECK_FCT_DO( fd_msg_search_avp ( msg, fd_dict_avp_OSI, &osi ), return );
+        CHECK_FCT( fd_msg_search_avp ( msg, fd_dict_avp_OSI, &osi ) );
         if (osi) {
                 /* Check the value is consistent with the saved one */
                 struct avp_hdr * hdr;
-                CHECK_FCT_DO(  fd_msg_avp_hdr( osi, &hdr ), return  );
+                CHECK_FCT(  fd_msg_avp_hdr( osi, &hdr )  );
                 if (hdr->avp_value == NULL) {
                         /* This is a sanity check */
@@ -56 +57 @@
 
                 if (peer->p_hdr.info.runtime.pir_orstate != hdr->avp_value->u32) {
-                        fd_log_debug("Received a new Origin-State-Id from peer %s! (%x / %x)\n", 
+                        TRACE_DEBUG(INFO, "Received a new Origin-State-Id from peer '%s'! (%x -> %x); resetting the connection.\n", 
                                 peer->p_hdr.info.pi_diamid, 
-                                hdr->avp_value->u32,
-                                peer->p_hdr.info.runtime.pir_orstate );
+                                peer->p_hdr.info.runtime.pir_orstate,
+                                hdr->avp_value->u32 );
+                        return EINVAL;
                 }
         }
+        return 0;
 }
 
@@ -92 +95 @@
         
         /* Check the value of OSI for information */
-        check_state_id(*msg, peer);
+        CHECK_FCT( check_state_id(*msg, peer) );
         
         if (req) {
@@ -102 +105 @@
                 
         } else {
-                /* Just discard the DWA */
+                /* Discard the DWA */
                 CHECK_FCT_DO( fd_msg_free(*msg), /* continue */ );
                 *msg = NULL;
@@ -123 +126 @@
         
         /* If we are in REOPEN state, increment the counter */
-        fd_cpu_flush_cache();
-        if (peer->p_hdr.info.runtime.pir_state == STATE_REOPEN) {
+        if (fd_peer_getstate(peer) == STATE_REOPEN) {
                 peer->p_flags.pf_reopen_cnt += 1;
                 

libfdcore/p_expiry.c

@@ -59 +59 @@
                 
                 for (li = fd_g_peers.next; li != &fd_g_peers; li = li->next) {
-                        struct fd_peer * peer = (struct fd_peer *)li;
-                        
-                        fd_cpu_flush_cache();
-                        if (peer->p_hdr.info.runtime.pir_state != STATE_ZOMBIE)
+                        struct fd_peer * peer = (struct fd_peer *)li->o;
+                        
+                        if (fd_peer_getstate(peer) != STATE_ZOMBIE)
                                 continue;
                         
@@ -78 +77 @@
                 /* Now delete peers that are in the purge list */
                 while (!FD_IS_LIST_EMPTY(&purge)) {
-                        struct fd_peer * peer = (struct fd_peer *)(purge.next);
+                        struct fd_peer * peer = (struct fd_peer *)(purge.next->o);
                         fd_list_unlink(&peer->p_hdr.chain);
                         TRACE_DEBUG(INFO, "Garbage Collect: delete zombie peer '%s'", peer->p_hdr.info.pi_diamid);
@@ -105 +104 @@
                 struct fd_peer * first;
                 
-                /* Check if there are expiring sessions available */
+                /* Check if there are expiring peers available */
                 if (FD_IS_LIST_EMPTY(&exp_list)) {
                         /* Just wait for a change or cancelation */
@@ -183 +182 @@
                 
                 /* update the p_exp_timer value */
-                CHECK_SYS(  clock_gettime(CLOCK_REALTIME, &peer->p_exp_timer)  );
+                CHECK_SYS_DO(  clock_gettime(CLOCK_REALTIME, &peer->p_exp_timer), { ASSERT(0); }  );
                 peer->p_exp_timer.tv_sec += peer->p_hdr.info.config.pic_lft;
                 

libfdcore/p_out.c

@@ -109 +109 @@
         {
                 char buf[48];
-                sprintf(buf, "OUT/%.*s", (int)sizeof(buf) - 5, peer->p_hdr.info.pi_diamid);
+                snprintf(buf, sizeof(buf), "OUT/%s", peer->p_hdr.info.pi_diamid);
                 fd_log_threadname ( buf );
         }
@@ -149 +149 @@
         CHECK_PARAMS( msg && *msg && (cnx || (peer && peer->p_cnxctx)));
         
-        fd_cpu_flush_cache();
-        if (peer && (peer->p_hdr.info.runtime.pir_state == STATE_OPEN)) {
+        if (fd_peer_getstate(peer) == STATE_OPEN) {
                 /* Normal case: just queue for the out thread to pick it up */
                 CHECK_FCT( fd_fifo_post(peer->p_tosend, msg) );

libfdcore/p_psm.c

@@ -36 +36 @@
 #include "fdcore-internal.h"
 
+/*
+This file implement a Peer State Machine which is a mix of:
+ - the state machine described in rfc3588bis
+ - the state machine described in rfc3539#section-3.4
+ - the following observations.
+ 
+The delivery of Diameter messages must not always be unordered: order is important at
+begining and end of a connection lifetime. It means we need agility to
+switch between "ordering enforced" and "ordering not enforced to counter
+HotLB" modes of operation.
+
+The connection state machine represented in RFC3588 (and rfc3588bis) is
+incomplete, because it lacks the SUSPECT state and the 3 DWR/DWA
+exchanges (section 5.1) when the peer recovers from this state.
+Personnally I don't see the rationale for exchanging 3 messages (why 3?)
+but, if we require at least 1 DWR/DWA exchange to be always performed
+after the CER/CEA exchange (and initiated by the peer that sent the
+CEA), we have a simple way to deal with our ordering problem, as resumed
+bellow. Peers are: [i]nitiator, [r]esponder.
+ (1) [i] SCTP connection attempt.
+ (2) [r] accept the connection.
+ (3) [i,r] (if secure port) DTLS handshake, close on failure.
+ (4) [i] Send CER
+ (5) [r] Receive CER, send CEA using stream 0, flag "unordered" cleared.
+       [r] Immediately send a DWR after the CEA, also using stream 0,
+flag "unordered" cleared.
+       [r] Move to STATE_OPEN_NEW state -- equivalent to OPEN except
+that all messages are sent ordered at the moment.
+ (6) [i] receive CEA, move to OPEN state. All messages can be sent
+unordered in OPEN state.
+       [i] As per normal operation, reply with DWA to the DWR.
+ (7) [r] Upon reception of the DWA, move to OPEN state, messages can be
+sent unordered from this point.
+
+Note about (5) and (6): if the Diameter Identity received in CER or CEA
+does not match the credentials from the certificate presented during
+DTLS handshake, we may need to specify a path of clean disconnection
+(not blocking the remote peer waiting for something).
+
+This proposed mechanism removes the problem of application messages
+received before the CEA by the initiator. Note that if the "old" inband
+TLS handshake is used, this handshake plays the same synchronization
+role than the new DWR/DWA, which becomes useless.
+
+
+The other time where ordering is important is by the end of connection
+lifetime, when one peer is shutting down the link for some reason
+(reboot, overload, no activity, etc...). In case of unordered delivery,
+we may have:
+- peer A sends an application message followed by a DPR. Peer B receives
+the DPR first and tears down the connection. Application message is lost.
+- Peer B sends an application message, then receives a DPR and answers a
+DPA. Peer A receives the DPA before the application message. The
+application message is lost.
+
+This situation is actually quite possible because DPR/DPA messages are
+very short, while application messages can be quite large. Therefore,
+they require much more time to deliver.
+
+I really cannot see a way to counter this effect by using the ordering
+of the messages, except by applying a timer (state STATE_CLOSING_GRACE).
+
+However, this problem must be balanced with the fact that the message
+that is lost will be in many cases sent again as the failover mechanism
+specifies.
+*/
+
 /* The actual declaration of peer_state_str */
 DECLARE_STATE_STR();
@@ -101 +168 @@
                 return 0;
         }
+        
         /* Insert in the active peers list */
         CHECK_POSIX( pthread_rwlock_wrlock(&fd_g_activ_peers_rw) );
         for (li = fd_g_activ_peers.next; li != &fd_g_activ_peers; li = li->next) {
                 struct fd_peer * next_p = (struct fd_peer *)li->o;
-                int cmp = strcmp(peer->p_hdr.info.pi_diamid, next_p->p_hdr.info.pi_diamid);
+                int cmp = fd_os_cmp(peer->p_hdr.info.pi_diamid, peer->p_hdr.info.pi_diamidlen, 
+                                        next_p->p_hdr.info.pi_diamid, next_p->p_hdr.info.pi_diamidlen);
                 if (cmp < 0)
                         break;
@@ -115 +184 @@
         if (peer->p_cb) {
                 TRACE_DEBUG(FULL, "Calling add callback for peer %s", peer->p_hdr.info.pi_diamid);
-                (*peer->p_cb)(&peer->p_hdr.info, peer->p_cb_data);
+                (*peer->p_cb)(&peer->p_hdr.info, peer->p_cb_data); /* TODO: do this in a separate detached thread? */
                 peer->p_cb = NULL;
                 peer->p_cb_data = NULL;
@@ -178 +247 @@
 }
 
+/* Read state */
+int fd_peer_get_state(struct peer_hdr *peer)
+{
+        int ret;
+        
+        struct fd_peer * p = (struct fd_peer *)peer;
+        
+        if (!CHECK_PEER(p))
+                return -1;
+        
+        CHECK_POSIX_DO( pthread_mutex_lock(&p->p_state_mtx), return -1 );
+        ret = p->p_state;
+        CHECK_POSIX_DO( pthread_mutex_unlock(&p->p_state_mtx), return -1 );
+        
+        return ret;
+}
+
 
 /* Change state */
@@ -186 +272 @@
         TRACE_ENTRY("%p %d(%s)", peer, new_state, STATE_STR(new_state));
         CHECK_PARAMS( CHECK_PEER(peer) );
-        fd_cpu_flush_cache();
-        old = peer->p_hdr.info.runtime.pir_state;
+        
+        old = fd_peer_getstate(peer);
         if (old == new_state)
                 return 0;
@@ -196 +282 @@
                         peer->p_hdr.info.pi_diamid);
         
-        peer->p_hdr.info.runtime.pir_state = new_state;
-        fd_cpu_flush_cache();
+        
+        CHECK_POSIX( pthread_mutex_lock(&peer->p_state_mtx) );
+        peer->p_state = new_state;
+        CHECK_POSIX( pthread_mutex_unlock(&peer->p_state_mtx) );
         
         if (old == STATE_OPEN) {
@@ -255 +343 @@
 {
         /* Move to CLOSED state: failover messages, stop OUT thread, unlink peer from active list */
-        fd_cpu_flush_cache();
-        if (peer->p_hdr.info.runtime.pir_state != STATE_ZOMBIE) {
+        if (fd_peer_getstate(peer) != STATE_ZOMBIE) {
                 CHECK_FCT_DO( fd_psm_change_state(peer, STATE_CLOSED), /* continue */ );
         }
@@ -285 +372 @@
         struct fd_peer * peer = (struct fd_peer *)arg;
         CHECK_PARAMS_DO( CHECK_PEER(peer), return );
-        peer->p_hdr.info.runtime.pir_state = STATE_ZOMBIE;
-        fd_cpu_flush_cache();
+        CHECK_POSIX_DO( pthread_mutex_lock(&peer->p_state_mtx), );
+        peer->p_state = STATE_ZOMBIE;
+        CHECK_POSIX_DO( pthread_mutex_unlock(&peer->p_state_mtx), );
         return;
 }
@@ -298 +386 @@
         size_t ev_sz;
         void * ev_data;
+        int cur_state;
         
         CHECK_PARAMS_DO( CHECK_PEER(peer), ASSERT(0) );
@@ -306 +395 @@
         {
                 char buf[48];
-                sprintf(buf, "PSM/%.*s", (int)sizeof(buf) - 5, peer->p_hdr.info.pi_diamid);
+                snprintf(buf, sizeof(buf), "PSM/%s", peer->p_hdr.info.pi_diamid);
                 fd_log_threadname ( buf );
         }
         
         /* The state machine starts in CLOSED state */
-        peer->p_hdr.info.runtime.pir_state = STATE_CLOSED;
+        CHECK_POSIX_DO( pthread_mutex_lock(&peer->p_state_mtx), goto psm_end );
+        peer->p_state = STATE_CLOSED;
+        CHECK_POSIX_DO( pthread_mutex_unlock(&peer->p_state_mtx), goto psm_end );
 
         /* Wait that the PSM are authorized to start in the daemon */
@@ -326 +417 @@
         /* Get next event */
         TRACE_DEBUG(FULL, "'%s' in state '%s' waiting for next event.",
-                        peer->p_hdr.info.pi_diamid, STATE_STR(peer->p_hdr.info.runtime.pir_state));
+                        peer->p_hdr.info.pi_diamid, STATE_STR(fd_peer_getstate(peer)));
         CHECK_FCT_DO( fd_event_timedget(peer->p_events, &peer->p_psm_timer, FDEVP_PSM_TIMEOUT, &event, &ev_sz, &ev_data), goto psm_end );
+        
+        cur_state = fd_peer_getstate(peer);
+        if (cur_state == -1)
+                goto psm_end;
+        
         TRACE_DEBUG(FULL, "'%s'\t<-- '%s'\t(%p,%zd)\t'%s'",
-                        STATE_STR(peer->p_hdr.info.runtime.pir_state),
+                        STATE_STR(cur_state),
                         fd_pev_str(event), ev_data, ev_sz,
                         peer->p_hdr.info.pi_diamid);
@@ -336 +432 @@
 
         /* The following states are impossible */
-        ASSERT( peer->p_hdr.info.runtime.pir_state != STATE_NEW );
-        ASSERT( peer->p_hdr.info.runtime.pir_state != STATE_ZOMBIE );
-        ASSERT( peer->p_hdr.info.runtime.pir_state != STATE_OPEN_HANDSHAKE ); /* because it should exist only between two loops */
+        ASSERT( cur_state != STATE_NEW );
+        ASSERT( cur_state != STATE_ZOMBIE );
+        ASSERT( cur_state != STATE_OPEN_HANDSHAKE ); /* because it should exist only between two loops */
 
         /* Purge invalid events */
         if (!CHECK_PEVENT(event)) {
                 TRACE_DEBUG(INFO, "Invalid event received in PSM '%s' : %d", peer->p_hdr.info.pi_diamid, event);
+                ASSERT(0); /* we should investigate this situation */
                 goto psm_loop;
         }
@@ -354 +451 @@
         /* Requests to terminate the peer object */
         if (event == FDEVP_TERMINATE) {
-                switch (peer->p_hdr.info.runtime.pir_state) {
+                switch (cur_state) {
                         case STATE_OPEN:
+                        case STATE_OPEN_NEW:
                         case STATE_REOPEN:
-                                /* We cannot just close the conenction, we have to send a DPR first */
+                                /* We cannot just close the connection, we have to send a DPR first */
                                 CHECK_FCT_DO( fd_p_dp_initiate(peer, ev_data), goto psm_end );
                                 goto psm_loop;
@@ -363 +461 @@
                         /*      
                         case STATE_CLOSING:
+                        case STATE_CLOSING_GRACE:
                         case STATE_WAITCNXACK:
                         case STATE_WAITCNXACK_ELEC:
@@ -380 +479 @@
                 struct msg_hdr * hdr;
                 
-                /* If the current state does not allow receiving messages, just drop it */
-                if (peer->p_hdr.info.runtime.pir_state == STATE_CLOSED) {
-                        TRACE_DEBUG(FULL, "Purging message in queue while in CLOSED state (%zdb)", ev_sz);
-                        free(ev_data);
-                        goto psm_loop;
-                }
-                
                 /* Parse the received buffer */
                 CHECK_FCT_DO( fd_msg_parse_buffer( (void *)&ev_data, ev_sz, &msg), 
@@ -396 +488 @@
                         } );
                 
+                /* If the current state does not allow receiving messages, just drop it */
+                if (cur_state == STATE_CLOSED) {
+                        /* In such case, just discard the message */
+                        fd_msg_log( FD_MSG_LOG_DROPPED, msg, "Purged from peer '%s''s queue (CLOSED state).", peer->p_hdr.info.pi_diamid );
+                        fd_msg_free(msg);
+                        goto psm_loop;
+                }
+                
                 /* Log incoming message */
-                fd_msg_log( FD_MSG_LOG_RECEIVED, msg, "Received %zdb from '%s'", ev_sz, peer->p_hdr.info.pi_diamid );
+                fd_msg_log( FD_MSG_LOG_RECEIVED, msg, "Received %zdb from '%s' (%s)", ev_sz, peer->p_hdr.info.pi_diamid, STATE_STR(cur_state) );
         
                 /* Extract the header */
@@ -417 +517 @@
                 }
                 
+                if (cur_state == STATE_OPEN_NEW) {
+                        /* OK, we have received something, so the connection is supposedly now in OPEN state at the remote site */
+                        fd_psm_change_state(peer, STATE_OPEN );
+                }
+                
                 /* Now handle non-link-local messages */
                 if (fd_msg_is_routable(msg)) {
-                        switch (peer->p_hdr.info.runtime.pir_state) {
+                        switch (cur_state) {
                                 /* To maximize compatibility -- should not be a security issue here */
                                 case STATE_REOPEN:
                                 case STATE_SUSPECT:
                                 case STATE_CLOSING:
+                                case STATE_CLOSING_GRACE:
                                         TRACE_DEBUG(FULL, "Accepted a message while not in OPEN state... ");
                                 /* The standard situation : */
+                                case STATE_OPEN_NEW:
                                 case STATE_OPEN:
                                         /* We received a valid routable message, update the expiry timer */
@@ -431 +538 @@
 
                                         /* Set the message source and add the Route-Record */
-                                        CHECK_FCT_DO( fd_msg_source_set( msg, peer->p_hdr.info.pi_diamid, 1, fd_g_config->cnf_dict ), goto psm_end);
+                                        CHECK_FCT_DO( fd_msg_source_set( msg, peer->p_hdr.info.pi_diamid, peer->p_hdr.info.pi_diamidlen, 1, fd_g_config->cnf_dict ), goto psm_end);
 
                                         /* Requeue to the global incoming queue */
@@ -437 +544 @@
 
                                         /* Update the peer timer (only in OPEN state) */
-                                        if ((peer->p_hdr.info.runtime.pir_state == STATE_OPEN) && (!peer->p_flags.pf_dw_pending)) {
+                                        if ((cur_state == STATE_OPEN) && (!peer->p_flags.pf_dw_pending)) {
                                                 fd_psm_next_timeout(peer, 1, peer->p_hdr.info.config.pic_twtimer ?: fd_g_config->cnf_timer_tw);
                                         }
@@ -449 +556 @@
                                 default:
                                         /* In such case, just discard the message */
-                                        fd_msg_log( FD_MSG_LOG_DROPPED, msg, "Received from peer '%s' while connection was not in OPEN state.", peer->p_hdr.info.pi_diamid );
+                                        fd_msg_log( FD_MSG_LOG_DROPPED, msg, "Received from peer '%s' while connection was not in state %s.", peer->p_hdr.info.pi_diamid, STATE_STR(cur_state) );
                                         fd_msg_free(msg);
                         }
@@ -485 +592 @@
                         case CC_DISCONNECT_PEER:
                                 CHECK_FCT_DO( fd_p_dp_handle(&msg, (hdr->msg_flags & CMD_FLAG_REQUEST), peer), goto psm_reset );
-                                if (peer->p_hdr.info.runtime.pir_state == STATE_CLOSING)
+                                if (fd_peer_getstate(peer) == STATE_CLOSING)
                                         goto psm_end;
+
                                 break;
                         
@@ -494 +602 @@
                         
                         default:
-                                /* Unknown / unexpected / invalid message */
+                                /* Unknown / unexpected / invalid message -- but validated by our dictionary */
                                 TRACE_DEBUG(INFO, "Invalid non-routable command received: %u.", hdr->msg_code);
                                 if (hdr->msg_flags & CMD_FLAG_REQUEST) {
@@ -502 +610 @@
 
                                                 /* Set the error code */
-                                                CHECK_FCT_DO( fd_msg_rescode_set(msg, "DIAMETER_INVALID_HDR_BITS", NULL, NULL, 1 ), break );
+                                                CHECK_FCT_DO( fd_msg_rescode_set(msg, "DIAMETER_COMMAND_UNSUPPORTED", "Or maybe the P-bit or application Id are erroneous.", NULL, 1 ), break );
 
                                                 /* Send the answer */
@@ -509 +617 @@
                                 } else {
                                         /* We did ASK for it ??? */
-                                        fd_log_debug("Invalid PXY flag in answer header ?\n");
+                                        TRACE_DEBUG(INFO, "Received answer with erroneous 'is_routable' result...");
                                 }
                                 
@@ -531 +639 @@
         /* The connection object is broken */
         if (event == FDEVP_CNX_ERROR) {
-                switch (peer->p_hdr.info.runtime.pir_state) {
+                switch (cur_state) {
                         case STATE_WAITCNXACK_ELEC:
                                 /* Abort the initiating side */
@@ -541 +649 @@
                         case STATE_WAITCEA:
                         case STATE_OPEN:
+                        case STATE_OPEN_NEW:
                         case STATE_REOPEN:
                         case STATE_WAITCNXACK:
@@ -558 +667 @@
                                 goto psm_end;
                                 
+                        case STATE_CLOSING_GRACE:
+                                if (peer->p_flags.pf_localterm) /* initiated here */
+                                        goto psm_end;
+                                
+                                fd_psm_cleanup(peer, 0);
+                                
+                                /* Reset the timer for next connection attempt */
+                                fd_psm_next_timeout(peer, 1, fd_p_dp_newdelay(peer));
+                                goto psm_loop;
                 }
                 goto psm_loop;
@@ -616 +734 @@
                 peer->p_ini_thr = (pthread_t)NULL;
                 
-                switch (peer->p_hdr.info.runtime.pir_state) {
+                switch (cur_state) {
                         case STATE_WAITCNXACK_ELEC:
                         case STATE_WAITCNXACK:
@@ -624 +742 @@
                         default:
                                 /* Just abort the attempt and continue */
-                                TRACE_DEBUG(FULL, "Connection attempt successful but current state is %s, closing...", STATE_STR(peer->p_hdr.info.runtime.pir_state));
+                                TRACE_DEBUG(FULL, "Connection attempt successful but current state is %s, closing... (too slow?)", STATE_STR(cur_state));
                                 fd_cnx_destroy(cnx);
                 }
@@ -638 +756 @@
                 peer->p_ini_thr = (pthread_t)NULL;
                 
-                switch (peer->p_hdr.info.runtime.pir_state) {
+                switch (cur_state) {
                         case STATE_WAITCNXACK_ELEC:
                                 /* Abort the initiating side */
@@ -653 +771 @@
                         default:
                                 /* Just ignore */
-                                TRACE_DEBUG(FULL, "Connection attempt failed but current state is %s, ignoring...", STATE_STR(peer->p_hdr.info.runtime.pir_state));
+                                TRACE_DEBUG(FULL, "Connection attempt failed but current state is %s, ignoring...", STATE_STR(cur_state));
                 }
                 
@@ -661 +779 @@
         /* The timeout for the current state has been reached */
         if (event == FDEVP_PSM_TIMEOUT) {
-                switch (peer->p_hdr.info.runtime.pir_state) {
+                switch (cur_state) {
                         case STATE_OPEN:
                         case STATE_REOPEN:
+                        case STATE_OPEN_NEW:
                                 CHECK_FCT_DO( fd_p_dw_timeout(peer), goto psm_end );
                                 goto psm_loop;
@@ -676 +795 @@
                                 /* Mark the connection problem */
                                 peer->p_flags.pf_cnx_pb = 1;
-                                
                         case STATE_CLOSING:
                         case STATE_WAITCNXACK:
@@ -683 +801 @@
                                 fd_psm_next_timeout(peer, 1, peer->p_hdr.info.config.pic_tctimer ?: fd_g_config->cnf_timer_tc);
                                 goto psm_reset;
+                                
+                        case STATE_CLOSING_GRACE:
+                                /* The grace period is completed, now close */
+                                if (peer->p_flags.pf_localterm)
+                                        goto psm_end;
+                                
+                                fd_psm_cleanup(peer, 0);
+                                /* Reset the timer for next connection attempt */
+                                fd_psm_next_timeout(peer, 1, fd_p_dp_newdelay(peer));
+                                goto psm_loop;
                                 
                         case STATE_WAITCNXACK_ELEC:
@@ -697 +825 @@
         
         /* Default action : the handling has not yet been implemented. [for debug only] */
-        TRACE_DEBUG(INFO, "Missing handler in PSM for '%s'\t<-- '%s'", STATE_STR(peer->p_hdr.info.runtime.pir_state), fd_pev_str(event));
+        TRACE_DEBUG(INFO, "Missing handler in PSM for '%s'\t<-- '%s'", STATE_STR(cur_state), fd_pev_str(event));
 psm_reset:
         if (peer->p_flags.pf_delete)
@@ -707 +835 @@
         fd_psm_cleanup(peer, 1);
         TRACE_DEBUG(INFO, "'%s'\t-> STATE_ZOMBIE (terminated)\t'%s'",
-                        STATE_STR(peer->p_hdr.info.runtime.pir_state),
+                        STATE_STR(fd_peer_getstate(peer)),
                         peer->p_hdr.info.pi_diamid);
         pthread_cleanup_pop(1); /* set STATE_ZOMBIE */
-        fd_cpu_flush_cache();
         peer->p_psm = (pthread_t)NULL;
         pthread_detach(pthread_self());
@@ -726 +853 @@
         
         /* Check the peer and state are OK */
-        CHECK_PARAMS( CHECK_PEER(peer) && (peer->p_hdr.info.runtime.pir_state == STATE_NEW) );
+        CHECK_PARAMS( fd_peer_getstate(peer) == STATE_NEW );
         
         /* Create the FIFO for events */
@@ -744 +871 @@
         CHECK_PARAMS( CHECK_PEER(peer) );
         
-        fd_cpu_flush_cache();
-        if (peer->p_hdr.info.runtime.pir_state != STATE_ZOMBIE) {
+        if (fd_peer_getstate(peer) != STATE_ZOMBIE) {
                 CHECK_FCT( fd_event_send(peer->p_events, FDEVP_TERMINATE, 0, reason) );
         } else {

libfdcore/p_sr.c

@@ -36 +36 @@
 #include "fdcore-internal.h"
 
-#ifndef SR_DEBUG_LVL
-#define SR_DEBUG_LVL ANNOYING
-#endif /* SR_DEBUG_LVL */
-
 /* Structure to store a sent request */
 struct sentreq {
@@ -69 +65 @@
         struct fd_list * li;
         struct timespec now;
-        if (!TRACE_BOOL(SR_DEBUG_LVL))
+        
+        if (!TRACE_BOOL(ANNOYING))
                 return;
+        
+        fd_log_debug("%sSentReq list @%p:\n", text, srlist);
+        
         CHECK_SYS_DO( clock_gettime(CLOCK_REALTIME, &now), );
-        fd_log_debug("%sSentReq list @%p:\n", text, srlist);
+        
         for (li = srlist->next; li != srlist; li = li->next) {
                 struct sentreq * sr = (struct sentreq *)li;
                 uint32_t * nexthbh = li->o;
-                fd_log_debug(" - Next req (%x): [since %ld.%06ld sec]\n", *nexthbh, 
+                
+                fd_log_debug(" - Next req (hbh:%x): [since %ld.%06ld sec]\n", *nexthbh, 
                         (now.tv_nsec >= sr->added_on.tv_nsec) ? (now.tv_sec - sr->added_on.tv_sec) : (now.tv_sec - sr->added_on.tv_sec - 1),
                         (now.tv_nsec >= sr->added_on.tv_nsec) ? (now.tv_nsec - sr->added_on.tv_nsec) / 1000 : (now.tv_nsec - sr->added_on.tv_nsec + 1000000000) / 1000);
-                fd_msg_dump_one(SR_DEBUG_LVL + 1, sr->req);
+                
+                fd_msg_dump_one(ANNOYING + 1, sr->req);
         }
 }
@@ -117 +119 @@
 }
 
-/* thread that handles messages expiring. The thread is started / cancelled only when needed */
+/* thread that handles messages expiring. The thread is started only when needed */
 static void * sr_expiry_th(void * arg) {
         struct sr_list * srlist = arg;
@@ -129 +131 @@
         {
                 char buf[48];
-                sprintf(buf, "ReqExp/%.*s", (int)sizeof(buf) - 8, ((struct fd_peer *)(srlist->exp.o))->p_hdr.info.pi_diamid);
+                snprintf(buf, sizeof(buf), "ReqExp/%s", ((struct fd_peer *)(srlist->exp.o))->p_hdr.info.pi_diamid);
                 fd_log_threadname ( buf );
         }
@@ -313 +315 @@
                 } else {
                         /* Just free the request. */
-                        fd_msg_log( FD_MSG_LOG_DROPPED, sr->req, "Local message discarded during failover" );
+                        fd_msg_log( FD_MSG_LOG_DROPPED, sr->req, "Sent & unanswered local message discarded during failover." );
                         CHECK_FCT_DO(fd_msg_free(sr->req), /* Ignore */);
                 }

libfdcore/peers.c

@@ -73 +73 @@
         
         p->p_eyec = EYEC_PEER;
+        CHECK_POSIX( pthread_mutex_init(&p->p_state_mtx, NULL) );
+        
         fd_list_init(&p->p_actives, p);
         fd_list_init(&p->p_expiry, p);
@@ -94 +96 @@
         struct fd_list * li;
         int ret = 0;
+        
         TRACE_ENTRY("%p %p %p %p", info, orig_dbg, cb, cb_data);
         CHECK_PARAMS(info && info->pi_diamid);
         
+        if (info->config.pic_realm) {
+                if (!fd_os_is_valid_DiameterIdentity((os0_t)info->config.pic_realm, strlen(info->config.pic_realm))) {
+                        TRACE_DEBUG(INFO, "'%s' is not a valid DiameterIdentity.", info->config.pic_realm);
+                        return EINVAL;
+                }
+        }
+        
         /* Create a structure to contain the new peer information */
         CHECK_FCT( fd_peer_alloc(&p) );
         
         /* Copy the informations from the parameters received */
-        CHECK_MALLOC( p->p_hdr.info.pi_diamid = strdup(info->pi_diamid) );
+        p->p_hdr.info.pi_diamid = info->pi_diamid;
+        CHECK_FCT( fd_os_validate_DiameterIdentity(&p->p_hdr.info.pi_diamid, &p->p_hdr.info.pi_diamidlen, 1) );
         
         memcpy( &p->p_hdr.info.config, &info->config, sizeof(p->p_hdr.info.config) );
+        
         /* Duplicate the strings if provided */
         if (info->config.pic_realm) {
@@ -109 +121 @@
         }
         if (info->config.pic_priority) {
-                CHECK_MALLOC( p->p_hdr.info.config.pic_realm = strdup(info->config.pic_priority) );
+                CHECK_MALLOC( p->p_hdr.info.config.pic_priority = strdup(info->config.pic_priority) );
         }
         
@@ -124 +136 @@
                 CHECK_MALLOC( p->p_dbgorig = strdup(orig_dbg) );
         } else {
-                CHECK_MALLOC( p->p_dbgorig = strdup("unknown") );
+                CHECK_MALLOC( p->p_dbgorig = strdup("unspecified") );
         }
         p->p_cb = cb;
@@ -134 +146 @@
         for (li = fd_g_peers.next; li != &fd_g_peers; li = li->next) {
                 struct fd_peer * next = (struct fd_peer *)li;
-                int cmp = strcasecmp( p->p_hdr.info.pi_diamid, next->p_hdr.info.pi_diamid );
+                int cmp = fd_os_almostcasecmp( p->p_hdr.info.pi_diamid, p->p_hdr.info.pi_diamidlen, 
+                                                next->p_hdr.info.pi_diamid, next->p_hdr.info.pi_diamidlen );
                 if (cmp > 0)
                         continue;
@@ -162 +175 @@
 
 /* Search for a peer */
-int fd_peer_getbyid( char * diamid, struct peer_hdr ** peer )
+int fd_peer_getbyid( DiamId_t diamid, size_t diamidlen, int igncase, struct peer_hdr ** peer )
 {
         struct fd_list * li;
-        
-        TRACE_ENTRY("%p %p", diamid, peer);
-        CHECK_PARAMS( diamid && peer );
+        TRACE_ENTRY("%p %zd %d %p", diamid, diamidlen, igncase, peer);
+        CHECK_PARAMS( diamid && diamidlen && peer );
         
         *peer = NULL;
@@ -175 +187 @@
         for (li = fd_g_peers.next; li != &fd_g_peers; li = li->next) {
                 struct fd_peer * next = (struct fd_peer *)li;
-                int cmp = strcasecmp( diamid, next->p_hdr.info.pi_diamid );
+                int cmp;
+                if (igncase)
+                        cmp = fd_os_almostcasecmp( diamid, diamidlen, next->p_hdr.info.pi_diamid, next->p_hdr.info.pi_diamidlen );
+                else
+                        cmp = fd_os_cmp( diamid, diamidlen, next->p_hdr.info.pi_diamid, next->p_hdr.info.pi_diamidlen );
                 if (cmp > 0)
                         continue;
@@ -255 +271 @@
         
         CHECK_FCT_DO( fd_fifo_del(&p->p_tosend), /* continue */ );
+        CHECK_POSIX_DO( pthread_mutex_destroy(&p->p_state_mtx), /* continue */);
         CHECK_POSIX_DO( pthread_mutex_destroy(&p->p_sr.mtx), /* continue */);
         CHECK_POSIX_DO( pthread_cond_destroy(&p->p_sr.cnd), /* continue */);
@@ -283 +300 @@
         CHECK_FCT_DO( pthread_rwlock_wrlock(&fd_g_peers_rw), /* continue */ );
         for (li = fd_g_peers.next; li != &fd_g_peers; li = li->next) {
-                struct fd_peer * peer = (struct fd_peer *)li;
-                
-                fd_cpu_flush_cache();
-                if (peer->p_hdr.info.runtime.pir_state != STATE_ZOMBIE) {
+                struct fd_peer * peer = (struct fd_peer *)li->o;
+                
+                if (fd_peer_getstate(peer) != STATE_ZOMBIE) {
                         CHECK_FCT_DO( fd_psm_terminate(peer, "REBOOTING"), /* continue */ );
                 } else {
@@ -307 +323 @@
                 
                 /* Allow the PSM(s) to execute */
-                sched_yield();
+                usleep(100000);
                 
                 /* Remove zombie peers */
                 CHECK_FCT_DO( pthread_rwlock_wrlock(&fd_g_peers_rw), /* continue */ );
                 for (li = fd_g_peers.next; li != &fd_g_peers; li = li->next) {
-                        struct fd_peer * peer = (struct fd_peer *)li;
-                        fd_cpu_flush_cache();
-                        if (peer->p_hdr.info.runtime.pir_state == STATE_ZOMBIE) {
+                        struct fd_peer * peer = (struct fd_peer *)li->o;
+                        if (fd_peer_getstate(peer) == STATE_ZOMBIE) {
                                 li = li->prev; /* to avoid breaking the loop */
                                 fd_list_unlink(&peer->p_hdr.chain);
@@ -329 +344 @@
                 CHECK_FCT_DO( pthread_rwlock_wrlock(&fd_g_peers_rw), /* continue */ );
                 while (!FD_IS_LIST_EMPTY(&fd_g_peers)) {
-                        struct fd_peer * peer = (struct fd_peer *)(fd_g_peers.next);
+                        struct fd_peer * peer = (struct fd_peer *)(fd_g_peers.next->o);
                         fd_psm_abord(peer);
                         fd_list_unlink(&peer->p_hdr.chain);
@@ -339 +354 @@
         /* Free memory objects of all peers */
         while (!FD_IS_LIST_EMPTY(&purge)) {
-                struct fd_peer * peer = (struct fd_peer *)(purge.next);
+                struct fd_peer * peer = (struct fd_peer *)(purge.next->o);
                 fd_list_unlink(&peer->p_hdr.chain);
                 fd_peer_free(&peer);
@@ -364 +379 @@
         }
 
-        fd_log_debug(">  %s\t%s", STATE_STR(peer->p_hdr.info.runtime.pir_state), peer->p_hdr.info.pi_diamid);
+        fd_log_debug(">  %s\t%s", STATE_STR(fd_peer_getstate(peer)), peer->p_hdr.info.pi_diamid);
         if (details > INFO) {
-                fd_log_debug("\t(rlm:%s)", peer->p_hdr.info.runtime.pir_realm ?: "(unknown)");
+                fd_log_debug("\t(rlm:%s)", peer->p_hdr.info.runtime.pir_realm ?: "<unknown>");
                 if (peer->p_hdr.info.runtime.pir_prodname)
                         fd_log_debug("\t['%s' %u]", peer->p_hdr.info.runtime.pir_prodname, peer->p_hdr.info.runtime.pir_firmrev);
@@ -398 +413 @@
         
         for (li = fd_g_peers.next; li != &fd_g_peers; li = li->next) {
-                struct fd_peer * np = (struct fd_peer *)li;
+                struct fd_peer * np = (struct fd_peer *)li->o;
                 fd_peer_dump(np, details);
         }
@@ -404 +419 @@
         CHECK_FCT_DO( pthread_rwlock_unlock(&fd_g_peers_rw), /* continue */ );
 }
+
+static struct dict_object *avp_oh_model = NULL;
+static pthread_mutex_t cache_avp_lock = PTHREAD_MUTEX_INITIALIZER;
 
 /* Handle an incoming CER request on a new connection */
@@ -409 +427 @@
 {
         struct msg * msg;
-        struct dict_object *avp_oh_model;
-        avp_code_t code = AC_ORIGIN_HOST;
         struct avp *avp_oh;
         struct avp_hdr * avp_hdr;
@@ -424 +440 @@
         msg = *cer; 
         
+        /* If needed, resolve the dictioanry model for Origin-Host */
+        CHECK_POSIX( pthread_mutex_lock(&cache_avp_lock) );
+        if (!avp_oh_model) {
+                avp_code_t code = AC_ORIGIN_HOST;
+                int ret;
+                CHECK_FCT_DO( ret = fd_dict_search ( fd_g_config->cnf_dict, DICT_AVP, AVP_BY_CODE, &code, &avp_oh_model, ENOENT),
+                        { CHECK_POSIX( pthread_mutex_unlock(&cache_avp_lock) ); return ret; } );
+        }
+        CHECK_POSIX( pthread_mutex_unlock(&cache_avp_lock) );
+        
         /* Find the Diameter Identity of the remote peer in the message */
-        CHECK_FCT( fd_dict_search ( fd_g_config->cnf_dict, DICT_AVP, AVP_BY_CODE, &code, &avp_oh_model, ENOENT) );
         CHECK_FCT( fd_msg_search_avp ( msg, avp_oh_model, &avp_oh ) );
+        ASSERT(avp_oh); /* otherwise it should not have passed rules validation, right? */
         CHECK_FCT( fd_msg_avp_hdr ( avp_oh, &avp_hdr ) );
+        
+        /* First, check if the Origin-Host value  */
+        if (!fd_os_is_valid_DiameterIdentity(avp_hdr->avp_value->os.data, avp_hdr->avp_value->os.len)) {
+                TRACE_DEBUG(INFO, "Received new CER with invalid \\0 in its Origin-Host");
+                CHECK_FCT( fd_msg_new_answer_from_req ( fd_g_config->cnf_dict, cer, MSGFL_ANSW_ERROR ) );
+                CHECK_FCT( fd_msg_rescode_set(*cer, "ER_DIAMETER_INVALID_AVP_VALUE", 
+                                                        "Your Origin-Host contains invalid characters.", avp_oh, 1 ) );
+                CHECK_FCT( fd_out_send(cer, *cnx, NULL, FD_CNX_ORDERED) );
+                return EINVAL;
+        }
         
         /* Search if we already have this peer id in our list. We take directly the write lock so that we don't need to upgrade if it is a new peer.
@@ -435 +471 @@
         
         for (li = fd_g_peers.next; li != &fd_g_peers; li = li->next) {
-                peer = (struct fd_peer *)li;
-                /* It is probably unwise to use strcasecmp on UTF8 data... To be improved! */
-                int cmp = strncasecmp( (char *)avp_hdr->avp_value->os.data, peer->p_hdr.info.pi_diamid, avp_hdr->avp_value->os.len );
+                int cmp;
+                peer = (struct fd_peer *)li->o;
+                cmp = fd_os_almostcasecmp( avp_hdr->avp_value->os.data, avp_hdr->avp_value->os.len, peer->p_hdr.info.pi_diamid, peer->p_hdr.info.pi_diamidlen );
                 if (cmp > 0)
                         continue;
@@ -451 +487 @@
                 
                 /* Set the peer Diameter Id and the responder flag parameters */
-                CHECK_MALLOC_DO( peer->p_hdr.info.pi_diamid = malloc(avp_hdr->avp_value->os.len + 1), { ret = ENOMEM; goto out; } );
-                memcpy(peer->p_hdr.info.pi_diamid, avp_hdr->avp_value->os.data, avp_hdr->avp_value->os.len);
-                peer->p_hdr.info.pi_diamid[avp_hdr->avp_value->os.len] = '\0';
+                CHECK_MALLOC_DO( peer->p_hdr.info.pi_diamid = os0dup(avp_hdr->avp_value->os.data, avp_hdr->avp_value->os.len), 
+                        { ret = ENOMEM; goto out; } );
+                peer->p_hdr.info.pi_diamidlen = avp_hdr->avp_value->os.len;
                 CHECK_MALLOC_DO( peer->p_dbgorig = strdup(fd_cnx_getid(*cnx)), { ret = ENOMEM; goto out; } );
                 peer->p_flags.pf_responder = 1;
@@ -470 +506 @@
         } else {
                 /* Check if the peer is in zombie state */
-                fd_cpu_flush_cache();
-                if (peer->p_hdr.info.runtime.pir_state == STATE_ZOMBIE) {
+                if (fd_peer_getstate(peer) == STATE_ZOMBIE) {
                         /* Re-activate the peer */
                         if (peer->p_hdr.info.config.pic_flags.exp)
                                 peer->p_flags.pf_responder = 1;
-                        peer->p_hdr.info.runtime.pir_state = STATE_NEW;
+                        CHECK_POSIX_DO( pthread_mutex_lock(&peer->p_state_mtx), );
+                        peer->p_state = STATE_NEW;
+                        CHECK_POSIX_DO( pthread_mutex_unlock(&peer->p_state_mtx), );
+                        peer->p_flags.pf_localterm = 0;
                         CHECK_FCT_DO( ret = fd_psm_begin(peer), goto out );
                 }
         }
-                
+        
         /* Send the new connection event to the PSM */
         CHECK_MALLOC_DO( ev_data = malloc(sizeof(struct cnx_incoming)), { ret = ENOMEM; goto out; } );

libfdcore/routing_dispatch.c

@@ -208 +208 @@
                 struct fd_peer * peer;
                 struct fd_app *found;
-                CHECK_FCT( fd_peer_getbyid( c->diamid, (void *)&peer ) );
-                if (peer && (peer->p_hdr.info.runtime.pir_relay == 0)) {
+                CHECK_FCT( fd_peer_getbyid( c->diamid, c->diamidlen, 0, (void *)&peer ) );
+                if (peer && !peer->p_hdr.info.runtime.pir_relay) {
                         /* Check if the remote peer advertised the message's appli */
                         CHECK_FCT( fd_app_check(&peer->p_hdr.info.runtime.pir_apps, hdr->msg_appl, &found) );
@@ -264 +264 @@
         for (li = candidates->next; li != candidates; li = li->next) {
                 struct rtd_candidate *c = (struct rtd_candidate *) li;
+                
+            #if 0 /* this is actually useless since the sending process will also ensure that the peer is still available */
                 struct fd_peer * peer;
-                CHECK_FCT( fd_peer_getbyid( c->diamid, (void *)&peer ) );
-                if (peer) {
-                        if (dh 
-                                && (dh->os.len == strlen(peer->p_hdr.info.pi_diamid)) 
-                                /* Here again we use strncasecmp on UTF8 data... This should probably be changed. */
-                                && (strncasecmp(peer->p_hdr.info.pi_diamid, (char *)dh->os.data, dh->os.len) == 0)) {
-                                /* The candidate is the Destination-Host */
-                                c->score += FD_SCORE_FINALDEST;
-                        } else {
-                                if (dr  && peer->p_hdr.info.runtime.pir_realm 
-                                        && (dr->os.len == strlen(peer->p_hdr.info.runtime.pir_realm)) 
-                                        /* Yet another case where we use strncasecmp on UTF8 data... Hmmm :-( */
-                                        && (strncasecmp(peer->p_hdr.info.runtime.pir_realm, (char *)dr->os.data, dr->os.len) == 0)) {
-                                        /* The candidate's realm matchs the Destination-Realm */
-                                        c->score += FD_SCORE_REALM;
-                                }
+                /* Since the candidates list comes from the peers list, we do not have any issue with upper/lower case to find the peer object */
+                CHECK_FCT( fd_peer_getbyid( c->diamid, c->diamidlen, 0, (void *)&peer ) );
+                if (!peer)
+                        continue; /* it has been deleted since the candidate list was generated; avoid sending to this one in that case. */
+            #endif /* 0 */
+                
+                /* In the AVPs, the value comes from the network, so let's be case permissive */
+                if (dh && !fd_os_almostcasecmp(dh->os.data, dh->os.len, c->diamid, c->diamidlen) ) {
+                        /* The candidate is the Destination-Host */
+                        c->score += FD_SCORE_FINALDEST;
+                } else {
+                        if (dr && !fd_os_almostcasecmp(dr->os.data, dr->os.len, c->realm, c->realmlen) ) {
+                                /* The candidate's realm matchs the Destination-Realm */
+                                c->score += FD_SCORE_REALM;
                         }
                 }
@@ -298 +298 @@
         
         TRACE_ENTRY("%p %p %p", un, excl_idx, at_idx);
-        CHECK_PARAMS_DO( un && excl_idx, return );
+        CHECK_PARAMS_DO( un && excl_idx && at_idx, return );
+        
         *excl_idx = 0;
+        *at_idx = 0;
         
         /* Search if there is a '!' before any '@' -- do we need to check it contains a '.' ? */
@@ -307 +309 @@
                         if (!*excl_idx)
                                 *excl_idx = i;
-                        if (!at_idx)
-                                return;
+                        continue;
                 }
                 /* If we reach the realm part, we can stop */
                 if ( un->os.data[i] == (unsigned char) '@' ) {
-                        if (at_idx)
-                                *at_idx = i;
+                        *at_idx = i;
                         break;
+                }
+                /* Stop if we find a \0 in the middle */
+                if ( un->os.data[i] == 0 ) {
+                        return;
                 }
                 /* Skip escaped characters */
@@ -321 +325 @@
                         continue;
                 }
-                /* Skip UTF-8 characters spanning on several bytes */
-                if ( (un->os.data[i] & 0xF8) == 0xF0 ) { /* 11110zzz */
-                        i += 3;
-                        continue;
-                }
-                if ( (un->os.data[i] & 0xF0) == 0xE0 ) { /* 1110yyyy */
-                        i += 2;
-                        continue;
-                }
-                if ( (un->os.data[i] & 0xE0) == 0xC0 ) { /* 110yyyxx */
-                        i += 1;
-                        continue;
-                }
         }
         
@@ -340 +331 @@
 
 /* Test if a User-Name AVP contains a Decorated NAI -- RFC4282, RFC5729 */
-static int is_decorated_NAI(union avp_value * un)
-{
-        int i;
-        TRACE_ENTRY("%p", un);
-        
-        /* If there was no User-Name, we return false */
-        if (un == NULL)
-                return 0;
-        
-        nai_get_indexes(un, &i, NULL);
-        
-        return i;
-}
-
 /* Create new User-Name and Destination-Realm values */
-static int process_decorated_NAI(union avp_value * un, union avp_value * dr)
-{
-        int at_idx = 0, sep_idx = 0;
+static int process_decorated_NAI(int * was_nai, union avp_value * un, union avp_value * dr)
+{
+        int at_idx, sep_idx;
         unsigned char * old_un;
-        TRACE_ENTRY("%p %p", un, dr);
-        CHECK_PARAMS(un && dr);
+        TRACE_ENTRY("%p %p %p", was_nai, un, dr);
+        CHECK_PARAMS(was_nai && un && dr);
         
         /* Save the decorated User-Name, for example 'homerealm.example.net!user@otherrealm.example.net' */
@@ -367 +344 @@
         /* Search the positions of the first '!' and the '@' in the string */
         nai_get_indexes(un, &sep_idx, &at_idx);
-        CHECK_PARAMS( (0 < sep_idx) && (sep_idx < at_idx) && (at_idx < un->os.len));
+        if ((!sep_idx) || (sep_idx > at_idx) || !fd_os_is_valid_DiameterIdentity(old_un, sep_idx /* this is the new realm part */)) {
+                *was_nai = 0;
+                return 0;
+        }
+        
+        *was_nai = 1;
         
         /* Create the new User-Name value */
@@ -390 +372 @@
 }
 
+
 /* Function to return an error to an incoming request */
 static int return_error(struct msg ** pmsg, char * error_code, char * error_message, struct avp * failedavp)
@@ -398 +381 @@
         /* Get the source of the message */
         {
-                char * id;
-                CHECK_FCT( fd_msg_source_get( *pmsg, &id ) );
+                DiamId_t id;
+                size_t   idlen;
+                CHECK_FCT( fd_msg_source_get( *pmsg, &id, &idlen ) );
                 
                 if (id == NULL) {
@@ -406 +390 @@
                 
                         /* Search the peer with this id */
-                        CHECK_FCT( fd_peer_getbyid( id, (void *)&peer ) );
+                        CHECK_FCT( fd_peer_getbyid( id, idlen, 0, (void *)&peer ) );
 
                         if (!peer) {
@@ -439 +423 @@
 /****************************************************************************/
 
-/* These are the functions of each threads: dispatch & routing */
 /* The DISPATCH message processing */
 static int msg_dispatch(struct msg ** pmsg)
@@ -447 +430 @@
         struct session * sess;
         enum disp_action action;
-        const char * ec = NULL;
-        const char * em = NULL;
+        char * ec = NULL;
+        char * em = NULL;
 
         /* Read the message header */
@@ -460 +443 @@
         CHECK_FCT_DO( ret = fd_msg_parse_or_error( pmsg ),
                 {
-                        /* in case of error, the message is already dump'd */
+                        /* in case of error */
                         if ((ret == EBADMSG) && (*pmsg != NULL)) {
                                 /* msg now contains the answer message to send back */
@@ -512 +495 @@
                                 em = "The message was not handled by any extension callback";
                                 ec = "DIAMETER_COMMAND_UNSUPPORTED";
-                        
+                                /* and continue as if an error occurred... */
                         case DISP_ACT_ERROR:
                                 /* We have a problem with delivering the message */
@@ -528 +511 @@
                                 /* Create an answer with the error code and message */
                                 CHECK_FCT( fd_msg_new_answer_from_req ( fd_g_config->cnf_dict, pmsg, 0 ) );
-                                CHECK_FCT( fd_msg_rescode_set(*pmsg, (char *)ec, (char *)em, NULL, 1 ) );
+                                CHECK_FCT( fd_msg_rescode_set(*pmsg, ec, em, NULL, 1 ) );
                                 
                         case DISP_ACT_SEND:
@@ -545 +528 @@
         int is_req = 0;
         int is_err = 0;
-        char * qry_src = NULL;
+        DiamId_t qry_src = NULL;
+        size_t   qry_src_len = 0;
 
         /* Read the message header */
@@ -572 +556 @@
                 /* Check if we have local support for the message application */
                 if ( (hdr->msg_appl == 0) || (hdr->msg_appl == AI_RELAY) ) {
-                        TRACE_DEBUG(INFO, "Received a routable message with application id 0, returning DIAMETER_APPLICATION_UNSUPPORTED");
+                        TRACE_DEBUG(INFO, "Received a routable message with application id 0 or " _stringize(AI_RELAY) " (relay),\n"
+                                          " returning DIAMETER_APPLICATION_UNSUPPORTED");
                         CHECK_FCT( return_error( pmsg, "DIAMETER_APPLICATION_UNSUPPORTED", "Routable message with application id 0 or relay", NULL) );
                         return 0;
@@ -587 +572 @@
                         struct fd_pei error_info;
                         int ret;
+                        
+                        memset(&error_info, 0, sizeof(struct fd_pei)); 
+                        
                         CHECK_FCT(  fd_msg_avp_hdr( avp, &ahdr )  );
 
@@ -604 +592 @@
                                                 ASSERT( ahdr->avp_value );
                                                 /* Compare the Destination-Host AVP of the message with our identity */
-                                                if (ahdr->avp_value->os.len != fd_g_config->cnf_diamid_len) {
+                                                if (!fd_os_almostcasecmp(ahdr->avp_value->os.data, ahdr->avp_value->os.len, fd_g_config->cnf_diamid, fd_g_config->cnf_diamid_len)) {
+                                                        is_dest_host = YES;
+                                                } else {
                                                         is_dest_host = NO;
-                                                } else {
-                                                        is_dest_host = (strncasecmp(fd_g_config->cnf_diamid, (char *)ahdr->avp_value->os.data, fd_g_config->cnf_diamid_len) 
-                                                                                ? NO : YES);
                                                 }
                                                 break;
@@ -626 +613 @@
                                                 dr_val = ahdr->avp_value;
                                                 /* Compare the Destination-Realm AVP of the message with our identity */
-                                                if (ahdr->avp_value->os.len != fd_g_config->cnf_diamrlm_len) {
+                                                if (!fd_os_almostcasecmp(dr_val->os.data, dr_val->os.len, fd_g_config->cnf_diamrlm, fd_g_config->cnf_diamrlm_len)) {
+                                                        is_dest_realm = YES;
+                                                } else {
                                                         is_dest_realm = NO;
-                                                } else {
-                                                        is_dest_realm = (strncasecmp(fd_g_config->cnf_diamrlm, (char *)ahdr->avp_value->os.data, fd_g_config->cnf_diamrlm_len) 
-                                                                                ? NO : YES);
                                                 }
                                                 break;
 
+                                        /* we also use User-Name for decorated NAI */
                                         case AC_USER_NAME:
                                                 /* Parse this AVP */
@@ -652 +639 @@
                         }
 
+                        /* Stop when we found all 3 AVPs -- they are supposed to be at the beginning of the message, so this should be fast */
                         if ((is_dest_host != UNKNOWN) && (is_dest_realm != UNKNOWN) && un)
                                 break;
@@ -682 +670 @@
                 if ((is_dest_host == NO) || (is_dest_realm == NO)) {
                         if (fd_g_config->cnf_flags.no_fwd) {
-                                CHECK_FCT( return_error( pmsg, "DIAMETER_UNABLE_TO_DELIVER", "This peer is not an agent", NULL) );
+                                CHECK_FCT( return_error( pmsg, "DIAMETER_UNABLE_TO_DELIVER", "I am not a Diameter agent", NULL) );
                                 return 0;
                         }
                 } else {
                 /* Destination-Host was not set, and Destination-Realm is matching : we may handle or pass to a fellow peer */
+                        int is_nai = 0;
 
                         /* test for decorated NAI  (RFC5729 section 4.4) */
-                        if (is_decorated_NAI(un_val)) {
-                                /* Handle the decorated NAI */
-                                CHECK_FCT_DO( process_decorated_NAI(un_val, dr_val),
-                                        {
-                                                /* If the process failed, we assume it is because of the AVP format */
-                                                CHECK_FCT( return_error( pmsg, "DIAMETER_INVALID_AVP_VALUE", "Failed to process decorated NAI", un) );
-                                                return 0;
-                                        } );
-
+                        /* Handle the decorated NAI */
+                        CHECK_FCT_DO( process_decorated_NAI(&is_nai, un_val, dr_val),
+                                {
+                                        /* If the process failed, we assume it is because of the AVP format */
+                                        CHECK_FCT( return_error( pmsg, "DIAMETER_INVALID_AVP_VALUE", "Failed to process decorated NAI", un) );
+                                        return 0;
+                                } );
+                                
+                        if (is_nai) {
                                 /* We have transformed the AVP, now submit it again in the queue */
                                 CHECK_FCT(fd_fifo_post(fd_g_incoming, pmsg) );
@@ -724 +713 @@
                 /* Retrieve the corresponding query and its origin */
                 CHECK_FCT( fd_msg_answ_getq( *pmsg, &qry ) );
-                CHECK_FCT( fd_msg_source_get( qry, &qry_src ) );
+                CHECK_FCT( fd_msg_source_get( qry, &qry_src, &qry_src_len ) );
 
                 if ((!qry_src) && (!is_err)) {
@@ -733 +722 @@
 
                 /* From that point, for answers, we will call the registered callbacks, then pass it to the dispatch module or forward it */
+                TODO("Callback for answers with a Redirect code?");
         }
 
@@ -800 +790 @@
         if ( ! is_req ) {
                 struct msg * qry;
-                char * qry_src = NULL;
+                DiamId_t qry_src = NULL;
+                size_t qry_src_len = 0;
                 struct msg_hdr * qry_hdr;
                 struct fd_peer * peer = NULL;
@@ -806 +797 @@
                 /* Retrieve the corresponding query and its origin */
                 CHECK_FCT( fd_msg_answ_getq( *pmsg, &qry ) );
-                CHECK_FCT( fd_msg_source_get( qry, &qry_src ) );
+                CHECK_FCT( fd_msg_source_get( qry, &qry_src, &qry_src_len ) );
 
                 ASSERT( qry_src ); /* if it is NULL, the message should have been in the LOCAL queue! */
 
                 /* Find the peer corresponding to this name */
-                CHECK_FCT( fd_peer_getbyid( qry_src, (void *) &peer ) );
-                fd_cpu_flush_cache();
-                if ((!peer) || (peer->p_hdr.info.runtime.pir_state != STATE_OPEN)) {
+                CHECK_FCT( fd_peer_getbyid( qry_src, qry_src_len, 0, (void *) &peer ) );
+                if (fd_peer_getstate(peer) != STATE_OPEN) {
                         fd_msg_log( FD_MSG_LOG_DROPPED, *pmsg, "Unable to forward answer to deleted / closed peer '%s'.", qry_src);
                         fd_msg_free(*pmsg);
@@ -844 +834 @@
                 for (li = fd_g_activ_peers.next; li != &fd_g_activ_peers; li = li->next) {
                         struct fd_peer * p = (struct fd_peer *)li->o;
-                        CHECK_FCT_DO( ret = fd_rtd_candidate_add(rtd, p->p_hdr.info.pi_diamid, p->p_hdr.info.runtime.pir_realm), { CHECK_FCT_DO( pthread_rwlock_unlock(&fd_g_activ_peers_rw), ); return ret; } );
+                        CHECK_FCT_DO( ret = fd_rtd_candidate_add(rtd, 
+                                                        p->p_hdr.info.pi_diamid, 
+                                                        p->p_hdr.info.pi_diamidlen, 
+                                                        p->p_hdr.info.runtime.pir_realm,
+                                                        p->p_hdr.info.runtime.pir_realmlen), 
+                                { CHECK_FCT_DO( pthread_rwlock_unlock(&fd_g_activ_peers_rw), ); return ret; } );
                 }
                 CHECK_FCT( pthread_rwlock_unlock(&fd_g_activ_peers_rw) );
@@ -867 +862 @@
                                         } );
                                 ASSERT( ahdr->avp_value );
-                                /* Remove this value from the list */
-                                fd_rtd_candidate_del(rtd, (char *)ahdr->avp_value->os.data, ahdr->avp_value->os.len);
+                                /* Remove this value from the list. We don't need to pay special attention to the contents here. */
+                                fd_rtd_candidate_del(rtd, ahdr->avp_value->os.data, ahdr->avp_value->os.len);
                         }
 
@@ -876 +871 @@
         }
 
-        /* Note: we reset the scores and pass the message to the callbacks, maybe we could re-use the saved scores when we have received an error ? */
+        /* Note: we reset the scores and pass the message to the callbacks, maybe we could re-use the saved scores when we have received an error ? -- TODO */
 
         /* Ok, we have our list in rtd now, let's (re)initialize the scores */
@@ -928 +923 @@
 
                 /* Search for the peer */
-                CHECK_FCT( fd_peer_getbyid( c->diamid, (void *)&peer ) );
-
-                fd_cpu_flush_cache();
-                if (peer && (peer->p_hdr.info.runtime.pir_state == STATE_OPEN)) {
+                CHECK_FCT( fd_peer_getbyid( c->diamid, c->diamidlen, 0, (void *)&peer ) );
+
+                if (fd_peer_getstate(peer) == STATE_OPEN) {
                         /* Send to this one */
                         CHECK_FCT_DO( fd_out_send(pmsg, NULL, peer, 0), continue );
@@ -964 +958 @@
 /* Control of the threads */
 static enum { RUN = 0, STOP = 1 } order_val = RUN;
-static pthread_mutex_t order_lock = PTHREAD_MUTEX_INITIALIZER;
+static pthread_mutex_t order_state_lock = PTHREAD_MUTEX_INITIALIZER;
 
 /* Threads report their status */
@@ -970 +964 @@
 static void cleanup_state(void * state_loc)
 {
-        if (state_loc)
-                *(enum thread_state *)state_loc = NOTRUNNING;
+        CHECK_POSIX_DO( pthread_mutex_lock(&order_state_lock), );
+        *(enum thread_state *)state_loc = NOTRUNNING;
+        CHECK_POSIX_DO( pthread_mutex_unlock(&order_state_lock), );
 }
 
@@ -991 +986 @@
         
         /* Mark the thread running */
+        CHECK_POSIX_DO( pthread_mutex_lock(&order_state_lock), );
         *(enum thread_state *)arg = RUNNING;
-        fd_cpu_flush_cache();
+        CHECK_POSIX_DO( pthread_mutex_unlock(&order_state_lock), );
         
         do {
@@ -1000 +996 @@
                 {
                         int must_stop;
-                        CHECK_POSIX_DO( pthread_mutex_lock(&order_lock), { ASSERT(0); } ); /* we lock to flush the caches */
+                        CHECK_POSIX_DO( pthread_mutex_lock(&order_state_lock), { ASSERT(0); } ); /* we lock to flush the caches */
                         must_stop = (order_val == STOP);
-                        CHECK_POSIX_DO( pthread_mutex_unlock(&order_lock), { ASSERT(0); } );
+                        CHECK_POSIX_DO( pthread_mutex_unlock(&order_state_lock), { ASSERT(0); } );
                         if (must_stop)
                                 goto end;
@@ -1014 +1010 @@
                 {
                         int ret;
-                        ret = fd_fifo_get ( queue, &msg );
+                        struct timespec ts;
+                        
+                        CHECK_SYS_DO( clock_gettime(CLOCK_REALTIME, &ts), goto fatal_error );
+                        ts.tv_sec += 1;
+                        
+                        ret = fd_fifo_timedget ( queue, &msg, &ts );
+                        if (ret == ETIMEDOUT)
+                                /* loop, check if the thread must stop now */
+                                continue;
                         if (ret == EPIPE)
                                 /* The queue was destroyed, we are probably exiting */
@@ -1100 +1104 @@
         CHECK_FCT( fd_rt_out_register( dont_send_if_no_common_app, NULL, 10, NULL ) );
         CHECK_FCT( fd_rt_out_register( score_destination_avp, NULL, 10, NULL ) );
+        
+        TODO("built-in callbacks for Redirect messages?");
+        
         return 0;
 }
@@ -1106 +1113 @@
 int fd_rtdisp_cleanstop(void)
 {
-        CHECK_POSIX( pthread_mutex_lock(&order_lock) );
+        CHECK_POSIX_DO( pthread_mutex_lock(&order_state_lock), );
         order_val = STOP;
-        CHECK_POSIX( pthread_mutex_unlock(&order_lock) );
+        CHECK_POSIX_DO( pthread_mutex_unlock(&order_state_lock), );
 
         return 0;
@@ -1117 +1124 @@
         TRACE_ENTRY("%p %p", st, thr);
         CHECK_PARAMS_DO(st && thr, return);
+        int terminated;
+        
+        CHECK_POSIX_DO( pthread_mutex_lock(&order_state_lock), );
+        terminated = (*st == NOTRUNNING);
+        CHECK_POSIX_DO( pthread_mutex_unlock(&order_state_lock), );
+        
 
         /* Wait for a second for the thread to complete, by monitoring my_state */
-        fd_cpu_flush_cache();
-        if (*st != NOTRUNNING) {
+        if (!terminated) {
                 TRACE_DEBUG(INFO, "Waiting for the %s thread to have a chance to terminate", th_name);
                 do {
@@ -1131 +1143 @@
                         
                         while (TS_IS_INFERIOR( &ts, &ts_final )) {
-                                fd_cpu_flush_cache();
-                                if (*st == NOTRUNNING)
+                        
+                                CHECK_POSIX_DO( pthread_mutex_lock(&order_state_lock), );
+                                terminated = (*st == NOTRUNNING);
+                                CHECK_POSIX_DO( pthread_mutex_unlock(&order_state_lock), );
+                                if (terminated)
                                         break;
                                 

libfdcore/sctp.c

@@ -999 +999 @@
 
 /* Send a buffer over a specified stream */
-int fd_sctp_sendstr(int sock, uint16_t strid, uint8_t * buf, size_t len, uint32_t * cc_status)
+int fd_sctp_sendstr(struct cnxctx * conn, uint16_t strid, uint8_t * buf, size_t len)
 {
         struct msghdr mhdr;
@@ -1009 +1009 @@
         int timedout = 0;
         
-        TRACE_ENTRY("%d %hu %p %zd %p", sock, strid, buf, len, cc_status);
-        CHECK_PARAMS(cc_status);
+        TRACE_ENTRY("%p %hu %p %zd", conn, strid, buf, len);
+        CHECK_PARAMS(conn && buf && len);
         
         memset(&mhdr, 0, sizeof(mhdr));
@@ -1037 +1037 @@
         mhdr.msg_controllen = sizeof(anci);
         
-        TRACE_DEBUG(FULL, "Sending %db data on stream %hu of socket %d", len, strid, sock);
+        TRACE_DEBUG(FULL, "Sending %db data on stream %hu of socket %d", len, strid, conn->cc_socket);
 again:  
-        ret = sendmsg(sock, &mhdr, 0);
+        ret = sendmsg(conn->cc_socket, &mhdr, 0);
         /* Handle special case of timeout */
         if ((ret < 0) && (errno == EAGAIN)) {
-                if (!(*cc_status & CC_STATUS_CLOSING))
+                if (! fd_cnx_teststate(conn, CC_STATUS_CLOSING ))
                         goto again; /* don't care, just ignore */
                 if (!timedout) {
@@ -1057 +1057 @@
 
 /* Receive the next data from the socket, or next notification */
-int fd_sctp_recvmeta(int sock, uint16_t * strid, uint8_t ** buf, size_t * len, int *event, uint32_t * cc_status)
+int fd_sctp_recvmeta(struct cnxctx * conn, uint16_t * strid, uint8_t ** buf, size_t * len, int *event)
 {
         ssize_t                  ret = 0;
@@ -1068 +1068 @@
         int                      timedout = 0;
         
-        TRACE_ENTRY("%d %p %p %p %p %p", sock, strid, buf, len, event, cc_status);
-        CHECK_PARAMS( (sock > 0) && buf && len && event && cc_status );
+        TRACE_ENTRY("%p %p %p %p %p", conn, strid, buf, len, event);
+        CHECK_PARAMS( conn && buf && len && event );
         
         /* Cleanup out parameters */
@@ -1098 +1098 @@
 again:
         pthread_cleanup_push(free, data);
-        ret = recvmsg(sock, &mhdr, 0);
+        ret = recvmsg(conn->cc_socket, &mhdr, 0);
         pthread_cleanup_pop(0);
         
         /* First, handle timeouts (same as fd_cnx_s_recv) */
         if ((ret < 0) && (errno == EAGAIN)) {
-                if (!(*cc_status & CC_STATUS_CLOSING))
+                if (! fd_cnx_teststate(conn, CC_STATUS_CLOSING ))
                         goto again; /* don't care, just ignore */
                 if (!timedout) {
@@ -1132 +1132 @@
                 union sctp_notification * notif = (union sctp_notification *) data;
                 
-                TRACE_DEBUG(FULL, "Received %db data of notification on socket %d", datasize, sock);
+                TRACE_DEBUG(FULL, "Received %db data of notification on socket %d", datasize, conn->cc_socket);
         
                 switch (notif->sn_header.sn_type) {
@@ -1226 +1226 @@
                         *strid = sndrcv->sinfo_stream;
                 }
-                TRACE_DEBUG(FULL, "Received %db data on socket %d, stream %hu", datasize, sock, *strid);
+                TRACE_DEBUG(FULL, "Received %db data on socket %d, stream %hu", datasize, conn->cc_socket, *strid);
         } else {
-                TRACE_DEBUG(FULL, "Received %db data on socket %d (stream ignored)", datasize, sock);
+                TRACE_DEBUG(FULL, "Received %db data on socket %d (stream ignored)", datasize, conn->cc_socket);
         }
         

libfdcore/sctps.c

@@ -57 +57 @@
 */
 
-
+/* TODO: change this whole wrapper to DTLS which should not require many different threads */
 
 /*************************************************************/
@@ -83 +83 @@
         
         ASSERT( conn->cc_proto == IPPROTO_SCTP );
-        ASSERT( Target_Queue(conn) );
+        ASSERT( fd_cnx_target_queue(conn) );
         ASSERT( conn->cc_sctps_data.array );
         
         do {
-                fd_cpu_flush_cache();
-                CHECK_FCT_DO( fd_sctp_recvmeta(conn->cc_socket, &strid, &buf, &bufsz, &event, &conn->cc_status), goto fatal );
+                CHECK_FCT_DO( fd_sctp_recvmeta(conn, &strid, &buf, &bufsz, &event), goto fatal );
                 switch (event) {
                         case FDEVP_CNX_MSG_RECV:
@@ -102 +101 @@
                         case FDEVP_CNX_EP_CHANGE:
                                 /* Send this event to the target queue */
-                                fd_cpu_flush_cache();
-                                CHECK_FCT_DO( fd_event_send( Target_Queue(conn), event, bufsz, buf), goto fatal );
+                                CHECK_FCT_DO( fd_event_send( fd_cnx_target_queue(conn), event, bufsz, buf), goto fatal );
                                 break;
                         
@@ -144 +142 @@
         CHECK_PARAMS_DO(ctx && ctx->raw_recv && ctx->parent, goto error);
         cnx = ctx->parent;
-        ASSERT( Target_Queue(cnx) );
+        ASSERT( fd_cnx_target_queue(cnx) );
         
         /* Set the thread name */
@@ -173 +171 @@
         CHECK_PARAMS_DO( tr && data, { errno = EINVAL; return -1; } );
         
-        fd_cpu_flush_cache();
-        CHECK_FCT_DO( fd_sctp_sendstr(ctx->parent->cc_socket, ctx->strid, (uint8_t *)data, len, &ctx->parent->cc_status), /* errno is already set */ return -1 );
+        CHECK_FCT_DO( fd_sctp_sendstr(ctx->parent, ctx->strid, (uint8_t *)data, len), /* errno is already set */ return -1 );
         
         return len;
@@ -306 +303 @@
                 struct sr_data * sr = (struct sr_data *)ret;
                 
-                if ( key.size < sr->key.size )
-                        break;
-                
-                if ( key.size > sr->key.size )
-                        continue;
-                
-                /* Key sizes are equal */
-                cmp = memcmp( key.data, sr->key.data, key.size );
-                
+                cmp = fd_os_cmp(key.data, key.size, sr->key.data, sr->key.size);
                 if (cmp > 0)
                         continue;
@@ -627 +616 @@
         /* End all TLS sessions, in series (not as efficient as paralel, but simpler) */
         for (i = 1; i < conn->cc_sctp_para.pairs; i++) {
-                fd_cpu_flush_cache();
-                if ( ! (conn->cc_status & CC_STATUS_ERROR)) {
+                if ( ! fd_cnx_teststate(conn, CC_STATUS_ERROR)) {
                         CHECK_GNUTLS_DO( gnutls_bye(conn->cc_sctps_data.array[i].session, GNUTLS_SHUT_WR), fd_cnx_markerror(conn) );
                 }

libfdcore/server.c

@@ -41 +41 @@
 /* We don't need to protect this list, it is only accessed from the main framework thread. */
 
+enum s_state {
+        NOT_CREATED=0,
+        RUNNING,
+        TERMINATED,
+        ERROR   /* an error occurred, this is not a valid status */
+};
+
 /* Servers information */
 struct server {
@@ -50 +57 @@
         
         pthread_t       thr;            /* The thread listening for new connections */
-        int             status;         /* 0 : not created; 1 : running; 2 : terminated */
+        enum s_state    state;          /* state of the thread */
         
         struct fd_list  clients;        /* List of clients connected to this server, not yet identified */
         pthread_mutex_t clients_mtx;    /* Mutex to protect the list of clients */
 };
+
 
 /* Client information (connecting peer for which we don't have the CER yet) */
@@ -65 +73 @@
 
 
+
+/* Micro functions to read/change the status thread-safely */
+static pthread_mutex_t s_lock = PTHREAD_MUTEX_INITIALIZER;
+static enum s_state get_status(struct server * s)
+{
+        enum s_state r;
+        CHECK_POSIX_DO( pthread_mutex_lock(&s_lock), return ERROR );
+        r = s->state;
+        CHECK_POSIX_DO( pthread_mutex_unlock(&s_lock), return ERROR );
+        return r;
+}
+static void set_status(struct server * s, enum s_state st)
+{
+        CHECK_POSIX_DO( pthread_mutex_lock(&s_lock), return );
+        s->state = st;
+        CHECK_POSIX_DO( pthread_mutex_unlock(&s_lock), return );
+}
+        
+
+
 /* Dump all servers information */
 void fd_servers_dump()
@@ -73 +101 @@
         for (li = FD_SERVERS.next; li != &FD_SERVERS; li = li->next) {
                 struct server * s = (struct server *)li;
-                fd_cpu_flush_cache();
+                enum s_state st = get_status(s);
                 fd_log_debug("  Serv %p '%s': %s, %s, %s\n", 
                                 s, fd_cnx_getid(s->conn), 
                                 IPPROTO_NAME( s->proto ),
                                 s->secur ? "Secur" : "NotSecur",
-                                (s->status == 0) ? "Thread not created" :
-                                ((s->status == 1) ? "Thread running" :
-                                ((s->status == 2) ? "Thread terminated" :
+                                (st == NOT_CREATED) ? "Thread not created" :
+                                ((st == RUNNING) ? "Thread running" :
+                                ((st == TERMINATED) ? "Thread terminated" :
                                                           "Thread status unknown")));
                 /* Dump the client list of this server */
@@ -192 +220 @@
         CHECK_PARAMS_DO(s, goto error);
         fd_log_threadname ( fd_cnx_getid(s->conn) );
-        s->status = 1;
-        fd_cpu_flush_cache();
+        set_status(s, RUNNING);
         
         /* Accept incoming connections */
@@ -225 +252 @@
 error:  
         if (s)
-                s->status = 2;
+                set_status(s, TERMINATED);
         /* Send error signal to the daemon */
         TRACE_DEBUG(INFO, "An error occurred in server module! Thread is terminating...");
@@ -266 +293 @@
                 
                 /* Create the server on unsecure port */
-                CHECK_MALLOC( s = new_serv(IPPROTO_SCTP, 0) );
-                CHECK_MALLOC( s->conn = fd_cnx_serv_sctp(fd_g_config->cnf_port, FD_IS_LIST_EMPTY(&fd_g_config->cnf_endpoints) ? NULL : &fd_g_config->cnf_endpoints) );
-                fd_list_insert_before( &FD_SERVERS, &s->chain );
-                CHECK_POSIX( pthread_create( &s->thr, NULL, serv_th, s ) );
+                if (fd_g_config->cnf_port) {
+                        CHECK_MALLOC( s = new_serv(IPPROTO_SCTP, 0) );
+                        CHECK_MALLOC( s->conn = fd_cnx_serv_sctp(fd_g_config->cnf_port, empty_conf_ep ? NULL : &fd_g_config->cnf_endpoints) );
+                        fd_list_insert_before( &FD_SERVERS, &s->chain );
+                        CHECK_POSIX( pthread_create( &s->thr, NULL, serv_th, s ) );
+                }
                 
                 /* Create the server on secure port */
-                CHECK_MALLOC( s = new_serv(IPPROTO_SCTP, 1) );
-                CHECK_MALLOC( s->conn = fd_cnx_serv_sctp(fd_g_config->cnf_port_tls, FD_IS_LIST_EMPTY(&fd_g_config->cnf_endpoints) ? NULL : &fd_g_config->cnf_endpoints) );
-                fd_list_insert_before( &FD_SERVERS, &s->chain );
-                CHECK_POSIX( pthread_create( &s->thr, NULL, serv_th, s ) );
+                if (fd_g_config->cnf_port_tls) {
+                        CHECK_MALLOC( s = new_serv(IPPROTO_SCTP, 1) );
+                        CHECK_MALLOC( s->conn = fd_cnx_serv_sctp(fd_g_config->cnf_port_tls, empty_conf_ep ? NULL : &fd_g_config->cnf_endpoints) );
+                        fd_list_insert_before( &FD_SERVERS, &s->chain );
+                        CHECK_POSIX( pthread_create( &s->thr, NULL, serv_th, s ) );
+                }
                 
 #endif /* DISABLE_SCTP */
@@ -287 +318 @@
                         if (!fd_g_config->cnf_flags.no_ip4) {
                                 
-                                CHECK_MALLOC( s = new_serv(IPPROTO_TCP, 0) );
-                                CHECK_MALLOC( s->conn = fd_cnx_serv_tcp(fd_g_config->cnf_port, AF_INET, NULL) );
-                                fd_list_insert_before( &FD_SERVERS, &s->chain );
-                                CHECK_POSIX( pthread_create( &s->thr, NULL, serv_th, s ) );
-
-                                CHECK_MALLOC( s = new_serv(IPPROTO_TCP, 1) );
-                                CHECK_MALLOC( s->conn = fd_cnx_serv_tcp(fd_g_config->cnf_port_tls, AF_INET, NULL) );
-                                fd_list_insert_before( &FD_SERVERS, &s->chain );
-                                CHECK_POSIX( pthread_create( &s->thr, NULL, serv_th, s ) );
+                                if (fd_g_config->cnf_port) {
+                                        CHECK_MALLOC( s = new_serv(IPPROTO_TCP, 0) );
+                                        CHECK_MALLOC( s->conn = fd_cnx_serv_tcp(fd_g_config->cnf_port, AF_INET, NULL) );
+                                        fd_list_insert_before( &FD_SERVERS, &s->chain );
+                                        CHECK_POSIX( pthread_create( &s->thr, NULL, serv_th, s ) );
+                                }
+
+                                if (fd_g_config->cnf_port_tls) {
+                                        CHECK_MALLOC( s = new_serv(IPPROTO_TCP, 1) );
+                                        CHECK_MALLOC( s->conn = fd_cnx_serv_tcp(fd_g_config->cnf_port_tls, AF_INET, NULL) );
+                                        fd_list_insert_before( &FD_SERVERS, &s->chain );
+                                        CHECK_POSIX( pthread_create( &s->thr, NULL, serv_th, s ) );
+                                }
                         }
+                        
                         /* Bind TCP servers on [::] */
                         if (!fd_g_config->cnf_flags.no_ip6) {
-                                
-                                CHECK_MALLOC( s = new_serv(IPPROTO_TCP, 0) );
-                                CHECK_MALLOC( s->conn = fd_cnx_serv_tcp(fd_g_config->cnf_port, AF_INET6, NULL) );
-                                fd_list_insert_before( &FD_SERVERS, &s->chain );
-                                CHECK_POSIX( pthread_create( &s->thr, NULL, serv_th, s ) );
-
-                                CHECK_MALLOC( s = new_serv(IPPROTO_TCP, 1) );
-                                CHECK_MALLOC( s->conn = fd_cnx_serv_tcp(fd_g_config->cnf_port_tls, AF_INET6, NULL) );
-                                fd_list_insert_before( &FD_SERVERS, &s->chain );
-                                CHECK_POSIX( pthread_create( &s->thr, NULL, serv_th, s ) );
+
+                                if (fd_g_config->cnf_port) {
+                                        CHECK_MALLOC( s = new_serv(IPPROTO_TCP, 0) );
+                                        CHECK_MALLOC( s->conn = fd_cnx_serv_tcp(fd_g_config->cnf_port, AF_INET6, NULL) );
+                                        fd_list_insert_before( &FD_SERVERS, &s->chain );
+                                        CHECK_POSIX( pthread_create( &s->thr, NULL, serv_th, s ) );
+                                }
+
+                                if (fd_g_config->cnf_port_tls) {
+                                        CHECK_MALLOC( s = new_serv(IPPROTO_TCP, 1) );
+                                        CHECK_MALLOC( s->conn = fd_cnx_serv_tcp(fd_g_config->cnf_port_tls, AF_INET6, NULL) );
+                                        fd_list_insert_before( &FD_SERVERS, &s->chain );
+                                        CHECK_POSIX( pthread_create( &s->thr, NULL, serv_th, s ) );
+                                }
                         }
                 } else {
@@ -323 +363 @@
                                         continue;
                                 
-                                CHECK_MALLOC( s = new_serv(IPPROTO_TCP, 0) );
-                                CHECK_MALLOC( s->conn = fd_cnx_serv_tcp(fd_g_config->cnf_port, sa->sa_family, ep) );
-                                fd_list_insert_before( &FD_SERVERS, &s->chain );
-                                CHECK_POSIX( pthread_create( &s->thr, NULL, serv_th, s ) );
-
-                                CHECK_MALLOC( s = new_serv(IPPROTO_TCP, 1) );
-                                CHECK_MALLOC( s->conn = fd_cnx_serv_tcp(fd_g_config->cnf_port_tls, sa->sa_family, ep) );
-                                fd_list_insert_before( &FD_SERVERS, &s->chain );
-                                CHECK_POSIX( pthread_create( &s->thr, NULL, serv_th, s ) );
+                                if (fd_g_config->cnf_port) {
+                                        CHECK_MALLOC( s = new_serv(IPPROTO_TCP, 0) );
+                                        CHECK_MALLOC( s->conn = fd_cnx_serv_tcp(fd_g_config->cnf_port, sa->sa_family, ep) );
+                                        fd_list_insert_before( &FD_SERVERS, &s->chain );
+                                        CHECK_POSIX( pthread_create( &s->thr, NULL, serv_th, s ) );
+                                }
+
+                                if (fd_g_config->cnf_port_tls) {
+                                        CHECK_MALLOC( s = new_serv(IPPROTO_TCP, 1) );
+                                        CHECK_MALLOC( s->conn = fd_cnx_serv_tcp(fd_g_config->cnf_port_tls, sa->sa_family, ep) );
+                                        fd_list_insert_before( &FD_SERVERS, &s->chain );
+                                        CHECK_POSIX( pthread_create( &s->thr, NULL, serv_th, s ) );
+                                }
                         }
                 }
         }
         
-        /* Now, if we still have not got the list of local adresses, try to read it from the kernel directly */
-        if (FD_IS_LIST_EMPTY(&fd_g_config->cnf_endpoints)) {
+        /* Now, if we had an empty list of local adresses (no address configured), try to read the real addresses from the kernel */
+        if (empty_conf_ep) {
                 CHECK_FCT(fd_cnx_get_local_eps(&fd_g_config->cnf_endpoints));
                 if (FD_IS_LIST_EMPTY(&fd_g_config->cnf_endpoints)) {
-                        TRACE_DEBUG(INFO, "Unable to find the addresses of the local system. Please use \"ListenOn\" parameter in the configuration.");
+                        TRACE_DEBUG(INFO, "Unable to find the address(es) of the local system.\n" 
+                                        "Please use \"ListenOn\" parameter in the configuration.\n"
+                                        "This information is required to generate the CER/CEA messages.\n");
                         return EINVAL;
                 }