Index: extensions/test_netemul/tne_process.c
===================================================================
--- extensions/test_netemul/tne_process.c	(revision 656)
+++ extensions/test_netemul/tne_process.c	(revision 691)
@@ -247,37 +247,33 @@
 	
 	CHECK_POSIX_DO( pthread_mutex_lock(&mtx), goto error );
+	pthread_cleanup_push( fd_cleanup_mutex, &mtx );
 	
 	/* The loop */
 	while (1) {
 		/* First, test if we are canceled */
-		CHECK_POSIX_DO( pthread_mutex_unlock(&mtx), goto error );
 		pthread_testcancel();
-		CHECK_POSIX_DO( pthread_mutex_lock(&mtx), goto error );
-		
-		pthread_cleanup_push( fd_cleanup_mutex, &mtx );
 		
 		/* Send all messages that are ready (free resources before using new ones) */
-		CHECK_FCT_DO( send_all_ready(), goto error_ul );
+		CHECK_FCT_DO( send_all_ready(), break );
 		
 		/* Now process the new messages in input list for duplicate filter */
-		CHECK_FCT_DO( do_duplicates(), goto error_ul );
+		CHECK_FCT_DO( do_duplicates(), break );
 		
 		/* Now compute the latency for each new item */
-		CHECK_FCT_DO( do_latency(), goto error_ul );
+		CHECK_FCT_DO( do_latency(), break );
 		
 		/* Now, wait then loop */
 		if (FD_IS_LIST_EMPTY(&waitlist)) {
-			CHECK_POSIX_DO( pthread_cond_wait(&cnd, &mtx), goto error_ul );
+			CHECK_POSIX_DO( pthread_cond_wait(&cnd, &mtx), break );
 		} else {
 			CHECK_POSIX_DO2( pthread_cond_timedwait(&cnd, &mtx, &((struct process_item *)(waitlist.next))->ts), 
 				ETIMEDOUT, /* ETIMEDOUT is a normal return value, continue */,
-					/* on other error, */ goto error_ul );
+					/* on other error, */ break );
 		}
 		
-		pthread_cleanup_pop( 0 );
 		/* loop */
 	}
 
-error_ul:
+	pthread_cleanup_pop( 0 );
 	CHECK_POSIX_DO( pthread_mutex_unlock(&mtx),  );
 error:
Index: libfdcore/core.c
===================================================================
--- libfdcore/core.c	(revision 689)
+++ libfdcore/core.c	(revision 691)
@@ -241,4 +241,6 @@
 int fd_core_waitstartcomplete(void)
 {
+	int ret = 0;
+	
 	TRACE_ENTRY("");
 	
@@ -246,6 +248,6 @@
 	pthread_cleanup_push( fd_cleanup_mutex, &is_ready_mtx );
 	
-	while (!is_ready) {
-		CHECK_POSIX( pthread_cond_wait( &is_ready_cnd, &is_ready_mtx ) );
+	while (!ret && !is_ready) {
+		CHECK_POSIX_DO( ret = pthread_cond_wait( &is_ready_cnd, &is_ready_mtx ),  );
 	}
 	
@@ -253,5 +255,5 @@
 	CHECK_POSIX( pthread_mutex_unlock( &is_ready_mtx ) );
 	
-	return 0;
+	return ret;
 }
 
Index: libfdcore/p_cnx.c
===================================================================
--- libfdcore/p_cnx.c	(revision 662)
+++ libfdcore/p_cnx.c	(revision 691)
@@ -211,4 +211,5 @@
 	struct next_conn * nc = NULL;
 	int rebuilt = 0;
+	int fatal_error=0;
 	
 	TRACE_ENTRY("%p", arg);
@@ -226,5 +227,5 @@
 		if (FD_IS_LIST_EMPTY(&peer->p_connparams)) {
 			if (! rebuilt) {
-				CHECK_FCT_DO( prepare_connection_list(peer), goto fatal_error );
+				CHECK_FCT_DO( fatal_error = prepare_connection_list(peer), goto out );
 				rebuilt ++;
 			}
@@ -232,5 +233,5 @@
 				/* We encountered an error or we have looped over all the addresses of the peer. */
 				TRACE_DEBUG(INFO, "Unable to connect to the peer %s, aborting attempts for now.", peer->p_hdr.info.pi_diamid);
-				CHECK_FCT_DO( fd_event_send(peer->p_events, FDEVP_CNX_FAILED, 0, NULL), goto fatal_error );
+				CHECK_FCT_DO( fatal_error = fd_event_send(peer->p_events, FDEVP_CNX_FAILED, 0, NULL), goto out );
 				return NULL;
 			}
@@ -277,25 +278,28 @@
 				empty_connection_list(peer);
 				fd_ep_filter(&peer->p_hdr.info.pi_endpoints, EP_FL_CONF);
-				return NULL;
+				goto out_pop;
 			} );
 	} else {
 		/* Prepare to receive the next message */
-		CHECK_FCT_DO( fd_cnx_start_clear(cnx, 0), goto fatal_error );
+		CHECK_FCT_DO( fatal_error = fd_cnx_start_clear(cnx, 0), goto out_pop );
 	}
 	
 	/* Upon success, generate FDEVP_CNX_ESTABLISHED */
-	CHECK_FCT_DO( fd_event_send(peer->p_events, FDEVP_CNX_ESTABLISHED, 0, cnx), goto fatal_error );
-	
+	CHECK_FCT_DO( fatal_error = fd_event_send(peer->p_events, FDEVP_CNX_ESTABLISHED, 0, cnx),  );
+out_pop:
+	;	
 	pthread_cleanup_pop(0);
 	
-	return NULL;
-	
-fatal_error:
-	/* Cleanup the connection */
-	if (cnx)
-		fd_cnx_destroy(cnx);
-
-	/* Generate a termination event */
-	CHECK_FCT_DO(fd_event_send(fd_g_config->cnf_main_ev, FDEV_TERMINATE, 0, NULL), );
+out:
+	
+	if (fatal_error) {
+	
+		/* Cleanup the connection */
+		if (cnx)
+			fd_cnx_destroy(cnx);
+
+		/* Generate a termination event */
+		CHECK_FCT_DO(fd_event_send(fd_g_config->cnf_main_ev, FDEV_TERMINATE, 0, NULL), );
+	}
 	
 	return NULL;
Index: libfdcore/p_expiry.c
===================================================================
--- libfdcore/p_expiry.c	(revision 686)
+++ libfdcore/p_expiry.c	(revision 691)
@@ -98,5 +98,5 @@
 	TRACE_ENTRY( "%p", arg );
 	
-	CHECK_POSIX_DO( pthread_mutex_lock(&exp_mtx),  goto error );
+	CHECK_POSIX_DO( pthread_mutex_lock(&exp_mtx), { ASSERT(0); } );
 	pthread_cleanup_push( fd_cleanup_mutex, &exp_mtx );
 	
@@ -108,5 +108,5 @@
 		if (FD_IS_LIST_EMPTY(&exp_list)) {
 			/* Just wait for a change or cancelation */
-			CHECK_POSIX_DO( pthread_cond_wait( &exp_cnd, &exp_mtx ), goto error );
+			CHECK_POSIX_DO( pthread_cond_wait( &exp_cnd, &exp_mtx ), { ASSERT(0); } );
 			/* Restart the loop on wakeup */
 			continue;
@@ -118,5 +118,5 @@
 		
 		/* Get the current time */
-		CHECK_SYS_DO(  clock_gettime(CLOCK_REALTIME, &now),  goto error  );
+		CHECK_SYS_DO(  clock_gettime(CLOCK_REALTIME, &now),  { ASSERT(0); }  );
 
 		/* If first peer is not expired, we just wait until it happens */
@@ -125,5 +125,5 @@
 			CHECK_POSIX_DO2(  pthread_cond_timedwait( &exp_cnd, &exp_mtx, &first->p_exp_timer ),  
 					ETIMEDOUT, /* ETIMEDOUT is a normal return value, continue */,
-					/* on other error, */ goto error );
+					/* on other error, */ { ASSERT(0); } );
 	
 			/* on wakeup, loop */
@@ -133,12 +133,11 @@
 		/* Now, the first peer in the list is expired; signal it */
 		fd_list_unlink( &first->p_expiry );
-		CHECK_FCT_DO( fd_event_send(first->p_events, FDEVP_TERMINATE, 0, "DO_NOT_WANT_TO_TALK_TO_YOU"), goto error );
+		CHECK_FCT_DO( fd_event_send(first->p_events, FDEVP_TERMINATE, 0, "DO_NOT_WANT_TO_TALK_TO_YOU"), break );
 		
 	} while (1);
 	
 	pthread_cleanup_pop( 1 );
-error:
+
 	TRACE_DEBUG(INFO, "An error occurred in peers module! Expiry thread is terminating...");
-	ASSERT(0);
 	CHECK_FCT_DO(fd_event_send(fd_g_config->cnf_main_ev, FDEV_TERMINATE, 0, NULL), );
 	return NULL;
Index: libfdcore/p_out.c
===================================================================
--- libfdcore/p_out.c	(revision 688)
+++ libfdcore/p_out.c	(revision 691)
@@ -69,10 +69,15 @@
 	/* Save a request before sending so that there is no race condition with the answer */
 	if (msg_is_a_req) {
-		CHECK_FCT_DO( ret = fd_p_sr_store(srl, msg, &hdr->msg_hbhid, bkp_hbh), { free(buf); return ret; } );
+		CHECK_FCT_DO( ret = fd_p_sr_store(srl, msg, &hdr->msg_hbhid, bkp_hbh), goto out );
 	}
 	
 	/* Send the message */
-	CHECK_FCT_DO( ret = fd_cnx_send(cnx, buf, sz, flags), { free(buf); return ret; } );
+	CHECK_FCT_DO( ret = fd_cnx_send(cnx, buf, sz, flags), );
+out:
+	;	
 	pthread_cleanup_pop(1);
+	
+	if (ret)
+		return ret;
 	
 	/* Free remaining messages (i.e. answers) */
Index: libfdcore/p_psm.c
===================================================================
--- libfdcore/p_psm.c	(revision 688)
+++ libfdcore/p_psm.c	(revision 691)
@@ -55,15 +55,16 @@
 static int fd_psm_waitstart()
 {
+	int ret = 0;
 	TRACE_ENTRY("");
 	CHECK_POSIX( pthread_mutex_lock(&started_mtx) );
 awake:	
-	if (! started) {
+	if (!ret && !started) {
 		pthread_cleanup_push( fd_cleanup_mutex, &started_mtx );
-		CHECK_POSIX( pthread_cond_wait(&started_cnd, &started_mtx) );
+		CHECK_POSIX_DO( ret = pthread_cond_wait(&started_cnd, &started_mtx), );
 		pthread_cleanup_pop( 0 );
 		goto awake;
 	}
 	CHECK_POSIX( pthread_mutex_unlock(&started_mtx) );
-	return 0;
+	return ret;
 }
 
Index: libfdcore/p_sr.c
===================================================================
--- libfdcore/p_sr.c	(revision 688)
+++ libfdcore/p_sr.c	(revision 691)
@@ -117,5 +117,5 @@
 }
 
-/* thread that handles messages expiring. The thread is started / stopped only when needed */
+/* thread that handles messages expiring. The thread is started / cancelled only when needed */
 static void * sr_expiry_th(void * arg) {
 	struct sr_list * srlist = arg;
@@ -184,4 +184,5 @@
 	; /* pthread_cleanup_pop sometimes expands as "} ..." and the label beofre this cause some compilers to complain... */
 	pthread_cleanup_pop( 1 );
+	ASSERT(0); /* we have encountered a problem, maybe time to signal the framework to terminate? */
 	return NULL;
 }
Index: libfdcore/peers.c
===================================================================
--- libfdcore/peers.c	(revision 688)
+++ libfdcore/peers.c	(revision 691)
@@ -533,6 +533,8 @@
 		int auth = 0;
 		pthread_cleanup_push(fd_cleanup_rwlock, &validators_rw);
-		CHECK_FCT_DO( ret = ((int(*)(struct peer_info *, int *, int (**)(struct peer_info *)))(v->o)) (&peer->p_hdr.info, &auth, &peer->p_cb2), goto out );
+		CHECK_FCT_DO( ret = ((int(*)(struct peer_info *, int *, int (**)(struct peer_info *)))(v->o)) (&peer->p_hdr.info, &auth, &peer->p_cb2),  );
 		pthread_cleanup_pop(0);
+		if (ret)
+			goto out;
 		if (auth) {
 			ret = (auth > 0) ? 0 : -1;
Index: libfdcore/routing_dispatch.c
===================================================================
--- libfdcore/routing_dispatch.c	(revision 688)
+++ libfdcore/routing_dispatch.c	(revision 691)
@@ -1000,7 +1000,7 @@
 		{
 			int must_stop;
-			CHECK_POSIX_DO( pthread_mutex_lock(&order_lock), goto end ); /* we lock to flush the caches */
+			CHECK_POSIX_DO( pthread_mutex_lock(&order_lock), { ASSERT(0); } ); /* we lock to flush the caches */
 			must_stop = (order_val == STOP);
-			CHECK_POSIX_DO( pthread_mutex_unlock(&order_lock), goto end );
+			CHECK_POSIX_DO( pthread_mutex_unlock(&order_lock), { ASSERT(0); } );
 			if (must_stop)
 				goto end;
Index: libfdcore/sctp.c
===================================================================
--- libfdcore/sctp.c	(revision 662)
+++ libfdcore/sctp.c	(revision 691)
@@ -822,10 +822,10 @@
 	
 	/* Set the socket options */
-	CHECK_FCT_DO( ret = fd_setsockopt_prebind(*sock), goto fail );
+	CHECK_FCT_DO( ret = fd_setsockopt_prebind(*sock), goto out );
 	
 	/* Create the array of addresses, add first the configured addresses, then the discovered, then the other ones */
-	CHECK_FCT_DO( ret = add_addresses_from_list_mask(&sar.buf, &size, &count, family, htons(port), list, EP_FL_CONF,              EP_FL_CONF	), goto fail );
-	CHECK_FCT_DO( ret = add_addresses_from_list_mask(&sar.buf, &size, &count, family, htons(port), list, EP_FL_CONF | EP_FL_DISC, EP_FL_DISC	), goto fail );
-	CHECK_FCT_DO( ret = add_addresses_from_list_mask(&sar.buf, &size, &count, family, htons(port), list, EP_FL_CONF | EP_FL_DISC, 0		), goto fail );
+	CHECK_FCT_DO( ret = add_addresses_from_list_mask(&sar.buf, &size, &count, family, htons(port), list, EP_FL_CONF,              EP_FL_CONF	), goto out );
+	CHECK_FCT_DO( ret = add_addresses_from_list_mask(&sar.buf, &size, &count, family, htons(port), list, EP_FL_CONF | EP_FL_DISC, EP_FL_DISC	), goto out );
+	CHECK_FCT_DO( ret = add_addresses_from_list_mask(&sar.buf, &size, &count, family, htons(port), list, EP_FL_CONF | EP_FL_DISC, 0		), goto out );
 	
 	/* Try connecting */
@@ -866,5 +866,5 @@
 		/* Some errors are expected, we log at different level */
 		TRACE_DEBUG( lvl, "sctp_connectx returned an error: %s", strerror(ret));
-		goto fail;
+		goto out;
 	}
 	
@@ -872,18 +872,20 @@
 	
 	/* Set the remaining sockopts */
-	CHECK_FCT_DO( ret = fd_setsockopt_postbind(*sock, 1), goto fail_deco );
-	
-	/* Done! */
+	CHECK_FCT_DO( ret = fd_setsockopt_postbind(*sock, 1), 
+		{ 
+			CHECK_SYS_DO( shutdown(*sock, SHUT_RDWR), /* continue */ );
+		} );
+	
+out:
+	;
 	pthread_cleanup_pop(0);
-	return 0;
-	
-fail_deco:
-	CHECK_SYS_DO( shutdown(*sock, SHUT_RDWR), /* continue */ );
-fail:
-	if (*sock > 0) {
-		CHECK_SYS_DO( close(*sock), /* continue */ );
-		*sock = -1;
-	}
-	free(sar.buf);
+	
+	if (ret) {
+		if (*sock > 0) {
+			CHECK_SYS_DO( close(*sock), /* continue */ );
+			*sock = -1;
+		}
+		free(sar.buf);
+	}
 	return ret;
 }
Index: libfdcore/server.c
===================================================================
--- libfdcore/server.c	(revision 688)
+++ libfdcore/server.c	(revision 691)
@@ -151,5 +151,5 @@
 	pthread_cleanup_push((void *)fd_cnx_destroy, c->conn);
 	pthread_cleanup_push((void *)fd_msg_free, msg);
-	CHECK_FCT_DO( fd_peer_handle_newCER( &msg, &c->conn ), goto cleanup );
+	CHECK_FCT_DO( fd_peer_handle_newCER( &msg, &c->conn ),  );
 	pthread_cleanup_pop(0);
 	pthread_cleanup_pop(0);
Index: libfdcore/tcp.c
===================================================================
--- libfdcore/tcp.c	(revision 662)
+++ libfdcore/tcp.c	(revision 691)
@@ -134,9 +134,9 @@
 	CHECK_SYS(  s = socket(sa->sa_family, SOCK_STREAM, IPPROTO_TCP)  );
 	
+	/* Set the socket options */
+	CHECK_FCT(  fd_tcp_setsockopt(sa->sa_family, s)  );
+	
 	/* Cleanup if we are cancelled */
 	pthread_cleanup_push(fd_cleanup_socket, &s);
-	
-	/* Set the socket options */
-	CHECK_FCT(  fd_tcp_setsockopt(sa->sa_family, s)  );
 	
 	TRACE_DEBUG_sSA(FULL, "Attempting TCP connection with peer: ", sa, NI_NUMERICHOST | NI_NUMERICSERV, "..." );
Index: libfdproto/sessions.c
===================================================================
--- libfdproto/sessions.c	(revision 686)
+++ libfdproto/sessions.c	(revision 691)
@@ -176,5 +176,5 @@
 		if (FD_IS_LIST_EMPTY(&exp_sentinel)) {
 			/* Just wait for a change or cancelation */
-			CHECK_POSIX_DO( pthread_cond_wait( &exp_cond, &exp_lock ), break );
+			CHECK_POSIX_DO( pthread_cond_wait( &exp_cond, &exp_lock ), break /* this might not pop the cleanup handler, but since we ASSERT(0), it is not the big issue... */ );
 			/* Restart the loop on wakeup */
 			goto again;
@@ -207,5 +207,5 @@
 	} while (1);
 	
-	TRACE_DEBUG(INFO, "An error occurred in session module! Expiry thread is terminating...");
+	TRACE_DEBUG(INFO, "A system error occurred in session module! Expiry thread is terminating...");
 	ASSERT(0);
 	return NULL;
@@ -442,5 +442,5 @@
 		/* We added a new expiring element, we must signal */
 		if (li == &exp_sentinel) {
-			CHECK_POSIX( pthread_cond_signal(&exp_cond) );
+			CHECK_POSIX_DO( pthread_cond_signal(&exp_cond), { ASSERT(0); } ); /* if it fails, we might not pop the cleanup handlers, but this should not happen -- and we'd have a serious problem otherwise */
 		}
 		
@@ -458,5 +458,5 @@
 		/* We're done */
 		pthread_cleanup_pop(0);
-		CHECK_POSIX( pthread_mutex_unlock( &exp_lock ) );
+		CHECK_POSIX_DO( pthread_mutex_unlock( &exp_lock ), { ASSERT(0); } ); /* if it fails, we might not pop the cleanup handler, but this should not happen -- and we'd have a serious problem otherwise */
 	}
 	
@@ -540,5 +540,5 @@
 	/* We added a new expiring element, we must signal if it was in first position */
 	if (session->expire.prev == &exp_sentinel) {
-		CHECK_POSIX( pthread_cond_signal(&exp_cond) );
+		CHECK_POSIX_DO( pthread_cond_signal(&exp_cond), { ASSERT(0); /* so that we don't have a pending cancellation handler */ } );
 	}
 
@@ -573,13 +573,13 @@
 	
 	/* Unlink and invalidate */
-	CHECK_FCT( pthread_mutex_lock( H_LOCK(sess->hash) ) );
+	CHECK_POSIX( pthread_mutex_lock( H_LOCK(sess->hash) ) );
 	pthread_cleanup_push( fd_cleanup_mutex, H_LOCK(sess->hash) );
-	CHECK_FCT( pthread_mutex_lock( &exp_lock ) );
+	CHECK_POSIX_DO( pthread_mutex_lock( &exp_lock ), { ASSERT(0); /* otherwise cleanup handler is not pop'd */ } );
 	fd_list_unlink( &sess->chain_h );
 	fd_list_unlink( &sess->expire ); /* no need to signal the condition here */
 	sess->eyec = 0xdead;
-	CHECK_FCT( pthread_mutex_unlock( &exp_lock ) );
+	CHECK_POSIX_DO( pthread_mutex_unlock( &exp_lock ), { ASSERT(0); /* otherwise cleanup handler is not pop'd */ } );
 	pthread_cleanup_pop(0);
-	CHECK_FCT( pthread_mutex_unlock( H_LOCK(sess->hash) ) );
+	CHECK_POSIX( pthread_mutex_unlock( H_LOCK(sess->hash) ) );
 	
 	/* Now destroy all states associated -- we don't take the lock since nobody can access this session anymore (in theory) */
@@ -614,5 +614,5 @@
 	CHECK_POSIX( pthread_mutex_lock( H_LOCK(sess->hash) ) );
 	pthread_cleanup_push( fd_cleanup_mutex, H_LOCK(sess->hash) );
-	CHECK_POSIX( pthread_mutex_lock( &exp_lock ) );
+	CHECK_POSIX_DO( pthread_mutex_lock( &exp_lock ), { ASSERT(0); /* otherwise, cleanup not poped on FreeBSD */ } );
 	if (FD_IS_LIST_EMPTY(&sess->states)) {
 		fd_list_unlink( &sess->chain_h );
@@ -622,5 +622,5 @@
 		free(sess);
 	}
-	CHECK_POSIX( pthread_mutex_unlock( &exp_lock ) );
+	CHECK_POSIX_DO( pthread_mutex_unlock( &exp_lock ), { ASSERT(0); /* otherwise, cleanup not poped on FreeBSD */ } );
 	pthread_cleanup_pop(0);
 	CHECK_POSIX( pthread_mutex_unlock( H_LOCK(hash) ) );
@@ -635,4 +635,5 @@
 	struct fd_list * li;
 	int already = 0;
+	int ret = 0;
 	
 	TRACE_ENTRY("%p %p %p", handler, session, state);
@@ -644,5 +645,5 @@
 			
 	/* Create the new state object */
-	CHECK_MALLOC(new = malloc(sizeof(struct state)) );
+	CHECK_MALLOC_DO(new = malloc(sizeof(struct state)), { ret = ENOMEM; goto out; } );
 	memset(new, 0, sizeof(struct state));
 	
@@ -661,5 +662,5 @@
 		if (st->hdl->id == handler->id) {
 			TRACE_DEBUG(INFO, "A state was already stored for session '%s' and handler '%p', at location %p", session->sid, st->hdl, st->state);
-			already = 1;
+			already = EALREADY;
 		}
 		
@@ -673,9 +674,10 @@
 		free(new);
 	}
-	
+out:
+	;	
 	pthread_cleanup_pop(0);
 	CHECK_POSIX( pthread_mutex_unlock(&session->stlock) );
 	
-	return already ? EALREADY : 0;
+	return ret ?: already;
 }
 
