CVE-2013-0281: Resource Management Errors (CWE-399)
value: Pacemaker 1.1.10, when remote Cluster Information Base (CIB) configuration or resource management is enabled, does not limit the duration of connections to the blocking sockets, which allows remote attackers to cause a denial of service (connection blocking).
CWE description: Weaknesses in this category are related to improper management of system resources.
Published: 2013-11-23T11:55Z
Repository: https://github.com/ClusterLabs/pacemaker
Language: C
Files changed: cib/callbacks.c, cib/callbacks.h, cib/notify.c, cib/remote.c, include/crm_internal.h, lib/cib/cib_remote.c, lib/common/mainloop.c, lib/common/remote.c, tools/crm_mon.c
Commit date: 2013-01-18 14:43:01 -0600
Commit message: High: core: Internal tls api improvements for reuse with future LRMD tls backend.

Diff: diff --git a/cib/callbacks.c b/cib/callbacks.c
index 86cfd9226..aa5f6a827 100644
--- a/cib/callbacks.c
+++ b/cib/callbacks.c
@@ -345,17 +345,22 @@ do_local_notify(xmlNode * notify_src, const char *client_id,
 
     } else {
         int rid = 0;
 
         if(sync_reply) {
-            CRM_LOG_ASSERT(client_obj->request_id);
+            if (client_obj->ipc) {
+                CRM_LOG_ASSERT(client_obj->request_id);
 
-            rid = client_obj->request_id;
-            client_obj->request_id = 0;
+                rid = client_obj->request_id;
+                client_obj->request_id = 0;
 
-            crm_trace("Sending response %d to %s %s",
+                crm_trace("Sending response %d to %s %s",
                       rid, client_obj->name, from_peer?"(originator of delegated request)":"");
+            } else {
+                crm_trace("Sending response to %s %s",
+                      client_obj->name, from_peer?"(originator of delegated request)":"");
+            }
 
         } else {
             crm_trace("Sending an event to %s %s",
                       client_obj->name, from_peer?"(originator of delegated request)":"");
         }
diff --git a/cib/callbacks.h b/cib/callbacks.h
index 99a5065bf..b8af997b6 100644
--- a/cib/callbacks.h
+++ b/cib/callbacks.h
@@ -39,29 +39,33 @@ extern GHashTable *config_hash;
 typedef struct cib_client_s {
     char *id;
     char *name;
     char *callback_id;
     char *user;
+    char *recv_buf;
     int request_id;
 
     qb_ipcs_connection_t *ipc;
 
 #ifdef HAVE_GNUTLS_GNUTLS_H
     gnutls_session *session;
+    gboolean handshake_complete;
 #else
     void *session;
 #endif
     gboolean encrypted;
+    gboolean remote_auth;
     mainloop_io_t *remote;
-        
+
     unsigned long num_calls;
 
     int pre_notify;
     int post_notify;
     int confirmations;
     int replace;
     int diffs;
+    int remote_auth_timeout;
 
     GList *delegated_calls;
 } cib_client_t;
 
 typedef struct cib_operation_s {
diff --git a/cib/notify.c b/cib/notify.c
index 9dcb6997a..1dcda8fcd 100644
--- a/cib/notify.c
+++ b/cib/notify.c
@@ -81,11 +81,11 @@ cib_notify_client(gpointer key, gpointer value, gpointer user_data)
     xmlNode *update_msg = user_data;
 
     CRM_CHECK(client != NULL, return TRUE);
     CRM_CHECK(update_msg != NULL, return TRUE);
 
-    if (client->ipc == NULL) {
+    if (client->ipc == NULL && client->session == NULL) {
         crm_warn("Skipping client with NULL channel");
         return FALSE;
     }
 
     type = crm_element_value(update_msg, F_SUBTYPE);
diff --git a/cib/remote.c b/cib/remote.c
index 7ad7132b2..f9b90a074 100644
--- a/cib/remote.c
+++ b/cib/remote.c
@@ -58,34 +58,29 @@
 #    include <pam/pam_appl.h>
 #    define HAVE_PAM 1
 #  endif
 #endif
 
-#ifdef HAVE_DECL_NANOSLEEP
-#  include <time.h>
-#endif
-
 extern int remote_tls_fd;
 extern gboolean cib_shutdown_flag;
 
 int init_remote_listener(int port, gboolean encrypted);
 void cib_remote_connection_destroy(gpointer user_data);
 
 #ifdef HAVE_GNUTLS_GNUTLS_H
 #  define DH_BITS 1024
 gnutls_dh_params dh_params;
-extern gnutls_anon_server_credentials anon_cred_s;
+gnutls_anon_server_credentials anon_cred_s;
 static void
 debug_log(int level, const char *str)
 {
     fputs(str, stderr);
 }
-
-extern gnutls_session *create_tls_session(int csock, int type);
-
 #endif
 
+#define REMOTE_AUTH_TIMEOUT 10000
+
 int num_clients;
 int authenticate_user(const char *user, const char *passwd);
 int cib_remote_listen(gpointer data);
 int cib_remote_msg(gpointer data);
 
@@ -119,11 +114,11 @@ init_remote_listener(int port, gboolean encrypted)
         crm_warn("TLS support is not available");
         return 0;
 #else
         crm_notice("Starting a tls listener on port %d.", port);
         gnutls_global_init();
-/* 	gnutls_global_set_log_level (10); */
+        /* gnutls_global_set_log_level (10); */
         gnutls_global_set_log_function(debug_log);
         gnutls_dh_params_init(&dh_params);
         gnutls_dh_params_generate2(dh_params, DH_BITS);
         gnutls_anon_allocate_server_credentials(&anon_cred_s);
         gnutls_anon_set_server_dh_params(anon_cred_s, dh_params);
@@ -213,41 +208,93 @@ check_group_membership(const char *usr, const char *grp)
     };
 
     return FALSE;
 }
 
+static gboolean
+cib_remote_auth(xmlNode *login)
+{
+    const char *user = NULL;
+    const char *pass = NULL;
+    const char *tmp = NULL;
+
+    crm_log_xml_info(login, "Login: ");
+    if (login == NULL) {
+        return FALSE;
+    }
+
+    tmp = crm_element_name(login);
+    if (safe_str_neq(tmp, "cib_command")) {
+        crm_err("Wrong tag: %s", tmp);
+        return FALSE;
+    }
+
+    tmp = crm_element_value(login, "op");
+    if (safe_str_neq(tmp, "authenticate")) {
+        crm_err("Wrong operation: %s", tmp);
+        return FALSE;
+    }
+
+    user = crm_element_value(login, "user");
+    pass = crm_element_value(login, "password");
+
+    if (!user || !pass) {
+        crm_err("missing auth credentials");
+        return FALSE;
+    }
+
+    /* Non-root daemons can only validate the password of the
+     * user they're running as
+     */
+    if (check_group_membership(user, CRM_DAEMON_GROUP) == FALSE) {
+        crm_err("User is not a member of the required group");
+        return FALSE;
+
+    } else if (authenticate_user(user, pass) == FALSE) {
+        crm_err("PAM auth failed");
+        return FALSE;
+    }
+
+    return TRUE;
+}
+
+static gboolean
+remote_auth_timeout_cb(gpointer data)
+{
+    cib_client_t *client = data;
+
+    client->remote_auth_timeout = 0;
+
+    if (client->remote_auth == TRUE) {
+        return FALSE;
+    }
+
+    mainloop_del_fd(client->remote);
+    crm_err("Remote client authentication timed out");
+
+    return FALSE;
+}
 int
 cib_remote_listen(gpointer data)
 {
-    int lpc = 0;
     int csock = 0;
     unsigned laddr;
-    time_t now = 0;
-    time_t start = time(NULL);
     struct sockaddr_in addr;
     int ssock = *(int *)data;
+    int flag;
 
 #ifdef HAVE_GNUTLS_GNUTLS_H
     gnutls_session *session = NULL;
 #endif
     cib_client_t *new_client = NULL;
 
-    xmlNode *login = NULL;
-    const char *user = NULL;
-    const char *pass = NULL;
-    const char *tmp = NULL;
-
-#ifdef HAVE_DECL_NANOSLEEP
-    const struct timespec sleepfast = { 0, 10000000 };  /* 10 millisec */
-#endif
-
     static struct mainloop_fd_callbacks remote_client_fd_callbacks = 
         {
             .dispatch = cib_remote_msg,
             .destroy = cib_remote_connection_destroy,
-        };    
-    
+        };
+
     /* accept the connection */
     laddr = sizeof(addr);
     csock = accept(ssock, (struct sockaddr *)&addr, &laddr);
     crm_debug("New %s connection from %s",
               ssock == remote_tls_fd ? "secure" : "clear-text", inet_ntoa(addr.sin_addr));
@@ -255,129 +302,63 @@ cib_remote_listen(gpointer data)
     if (csock == -1) {
         crm_err("accept socket failed");
         return TRUE;
     }
 
+    if ((flag = fcntl(csock, F_GETFL)) >= 0) {
+        if (fcntl(csock, F_SETFL, flag | O_NONBLOCK) < 0) {
+            crm_err( "fcntl() write failed");
+            close(csock);
+            return TRUE;
+        }
+    } else {
+        crm_err( "fcntl() read failed");
+        close(csock);
+        return TRUE;
+    }
+
     if (ssock == remote_tls_fd) {
 #ifdef HAVE_GNUTLS_GNUTLS_H
         /* create gnutls session for the server socket */
-        session = create_tls_session(csock, GNUTLS_SERVER);
+        session = crm_create_anon_tls_session(csock, GNUTLS_SERVER, anon_cred_s);
         if (session == NULL) {
             crm_err("TLS session creation failed");
             close(csock);
             return TRUE;
         }
 #endif
     }
 
-    do {
-        crm_trace("Iter: %d", lpc++);
-        if (ssock == remote_tls_fd) {
-#ifdef HAVE_GNUTLS_GNUTLS_H
-            login = crm_recv_remote_msg(session, TRUE);
-#endif
-        } else {
-            login = crm_recv_remote_msg(GINT_TO_POINTER(csock), FALSE);
-        }
-        if (login != NULL) {
-            break;
-        }
-#ifdef HAVE_DECL_NANOSLEEP
-        nanosleep(&sleepfast, NULL);
-#else
-        sleep(1);
-#endif
-        now = time(NULL);
-
-        /* Peers have 3s to connect */
-    } while (login == NULL && (start - now) < 4);
-
-    crm_log_xml_info(login, "Login: ");
-    if (login == NULL) {
-        goto bail;
-    }
-
-    tmp = crm_element_name(login);
-    if (safe_str_neq(tmp, "cib_command")) {
-        crm_err("Wrong tag: %s", tmp);
-        goto bail;
-    }
-
-    tmp = crm_element_value(login, "op");
-    if (safe_str_neq(tmp, "authenticate")) {
-        crm_err("Wrong operation: %s", tmp);
-        goto bail;
-    }
-
-    user = crm_element_value(login, "user");
-    pass = crm_element_value(login, "password");
-
-    /* Non-root daemons can only validate the password of the
-     * user they're running as
-     */
-    if (check_group_membership(user, CRM_DAEMON_GROUP) == FALSE) {
-        crm_err("User is not a member of the required group");
-        goto bail;
-
-    } else if (authenticate_user(user, pass) == FALSE) {
-        crm_err("PAM auth failed");
-        goto bail;
-    }
-
-    /* send ACK */
     num_clients++;
     new_client = calloc(1, sizeof(cib_client_t));
-    new_client->name = crm_element_value_copy(login, "name");
-
-    CRM_CHECK(new_client->id == NULL, free(new_client->id));
     new_client->id = crm_generate_uuid();
-
-#if ENABLE_ACL
-    new_client->user = strdup(user);
-#endif
-
     new_client->callback_id = NULL;
+    /* clients have a few seconds to perform handshake. */
+    new_client->remote_auth_timeout = g_timeout_add(REMOTE_AUTH_TIMEOUT, remote_auth_timeout_cb, new_client);
+
     if (ssock == remote_tls_fd) {
 #ifdef HAVE_GNUTLS_GNUTLS_H
         new_client->encrypted = TRUE;
         new_client->session = session;
 #endif
     } else {
         new_client->session = GINT_TO_POINTER(csock);
     }
 
-    free_xml(login);
-    login = create_xml_node(NULL, "cib_result");
-    crm_xml_add(login, F_CIB_OPERATION, CRM_OP_REGISTER);
-    crm_xml_add(login, F_CIB_CLIENTID, new_client->id);
-    crm_send_remote_msg(new_client->session, login, new_client->encrypted);
-    free_xml(login);
-
     new_client->remote = mainloop_add_fd(
         "cib-remote-client", G_PRIORITY_DEFAULT, csock, new_client, &remote_client_fd_callbacks);
 
     g_hash_table_insert(client_list, new_client->id, new_client);
 
     return TRUE;
-
-  bail:
-    if (ssock == remote_tls_fd) {
-#ifdef HAVE_GNUTLS_GNUTLS_H
-        gnutls_bye(*session, GNUTLS_SHUT_RDWR);
-        gnutls_deinit(*session);
-        gnutls_free(session);
-#endif
-    }
-    close(csock);
-    free_xml(login);
-    return TRUE;
 }
 
 void
 cib_remote_connection_destroy(gpointer user_data)
 {
     cib_client_t *client = user_data;
+    int csock = 0;
 
     if (client == NULL) {
         return;
     }
 
@@ -391,41 +372,58 @@ cib_remote_connection_destroy(gpointer user_data)
     }
 
     crm_trace("Destroying %s (%p)", client->name, user_data);
     num_clients--;
     crm_trace("Num unfree'd clients: %d", num_clients);
+    if (client->remote_auth_timeout) {
+        g_source_remove(client->remote_auth_timeout);
+    }
+
+    if (client->encrypted) {
+#ifdef HAVE_GNUTLS_GNUTLS_H
+        if (client->session) {
+            void *sock_ptr = gnutls_transport_get_ptr(*client->session);
+            csock = GPOINTER_TO_INT(sock_ptr);
+            if (client->handshake_complete) {
+                gnutls_bye(*client->session, GNUTLS_SHUT_WR);
+            }
+            gnutls_deinit(*client->session);
+            gnutls_free(client->session);
+        }
+#endif
+    } else {
+        csock = GPOINTER_TO_INT(client->session);
+    }
+    client->session = NULL;
+
+    if (csock > 0) {
+        close(csock);
+    }
+
     free(client->name);
     free(client->callback_id);
     free(client->id);
     free(client->user);
+    free(client->recv_buf);
     free(client);
     crm_trace("Freed the cib client");
 
     if (cib_shutdown_flag) {
         cib_shutdown(0);
     }
     return;
 }
 
-int
-cib_remote_msg(gpointer data)
+static void
+cib_handle_remote_msg(cib_client_t *client, xmlNode *command)
 {
     const char *value = NULL;
-    xmlNode *command = NULL;
-    cib_client_t *client = data;
-
-    crm_trace("%s callback", client->encrypted ? "secure" : "clear-text");
-
-    command = crm_recv_remote_msg(client->session, client->encrypted);
-    if (command == NULL) {
-        return -1;
-    }
 
     value = crm_element_name(command);
     if (safe_str_neq(value, "cib_command")) {
         crm_log_xml_trace(command, "Bad command: ");
-        goto bail;
+        return;
     }
 
     if (client->name == NULL) {
         value = crm_element_value(command, F_CLIENTNAME);
         if (value == NULL) {
@@ -470,13 +468,99 @@ cib_remote_msg(gpointer data)
         crm_xml_add_int(command, F_CIB_CALLOPTS, 0);
     }
 
     crm_log_xml_trace(command, "Remote command: ");
     cib_common_callback_worker(0, 0, command, client, TRUE);
-  bail:
-    free_xml(command);
-    command = NULL;
+}
+
+int
+cib_remote_msg(gpointer data)
+{
+    xmlNode *command = NULL;
+    cib_client_t *client = data;
+    int disconnected = 0;
+    int timeout = client->remote_auth ? -1 : 1000;
+
+    crm_trace("%s callback", client->encrypted ? "secure" : "clear-text");
+
+#ifdef HAVE_GNUTLS_GNUTLS_H
+    if (client->encrypted && (client->handshake_complete == FALSE)) {
+        int rc = 0;
+
+        /* Muliple calls to handshake will be required, this callback
+         * will be invoked once the client sends more handshake data. */
+        do {
+            rc = gnutls_handshake(*client->session);
+
+            if (rc < 0 && rc != GNUTLS_E_AGAIN) {
+                crm_err("Remote cib tls handshake failed");
+                return -1;
+            }
+        } while (rc == GNUTLS_E_INTERRUPTED);
+
+        if (rc == 0) {
+            crm_debug("Remote cib tls handshake completed");
+            client->handshake_complete = TRUE;
+            if (client->remote_auth_timeout) {
+                g_source_remove(client->remote_auth_timeout);
+            }
+            /* after handshake, clients must send auth in a few seconds */
+            client->remote_auth_timeout = g_timeout_add(REMOTE_AUTH_TIMEOUT, remote_auth_timeout_cb, client);
+        }
+        return 0;
+    }
+#endif
+
+    crm_recv_remote_msg(client->session, &client->recv_buf, client->encrypted, timeout, &disconnected);
+
+    /* must pass auth before we will process anything else */
+    if (client->remote_auth == FALSE) {
+        xmlNode *reg;
+#if ENABLE_ACL
+        const char *user = NULL;
+#endif
+        command = crm_parse_remote_buffer(&client->recv_buf);
+        if (cib_remote_auth(command) == FALSE) {
+            free_xml(command);
+            return -1;
+        }
+
+        crm_debug("remote connection authenticated successfully");
+        client->remote_auth = TRUE;
+        g_source_remove(client->remote_auth_timeout);
+        client->remote_auth_timeout = 0;
+        client->name = crm_element_value_copy(command, "name");
+
+#if ENABLE_ACL
+        user = crm_element_value(command, "user");
+        if (user) {
+           new_client->user = strdup(user);
+        }
+#endif
+
+        /* send ACK */
+        reg = create_xml_node(NULL, "cib_result");
+        crm_xml_add(reg, F_CIB_OPERATION, CRM_OP_REGISTER);
+        crm_xml_add(reg, F_CIB_CLIENTID, client->id);
+        crm_send_remote_msg(client->session, reg, client->encrypted);
+        free_xml(reg);
+        free_xml(command);
+    }
+
+    command = crm_parse_remote_buffer(&client->recv_buf);
+    while (command) {
+        crm_trace("command received");
+        cib_handle_remote_msg(client, command);
+        free_xml(command);
+        command = crm_parse_remote_buffer(&client->recv_buf);
+    }
+
+    if (disconnected) {
+        crm_trace("disconnected while receiving remote cib msg.");
+        return -1;
+    }
+
     return 0;
 }
 
 #ifdef HAVE_PAM
 /* 
diff --git a/include/crm_internal.h b/include/crm_internal.h
index cf6d95d37..388af5931 100644
--- a/include/crm_internal.h
+++ b/include/crm_internal.h
@@ -197,12 +197,44 @@ crm_set_bit(const char *function, const char *target, long long word, long long
 void g_hash_destroy_str(gpointer data);
 
 long long crm_int_helper(const char *text, char **end_text);
 char *crm_concat(const char *prefix, const char *suffix, char join);
 char *generate_hash_key(const char *crm_msg_reference, const char *sys);
-xmlNode *crm_recv_remote_msg(void *session, gboolean encrypted);
-void crm_send_remote_msg(void *session, xmlNode * msg, gboolean encrypted);
+
+
+/*! remote tcp/tls helper functions */
+gboolean crm_recv_remote_msg(void *session, char **recv_buf, gboolean encrypted, int total_timeout_ms, int *disconnected);
+char *crm_recv_remote_raw(void *data, gboolean encrypted, size_t max_recv, size_t *recv_len, int *disconnected);
+int crm_send_remote_msg(void *session, xmlNode * msg, gboolean encrypted);
+int crm_recv_remote_ready(void *session, gboolean encrypted, int timeout_ms);
+xmlNode *crm_parse_remote_buffer(char **msg_buf);
+int crm_remote_tcp_connect(const char *host, int port);
+
+#ifdef HAVE_GNUTLS_GNUTLS_H
+/*!
+ * \internal
+ * \brief Initiate the client handshake after establishing the tcp socket.
+ * \note This is a blocking function, it will block until the entire handshake
+ *       is complete or until the timeout period is reached.
+ * \retval 0 success
+ * \retval negative, failure
+ */
+int crm_initiate_client_tls_handshake(void *session_data, int timeout_ms);
+/*!
+ * \internal
+ * \brief Create client or server session for anon DH encryption credentials
+ * \param sock, the socket the session will use for transport
+ * \param type, GNUTLS_SERVER or GNUTLS_CLIENT
+ * \param credentials, gnutls_anon_server_credentials_t or gnutls_anon_client_credentials_t
+ *
+ * \retval gnutls_session * on success
+ * \retval NULL on failure
+ */
+void *crm_create_anon_tls_session(int sock, int type, void *credentials);
+#endif
+
+#define REMOTE_MSG_TERMINATOR "\r\n\r\n"
 
 const char *daemon_option(const char *option);
 void set_daemon_option(const char *option, const char *value);
 gboolean daemon_option_enabled(const char *daemon, const char *option);
 void strip_text_nodes(xmlNode *xml);
diff --git a/lib/cib/cib_remote.c b/lib/cib/cib_remote.c
index 91bca96eb..ca80c0e07 100644
--- a/lib/cib/cib_remote.c
+++ b/lib/cib/cib_remote.c
@@ -36,18 +36,19 @@
 #include <crm/common/mainloop.h>
 
 #ifdef HAVE_GNUTLS_GNUTLS_H
 #  undef KEYFILE
 #  include <gnutls/gnutls.h>
-extern gnutls_anon_client_credentials anon_cred_c;
-extern gnutls_session *create_tls_session(int csock, int type);
+gnutls_anon_client_credentials anon_cred_c;
+#define DEFAULT_CLIENT_HANDSHAKE_TIMEOUT 5000 /* 5 seconds */
 
 const int kx_prio[] = {
     GNUTLS_KX_ANON_DH,
     0
 };
 
+static gboolean remote_gnutls_credentials_init = FALSE;
 #else
 typedef void gnutls_session;
 #endif
 
 #include <arpa/inet.h>
@@ -59,10 +60,11 @@ struct remote_connection_s {
     int socket;
     gboolean encrypted;
     gnutls_session *session;
     mainloop_io_t *source;
     char *token;
+    char *recv_buf;
 };
 
 typedef struct cib_remote_opaque_s {
     int flags;
     int socket;
@@ -74,11 +76,12 @@ typedef struct cib_remote_opaque_s {
     struct remote_connection_s callback;
 
 } cib_remote_opaque_t;
 
 void cib_remote_connection_destroy(gpointer user_data);
-int cib_remote_dispatch(gpointer user_data);
+int cib_remote_callback_dispatch(gpointer user_data);
+int cib_remote_command_dispatch(gpointer user_data);
 int cib_remote_signon(cib_t * cib, const char *name, enum cib_conn_type type);
 int cib_remote_signoff(cib_t * cib);
 int cib_remote_free(cib_t * cib);
 
 int cib_remote_perform_op(cib_t * cib, const char *op, const char *host, const char *section,
@@ -156,121 +159,95 @@ cib_remote_new(const char *server, const char *user, const char *passwd, int por
 static int
 cib_tls_close(cib_t * cib)
 {
     cib_remote_opaque_t *private = cib->variant_opaque;
 
-    shutdown(private->command.socket, SHUT_RDWR);       /* no more receptions */
-    shutdown(private->callback.socket, SHUT_RDWR);      /* no more receptions */
-    close(private->command.socket);
-    close(private->callback.socket);
-
 #ifdef HAVE_GNUTLS_GNUTLS_H
     if (private->command.encrypted) {
-        gnutls_bye(*(private->command.session), GNUTLS_SHUT_RDWR);
-        gnutls_deinit(*(private->command.session));
-        gnutls_free(private->command.session);
-
-        gnutls_bye(*(private->callback.session), GNUTLS_SHUT_RDWR);
-        gnutls_deinit(*(private->callback.session));
-        gnutls_free(private->callback.session);
+        if (private->command.session) {
+            gnutls_bye(*(private->command.session), GNUTLS_SHUT_RDWR);
+            gnutls_deinit(*(private->command.session));
+            gnutls_free(private->command.session);
+        }
 
-        gnutls_anon_free_client_credentials(anon_cred_c);
-        gnutls_global_deinit();
+        if (private->callback.session) {
+            gnutls_bye(*(private->callback.session), GNUTLS_SHUT_RDWR);
+            gnutls_deinit(*(private->callback.session));
+            gnutls_free(private->callback.session);
+        }
+        private->command.session = NULL;
+        private->callback.session = NULL;
+        if (remote_gnutls_credentials_init) {
+            gnutls_anon_free_client_credentials(anon_cred_c);
+            gnutls_global_deinit();
+            remote_gnutls_credentials_init = FALSE;
+        }
     }
 #endif
+
+    if (private->command.socket) {
+        shutdown(private->command.socket, SHUT_RDWR);       /* no more receptions */
+        close(private->command.socket);
+    }
+    if (private->callback.socket) {
+        shutdown(private->callback.socket, SHUT_RDWR);      /* no more receptions */
+        close(private->callback.socket);
+    }
+    private->command.socket = 0;
+    private->callback.socket = 0;
+
+    free(private->command.recv_buf);
+    free(private->callback.recv_buf);
+    private->command.recv_buf = NULL;
+    private->callback.recv_buf = NULL;
+
     return 0;
 }
 
 static int
-cib_tls_signon(cib_t * cib, struct remote_connection_s *connection)
+cib_tls_signon(cib_t * cib, struct remote_connection_s *connection, gboolean event_channel)
 {
     int sock;
     cib_remote_opaque_t *private = cib->variant_opaque;
-    struct sockaddr_in addr;
     int rc = 0;
-    char *server = private->server;
-
-    int ret_ga;
-    struct addrinfo *res;
-    struct addrinfo hints;
+    int disconnected = 0;
 
     xmlNode *answer = NULL;
     xmlNode *login = NULL;
 
-    static struct mainloop_fd_callbacks cib_fd_callbacks = 
-        {
-            .dispatch = cib_remote_dispatch,
-            .destroy = cib_remote_connection_destroy,
-        };
+    static struct mainloop_fd_callbacks cib_fd_callbacks = { 0, };
+
+    cib_fd_callbacks.dispatch = event_channel ? cib_remote_callback_dispatch : cib_remote_command_dispatch;
+    cib_fd_callbacks.destroy = cib_remote_connection_destroy;
 
     connection->socket = 0;
     connection->session = NULL;
 
-    /* create socket */
-    sock = socket(PF_INET, SOCK_STREAM, IPPROTO_TCP);
-    if (sock == -1) {
-        crm_perror(LOG_ERR, "Socket creation failed");
-        return -1;
-    }
-
-    /* getaddrinfo */
-    bzero(&hints, sizeof(struct addrinfo));
-    hints.ai_flags = AI_CANONNAME;
-    hints.ai_family = AF_INET;
-    hints.ai_socktype = SOCK_RAW;
-
-    if (hints.ai_family == AF_INET6) {
-        hints.ai_protocol = IPPROTO_ICMPV6;
-    } else {
-        hints.ai_protocol = IPPROTO_ICMP;
-    }
-
-    crm_debug("Looking up %s", server);
-    ret_ga = getaddrinfo(server, NULL, &hints, &res);
-    if (ret_ga) {
-        crm_err("getaddrinfo: %s", gai_strerror(ret_ga));
-        close(sock);
-        return -1;
-    }
-
-    if (res->ai_canonname) {
-        server = res->ai_canonname;
-    }
-
-    crm_debug("Got address %s for %s", server, private->server);
-
-    if (!res->ai_addr) {
-        fprintf(stderr, "getaddrinfo failed");
-        crm_exit(1);
-    }
-#if 1
-    memcpy(&addr, res->ai_addr, res->ai_addrlen);
-#else
-    /* connect to server */
-    memset(&addr, 0, sizeof(addr));
-    addr.sin_family = AF_INET;
-    addr.sin_addr.s_addr = inet_addr(server);
-#endif
-    addr.sin_port = htons(private->port);
-
-    if (connect(sock, (struct sockaddr *)&addr, sizeof(addr)) == -1) {
-        crm_perror(LOG_ERR, "Connection to %s:%d failed", server, private->port);
-        close(sock);
-        return -1;
+    sock = crm_remote_tcp_connect(private->server, private->port);
+    if (sock <= 0) {
+        crm_perror(LOG_ERR, "remote tcp connection to %s:%d failed", private->server, private->port);
     }
 
+    connection->socket = sock;
     if (connection->encrypted) {
         /* initialize GnuTls lib */
 #ifdef HAVE_GNUTLS_GNUTLS_H
-        gnutls_global_init();
-        gnutls_anon_allocate_client_credentials(&anon_cred_c);
+        if (remote_gnutls_credentials_init == FALSE) {
+            gnutls_global_init();
+            gnutls_anon_allocate_client_credentials(&anon_cred_c);
+            remote_gnutls_credentials_init = TRUE;
+        }
 
         /* bind the socket to GnuTls lib */
-        connection->session = create_tls_session(sock, GNUTLS_CLIENT);
-        if (connection->session == NULL) {
-            crm_perror(LOG_ERR, "Session creation for %s:%d failed", server, private->port);
-            close(sock);
+        connection->session = crm_create_anon_tls_session(sock, GNUTLS_CLIENT, anon_cred_c);
+
+        if (crm_initiate_client_tls_handshake(connection->session, DEFAULT_CLIENT_HANDSHAKE_TIMEOUT) != 0) {
+            crm_err("Session creation for %s:%d failed", private->server, private->port);
+
+            gnutls_deinit(*connection->session);
+            gnutls_free(connection->session);
+            connection->session = NULL;
             cib_tls_close(cib);
             return -1;
         }
 #else
         return -EPROTONOSUPPORT;
@@ -287,11 +264,18 @@ cib_tls_signon(cib_t * cib, struct remote_connection_s *connection)
     crm_xml_add(login, "hidden", "password");
 
     crm_send_remote_msg(connection->session, login, connection->encrypted);
     free_xml(login);
 
-    answer = crm_recv_remote_msg(connection->session, connection->encrypted);
+    crm_recv_remote_msg(connection->session, &connection->recv_buf, connection->encrypted, -1, &disconnected);
+
+    if (disconnected) {
+        rc = -ENOTCONN;
+    }
+
+    answer = crm_parse_remote_buffer(&connection->recv_buf);
+
     crm_log_xml_trace(answer, "Reply");
     if (answer == NULL) {
         rc = -EPROTO;
 
     } else {
@@ -308,16 +292,19 @@ cib_tls_signon(cib_t * cib, struct remote_connection_s *connection)
 
         } else {
             connection->token = strdup(tmp_ticket);
         }
     }
+    free_xml(answer);
+    answer = NULL;
 
     if (rc != 0) {
         cib_tls_close(cib);
+        return rc;
     }
 
-    connection->socket = sock;
+    crm_trace("remote client connection established");
     connection->source = mainloop_add_fd("cib-remote", G_PRIORITY_HIGH, connection->socket, cib, &cib_fd_callbacks);
     return rc;
 }
 
 void
@@ -329,39 +316,65 @@ cib_remote_connection_destroy(gpointer user_data)
 #endif
     return;
 }
 
 int
-cib_remote_dispatch(gpointer user_data)
+cib_remote_command_dispatch(gpointer user_data)
+{
+    int disconnected = 0;
+    cib_t *cib = user_data;
+    cib_remote_opaque_t *private = cib->variant_opaque;
+
+    crm_recv_remote_msg(private->command.session, &private->command.recv_buf, private->command.encrypted, -1, &disconnected);
+
+    free(private->command.recv_buf);
+    private->command.recv_buf = NULL;
+    crm_err("received late reply for remote cib connection, discarding");
+
+    if (disconnected) {
+        return -1;
+    }
+    return 0;
+}
+
+int
+cib_remote_callback_dispatch(gpointer user_data)
 {
     cib_t *cib = user_data;
     cib_remote_opaque_t *private = cib->variant_opaque;
 
     xmlNode *msg = NULL;
-    const char *type = NULL;
+    int disconnected = 0;
 
     crm_info("Message on callback channel");
-    msg = crm_recv_remote_msg(private->callback.session, private->callback.encrypted);
 
-    type = crm_element_value(msg, F_TYPE);
-    crm_trace("Activating %s callbacks...", type);
+    crm_recv_remote_msg(private->callback.session, &private->callback.recv_buf, private->callback.encrypted, -1, &disconnected);
 
-    if (safe_str_eq(type, T_CIB)) {
-        cib_native_callback(cib, msg, 0, 0);
+    msg = crm_parse_remote_buffer(&private->callback.recv_buf);
+    while (msg) {
+        const char *type = crm_element_value(msg, F_TYPE);
+        crm_trace("Activating %s callbacks...", type);
 
-    } else if (safe_str_eq(type, T_CIB_NOTIFY)) {
-        g_list_foreach(cib->notify_list, cib_native_notify, msg);
+        if (safe_str_eq(type, T_CIB)) {
+            cib_native_callback(cib, msg, 0, 0);
 
-    } else {
-        crm_err("Unknown message type: %s", type);
-    }
+        } else if (safe_str_eq(type, T_CIB_NOTIFY)) {
+            g_list_foreach(cib->notify_list, cib_native_notify, msg);
+
+        } else {
+            crm_err("Unknown message type: %s", type);
+        }
 
-    if (msg != NULL) {
         free_xml(msg);
-        return 0;
+        msg = crm_parse_remote_buffer(&private->callback.recv_buf);
+    }
+
+    if (disconnected) {
+        return -1;
     }
-    return -1;
+
+    return 0;
 }
 
 int
 cib_remote_signon(cib_t * cib, const char *name, enum cib_conn_type type)
 {
@@ -392,15 +405,15 @@ cib_remote_signon(cib_t * cib, const char *name, enum cib_conn_type type)
     if (private->server == NULL || private->user == NULL) {
         rc = -EINVAL;
     }
 
     if (rc == pcmk_ok) {
-        rc = cib_tls_signon(cib, &(private->command));
+        rc = cib_tls_signon(cib, &(private->command), FALSE);
     }
 
     if (rc == pcmk_ok) {
-        rc = cib_tls_signon(cib, &(private->callback));
+        rc = cib_tls_signon(cib, &(private->callback), TRUE);
     }
 
     if (rc == pcmk_ok) {
         xmlNode *hello =
             cib_create_op(0, private->callback.token, CRM_OP_REGISTER, NULL, NULL, NULL, 0, NULL);
@@ -461,41 +474,24 @@ cib_remote_free(cib_t * cib)
     }
 
     return rc;
 }
 
-static gboolean timer_expired = FALSE;
-static struct timer_rec_s *sync_timer = NULL;
-static gboolean
-cib_timeout_handler(gpointer data)
-{
-    struct timer_rec_s *timer = data;
-
-    timer_expired = TRUE;
-    crm_err("Call %d timed out after %ds", timer->call_id, timer->timeout);
-
-    /* Always return TRUE, never remove the handler
-     * We do that after the while-loop in cib_native_perform_op()
-     */
-    return TRUE;
-}
-
 int
 cib_remote_perform_op(cib_t * cib, const char *op, const char *host, const char *section,
                       xmlNode * data, xmlNode ** output_data, int call_options, const char *name)
 {
     int rc = pcmk_ok;
+    int disconnected = 0;
+    int remaining_time = 0;
+    time_t start_time;
 
     xmlNode *op_msg = NULL;
     xmlNode *op_reply = NULL;
 
     cib_remote_opaque_t *private = cib->variant_opaque;
 
-    if (sync_timer == NULL) {
-        sync_timer = calloc(1, sizeof(struct timer_rec_s));
-    }
-
     if (cib->state == cib_disconnected) {
         return -ENOTCONN;
     }
 
     if (output_data != NULL) {
@@ -522,11 +518,15 @@ cib_remote_perform_op(cib_t * cib, const char *op, const char *host, const char
     if (op_msg == NULL) {
         return -EPROTO;
     }
 
     crm_trace("Sending %s message to CIB service", op);
-    crm_send_remote_msg(private->command.session, op_msg, private->command.encrypted);
+    if (!(call_options & cib_sync_call)) {
+        crm_send_remote_msg(private->callback.session, op_msg, private->command.encrypted);
+    } else {
+        crm_send_remote_msg(private->command.session, op_msg, private->command.encrypted);
+    }
     free_xml(op_msg);
 
     if ((call_options & cib_discard_reply)) {
         crm_trace("Discarding reply");
         return pcmk_ok;
@@ -535,34 +535,25 @@ cib_remote_perform_op(cib_t * cib, const char *op, const char *host, const char
         return cib->call_id;
     }
 
     crm_trace("Waiting for a syncronous reply");
 
-    if (cib->call_timeout > 0) {
-        /* We need this, even with msgfromIPC_timeout(), because we might
-         * get other/older replies that don't match the active request
-         */
-        timer_expired = FALSE;
-        sync_timer->call_id = cib->call_id;
-        sync_timer->timeout = cib->call_timeout * 1000;
-        sync_timer->ref = g_timeout_add(sync_timer->timeout, cib_timeout_handler, sync_timer);
-    }
+    start_time = time(NULL);
+    remaining_time = cib->call_timeout ? cib->call_timeout : 60;
 
-    while (timer_expired == FALSE) {
+    while (remaining_time > 0 && !disconnected) {
         int reply_id = -1;
         int msg_id = cib->call_id;
 
-        op_reply = crm_recv_remote_msg(private->command.session, private->command.encrypted);
-        if (op_reply == NULL) {
+        crm_recv_remote_msg(private->command.session, &private->command.recv_buf, private->command.encrypted, remaining_time * 1000, &disconnected);
+        op_reply = crm_parse_remote_buffer(&private->command.recv_buf);
+
+        if (!op_reply) {
             break;
         }
 
         crm_element_value_int(op_reply, F_CIB_CALLID, &reply_id);
-        CRM_CHECK(reply_id > 0, free_xml(op_reply);
-                  if (sync_timer->ref > 0) {
-                  g_source_remove(sync_timer->ref); sync_timer->ref = 0;}
-                  return -ENOMSG) ;
 
         if (reply_id == msg_id) {
             break;
 
         } else if (reply_id < msg_id) {
@@ -577,28 +568,25 @@ cib_remote_perform_op(cib_t * cib, const char *op, const char *host, const char
             crm_err("Received a __future__ reply:" " %d (wanted %d)", reply_id, msg_id);
         }
 
         free_xml(op_reply);
         op_reply = NULL;
-    }
-
-    if (sync_timer->ref > 0) {
-        g_source_remove(sync_timer->ref);
-        sync_timer->ref = 0;
-    }
 
-    if (timer_expired) {
-        return -ETIME;
+        /* wasn't the right reply, try and read some more */
+        remaining_time = time(NULL) - start_time;
     }
 
     /* if(IPC_ISRCONN(native->command_channel) == FALSE) { */
     /*      crm_err("CIB disconnected: %d",  */
     /*              native->command_channel->ch_status); */
     /*      cib->state = cib_disconnected; */
     /* } */
 
-    if (op_reply == NULL) {
+    if (disconnected) {
+        crm_err("Disconnected while waiting for reply.");
+        return -ENOTCONN;
+    } else if (op_reply == NULL) {
         crm_err("No reply message - empty");
         return -ENOMSG;
     }
 
     crm_trace("Syncronous reply received");
diff --git a/lib/common/mainloop.c b/lib/common/mainloop.c
index 09cf6e92b..1e413b60f 100644
--- a/lib/common/mainloop.c
+++ b/lib/common/mainloop.c
@@ -176,10 +176,11 @@ gboolean
 mainloop_destroy_trigger(crm_trigger_t * source)
 {
     source->trigger = FALSE;
     if (source->id > 0) {
         g_source_remove(source->id);
+        source->id = 0;
     }
     return TRUE;
 }
 
 typedef struct signal_s {
diff --git a/lib/common/remote.c b/lib/common/remote.c
index 7f04097c3..ae61481bc 100644
--- a/lib/common/remote.c
+++ b/lib/common/remote.c
@@ -23,12 +23,14 @@
 #include <stdio.h>
 #include <sys/types.h>
 #include <sys/stat.h>
 #include <unistd.h>
 #include <sys/socket.h>
-
+#include <arpa/inet.h>
 #include <netinet/ip.h>
+#include <netdb.h>
+
 
 #include <stdlib.h>
 #include <errno.h>
 #include <fcntl.h>
 #include <glib.h>
@@ -40,301 +42,664 @@
 #  undef KEYFILE
 #  include <gnutls/gnutls.h>
 #endif
 
 #ifdef HAVE_GNUTLS_GNUTLS_H
-const int tls_kx_order[] = {
+const int anon_tls_kx_order[] = {
     GNUTLS_KX_ANON_DH,
     GNUTLS_KX_DHE_RSA,
     GNUTLS_KX_DHE_DSS,
     GNUTLS_KX_RSA,
     0
 };
 
-gnutls_anon_client_credentials anon_cred_c;
-gnutls_anon_server_credentials anon_cred_s;
-static char *cib_send_tls(gnutls_session * session, xmlNode * msg);
-static char *cib_recv_tls(gnutls_session * session);
-#endif
+int
+crm_initiate_client_tls_handshake(void *session_data, int timeout_ms)
+{
+    int rc = 0;
+    int pollrc = 0;
+    time_t start = time(NULL);
+    gnutls_session *session = session_data;
 
-char *cib_recv_plaintext(int sock);
-char *cib_send_plaintext(int sock, xmlNode * msg);
+    do {
+        rc = gnutls_handshake(*session);
+        if (rc == GNUTLS_E_INTERRUPTED || rc == GNUTLS_E_AGAIN) {
+            pollrc = crm_recv_remote_ready(session, TRUE, 1000);
+            if (pollrc < 0) {
+                /* poll returned error, there is no hope */
+                rc = -1;
+            }
+        }
+    } while (((time(NULL) - start) < (timeout_ms/1000)) &&
+            (rc == GNUTLS_E_INTERRUPTED || rc == GNUTLS_E_AGAIN));
 
-#ifdef HAVE_GNUTLS_GNUTLS_H
-gnutls_session *create_tls_session(int csock, int type);
+    return rc;
+}
 
-gnutls_session *
-create_tls_session(int csock, int type /* GNUTLS_SERVER, GNUTLS_CLIENT */ )
+void *
+crm_create_anon_tls_session(int csock, int type /* GNUTLS_SERVER, GNUTLS_CLIENT */, void *credentials)
 {
-    int rc = 0;
     gnutls_session *session = gnutls_malloc(sizeof(gnutls_session));
 
     gnutls_init(session, type);
 #  ifdef HAVE_GNUTLS_PRIORITY_SET_DIRECT
 /*      http://www.manpagez.com/info/gnutls/gnutls-2.10.4/gnutls_81.php#Echo-Server-with-anonymous-authentication */
     gnutls_priority_set_direct(*session, "NORMAL:+ANON-DH", NULL);
 /*	gnutls_priority_set_direct (*session, "NONE:+VERS-TLS-ALL:+CIPHER-ALL:+MAC-ALL:+SIGN-ALL:+COMP-ALL:+ANON-DH", NULL); */
 #  else
     gnutls_set_default_priority(*session);
-    gnutls_kx_set_priority(*session, tls_kx_order);
+    gnutls_kx_set_priority(*session, anon_tls_kx_order);
 #  endif
     gnutls_transport_set_ptr(*session, (gnutls_transport_ptr) GINT_TO_POINTER(csock));
     switch (type) {
-        case GNUTLS_SERVER:
-            gnutls_credentials_set(*session, GNUTLS_CRD_ANON, anon_cred_s);
-            break;
-        case GNUTLS_CLIENT:
-            gnutls_credentials_set(*session, GNUTLS_CRD_ANON, anon_cred_c);
-            break;
+    case GNUTLS_SERVER:
+        gnutls_credentials_set(*session, GNUTLS_CRD_ANON, (gnutls_anon_server_credentials_t) credentials);
+        break;
+    case GNUTLS_CLIENT:
+        gnutls_credentials_set(*session, GNUTLS_CRD_ANON, (gnutls_anon_client_credentials_t) credentials);
+        break;
     }
 
-    do {
-        rc = gnutls_handshake(*session);
-    } while (rc == GNUTLS_E_INTERRUPTED || rc == GNUTLS_E_AGAIN);
-
-    if (rc < 0) {
-        crm_err("Handshake failed: %s", gnutls_strerror(rc));
-        gnutls_deinit(*session);
-        gnutls_free(session);
-        return NULL;
-    }
     return session;
 }
 
-static char *
-cib_send_tls(gnutls_session * session, xmlNode * msg)
+static int
+crm_send_tls(gnutls_session * session, const char *buf, size_t len)
 {
-    char *xml_text = NULL;
-
-#  if 0
-    const char *name = crm_element_name(msg);
+    const char *unsent = buf;
+    int rc = 0;
+    int total_send;
 
-    if (safe_str_neq(name, "cib_command")) {
-        xmlNodeSetName(msg, "cib_result");
+    if (buf == NULL) {
+        return -1;
     }
-#  endif
-    xml_text = dump_xml_unformatted(msg);
-    if (xml_text != NULL) {
-        char *unsent = xml_text;
-        int len = strlen(xml_text);
-        int rc = 0;
 
-        len++;                  /* null char */
-        crm_trace("Message size: %d", len);
+    total_send = len;
+    crm_trace("Message size: %d", len);
 
-        while (TRUE) {
-            rc = gnutls_record_send(*session, unsent, len);
-            crm_debug("Sent %d bytes", rc);
+    while (TRUE) {
+        rc = gnutls_record_send(*session, unsent, len);
 
-            if (rc == GNUTLS_E_INTERRUPTED || rc == GNUTLS_E_AGAIN) {
-                crm_debug("Retry");
+        if (rc == GNUTLS_E_INTERRUPTED || rc == GNUTLS_E_AGAIN) {
+            crm_debug("Retry");
 
-            } else if (rc < 0) {
-                crm_debug("Connection terminated");
-                break;
+        } else if (rc < 0) {
+            crm_err("Connection terminated rc = %d", rc);
+            break;
 
-            } else if (rc < len) {
-                crm_debug("Only sent %d of %d bytes", rc, len);
-                len -= rc;
-                unsent += rc;
-            } else {
-                break;
-            }
+        } else if (rc < len) {
+            crm_debug("Only sent %d of %d bytes", rc, len);
+            len -= rc;
+            unsent += rc;
+        } else {
+            crm_debug("Sent %d bytes", rc);
+            break;
         }
-
     }
-    free(xml_text);
-    return NULL;
 
+    return rc < 0 ? rc : total_send;
 }
 
+
+/*!
+ * \internal
+ * \brief Read bytes off non blocking tls session.
+ *
+ * \param session - tls session to read
+ * \param max_size - max bytes allowed to read for buffer. 0 assumes no limit
+ *
+ * \note only use with NON-Blocking sockets. Should only be used after polling socket.
+ *       This function will return once max_size is met, the socket read buffer
+ *       is empty, or an error is encountered.
+ *
+ * \retval '\0' terminated buffer on success
+ */
 static char *
-cib_recv_tls(gnutls_session * session)
+crm_recv_tls(gnutls_session * session, size_t max_size, size_t *recv_len, int *disconnected)
 {
     char *buf = NULL;
-
     int rc = 0;
-    int len = 0;
-    int chunk_size = 1024;
+    size_t len = 0;
+    size_t chunk_size = max_size ? max_size : 1024;
+    size_t buf_size = 0;
+    size_t read_size = 0;
 
     if (session == NULL) {
-        return NULL;
+        if (disconnected) {
+            *disconnected = 1;
+        }
+        goto done;
     }
 
-    buf = calloc(1, chunk_size);
+    buf = calloc(1, chunk_size + 1);
+    buf_size = chunk_size;
 
     while (TRUE) {
-        errno = 0;
-        rc = gnutls_record_recv(*session, buf + len, chunk_size);
-        crm_trace("Got %d more bytes. errno=%d", rc, errno);
+        read_size = buf_size - len;
 
-        if (rc == GNUTLS_E_INTERRUPTED || rc == GNUTLS_E_AGAIN) {
-            crm_trace("Retry");
+        /* automatically grow the buffer when needed if max_size is not set.*/
+        if (!max_size && (read_size < (chunk_size / 2))) {
+            buf_size += chunk_size;
+            crm_trace("Grow buffer by %d more bytes. buf is now %d bytes", (int)chunk_size, buf_size);
+            buf = realloc(buf, buf_size + 1);
+            CRM_ASSERT(buf != NULL);
 
-        } else if (rc == GNUTLS_E_UNEXPECTED_PACKET_LENGTH) {
-            crm_trace("Session disconnected");
-            goto bail;
+            read_size = buf_size - len;
+        }
 
-        } else if (rc < 0) {
-            crm_err("Error receiving message: %s (%d)", gnutls_strerror(rc), rc);
-            goto bail;
+        rc = gnutls_record_recv(*session, buf + len, read_size);
 
-        } else if (rc == chunk_size) {
+        if (rc > 0) {
+            crm_trace("Got %d more bytes.", rc);
             len += rc;
-            chunk_size *= 2;
-            buf = realloc(buf, len + chunk_size);
-            crm_trace("Retry with %d more bytes", (int)chunk_size);
-            CRM_ASSERT(buf != NULL);
-
-        } else if (buf[len + rc - 1] != 0) {
-            crm_trace("Last char is %d '%c'", buf[len + rc - 1], buf[len + rc - 1]);
-            crm_trace("Retry with %d more bytes", (int)chunk_size);
-            len += rc;
-            buf = realloc(buf, len + chunk_size);
-            CRM_ASSERT(buf != NULL);
+            /* always null terminate buffer, the +1 to alloc always allows for this.*/
+            buf[len] = '\0';
+        }
+        if (max_size && (max_size == read_size)) {
+            crm_trace("Buffer max read size %d met" , max_size);
+            goto done;
+        }
 
-        } else {
-            crm_trace("Got %d more bytes", (int)rc);
-            return buf;
+        /* process any errors. */
+        if (rc == GNUTLS_E_INTERRUPTED) {
+            crm_trace("EINTR encoutered, retry tls read");
+        } else if (rc == GNUTLS_E_AGAIN) {
+            crm_trace("non-blocking, exiting read on rc = %d", rc);
+            goto done;
+        } else if (rc <= 0) {
+            if (rc == 0) {
+                crm_debug("EOF encoutered during TLS read");
+            } else {
+                crm_debug("Error receiving message: %s (%d)", gnutls_strerror(rc), rc);
+            }
+            if (disconnected) {
+                *disconnected = 1;
+            }
+            goto done;
         }
     }
-  bail:
-    free(buf);
-    return NULL;
+
+done:
+    if (recv_len) {
+        *recv_len = len;
+    }
+    if (!len) {
+        free(buf);
+        buf = NULL;
+    }
+    return buf;
 
 }
 #endif
 
-char *
-cib_send_plaintext(int sock, xmlNode * msg)
+static int
+crm_send_plaintext(int sock, const char *buf, size_t len)
 {
-    char *xml_text = dump_xml_unformatted(msg);
 
-    if (xml_text != NULL) {
-        int rc = 0;
-        char *unsent = xml_text;
-        int len = strlen(xml_text);
+    int rc = 0;
+    const char *unsent = buf;
+    int total_send;
 
-        len++;                  /* null char */
-        crm_trace("Message on socket %d: size=%d", sock, len);
-  retry:
-        rc = write(sock, unsent, len);
-        if (rc < 0) {
-            switch (errno) {
-                case EINTR:
-                case EAGAIN:
-                    crm_trace("Retry");
-                    goto retry;
-                default:
-                    crm_perror(LOG_ERR, "Could only write %d of the remaining %d bytes", rc, len);
-                    break;
-            }
+    if (buf == NULL) {
+        return -1;
+    }
+    total_send = len;
 
-        } else if (rc < len) {
-            crm_trace("Only sent %d of %d remaining bytes", rc, len);
-            len -= rc;
-            unsent += rc;
+    crm_trace("Message on socket %d: size=%d", sock, len);
+  retry:
+    rc = write(sock, unsent, len);
+    if (rc < 0) {
+        switch (errno) {
+        case EINTR:
+        case EAGAIN:
+            crm_trace("Retry");
             goto retry;
-
-        } else {
-            crm_trace("Sent %d bytes: %.100s", rc, xml_text);
+        default:
+            crm_perror(LOG_ERR, "Could only write %d of the remaining %d bytes", rc, (int) len);
+            break;
         }
+
+    } else if (rc < len) {
+        crm_trace("Only sent %d of %d remaining bytes", rc, len);
+        len -= rc;
+        unsent += rc;
+        goto retry;
+
+     } else {
+        crm_trace("Sent %d bytes: %.100s", rc, buf);
     }
-    free(xml_text);
-    return NULL;
+
+    return rc < 0 ? rc : total_send;
 
 }
 
-char *
-cib_recv_plaintext(int sock)
+/*!
+ * \internal
+ * \brief Read bytes off non blocking socket.
+ *
+ * \param session - tls session to read
+ * \param max_size - max bytes allowed to read for buffer. 0 assumes no limit
+ *
+ * \note only use with NON-Blocking sockets. Should only be used after polling socket.
+ *       This function will return once max_size is met, the socket read buffer
+ *       is empty, or an error is encountered.
+ *
+ * \retval '\0' terminated buffer on success
+ */
+static char *
+crm_recv_plaintext(int sock, size_t max_size, size_t *recv_len, int *disconnected)
 {
     char *buf = NULL;
-
     ssize_t rc = 0;
     ssize_t len = 0;
-    ssize_t chunk_size = 512;
+    ssize_t chunk_size = max_size ? max_size : 1024;
+    size_t buf_size = 0;
+    size_t read_size = 0;
 
-    buf = calloc(1, chunk_size);
+    if (sock <= 0) {
+        if (disconnected) {
+            *disconnected = 1;
+        }
+        goto done;
+    }
 
-    while (1) {
-        errno = 0;
-        rc = read(sock, buf + len, chunk_size);
-        crm_trace("Got %d more bytes. errno=%d", (int)rc, errno);
-
-        if (errno == EINTR || errno == EAGAIN) {
-            crm_trace("Retry: %d", (int)rc);
-            if (rc > 0) {
-                len += rc;
-                buf = realloc(buf, len + chunk_size);
-                CRM_ASSERT(buf != NULL);
-            }
+    buf = calloc(1, chunk_size + 1);
+    buf_size = chunk_size;
 
-        } else if (rc < 0) {
-            crm_perror(LOG_ERR, "Error receiving message: %d", (int)rc);
-            goto bail;
+    while (TRUE) {
+        errno = 0;
+        read_size = buf_size - len;
 
-        } else if (rc == chunk_size) {
-            len += rc;
-            chunk_size *= 2;
-            buf = realloc(buf, len + chunk_size);
-            crm_trace("Retry with %d more bytes", (int)chunk_size);
+        /* automatically grow the buffer when needed if max_size is not set.*/
+        if (!max_size && (read_size < (chunk_size / 2))) {
+            buf_size += chunk_size;
+            crm_trace("Grow buffer by %d more bytes. buf is now %d bytes", (int)chunk_size, buf_size);
+            buf = realloc(buf, buf_size + 1);
             CRM_ASSERT(buf != NULL);
 
-        } else if (buf[len + rc - 1] != 0) {
-            crm_trace("Last char is %d '%c'", buf[len + rc - 1], buf[len + rc - 1]);
-            crm_trace("Retry with %d more bytes", (int)chunk_size);
+            read_size = buf_size - len;
+        }
+
+        rc = read(sock, buf + len, chunk_size);
+
+        if (rc > 0) {
+            crm_trace("Got %d more bytes. errno=%d", (int)rc, errno);
             len += rc;
-            buf = realloc(buf, len + chunk_size);
-            CRM_ASSERT(buf != NULL);
+            /* always null terminate buffer, the +1 to alloc always allows for this.*/
+            buf[len] = '\0';
+        }
+        if (max_size && (max_size == read_size)) {
+            crm_trace("Buffer max read size %d met" , max_size);
+            goto done;
+        }
 
-        } else {
-            return buf;
+        if (rc > 0) {
+            continue;
+        } else if (rc == 0) {
+            if (disconnected) {
+                *disconnected = 1;
+            }
+            crm_trace("EOF encoutered during read");
+            goto done;
+        }
+
+        /* process errors */
+        if (errno == EINTR) {
+            crm_trace("EINTER encoutered, retry socket read.");
+        } else if (errno == EAGAIN) {
+            crm_trace("non-blocking, exiting read on rc = %d", rc);
+            goto done;
+        } else if (errno <= 0) {
+            if (disconnected) {
+                *disconnected = 1;
+            }
+            crm_debug("Error receiving message: %d", (int)rc);
+            goto done;
         }
     }
-  bail:
-    free(buf);
-    return NULL;
 
+done:
+    if (recv_len) {
+        *recv_len = len;
+    }
+    if (!len) {
+        free(buf);
+        buf = NULL;
+    }
+    return buf;
 }
 
-void
-crm_send_remote_msg(void *session, xmlNode * msg, gboolean encrypted)
+static int
+crm_send_remote_msg_raw(void *session, const char *buf, size_t len, gboolean encrypted)
 {
+    int rc = -1;
     if (encrypted) {
 #ifdef HAVE_GNUTLS_GNUTLS_H
-        cib_send_tls(session, msg);
+        rc = crm_send_tls(session, buf, len);
 #else
         CRM_ASSERT(encrypted == FALSE);
 #endif
     } else {
-        cib_send_plaintext(GPOINTER_TO_INT(session), msg);
+        rc = crm_send_plaintext(GPOINTER_TO_INT(session), buf, len);
     }
+    return rc;
 }
 
+int
+crm_send_remote_msg(void *session, xmlNode * msg, gboolean encrypted)
+{
+    int rc = -1;
+    char *xml_text = NULL;
+    int len = 0;
+
+    xml_text = dump_xml_unformatted(msg);
+    if (xml_text) {
+        len = strlen(xml_text);
+    } else {
+        crm_err("Invalid XML, can not send msg");
+        return -1;
+    }
+
+    rc = crm_send_remote_msg_raw(session, xml_text, len, encrypted);
+    if (rc < 0) {
+        goto done;
+    }
+    rc = crm_send_remote_msg_raw(session, REMOTE_MSG_TERMINATOR, strlen(REMOTE_MSG_TERMINATOR), encrypted);
+
+done:
+    if (rc < 0) {
+        crm_err("Failed to send remote msg, rc = %d", rc);
+    }
+
+    free(xml_text);
+    return rc;
+}
+
+/*!
+ * \internal
+ * \brief handles the recv buffer and parsing out msgs.
+ * \note new_data is owned by this function once it is passed in.
+ */
 xmlNode *
-crm_recv_remote_msg(void *session, gboolean encrypted)
+crm_parse_remote_buffer(char **msg_buf)
 {
-    char *reply = NULL;
+    char *buf = NULL;
+    char *start = NULL;
+    char *end = NULL;
     xmlNode *xml = NULL;
 
+    if (*msg_buf == NULL) {
+        return NULL;
+    }
+
+    /* take ownership of the buffer */
+    buf = *msg_buf;
+    *msg_buf = NULL;
+
+    /* MSGS are separated by a '\r\n\r\n'. Split a message off the buffer and return it. */
+    start = buf;
+    end = strstr(start, REMOTE_MSG_TERMINATOR);
+
+    while (!xml && end) {
+
+        /* grab the message */
+        end[0] = '\0';
+        end += strlen(REMOTE_MSG_TERMINATOR);
+
+        xml = string2xml(start);
+        if (xml == NULL) {
+            crm_err("Couldn't parse: '%.120s'", start);
+        }
+        start = end;
+        end = strstr(start, REMOTE_MSG_TERMINATOR);
+    }
+
+    if (xml && start) {
+        /* we have msgs left over, save it until next time */
+        *msg_buf = strdup(start);
+        free(buf);
+    } else if (!xml) {
+        /* no msg present */
+        *msg_buf = buf;
+    }
+
+    return xml;
+}
+
+/*!
+ * \internal
+ * \brief Determine if a remote session has data to read
+ *
+ * \retval 0, timeout occured.
+ * \retval positive, data is ready to be read
+ * \retval negative, session has ended
+ */
+int
+crm_recv_remote_ready(void *session, gboolean encrypted, int timeout /* ms */)
+{
+    struct pollfd fds = { 0, };
+    int sock = 0;
+    void *sock_ptr = NULL;
+    int rc = 0;
+    time_t start;
+
+    if (encrypted) {
+#ifdef HAVE_GNUTLS_GNUTLS_H
+        gnutls_session *tls_session = session;
+        sock_ptr = gnutls_transport_get_ptr(*tls_session);
+#else
+        CRM_ASSERT(encrypted == FALSE);
+#endif
+    } else {
+        sock_ptr = session;
+    }
+
+    sock = GPOINTER_TO_INT(sock_ptr);
+    if (sock <= 0) {
+        return -ENOTCONN;
+    }
+
+    start = time(NULL);
+    errno = 0;
+    do {
+        fds.fd = sock;
+        fds.events = POLLIN;
+
+        /* If we got an EINTR while polling, and we have a
+         * specific timeout we are trying to honor, attempt
+         * to adjust the timeout to the closest second. */
+        if (errno == EINTR && (timeout > 0)) {
+            timeout = timeout - ((time(NULL) - start) * 1000);
+            if (timeout < 1000) {
+                timeout = 1000;
+            }
+        }
+
+        rc = poll(&fds, 1, timeout);
+    } while (rc < 0 && errno == EINTR);
+
+    return rc;
+}
+
+char *
+crm_recv_remote_raw(void *session, gboolean encrypted, size_t max_recv, size_t *recv_len, int *disconnected)
+{
+    char *reply = NULL;
+    if (recv_len) {
+        *recv_len = 0;
+    }
+
+    if (disconnected) {
+        *disconnected = 0;
+    }
+
     if (encrypted) {
 #ifdef HAVE_GNUTLS_GNUTLS_H
-        reply = cib_recv_tls(session);
+        reply = crm_recv_tls(session, max_recv, recv_len, disconnected);
 #else
         CRM_ASSERT(encrypted == FALSE);
 #endif
     } else {
-        reply = cib_recv_plaintext(GPOINTER_TO_INT(session));
+        reply = crm_recv_plaintext(GPOINTER_TO_INT(session), max_recv, recv_len, disconnected);
     }
     if (reply == NULL || strlen(reply) == 0) {
         crm_trace("Empty reply");
+    }
 
-    } else {
-        xml = string2xml(reply);
-        if (xml == NULL) {
-            crm_err("Couldn't parse: '%.120s'", reply);
+    return reply;
+}
+
+/*!
+ * \internal
+ * \brief Read data off the socket until at least one full message is present or timeout occures.
+ * \retval TRUE message read
+ * \retval FALSE full message not read
+ */
+
+gboolean
+crm_recv_remote_msg(void *session, char **recv_buf, gboolean encrypted, int total_timeout /*ms */, int *disconnected)
+{
+    int ret;
+    size_t request_len = 0;
+    time_t start = time(NULL);
+    char *raw_request = NULL;
+    int remaining_timeout = 0;
+
+    if (total_timeout == 0) {
+        total_timeout = 10000;
+    } else if (total_timeout < 0) {
+        total_timeout = 60000;
+    }
+    *disconnected = 0;
+
+    remaining_timeout = total_timeout;
+    while ((remaining_timeout > 0) && !(*disconnected)) {
+
+        /* read some more off the tls buffer if we still have time left. */
+        crm_trace("waiting to receive remote msg, starting timeout %d, remaining_timeout %d", total_timeout, remaining_timeout);
+        ret = crm_recv_remote_ready(session, encrypted, remaining_timeout);
+        raw_request = NULL;
+
+        if (ret == 0) {
+            crm_err("poll timed out (%d ms) while waiting to receive msg", remaining_timeout);
+            return FALSE;
+
+        } else if (ret < 0) {
+            if (errno != EINTR) {
+                crm_debug("poll returned error while waiting for msg, rc: %d, errno: %d", ret, errno);
+                *disconnected = 1;
+                return FALSE;
+            }
+            crm_debug("poll EINTR encountered during poll, retrying");
+        } else {
+            raw_request = crm_recv_remote_raw(session, encrypted, 0, &request_len, disconnected);
+        }
+
+        remaining_timeout = remaining_timeout - ((time(NULL) - start) * 1000);
+
+        if (!raw_request) {
+            crm_debug("Empty msg received after poll");
+            continue;
+        }
+
+        if (*recv_buf) {
+            int old_len = strlen(*recv_buf);
+
+            crm_trace("Expanding recv buffer from %d to %d", old_len, old_len+request_len);
+
+            *recv_buf = realloc(*recv_buf, old_len + request_len + 1);
+            memcpy(*recv_buf + old_len, raw_request, request_len);
+            *(*recv_buf+old_len+request_len) = '\0';
+            free(raw_request);
+        } else {
+            *recv_buf = raw_request;
+        }
+
+        if (strstr(*recv_buf, REMOTE_MSG_TERMINATOR)) {
+            return TRUE;
         }
     }
 
-    free(reply);
-    return xml;
+    return FALSE;
 }
+
+/*!
+ * \internal
+ * \brief tcp connection to server at specified port
+ * \retval positive, socket fd.
+ * \retval negative, failed to connect.
+ */
+int
+crm_remote_tcp_connect(const char *host, int port)
+{
+    struct addrinfo *res;
+    struct addrinfo *rp;
+    struct addrinfo hints;
+    const char *server = host;
+    int ret_ga;
+    int sock;
+
+    /* getaddrinfo */
+    memset(&hints, 0, sizeof(struct addrinfo));
+    hints.ai_family = AF_UNSPEC;    /* Allow IPv4 or IPv6 */
+    hints.ai_socktype = SOCK_STREAM;
+    hints.ai_flags = AI_CANONNAME;
+
+    crm_debug("Looking up %s", server);
+    ret_ga = getaddrinfo(server, NULL, &hints, &res);
+    if (ret_ga) {
+        crm_err("getaddrinfo: %s", gai_strerror(ret_ga));
+        return -1;
+    }
+
+    if (!res || !res->ai_addr) {
+        crm_err("getaddrinfo failed");
+        return -1;
+    }
+
+    for (rp = res; rp != NULL; rp = rp->ai_next) {
+        struct sockaddr *addr = rp->ai_addr;
+        int flag = 0;
+        if (!addr) {
+            continue;
+        }
+
+        if (rp->ai_canonname) {
+            server = res->ai_canonname;
+        }
+        crm_debug("Got address %s for %s", server, host);
+
+        /* create socket */
+        sock = socket(rp->ai_family, SOCK_STREAM, IPPROTO_TCP);
+        if (sock == -1) {
+            crm_err("Socket creation failed for remote client connection.");
+            continue;
+        }
+        if (addr->sa_family == AF_INET6) {
+            struct sockaddr_in6 *addr_in = (struct sockaddr_in6 *) addr;
+            addr_in->sin6_port = htons(port);
+        } else {
+            struct sockaddr_in *addr_in = (struct sockaddr_in *) addr;
+            addr_in->sin_port = htons(port);
+            crm_info("Attempting to connect to remote server at %s:%d", inet_ntoa(addr_in->sin_addr), port);
+        }
+
+        if (connect(sock, rp->ai_addr, rp->ai_addrlen) == 0) {
+            if ((flag = fcntl(sock, F_GETFL)) >= 0) {
+                if (fcntl(sock, F_SETFL, flag | O_NONBLOCK) < 0) {
+                    crm_err( "fcntl() write failed");
+                    close(sock);
+                    sock = -1;
+                    continue;
+                }
+            }
+            break;                  /* Success */
+        }
+
+        close(sock);
+        sock = -1;
+    }
+    freeaddrinfo(res);
+
+    return sock;
+}
+
diff --git a/tools/crm_mon.c b/tools/crm_mon.c
index 5c2e68722..fe592649f 100644
--- a/tools/crm_mon.c
+++ b/tools/crm_mon.c
@@ -273,11 +273,11 @@ cib_connect(gboolean full)
 
         if (full) {
             if (rc == pcmk_ok) {
                 rc = cib->cmds->set_connection_dnotify(cib, mon_cib_connection_destroy);
                 if (rc == -EPROTONOSUPPORT) {
-                    print_as("Notification setup failed, won't be able to reconnect after failure");
+                    print_as("Notification setup not supported, won't be able to reconnect after failure");
                     if (as_console) {
                         sleep(2);
                     }
                     rc = pcmk_ok;
                 }

Vulnerable code:             CRM_LOG_ASSERT(client_obj->request_id);
            rid = client_obj->request_id;
            client_obj->request_id = 0;
            crm_trace("Sending response %d to %s %s",
        
    if (client->ipc == NULL) {
#ifdef HAVE_DECL_NANOSLEEP
#  include <time.h>
#endif

extern gnutls_anon_server_credentials anon_cred_s;

extern gnutls_session *create_tls_session(int csock, int type);

/* 	gnutls_global_set_log_level (10); */
    int lpc = 0;
    time_t now = 0;
    time_t start = time(NULL);
    xmlNode *login = NULL;
    const char *user = NULL;
    const char *pass = NULL;
    const char *tmp = NULL;

#ifdef HAVE_DECL_NANOSLEEP
    const struct timespec sleepfast = { 0, 10000000 };  /* 10 millisec */
#endif

        };    
    
        session = create_tls_session(csock, GNUTLS_SERVER);
    do {
        crm_trace("Iter: %d", lpc++);
        if (ssock == remote_tls_fd) {
#ifdef HAVE_GNUTLS_GNUTLS_H
            login = crm_recv_remote_msg(session, TRUE);
#endif
        } else {
            login = crm_recv_remote_msg(GINT_TO_POINTER(csock), FALSE);
        }
        if (login != NULL) {
            break;
        }
#ifdef HAVE_DECL_NANOSLEEP
        nanosleep(&sleepfast, NULL);
#else
        sleep(1);
#endif
        now = time(NULL);

        /* Peers have 3s to connect */
    } while (login == NULL && (start - now) < 4);

    crm_log_xml_info(login, "Login: ");
    if (login == NULL) {
        goto bail;
    }

    tmp = crm_element_name(login);
    if (safe_str_neq(tmp, "cib_command")) {
        crm_err("Wrong tag: %s", tmp);
        goto bail;
    }

    tmp = crm_element_value(login, "op");
    if (safe_str_neq(tmp, "authenticate")) {
        crm_err("Wrong operation: %s", tmp);
        goto bail;
    }

    user = crm_element_value(login, "user");
    pass = crm_element_value(login, "password");

    /* Non-root daemons can only validate the password of the
     * user they're running as
     */
    if (check_group_membership(user, CRM_DAEMON_GROUP) == FALSE) {
        crm_err("User is not a member of the required group");
        goto bail;

    } else if (authenticate_user(user, pass) == FALSE) {
        crm_err("PAM auth failed");
        goto bail;
    }

    /* send ACK */
    new_client->name = crm_element_value_copy(login, "name");

    CRM_CHECK(new_client->id == NULL, free(new_client->id));

#if ENABLE_ACL
    new_client->user = strdup(user);
#endif

    free_xml(login);
    login = create_xml_node(NULL, "cib_result");
    crm_xml_add(login, F_CIB_OPERATION, CRM_OP_REGISTER);
    crm_xml_add(login, F_CIB_CLIENTID, new_client->id);
    crm_send_remote_msg(new_client->session, login, new_client->encrypted);
    free_xml(login);


  bail:
    if (ssock == remote_tls_fd) {
#ifdef HAVE_GNUTLS_GNUTLS_H
        gnutls_bye(*session, GNUTLS_SHUT_RDWR);
        gnutls_deinit(*session);
        gnutls_free(session);
#endif
    }
    close(csock);
    free_xml(login);
    return TRUE;
int
cib_remote_msg(gpointer data)
    xmlNode *command = NULL;
    cib_client_t *client = data;

    crm_trace("%s callback", client->encrypted ? "secure" : "clear-text");

    command = crm_recv_remote_msg(client->session, client->encrypted);
    if (command == NULL) {
        return -1;
    }
        goto bail;
  bail:
    free_xml(command);
    command = NULL;
xmlNode *crm_recv_remote_msg(void *session, gboolean encrypted);
void crm_send_remote_msg(void *session, xmlNode * msg, gboolean encrypted);
extern gnutls_anon_client_credentials anon_cred_c;
extern gnutls_session *create_tls_session(int csock, int type);
int cib_remote_dispatch(gpointer user_data);
    shutdown(private->command.socket, SHUT_RDWR);       /* no more receptions */
    shutdown(private->callback.socket, SHUT_RDWR);      /* no more receptions */
    close(private->command.socket);
    close(private->callback.socket);

        gnutls_bye(*(private->command.session), GNUTLS_SHUT_RDWR);
        gnutls_deinit(*(private->command.session));
        gnutls_free(private->command.session);

        gnutls_bye(*(private->callback.session), GNUTLS_SHUT_RDWR);
        gnutls_deinit(*(private->callback.session));
        gnutls_free(private->callback.session);
        gnutls_anon_free_client_credentials(anon_cred_c);
        gnutls_global_deinit();
cib_tls_signon(cib_t * cib, struct remote_connection_s *connection)
    struct sockaddr_in addr;
    char *server = private->server;

    int ret_ga;
    struct addrinfo *res;
    struct addrinfo hints;
    static struct mainloop_fd_callbacks cib_fd_callbacks = 
        {
            .dispatch = cib_remote_dispatch,
            .destroy = cib_remote_connection_destroy,
        };
    /* create socket */
    sock = socket(PF_INET, SOCK_STREAM, IPPROTO_TCP);
    if (sock == -1) {
        crm_perror(LOG_ERR, "Socket creation failed");
        return -1;
    }

    /* getaddrinfo */
    bzero(&hints, sizeof(struct addrinfo));
    hints.ai_flags = AI_CANONNAME;
    hints.ai_family = AF_INET;
    hints.ai_socktype = SOCK_RAW;

    if (hints.ai_family == AF_INET6) {
        hints.ai_protocol = IPPROTO_ICMPV6;
    } else {
        hints.ai_protocol = IPPROTO_ICMP;
    }

    crm_debug("Looking up %s", server);
    ret_ga = getaddrinfo(server, NULL, &hints, &res);
    if (ret_ga) {
        crm_err("getaddrinfo: %s", gai_strerror(ret_ga));
        close(sock);
        return -1;
    }

    if (res->ai_canonname) {
        server = res->ai_canonname;
    }

    crm_debug("Got address %s for %s", server, private->server);

    if (!res->ai_addr) {
        fprintf(stderr, "getaddrinfo failed");
        crm_exit(1);
    }
#if 1
    memcpy(&addr, res->ai_addr, res->ai_addrlen);
#else
    /* connect to server */
    memset(&addr, 0, sizeof(addr));
    addr.sin_family = AF_INET;
    addr.sin_addr.s_addr = inet_addr(server);
#endif
    addr.sin_port = htons(private->port);

    if (connect(sock, (struct sockaddr *)&addr, sizeof(addr)) == -1) {
        crm_perror(LOG_ERR, "Connection to %s:%d failed", server, private->port);
        close(sock);
        return -1;
        gnutls_global_init();
        gnutls_anon_allocate_client_credentials(&anon_cred_c);
        connection->session = create_tls_session(sock, GNUTLS_CLIENT);
        if (connection->session == NULL) {
            crm_perror(LOG_ERR, "Session creation for %s:%d failed", server, private->port);
            close(sock);
    answer = crm_recv_remote_msg(connection->session, connection->encrypted);
    connection->socket = sock;
cib_remote_dispatch(gpointer user_data)
    const char *type = NULL;
    msg = crm_recv_remote_msg(private->callback.session, private->callback.encrypted);
    type = crm_element_value(msg, F_TYPE);
    crm_trace("Activating %s callbacks...", type);
    if (safe_str_eq(type, T_CIB)) {
        cib_native_callback(cib, msg, 0, 0);
    } else if (safe_str_eq(type, T_CIB_NOTIFY)) {
        g_list_foreach(cib->notify_list, cib_native_notify, msg);
    } else {
        crm_err("Unknown message type: %s", type);
    }
    if (msg != NULL) {
        return 0;
    return -1;
        rc = cib_tls_signon(cib, &(private->command));
        rc = cib_tls_signon(cib, &(private->callback));
static gboolean timer_expired = FALSE;
static struct timer_rec_s *sync_timer = NULL;
static gboolean
cib_timeout_handler(gpointer data)
{
    struct timer_rec_s *timer = data;

    timer_expired = TRUE;
    crm_err("Call %d timed out after %ds", timer->call_id, timer->timeout);

    /* Always return TRUE, never remove the handler
     * We do that after the while-loop in cib_native_perform_op()
     */
    return TRUE;
}

    if (sync_timer == NULL) {
        sync_timer = calloc(1, sizeof(struct timer_rec_s));
    }

    crm_send_remote_msg(private->command.session, op_msg, private->command.encrypted);
    if (cib->call_timeout > 0) {
        /* We need this, even with msgfromIPC_timeout(), because we might
         * get other/older replies that don't match the active request
         */
        timer_expired = FALSE;
        sync_timer->call_id = cib->call_id;
        sync_timer->timeout = cib->call_timeout * 1000;
        sync_timer->ref = g_timeout_add(sync_timer->timeout, cib_timeout_handler, sync_timer);
    }
    while (timer_expired == FALSE) {
        op_reply = crm_recv_remote_msg(private->command.session, private->command.encrypted);
        if (op_reply == NULL) {
        CRM_CHECK(reply_id > 0, free_xml(op_reply);
                  if (sync_timer->ref > 0) {
                  g_source_remove(sync_timer->ref); sync_timer->ref = 0;}
                  return -ENOMSG) ;
    }

    if (sync_timer->ref > 0) {
        g_source_remove(sync_timer->ref);
        sync_timer->ref = 0;
    }
    if (timer_expired) {
        return -ETIME;
    if (op_reply == NULL) {

const int tls_kx_order[] = {
gnutls_anon_client_credentials anon_cred_c;
gnutls_anon_server_credentials anon_cred_s;
static char *cib_send_tls(gnutls_session * session, xmlNode * msg);
static char *cib_recv_tls(gnutls_session * session);
#endif
char *cib_recv_plaintext(int sock);
char *cib_send_plaintext(int sock, xmlNode * msg);
#ifdef HAVE_GNUTLS_GNUTLS_H
gnutls_session *create_tls_session(int csock, int type);
gnutls_session *
create_tls_session(int csock, int type /* GNUTLS_SERVER, GNUTLS_CLIENT */ )
    int rc = 0;
    gnutls_kx_set_priority(*session, tls_kx_order);
        case GNUTLS_SERVER:
            gnutls_credentials_set(*session, GNUTLS_CRD_ANON, anon_cred_s);
            break;
        case GNUTLS_CLIENT:
            gnutls_credentials_set(*session, GNUTLS_CRD_ANON, anon_cred_c);
            break;
    do {
        rc = gnutls_handshake(*session);
    } while (rc == GNUTLS_E_INTERRUPTED || rc == GNUTLS_E_AGAIN);

    if (rc < 0) {
        crm_err("Handshake failed: %s", gnutls_strerror(rc));
        gnutls_deinit(*session);
        gnutls_free(session);
        return NULL;
    }
static char *
cib_send_tls(gnutls_session * session, xmlNode * msg)
    char *xml_text = NULL;

#  if 0
    const char *name = crm_element_name(msg);
    if (safe_str_neq(name, "cib_command")) {
        xmlNodeSetName(msg, "cib_result");
#  endif
    xml_text = dump_xml_unformatted(msg);
    if (xml_text != NULL) {
        char *unsent = xml_text;
        int len = strlen(xml_text);
        int rc = 0;
        len++;                  /* null char */
        crm_trace("Message size: %d", len);
        while (TRUE) {
            rc = gnutls_record_send(*session, unsent, len);
            crm_debug("Sent %d bytes", rc);
            if (rc == GNUTLS_E_INTERRUPTED || rc == GNUTLS_E_AGAIN) {
                crm_debug("Retry");
            } else if (rc < 0) {
                crm_debug("Connection terminated");
                break;
            } else if (rc < len) {
                crm_debug("Only sent %d of %d bytes", rc, len);
                len -= rc;
                unsent += rc;
            } else {
                break;
            }

    free(xml_text);
    return NULL;
cib_recv_tls(gnutls_session * session)

    int len = 0;
    int chunk_size = 1024;
        return NULL;
    buf = calloc(1, chunk_size);
        errno = 0;
        rc = gnutls_record_recv(*session, buf + len, chunk_size);
        crm_trace("Got %d more bytes. errno=%d", rc, errno);
        if (rc == GNUTLS_E_INTERRUPTED || rc == GNUTLS_E_AGAIN) {
            crm_trace("Retry");
        } else if (rc == GNUTLS_E_UNEXPECTED_PACKET_LENGTH) {
            crm_trace("Session disconnected");
            goto bail;
        } else if (rc < 0) {
            crm_err("Error receiving message: %s (%d)", gnutls_strerror(rc), rc);
            goto bail;
        } else if (rc == chunk_size) {
            chunk_size *= 2;
            buf = realloc(buf, len + chunk_size);
            crm_trace("Retry with %d more bytes", (int)chunk_size);
            CRM_ASSERT(buf != NULL);

        } else if (buf[len + rc - 1] != 0) {
            crm_trace("Last char is %d '%c'", buf[len + rc - 1], buf[len + rc - 1]);
            crm_trace("Retry with %d more bytes", (int)chunk_size);
            len += rc;
            buf = realloc(buf, len + chunk_size);
            CRM_ASSERT(buf != NULL);
        } else {
            crm_trace("Got %d more bytes", (int)rc);
            return buf;
  bail:
    free(buf);
    return NULL;
char *
cib_send_plaintext(int sock, xmlNode * msg)
    char *xml_text = dump_xml_unformatted(msg);
    if (xml_text != NULL) {
        int rc = 0;
        char *unsent = xml_text;
        int len = strlen(xml_text);
        len++;                  /* null char */
        crm_trace("Message on socket %d: size=%d", sock, len);
  retry:
        rc = write(sock, unsent, len);
        if (rc < 0) {
            switch (errno) {
                case EINTR:
                case EAGAIN:
                    crm_trace("Retry");
                    goto retry;
                default:
                    crm_perror(LOG_ERR, "Could only write %d of the remaining %d bytes", rc, len);
                    break;
            }
        } else if (rc < len) {
            crm_trace("Only sent %d of %d remaining bytes", rc, len);
            len -= rc;
            unsent += rc;

        } else {
            crm_trace("Sent %d bytes: %.100s", rc, xml_text);
    free(xml_text);
    return NULL;
char *
cib_recv_plaintext(int sock)

    ssize_t chunk_size = 512;
    buf = calloc(1, chunk_size);
    while (1) {
        errno = 0;
        rc = read(sock, buf + len, chunk_size);
        crm_trace("Got %d more bytes. errno=%d", (int)rc, errno);

        if (errno == EINTR || errno == EAGAIN) {
            crm_trace("Retry: %d", (int)rc);
            if (rc > 0) {
                len += rc;
                buf = realloc(buf, len + chunk_size);
                CRM_ASSERT(buf != NULL);
            }
        } else if (rc < 0) {
            crm_perror(LOG_ERR, "Error receiving message: %d", (int)rc);
            goto bail;
        } else if (rc == chunk_size) {
            len += rc;
            chunk_size *= 2;
            buf = realloc(buf, len + chunk_size);
            crm_trace("Retry with %d more bytes", (int)chunk_size);
        } else if (buf[len + rc - 1] != 0) {
            crm_trace("Last char is %d '%c'", buf[len + rc - 1], buf[len + rc - 1]);
            crm_trace("Retry with %d more bytes", (int)chunk_size);
            buf = realloc(buf, len + chunk_size);
            CRM_ASSERT(buf != NULL);
        } else {
            return buf;
  bail:
    free(buf);
    return NULL;
void
crm_send_remote_msg(void *session, xmlNode * msg, gboolean encrypted)
        cib_send_tls(session, msg);
        cib_send_plaintext(GPOINTER_TO_INT(session), msg);
crm_recv_remote_msg(void *session, gboolean encrypted)
    char *reply = NULL;
        reply = cib_recv_tls(session);
        reply = cib_recv_plaintext(GPOINTER_TO_INT(session));
    } else {
        xml = string2xml(reply);
        if (xml == NULL) {
            crm_err("Couldn't parse: '%.120s'", reply);
    free(reply);
    return xml;
                    print_as("Notification setup failed, won't be able to reconnect after failure");
Fixed code:             if (client_obj->ipc) {
                CRM_LOG_ASSERT(client_obj->request_id);
                rid = client_obj->request_id;
                client_obj->request_id = 0;
                crm_trace("Sending response %d to %s %s",
            } else {
                crm_trace("Sending response to %s %s",
                      client_obj->name, from_peer?"(originator of delegated request)":"");
            }
    char *recv_buf;
    gboolean handshake_complete;
    gboolean remote_auth;

    int remote_auth_timeout;
    if (client->ipc == NULL && client->session == NULL) {
gnutls_anon_server_credentials anon_cred_s;
#define REMOTE_AUTH_TIMEOUT 10000

        /* gnutls_global_set_log_level (10); */
static gboolean
cib_remote_auth(xmlNode *login)
{
    const char *user = NULL;
    const char *pass = NULL;
    const char *tmp = NULL;

    crm_log_xml_info(login, "Login: ");
    if (login == NULL) {
        return FALSE;
    }

    tmp = crm_element_name(login);
    if (safe_str_neq(tmp, "cib_command")) {
        crm_err("Wrong tag: %s", tmp);
        return FALSE;
    }

    tmp = crm_element_value(login, "op");
    if (safe_str_neq(tmp, "authenticate")) {
        crm_err("Wrong operation: %s", tmp);
        return FALSE;
    }

    user = crm_element_value(login, "user");
    pass = crm_element_value(login, "password");

    if (!user || !pass) {
        crm_err("missing auth credentials");
        return FALSE;
    }

    /* Non-root daemons can only validate the password of the
     * user they're running as
     */
    if (check_group_membership(user, CRM_DAEMON_GROUP) == FALSE) {
        crm_err("User is not a member of the required group");
        return FALSE;

    } else if (authenticate_user(user, pass) == FALSE) {
        crm_err("PAM auth failed");
        return FALSE;
    }

    return TRUE;
}

static gboolean
remote_auth_timeout_cb(gpointer data)
{
    cib_client_t *client = data;

    client->remote_auth_timeout = 0;

    if (client->remote_auth == TRUE) {
        return FALSE;
    }

    mainloop_del_fd(client->remote);
    crm_err("Remote client authentication timed out");

    return FALSE;
}
    int flag;
        };

    if ((flag = fcntl(csock, F_GETFL)) >= 0) {
        if (fcntl(csock, F_SETFL, flag | O_NONBLOCK) < 0) {
            crm_err( "fcntl() write failed");
            close(csock);
            return TRUE;
        }
    } else {
        crm_err( "fcntl() read failed");
        close(csock);
        return TRUE;
    }

        session = crm_create_anon_tls_session(csock, GNUTLS_SERVER, anon_cred_s);
    /* clients have a few seconds to perform handshake. */
    new_client->remote_auth_timeout = g_timeout_add(REMOTE_AUTH_TIMEOUT, remote_auth_timeout_cb, new_client);

    int csock = 0;
    if (client->remote_auth_timeout) {
        g_source_remove(client->remote_auth_timeout);
    }

    if (client->encrypted) {
#ifdef HAVE_GNUTLS_GNUTLS_H
        if (client->session) {
            void *sock_ptr = gnutls_transport_get_ptr(*client->session);
            csock = GPOINTER_TO_INT(sock_ptr);
            if (client->handshake_complete) {
                gnutls_bye(*client->session, GNUTLS_SHUT_WR);
            }
            gnutls_deinit(*client->session);
            gnutls_free(client->session);
        }
#endif
    } else {
        csock = GPOINTER_TO_INT(client->session);
    }
    client->session = NULL;

    if (csock > 0) {
        close(csock);
    }

    free(client->recv_buf);
static void
cib_handle_remote_msg(cib_client_t *client, xmlNode *command)
        return;
}

int
cib_remote_msg(gpointer data)
{
    xmlNode *command = NULL;
    cib_client_t *client = data;
    int disconnected = 0;
    int timeout = client->remote_auth ? -1 : 1000;

    crm_trace("%s callback", client->encrypted ? "secure" : "clear-text");

#ifdef HAVE_GNUTLS_GNUTLS_H
    if (client->encrypted && (client->handshake_complete == FALSE)) {
        int rc = 0;

        /* Muliple calls to handshake will be required, this callback
         * will be invoked once the client sends more handshake data. */
        do {
            rc = gnutls_handshake(*client->session);

            if (rc < 0 && rc != GNUTLS_E_AGAIN) {
                crm_err("Remote cib tls handshake failed");
                return -1;
            }
        } while (rc == GNUTLS_E_INTERRUPTED);

        if (rc == 0) {
            crm_debug("Remote cib tls handshake completed");
            client->handshake_complete = TRUE;
            if (client->remote_auth_timeout) {
                g_source_remove(client->remote_auth_timeout);
            }
            /* after handshake, clients must send auth in a few seconds */
            client->remote_auth_timeout = g_timeout_add(REMOTE_AUTH_TIMEOUT, remote_auth_timeout_cb, client);
        }
        return 0;
    }
#endif

    crm_recv_remote_msg(client->session, &client->recv_buf, client->encrypted, timeout, &disconnected);

    /* must pass auth before we will process anything else */
    if (client->remote_auth == FALSE) {
        xmlNode *reg;
#if ENABLE_ACL
        const char *user = NULL;
#endif
        command = crm_parse_remote_buffer(&client->recv_buf);
        if (cib_remote_auth(command) == FALSE) {
            free_xml(command);
            return -1;
        }

        crm_debug("remote connection authenticated successfully");
        client->remote_auth = TRUE;
        g_source_remove(client->remote_auth_timeout);
        client->remote_auth_timeout = 0;
        client->name = crm_element_value_copy(command, "name");

#if ENABLE_ACL
        user = crm_element_value(command, "user");
        if (user) {
           new_client->user = strdup(user);
        }
#endif

        /* send ACK */
        reg = create_xml_node(NULL, "cib_result");
        crm_xml_add(reg, F_CIB_OPERATION, CRM_OP_REGISTER);
        crm_xml_add(reg, F_CIB_CLIENTID, client->id);
        crm_send_remote_msg(client->session, reg, client->encrypted);
        free_xml(reg);
        free_xml(command);
    }

    command = crm_parse_remote_buffer(&client->recv_buf);
    while (command) {
        crm_trace("command received");
        cib_handle_remote_msg(client, command);
        free_xml(command);
        command = crm_parse_remote_buffer(&client->recv_buf);
    }

    if (disconnected) {
        crm_trace("disconnected while receiving remote cib msg.");
        return -1;
    }



/*! remote tcp/tls helper functions */
gboolean crm_recv_remote_msg(void *session, char **recv_buf, gboolean encrypted, int total_timeout_ms, int *disconnected);
char *crm_recv_remote_raw(void *data, gboolean encrypted, size_t max_recv, size_t *recv_len, int *disconnected);
int crm_send_remote_msg(void *session, xmlNode * msg, gboolean encrypted);
int crm_recv_remote_ready(void *session, gboolean encrypted, int timeout_ms);
xmlNode *crm_parse_remote_buffer(char **msg_buf);
int crm_remote_tcp_connect(const char *host, int port);

#ifdef HAVE_GNUTLS_GNUTLS_H
/*!
 * \internal
 * \brief Initiate the client handshake after establishing the tcp socket.
 * \note This is a blocking function, it will block until the entire handshake
 *       is complete or until the timeout period is reached.
 * \retval 0 success
 * \retval negative, failure
 */
int crm_initiate_client_tls_handshake(void *session_data, int timeout_ms);
/*!
 * \internal
 * \brief Create client or server session for anon DH encryption credentials
 * \param sock, the socket the session will use for transport
 * \param type, GNUTLS_SERVER or GNUTLS_CLIENT
 * \param credentials, gnutls_anon_server_credentials_t or gnutls_anon_client_credentials_t
 *
 * \retval gnutls_session * on success
 * \retval NULL on failure
 */
void *crm_create_anon_tls_session(int sock, int type, void *credentials);
#endif

#define REMOTE_MSG_TERMINATOR "\r\n\r\n"
gnutls_anon_client_credentials anon_cred_c;
#define DEFAULT_CLIENT_HANDSHAKE_TIMEOUT 5000 /* 5 seconds */
static gboolean remote_gnutls_credentials_init = FALSE;
    char *recv_buf;
int cib_remote_callback_dispatch(gpointer user_data);
int cib_remote_command_dispatch(gpointer user_data);
        if (private->command.session) {
            gnutls_bye(*(private->command.session), GNUTLS_SHUT_RDWR);
            gnutls_deinit(*(private->command.session));
            gnutls_free(private->command.session);
        }
        if (private->callback.session) {
            gnutls_bye(*(private->callback.session), GNUTLS_SHUT_RDWR);
            gnutls_deinit(*(private->callback.session));
            gnutls_free(private->callback.session);
        }
        private->command.session = NULL;
        private->callback.session = NULL;
        if (remote_gnutls_credentials_init) {
            gnutls_anon_free_client_credentials(anon_cred_c);
            gnutls_global_deinit();
            remote_gnutls_credentials_init = FALSE;
        }

    if (private->command.socket) {
        shutdown(private->command.socket, SHUT_RDWR);       /* no more receptions */
        close(private->command.socket);
    }
    if (private->callback.socket) {
        shutdown(private->callback.socket, SHUT_RDWR);      /* no more receptions */
        close(private->callback.socket);
    }
    private->command.socket = 0;
    private->callback.socket = 0;

    free(private->command.recv_buf);
    free(private->callback.recv_buf);
    private->command.recv_buf = NULL;
    private->callback.recv_buf = NULL;

cib_tls_signon(cib_t * cib, struct remote_connection_s *connection, gboolean event_channel)
    int disconnected = 0;
    static struct mainloop_fd_callbacks cib_fd_callbacks = { 0, };

    cib_fd_callbacks.dispatch = event_channel ? cib_remote_callback_dispatch : cib_remote_command_dispatch;
    cib_fd_callbacks.destroy = cib_remote_connection_destroy;
    sock = crm_remote_tcp_connect(private->server, private->port);
    if (sock <= 0) {
        crm_perror(LOG_ERR, "remote tcp connection to %s:%d failed", private->server, private->port);
    connection->socket = sock;
        if (remote_gnutls_credentials_init == FALSE) {
            gnutls_global_init();
            gnutls_anon_allocate_client_credentials(&anon_cred_c);
            remote_gnutls_credentials_init = TRUE;
        }
        connection->session = crm_create_anon_tls_session(sock, GNUTLS_CLIENT, anon_cred_c);

        if (crm_initiate_client_tls_handshake(connection->session, DEFAULT_CLIENT_HANDSHAKE_TIMEOUT) != 0) {
            crm_err("Session creation for %s:%d failed", private->server, private->port);

            gnutls_deinit(*connection->session);
            gnutls_free(connection->session);
            connection->session = NULL;
    crm_recv_remote_msg(connection->session, &connection->recv_buf, connection->encrypted, -1, &disconnected);

    if (disconnected) {
        rc = -ENOTCONN;
    }

    answer = crm_parse_remote_buffer(&connection->recv_buf);

    free_xml(answer);
    answer = NULL;
        return rc;
    crm_trace("remote client connection established");
cib_remote_command_dispatch(gpointer user_data)
{
    int disconnected = 0;
    cib_t *cib = user_data;
    cib_remote_opaque_t *private = cib->variant_opaque;

    crm_recv_remote_msg(private->command.session, &private->command.recv_buf, private->command.encrypted, -1, &disconnected);

    free(private->command.recv_buf);
    private->command.recv_buf = NULL;
    crm_err("received late reply for remote cib connection, discarding");

    if (disconnected) {
        return -1;
    }
    return 0;
}

int
cib_remote_callback_dispatch(gpointer user_data)
    int disconnected = 0;
    crm_recv_remote_msg(private->callback.session, &private->callback.recv_buf, private->callback.encrypted, -1, &disconnected);
    msg = crm_parse_remote_buffer(&private->callback.recv_buf);
    while (msg) {
        const char *type = crm_element_value(msg, F_TYPE);
        crm_trace("Activating %s callbacks...", type);
        if (safe_str_eq(type, T_CIB)) {
            cib_native_callback(cib, msg, 0, 0);
        } else if (safe_str_eq(type, T_CIB_NOTIFY)) {
            g_list_foreach(cib->notify_list, cib_native_notify, msg);

        } else {
            crm_err("Unknown message type: %s", type);
        }
        msg = crm_parse_remote_buffer(&private->callback.recv_buf);
    }

    if (disconnected) {
        return -1;

    return 0;
        rc = cib_tls_signon(cib, &(private->command), FALSE);
        rc = cib_tls_signon(cib, &(private->callback), TRUE);
    int disconnected = 0;
    int remaining_time = 0;
    time_t start_time;
    if (!(call_options & cib_sync_call)) {
        crm_send_remote_msg(private->callback.session, op_msg, private->command.encrypted);
    } else {
        crm_send_remote_msg(private->command.session, op_msg, private->command.encrypted);
    }
    start_time = time(NULL);
    remaining_time = cib->call_timeout ? cib->call_timeout : 60;
    while (remaining_time > 0 && !disconnected) {
        crm_recv_remote_msg(private->command.session, &private->command.recv_buf, private->command.encrypted, remaining_time * 1000, &disconnected);
        op_reply = crm_parse_remote_buffer(&private->command.recv_buf);

        if (!op_reply) {
        /* wasn't the right reply, try and read some more */
        remaining_time = time(NULL) - start_time;
    if (disconnected) {
        crm_err("Disconnected while waiting for reply.");
        return -ENOTCONN;
    } else if (op_reply == NULL) {
        source->id = 0;
#include <arpa/inet.h>
#include <netdb.h>

const int anon_tls_kx_order[] = {
int
crm_initiate_client_tls_handshake(void *session_data, int timeout_ms)
{
    int rc = 0;
    int pollrc = 0;
    time_t start = time(NULL);
    gnutls_session *session = session_data;
    do {
        rc = gnutls_handshake(*session);
        if (rc == GNUTLS_E_INTERRUPTED || rc == GNUTLS_E_AGAIN) {
            pollrc = crm_recv_remote_ready(session, TRUE, 1000);
            if (pollrc < 0) {
                /* poll returned error, there is no hope */
                rc = -1;
            }
        }
    } while (((time(NULL) - start) < (timeout_ms/1000)) &&
            (rc == GNUTLS_E_INTERRUPTED || rc == GNUTLS_E_AGAIN));
    return rc;
}
void *
crm_create_anon_tls_session(int csock, int type /* GNUTLS_SERVER, GNUTLS_CLIENT */, void *credentials)
    gnutls_kx_set_priority(*session, anon_tls_kx_order);
    case GNUTLS_SERVER:
        gnutls_credentials_set(*session, GNUTLS_CRD_ANON, (gnutls_anon_server_credentials_t) credentials);
        break;
    case GNUTLS_CLIENT:
        gnutls_credentials_set(*session, GNUTLS_CRD_ANON, (gnutls_anon_client_credentials_t) credentials);
        break;
static int
crm_send_tls(gnutls_session * session, const char *buf, size_t len)
    const char *unsent = buf;
    int rc = 0;
    int total_send;
    if (buf == NULL) {
        return -1;
    total_send = len;
    crm_trace("Message size: %d", len);
    while (TRUE) {
        rc = gnutls_record_send(*session, unsent, len);
        if (rc == GNUTLS_E_INTERRUPTED || rc == GNUTLS_E_AGAIN) {
            crm_debug("Retry");
        } else if (rc < 0) {
            crm_err("Connection terminated rc = %d", rc);
            break;
        } else if (rc < len) {
            crm_debug("Only sent %d of %d bytes", rc, len);
            len -= rc;
            unsent += rc;
        } else {
            crm_debug("Sent %d bytes", rc);
            break;
    return rc < 0 ? rc : total_send;

/*!
 * \internal
 * \brief Read bytes off non blocking tls session.
 *
 * \param session - tls session to read
 * \param max_size - max bytes allowed to read for buffer. 0 assumes no limit
 *
 * \note only use with NON-Blocking sockets. Should only be used after polling socket.
 *       This function will return once max_size is met, the socket read buffer
 *       is empty, or an error is encountered.
 *
 * \retval '\0' terminated buffer on success
 */
crm_recv_tls(gnutls_session * session, size_t max_size, size_t *recv_len, int *disconnected)
    size_t len = 0;
    size_t chunk_size = max_size ? max_size : 1024;
    size_t buf_size = 0;
    size_t read_size = 0;
        if (disconnected) {
            *disconnected = 1;
        }
        goto done;
    buf = calloc(1, chunk_size + 1);
    buf_size = chunk_size;
        read_size = buf_size - len;
        /* automatically grow the buffer when needed if max_size is not set.*/
        if (!max_size && (read_size < (chunk_size / 2))) {
            buf_size += chunk_size;
            crm_trace("Grow buffer by %d more bytes. buf is now %d bytes", (int)chunk_size, buf_size);
            buf = realloc(buf, buf_size + 1);
            CRM_ASSERT(buf != NULL);
            read_size = buf_size - len;
        }
        rc = gnutls_record_recv(*session, buf + len, read_size);
        if (rc > 0) {
            crm_trace("Got %d more bytes.", rc);
            /* always null terminate buffer, the +1 to alloc always allows for this.*/
            buf[len] = '\0';
        }
        if (max_size && (max_size == read_size)) {
            crm_trace("Buffer max read size %d met" , max_size);
            goto done;
        }
        /* process any errors. */
        if (rc == GNUTLS_E_INTERRUPTED) {
            crm_trace("EINTR encoutered, retry tls read");
        } else if (rc == GNUTLS_E_AGAIN) {
            crm_trace("non-blocking, exiting read on rc = %d", rc);
            goto done;
        } else if (rc <= 0) {
            if (rc == 0) {
                crm_debug("EOF encoutered during TLS read");
            } else {
                crm_debug("Error receiving message: %s (%d)", gnutls_strerror(rc), rc);
            }
            if (disconnected) {
                *disconnected = 1;
            }
            goto done;

done:
    if (recv_len) {
        *recv_len = len;
    }
    if (!len) {
        free(buf);
        buf = NULL;
    }
    return buf;
static int
crm_send_plaintext(int sock, const char *buf, size_t len)
    int rc = 0;
    const char *unsent = buf;
    int total_send;
    if (buf == NULL) {
        return -1;
    }
    total_send = len;
    crm_trace("Message on socket %d: size=%d", sock, len);
  retry:
    rc = write(sock, unsent, len);
    if (rc < 0) {
        switch (errno) {
        case EINTR:
        case EAGAIN:
            crm_trace("Retry");
        default:
            crm_perror(LOG_ERR, "Could only write %d of the remaining %d bytes", rc, (int) len);
            break;

    } else if (rc < len) {
        crm_trace("Only sent %d of %d remaining bytes", rc, len);
        len -= rc;
        unsent += rc;
        goto retry;

     } else {
        crm_trace("Sent %d bytes: %.100s", rc, buf);

    return rc < 0 ? rc : total_send;
/*!
 * \internal
 * \brief Read bytes off non blocking socket.
 *
 * \param session - tls session to read
 * \param max_size - max bytes allowed to read for buffer. 0 assumes no limit
 *
 * \note only use with NON-Blocking sockets. Should only be used after polling socket.
 *       This function will return once max_size is met, the socket read buffer
 *       is empty, or an error is encountered.
 *
 * \retval '\0' terminated buffer on success
 */
static char *
crm_recv_plaintext(int sock, size_t max_size, size_t *recv_len, int *disconnected)
    ssize_t chunk_size = max_size ? max_size : 1024;
    size_t buf_size = 0;
    size_t read_size = 0;
    if (sock <= 0) {
        if (disconnected) {
            *disconnected = 1;
        }
        goto done;
    }
    buf = calloc(1, chunk_size + 1);
    buf_size = chunk_size;
    while (TRUE) {
        errno = 0;
        read_size = buf_size - len;
        /* automatically grow the buffer when needed if max_size is not set.*/
        if (!max_size && (read_size < (chunk_size / 2))) {
            buf_size += chunk_size;
            crm_trace("Grow buffer by %d more bytes. buf is now %d bytes", (int)chunk_size, buf_size);
            buf = realloc(buf, buf_size + 1);
            read_size = buf_size - len;
        }

        rc = read(sock, buf + len, chunk_size);

        if (rc > 0) {
            crm_trace("Got %d more bytes. errno=%d", (int)rc, errno);
            /* always null terminate buffer, the +1 to alloc always allows for this.*/
            buf[len] = '\0';
        }
        if (max_size && (max_size == read_size)) {
            crm_trace("Buffer max read size %d met" , max_size);
            goto done;
        }
        if (rc > 0) {
            continue;
        } else if (rc == 0) {
            if (disconnected) {
                *disconnected = 1;
            }
            crm_trace("EOF encoutered during read");
            goto done;
        }

        /* process errors */
        if (errno == EINTR) {
            crm_trace("EINTER encoutered, retry socket read.");
        } else if (errno == EAGAIN) {
            crm_trace("non-blocking, exiting read on rc = %d", rc);
            goto done;
        } else if (errno <= 0) {
            if (disconnected) {
                *disconnected = 1;
            }
            crm_debug("Error receiving message: %d", (int)rc);
            goto done;
done:
    if (recv_len) {
        *recv_len = len;
    }
    if (!len) {
        free(buf);
        buf = NULL;
    }
    return buf;
static int
crm_send_remote_msg_raw(void *session, const char *buf, size_t len, gboolean encrypted)
    int rc = -1;
        rc = crm_send_tls(session, buf, len);
        rc = crm_send_plaintext(GPOINTER_TO_INT(session), buf, len);
    return rc;
int
crm_send_remote_msg(void *session, xmlNode * msg, gboolean encrypted)
{
    int rc = -1;
    char *xml_text = NULL;
    int len = 0;

    xml_text = dump_xml_unformatted(msg);
    if (xml_text) {
        len = strlen(xml_text);
    } else {
        crm_err("Invalid XML, can not send msg");
        return -1;
    }

    rc = crm_send_remote_msg_raw(session, xml_text, len, encrypted);
    if (rc < 0) {
        goto done;
    }
    rc = crm_send_remote_msg_raw(session, REMOTE_MSG_TERMINATOR, strlen(REMOTE_MSG_TERMINATOR), encrypted);

done:
    if (rc < 0) {
        crm_err("Failed to send remote msg, rc = %d", rc);
    }

    free(xml_text);
    return rc;
}

/*!
 * \internal
 * \brief handles the recv buffer and parsing out msgs.
 * \note new_data is owned by this function once it is passed in.
 */
crm_parse_remote_buffer(char **msg_buf)
    char *buf = NULL;
    char *start = NULL;
    char *end = NULL;
    if (*msg_buf == NULL) {
        return NULL;
    }

    /* take ownership of the buffer */
    buf = *msg_buf;
    *msg_buf = NULL;

    /* MSGS are separated by a '\r\n\r\n'. Split a message off the buffer and return it. */
    start = buf;
    end = strstr(start, REMOTE_MSG_TERMINATOR);

    while (!xml && end) {

        /* grab the message */
        end[0] = '\0';
        end += strlen(REMOTE_MSG_TERMINATOR);

        xml = string2xml(start);
        if (xml == NULL) {
            crm_err("Couldn't parse: '%.120s'", start);
        }
        start = end;
        end = strstr(start, REMOTE_MSG_TERMINATOR);
    }

    if (xml && start) {
        /* we have msgs left over, save it until next time */
        *msg_buf = strdup(start);
        free(buf);
    } else if (!xml) {
        /* no msg present */
        *msg_buf = buf;
    }

    return xml;
}

/*!
 * \internal
 * \brief Determine if a remote session has data to read
 *
 * \retval 0, timeout occured.
 * \retval positive, data is ready to be read
 * \retval negative, session has ended
 */
int
crm_recv_remote_ready(void *session, gboolean encrypted, int timeout /* ms */)
{
    struct pollfd fds = { 0, };
    int sock = 0;
    void *sock_ptr = NULL;
    int rc = 0;
    time_t start;

    if (encrypted) {
#ifdef HAVE_GNUTLS_GNUTLS_H
        gnutls_session *tls_session = session;
        sock_ptr = gnutls_transport_get_ptr(*tls_session);
#else
        CRM_ASSERT(encrypted == FALSE);
#endif
    } else {
        sock_ptr = session;
    }

    sock = GPOINTER_TO_INT(sock_ptr);
    if (sock <= 0) {
        return -ENOTCONN;
    }

    start = time(NULL);
    errno = 0;
    do {
        fds.fd = sock;
        fds.events = POLLIN;

        /* If we got an EINTR while polling, and we have a
         * specific timeout we are trying to honor, attempt
         * to adjust the timeout to the closest second. */
        if (errno == EINTR && (timeout > 0)) {
            timeout = timeout - ((time(NULL) - start) * 1000);
            if (timeout < 1000) {
                timeout = 1000;
            }
        }

        rc = poll(&fds, 1, timeout);
    } while (rc < 0 && errno == EINTR);

    return rc;
}

char *
crm_recv_remote_raw(void *session, gboolean encrypted, size_t max_recv, size_t *recv_len, int *disconnected)
{
    char *reply = NULL;
    if (recv_len) {
        *recv_len = 0;
    }

    if (disconnected) {
        *disconnected = 0;
    }

        reply = crm_recv_tls(session, max_recv, recv_len, disconnected);
        reply = crm_recv_plaintext(GPOINTER_TO_INT(session), max_recv, recv_len, disconnected);
    }
    return reply;
}

/*!
 * \internal
 * \brief Read data off the socket until at least one full message is present or timeout occures.
 * \retval TRUE message read
 * \retval FALSE full message not read
 */

gboolean
crm_recv_remote_msg(void *session, char **recv_buf, gboolean encrypted, int total_timeout /*ms */, int *disconnected)
{
    int ret;
    size_t request_len = 0;
    time_t start = time(NULL);
    char *raw_request = NULL;
    int remaining_timeout = 0;

    if (total_timeout == 0) {
        total_timeout = 10000;
    } else if (total_timeout < 0) {
        total_timeout = 60000;
    }
    *disconnected = 0;

    remaining_timeout = total_timeout;
    while ((remaining_timeout > 0) && !(*disconnected)) {

        /* read some more off the tls buffer if we still have time left. */
        crm_trace("waiting to receive remote msg, starting timeout %d, remaining_timeout %d", total_timeout, remaining_timeout);
        ret = crm_recv_remote_ready(session, encrypted, remaining_timeout);
        raw_request = NULL;

        if (ret == 0) {
            crm_err("poll timed out (%d ms) while waiting to receive msg", remaining_timeout);
            return FALSE;

        } else if (ret < 0) {
            if (errno != EINTR) {
                crm_debug("poll returned error while waiting for msg, rc: %d, errno: %d", ret, errno);
                *disconnected = 1;
                return FALSE;
            }
            crm_debug("poll EINTR encountered during poll, retrying");
        } else {
            raw_request = crm_recv_remote_raw(session, encrypted, 0, &request_len, disconnected);
        }

        remaining_timeout = remaining_timeout - ((time(NULL) - start) * 1000);

        if (!raw_request) {
            crm_debug("Empty msg received after poll");
            continue;
        }

        if (*recv_buf) {
            int old_len = strlen(*recv_buf);

            crm_trace("Expanding recv buffer from %d to %d", old_len, old_len+request_len);

            *recv_buf = realloc(*recv_buf, old_len + request_len + 1);
            memcpy(*recv_buf + old_len, raw_request, request_len);
            *(*recv_buf+old_len+request_len) = '\0';
            free(raw_request);
        } else {
            *recv_buf = raw_request;
        }

        if (strstr(*recv_buf, REMOTE_MSG_TERMINATOR)) {
            return TRUE;
    return FALSE;

/*!
 * \internal
 * \brief tcp connection to server at specified port
 * \retval positive, socket fd.
 * \retval negative, failed to connect.
 */
int
crm_remote_tcp_connect(const char *host, int port)
{
    struct addrinfo *res;
    struct addrinfo *rp;
    struct addrinfo hints;
    const char *server = host;
    int ret_ga;
    int sock;

    /* getaddrinfo */
    memset(&hints, 0, sizeof(struct addrinfo));
    hints.ai_family = AF_UNSPEC;    /* Allow IPv4 or IPv6 */
    hints.ai_socktype = SOCK_STREAM;
    hints.ai_flags = AI_CANONNAME;

    crm_debug("Looking up %s", server);
    ret_ga = getaddrinfo(server, NULL, &hints, &res);
    if (ret_ga) {
        crm_err("getaddrinfo: %s", gai_strerror(ret_ga));
        return -1;
    }

    if (!res || !res->ai_addr) {
        crm_err("getaddrinfo failed");
        return -1;
    }

    for (rp = res; rp != NULL; rp = rp->ai_next) {
        struct sockaddr *addr = rp->ai_addr;
        int flag = 0;
        if (!addr) {
            continue;
        }

        if (rp->ai_canonname) {
            server = res->ai_canonname;
        }
        crm_debug("Got address %s for %s", server, host);

        /* create socket */
        sock = socket(rp->ai_family, SOCK_STREAM, IPPROTO_TCP);
        if (sock == -1) {
            crm_err("Socket creation failed for remote client connection.");
            continue;
        }
        if (addr->sa_family == AF_INET6) {
            struct sockaddr_in6 *addr_in = (struct sockaddr_in6 *) addr;
            addr_in->sin6_port = htons(port);
        } else {
            struct sockaddr_in *addr_in = (struct sockaddr_in *) addr;
            addr_in->sin_port = htons(port);
            crm_info("Attempting to connect to remote server at %s:%d", inet_ntoa(addr_in->sin_addr), port);
        }

        if (connect(sock, rp->ai_addr, rp->ai_addrlen) == 0) {
            if ((flag = fcntl(sock, F_GETFL)) >= 0) {
                if (fcntl(sock, F_SETFL, flag | O_NONBLOCK) < 0) {
                    crm_err( "fcntl() write failed");
                    close(sock);
                    sock = -1;
                    continue;
                }
            }
            break;                  /* Success */
        }

        close(sock);
        sock = -1;
    }
    freeaddrinfo(res);

    return sock;
}

                    print_as("Notification setup not supported, won't be able to reconnect after failure");