CVE-2013-0281 (C, published 2013-11-23T11:55Z)
Description: value: Pacemaker 1.1.10, when remote Cluster Information Base (CIB) configuration or resource management is enabled, does not limit the duration of connections to the blocking sockets, which allows remote attackers to cause a denial of service (connection blocking).
Severity: MEDIUM (CVSS v3 base score: )
Weakness: CWE-399 \u2014 Resource Management Errors
Weaknesses in this category are related to improper management of system resources.
Repository: https://github.com/ClusterLabs/pacemaker
Files changed:
cib/callbacks.c
cib/callbacks.h
cib/notify.c
cib/remote.c
include/crm_internal.h
lib/cib/cib_remote.c
lib/common/mainloop.c
lib/common/remote.c
tools/crm_mon.c
Commit message: High: core: Internal tls api improvements for reuse with future LRMD tls backend.


Vulnerable code:
            CRM_LOG_ASSERT(client_obj->request_id);
            rid = client_obj->request_id;
            client_obj->request_id = 0;
            crm_trace("Sending response %d to %s %s",
        
    if (client->ipc == NULL) {
#ifdef HAVE_DECL_NANOSLEEP
#  include <time.h>
#endif

extern gnutls_anon_server_credentials anon_cred_s;

extern gnutls_session *create_tls_session(int csock, int type);

/* 	gnutls_global_set_log_level (10); */
    int lpc = 0;
    time_t now = 0;
    time_t start = time(NULL);
    xmlNode *login = NULL;
    const char *user = NULL;
    const char *pass = NULL;
    const char *tmp = NULL;

#ifdef HAVE_DECL_NANOSLEEP
    const struct timespec sleepfast = { 0, 10000000 };  /* 10 millisec */
#endif

        };    
    
        session = create_tls_session(csock, GNUTLS_SERVER);
    do {
        crm_trace("Iter: %d", lpc++);
        if (ssock == remote_tls_fd) {
#ifdef HAVE_GNUTLS_GNUTLS_H
            login = crm_recv_remote_msg(session, TRUE);
#endif
        } else {
            login = crm_recv_remote_msg(GINT_TO_POINTER(csock), FALSE);
        }
        if (login != NULL) {
            break;
        }
#ifdef HAVE_DECL_NANOSLEEP
        nanosleep(&sleepfast, NULL);
#else
        sleep(1);
#endif
        now = time(NULL);

        /* Peers have 3s to connect */
    } while (login == NULL && (start - now) < 4);

    crm_log_xml_info(login, "Login: ");
    if (login == NULL) {
        goto bail;
    }

    tmp = crm_element_name(login);
    if (safe_str_neq(tmp, "cib_command")) {
        crm_err("Wrong tag: %s", tmp);
        goto bail;
    }

    tmp = crm_element_value(login, "op");
    if (safe_str_neq(tmp, "authenticate")) {
        crm_err("Wrong operation: %s", tmp);
        goto bail;
    }

    user = crm_element_value(login, "user");
    pass = crm_element_value(login, "password");

    /* Non-root daemons can only validate the password of the
     * user they're running as
     */
    if (check_group_membership(user, CRM_DAEMON_GROUP) == FALSE) {
        crm_err("User is not a member of the required group");
        goto bail;

    } else if (authenticate_user(user, pass) == FALSE) {
        crm_err("PAM auth failed");
        goto bail;
    }

    /* send ACK */
    new_client->name = crm_element_value_copy(login, "name");

    CRM_CHECK(new_client->id == NULL, free(new_client->id));

#if ENABLE_ACL
    new_client->user = strdup(user);
#endif

    free_xml(login);
    login = create_xml_node(NULL, "cib_result");
    crm_xml_add(login, F_CIB_OPERATION, CRM_OP_REGISTER);
    crm_xml_add(login, F_CIB_CLIENTID, new_client->id);
    crm_send_remote_msg(new_client->session, login, new_client->encrypted);
    free_xml(login);


  bail:
    if (ssock == remote_tls_fd) {
#ifdef HAVE_GNUTLS_GNUTLS_H
        gnutls_bye(*session, GNUTLS_SHUT_RDWR);
        gnutls_deinit(*session);
        gnutls_free(session);
#endif
    }
    close(csock);
    free_xml(login);
    return TRUE;
int
cib_remote_msg(gpointer data)
    xmlNode *command = NULL;
    cib_client_t *client = data;

    crm_trace("%s callback", client->encrypted ? "secure" : "clear-text");

    command = crm_recv_remote_msg(client->session, client->encrypted);
    if (command == NULL) {
        return -1;
    }
        goto bail;
  bail:
    free_xml(command);
    command = NULL;
xmlNode *crm_recv_remote_msg(void *session, gboolean encrypted);
void crm_send_remote_msg(void *session, xmlNode * msg, gboolean encrypted);
extern gnutls_anon_client_credentials anon_cred_c;
extern gnutls_session *create_tls_session(int csock, int type);
int cib_remote_dispatch(gpointer user_data);
    shutdown(private->command.socket, SHUT_RDWR);       /* no more receptions */
    shutdown(private->callback.socket, SHUT_RDWR);      /* no more receptions */
    close(private->command.socket);
    close(private->callback.socket);

        gnutls_bye(*(private->command.session), GNUTLS_SHUT_RDWR);
        gnutls_deinit(*(private->command.session));
        gnutls_free(private->command.session);

        gnutls_bye(*(private->callback.session), GNUTLS_SHUT_RDWR);
        gnutls_deinit(*(private->callback.session));
        gnutls_free(private->callback.session);
        gnutls_anon_free_client_credentials(anon_cred_c);
        gnutls_global_deinit();
cib_tls_signon(cib_t * cib, struct remote_connection_s *connection)
    struct sockaddr_in addr;
    char *server = private->server;

    int ret_ga;
    struct addrinfo *res;
    struct addrinfo hints;
    static struct mainloop_fd_callbacks cib_fd_callbacks = 
        {
            .dispatch = cib_remote_dispatch,
            .destroy = cib_remote_connection_destroy,
        };
    /* create socket */
    sock = socket(PF_INET, SOCK_STREAM, IPPROTO_TCP);
    if (sock == -1) {
        crm_perror(LOG_ERR, "Socket creation failed");
        return -1;
    }

    /* getaddrinfo */
    bzero(&hints, sizeof(struct addrinfo));
    hints.ai_flags = AI_CANONNAME;
    hints.ai_family = AF_INET;
    hints.ai_socktype = SOCK_RAW;

    if (hints.ai_family == AF_INET6) {
        hints.ai_protocol = IPPROTO_ICMPV6;
    } else {
        hints.ai_protocol = IPPROTO_ICMP;
    }

    crm_debug("Looking up %s", server);
    ret_ga = getaddrinfo(server, NULL, &hints, &res);
    if (ret_ga) {
        crm_err("getaddrinfo: %s", gai_strerror(ret_ga));
        close(sock);
        return -1;
    }

    if (res->ai_canonname) {
        server = res->ai_canonname;
    }

    crm_debug("Got address %s for %s", server, private->server);

    if (!res->ai_addr) {
        fprintf(stderr, "getaddrinfo failed");
        crm_exit(1);
    }
#if 1
    memcpy(&addr, res->ai_addr, res->ai_addrlen);
#else
    /* connect to server */
    memset(&addr, 0, sizeof(addr));
    addr.sin_family = AF_INET;
    addr.sin_addr.s_addr = inet_addr(server);
#endif
    addr.sin_port = htons(private->port);

    if (connect(sock, (struct sockaddr *)&addr, sizeof(addr)) == -1) {
        crm_perror(LOG_ERR, "Connection to %s:%d failed", server, private->port);
        close(sock);
        return -1;
        gnutls_global_init();
        gnutls_anon_allocate_client_credentials(&anon_cred_c);
        connection->session = create_tls_session(sock, GNUTLS_CLIENT);
        if (connection->session == NULL) {
            crm_perror(LOG_ERR, "Session creation for %s:%d failed", server, private->port);
            close(sock);
    answer = crm_recv_remote_msg(connection->session, connection->encrypted);
    connection->socket = sock;
cib_remote_dispatch(gpointer user_data)
    const char *type = NULL;
    msg = crm_recv_remote_msg(private->callback.session, private->callback.encrypted);
    type = crm_element_value(msg, F_TYPE);
    crm_trace("Activating %s callbacks...", type);
    if (safe_str_eq(type, T_CIB)) {
        cib_native_callback(cib, msg, 0, 0);
    } else if (safe_str_eq(type, T_CIB_NOTIFY)) {
        g_list_foreach(cib->notify_list, cib_native_notify, msg);
    } else {
        crm_err("Unknown message type: %s", type);
    }
    if (msg != NULL) {
        return 0;
    return -1;
        rc = cib_tls_signon(cib, &(private->command));
        rc = cib_tls_signon(cib, &(private->callback));
static gboolean timer_expired = FALSE;
static struct timer_rec_s *sync_timer = NULL;
static gboolean
cib_timeout_handler(gpointer data)
{
    struct timer_rec_s *timer = data;

    timer_expired = TRUE;
    crm_err("Call %d timed out after %ds", timer->call_id, timer->timeout);

    /* Always return TRUE, never remove the handler
     * We do that after the while-loop in cib_native_perform_op()
     */
    return TRUE;
}

    if (sync_timer == NULL) {
        sync_timer = calloc(1, sizeof(struct timer_rec_s));
    }

    crm_send_remote_msg(private->command.session, op_msg, private->command.encrypted);
    if (cib->call_timeout > 0) {
        /* We need this, even with msgfromIPC_timeout(), because we might
         * get other/older replies that don't match the active request
         */
        timer_expired = FALSE;
        sync_timer->call_id = cib->call_id;
        sync_timer->timeout = cib->call_timeout * 1000;
        sync_timer->ref = g_timeout_add(sync_timer->timeout, cib_timeout_handler, sync_timer);
    }
    while (timer_expired == FALSE) {
        op_reply = crm_recv_remote_msg(private->command.session, private->command.encrypted);
        if (op_reply == NULL) {
        CRM_CHECK(reply_id > 0, free_xml(op_reply);
                  if (sync_timer->ref > 0) {
                  g_source_remove(sync_timer->ref); sync_timer->ref = 0;}
                  return -ENOMSG) ;
    }

    if (sync_timer->ref > 0) {
        g_source_remove(sync_timer->ref);
        sync_timer->ref = 0;
    }
    if (timer_expired) {
        return -ETIME;
    if (op_reply == NULL) {

const int tls_kx_order[] = {
gnutls_anon_client_credentials anon_cred_c;
gnutls_anon_server_credentials anon_cred_s;
static char *cib_send_tls(gnutls_session * session, xmlNode * msg);
static char *cib_recv_tls(gnutls_session * session);
#endif
char *cib_recv_plaintext(int sock);
char *cib_send_plaintext(int sock, xmlNode * msg);
#ifdef HAVE_GNUTLS_GNUTLS_H
gnutls_session *create_tls_session(int csock, int type);
gnutls_session *
create_tls_session(int csock, int type /* GNUTLS_SERVER, GNUTLS_CLIENT */ )
    int rc = 0;
    gnutls_kx_set_priority(*session, tls_kx_order);
        case GNUTLS_SERVER:
            gnutls_credentials_set(*session, GNUTLS_CRD_ANON, anon_cred_s);
            break;
        case GNUTLS_CLIENT:
            gnutls_credentials_set(*session, GNUTLS_CRD_ANON, anon_cred_c);
            break;
    do {
        rc = gnutls_handshake(*session);
    } while (rc == GNUTLS_E_INTERRUPTED || rc == GNUTLS_E_AGAIN);

    if (rc < 0) {
        crm_err("Handshake failed: %s", gnutls_strerror(rc));
        gnutls_deinit(*session);
        gnutls_free(session);
        return NULL;
    }
static char *
cib_send_tls(gnutls_session * session, xmlNode * msg)
    char *xml_text = NULL;

#  if 0
    const char *name = crm_element_name(msg);
    if (safe_str_neq(name, "cib_command")) {
        xmlNodeSetName(msg, "cib_result");
#  endif
    xml_text = dump_xml_unformatted(msg);
    if (xml_text != NULL) {
        char *unsent = xml_text;
        int len = strlen(xml_text);
        int rc = 0;
        len++;                  /* null char */
        crm_trace("Message size: %d", len);
        while (TRUE) {
            rc = gnutls_record_send(*session, unsent, len);
            crm_debug("Sent %d bytes", rc);
            if (rc == GNUTLS_E_INTERRUPTED || rc == GNUTLS_E_AGAIN) {
                crm_debug("Retry");
            } else if (rc < 0) {
                crm_debug("Connection terminated");
                break;
            } else if (rc < len) {
                crm_debug("Only sent %d of %d bytes", rc, len);
                len -= rc;
                unsent += rc;
            } else {
                break;
            }

    free(xml_text);
    return NULL;
cib_recv_tls(gnutls_session * session)

    int len = 0;
    int chunk_size = 1024;
        return NULL;
    buf = calloc(1, chunk_size);
        errno = 0;
        rc = gnutls_record_recv(*session, buf + len, chunk_size);
        crm_trace("Got %d more bytes. errno=%d", rc, errno);
        if (rc == GNUTLS_E_INTERRUPTED || rc == GNUTLS_E_AGAIN) {
            crm_trace("Retry");
        } else if (rc == GNUTLS_E_UNEXPECTED_PACKET_LENGTH) {
            crm_trace("Session disconnected");
            goto bail;
        } else if (rc < 0) {
            crm_err("Error receiving message: %s (%d)", gnutls_strerror(rc), rc);
            goto bail;
        } else if (rc == chunk_size) {
            chunk_size *= 2;
            buf = realloc(buf, len + chunk_size);
            crm_trace("Retry with %d more bytes", (int)chunk_size);
            CRM_ASSERT(buf != NULL);

        } else if (buf[len + rc - 1] != 0) {
            crm_trace("Last char is %d '%c'", buf[len + rc - 1], buf[len + rc - 1]);
            crm_trace("Retry with %d more bytes", (int)chunk_size);
            len += rc;
            buf = realloc(buf, len + chunk_size);
            CRM_ASSERT(buf != NULL);
        } else {
            crm_trace("Got %d more bytes", (int)rc);
            return buf;
  bail:
    free(buf);
    return NULL;
char *
cib_send_plaintext(int sock, xmlNode * msg)
    char *xml_text = dump_xml_unformatted(msg);
    if (xml_text != NULL) {
        int rc = 0;
        char *unsent = xml_text;
        int len = strlen(xml_text);
        len++;                  /* null char */
        crm_trace("Message on socket %d: size=%d", sock, len);
  retry:
        rc = write(sock, unsent, len);
        if (rc < 0) {
            switch (errno) {
                case EINTR:
                case EAGAIN:
                    crm_trace("Retry");
                    goto retry;
                default:
                    crm_perror(LOG_ERR, "Could only write %d of the remaining %d bytes", rc, len);
                    break;
            }
        } else if (rc < len) {
            crm_trace("Only sent %d of %d remaining bytes", rc, len);
            len -= rc;
            unsent += rc;

        } else {
            crm_trace("Sent %d bytes: %.100s", rc, xml_text);
    free(xml_text);
    return NULL;
char *
cib_recv_plaintext(int sock)

    ssize_t chunk_size = 512;
    buf = calloc(1, chunk_size);
    while (1) {
        errno = 0;
        rc = read(sock, buf + len, chunk_size);
        crm_trace("Got %d more bytes. errno=%d", (int)rc, errno);

        if (errno == EINTR || errno == EAGAIN) {
            crm_trace("Retry: %d", (int)rc);
            if (rc > 0) {
                len += rc;
                buf = realloc(buf, len + chunk_size);
                CRM_ASSERT(buf != NULL);
            }
        } else if (rc < 0) {
            crm_perror(LOG_ERR, "Error receiving message: %d", (int)rc);
            goto bail;
        } else if (rc == chunk_size) {
            len += rc;
            chunk_size *= 2;
            buf = realloc(buf, len + chunk_size);
            crm_trace("Retry with %d more bytes", (int)chunk_size);
        } else if (buf[len + rc - 1] != 0) {
            crm_trace("Last char is %d '%c'", buf[len + rc - 1], buf[len + rc - 1]);
            crm_trace("Retry with %d more bytes", (int)chunk_size);
            buf = realloc(buf, len + chunk_size);
            CRM_ASSERT(buf != NULL);
        } else {
            return buf;
  bail:
    free(buf);
    return NULL;
void
crm_send_remote_msg(void *session, xmlNode * msg, gboolean encrypted)
        cib_send_tls(session, msg);
        cib_send_plaintext(GPOINTER_TO_INT(session), msg);
crm_recv_remote_msg(void *session, gboolean encrypted)
    char *reply = NULL;
        reply = cib_recv_tls(session);
        reply = cib_recv_plaintext(GPOINTER_TO_INT(session));
    } else {
        xml = string2xml(reply);
        if (xml == NULL) {
            crm_err("Couldn't parse: '%.120s'", reply);
    free(reply);
    return xml;
                    print_as("Notification setup failed, won't be able to reconnect after failure");

Fixed code:
            if (client_obj->ipc) {
                CRM_LOG_ASSERT(client_obj->request_id);
                rid = client_obj->request_id;
                client_obj->request_id = 0;
                crm_trace("Sending response %d to %s %s",
            } else {
                crm_trace("Sending response to %s %s",
                      client_obj->name, from_peer?"(originator of delegated request)":"");
            }
    char *recv_buf;
    gboolean handshake_complete;
    gboolean remote_auth;

    int remote_auth_timeout;
    if (client->ipc == NULL && client->session == NULL) {
gnutls_anon_server_credentials anon_cred_s;
#define REMOTE_AUTH_TIMEOUT 10000

        /* gnutls_global_set_log_level (10); */
static gboolean
cib_remote_auth(xmlNode *login)
{
    const char *user = NULL;
    const char *pass = NULL;
    const char *tmp = NULL;

    crm_log_xml_info(login, "Login: ");
    if (login == NULL) {
        return FALSE;
    }

    tmp = crm_element_name(login);
    if (safe_str_neq(tmp, "cib_command")) {
        crm_err("Wrong tag: %s", tmp);
        return FALSE;
    }

    tmp = crm_element_value(login, "op");
    if (safe_str_neq(tmp, "authenticate")) {
        crm_err("Wrong operation: %s", tmp);
        return FALSE;
    }

    user = crm_element_value(login, "user");
    pass = crm_element_value(login, "password");

    if (!user || !pass) {
        crm_err("missing auth credentials");
        return FALSE;
    }

    /* Non-root daemons can only validate the password of the
     * user they're running as
     */
    if (check_group_membership(user, CRM_DAEMON_GROUP) == FALSE) {
        crm_err("User is not a member of the required group");
        return FALSE;

    } else if (authenticate_user(user, pass) == FALSE) {
        crm_err("PAM auth failed");
        return FALSE;
    }

    return TRUE;
}

static gboolean
remote_auth_timeout_cb(gpointer data)
{
    cib_client_t *client = data;

    client->remote_auth_timeout = 0;

    if (client->remote_auth == TRUE) {
        return FALSE;
    }

    mainloop_del_fd(client->remote);
    crm_err("Remote client authentication timed out");

    return FALSE;
}
    int flag;
        };

    if ((flag = fcntl(csock, F_GETFL)) >= 0) {
        if (fcntl(csock, F_SETFL, flag | O_NONBLOCK) < 0) {
            crm_err( "fcntl() write failed");
            close(csock);
            return TRUE;
        }
    } else {
        crm_err( "fcntl() read failed");
        close(csock);
        return TRUE;
    }

        session = crm_create_anon_tls_session(csock, GNUTLS_SERVER, anon_cred_s);
    /* clients have a few seconds to perform handshake. */
    new_client->remote_auth_timeout = g_timeout_add(REMOTE_AUTH_TIMEOUT, remote_auth_timeout_cb, new_client);

    int csock = 0;
    if (client->remote_auth_timeout) {
        g_source_remove(client->remote_auth_timeout);
    }

    if (client->encrypted) {
#ifdef HAVE_GNUTLS_GNUTLS_H
        if (client->session) {
            void *sock_ptr = gnutls_transport_get_ptr(*client->session);
            csock = GPOINTER_TO_INT(sock_ptr);
            if (client->handshake_complete) {
                gnutls_bye(*client->session, GNUTLS_SHUT_WR);
            }
            gnutls_deinit(*client->session);
            gnutls_free(client->session);
        }
#endif
    } else {
        csock = GPOINTER_TO_INT(client->session);
    }
    client->session = NULL;

    if (csock > 0) {
        close(csock);
    }

    free(client->recv_buf);
static void
cib_handle_remote_msg(cib_client_t *client, xmlNode *command)
        return;
}

int
cib_remote_msg(gpointer data)
{
    xmlNode *command = NULL;
    cib_client_t *client = data;
    int disconnected = 0;
    int timeout = client->remote_auth ? -1 : 1000;

    crm_trace("%s callback", client->encrypted ? "secure" : "clear-text");

#ifdef HAVE_GNUTLS_GNUTLS_H
    if (client->encrypted && (client->handshake_complete == FALSE)) {
        int rc = 0;

        /* Muliple calls to handshake will be required, this callback
         * will be invoked once the client sends more handshake data. */
        do {
            rc = gnutls_handshake(*client->session);

            if (rc < 0 && rc != GNUTLS_E_AGAIN) {
                crm_err("Remote cib tls handshake failed");
                return -1;
            }
        } while (rc == GNUTLS_E_INTERRUPTED);

        if (rc == 0) {
            crm_debug("Remote cib tls handshake completed");
            client->handshake_complete = TRUE;
            if (client->remote_auth_timeout) {
                g_source_remove(client->remote_auth_timeout);
            }
            /* after handshake, clients must send auth in a few seconds */
            client->remote_auth_timeout = g_timeout_add(REMOTE_AUTH_TIMEOUT, remote_auth_timeout_cb, client);
        }
        return 0;
    }
#endif

    crm_recv_remote_msg(client->session, &client->recv_buf, client->encrypted, timeout, &disconnected);

    /* must pass auth before we will process anything else */
    if (client->remote_auth == FALSE) {
        xmlNode *reg;
#if ENABLE_ACL
        const char *user = NULL;
#endif
        command = crm_parse_remote_buffer(&client->recv_buf);
        if (cib_remote_auth(command) == FALSE) {
            free_xml(command);
            return -1;
        }

        crm_debug("remote connection authenticated successfully");
        client->remote_auth = TRUE;
        g_source_remove(client->remote_auth_timeout);
        client->remote_auth_timeout = 0;
        client->name = crm_element_value_copy(command, "name");

#if ENABLE_ACL
        user = crm_element_value(command, "user");
        if (user) {
           new_client->user = strdup(user);
        }
#endif

        /* send ACK */
        reg = create_xml_node(NULL, "cib_result");
        crm_xml_add(reg, F_CIB_OPERATION, CRM_OP_REGISTER);
        crm_xml_add(reg, F_CIB_CLIENTID, client->id);
        crm_send_remote_msg(client->session, reg, client->encrypted);
        free_xml(reg);
        free_xml(command);
    }

    command = crm_parse_remote_buffer(&client->recv_buf);
    while (command) {
        crm_trace("command received");
        cib_handle_remote_msg(client, command);
        free_xml(command);
        command = crm_parse_remote_buffer(&client->recv_buf);
    }

    if (disconnected) {
        crm_trace("disconnected while receiving remote cib msg.");
        return -1;
    }



/*! remote tcp/tls helper functions */
gboolean crm_recv_remote_msg(void *session, char **recv_buf, gboolean encrypted, int total_timeout_ms, int *disconnected);
char *crm_recv_remote_raw(void *data, gboolean encrypted, size_t max_recv, size_t *recv_len, int *disconnected);
int crm_send_remote_msg(void *session, xmlNode * msg, gboolean encrypted);
int crm_recv_remote_ready(void *session, gboolean encrypted, int timeout_ms);
xmlNode *crm_parse_remote_buffer(char **msg_buf);
int crm_remote_tcp_connect(const char *host, int port);

#ifdef HAVE_GNUTLS_GNUTLS_H
/*!
 * \internal
 * \brief Initiate the client handshake after establishing the tcp socket.
 * \note This is a blocking function, it will block until the entire handshake
 *       is complete or until the timeout period is reached.
 * \retval 0 success
 * \retval negative, failure
 */
int crm_initiate_client_tls_handshake(void *session_data, int timeout_ms);
/*!
 * \internal
 * \brief Create client or server session for anon DH encryption credentials
 * \param sock, the socket the session will use for transport
 * \param type, GNUTLS_SERVER or GNUTLS_CLIENT
 * \param credentials, gnutls_anon_server_credentials_t or gnutls_anon_client_credentials_t
 *
 * \retval gnutls_session * on success
 * \retval NULL on failure
 */
void *crm_create_anon_tls_session(int sock, int type, void *credentials);
#endif

#define REMOTE_MSG_TERMINATOR "\r\n\r\n"
gnutls_anon_client_credentials anon_cred_c;
#define DEFAULT_CLIENT_HANDSHAKE_TIMEOUT 5000 /* 5 seconds */
static gboolean remote_gnutls_credentials_init = FALSE;
    char *recv_buf;
int cib_remote_callback_dispatch(gpointer user_data);
int cib_remote_command_dispatch(gpointer user_data);
        if (private->command.session) {
            gnutls_bye(*(private->command.session), GNUTLS_SHUT_RDWR);
            gnutls_deinit(*(private->command.session));
            gnutls_free(private->command.session);
        }
        if (private->callback.session) {
            gnutls_bye(*(private->callback.session), GNUTLS_SHUT_RDWR);
            gnutls_deinit(*(private->callback.session));
            gnutls_free(private->callback.session);
        }
        private->command.session = NULL;
        private->callback.session = NULL;
        if (remote_gnutls_credentials_init) {
            gnutls_anon_free_client_credentials(anon_cred_c);
            gnutls_global_deinit();
            remote_gnutls_credentials_init = FALSE;
        }

    if (private->command.socket) {
        shutdown(private->command.socket, SHUT_RDWR);       /* no more receptions */
        close(private->command.socket);
    }
    if (private->callback.socket) {
        shutdown(private->callback.socket, SHUT_RDWR);      /* no more receptions */
        close(private->callback.socket);
    }
    private->command.socket = 0;
    private->callback.socket = 0;

    free(private->command.recv_buf);
    free(private->callback.recv_buf);
    private->command.recv_buf = NULL;
    private->callback.recv_buf = NULL;

cib_tls_signon(cib_t * cib, struct remote_connection_s *connection, gboolean event_channel)
    int disconnected = 0;
    static struct mainloop_fd_callbacks cib_fd_callbacks = { 0, };

    cib_fd_callbacks.dispatch = event_channel ? cib_remote_callback_dispatch : cib_remote_command_dispatch;
    cib_fd_callbacks.destroy = cib_remote_connection_destroy;
    sock = crm_remote_tcp_connect(private->server, private->port);
    if (sock <= 0) {
        crm_perror(LOG_ERR, "remote tcp connection to %s:%d failed", private->server, private->port);
    connection->socket = sock;
        if (remote_gnutls_credentials_init == FALSE) {
            gnutls_global_init();
            gnutls_anon_allocate_client_credentials(&anon_cred_c);
            remote_gnutls_credentials_init = TRUE;
        }
        connection->session = crm_create_anon_tls_session(sock, GNUTLS_CLIENT, anon_cred_c);

        if (crm_initiate_client_tls_handshake(connection->session, DEFAULT_CLIENT_HANDSHAKE_TIMEOUT) != 0) {
            crm_err("Session creation for %s:%d failed", private->server, private->port);

            gnutls_deinit(*connection->session);
            gnutls_free(connection->session);
            connection->session = NULL;
    crm_recv_remote_msg(connection->session, &connection->recv_buf, connection->encrypted, -1, &disconnected);

    if (disconnected) {
        rc = -ENOTCONN;
    }

    answer = crm_parse_remote_buffer(&connection->recv_buf);

    free_xml(answer);
    answer = NULL;
        return rc;
    crm_trace("remote client connection established");
cib_remote_command_dispatch(gpointer user_data)
{
    int disconnected = 0;
    cib_t *cib = user_data;
    cib_remote_opaque_t *private = cib->variant_opaque;

    crm_recv_remote_msg(private->command.session, &private->command.recv_buf, private->command.encrypted, -1, &disconnected);

    free(private->command.recv_buf);
    private->command.recv_buf = NULL;
    crm_err("received late reply for remote cib connection, discarding");

    if (disconnected) {
        return -1;
    }
    return 0;
}

int
cib_remote_callback_dispatch(gpointer user_data)
    int disconnected = 0;
    crm_recv_remote_msg(private->callback.session, &private->callback.recv_buf, private->callback.encrypted, -1, &disconnected);
    msg = crm_parse_remote_buffer(&private->callback.recv_buf);
    while (msg) {
        const char *type = crm_element_value(msg, F_TYPE);
        crm_trace("Activating %s callbacks...", type);
        if (safe_str_eq(type, T_CIB)) {
            cib_native_callback(cib, msg, 0, 0);
        } else if (safe_str_eq(type, T_CIB_NOTIFY)) {
            g_list_foreach(cib->notify_list, cib_native_notify, msg);

        } else {
            crm_err("Unknown message type: %s", type);
        }
        msg = crm_parse_remote_buffer(&private->callback.recv_buf);
    }

    if (disconnected) {
        return -1;

    return 0;
        rc = cib_tls_signon(cib, &(private->command), FALSE);
        rc = cib_tls_signon(cib, &(private->callback), TRUE);
    int disconnected = 0;
    int remaining_time = 0;
    time_t start_time;
    if (!(call_options & cib_sync_call)) {
        crm_send_remote_msg(private->callback.session, op_msg, private->command.encrypted);
    } else {
        crm_send_remote_msg(private->command.session, op_msg, private->command.encrypted);
    }
    start_time = time(NULL);
    remaining_time = cib->call_timeout ? cib->call_timeout : 60;
    while (remaining_time > 0 && !disconnected) {
        crm_recv_remote_msg(private->command.session, &private->command.recv_buf, private->command.encrypted, remaining_time * 1000, &disconnected);
        op_reply = crm_parse_remote_buffer(&private->command.recv_buf);

        if (!op_reply) {
        /* wasn't the right reply, try and read some more */
        remaining_time = time(NULL) - start_time;
    if (disconnected) {
        crm_err("Disconnected while waiting for reply.");
        return -ENOTCONN;
    } else if (op_reply == NULL) {
        source->id = 0;
#include <arpa/inet.h>
#include <netdb.h>

const int anon_tls_kx_order[] = {
int
crm_initiate_client_tls_handshake(void *session_data, int timeout_ms)
{
    int rc = 0;
    int pollrc = 0;
    time_t start = time(NULL);
    gnutls_session *session = session_data;
    do {
        rc = gnutls_handshake(*session);
        if (rc == GNUTLS_E_INTERRUPTED || rc == GNUTLS_E_AGAIN) {
            pollrc = crm_recv_remote_ready(session, TRUE, 1000);
            if (pollrc < 0) {
                /* poll returned error, there is no hope */
                rc = -1;
            }
        }
    } while (((time(NULL) - start) < (timeout_ms/1000)) &&
            (rc == GNUTLS_E_INTERRUPTED || rc == GNUTLS_E_AGAIN));
    return rc;
}
void *
crm_create_anon_tls_session(int csock, int type /* GNUTLS_SERVER, GNUTLS_CLIENT */, void *credentials)
    gnutls_kx_set_priority(*session, anon_tls_kx_order);
    case GNUTLS_SERVER:
        gnutls_credentials_set(*session, GNUTLS_CRD_ANON, (gnutls_anon_server_credentials_t) credentials);
        break;
    case GNUTLS_CLIENT:
        gnutls_credentials_set(*session, GNUTLS_CRD_ANON, (gnutls_anon_client_credentials_t) credentials);
        break;
static int
crm_send_tls(gnutls_session * session, const char *buf, size_t len)
    const char *unsent = buf;
    int rc = 0;
    int total_send;
    if (buf == NULL) {
        return -1;
    total_send = len;
    crm_trace("Message size: %d", len);
    while (TRUE) {
        rc = gnutls_record_send(*session, unsent, len);
        if (rc == GNUTLS_E_INTERRUPTED || rc == GNUTLS_E_AGAIN) {
            crm_debug("Retry");
        } else if (rc < 0) {
            crm_err("Connection terminated rc = %d", rc);
            break;
        } else if (rc < len) {
            crm_debug("Only sent %d of %d bytes", rc, len);
            len -= rc;
            unsent += rc;
        } else {
            crm_debug("Sent %d bytes", rc);
            break;
    return rc < 0 ? rc : total_send;

/*!
 * \internal
 * \brief Read bytes off non blocking tls session.
 *
 * \param session - tls session to read
 * \param max_size - max bytes allowed to read for buffer. 0 assumes no limit
 *
 * \note only use with NON-Blocking sockets. Should only be used after polling socket.
 *       This function will return once max_size is met, the socket read buffer
 *       is empty, or an error is encountered.
 *
 * \retval '\0' terminated buffer on success
 */
crm_recv_tls(gnutls_session * session, size_t max_size, size_t *recv_len, int *disconnected)
    size_t len = 0;
    size_t chunk_size = max_size ? max_size : 1024;
    size_t buf_size = 0;
    size_t read_size = 0;
        if (disconnected) {
            *disconnected = 1;
        }
        goto done;
    buf = calloc(1, chunk_size + 1);
    buf_size = chunk_size;
        read_size = buf_size - len;
        /* automatically grow the buffer when needed if max_size is not set.*/
        if (!max_size && (read_size < (chunk_size / 2))) {
            buf_size += chunk_size;
            crm_trace("Grow buffer by %d more bytes. buf is now %d bytes", (int)chunk_size, buf_size);
            buf = realloc(buf, buf_size + 1);
            CRM_ASSERT(buf != NULL);
            read_size = buf_size - len;
        }
        rc = gnutls_record_recv(*session, buf + len, read_size);
        if (rc > 0) {
            crm_trace("Got %d more bytes.", rc);
            /* always null terminate buffer, the +1 to alloc always allows for this.*/
            buf[len] = '\0';
        }
        if (max_size && (max_size == read_size)) {
            crm_trace("Buffer max read size %d met" , max_size);
            goto done;
        }
        /* process any errors. */
        if (rc == GNUTLS_E_INTERRUPTED) {
            crm_trace("EINTR encoutered, retry tls read");
        } else if (rc == GNUTLS_E_AGAIN) {
            crm_trace("non-blocking, exiting read on rc = %d", rc);
            goto done;
        } else if (rc <= 0) {
            if (rc == 0) {
                crm_debug("EOF encoutered during TLS read");
            } else {
                crm_debug("Error receiving message: %s (%d)", gnutls_strerror(rc), rc);
            }
            if (disconnected) {
                *disconnected = 1;
            }
            goto done;

done:
    if (recv_len) {
        *recv_len = len;
    }
    if (!len) {
        free(buf);
        buf = NULL;
    }
    return buf;
static int
crm_send_plaintext(int sock, const char *buf, size_t len)
    int rc = 0;
    const char *unsent = buf;
    int total_send;
    if (buf == NULL) {
        return -1;
    }
    total_send = len;
    crm_trace("Message on socket %d: size=%d", sock, len);
  retry:
    rc = write(sock, unsent, len);
    if (rc < 0) {
        switch (errno) {
        case EINTR:
        case EAGAIN:
            crm_trace("Retry");
        default:
            crm_perror(LOG_ERR, "Could only write %d of the remaining %d bytes", rc, (int) len);
            break;

    } else if (rc < len) {
        crm_trace("Only sent %d of %d remaining bytes", rc, len);
        len -= rc;
        unsent += rc;
        goto retry;

     } else {
        crm_trace("Sent %d bytes: %.100s", rc, buf);

    return rc < 0 ? rc : total_send;
/*!
 * \internal
 * \brief Read bytes off non blocking socket.
 *
 * \param session - tls session to read
 * \param max_size - max bytes allowed to read for buffer. 0 assumes no limit
 *
 * \note only use with NON-Blocking sockets. Should only be used after polling socket.
 *       This function will return once max_size is met, the socket read buffer
 *       is empty, or an error is encountered.
 *
 * \retval '\0' terminated buffer on success
 */
static char *
crm_recv_plaintext(int sock, size_t max_size, size_t *recv_len, int *disconnected)
    ssize_t chunk_size = max_size ? max_size : 1024;
    size_t buf_size = 0;
    size_t read_size = 0;
    if (sock <= 0) {
        if (disconnected) {
            *disconnected = 1;
        }
        goto done;
    }
    buf = calloc(1, chunk_size + 1);
    buf_size = chunk_size;
    while (TRUE) {
        errno = 0;
        read_size = buf_size - len;
        /* automatically grow the buffer when needed if max_size is not set.*/
        if (!max_size && (read_size < (chunk_size / 2))) {
            buf_size += chunk_size;
            crm_trace("Grow buffer by %d more bytes. buf is now %d bytes", (int)chunk_size, buf_size);
            buf = realloc(buf, buf_size + 1);
            read_size = buf_size - len;
        }

        rc = read(sock, buf + len, chunk_size);

        if (rc > 0) {
            crm_trace("Got %d more bytes. errno=%d", (int)rc, errno);
            /* always null terminate buffer, the +1 to alloc always allows for this.*/
            buf[len] = '\0';
        }
        if (max_size && (max_size == read_size)) {
            crm_trace("Buffer max read size %d met" , max_size);
            goto done;
        }
        if (rc > 0) {
            continue;
        } else if (rc == 0) {
            if (disconnected) {
                *disconnected = 1;
            }
            crm_trace("EOF encoutered during read");
            goto done;
        }

        /* process errors */
        if (errno == EINTR) {
            crm_trace("EINTER encoutered, retry socket read.");
        } else if (errno == EAGAIN) {
            crm_trace("non-blocking, exiting read on rc = %d", rc);
            goto done;
        } else if (errno <= 0) {
            if (disconnected) {
                *disconnected = 1;
            }
            crm_debug("Error receiving message: %d", (int)rc);
            goto done;
done:
    if (recv_len) {
        *recv_len = len;
    }
    if (!len) {
        free(buf);
        buf = NULL;
    }
    return buf;
static int
crm_send_remote_msg_raw(void *session, const char *buf, size_t len, gboolean encrypted)
    int rc = -1;
        rc = crm_send_tls(session, buf, len);
        rc = crm_send_plaintext(GPOINTER_TO_INT(session), buf, len);
    return rc;
int
crm_send_remote_msg(void *session, xmlNode * msg, gboolean encrypted)
{
    int rc = -1;
    char *xml_text = NULL;
    int len = 0;

    xml_text = dump_xml_unformatted(msg);
    if (xml_text) {
        len = strlen(xml_text);
    } else {
        crm_err("Invalid XML, can not send msg");
        return -1;
    }

    rc = crm_send_remote_msg_raw(session, xml_text, len, encrypted);
    if (rc < 0) {
        goto done;
    }
    rc = crm_send_remote_msg_raw(session, REMOTE_MSG_TERMINATOR, strlen(REMOTE_MSG_TERMINATOR), encrypted);

done:
    if (rc < 0) {
        crm_err("Failed to send remote msg, rc = %d", rc);
    }

    free(xml_text);
    return rc;
}

/*!
 * \internal
 * \brief handles the recv buffer and parsing out msgs.
 * \note new_data is owned by this function once it is passed in.
 */
crm_parse_remote_buffer(char **msg_buf)
    char *buf = NULL;
    char *start = NULL;
    char *end = NULL;
    if (*msg_buf == NULL) {
        return NULL;
    }

    /* take ownership of the buffer */
    buf = *msg_buf;
    *msg_buf = NULL;

    /* MSGS are separated by a '\r\n\r\n'. Split a message off the buffer and return it. */
    start = buf;
    end = strstr(start, REMOTE_MSG_TERMINATOR);

    while (!xml && end) {

        /* grab the message */
        end[0] = '\0';
        end += strlen(REMOTE_MSG_TERMINATOR);

        xml = string2xml(start);
        if (xml == NULL) {
            crm_err("Couldn't parse: '%.120s'", start);
        }
        start = end;
        end = strstr(start, REMOTE_MSG_TERMINATOR);
    }

    if (xml && start) {
        /* we have msgs left over, save it until next time */
        *msg_buf = strdup(start);
        free(buf);
    } else if (!xml) {
        /* no msg present */
        *msg_buf = buf;
    }

    return xml;
}

/*!
 * \internal
 * \brief Determine if a remote session has data to read
 *
 * \retval 0, timeout occured.
 * \retval positive, data is ready to be read
 * \retval negative, session has ended
 */
int
crm_recv_remote_ready(void *session, gboolean encrypted, int timeout /* ms */)
{
    struct pollfd fds = { 0, };
    int sock = 0;
    void *sock_ptr = NULL;
    int rc = 0;
    time_t start;

    if (encrypted) {
#ifdef HAVE_GNUTLS_GNUTLS_H
        gnutls_session *tls_session = session;
        sock_ptr = gnutls_transport_get_ptr(*tls_session);
#else
        CRM_ASSERT(encrypted == FALSE);
#endif
    } else {
        sock_ptr = session;
    }

    sock = GPOINTER_TO_INT(sock_ptr);
    if (sock <= 0) {
        return -ENOTCONN;
    }

    start = time(NULL);
    errno = 0;
    do {
        fds.fd = sock;
        fds.events = POLLIN;

        /* If we got an EINTR while polling, and we have a
         * specific timeout we are trying to honor, attempt
         * to adjust the timeout to the closest second. */
        if (errno == EINTR && (timeout > 0)) {
            timeout = timeout - ((time(NULL) - start) * 1000);
            if (timeout < 1000) {
                timeout = 1000;
            }
        }

        rc = poll(&fds, 1, timeout);
    } while (rc < 0 && errno == EINTR);

    return rc;
}

char *
crm_recv_remote_raw(void *session, gboolean encrypted, size_t max_recv, size_t *recv_len, int *disconnected)
{
    char *reply = NULL;
    if (recv_len) {
        *recv_len = 0;
    }

    if (disconnected) {
        *disconnected = 0;
    }

        reply = crm_recv_tls(session, max_recv, recv_len, disconnected);
        reply = crm_recv_plaintext(GPOINTER_TO_INT(session), max_recv, recv_len, disconnected);
    }
    return reply;
}

/*!
 * \internal
 * \brief Read data off the socket until at least one full message is present or timeout occures.
 * \retval TRUE message read
 * \retval FALSE full message not read
 */

gboolean
crm_recv_remote_msg(void *session, char **recv_buf, gboolean encrypted, int total_timeout /*ms */, int *disconnected)
{
    int ret;
    size_t request_len = 0;
    time_t start = time(NULL);
    char *raw_request = NULL;
    int remaining_timeout = 0;

    if (total_timeout == 0) {
        total_timeout = 10000;
    } else if (total_timeout < 0) {
        total_timeout = 60000;
    }
    *disconnected = 0;

    remaining_timeout = total_timeout;
    while ((remaining_timeout > 0) && !(*disconnected)) {

        /* read some more off the tls buffer if we still have time left. */
        crm_trace("waiting to receive remote msg, starting timeout %d, remaining_timeout %d", total_timeout, remaining_timeout);
        ret = crm_recv_remote_ready(session, encrypted, remaining_timeout);
        raw_request = NULL;

        if (ret == 0) {
            crm_err("poll timed out (%d ms) while waiting to receive msg", remaining_timeout);
            return FALSE;

        } else if (ret < 0) {
            if (errno != EINTR) {
                crm_debug("poll returned error while waiting for msg, rc: %d, errno: %d", ret, errno);
                *disconnected = 1;
                return FALSE;
            }
            crm_debug("poll EINTR encountered during poll, retrying");
        } else {
            raw_request = crm_recv_remote_raw(session, encrypted, 0, &request_len, disconnected);
        }

        remaining_timeout = remaining_timeout - ((time(NULL) - start) * 1000);

        if (!raw_request) {
            crm_debug("Empty msg received after poll");
            continue;
        }

        if (*recv_buf) {
            int old_len = strlen(*recv_buf);

            crm_trace("Expanding recv buffer from %d to %d", old_len, old_len+request_len);

            *recv_buf = realloc(*recv_buf, old_len + request_len + 1);
            memcpy(*recv_buf + old_len, raw_request, request_len);
            *(*recv_buf+old_len+request_len) = '\0';
            free(raw_request);
        } else {
            *recv_buf = raw_request;
        }

        if (strstr(*recv_buf, REMOTE_MSG_TERMINATOR)) {
            return TRUE;
    return FALSE;

/*!
 * \internal
 * \brief tcp connection to server at specified port
 * \retval positive, socket fd.
 * \retval negative, failed to connect.
 */
int
crm_remote_tcp_connect(const char *host, int port)
{
    struct addrinfo *res;
    struct addrinfo *rp;
    struct addrinfo hints;
    const char *server = host;
    int ret_ga;
    int sock;

    /* getaddrinfo */
    memset(&hints, 0, sizeof(struct addrinfo));
    hints.ai_family = AF_UNSPEC;    /* Allow IPv4 or IPv6 */
    hints.ai_socktype = SOCK_STREAM;
    hints.ai_flags = AI_CANONNAME;

    crm_debug("Looking up %s", server);
    ret_ga = getaddrinfo(server, NULL, &hints, &res);
    if (ret_ga) {
        crm_err("getaddrinfo: %s", gai_strerror(ret_ga));
        return -1;
    }

    if (!res || !res->ai_addr) {
        crm_err("getaddrinfo failed");
        return -1;
    }

    for (rp = res; rp != NULL; rp = rp->ai_next) {
        struct sockaddr *addr = rp->ai_addr;
        int flag = 0;
        if (!addr) {
            continue;
        }

        if (rp->ai_canonname) {
            server = res->ai_canonname;
        }
        crm_debug("Got address %s for %s", server, host);

        /* create socket */
        sock = socket(rp->ai_family, SOCK_STREAM, IPPROTO_TCP);
        if (sock == -1) {
            crm_err("Socket creation failed for remote client connection.");
            continue;
        }
        if (addr->sa_family == AF_INET6) {
            struct sockaddr_in6 *addr_in = (struct sockaddr_in6 *) addr;
            addr_in->sin6_port = htons(port);
        } else {
            struct sockaddr_in *addr_in = (struct sockaddr_in *) addr;
            addr_in->sin_port = htons(port);
            crm_info("Attempting to connect to remote server at %s:%d", inet_ntoa(addr_in->sin_addr), port);
        }

        if (connect(sock, rp->ai_addr, rp->ai_addrlen) == 0) {
            if ((flag = fcntl(sock, F_GETFL)) >= 0) {
                if (fcntl(sock, F_SETFL, flag | O_NONBLOCK) < 0) {
                    crm_err( "fcntl() write failed");
                    close(sock);
                    sock = -1;
                    continue;
                }
            }
            break;                  /* Success */
        }

        close(sock);
        sock = -1;
    }
    freeaddrinfo(res);

    return sock;
}

                    print_as("Notification setup not supported, won't be able to reconnect after failure");