Commit message: LibVNCClient: remove now-useless cast

re #273
CVE: CVE-2018-20748
CWE: CWE-787
Vulnerability description: LibVNC before 0.9.12 contains multiple heap out-of-bounds write vulnerabilities in libvncclient/rfbproto.c. The fix for CVE-2018-20019 was incomplete.

Project: libvncserver
File: rfbproto.c

ReadSupportedSecurityType(rfbClient* client, uint32_t *result, rfbBool subAuth)
{
    uint8_t count=0;
    uint8_t loop=0;
    uint8_t flag=0;
    rfbBool extAuthHandler;
    uint8_t tAuth[256];
    char buf1[500],buf2[10];
    uint32_t authScheme;
    rfbClientProtocolExtension* e;

    if (!ReadFromRFBServer(client, (char *)&count, 1)) return FALSE;

    if (count==0)
    {
        rfbClientLog("List of security types is ZERO, expecting an error to follow\n");
        ReadReason(client);
        return FALSE;
    }

    rfbClientLog("We have %d security types to read\n", count);
    authScheme=0;
    /* now, we have a list of available security types to read ( uint8_t[] ) */
    for (loop=0;loop<count;loop++)
    {
        if (!ReadFromRFBServer(client, (char *)&tAuth[loop], 1)) return FALSE;
        rfbClientLog("%d) Received security type %d\n", loop, tAuth[loop]);
        if (flag) continue;
        extAuthHandler=FALSE;
        for (e = rfbClientExtensions; e; e = e->next) {
            if (!e->handleAuthentication) continue;
            uint32_t const* secType;
            for (secType = e->securityTypes; secType && *secType; secType++) {
                if (tAuth[loop]==*secType) {
                    extAuthHandler=TRUE;
                }
            }
        }
        if (tAuth[loop]==rfbVncAuth || tAuth[loop]==rfbNoAuth ||
			extAuthHandler ||
#if defined(LIBVNCSERVER_HAVE_GNUTLS) || defined(LIBVNCSERVER_HAVE_LIBSSL)
            tAuth[loop]==rfbVeNCrypt ||
#endif
#ifdef LIBVNCSERVER_HAVE_SASL
            tAuth[loop]==rfbSASL ||
#endif /* LIBVNCSERVER_HAVE_SASL */
            (tAuth[loop]==rfbARD && client->GetCredential) ||
            (!subAuth && (tAuth[loop]==rfbTLS || (tAuth[loop]==rfbVeNCrypt && client->GetCredential))))
        {
            if (!subAuth && client->clientAuthSchemes)
            {
                int i;
                for (i=0;client->clientAuthSchemes[i];i++)
                {
                    if (client->clientAuthSchemes[i]==(uint32_t)tAuth[loop])
                    {
                        flag++;
                        authScheme=tAuth[loop];
                        break;
                    }
                }
            }
            else
            {
                flag++;
                authScheme=tAuth[loop];
            }
            if (flag)
            {
                rfbClientLog("Selecting security type %d (%d/%d in the list)\n", authScheme, loop, count);
                /* send back a single byte indicating which security type to use */
                if (!WriteToRFBServer(client, (char *)&tAuth[loop], 1)) return FALSE;
            }
        }
    }
    if (authScheme==0)
    {
        memset(buf1, 0, sizeof(buf1));
        for (loop=0;loop<count;loop++)
        {
            if (strlen(buf1)>=sizeof(buf1)-1) break;
            snprintf(buf2, sizeof(buf2), (loop>0 ? ", %d" : "%d"), (int)tAuth[loop]);
            strncat(buf1, buf2, sizeof(buf1)-strlen(buf1)-1);
        }
        rfbClientLog("Unknown authentication scheme from VNC server: %s\n",
               buf1);
        return FALSE;
    }
    *result = authScheme;
    return TRUE;
}

Project: libvncserver
File: rfbproto.c

DefaultSupportedMessages(rfbClient* client)
{
    memset((char *)&client->supportedMessages,0,sizeof(client->supportedMessages));

    /* Default client supported messages (universal RFB 3.3 protocol) */
    SetClient2Server(client, rfbSetPixelFormat);
    /* SetClient2Server(client, rfbFixColourMapEntries); Not currently supported */
    SetClient2Server(client, rfbSetEncodings);
    SetClient2Server(client, rfbFramebufferUpdateRequest);
    SetClient2Server(client, rfbKeyEvent);
    SetClient2Server(client, rfbPointerEvent);
    SetClient2Server(client, rfbClientCutText);
    /* technically, we only care what we can *send* to the server
     * but, we set Server2Client Just in case it ever becomes useful
     */
    SetServer2Client(client, rfbFramebufferUpdate);
    SetServer2Client(client, rfbSetColourMapEntries);
    SetServer2Client(client, rfbBell);
    SetServer2Client(client, rfbServerCutText);
}

Project: libvncserver
File: rfbproto.c

void* rfbClientGetClientData(rfbClient* client, void* tag)
{
	rfbClientData* clientData = client->clientData;

	while(clientData) {
		if(clientData->tag == tag)
			return clientData->data;
		clientData = clientData->next;
	}

	return NULL;
}

Project: libvncserver
File: rfbproto.c

InitialiseRFBConnection(rfbClient* client)
{
  rfbProtocolVersionMsg pv;
  int major,minor;
  uint32_t authScheme;
  uint32_t subAuthScheme;
  rfbClientInitMsg ci;

  /* if the connection is immediately closed, don't report anything, so
       that pmw's monitor can make test connections */

  if (client->listenSpecified)
    errorMessageOnReadFailure = FALSE;

  if (!ReadFromRFBServer(client, pv, sz_rfbProtocolVersionMsg)) return FALSE;
  pv[sz_rfbProtocolVersionMsg]=0;

  errorMessageOnReadFailure = TRUE;

  pv[sz_rfbProtocolVersionMsg] = 0;

  if (sscanf(pv,rfbProtocolVersionFormat,&major,&minor) != 2) {
    rfbClientLog("Not a valid VNC server (%s)\n",pv);
    return FALSE;
  }


  DefaultSupportedMessages(client);
  client->major = major;
  client->minor = minor;

  /* fall back to viewer supported version */
  if ((major==rfbProtocolMajorVersion) && (minor>rfbProtocolMinorVersion))
    client->minor = rfbProtocolMinorVersion;

  /* UltraVNC uses minor codes 4 and 6 for the server */
  if (major==3 && (minor==4 || minor==6)) {
      rfbClientLog("UltraVNC server detected, enabling UltraVNC specific messages\n",pv);
      DefaultSupportedMessagesUltraVNC(client);
  }

  /* UltraVNC Single Click uses minor codes 14 and 16 for the server */
  if (major==3 && (minor==14 || minor==16)) {
     minor = minor - 10;
     client->minor = minor;
     rfbClientLog("UltraVNC Single Click server detected, enabling UltraVNC specific messages\n",pv);
     DefaultSupportedMessagesUltraVNC(client);
  }

  /* TightVNC uses minor codes 5 for the server */
  if (major==3 && minor==5) {
      rfbClientLog("TightVNC server detected, enabling TightVNC specific messages\n",pv);
      DefaultSupportedMessagesTightVNC(client);
  }

  /* we do not support > RFB3.8 */
  if ((major==3 && minor>8) || major>3)
  {
    client->major=3;
    client->minor=8;
  }

  rfbClientLog("VNC server supports protocol version %d.%d (viewer %d.%d)\n",
	  major, minor, rfbProtocolMajorVersion, rfbProtocolMinorVersion);

  sprintf(pv,rfbProtocolVersionFormat,client->major,client->minor);

  if (!WriteToRFBServer(client, pv, sz_rfbProtocolVersionMsg)) return FALSE;


  /* 3.7 and onwards sends a # of security types first */
  if (client->major==3 && client->minor > 6)
  {
    if (!ReadSupportedSecurityType(client, &authScheme, FALSE)) return FALSE;
  }
  else
  {
    if (!ReadFromRFBServer(client, (char *)&authScheme, 4)) return FALSE;
    authScheme = rfbClientSwap32IfLE(authScheme);
  }
  
  rfbClientLog("Selected Security Scheme %d\n", authScheme);
  client->authScheme = authScheme;
  
  switch (authScheme) {

  case rfbConnFailed:
    ReadReason(client);
    return FALSE;

  case rfbNoAuth:
    rfbClientLog("No authentication needed\n");

    /* 3.8 and upwards sends a Security Result for rfbNoAuth */
    if ((client->major==3 && client->minor > 7) || client->major>3)
        if (!rfbHandleAuthResult(client)) return FALSE;        

    break;

  case rfbVncAuth:
    if (!HandleVncAuth(client)) return FALSE;
    break;

#ifdef LIBVNCSERVER_HAVE_SASL
  case rfbSASL:
    if (!HandleSASLAuth(client)) return FALSE;
    break;
#endif /* LIBVNCSERVER_HAVE_SASL */

  case rfbMSLogon:
    if (!HandleMSLogonAuth(client)) return FALSE;
    break;

  case rfbARD:
#ifndef LIBVNCSERVER_WITH_CLIENT_GCRYPT
    rfbClientLog("GCrypt support was not compiled in\n");
    return FALSE;
#else
    if (!HandleARDAuth(client)) return FALSE;
#endif
    break;

  case rfbTLS:
    if (!HandleAnonTLSAuth(client)) return FALSE;
    /* After the TLS session is established, sub auth types are expected.
     * Note that all following reading/writing are through the TLS session from here.
     */
    if (!ReadSupportedSecurityType(client, &subAuthScheme, TRUE)) return FALSE;
    client->subAuthScheme = subAuthScheme;

    switch (subAuthScheme) {

      case rfbConnFailed:
        ReadReason(client);
        return FALSE;

      case rfbNoAuth:
        rfbClientLog("No sub authentication needed\n");
        /* 3.8 and upwards sends a Security Result for rfbNoAuth */
        if ((client->major==3 && client->minor > 7) || client->major>3)
            if (!rfbHandleAuthResult(client)) return FALSE;
        break;

      case rfbVncAuth:
        if (!HandleVncAuth(client)) return FALSE;
        break;

#ifdef LIBVNCSERVER_HAVE_SASL
      case rfbSASL:
        if (!HandleSASLAuth(client)) return FALSE;
        break;
#endif /* LIBVNCSERVER_HAVE_SASL */

      default:
        rfbClientLog("Unknown sub authentication scheme from VNC server: %d\n",
            (int)subAuthScheme);
        return FALSE;
    }

    break;

  case rfbVeNCrypt:
    if (!HandleVeNCryptAuth(client)) return FALSE;

    switch (client->subAuthScheme) {

      case rfbVeNCryptTLSNone:
      case rfbVeNCryptX509None:
        rfbClientLog("No sub authentication needed\n");
        if (!rfbHandleAuthResult(client)) return FALSE;
        break;

      case rfbVeNCryptTLSVNC:
      case rfbVeNCryptX509VNC:
        if (!HandleVncAuth(client)) return FALSE;
        break;

      case rfbVeNCryptTLSPlain:
      case rfbVeNCryptX509Plain:
        if (!HandlePlainAuth(client)) return FALSE;
        break;

#ifdef LIBVNCSERVER_HAVE_SASL
      case rfbVeNCryptX509SASL:
      case rfbVeNCryptTLSSASL:
        if (!HandleSASLAuth(client)) return FALSE;
        break;
#endif /* LIBVNCSERVER_HAVE_SASL */

      default:
        rfbClientLog("Unknown sub authentication scheme from VNC server: %d\n",
            client->subAuthScheme);
        return FALSE;
    }

    break;

  default:
    {
      rfbBool authHandled=FALSE;
      rfbClientProtocolExtension* e;
      for (e = rfbClientExtensions; e; e = e->next) {
        uint32_t const* secType;
        if (!e->handleAuthentication) continue;
        for (secType = e->securityTypes; secType && *secType; secType++) {
          if (authScheme==*secType) {
            if (!e->handleAuthentication(client, authScheme)) return FALSE;
            if (!rfbHandleAuthResult(client)) return FALSE;
            authHandled=TRUE;
          }
        }
      }
      if (authHandled) break;
    }
    rfbClientLog("Unknown authentication scheme from VNC server: %d\n",
	    (int)authScheme);
    return FALSE;
  }

  ci.shared = (client->appData.shareDesktop ? 1 : 0);

  if (!WriteToRFBServer(client,  (char *)&ci, sz_rfbClientInitMsg)) return FALSE;

  if (!ReadFromRFBServer(client, (char *)&client->si, sz_rfbServerInitMsg)) return FALSE;

  client->si.framebufferWidth = rfbClientSwap16IfLE(client->si.framebufferWidth);
  client->si.framebufferHeight = rfbClientSwap16IfLE(client->si.framebufferHeight);
  client->si.format.redMax = rfbClientSwap16IfLE(client->si.format.redMax);
  client->si.format.greenMax = rfbClientSwap16IfLE(client->si.format.greenMax);
  client->si.format.blueMax = rfbClientSwap16IfLE(client->si.format.blueMax);
  client->si.nameLength = rfbClientSwap32IfLE(client->si.nameLength);

  if (client->si.nameLength > 1<<20) {
      rfbClientErr("Too big desktop name length sent by server: %u B > 1 MB\n", (unsigned int)client->si.nameLength);
      return FALSE;
  }

  client->desktopName = malloc(client->si.nameLength + 1);
  if (!client->desktopName) {
    rfbClientLog("Error allocating memory for desktop name, %lu bytes\n",
            (unsigned long)client->si.nameLength);
    return FALSE;
  }

  if (!ReadFromRFBServer(client, client->desktopName, client->si.nameLength)) return FALSE;

  client->desktopName[client->si.nameLength] = 0;

  rfbClientLog("Desktop name \"%s\"\n",client->desktopName);

  rfbClientLog("Connected to VNC server, using protocol version %d.%d\n",
	  client->major, client->minor);

  rfbClientLog("VNC server default format:\n");
  PrintPixelFormat(&client->si.format);

  return TRUE;
}

Project: libvncserver
File: rfbproto.c

void rfbClientRegisterExtension(rfbClientProtocolExtension* e)
{
	e->next = rfbClientExtensions;
	rfbClientExtensions = e;
}

Project: libvncserver
File: rfbproto.c

SendPointerEvent(rfbClient* client,int x, int y, int buttonMask)
{
  rfbPointerEventMsg pe;

  if (!SupportsClient2Server(client, rfbPointerEvent)) return TRUE;

  pe.type = rfbPointerEvent;
  pe.buttonMask = buttonMask;
  if (x < 0) x = 0;
  if (y < 0) y = 0;

  pe.x = rfbClientSwap16IfLE(x);
  pe.y = rfbClientSwap16IfLE(y);
  return WriteToRFBServer(client, (char *)&pe, sz_rfbPointerEventMsg);
}

Project: libvncserver
File: rfbproto.c

DefaultSupportedMessagesTightVNC(rfbClient* client)
{
    DefaultSupportedMessages(client);
    SetClient2Server(client, rfbFileTransfer);
    SetClient2Server(client, rfbSetServerInput);
    SetClient2Server(client, rfbSetSW);
    /* SetClient2Server(client, rfbTextChat); */
    /* technically, we only care what we can *send* to the server */
    SetServer2Client(client, rfbFileTransfer);
    SetServer2Client(client, rfbTextChat);
}

Project: libvncserver
File: rfbproto.c

IsUnixSocket(const char *name)
{
  struct stat sb;
  if(stat(name, &sb) == 0 && (sb.st_mode & S_IFMT) == S_IFSOCK)
    return TRUE;
  return FALSE;
}

Project: libvncserver
File: rfbproto.c

HandleMSLogonAuth(rfbClient *client)
{
  uint64_t gen, mod, resp, priv, pub, key;
  uint8_t username[256], password[64];
  rfbCredential *cred;

  if (!ReadFromRFBServer(client, (char *)&gen, 8)) return FALSE;
  if (!ReadFromRFBServer(client, (char *)&mod, 8)) return FALSE;
  if (!ReadFromRFBServer(client, (char *)&resp, 8)) return FALSE;
  gen = rfbClientSwap64IfLE(gen);
  mod = rfbClientSwap64IfLE(mod);
  resp = rfbClientSwap64IfLE(resp);

  if (!client->GetCredential)
  {
    rfbClientLog("GetCredential callback is not set.\n");
    return FALSE;
  }
  rfbClientLog("WARNING! MSLogon security type has very low password encryption! "\
    "Use it only with SSH tunnel or trusted network.\n");
  cred = client->GetCredential(client, rfbCredentialTypeUser);
  if (!cred)
  {
    rfbClientLog("Reading credential failed\n");
    return FALSE;
  }

  memset(username, 0, sizeof(username));
  strncpy((char *)username, cred->userCredential.username, sizeof(username));
  memset(password, 0, sizeof(password));
  strncpy((char *)password, cred->userCredential.password, sizeof(password));
  FreeUserCredential(cred);

  srand(time(NULL));
  priv = ((uint64_t)rand())<<32;
  priv |= (uint64_t)rand();

  pub = rfbPowM64(gen, priv, mod);
  key = rfbPowM64(resp, priv, mod);
  pub = rfbClientSwap64IfLE(pub);
  key = rfbClientSwap64IfLE(key);

  rfbClientEncryptBytes2(username, sizeof(username), (unsigned char *)&key);
  rfbClientEncryptBytes2(password, sizeof(password), (unsigned char *)&key);

  if (!WriteToRFBServer(client, (char *)&pub, 8)) return FALSE;
  if (!WriteToRFBServer(client, (char *)username, sizeof(username))) return FALSE;
  if (!WriteToRFBServer(client, (char *)password, sizeof(password))) return FALSE;

  /* Handle the SecurityResult message */
  if (!rfbHandleAuthResult(client)) return FALSE;

  return TRUE;
}

Project: libvncserver
File: rfbproto.c

SupportsServer2Client(rfbClient* client, int messageType)
{
    return (client->supportedMessages.server2client[((messageType & 0xFF)/8)] & (1<<(messageType % 8)) ? TRUE : FALSE);
}

Project: libvncserver
File: rfbproto.c

SetServer2Client(rfbClient* client, int messageType)
{
  client->supportedMessages.server2client[((messageType & 0xFF)/8)] |= (1<<(messageType % 8));
}

Project: libvncserver
File: rfbproto.c

SendScaleSetting(rfbClient* client,int scaleSetting)
{
  rfbSetScaleMsg ssm;

  ssm.scale = scaleSetting;
  ssm.pad = 0;
  
  /* favor UltraVNC SetScale if both are supported */
  if (SupportsClient2Server(client, rfbSetScale)) {
      ssm.type = rfbSetScale;
      if (!WriteToRFBServer(client, (char *)&ssm, sz_rfbSetScaleMsg))
          return FALSE;
  }
  
  if (SupportsClient2Server(client, rfbPalmVNCSetScaleFactor)) {
      ssm.type = rfbPalmVNCSetScaleFactor;
      if (!WriteToRFBServer(client, (char *)&ssm, sz_rfbSetScaleMsg))
          return FALSE;
  }

  return TRUE;
}

Project: libvncserver
File: rfbproto.c

rfbBool TextChatFinish(rfbClient* client)
{
    rfbTextChatMsg chat;
    if (!SupportsClient2Server(client, rfbTextChat)) return TRUE;
    chat.type = rfbTextChat;
    chat.pad1 = 0;
    chat.pad2 = 0;
    chat.length = rfbClientSwap32IfLE(rfbTextChatFinished);
    return  (WriteToRFBServer(client, (char *)&chat, sz_rfbTextChatMsg) ? TRUE : FALSE);
}

Project: libvncserver
File: rfbproto.c

SetClientAuthSchemes(rfbClient* client,const uint32_t *authSchemes, int size)
{
  int i;

  if (client->clientAuthSchemes)
  {
    free(client->clientAuthSchemes);
    client->clientAuthSchemes = NULL;
  }
  if (authSchemes)
  {
    if (size<0)
    {
      /* If size<0 we assume the passed-in list is also 0-terminate, so we
       * calculate the size here */
      for (size=0;authSchemes[size];size++) ;
    }
    client->clientAuthSchemes = (uint32_t*)malloc(sizeof(uint32_t)*(size+1));
    for (i=0;i<size;i++)
      client->clientAuthSchemes[i] = authSchemes[i];
    client->clientAuthSchemes[size] = 0;
  }
}

Project: libvncserver
File: rfbproto.c

HandleVncAuth(rfbClient *client)
{
    uint8_t challenge[CHALLENGESIZE];
    char *passwd=NULL;
    int i;

    if (!ReadFromRFBServer(client, (char *)challenge, CHALLENGESIZE)) return FALSE;

    if (client->serverPort!=-1) { /* if not playing a vncrec file */
      if (client->GetPassword)
        passwd = client->GetPassword(client);

      if ((!passwd) || (strlen(passwd) == 0)) {
        rfbClientLog("Reading password failed\n");
        return FALSE;
      }
      if (strlen(passwd) > 8) {
        passwd[8] = '\0';
      }

      rfbClientEncryptBytes(challenge, passwd);

      /* Lose the password from memory */
      for (i = strlen(passwd); i >= 0; i--) {
        passwd[i] = '\0';
      }
      free(passwd);

      if (!WriteToRFBServer(client, (char *)challenge, CHALLENGESIZE)) return FALSE;
    }

    /* Handle the SecurityResult message */
    if (!rfbHandleAuthResult(client)) return FALSE;

    return TRUE;
}

Project: libvncserver
File: rfbproto.c

void rfbClientSetClientData(rfbClient* client, void* tag, void* data)
{
	rfbClientData* clientData = client->clientData;

	while(clientData && clientData->tag != tag)
		clientData = clientData->next;
	if(clientData == NULL) {
		clientData = calloc(sizeof(rfbClientData), 1);
		clientData->next = client->clientData;
		client->clientData = clientData;
		clientData->tag = tag;
	}

	clientData->data = data;
}

Project: libvncserver
File: rfbproto.c

HandleARDAuth(rfbClient *client)
{
  uint8_t gen[2], len[2];
  size_t keylen;
  uint8_t *mod = NULL, *resp, *pub, *key, *shared;
  gcry_mpi_t genmpi = NULL, modmpi = NULL, respmpi = NULL;
  gcry_mpi_t privmpi = NULL, pubmpi = NULL, keympi = NULL;
  gcry_md_hd_t md5 = NULL;
  gcry_cipher_hd_t aes = NULL;
  gcry_error_t error;
  uint8_t userpass[128], ciphertext[128];
  int passwordLen, usernameLen;
  rfbCredential *cred = NULL;
  rfbBool result = FALSE;

  if (!gcry_control(GCRYCTL_INITIALIZATION_FINISHED_P))
  {
    /* Application did not initialize gcrypt, so we should */
    if (!gcry_check_version(GCRYPT_VERSION))
    {
      /* Older version of libgcrypt is installed on system than compiled against */
      rfbClientLog("libgcrypt version mismatch.\n");
    }
  }

  while (1)
  {
    if (!ReadFromRFBServer(client, (char *)gen, 2))
      break;
    if (!ReadFromRFBServer(client, (char *)len, 2))
      break;

    if (!client->GetCredential)
    {
      rfbClientLog("GetCredential callback is not set.\n");
      break;
    }
    cred = client->GetCredential(client, rfbCredentialTypeUser);
    if (!cred)
    {
      rfbClientLog("Reading credential failed\n");
      break;
    }

    keylen = 256*len[0]+len[1];
    mod = (uint8_t*)malloc(keylen*4);
    if (!mod)
    {
      rfbClientLog("malloc out of memory\n");
      break;
    }
    resp = mod+keylen;
    pub = resp+keylen;
    key = pub+keylen;

    if (!ReadFromRFBServer(client, (char *)mod, keylen))
      break;
    if (!ReadFromRFBServer(client, (char *)resp, keylen))
      break;

    error = gcry_mpi_scan(&genmpi, GCRYMPI_FMT_USG, gen, 2, NULL);
    if (gcry_err_code(error) != GPG_ERR_NO_ERROR)
    {
      rfbClientLog("gcry_mpi_scan error: %s\n", gcry_strerror(error));
      break;
    }
    error = gcry_mpi_scan(&modmpi, GCRYMPI_FMT_USG, mod, keylen, NULL);
    if (gcry_err_code(error) != GPG_ERR_NO_ERROR)
    {
      rfbClientLog("gcry_mpi_scan error: %s\n", gcry_strerror(error));
      break;
    }
    error = gcry_mpi_scan(&respmpi, GCRYMPI_FMT_USG, resp, keylen, NULL);
    if (gcry_err_code(error) != GPG_ERR_NO_ERROR)
    {
      rfbClientLog("gcry_mpi_scan error: %s\n", gcry_strerror(error));
      break;
    }

    privmpi = gcry_mpi_new(keylen);
    if (!privmpi)
    {
      rfbClientLog("gcry_mpi_new out of memory\n");
      break;
    }
    gcry_mpi_randomize(privmpi, (keylen/8)*8, GCRY_STRONG_RANDOM);

    pubmpi = gcry_mpi_new(keylen);
    if (!pubmpi)
    {
      rfbClientLog("gcry_mpi_new out of memory\n");
      break;
    }
    gcry_mpi_powm(pubmpi, genmpi, privmpi, modmpi);

    keympi = gcry_mpi_new(keylen);
    if (!keympi)
    {
      rfbClientLog("gcry_mpi_new out of memory\n");
      break;
    }
    gcry_mpi_powm(keympi, respmpi, privmpi, modmpi);

    if (!rfbMpiToBytes(pubmpi, pub, keylen))
      break;
    if (!rfbMpiToBytes(keympi, key, keylen))
      break;

    error = gcry_md_open(&md5, GCRY_MD_MD5, 0);
    if (gcry_err_code(error) != GPG_ERR_NO_ERROR)
    {
      rfbClientLog("gcry_md_open error: %s\n", gcry_strerror(error));
      break;
    }
    gcry_md_write(md5, key, keylen);
    error = gcry_md_final(md5);
    if (gcry_err_code(error) != GPG_ERR_NO_ERROR)
    {
      rfbClientLog("gcry_md_final error: %s\n", gcry_strerror(error));
      break;
    }
    shared = gcry_md_read(md5, GCRY_MD_MD5);

    passwordLen = strlen(cred->userCredential.password)+1;
    usernameLen = strlen(cred->userCredential.username)+1;
    if (passwordLen > sizeof(userpass)/2)
      passwordLen = sizeof(userpass)/2;
    if (usernameLen > sizeof(userpass)/2)
      usernameLen = sizeof(userpass)/2;

    gcry_randomize(userpass, sizeof(userpass), GCRY_STRONG_RANDOM);
    memcpy(userpass, cred->userCredential.username, usernameLen);
    memcpy(userpass+sizeof(userpass)/2, cred->userCredential.password, passwordLen);

    error = gcry_cipher_open(&aes, GCRY_CIPHER_AES128, GCRY_CIPHER_MODE_ECB, 0);
    if (gcry_err_code(error) != GPG_ERR_NO_ERROR)
    {
      rfbClientLog("gcry_cipher_open error: %s\n", gcry_strerror(error));
      break;
    }
    error = gcry_cipher_setkey(aes, shared, 16);
    if (gcry_err_code(error) != GPG_ERR_NO_ERROR)
    {
      rfbClientLog("gcry_cipher_setkey error: %s\n", gcry_strerror(error));
      break;
    }
    error = gcry_cipher_encrypt(aes, ciphertext, sizeof(ciphertext), userpass, sizeof(userpass));
    if (gcry_err_code(error) != GPG_ERR_NO_ERROR)
    {
      rfbClientLog("gcry_cipher_encrypt error: %s\n", gcry_strerror(error));
      break;
    }

    if (!WriteToRFBServer(client, (char *)ciphertext, sizeof(ciphertext)))
      break;
    if (!WriteToRFBServer(client, (char *)pub, keylen))
      break;

    /* Handle the SecurityResult message */
    if (!rfbHandleAuthResult(client))
      break;

    result = TRUE;
    break;
  }

  if (cred)
    FreeUserCredential(cred);
  if (mod)
    free(mod);
  if (genmpi)
    gcry_mpi_release(genmpi);
  if (modmpi)
    gcry_mpi_release(modmpi);
  if (respmpi)
    gcry_mpi_release(respmpi);
  if (privmpi)
    gcry_mpi_release(privmpi);
  if (pubmpi)
    gcry_mpi_release(pubmpi);
  if (keympi)
    gcry_mpi_release(keympi);
  if (md5)
    gcry_md_close(md5);
  if (aes)
    gcry_cipher_close(aes);
  return result;
}

Project: libvncserver
File: rfbproto.c

SendFramebufferUpdateRequest(rfbClient* client, int x, int y, int w, int h, rfbBool incremental)
{
  rfbFramebufferUpdateRequestMsg fur;

  if (!SupportsClient2Server(client, rfbFramebufferUpdateRequest)) return TRUE;
  
  fur.type = rfbFramebufferUpdateRequest;
  fur.incremental = incremental ? 1 : 0;
  fur.x = rfbClientSwap16IfLE(x);
  fur.y = rfbClientSwap16IfLE(y);
  fur.w = rfbClientSwap16IfLE(w);
  fur.h = rfbClientSwap16IfLE(h);

  if (!WriteToRFBServer(client, (char *)&fur, sz_rfbFramebufferUpdateRequestMsg))
    return FALSE;

  return TRUE;
}

Project: libvncserver
File: rfbproto.c

SendClientCutText(rfbClient* client, char *str, int len)
{
  rfbClientCutTextMsg cct;

  if (!SupportsClient2Server(client, rfbClientCutText)) return TRUE;

  memset(&cct, 0, sizeof(cct));
  cct.type = rfbClientCutText;
  cct.length = rfbClientSwap32IfLE(len);
  return  (WriteToRFBServer(client, (char *)&cct, sz_rfbClientCutTextMsg) &&
	   WriteToRFBServer(client, str, len));
}

Project: libvncserver
File: rfbproto.c

ReadReason(rfbClient* client)
{
    uint32_t reasonLen;
    char *reason;

    if (!ReadFromRFBServer(client, (char *)&reasonLen, 4)) return;
    reasonLen = rfbClientSwap32IfLE(reasonLen);
    if(reasonLen > 1<<20) {
      rfbClientLog("VNC connection failed, but sent reason length of %u exceeds limit of 1MB",(unsigned int)reasonLen);
      return;
    }
    reason = malloc(reasonLen+1);
    if (!ReadFromRFBServer(client, reason, reasonLen)) { free(reason); return; }
    reason[reasonLen]=0;
    rfbClientLog("VNC connection failed: %s\n",reason);
    free(reason);
}

Project: libvncserver
File: rfbproto.c

SupportsClient2Server(rfbClient* client, int messageType)
{
    return (client->supportedMessages.client2server[((messageType & 0xFF)/8)] & (1<<(messageType % 8)) ? TRUE : FALSE);
}

Project: libvncserver
File: rfbproto.c

rfbBool PermitServerInput(rfbClient* client, int enabled)
{
    rfbSetServerInputMsg msg;

    if (!SupportsClient2Server(client, rfbSetServerInput)) return TRUE;
    /* enabled==1, then server input from local keyboard is disabled */
    msg.type = rfbSetServerInput;
    msg.status = (enabled ? 1 : 0);
    msg.pad = 0;
    return  (WriteToRFBServer(client, (char *)&msg, sz_rfbSetServerInputMsg) ? TRUE : FALSE);
}

Project: libvncserver
File: rfbproto.c

rfbBool SendXvpMsg(rfbClient* client, uint8_t version, uint8_t code)
{
    rfbXvpMsg xvp;

    if (!SupportsClient2Server(client, rfbXvp)) return TRUE;
    xvp.type = rfbXvp;
    xvp.pad = 0;
    xvp.version = version;
    xvp.code = code;

    if (!WriteToRFBServer(client, (char *)&xvp, sz_rfbXvpMsg))
        return FALSE;

    return TRUE;
}

Project: libvncserver
File: rfbproto.c

PrintPixelFormat(rfbPixelFormat *format)
{
  if (format->bitsPerPixel == 1) {
    rfbClientLog("  Single bit per pixel.\n");
    rfbClientLog(
	    "  %s significant bit in each byte is leftmost on the screen.\n",
	    (format->bigEndian ? "Most" : "Least"));
  } else {
    rfbClientLog("  %d bits per pixel.\n",format->bitsPerPixel);
    if (format->bitsPerPixel != 8) {
      rfbClientLog("  %s significant byte first in each pixel.\n",
	      (format->bigEndian ? "Most" : "Least"));
    }
    if (format->trueColour) {
      rfbClientLog("  TRUE colour: max red %d green %d blue %d"
		   ", shift red %d green %d blue %d\n",
		   format->redMax, format->greenMax, format->blueMax,
		   format->redShift, format->greenShift, format->blueShift);
    } else {
      rfbClientLog("  Colour map (not true colour).\n");
    }
  }
}

Project: libvncserver
File: rfbproto.c

rfbDefaultClientLog(const char *format, ...)
{
    va_list args;
    char buf[256];
    time_t log_clock;

    if(!rfbEnableClientLogging)
      return;

    va_start(args, format);

    time(&log_clock);
    strftime(buf, 255, "%d/%m/%Y %X ", localtime(&log_clock));
    fprintf(stderr, "%s", buf);

    vfprintf(stderr, format, args);
    fflush(stderr);

    va_end(args);
}

Project: libvncserver
File: rfbproto.c

HandlePlainAuth(rfbClient *client)
{
  uint32_t ulen, ulensw;
  uint32_t plen, plensw;
  rfbCredential *cred;

  if (!client->GetCredential)
  {
    rfbClientLog("GetCredential callback is not set.\n");
    return FALSE;
  }
  cred = client->GetCredential(client, rfbCredentialTypeUser);
  if (!cred)
  {
    rfbClientLog("Reading credential failed\n");
    return FALSE;
  }

  ulen = (cred->userCredential.username ? strlen(cred->userCredential.username) : 0);
  ulensw = rfbClientSwap32IfLE(ulen);
  plen = (cred->userCredential.password ? strlen(cred->userCredential.password) : 0);
  plensw = rfbClientSwap32IfLE(plen);
  if (!WriteToRFBServer(client, (char *)&ulensw, 4) ||
      !WriteToRFBServer(client, (char *)&plensw, 4))
  {
    FreeUserCredential(cred);
    return FALSE;
  }
  if (ulen > 0)
  {
    if (!WriteToRFBServer(client, cred->userCredential.username, ulen))
    {
      FreeUserCredential(cred);
      return FALSE;
    }
  }
  if (plen > 0)
  {
    if (!WriteToRFBServer(client, cred->userCredential.password, plen))
    {
      FreeUserCredential(cred);
      return FALSE;
    }
  }

  FreeUserCredential(cred);

  /* Handle the SecurityResult message */
  if (!rfbHandleAuthResult(client)) return FALSE;

  return TRUE;
}

Project: libvncserver
File: rfbproto.c

rfbHandleAuthResult(rfbClient* client)
{
    uint32_t authResult=0;

    if (!ReadFromRFBServer(client, (char *)&authResult, 4)) return FALSE;

    authResult = rfbClientSwap32IfLE(authResult);

    switch (authResult) {
    case rfbVncAuthOK:
      rfbClientLog("VNC authentication succeeded\n");
      return TRUE;
      break;
    case rfbVncAuthFailed:
      if (client->major==3 && client->minor>7)
      {
        /* we have an error following */
        ReadReason(client);
        return FALSE;
      }
      rfbClientLog("VNC authentication failed\n");
      return FALSE;
    case rfbVncAuthTooMany:
      rfbClientLog("VNC authentication failed - too many tries\n");
      return FALSE;
    }

    rfbClientLog("Unknown VNC authentication result: %d\n",
                 (int)authResult);
    return FALSE;
}

Project: libvncserver
File: rfbproto.c

SendKeyEvent(rfbClient* client, uint32_t key, rfbBool down)
{
  rfbKeyEventMsg ke;

  if (!SupportsClient2Server(client, rfbKeyEvent)) return TRUE;

  memset(&ke, 0, sizeof(ke));
  ke.type = rfbKeyEvent;
  ke.down = down ? 1 : 0;
  ke.key = rfbClientSwap32IfLE(key);
  return WriteToRFBServer(client, (char *)&ke, sz_rfbKeyEventMsg);
}

Project: libvncserver
File: rfbproto.c

rfbMpiToBytes(const gcry_mpi_t value, uint8_t *result, size_t size)
{
  gcry_error_t error;
  size_t len;
  int i;

  error = gcry_mpi_print(GCRYMPI_FMT_USG, result, size, &len, value);
  if (gcry_err_code(error) != GPG_ERR_NO_ERROR)
  {
    rfbClientLog("gcry_mpi_print error: %s\n", gcry_strerror(error));
    return FALSE;
  }
  for (i=size-1;i>(int)size-1-(int)len;--i)
    result[i] = result[i-size+len];
  for (;i>=0;--i)
    result[i] = 0;
  return TRUE;
}

Project: libvncserver
File: rfbproto.c

rfbBool TextChatClose(rfbClient* client)
{
    rfbTextChatMsg chat;
    if (!SupportsClient2Server(client, rfbTextChat)) return TRUE;
    chat.type = rfbTextChat;
    chat.pad1 = 0;
    chat.pad2 = 0;
    chat.length = rfbClientSwap32IfLE(rfbTextChatClose);
    return  (WriteToRFBServer(client, (char *)&chat, sz_rfbTextChatMsg) ? TRUE : FALSE);
}

Project: libvncserver
File: rfbproto.c

rfbBool TextChatSend(rfbClient* client, char *text)
{
    rfbTextChatMsg chat;
    int count = strlen(text);

    if (!SupportsClient2Server(client, rfbTextChat)) return TRUE;
    chat.type = rfbTextChat;
    chat.pad1 = 0;
    chat.pad2 = 0;
    chat.length = (uint32_t)count;
    chat.length = rfbClientSwap32IfLE(chat.length);

    if (!WriteToRFBServer(client, (char *)&chat, sz_rfbTextChatMsg))
        return FALSE;

    if (count>0) {
        if (!WriteToRFBServer(client, text, count))
            return FALSE;
    }
    return TRUE;
}

Project: libvncserver
File: rfbproto.c

SendIncrementalFramebufferUpdateRequest(rfbClient* client)
{
	return SendFramebufferUpdateRequest(client,
			client->updateRect.x, client->updateRect.y,
			client->updateRect.w, client->updateRect.h, TRUE);
}

Project: libvncserver
File: rfbproto.c

SetClient2Server(rfbClient* client, int messageType)
{
  client->supportedMessages.client2server[((messageType & 0xFF)/8)] |= (1<<(messageType % 8));
}

Project: libvncserver
File: rfbproto.c

DefaultSupportedMessagesUltraVNC(rfbClient* client)
{
    DefaultSupportedMessages(client);
    SetClient2Server(client, rfbFileTransfer);
    SetClient2Server(client, rfbSetScale);
    SetClient2Server(client, rfbSetServerInput);
    SetClient2Server(client, rfbSetSW);
    SetClient2Server(client, rfbTextChat);
    SetClient2Server(client, rfbPalmVNCSetScaleFactor);
    /* technically, we only care what we can *send* to the server */
    SetServer2Client(client, rfbResizeFrameBuffer);
    SetServer2Client(client, rfbPalmVNCReSizeFrameBuffer);
    SetServer2Client(client, rfbFileTransfer);
    SetServer2Client(client, rfbTextChat);
}

Project: libvncserver
File: rfbproto.c

rfbBool TextChatOpen(rfbClient* client)
{
    rfbTextChatMsg chat;

    if (!SupportsClient2Server(client, rfbTextChat)) return TRUE;
    chat.type = rfbTextChat;
    chat.pad1 = 0;
    chat.pad2 = 0;
    chat.length = rfbClientSwap32IfLE(rfbTextChatOpen);
    return  (WriteToRFBServer(client, (char *)&chat, sz_rfbTextChatMsg) ? TRUE : FALSE);
}

Project: libvncserver
File: rfbproto.c

ConnectToRFBServer(rfbClient* client,const char *hostname, int port)
{
  if (client->serverPort==-1) {
    /* serverHost is a file recorded by vncrec. */
    const char* magic="vncLog0.0";
    char buffer[10];
    rfbVNCRec* rec = (rfbVNCRec*)malloc(sizeof(rfbVNCRec));
    client->vncRec = rec;

    rec->file = fopen(client->serverHost,"rb");
    rec->tv.tv_sec = 0;
    rec->readTimestamp = FALSE;
    rec->doNotSleep = FALSE;
    
    if (!rec->file) {
      rfbClientLog("Could not open %s.\n",client->serverHost);
      return FALSE;
    }
    setbuf(rec->file,NULL);

    if (fread(buffer,1,strlen(magic),rec->file) != strlen(magic) || strncmp(buffer,magic,strlen(magic))) {
      rfbClientLog("File %s was not recorded by vncrec.\n",client->serverHost);
      fclose(rec->file);
      return FALSE;
    }
    client->sock = -1;
    return TRUE;
  }

#ifndef WIN32
  if(IsUnixSocket(hostname))
    /* serverHost is a UNIX socket. */
    client->sock = ConnectClientToUnixSock(hostname);
  else
#endif
  {
#ifdef LIBVNCSERVER_IPv6
    client->sock = ConnectClientToTcpAddr6(hostname, port);
    if (client->sock == -1)
#endif
    {
      unsigned int host;

      /* serverHost is a hostname */
      if (!StringToIPAddr(hostname, &host)) {
        rfbClientLog("Couldn't convert '%s' to host address\n", hostname);
        return FALSE;
      }
      client->sock = ConnectClientToTcpAddr(host, port);
    }
  }

  if (client->sock < 0) {
    rfbClientLog("Unable to connect to VNC server\n");
    return FALSE;
  }

  if(client->QoS_DSCP && !SetDSCP(client->sock, client->QoS_DSCP))
     return FALSE;

  return SetNonBlocking(client->sock);
}

Project: libvncserver
File: rfbproto.c

rfbMulM64(uint64_t x, uint64_t y, uint64_t m)
{
  uint64_t r;
  for(r=0;x>0;x>>=1)
  {
    if (x&1) r=rfbAddM64(r,y,m);
    y=rfbAddM64(y,y,m);
  }
  return r;
}

Project: libvncserver
File: rfbproto.c

ClearServer2Client(rfbClient* client, int messageType)
{
  client->supportedMessages.server2client[((messageType & 0xFF)/8)] &= (!(1<<(messageType % 8)));
}

Project: libvncserver
File: rfbproto.c

rfbBool ConnectToRFBRepeater(rfbClient* client,const char *repeaterHost, int repeaterPort, const char *destHost, int destPort)
{
  rfbProtocolVersionMsg pv;
  int major,minor;
  char tmphost[250];
  int tmphostlen;

#ifdef LIBVNCSERVER_IPv6
  client->sock = ConnectClientToTcpAddr6(repeaterHost, repeaterPort);
  if (client->sock == -1)
#endif
  {
    unsigned int host;
    if (!StringToIPAddr(repeaterHost, &host)) {
      rfbClientLog("Couldn't convert '%s' to host address\n", repeaterHost);
      return FALSE;
    }

    client->sock = ConnectClientToTcpAddr(host, repeaterPort);
  }

  if (client->sock < 0) {
    rfbClientLog("Unable to connect to VNC repeater\n");
    return FALSE;
  }

  if (!SetNonBlocking(client->sock))
    return FALSE;

  if (!ReadFromRFBServer(client, pv, sz_rfbProtocolVersionMsg))
    return FALSE;
  pv[sz_rfbProtocolVersionMsg] = 0;

  /* UltraVNC repeater always report version 000.000 to identify itself */
  if (sscanf(pv,rfbProtocolVersionFormat,&major,&minor) != 2 || major != 0 || minor != 0) {
    rfbClientLog("Not a valid VNC repeater (%s)\n",pv);
    return FALSE;
  }

  rfbClientLog("Connected to VNC repeater, using protocol version %d.%d\n", major, minor);

  tmphostlen = snprintf(tmphost, sizeof(tmphost), "%s:%d", destHost, destPort);
  if(tmphostlen < 0 || tmphostlen >= (int)sizeof(tmphost))
    return FALSE; /* snprintf error or output truncated */

  if (!WriteToRFBServer(client, tmphost, tmphostlen + 1))
    return FALSE;

  return TRUE;
}

Project: libvncserver
File: rfbproto.c

FreeUserCredential(rfbCredential *cred)
{
  if (cred->userCredential.username) free(cred->userCredential.username);
  if (cred->userCredential.password) free(cred->userCredential.password);
  free(cred);
}

Project: libvncserver
File: rfbproto.c

rfbPowM64(uint64_t b, uint64_t e, uint64_t m)
{
  uint64_t r;
  for(r=1;e>0;e>>=1)
  {
    if(e&1) r=rfbMulM64(r,b,m);
    b=rfbMulM64(b,b,m);
  }
  return r;
}

Project: libvncserver
File: rfbproto.c

ClearClient2Server(rfbClient* client, int messageType)
{
  client->supportedMessages.client2server[((messageType & 0xFF)/8)] &= (!(1<<(messageType % 8)));
}