Commit message: Fix integer overflow in MallocFrameBuffer()

Promote integers to uint64_t to avoid integer overflow issue during
frame buffer allocation for very large screen sizes
CVE: CVE-2014-6051
CWE: CWE-189
Vulnerability description: Integer overflow in the MallocFrameBuffer function in vncviewer.c in LibVNCServer 0.9.9 and earlier allows remote VNC servers to cause a denial of service (crash) and possibly execute arbitrary code via an advertisement for a large screen size, which triggers a heap-based buffer overflow.

Project: libvncserver
File: vncviewer.c

static void Dummy(rfbClient* client) {
}

Project: libvncserver
File: vncviewer.c

static void DummyRect(rfbClient* client, int x, int y, int w, int h) {
}

Project: libvncserver
File: vncviewer.c

static rfbBool MallocFrameBuffer(rfbClient* client) {
uint64_t allocSize;

  if(client->frameBuffer)
    free(client->frameBuffer);

  /* SECURITY: promote 'width' into uint64_t so that the multiplication does not overflow
     'width' and 'height' are 16-bit integers per RFB protocol design
     SIZE_MAX is the maximum value that can fit into size_t
  */
  allocSize = (uint64_t)client->width * client->height * client->format.bitsPerPixel/8;

  if (allocSize >= SIZE_MAX) {
    rfbClientErr("CRITICAL: cannot allocate frameBuffer, requested size is too large\n");
    return FALSE;
  }

  client->frameBuffer=malloc( (size_t)allocSize );

  if (client->frameBuffer == NULL)
    rfbClientErr("CRITICAL: frameBuffer allocation failed, requested size too large or not enough memory?\n");

  return client->frameBuffer?TRUE:FALSE;
}

Project: libvncserver
File: vncviewer.c

static char* NoPassword(rfbClient* client) {
  return strdup("");
}

Project: libvncserver
File: vncviewer.c

static void initAppData(AppData* data) {
	data->shareDesktop=TRUE;
	data->viewOnly=FALSE;
#ifdef LIBVNCSERVER_CONFIG_LIBVA
	data->encodingsString="h264 tight zrle ultra copyrect hextile zlib corre rre raw";
#else
	data->encodingsString="tight zrle ultra copyrect hextile zlib corre rre raw";
#endif
	data->useBGR233=FALSE;
	data->nColours=0;
	data->forceOwnCmap=FALSE;
	data->forceTrueColour=FALSE;
	data->requestedDepth=0;
	data->compressLevel=3;
	data->qualityLevel=5;
#ifdef LIBVNCSERVER_HAVE_LIBJPEG
	data->enableJPEG=TRUE;
#else
	data->enableJPEG=FALSE;
#endif
	data->useRemoteCursor=FALSE;
}

Project: libvncserver
File: vncviewer.c

rfbClient* rfbGetClient(int bitsPerSample,int samplesPerPixel,
			int bytesPerPixel) {
  rfbClient* client=(rfbClient*)calloc(sizeof(rfbClient),1);
  if(!client) {
    rfbClientErr("Couldn't allocate client structure!\n");
    return NULL;
  }
  initAppData(&client->appData);
  client->endianTest = 1;
  client->programName="";
  client->serverHost=strdup("");
  client->serverPort=5900;

  client->destHost = NULL;
  client->destPort = 5900;
  
  client->CurrentKeyboardLedState = 0;
  client->HandleKeyboardLedState = (HandleKeyboardLedStateProc)DummyPoint;

  /* default: use complete frame buffer */ 
  client->updateRect.x = -1;
 
  client->frameBuffer = NULL;
  client->outputWindow = 0;
 
  client->format.bitsPerPixel = bytesPerPixel*8;
  client->format.depth = bitsPerSample*samplesPerPixel;
  client->appData.requestedDepth=client->format.depth;
  client->format.bigEndian = *(char *)&client->endianTest?FALSE:TRUE;
  client->format.trueColour = TRUE;

  if (client->format.bitsPerPixel == 8) {
    client->format.redMax = 7;
    client->format.greenMax = 7;
    client->format.blueMax = 3;
    client->format.redShift = 0;
    client->format.greenShift = 3;
    client->format.blueShift = 6;
  } else {
    client->format.redMax = (1 << bitsPerSample) - 1;
    client->format.greenMax = (1 << bitsPerSample) - 1;
    client->format.blueMax = (1 << bitsPerSample) - 1;
    if(!client->format.bigEndian) {
      client->format.redShift = 0;
      client->format.greenShift = bitsPerSample;
      client->format.blueShift = bitsPerSample * 2;
    } else {
      if(client->format.bitsPerPixel==8*3) {
	client->format.redShift = bitsPerSample*2;
	client->format.greenShift = bitsPerSample*1;
	client->format.blueShift = 0;
      } else {
	client->format.redShift = bitsPerSample*3;
	client->format.greenShift = bitsPerSample*2;
	client->format.blueShift = bitsPerSample;
      }
    }
  }

  client->bufoutptr=client->buf;
  client->buffered=0;

#ifdef LIBVNCSERVER_HAVE_LIBZ
  client->raw_buffer_size = -1;
  client->decompStreamInited = FALSE;

#ifdef LIBVNCSERVER_HAVE_LIBJPEG
  memset(client->zlibStreamActive,0,sizeof(rfbBool)*4);
  client->jpegSrcManager = NULL;
#endif
#endif

  client->HandleCursorPos = DummyPoint;
  client->SoftCursorLockArea = DummyRect;
  client->SoftCursorUnlockScreen = Dummy;
  client->GotFrameBufferUpdate = DummyRect;
  client->FinishedFrameBufferUpdate = NULL;
  client->GetPassword = ReadPassword;
  client->MallocFrameBuffer = MallocFrameBuffer;
  client->Bell = Dummy;
  client->CurrentKeyboardLedState = 0;
  client->HandleKeyboardLedState = (HandleKeyboardLedStateProc)DummyPoint;
  client->QoS_DSCP = 0;

  client->authScheme = 0;
  client->subAuthScheme = 0;
  client->GetCredential = NULL;
  client->tlsSession = NULL;
  client->sock = -1;
  client->listenSock = -1;
  client->listenAddress = NULL;
  client->listen6Sock = -1;
  client->listen6Address = NULL;
  client->clientAuthSchemes = NULL;
  return client;
}

Project: libvncserver
File: vncviewer.c

rfbBool rfbInitClient(rfbClient* client,int* argc,char** argv) {
  int i,j;

  if(argv && argc && *argc) {
    if(client->programName==0)
      client->programName=argv[0];

    for (i = 1; i < *argc; i++) {
      j = i;
      if (strcmp(argv[i], "-listen") == 0) {
	listenForIncomingConnections(client);
	break;
      } else if (strcmp(argv[i], "-listennofork") == 0) {
	listenForIncomingConnectionsNoFork(client, -1);
	break;
      } else if (strcmp(argv[i], "-play") == 0) {
	client->serverPort = -1;
	j++;
      } else if (i+1<*argc && strcmp(argv[i], "-encodings") == 0) {
	client->appData.encodingsString = argv[i+1];
	j+=2;
      } else if (i+1<*argc && strcmp(argv[i], "-compress") == 0) {
	client->appData.compressLevel = atoi(argv[i+1]);
	j+=2;
      } else if (i+1<*argc && strcmp(argv[i], "-quality") == 0) {
	client->appData.qualityLevel = atoi(argv[i+1]);
	j+=2;
      } else if (i+1<*argc && strcmp(argv[i], "-scale") == 0) {
        client->appData.scaleSetting = atoi(argv[i+1]);
        j+=2;
      } else if (i+1<*argc && strcmp(argv[i], "-qosdscp") == 0) {
        client->QoS_DSCP = atoi(argv[i+1]);
        j+=2;
      } else if (i+1<*argc && strcmp(argv[i], "-repeaterdest") == 0) {
	char* colon=strchr(argv[i+1],':');

	if(client->destHost)
	  free(client->destHost);
        client->destPort = 5900;

	client->destHost = strdup(argv[i+1]);
	if(colon) {
	  client->destHost[(int)(colon-argv[i+1])] = '\0';
	  client->destPort = atoi(colon+1);
	}
        j+=2;
      } else {
	char* colon=strchr(argv[i],':');

	if(client->serverHost)
	  free(client->serverHost);

	if(colon) {
	  client->serverHost = strdup(argv[i]);
	  client->serverHost[(int)(colon-argv[i])] = '\0';
	  client->serverPort = atoi(colon+1);
	} else {
	  client->serverHost = strdup(argv[i]);
	}
	if(client->serverPort >= 0 && client->serverPort < 5900)
	  client->serverPort += 5900;
      }
      /* purge arguments */
      if (j>i) {
	*argc-=j-i;
	memmove(argv+i,argv+j,(*argc-i)*sizeof(char*));
	i--;
      }
    }
  }

  if(!rfbInitConnection(client)) {
    rfbClientCleanup(client);
    return FALSE;
  }

  return TRUE;
}

Project: libvncserver
File: vncviewer.c

static char* ReadPassword(rfbClient* client) {
#ifdef __MINGW32__
	/* FIXME */
	rfbClientErr("ReadPassword on MinGW32 NOT IMPLEMENTED\n");
	return NoPassword(client);
#else
	int i;
	char* p=malloc(9);
	struct termios save,noecho;
	p[0]=0;
	if(tcgetattr(fileno(stdin),&save)!=0) return p;
	noecho=save; noecho.c_lflag &= ~ECHO;
	if(tcsetattr(fileno(stdin),TCSAFLUSH,&noecho)!=0) return p;
	fprintf(stderr,"Password: ");
	i=0;
	while(1) {
		int c=fgetc(stdin);
		if(c=='\n')
			break;
		if(i<8) {
			p[i]=c;
			i++;
			p[i]=0;
		}
	}
	tcsetattr(fileno(stdin),TCSAFLUSH,&save);
	return p;
#endif
}

Project: libvncserver
File: vncviewer.c

static rfbBool DummyPoint(rfbClient* client, int x, int y) {
  return TRUE;
}

Project: libvncserver
File: vncviewer.c

void rfbClientCleanup(rfbClient* client) {
#ifdef LIBVNCSERVER_HAVE_LIBZ
#ifdef LIBVNCSERVER_HAVE_LIBJPEG
  int i;

  for ( i = 0; i < 4; i++ ) {
    if (client->zlibStreamActive[i] == TRUE ) {
      if (inflateEnd (&client->zlibStream[i]) != Z_OK &&
	  client->zlibStream[i].msg != NULL)
	rfbClientLog("inflateEnd: %s\n", client->zlibStream[i].msg);
    }
  }

  if ( client->decompStreamInited == TRUE ) {
    if (inflateEnd (&client->decompStream) != Z_OK &&
	client->decompStream.msg != NULL)
      rfbClientLog("inflateEnd: %s\n", client->decompStream.msg );
  }

  if (client->jpegSrcManager)
    free(client->jpegSrcManager);
#endif
#endif

  FreeTLS(client);

  while (client->clientData) {
    rfbClientData* next = client->clientData->next;
    free(client->clientData);
    client->clientData = next;
  }

  if (client->sock >= 0)
    close(client->sock);
  if (client->listenSock >= 0)
    close(client->listenSock);
  free(client->desktopName);
  free(client->serverHost);
  if (client->destHost)
    free(client->destHost);
  if (client->clientAuthSchemes)
    free(client->clientAuthSchemes);
  free(client);
}