Home | History | Annotate | Download | only in libvncclient 
      1 /*
      2  *  Copyright (C) 2005 Johannes E. Schindelin.  All Rights Reserved.
      3  *
      4  *  This is free software; you can redistribute it and/or modify
      5  *  it under the terms of the GNU General Public License as published by
      6  *  the Free Software Foundation; either version 2 of the License, or
      7  *  (at your option) any later version.
      8  *
      9  *  This software is distributed in the hope that it will be useful,
     10  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
     11  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
     12  *  GNU General Public License for more details.
     13  *
     14  *  You should have received a copy of the GNU General Public License
     15  *  along with this software; if not, write to the Free Software
     16  *  Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307,
     17  *  USA.
     18  */
     19 
     20 #ifdef LIBVNCSERVER_HAVE_LIBZ
     21 
     22 /*
     23  * zrle.c - handle zrle encoding.
     24  *
     25  * This file shouldn't be compiled directly.  It is included multiple times by
     26  * rfbproto.c, each time with a different definition of the macro BPP.  For
     27  * each value of BPP, this file defines a function which handles an zrle
     28  * encoded rectangle with BPP bits per pixel.
     29  */
     30 
     31 #ifndef REALBPP
     32 #define REALBPP BPP
     33 #endif
     34 
     35 #if !defined(UNCOMP) || UNCOMP==0
     36 #define HandleZRLE CONCAT2E(HandleZRLE,REALBPP)
     37 #define HandleZRLETile CONCAT2E(HandleZRLETile,REALBPP)
     38 #elif UNCOMP>0
     39 #define HandleZRLE CONCAT3E(HandleZRLE,REALBPP,Down)
     40 #define HandleZRLETile CONCAT3E(HandleZRLETile,REALBPP,Down)
     41 #else
     42 #define HandleZRLE CONCAT3E(HandleZRLE,REALBPP,Up)
     43 #define HandleZRLETile CONCAT3E(HandleZRLETile,REALBPP,Up)
     44 #endif
     45 #define CARDBPP CONCAT3E(uint,BPP,_t)
     46 #define CARDREALBPP CONCAT3E(uint,REALBPP,_t)
     47 
     48 #define ENDIAN_LITTLE 0
     49 #define ENDIAN_BIG 1
     50 #define ENDIAN_NO 2
     51 #define ZYWRLE_ENDIAN ENDIAN_LITTLE
     52 #undef END_FIX
     53 #if ZYWRLE_ENDIAN == ENDIAN_LITTLE
     54 #  define END_FIX LE
     55 #elif ZYWRLE_ENDIAN == ENDIAN_BIG
     56 #  define END_FIX BE
     57 #else
     58 #  define END_FIX NE
     59 #endif
     60 #define __RFB_CONCAT3E(a,b,c) CONCAT3E(a,b,c)
     61 #define __RFB_CONCAT2E(a,b) CONCAT2E(a,b)
     62 #undef CPIXEL
     63 #if REALBPP != BPP
     64 #if UNCOMP == 0
     65 #define CPIXEL REALBPP
     66 #elif UNCOMP>0
     67 #define CPIXEL CONCAT2E(REALBPP,Down)
     68 #else
     69 #define CPIXEL CONCAT2E(REALBPP,Up)
     70 #endif
     71 #endif
     72 #define PIXEL_T __RFB_CONCAT3E(uint,BPP,_t)
     73 #if BPP!=8
     74 #define ZYWRLE_DECODE 1
     75 #include "zywrletemplate.c"
     76 #endif
     77 #undef CPIXEL
     78 
     79 static int HandleZRLETile(rfbClient* client,
     80 	uint8_t* buffer,size_t buffer_length,
     81 	int x,int y,int w,int h);
     82 
     83 static rfbBool
     84 HandleZRLE (rfbClient* client, int rx, int ry, int rw, int rh)
     85 {
     86 	rfbZRLEHeader header;
     87 	int remaining;
     88 	int inflateResult;
     89 	int toRead;
     90 	int min_buffer_size = rw * rh * (REALBPP / 8) * 2;
     91 
     92 	/* First make sure we have a large enough raw buffer to hold the
     93 	 * decompressed data.  In practice, with a fixed REALBPP, fixed frame
     94 	 * buffer size and the first update containing the entire frame
     95 	 * buffer, this buffer allocation should only happen once, on the
     96 	 * first update.
     97 	 */
     98 	if ( client->raw_buffer_size < min_buffer_size) {
     99 
    100 		if ( client->raw_buffer != NULL ) {
    101 
    102 			free( client->raw_buffer );
    103 
    104 		}
    105 
    106 		client->raw_buffer_size = min_buffer_size;
    107 		client->raw_buffer = (char*) malloc( client->raw_buffer_size );
    108 
    109 	}
    110 
    111 	if (!ReadFromRFBServer(client, (char *)&header, sz_rfbZRLEHeader))
    112 		return FALSE;
    113 
    114 	remaining = rfbClientSwap32IfLE(header.length);
    115 
    116 	/* Need to initialize the decompressor state. */
    117 	client->decompStream.next_in   = ( Bytef * )client->buffer;
    118 	client->decompStream.avail_in  = 0;
    119 	client->decompStream.next_out  = ( Bytef * )client->raw_buffer;
    120 	client->decompStream.avail_out = client->raw_buffer_size;
    121 	client->decompStream.data_type = Z_BINARY;
    122 
    123 	/* Initialize the decompression stream structures on the first invocation. */
    124 	if ( client->decompStreamInited == FALSE ) {
    125 
    126 		inflateResult = inflateInit( &client->decompStream );
    127 
    128 		if ( inflateResult != Z_OK ) {
    129 			rfbClientLog(
    130 					"inflateInit returned error: %d, msg: %s\n",
    131 					inflateResult,
    132 					client->decompStream.msg);
    133 			return FALSE;
    134 		}
    135 
    136 		client->decompStreamInited = TRUE;
    137 
    138 	}
    139 
    140 	inflateResult = Z_OK;
    141 
    142 	/* Process buffer full of data until no more to process, or
    143 	 * some type of inflater error, or Z_STREAM_END.
    144 	 */
    145 	while (( remaining > 0 ) &&
    146 			( inflateResult == Z_OK )) {
    147 
    148 		if ( remaining > RFB_BUFFER_SIZE ) {
    149 			toRead = RFB_BUFFER_SIZE;
    150 		}
    151 		else {
    152 			toRead = remaining;
    153 		}
    154 
    155 		/* Fill the buffer, obtaining data from the server. */
    156 		if (!ReadFromRFBServer(client, client->buffer,toRead))
    157 			return FALSE;
    158 
    159 		client->decompStream.next_in  = ( Bytef * )client->buffer;
    160 		client->decompStream.avail_in = toRead;
    161 
    162 		/* Need to uncompress buffer full. */
    163 		inflateResult = inflate( &client->decompStream, Z_SYNC_FLUSH );
    164 
    165 		/* We never supply a dictionary for compression. */
    166 		if ( inflateResult == Z_NEED_DICT ) {
    167 			rfbClientLog("zlib inflate needs a dictionary!\n");
    168 			return FALSE;
    169 		}
    170 		if ( inflateResult < 0 ) {
    171 			rfbClientLog(
    172 					"zlib inflate returned error: %d, msg: %s\n",
    173 					inflateResult,
    174 					client->decompStream.msg);
    175 			return FALSE;
    176 		}
    177 
    178 		/* Result buffer allocated to be at least large enough.  We should
    179 		 * never run out of space!
    180 		 */
    181 		if (( client->decompStream.avail_in > 0 ) &&
    182 				( client->decompStream.avail_out <= 0 )) {
    183 			rfbClientLog("zlib inflate ran out of space!\n");
    184 			return FALSE;
    185 		}
    186 
    187 		remaining -= toRead;
    188 
    189 	} /* while ( remaining > 0 ) */
    190 
    191 	if ( inflateResult == Z_OK ) {
    192 		char* buf=client->raw_buffer;
    193 		int i,j;
    194 
    195 		remaining = client->raw_buffer_size-client->decompStream.avail_out;
    196 
    197 		for(j=0; j<rh; j+=rfbZRLETileHeight)
    198 			for(i=0; i<rw; i+=rfbZRLETileWidth) {
    199 				int subWidth=(i+rfbZRLETileWidth>rw)?rw-i:rfbZRLETileWidth;
    200 				int subHeight=(j+rfbZRLETileHeight>rh)?rh-j:rfbZRLETileHeight;
    201 				int result=HandleZRLETile(client,buf,remaining,rx+i,ry+j,subWidth,subHeight);
    202 
    203 				if(result<0) {
    204 					rfbClientLog("ZRLE decoding failed (%d)\n",result);
    205 return TRUE;
    206 					return FALSE;
    207 				}
    208 
    209 				buf+=result;
    210 				remaining-=result;
    211 			}
    212 	}
    213 	else {
    214 
    215 		rfbClientLog(
    216 				"zlib inflate returned error: %d, msg: %s\n",
    217 				inflateResult,
    218 				client->decompStream.msg);
    219 		return FALSE;
    220 
    221 	}
    222 
    223 	return TRUE;
    224 }
    225 
    226 #if REALBPP!=BPP && defined(UNCOMP) && UNCOMP!=0
    227 #if UNCOMP>0
    228 #define UncompressCPixel(pointer) ((*(CARDBPP*)pointer)>>UNCOMP)
    229 #else
    230 #define UncompressCPixel(pointer) ((*(CARDBPP*)pointer)<<(-(UNCOMP)))
    231 #endif
    232 #else
    233 #define UncompressCPixel(pointer) (*(CARDBPP*)pointer)
    234 #endif
    235 
    236 static int HandleZRLETile(rfbClient* client,
    237 		uint8_t* buffer,size_t buffer_length,
    238 		int x,int y,int w,int h) {
    239 	uint8_t* buffer_copy = buffer;
    240 	uint8_t* buffer_end = buffer+buffer_length;
    241 	uint8_t type;
    242 #if BPP!=8
    243 	uint8_t zywrle_level = (client->appData.qualityLevel & 0x80) ?
    244 		0 : (3 - client->appData.qualityLevel / 3);
    245 #endif
    246 
    247 	if(buffer_length<1)
    248 		return -2;
    249 
    250 	type = *buffer;
    251 	buffer++;
    252 	{
    253 		if( type == 0 ) /* raw */
    254 #if BPP!=8
    255           if( zywrle_level > 0 ){
    256 			CARDBPP* pFrame = (CARDBPP*)client->frameBuffer + y*client->width+x;
    257 			int ret;
    258 			client->appData.qualityLevel |= 0x80;
    259 			ret = HandleZRLETile(client, buffer, buffer_end-buffer, x, y, w, h);
    260 		    client->appData.qualityLevel &= 0x7F;
    261 			if( ret < 0 ){
    262 				return ret;
    263 			}
    264 			ZYWRLE_SYNTHESIZE( pFrame, pFrame, w, h, client->width, zywrle_level, (int*)client->zlib_buffer );
    265 			buffer += ret;
    266 		  }else
    267 #endif
    268 		{
    269 #if REALBPP!=BPP
    270 			int i,j;
    271 
    272 			if(1+w*h*REALBPP/8>buffer_length) {
    273 				rfbClientLog("expected %d bytes, got only %d (%dx%d)\n",1+w*h*REALBPP/8,buffer_length,w,h);
    274 				return -3;
    275 			}
    276 
    277 			for(j=y*client->width; j<(y+h)*client->width; j+=client->width)
    278 				for(i=x; i<x+w; i++,buffer+=REALBPP/8)
    279 					((CARDBPP*)client->frameBuffer)[j+i] = UncompressCPixel(buffer);
    280 #else
    281 			CopyRectangle(client, buffer, x, y, w, h);
    282 			buffer+=w*h*REALBPP/8;
    283 #endif
    284 		}
    285 		else if( type == 1 ) /* solid */
    286 		{
    287 			CARDBPP color = UncompressCPixel(buffer);
    288 
    289 			if(1+REALBPP/8>buffer_length)
    290 				return -4;
    291 
    292 			FillRectangle(client, x, y, w, h, color);
    293 
    294 			buffer+=REALBPP/8;
    295 
    296 		}
    297 		else if( (type >= 2)&&(type <= 127) ) /* packed Palette */
    298 		{
    299 			CARDBPP palette[16];
    300 			int i,j,shift,
    301 				bpp=(type>4?(type>16?8:4):(type>2?2:1)),
    302 				mask=(1<<bpp)-1,
    303 				divider=(8/bpp);
    304 
    305 			if(1+type*REALBPP/8+((w+divider-1)/divider)*h>buffer_length)
    306 				return -5;
    307 
    308 			/* read palette */
    309 			for(i=0; i<type; i++,buffer+=REALBPP/8)
    310 				palette[i] = UncompressCPixel(buffer);
    311 
    312 			/* read palettized pixels */
    313 			for(j=y*client->width; j<(y+h)*client->width; j+=client->width) {
    314 				for(i=x,shift=8-bpp; i<x+w; i++) {
    315 					((CARDBPP*)client->frameBuffer)[j+i] = palette[((*buffer)>>shift)&mask];
    316 					shift-=bpp;
    317 					if(shift<0) {
    318 						shift=8-bpp;
    319 						buffer++;
    320 					}
    321 				}
    322 				if(shift<8-bpp)
    323 					buffer++;
    324 			}
    325 
    326 		}
    327 		/* case 17 ... 127: not used, but valid */
    328 		else if( type == 128 ) /* plain RLE */
    329 		{
    330 			int i=0,j=0;
    331 			while(j<h) {
    332 				int color,length;
    333 				/* read color */
    334 				if(buffer+REALBPP/8+1>buffer_end)
    335 					return -7;
    336 				color = UncompressCPixel(buffer);
    337 				buffer+=REALBPP/8;
    338 				/* read run length */
    339 				length=1;
    340 				while(*buffer==0xff) {
    341 					if(buffer+1>=buffer_end)
    342 						return -8;
    343 					length+=*buffer;
    344 					buffer++;
    345 				}
    346 				length+=*buffer;
    347 				buffer++;
    348 				while(j<h && length>0) {
    349 					((CARDBPP*)client->frameBuffer)[(y+j)*client->width+x+i] = color;
    350 					length--;
    351 					i++;
    352 					if(i>=w) {
    353 						i=0;
    354 						j++;
    355 					}
    356 				}
    357 				if(length>0)
    358 					rfbClientLog("Warning: possible ZRLE corruption\n");
    359 			}
    360 
    361 		}
    362 		else if( type == 129 ) /* unused */
    363 		{
    364 			return -8;
    365 		}
    366 		else if( type >= 130 ) /* palette RLE */
    367 		{
    368 			CARDBPP palette[128];
    369 			int i,j;
    370 
    371 			if(2+(type-128)*REALBPP/8>buffer_length)
    372 				return -9;
    373 
    374 			/* read palette */
    375 			for(i=0; i<type-128; i++,buffer+=REALBPP/8)
    376 				palette[i] = UncompressCPixel(buffer);
    377 			/* read palettized pixels */
    378 			i=j=0;
    379 			while(j<h) {
    380 				int color,length;
    381 				/* read color */
    382 				if(buffer>=buffer_end)
    383 					return -10;
    384 				color = palette[(*buffer)&0x7f];
    385 				length=1;
    386 				if(*buffer&0x80) {
    387 					if(buffer+1>=buffer_end)
    388 						return -11;
    389 					buffer++;
    390 					/* read run length */
    391 					while(*buffer==0xff) {
    392 						if(buffer+1>=buffer_end)
    393 							return -8;
    394 						length+=*buffer;
    395 						buffer++;
    396 					}
    397 					length+=*buffer;
    398 				}
    399 				buffer++;
    400 				while(j<h && length>0) {
    401 					((CARDBPP*)client->frameBuffer)[(y+j)*client->width+x+i] = color;
    402 					length--;
    403 					i++;
    404 					if(i>=w) {
    405 						i=0;
    406 						j++;
    407 					}
    408 				}
    409 				if(length>0)
    410 					rfbClientLog("Warning: possible ZRLE corruption\n");
    411 			}
    412 		}
    413 	}
    414 
    415 	return buffer-buffer_copy;
    416 }
    417 
    418 #undef CARDBPP
    419 #undef CARDREALBPP
    420 #undef HandleZRLE
    421 #undef HandleZRLETile
    422 #undef UncompressCPixel
    423 #undef REALBPP
    424 
    425 #endif
    426 
    427 #undef UNCOMP
    428