1 /* 2 * Copyright (C) 2005 Johannes E. Schindelin. All Rights Reserved. 3 * 4 * This is free software; you can redistribute it and/or modify 5 * it under the terms of the GNU General Public License as published by 6 * the Free Software Foundation; either version 2 of the License, or 7 * (at your option) any later version. 8 * 9 * This software is distributed in the hope that it will be useful, 10 * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 * GNU General Public License for more details. 13 * 14 * You should have received a copy of the GNU General Public License 15 * along with this software; if not, write to the Free Software 16 * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, 17 * USA. 18 */ 19 20 #ifdef LIBVNCSERVER_HAVE_LIBZ 21 22 /* 23 * zrle.c - handle zrle encoding. 24 * 25 * This file shouldn't be compiled directly. It is included multiple times by 26 * rfbproto.c, each time with a different definition of the macro BPP. For 27 * each value of BPP, this file defines a function which handles an zrle 28 * encoded rectangle with BPP bits per pixel. 29 */ 30 31 #ifndef REALBPP 32 #define REALBPP BPP 33 #endif 34 35 #if !defined(UNCOMP) || UNCOMP==0 36 #define HandleZRLE CONCAT2E(HandleZRLE,REALBPP) 37 #define HandleZRLETile CONCAT2E(HandleZRLETile,REALBPP) 38 #elif UNCOMP>0 39 #define HandleZRLE CONCAT3E(HandleZRLE,REALBPP,Down) 40 #define HandleZRLETile CONCAT3E(HandleZRLETile,REALBPP,Down) 41 #else 42 #define HandleZRLE CONCAT3E(HandleZRLE,REALBPP,Up) 43 #define HandleZRLETile CONCAT3E(HandleZRLETile,REALBPP,Up) 44 #endif 45 #define CARDBPP CONCAT3E(uint,BPP,_t) 46 #define CARDREALBPP CONCAT3E(uint,REALBPP,_t) 47 48 #define ENDIAN_LITTLE 0 49 #define ENDIAN_BIG 1 50 #define ENDIAN_NO 2 51 #define ZYWRLE_ENDIAN ENDIAN_LITTLE 52 #undef END_FIX 53 #if ZYWRLE_ENDIAN == ENDIAN_LITTLE 54 # define END_FIX LE 55 #elif ZYWRLE_ENDIAN == ENDIAN_BIG 56 # define END_FIX BE 57 #else 58 # define END_FIX NE 59 #endif 60 #define __RFB_CONCAT3E(a,b,c) CONCAT3E(a,b,c) 61 #define __RFB_CONCAT2E(a,b) CONCAT2E(a,b) 62 #undef CPIXEL 63 #if REALBPP != BPP 64 #if UNCOMP == 0 65 #define CPIXEL REALBPP 66 #elif UNCOMP>0 67 #define CPIXEL CONCAT2E(REALBPP,Down) 68 #else 69 #define CPIXEL CONCAT2E(REALBPP,Up) 70 #endif 71 #endif 72 #define PIXEL_T __RFB_CONCAT3E(uint,BPP,_t) 73 #if BPP!=8 74 #define ZYWRLE_DECODE 1 75 #include "zywrletemplate.c" 76 #endif 77 #undef CPIXEL 78 79 static int HandleZRLETile(rfbClient* client, 80 uint8_t* buffer,size_t buffer_length, 81 int x,int y,int w,int h); 82 83 static rfbBool 84 HandleZRLE (rfbClient* client, int rx, int ry, int rw, int rh) 85 { 86 rfbZRLEHeader header; 87 int remaining; 88 int inflateResult; 89 int toRead; 90 int min_buffer_size = rw * rh * (REALBPP / 8) * 2; 91 92 /* First make sure we have a large enough raw buffer to hold the 93 * decompressed data. In practice, with a fixed REALBPP, fixed frame 94 * buffer size and the first update containing the entire frame 95 * buffer, this buffer allocation should only happen once, on the 96 * first update. 97 */ 98 if ( client->raw_buffer_size < min_buffer_size) { 99 100 if ( client->raw_buffer != NULL ) { 101 102 free( client->raw_buffer ); 103 104 } 105 106 client->raw_buffer_size = min_buffer_size; 107 client->raw_buffer = (char*) malloc( client->raw_buffer_size ); 108 109 } 110 111 if (!ReadFromRFBServer(client, (char *)&header, sz_rfbZRLEHeader)) 112 return FALSE; 113 114 remaining = rfbClientSwap32IfLE(header.length); 115 116 /* Need to initialize the decompressor state. */ 117 client->decompStream.next_in = ( Bytef * )client->buffer; 118 client->decompStream.avail_in = 0; 119 client->decompStream.next_out = ( Bytef * )client->raw_buffer; 120 client->decompStream.avail_out = client->raw_buffer_size; 121 client->decompStream.data_type = Z_BINARY; 122 123 /* Initialize the decompression stream structures on the first invocation. */ 124 if ( client->decompStreamInited == FALSE ) { 125 126 inflateResult = inflateInit( &client->decompStream ); 127 128 if ( inflateResult != Z_OK ) { 129 rfbClientLog( 130 "inflateInit returned error: %d, msg: %s\n", 131 inflateResult, 132 client->decompStream.msg); 133 return FALSE; 134 } 135 136 client->decompStreamInited = TRUE; 137 138 } 139 140 inflateResult = Z_OK; 141 142 /* Process buffer full of data until no more to process, or 143 * some type of inflater error, or Z_STREAM_END. 144 */ 145 while (( remaining > 0 ) && 146 ( inflateResult == Z_OK )) { 147 148 if ( remaining > RFB_BUFFER_SIZE ) { 149 toRead = RFB_BUFFER_SIZE; 150 } 151 else { 152 toRead = remaining; 153 } 154 155 /* Fill the buffer, obtaining data from the server. */ 156 if (!ReadFromRFBServer(client, client->buffer,toRead)) 157 return FALSE; 158 159 client->decompStream.next_in = ( Bytef * )client->buffer; 160 client->decompStream.avail_in = toRead; 161 162 /* Need to uncompress buffer full. */ 163 inflateResult = inflate( &client->decompStream, Z_SYNC_FLUSH ); 164 165 /* We never supply a dictionary for compression. */ 166 if ( inflateResult == Z_NEED_DICT ) { 167 rfbClientLog("zlib inflate needs a dictionary!\n"); 168 return FALSE; 169 } 170 if ( inflateResult < 0 ) { 171 rfbClientLog( 172 "zlib inflate returned error: %d, msg: %s\n", 173 inflateResult, 174 client->decompStream.msg); 175 return FALSE; 176 } 177 178 /* Result buffer allocated to be at least large enough. We should 179 * never run out of space! 180 */ 181 if (( client->decompStream.avail_in > 0 ) && 182 ( client->decompStream.avail_out <= 0 )) { 183 rfbClientLog("zlib inflate ran out of space!\n"); 184 return FALSE; 185 } 186 187 remaining -= toRead; 188 189 } /* while ( remaining > 0 ) */ 190 191 if ( inflateResult == Z_OK ) { 192 char* buf=client->raw_buffer; 193 int i,j; 194 195 remaining = client->raw_buffer_size-client->decompStream.avail_out; 196 197 for(j=0; j<rh; j+=rfbZRLETileHeight) 198 for(i=0; i<rw; i+=rfbZRLETileWidth) { 199 int subWidth=(i+rfbZRLETileWidth>rw)?rw-i:rfbZRLETileWidth; 200 int subHeight=(j+rfbZRLETileHeight>rh)?rh-j:rfbZRLETileHeight; 201 int result=HandleZRLETile(client,buf,remaining,rx+i,ry+j,subWidth,subHeight); 202 203 if(result<0) { 204 rfbClientLog("ZRLE decoding failed (%d)\n",result); 205 return TRUE; 206 return FALSE; 207 } 208 209 buf+=result; 210 remaining-=result; 211 } 212 } 213 else { 214 215 rfbClientLog( 216 "zlib inflate returned error: %d, msg: %s\n", 217 inflateResult, 218 client->decompStream.msg); 219 return FALSE; 220 221 } 222 223 return TRUE; 224 } 225 226 #if REALBPP!=BPP && defined(UNCOMP) && UNCOMP!=0 227 #if UNCOMP>0 228 #define UncompressCPixel(pointer) ((*(CARDBPP*)pointer)>>UNCOMP) 229 #else 230 #define UncompressCPixel(pointer) ((*(CARDBPP*)pointer)<<(-(UNCOMP))) 231 #endif 232 #else 233 #define UncompressCPixel(pointer) (*(CARDBPP*)pointer) 234 #endif 235 236 static int HandleZRLETile(rfbClient* client, 237 uint8_t* buffer,size_t buffer_length, 238 int x,int y,int w,int h) { 239 uint8_t* buffer_copy = buffer; 240 uint8_t* buffer_end = buffer+buffer_length; 241 uint8_t type; 242 #if BPP!=8 243 uint8_t zywrle_level = (client->appData.qualityLevel & 0x80) ? 244 0 : (3 - client->appData.qualityLevel / 3); 245 #endif 246 247 if(buffer_length<1) 248 return -2; 249 250 type = *buffer; 251 buffer++; 252 { 253 if( type == 0 ) /* raw */ 254 #if BPP!=8 255 if( zywrle_level > 0 ){ 256 CARDBPP* pFrame = (CARDBPP*)client->frameBuffer + y*client->width+x; 257 int ret; 258 client->appData.qualityLevel |= 0x80; 259 ret = HandleZRLETile(client, buffer, buffer_end-buffer, x, y, w, h); 260 client->appData.qualityLevel &= 0x7F; 261 if( ret < 0 ){ 262 return ret; 263 } 264 ZYWRLE_SYNTHESIZE( pFrame, pFrame, w, h, client->width, zywrle_level, (int*)client->zlib_buffer ); 265 buffer += ret; 266 }else 267 #endif 268 { 269 #if REALBPP!=BPP 270 int i,j; 271 272 if(1+w*h*REALBPP/8>buffer_length) { 273 rfbClientLog("expected %d bytes, got only %d (%dx%d)\n",1+w*h*REALBPP/8,buffer_length,w,h); 274 return -3; 275 } 276 277 for(j=y*client->width; j<(y+h)*client->width; j+=client->width) 278 for(i=x; i<x+w; i++,buffer+=REALBPP/8) 279 ((CARDBPP*)client->frameBuffer)[j+i] = UncompressCPixel(buffer); 280 #else 281 CopyRectangle(client, buffer, x, y, w, h); 282 buffer+=w*h*REALBPP/8; 283 #endif 284 } 285 else if( type == 1 ) /* solid */ 286 { 287 CARDBPP color = UncompressCPixel(buffer); 288 289 if(1+REALBPP/8>buffer_length) 290 return -4; 291 292 FillRectangle(client, x, y, w, h, color); 293 294 buffer+=REALBPP/8; 295 296 } 297 else if( (type >= 2)&&(type <= 127) ) /* packed Palette */ 298 { 299 CARDBPP palette[16]; 300 int i,j,shift, 301 bpp=(type>4?(type>16?8:4):(type>2?2:1)), 302 mask=(1<<bpp)-1, 303 divider=(8/bpp); 304 305 if(1+type*REALBPP/8+((w+divider-1)/divider)*h>buffer_length) 306 return -5; 307 308 /* read palette */ 309 for(i=0; i<type; i++,buffer+=REALBPP/8) 310 palette[i] = UncompressCPixel(buffer); 311 312 /* read palettized pixels */ 313 for(j=y*client->width; j<(y+h)*client->width; j+=client->width) { 314 for(i=x,shift=8-bpp; i<x+w; i++) { 315 ((CARDBPP*)client->frameBuffer)[j+i] = palette[((*buffer)>>shift)&mask]; 316 shift-=bpp; 317 if(shift<0) { 318 shift=8-bpp; 319 buffer++; 320 } 321 } 322 if(shift<8-bpp) 323 buffer++; 324 } 325 326 } 327 /* case 17 ... 127: not used, but valid */ 328 else if( type == 128 ) /* plain RLE */ 329 { 330 int i=0,j=0; 331 while(j<h) { 332 int color,length; 333 /* read color */ 334 if(buffer+REALBPP/8+1>buffer_end) 335 return -7; 336 color = UncompressCPixel(buffer); 337 buffer+=REALBPP/8; 338 /* read run length */ 339 length=1; 340 while(*buffer==0xff) { 341 if(buffer+1>=buffer_end) 342 return -8; 343 length+=*buffer; 344 buffer++; 345 } 346 length+=*buffer; 347 buffer++; 348 while(j<h && length>0) { 349 ((CARDBPP*)client->frameBuffer)[(y+j)*client->width+x+i] = color; 350 length--; 351 i++; 352 if(i>=w) { 353 i=0; 354 j++; 355 } 356 } 357 if(length>0) 358 rfbClientLog("Warning: possible ZRLE corruption\n"); 359 } 360 361 } 362 else if( type == 129 ) /* unused */ 363 { 364 return -8; 365 } 366 else if( type >= 130 ) /* palette RLE */ 367 { 368 CARDBPP palette[128]; 369 int i,j; 370 371 if(2+(type-128)*REALBPP/8>buffer_length) 372 return -9; 373 374 /* read palette */ 375 for(i=0; i<type-128; i++,buffer+=REALBPP/8) 376 palette[i] = UncompressCPixel(buffer); 377 /* read palettized pixels */ 378 i=j=0; 379 while(j<h) { 380 int color,length; 381 /* read color */ 382 if(buffer>=buffer_end) 383 return -10; 384 color = palette[(*buffer)&0x7f]; 385 length=1; 386 if(*buffer&0x80) { 387 if(buffer+1>=buffer_end) 388 return -11; 389 buffer++; 390 /* read run length */ 391 while(*buffer==0xff) { 392 if(buffer+1>=buffer_end) 393 return -8; 394 length+=*buffer; 395 buffer++; 396 } 397 length+=*buffer; 398 } 399 buffer++; 400 while(j<h && length>0) { 401 ((CARDBPP*)client->frameBuffer)[(y+j)*client->width+x+i] = color; 402 length--; 403 i++; 404 if(i>=w) { 405 i=0; 406 j++; 407 } 408 } 409 if(length>0) 410 rfbClientLog("Warning: possible ZRLE corruption\n"); 411 } 412 } 413 } 414 415 return buffer-buffer_copy; 416 } 417 418 #undef CARDBPP 419 #undef CARDREALBPP 420 #undef HandleZRLE 421 #undef HandleZRLETile 422 #undef UncompressCPixel 423 #undef REALBPP 424 425 #endif 426 427 #undef UNCOMP 428