1 2 /* pnggccrd.c - mixed C/assembler version of utilities to read a PNG file 3 * 4 * For Intel/AMD x86 or x86-64 CPU (Pentium-MMX or later) and GNU C compiler. 5 * 6 * Last changed in libpng 1.2.19 August 18, 2007 7 * For conditions of distribution and use, see copyright notice in png.h 8 * Copyright (c) 1998 Intel Corporation 9 * Copyright (c) 1999-2002,2007 Greg Roelofs 10 * Copyright (c) 1998-2007 Glenn Randers-Pehrson 11 * 12 * Based on MSVC code contributed by Nirav Chhatrapati, Intel Corp., 1998. 13 * Interface to libpng contributed by Gilles Vollant, 1999. 14 * GNU C port by Greg Roelofs, 1999-2001. 15 * 16 * References: 17 * 18 * http://www.intel.com/drg/pentiumII/appnotes/916/916.htm 19 * http://www.intel.com/drg/pentiumII/appnotes/923/923.htm 20 * [Intel's performance analysis of the MMX vs. non-MMX code; 21 * moved/deleted as of 2006, but text and some graphs still 22 * available via WayBack Machine at archive.org] 23 * 24 * http://www.ibiblio.org/gferg/ldp/GCC-Inline-Assembly-HOWTO.html 25 * http://sam.zoy.org/blog/2007-04-13-shlib-with-non-pic-code-have-inline-assembly-and-pic-mix-well 26 * http://gcc.gnu.org/onlinedocs/gcc/Extended-Asm.html 27 * http://gcc.gnu.org/onlinedocs/gcc/Variable-Attributes.html 28 * http://gcc.gnu.org/onlinedocs/gcc/Function-Attributes.html 29 * AMD64 Architecture Programmer's Manual, volumes 1 and 5 30 * [http://www.amd.com/us-en/Processors/TechnicalResources/0,,30_182_739_7044,00.html] 31 * Intel 64 and IA-32 Software Developer's Manuals 32 * [http://developer.intel.com/products/processor/manuals/] 33 * 34 * png_read_filter_row_mmx_*() were converted in place with intel2gas 1.3.1: 35 * 36 * intel2gas -mdI pnggccrd.c.partially-msvc -o pnggccrd.c 37 * 38 * and then cleaned up by hand. See http://hermes.terminal.at/intel2gas/ . 39 * 40 * NOTE: A sufficiently recent version of GNU as (or as.exe under DOS/Windows) 41 * is required to assemble the newer asm instructions such as movq. (Version 42 * 2.5.2l.15 is definitely too old.) See ftp://ftp.gnu.org/pub/gnu/binutils/ . 43 */ 44 45 /* 46 * PORTING NOTES AND CHANGELOG (mostly by Greg Roelofs) 47 * =========================== 48 * 49 * 19991006: 50 * - fixed sign error in post-MMX cleanup code (16- & 32-bit cases) 51 * 52 * 19991007: 53 * - additional optimizations (possible or definite): 54 * x [DONE] write MMX code for 64-bit case (pixel_bytes == 8) [not tested] 55 * - write MMX code for 48-bit case (pixel_bytes == 6) 56 * - figure out what's up with 24-bit case (pixel_bytes == 3): 57 * why subtract 8 from width_mmx in the pass 4/5 case? 58 * (only width_mmx case) (near line 2335) 59 * x [DONE] replace pixel_bytes within each block with the true 60 * constant value (or are compilers smart enough to do that?) 61 * - rewrite all MMX interlacing code so it's aligned with 62 * the *beginning* of the row buffer, not the end. This 63 * would not only allow one to eliminate half of the memory 64 * writes for odd passes (that is, pass == odd), it may also 65 * eliminate some unaligned-data-access exceptions (assuming 66 * there's a penalty for not aligning 64-bit accesses on 67 * 64-bit boundaries). The only catch is that the "leftover" 68 * pixel(s) at the end of the row would have to be saved, 69 * but there are enough unused MMX registers in every case, 70 * so this is not a problem. A further benefit is that the 71 * post-MMX cleanup code (C code) in at least some of the 72 * cases could be done within the assembler block. 73 * x [DONE] the "v3 v2 v1 v0 v7 v6 v5 v4" comments are confusing, 74 * inconsistent, and don't match the MMX Programmer's Reference 75 * Manual conventions anyway. They should be changed to 76 * "b7 b6 b5 b4 b3 b2 b1 b0," where b0 indicates the byte that 77 * was lowest in memory (i.e., corresponding to a left pixel) 78 * and b7 is the byte that was highest (i.e., a right pixel). 79 * 80 * 19991016: 81 * - Brennan's Guide notwithstanding, gcc under Linux does *not* 82 * want globals prefixed by underscores when referencing them-- 83 * i.e., if the variable is const4, then refer to it as const4, 84 * not _const4. This seems to be a djgpp-specific requirement. 85 * Also, such variables apparently *must* be declared outside 86 * of functions; neither static nor automatic variables work if 87 * defined within the scope of a single function, but both 88 * static and truly global (multi-module) variables work fine. 89 * 90 * 19991017: 91 * - replaced pixel_bytes in each png_memcpy() call with constant value for 92 * inlining (png_do_read_interlace() "non-MMX/modified C code" block) 93 * 94 * 19991023: 95 * - fixed png_combine_row() non-MMX replication bug (odd passes only?) 96 * - switched from string-concatenation-with-macros to cleaner method of 97 * renaming global variables for djgpp--i.e., always use prefixes in 98 * inlined assembler code (== strings) and conditionally rename the 99 * variables, not the other way around. Hence _const4, _mask8_0, etc. 100 * 101 * 19991024: 102 * - fixed mmxsupport()/png_do_read_interlace() first-row bug 103 * This one was severely weird: even though mmxsupport() doesn't touch 104 * ebx (where "row" pointer was stored), it nevertheless managed to zero 105 * the register (even in static/non-fPIC code--see below), which in turn 106 * caused png_do_read_interlace() to return prematurely on the first row of 107 * interlaced images (i.e., without expanding the interlaced pixels). 108 * Inspection of the generated assembly code didn't turn up any clues, 109 * although it did point at a minor optimization (i.e., get rid of 110 * mmx_supported_local variable and just use eax). Possibly the CPUID 111 * instruction is more destructive than it looks? (Not yet checked.) 112 * - "info gcc" was next to useless, so compared fPIC and non-fPIC assembly 113 * listings... Apparently register spillage has to do with ebx, since 114 * it's used to index the global offset table. Commenting it out of the 115 * input-reg lists in png_combine_row() eliminated compiler barfage, so 116 * ifdef'd with __PIC__ macro: if defined, use a global for unmask 117 * 118 * 19991107: 119 * - verified CPUID clobberage: 12-char string constant ("GenuineIntel", 120 * "AuthenticAMD", etc.) placed in ebx:ecx:edx. Still need to polish. 121 * 122 * 19991120: 123 * - made "diff" variable (now "_dif") global to simplify conversion of 124 * filtering routines (running out of regs, sigh). "diff" is still used 125 * in interlacing routines, however. 126 * - fixed up both versions of mmxsupport() (ORIG_THAT_USED_TO_CLOBBER_EBX 127 * macro determines which is used); original not yet tested. 128 * 129 * 20000213: 130 * - when compiling with gcc, be sure to use -fomit-frame-pointer 131 * 132 * 20000319: 133 * - fixed a register-name typo in png_do_read_interlace(), default (MMX) case, 134 * pass == 4 or 5, that caused visible corruption of interlaced images 135 * 136 * 20000623: 137 * - Various problems were reported with gcc 2.95.2 in the Cygwin environment, 138 * many of the form "forbidden register 0 (ax) was spilled for class AREG." 139 * This is explained at http://gcc.gnu.org/fom_serv/cache/23.html, and 140 * Chuck Wilson supplied a patch involving dummy output registers. See 141 * http://sourceforge.net/bugs/?func=detailbug&bug_id=108741&group_id=5624 142 * for the original (anonymous) SourceForge bug report. 143 * 144 * 20000706: 145 * - Chuck Wilson passed along these remaining gcc 2.95.2 errors: 146 * pnggccrd.c: In function `png_combine_row': 147 * pnggccrd.c:525: more than 10 operands in `asm' 148 * pnggccrd.c:669: more than 10 operands in `asm' 149 * pnggccrd.c:828: more than 10 operands in `asm' 150 * pnggccrd.c:994: more than 10 operands in `asm' 151 * pnggccrd.c:1177: more than 10 operands in `asm' 152 * They are all the same problem and can be worked around by using the 153 * global _unmask variable unconditionally, not just in the -fPIC case. 154 * Reportedly earlier versions of gcc also have the problem with more than 155 * 10 operands; they just don't report it. Much strangeness ensues, etc. 156 * 157 * 20000729: 158 * - enabled png_read_filter_row_mmx_up() (shortest remaining unconverted 159 * MMX routine); began converting png_read_filter_row_mmx_sub() 160 * - to finish remaining sections: 161 * - clean up indentation and comments 162 * - preload local variables 163 * - add output and input regs (order of former determines numerical 164 * mapping of latter) 165 * - avoid all usage of ebx (including bx, bh, bl) register [20000823] 166 * - remove "$" from addressing of Shift and Mask variables [20000823] 167 * 168 * 20000731: 169 * - global union vars causing segfaults in png_read_filter_row_mmx_sub()? 170 * 171 * 20000822: 172 * - ARGH, stupid png_read_filter_row_mmx_sub() segfault only happens with 173 * shared-library (-fPIC) version! Code works just fine as part of static 174 * library. Should have tested that sooner. 175 * ebx is getting clobbered again (explicitly this time); need to save it 176 * on stack or rewrite asm code to avoid using it altogether. Blargh! 177 * 178 * 20000823: 179 * - first section was trickiest; all remaining sections have ebx -> edx now. 180 * (-fPIC works again.) Also added missing underscores to various Shift* 181 * and *Mask* globals and got rid of leading "$" signs. 182 * 183 * 20000826: 184 * - added visual separators to help navigate microscopic printed copies 185 * (http://pobox.com/~newt/code/gpr-latest.zip, mode 10); started working 186 * on png_read_filter_row_mmx_avg() 187 * 188 * 20000828: 189 * - finished png_read_filter_row_mmx_avg(): only Paeth left! (930 lines...) 190 * What the hell, did png_read_filter_row_mmx_paeth(), too. Comments not 191 * cleaned up/shortened in either routine, but functionality is complete 192 * and seems to be working fine. 193 * 194 * 20000829: 195 * - ahhh, figured out last(?) bit of gcc/gas asm-fu: if register is listed 196 * as an input reg (with dummy output variables, etc.), then it *cannot* 197 * also appear in the clobber list or gcc 2.95.2 will barf. The solution 198 * is simple enough... 199 * 200 * 20000914: 201 * - bug in png_read_filter_row_mmx_avg(): 16-bit grayscale not handled 202 * correctly (but 48-bit RGB just fine) 203 * 204 * 20000916: 205 * - fixed bug in png_read_filter_row_mmx_avg(), bpp == 2 case; three errors: 206 * - "_ShiftBpp.use = 24;" should have been "_ShiftBpp.use = 16;" 207 * - "_ShiftRem.use = 40;" should have been "_ShiftRem.use = 48;" 208 * - "psllq _ShiftRem, %%mm2" should have been "psrlq _ShiftRem, %%mm2" 209 * 210 * 20010101: 211 * - added new png_init_mmx_flags() function (here only because it needs to 212 * call mmxsupport(), which should probably become global png_mmxsupport()); 213 * modified other MMX routines to run conditionally (png_ptr->asm_flags) 214 * 215 * 20010103: 216 * - renamed mmxsupport() to png_mmx_support(), with auto-set of mmx_supported, 217 * and made it public; moved png_init_mmx_flags() to png.c as internal func 218 * 219 * 20010104: 220 * - removed dependency on png_read_filter_row_c() (C code already duplicated 221 * within MMX version of png_read_filter_row()) so no longer necessary to 222 * compile it into pngrutil.o 223 * 224 * 20010310: 225 * - fixed buffer-overrun bug in png_combine_row() C code (non-MMX) 226 * 227 * 20010808: 228 * - added PNG_THREAD_UNSAFE_OK around code using global variables [GR-P] 229 * 230 * 20011124: 231 * - fixed missing save of Eflag in png_mmx_support() [Maxim Sobolev] 232 * 233 * 20020304: 234 * - eliminated incorrect use of width_mmx in pixel_bytes == 8 case 235 * 236 * 20020407: 237 * - fixed insufficient preservation of ebx register [Sami Farin] 238 * 239 * 20040724: 240 * - more tinkering with clobber list at lines 4529 and 5033 to get it to 241 * compile with gcc 3.4 [GR-P] 242 * 243 * 20040809: 244 * - added "rim" definitions for CONST4 and CONST6 [GR-P] 245 * 246 * 20060303: 247 * - added "OS2" to list of systems that don't need leading underscores [GR-P] 248 * 249 * 20060320: 250 * - made PIC-compliant [Christian Aichinger] 251 * 252 * 20070313: 253 * - finally applied Giuseppe Ghib's 64-bit patch of 20060803 (completely 254 * overlooked Dylan Alex Simon's similar patch of 20060414, oops...) 255 * 256 * 20070524: 257 * - fixed link failure caused by asm-only variables being optimized out 258 * (identified by Dimitri of Trolltech) with __attribute__((used)), which 259 * also gets rid of warnings => nuked ugly png_squelch_warnings() hack 260 * - dropped redundant ifdef 261 * - moved png_mmx_support() back up where originally intended (as in 262 * pngvcrd.c), using __attribute__((noinline)) in extra prototype 263 * 264 * 20070527: 265 * - revised png_combine_row() to reuse mask in lieu of external _unmask 266 * - moved 32-bit (RGBA) case to top of png_combine_row(): most common 267 * - just about ready to give up on x86-64 -fPIC mode; can't even access 16 268 * _mask*_* constants without triggering link error on shared library: 269 * /usr/bin/ld: pnggccrd.pic.o: relocation R_X86_64_32S against `a local 270 * symbol' can not be used when making a shared object; recompile with 271 * -fPIC 272 * pnggccrd.pic.o: could not read symbols: Bad value 273 * ("objdump -x pnggccrd.pic.o | grep rodata" to verify) 274 * [might be able to work around by doing within assembly code whatever 275 * -fPIC does, but given problems to date, seems like long shot...] 276 * [relevant ifdefs: __x86_64__ && __PIC__ => C code only] 277 * - changed #if 0 to #ifdef PNG_CLOBBER_MMX_REGS_SUPPORTED in case gcc ever 278 * supports MMX regs (%mm0, etc.) in clobber list (not supported by gcc 279 * 2.7.2.3, 2.91.66 (egcs 1.1.2), 3.x, or 4.1.2) 280 * 281 * 20070603: 282 * - revised png_combine_row() to use @GOTPCREL(%%rip) addressing on _c64 283 * struct of _mask*_* constants for x86-64 -fPIC; see sam.zoy.org link 284 * above for details 285 * - moved _const4 and _const6 into _c64 struct, renamed to _amask5_3_0 and 286 * _amask7_1_0, respectively 287 * - can't figure out how to use _c64._mask*_* vars within asm code, so still 288 * need single variables for non-x86-64/-fPIC half :-( 289 * - replaced various __PIC__ ifdefs with *_GOT_ebx macros 290 * - moved _LBCarryMask and _HBClearMask into _c64 struct 291 * - conditionally replaced _p*temp variables with %r11d-%r13d (via p*_TEMP 292 * and CLOBBER_r1*d macros) 293 * 294 * 20070604: 295 * - replaced all _ActiveMask and _ActiveMaskEnd with new _amask*_*_* consts 296 * (_amask naming convention: numbers of 00-bytes, ff-bytes, 00-bytes) 297 * - _ActiveMask // (10) // avg/paeth/sub; read-only; consts; movq/pand 298 * 0x0000000000ffffffLL (bpp 3, avg) _amask5_3_0 299 * 0xffffffffffffffffLL (bpp 4, 6, avg) _amask0_8_0 300 * 0x000000000000ffffLL (bpp 2, avg) _amask6_2_0 301 * 0x0000000000ffffffLL (bpp 3, paeth) _amask5_3_0 302 * 0x00000000ffffffffLL (bpp 6, paeth) _amask4_4_0 303 * 0x00000000ffffffffLL (bpp 4, paeth) _amask4_4_0 304 * 0x00000000ffffffffLL (bpp 8, paeth) _amask4_4_0 305 * 0x0000ffffff000000LL (bpp 3, sub) _amask2_3_3 306 * 0x00000000ffff0000LL (bpp 2, sub) _amask4_2_2 307 * - _ActiveMaskEnd // (1) // paeth only; read-only; const; pand 308 * 0xffff000000000000LL (bpp 3, paeth) _amask0_2_6 309 * - changed all "#if defined(__x86_64__) // later // && defined(__PIC__)" 310 * lines to "#ifdef PNG_x86_64_USE_GOTPCREL" for easier/safer testing 311 * 312 * 20070605: 313 * - merged PNG_x86_64_USE_GOTPCREL, non-PNG_x86_64_USE_GOTPCREL code via 314 * *MASK* and LOAD/RESTORE macros 315 * 316 * 20070607: 317 * - replaced all constant instances of _ShiftBpp, _ShiftRem with immediates 318 * (still have two shared cases in avg, sub routines) 319 * 320 * 20070609: 321 * - replaced remaining instances of _ShiftBpp, _ShiftRem with immediates 322 * (split sub and avg 4/6-bpp cases into separate blocks) 323 * - fixed paeth bug due to clobbered r11/r12/r13 regs 324 * 325 * 20070610: 326 * - made global "_dif" variable (avg/paeth/sub routines) local again (now 327 * "diff"--see 19991120 entry above), using register constraints 328 * - note that %ebp in clobber list doesn't actually work, at least for 32-bit 329 * version and gcc 4.1.2; must save and restore manually. (Seems to work 330 * OK for 64-bit version and gcc 3.4.3, but gcc may not be using ebp/rbp 331 * in that case.) 332 * - started replacing direct _MMXLength accesses with register constraints 333 * 334 * 20070612: 335 * - continued replacing direct _MMXLength accesses with register constraints 336 * 337 * 20070613: 338 * - finished replacing direct _MMXLength accesses with register constraints; 339 * switched to local variable (and renamed back to MMXLength) 340 * 341 * 20070614: 342 * - fixed sub bpp = 1 bug 343 * - started replacing direct _FullLength accesses with register constraints 344 * 345 * 20070615: 346 * - fixed 64-bit paeth bpp 3 crash bug (misplaced LOAD_GOT_rbp) 347 * - fixed 64-bit paeth bpp 1/2 and cleanup-block crash bugs (misplaced 348 * RESTORE_r11_r12_r13) 349 * - slightly optimized avg/paeth cleanup blocks and paeth bpp 1/2 block 350 * (save/restore ebx only if needed) 351 * - continued replacing direct _FullLength accesses with register constraints 352 * 353 * 20070616: 354 * - finished replacing direct _FullLength accesses with register constraints 355 * (*ugly* conditional clobber-separator macros for avg and paeth, sigh) 356 * 357 * 20070618: 358 * - fixed misplaced PNG_THREAD_UNSAFE_OK endif (was missing LOAD_GOT_rbp/ 359 * RESTORE_rbp in 32-bit thread-safe case) 360 * - changed all "ifdef *" to "if defined(*)" [GR-P] 361 * 362 * 20070619: 363 * - rearranged most bitdepth-related case statements to put most frequent 364 * cases at top (24-bit, 32-bit, 8-bit, rest) 365 * 366 * 20070623: 367 * - cleaned up png_debug() warnings/formatting 368 * - removed PNG_MMX_CODE_SUPPORTED ifdefs and added outer __GNUC__ ifdef 369 * (module no longer used by non-x86/non-GCC builds as of libpng 1.2.19) 370 * - removed single libpng-1.2.x PNG_DEBUG dependency on 1.0.x png_struct 371 * member (row_buf_size) 372 * - rearranged pass-related if-blocks in png_do_read_interlace() to put most 373 * frequent cases (4, 5) at top [GR-P suggestion] 374 * 375 * 20070624-29: 376 * - fixed 64-bit crash bug: pointers -> rsi/rdi, not esi/edi (switched to 377 * %0/%1/%2/%3/%4 notation; eliminated size suffixes from relevant add/ 378 * inc/sub/mov instructions; changed dummy vars to pointers) 379 * - png_combine_row() 380 * - png_do_read_interlace() 381 * - png_read_filter_row_mmx_avg() 382 * - png_read_filter_row_mmx_paeth() 383 * - png_read_filter_row_mmx_sub() 384 * - png_read_filter_row_mmx_up() 385 * - NOTE: this fix makes use of the fact that modifying a 32-bit reg (e.g., 386 * %%ebx) clears the top half of its corresponding 64-bit reg (%%rbx), so 387 * it's safe to mix 32-bit operations with 64-bit base/index addressing 388 * (see new PSI/PAX/PBX/PDX/PBP/etc. "pointer-register" macros); applies 389 * also to clobber lists 390 * 391 * 20070630: 392 * - cleaned up formatting, macros, minor png_read_filter_row_mmx_sub() 8-bpp 393 * register-usage inefficiency 394 * - fixed 32-bit png_do_read_interlace() bug (was using pointer size for 395 * 64-bit dummy values) 396 * 397 * 20070703: 398 * - added check for (manual) PIC macro to fix OpenBSD crash bug 399 * 400 * 20070717: 401 * - fixed 48-bit png_combine_row() bug (was acting like 32-bit): copy 6 402 * bytes per pixel, not 4, and use stride of 6, not 4, in the second loop 403 * of interlace processing of 48-bit pixels [GR-P] 404 * 405 * 20070722: 406 * - fixed 64-bit png_uint_32 bug with MMXLength/FullLength temp vars 407 * 408 * [still broken: tops of all row-filter blocks (input/output constraints); 409 * shows up on 64-bit dynamic (-fPIC) version with -O2, especially if debug- 410 * printfs enabled, but at right edge of odd-width images even if disabled] 411 * 412 * 413 * STILL TO DO: 414 * - fix final thread-unsafe code using stack vars and pointer? (paeth top, 415 * default, bottom only: default, bottom already 5 reg constraints; could 416 * replace bpp with pointer and group bpp/patemp/pbtemp/pctemp in array) 417 * - fix ebp/no-reg-constraint inefficiency (avg/paeth/sub top) 418 * - test png_do_read_interlace() 64-bit case (pixel_bytes == 8) 419 * - write MMX code for 48-bit case (pixel_bytes == 6) 420 * - figure out what's up with 24-bit case (pixel_bytes == 3): 421 * why subtract 8 from width_mmx in the pass 4/5 case? due to 422 * odd number of bytes? (only width_mmx case) (near line 2335) 423 * - rewrite all MMX interlacing code so it's aligned with beginning 424 * of the row buffer, not the end (see 19991007 for details) 425 * - add error messages to any remaining bogus default cases 426 * - enable pixel_depth == 8 cases in png_read_filter_row()? (test speed) 427 * - try =r, etc., as reg constraints? (would gcc use 64-bit ones on x86-64?) 428 * - need full, non-graphical, CRC-based test suite... maybe autogenerate 429 * random data of various height/width/depth, compute CRCs, write (C 430 * funcs), read (asm/MMX), recompute CRCs, and compare? 431 * - write true x86-64 version using 128-bit "media instructions", %xmm0-15, 432 * and extra general-purpose registers 433 */ 434 435 #if defined(__GNUC__) 436 437 #define PNG_INTERNAL 438 #include "png.h" 439 440 441 /* for some inexplicable reason, gcc 3.3.5 on OpenBSD (and elsewhere?) does 442 * *not* define __PIC__ when the -fPIC option is used, so we have to rely on 443 * makefiles and whatnot to define the PIC macro explicitly */ 444 #if defined(PIC) && !defined(__PIC__) // (this can/should move to pngconf.h) 445 # define __PIC__ 446 #endif 447 448 #if defined(PNG_ASSEMBLER_CODE_SUPPORTED) && defined(PNG_USE_PNGGCCRD) 449 450 /* if you want/need full thread-safety on x86-64 even when linking statically, 451 * comment out the "&& defined(__PIC__)" part here: */ 452 #if defined(__x86_64__) && defined(__PIC__) 453 # define PNG_x86_64_USE_GOTPCREL // GOTPCREL => full thread-safety 454 # define PNG_CLOBBER_x86_64_REGS_SUPPORTED // works as of gcc 3.4.3 ... 455 #endif 456 457 int PNGAPI png_mmx_support(void); 458 459 #if defined(PNG_USE_LOCAL_ARRAYS) 460 static PNG_CONST int FARDATA png_pass_start[7] = {0, 4, 0, 2, 0, 1, 0}; 461 static PNG_CONST int FARDATA png_pass_inc[7] = {8, 8, 4, 4, 2, 2, 1}; 462 static PNG_CONST int FARDATA png_pass_width[7] = {8, 4, 4, 2, 2, 1, 1}; 463 #endif 464 465 /* djgpp, Win32, Cygwin, and OS2 add their own underscores to global variables, 466 * so define them without: */ 467 #if defined(__DJGPP__) || defined(WIN32) || defined(__CYGWIN__) || \ 468 defined(__OS2__) 469 # define _mmx_supported mmx_supported 470 # define _mask8_0 mask8_0 471 # define _mask16_1 mask16_1 472 # define _mask16_0 mask16_0 473 # define _mask24_2 mask24_2 474 # define _mask24_1 mask24_1 475 # define _mask24_0 mask24_0 476 # define _mask32_3 mask32_3 477 # define _mask32_2 mask32_2 478 # define _mask32_1 mask32_1 479 # define _mask32_0 mask32_0 480 # define _mask48_5 mask48_5 481 # define _mask48_4 mask48_4 482 # define _mask48_3 mask48_3 483 # define _mask48_2 mask48_2 484 # define _mask48_1 mask48_1 485 # define _mask48_0 mask48_0 486 # define _amask5_3_0 amask5_3_0 487 # define _amask7_1_0 amask7_1_0 488 # define _LBCarryMask LBCarryMask 489 # define _HBClearMask HBClearMask 490 # define _amask0_8_0 amask0_8_0 491 # define _amask6_2_0 amask6_2_0 492 # define _amask4_4_0 amask4_4_0 493 # define _amask0_2_6 amask0_2_6 494 # define _amask2_3_3 amask2_3_3 495 # define _amask4_2_2 amask4_2_2 496 # if defined(PNG_THREAD_UNSAFE_OK) 497 # define _patemp patemp 498 # define _pbtemp pbtemp 499 # define _pctemp pctemp 500 # endif 501 #endif // djgpp, Win32, Cygwin, OS2 502 503 504 /* These constants are used in the inlined MMX assembly code. */ 505 506 typedef unsigned long long ull; 507 508 #if defined(PNG_x86_64_USE_GOTPCREL) 509 static PNG_CONST struct { 510 //ull _mask_array[26]; 511 512 // png_combine_row() constants: 513 ull _mask8_0; 514 ull _mask16_0, _mask16_1; 515 ull _mask24_0, _mask24_1, _mask24_2; 516 ull _mask32_0, _mask32_1, _mask32_2, _mask32_3; 517 ull _mask48_0, _mask48_1, _mask48_2, _mask48_3, _mask48_4, _mask48_5; 518 519 // png_do_read_interlace() constants: 520 ull _amask5_3_0, _amask7_1_0; // was _const4 and _const6, respectively 521 522 // png_read_filter_row_mmx_avg() constants (also uses _amask5_3_0): 523 ull _LBCarryMask, _HBClearMask; 524 ull _amask0_8_0, _amask6_2_0; // was ActiveMask for bpp 4/6 and 2 cases 525 526 // png_read_filter_row_mmx_paeth() constants (also uses _amask5_3_0): 527 ull _amask4_4_0, _amask0_2_6; // was ActiveMask{,End} for bpp 6/4/8 and 3 528 529 // png_read_filter_row_mmx_sub() constants: 530 ull _amask2_3_3, _amask4_2_2; // was ActiveMask for bpp 3 and 2 cases 531 532 } _c64 __attribute__((used, aligned(8))) = { 533 534 // png_combine_row() constants: 535 0x0102040810204080LL, // _mask8_0 offset 0 536 537 0x1010202040408080LL, // _mask16_0 offset 8 538 0x0101020204040808LL, // _mask16_1 offset 16 539 540 0x2020404040808080LL, // _mask24_0 offset 24 541 0x0408080810101020LL, // _mask24_1 offset 32 542 0x0101010202020404LL, // _mask24_2 offset 40 543 544 0x4040404080808080LL, // _mask32_0 offset 48 545 0x1010101020202020LL, // _mask32_1 offset 56 546 0x0404040408080808LL, // _mask32_2 offset 64 547 0x0101010102020202LL, // _mask32_3 offset 72 548 549 0x4040808080808080LL, // _mask48_0 offset 80 550 0x2020202040404040LL, // _mask48_1 offset 88 551 0x1010101010102020LL, // _mask48_2 offset 96 552 0x0404080808080808LL, // _mask48_3 offset 104 553 0x0202020204040404LL, // _mask48_4 offset 112 554 0x0101010101010202LL, // _mask48_5 offset 120 555 556 // png_do_read_interlace() constants: 557 0x0000000000FFFFFFLL, // _amask5_3_0 offset 128 (bpp 3, avg/paeth) const4 558 0x00000000000000FFLL, // _amask7_1_0 offset 136 const6 559 560 // png_read_filter_row_mmx_avg() constants: 561 0x0101010101010101LL, // _LBCarryMask offset 144 562 0x7F7F7F7F7F7F7F7FLL, // _HBClearMask offset 152 563 0xFFFFFFFFFFFFFFFFLL, // _amask0_8_0 offset 160 (bpp 4/6, avg) 564 0x000000000000FFFFLL, // _amask6_2_0 offset 168 (bpp 2, avg) 565 566 // png_read_filter_row_mmx_paeth() constants: 567 0x00000000FFFFFFFFLL, // _amask4_4_0 offset 176 (bpp 6/4/8, paeth) 568 0xFFFF000000000000LL, // _amask0_2_6 offset 184 (bpp 3, paeth) A.M.End 569 570 // png_read_filter_row_mmx_sub() constants: 571 0x0000FFFFFF000000LL, // _amask2_3_3 offset 192 (bpp 3, sub) 572 0x00000000FFFF0000LL, // _amask4_2_2 offset 200 (bpp 2, sub) 573 574 }; 575 576 #define MASK8_0 "(%%rbp)" 577 #define MASK16_0 "8(%%rbp)" 578 #define MASK16_1 "16(%%rbp)" 579 #define MASK24_0 "24(%%rbp)" 580 #define MASK24_1 "32(%%rbp)" 581 #define MASK24_2 "40(%%rbp)" 582 #define MASK32_0 "48(%%rbp)" 583 #define MASK32_1 "56(%%rbp)" 584 #define MASK32_2 "64(%%rbp)" 585 #define MASK32_3 "72(%%rbp)" 586 #define MASK48_0 "80(%%rbp)" 587 #define MASK48_1 "88(%%rbp)" 588 #define MASK48_2 "96(%%rbp)" 589 #define MASK48_3 "104(%%rbp)" 590 #define MASK48_4 "112(%%rbp)" 591 #define MASK48_5 "120(%%rbp)" 592 #define AMASK5_3_0 "128(%%rbp)" 593 #define AMASK7_1_0 "136(%%rbp)" 594 #define LB_CARRY_MASK "144(%%rbp)" 595 #define HB_CLEAR_MASK "152(%%rbp)" 596 #define AMASK0_8_0 "160(%%rbp)" 597 #define AMASK6_2_0 "168(%%rbp)" 598 #define AMASK4_4_0 "176(%%rbp)" 599 #define AMASK0_2_6 "184(%%rbp)" 600 #define AMASK2_3_3 "192(%%rbp)" 601 #define AMASK4_2_2 "200(%%rbp)" 602 603 #else // !PNG_x86_64_USE_GOTPCREL 604 605 static PNG_CONST ull _mask8_0 __attribute__((used, aligned(8))) = 0x0102040810204080LL; 606 607 static PNG_CONST ull _mask16_1 __attribute__((used, aligned(8))) = 0x0101020204040808LL; 608 static PNG_CONST ull _mask16_0 __attribute__((used, aligned(8))) = 0x1010202040408080LL; 609 610 static PNG_CONST ull _mask24_2 __attribute__((used, aligned(8))) = 0x0101010202020404LL; 611 static PNG_CONST ull _mask24_1 __attribute__((used, aligned(8))) = 0x0408080810101020LL; 612 static PNG_CONST ull _mask24_0 __attribute__((used, aligned(8))) = 0x2020404040808080LL; 613 614 static PNG_CONST ull _mask32_3 __attribute__((used, aligned(8))) = 0x0101010102020202LL; 615 static PNG_CONST ull _mask32_2 __attribute__((used, aligned(8))) = 0x0404040408080808LL; 616 static PNG_CONST ull _mask32_1 __attribute__((used, aligned(8))) = 0x1010101020202020LL; 617 static PNG_CONST ull _mask32_0 __attribute__((used, aligned(8))) = 0x4040404080808080LL; 618 619 static PNG_CONST ull _mask48_5 __attribute__((used, aligned(8))) = 0x0101010101010202LL; 620 static PNG_CONST ull _mask48_4 __attribute__((used, aligned(8))) = 0x0202020204040404LL; 621 static PNG_CONST ull _mask48_3 __attribute__((used, aligned(8))) = 0x0404080808080808LL; 622 static PNG_CONST ull _mask48_2 __attribute__((used, aligned(8))) = 0x1010101010102020LL; 623 static PNG_CONST ull _mask48_1 __attribute__((used, aligned(8))) = 0x2020202040404040LL; 624 static PNG_CONST ull _mask48_0 __attribute__((used, aligned(8))) = 0x4040808080808080LL; 625 626 // png_do_read_interlace() constants: 627 static PNG_CONST ull _amask5_3_0 __attribute__((aligned(8))) = 0x0000000000FFFFFFLL; // was _const4 628 static PNG_CONST ull _amask7_1_0 __attribute__((aligned(8))) = 0x00000000000000FFLL; // was _const6 629 630 // png_read_filter_row_mmx_avg() constants: 631 static PNG_CONST ull _LBCarryMask __attribute__((used, aligned(8))) = 0x0101010101010101LL; 632 static PNG_CONST ull _HBClearMask __attribute__((used, aligned(8))) = 0x7f7f7f7f7f7f7f7fLL; 633 static PNG_CONST ull _amask0_8_0 __attribute__((used, aligned(8))) = 0xFFFFFFFFFFFFFFFFLL; 634 static PNG_CONST ull _amask6_2_0 __attribute__((used, aligned(8))) = 0x000000000000FFFFLL; 635 636 // png_read_filter_row_mmx_paeth() constants: 637 static PNG_CONST ull _amask4_4_0 __attribute__((used, aligned(8))) = 0x00000000FFFFFFFFLL; 638 static PNG_CONST ull _amask0_2_6 __attribute__((used, aligned(8))) = 0xFFFF000000000000LL; 639 640 // png_read_filter_row_mmx_sub() constants: 641 static PNG_CONST ull _amask2_3_3 __attribute__((used, aligned(8))) = 0x0000FFFFFF000000LL; 642 static PNG_CONST ull _amask4_2_2 __attribute__((used, aligned(8))) = 0x00000000FFFF0000LL; 643 644 #define MASK8_0 "_mask8_0" 645 #define MASK16_0 "_mask16_0" 646 #define MASK16_1 "_mask16_1" 647 #define MASK24_0 "_mask24_0" 648 #define MASK24_1 "_mask24_1" 649 #define MASK24_2 "_mask24_2" 650 #define MASK32_0 "_mask32_0" 651 #define MASK32_1 "_mask32_1" 652 #define MASK32_2 "_mask32_2" 653 #define MASK32_3 "_mask32_3" 654 #define MASK48_0 "_mask48_0" 655 #define MASK48_1 "_mask48_1" 656 #define MASK48_2 "_mask48_2" 657 #define MASK48_3 "_mask48_3" 658 #define MASK48_4 "_mask48_4" 659 #define MASK48_5 "_mask48_5" 660 #define AMASK5_3_0 "_amask5_3_0" 661 #define AMASK7_1_0 "_amask7_1_0" 662 #define LB_CARRY_MASK "_LBCarryMask" 663 #define HB_CLEAR_MASK "_HBClearMask" 664 #define AMASK0_8_0 "_amask0_8_0" 665 #define AMASK6_2_0 "_amask6_2_0" 666 #define AMASK4_4_0 "_amask4_4_0" 667 #define AMASK0_2_6 "_amask0_2_6" 668 #define AMASK2_3_3 "_amask2_3_3" 669 #define AMASK4_2_2 "_amask4_2_2" 670 671 #endif // ?PNG_x86_64_USE_GOTPCREL 672 673 674 #if defined(PNG_HAVE_MMX_READ_FILTER_ROW) || defined(PNG_HAVE_MMX_COMBINE_ROW) 675 676 // this block is specific to png_read_filter_row_mmx_paeth() except for 677 // LOAD_GOT_rbp and RESTORE_rbp, which are also used in png_combine_row() 678 #if defined(PNG_x86_64_USE_GOTPCREL) 679 # define pa_TEMP "%%r11d" 680 # define pb_TEMP "%%r12d" 681 # define pc_TEMP "%%r13d" 682 # if defined(PNG_CLOBBER_x86_64_REGS_SUPPORTED) // works as of gcc 3.4.3 ... 683 # define SAVE_r11_r12_r13 684 # define RESTORE_r11_r12_r13 685 # define _CLOBBER_r11_r12_r13 ,"%r11", "%r12", "%r13" 686 # define CLOBBER_r11_r12_r13 "%r11", "%r12", "%r13" 687 # else // !PNG_CLOBBER_x86_64_REGS_SUPPORTED 688 # define SAVE_r11_r12_r13 "pushq %%r11 \n\t" \ 689 "pushq %%r12 \n\t" \ 690 "pushq %%r13 \n\t" // "normally 0-extended" 691 # define RESTORE_r11_r12_r13 "popq %%r13 \n\t" \ 692 "popq %%r12 \n\t" \ 693 "popq %%r11 \n\t" 694 # define _CLOBBER_r11_r12_r13 695 # define CLOBBER_r11_r12_r13 696 # endif 697 # define LOAD_GOT_rbp "pushq %%rbp \n\t" \ 698 "movq _c64@GOTPCREL(%%rip), %%rbp \n\t" 699 # define RESTORE_rbp "popq %%rbp \n\t" 700 #else // 32-bit and/or non-PIC 701 # if defined(PNG_THREAD_UNSAFE_OK) 702 // These variables are used in png_read_filter_row_mmx_paeth() and would be 703 // local variables if not for gcc-inline-assembly addressing limitations 704 // (some apparently related to ELF format, others to CPU type). 705 // 706 // WARNING: Their presence defeats the thread-safety of libpng. 707 static int _patemp __attribute__((used)); 708 static int _pbtemp __attribute__((used)); 709 static int _pctemp __attribute__((used)); 710 # define pa_TEMP "_patemp" 711 # define pb_TEMP "_pbtemp" // temp variables for 712 # define pc_TEMP "_pctemp" // Paeth routine 713 # define SAVE_r11_r12_r13 714 # define RESTORE_r11_r12_r13 715 # define _CLOBBER_r11_r12_r13 // not using regs => not clobbering 716 # define CLOBBER_r11_r12_r13 717 # endif // PNG_THREAD_UNSAFE_OK 718 # define LOAD_GOT_rbp 719 # define RESTORE_rbp 720 #endif 721 722 #if defined(__x86_64__) 723 # define SAVE_ebp 724 # define RESTORE_ebp 725 # define _CLOBBER_ebp ,"%ebp" 726 # define CLOBBER_ebp "%ebp" 727 # define SAVE_FullLength "movl %%eax, %%r15d \n\t" 728 # define RESTORE_FullLength "movl %%r15d, " // may go into eax or ecx 729 # if defined(PNG_CLOBBER_x86_64_REGS_SUPPORTED) // works as of gcc 3.4.3 ... 730 # define SAVE_r15 731 # define RESTORE_r15 732 # define _CLOBBER_r15 ,"%r15" 733 # else 734 # define SAVE_r15 "pushq %%r15 \n\t" 735 # define RESTORE_r15 "popq %%r15 \n\t" 736 # define _CLOBBER_r15 737 # endif 738 # define PBP "%%rbp" // regs used for 64-bit 739 # define PAX "%%rax" // pointers or in 740 # define PBX "%%rbx" // combination with 741 # define PCX "%%rcx" // 64-bit pointer-regs 742 # define PDX "%%rdx" // (base/index pairs, 743 # define PSI "%%rsi" // add/sub/mov pairs) 744 # define CLEAR_BOTTOM_3_BITS "and $0xfffffffffffffff8, " 745 #else 746 # define SAVE_ebp "pushl %%ebp \n\t" // clobber list doesn't work 747 # define RESTORE_ebp "popl %%ebp \n\t" // for %ebp on 32-bit; not 748 # define _CLOBBER_ebp // clear why not 749 # define CLOBBER_ebp 750 # define SAVE_FullLength "pushl %%eax \n\t" 751 # define RESTORE_FullLength "popl " // eax (avg) or ecx (paeth) 752 # define SAVE_r15 753 # define RESTORE_r15 754 # define _CLOBBER_r15 755 # define PBP "%%ebp" // regs used for or in 756 # define PAX "%%eax" // combination with 757 # define PBX "%%ebx" // "normal," 32-bit 758 # define PCX "%%ecx" // pointers 759 # define PDX "%%edx" 760 # define PSI "%%esi" 761 # define CLEAR_BOTTOM_3_BITS "and $0xfffffff8, " 762 #endif 763 764 // CLOB_COMMA_ebx_ebp: need comma ONLY if both CLOBBER_ebp and CLOBBER_GOT_ebx 765 // have values, i.e., only if __x86_64__ AND !__PIC__ 766 #if defined(__x86_64__) && !defined(__PIC__) 767 # define CLOB_COMMA_ebx_ebp , // clobbering both ebp and ebx => need comma 768 #else 769 # define CLOB_COMMA_ebx_ebp 770 #endif 771 772 // CLOB_COMMA_ebX_r1X: need comma UNLESS both CLOBBER_ebp and CLOBBER_GOT_ebx 773 // are empty OR CLOBBER_r11_r12_r13 is empty--i.e., NO comma 774 // if (!__x86_64__ AND __PIC__) OR !(PNG_x86_64_USE_GOTPCREL 775 // AND PNG_CLOBBER_x86_64_REGS_SUPPORTED) (double sigh...) 776 #if (!defined(__x86_64__) && defined(__PIC__)) || \ 777 !defined(PNG_x86_64_USE_GOTPCREL) || \ 778 !defined(PNG_CLOBBER_x86_64_REGS_SUPPORTED) 779 # define CLOB_COMMA_ebX_r1X 780 #else 781 # define CLOB_COMMA_ebX_r1X , // clobbering (ebp OR ebx) AND r11_r12_r13 782 #endif 783 784 // CLOB_COLON_ebx_ebp: need colon unless CLOBBER_ebp and CLOBBER_GOT_ebx are 785 // BOTH empty--i.e., NO colon if (!__x86_64__ AND __PIC__) 786 // CLOB_COLON_ebx_ebp_r1X: if, in addition, CLOBBER_r11_r12_r13 is empty, then 787 // no colon for Paeth blocks, either--i.e., NO colon 788 // if !(PNG_x86_64_USE_GOTPCREL AND 789 // PNG_CLOBBER_x86_64_REGS_SUPPORTED) 790 #if (!defined(__x86_64__) && defined(__PIC__)) 791 # define CLOB_COLON_ebx_ebp 792 # if !(defined(PNG_x86_64_USE_GOTPCREL) && \ 793 defined(PNG_CLOBBER_x86_64_REGS_SUPPORTED)) 794 # define CLOB_COLON_ebx_ebp_r1X 795 # else 796 # define CLOB_COLON_ebx_ebp_r1X : // clobbering ebp OR ebx OR r11_r12_r13 797 # endif 798 #else 799 # define CLOB_COLON_ebx_ebp : // clobbering ebp OR ebx 800 # define CLOB_COLON_ebx_ebp_r1X : // clobbering ebp OR ebx OR r11_r12_r13 801 #endif 802 803 #endif // PNG_HAVE_MMX_READ_FILTER_ROW 804 805 #if defined(__PIC__) // macros to save, restore index to Global Offset Table 806 # if defined(__x86_64__) 807 # define SAVE_GOT_ebx "pushq %%rbx \n\t" 808 # define RESTORE_GOT_ebx "popq %%rbx \n\t" 809 # else 810 # define SAVE_GOT_ebx "pushl %%ebx \n\t" 811 # define RESTORE_GOT_ebx "popl %%ebx \n\t" 812 # endif 813 # define _CLOBBER_GOT_ebx // explicitly saved, restored => not clobbered 814 # define CLOBBER_GOT_ebx 815 #else 816 # define SAVE_GOT_ebx 817 # define RESTORE_GOT_ebx 818 # define _CLOBBER_GOT_ebx ,"%ebx" 819 # define CLOBBER_GOT_ebx "%ebx" 820 #endif 821 822 #if defined(PNG_HAVE_MMX_COMBINE_ROW) || defined(PNG_HAVE_MMX_READ_INTERLACE) 823 # define BPP2 2 824 # define BPP3 3 // bytes per pixel (a.k.a. pixel_bytes) 825 # define BPP4 4 // (defined only to help avoid cut-and-paste errors) 826 # define BPP6 6 827 # define BPP8 8 828 #endif 829 830 831 832 static int _mmx_supported = 2; // 0: no MMX; 1: MMX supported; 2: not tested 833 834 /*===========================================================================*/ 835 /* */ 836 /* P N G _ M M X _ S U P P O R T */ 837 /* */ 838 /*===========================================================================*/ 839 840 // GRR NOTES: (1) the following code assumes 386 or better (pushfl/popfl) 841 // (2) all instructions compile with gcc 2.7.2.3 and later 842 // x (3) the function is moved down here to prevent gcc from 843 // x inlining it in multiple places and then barfing be- 844 // x cause the ".NOT_SUPPORTED" label is multiply defined 845 // [need to retest with gcc 2.7.2.3] 846 847 // GRR 20070524: This declaration apparently is compatible with but supersedes 848 // the one in png.h; in any case, the generated object file is slightly 849 // smaller. It is unnecessary with gcc 4.1.2, but gcc 2.x apparently 850 // replicated the ".NOT_SUPPORTED" label in each location the function was 851 // inlined, leading to compilation errors due to the "multiply defined" 852 // label. Old workaround was to leave the function at the end of this 853 // file; new one (still testing) is to use a gcc-specific function attribute 854 // to prevent local inlining. 855 int PNGAPI 856 png_mmx_support(void) __attribute__((noinline)); 857 858 int PNGAPI 859 png_mmx_support(void) 860 { 861 #if defined(PNG_MMX_CODE_SUPPORTED) // superfluous, but what the heck 862 int result; 863 __asm__ __volatile__ ( 864 #if defined(__x86_64__) 865 "pushq %%rbx \n\t" // rbx gets clobbered by CPUID instruction 866 "pushq %%rcx \n\t" // so does rcx... 867 "pushq %%rdx \n\t" // ...and rdx (but rcx & rdx safe on Linux) 868 "pushfq \n\t" // save Eflag to stack 869 "popq %%rax \n\t" // get Eflag from stack into rax 870 "movq %%rax, %%rcx \n\t" // make another copy of Eflag in rcx 871 "xorl $0x200000, %%eax \n\t" // toggle ID bit in Eflag (i.e., bit 21) 872 "pushq %%rax \n\t" // save modified Eflag back to stack 873 "popfq \n\t" // restore modified value to Eflag reg 874 "pushfq \n\t" // save Eflag to stack 875 "popq %%rax \n\t" // get Eflag from stack 876 "pushq %%rcx \n\t" // save original Eflag to stack 877 "popfq \n\t" // restore original Eflag 878 #else 879 "pushl %%ebx \n\t" // ebx gets clobbered by CPUID instruction 880 "pushl %%ecx \n\t" // so does ecx... 881 "pushl %%edx \n\t" // ...and edx (but ecx & edx safe on Linux) 882 "pushfl \n\t" // save Eflag to stack 883 "popl %%eax \n\t" // get Eflag from stack into eax 884 "movl %%eax, %%ecx \n\t" // make another copy of Eflag in ecx 885 "xorl $0x200000, %%eax \n\t" // toggle ID bit in Eflag (i.e., bit 21) 886 "pushl %%eax \n\t" // save modified Eflag back to stack 887 "popfl \n\t" // restore modified value to Eflag reg 888 "pushfl \n\t" // save Eflag to stack 889 "popl %%eax \n\t" // get Eflag from stack 890 "pushl %%ecx \n\t" // save original Eflag to stack 891 "popfl \n\t" // restore original Eflag 892 #endif 893 "xorl %%ecx, %%eax \n\t" // compare new Eflag with original Eflag 894 "jz 0f \n\t" // if same, CPUID instr. is not supported 895 896 "xorl %%eax, %%eax \n\t" // set eax to zero 897 // ".byte 0x0f, 0xa2 \n\t" // CPUID instruction (two-byte opcode) 898 "cpuid \n\t" // get the CPU identification info 899 "cmpl $1, %%eax \n\t" // make sure eax return non-zero value 900 "jl 0f \n\t" // if eax is zero, MMX is not supported 901 902 "xorl %%eax, %%eax \n\t" // set eax to zero and... 903 "incl %%eax \n\t" // ...increment eax to 1. This pair is 904 // faster than the instruction "mov eax, 1" 905 "cpuid \n\t" // get the CPU identification info again 906 "andl $0x800000, %%edx \n\t" // mask out all bits but MMX bit (23) 907 "cmpl $0, %%edx \n\t" // 0 = MMX not supported 908 "jz 0f \n\t" // non-zero = yes, MMX IS supported 909 910 "movl $1, %%eax \n\t" // set return value to 1 911 "jmp 1f \n\t" // DONE: have MMX support 912 913 "0: \n\t" // .NOT_SUPPORTED: target label for jump instructions 914 "movl $0, %%eax \n\t" // set return value to 0 915 "1: \n\t" // .RETURN: target label for jump instructions 916 #if defined(__x86_64__) 917 "popq %%rdx \n\t" // restore rdx 918 "popq %%rcx \n\t" // restore rcx 919 "popq %%rbx \n\t" // restore rbx 920 #else 921 "popl %%edx \n\t" // restore edx 922 "popl %%ecx \n\t" // restore ecx 923 "popl %%ebx \n\t" // restore ebx 924 #endif 925 926 // "ret \n\t" // DONE: no MMX support 927 // (fall through to standard C "ret") 928 929 : "=a" (result) // output list 930 931 : // any variables used on input (none) 932 933 // no clobber list 934 // , "%ebx", "%ecx", "%edx" // GRR: we handle these manually 935 // , "memory" // if write to a variable gcc thought was in a reg 936 // , "cc" // "condition codes" (flag bits) 937 ); 938 _mmx_supported = result; 939 #else 940 _mmx_supported = 0; 941 #endif /* PNG_MMX_CODE_SUPPORTED */ 942 943 return _mmx_supported; 944 } 945 946 947 /*===========================================================================*/ 948 /* */ 949 /* P N G _ C O M B I N E _ R O W */ 950 /* */ 951 /*===========================================================================*/ 952 953 #if defined(PNG_HAVE_MMX_COMBINE_ROW) 954 955 /* Combines the row recently read in with the previous row. 956 This routine takes care of alpha and transparency if requested. 957 This routine also handles the two methods of progressive display 958 of interlaced images, depending on the mask value. 959 The mask value describes which pixels are to be combined with 960 the row. The pattern always repeats every 8 pixels, so just 8 961 bits are needed. A one indicates the pixel is to be combined; a 962 zero indicates the pixel is to be skipped. This is in addition 963 to any alpha or transparency value associated with the pixel. 964 If you want all pixels to be combined, pass 0xff (255) in mask. */ 965 966 /* Use this routine for the x86 platform - it uses a faster MMX routine 967 if the machine supports MMX. */ 968 969 void /* PRIVATE */ 970 png_combine_row(png_structp png_ptr, png_bytep row, int mask) 971 { 972 int dummy_value_a; // fix 'forbidden register spilled' error 973 int dummy_value_c; 974 int dummy_value_d; 975 png_bytep dummy_value_S; 976 png_bytep dummy_value_D; 977 978 png_debug(1, "in png_combine_row (pnggccrd.c)\n"); 979 980 if (_mmx_supported == 2) { 981 #if !defined(PNG_1_0_X) 982 /* this should have happened in png_init_mmx_flags() already */ 983 png_warning(png_ptr, "asm_flags may not have been initialized"); 984 #endif 985 png_mmx_support(); 986 } 987 988 if (mask == 0xff) 989 { 990 png_debug(2,"mask == 0xff: doing single png_memcpy()\n"); 991 png_memcpy(row, png_ptr->row_buf + 1, 992 (png_size_t)PNG_ROWBYTES(png_ptr->row_info.pixel_depth,png_ptr->width)); 993 } 994 else /* (png_combine_row() is never called with mask == 0) */ 995 { 996 switch (png_ptr->row_info.pixel_depth) 997 { 998 case 24: /* png_ptr->row_info.pixel_depth */ 999 { 1000 png_bytep srcptr; 1001 png_bytep dstptr; 1002 1003 #if !defined(PNG_1_0_X) 1004 if (png_ptr->asm_flags & PNG_ASM_FLAG_MMX_READ_COMBINE_ROW) 1005 #else 1006 if (_mmx_supported) 1007 #endif 1008 { 1009 png_uint_32 len; 1010 int diff; 1011 1012 srcptr = png_ptr->row_buf + 1; 1013 dstptr = row; 1014 len = png_ptr->width & ~7; // reduce to multiple of 8 1015 diff = (int) (png_ptr->width & 7); // amount lost 1016 1017 __asm__ __volatile__ ( 1018 "not %%edx \n\t" // mask => unmask 1019 "movd %%edx, %%mm7 \n\t" // load bit pattern 1020 "not %%edx \n\t" // unmask => mask for later 1021 "psubb %%mm6, %%mm6 \n\t" // zero mm6 1022 "punpcklbw %%mm7, %%mm7 \n\t" 1023 "punpcklwd %%mm7, %%mm7 \n\t" 1024 "punpckldq %%mm7, %%mm7 \n\t" // fill reg with 8 masks 1025 1026 LOAD_GOT_rbp 1027 "movq " MASK24_0 ", %%mm0 \n\t" // _mask24_0 -> mm0 1028 "movq " MASK24_1 ", %%mm1 \n\t" // _mask24_1 -> mm1 1029 "movq " MASK24_2 ", %%mm2 \n\t" // _mask24_2 -> mm2 1030 RESTORE_rbp 1031 1032 "pand %%mm7, %%mm0 \n\t" 1033 "pand %%mm7, %%mm1 \n\t" 1034 "pand %%mm7, %%mm2 \n\t" 1035 1036 "pcmpeqb %%mm6, %%mm0 \n\t" 1037 "pcmpeqb %%mm6, %%mm1 \n\t" 1038 "pcmpeqb %%mm6, %%mm2 \n\t" 1039 1040 // preload "movl len, %%ecx \n\t" // load length of line 1041 // preload "movl srcptr, %3 \n\t" // load source 1042 // preload "movl dstptr, %4 \n\t" // load dest 1043 1044 "cmpl $0, %%ecx \n\t" 1045 "jz mainloop24end \n\t" 1046 1047 "mainloop24: \n\t" 1048 "movq (%3), %%mm4 \n\t" 1049 "pand %%mm0, %%mm4 \n\t" 1050 "movq %%mm0, %%mm6 \n\t" 1051 "movq (%4), %%mm7 \n\t" 1052 "pandn %%mm7, %%mm6 \n\t" 1053 "por %%mm6, %%mm4 \n\t" 1054 "movq %%mm4, (%4) \n\t" 1055 1056 "movq 8(%3), %%mm5 \n\t" 1057 "pand %%mm1, %%mm5 \n\t" 1058 "movq %%mm1, %%mm7 \n\t" 1059 "movq 8(%4), %%mm6 \n\t" 1060 "pandn %%mm6, %%mm7 \n\t" 1061 "por %%mm7, %%mm5 \n\t" 1062 "movq %%mm5, 8(%4) \n\t" 1063 1064 "movq 16(%3), %%mm6 \n\t" 1065 "pand %%mm2, %%mm6 \n\t" 1066 "movq %%mm2, %%mm4 \n\t" 1067 "movq 16(%4), %%mm7 \n\t" 1068 "pandn %%mm7, %%mm4 \n\t" 1069 "por %%mm4, %%mm6 \n\t" 1070 "movq %%mm6, 16(%4) \n\t" 1071 1072 "add $24, %3 \n\t" // inc by 24 bytes processed 1073 "add $24, %4 \n\t" 1074 "subl $8, %%ecx \n\t" // dec by 8 pixels processed 1075 1076 "ja mainloop24 \n\t" 1077 1078 "mainloop24end: \n\t" 1079 // preload "movl diff, %%ecx \n\t" // (diff is in eax) 1080 "movl %%eax, %%ecx \n\t" 1081 "cmpl $0, %%ecx \n\t" 1082 "jz end24 \n\t" 1083 // preload "movl mask, %%edx \n\t" 1084 "sall $24, %%edx \n\t" // make low byte, high byte 1085 1086 "secondloop24: \n\t" 1087 "sall %%edx \n\t" // move high bit to CF 1088 "jnc skip24 \n\t" // if CF = 0 1089 "movw (%3), %%ax \n\t" 1090 "movw %%ax, (%4) \n\t" 1091 "xorl %%eax, %%eax \n\t" 1092 "movb 2(%3), %%al \n\t" 1093 "movb %%al, 2(%4) \n\t" 1094 1095 "skip24: \n\t" 1096 "add $3, %3 \n\t" 1097 "add $3, %4 \n\t" 1098 "decl %%ecx \n\t" 1099 "jnz secondloop24 \n\t" 1100 1101 "end24: \n\t" 1102 "EMMS \n\t" // DONE 1103 1104 : "=a" (dummy_value_a), // output regs (dummy) 1105 "=d" (dummy_value_d), 1106 "=c" (dummy_value_c), 1107 "=S" (dummy_value_S), 1108 "=D" (dummy_value_D) 1109 1110 : "0" (diff), // eax // input regs 1111 "1" (mask), // edx 1112 "2" (len), // ecx 1113 // was (unmask) "b" RESERVED // ebx // Global Offset Table idx 1114 "3" (srcptr), // esi/rsi 1115 "4" (dstptr) // edi/rdi 1116 1117 #if defined(PNG_CLOBBER_MMX_REGS_SUPPORTED) 1118 : "%mm0", "%mm1", "%mm2" // clobber list 1119 , "%mm4", "%mm5", "%mm6", "%mm7" 1120 #endif 1121 ); 1122 } 1123 else /* not _mmx_supported - use modified C routine */ 1124 { 1125 register png_uint_32 i; 1126 png_uint_32 initial_val = BPP3 * png_pass_start[png_ptr->pass]; 1127 /* png.c: png_pass_start[] = {0, 4, 0, 2, 0, 1, 0}; */ 1128 register int stride = BPP3 * png_pass_inc[png_ptr->pass]; 1129 /* png.c: png_pass_inc[] = {8, 8, 4, 4, 2, 2, 1}; */ 1130 register int rep_bytes = BPP3 * png_pass_width[png_ptr->pass]; 1131 /* png.c: png_pass_width[] = {8, 4, 4, 2, 2, 1, 1}; */ 1132 png_uint_32 len = png_ptr->width &~7; /* reduce to mult. of 8 */ 1133 int diff = (int) (png_ptr->width & 7); /* amount lost */ 1134 register png_uint_32 final_val = BPP3 * len; /* GRR bugfix */ 1135 1136 srcptr = png_ptr->row_buf + 1 + initial_val; 1137 dstptr = row + initial_val; 1138 1139 for (i = initial_val; i < final_val; i += stride) 1140 { 1141 png_memcpy(dstptr, srcptr, rep_bytes); 1142 srcptr += stride; 1143 dstptr += stride; 1144 } 1145 if (diff) /* number of leftover pixels: 3 for pngtest */ 1146 { 1147 final_val += diff*BPP3; 1148 for (; i < final_val; i += stride) 1149 { 1150 if (rep_bytes > (int)(final_val-i)) 1151 rep_bytes = (int)(final_val-i); 1152 png_memcpy(dstptr, srcptr, rep_bytes); 1153 srcptr += stride; 1154 dstptr += stride; 1155 } 1156 } 1157 } /* end of else (_mmx_supported) */ 1158 1159 break; 1160 } /* end 24 bpp */ 1161 1162 // formerly claimed to be most common case (combining 32-bit RGBA), 1163 // but almost certainly less common than 24-bit RGB case 1164 case 32: /* png_ptr->row_info.pixel_depth */ 1165 { 1166 png_bytep srcptr; 1167 png_bytep dstptr; 1168 1169 #if !defined(PNG_1_0_X) 1170 if (png_ptr->asm_flags & PNG_ASM_FLAG_MMX_READ_COMBINE_ROW) 1171 #else 1172 if (_mmx_supported) 1173 #endif 1174 { 1175 png_uint_32 len; 1176 int diff; 1177 1178 srcptr = png_ptr->row_buf + 1; 1179 dstptr = row; 1180 len = png_ptr->width & ~7; // reduce to multiple of 8 1181 diff = (int) (png_ptr->width & 7); // amount lost 1182 1183 __asm__ __volatile__ ( 1184 "not %%edx \n\t" // mask => unmask 1185 "movd %%edx, %%mm7 \n\t" // load bit pattern 1186 "not %%edx \n\t" // unmask => mask for later 1187 "psubb %%mm6, %%mm6 \n\t" // zero mm6 1188 "punpcklbw %%mm7, %%mm7 \n\t" 1189 "punpcklwd %%mm7, %%mm7 \n\t" 1190 "punpckldq %%mm7, %%mm7 \n\t" // fill reg with 8 masks 1191 1192 LOAD_GOT_rbp 1193 "movq " MASK32_0 ", %%mm0 \n\t" // _mask32_0 1194 "movq " MASK32_1 ", %%mm1 \n\t" // _mask32_1 1195 "movq " MASK32_2 ", %%mm2 \n\t" // _mask32_2 1196 "movq " MASK32_3 ", %%mm3 \n\t" // _mask32_3 1197 RESTORE_rbp 1198 1199 "pand %%mm7, %%mm0 \n\t" 1200 "pand %%mm7, %%mm1 \n\t" 1201 "pand %%mm7, %%mm2 \n\t" 1202 "pand %%mm7, %%mm3 \n\t" 1203 1204 "pcmpeqb %%mm6, %%mm0 \n\t" 1205 "pcmpeqb %%mm6, %%mm1 \n\t" 1206 "pcmpeqb %%mm6, %%mm2 \n\t" 1207 "pcmpeqb %%mm6, %%mm3 \n\t" 1208 1209 // preload "movl len, %%ecx \n\t" // load length of line 1210 // preload "movl srcptr, %3 \n\t" // load source 1211 // preload "movl dstptr, %4 \n\t" // load dest 1212 1213 "cmpl $0, %%ecx \n\t" // lcr 1214 "jz mainloop32end \n\t" 1215 1216 "mainloop32: \n\t" 1217 "movq (%3), %%mm4 \n\t" 1218 "pand %%mm0, %%mm4 \n\t" 1219 "movq %%mm0, %%mm6 \n\t" 1220 "movq (%4), %%mm7 \n\t" 1221 "pandn %%mm7, %%mm6 \n\t" 1222 "por %%mm6, %%mm4 \n\t" 1223 "movq %%mm4, (%4) \n\t" 1224 1225 "movq 8(%3), %%mm5 \n\t" 1226 "pand %%mm1, %%mm5 \n\t" 1227 "movq %%mm1, %%mm7 \n\t" 1228 "movq 8(%4), %%mm6 \n\t" 1229 "pandn %%mm6, %%mm7 \n\t" 1230 "por %%mm7, %%mm5 \n\t" 1231 "movq %%mm5, 8(%4) \n\t" 1232 1233 "movq 16(%3), %%mm6 \n\t" 1234 "pand %%mm2, %%mm6 \n\t" 1235 "movq %%mm2, %%mm4 \n\t" 1236 "movq 16(%4), %%mm7 \n\t" 1237 "pandn %%mm7, %%mm4 \n\t" 1238 "por %%mm4, %%mm6 \n\t" 1239 "movq %%mm6, 16(%4) \n\t" 1240 1241 "movq 24(%3), %%mm7 \n\t" 1242 "pand %%mm3, %%mm7 \n\t" 1243 "movq %%mm3, %%mm5 \n\t" 1244 "movq 24(%4), %%mm4 \n\t" 1245 "pandn %%mm4, %%mm5 \n\t" 1246 "por %%mm5, %%mm7 \n\t" 1247 "movq %%mm7, 24(%4) \n\t" 1248 1249 "add $32, %3 \n\t" // inc by 32 bytes processed 1250 "add $32, %4 \n\t" 1251 "subl $8, %%ecx \n\t" // dec by 8 pixels processed 1252 "ja mainloop32 \n\t" 1253 1254 "mainloop32end: \n\t" 1255 // preload "movl diff, %%ecx \n\t" // (diff is in eax) 1256 "movl %%eax, %%ecx \n\t" 1257 "cmpl $0, %%ecx \n\t" 1258 "jz end32 \n\t" 1259 // preload "movl mask, %%edx \n\t" 1260 "sall $24, %%edx \n\t" // low byte => high byte 1261 1262 "secondloop32: \n\t" 1263 "sall %%edx \n\t" // move high bit to CF 1264 "jnc skip32 \n\t" // if CF = 0 1265 "movl (%3), %%eax \n\t" 1266 "movl %%eax, (%4) \n\t" 1267 1268 "skip32: \n\t" 1269 "add $4, %3 \n\t" 1270 "add $4, %4 \n\t" 1271 "decl %%ecx \n\t" 1272 "jnz secondloop32 \n\t" 1273 1274 "end32: \n\t" 1275 "EMMS \n\t" // DONE 1276 1277 : "=a" (dummy_value_a), // output regs (dummy) 1278 "=d" (dummy_value_d), 1279 "=c" (dummy_value_c), 1280 "=S" (dummy_value_S), 1281 "=D" (dummy_value_D) 1282 1283 : "0" (diff), // eax // input regs 1284 "1" (mask), // edx 1285 "2" (len), // ecx 1286 // was (unmask) "b" RESERVED // ebx // Global Offset Table idx 1287 "3" (srcptr), // esi/rsi 1288 "4" (dstptr) // edi/rdi 1289 1290 #if defined(PNG_CLOBBER_MMX_REGS_SUPPORTED) 1291 : "%mm0", "%mm1", "%mm2", "%mm3" // clobber list 1292 , "%mm4", "%mm5", "%mm6", "%mm7" 1293 #endif 1294 ); 1295 } 1296 else /* not _mmx_supported - use modified C routine */ 1297 { 1298 register png_uint_32 i; 1299 png_uint_32 initial_val = BPP4 * png_pass_start[png_ptr->pass]; 1300 /* png.c: png_pass_start[] = {0, 4, 0, 2, 0, 1, 0}; */ 1301 register int stride = BPP4 * png_pass_inc[png_ptr->pass]; 1302 /* png.c: png_pass_inc[] = {8, 8, 4, 4, 2, 2, 1}; */ 1303 register int rep_bytes = BPP4 * png_pass_width[png_ptr->pass]; 1304 /* png.c: png_pass_width[] = {8, 4, 4, 2, 2, 1, 1}; */ 1305 png_uint_32 len = png_ptr->width &~7; /* reduce to mult. of 8 */ 1306 int diff = (int) (png_ptr->width & 7); /* amount lost */ 1307 register png_uint_32 final_val = BPP4 * len; /* GRR bugfix */ 1308 1309 srcptr = png_ptr->row_buf + 1 + initial_val; 1310 dstptr = row + initial_val; 1311 1312 for (i = initial_val; i < final_val; i += stride) 1313 { 1314 png_memcpy(dstptr, srcptr, rep_bytes); 1315 srcptr += stride; 1316 dstptr += stride; 1317 } 1318 if (diff) /* number of leftover pixels: 3 for pngtest */ 1319 { 1320 final_val += diff*BPP4; 1321 for (; i < final_val; i += stride) 1322 { 1323 if (rep_bytes > (int)(final_val-i)) 1324 rep_bytes = (int)(final_val-i); 1325 png_memcpy(dstptr, srcptr, rep_bytes); 1326 srcptr += stride; 1327 dstptr += stride; 1328 } 1329 } 1330 } /* end of else (_mmx_supported) */ 1331 1332 break; 1333 } /* end 32 bpp */ 1334 1335 case 8: /* png_ptr->row_info.pixel_depth */ 1336 { 1337 png_bytep srcptr; 1338 png_bytep dstptr; 1339 1340 #if !defined(PNG_1_0_X) 1341 if (png_ptr->asm_flags & PNG_ASM_FLAG_MMX_READ_COMBINE_ROW) 1342 #else 1343 if (_mmx_supported) 1344 #endif 1345 { 1346 png_uint_32 len; 1347 int diff; 1348 1349 srcptr = png_ptr->row_buf + 1; 1350 dstptr = row; 1351 len = png_ptr->width & ~7; // reduce to multiple of 8 1352 diff = (int) (png_ptr->width & 7); // amount lost 1353 1354 __asm__ __volatile__ ( 1355 "not %%edx \n\t" // mask => unmask 1356 "movd %%edx, %%mm7 \n\t" // load bit pattern 1357 "not %%edx \n\t" // unmask => mask for later 1358 "psubb %%mm6, %%mm6 \n\t" // zero mm6 1359 "punpcklbw %%mm7, %%mm7 \n\t" 1360 "punpcklwd %%mm7, %%mm7 \n\t" 1361 "punpckldq %%mm7, %%mm7 \n\t" // fill reg with 8 masks 1362 1363 LOAD_GOT_rbp 1364 "movq " MASK8_0 ", %%mm0 \n\t" // _mask8_0 -> mm0 1365 RESTORE_rbp 1366 1367 "pand %%mm7, %%mm0 \n\t" // nonzero if keep byte 1368 "pcmpeqb %%mm6, %%mm0 \n\t" // zeros->1s, v versa 1369 1370 // preload "movl len, %%ecx \n\t" // load length of line 1371 // preload "movl srcptr, %3 \n\t" // load source 1372 // preload "movl dstptr, %4 \n\t" // load dest 1373 1374 "cmpl $0, %%ecx \n\t" // len == 0 ? 1375 "je mainloop8end \n\t" 1376 1377 "mainloop8: \n\t" 1378 "movq (%3), %%mm4 \n\t" // *srcptr 1379 "pand %%mm0, %%mm4 \n\t" 1380 "movq %%mm0, %%mm6 \n\t" 1381 "pandn (%4), %%mm6 \n\t" // *dstptr 1382 "por %%mm6, %%mm4 \n\t" 1383 "movq %%mm4, (%4) \n\t" 1384 "add $8, %3 \n\t" // inc by 8 bytes processed 1385 "add $8, %4 \n\t" 1386 "subl $8, %%ecx \n\t" // dec by 8 pixels processed 1387 "ja mainloop8 \n\t" 1388 1389 "mainloop8end: \n\t" 1390 // preload "movl diff, %%ecx \n\t" // (diff is in eax) 1391 "movl %%eax, %%ecx \n\t" 1392 "cmpl $0, %%ecx \n\t" 1393 "jz end8 \n\t" 1394 // preload "movl mask, %%edx \n\t" 1395 "sall $24, %%edx \n\t" // make low byte, high byte 1396 1397 "secondloop8: \n\t" 1398 "sall %%edx \n\t" // move high bit to CF 1399 "jnc skip8 \n\t" // if CF = 0 1400 "movb (%3), %%al \n\t" 1401 "movb %%al, (%4) \n\t" 1402 1403 "skip8: \n\t" 1404 "inc %3 \n\t" 1405 "inc %4 \n\t" 1406 "decl %%ecx \n\t" 1407 "jnz secondloop8 \n\t" 1408 1409 "end8: \n\t" 1410 "EMMS \n\t" // DONE 1411 1412 : "=a" (dummy_value_a), // output regs (dummy) 1413 "=d" (dummy_value_d), 1414 "=c" (dummy_value_c), 1415 "=S" (dummy_value_S), 1416 "=D" (dummy_value_D) 1417 1418 : "0" (diff), // eax // input regs 1419 "1" (mask), // edx 1420 "2" (len), // ecx 1421 // was (unmask) "b" RESERVED // ebx // Global Offset Table idx 1422 "3" (srcptr), // esi/rsi 1423 "4" (dstptr) // edi/rdi 1424 1425 #if defined(PNG_CLOBBER_MMX_REGS_SUPPORTED) 1426 : "%mm0", "%mm4", "%mm6", "%mm7" // clobber list 1427 #endif 1428 ); 1429 } 1430 else /* not _mmx_supported - use modified C routine */ 1431 { 1432 register png_uint_32 i; 1433 png_uint_32 initial_val = png_pass_start[png_ptr->pass]; 1434 /* png.c: png_pass_start[] = {0, 4, 0, 2, 0, 1, 0}; */ 1435 register int stride = png_pass_inc[png_ptr->pass]; 1436 /* png.c: png_pass_inc[] = {8, 8, 4, 4, 2, 2, 1}; */ 1437 register int rep_bytes = png_pass_width[png_ptr->pass]; 1438 /* png.c: png_pass_width[] = {8, 4, 4, 2, 2, 1, 1}; */ 1439 png_uint_32 len = png_ptr->width &~7; /* reduce to mult. of 8 */ 1440 int diff = (int) (png_ptr->width & 7); /* amount lost */ 1441 register png_uint_32 final_val = len; /* GRR bugfix */ 1442 1443 srcptr = png_ptr->row_buf + 1 + initial_val; 1444 dstptr = row + initial_val; 1445 1446 for (i = initial_val; i < final_val; i += stride) 1447 { 1448 png_memcpy(dstptr, srcptr, rep_bytes); 1449 srcptr += stride; 1450 dstptr += stride; 1451 } 1452 if (diff) /* number of leftover pixels: 3 for pngtest */ 1453 { 1454 final_val += diff /* *BPP1 */ ; 1455 for (; i < final_val; i += stride) 1456 { 1457 if (rep_bytes > (int)(final_val-i)) 1458 rep_bytes = (int)(final_val-i); 1459 png_memcpy(dstptr, srcptr, rep_bytes); 1460 srcptr += stride; 1461 dstptr += stride; 1462 } 1463 } 1464 1465 } /* end of else (_mmx_supported) */ 1466 1467 break; 1468 } /* end 8 bpp */ 1469 1470 case 1: /* png_ptr->row_info.pixel_depth */ 1471 { 1472 png_bytep sp; 1473 png_bytep dp; 1474 int s_inc, s_start, s_end; 1475 int m; 1476 int shift; 1477 png_uint_32 i; 1478 1479 sp = png_ptr->row_buf + 1; 1480 dp = row; 1481 m = 0x80; 1482 #if defined(PNG_READ_PACKSWAP_SUPPORTED) 1483 if (png_ptr->transformations & PNG_PACKSWAP) 1484 { 1485 s_start = 0; 1486 s_end = 7; 1487 s_inc = 1; 1488 } 1489 else 1490 #endif 1491 { 1492 s_start = 7; 1493 s_end = 0; 1494 s_inc = -1; 1495 } 1496 1497 shift = s_start; 1498 1499 for (i = 0; i < png_ptr->width; i++) 1500 { 1501 if (m & mask) 1502 { 1503 int value; 1504 1505 value = (*sp >> shift) & 0x1; 1506 *dp &= (png_byte)((0x7f7f >> (7 - shift)) & 0xff); 1507 *dp |= (png_byte)(value << shift); 1508 } 1509 1510 if (shift == s_end) 1511 { 1512 shift = s_start; 1513 sp++; 1514 dp++; 1515 } 1516 else 1517 shift += s_inc; 1518 1519 if (m == 1) 1520 m = 0x80; 1521 else 1522 m >>= 1; 1523 } 1524 break; 1525 } /* end 1 bpp */ 1526 1527 case 2: /* png_ptr->row_info.pixel_depth */ 1528 { 1529 png_bytep sp; 1530 png_bytep dp; 1531 int s_start, s_end, s_inc; 1532 int m; 1533 int shift; 1534 png_uint_32 i; 1535 int value; 1536 1537 sp = png_ptr->row_buf + 1; 1538 dp = row; 1539 m = 0x80; 1540 #if defined(PNG_READ_PACKSWAP_SUPPORTED) 1541 if (png_ptr->transformations & PNG_PACKSWAP) 1542 { 1543 s_start = 0; 1544 s_end = 6; 1545 s_inc = 2; 1546 } 1547 else 1548 #endif 1549 { 1550 s_start = 6; 1551 s_end = 0; 1552 s_inc = -2; 1553 } 1554 1555 shift = s_start; 1556 1557 for (i = 0; i < png_ptr->width; i++) 1558 { 1559 if (m & mask) 1560 { 1561 value = (*sp >> shift) & 0x3; 1562 *dp &= (png_byte)((0x3f3f >> (6 - shift)) & 0xff); 1563 *dp |= (png_byte)(value << shift); 1564 } 1565 1566 if (shift == s_end) 1567 { 1568 shift = s_start; 1569 sp++; 1570 dp++; 1571 } 1572 else 1573 shift += s_inc; 1574 if (m == 1) 1575 m = 0x80; 1576 else 1577 m >>= 1; 1578 } 1579 break; 1580 } /* end 2 bpp */ 1581 1582 case 4: /* png_ptr->row_info.pixel_depth */ 1583 { 1584 png_bytep sp; 1585 png_bytep dp; 1586 int s_start, s_end, s_inc; 1587 int m; 1588 int shift; 1589 png_uint_32 i; 1590 int value; 1591 1592 sp = png_ptr->row_buf + 1; 1593 dp = row; 1594 m = 0x80; 1595 #if defined(PNG_READ_PACKSWAP_SUPPORTED) 1596 if (png_ptr->transformations & PNG_PACKSWAP) 1597 { 1598 s_start = 0; 1599 s_end = 4; 1600 s_inc = 4; 1601 } 1602 else 1603 #endif 1604 { 1605 s_start = 4; 1606 s_end = 0; 1607 s_inc = -4; 1608 } 1609 1610 shift = s_start; 1611 1612 for (i = 0; i < png_ptr->width; i++) 1613 { 1614 if (m & mask) 1615 { 1616 value = (*sp >> shift) & 0xf; 1617 *dp &= (png_byte)((0xf0f >> (4 - shift)) & 0xff); 1618 *dp |= (png_byte)(value << shift); 1619 } 1620 1621 if (shift == s_end) 1622 { 1623 shift = s_start; 1624 sp++; 1625 dp++; 1626 } 1627 else 1628 shift += s_inc; 1629 if (m == 1) 1630 m = 0x80; 1631 else 1632 m >>= 1; 1633 } 1634 break; 1635 } /* end 4 bpp */ 1636 1637 case 16: /* png_ptr->row_info.pixel_depth */ 1638 { 1639 png_bytep srcptr; 1640 png_bytep dstptr; 1641 1642 #if !defined(PNG_1_0_X) 1643 if (png_ptr->asm_flags & PNG_ASM_FLAG_MMX_READ_COMBINE_ROW) 1644 #else 1645 if (_mmx_supported) 1646 #endif 1647 { 1648 png_uint_32 len; 1649 int diff; 1650 1651 srcptr = png_ptr->row_buf + 1; 1652 dstptr = row; 1653 len = png_ptr->width & ~7; // reduce to multiple of 8 1654 diff = (int) (png_ptr->width & 7); // amount lost 1655 1656 __asm__ __volatile__ ( 1657 "not %%edx \n\t" // mask => unmask 1658 "movd %%edx, %%mm7 \n\t" // load bit pattern 1659 "not %%edx \n\t" // unmask => mask for later 1660 "psubb %%mm6, %%mm6 \n\t" // zero mm6 1661 "punpcklbw %%mm7, %%mm7 \n\t" 1662 "punpcklwd %%mm7, %%mm7 \n\t" 1663 "punpckldq %%mm7, %%mm7 \n\t" // fill reg with 8 masks 1664 1665 LOAD_GOT_rbp 1666 "movq " MASK16_0 ", %%mm0 \n\t" // _mask16_0 -> mm0 1667 "movq " MASK16_1 ", %%mm1 \n\t" // _mask16_1 -> mm1 1668 RESTORE_rbp 1669 1670 "pand %%mm7, %%mm0 \n\t" 1671 "pand %%mm7, %%mm1 \n\t" 1672 1673 "pcmpeqb %%mm6, %%mm0 \n\t" 1674 "pcmpeqb %%mm6, %%mm1 \n\t" 1675 1676 // preload "movl len, %%ecx \n\t" // load length of line 1677 // preload "movl srcptr, %3 \n\t" // load source 1678 // preload "movl dstptr, %4 \n\t" // load dest 1679 1680 "cmpl $0, %%ecx \n\t" 1681 "jz mainloop16end \n\t" 1682 1683 "mainloop16: \n\t" 1684 "movq (%3), %%mm4 \n\t" 1685 "pand %%mm0, %%mm4 \n\t" 1686 "movq %%mm0, %%mm6 \n\t" 1687 "movq (%4), %%mm7 \n\t" 1688 "pandn %%mm7, %%mm6 \n\t" 1689 "por %%mm6, %%mm4 \n\t" 1690 "movq %%mm4, (%4) \n\t" 1691 1692 "movq 8(%3), %%mm5 \n\t" 1693 "pand %%mm1, %%mm5 \n\t" 1694 "movq %%mm1, %%mm7 \n\t" 1695 "movq 8(%4), %%mm6 \n\t" 1696 "pandn %%mm6, %%mm7 \n\t" 1697 "por %%mm7, %%mm5 \n\t" 1698 "movq %%mm5, 8(%4) \n\t" 1699 1700 "add $16, %3 \n\t" // inc by 16 bytes processed 1701 "add $16, %4 \n\t" 1702 "subl $8, %%ecx \n\t" // dec by 8 pixels processed 1703 "ja mainloop16 \n\t" 1704 1705 "mainloop16end: \n\t" 1706 // preload "movl diff, %%ecx \n\t" // (diff is in eax) 1707 "movl %%eax, %%ecx \n\t" 1708 "cmpl $0, %%ecx \n\t" 1709 "jz end16 \n\t" 1710 // preload "movl mask, %%edx \n\t" 1711 "sall $24, %%edx \n\t" // make low byte, high byte 1712 1713 "secondloop16: \n\t" 1714 "sall %%edx \n\t" // move high bit to CF 1715 "jnc skip16 \n\t" // if CF = 0 1716 "movw (%3), %%ax \n\t" 1717 "movw %%ax, (%4) \n\t" 1718 1719 "skip16: \n\t" 1720 "add $2, %3 \n\t" 1721 "add $2, %4 \n\t" 1722 "decl %%ecx \n\t" 1723 "jnz secondloop16 \n\t" 1724 1725 "end16: \n\t" 1726 "EMMS \n\t" // DONE 1727 1728 : "=a" (dummy_value_a), // output regs (dummy) 1729 "=d" (dummy_value_d), 1730 "=c" (dummy_value_c), 1731 "=S" (dummy_value_S), 1732 "=D" (dummy_value_D) 1733 1734 : "0" (diff), // eax // input regs 1735 "1" (mask), // edx 1736 "2" (len), // ecx 1737 // was (unmask) "b" RESERVED // ebx // Global Offset Table idx 1738 "3" (srcptr), // esi/rsi 1739 "4" (dstptr) // edi/rdi 1740 1741 #if defined(PNG_CLOBBER_MMX_REGS_SUPPORTED) 1742 : "%mm0", "%mm1", "%mm4" // clobber list 1743 , "%mm5", "%mm6", "%mm7" 1744 #endif 1745 ); 1746 } 1747 else /* not _mmx_supported - use modified C routine */ 1748 { 1749 register png_uint_32 i; 1750 png_uint_32 initial_val = BPP2 * png_pass_start[png_ptr->pass]; 1751 /* png.c: png_pass_start[] = {0, 4, 0, 2, 0, 1, 0}; */ 1752 register int stride = BPP2 * png_pass_inc[png_ptr->pass]; 1753 /* png.c: png_pass_inc[] = {8, 8, 4, 4, 2, 2, 1}; */ 1754 register int rep_bytes = BPP2 * png_pass_width[png_ptr->pass]; 1755 /* png.c: png_pass_width[] = {8, 4, 4, 2, 2, 1, 1}; */ 1756 png_uint_32 len = png_ptr->width &~7; /* reduce to mult. of 8 */ 1757 int diff = (int) (png_ptr->width & 7); /* amount lost */ 1758 register png_uint_32 final_val = BPP2 * len; /* GRR bugfix */ 1759 1760 srcptr = png_ptr->row_buf + 1 + initial_val; 1761 dstptr = row + initial_val; 1762 1763 for (i = initial_val; i < final_val; i += stride) 1764 { 1765 png_memcpy(dstptr, srcptr, rep_bytes); 1766 srcptr += stride; 1767 dstptr += stride; 1768 } 1769 if (diff) /* number of leftover pixels: 3 for pngtest */ 1770 { 1771 final_val += diff*BPP2; 1772 for (; i < final_val; i += stride) 1773 { 1774 if (rep_bytes > (int)(final_val-i)) 1775 rep_bytes = (int)(final_val-i); 1776 png_memcpy(dstptr, srcptr, rep_bytes); 1777 srcptr += stride; 1778 dstptr += stride; 1779 } 1780 } 1781 } /* end of else (_mmx_supported) */ 1782 1783 break; 1784 } /* end 16 bpp */ 1785 1786 case 48: /* png_ptr->row_info.pixel_depth */ 1787 { 1788 png_bytep srcptr; 1789 png_bytep dstptr; 1790 1791 #if !defined(PNG_1_0_X) 1792 if (png_ptr->asm_flags & PNG_ASM_FLAG_MMX_READ_COMBINE_ROW) 1793 #else 1794 if (_mmx_supported) 1795 #endif 1796 { 1797 png_uint_32 len; 1798 int diff; 1799 1800 srcptr = png_ptr->row_buf + 1; 1801 dstptr = row; 1802 len = png_ptr->width & ~7; // reduce to multiple of 8 1803 diff = (int) (png_ptr->width & 7); // amount lost 1804 1805 __asm__ __volatile__ ( 1806 "not %%edx \n\t" // mask => unmask 1807 "movd %%edx, %%mm7 \n\t" // load bit pattern 1808 "not %%edx \n\t" // unmask => mask for later 1809 "psubb %%mm6, %%mm6 \n\t" // zero mm6 1810 "punpcklbw %%mm7, %%mm7 \n\t" 1811 "punpcklwd %%mm7, %%mm7 \n\t" 1812 "punpckldq %%mm7, %%mm7 \n\t" // fill reg with 8 masks 1813 1814 LOAD_GOT_rbp 1815 "movq " MASK48_0 ", %%mm0 \n\t" // _mask48_0 -> mm0 1816 "movq " MASK48_1 ", %%mm1 \n\t" // _mask48_1 -> mm1 1817 "movq " MASK48_2 ", %%mm2 \n\t" // _mask48_2 -> mm2 1818 "movq " MASK48_3 ", %%mm3 \n\t" // _mask48_3 -> mm3 1819 "movq " MASK48_4 ", %%mm4 \n\t" // _mask48_4 -> mm4 1820 "movq " MASK48_5 ", %%mm5 \n\t" // _mask48_5 -> mm5 1821 RESTORE_rbp 1822 1823 "pand %%mm7, %%mm0 \n\t" 1824 "pand %%mm7, %%mm1 \n\t" 1825 "pand %%mm7, %%mm2 \n\t" 1826 "pand %%mm7, %%mm3 \n\t" 1827 "pand %%mm7, %%mm4 \n\t" 1828 "pand %%mm7, %%mm5 \n\t" 1829 1830 "pcmpeqb %%mm6, %%mm0 \n\t" 1831 "pcmpeqb %%mm6, %%mm1 \n\t" 1832 "pcmpeqb %%mm6, %%mm2 \n\t" 1833 "pcmpeqb %%mm6, %%mm3 \n\t" 1834 "pcmpeqb %%mm6, %%mm4 \n\t" 1835 "pcmpeqb %%mm6, %%mm5 \n\t" 1836 1837 // preload "movl len, %%ecx \n\t" // load length of line 1838 // preload "movl srcptr, %3 \n\t" // load source 1839 // preload "movl dstptr, %4 \n\t" // load dest 1840 1841 "cmpl $0, %%ecx \n\t" 1842 "jz mainloop48end \n\t" 1843 1844 "mainloop48: \n\t" 1845 "movq (%3), %%mm7 \n\t" 1846 "pand %%mm0, %%mm7 \n\t" 1847 "movq %%mm0, %%mm6 \n\t" 1848 "pandn (%4), %%mm6 \n\t" 1849 "por %%mm6, %%mm7 \n\t" 1850 "movq %%mm7, (%4) \n\t" 1851 1852 "movq 8(%3), %%mm6 \n\t" 1853 "pand %%mm1, %%mm6 \n\t" 1854 "movq %%mm1, %%mm7 \n\t" 1855 "pandn 8(%4), %%mm7 \n\t" 1856 "por %%mm7, %%mm6 \n\t" 1857 "movq %%mm6, 8(%4) \n\t" 1858 1859 "movq 16(%3), %%mm6 \n\t" 1860 "pand %%mm2, %%mm6 \n\t" 1861 "movq %%mm2, %%mm7 \n\t" 1862 "pandn 16(%4), %%mm7 \n\t" 1863 "por %%mm7, %%mm6 \n\t" 1864 "movq %%mm6, 16(%4) \n\t" 1865 1866 "movq 24(%3), %%mm7 \n\t" 1867 "pand %%mm3, %%mm7 \n\t" 1868 "movq %%mm3, %%mm6 \n\t" 1869 "pandn 24(%4), %%mm6 \n\t" 1870 "por %%mm6, %%mm7 \n\t" 1871 "movq %%mm7, 24(%4) \n\t" 1872 1873 "movq 32(%3), %%mm6 \n\t" 1874 "pand %%mm4, %%mm6 \n\t" 1875 "movq %%mm4, %%mm7 \n\t" 1876 "pandn 32(%4), %%mm7 \n\t" 1877 "por %%mm7, %%mm6 \n\t" 1878 "movq %%mm6, 32(%4) \n\t" 1879 1880 "movq 40(%3), %%mm7 \n\t" 1881 "pand %%mm5, %%mm7 \n\t" 1882 "movq %%mm5, %%mm6 \n\t" 1883 "pandn 40(%4), %%mm6 \n\t" 1884 "por %%mm6, %%mm7 \n\t" 1885 "movq %%mm7, 40(%4) \n\t" 1886 1887 "add $48, %3 \n\t" // inc by 48 bytes processed 1888 "add $48, %4 \n\t" 1889 "subl $8, %%ecx \n\t" // dec by 8 pixels processed 1890 1891 "ja mainloop48 \n\t" 1892 1893 "mainloop48end: \n\t" 1894 // preload "movl diff, %%ecx \n\t" // (diff is in eax) 1895 "movl %%eax, %%ecx \n\t" 1896 "cmpl $0, %%ecx \n\t" 1897 "jz end48 \n\t" 1898 // preload "movl mask, %%edx \n\t" 1899 "sall $24, %%edx \n\t" // make low byte, high byte 1900 1901 "secondloop48: \n\t" 1902 "sall %%edx \n\t" // move high bit to CF 1903 "jnc skip48 \n\t" // if CF = 0 1904 "movl (%3), %%eax \n\t" 1905 "movl %%eax, (%4) \n\t" 1906 "movw 4(%3), %%ax \n\t" // GR-P bugfix 20070717 1907 "movw %%ax, 4(%4) \n\t" // GR-P bugfix 20070717 1908 1909 "skip48: \n\t" 1910 "add $6, %3 \n\t" // GR-P bugfix 20070717 1911 "add $6, %4 \n\t" // GR-P bugfix 20070717 1912 "decl %%ecx \n\t" 1913 "jnz secondloop48 \n\t" 1914 1915 "end48: \n\t" 1916 "EMMS \n\t" // DONE 1917 1918 : "=a" (dummy_value_a), // output regs (dummy) 1919 "=d" (dummy_value_d), 1920 "=c" (dummy_value_c), 1921 "=S" (dummy_value_S), 1922 "=D" (dummy_value_D) 1923 1924 : "0" (diff), // eax // input regs 1925 "1" (mask), // edx 1926 "2" (len), // ecx 1927 // was (unmask) "b" RESERVED // ebx // Global Offset Table idx 1928 "3" (srcptr), // esi/rsi 1929 "4" (dstptr) // edi/rdi 1930 1931 #if defined(PNG_CLOBBER_MMX_REGS_SUPPORTED) 1932 : "%mm0", "%mm1", "%mm2", "%mm3" // clobber list 1933 , "%mm4", "%mm5", "%mm6", "%mm7" 1934 #endif 1935 ); 1936 } 1937 else /* not _mmx_supported - use modified C routine */ 1938 { 1939 register png_uint_32 i; 1940 png_uint_32 initial_val = BPP6 * png_pass_start[png_ptr->pass]; 1941 /* png.c: png_pass_start[] = {0, 4, 0, 2, 0, 1, 0}; */ 1942 register int stride = BPP6 * png_pass_inc[png_ptr->pass]; 1943 /* png.c: png_pass_inc[] = {8, 8, 4, 4, 2, 2, 1}; */ 1944 register int rep_bytes = BPP6 * png_pass_width[png_ptr->pass]; 1945 /* png.c: png_pass_width[] = {8, 4, 4, 2, 2, 1, 1}; */ 1946 png_uint_32 len = png_ptr->width &~7; /* reduce to mult. of 8 */ 1947 int diff = (int) (png_ptr->width & 7); /* amount lost */ 1948 register png_uint_32 final_val = BPP6 * len; /* GRR bugfix */ 1949 1950 srcptr = png_ptr->row_buf + 1 + initial_val; 1951 dstptr = row + initial_val; 1952 1953 for (i = initial_val; i < final_val; i += stride) 1954 { 1955 png_memcpy(dstptr, srcptr, rep_bytes); 1956 srcptr += stride; 1957 dstptr += stride; 1958 } 1959 if (diff) /* number of leftover pixels: 3 for pngtest */ 1960 { 1961 final_val += diff*BPP6; 1962 for (; i < final_val; i += stride) 1963 { 1964 if (rep_bytes > (int)(final_val-i)) 1965 rep_bytes = (int)(final_val-i); 1966 png_memcpy(dstptr, srcptr, rep_bytes); 1967 srcptr += stride; 1968 dstptr += stride; 1969 } 1970 } 1971 } /* end of else (_mmx_supported) */ 1972 1973 break; 1974 } /* end 48 bpp */ 1975 1976 case 64: /* png_ptr->row_info.pixel_depth */ 1977 { 1978 png_bytep srcptr; 1979 png_bytep dstptr; 1980 register png_uint_32 i; 1981 png_uint_32 initial_val = BPP8 * png_pass_start[png_ptr->pass]; 1982 /* png.c: png_pass_start[] = {0, 4, 0, 2, 0, 1, 0}; */ 1983 register int stride = BPP8 * png_pass_inc[png_ptr->pass]; 1984 /* png.c: png_pass_inc[] = {8, 8, 4, 4, 2, 2, 1}; */ 1985 register int rep_bytes = BPP8 * png_pass_width[png_ptr->pass]; 1986 /* png.c: png_pass_width[] = {8, 4, 4, 2, 2, 1, 1}; */ 1987 png_uint_32 len = png_ptr->width &~7; /* reduce to mult. of 8 */ 1988 int diff = (int) (png_ptr->width & 7); /* amount lost */ 1989 register png_uint_32 final_val = BPP8 * len; /* GRR bugfix */ 1990 1991 srcptr = png_ptr->row_buf + 1 + initial_val; 1992 dstptr = row + initial_val; 1993 1994 for (i = initial_val; i < final_val; i += stride) 1995 { 1996 png_memcpy(dstptr, srcptr, rep_bytes); 1997 srcptr += stride; 1998 dstptr += stride; 1999 } 2000 if (diff) /* number of leftover pixels: 3 for pngtest */ 2001 { 2002 final_val += diff*BPP8; 2003 for (; i < final_val; i += stride) 2004 { 2005 if (rep_bytes > (int)(final_val-i)) 2006 rep_bytes = (int)(final_val-i); 2007 png_memcpy(dstptr, srcptr, rep_bytes); 2008 srcptr += stride; 2009 dstptr += stride; 2010 } 2011 } 2012 2013 break; 2014 } /* end 64 bpp */ 2015 2016 default: /* png_ptr->row_info.pixel_depth != 1,2,4,8,16,24,32,48,64 */ 2017 { 2018 // ERROR: SHOULD NEVER BE REACHED 2019 #if defined(PNG_DEBUG) 2020 png_debug(1, "Internal libpng logic error (GCC " 2021 "png_combine_row() pixel_depth)\n"); 2022 #endif 2023 break; 2024 } 2025 } /* end switch (png_ptr->row_info.pixel_depth) */ 2026 2027 } /* end if (non-trivial mask) */ 2028 2029 } /* end png_combine_row() */ 2030 2031 #endif /* PNG_HAVE_MMX_COMBINE_ROW */ 2032 2033 2034 2035 2036 /*===========================================================================*/ 2037 /* */ 2038 /* P N G _ D O _ R E A D _ I N T E R L A C E */ 2039 /* */ 2040 /*===========================================================================*/ 2041 2042 #if defined(PNG_READ_INTERLACING_SUPPORTED) 2043 #if defined(PNG_HAVE_MMX_READ_INTERLACE) 2044 2045 /* png_do_read_interlace() is called after any 16-bit to 8-bit conversion 2046 * has taken place. [GRR: what other steps come before and/or after?] 2047 */ 2048 2049 void /* PRIVATE */ 2050 png_do_read_interlace(png_structp png_ptr) 2051 { 2052 png_row_infop row_info = &(png_ptr->row_info); 2053 png_bytep row = png_ptr->row_buf + 1; 2054 int pass = png_ptr->pass; 2055 #if defined(PNG_READ_PACKSWAP_SUPPORTED) 2056 png_uint_32 transformations = png_ptr->transformations; 2057 #endif 2058 2059 png_debug(1, "in png_do_read_interlace (pnggccrd.c)\n"); 2060 2061 if (_mmx_supported == 2) { 2062 #if !defined(PNG_1_0_X) 2063 /* this should have happened in png_init_mmx_flags() already */ 2064 png_warning(png_ptr, "asm_flags may not have been initialized"); 2065 #endif 2066 png_mmx_support(); 2067 } 2068 2069 if (row != NULL && row_info != NULL) 2070 { 2071 png_uint_32 final_width; 2072 2073 final_width = row_info->width * png_pass_inc[pass]; 2074 2075 switch (row_info->pixel_depth) 2076 { 2077 case 1: 2078 { 2079 png_bytep sp, dp; 2080 int sshift, dshift; 2081 int s_start, s_end, s_inc; 2082 png_byte v; 2083 png_uint_32 i; 2084 int j; 2085 2086 sp = row + (png_size_t)((row_info->width - 1) >> 3); 2087 dp = row + (png_size_t)((final_width - 1) >> 3); 2088 #if defined(PNG_READ_PACKSWAP_SUPPORTED) 2089 if (transformations & PNG_PACKSWAP) 2090 { 2091 sshift = (int)((row_info->width + 7) & 7); 2092 dshift = (int)((final_width + 7) & 7); 2093 s_start = 7; 2094 s_end = 0; 2095 s_inc = -1; 2096 } 2097 else 2098 #endif 2099 { 2100 sshift = 7 - (int)((row_info->width + 7) & 7); 2101 dshift = 7 - (int)((final_width + 7) & 7); 2102 s_start = 0; 2103 s_end = 7; 2104 s_inc = 1; 2105 } 2106 2107 for (i = row_info->width; i; i--) 2108 { 2109 v = (png_byte)((*sp >> sshift) & 0x1); 2110 for (j = 0; j < png_pass_inc[pass]; j++) 2111 { 2112 *dp &= (png_byte)((0x7f7f >> (7 - dshift)) & 0xff); 2113 *dp |= (png_byte)(v << dshift); 2114 if (dshift == s_end) 2115 { 2116 dshift = s_start; 2117 dp--; 2118 } 2119 else 2120 dshift += s_inc; 2121 } 2122 if (sshift == s_end) 2123 { 2124 sshift = s_start; 2125 sp--; 2126 } 2127 else 2128 sshift += s_inc; 2129 } 2130 break; 2131 } 2132 2133 case 2: 2134 { 2135 png_bytep sp, dp; 2136 int sshift, dshift; 2137 int s_start, s_end, s_inc; 2138 png_uint_32 i; 2139 2140 sp = row + (png_size_t)((row_info->width - 1) >> 2); 2141 dp = row + (png_size_t)((final_width - 1) >> 2); 2142 #if defined(PNG_READ_PACKSWAP_SUPPORTED) 2143 if (transformations & PNG_PACKSWAP) 2144 { 2145 sshift = (png_size_t)(((row_info->width + 3) & 3) << 1); 2146 dshift = (png_size_t)(((final_width + 3) & 3) << 1); 2147 s_start = 6; 2148 s_end = 0; 2149 s_inc = -2; 2150 } 2151 else 2152 #endif 2153 { 2154 sshift = (png_size_t)((3 - ((row_info->width + 3) & 3)) << 1); 2155 dshift = (png_size_t)((3 - ((final_width + 3) & 3)) << 1); 2156 s_start = 0; 2157 s_end = 6; 2158 s_inc = 2; 2159 } 2160 2161 for (i = row_info->width; i; i--) 2162 { 2163 png_byte v; 2164 int j; 2165 2166 v = (png_byte)((*sp >> sshift) & 0x3); 2167 for (j = 0; j < png_pass_inc[pass]; j++) 2168 { 2169 *dp &= (png_byte)((0x3f3f >> (6 - dshift)) & 0xff); 2170 *dp |= (png_byte)(v << dshift); 2171 if (dshift == s_end) 2172 { 2173 dshift = s_start; 2174 dp--; 2175 } 2176 else 2177 dshift += s_inc; 2178 } 2179 if (sshift == s_end) 2180 { 2181 sshift = s_start; 2182 sp--; 2183 } 2184 else 2185 sshift += s_inc; 2186 } 2187 break; 2188 } 2189 2190 case 4: 2191 { 2192 png_bytep sp, dp; 2193 int sshift, dshift; 2194 int s_start, s_end, s_inc; 2195 png_uint_32 i; 2196 2197 sp = row + (png_size_t)((row_info->width - 1) >> 1); 2198 dp = row + (png_size_t)((final_width - 1) >> 1); 2199 #if defined(PNG_READ_PACKSWAP_SUPPORTED) 2200 if (transformations & PNG_PACKSWAP) 2201 { 2202 sshift = (png_size_t)(((row_info->width + 1) & 1) << 2); 2203 dshift = (png_size_t)(((final_width + 1) & 1) << 2); 2204 s_start = 4; 2205 s_end = 0; 2206 s_inc = -4; 2207 } 2208 else 2209 #endif 2210 { 2211 sshift = (png_size_t)((1 - ((row_info->width + 1) & 1)) << 2); 2212 dshift = (png_size_t)((1 - ((final_width + 1) & 1)) << 2); 2213 s_start = 0; 2214 s_end = 4; 2215 s_inc = 4; 2216 } 2217 2218 for (i = row_info->width; i; i--) 2219 { 2220 png_byte v; 2221 int j; 2222 2223 v = (png_byte)((*sp >> sshift) & 0xf); 2224 for (j = 0; j < png_pass_inc[pass]; j++) 2225 { 2226 *dp &= (png_byte)((0xf0f >> (4 - dshift)) & 0xff); 2227 *dp |= (png_byte)(v << dshift); 2228 if (dshift == s_end) 2229 { 2230 dshift = s_start; 2231 dp--; 2232 } 2233 else 2234 dshift += s_inc; 2235 } 2236 if (sshift == s_end) 2237 { 2238 sshift = s_start; 2239 sp--; 2240 } 2241 else 2242 sshift += s_inc; 2243 } 2244 break; 2245 } 2246 2247 /*====================================================================*/ 2248 2249 default: /* 8-bit or larger (this is where the routine is modified) */ 2250 { 2251 png_bytep sptr, dp; 2252 png_uint_32 i; 2253 png_size_t pixel_bytes; 2254 int width = (int)row_info->width; 2255 2256 pixel_bytes = (row_info->pixel_depth >> 3); 2257 2258 /* point sptr at the last pixel in the pre-expanded row: */ 2259 sptr = row + (width - 1) * pixel_bytes; 2260 2261 /* point dp at the last pixel position in the expanded row: */ 2262 dp = row + (final_width - 1) * pixel_bytes; 2263 2264 /* New code by Nirav Chhatrapati - Intel Corporation */ 2265 2266 #if !defined(PNG_1_0_X) 2267 if (png_ptr->asm_flags & PNG_ASM_FLAG_MMX_READ_INTERLACE) 2268 #else 2269 if (_mmx_supported) 2270 #endif 2271 { 2272 int dummy_value_c; // fix 'forbidden register spilled' 2273 png_bytep dummy_value_S; 2274 png_bytep dummy_value_D; 2275 png_bytep dummy_value_a; 2276 png_bytep dummy_value_d; 2277 2278 //-------------------------------------------------------------- 2279 if (pixel_bytes == BPP3) 2280 { 2281 if (((pass == 4) || (pass == 5)) && width) 2282 { 2283 int width_mmx = ((width >> 1) << 1) - 8; // GRR: huh? 2284 if (width_mmx < 0) 2285 width_mmx = 0; 2286 width -= width_mmx; // 8 or 9 pix, 24 or 27 bytes 2287 if (width_mmx) 2288 { 2289 // png_pass_inc[] = {8, 8, 4, 4, 2, 2, 1}; 2290 // sptr points at last pixel in pre-expanded row 2291 // dp points at last pixel position in expanded row 2292 __asm__ __volatile__ ( 2293 "sub $3, %1 \n\t" 2294 "sub $9, %2 \n\t" 2295 // (png_pass_inc[pass] + 1)*pixel_bytes 2296 2297 ".loop3_pass4: \n\t" 2298 "movq (%1), %%mm0 \n\t" // x x 5 4 3 2 1 0 2299 "movq %%mm0, %%mm1 \n\t" // x x 5 4 3 2 1 0 2300 "movq %%mm0, %%mm2 \n\t" // x x 5 4 3 2 1 0 2301 "psllq $24, %%mm0 \n\t" // 4 3 2 1 0 z z z 2302 "pand (%3), %%mm1 \n\t" // z z z z z 2 1 0 2303 "psrlq $24, %%mm2 \n\t" // z z z x x 5 4 3 2304 "por %%mm1, %%mm0 \n\t" // 4 3 2 1 0 2 1 0 2305 "movq %%mm2, %%mm3 \n\t" // z z z x x 5 4 3 2306 "psllq $8, %%mm2 \n\t" // z z x x 5 4 3 z 2307 "movq %%mm0, (%2) \n\t" 2308 "psrlq $16, %%mm3 \n\t" // z z z z z x x 5 2309 "pand (%4), %%mm3 \n\t" // z z z z z z z 5 2310 "por %%mm3, %%mm2 \n\t" // z z x x 5 4 3 5 2311 "sub $6, %1 \n\t" 2312 "movd %%mm2, 8(%2) \n\t" 2313 "sub $12, %2 \n\t" 2314 "subl $2, %%ecx \n\t" 2315 "jnz .loop3_pass4 \n\t" 2316 "EMMS \n\t" // DONE 2317 2318 : "=c" (dummy_value_c), // output regs (dummy) 2319 "=S" (dummy_value_S), 2320 "=D" (dummy_value_D), 2321 "=a" (dummy_value_a), 2322 "=d" (dummy_value_d) 2323 2324 : "0" (width_mmx), // ecx // input regs 2325 "1" (sptr), // esi/rsi 2326 "2" (dp), // edi/rdi 2327 #if defined(PNG_x86_64_USE_GOTPCREL) // formerly _const4 and _const6: 2328 "3" (&_c64._amask5_3_0), // (0x0000000000FFFFFFLL) 2329 "4" (&_c64._amask7_1_0) // (0x00000000000000FFLL) 2330 #else 2331 "3" (&_amask5_3_0), // eax (0x0000000000FFFFFFLL) 2332 "4" (&_amask7_1_0) // edx (0x00000000000000FFLL) 2333 #endif 2334 2335 #if defined(PNG_CLOBBER_MMX_REGS_SUPPORTED) 2336 : "%mm0", "%mm1" // clobber list 2337 , "%mm2", "%mm3" 2338 #endif 2339 ); 2340 } 2341 2342 sptr -= width_mmx*BPP3; 2343 dp -= width_mmx*2*BPP3; 2344 for (i = width; i; i--) 2345 { 2346 png_byte v[8]; 2347 int j; 2348 2349 png_memcpy(v, sptr, BPP3); 2350 for (j = 0; j < png_pass_inc[pass]; j++) 2351 { 2352 png_memcpy(dp, v, BPP3); 2353 dp -= BPP3; 2354 } 2355 sptr -= BPP3; 2356 } 2357 } 2358 else if (((pass == 2) || (pass == 3)) && width) 2359 { 2360 __asm__ __volatile__ ( 2361 "sub $9, %2 \n\t" 2362 // (png_pass_inc[pass] - 1)*pixel_bytes 2363 2364 ".loop3_pass2: \n\t" 2365 "movd (%1), %%mm0 \n\t" // x x x x x 2 1 0 2366 "pand (%3), %%mm0 \n\t" // z z z z z 2 1 0 2367 "movq %%mm0, %%mm1 \n\t" // z z z z z 2 1 0 2368 "psllq $16, %%mm0 \n\t" // z z z 2 1 0 z z 2369 "movq %%mm0, %%mm2 \n\t" // z z z 2 1 0 z z 2370 "psllq $24, %%mm0 \n\t" // 2 1 0 z z z z z 2371 "psrlq $8, %%mm1 \n\t" // z z z z z z 2 1 2372 "por %%mm2, %%mm0 \n\t" // 2 1 0 2 1 0 z z 2373 "por %%mm1, %%mm0 \n\t" // 2 1 0 2 1 0 2 1 2374 "movq %%mm0, 4(%2) \n\t" 2375 "psrlq $16, %%mm0 \n\t" // z z 2 1 0 2 1 0 2376 "sub $3, %1 \n\t" 2377 "movd %%mm0, (%2) \n\t" 2378 "sub $12, %2 \n\t" 2379 "decl %%ecx \n\t" 2380 "jnz .loop3_pass2 \n\t" 2381 "EMMS \n\t" // DONE 2382 2383 : "=c" (dummy_value_c), // output regs (dummy) 2384 "=S" (dummy_value_S), 2385 "=D" (dummy_value_D), 2386 "=a" (dummy_value_a) 2387 2388 : "0" (width), // ecx // input regs 2389 "1" (sptr), // esi/rsi 2390 "2" (dp), // edi/rdi 2391 #if defined(PNG_x86_64_USE_GOTPCREL) // formerly _const4: 2392 "3" (&_c64._amask5_3_0) // (0x0000000000FFFFFFLL) 2393 #else 2394 "3" (&_amask5_3_0) // eax (0x0000000000FFFFFFLL) 2395 #endif 2396 2397 #if defined(CLOBBER_MMX_REGS_SUPPORTED) 2398 : "%mm0", "%mm1", "%mm2" // clobber list 2399 #endif 2400 ); 2401 } 2402 else if (width) // && ((pass == 0) || (pass == 1)) 2403 { 2404 __asm__ __volatile__ ( 2405 "sub $21, %2 \n\t" 2406 // (png_pass_inc[pass] - 1)*pixel_bytes 2407 2408 ".loop3_pass0: \n\t" 2409 "movd (%1), %%mm0 \n\t" // x x x x x 2 1 0 2410 "pand (%3), %%mm0 \n\t" // z z z z z 2 1 0 2411 "movq %%mm0, %%mm1 \n\t" // z z z z z 2 1 0 2412 "psllq $16, %%mm0 \n\t" // z z z 2 1 0 z z 2413 "movq %%mm0, %%mm2 \n\t" // z z z 2 1 0 z z 2414 "psllq $24, %%mm0 \n\t" // 2 1 0 z z z z z 2415 "psrlq $8, %%mm1 \n\t" // z z z z z z 2 1 2416 "por %%mm2, %%mm0 \n\t" // 2 1 0 2 1 0 z z 2417 "por %%mm1, %%mm0 \n\t" // 2 1 0 2 1 0 2 1 2418 "movq %%mm0, %%mm3 \n\t" // 2 1 0 2 1 0 2 1 2419 "psllq $16, %%mm0 \n\t" // 0 2 1 0 2 1 z z 2420 "movq %%mm3, %%mm4 \n\t" // 2 1 0 2 1 0 2 1 2421 "punpckhdq %%mm0, %%mm3 \n\t" // 0 2 1 0 2 1 0 2 2422 "movq %%mm4, 16(%2) \n\t" 2423 "psrlq $32, %%mm0 \n\t" // z z z z 0 2 1 0 2424 "movq %%mm3, 8(%2) \n\t" 2425 "punpckldq %%mm4, %%mm0 \n\t" // 1 0 2 1 0 2 1 0 2426 "sub $3, %1 \n\t" 2427 "movq %%mm0, (%2) \n\t" 2428 "sub $24, %2 \n\t" 2429 "decl %%ecx \n\t" 2430 "jnz .loop3_pass0 \n\t" 2431 "EMMS \n\t" // DONE 2432 2433 : "=c" (dummy_value_c), // output regs (dummy) 2434 "=S" (dummy_value_S), 2435 "=D" (dummy_value_D), 2436 "=a" (dummy_value_a) 2437 2438 : "0" (width), // ecx // input regs 2439 "1" (sptr), // esi/rsi 2440 "2" (dp), // edi/rdi 2441 #if defined(PNG_x86_64_USE_GOTPCREL) // formerly _const4: 2442 "3" (&_c64._amask5_3_0) // (0x0000000000FFFFFFLL) 2443 #else 2444 "3" (&_amask5_3_0) // eax (0x0000000000FFFFFFLL) 2445 #endif 2446 2447 #if defined(PNG_CLOBBER_MMX_REGS_SUPPORTED) 2448 : "%mm0", "%mm1", "%mm2" // clobber list 2449 , "%mm3", "%mm4" 2450 #endif 2451 ); 2452 } 2453 } /* end of pixel_bytes == 3 */ 2454 2455 //-------------------------------------------------------------- 2456 else if (pixel_bytes == BPP4) 2457 { 2458 if (((pass == 4) || (pass == 5)) && width) 2459 { 2460 int width_mmx = ((width >> 1) << 1) ; 2461 width -= width_mmx; // 0,1 pixels => 0,4 bytes 2462 if (width_mmx) 2463 { 2464 __asm__ __volatile__ ( 2465 "sub $4, %1 \n\t" 2466 "sub $12, %2 \n\t" 2467 2468 ".loop4_pass4: \n\t" 2469 "movq (%1), %%mm0 \n\t" // 7 6 5 4 3 2 1 0 2470 "movq %%mm0, %%mm1 \n\t" // 7 6 5 4 3 2 1 0 2471 "punpckldq %%mm0, %%mm0 \n\t" // 3 2 1 0 3 2 1 0 2472 "punpckhdq %%mm1, %%mm1 \n\t" // 7 6 5 4 7 6 5 4 2473 "movq %%mm0, (%2) \n\t" 2474 "sub $8, %1 \n\t" 2475 "movq %%mm1, 8(%2) \n\t" 2476 "sub $16, %2 \n\t" 2477 "subl $2, %%ecx \n\t" 2478 "jnz .loop4_pass4 \n\t" 2479 "EMMS \n\t" // DONE 2480 2481 : "=c" (dummy_value_c), // output regs (dummy) 2482 "=S" (dummy_value_S), 2483 "=D" (dummy_value_D) 2484 2485 : "0" (width_mmx), // ecx // input regs 2486 "1" (sptr), // esi/rsi 2487 "2" (dp) // edi/rdi 2488 2489 #if defined(CLOBBER_MMX_REGS_SUPPORTED) 2490 : "%mm0", "%mm1" // clobber list 2491 #endif 2492 ); 2493 } 2494 2495 sptr -= (width_mmx*BPP4 - BPP4); // sign fixed 2496 dp -= (width_mmx*2*BPP4 - BPP4); // sign fixed 2497 for (i = width; i; i--) 2498 { 2499 png_byte v[8]; 2500 int j; 2501 sptr -= BPP4; 2502 png_memcpy(v, sptr, BPP4); 2503 for (j = 0; j < png_pass_inc[pass]; j++) 2504 { 2505 dp -= BPP4; 2506 png_memcpy(dp, v, BPP4); 2507 } 2508 } 2509 } 2510 else if (((pass == 2) || (pass == 3)) && width) 2511 { 2512 int width_mmx = ((width >> 1) << 1); 2513 width -= width_mmx; // 0,1 pixels => 0,4 bytes 2514 if (width_mmx) 2515 { 2516 __asm__ __volatile__ ( 2517 "sub $4, %1 \n\t" 2518 "sub $28, %2 \n\t" 2519 2520 ".loop4_pass2: \n\t" 2521 "movq (%1), %%mm0 \n\t" // 7 6 5 4 3 2 1 0 2522 "movq %%mm0, %%mm1 \n\t" // 7 6 5 4 3 2 1 0 2523 "punpckldq %%mm0, %%mm0 \n\t" // 3 2 1 0 3 2 1 0 2524 "punpckhdq %%mm1, %%mm1 \n\t" // 7 6 5 4 7 6 5 4 2525 "movq %%mm0, (%2) \n\t" 2526 "movq %%mm0, 8(%2) \n\t" 2527 "movq %%mm1, 16(%2) \n\t" 2528 "movq %%mm1, 24(%2) \n\t" 2529 "sub $8, %1 \n\t" 2530 "sub $32, %2 \n\t" 2531 "subl $2, %%ecx \n\t" 2532 "jnz .loop4_pass2 \n\t" 2533 "EMMS \n\t" // DONE 2534 2535 : "=c" (dummy_value_c), // output regs (dummy) 2536 "=S" (dummy_value_S), 2537 "=D" (dummy_value_D) 2538 2539 : "0" (width_mmx), // ecx // input regs 2540 "1" (sptr), // esi/rsi 2541 "2" (dp) // edi/rdi 2542 2543 #if defined(CLOBBER_MMX_REGS_SUPPORTED) 2544 : "%mm0", "%mm1" // clobber list 2545 #endif 2546 ); 2547 } 2548 2549 sptr -= (width_mmx*4 - 4); // sign fixed 2550 dp -= (width_mmx*16 - 4); // sign fixed 2551 for (i = width; i; i--) 2552 { 2553 png_byte v[8]; 2554 int j; 2555 sptr -= 4; 2556 png_memcpy(v, sptr, 4); 2557 for (j = 0; j < png_pass_inc[pass]; j++) 2558 { 2559 dp -= 4; 2560 png_memcpy(dp, v, 4); 2561 } 2562 } 2563 } 2564 else if (width) // && ((pass == 0) || (pass == 1)) 2565 { 2566 int width_mmx = ((width >> 1) << 1); 2567 width -= width_mmx; // 0,1 pixels => 0,4 bytes 2568 if (width_mmx) 2569 { 2570 __asm__ __volatile__ ( 2571 "sub $4, %1 \n\t" 2572 "sub $60, %2 \n\t" 2573 2574 ".loop4_pass0: \n\t" 2575 "movq (%1), %%mm0 \n\t" // 7 6 5 4 3 2 1 0 2576 "movq %%mm0, %%mm1 \n\t" // 7 6 5 4 3 2 1 0 2577 "punpckldq %%mm0, %%mm0 \n\t" // 3 2 1 0 3 2 1 0 2578 "punpckhdq %%mm1, %%mm1 \n\t" // 7 6 5 4 7 6 5 4 2579 "movq %%mm0, (%2) \n\t" 2580 "movq %%mm0, 8(%2) \n\t" 2581 "movq %%mm0, 16(%2) \n\t" 2582 "movq %%mm0, 24(%2) \n\t" 2583 "movq %%mm1, 32(%2) \n\t" 2584 "movq %%mm1, 40(%2) \n\t" 2585 "movq %%mm1, 48(%2) \n\t" 2586 "sub $8, %1 \n\t" 2587 "movq %%mm1, 56(%2) \n\t" 2588 "sub $64, %2 \n\t" 2589 "subl $2, %%ecx \n\t" 2590 "jnz .loop4_pass0 \n\t" 2591 "EMMS \n\t" // DONE 2592 2593 : "=c" (dummy_value_c), // output regs (dummy) 2594 "=S" (dummy_value_S), 2595 "=D" (dummy_value_D) 2596 2597 : "0" (width_mmx), // ecx // input regs 2598 "1" (sptr), // esi/rsi 2599 "2" (dp) // edi/rdi 2600 2601 #if defined(PNG_CLOBBER_MMX_REGS_SUPPORTED) 2602 : "%mm0", "%mm1" // clobber list 2603 #endif 2604 ); 2605 } 2606 2607 sptr -= (width_mmx*4 - 4); // sign fixed 2608 dp -= (width_mmx*32 - 4); // sign fixed 2609 for (i = width; i; i--) 2610 { 2611 png_byte v[8]; 2612 int j; 2613 sptr -= 4; 2614 png_memcpy(v, sptr, 4); 2615 for (j = 0; j < png_pass_inc[pass]; j++) 2616 { 2617 dp -= 4; 2618 png_memcpy(dp, v, 4); 2619 } 2620 } 2621 } 2622 } /* end of pixel_bytes == 4 */ 2623 2624 //-------------------------------------------------------------- 2625 else if (pixel_bytes == 1) 2626 { 2627 if (((pass == 4) || (pass == 5)) && width) 2628 { 2629 int width_mmx = ((width >> 3) << 3); 2630 width -= width_mmx; // 0-3 pixels => 0-3 bytes 2631 if (width_mmx) 2632 { 2633 __asm__ __volatile__ ( 2634 "sub $7, %1 \n\t" 2635 "sub $15, %2 \n\t" 2636 2637 ".loop1_pass4: \n\t" 2638 "movq (%1), %%mm0 \n\t" // 7 6 5 4 3 2 1 0 2639 "movq %%mm0, %%mm1 \n\t" // 7 6 5 4 3 2 1 0 2640 "punpcklbw %%mm0, %%mm0 \n\t" // 3 3 2 2 1 1 0 0 2641 "punpckhbw %%mm1, %%mm1 \n\t" // 7 7 6 6 5 5 4 4 2642 "movq %%mm1, 8(%2) \n\t" 2643 "sub $8, %1 \n\t" 2644 "movq %%mm0, (%2) \n\t" 2645 "sub $16, %2 \n\t" 2646 "subl $8, %%ecx \n\t" 2647 "jnz .loop1_pass4 \n\t" 2648 "EMMS \n\t" // DONE 2649 2650 : "=c" (dummy_value_c), // output regs (dummy) 2651 "=S" (dummy_value_S), 2652 "=D" (dummy_value_D) 2653 2654 : "0" (width_mmx), // ecx // input regs 2655 "1" (sptr), // esi/rsi 2656 "2" (dp) // edi/rdi 2657 2658 #if defined(PNG_CLOBBER_MMX_REGS_SUPPORTED) 2659 : "%mm0", "%mm1" // clobber list 2660 #endif 2661 ); 2662 } 2663 2664 sptr -= width_mmx; 2665 dp -= width_mmx*2; 2666 for (i = width; i; i--) 2667 { 2668 int j; 2669 2670 for (j = 0; j < png_pass_inc[pass]; j++) 2671 { 2672 *dp-- = *sptr; 2673 } 2674 --sptr; 2675 } 2676 } 2677 else if (((pass == 2) || (pass == 3)) && width) 2678 { 2679 int width_mmx = ((width >> 2) << 2); 2680 width -= width_mmx; // 0-3 pixels => 0-3 bytes 2681 if (width_mmx) 2682 { 2683 __asm__ __volatile__ ( 2684 "sub $3, %1 \n\t" 2685 "sub $15, %2 \n\t" 2686 2687 ".loop1_pass2: \n\t" 2688 "movd (%1), %%mm0 \n\t" // x x x x 3 2 1 0 2689 "punpcklbw %%mm0, %%mm0 \n\t" // 3 3 2 2 1 1 0 0 2690 "movq %%mm0, %%mm1 \n\t" // 3 3 2 2 1 1 0 0 2691 "punpcklwd %%mm0, %%mm0 \n\t" // 1 1 1 1 0 0 0 0 2692 "punpckhwd %%mm1, %%mm1 \n\t" // 3 3 3 3 2 2 2 2 2693 "movq %%mm0, (%2) \n\t" 2694 "sub $4, %1 \n\t" 2695 "movq %%mm1, 8(%2) \n\t" 2696 "sub $16, %2 \n\t" 2697 "subl $4, %%ecx \n\t" 2698 "jnz .loop1_pass2 \n\t" 2699 "EMMS \n\t" // DONE 2700 2701 : "=c" (dummy_value_c), // output regs (dummy) 2702 "=S" (dummy_value_S), 2703 "=D" (dummy_value_D) 2704 2705 : "0" (width_mmx), // ecx // input regs 2706 "1" (sptr), // esi/rsi 2707 "2" (dp) // edi/rdi 2708 2709 #if defined(PNG_CLOBBER_MMX_REGS_SUPPORTED) 2710 : "%mm0", "%mm1" // clobber list 2711 #endif 2712 ); 2713 } 2714 2715 sptr -= width_mmx; 2716 dp -= width_mmx*4; 2717 for (i = width; i; i--) 2718 { 2719 int j; 2720 2721 for (j = 0; j < png_pass_inc[pass]; j++) 2722 { 2723 *dp-- = *sptr; 2724 } 2725 --sptr; 2726 } 2727 } 2728 else if (width) // && ((pass == 0) || (pass == 1)) 2729 { 2730 int width_mmx = ((width >> 2) << 2); 2731 width -= width_mmx; // 0-3 pixels => 0-3 bytes 2732 if (width_mmx) 2733 { 2734 __asm__ __volatile__ ( 2735 "sub $3, %1 \n\t" 2736 "sub $31, %2 \n\t" 2737 2738 ".loop1_pass0: \n\t" 2739 "movd (%1), %%mm0 \n\t" // x x x x 3 2 1 0 2740 "movq %%mm0, %%mm1 \n\t" // x x x x 3 2 1 0 2741 "punpcklbw %%mm0, %%mm0 \n\t" // 3 3 2 2 1 1 0 0 2742 "movq %%mm0, %%mm2 \n\t" // 3 3 2 2 1 1 0 0 2743 "punpcklwd %%mm0, %%mm0 \n\t" // 1 1 1 1 0 0 0 0 2744 "movq %%mm0, %%mm3 \n\t" // 1 1 1 1 0 0 0 0 2745 "punpckldq %%mm0, %%mm0 \n\t" // 0 0 0 0 0 0 0 0 2746 "punpckhdq %%mm3, %%mm3 \n\t" // 1 1 1 1 1 1 1 1 2747 "movq %%mm0, (%2) \n\t" 2748 "punpckhwd %%mm2, %%mm2 \n\t" // 3 3 3 3 2 2 2 2 2749 "movq %%mm3, 8(%2) \n\t" 2750 "movq %%mm2, %%mm4 \n\t" // 3 3 3 3 2 2 2 2 2751 "punpckldq %%mm2, %%mm2 \n\t" // 2 2 2 2 2 2 2 2 2752 "punpckhdq %%mm4, %%mm4 \n\t" // 3 3 3 3 3 3 3 3 2753 "movq %%mm2, 16(%2) \n\t" 2754 "sub $4, %1 \n\t" 2755 "movq %%mm4, 24(%2) \n\t" 2756 "sub $32, %2 \n\t" 2757 "subl $4, %%ecx \n\t" 2758 "jnz .loop1_pass0 \n\t" 2759 "EMMS \n\t" // DONE 2760 2761 : "=c" (dummy_value_c), // output regs (dummy) 2762 "=S" (dummy_value_S), 2763 "=D" (dummy_value_D) 2764 2765 : "0" (width_mmx), // ecx // input regs 2766 "1" (sptr), // esi/rsi 2767 "2" (dp) // edi/rdi 2768 2769 #if defined(PNG_CLOBBER_MMX_REGS_SUPPORTED) 2770 : "%mm0", "%mm1", "%mm2" // clobber list 2771 , "%mm3", "%mm4" 2772 #endif 2773 ); 2774 } 2775 2776 sptr -= width_mmx; 2777 dp -= width_mmx*8; 2778 for (i = width; i; i--) 2779 { 2780 int j; 2781 2782 /* I simplified this part in version 1.0.4e 2783 * here and in several other instances where 2784 * pixel_bytes == 1 -- GR-P 2785 * 2786 * Original code: 2787 * 2788 * png_byte v[8]; 2789 * png_memcpy(v, sptr, pixel_bytes); 2790 * for (j = 0; j < png_pass_inc[pass]; j++) 2791 * { 2792 * png_memcpy(dp, v, pixel_bytes); 2793 * dp -= pixel_bytes; 2794 * } 2795 * sptr -= pixel_bytes; 2796 * 2797 * Replacement code is in the next three lines: 2798 */ 2799 2800 for (j = 0; j < png_pass_inc[pass]; j++) 2801 { 2802 *dp-- = *sptr; 2803 } 2804 --sptr; 2805 } 2806 } 2807 } /* end of pixel_bytes == 1 */ 2808 2809 //-------------------------------------------------------------- 2810 else if (pixel_bytes == BPP2) 2811 { 2812 if (((pass == 4) || (pass == 5)) && width) 2813 { 2814 int width_mmx = ((width >> 1) << 1) ; 2815 width -= width_mmx; // 0,1 pixels => 0,2 bytes 2816 if (width_mmx) 2817 { 2818 __asm__ __volatile__ ( 2819 "sub $2, %1 \n\t" 2820 "sub $6, %2 \n\t" 2821 2822 ".loop2_pass4: \n\t" 2823 "movd (%1), %%mm0 \n\t" // x x x x 3 2 1 0 2824 "punpcklwd %%mm0, %%mm0 \n\t" // 3 2 3 2 1 0 1 0 2825 "sub $4, %1 \n\t" 2826 "movq %%mm0, (%2) \n\t" 2827 "sub $8, %2 \n\t" 2828 "subl $2, %%ecx \n\t" 2829 "jnz .loop2_pass4 \n\t" 2830 "EMMS \n\t" // DONE 2831 2832 : "=c" (dummy_value_c), // output regs (dummy) 2833 "=S" (dummy_value_S), 2834 "=D" (dummy_value_D) 2835 2836 : "0" (width_mmx), // ecx // input regs 2837 "1" (sptr), // esi/rsi 2838 "2" (dp) // edi/rdi 2839 2840 #if defined(PNG_CLOBBER_MMX_REGS_SUPPORTED) 2841 : "%mm0" // clobber list 2842 #endif 2843 ); 2844 } 2845 2846 sptr -= (width_mmx*BPP2 - BPP2); // sign fixed 2847 dp -= (width_mmx*2*BPP2 - BPP2); // sign fixed 2848 for (i = width; i; i--) 2849 { 2850 png_byte v[8]; 2851 int j; 2852 sptr -= BPP2; 2853 png_memcpy(v, sptr, BPP2); 2854 for (j = 0; j < png_pass_inc[pass]; j++) 2855 { 2856 dp -= BPP2; 2857 png_memcpy(dp, v, BPP2); 2858 } 2859 } 2860 } 2861 else if (((pass == 2) || (pass == 3)) && width) 2862 { 2863 int width_mmx = ((width >> 1) << 1) ; 2864 width -= width_mmx; // 0,1 pixels => 0,2 bytes 2865 if (width_mmx) 2866 { 2867 __asm__ __volatile__ ( 2868 "sub $2, %1 \n\t" 2869 "sub $14, %2 \n\t" 2870 2871 ".loop2_pass2: \n\t" 2872 "movd (%1), %%mm0 \n\t" // x x x x 3 2 1 0 2873 "punpcklwd %%mm0, %%mm0 \n\t" // 3 2 3 2 1 0 1 0 2874 "movq %%mm0, %%mm1 \n\t" // 3 2 3 2 1 0 1 0 2875 "punpckldq %%mm0, %%mm0 \n\t" // 1 0 1 0 1 0 1 0 2876 "punpckhdq %%mm1, %%mm1 \n\t" // 3 2 3 2 3 2 3 2 2877 "movq %%mm0, (%2) \n\t" 2878 "sub $4, %1 \n\t" 2879 "movq %%mm1, 8(%2) \n\t" 2880 "sub $16, %2 \n\t" 2881 "subl $2, %%ecx \n\t" 2882 "jnz .loop2_pass2 \n\t" 2883 "EMMS \n\t" // DONE 2884 2885 : "=c" (dummy_value_c), // output regs (dummy) 2886 "=S" (dummy_value_S), 2887 "=D" (dummy_value_D) 2888 2889 : "0" (width_mmx), // ecx // input regs 2890 "1" (sptr), // esi/rsi 2891 "2" (dp) // edi/rdi 2892 2893 #if defined(CLOBBER_MMX_REGS_SUPPORTED) 2894 : "%mm0", "%mm1" // clobber list 2895 #endif 2896 ); 2897 } 2898 2899 sptr -= (width_mmx*2 - 2); // sign fixed 2900 dp -= (width_mmx*8 - 2); // sign fixed 2901 for (i = width; i; i--) 2902 { 2903 png_byte v[8]; 2904 int j; 2905 sptr -= 2; 2906 png_memcpy(v, sptr, 2); 2907 for (j = 0; j < png_pass_inc[pass]; j++) 2908 { 2909 dp -= 2; 2910 png_memcpy(dp, v, 2); 2911 } 2912 } 2913 } 2914 else if (width) // && ((pass == 0) || (pass == 1)) 2915 { 2916 int width_mmx = ((width >> 1) << 1); 2917 width -= width_mmx; // 0,1 pixels => 0,2 bytes 2918 if (width_mmx) 2919 { 2920 __asm__ __volatile__ ( 2921 "sub $2, %1 \n\t" 2922 "sub $30, %2 \n\t" 2923 2924 ".loop2_pass0: \n\t" 2925 "movd (%1), %%mm0 \n\t" // x x x x 3 2 1 0 2926 "punpcklwd %%mm0, %%mm0 \n\t" // 3 2 3 2 1 0 1 0 2927 "movq %%mm0, %%mm1 \n\t" // 3 2 3 2 1 0 1 0 2928 "punpckldq %%mm0, %%mm0 \n\t" // 1 0 1 0 1 0 1 0 2929 "punpckhdq %%mm1, %%mm1 \n\t" // 3 2 3 2 3 2 3 2 2930 "movq %%mm0, (%2) \n\t" 2931 "movq %%mm0, 8(%2) \n\t" 2932 "movq %%mm1, 16(%2) \n\t" 2933 "sub $4, %1 \n\t" 2934 "movq %%mm1, 24(%2) \n\t" 2935 "sub $32, %2 \n\t" 2936 "subl $2, %%ecx \n\t" 2937 "jnz .loop2_pass0 \n\t" 2938 "EMMS \n\t" // DONE 2939 2940 : "=c" (dummy_value_c), // output regs (dummy) 2941 "=S" (dummy_value_S), 2942 "=D" (dummy_value_D) 2943 2944 : "0" (width_mmx), // ecx // input regs 2945 "1" (sptr), // esi/rsi 2946 "2" (dp) // edi/rdi 2947 2948 #if defined(CLOBBER_MMX_REGS_SUPPORTED) 2949 : "%mm0", "%mm1" // clobber list 2950 #endif 2951 ); 2952 } 2953 2954 sptr -= (width_mmx*2 - 2); // sign fixed 2955 dp -= (width_mmx*16 - 2); // sign fixed 2956 for (i = width; i; i--) 2957 { 2958 png_byte v[8]; 2959 int j; 2960 sptr -= 2; 2961 png_memcpy(v, sptr, 2); 2962 for (j = 0; j < png_pass_inc[pass]; j++) 2963 { 2964 dp -= 2; 2965 png_memcpy(dp, v, 2); 2966 } 2967 } 2968 } 2969 } /* end of pixel_bytes == 2 */ 2970 2971 //-------------------------------------------------------------- 2972 else if (pixel_bytes == BPP8) 2973 { 2974 // GRR TEST: should work, but needs testing (special 64-bit version of rpng2?) 2975 // GRR NOTE: no need to combine passes here! 2976 if (((pass == 4) || (pass == 5)) && width) 2977 { 2978 // source is 8-byte RRGGBBAA 2979 // dest is 16-byte RRGGBBAA RRGGBBAA 2980 __asm__ __volatile__ ( 2981 "sub $8, %2 \n\t" // start of last block 2982 2983 ".loop8_pass4: \n\t" 2984 "movq (%1), %%mm0 \n\t" // 7 6 5 4 3 2 1 0 2985 "movq %%mm0, (%2) \n\t" 2986 "sub $8, %1 \n\t" 2987 "movq %%mm0, 8(%2) \n\t" 2988 "sub $16, %2 \n\t" 2989 "decl %%ecx \n\t" 2990 "jnz .loop8_pass4 \n\t" 2991 "EMMS \n\t" // DONE 2992 2993 : "=c" (dummy_value_c), // output regs (dummy) 2994 "=S" (dummy_value_S), 2995 "=D" (dummy_value_D) 2996 2997 : "0" (width), // ecx // input regs 2998 "1" (sptr), // esi/rsi 2999 "2" (dp) // edi/rdi 3000 3001 #if defined(CLOBBER_MMX_REGS_SUPPORTED) 3002 : "%mm0" // clobber list 3003 #endif 3004 ); 3005 } 3006 else if (((pass == 2) || (pass == 3)) && width) 3007 { 3008 // source is 8-byte RRGGBBAA 3009 // dest is 32-byte RRGGBBAA RRGGBBAA RRGGBBAA RRGGBBAA 3010 // (recall that expansion is _in place_: sptr and dp 3011 // both point at locations within same row buffer) 3012 __asm__ __volatile__ ( 3013 "sub $24, %2 \n\t" // start of last block 3014 3015 ".loop8_pass2: \n\t" 3016 "movq (%1), %%mm0 \n\t" // 7 6 5 4 3 2 1 0 3017 "movq %%mm0, (%2) \n\t" 3018 "movq %%mm0, 8(%2) \n\t" 3019 "movq %%mm0, 16(%2) \n\t" 3020 "sub $8, %1 \n\t" 3021 "movq %%mm0, 24(%2) \n\t" 3022 "sub $32, %2 \n\t" 3023 "decl %%ecx \n\t" 3024 "jnz .loop8_pass2 \n\t" 3025 "EMMS \n\t" // DONE 3026 3027 : "=c" (dummy_value_c), // output regs (dummy) 3028 "=S" (dummy_value_S), 3029 "=D" (dummy_value_D) 3030 3031 : "0" (width), // ecx // input regs 3032 "1" (sptr), // esi/rsi 3033 "2" (dp) // edi/rdi 3034 3035 #if defined(PNG_CLOBBER_MMX_REGS_SUPPORTED) 3036 : "%mm0" // clobber list 3037 #endif 3038 ); 3039 } 3040 else if (width) // && ((pass == 0) || (pass == 1)) 3041 { 3042 // source is 8-byte RRGGBBAA 3043 // dest is 64-byte RRGGBBAA RRGGBBAA RRGGBBAA RRGGBBAA ... 3044 __asm__ __volatile__ ( 3045 "sub $56, %2 \n\t" // start of last block 3046 3047 ".loop8_pass0: \n\t" 3048 "movq (%1), %%mm0 \n\t" // 7 6 5 4 3 2 1 0 3049 "movq %%mm0, (%2) \n\t" 3050 "movq %%mm0, 8(%2) \n\t" 3051 "movq %%mm0, 16(%2) \n\t" 3052 "movq %%mm0, 24(%2) \n\t" 3053 "movq %%mm0, 32(%2) \n\t" 3054 "movq %%mm0, 40(%2) \n\t" 3055 "movq %%mm0, 48(%2) \n\t" 3056 "sub $8, %1 \n\t" 3057 "movq %%mm0, 56(%2) \n\t" 3058 "sub $64, %2 \n\t" 3059 "decl %%ecx \n\t" 3060 "jnz .loop8_pass0 \n\t" 3061 "EMMS \n\t" // DONE 3062 3063 : "=c" (dummy_value_c), // output regs (dummy) 3064 "=S" (dummy_value_S), 3065 "=D" (dummy_value_D) 3066 3067 : "0" (width), // ecx // input regs 3068 "1" (sptr), // esi/rsi 3069 "2" (dp) // edi/rdi 3070 3071 #if defined(PNG_CLOBBER_MMX_REGS_SUPPORTED) 3072 : "%mm0" // clobber list 3073 #endif 3074 ); 3075 } 3076 } /* end of pixel_bytes == 8 */ 3077 3078 //-------------------------------------------------------------- 3079 else if (pixel_bytes == BPP6) // why no MMX for this case? 3080 { 3081 for (i = width; i; i--) 3082 { 3083 png_byte v[8]; 3084 int j; 3085 png_memcpy(v, sptr, BPP6); 3086 for (j = 0; j < png_pass_inc[pass]; j++) 3087 { 3088 png_memcpy(dp, v, BPP6); 3089 dp -= BPP6; 3090 } 3091 sptr -= BPP6; 3092 } 3093 } /* end of pixel_bytes == 6 */ 3094 3095 //-------------------------------------------------------------- 3096 else 3097 { 3098 // ERROR: SHOULD NEVER BE REACHED 3099 #if defined(PNG_DEBUG) 3100 png_debug(1, "Internal libpng logic error (GCC " 3101 "png_do_read_interlace() _mmx_supported)\n"); 3102 #endif 3103 } 3104 3105 } // end of _mmx_supported ======================================== 3106 3107 else /* MMX not supported: use modified C code - takes advantage 3108 * of inlining of png_memcpy for a constant */ 3109 { 3110 if (pixel_bytes == BPP3) 3111 { 3112 for (i = width; i; i--) 3113 { 3114 png_byte v[8]; 3115 int j; 3116 png_memcpy(v, sptr, BPP3); 3117 for (j = 0; j < png_pass_inc[pass]; j++) 3118 { 3119 png_memcpy(dp, v, BPP3); 3120 dp -= BPP3; 3121 } 3122 sptr -= BPP3; 3123 } 3124 } 3125 else if (pixel_bytes == BPP4) 3126 { 3127 for (i = width; i; i--) 3128 { 3129 png_byte v[8]; 3130 int j; 3131 png_memcpy(v, sptr, BPP4); 3132 for (j = 0; j < png_pass_inc[pass]; j++) 3133 { 3134 #if defined(PNG_DEBUG) && defined(PNG_1_0_X) // row_buf_size gone in 1.2.x 3135 if (dp < row || dp+3 > row+png_ptr->row_buf_size) 3136 { 3137 printf("dp out of bounds: row=%10p, dp=%10p, " 3138 "rp=%10p\n", row, dp, row+png_ptr->row_buf_size); 3139 printf("row_buf_size=%lu\n", png_ptr->row_buf_size); 3140 } 3141 #endif 3142 png_memcpy(dp, v, BPP4); 3143 dp -= BPP4; 3144 } 3145 sptr -= BPP4; 3146 } 3147 } 3148 else if (pixel_bytes == 1) 3149 { 3150 for (i = width; i; i--) 3151 { 3152 int j; 3153 for (j = 0; j < png_pass_inc[pass]; j++) 3154 { 3155 *dp-- = *sptr; 3156 } 3157 --sptr; 3158 } 3159 } 3160 else if (pixel_bytes == BPP2) 3161 { 3162 for (i = width; i; i--) 3163 { 3164 png_byte v[8]; 3165 int j; 3166 png_memcpy(v, sptr, BPP2); 3167 for (j = 0; j < png_pass_inc[pass]; j++) 3168 { 3169 png_memcpy(dp, v, BPP2); 3170 dp -= BPP2; 3171 } 3172 sptr -= BPP2; 3173 } 3174 } 3175 else if (pixel_bytes == BPP6) 3176 { 3177 for (i = width; i; i--) 3178 { 3179 png_byte v[8]; 3180 int j; 3181 png_memcpy(v, sptr, BPP6); 3182 for (j = 0; j < png_pass_inc[pass]; j++) 3183 { 3184 png_memcpy(dp, v, BPP6); 3185 dp -= BPP6; 3186 } 3187 sptr -= BPP6; 3188 } 3189 } 3190 else if (pixel_bytes == BPP8) 3191 { 3192 for (i = width; i; i--) 3193 { 3194 png_byte v[8]; 3195 int j; 3196 png_memcpy(v, sptr, BPP8); 3197 for (j = 0; j < png_pass_inc[pass]; j++) 3198 { 3199 png_memcpy(dp, v, BPP8); 3200 dp -= BPP8; 3201 } 3202 sptr -= BPP8; 3203 } 3204 } 3205 else 3206 { 3207 // ERROR: SHOULD NEVER BE REACHED 3208 #if defined(PNG_DEBUG) 3209 png_debug(1, "Internal libpng logic error (GCC " 3210 "png_do_read_interlace() !_mmx_supported)\n"); 3211 #endif 3212 } 3213 3214 } /* end if (MMX not supported) */ 3215 break; 3216 } /* end default (8-bit or larger) */ 3217 } /* end switch (row_info->pixel_depth) */ 3218 3219 row_info->width = final_width; 3220 3221 row_info->rowbytes = PNG_ROWBYTES(row_info->pixel_depth,final_width); 3222 } 3223 3224 } /* end png_do_read_interlace() */ 3225 3226 #endif /* PNG_HAVE_MMX_READ_INTERLACE */ 3227 #endif /* PNG_READ_INTERLACING_SUPPORTED */ 3228 3229 3230 3231 #if defined(PNG_HAVE_MMX_READ_FILTER_ROW) 3232 #if defined(PNG_MMX_READ_FILTER_AVG_SUPPORTED) 3233 3234 //===========================================================================// 3235 // // 3236 // P N G _ R E A D _ F I L T E R _ R O W _ M M X _ A V G // 3237 // // 3238 //===========================================================================// 3239 3240 // Optimized code for PNG Average filter decoder 3241 3242 static void /* PRIVATE */ 3243 png_read_filter_row_mmx_avg(png_row_infop row_info, png_bytep row, 3244 png_bytep prev_row) 3245 { 3246 unsigned FullLength, MMXLength; // png_uint_32 is actually 64-bit on x86-64 3247 int bpp; 3248 int dummy_value_a; 3249 int dummy_value_c; // fix 'forbidden register 2 (cx) was spilled' error 3250 int dummy_value_d; 3251 png_bytep dummy_value_S; 3252 png_bytep dummy_value_D; 3253 int diff; // __attribute__((used)); 3254 3255 bpp = (row_info->pixel_depth + 7) >> 3; // calc number of bytes per pixel 3256 FullLength = row_info->rowbytes; // number of bytes to filter 3257 3258 __asm__ __volatile__ ( 3259 "avg_top: \n\t" 3260 SAVE_GOT_ebx 3261 SAVE_r15 3262 SAVE_ebp 3263 // initialize address pointers and offset 3264 //pre "movl row, %5 \n\t" // edi/rdi: ptr to Avg(x) 3265 "xorl %%ebx, %%ebx \n\t" // ebx: x 3266 //pre "movl prev_row, %4 \n\t" // esi/rsi: ptr to Prior(x) 3267 "mov %5, " PDX " \n\t" // copy of row ptr... 3268 //pre "subl bpp, " PDX " \n\t" // (bpp is preloaded into ecx) 3269 "sub " PCX "," PDX " \n\t" // edx/rdx: ptr to Raw(x-bpp) 3270 //pre "movl FullLength, %%eax \n\t" // bring in via eax... 3271 SAVE_FullLength // ...but store for later use 3272 "xorl %%eax, %%eax \n\t" 3273 3274 // Compute the Raw value for the first bpp bytes 3275 // Raw(x) = Avg(x) + (Prior(x)/2) 3276 "avg_rlp: \n\t" 3277 "movb (%4," PBX ",), %%al \n\t" // load al with Prior(x) 3278 "incl %%ebx \n\t" 3279 "shrb %%al \n\t" // divide by 2 3280 "addb -1(%5," PBX ",), %%al \n\t" // add Avg(x); -1 to offset inc ebx 3281 //pre "cmpl bpp, %%ebx \n\t" // (bpp is preloaded into ecx) 3282 "cmpl %%ecx, %%ebx \n\t" 3283 "movb %%al, -1(%5," PBX ",) \n\t" // write Raw(x); -1 to offset inc ebx 3284 "jb avg_rlp \n\t" // mov does not affect flags 3285 3286 // get # of bytes to alignment (32-bit mask _would_ be good enough 3287 // [computing delta], but 32-bit ops are zero-extended on 64-bit, argh) 3288 // (if swapped edx and ebp, could do 8-bit or 16-bit mask...FIXME?) 3289 "mov %5, " PBP " \n\t" // take start of row 3290 "add " PBX "," PBP " \n\t" // add bpp 3291 "add $0xf, " PBP " \n\t" // add 7+8 to incr past alignment bdry 3292 // "andl $0xfffffff8, %%ebp \n\t" // mask to alignment boundary (32-bit!) 3293 CLEAR_BOTTOM_3_BITS PBP "\n\t" // mask to alignment boundary 3294 "sub %5, " PBP " \n\t" // subtract row ptr again => ebp = 3295 "jz avg_go \n\t" // target value of ebx at alignment 3296 3297 "xorl %%ecx, %%ecx \n\t" 3298 3299 // fix alignment 3300 // Compute the Raw value for the bytes up to the alignment boundary 3301 // Raw(x) = Avg(x) + ((Raw(x-bpp) + Prior(x))/2) 3302 "avg_lp1: \n\t" 3303 "xorl %%eax, %%eax \n\t" 3304 "movb (%4," PBX ",), %%cl \n\t" // load cl with Prior(x) 3305 "movb (" PDX "," PBX ",), %%al \n\t" // load al with Raw(x-bpp) 3306 "addw %%cx, %%ax \n\t" 3307 "incl %%ebx \n\t" 3308 "shrw %%ax \n\t" // divide by 2 3309 "addb -1(%5," PBX ",), %%al \n\t" // add Avg(x); -1 to offset inc ebx 3310 "cmpl %%ebp, %%ebx \n\t" // check if at alignment boundary 3311 "movb %%al, -1(%5," PBX ",) \n\t" // write Raw(x); -1 to offset inc ebx 3312 "jb avg_lp1 \n\t" // repeat until at alignment boundary 3313 3314 "avg_go: \n\t" 3315 RESTORE_FullLength "%%eax \n\t" // FullLength -> eax 3316 "movl %%eax, %%ecx \n\t" // copy -> ecx 3317 "subl %%ebx, %%eax \n\t" // subtract alignment fix 3318 "andl $0x00000007, %%eax \n\t" // calc bytes over mult of 8 3319 "subl %%eax, %%ecx \n\t" // sub over-bytes from original length 3320 //out "movl %%ecx, MMXLength \n\t" 3321 "movl %%ebp, %%eax \n\t" // ebp = diff, but no reg constraint(?) 3322 RESTORE_ebp // (could swap ebp and edx functions) 3323 RESTORE_r15 3324 RESTORE_GOT_ebx 3325 3326 // "There is no way for you to specify that an input operand is modified 3327 // without also specifying it as an output operand." [makes sense] 3328 3329 // "Unless an output operand has the `&' constraint modifier, GCC may 3330 // allocate it in the same register as an unrelated input operand, on the 3331 // assumption the inputs are consumed before the outputs are produced." 3332 // [trying to _force_ this] 3333 3334 // "`=' Means that this operand is write-only for this instruction: 3335 // the previous value is discarded and replaced by output data." 3336 // [operand == variable name, presumably] 3337 3338 // output regs 3339 // these are operands 0-1 (originally 0-3): 3340 : "=c" (MMXLength), // %0 -> %0 3341 "=a" (diff) // %3 -> %1 3342 // "=S" (dummy_value_S), // %1 -> GONE 3343 // "=D" (dummy_value_D), // %2 -> GONE 3344 3345 // input regs 3346 // these are operands 2-5 (originally 4-7); two of their constraints say 3347 // they must go in same places as operands 0-1 (originally 0-3) above: 3348 : "0" (bpp), // %4 -> %2 ecx 3349 "1" (FullLength), // %7 -> %3 eax 3350 "S" (prev_row), // %5 -> %4 esi/rsi 3351 "D" (row) // %6 -> %5 edi/rdi 3352 3353 : "%edx" // clobber list 3354 _CLOBBER_r15 3355 _CLOBBER_ebp 3356 _CLOBBER_GOT_ebx 3357 ); 3358 3359 // now do the math for the rest of the row 3360 switch (bpp) 3361 { 3362 case 3: 3363 { 3364 // _ShiftBpp = 24; // == 3 * 8 3365 // _ShiftRem = 40; // == 64 - 24 3366 3367 __asm__ __volatile__ ( 3368 // re-init address pointers and offset 3369 LOAD_GOT_rbp 3370 "movq " AMASK5_3_0 ", %%mm7 \n\t" // _amask5_3_0 -> mm7 3371 // preload "movl diff, %%ecx \n\t" // ecx: x = offset to 3372 // alignment boundary 3373 "movq " LB_CARRY_MASK ", %%mm5 \n\t" // [interleave for parallel.?] 3374 // preload "movl row, %1 \n\t" // edi: Avg(x) 3375 "movq " HB_CLEAR_MASK ", %%mm4 \n\t" // _HBClearMask -> mm4 3376 // preload "movl prev_row, %0 \n\t" // esi: Prior(x) 3377 RESTORE_rbp 3378 3379 // prime the pump: load the first Raw(x-bpp) data set 3380 "movq -8(%1," PCX ",), %%mm2 \n\t"// load previous aligned 8 bytes 3381 // (correct pos. in loop below) 3382 "avg_3lp: \n\t" 3383 "movq (%1," PCX ",), %%mm0 \n\t" // load mm0 with Avg(x) 3384 "movq %%mm5, %%mm3 \n\t" 3385 "psrlq $40, %%mm2 \n\t" // correct position Raw(x-bpp) 3386 // data 3387 "movq (%0," PCX ",), %%mm1 \n\t" // load mm1 with Prior(x) 3388 "movq %%mm7, %%mm6 \n\t" 3389 "pand %%mm1, %%mm3 \n\t" // get lsb for each prevrow byte 3390 "psrlq $1, %%mm1 \n\t" // divide prev_row bytes by 2 3391 "pand %%mm4, %%mm1 \n\t" // clear invalid bit 7 of each 3392 // byte 3393 "paddb %%mm1, %%mm0 \n\t" // add (Prev_row/2) to Avg for 3394 // each byte 3395 // add 1st active group (Raw(x-bpp)/2) to average with LBCarry 3396 "movq %%mm3, %%mm1 \n\t" // now use mm1 for getting 3397 // LBCarrys 3398 "pand %%mm2, %%mm1 \n\t" // get LBCarrys for each byte 3399 // where both lsb's were == 1 3400 // (valid only for active group) 3401 "psrlq $1, %%mm2 \n\t" // divide raw bytes by 2 3402 "pand %%mm4, %%mm2 \n\t" // clear invalid bit 7 of each 3403 // byte 3404 "paddb %%mm1, %%mm2 \n\t" // add LBCarrys to Raw(x-bpp)/2 3405 // for each byte 3406 "pand %%mm6, %%mm2 \n\t" // leave only Active Group 1 3407 // bytes to add to Avg 3408 "paddb %%mm2, %%mm0 \n\t" // add (Raw/2) + LBCarrys to 3409 // Avg for each Active byte 3410 // add 2nd active group (Raw(x-bpp)/2) to average with _LBCarry 3411 "psllq $24, %%mm6 \n\t" // shift the mm6 mask to cover 3412 // bytes 3-5 3413 "movq %%mm0, %%mm2 \n\t" // mov updated Raws to mm2 3414 "psllq $24, %%mm2 \n\t" // shift data to pos. correctly 3415 "movq %%mm3, %%mm1 \n\t" // now use mm1 for getting 3416 // LBCarrys 3417 "pand %%mm2, %%mm1 \n\t" // get LBCarrys for each byte 3418 // where both lsb's were == 1 3419 // (valid only for active group) 3420 "psrlq $1, %%mm2 \n\t" // divide raw bytes by 2 3421 "pand %%mm4, %%mm2 \n\t" // clear invalid bit 7 of each 3422 // byte 3423 "paddb %%mm1, %%mm2 \n\t" // add LBCarrys to Raw(x-bpp)/2 3424 // for each byte 3425 "pand %%mm6, %%mm2 \n\t" // leave only Active Group 2 3426 // bytes to add to Avg 3427 "paddb %%mm2, %%mm0 \n\t" // add (Raw/2) + LBCarrys to 3428 // Avg for each Active byte 3429 3430 // add 3rd active group (Raw(x-bpp)/2) to average with _LBCarry 3431 "psllq $24, %%mm6 \n\t" // shift mm6 mask to cover last 3432 // two bytes 3433 "movq %%mm0, %%mm2 \n\t" // mov updated Raws to mm2 3434 "psllq $24, %%mm2 \n\t" // shift data to pos. correctly 3435 // Data need be shifted only once here to 3436 // get the correct x-bpp offset. 3437 "movq %%mm3, %%mm1 \n\t" // now use mm1 for getting 3438 // LBCarrys 3439 "pand %%mm2, %%mm1 \n\t" // get LBCarrys for each byte 3440 // where both 3441 // lsb's were == 1 (only valid for active group) 3442 "psrlq $1, %%mm2 \n\t" // divide raw bytes by 2 3443 "pand %%mm4, %%mm2 \n\t" // clear invalid bit 7 of each 3444 // byte 3445 "paddb %%mm1, %%mm2 \n\t" // add LBCarrys to Raw(x-bpp)/2 3446 // for each byte 3447 "pand %%mm6, %%mm2 \n\t" // leave only Active Group 2 3448 // bytes to add to Avg 3449 "addl $8, %%ecx \n\t" 3450 "paddb %%mm2, %%mm0 \n\t" // add (Raw/2) + LBCarrys to 3451 // Avg for each Active byte 3452 // now ready to write back to memory 3453 "movq %%mm0, -8(%1," PCX ",) \n\t" 3454 // move updated Raw(x) to use as Raw(x-bpp) for next loop 3455 "cmpl %%eax, %%ecx \n\t" // MMXLength 3456 "movq %%mm0, %%mm2 \n\t" // mov updated Raw(x) to mm2 3457 "jb avg_3lp \n\t" 3458 3459 : "=S" (dummy_value_S), // output regs (dummy) 3460 "=D" (dummy_value_D), 3461 "=c" (dummy_value_c), 3462 "=a" (dummy_value_a) 3463 3464 : "0" (prev_row), // esi/rsi // input regs 3465 "1" (row), // edi/rdi 3466 "2" (diff), // ecx 3467 "3" (MMXLength) // eax 3468 3469 #if defined(CLOBBER_MMX_REGS_SUPPORTED) 3470 : "%mm0", "%mm1", "%mm2", "%mm3" // clobber list 3471 , "%mm4", "%mm5", "%mm6", "%mm7" 3472 #endif 3473 ); 3474 } 3475 break; // end 3 bpp 3476 3477 case 4: // formerly shared with 6 bpp case via _ShiftBpp and _ShiftRem, 3478 { // but loop uses all 8 MMX regs, and psrlq/psllq require 64-bit 3479 // mem (PIC/.so problems), MMX reg (none left), or immediate 3480 // _ShiftBpp = bpp << 3; // 32 (psllq) 3481 // _ShiftRem = 64 - _ShiftBpp; // 32 (psrlq) 3482 3483 __asm__ __volatile__ ( 3484 LOAD_GOT_rbp 3485 "movq " HB_CLEAR_MASK ", %%mm4 \n\t" // _HBClearMask -> mm4 3486 "movq " LB_CARRY_MASK ", %%mm5 \n\t" // _LBCarryMask -> mm5 3487 // re-init address pointers and offset 3488 // preload "movl diff, %%ecx \n\t" // ecx: x = offset to 3489 // alignment boundary 3490 "movq " AMASK0_8_0 ", %%mm7 \n\t" // _amask0_8_0 -> mm7 3491 RESTORE_rbp 3492 3493 // ... and clear all bytes except for 1st active group 3494 // preload "movl row, %1 \n\t" // edi: Avg(x) 3495 "psrlq $32, %%mm7 \n\t" // was _ShiftRem 3496 // preload "movl prev_row, %0 \n\t" // esi: Prior(x) 3497 "movq %%mm7, %%mm6 \n\t" 3498 "psllq $32, %%mm6 \n\t" // mask for 2nd active group