1 /* 2 * Copyright 2015 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package android.hardware.camera2.cts.rs; 18 19 import android.graphics.Bitmap; 20 import android.hardware.camera2.CameraCharacteristics; 21 import android.hardware.camera2.CameraMetadata; 22 import android.hardware.camera2.CaptureResult; 23 import android.hardware.camera2.params.ColorSpaceTransform; 24 import android.hardware.camera2.params.LensShadingMap; 25 import android.renderscript.Allocation; 26 import android.renderscript.Element; 27 import android.renderscript.Float3; 28 import android.renderscript.Float4; 29 import android.renderscript.Int4; 30 import android.renderscript.Matrix3f; 31 import android.renderscript.RenderScript; 32 import android.renderscript.Type; 33 34 import android.hardware.camera2.cts.ScriptC_raw_converter; 35 import android.util.Log; 36 import android.util.Rational; 37 import android.util.SparseIntArray; 38 39 import java.util.Arrays; 40 41 /** 42 * Utility class providing methods for rendering RAW16 images into other colorspaces. 43 */ 44 public class RawConverter { 45 private static final String TAG = "RawConverter"; 46 private static final boolean DEBUG = Log.isLoggable(TAG, Log.DEBUG); 47 48 /** 49 * Matrix to convert from CIE XYZ colorspace to sRGB, Bradford-adapted to D65. 50 */ 51 private static final float[] sXYZtoRGBBradford = new float[] { 52 3.1338561f, -1.6168667f, -0.4906146f, 53 -0.9787684f, 1.9161415f, 0.0334540f, 54 0.0719453f, -0.2289914f, 1.4052427f 55 }; 56 57 /** 58 * Matrix to convert from the ProPhoto RGB colorspace to CIE XYZ colorspace. 59 */ 60 private static final float[] sProPhotoToXYZ = new float[] { 61 0.797779f, 0.135213f, 0.031303f, 62 0.288000f, 0.711900f, 0.000100f, 63 0.000000f, 0.000000f, 0.825105f 64 }; 65 66 /** 67 * Matrix to convert from CIE XYZ colorspace to ProPhoto RGB colorspace. 68 */ 69 private static final float[] sXYZtoProPhoto = new float[] { 70 1.345753f, -0.255603f, -0.051025f, 71 -0.544426f, 1.508096f, 0.020472f, 72 0.000000f, 0.000000f, 1.211968f 73 }; 74 75 /** 76 * Coefficients for a 3rd order polynomial, ordered from highest to lowest power. This 77 * polynomial approximates the default tonemapping curve used for ACR3. 78 */ 79 private static final float[] DEFAULT_ACR3_TONEMAP_CURVE_COEFFS = new float[] { 80 -0.7836f, 0.8469f, 0.943f, 0.0209f 81 }; 82 83 /** 84 * The D50 whitepoint coordinates in CIE XYZ colorspace. 85 */ 86 private static final float[] D50_XYZ = new float[] { 0.9642f, 1, 0.8249f }; 87 88 /** 89 * An array containing the color temperatures for standard reference illuminants. 90 */ 91 private static final SparseIntArray sStandardIlluminants = new SparseIntArray(); 92 private static final int NO_ILLUMINANT = -1; 93 static { 94 sStandardIlluminants.append(CameraMetadata.SENSOR_REFERENCE_ILLUMINANT1_DAYLIGHT, 6504); 95 sStandardIlluminants.append(CameraMetadata.SENSOR_REFERENCE_ILLUMINANT1_D65, 6504); 96 sStandardIlluminants.append(CameraMetadata.SENSOR_REFERENCE_ILLUMINANT1_D50, 5003); 97 sStandardIlluminants.append(CameraMetadata.SENSOR_REFERENCE_ILLUMINANT1_D55, 5503); 98 sStandardIlluminants.append(CameraMetadata.SENSOR_REFERENCE_ILLUMINANT1_D75, 7504); 99 sStandardIlluminants.append(CameraMetadata.SENSOR_REFERENCE_ILLUMINANT1_STANDARD_A, 2856); 100 sStandardIlluminants.append(CameraMetadata.SENSOR_REFERENCE_ILLUMINANT1_STANDARD_B, 4874); 101 sStandardIlluminants.append(CameraMetadata.SENSOR_REFERENCE_ILLUMINANT1_STANDARD_C, 6774); 102 sStandardIlluminants.append( 103 CameraMetadata.SENSOR_REFERENCE_ILLUMINANT1_DAYLIGHT_FLUORESCENT, 6430); 104 sStandardIlluminants.append( 105 CameraMetadata.SENSOR_REFERENCE_ILLUMINANT1_COOL_WHITE_FLUORESCENT, 4230); 106 sStandardIlluminants.append( 107 CameraMetadata.SENSOR_REFERENCE_ILLUMINANT1_WHITE_FLUORESCENT, 3450); 108 // TODO: Add the rest of the illuminants included in the LightSource EXIF tag. 109 } 110 111 /** 112 * Convert a RAW16 buffer into an sRGB buffer, and write the result into a bitmap. 113 * 114 * <p> This function applies the operations roughly outlined in the Adobe DNG specification 115 * using the provided metadata about the image sensor. Sensor data for Android devices is 116 * assumed to be relatively linear, and no extra linearization step is applied here. The 117 * following operations are applied in the given order:</p> 118 * 119 * <ul> 120 * <li> 121 * Black level subtraction - the black levels given in the SENSOR_BLACK_LEVEL_PATTERN 122 * tag are subtracted from the corresponding raw pixels. 123 * </li> 124 * <li> 125 * Rescaling - each raw pixel is scaled by 1/(white level - black level). 126 * </li> 127 * <li> 128 * Lens shading correction - the interpolated gains from the gain map defined in the 129 * STATISTICS_LENS_SHADING_CORRECTION_MAP are applied to each raw pixel. 130 * </li> 131 * <li> 132 * Clipping - each raw pixel is clipped to a range of [0.0, 1.0]. 133 * </li> 134 * <li> 135 * Demosaic - the RGB channels for each pixel are retrieved from the Bayer mosaic 136 * of raw pixels using a simple bilinear-interpolation demosaicing algorithm. 137 * </li> 138 * <li> 139 * Colorspace transform to wide-gamut RGB - each pixel is mapped into a 140 * wide-gamut colorspace (in this case ProPhoto RGB is used) from the sensor 141 * colorspace. 142 * </li> 143 * <li> 144 * Tonemapping - A basic tonemapping curve using the default from ACR3 is applied 145 * (no further exposure compensation is applied here, though this could be improved). 146 * </li> 147 * <li> 148 * Colorspace transform to final RGB - each pixel is mapped into linear sRGB colorspace. 149 * </li> 150 * <li> 151 * Gamma correction - each pixel is gamma corrected using =2.2 to map into sRGB 152 * colorspace for viewing. 153 * </li> 154 * <li> 155 * Packing - each pixel is scaled so that each color channel has a range of [0, 255], 156 * and is packed into an Android bitmap. 157 * </li> 158 * </ul> 159 * 160 * <p> Arguments given here are assumed to come from the values for the corresponding 161 * {@link CameraCharacteristics.Key}s defined for the camera that produced this RAW16 buffer. 162 * </p> 163 * @param rs a {@link RenderScript} context to use. 164 * @param inputWidth width of the input RAW16 image in pixels. 165 * @param inputHeight height of the input RAW16 image in pixels. 166 * @param inputStride stride of the input RAW16 image in bytes. 167 * @param rawImageInput a byte array containing a RAW16 image. 168 * @param staticMetadata the {@link CameraCharacteristics} for this RAW capture. 169 * @param dynamicMetadata the {@link CaptureResult} for this RAW capture. 170 * @param outputOffsetX the offset width into the raw image of the left side of the output 171 * rectangle. 172 * @param outputOffsetY the offset height into the raw image of the top side of the output 173 * rectangle. 174 * @param argbOutput a {@link Bitmap} to output the rendered RAW image into. The height and 175 * width of this bitmap along with the output offsets are used to determine 176 * the dimensions and offset of the output rectangle contained in the RAW 177 * image to be rendered. 178 */ 179 public static void convertToSRGB(RenderScript rs, int inputWidth, int inputHeight, 180 int inputStride, byte[] rawImageInput, CameraCharacteristics staticMetadata, 181 CaptureResult dynamicMetadata, int outputOffsetX, int outputOffsetY, 182 /*out*/Bitmap argbOutput) { 183 int cfa = staticMetadata.get(CameraCharacteristics.SENSOR_INFO_COLOR_FILTER_ARRANGEMENT); 184 int[] blackLevelPattern = new int[4]; 185 staticMetadata.get(CameraCharacteristics.SENSOR_BLACK_LEVEL_PATTERN). 186 copyTo(blackLevelPattern, /*offset*/0); 187 int whiteLevel = staticMetadata.get(CameraCharacteristics.SENSOR_INFO_WHITE_LEVEL); 188 int ref1 = staticMetadata.get(CameraCharacteristics.SENSOR_REFERENCE_ILLUMINANT1); 189 int ref2; 190 if (staticMetadata.get(CameraCharacteristics.SENSOR_REFERENCE_ILLUMINANT2) != null) { 191 ref2 = staticMetadata.get(CameraCharacteristics.SENSOR_REFERENCE_ILLUMINANT2); 192 } 193 else { 194 ref2 = ref1; 195 } 196 float[] calib1 = new float[9]; 197 float[] calib2 = new float[9]; 198 convertColorspaceTransform( 199 staticMetadata.get(CameraCharacteristics.SENSOR_CALIBRATION_TRANSFORM1), calib1); 200 if (staticMetadata.get(CameraCharacteristics.SENSOR_CALIBRATION_TRANSFORM2) != null) { 201 convertColorspaceTransform( 202 staticMetadata.get(CameraCharacteristics.SENSOR_CALIBRATION_TRANSFORM2), calib2); 203 } 204 else { 205 convertColorspaceTransform( 206 staticMetadata.get(CameraCharacteristics.SENSOR_CALIBRATION_TRANSFORM1), calib2); 207 } 208 float[] color1 = new float[9]; 209 float[] color2 = new float[9]; 210 convertColorspaceTransform( 211 staticMetadata.get(CameraCharacteristics.SENSOR_COLOR_TRANSFORM1), color1); 212 if (staticMetadata.get(CameraCharacteristics.SENSOR_COLOR_TRANSFORM2) != null) { 213 convertColorspaceTransform( 214 staticMetadata.get(CameraCharacteristics.SENSOR_COLOR_TRANSFORM2), color2); 215 } 216 else { 217 convertColorspaceTransform( 218 staticMetadata.get(CameraCharacteristics.SENSOR_COLOR_TRANSFORM1), color2); 219 } 220 float[] forward1 = new float[9]; 221 float[] forward2 = new float[9]; 222 convertColorspaceTransform( 223 staticMetadata.get(CameraCharacteristics.SENSOR_FORWARD_MATRIX1), forward1); 224 if (staticMetadata.get(CameraCharacteristics.SENSOR_FORWARD_MATRIX2) != null) { 225 convertColorspaceTransform( 226 staticMetadata.get(CameraCharacteristics.SENSOR_FORWARD_MATRIX2), forward2); 227 } 228 else { 229 convertColorspaceTransform( 230 staticMetadata.get(CameraCharacteristics.SENSOR_FORWARD_MATRIX1), forward2); 231 } 232 233 Rational[] neutral = dynamicMetadata.get(CaptureResult.SENSOR_NEUTRAL_COLOR_POINT); 234 235 LensShadingMap shadingMap = dynamicMetadata.get(CaptureResult.STATISTICS_LENS_SHADING_CORRECTION_MAP); 236 237 convertToSRGB(rs, inputWidth, inputHeight, inputStride, cfa, blackLevelPattern, whiteLevel, 238 rawImageInput, ref1, ref2, calib1, calib2, color1, color2, 239 forward1, forward2, neutral, shadingMap, outputOffsetX, outputOffsetY, argbOutput); 240 } 241 242 /** 243 * Convert a RAW16 buffer into an sRGB buffer, and write the result into a bitmap. 244 * 245 * @see #convertToSRGB 246 */ 247 private static void convertToSRGB(RenderScript rs, int inputWidth, int inputHeight, 248 int inputStride, int cfa, int[] blackLevelPattern, int whiteLevel, byte[] rawImageInput, 249 int referenceIlluminant1, int referenceIlluminant2, float[] calibrationTransform1, 250 float[] calibrationTransform2, float[] colorMatrix1, float[] colorMatrix2, 251 float[] forwardTransform1, float[] forwardTransform2, Rational[/*3*/] neutralColorPoint, 252 LensShadingMap lensShadingMap, int outputOffsetX, int outputOffsetY, 253 /*out*/Bitmap argbOutput) { 254 255 // Validate arguments 256 if (argbOutput == null || rs == null || rawImageInput == null) { 257 throw new IllegalArgumentException("Null argument to convertToSRGB"); 258 } 259 if (argbOutput.getConfig() != Bitmap.Config.ARGB_8888) { 260 throw new IllegalArgumentException( 261 "Output bitmap passed to convertToSRGB is not ARGB_8888 format"); 262 } 263 if (outputOffsetX < 0 || outputOffsetY < 0) { 264 throw new IllegalArgumentException("Negative offset passed to convertToSRGB"); 265 } 266 if ((inputStride / 2) < inputWidth) { 267 throw new IllegalArgumentException("Stride too small."); 268 } 269 if ((inputStride % 2) != 0) { 270 throw new IllegalArgumentException("Invalid stride for RAW16 format, see graphics.h."); 271 } 272 int outWidth = argbOutput.getWidth(); 273 int outHeight = argbOutput.getHeight(); 274 if (outWidth + outputOffsetX > inputWidth || outHeight + outputOffsetY > inputHeight) { 275 throw new IllegalArgumentException("Raw image with dimensions (w=" + inputWidth + 276 ", h=" + inputHeight + "), cannot converted into sRGB image with dimensions (w=" 277 + outWidth + ", h=" + outHeight + ")."); 278 } 279 if (cfa < 0 || cfa > 3) { 280 throw new IllegalArgumentException("Unsupported cfa pattern " + cfa + " used."); 281 } 282 if (DEBUG) { 283 Log.d(TAG, "Metadata Used:"); 284 Log.d(TAG, "Input width,height: " + inputWidth + "," + inputHeight); 285 Log.d(TAG, "Output offset x,y: " + outputOffsetX + "," + outputOffsetY); 286 Log.d(TAG, "Output width,height: " + outWidth + "," + outHeight); 287 Log.d(TAG, "CFA: " + cfa); 288 Log.d(TAG, "BlackLevelPattern: " + Arrays.toString(blackLevelPattern)); 289 Log.d(TAG, "WhiteLevel: " + whiteLevel); 290 Log.d(TAG, "ReferenceIlluminant1: " + referenceIlluminant1); 291 Log.d(TAG, "ReferenceIlluminant2: " + referenceIlluminant2); 292 Log.d(TAG, "CalibrationTransform1: " + Arrays.toString(calibrationTransform1)); 293 Log.d(TAG, "CalibrationTransform2: " + Arrays.toString(calibrationTransform2)); 294 Log.d(TAG, "ColorMatrix1: " + Arrays.toString(colorMatrix1)); 295 Log.d(TAG, "ColorMatrix2: " + Arrays.toString(colorMatrix2)); 296 Log.d(TAG, "ForwardTransform1: " + Arrays.toString(forwardTransform1)); 297 Log.d(TAG, "ForwardTransform2: " + Arrays.toString(forwardTransform2)); 298 Log.d(TAG, "NeutralColorPoint: " + Arrays.toString(neutralColorPoint)); 299 } 300 301 Allocation gainMap = null; 302 if (lensShadingMap != null) { 303 float[] lsm = new float[lensShadingMap.getGainFactorCount()]; 304 lensShadingMap.copyGainFactors(/*inout*/lsm, /*offset*/0); 305 gainMap = createFloat4Allocation(rs, lsm, lensShadingMap.getColumnCount(), 306 lensShadingMap.getRowCount()); 307 } 308 309 float[] normalizedForwardTransform1 = Arrays.copyOf(forwardTransform1, 310 forwardTransform1.length); 311 normalizeFM(normalizedForwardTransform1); 312 float[] normalizedForwardTransform2 = Arrays.copyOf(forwardTransform2, 313 forwardTransform2.length); 314 normalizeFM(normalizedForwardTransform2); 315 316 float[] normalizedColorMatrix1 = Arrays.copyOf(colorMatrix1, colorMatrix1.length); 317 normalizeCM(normalizedColorMatrix1); 318 float[] normalizedColorMatrix2 = Arrays.copyOf(colorMatrix2, colorMatrix2.length); 319 normalizeCM(normalizedColorMatrix2); 320 321 if (DEBUG) { 322 Log.d(TAG, "Normalized ForwardTransform1: " + Arrays.toString(normalizedForwardTransform1)); 323 Log.d(TAG, "Normalized ForwardTransform2: " + Arrays.toString(normalizedForwardTransform2)); 324 Log.d(TAG, "Normalized ColorMatrix1: " + Arrays.toString(normalizedColorMatrix1)); 325 Log.d(TAG, "Normalized ColorMatrix2: " + Arrays.toString(normalizedColorMatrix2)); 326 } 327 328 // Calculate full sensor colorspace to sRGB colorspace transform. 329 double interpolationFactor = findDngInterpolationFactor(referenceIlluminant1, 330 referenceIlluminant2, calibrationTransform1, calibrationTransform2, 331 normalizedColorMatrix1, normalizedColorMatrix2, neutralColorPoint); 332 if (DEBUG) Log.d(TAG, "Interpolation factor used: " + interpolationFactor); 333 float[] sensorToXYZ = new float[9]; 334 calculateCameraToXYZD50Transform(normalizedForwardTransform1, normalizedForwardTransform2, 335 calibrationTransform1, calibrationTransform2, neutralColorPoint, 336 interpolationFactor, /*out*/sensorToXYZ); 337 if (DEBUG) Log.d(TAG, "CameraToXYZ xform used: " + Arrays.toString(sensorToXYZ)); 338 float[] sensorToProPhoto = new float[9]; 339 multiply(sXYZtoProPhoto, sensorToXYZ, /*out*/sensorToProPhoto); 340 if (DEBUG) Log.d(TAG, "CameraToIntemediate xform used: " + Arrays.toString(sensorToProPhoto)); 341 Allocation output = Allocation.createFromBitmap(rs, argbOutput); 342 343 float[] proPhotoToSRGB = new float[9]; 344 multiply(sXYZtoRGBBradford, sProPhotoToXYZ, /*out*/proPhotoToSRGB); 345 346 // Setup input allocation (16-bit raw pixels) 347 Type.Builder typeBuilder = new Type.Builder(rs, Element.U16(rs)); 348 typeBuilder.setX((inputStride / 2)); 349 typeBuilder.setY(inputHeight); 350 Type inputType = typeBuilder.create(); 351 Allocation input = Allocation.createTyped(rs, inputType); 352 input.copyFromUnchecked(rawImageInput); 353 354 // Setup RS kernel globals 355 ScriptC_raw_converter converterKernel = new ScriptC_raw_converter(rs); 356 converterKernel.set_inputRawBuffer(input); 357 converterKernel.set_whiteLevel(whiteLevel); 358 converterKernel.set_sensorToIntermediate(new Matrix3f(transpose(sensorToProPhoto))); 359 converterKernel.set_intermediateToSRGB(new Matrix3f(transpose(proPhotoToSRGB))); 360 converterKernel.set_offsetX(outputOffsetX); 361 converterKernel.set_offsetY(outputOffsetY); 362 converterKernel.set_rawHeight(inputHeight); 363 converterKernel.set_rawWidth(inputWidth); 364 converterKernel.set_neutralPoint(new Float3(neutralColorPoint[0].floatValue(), 365 neutralColorPoint[1].floatValue(), neutralColorPoint[2].floatValue())); 366 converterKernel.set_toneMapCoeffs(new Float4(DEFAULT_ACR3_TONEMAP_CURVE_COEFFS[0], 367 DEFAULT_ACR3_TONEMAP_CURVE_COEFFS[1], DEFAULT_ACR3_TONEMAP_CURVE_COEFFS[2], 368 DEFAULT_ACR3_TONEMAP_CURVE_COEFFS[3])); 369 converterKernel.set_hasGainMap(gainMap != null); 370 if (gainMap != null) { 371 converterKernel.set_gainMap(gainMap); 372 converterKernel.set_gainMapWidth(lensShadingMap.getColumnCount()); 373 converterKernel.set_gainMapHeight(lensShadingMap.getRowCount()); 374 } 375 376 converterKernel.set_cfaPattern(cfa); 377 converterKernel.set_blackLevelPattern(new Int4(blackLevelPattern[0], 378 blackLevelPattern[1], blackLevelPattern[2], blackLevelPattern[3])); 379 converterKernel.forEach_convert_RAW_To_ARGB(output); 380 output.copyTo(argbOutput); // Force RS sync with bitmap (does not do an extra copy). 381 } 382 383 /** 384 * Create a float-backed renderscript {@link Allocation} with the given dimensions, containing 385 * the contents of the given float array. 386 * 387 * @param rs a {@link RenderScript} context to use. 388 * @param fArray the float array to copy into the {@link Allocation}. 389 * @param width the width of the {@link Allocation}. 390 * @param height the height of the {@link Allocation}. 391 * @return an {@link Allocation} containing the given floats. 392 */ 393 private static Allocation createFloat4Allocation(RenderScript rs, float[] fArray, 394 int width, int height) { 395 if (fArray.length != width * height * 4) { 396 throw new IllegalArgumentException("Invalid float array of length " + fArray.length + 397 ", must be correct size for Allocation of dimensions " + width + "x" + height); 398 } 399 Type.Builder builder = new Type.Builder(rs, Element.F32_4(rs)); 400 builder.setX(width); 401 builder.setY(height); 402 Allocation fAlloc = Allocation.createTyped(rs, builder.create()); 403 fAlloc.copyFrom(fArray); 404 return fAlloc; 405 } 406 407 /** 408 * Calculate the correlated color temperature (CCT) for a given x,y chromaticity in CIE 1931 x,y 409 * chromaticity space using McCamy's cubic approximation algorithm given in: 410 * 411 * McCamy, Calvin S. (April 1992). 412 * "Correlated color temperature as an explicit function of chromaticity coordinates". 413 * Color Research & Application 17 (2): 142144 414 * 415 * @param x x chromaticity component. 416 * @param y y chromaticity component. 417 * 418 * @return the CCT associated with this chromaticity coordinate. 419 */ 420 private static double calculateColorTemperature(double x, double y) { 421 double n = (x - 0.332) / (y - 0.1858); 422 return -449 * Math.pow(n, 3) + 3525 * Math.pow(n, 2) - 6823.3 * n + 5520.33; 423 } 424 425 /** 426 * Calculate the x,y chromaticity coordinates in CIE 1931 x,y chromaticity space from the given 427 * CIE XYZ coordinates. 428 * 429 * @param X the CIE XYZ X coordinate. 430 * @param Y the CIE XYZ Y coordinate. 431 * @param Z the CIE XYZ Z coordinate. 432 * 433 * @return the [x, y] chromaticity coordinates as doubles. 434 */ 435 private static double[] calculateCIExyCoordinates(double X, double Y, double Z) { 436 double[] ret = new double[] { 0, 0 }; 437 ret[0] = X / (X + Y + Z); 438 ret[1] = Y / (X + Y + Z); 439 return ret; 440 } 441 442 /** 443 * Linearly interpolate between a and b given fraction f. 444 * 445 * @param a first term to interpolate between, a will be returned when f == 0. 446 * @param b second term to interpolate between, b will be returned when f == 1. 447 * @param f the fraction to interpolate by. 448 * 449 * @return interpolated result as double. 450 */ 451 private static double lerp(double a, double b, double f) { 452 return (a * (1.0f - f)) + (b * f); 453 } 454 455 /** 456 * Linearly interpolate between 3x3 matrices a and b given fraction f. 457 * 458 * @param a first 3x3 matrix to interpolate between, a will be returned when f == 0. 459 * @param b second 3x3 matrix to interpolate between, b will be returned when f == 1. 460 * @param f the fraction to interpolate by. 461 * @param result will be set to contain the interpolated matrix. 462 */ 463 private static void lerp(float[] a, float[] b, double f, /*out*/float[] result) { 464 for (int i = 0; i < 9; i++) { 465 result[i] = (float) lerp(a[i], b[i], f); 466 } 467 } 468 469 /** 470 * Convert a 9x9 {@link ColorSpaceTransform} to a matrix and write the matrix into the 471 * output. 472 * 473 * @param xform a {@link ColorSpaceTransform} to transform. 474 * @param output the 3x3 matrix to overwrite. 475 */ 476 private static void convertColorspaceTransform(ColorSpaceTransform xform, /*out*/float[] output) { 477 for (int i = 0; i < 3; i++) { 478 for (int j = 0; j < 3; j++) { 479 output[i * 3 + j] = xform.getElement(j, i).floatValue(); 480 } 481 } 482 } 483 484 /** 485 * Find the interpolation factor to use with the RAW matrices given a neutral color point. 486 * 487 * @param referenceIlluminant1 first reference illuminant. 488 * @param referenceIlluminant2 second reference illuminant. 489 * @param calibrationTransform1 calibration matrix corresponding to the first reference 490 * illuminant. 491 * @param calibrationTransform2 calibration matrix corresponding to the second reference 492 * illuminant. 493 * @param colorMatrix1 color matrix corresponding to the first reference illuminant. 494 * @param colorMatrix2 color matrix corresponding to the second reference illuminant. 495 * @param neutralColorPoint the neutral color point used to calculate the interpolation factor. 496 * 497 * @return the interpolation factor corresponding to the given neutral color point. 498 */ 499 private static double findDngInterpolationFactor(int referenceIlluminant1, 500 int referenceIlluminant2, float[] calibrationTransform1, float[] calibrationTransform2, 501 float[] colorMatrix1, float[] colorMatrix2, Rational[/*3*/] neutralColorPoint) { 502 503 int colorTemperature1 = sStandardIlluminants.get(referenceIlluminant1, NO_ILLUMINANT); 504 if (colorTemperature1 == NO_ILLUMINANT) { 505 throw new IllegalArgumentException("No such illuminant for reference illuminant 1: " + 506 referenceIlluminant1); 507 } 508 509 int colorTemperature2 = sStandardIlluminants.get(referenceIlluminant2, NO_ILLUMINANT); 510 if (colorTemperature2 == NO_ILLUMINANT) { 511 throw new IllegalArgumentException("No such illuminant for reference illuminant 2: " + 512 referenceIlluminant2); 513 } 514 515 if (DEBUG) Log.d(TAG, "ColorTemperature1: " + colorTemperature1); 516 if (DEBUG) Log.d(TAG, "ColorTemperature2: " + colorTemperature2); 517 518 double interpFactor = 0.5; // Initial guess for interpolation factor 519 double oldInterpFactor = interpFactor; 520 521 double lastDiff = Double.MAX_VALUE; 522 double tolerance = 0.0001; 523 float[] XYZToCamera1 = new float[9]; 524 float[] XYZToCamera2 = new float[9]; 525 multiply(calibrationTransform1, colorMatrix1, /*out*/XYZToCamera1); 526 multiply(calibrationTransform2, colorMatrix2, /*out*/XYZToCamera2); 527 528 float[] cameraNeutral = new float[] { neutralColorPoint[0].floatValue(), 529 neutralColorPoint[1].floatValue(), neutralColorPoint[2].floatValue()}; 530 531 float[] neutralGuess = new float[3]; 532 float[] interpXYZToCamera = new float[9]; 533 float[] interpXYZToCameraInverse = new float[9]; 534 535 536 double lower = Math.min(colorTemperature1, colorTemperature2); 537 double upper = Math.max(colorTemperature1, colorTemperature2); 538 539 if(DEBUG) { 540 Log.d(TAG, "XYZtoCamera1: " + Arrays.toString(XYZToCamera1)); 541 Log.d(TAG, "XYZtoCamera2: " + Arrays.toString(XYZToCamera2)); 542 Log.d(TAG, "Finding interpolation factor, initial guess 0.5..."); 543 } 544 // Iteratively guess xy value, find new CCT, and update interpolation factor. 545 int loopLimit = 30; 546 int count = 0; 547 while (lastDiff > tolerance && loopLimit > 0) { 548 if (DEBUG) Log.d(TAG, "Loop count " + count); 549 lerp(XYZToCamera1, XYZToCamera2, interpFactor, interpXYZToCamera); 550 if (!invert(interpXYZToCamera, /*out*/interpXYZToCameraInverse)) { 551 throw new IllegalArgumentException( 552 "Cannot invert XYZ to Camera matrix, input matrices are invalid."); 553 } 554 555 map(interpXYZToCameraInverse, cameraNeutral, /*out*/neutralGuess); 556 double[] xy = calculateCIExyCoordinates(neutralGuess[0], neutralGuess[1], 557 neutralGuess[2]); 558 559 double colorTemperature = calculateColorTemperature(xy[0], xy[1]); 560 561 if (colorTemperature <= lower) { 562 interpFactor = 1; 563 } else if (colorTemperature >= upper) { 564 interpFactor = 0; 565 } else { 566 double invCT = 1.0 / colorTemperature; 567 interpFactor = (invCT - 1.0 / upper) / ( 1.0 / lower - 1.0 / upper); 568 } 569 570 if (lower == colorTemperature1) { 571 interpFactor = 1.0 - interpFactor; 572 } 573 574 interpFactor = (interpFactor + oldInterpFactor) / 2; 575 lastDiff = Math.abs(oldInterpFactor - interpFactor); 576 oldInterpFactor = interpFactor; 577 loopLimit--; 578 count++; 579 580 if (DEBUG) { 581 Log.d(TAG, "CameraToXYZ chosen: " + Arrays.toString(interpXYZToCameraInverse)); 582 Log.d(TAG, "XYZ neutral color guess: " + Arrays.toString(neutralGuess)); 583 Log.d(TAG, "xy coordinate: " + Arrays.toString(xy)); 584 Log.d(TAG, "xy color temperature: " + colorTemperature); 585 Log.d(TAG, "New interpolation factor: " + interpFactor); 586 } 587 } 588 589 if (loopLimit == 0) { 590 Log.w(TAG, "Could not converge on interpolation factor, using factor " + interpFactor + 591 " with remaining error factor of " + lastDiff); 592 } 593 return interpFactor; 594 } 595 596 /** 597 * Calculate the transform from the raw camera sensor colorspace to CIE XYZ colorspace with a 598 * D50 whitepoint. 599 * 600 * @param forwardTransform1 forward transform matrix corresponding to the first reference 601 * illuminant. 602 * @param forwardTransform2 forward transform matrix corresponding to the second reference 603 * illuminant. 604 * @param calibrationTransform1 calibration transform matrix corresponding to the first 605 * reference illuminant. 606 * @param calibrationTransform2 calibration transform matrix corresponding to the second 607 * reference illuminant. 608 * @param neutralColorPoint the neutral color point used to calculate the interpolation factor. 609 * @param interpolationFactor the interpolation factor to use for the forward and 610 * calibration transforms. 611 * @param outputTransform set to the full sensor to XYZ colorspace transform. 612 */ 613 private static void calculateCameraToXYZD50Transform(float[] forwardTransform1, 614 float[] forwardTransform2, float[] calibrationTransform1, float[] calibrationTransform2, 615 Rational[/*3*/] neutralColorPoint, double interpolationFactor, 616 /*out*/float[] outputTransform) { 617 float[] cameraNeutral = new float[] { neutralColorPoint[0].floatValue(), 618 neutralColorPoint[1].floatValue(), neutralColorPoint[2].floatValue()}; 619 if (DEBUG) Log.d(TAG, "Camera neutral: " + Arrays.toString(cameraNeutral)); 620 621 float[] interpolatedCC = new float[9]; 622 lerp(calibrationTransform1, calibrationTransform2, interpolationFactor, 623 interpolatedCC); 624 float[] inverseInterpolatedCC = new float[9]; 625 if (!invert(interpolatedCC, /*out*/inverseInterpolatedCC)) { 626 throw new IllegalArgumentException( "Cannot invert interpolated calibration transform" + 627 ", input matrices are invalid."); 628 } 629 if (DEBUG) Log.d(TAG, "Inverted interpolated CalibrationTransform: " + 630 Arrays.toString(inverseInterpolatedCC)); 631 632 float[] referenceNeutral = new float[3]; 633 map(inverseInterpolatedCC, cameraNeutral, /*out*/referenceNeutral); 634 if (DEBUG) Log.d(TAG, "Reference neutral: " + Arrays.toString(referenceNeutral)); 635 float maxNeutral = Math.max(Math.max(referenceNeutral[0], referenceNeutral[1]), 636 referenceNeutral[2]); 637 float[] D = new float[] { maxNeutral/referenceNeutral[0], 0, 0, 638 0, maxNeutral/referenceNeutral[1], 0, 639 0, 0, maxNeutral/referenceNeutral[2] }; 640 if (DEBUG) Log.d(TAG, "Reference Neutral Diagonal: " + Arrays.toString(D)); 641 642 float[] intermediate = new float[9]; 643 float[] intermediate2 = new float[9]; 644 645 lerp(forwardTransform1, forwardTransform2, interpolationFactor, /*out*/intermediate); 646 if (DEBUG) Log.d(TAG, "Interpolated ForwardTransform: " + Arrays.toString(intermediate)); 647 648 multiply(D, inverseInterpolatedCC, /*out*/intermediate2); 649 multiply(intermediate, intermediate2, /*out*/outputTransform); 650 } 651 652 /** 653 * Map a 3d column vector using the given matrix. 654 * 655 * @param matrix float array containing 3x3 matrix to map vector by. 656 * @param input 3 dimensional vector to map. 657 * @param output 3 dimensional vector result. 658 */ 659 private static void map(float[] matrix, float[] input, /*out*/float[] output) { 660 output[0] = input[0] * matrix[0] + input[1] * matrix[1] + input[2] * matrix[2]; 661 output[1] = input[0] * matrix[3] + input[1] * matrix[4] + input[2] * matrix[5]; 662 output[2] = input[0] * matrix[6] + input[1] * matrix[7] + input[2] * matrix[8]; 663 } 664 665 /** 666 * Multiply two 3x3 matrices together: A * B 667 * 668 * @param a left matrix. 669 * @param b right matrix. 670 */ 671 private static void multiply(float[] a, float[] b, /*out*/float[] output) { 672 output[0] = a[0] * b[0] + a[1] * b[3] + a[2] * b[6]; 673 output[3] = a[3] * b[0] + a[4] * b[3] + a[5] * b[6]; 674 output[6] = a[6] * b[0] + a[7] * b[3] + a[8] * b[6]; 675 output[1] = a[0] * b[1] + a[1] * b[4] + a[2] * b[7]; 676 output[4] = a[3] * b[1] + a[4] * b[4] + a[5] * b[7]; 677 output[7] = a[6] * b[1] + a[7] * b[4] + a[8] * b[7]; 678 output[2] = a[0] * b[2] + a[1] * b[5] + a[2] * b[8]; 679 output[5] = a[3] * b[2] + a[4] * b[5] + a[5] * b[8]; 680 output[8] = a[6] * b[2] + a[7] * b[5] + a[8] * b[8]; 681 } 682 683 /** 684 * Transpose a 3x3 matrix in-place. 685 * 686 * @param m the matrix to transpose. 687 * @return the transposed matrix. 688 */ 689 private static float[] transpose(/*inout*/float[/*9*/] m) { 690 float t = m[1]; 691 m[1] = m[3]; 692 m[3] = t; 693 t = m[2]; 694 m[2] = m[6]; 695 m[6] = t; 696 t = m[5]; 697 m[5] = m[7]; 698 m[7] = t; 699 return m; 700 } 701 702 /** 703 * Invert a 3x3 matrix, or return false if the matrix is singular. 704 * 705 * @param m matrix to invert. 706 * @param output set the output to be the inverse of m. 707 */ 708 private static boolean invert(float[] m, /*out*/float[] output) { 709 double a00 = m[0]; 710 double a01 = m[1]; 711 double a02 = m[2]; 712 double a10 = m[3]; 713 double a11 = m[4]; 714 double a12 = m[5]; 715 double a20 = m[6]; 716 double a21 = m[7]; 717 double a22 = m[8]; 718 719 double t00 = a11 * a22 - a21 * a12; 720 double t01 = a21 * a02 - a01 * a22; 721 double t02 = a01 * a12 - a11 * a02; 722 double t10 = a20 * a12 - a10 * a22; 723 double t11 = a00 * a22 - a20 * a02; 724 double t12 = a10 * a02 - a00 * a12; 725 double t20 = a10 * a21 - a20 * a11; 726 double t21 = a20 * a01 - a00 * a21; 727 double t22 = a00 * a11 - a10 * a01; 728 729 double det = a00 * t00 + a01 * t10 + a02 * t20; 730 if (Math.abs(det) < 1e-9) { 731 return false; // Inverse too close to zero, not invertible. 732 } 733 734 output[0] = (float) (t00 / det); 735 output[1] = (float) (t01 / det); 736 output[2] = (float) (t02 / det); 737 output[3] = (float) (t10 / det); 738 output[4] = (float) (t11 / det); 739 output[5] = (float) (t12 / det); 740 output[6] = (float) (t20 / det); 741 output[7] = (float) (t21 / det); 742 output[8] = (float) (t22 / det); 743 return true; 744 } 745 746 /** 747 * Scale each element in a matrix by the given scaling factor. 748 * 749 * @param factor factor to scale by. 750 * @param matrix the float array containing a 3x3 matrix to scale. 751 */ 752 private static void scale(float factor, /*inout*/float[] matrix) { 753 for (int i = 0; i < 9; i++) { 754 matrix[i] *= factor; 755 } 756 } 757 758 /** 759 * Clamp a value to a given range. 760 * 761 * @param low lower bound to clamp to. 762 * @param high higher bound to clamp to. 763 * @param value the value to clamp. 764 * @return the clamped value. 765 */ 766 private static double clamp(double low, double high, double value) { 767 return Math.max(low, Math.min(high, value)); 768 } 769 770 /** 771 * Return the max float in the array. 772 * 773 * @param array array of floats to search. 774 * @return max float in the array. 775 */ 776 private static float max(float[] array) { 777 float val = array[0]; 778 for (float f : array) { 779 val = (f > val) ? f : val; 780 } 781 return val; 782 } 783 784 /** 785 * Normalize ColorMatrix to eliminate headroom for input space scaled to [0, 1] using 786 * the D50 whitepoint. This maps the D50 whitepoint into the colorspace used by the 787 * ColorMatrix, then uses the resulting whitepoint to renormalize the ColorMatrix so 788 * that the channel values in the resulting whitepoint for this operation are clamped 789 * to the range [0, 1]. 790 * 791 * @param colorMatrix a 3x3 matrix containing a DNG ColorMatrix to be normalized. 792 */ 793 private static void normalizeCM(/*inout*/float[] colorMatrix) { 794 float[] tmp = new float[3]; 795 map(colorMatrix, D50_XYZ, /*out*/tmp); 796 float maxVal = max(tmp); 797 if (maxVal > 0) { 798 scale(1.0f / maxVal, colorMatrix); 799 } 800 } 801 802 /** 803 * Normalize ForwardMatrix to ensure that sensor whitepoint [1, 1, 1] maps to D50 in CIE XYZ 804 * colorspace. 805 * 806 * @param forwardMatrix a 3x3 matrix containing a DNG ForwardTransform to be normalized. 807 */ 808 private static void normalizeFM(/*inout*/float[] forwardMatrix) { 809 float[] tmp = new float[] {1, 1, 1}; 810 float[] xyz = new float[3]; 811 map(forwardMatrix, tmp, /*out*/xyz); 812 813 float[] intermediate = new float[9]; 814 float[] m = new float[] {1.0f / xyz[0], 0, 0, 0, 1.0f / xyz[1], 0, 0, 0, 1.0f / xyz[2]}; 815 816 multiply(m, forwardMatrix, /*out*/ intermediate); 817 float[] m2 = new float[] {D50_XYZ[0], 0, 0, 0, D50_XYZ[1], 0, 0, 0, D50_XYZ[2]}; 818 multiply(m2, intermediate, /*out*/forwardMatrix); 819 } 820 } 821
