1 /* 2 * Copyright (C) 2012 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package com.android.sdklib.build; 18 19 import java.io.BufferedReader; 20 import java.io.File; 21 import java.io.FileInputStream; 22 import java.io.FileNotFoundException; 23 import java.io.FileOutputStream; 24 import java.io.IOException; 25 import java.io.InputStreamReader; 26 import java.io.OutputStreamWriter; 27 import java.io.PrintStream; 28 import java.io.UnsupportedEncodingException; 29 import java.security.MessageDigest; 30 import java.util.ArrayList; 31 import java.util.Collection; 32 import java.util.Formatter; 33 import java.util.HashMap; 34 import java.util.List; 35 import java.util.Map; 36 import java.util.Map.Entry; 37 import java.util.regex.Matcher; 38 import java.util.regex.Pattern; 39 40 /** 41 * A Class to handle a list of jar files, finding and removing duplicates. 42 * 43 * Right now duplicates are based on: 44 * - same filename 45 * - same length 46 * - same content: using sha1 comparison. 47 * 48 * The length/sha1 are kept in a cache and only updated if the library is changed. 49 */ 50 public class JarListSanitizer { 51 52 private static final byte[] sBuffer = new byte[4096]; 53 private static final String CACHE_FILENAME = "jarlist.cache"; 54 private static final Pattern READ_PATTERN = Pattern.compile("^(\\d+) (\\d+) ([0-9a-f]+) (.+)$"); 55 56 /** 57 * Simple class holding the data regarding a jar dependency. 58 * 59 */ 60 private static final class JarEntity { 61 private final File mFile; 62 private final long mLastModified; 63 private long mLength; 64 private String mSha1; 65 66 /** 67 * Creates an entity from cached data. 68 * @param path the file path 69 * @param lastModified when it was last modified 70 * @param length its length 71 * @param sha1 its sha1 72 */ 73 private JarEntity(String path, long lastModified, long length, String sha1) { 74 mFile = new File(path); 75 mLastModified = lastModified; 76 mLength = length; 77 mSha1 = sha1; 78 } 79 80 /** 81 * Creates an entity from a {@link File}. 82 * @param file the file. 83 */ 84 private JarEntity(File file) { 85 mFile = file; 86 mLastModified = file.lastModified(); 87 mLength = file.length(); 88 } 89 90 /** 91 * Checks whether the {@link File#lastModified()} matches the cached value. If not, length 92 * is updated and the sha1 is reset (but not recomputed, this is done on demand). 93 * @return return whether the file was changed. 94 */ 95 private boolean checkValidity() { 96 if (mLastModified != mFile.lastModified()) { 97 mLength = mFile.length(); 98 mSha1 = null; 99 return true; 100 } 101 102 return false; 103 } 104 105 private File getFile() { 106 return mFile; 107 } 108 109 private long getLastModified() { 110 return mLastModified; 111 } 112 113 private long getLength() { 114 return mLength; 115 } 116 117 /** 118 * Returns the file's sha1, computing it if necessary. 119 * @return the sha1 120 * @throws Sha1Exception 121 */ 122 private String getSha1() throws Sha1Exception { 123 if (mSha1 == null) { 124 mSha1 = JarListSanitizer.getSha1(mFile); 125 } 126 return mSha1; 127 } 128 129 private boolean hasSha1() { 130 return mSha1 != null; 131 } 132 } 133 134 /** 135 * Exception used to indicate the sanitized list of jar dependency cannot be computed due 136 * to inconsistency in duplicate jar files. 137 */ 138 public static final class DifferentLibException extends Exception { 139 private static final long serialVersionUID = 1L; 140 private final String[] mDetails; 141 142 public DifferentLibException(String message, String[] details) { 143 super(message); 144 mDetails = details; 145 } 146 147 public String[] getDetails() { 148 return mDetails; 149 } 150 } 151 152 /** 153 * Exception to indicate a failure to check a jar file's content. 154 */ 155 public static final class Sha1Exception extends Exception { 156 private static final long serialVersionUID = 1L; 157 private final File mJarFile; 158 159 public Sha1Exception(File jarFile, Throwable cause) { 160 super(cause); 161 mJarFile = jarFile; 162 } 163 164 public File getJarFile() { 165 return mJarFile; 166 } 167 } 168 169 private final File mOut; 170 private final PrintStream mOutStream; 171 172 /** 173 * Creates a sanitizer. 174 * @param out the project output where the cache is to be stored. 175 */ 176 public JarListSanitizer(File out) { 177 mOut = out; 178 mOutStream = System.out; 179 } 180 181 public JarListSanitizer(File out, PrintStream outStream) { 182 mOut = out; 183 mOutStream = outStream; 184 } 185 186 /** 187 * Sanitize a given list of files 188 * @param files the list to sanitize 189 * @return a new list containing no duplicates. 190 * @throws DifferentLibException 191 * @throws Sha1Exception 192 */ 193 public List<File> sanitize(Collection<File> files) throws DifferentLibException, Sha1Exception { 194 List<File> results = new ArrayList<File>(); 195 196 // get the cache list. 197 Map<String, JarEntity> jarList = getCachedJarList(); 198 199 boolean updateJarList = false; 200 201 // clean it up of removed files. 202 // use results as a temp storage to store the files to remove as we go through the map. 203 for (JarEntity entity : jarList.values()) { 204 if (entity.getFile().exists() == false) { 205 results.add(entity.getFile()); 206 } 207 } 208 209 // the actual clean up. 210 if (results.size() > 0) { 211 for (File f : results) { 212 jarList.remove(f.getAbsolutePath()); 213 } 214 215 results.clear(); 216 updateJarList = true; 217 } 218 219 Map<String, List<JarEntity>> nameMap = new HashMap<String, List<JarEntity>>(); 220 221 // update the current jar list if needed, while building a 2ndary map based on 222 // filename only. 223 for (File file : files) { 224 String path = file.getAbsolutePath(); 225 JarEntity entity = jarList.get(path); 226 227 if (entity == null) { 228 entity = new JarEntity(file); 229 jarList.put(path, entity); 230 updateJarList = true; 231 } else { 232 updateJarList |= entity.checkValidity(); 233 } 234 235 String filename = file.getName(); 236 List<JarEntity> nameList = nameMap.get(filename); 237 if (nameList == null) { 238 nameList = new ArrayList<JarEntity>(); 239 nameMap.put(filename, nameList); 240 } 241 nameList.add(entity); 242 } 243 244 try { 245 // now look for dups. Each name list can have more than one file but they must 246 // have the same size/sha1 247 for (Entry<String, List<JarEntity>> entry : nameMap.entrySet()) { 248 List<JarEntity> list = entry.getValue(); 249 checkEntities(entry.getKey(), list); 250 251 // if we are here, there's no issue. Add the first of the list to the results. 252 results.add(list.get(0).getFile()); 253 } 254 255 // special case for android-support-v4/13 256 checkSupportLibs(nameMap, results); 257 } finally { 258 if (updateJarList) { 259 writeJarList(nameMap); 260 } 261 } 262 263 return results; 264 } 265 266 /** 267 * Checks whether a given list of duplicates can be replaced by a single one. 268 * @param filename the filename of the files 269 * @param list the list of dup files 270 * @throws DifferentLibException 271 * @throws Sha1Exception 272 */ 273 private void checkEntities(String filename, List<JarEntity> list) 274 throws DifferentLibException, Sha1Exception { 275 if (list.size() == 1) { 276 return; 277 } 278 279 JarEntity baseEntity = list.get(0); 280 long baseLength = baseEntity.getLength(); 281 String baseSha1 = baseEntity.getSha1(); 282 283 final int count = list.size(); 284 for (int i = 1; i < count ; i++) { 285 JarEntity entity = list.get(i); 286 if (entity.getLength() != baseLength || entity.getSha1().equals(baseSha1) == false) { 287 throw new DifferentLibException("Jar mismatch! Fix your dependencies", 288 getEntityDetails(filename, list)); 289 } 290 291 } 292 } 293 294 /** 295 * Checks for present of both support libraries in v4 and v13. If both are detected, 296 * v4 is removed from <var>results</var> 297 * @param nameMap the list of jar as a map of (filename, list of files). 298 * @param results the current list of jar file set to be used. it's already been cleaned of 299 * duplicates. 300 */ 301 private void checkSupportLibs(Map<String, List<JarEntity>> nameMap, List<File> results) { 302 List<JarEntity> v4 = nameMap.get("android-support-v4.jar"); 303 List<JarEntity> v13 = nameMap.get("android-support-v13.jar"); 304 305 if (v13 != null && v4 != null) { 306 mOutStream.println("WARNING: Found both android-support-v4 and android-support-v13 in the dependency list."); 307 mOutStream.println("Because v13 includes v4, using only v13."); 308 results.remove(v4.get(0).getFile()); 309 } 310 } 311 312 private Map<String, JarEntity> getCachedJarList() { 313 Map<String, JarEntity> cache = new HashMap<String, JarListSanitizer.JarEntity>(); 314 315 File cacheFile = new File(mOut, CACHE_FILENAME); 316 if (cacheFile.exists() == false) { 317 return cache; 318 } 319 320 BufferedReader reader = null; 321 try { 322 reader = new BufferedReader(new InputStreamReader(new FileInputStream(cacheFile), 323 "UTF-8")); 324 325 String line = null; 326 while ((line = reader.readLine()) != null) { 327 // skip comments 328 if (line.charAt(0) == '#') { 329 continue; 330 } 331 332 // get the data with a regexp 333 Matcher m = READ_PATTERN.matcher(line); 334 if (m.matches()) { 335 String path = m.group(4); 336 337 JarEntity entity = new JarEntity( 338 path, 339 Long.parseLong(m.group(1)), 340 Long.parseLong(m.group(2)), 341 m.group(3)); 342 343 cache.put(path, entity); 344 } 345 } 346 347 } catch (FileNotFoundException e) { 348 // won't happen, we check up front. 349 } catch (UnsupportedEncodingException e) { 350 // shouldn't happen, but if it does, we just won't have a cache. 351 } catch (IOException e) { 352 // shouldn't happen, but if it does, we just won't have a cache. 353 } finally { 354 if (reader != null) { 355 try { 356 reader.close(); 357 } catch (IOException e) { 358 } 359 } 360 } 361 362 return cache; 363 } 364 365 private void writeJarList(Map<String, List<JarEntity>> nameMap) { 366 File cacheFile = new File(mOut, CACHE_FILENAME); 367 OutputStreamWriter writer = null; 368 try { 369 writer = new OutputStreamWriter( 370 new FileOutputStream(cacheFile), "UTF-8"); 371 372 writer.write("# cache for current jar dependecy. DO NOT EDIT.\n"); 373 writer.write("# format is <lastModified> <length> <SHA-1> <path>\n"); 374 writer.write("# Encoding is UTF-8\n"); 375 376 for (List<JarEntity> list : nameMap.values()) { 377 // clean up the list of files that don't have a sha1. 378 for (int i = 0 ; i < list.size() ; ) { 379 JarEntity entity = list.get(i); 380 if (entity.hasSha1()) { 381 i++; 382 } else { 383 list.remove(i); 384 } 385 } 386 387 if (list.size() > 1) { 388 for (JarEntity entity : list) { 389 writer.write(String.format("%d %d %s %s\n", 390 entity.getLastModified(), 391 entity.getLength(), 392 entity.getSha1(), 393 entity.getFile().getAbsolutePath())); 394 } 395 } 396 } 397 } catch (IOException e) { 398 mOutStream.println("WARNING: unable to write jarlist cache file " + 399 cacheFile.getAbsolutePath()); 400 } catch (Sha1Exception e) { 401 // shouldn't happen here since we check that the sha1 is present first, meaning it's 402 // already been computing. 403 } finally { 404 if (writer != null) { 405 try { 406 writer.close(); 407 } catch (IOException e) { 408 } 409 } 410 } 411 } 412 413 private String[] getEntityDetails(String filename, List<JarEntity> list) throws Sha1Exception { 414 ArrayList<String> result = new ArrayList<String>(); 415 result.add( 416 String.format("Found %d versions of %s in the dependency list,", 417 list.size(), filename)); 418 result.add("but not all the versions are identical (check is based on SHA-1 only at this time)."); 419 result.add("All versions of the libraries must be the same at this time."); 420 result.add("Versions found are:"); 421 for (JarEntity entity : list) { 422 result.add("Path: " + entity.getFile().getAbsolutePath()); 423 result.add("\tLength: " + entity.getLength()); 424 result.add("\tSHA-1: " + entity.getSha1()); 425 } 426 427 return result.toArray(new String[result.size()]); 428 } 429 430 /** 431 * Computes the sha1 of a file and returns it. 432 * @param f the file to compute the sha1 for. 433 * @return the sha1 value 434 * @throws Sha1Exception if the sha1 value cannot be computed. 435 */ 436 private static String getSha1(File f) throws Sha1Exception { 437 synchronized (sBuffer) { 438 try { 439 MessageDigest md = MessageDigest.getInstance("SHA-1"); 440 441 FileInputStream fis = new FileInputStream(f); 442 while (true) { 443 int length = fis.read(sBuffer); 444 if (length > 0) { 445 md.update(sBuffer, 0, length); 446 } else { 447 break; 448 } 449 } 450 451 return byteArray2Hex(md.digest()); 452 453 } catch (Exception e) { 454 throw new Sha1Exception(f, e); 455 } 456 } 457 } 458 459 private static String byteArray2Hex(final byte[] hash) { 460 Formatter formatter = new Formatter(); 461 for (byte b : hash) { 462 formatter.format("%02x", b); 463 } 464 return formatter.toString(); 465 } 466 } 467