1 /** 2 * Copyright 2016 Google Inc. All Rights Reserved. 3 * 4 * <p>Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file 5 * except in compliance with the License. You may obtain a copy of the License at 6 * 7 * <p>http://www.apache.org/licenses/LICENSE-2.0 8 * 9 * <p>Unless required by applicable law or agreed to in writing, software distributed under the 10 * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 * express or implied. See the License for the specific language governing permissions and 12 * limitations under the License. 13 */ 14 package com.android.vts.util; 15 16 import com.android.vts.entity.BranchEntity; 17 import com.android.vts.entity.BuildTargetEntity; 18 import com.android.vts.entity.CoverageEntity; 19 import com.android.vts.entity.DeviceInfoEntity; 20 import com.android.vts.entity.ProfilingPointRunEntity; 21 import com.android.vts.entity.TestCaseRunEntity; 22 import com.android.vts.entity.TestEntity; 23 import com.android.vts.entity.TestPlanEntity; 24 import com.android.vts.entity.TestPlanRunEntity; 25 import com.android.vts.entity.TestRunEntity; 26 import com.android.vts.entity.TestRunEntity.TestRunType; 27 import com.android.vts.job.VtsAlertJobServlet; 28 import com.android.vts.job.VtsCoverageAlertJobServlet; 29 import com.android.vts.job.VtsProfilingStatsJobServlet; 30 import com.android.vts.proto.VtsReportMessage.AndroidDeviceInfoMessage; 31 import com.android.vts.proto.VtsReportMessage.CoverageReportMessage; 32 import com.android.vts.proto.VtsReportMessage.LogMessage; 33 import com.android.vts.proto.VtsReportMessage.ProfilingReportMessage; 34 import com.android.vts.proto.VtsReportMessage.TestCaseReportMessage; 35 import com.android.vts.proto.VtsReportMessage.TestCaseResult; 36 import com.android.vts.proto.VtsReportMessage.TestPlanReportMessage; 37 import com.android.vts.proto.VtsReportMessage.TestReportMessage; 38 import com.android.vts.proto.VtsReportMessage.UrlResourceMessage; 39 import com.google.appengine.api.datastore.DatastoreFailureException; 40 import com.google.appengine.api.datastore.DatastoreService; 41 import com.google.appengine.api.datastore.DatastoreServiceFactory; 42 import com.google.appengine.api.datastore.DatastoreTimeoutException; 43 import com.google.appengine.api.datastore.Entity; 44 import com.google.appengine.api.datastore.EntityNotFoundException; 45 import com.google.appengine.api.datastore.FetchOptions; 46 import com.google.appengine.api.datastore.Key; 47 import com.google.appengine.api.datastore.KeyFactory; 48 import com.google.appengine.api.datastore.Query; 49 import com.google.appengine.api.datastore.Query.Filter; 50 import com.google.appengine.api.datastore.Query.FilterOperator; 51 import com.google.appengine.api.datastore.Query.FilterPredicate; 52 import com.google.appengine.api.datastore.Transaction; 53 import com.google.appengine.api.datastore.TransactionOptions; 54 import com.google.common.collect.Lists; 55 import java.io.IOException; 56 import java.util.ArrayList; 57 import java.util.Arrays; 58 import java.util.ConcurrentModificationException; 59 import java.util.HashSet; 60 import java.util.List; 61 import java.util.Map; 62 import java.util.Set; 63 import java.util.logging.Level; 64 import java.util.logging.Logger; 65 66 /** DatastoreHelper, a helper class for interacting with Cloud Datastore. */ 67 public class DatastoreHelper { 68 /** The default kind name for datastore */ 69 public static final String NULL_ENTITY_KIND = "nullEntity"; 70 71 public static final int MAX_WRITE_RETRIES = 5; 72 /** 73 * This variable is for maximum number of entities per transaction You can find the detail here 74 * (https://cloud.google.com/datastore/docs/concepts/limits) 75 */ 76 public static final int MAX_ENTITY_SIZE_PER_TRANSACTION = 300; 77 78 protected static final Logger logger = Logger.getLogger(DatastoreHelper.class.getName()); 79 private static final DatastoreService datastore = DatastoreServiceFactory.getDatastoreService(); 80 81 /** 82 * Get query fetch options for large batches of entities. 83 * 84 * @return FetchOptions with a large chunk and prefetch size. 85 */ 86 public static FetchOptions getLargeBatchOptions() { 87 return FetchOptions.Builder.withChunkSize(1000).prefetchSize(1000); 88 } 89 90 /** 91 * Returns true if there are data points newer than lowerBound in the results table. 92 * 93 * @param parentKey The parent key to use in the query. 94 * @param kind The query entity kind. 95 * @param lowerBound The (exclusive) lower time bound, long, microseconds. 96 * @return boolean True if there are newer data points. 97 * @throws IOException 98 */ 99 public static boolean hasNewer(Key parentKey, String kind, Long lowerBound) throws IOException { 100 if (lowerBound == null || lowerBound <= 0) return false; 101 DatastoreService datastore = DatastoreServiceFactory.getDatastoreService(); 102 Key startKey = KeyFactory.createKey(parentKey, kind, lowerBound); 103 Filter startFilter = 104 new FilterPredicate( 105 Entity.KEY_RESERVED_PROPERTY, FilterOperator.GREATER_THAN, startKey); 106 Query q = new Query(kind).setAncestor(parentKey).setFilter(startFilter).setKeysOnly(); 107 return datastore.prepare(q).countEntities(FetchOptions.Builder.withLimit(1)) > 0; 108 } 109 110 /** 111 * Returns true if there are data points older than upperBound in the table. 112 * 113 * @param parentKey The parent key to use in the query. 114 * @param kind The query entity kind. 115 * @param upperBound The (exclusive) upper time bound, long, microseconds. 116 * @return boolean True if there are older data points. 117 * @throws IOException 118 */ 119 public static boolean hasOlder(Key parentKey, String kind, Long upperBound) throws IOException { 120 if (upperBound == null || upperBound <= 0) return false; 121 Key endKey = KeyFactory.createKey(parentKey, kind, upperBound); 122 Filter endFilter = 123 new FilterPredicate(Entity.KEY_RESERVED_PROPERTY, FilterOperator.LESS_THAN, endKey); 124 Query q = new Query(kind).setAncestor(parentKey).setFilter(endFilter).setKeysOnly(); 125 return datastore.prepare(q).countEntities(FetchOptions.Builder.withLimit(1)) > 0; 126 } 127 128 /** 129 * Get all of the devices branches. 130 * 131 * @return a list of all branches. 132 */ 133 public static List<String> getAllBranches() { 134 Query query = new Query(BranchEntity.KIND).setKeysOnly(); 135 List<String> branches = new ArrayList<>(); 136 for (Entity e : datastore.prepare(query).asIterable(getLargeBatchOptions())) { 137 branches.add(e.getKey().getName()); 138 } 139 return branches; 140 } 141 142 /** 143 * Get all of the device build flavors. 144 * 145 * @return a list of all device build flavors. 146 */ 147 public static List<String> getAllBuildFlavors() { 148 Query query = new Query(BuildTargetEntity.KIND).setKeysOnly(); 149 List<String> devices = new ArrayList<>(); 150 for (Entity e : datastore.prepare(query).asIterable(getLargeBatchOptions())) { 151 devices.add(e.getKey().getName()); 152 } 153 return devices; 154 } 155 156 /** 157 * Upload data from a test report message 158 * 159 * @param report The test report containing data to upload. 160 */ 161 public static void insertTestReport(TestReportMessage report) { 162 163 List<Entity> testEntityList = new ArrayList<>(); 164 List<Entity> branchEntityList = new ArrayList<>(); 165 List<Entity> buildTargetEntityList = new ArrayList<>(); 166 List<Entity> coverageEntityList = new ArrayList<>(); 167 List<Entity> profilingPointRunEntityList = new ArrayList<>(); 168 169 if (!report.hasStartTimestamp() 170 || !report.hasEndTimestamp() 171 || !report.hasTest() 172 || !report.hasHostInfo() 173 || !report.hasBuildInfo()) { 174 // missing information 175 return; 176 } 177 long startTimestamp = report.getStartTimestamp(); 178 long endTimestamp = report.getEndTimestamp(); 179 String testName = report.getTest().toStringUtf8(); 180 String testBuildId = report.getBuildInfo().getId().toStringUtf8(); 181 String hostName = report.getHostInfo().getHostname().toStringUtf8(); 182 183 TestEntity testEntity = new TestEntity(testName); 184 185 Key testRunKey = 186 KeyFactory.createKey( 187 testEntity.key, TestRunEntity.KIND, report.getStartTimestamp()); 188 189 long passCount = 0; 190 long failCount = 0; 191 long coveredLineCount = 0; 192 long totalLineCount = 0; 193 194 Set<Key> buildTargetKeys = new HashSet<>(); 195 Set<Key> branchKeys = new HashSet<>(); 196 List<TestCaseRunEntity> testCases = new ArrayList<>(); 197 List<Key> profilingPointKeys = new ArrayList<>(); 198 List<String> links = new ArrayList<>(); 199 200 // Process test cases 201 for (TestCaseReportMessage testCase : report.getTestCaseList()) { 202 String testCaseName = testCase.getName().toStringUtf8(); 203 TestCaseResult result = testCase.getTestResult(); 204 // Track global pass/fail counts 205 if (result == TestCaseResult.TEST_CASE_RESULT_PASS) { 206 ++passCount; 207 } else if (result != TestCaseResult.TEST_CASE_RESULT_SKIP) { 208 ++failCount; 209 } 210 if (testCase.getSystraceCount() > 0 211 && testCase.getSystraceList().get(0).getUrlCount() > 0) { 212 String systraceLink = testCase.getSystraceList().get(0).getUrl(0).toStringUtf8(); 213 links.add(systraceLink); 214 } 215 // Process coverage data for test case 216 for (CoverageReportMessage coverage : testCase.getCoverageList()) { 217 CoverageEntity coverageEntity = 218 CoverageEntity.fromCoverageReport(testRunKey, testCaseName, coverage); 219 if (coverageEntity == null) { 220 logger.log(Level.WARNING, "Invalid coverage report in test run " + testRunKey); 221 } else { 222 coveredLineCount += coverageEntity.coveredLineCount; 223 totalLineCount += coverageEntity.totalLineCount; 224 coverageEntityList.add(coverageEntity.toEntity()); 225 } 226 } 227 // Process profiling data for test case 228 for (ProfilingReportMessage profiling : testCase.getProfilingList()) { 229 ProfilingPointRunEntity profilingPointRunEntity = 230 ProfilingPointRunEntity.fromProfilingReport(testRunKey, profiling); 231 if (profilingPointRunEntity == null) { 232 logger.log(Level.WARNING, "Invalid profiling report in test run " + testRunKey); 233 } else { 234 profilingPointRunEntityList.add(profilingPointRunEntity.toEntity()); 235 profilingPointKeys.add(profilingPointRunEntity.key); 236 testEntity.setHasProfilingData(true); 237 } 238 } 239 240 int lastIndex = testCases.size() - 1; 241 if (lastIndex < 0 || testCases.get(lastIndex).isFull()) { 242 testCases.add(new TestCaseRunEntity()); 243 ++lastIndex; 244 } 245 TestCaseRunEntity testCaseEntity = testCases.get(lastIndex); 246 testCaseEntity.addTestCase(testCaseName, result.getNumber()); 247 } 248 249 List<Entity> testCasePuts = new ArrayList<>(); 250 for (TestCaseRunEntity testCaseEntity : testCases) { 251 testCasePuts.add(testCaseEntity.toEntity()); 252 } 253 List<Key> testCaseKeys = datastore.put(testCasePuts); 254 255 List<Long> testCaseIds = new ArrayList<>(); 256 for (Key key : testCaseKeys) { 257 testCaseIds.add(key.getId()); 258 } 259 260 // Process device information 261 TestRunType testRunType = null; 262 for (AndroidDeviceInfoMessage device : report.getDeviceInfoList()) { 263 DeviceInfoEntity deviceInfoEntity = 264 DeviceInfoEntity.fromDeviceInfoMessage(testRunKey, device); 265 if (deviceInfoEntity == null) { 266 logger.log(Level.WARNING, "Invalid device info in test run " + testRunKey); 267 } else { 268 // Run type on devices must be the same, else set to OTHER 269 TestRunType runType = TestRunType.fromBuildId(deviceInfoEntity.buildId); 270 if (testRunType == null) { 271 testRunType = runType; 272 } else if (runType != testRunType) { 273 testRunType = TestRunType.OTHER; 274 } 275 testEntityList.add(deviceInfoEntity.toEntity()); 276 BuildTargetEntity target = new BuildTargetEntity(deviceInfoEntity.buildFlavor); 277 if (buildTargetKeys.add(target.key)) { 278 buildTargetEntityList.add(target.toEntity()); 279 } 280 BranchEntity branch = new BranchEntity(deviceInfoEntity.branch); 281 if (branchKeys.add(branch.key)) { 282 branchEntityList.add(branch.toEntity()); 283 } 284 } 285 } 286 287 // Overall run type should be determined by the device builds unless test build is OTHER 288 if (testRunType == null) { 289 testRunType = TestRunType.fromBuildId(testBuildId); 290 } else if (TestRunType.fromBuildId(testBuildId) == TestRunType.OTHER) { 291 testRunType = TestRunType.OTHER; 292 } 293 294 // Process global coverage data 295 for (CoverageReportMessage coverage : report.getCoverageList()) { 296 CoverageEntity coverageEntity = 297 CoverageEntity.fromCoverageReport(testRunKey, new String(), coverage); 298 if (coverageEntity == null) { 299 logger.log(Level.WARNING, "Invalid coverage report in test run " + testRunKey); 300 } else { 301 coveredLineCount += coverageEntity.coveredLineCount; 302 totalLineCount += coverageEntity.totalLineCount; 303 coverageEntityList.add(coverageEntity.toEntity()); 304 } 305 } 306 307 // Process global profiling data 308 for (ProfilingReportMessage profiling : report.getProfilingList()) { 309 ProfilingPointRunEntity profilingPointRunEntity = 310 ProfilingPointRunEntity.fromProfilingReport(testRunKey, profiling); 311 if (profilingPointRunEntity == null) { 312 logger.log(Level.WARNING, "Invalid profiling report in test run " + testRunKey); 313 } else { 314 profilingPointRunEntityList.add(profilingPointRunEntity.toEntity()); 315 profilingPointKeys.add(profilingPointRunEntity.key); 316 testEntity.setHasProfilingData(true); 317 } 318 } 319 320 // Process log data 321 for (LogMessage log : report.getLogList()) { 322 if (log.hasUrl()) links.add(log.getUrl().toStringUtf8()); 323 } 324 // Process url resource 325 for (UrlResourceMessage resource : report.getLinkResourceList()) { 326 if (resource.hasUrl()) links.add(resource.getUrl().toStringUtf8()); 327 } 328 329 TestRunEntity testRunEntity = 330 new TestRunEntity( 331 testEntity.key, 332 testRunType, 333 startTimestamp, 334 endTimestamp, 335 testBuildId, 336 hostName, 337 passCount, 338 failCount, 339 testCaseIds, 340 links, 341 coveredLineCount, 342 totalLineCount); 343 testEntityList.add(testRunEntity.toEntity()); 344 345 Entity test = testEntity.toEntity(); 346 347 if (datastoreTransactionalRetry(test, testEntityList)) { 348 List<List<Entity>> auxiliaryEntityList = 349 Arrays.asList( 350 profilingPointRunEntityList, 351 coverageEntityList, 352 branchEntityList, 353 buildTargetEntityList); 354 int indexCount = 0; 355 for (List<Entity> entityList : auxiliaryEntityList) { 356 switch (indexCount) { 357 case 0: 358 case 1: 359 if (entityList.size() > MAX_ENTITY_SIZE_PER_TRANSACTION) { 360 List<List<Entity>> partitionedList = 361 Lists.partition(entityList, MAX_ENTITY_SIZE_PER_TRANSACTION); 362 partitionedList.forEach( 363 subEntityList -> { 364 datastoreTransactionalRetry( 365 new Entity(NULL_ENTITY_KIND), subEntityList); 366 }); 367 } else { 368 datastoreTransactionalRetry(new Entity(NULL_ENTITY_KIND), entityList); 369 } 370 break; 371 case 2: 372 case 3: 373 datastoreTransactionalRetryWithXG( 374 new Entity(NULL_ENTITY_KIND), entityList, true); 375 break; 376 default: 377 break; 378 } 379 indexCount++; 380 } 381 382 if (testRunEntity.type == TestRunType.POSTSUBMIT) { 383 VtsAlertJobServlet.addTask(testRunKey); 384 if (testRunEntity.hasCoverage) { 385 VtsCoverageAlertJobServlet.addTask(testRunKey); 386 } 387 if (profilingPointKeys.size() > 0) { 388 VtsProfilingStatsJobServlet.addTasks(profilingPointKeys); 389 } 390 } else { 391 logger.log( 392 Level.WARNING, 393 "The alert email was not sent as testRunEntity type is not POSTSUBMIT!" + 394 " \n " + " testRunEntity type => " + testRunEntity.type); 395 } 396 } 397 } 398 399 /** 400 * Upload data from a test plan report message 401 * 402 * @param report The test plan report containing data to upload. 403 */ 404 public static void insertTestPlanReport(TestPlanReportMessage report) { 405 List<Entity> testEntityList = new ArrayList<>(); 406 407 List<String> testModules = report.getTestModuleNameList(); 408 List<Long> testTimes = report.getTestModuleStartTimestampList(); 409 if (testModules.size() != testTimes.size() || !report.hasTestPlanName()) { 410 logger.log(Level.WARNING, "TestPlanReportMessage is missing information."); 411 return; 412 } 413 414 String testPlanName = report.getTestPlanName(); 415 Entity testPlanEntity = new TestPlanEntity(testPlanName).toEntity(); 416 List<Key> testRunKeys = new ArrayList<>(); 417 for (int i = 0; i < testModules.size(); i++) { 418 String test = testModules.get(i); 419 long time = testTimes.get(i); 420 Key parentKey = KeyFactory.createKey(TestEntity.KIND, test); 421 Key testRunKey = KeyFactory.createKey(parentKey, TestRunEntity.KIND, time); 422 testRunKeys.add(testRunKey); 423 } 424 Map<Key, Entity> testRuns = datastore.get(testRunKeys); 425 long passCount = 0; 426 long failCount = 0; 427 long startTimestamp = -1; 428 long endTimestamp = -1; 429 String testBuildId = null; 430 TestRunType type = null; 431 Set<DeviceInfoEntity> deviceInfoEntitySet = new HashSet<>(); 432 for (Key testRunKey : testRuns.keySet()) { 433 TestRunEntity testRun = TestRunEntity.fromEntity(testRuns.get(testRunKey)); 434 if (testRun == null) { 435 continue; // not a valid test run 436 } 437 passCount += testRun.passCount; 438 failCount += testRun.failCount; 439 if (startTimestamp < 0 || testRunKey.getId() < startTimestamp) { 440 startTimestamp = testRunKey.getId(); 441 } 442 if (endTimestamp < 0 || testRun.endTimestamp > endTimestamp) { 443 endTimestamp = testRun.endTimestamp; 444 } 445 if (type == null) { 446 type = testRun.type; 447 } else if (type != testRun.type) { 448 type = TestRunType.OTHER; 449 } 450 testBuildId = testRun.testBuildId; 451 Query deviceInfoQuery = new Query(DeviceInfoEntity.KIND).setAncestor(testRunKey); 452 for (Entity deviceInfoEntity : datastore.prepare(deviceInfoQuery).asIterable()) { 453 DeviceInfoEntity device = DeviceInfoEntity.fromEntity(deviceInfoEntity); 454 if (device == null) { 455 continue; // invalid entity 456 } 457 deviceInfoEntitySet.add(device); 458 } 459 } 460 if (startTimestamp < 0 || testBuildId == null || type == null) { 461 logger.log(Level.WARNING, "Couldn't infer test run information from runs."); 462 return; 463 } 464 TestPlanRunEntity testPlanRun = 465 new TestPlanRunEntity( 466 testPlanEntity.getKey(), 467 testPlanName, 468 type, 469 startTimestamp, 470 endTimestamp, 471 testBuildId, 472 passCount, 473 failCount, 474 testRunKeys); 475 476 // Create the device infos. 477 for (DeviceInfoEntity device : deviceInfoEntitySet) { 478 testEntityList.add(device.copyWithParent(testPlanRun.key).toEntity()); 479 } 480 testEntityList.add(testPlanRun.toEntity()); 481 482 datastoreTransactionalRetry(testPlanEntity, testEntityList); 483 } 484 485 /** 486 * Datastore Transactional process for data insertion with MAX_WRITE_RETRIES times and withXG of 487 * false value 488 * 489 * @param entity The entity that you want to insert to datastore. 490 * @param entityList The list of entity for using datastore put method. 491 */ 492 private static boolean datastoreTransactionalRetry(Entity entity, List<Entity> entityList) { 493 return datastoreTransactionalRetryWithXG(entity, entityList, false); 494 } 495 496 /** 497 * Datastore Transactional process for data insertion with MAX_WRITE_RETRIES times 498 * 499 * @param entity The entity that you want to insert to datastore. 500 * @param entityList The list of entity for using datastore put method. 501 */ 502 private static boolean datastoreTransactionalRetryWithXG( 503 Entity entity, List<Entity> entityList, boolean withXG) { 504 int retries = 0; 505 while (true) { 506 Transaction txn; 507 if (withXG) { 508 TransactionOptions options = TransactionOptions.Builder.withXG(withXG); 509 txn = datastore.beginTransaction(options); 510 } else { 511 txn = datastore.beginTransaction(); 512 } 513 514 try { 515 // Check if test already exists in the database 516 if (!entity.getKind().equalsIgnoreCase(NULL_ENTITY_KIND)) { 517 try { 518 if (entity.getKind().equalsIgnoreCase("Test")) { 519 Entity datastoreEntity = datastore.get(entity.getKey()); 520 TestEntity datastoreTestEntity = TestEntity.fromEntity(datastoreEntity); 521 if (datastoreTestEntity == null 522 || !datastoreTestEntity.equals(entity)) { 523 entityList.add(entity); 524 } 525 } else if (entity.getKind().equalsIgnoreCase("TestPlan")) { 526 datastore.get(entity.getKey()); 527 } else { 528 datastore.get(entity.getKey()); 529 } 530 } catch (EntityNotFoundException e) { 531 entityList.add(entity); 532 } 533 } 534 datastore.put(txn, entityList); 535 txn.commit(); 536 break; 537 } catch (ConcurrentModificationException 538 | DatastoreFailureException 539 | DatastoreTimeoutException e) { 540 entityList.remove(entity); 541 logger.log( 542 Level.WARNING, 543 "Retrying insert kind: " + entity.getKind() + " key: " + entity.getKey()); 544 if (retries++ >= MAX_WRITE_RETRIES) { 545 logger.log( 546 Level.SEVERE, 547 "Exceeded maximum retries kind: " 548 + entity.getKind() 549 + " key: " 550 + entity.getKey()); 551 return false; 552 } 553 } finally { 554 if (txn.isActive()) { 555 logger.log( 556 Level.WARNING, "Transaction rollback forced for : " + entity.getKind()); 557 txn.rollback(); 558 } 559 } 560 } 561 return true; 562 } 563 } 564