1 /* 2 * Copyright (C) 2017 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package com.android.tradefed.testtype; 18 19 import com.android.compatibility.common.tradefed.build.CompatibilityBuildHelper; 20 import com.android.ddmlib.Log.LogLevel; 21 import com.android.tradefed.build.IBuildInfo; 22 import com.android.tradefed.config.Option; 23 import com.android.tradefed.config.OptionClass; 24 import com.android.tradefed.device.DeviceNotAvailableException; 25 import com.android.tradefed.device.ITestDevice; 26 import com.android.tradefed.invoker.IInvocationContext; 27 import com.android.tradefed.log.LogUtil.CLog; 28 import com.android.tradefed.result.ITestInvocationListener; 29 import com.android.tradefed.result.ITestLifeCycleReceiver; 30 import com.android.tradefed.result.TestDescription; 31 import com.android.tradefed.targetprep.VtsCoveragePreparer; 32 import com.android.tradefed.util.ArrayUtil; 33 import com.android.tradefed.util.CommandResult; 34 import com.android.tradefed.util.CommandStatus; 35 import com.android.tradefed.util.FileUtil; 36 import com.android.tradefed.util.IRunUtil; 37 import com.android.tradefed.util.JsonUtil; 38 import com.android.tradefed.util.RunInterruptedException; 39 import com.android.tradefed.util.RunUtil; 40 import com.android.tradefed.util.VtsDashboardUtil; 41 import com.android.tradefed.util.VtsPythonRunnerHelper; 42 import com.android.tradefed.util.VtsVendorConfigFileUtil; 43 44 import org.json.JSONArray; 45 import org.json.JSONException; 46 import org.json.JSONObject; 47 48 import java.io.BufferedWriter; 49 import java.io.File; 50 import java.io.FileNotFoundException; 51 import java.io.FileWriter; 52 import java.io.IOException; 53 import java.io.PrintWriter; 54 import java.nio.file.Paths; 55 import java.util.ArrayList; 56 import java.util.Arrays; 57 import java.util.Collection; 58 import java.util.Set; 59 import java.util.TreeSet; 60 61 /** 62 * A Test that runs a vts multi device test package (part of Vendor Test Suite, 63 * VTS) on given device. 64 */ 65 66 @OptionClass(alias = "vtsmultidevicetest") 67 public class VtsMultiDeviceTest 68 implements IDeviceTest, IRemoteTest, ITestFilterReceiver, IRuntimeHintProvider, 69 ITestCollector, IBuildReceiver, IAbiReceiver, IInvocationContextReceiver { 70 static final String ANDROIDDEVICE = "AndroidDevice"; 71 static final String BUILD = "build"; 72 static final String BUILD_ID = "build_id"; 73 static final String BUILD_TARGET = "build_target"; 74 static final String COVERAGE_PROPERTY = "ro.vts.coverage"; 75 static final String DATA_FILE_PATH = "data_file_path"; 76 static final String LOG_PATH = "log_path"; 77 static final String LOG_SEVERITY = "log_severity"; 78 static final String NAME = "name"; 79 static final String SERIAL = "serial"; 80 static final String TESTMODULE = "TestModule"; 81 static final String TEST_BED = "test_bed"; 82 static final String TEST_PLAN_REPORT_FILE = "TEST_PLAN_REPORT_FILE"; 83 static final String TEST_SUITE = "test_suite"; 84 static final String TEST_MAX_TIMEOUT = "test_max_timeout"; 85 static final String VIRTUAL_ENV_PATH = "VIRTUALENVPATH"; 86 static final String ABI_NAME = "abi_name"; 87 static final String ABI_BITNESS = "abi_bitness"; 88 static final String SKIP_ON_32BIT_ABI = "skip_on_32bit_abi"; 89 static final String SKIP_ON_64BIT_ABI = "skip_on_64bit_abi"; 90 static final String SKIP_IF_THERMAL_THROTTLING = "skip_if_thermal_throttling"; 91 static final String DISABLE_CPU_FREQUENCY_SCALING = "disable_cpu_frequency_scaling"; 92 static final String RUN_32BIT_ON_64BIT_ABI = "run_32bit_on_64bit_abi"; 93 static final String CONFIG_FILE_EXTENSION = ".config"; 94 static final String INCLUDE_FILTER = "include_filter"; 95 static final String EXCLUDE_FILTER = "exclude_filter"; 96 static final String EXCLUDE_OVER_INCLUDE = "exclude_over_include"; 97 static final String BINARY_TEST_SOURCE = "binary_test_source"; 98 static final String BINARY_TEST_WORKING_DIRECTORY = "binary_test_working_directory"; 99 static final String BINARY_TEST_ENVP = "binary_test_envp"; 100 static final String BINARY_TEST_ARGS = "binary_test_args"; 101 static final String BINARY_TEST_LD_LIBRARY_PATH = "binary_test_ld_library_path"; 102 static final String BINARY_TEST_PROFILING_LIBRARY_PATH = "binary_test_profiling_library_path"; 103 static final String BINARY_TEST_DISABLE_FRAMEWORK = "binary_test_disable_framework"; 104 static final String BINARY_TEST_STOP_NATIVE_SERVERS = "binary_test_stop_native_servers"; 105 static final String BINARY_TEST_TYPE_GTEST = "gtest"; 106 static final String BINARY_TEST_TYPE_LLVMFUZZER = "llvmfuzzer"; 107 static final String BINARY_TEST_TYPE_HAL_HIDL_GTEST = "hal_hidl_gtest"; 108 static final String BINARY_TEST_TYPE_HAL_HIDL_REPLAY_TEST = "hal_hidl_replay_test"; 109 static final String BINARY_TEST_TYPE_HOST_BINARY_TEST = "host_binary_test"; 110 static final String BUG_REPORT_ON_FAILURE = "BUG_REPORT_ON_FAILURE"; 111 static final String COLLECT_TESTS_ONLY = "collect_tests_only"; 112 static final String LOGCAT_ON_FAILURE = "LOGCAT_ON_FAILURE"; 113 static final String ENABLE_COVERAGE = "enable_coverage"; 114 static final String EXCLUDE_COVERAGE_PATH = "exclude_coverage_path"; 115 static final String ENABLE_PROFILING = "enable_profiling"; 116 static final String ENABLE_SANCOV = "enable_sancov"; 117 static final String GTEST_BATCH_MODE = "gtest_batch_mode"; 118 static final String SAVE_TRACE_FIEL_REMOTE = "save_trace_file_remote"; 119 static final String OUTPUT_COVERAGE_REPORT = "output_coverage_report"; 120 static final String COVERAGE_REPORT_PATH = "coverage_report_path"; 121 static final String GLOBAL_COVERAGE = "global_coverage"; 122 static final String LTP_NUMBER_OF_THREADS = "ltp_number_of_threads"; 123 static final String MOBLY_TEST_MODULE = "MOBLY_TEST_MODULE"; 124 static final String NATIVE_SERVER_PROCESS_NAME = "native_server_process_name"; 125 static final String PASSTHROUGH_MODE = "passthrough_mode"; 126 static final String PRECONDITION_HWBINDER_SERVICE = "precondition_hwbinder_service"; 127 static final String PRECONDITION_FEATURE = "precondition_feature"; 128 static final String PRECONDITION_FILE_PATH_PREFIX = "precondition_file_path_prefix"; 129 static final String PRECONDITION_FIRST_API_LEVEL = "precondition_first_api_level"; 130 static final String PRECONDITION_LSHAL = "precondition_lshal"; 131 static final String PRECONDITION_SYSPROP = "precondition_sysprop"; 132 static final String PRECONDITION_VINTF = "precondition_vintf"; 133 static final String ENABLE_SYSTRACE = "enable_systrace"; 134 static final String HAL_HIDL_REPLAY_TEST_TRACE_PATHS = "hal_hidl_replay_test_trace_paths"; 135 static final String HAL_HIDL_PACKAGE_NAME = "hal_hidl_package_name"; 136 static final String REPORT_MESSAGE_FILE_NAME = "report_proto.msg"; 137 static final String RUN_AS_VTS_SELF_TEST = "run_as_vts_self_test"; 138 static final String RUN_AS_COMPLIANCE_TEST = "run_as_compliance_test"; 139 static final String SYSTRACE_PROCESS_NAME = "systrace_process_name"; 140 static final String TEMPLATE_BINARY_TEST_PATH = "vts/testcases/template/binary_test/binary_test"; 141 static final String TEMPLATE_GTEST_BINARY_TEST_PATH = "vts/testcases/template/gtest_binary_test/gtest_binary_test"; 142 static final String TEMPLATE_LLVMFUZZER_TEST_PATH = "vts/testcases/template/llvmfuzzer_test/llvmfuzzer_test"; 143 static final String TEMPLATE_MOBLY_TEST_PATH = "vts/testcases/template/mobly/mobly_test"; 144 static final String TEMPLATE_HAL_HIDL_GTEST_PATH = "vts/testcases/template/hal_hidl_gtest/hal_hidl_gtest"; 145 static final String TEMPLATE_HAL_HIDL_REPLAY_TEST_PATH = "vts/testcases/template/hal_hidl_replay_test/hal_hidl_replay_test"; 146 static final String TEMPLATE_HOST_BINARY_TEST_PATH = "vts/testcases/template/host_binary_test/host_binary_test"; 147 static final long TEST_ABORT_TIMEOUT_MSECS = 1000 * 15; 148 static final String TEST_RUN_SUMMARY_FILE_NAME = "test_run_summary.json"; 149 static final float DEFAULT_TARGET_VERSION = -1; 150 static final String DEFAULT_TESTCASE_CONFIG_PATH = 151 "vts/tools/vts-tradefed/res/default/DefaultTestCase.runner_conf"; 152 153 private ITestDevice mDevice = null; 154 private IAbi mAbi = null; 155 156 @Option(name = "test-timeout", 157 description = "The amount of time (in milliseconds) for a test invocation. " 158 + "If the test cannot finish before timeout, it should interrupt itself and " 159 + "clean up in " + TEST_ABORT_TIMEOUT_MSECS + "ms. Hence the actual timeout " 160 + "is the specified value + " + TEST_ABORT_TIMEOUT_MSECS + "ms.", 161 isTimeVal = true) 162 private long mTestTimeout = 1000 * 60 * 60 * 3; 163 164 @Option(name = "test-module-name", 165 description = "The name for a test module.") 166 private String mTestModuleName = null; 167 168 @Option(name = "test-case-path", 169 description = "The path for test case.") 170 private String mTestCasePath = null; 171 172 @Option(name = "test-case-path-type", 173 description = "The type of test case path ('module' by default or 'file').") 174 private String mTestCasePathType = null; 175 176 @Option(name = "python-version", description = "The version of a Python interpreter to use.") 177 private String mPythonVersion = ""; 178 179 @Option(name = "test-config-path", 180 description = "The path for test case config file.") 181 private String mTestConfigPath = null; 182 183 @Option(name = "precondition-hwbinder-service", 184 description = "The name of a HW binder service needed to run the test.") 185 private String mPreconditionHwBinderServiceName = null; 186 187 @Option(name = "precondition-feature", 188 description = "The name of a `pm`-listable feature needed to run the test.") 189 private String mPreconditionFeature = null; 190 191 @Option(name = "precondition-file-path-prefix", 192 description = "The path prefix of a target-side file needed to run the test." 193 + "Format of tags:" 194 + " <source>: source without tag." 195 + " <tag>::<source>: <tag> specifies bitness of testcase: _32bit or _64bit" 196 + " Note: multiple sources are ANDed" 197 + "Format of each source string:" 198 + " <source>: absolute path of file prefix on device") 199 private Collection<String> mPreconditionFilePathPrefix = new ArrayList<>(); 200 201 @Option(name = "precondition-first-api-level", 202 description = "The lowest first API level required to run the test.") 203 private int mPreconditionFirstApiLevel = 0; 204 205 @Option(name = "precondition-lshal", 206 description = "The name of a `lshal`-listable feature needed to run the test.") 207 private String mPreconditionLshal = null; 208 209 @Option(name = "precondition-sysprop", 210 description = "The name=value for a system property configuration that needs " 211 + "to be met to run the test.") 212 private String mPreconditionSysProp = null; 213 214 @Option(name = "precondition-vintf", 215 description = "The full name of a HAL specified in vendor/manifest.xml and " 216 + "needed to run the test (e.g., android.hardware.graphics.mapper (at) 2.0). " 217 + "this can override precondition-lshal option.") 218 private String mPreconditionVintf = null; 219 220 @Option(name = "use-stdout-logs", 221 description = "Flag that determines whether to use std:out to parse output.") 222 private boolean mUseStdoutLogs = false; 223 224 @Option(name = "include-filter", 225 description = "The positive filter of the test names to run.") 226 private Set<String> mIncludeFilters = new TreeSet<>(); 227 228 @Option(name = "exclude-filter", 229 description = "The negative filter of the test names to run.") 230 private Set<String> mExcludeFilters = new TreeSet<>(); 231 232 @Option(name = "exclude-over-include", 233 description = "The negative filter of the test names to run.") 234 private boolean mExcludeOverInclude = false; 235 236 @Option(name = "runtime-hint", description = "The hint about the test's runtime.", 237 isTimeVal = true) 238 private long mRuntimeHint = 60000; // 1 minute 239 240 @Option(name = "enable-profiling", description = "Enable profiling for the tests.") 241 private boolean mEnableProfiling = false; 242 243 @Option(name = "save-trace-file-remote", 244 description = "Whether to save the trace file in remote storage.") 245 private boolean mSaveTraceFileRemote = false; 246 247 @Option(name = "enable-systrace", description = "Enable systrace for the tests.") 248 private boolean mEnableSystrace = false; 249 250 @Option(name = "enable-coverage", 251 description = "Enable coverage for the tests. In order for coverage to be measured, " + 252 "ro.vts.coverage system must have value \"1\" to indicate the target " + 253 "build is coverage instrumented.") 254 private boolean mEnableCoverage = true; 255 256 @Option(name = "global-coverage", description = "True to measure coverage for entire test, " 257 + "measure coverage for each test case otherwise. Currently, only global " 258 + "coverage is supported for binary tests") 259 private boolean mGlobalCoverage = true; 260 261 @Option(name = "enable-sancov", 262 description = "Enable sanitizer coverage for the tests. In order for coverage to be " 263 + "measured, the device must be a sancov build with its build info and " 264 + "unstripped binaries available to the sancov preparer class.") 265 private boolean mEnableSancov = true; 266 267 @Option(name = "output-coverage-report", description = "Whether to store raw coverage report.") 268 private boolean mOutputCoverageReport = false; 269 270 // Another design option is to parse a string or use enum for host preference on BINDER, 271 // PASSTHROUGH and DEFAULT(which is BINDER). Also in the future, we might want to deal with 272 // the case of target preference on PASSTHROUGH (if host does not specify to use BINDER mode). 273 @Option(name = "passthrough-mode", description = "Set getStub to use passthrough mode. " 274 + "Value true means use passthrough mode if available; false for binderized mode if " 275 + "available. Default is false") 276 private boolean mPassthroughMode = false; 277 278 @Option(name = "ltp-number-of-threads", 279 description = "Number of threads to run the LTP test cases. " 280 + "0 means using number of avaiable CPU threads.") 281 private int mLtpNumberOfThreads = -1; 282 283 @Option(name = "skip-on-32bit-abi", 284 description = "Whether to skip tests on 32bit ABI.") 285 private boolean mSkipOn32BitAbi = false; 286 287 @Option(name = "skip-on-64bit-abi", 288 description = "Whether to skip tests on 64bit ABI.") 289 private boolean mSkipOn64BitAbi = false; 290 291 @Option(name = "skip-if-thermal-throttling", 292 description = "Whether to skip tests if target device suffers from thermal throttling.") 293 private boolean mSkipIfThermalThrottling = false; 294 295 @Option(name = "disable-cpu-frequency-scaling", 296 description = "Whether to disable cpu frequency scaling for test.") 297 private boolean mDisableCpuFrequencyScaling = true; 298 299 @Option(name = "run-32bit-on-64bit-abi", 300 description = "Whether to run 32bit tests on 64bit ABI.") 301 private boolean mRun32bBitOn64BitAbi = false; 302 303 @Option(name = "binary-test-source", 304 description = "Binary test source paths relative to vts testcase directory on host." 305 + "Format of tags:" 306 + " <source>: source without tag." 307 + " <tag>::<source>: source with tag. Can be used to separate 32bit and 64" 308 + " bit tests with same file name." 309 + " <tag1>::<source1>, <tag1>::<source2>, <tag2>::<source3>: multiple" 310 + " sources connected by comma. White spaces in-between" 311 + " will be ignored." 312 + "Format of each source string:" 313 + " <source file>: push file and create test case." 314 + " Source is relative path of source file under vts's testcases" 315 + " folder. Source file will be pushed to working directory" 316 + " discarding original directory structure, and test cases will" 317 + " be created using the pushed file." 318 + " <source file>-><destination file>: push file and create test case." 319 + " Destination path is absolute path on device. Test cases will" 320 + " be created using the pushed file." 321 + " <source file>->: push file only." 322 + " Push the source file to its' tag's corresponding" 323 + " working directory. Test case will not be created on" 324 + " this file. This is equivalent to but simpler than specifying a" 325 + " working directory for the tag and use VtsFilePusher to push the" 326 + " file to the directory." 327 + " -><destination file>: create test case only." 328 + " Destination is absolute device side path." 329 + " Note: each path in source string can be a directory. However, the" 330 + " default binary test runner and gtest binary test runner does not" 331 + " support creating test cases from a directory. You will need to" 332 + " override the binary test runner's CreateTestCase method in python." 333 + " If you wish to push a source file to a specific destination and not" 334 + " create a test case from it, please use VtsFilePusher.") 335 private Collection<String> mBinaryTestSource = new ArrayList<>(); 336 337 @Option(name = "binary-test-working-directory", description = "Working directories for binary " 338 + "tests. Tags can be added to the front of each directory using '::' as delimiter. " 339 + "However, each tag should only has one working directory. This option is optional for " 340 + "binary tests. If not specified, different directories will be used for files with " 341 + "different tags.") 342 private Collection<String> mBinaryTestWorkingDirectory = new ArrayList<>(); 343 344 @Option(name = "binary-test-envp", description = "Additional environment path for binary " 345 + "tests. Tags can be added to the front of each directory using '::' as delimiter. " 346 + "There can be multiple instances of binary-test-envp for a same tag, which will " 347 + "later automatically be combined.") 348 private Collection<String> mBinaryTestEnvp = new ArrayList<>(); 349 350 @Option(name = "binary-test-args", description = "Additional args or flags for binary " 351 + "tests. Tags can be added to the front of each directory using '::' as delimiter. " 352 + "There can be multiple instances of binary-test-args for a same tag, which will " 353 + "later automatically be combined.") 354 private Collection<String> mBinaryTestArgs = new ArrayList<>(); 355 356 @Option(name = "binary-test-ld-library-path", description = "LD_LIBRARY_PATH for binary " 357 + "tests. Tags can be added to the front of each instance using '::' as delimiter. " 358 + "Multiple directories can be added under a same tag using ':' as delimiter. " 359 + "There can be multiple instances of ld-library-path for a same tag, which will " 360 + "later automatically be combined using ':' as delimiter. Paths without a tag " 361 + "will only used for binaries without tag. This option is optional for binary tests.") 362 private Collection<String> mBinaryTestLdLibraryPath = new ArrayList<>(); 363 364 @Option(name = "binary-test-profiling-library-path", description = "Path to lookup and load " 365 + "profiling libraries for tests with profiling enabled. Tags can be added to the " 366 + "front of each directory using '::' as delimiter. Only one directory could be " 367 + "specified for the same tag. This option is optional for binary tests. If not " 368 + "specified, default directories will be used for files with different tags.") 369 private Collection<String> mBinaryTestProfilingLibraryPath = new ArrayList<>(); 370 371 @Option(name = "binary-test-disable-framework", description = "Adb stop/start before/after test.") 372 private boolean mBinaryTestDisableFramework = false; 373 374 @Option(name = "binary-test-stop-native-servers", 375 description = "Set to stop all properly configured native servers during the testing.") 376 private boolean mBinaryTestStopNativeServers = false; 377 378 @Option(name = "bug-report-on-failure", 379 description = "To catch bugreport zip file at the end of failed test cases. " 380 + "If set to true, a report will be caught through adh shell command at the " 381 + "end of each failed test cases.") 382 private boolean mBugReportOnFailure = false; 383 384 @Option(name = "logcat-on-failure", 385 description = "To catch logcat from each buffers at the end of failed test cases. " 386 + "If set to true, a report will be caught through adh shell command at the " 387 + "end of each failed test cases.") 388 private boolean mLogcatOnFailure = true; 389 390 @Option(name = "native-server-process-name", 391 description = "Name of a native server process. The runner checks to make sure " 392 + "each specified native server process is not running after the framework stop.") 393 private Collection<String> mNativeServerProcessName = new ArrayList<>(); 394 395 @Option(name = "binary-test-type", description = "Binary test type. Only specify this when " 396 + "running an extended binary test without a python test file. Available options: gtest") 397 private String mBinaryTestType = ""; 398 399 @Option(name = "hal-hidl-replay-test-trace-path", description = "The path of a trace file to replay.") 400 private Collection<String> mHalHidlReplayTestTracePaths = new ArrayList<>(); 401 402 @Option(name = "hal-hidl-package-name", description = "The name of a target HIDL HAL package " 403 + "e.g., 'android.hardware.light (at) 2.0'.") 404 private String mHalHidlPackageName = null; 405 406 @Option(name = "systrace-process-name", description = "Process name for systrace.") 407 private String mSystraceProcessName = null; 408 409 @Option(name = "collect-tests-only", 410 description = "Only invoke setUpClass, generate*, and tearDownClass to collect list " 411 + "of applicable test cases. All collected tests pass without being executed.") 412 private boolean mCollectTestsOnly = false; 413 414 @Option(name = "gtest-batch-mode", description = "Run Gtest binaries in batch mode.") 415 private boolean mGtestBatchMode = false; 416 417 @Option(name = "log-severity", description = "Set the log severity level.") 418 private String mLogSeverity = "INFO"; 419 420 @Option(name = "python-binary", description = "python binary to use " 421 + "(optional)") 422 private String mPythonBin = null; 423 424 @Option(name = "run-as-vts-self-test", 425 description = "Run the module as vts-selftest. " 426 + "When the value is set to true, only setUpClass and tearDownClass function " 427 + "of the module will be called to ensure the framework is free of bug. " 428 + "Note that exception in tearDownClass will not be reported as failure.") 429 private boolean mRunAsVtsSelfTest = false; 430 431 @Option(name = "exclude-coverage-path", 432 description = "The coverage path that should be excluded in results. " 433 + "Used only when enable-coverage is true.") 434 private Collection<String> mExcludeCoveragePath = new ArrayList<>(); 435 436 @Option(name = "mobly-test-module", 437 description = "Mobly test module name. " 438 + "If this value is specified, VTS will use mobly test template " 439 + "with the configurations." 440 + "Multiple values can be added by repeatly using this option.") 441 private Collection<String> mMoblyTestModule = new ArrayList<>(); 442 443 private IRunUtil mRunUtil = null; 444 private IBuildInfo mBuildInfo = null; 445 private String mRunName = "VtsHostDrivenTest"; 446 // the path of a dir which contains the test data files. 447 private String mTestCaseDataDir = "./"; 448 449 private VtsVendorConfigFileUtil configReader = null; 450 private IInvocationContext mInvocationContext = null; 451 452 /** 453 * {@inheritDoc} 454 */ 455 @Override 456 public void setInvocationContext(IInvocationContext context) { 457 mInvocationContext = context; 458 setDevice(context.getDevices().get(0)); 459 setBuild(context.getBuildInfos().get(0)); 460 } 461 462 /** 463 * @return the mInvocationContext 464 */ 465 public IInvocationContext getInvocationContext() { 466 return mInvocationContext; 467 } 468 469 /** 470 * @return the mRunUtil 471 */ 472 public IRunUtil getRunUtil() { 473 return mRunUtil; 474 } 475 476 /** 477 * @param mRunUtil the mRunUtil to set 478 */ 479 public void setRunUtil(IRunUtil mRunUtil) { 480 this.mRunUtil = mRunUtil; 481 } 482 483 /** 484 * {@inheritDoc} 485 */ 486 @Override 487 public void setDevice(ITestDevice device) { 488 mDevice = device; 489 } 490 491 /** 492 * {@inheritDoc} 493 */ 494 @Override 495 public ITestDevice getDevice() { 496 return mDevice; 497 } 498 499 public void setTestCasePath(String path){ 500 mTestCasePath = path; 501 } 502 503 public void setTestConfigPath(String path){ 504 mTestConfigPath = path; 505 } 506 507 /** 508 * {@inheritDoc} 509 */ 510 @Override 511 public void addIncludeFilter(String filter) { 512 mIncludeFilters.add(cleanFilter(filter)); 513 } 514 515 /** 516 * {@inheritDoc} 517 */ 518 @Override 519 public void addAllIncludeFilters(Set<String> filters) { 520 for (String filter : filters) { 521 mIncludeFilters.add(cleanFilter(filter)); 522 } 523 } 524 525 /** 526 * {@inheritDoc} 527 */ 528 @Override 529 public void addExcludeFilter(String filter) { 530 mExcludeFilters.add(cleanFilter(filter)); 531 } 532 533 /** 534 * {@inheritDoc} 535 */ 536 @Override 537 public void addAllExcludeFilters(Set<String> filters) { 538 for (String filter : filters) { 539 mExcludeFilters.add(cleanFilter(filter)); 540 } 541 } 542 543 /** 544 * Conforms filters using a {@link TestDescription} format 545 * to be recognized by the GTest executable. 546 */ 547 private String cleanFilter(String filter) { 548 return filter.replace('#', '.'); 549 } 550 551 /** 552 * {@inheritDoc} 553 */ 554 @Override 555 public long getRuntimeHint() { 556 return mRuntimeHint; 557 } 558 559 /** 560 * {@inheritDoc} 561 */ 562 @Override 563 public void setCollectTestsOnly(boolean shouldCollectTest) { 564 mCollectTestsOnly = shouldCollectTest; 565 } 566 567 /** 568 * Generate a device json object from ITestDevice object. 569 * 570 * @param device device object 571 * @throws RuntimeException 572 * @throws JSONException 573 */ 574 private JSONObject generateJsonDeviceItem(ITestDevice device) throws JSONException { 575 JSONObject deviceItemObject = new JSONObject(); 576 deviceItemObject.put(SERIAL, device.getSerialNumber()); 577 try { 578 deviceItemObject.put("product_type", device.getProductType()); 579 deviceItemObject.put("product_variant", device.getProductVariant()); 580 deviceItemObject.put("build_alias", device.getBuildAlias()); 581 deviceItemObject.put("build_id", device.getBuildId()); 582 deviceItemObject.put("build_flavor", device.getBuildFlavor()); 583 } catch (DeviceNotAvailableException e) { 584 CLog.e("Device %s not available.", device.getSerialNumber()); 585 throw new RuntimeException("Failed to get device information"); 586 } 587 return deviceItemObject; 588 } 589 590 /** 591 * {@inheritDoc} 592 */ 593 @SuppressWarnings("deprecation") 594 @Override 595 public void run(ITestInvocationListener listener) 596 throws IllegalArgumentException, DeviceNotAvailableException { 597 if (mDevice == null) { 598 throw new DeviceNotAvailableException("Device has not been set."); 599 } 600 601 if (mBuildInfo == null) { 602 throw new RuntimeException("BuildInfo has not been set."); 603 } 604 605 if (mTestCasePath == null) { 606 if (!mBinaryTestSource.isEmpty()) { 607 String template; 608 switch (mBinaryTestType) { 609 case BINARY_TEST_TYPE_GTEST: 610 template = TEMPLATE_GTEST_BINARY_TEST_PATH; 611 break; 612 case BINARY_TEST_TYPE_HAL_HIDL_GTEST: 613 template = TEMPLATE_HAL_HIDL_GTEST_PATH; 614 break; 615 case BINARY_TEST_TYPE_HOST_BINARY_TEST: 616 template = TEMPLATE_HOST_BINARY_TEST_PATH; 617 break; 618 default: 619 template = TEMPLATE_BINARY_TEST_PATH; 620 } 621 CLog.i("Using default test case template at %s.", template); 622 setTestCasePath(template); 623 if (mEnableCoverage && !mGlobalCoverage) { 624 CLog.e("Only global coverage is supported for test type %s.", mBinaryTestType); 625 throw new RuntimeException("Failed to produce VTS runner test config"); 626 } 627 } else if (mBinaryTestType.equals(BINARY_TEST_TYPE_HAL_HIDL_REPLAY_TEST)) { 628 setTestCasePath(TEMPLATE_HAL_HIDL_REPLAY_TEST_PATH); 629 } else if (mBinaryTestType.equals(BINARY_TEST_TYPE_LLVMFUZZER)) { 630 // Fuzz test don't need test-case-path. 631 setTestCasePath(TEMPLATE_LLVMFUZZER_TEST_PATH); 632 } else if (!mMoblyTestModule.isEmpty()) { 633 setTestCasePath(TEMPLATE_MOBLY_TEST_PATH); 634 } else { 635 throw new IllegalArgumentException("test-case-path is not set."); 636 } 637 } 638 639 doRunTest(listener); 640 } 641 642 /** 643 * {@inheritDoc} 644 */ 645 @Override 646 public void setBuild(IBuildInfo buildInfo) { 647 mBuildInfo = buildInfo; 648 } 649 650 /** 651 * Populate a jsonObject with default fields. 652 * This method uses deepMergeJsonObjects method from JsonUtil to merge the default config file with the target 653 * config file. Field already defined in target config file will not be overwritten. Also, JSONArray will not be 654 * deep merged. 655 * 656 * @param jsonObject the target json object to populate 657 * @param testCaseDataDir data file path 658 * @throws IOException 659 * @throws JSONException 660 */ 661 private void populateDefaultJsonFields(JSONObject jsonObject, String testCaseDataDir) 662 throws IOException, JSONException { 663 CLog.i("Populating default fields to json object from %s", DEFAULT_TESTCASE_CONFIG_PATH); 664 String content = FileUtil.readStringFromFile(new File(mTestCaseDataDir, DEFAULT_TESTCASE_CONFIG_PATH)); 665 JSONObject defaultJsonObject = new JSONObject(content); 666 667 JsonUtil.deepMergeJsonObjects(jsonObject, defaultJsonObject); 668 } 669 670 /** 671 * This method reads the provided VTS runner test json config, adds or updates some of its 672 * fields (e.g., to add build info and device serial IDs), and returns the updated json object. 673 * This method calls populateDefaultJsonFields to populate the config JSONObject if the config file is missing 674 * or some required fields is missing from the JSONObject. If test name is not specified, this method 675 * will use the config file's file name file without extension as test name. If config file is missing, this method 676 * will use the test case's class name as test name. 677 * 678 * @param log_path the path of a directory to store the VTS runner logs. 679 * @return the updated JSONObject as the new test config. 680 */ 681 protected void updateVtsRunnerTestConfig(JSONObject jsonObject) 682 throws IOException, JSONException, RuntimeException { 683 configReader = new VtsVendorConfigFileUtil(); 684 if (configReader.LoadVendorConfig(mBuildInfo)) { 685 JSONObject vendorConfigJson = configReader.GetVendorConfigJson(); 686 if (vendorConfigJson != null) { 687 JsonUtil.deepMergeJsonObjects(jsonObject, vendorConfigJson); 688 } 689 } 690 691 CLog.i("Load original test config %s %s", mTestCaseDataDir, mTestConfigPath); 692 String content = null; 693 694 if (mTestConfigPath != null) { 695 content = FileUtil.readStringFromFile( 696 new File(Paths.get(mTestCaseDataDir, mTestConfigPath).toString())); 697 CLog.i("Loaded original test config %s", content); 698 if (content != null) { 699 JsonUtil.deepMergeJsonObjects(jsonObject, new JSONObject(content)); 700 } 701 } 702 703 populateDefaultJsonFields(jsonObject, mTestCaseDataDir); 704 CLog.i("Built a Json object using the loaded original test config"); 705 706 JSONArray deviceArray = new JSONArray(); 707 708 boolean coverageBuild = false; 709 boolean sancovBuild = false; 710 711 boolean first_device = true; 712 for (ITestDevice device : mInvocationContext.getDevices()) { 713 JSONObject deviceJson = generateJsonDeviceItem(device); 714 try { 715 String coverageProperty = device.getProperty(COVERAGE_PROPERTY); 716 boolean enable_coverage_for_device = 717 coverageProperty != null && coverageProperty.equals("1"); 718 if (first_device) { 719 coverageBuild = enable_coverage_for_device; 720 first_device = false; 721 } else { 722 if (coverageBuild && (!enable_coverage_for_device)) { 723 CLog.e("Device %s is not coverage build while others are.", 724 device.getSerialNumber()); 725 throw new RuntimeException("Device build not the same."); 726 } 727 } 728 } catch (DeviceNotAvailableException e) { 729 CLog.e("Device %s not available.", device.getSerialNumber()); 730 throw new RuntimeException("Failed to get device information"); 731 } 732 733 File sancovDir = 734 mBuildInfo.getFile(VtsCoveragePreparer.getSancovResourceDirKey(device)); 735 if (sancovDir != null) { 736 deviceJson.put("sancov_resources_path", sancovDir.getAbsolutePath()); 737 sancovBuild = true; 738 } 739 File gcovDir = mBuildInfo.getFile(VtsCoveragePreparer.getGcovResourceDirKey(device)); 740 if (gcovDir != null) { 741 deviceJson.put("gcov_resources_path", gcovDir.getAbsolutePath()); 742 coverageBuild = true; 743 } 744 deviceArray.put(deviceJson); 745 } 746 747 JSONArray testBedArray = (JSONArray) jsonObject.get(TEST_BED); 748 if (testBedArray.length() == 0) { 749 JSONObject testBedItemObject = new JSONObject(); 750 String testName; 751 if (mTestModuleName != null) { 752 testName = mTestModuleName; 753 } else { 754 CLog.w("--test-module-name not set (not recommended); deriving automatically"); 755 if (mTestConfigPath != null) { 756 testName = new File(mTestConfigPath).getName(); 757 testName = testName.replace(CONFIG_FILE_EXTENSION, ""); 758 } else if (mTestCasePath != null) { 759 testName = new File(mTestCasePath).getName(); 760 } else { 761 throw new RuntimeException( 762 "Failed to derive test module name; use --test-module-name option"); 763 } 764 } 765 CLog.logAndDisplay(LogLevel.INFO, "Setting test name as %s", testName); 766 testBedItemObject.put(NAME, testName); 767 testBedItemObject.put(ANDROIDDEVICE, deviceArray); 768 testBedArray.put(testBedItemObject); 769 } else if (testBedArray.length() == 1) { 770 JSONObject testBedItemObject = (JSONObject) testBedArray.get(0); 771 JSONArray androidDeviceArray = (JSONArray) testBedItemObject.get(ANDROIDDEVICE); 772 int length; 773 length = (androidDeviceArray.length() > deviceArray.length()) 774 ? androidDeviceArray.length() 775 : deviceArray.length(); 776 for (int index = 0; index < length; index++) { 777 if (index < androidDeviceArray.length()) { 778 if (index < deviceArray.length()) { 779 JsonUtil.deepMergeJsonObjects((JSONObject) androidDeviceArray.get(index), 780 (JSONObject) deviceArray.get(index)); 781 } 782 } else if (index < deviceArray.length()) { 783 androidDeviceArray.put(index, deviceArray.get(index)); 784 } 785 } 786 } else { 787 CLog.e("Multi-device not yet supported: %d devices requested", 788 testBedArray.length()); 789 throw new RuntimeException("Failed to produce VTS runner test config"); 790 } 791 jsonObject.put(DATA_FILE_PATH, mTestCaseDataDir); 792 CLog.i("Added %s = %s to the Json object", DATA_FILE_PATH, mTestCaseDataDir); 793 794 JSONObject build = new JSONObject(); 795 build.put(BUILD_ID, mBuildInfo.getBuildId()); 796 build.put(BUILD_TARGET, mBuildInfo.getBuildTargetName()); 797 jsonObject.put(BUILD, build); 798 CLog.i("Added %s to the Json object", BUILD); 799 800 JSONObject suite = new JSONObject(); 801 suite.put(NAME, mBuildInfo.getTestTag()); 802 suite.put(INCLUDE_FILTER, new JSONArray(mIncludeFilters)); 803 CLog.i("Added include filter to test suite: %s", mIncludeFilters); 804 suite.put(EXCLUDE_FILTER, new JSONArray(mExcludeFilters)); 805 CLog.i("Added exclude filter to test suite: %s", mExcludeFilters); 806 807 String coverageReportPath = mBuildInfo.getBuildAttributes().get("coverage_report_path"); 808 if (coverageReportPath != null) { 809 jsonObject.put(OUTPUT_COVERAGE_REPORT, true); 810 CLog.i("Added %s to the Json object", OUTPUT_COVERAGE_REPORT); 811 jsonObject.put(COVERAGE_REPORT_PATH, coverageReportPath); 812 CLog.i("Added %s to the Json object", COVERAGE_REPORT_PATH); 813 } 814 815 if (mExcludeOverInclude) { 816 jsonObject.put(EXCLUDE_OVER_INCLUDE, mExcludeOverInclude); 817 CLog.i("Added %s to the Json object", EXCLUDE_OVER_INCLUDE); 818 } 819 820 jsonObject.put(TEST_SUITE, suite); 821 CLog.i("Added %s to the Json object", TEST_SUITE); 822 823 jsonObject.put(TEST_MAX_TIMEOUT, mTestTimeout); 824 CLog.i("Added %s to the Json object: %d", TEST_MAX_TIMEOUT, mTestTimeout); 825 826 if (!mLogSeverity.isEmpty()) { 827 String logSeverity = mLogSeverity.toUpperCase(); 828 ArrayList<String> severityList = 829 new ArrayList<String>(Arrays.asList("ERROR", "WARNING", "INFO", "DEBUG")); 830 if (!severityList.contains(logSeverity)) { 831 CLog.w("Unsupported log severity %s, use default log_severity:INFO instead.", 832 logSeverity); 833 logSeverity = "INFO"; 834 } 835 jsonObject.put(LOG_SEVERITY, logSeverity); 836 CLog.i("Added %s to the Json object: %s", LOG_SEVERITY, logSeverity); 837 } 838 839 if (mAbi != null) { 840 jsonObject.put(ABI_NAME, mAbi.getName()); 841 CLog.i("Added %s to the Json object", ABI_NAME); 842 jsonObject.put(ABI_BITNESS, mAbi.getBitness()); 843 CLog.i("Added %s to the Json object", ABI_BITNESS); 844 } 845 846 if (mSkipOn32BitAbi) { 847 jsonObject.put(SKIP_ON_32BIT_ABI, mSkipOn32BitAbi); 848 CLog.i("Added %s to the Json object", SKIP_ON_32BIT_ABI); 849 } 850 851 if (mSkipOn64BitAbi) { 852 jsonObject.put(SKIP_ON_64BIT_ABI, mSkipOn64BitAbi); 853 CLog.i("Added %s to the Json object", SKIP_ON_64BIT_ABI); 854 } else if (mRun32bBitOn64BitAbi) { 855 jsonObject.put(RUN_32BIT_ON_64BIT_ABI, mRun32bBitOn64BitAbi); 856 CLog.i("Added %s to the Json object", RUN_32BIT_ON_64BIT_ABI); 857 } 858 859 if (mSkipIfThermalThrottling) { 860 jsonObject.put(SKIP_IF_THERMAL_THROTTLING, mSkipIfThermalThrottling); 861 CLog.i("Added %s to the Json object", SKIP_IF_THERMAL_THROTTLING); 862 } 863 864 jsonObject.put(DISABLE_CPU_FREQUENCY_SCALING, mDisableCpuFrequencyScaling); 865 CLog.i("Added %s to the Json object, value: %s", DISABLE_CPU_FREQUENCY_SCALING, 866 mDisableCpuFrequencyScaling); 867 868 if (!mBinaryTestSource.isEmpty()) { 869 jsonObject.put(BINARY_TEST_SOURCE, new JSONArray(mBinaryTestSource)); 870 CLog.i("Added %s to the Json object", BINARY_TEST_SOURCE); 871 } 872 873 if (!mBinaryTestWorkingDirectory.isEmpty()) { 874 jsonObject.put(BINARY_TEST_WORKING_DIRECTORY, 875 new JSONArray(mBinaryTestWorkingDirectory)); 876 CLog.i("Added %s to the Json object", BINARY_TEST_WORKING_DIRECTORY); 877 } 878 879 if (!mBinaryTestEnvp.isEmpty()) { 880 jsonObject.put(BINARY_TEST_ENVP, new JSONArray(mBinaryTestEnvp)); 881 CLog.i("Added %s to the Json object", BINARY_TEST_ENVP); 882 } 883 884 if (!mBinaryTestArgs.isEmpty()) { 885 jsonObject.put(BINARY_TEST_ARGS, new JSONArray(mBinaryTestArgs)); 886 CLog.i("Added %s to the Json object", BINARY_TEST_ARGS); 887 } 888 889 if (!mBinaryTestLdLibraryPath.isEmpty()) { 890 jsonObject.put(BINARY_TEST_LD_LIBRARY_PATH, 891 new JSONArray(mBinaryTestLdLibraryPath)); 892 CLog.i("Added %s to the Json object", BINARY_TEST_LD_LIBRARY_PATH); 893 } 894 895 if (mBugReportOnFailure) { 896 jsonObject.put(BUG_REPORT_ON_FAILURE, mBugReportOnFailure); 897 CLog.i("Added %s to the Json object", BUG_REPORT_ON_FAILURE); 898 } 899 900 if (!mLogcatOnFailure) { 901 jsonObject.put(LOGCAT_ON_FAILURE, mLogcatOnFailure); 902 CLog.i("Added %s to the Json object", LOGCAT_ON_FAILURE); 903 } 904 905 if (mEnableProfiling) { 906 jsonObject.put(ENABLE_PROFILING, mEnableProfiling); 907 CLog.i("Added %s to the Json object", ENABLE_PROFILING); 908 } 909 910 if (mSaveTraceFileRemote) { 911 jsonObject.put(SAVE_TRACE_FIEL_REMOTE, mSaveTraceFileRemote); 912 CLog.i("Added %s to the Json object", SAVE_TRACE_FIEL_REMOTE); 913 } 914 915 if (mEnableSystrace) { 916 jsonObject.put(ENABLE_SYSTRACE, mEnableSystrace); 917 CLog.i("Added %s to the Json object", ENABLE_SYSTRACE); 918 } 919 920 if (mEnableCoverage) { 921 jsonObject.put(GLOBAL_COVERAGE, mGlobalCoverage); 922 if (!mExcludeCoveragePath.isEmpty()) { 923 jsonObject.put(EXCLUDE_COVERAGE_PATH, new JSONArray(mExcludeCoveragePath)); 924 CLog.i("Added %s to the Json object", EXCLUDE_COVERAGE_PATH); 925 } 926 if (coverageBuild) { 927 jsonObject.put(ENABLE_COVERAGE, mEnableCoverage); 928 CLog.i("Added %s to the Json object", ENABLE_COVERAGE); 929 } else { 930 CLog.i("Device build has coverage disabled"); 931 } 932 } 933 934 if (mEnableSancov) { 935 if (sancovBuild) { 936 jsonObject.put(ENABLE_SANCOV, mEnableSancov); 937 CLog.i("Added %s to the Json object", ENABLE_SANCOV); 938 } else { 939 CLog.i("Device build has sancov disabled"); 940 } 941 } 942 943 if (mPreconditionHwBinderServiceName != null) { 944 jsonObject.put(PRECONDITION_HWBINDER_SERVICE, mPreconditionHwBinderServiceName); 945 CLog.i("Added %s to the Json object", PRECONDITION_HWBINDER_SERVICE); 946 } 947 948 if (mPreconditionFeature != null) { 949 jsonObject.put(PRECONDITION_FEATURE, mPreconditionFeature); 950 CLog.i("Added %s to the Json object", PRECONDITION_FEATURE); 951 } 952 953 if (!mPreconditionFilePathPrefix.isEmpty()) { 954 jsonObject.put( 955 PRECONDITION_FILE_PATH_PREFIX, new JSONArray(mPreconditionFilePathPrefix)); 956 CLog.i("Added %s to the Json object", PRECONDITION_FILE_PATH_PREFIX); 957 } 958 959 if (mPreconditionFirstApiLevel != 0) { 960 jsonObject.put(PRECONDITION_FIRST_API_LEVEL, mPreconditionFirstApiLevel); 961 CLog.i("Added %s to the Json object", PRECONDITION_FIRST_API_LEVEL); 962 } 963 964 if (mPreconditionLshal != null) { 965 jsonObject.put(PRECONDITION_LSHAL, mPreconditionLshal); 966 CLog.i("Added %s to the Json object", PRECONDITION_LSHAL); 967 } 968 969 if (mPreconditionVintf != null) { 970 jsonObject.put(PRECONDITION_VINTF, mPreconditionVintf); 971 CLog.i("Added %s to the Json object", PRECONDITION_VINTF); 972 } 973 974 if (mPreconditionSysProp != null) { 975 jsonObject.put(PRECONDITION_SYSPROP, mPreconditionSysProp); 976 CLog.i("Added %s to the Json object", PRECONDITION_SYSPROP); 977 } 978 979 if (!mBinaryTestProfilingLibraryPath.isEmpty()) { 980 jsonObject.put(BINARY_TEST_PROFILING_LIBRARY_PATH, 981 new JSONArray(mBinaryTestProfilingLibraryPath)); 982 CLog.i("Added %s to the Json object", BINARY_TEST_PROFILING_LIBRARY_PATH); 983 } 984 985 if (mBinaryTestDisableFramework) { 986 jsonObject.put(BINARY_TEST_DISABLE_FRAMEWORK, mBinaryTestDisableFramework); 987 CLog.i("Added %s to the Json object", BINARY_TEST_DISABLE_FRAMEWORK); 988 } 989 990 if (mBinaryTestStopNativeServers) { 991 jsonObject.put(BINARY_TEST_STOP_NATIVE_SERVERS, mBinaryTestStopNativeServers); 992 CLog.i("Added %s to the Json object", BINARY_TEST_STOP_NATIVE_SERVERS); 993 } 994 995 if (!mNativeServerProcessName.isEmpty()) { 996 jsonObject.put(NATIVE_SERVER_PROCESS_NAME, new JSONArray(mNativeServerProcessName)); 997 CLog.i("Added %s to the Json object", NATIVE_SERVER_PROCESS_NAME); 998 } 999 1000 if (!mHalHidlReplayTestTracePaths.isEmpty()) { 1001 jsonObject.put(HAL_HIDL_REPLAY_TEST_TRACE_PATHS, 1002 new JSONArray(mHalHidlReplayTestTracePaths)); 1003 CLog.i("Added %s to the Json object", HAL_HIDL_REPLAY_TEST_TRACE_PATHS); 1004 } 1005 1006 if (mHalHidlPackageName != null) { 1007 jsonObject.put(HAL_HIDL_PACKAGE_NAME, mHalHidlPackageName); 1008 CLog.i("Added %s to the Json object", SYSTRACE_PROCESS_NAME); 1009 } 1010 1011 if (mSystraceProcessName != null) { 1012 jsonObject.put(SYSTRACE_PROCESS_NAME, mSystraceProcessName); 1013 CLog.i("Added %s to the Json object", SYSTRACE_PROCESS_NAME); 1014 } 1015 1016 if (mPassthroughMode) { 1017 jsonObject.put(PASSTHROUGH_MODE, mPassthroughMode); 1018 CLog.i("Added %s to the Json object", PASSTHROUGH_MODE); 1019 } 1020 1021 if (mCollectTestsOnly) { 1022 jsonObject.put(COLLECT_TESTS_ONLY, mCollectTestsOnly); 1023 CLog.i("Added %s to the Json object", COLLECT_TESTS_ONLY); 1024 } 1025 1026 if (mGtestBatchMode) { 1027 jsonObject.put(GTEST_BATCH_MODE, mGtestBatchMode); 1028 CLog.i("Added %s to the Json object", GTEST_BATCH_MODE); 1029 } 1030 1031 if (mLtpNumberOfThreads >= 0) { 1032 jsonObject.put(LTP_NUMBER_OF_THREADS, mLtpNumberOfThreads); 1033 CLog.i("Added %s to the Json object", LTP_NUMBER_OF_THREADS); 1034 } 1035 1036 if (mRunAsVtsSelfTest) { 1037 jsonObject.put(RUN_AS_VTS_SELF_TEST, mRunAsVtsSelfTest); 1038 CLog.i("Added %s to the Json object", RUN_AS_VTS_SELF_TEST); 1039 } 1040 1041 if ("vts".equals(mBuildInfo.getTestTag())) { 1042 jsonObject.put(RUN_AS_COMPLIANCE_TEST, true); 1043 CLog.i("Added %s to the Json object", RUN_AS_COMPLIANCE_TEST); 1044 } 1045 1046 if (!mMoblyTestModule.isEmpty()) { 1047 jsonObject.put(MOBLY_TEST_MODULE, new JSONArray(mMoblyTestModule)); 1048 CLog.i("Added %s to the Json object", MOBLY_TEST_MODULE); 1049 } 1050 } 1051 1052 /** 1053 * Log a test module execution status to device logcat. 1054 * 1055 * @param status 1056 * @return true if succesful, false otherwise 1057 */ 1058 private boolean printToDeviceLogcatAboutTestModuleStatus(String status) { 1059 try { 1060 mDevice.executeShellCommand(String.format( 1061 "log -p i -t \"VTS\" \"[Test Module] %s %s\"", mTestModuleName, status)); 1062 } catch (DeviceNotAvailableException e) { 1063 CLog.w("Device unavailable while trying to write a message to logcat."); 1064 return false; 1065 } 1066 return true; 1067 } 1068 1069 private boolean AddTestModuleKeys(String test_module_name, long test_module_timestamp) { 1070 if (test_module_name.length() == 0 || test_module_timestamp == -1) { 1071 CLog.e(String.format("Test module keys (%s,%d) are invalid.", test_module_name, 1072 test_module_timestamp)); 1073 return false; 1074 } 1075 File reportFile = mBuildInfo.getFile(TEST_PLAN_REPORT_FILE); 1076 1077 try (FileWriter fw = new FileWriter(reportFile.getAbsoluteFile(), true); 1078 BufferedWriter bw = new BufferedWriter(fw); PrintWriter out = new PrintWriter(bw)) { 1079 out.println(String.format("%s %s", test_module_name, test_module_timestamp)); 1080 } catch (IOException e) { 1081 CLog.e(String.format( 1082 "Can't write to the test plan result file, %s", TEST_PLAN_REPORT_FILE)); 1083 return false; 1084 } 1085 return true; 1086 } 1087 1088 /** 1089 * This method prepares a command for the test and runs the python file as 1090 * given in the arguments. 1091 * 1092 * @param listener 1093 * @throws RuntimeException 1094 * @throws IllegalArgumentException 1095 */ 1096 private void doRunTest(ITestLifeCycleReceiver listener) 1097 throws RuntimeException, IllegalArgumentException { 1098 CLog.i("Device serial number: " + mDevice.getSerialNumber()); 1099 1100 setTestCaseDataDir(); 1101 1102 JSONObject jsonObject = new JSONObject(); 1103 File vtsRunnerLogDir = null; 1104 try { 1105 vtsRunnerLogDir = FileUtil.createTempDir("vts-runner-log"); 1106 updateVtsRunnerTestConfig(jsonObject); 1107 1108 jsonObject.put(LOG_PATH, vtsRunnerLogDir.getAbsolutePath()); 1109 CLog.i("Added %s to the Json object", LOG_PATH); 1110 } catch(IOException e) { 1111 throw new RuntimeException("Failed to read test config json file"); 1112 } catch(JSONException e) { 1113 throw new RuntimeException("Failed to build updated test config json data"); 1114 } 1115 1116 CLog.i("config json: %s", jsonObject.toString()); 1117 1118 String jsonFilePath = null; 1119 try { 1120 File tmpFile = FileUtil.createTempFile( 1121 mBuildInfo.getTestTag() + "-config-" + mBuildInfo.getDeviceSerial(), ".json"); 1122 jsonFilePath = tmpFile.getAbsolutePath(); 1123 CLog.i("config json file path: %s", jsonFilePath); 1124 FileWriter fw = new FileWriter(jsonFilePath); 1125 fw.write(jsonObject.toString()); 1126 fw.close(); 1127 } catch(IOException e) { 1128 throw new RuntimeException("Failed to create device config json file"); 1129 } 1130 1131 VtsPythonRunnerHelper vtsPythonRunnerHelper = createVtsPythonRunnerHelper(); 1132 vtsPythonRunnerHelper.setPythonVersion(mPythonVersion); 1133 if (mPythonBin == null){ 1134 mPythonBin = vtsPythonRunnerHelper.getPythonBinary(); 1135 } 1136 1137 String[] baseOpts = { 1138 mPythonBin, 1139 }; 1140 String[] testModule = new String[2]; 1141 String[] cmd; 1142 if (mTestCasePathType != null && mTestCasePathType.toLowerCase().equals("file")) { 1143 testModule[0] = mTestCasePath; 1144 if (!mTestCasePath.endsWith(".py")) { 1145 testModule[0] += ".py"; 1146 } 1147 } else { 1148 baseOpts = new String[2]; 1149 baseOpts[0] = mPythonBin; 1150 baseOpts[1] = "-m"; 1151 testModule[0] = mTestCasePath.replace("/", "."); 1152 } 1153 testModule[1] = jsonFilePath; 1154 cmd = ArrayUtil.buildArray(baseOpts, testModule); 1155 1156 printToDeviceLogcatAboutTestModuleStatus("BEGIN"); 1157 1158 CommandResult commandResult = new CommandResult(); 1159 String interruptMessage = 1160 vtsPythonRunnerHelper.runPythonRunner(cmd, commandResult, mTestTimeout); 1161 1162 if (commandResult != null) { 1163 CommandStatus commandStatus = commandResult.getStatus(); 1164 if (commandStatus != CommandStatus.SUCCESS 1165 && commandStatus != CommandStatus.TIMED_OUT) { 1166 CLog.e("Python process failed"); 1167 CLog.e("Python path: %s", vtsPythonRunnerHelper.getPythonPath()); 1168 CLog.e("Command stdout: " + commandResult.getStdout()); 1169 CLog.e("Command stderr: " + commandResult.getStderr()); 1170 CLog.e("Command status: " + commandStatus); 1171 CLog.e("Python log: "); 1172 printVtsLogs(vtsRunnerLogDir); 1173 printToDeviceLogcatAboutTestModuleStatus("ERROR"); 1174 throw new RuntimeException("Failed to run VTS test"); 1175 } 1176 printToDeviceLogcatAboutTestModuleStatus("END"); 1177 } 1178 1179 VtsMultiDeviceTestResultParser parser = 1180 new VtsMultiDeviceTestResultParser(listener, mRunName); 1181 1182 if (mUseStdoutLogs) { 1183 if (commandResult.getStdout() == null) { 1184 CLog.e("The std:out is null for CommandResult."); 1185 throw new RuntimeException("The std:out is null for CommandResult."); 1186 } 1187 parser.processNewLines(commandResult.getStdout().split("\n")); 1188 } else { 1189 // parse from test_run_summary.json instead of stdout 1190 String jsonData = null; 1191 JSONObject object = null; 1192 File testRunSummary = getFileTestRunSummary(vtsRunnerLogDir); 1193 if (testRunSummary == null) { 1194 CLog.e("Couldn't locate the file : " + TEST_RUN_SUMMARY_FILE_NAME); 1195 } else { 1196 try { 1197 jsonData = FileUtil.readStringFromFile(testRunSummary); 1198 CLog.i("Test Result Summary: %s", jsonData); 1199 object = new JSONObject(jsonData); 1200 } catch (IOException e) { 1201 CLog.e("Error occurred in parsing Json file : %s", testRunSummary.toPath()); 1202 } catch (JSONException e) { 1203 CLog.e("Error occurred in parsing Json String : %s", jsonData); 1204 } 1205 if (object == null) { 1206 CLog.e("Json object is null."); 1207 throw new RuntimeException("Json object is null."); 1208 } 1209 parser.processJsonFile(object); 1210 1211 try { 1212 JSONObject planObject = object.getJSONObject(TESTMODULE); 1213 String test_module_name = planObject.getString("Name"); 1214 long test_module_timestamp = planObject.getLong("Timestamp"); 1215 AddTestModuleKeys(test_module_name, test_module_timestamp); 1216 } catch (JSONException e) { 1217 CLog.d("Key '%s' not found in result json summary", TESTMODULE); 1218 } 1219 } 1220 } 1221 printVtsLogs(vtsRunnerLogDir); 1222 1223 File reportMsg = FileUtil.findFile(vtsRunnerLogDir, REPORT_MESSAGE_FILE_NAME); 1224 CLog.i("Report message path: %s", reportMsg); 1225 1226 if (reportMsg == null) { 1227 CLog.e("Cannot find report message proto file."); 1228 } else if (reportMsg.length() > 0) { 1229 CLog.i("Uploading report message. File size: %s", reportMsg.length()); 1230 VtsDashboardUtil dashboardUtil = new VtsDashboardUtil(configReader); 1231 dashboardUtil.Upload(reportMsg.getAbsolutePath()); 1232 } else { 1233 CLog.i("Result uploading is not enabled."); 1234 } 1235 1236 FileUtil.recursiveDelete(vtsRunnerLogDir); 1237 CLog.i("Deleted the runner log dir, %s.", vtsRunnerLogDir); 1238 if (jsonFilePath != null) { 1239 FileUtil.deleteFile(new File(jsonFilePath)); 1240 CLog.i("Deleted the runner json config file, %s.", jsonFilePath); 1241 } 1242 1243 if (interruptMessage != null) { 1244 throw new RunInterruptedException(interruptMessage); 1245 } 1246 } 1247 1248 /** 1249 * This method return the file test_run_summary.json which is then used to parse logs. 1250 * 1251 * @param logDir : The file that needs to be converted 1252 * @return the file named test_run_summary.json 1253 * @throws IllegalArgumentException 1254 */ 1255 private File getFileTestRunSummary(File logDir) throws IllegalArgumentException { 1256 File[] children; 1257 if (logDir == null) { 1258 throw new IllegalArgumentException("Argument logDir is null."); 1259 } 1260 children = logDir.listFiles(); 1261 if (children != null) { 1262 for (File child : children) { 1263 if (!child.isDirectory()) { 1264 if (child.getName().equals(TEST_RUN_SUMMARY_FILE_NAME)) { 1265 return child; 1266 } 1267 } else { 1268 File file = getFileTestRunSummary(child); 1269 if (file != null) { 1270 return file; 1271 } 1272 } 1273 } 1274 } 1275 return null; 1276 } 1277 1278 /** 1279 * The method prints all VTS runner log files 1280 * 1281 * @param logDir the File instance of the base log dir. 1282 */ 1283 private void printVtsLogs(File logDir) { 1284 File[] children; 1285 if (logDir == null) { 1286 CLog.e("Scan VTS log dir: null\n"); 1287 return; 1288 } 1289 CLog.i("Scan VTS log dir %s\n", logDir.getAbsolutePath()); 1290 children = logDir.listFiles(); 1291 if (children != null) { 1292 for (File child : children) { 1293 if (child.isDirectory()) { 1294 if (!child.getName().equals("temp")) { 1295 // temp in python log directory is for temp files produced by test module 1296 // and thus should not be included in log printout 1297 printVtsLogs(child); 1298 } 1299 } else { 1300 CLog.i("VTS log file %s\n", child.getAbsolutePath()); 1301 try { 1302 if (child.getName().startsWith("vts_agent") && 1303 child.getName().endsWith(".log")) { 1304 CLog.i("Content: %s\n", FileUtil.readStringFromFile(child)); 1305 } else { 1306 CLog.i("skip %s\n", child.getName()); 1307 } 1308 } catch (IOException e) { 1309 CLog.e("I/O error\n"); 1310 } 1311 } 1312 } 1313 } 1314 } 1315 1316 /** 1317 * Creates VtsPythonRunnerHelper. 1318 */ 1319 protected VtsPythonRunnerHelper createVtsPythonRunnerHelper() { 1320 return new VtsPythonRunnerHelper(mBuildInfo); 1321 } 1322 1323 /** 1324 * Set the path for android-vts/testcases/ which keeps the VTS python code under vts. 1325 */ 1326 private void setTestCaseDataDir() { 1327 CompatibilityBuildHelper buildHelper = new CompatibilityBuildHelper(mBuildInfo); 1328 File testDir = null; 1329 try { 1330 testDir = buildHelper.getTestsDir(); 1331 } catch (FileNotFoundException e) { 1332 /* pass */ 1333 } 1334 if (testDir != null) { 1335 mTestCaseDataDir = testDir.getAbsolutePath(); 1336 } 1337 } 1338 1339 /** 1340 * {@inheritDoc} 1341 */ 1342 @Override 1343 public void setAbi(IAbi abi){ 1344 mAbi = abi; 1345 } 1346 } 1347
