1 #!/usr/bin/env python 2 # 3 # Copyright (C) 2016 The Android Open Source Project 4 # 5 # Licensed under the Apache License, Version 2.0 (the "License"); 6 # you may not use this file except in compliance with the License. 7 # You may obtain a copy of the License at 8 # 9 # http://www.apache.org/licenses/LICENSE-2.0 10 # 11 # Unless required by applicable law or agreed to in writing, software 12 # distributed under the License is distributed on an "AS IS" BASIS, 13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 # See the License for the specific language governing permissions and 15 # limitations under the License. 16 # 17 18 import logging 19 20 from vts.proto import VtsReportMessage_pb2 as ReportMsg 21 from vts.runners.host import asserts 22 from vts.runners.host import base_test 23 from vts.runners.host import const 24 from vts.runners.host import test_runner 25 from vts.utils.python.controllers import android_device 26 from vts.utils.python.cpu import cpu_frequency_scaling 27 28 # number of threads to use when running the throughput tests on target. 29 _THREAD_LIST = [2, 3, 4, 5, 7, 10, 30, 50, 70, 100] 30 31 _ITERATIONS_PER_SECOND = "iterations_per_second" 32 _TIME_AVERAGE = "time_average" 33 _TIME_WORST = "time_worst" 34 _TIME_BEST = "time_best" 35 _TIME_PERCENTILE = "time_percentile" 36 37 38 class HwBinderThroughputBenchmark(base_test.BaseTestClass): 39 """A test case for the binder throughput benchmarking.""" 40 41 def setUpClass(self): 42 required_params = ["hidl_hal_mode"] 43 self.getUserParams(required_params) 44 self.dut = self.registerController(android_device)[0] 45 self.dut.shell.InvokeTerminal("one") 46 self._cpu_freq = cpu_frequency_scaling.CpuFrequencyScalingController(self.dut) 47 self._cpu_freq.DisableCpuScaling() 48 49 def setUp(self): 50 self._cpu_freq.SkipIfThermalThrottling(retry_delay_secs=30) 51 52 def tearDown(self): 53 self._cpu_freq.SkipIfThermalThrottling() 54 55 def tearDownClass(self): 56 self._cpu_freq.EnableCpuScaling() 57 58 def testRunBenchmark32Bit(self): 59 """A test case which runs the 32-bit benchmark.""" 60 self.RunBenchmarkAndReportResult(32) 61 62 def testRunBenchmark64Bit(self): 63 """A test case which runs the 64-bit benchmark.""" 64 self.RunBenchmarkAndReportResult(64) 65 66 def RunBenchmarkAndReportResult(self, bits): 67 """Runs the native binary and stores its result to the web DB. 68 69 Args: 70 bits: integer (32 or 64), the number of bits in a word chosen 71 at the compile time (e.g., 32- vs. 64-bit library). 72 """ 73 labels = [] 74 iterations_per_second = [] 75 time_average = [] 76 time_best = [] 77 time_worst = [] 78 time_percentile_50 = [] 79 time_percentile_90 = [] 80 time_percentile_95 = [] 81 time_percentile_99 = [] 82 83 for thread in _THREAD_LIST: 84 result = self.RunBenchmark(bits, thread) 85 labels.append("%s_thread" % thread) 86 iterations_per_second.append(result["iterations_per_second"]) 87 time_average.append(result["time_average"]) 88 time_best.append(result["time_best"]) 89 time_worst.append(result["time_worst"]) 90 time_percentile_50.append(result["time_percentile"][50]) 91 time_percentile_90.append(result["time_percentile"][90]) 92 time_percentile_95.append(result["time_percentile"][95]) 93 time_percentile_99.append(result["time_percentile"][99]) 94 95 # To upload to the web DB. 96 self.web.AddProfilingDataLabeledVector( 97 "hwbinder_throughput_iterations_per_second_%sbits" % bits, 98 labels, iterations_per_second, x_axis_label="Number of Threads", 99 y_axis_label="HwBinder RPC Iterations Per Second", 100 regression_mode=ReportMsg.VTS_REGRESSION_MODE_DISABLED) 101 102 self.web.AddProfilingDataLabeledVector( 103 "hwbinder_throughput_time_average_ns_%sbits" % bits, 104 labels, time_average, x_axis_label="Number of Threads", 105 y_axis_label="HwBinder RPC Time - Average (nanoseconds)", 106 regression_mode=ReportMsg.VTS_REGRESSION_MODE_DISABLED) 107 self.web.AddProfilingDataLabeledVector( 108 "hwbinder_throughput_time_best_ns_%sbits" % bits, 109 labels, time_best, x_axis_label="Number of Threads", 110 y_axis_label="HwBinder RPC Time - Best Case (nanoseconds)") 111 self.web.AddProfilingDataLabeledVector( 112 "hwbinder_throughput_time_worst_ns_%sbits" % bits, 113 labels, time_worst, x_axis_label="Number of Threads", 114 y_axis_label="HwBinder RPC Time - Worst Case (nanoseconds)", 115 regression_mode=ReportMsg.VTS_REGRESSION_MODE_DISABLED) 116 117 self.web.AddProfilingDataLabeledVector( 118 "hwbinder_throughput_time_50percentile_ns_%sbits" % bits, 119 labels, time_percentile_50, x_axis_label="Number of Threads", 120 y_axis_label="HwBinder RPC Time - 50 Percentile (nanoseconds)", 121 regression_mode=ReportMsg.VTS_REGRESSION_MODE_DISABLED) 122 self.web.AddProfilingDataLabeledVector( 123 "hwbinder_throughput_time_90percentile_ns_%sbits" % bits, 124 labels, time_percentile_90, x_axis_label="Number of Threads", 125 y_axis_label="HwBinder RPC Time - 90 Percentile (nanoseconds)", 126 regression_mode=ReportMsg.VTS_REGRESSION_MODE_DISABLED) 127 self.web.AddProfilingDataLabeledVector( 128 "hwbinder_throughput_time_95percentile_ns_%sbits" % bits, 129 labels, time_percentile_95, x_axis_label="Number of Threads", 130 y_axis_label="HwBinder RPC Time - 95 Percentile (nanoseconds)", 131 regression_mode=ReportMsg.VTS_REGRESSION_MODE_DISABLED) 132 self.web.AddProfilingDataLabeledVector( 133 "hwbinder_throughput_time_99percentile_ns_%sbits" % bits, 134 labels, time_percentile_99, x_axis_label="Number of Threads", 135 y_axis_label="HwBinder RPC Time - 99 Percentile (nanoseconds)", 136 regression_mode=ReportMsg.VTS_REGRESSION_MODE_DISABLED) 137 138 def RunBenchmark(self, bits, threads): 139 """Runs the native binary and parses its result. 140 141 Args: 142 bits: integer (32 or 64), the number of bits in a word chosen 143 at the compile time (e.g., 32- vs. 64-bit library). 144 threads: positive integer, the number of threads to use. 145 146 Returns: 147 a dict which contains the benchmarking result where the keys are: 148 'iterations_per_second', 'time_average', 'time_worst', 149 'time_best', 'time_percentile'. 150 """ 151 # Runs the benchmark. 152 logging.info("Start to run the benchmark with HIDL mode %s (%s bit mode)", 153 self.hidl_hal_mode, bits) 154 binary = "/data/local/tmp/%s/hwbinderThroughputTest%s" % (bits, bits) 155 156 results = self.dut.shell.one.Execute( 157 ["chmod 755 %s" % binary, 158 "VTS_ROOT_PATH=/data/local/tmp " \ 159 "LD_LIBRARY_PATH=/system/lib%s:/data/local/tmp/%s/hw:" 160 "/data/local/tmp/%s:" 161 "$LD_LIBRARY_PATH %s -m %s -w %s" % (bits, bits, bits, binary, self.hidl_hal_mode.encode("utf-8"), threads)]) 162 163 # Parses the result. 164 asserts.assertEqual(len(results[const.STDOUT]), 2) 165 logging.info("stderr: %s", results[const.STDERR][1]) 166 stdout_lines = results[const.STDOUT][1].split("\n") 167 logging.info("stdout: %s", stdout_lines) 168 169 asserts.assertFalse( 170 any(results[const.EXIT_CODE]), 171 "testRunBenchmark%sBit(%s thread) failed." % (bits, threads)) 172 173 # To upload to the web DB. 174 summary = {} 175 index = next(i for i, string in enumerate(stdout_lines) 176 if "iterations per sec:" in string) 177 summary[_ITERATIONS_PER_SECOND] = int(float( 178 stdout_lines[index].replace("iterations per sec: ", ""))) 179 # an example is 'iterations per sec: 34868.7' 180 181 index = next(i for i, string in enumerate(stdout_lines) 182 if "average:" in string) 183 stats_string = stdout_lines[index].split() 184 # an example is 'average:0.0542985ms worst:0.314584ms best:0.02651ms' 185 summary[_TIME_AVERAGE] = int(float( 186 stats_string[0].replace( 187 "average:", "").replace("ms", "")) * 1000000) 188 summary[_TIME_WORST] = int(float( 189 stats_string[1].replace("worst:", "").replace("ms", "")) * 1000000) 190 summary[_TIME_BEST] = int(float( 191 stats_string[2].replace("best:", "").replace("ms", "")) * 1000000) 192 193 index = next(i for i, string in enumerate(stdout_lines) 194 if "50%: " in string) 195 percentiles_string = stdout_lines[index].split() 196 summary[_TIME_PERCENTILE] = {} 197 summary[_TIME_PERCENTILE][50] = int(float(percentiles_string[1]) 198 * 1000000) 199 summary[_TIME_PERCENTILE][90] = int(float(percentiles_string[3]) 200 * 1000000) 201 summary[_TIME_PERCENTILE][95] = int(float(percentiles_string[5]) 202 * 1000000) 203 summary[_TIME_PERCENTILE][99] = int(float(percentiles_string[7]) 204 * 1000000) 205 return summary 206 207 if __name__ == "__main__": 208 test_runner.main() 209
