1 #!/usr/bin/env python 2 # 3 # Copyright (C) 2016 The Android Open Source Project 4 # 5 # Licensed under the Apache License, Version 2.0 (the "License"); 6 # you may not use this file except in compliance with the License. 7 # You may obtain a copy of the License at 8 # 9 # http://www.apache.org/licenses/LICENSE-2.0 10 # 11 # Unless required by applicable law or agreed to in writing, software 12 # distributed under the License is distributed on an "AS IS" BASIS, 13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 # See the License for the specific language governing permissions and 15 # limitations under the License. 16 # 17 18 import logging 19 20 from vts.proto import VtsReportMessage_pb2 as ReportMsg 21 from vts.runners.host import asserts 22 from vts.runners.host import base_test 23 from vts.runners.host import const 24 from vts.runners.host import test_runner 25 from vts.utils.python.controllers import android_device 26 from vts.utils.python.cpu import cpu_frequency_scaling 27 28 # number of threads to use when running the throughput tests on target. 29 _THREAD_LIST = [2, 3, 4, 5, 7, 10, 30, 50, 70, 100, 200] 30 31 _ITERATIONS_PER_SECOND = "iterations_per_second" 32 _TIME_AVERAGE = "time_average" 33 _TIME_WORST = "time_worst" 34 _TIME_BEST = "time_best" 35 _TIME_PERCENTILE = "time_percentile" 36 37 38 class BinderThroughputBenchmark(base_test.BaseTestClass): 39 """A test case for the binder throughput benchmarking.""" 40 41 def setUpClass(self): 42 self.dut = self.registerController(android_device)[0] 43 self.dut.shell.InvokeTerminal("one") 44 self._cpu_freq = cpu_frequency_scaling.CpuFrequencyScalingController(self.dut) 45 self._cpu_freq.DisableCpuScaling() 46 47 def setUp(self): 48 self._cpu_freq.SkipIfThermalThrottling(retry_delay_secs=30) 49 50 def tearDown(self): 51 self._cpu_freq.SkipIfThermalThrottling() 52 53 def tearDownClass(self): 54 self._cpu_freq.EnableCpuScaling() 55 56 def testRunBenchmark32Bit(self): 57 """A test case which runs the 32-bit benchmark.""" 58 self.RunBenchmarkAndReportResult(32) 59 60 def testRunBenchmark64Bit(self): 61 """A test case which runs the 64-bit benchmark.""" 62 self.RunBenchmarkAndReportResult(64) 63 64 def RunBenchmarkAndReportResult(self, bits): 65 """Runs the native binary and stores its result to the web DB. 66 67 Args: 68 bits: integer (32 or 64), the number of bits in a word chosen 69 at the compile time (e.g., 32- vs. 64-bit library). 70 """ 71 labels = [] 72 iterations_per_second = [] 73 time_average = [] 74 time_best = [] 75 time_worst = [] 76 time_percentile_50 = [] 77 time_percentile_90 = [] 78 time_percentile_95 = [] 79 time_percentile_99 = [] 80 81 for thread in _THREAD_LIST: 82 result = self.RunBenchmark(bits, thread) 83 labels.append("%s_thread" % thread) 84 iterations_per_second.append(result["iterations_per_second"]) 85 time_average.append(result["time_average"]) 86 time_best.append(result["time_best"]) 87 time_worst.append(result["time_worst"]) 88 time_percentile_50.append(result["time_percentile"][50]) 89 time_percentile_90.append(result["time_percentile"][90]) 90 time_percentile_95.append(result["time_percentile"][95]) 91 time_percentile_99.append(result["time_percentile"][99]) 92 93 # To upload to the web DB. 94 self.web.AddProfilingDataLabeledVector( 95 "binder_throughput_iterations_per_second_%sbits" % bits, 96 labels, iterations_per_second, x_axis_label="Number of Threads", 97 y_axis_label="Binder RPC Iterations Per Second", 98 regression_mode=ReportMsg.VTS_REGRESSION_MODE_DISABLED) 99 100 self.web.AddProfilingDataLabeledVector( 101 "binder_throughput_time_average_ns_%sbits" % bits, 102 labels, time_average, x_axis_label="Number of Threads", 103 y_axis_label="Binder RPC Time - Average (nanoseconds)", 104 regression_mode=ReportMsg.VTS_REGRESSION_MODE_DISABLED) 105 self.web.AddProfilingDataLabeledVector( 106 "binder_throughput_time_best_ns_%sbits" % bits, 107 labels, time_best, x_axis_label="Number of Threads", 108 y_axis_label="Binder RPC Time - Best Case (nanoseconds)") 109 self.web.AddProfilingDataLabeledVector( 110 "binder_throughput_time_worst_ns_%sbits" % bits, 111 labels, time_worst, x_axis_label="Number of Threads", 112 y_axis_label="Binder RPC Time - Worst Case (nanoseconds)", 113 regression_mode=ReportMsg.VTS_REGRESSION_MODE_DISABLED) 114 115 self.web.AddProfilingDataLabeledVector( 116 "binder_throughput_time_50percentile_ns_%sbits" % bits, 117 labels, time_percentile_50, x_axis_label="Number of Threads", 118 y_axis_label="Binder RPC Time - 50 Percentile (nanoseconds)", 119 regression_mode=ReportMsg.VTS_REGRESSION_MODE_DISABLED) 120 self.web.AddProfilingDataLabeledVector( 121 "binder_throughput_time_90percentile_ns_%sbits" % bits, 122 labels, time_percentile_90, x_axis_label="Number of Threads", 123 y_axis_label="Binder RPC Time - 90 Percentile (nanoseconds)", 124 regression_mode=ReportMsg.VTS_REGRESSION_MODE_DISABLED) 125 self.web.AddProfilingDataLabeledVector( 126 "binder_throughput_time_95percentile_ns_%sbits" % bits, 127 labels, time_percentile_95, x_axis_label="Number of Threads", 128 y_axis_label="Binder RPC Time - 95 Percentile (nanoseconds)", 129 regression_mode=ReportMsg.VTS_REGRESSION_MODE_DISABLED) 130 self.web.AddProfilingDataLabeledVector( 131 "binder_throughput_time_99percentile_ns_%sbits" % bits, 132 labels, time_percentile_99, x_axis_label="Number of Threads", 133 y_axis_label="Binder RPC Time - 99 Percentile (nanoseconds)", 134 regression_mode=ReportMsg.VTS_REGRESSION_MODE_DISABLED) 135 136 def RunBenchmark(self, bits, threads): 137 """Runs the native binary and parses its result. 138 139 Args: 140 bits: integer (32 or 64), the number of bits in a word chosen 141 at the compile time (e.g., 32- vs. 64-bit library). 142 threads: positive integer, the number of threads to use. 143 144 Returns: 145 a dict which contains the benchmarking result where the keys are: 146 'iterations_per_second', 'time_average', 'time_worst', 147 'time_best', 'time_percentile'. 148 """ 149 # Runs the benchmark. 150 logging.info("Start to run the benchmark (%s bit mode)", bits) 151 binary = "/data/local/tmp/%s/binderThroughputTest%s" % (bits, bits) 152 153 results = self.dut.shell.one.Execute( 154 ["chmod 755 %s" % binary, 155 "LD_LIBRARY_PATH=/data/local/tmp/%s/hw:" 156 "/data/local/tmp/%s:" 157 "$LD_LIBRARY_PATH %s -w %s" % (bits, bits, binary, threads)]) 158 159 # Parses the result. 160 asserts.assertEqual(len(results[const.STDOUT]), 2) 161 logging.info("stderr: %s", results[const.STDERR][1]) 162 stdout_lines = results[const.STDOUT][1].split("\n") 163 logging.info("stdout: %s", stdout_lines) 164 165 asserts.assertFalse( 166 any(results[const.EXIT_CODE]), 167 "testRunBenchmark%sBit(%s thread) failed." % (bits, threads)) 168 169 # To upload to the web DB. 170 summary = {} 171 index = next(i for i, string in enumerate(stdout_lines) 172 if "iterations per sec:" in string) 173 summary[_ITERATIONS_PER_SECOND] = int(float( 174 stdout_lines[index].replace("iterations per sec: ", ""))) 175 # an example is 'iterations per sec: 34868.7' 176 177 index = next(i for i, string in enumerate(stdout_lines) 178 if "average:" in string) 179 stats_string = stdout_lines[index].split() 180 # an example is 'average:0.0542985ms worst:0.314584ms best:0.02651ms' 181 summary[_TIME_AVERAGE] = int(float( 182 stats_string[0].replace( 183 "average:", "").replace("ms", "")) * 1000000) 184 summary[_TIME_WORST] = int(float( 185 stats_string[1].replace("worst:", "").replace("ms", "")) * 1000000) 186 summary[_TIME_BEST] = int(float( 187 stats_string[2].replace("best:", "").replace("ms", "")) * 1000000) 188 189 index = next(i for i, string in enumerate(stdout_lines) 190 if "50%: " in string) 191 percentiles_string = stdout_lines[index].split() 192 summary[_TIME_PERCENTILE] = {} 193 summary[_TIME_PERCENTILE][50] = int(float(percentiles_string[1]) 194 * 1000000) 195 summary[_TIME_PERCENTILE][90] = int(float(percentiles_string[3]) 196 * 1000000) 197 summary[_TIME_PERCENTILE][95] = int(float(percentiles_string[5]) 198 * 1000000) 199 summary[_TIME_PERCENTILE][99] = int(float(percentiles_string[7]) 200 * 1000000) 201 return summary 202 203 if __name__ == "__main__": 204 test_runner.main() 205
