1 # Copyright (c) 2017 The Chromium OS Authors. All rights reserved. 2 # Use of this source code is governed by a BSD-style license that can be 3 # found in the LICENSE file. 4 5 import datetime 6 import glob 7 import os 8 import time 9 10 from autotest_lib.client.common_lib import error 11 from autotest_lib.client.common_lib.cros import system_metrics_collector 12 from autotest_lib.client.common_lib.cros.cfm.metrics import ( 13 media_metrics_collector) 14 from autotest_lib.server.cros import cfm_jmidata_log_collector 15 from autotest_lib.server.cros.cfm import cfm_base_test 16 17 _SHORT_TIMEOUT = 5 18 _MEASUREMENT_DURATION_SECONDS = 10 19 _TOTAL_TEST_DURATION_SECONDS = 900 20 21 _BASE_DIR = '/home/chronos/user/Storage/ext/' 22 _EXT_ID = 'ikfcpmgefdpheiiomgmhlmmkihchmdlj' 23 _JMI_DIR = '/0*/File\ System/000/t/00/*' 24 _JMI_SOURCE_DIR = _BASE_DIR + _EXT_ID + _JMI_DIR 25 26 class ParticipantCountMetric(system_metrics_collector.Metric): 27 """ 28 Metric for getting the current participant count in a call. 29 """ 30 def __init__(self, cfm_facade): 31 """ 32 Initializes with a cfm_facade. 33 34 @param cfm_facade object having a get_participant_count() method. 35 """ 36 super(ParticipantCountMetric, self).__init__( 37 'participant_count', 38 'participants', 39 higher_is_better=True) 40 self.cfm_facade = cfm_facade 41 42 def collect_metric(self): 43 """ 44 Collects one metric value. 45 """ 46 self.values.append(self.cfm_facade.get_participant_count()) 47 48 49 class enterprise_CFM_Perf(cfm_base_test.CfmBaseTest): 50 """This is a server test which clears device TPM and runs 51 enterprise_RemoraRequisition client test to enroll the device in to hotrod 52 mode. After enrollment is successful, it collects and logs cpu, memory and 53 temperature data from the device under test.""" 54 version = 1 55 56 def start_hangout(self): 57 """Waits for the landing page and starts a hangout session.""" 58 self.cfm_facade.wait_for_hangouts_telemetry_commands() 59 current_date = datetime.datetime.now().strftime("%Y-%m-%d") 60 hangout_name = current_date + '-cfm-perf' 61 self.cfm_facade.start_new_hangout_session(hangout_name) 62 63 64 def join_meeting(self): 65 """Waits for the landing page and joins a meeting session.""" 66 self.cfm_facade.wait_for_meetings_landing_page() 67 # Daily meeting for perf testing with 9 remote participants. 68 meeting_code = 'nis-rhmz-dyh' 69 self.cfm_facade.join_meeting_session(meeting_code) 70 71 72 def collect_perf_data(self): 73 """ 74 Collects run time data from the DUT using system_metrics_collector. 75 Writes the data to the chrome perf dashboard. 76 """ 77 start_time = time.time() 78 while (time.time() - start_time) < _TOTAL_TEST_DURATION_SECONDS: 79 time.sleep(_MEASUREMENT_DURATION_SECONDS) 80 self.metrics_collector.collect_snapshot() 81 if self.is_meeting: 82 # Media metrics collector is only available for Meet. 83 self.media_metrics_collector.collect_snapshot() 84 self.metrics_collector.write_metrics(self.output_perf_value) 85 86 def _get_average(self, data_type, jmidata): 87 """Computes mean of a list of numbers. 88 89 @param data_type: Type of data to be retrieved from jmi data log. 90 @param jmidata: Raw jmi data log to parse. 91 @return Mean computed from the list of numbers. 92 """ 93 data = self._get_data_from_jmifile(data_type, jmidata) 94 if not data: 95 return 0 96 return float(sum(data)) / len(data) 97 98 99 def _get_max_value(self, data_type, jmidata): 100 """Computes maximum value of a list of numbers. 101 102 @param data_type: Type of data to be retrieved from jmi data log. 103 @param jmidata: Raw jmi data log to parse. 104 @return Maxium value from the list of numbers. 105 """ 106 data = self._get_data_from_jmifile(data_type, jmidata) 107 if not data: 108 return 0 109 return max(data) 110 111 112 def _get_sum(self, data_type, jmidata): 113 """Computes sum of a list of numbers. 114 115 @param data_type: Type of data to be retrieved from jmi data log. 116 @param jmidata: Raw jmi data log to parse. 117 @return Sum computed from the list of numbers. 118 """ 119 data = self._get_data_from_jmifile(data_type, jmidata) 120 if not data: 121 return 0 122 return sum(data) 123 124 125 def _get_last_value(self, data_type, jmidata): 126 """Gets last value of a list of numbers. 127 128 @param data_type: Type of data to be retrieved from jmi data log. 129 @param jmidata: Raw jmi data log to parse. 130 @return The last value in the jmidata for the specified data_type. 0 if 131 there are no values in the jmidata for this data_type. 132 """ 133 data = self._get_data_from_jmifile(data_type, jmidata) 134 if not data: 135 return 0 136 return data[-1] 137 138 139 def _get_data_from_jmifile(self, data_type, jmidata): 140 """Gets data from jmidata log for given data type. 141 142 @param data_type: Type of data to be retrieved from jmi data log. 143 @param jmidata: Raw jmi data log to parse. 144 @return Data for given data type from jmidata log. 145 """ 146 if self.is_meeting: 147 try: 148 timestamped_values = self.media_metrics_collector.get_metric( 149 data_type) 150 except KeyError: 151 # Ensure we always return at least one element, or perf uploads 152 # will be sad. 153 return [0] 154 # Strip timestamps. 155 values = [x[1] for x in timestamped_values] 156 # Each entry in values is a list, extract the raw values: 157 res = [] 158 for value_list in values: 159 res.extend(value_list) 160 # Ensure we always return at least one element, or perf uploads will 161 # be sad. 162 return res or [0] 163 else: 164 return cfm_jmidata_log_collector.GetDataFromLogs( 165 self, data_type, jmidata) 166 167 168 def _get_file_to_parse(self): 169 """Copy jmi logs from client to test's results directory. 170 171 @return The newest jmi log file. 172 """ 173 self._host.get_file(_JMI_SOURCE_DIR, self.resultsdir) 174 source_jmi_files = self.resultsdir + '/0*' 175 if not source_jmi_files: 176 raise error.TestNAError('JMI data file not found.') 177 newest_file = max(glob.iglob(source_jmi_files), key=os.path.getctime) 178 return newest_file 179 180 def upload_jmidata(self): 181 """ 182 Write jmidata results to results-chart.json file for Perf Dashboard. 183 """ 184 jmi_file = self._get_file_to_parse() 185 jmifile_to_parse = open(jmi_file, 'r') 186 jmidata = jmifile_to_parse.read() 187 188 # Compute and save aggregated stats from JMI. 189 self.output_perf_value(description='sum_vid_in_frames_decoded', 190 value=self._get_sum('frames_decoded', jmidata), units='frames', 191 higher_is_better=True) 192 193 self.output_perf_value(description='sum_vid_out_frames_encoded', 194 value=self._get_sum('frames_encoded', jmidata), units='frames', 195 higher_is_better=True) 196 197 self.output_perf_value(description='vid_out_adapt_changes', 198 value=self._get_last_value('adaptation_changes', jmidata), 199 units='count', higher_is_better=False) 200 201 self.output_perf_value(description='video_out_encode_time', 202 value=self._get_data_from_jmifile( 203 'average_encode_time', jmidata), 204 units='ms', higher_is_better=False) 205 206 self.output_perf_value(description='max_video_out_encode_time', 207 value=self._get_max_value('average_encode_time', jmidata), 208 units='ms', higher_is_better=False) 209 210 self.output_perf_value(description='vid_out_bandwidth_adapt', 211 value=self._get_average('bandwidth_adaptation', jmidata), 212 units='bool', higher_is_better=False) 213 214 self.output_perf_value(description='vid_out_cpu_adapt', 215 value=self._get_average('cpu_adaptation', jmidata), 216 units='bool', higher_is_better=False) 217 218 self.output_perf_value(description='video_in_res', 219 value=self._get_data_from_jmifile( 220 'video_received_frame_height', jmidata), 221 units='px', higher_is_better=True) 222 223 self.output_perf_value(description='video_out_res', 224 value=self._get_data_from_jmifile( 225 'video_sent_frame_height', jmidata), 226 units='resolution', higher_is_better=True) 227 228 self.output_perf_value(description='vid_in_framerate_decoded', 229 value=self._get_data_from_jmifile( 230 'framerate_decoded', jmidata), 231 units='fps', higher_is_better=True) 232 233 self.output_perf_value(description='vid_out_framerate_input', 234 value=self._get_data_from_jmifile( 235 'framerate_outgoing', jmidata), 236 units='fps', higher_is_better=True) 237 238 self.output_perf_value(description='vid_in_framerate_to_renderer', 239 value=self._get_data_from_jmifile( 240 'framerate_to_renderer', jmidata), 241 units='fps', higher_is_better=True) 242 243 self.output_perf_value(description='vid_in_framerate_received', 244 value=self._get_data_from_jmifile( 245 'framerate_received', jmidata), 246 units='fps', higher_is_better=True) 247 248 self.output_perf_value(description='vid_out_framerate_sent', 249 value=self._get_data_from_jmifile('framerate_sent', jmidata), 250 units='fps', higher_is_better=True) 251 252 self.output_perf_value(description='vid_in_frame_width', 253 value=self._get_data_from_jmifile( 254 'video_received_frame_width', jmidata), 255 units='px', higher_is_better=True) 256 257 self.output_perf_value(description='vid_out_frame_width', 258 value=self._get_data_from_jmifile( 259 'video_sent_frame_width', jmidata), 260 units='px', higher_is_better=True) 261 262 self.output_perf_value(description='vid_out_encode_cpu_usage', 263 value=self._get_data_from_jmifile( 264 'video_encode_cpu_usage', jmidata), 265 units='percent', higher_is_better=False) 266 267 total_vid_packets_sent = self._get_sum('video_packets_sent', jmidata) 268 total_vid_packets_lost = self._get_sum('video_packets_lost', jmidata) 269 lost_packet_percentage = float(total_vid_packets_lost)*100/ \ 270 float(total_vid_packets_sent) if \ 271 total_vid_packets_sent else 0 272 273 self.output_perf_value(description='lost_packet_percentage', 274 value=lost_packet_percentage, units='percent', 275 higher_is_better=False) 276 self.output_perf_value(description='cpu_usage_jmi', 277 value=self._get_data_from_jmifile('cpu_percent', jmidata), 278 units='percent', higher_is_better=False) 279 self.output_perf_value(description='renderer_cpu_usage', 280 value=self._get_data_from_jmifile( 281 'renderer_cpu_percent', jmidata), 282 units='percent', higher_is_better=False) 283 self.output_perf_value(description='browser_cpu_usage', 284 value=self._get_data_from_jmifile( 285 'browser_cpu_percent', jmidata), 286 units='percent', higher_is_better=False) 287 288 self.output_perf_value(description='gpu_cpu_usage', 289 value=self._get_data_from_jmifile( 290 'gpu_cpu_percent', jmidata), 291 units='percent', higher_is_better=False) 292 293 self.output_perf_value(description='active_streams', 294 value=self._get_data_from_jmifile( 295 'num_active_vid_in_streams', jmidata), 296 units='count', higher_is_better=True) 297 298 299 def initialize(self, host, run_test_only=False): 300 """ 301 Initializes common test properties. 302 303 @param host: a host object representing the DUT. 304 @param run_test_only: Wheter to run only the test or to also perform 305 deprovisioning, enrollment and system reboot. See cfm_base_test. 306 """ 307 super(enterprise_CFM_Perf, self).initialize(host, run_test_only) 308 self.system_facade = self._facade_factory.create_system_facade() 309 metrics = system_metrics_collector.create_default_metric_set( 310 self.system_facade) 311 metrics.append(ParticipantCountMetric(self.cfm_facade)) 312 self.metrics_collector = (system_metrics_collector. 313 SystemMetricsCollector(self.system_facade, 314 metrics)) 315 data_point_collector = media_metrics_collector.DataPointCollector( 316 self.cfm_facade) 317 self.media_metrics_collector = (media_metrics_collector 318 .MetricsCollector(data_point_collector)) 319 320 def run_once(self, is_meeting=False): 321 """Stays in a meeting/hangout and collects perf data.""" 322 self.is_meeting = is_meeting 323 if is_meeting: 324 self.join_meeting() 325 else: 326 self.start_hangout() 327 328 self.collect_perf_data() 329 330 if is_meeting: 331 self.cfm_facade.end_meeting_session() 332 else: 333 self.cfm_facade.end_hangout_session() 334 335 self.upload_jmidata() 336 337
