1 /* 2 * Copyright (C) 2016 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 package com.android.tradefed.testtype; 17 18 import static org.junit.Assert.assertEquals; 19 20 import com.android.tradefed.device.CollectingOutputReceiver; 21 import com.android.tradefed.log.LogUtil.CLog; 22 import com.android.tradefed.metrics.proto.MetricMeasurement.Metric; 23 import com.android.tradefed.result.ITestInvocationListener; 24 import com.android.tradefed.result.TestDescription; 25 26 import org.easymock.Capture; 27 import org.easymock.EasyMock; 28 import org.json.JSONException; 29 import org.json.JSONObject; 30 import org.junit.Test; 31 import org.junit.runner.RunWith; 32 import org.junit.runners.JUnit4; 33 34 import java.io.BufferedReader; 35 import java.io.File; 36 import java.io.IOException; 37 import java.io.InputStream; 38 import java.io.InputStreamReader; 39 import java.util.HashMap; 40 import java.util.Map; 41 42 /** Unit tests for {@link GoogleBenchmarkResultParser}. */ 43 @RunWith(JUnit4.class) 44 public class GoogleBenchmarkResultParserTest { 45 46 private static final String TEST_TYPE_DIR = "testtype"; 47 private static final String GBENCH_OUTPUT_FILE_1 = "gbench_output1.json"; 48 private static final String GBENCH_OUTPUT_FILE_2 = "gbench_output2.json"; 49 private static final String GBENCH_OUTPUT_FILE_3 = "gbench_output3.json"; 50 private static final String GBENCH_OUTPUT_FILE_4 = "gbench_output4.json"; 51 private static final String GBENCH_OUTPUT_FILE_5 = "gbench_output5.json"; 52 53 private static final String TEST_RUN = "test_run"; 54 55 /** 56 * Helper to read a file from the res/testtype directory and return its contents as a String[] 57 * 58 * @param filename the name of the file (without the extension) in the res/testtype directory 59 * @return a String[] of the 60 */ 61 private CollectingOutputReceiver readInFile(String filename) { 62 CollectingOutputReceiver output = null; 63 try { 64 output = new CollectingOutputReceiver(); 65 InputStream gtestResultStream1 = getClass().getResourceAsStream(File.separator + 66 TEST_TYPE_DIR + File.separator + filename); 67 BufferedReader reader = new BufferedReader(new InputStreamReader(gtestResultStream1)); 68 String line = null; 69 while ((line = reader.readLine()) != null) { 70 output.addOutput(line.getBytes(), 0, line.length()); 71 } 72 } 73 catch (NullPointerException e) { 74 CLog.e("GTest output file does not exist: " + filename); 75 } 76 catch (IOException e) { 77 CLog.e("Unable to read contents of gtest output file: " + filename); 78 } 79 return output; 80 } 81 82 /** Tests the parser for a simple test output for 2 tests. */ 83 @Test 84 public void testParseSimpleFile() throws Exception { 85 ITestInvocationListener mMockInvocationListener = 86 EasyMock.createMock(ITestInvocationListener.class); 87 mMockInvocationListener.testStarted((TestDescription) EasyMock.anyObject()); 88 EasyMock.expectLastCall().times(3); 89 Capture<HashMap<String, Metric>> capture1 = new Capture<>(); 90 Capture<HashMap<String, Metric>> capture2 = new Capture<>(); 91 Capture<HashMap<String, Metric>> capture3 = new Capture<>(); 92 mMockInvocationListener.testEnded( 93 (TestDescription) EasyMock.anyObject(), EasyMock.capture(capture1)); 94 mMockInvocationListener.testEnded( 95 (TestDescription) EasyMock.anyObject(), EasyMock.capture(capture2)); 96 mMockInvocationListener.testEnded( 97 (TestDescription) EasyMock.anyObject(), EasyMock.capture(capture3)); 98 EasyMock.replay(mMockInvocationListener); 99 CollectingOutputReceiver contents = readInFile(GBENCH_OUTPUT_FILE_1); 100 GoogleBenchmarkResultParser resultParser = 101 new GoogleBenchmarkResultParser(TEST_RUN, mMockInvocationListener); 102 Map<String, String> results = resultParser.parse(contents); 103 Map<String, String> expectedRes = new HashMap<String, String>(); 104 expectedRes.put("date", "2016-03-07 10:59:01"); 105 expectedRes.put("num_cpus", "48"); 106 expectedRes.put("mhz_per_cpu", "2601"); 107 expectedRes.put("cpu_scaling_enabled", "true"); 108 expectedRes.put("library_build_type", "debug"); 109 110 expectedRes.put("Pass", "3"); 111 assertEquals(expectedRes, results); 112 EasyMock.verify(mMockInvocationListener); 113 114 // Test 1 115 HashMap<String, Metric> resultTest1 = capture1.getValue(); 116 assertEquals(4, resultTest1.size()); 117 assertEquals("5", resultTest1.get("cpu_time").getMeasurements().getSingleString()); 118 assertEquals("5", resultTest1.get("real_time").getMeasurements().getSingleString()); 119 assertEquals("BM_one", resultTest1.get("name").getMeasurements().getSingleString()); 120 assertEquals( 121 "109451958", resultTest1.get("iterations").getMeasurements().getSingleString()); 122 123 // Test 2 124 HashMap<String, Metric> resultTest2 = capture2.getValue(); 125 assertEquals(4, resultTest2.size()); 126 assertEquals("11", resultTest2.get("cpu_time").getMeasurements().getSingleString()); 127 assertEquals("1", resultTest2.get("real_time").getMeasurements().getSingleString()); 128 assertEquals("BM_two", resultTest2.get("name").getMeasurements().getSingleString()); 129 assertEquals("50906784", resultTest2.get("iterations").getMeasurements().getSingleString()); 130 131 // Test 3 132 HashMap<String, Metric> resultTest3 = capture3.getValue(); 133 assertEquals(5, resultTest3.size()); 134 assertEquals("60", resultTest3.get("cpu_time").getMeasurements().getSingleString()); 135 assertEquals("60", resultTest3.get("real_time").getMeasurements().getSingleString()); 136 assertEquals( 137 "BM_string_strlen/64", resultTest3.get("name").getMeasurements().getSingleString()); 138 assertEquals("10499948", resultTest3.get("iterations").getMeasurements().getSingleString()); 139 assertEquals( 140 "1061047935", 141 resultTest3.get("bytes_per_second").getMeasurements().getSingleString()); 142 } 143 144 /** Tests the parser for a two simple test with different keys on the second test. */ 145 @Test 146 public void testParseSimpleFile_twoTests() throws Exception { 147 ITestInvocationListener mMockInvocationListener = 148 EasyMock.createMock(ITestInvocationListener.class); 149 mMockInvocationListener.testStarted((TestDescription) EasyMock.anyObject()); 150 EasyMock.expectLastCall().times(2); 151 Capture<HashMap<String, Metric>> capture = new Capture<>(); 152 mMockInvocationListener.testEnded( 153 (TestDescription) EasyMock.anyObject(), EasyMock.capture(capture)); 154 mMockInvocationListener.testEnded( 155 EasyMock.anyObject(), (HashMap<String, Metric>) EasyMock.anyObject()); 156 EasyMock.replay(mMockInvocationListener); 157 CollectingOutputReceiver contents = readInFile(GBENCH_OUTPUT_FILE_2); 158 GoogleBenchmarkResultParser resultParser = 159 new GoogleBenchmarkResultParser(TEST_RUN, mMockInvocationListener); 160 resultParser.parse(contents); 161 EasyMock.verify(mMockInvocationListener); 162 163 HashMap<String, Metric> results = capture.getValue(); 164 assertEquals(4, results.size()); 165 assertEquals("5", results.get("cpu_time").getMeasurements().getSingleString()); 166 assertEquals("5", results.get("real_time").getMeasurements().getSingleString()); 167 assertEquals("BM_one", results.get("name").getMeasurements().getSingleString()); 168 assertEquals("109451958", results.get("iterations").getMeasurements().getSingleString()); 169 } 170 171 /** 172 * Tests the parser with an error in the context, should stop parsing because format is 173 * unexpected. 174 */ 175 @Test 176 public void testParse_contextError() throws Exception { 177 ITestInvocationListener mMockInvocationListener = 178 EasyMock.createMock(ITestInvocationListener.class); 179 mMockInvocationListener.testRunFailed((String)EasyMock.anyObject()); 180 EasyMock.replay(mMockInvocationListener); 181 CollectingOutputReceiver contents = readInFile(GBENCH_OUTPUT_FILE_3); 182 GoogleBenchmarkResultParser resultParser = 183 new GoogleBenchmarkResultParser(TEST_RUN, mMockInvocationListener); 184 resultParser.parse(contents); 185 EasyMock.verify(mMockInvocationListener); 186 } 187 188 /** Tests the parser with an error: context is fine but no benchmark results */ 189 @Test 190 public void testParse_noBenchmarkResults() throws Exception { 191 ITestInvocationListener mMockInvocationListener = 192 EasyMock.createMock(ITestInvocationListener.class); 193 mMockInvocationListener.testRunFailed((String)EasyMock.anyObject()); 194 EasyMock.replay(mMockInvocationListener); 195 CollectingOutputReceiver contents = readInFile(GBENCH_OUTPUT_FILE_4); 196 GoogleBenchmarkResultParser resultParser = 197 new GoogleBenchmarkResultParser(TEST_RUN, mMockInvocationListener); 198 resultParser.parse(contents); 199 EasyMock.verify(mMockInvocationListener); 200 } 201 202 /** Test for {@link GoogleBenchmarkResultParser#parseJsonToMap(JSONObject)} */ 203 @Test 204 public void testJsonParse() throws JSONException { 205 ITestInvocationListener mMockInvocationListener = 206 EasyMock.createMock(ITestInvocationListener.class); 207 String jsonTest = "{ \"key1\": \"value1\", \"key2\": 2, \"key3\": [\"one\", \"two\"]}"; 208 JSONObject test = new JSONObject(jsonTest); 209 GoogleBenchmarkResultParser resultParser = 210 new GoogleBenchmarkResultParser(TEST_RUN, mMockInvocationListener); 211 Map<String, String> results = resultParser.parseJsonToMap(test); 212 assertEquals(results.get("key1"), "value1"); 213 assertEquals(results.get("key2"), "2"); 214 assertEquals(results.get("key3"), "[\"one\",\"two\"]"); 215 } 216 217 /** 218 * Test when a warning is printed before the JSON output. As long as the JSON is fine, we should 219 * still parse the output. 220 */ 221 @Test 222 public void testParseSimpleFile_withWarning() throws Exception { 223 ITestInvocationListener mMockInvocationListener = 224 EasyMock.createMock(ITestInvocationListener.class); 225 mMockInvocationListener.testStarted((TestDescription) EasyMock.anyObject()); 226 Capture<HashMap<String, Metric>> capture = new Capture<>(); 227 mMockInvocationListener.testEnded( 228 (TestDescription) EasyMock.anyObject(), EasyMock.capture(capture)); 229 EasyMock.replay(mMockInvocationListener); 230 CollectingOutputReceiver contents = readInFile(GBENCH_OUTPUT_FILE_5); 231 GoogleBenchmarkResultParser resultParser = 232 new GoogleBenchmarkResultParser(TEST_RUN, mMockInvocationListener); 233 resultParser.parse(contents); 234 EasyMock.verify(mMockInvocationListener); 235 236 HashMap<String, Metric> results = capture.getValue(); 237 assertEquals(5, results.size()); 238 assertEquals("19361", results.get("cpu_time").getMeasurements().getSingleString()); 239 assertEquals("44930", results.get("real_time").getMeasurements().getSingleString()); 240 assertEquals("BM_addInts", results.get("name").getMeasurements().getSingleString()); 241 assertEquals("36464", results.get("iterations").getMeasurements().getSingleString()); 242 assertEquals("ns", results.get("time_unit").getMeasurements().getSingleString()); 243 } 244 } 245
