1 #!/usr/local/bin/perl 2 # ******************************************************************************* 3 # * Copyright (C) 2002-2012 International Business Machines Corporation and * 4 # * others. All Rights Reserved. * 5 # ******************************************************************************* 6 7 use XML::LibXML; 8 9 # Assume we are running within the icu4j root directory 10 use lib 'src/com/ibm/icu/dev/test/perf'; 11 use Dataset; 12 my $OS=$^O; 13 14 my $CLASSPATH; 15 if ($OS eq "linux" || $OS eq "darwin") { 16 $CLASSPATH="../icu4j.jar:../tools/misc/out/lib/icu4j-tools.jar:out/bin"; 17 } else { 18 $CLASSPATH="../icu4j.jar;../tools/misc/out/lib/icu4j-tools.jar;out/bin"; 19 } 20 #--------------------------------------------------------------------- 21 22 # Methods to be tested. Each pair represents a test method and 23 # a baseline method which is used for comparison. 24 my @METHODS = ( 25 ['TestJDKConstruction', 'TestICUConstruction'], 26 ['TestJDKParse', 'TestICUParse'], 27 ['TestJDKFormat', 'TestICUFormat'] 28 ); 29 # Patterns which define the set of characters used for testing. 30 my @OPTIONS = ( 31 # locale pattern date string 32 [ "en_US", "dddd MMM yyyy", "15 Jan 2007"], 33 [ "sw_KE", "dddd MMM yyyy", "15 Jan 2007"], 34 [ "en_US", "HH:mm", "13:13"], 35 [ "en_US", "HH:mm zzzz", "13:13 Pacific Standard Time"], 36 [ "en_US", "HH:mm z", "13:13 PST"], 37 [ "en_US", "HH:mm Z", "13:13 -0800"], 38 ); 39 40 my $THREADS; # number of threads (input from command-line args) 41 my $CALIBRATE = 2; # duration in seconds for initial calibration 42 my $DURATION = 10; # duration in seconds for each pass 43 my $NUMPASSES = 4; # number of passes. If > 1 then the first pass 44 # is discarded as a JIT warm-up pass. 45 46 my $TABLEATTR = 'BORDER="1" CELLPADDING="4" CELLSPACING="0"'; 47 48 my $PLUS_MINUS = "±"; 49 50 if ($NUMPASSES < 3) { 51 die "Need at least 3 passes. One is discarded (JIT warmup) and need two to have 1 degree of freedom (t distribution)."; 52 } 53 54 55 # run all tests with the specified number of threads from command-line input 56 # (if there is no arguments, use $THREADS = 1) 57 foreach my $arg ($#ARGV >= 0 ? @ARGV : "1") { 58 $THREADS = $arg; 59 main(); 60 } 61 62 63 #--------------------------------------------------------------------- 64 sub main { 65 66 #-----------DATE FORMAT PERFORMANCE TESTS----------------------------- 67 my $testclass = 'com.ibm.icu.dev.test.perf.DateFormatPerformanceTest'; 68 #my $threads = ($THREADS > 1) ? "($THREADS threads)" : ""; 69 70 my $doc = XML::LibXML::Document->new('1.0', 'utf-8'); 71 my $root = $doc->createElement("perfTestResults"); 72 73 # my $raw = ""; 74 my @shortNames = ( "open" , "parse", "fmt"); 75 my $index=0; 76 77 for my $methodPair (@METHODS) { 78 79 my $testMethod = $methodPair->[0]; 80 my $baselineMethod = $methodPair->[1]; 81 my $testname = $shortNames[$index]; 82 $index++; 83 84 $OUT = ''; 85 my $patternCounter=1; 86 87 for my $pat (@OPTIONS) { 88 89 # measure the test method 90 my $t = measure2($testclass, $testMethod, $pat, -$DURATION); 91 my $testResult = $t->getMean(); 92 my $jdkElement = $doc->createElement("perfTestResult"); 93 my $testName = "DateFmt-$testname-pat$patternCounter-JDK"; 94 $jdkElement->setAttribute("test" => $testName); 95 $jdkElement->setAttribute("iterations" => 1); 96 $jdkElement->setAttribute("time" => $testResult); 97 $root->appendChild($jdkElement); 98 99 # measure baseline method 100 my $b = measure2($testclass, $baselineMethod, $pat, -$DURATION); 101 my $baseResult = $b->getMean(); 102 my $icuElement = $doc->createElement("perfTestResult"); 103 my $testName = "DateFmt-$testname-pat$patternCounter"; 104 $patternCounter++; 105 $icuElement->setAttribute("test"=> $testName); 106 $icuElement->setAttribute("iterations" => 1); 107 $icuElement->setAttribute("time" => $baseResult); 108 $root->appendChild($icuElement); 109 110 } 111 } 112 113 #------------------DECIMAL FORMAT TESTS--------------------------------- 114 115 my $testclass = 'com.ibm.icu.dev.test.perf.DecimalFormatPerformanceTest'; 116 my @OPTIONS = ( 117 # locale pattern date string 118 [ "en_US", "#,###.##", "1,234.56"], 119 [ "de_DE", "#,###.##", "1.234,56"], 120 ); 121 my $index=0; 122 for my $methodPair (@METHODS) { 123 124 my $testMethod = $methodPair->[0]; 125 my $baselineMethod = $methodPair->[1]; 126 my $testname = $shortNames[$index]; 127 $index++; 128 129 130 for my $pat (@OPTIONS) { 131 my $patternName = $pat->[0]; 132 133 # measure the test method 134 my $t = measure2($testclass, $testMethod, $pat, -$DURATION); 135 my $testResult = $t->getMean(); 136 my $jdkElement = $doc->createElement("perfTestResult"); 137 my $testName = "NumFmt-$testname-$patternName-JDK"; 138 $jdkElement->setAttribute("test" => $testName); 139 $jdkElement->setAttribute("iterations"=>1); 140 $jdkElement->setAttribute("time" => $testResult); 141 $root->appendChild($jdkElement); 142 143 # measure baseline method 144 my $b = measure2($testclass, $baselineMethod, $pat, -$DURATION); 145 my $baseResult = $b->getMean(); 146 my $icuElement = $doc->createElement("perfTestResult"); 147 my $testName = "NumFmt-$testname-$patternName"; 148 $icuElement->setAttribute("test"=> $testName); 149 $icuElement->setAttribute("iterations"=>1); 150 $icuElement->setAttribute("time" => $baseResult); 151 $root->appendChild($icuElement); 152 } 153 } 154 155 #----------------COLLATION PERFORMANCE TESTS--------------------------_ 156 157 %dataFiles = ( 158 "en_US", "TestNames_Latin.txt", 159 "da_DK", "TestNames_Latin.txt", 160 "de_DE", "TestNames_Latin.txt", 161 "de__PHONEBOOK", "TestNames_Latin.txt", 162 "fr_FR", "TestNames_Latin.txt", 163 "ja_JP", "TestNames_Latin.txt TestNames_Japanese_h.txt TestNames_Japanese_k.txt TestNames_Asian.txt", 164 "zh_CN", "TestNames_Latin.txt TestNames_Chinese.txt", 165 "zh_TW", "TestNames_Latin.txt TestNames_Chinese.txt", 166 "zh__PINYIN", "TestNames_Latin.txt TestNames_Chinese.txt", 167 "ru_RU", "TestNames_Latin.txt TestNames_Russian.txt", 168 "th", "TestNames_Latin.txt TestNames_Thai.txt", 169 "ko_KR", "TestNames_Latin.txt TestNames_Korean.txt", 170 ); 171 172 # Outer loop runs through the locales to test 173 # (Edit this list dirctly to make changes) 174 # 175 foreach $locale ( 176 "en_US", 177 "da_DK", 178 "de_DE", 179 "de__PHONEBOOK", 180 "fr_FR", 181 "ja_JP", 182 "zh_CN", 183 "zh_TW", 184 "zh__PINYIN", 185 "ko_KR", 186 "ru_RU", 187 "th", 188 ) 189 { 190 191 192 # 193 # Inner loop runs over the set of data files specified for each locale. 194 # (Edit the %datafiles initialization, above, to make changes. 195 # 196 $ff = $dataFiles{$locale}; 197 @ff = split(/[\s]+/, $ff); 198 $counter = 1; 199 foreach $data (@ff) { 200 # 201 # Run ICU Test for this (locale, data file) pair. 202 # 203 $iStrCol = `java -classpath $CLASSPATH com.ibm.icu.dev.test.perf.CollationPerformanceTest -terse -file data/collation/$data -locale $locale -loop 1000 -binsearch`; 204 print "java -classpath $CLASSPATH com.ibm.icu.dev.test.perf.CollationPerformanceTest -terse -file data/collation/$data -locale $locale -loop 1000 -binsearch\n"; 205 $iStrCol =~s/[,\s]*//g; # whack off the leading " ," in the returned result. 206 doKeyTimes("java -classpath $CLASSPATH com.ibm.icu.dev.test.perf.CollationPerformanceTest -terse -file data/collation/$data -locale $locale -loop 1000 -keygen", 207 my $iKeyGen, my $iKeyLen); 208 209 # 210 # Run Windows test for this (locale, data file) pair. Only do if 211 # we are not on Windows 98/ME and we hava a windows langID 212 # for the locale. 213 # 214 $wStrCol = $wKeyGen = $wKeyLen = 0; 215 my $wStrCol = `java -classpath $CLASSPATH com.ibm.icu.dev.test.perf.CollationPerformanceTest -terse -file data/collation/$data -locale $locale -loop 1000 -binsearch -java`; 216 $wStrCol =~s/[,\s]*//g; # whack off the leading " ," in the returned result. 217 doKeyTimes("java -classpath $CLASSPATH com.ibm.icu.dev.test.perf.CollationPerformanceTest -terse -file data/collation/$data -locale $locale -loop 1000 -keygen -java", 218 $wKeyGen, $wKeyLen); 219 220 $collDiff = $keyGenDiff = $keyLenDiff = 0; 221 if ($wKeyLen > 0) { 222 $collDiff = (($wStrCol - $iStrCol) / $iStrCol) * 100; 223 $keyGenDiff = (($wKeyGen - $iKeyGen) / $iKeyGen) * 100; 224 $keyLenDiff = (($wKeyLen - $iKeyLen) / $iKeyLen) * 100; 225 } 226 227 my $ICU = $doc->createElement("perfTestResult"); 228 my $testname = "Coll-$locale-data$counter-StrCol"; 229 #write the results corresponding to this local,data pair 230 $ICU->setAttribute("test"=> $testname); 231 $ICU->setAttribute("iterations"=>1000); 232 $ICU->setAttribute("time"=> $iStrCol); 233 $root->appendChild($ICU); 234 235 my $Key = $doc->createElement("perfTestResult"); 236 my $testname = "Coll-$locale-data$counter-keyGen"; 237 $Key->setAttribute("test"=> $testname); 238 $Key->setAttribute("iterations"=>1000); 239 $Key->setAttribute("time"=>$iKeyGen); 240 $root->appendChild($Key); 241 242 my $JDK = $doc->createElement("perfTestResult"); 243 my $testname = "Coll-$locale-data$counter-StrCol-JDK"; 244 $JDK->setAttribute("test"=>$testname); 245 $JDK->setAttribute("iterations"=>1000); 246 $JDK->setAttribute("time"=>$wStrCol); 247 $root->appendChild($JDK); 248 249 my $Key = $doc->createElement("perfTestResult"); 250 my $testname = "Coll-$locale-data$counter-keyGen-JDK"; 251 $Key->setAttribute("test"=>$testname); 252 $Key->setAttribute("iterations"=>1000); 253 $Key->setAttribute("time"=>$wKeyGen); 254 $root->appendChild($Key); 255 $counter++; 256 } 257 } 258 259 260 261 #----------WRITE RESULTS TO perf.xml----------------------- 262 $doc->setDocumentElement($root); 263 open my $out_fh, '>', "perf.xml"; 264 print {$out_fh} $doc->toString; 265 } 266 267 268 #--------------------------------------------------------------------- 269 # Append text to the global variable $OUT 270 sub out { 271 $OUT .= join('', @_); 272 } 273 274 275 #--------------------------------------------------------------------- 276 # Measure a given test method with a give test pattern using the 277 # global run parameters. 278 # 279 # @param the method to run 280 # @param the pattern defining characters to test 281 # @param if >0 then the number of iterations per pass. If <0 then 282 # (negative of) the number of seconds per pass. 283 # 284 # @return a Dataset object, scaled by iterations per pass and 285 # events per iteration, to give time per event 286 # 287 sub measure2 { 288 my @data = measure1(@_); 289 my $iterPerPass = shift(@data); 290 my $eventPerIter = shift(@data); 291 292 shift(@data) if (@data > 1); # discard first run 293 294 my $ds = Dataset->new(@data); 295 $ds->setScale(1.0e-3 / ($iterPerPass * $eventPerIter)); 296 $ds; 297 } 298 299 #--------------------------------------------------------------------- 300 # Measure a given test method with a give test pattern using the 301 # global run parameters. 302 # 303 # @param the method to run 304 # @param the pattern defining characters to test 305 # @param if >0 then the number of iterations per pass. If <0 then 306 # (negative of) the number of seconds per pass. 307 # 308 # @return array of: 309 # [0] iterations per pass 310 # [1] events per iteration 311 # [2..] ms reported for each pass, in order 312 # 313 sub measure1 { 314 my $testclass = shift; 315 my $method = shift; 316 my $pat = shift; 317 my $iterCount = shift; # actually might be -seconds/pass 318 319 # is $iterCount actually -seconds/pass? 320 if ($iterCount < 0) { 321 322 # calibrate: estimate ms/iteration 323 print "Calibrating..."; 324 my @t = callJava($testclass, $method, $pat, -$CALIBRATE, 1); 325 print "done.\n"; 326 327 my @data = split(/\s+/, $t[0]->[2]); 328 $data[0] *= 1.0e+3; 329 330 my $timePerIter = 1.0e-3 * $data[0] / $data[1]; 331 332 # determine iterations/pass 333 $iterCount = int(-$iterCount / $timePerIter + 0.5); 334 } 335 336 # run passes 337 print "Measuring $iterCount iterations x $NUMPASSES passes..."; 338 my @t = callJava($testclass, $method, $pat, $iterCount, $NUMPASSES); 339 print "done.\n"; 340 my @ms = (); 341 my @b; # scratch 342 for my $a (@t) { 343 # $a->[0]: method name, corresponds to $method 344 # $a->[1]: 'begin' data, == $iterCount 345 # $a->[2]: 'end' data, of the form <ms> <loops> <eventsPerIter> 346 # $a->[3...]: gc messages from JVM during pass 347 @b = split(/\s+/, $a->[2]); 348 push(@ms, $b[0] * 1.0e+3); 349 } 350 my $eventsPerIter = $b[2]; 351 352 my @ms_str = @ms; 353 $ms_str[0] .= " (discarded)" if (@ms_str > 1); 354 355 ($iterCount, $eventsPerIter, @ms); 356 } 357 358 #--------------------------------------------------------------------- 359 # Invoke java to run $TESTCLASS, passing it the given parameters. 360 # 361 # @param the method to run 362 # @param the number of iterations, or if negative, the duration 363 # in seconds. If more than on pass is desired, pass in 364 # a string, e.g., "100 100 100". 365 # @param the pattern defining characters to test 366 # 367 # @return an array of results. Each result is an array REF 368 # describing one pass. The array REF contains: 369 # ->[0]: The method name as reported 370 # ->[1]: The params on the '= <meth> begin ...' line 371 # ->[2]: The params on the '= <meth> end ...' line 372 # ->[3..]: GC messages from the JVM, if any 373 # 374 sub callJava { 375 my $testclass = shift; 376 my $method = shift; 377 my $pat = shift; 378 my $n = shift; 379 my $passes = shift; 380 381 my $n = ($n < 0) ? "-t ".(-$n) : "-i ".$n; 382 383 my $cmd = "java -classpath $CLASSPATH $testclass $method $n -p $passes -L @$pat[0] \"@$pat[1]\" \"@$pat[2]\" -r $THREADS"; 384 print "[$cmd]\n"; # for debugging 385 open(PIPE, "$cmd|") or die "Can't run \"$cmd\""; 386 my @out; 387 while (<PIPE>) { 388 push(@out, $_); 389 } 390 close(PIPE) or die "Java failed: \"$cmd\""; 391 392 @out = grep(!/^\#/, @out); # filter out comments 393 394 #print "[", join("\n", @out), "]\n"; 395 396 my @results; 397 my $method = ''; 398 my $data = []; 399 foreach (@out) { 400 next unless (/\S/); 401 402 if (/^=\s*(\w+)\s*(\w+)\s*(.*)/) { 403 my ($m, $state, $d) = ($1, $2, $3); 404 #print "$_ => [[$m $state $data]]\n"; 405 if ($state eq 'begin') { 406 die "$method was begun but not finished" if ($method); 407 $method = $m; 408 push(@$data, $d); 409 push(@$data, ''); # placeholder for end data 410 } elsif ($state eq 'end') { 411 if ($m ne $method) { 412 die "$method end does not match: $_"; 413 } 414 $data->[1] = $d; # insert end data at [1] 415 #print "#$method:", join(";",@$data), "\n"; 416 unshift(@$data, $method); # add method to start 417 418 push(@results, $data); 419 $method = ''; 420 $data = []; 421 } else { 422 die "Can't parse: $_"; 423 } 424 } 425 elsif (/^\[/) { 426 if ($method) { 427 push(@$data, $_); 428 } else { 429 # ignore extraneous GC notices 430 } 431 } 432 else { 433 die "Can't parse: $_"; 434 } 435 } 436 437 die "$method was begun but not finished" if ($method); 438 439 @results; 440 } 441 442 #----------------------------------------------------------------------------------- 443 # doKeyGenTimes($Command_to_run, $time, $key_length) 444 # Do a key-generation test and return the time and key length/char values. 445 # 446 sub doKeyTimes($$$) { 447 # print "$_[0]\n"; 448 local($x) = `$_[0]`; # execute the collperf command. 449 ($_[1], $_[2]) = split(/\,/, $x); # collperf returns "time, keylength" string. 450 } 451 452 453 #eof 454
