Home | History | Annotate | Download | only in tools 
      1 #!/usr/bin/env python
      2 # Copyright 2017 The PDFium Authors. All rights reserved.
      3 # Use of this source code is governed by a BSD-style license that can be
      4 # found in the LICENSE file.
      5 
      6 """Looks for performance regressions on all pushes since the last run.
      7 
      8 Run this nightly to have a periodical check for performance regressions.
      9 
     10 Stores the results for each run and last checkpoint in a results directory.
     11 """
     12 
     13 import argparse
     14 import datetime
     15 import json
     16 import os
     17 import sys
     18 
     19 from common import PrintWithTime
     20 from common import RunCommandPropagateErr
     21 from githelper import GitHelper
     22 from safetynet_conclusions import PrintConclusionsDictHumanReadable
     23 
     24 
     25 class JobContext(object):
     26   """Context for a single run, including name and directory paths."""
     27 
     28   def __init__(self, args):
     29     self.datetime = datetime.datetime.now().strftime('%Y-%m-%d-%H-%M-%S')
     30     self.results_dir = args.results_dir
     31     self.last_revision_covered_file = os.path.join(self.results_dir,
     32                                                    'last_revision_covered')
     33     self.run_output_dir = os.path.join(self.results_dir,
     34                                        'profiles_%s' % self.datetime)
     35     self.run_output_log_file = os.path.join(self.results_dir,
     36                                             '%s.log' % self.datetime)
     37 
     38 
     39 class JobRun(object):
     40   """A single run looking for regressions since the last one."""
     41 
     42   def __init__(self, args, context):
     43     """Constructor.
     44 
     45     Args:
     46       args: Namespace with arguments passed to the script.
     47       context: JobContext for this run.
     48     """
     49     self.git = GitHelper()
     50     self.args = args
     51     self.context = context
     52 
     53   def Run(self):
     54     """Searches for regressions.
     55 
     56     Will only write a checkpoint when first run, and on all subsequent runs
     57     a comparison is done against the last checkpoint.
     58 
     59     Returns:
     60       Exit code for the script: 0 if no significant changes are found; 1 if
     61       there was an error in the comparison; 3 if there was a regression; 4 if
     62       there was an improvement and no regression.
     63     """
     64     pdfium_src_dir = os.path.join(
     65         os.path.dirname(__file__),
     66         os.path.pardir,
     67         os.path.pardir)
     68     os.chdir(pdfium_src_dir)
     69 
     70     if not self.git.IsCurrentBranchClean() and not self.args.no_checkout:
     71       PrintWithTime('Current branch is not clean, aborting')
     72       return 1
     73 
     74     branch_to_restore = self.git.GetCurrentBranchName()
     75 
     76     if not self.args.no_checkout:
     77       self.git.FetchOriginMaster()
     78       self.git.Checkout('origin/master')
     79 
     80     # Make sure results dir exists
     81     if not os.path.exists(self.context.results_dir):
     82       os.makedirs(self.context.results_dir)
     83 
     84     if not os.path.exists(self.context.last_revision_covered_file):
     85       result = self._InitialRun()
     86     else:
     87       with open(self.context.last_revision_covered_file) as f:
     88         last_revision_covered = f.read().strip()
     89       result = self._IncrementalRun(last_revision_covered)
     90 
     91     self.git.Checkout(branch_to_restore)
     92     return result
     93 
     94   def _InitialRun(self):
     95     """Initial run, just write a checkpoint.
     96 
     97     Returns:
     98       Exit code for the script.
     99     """
    100     current = self.git.GetCurrentBranchHash()
    101 
    102     PrintWithTime('Initial run, current is %s' % current)
    103 
    104     self._WriteCheckpoint(current)
    105 
    106     PrintWithTime('All set up, next runs will be incremental and perform '
    107                   'comparisons')
    108     return 0
    109 
    110   def _IncrementalRun(self, last_revision_covered):
    111     """Incremental run, compare against last checkpoint and update it.
    112 
    113     Args:
    114       last_revision_covered: String with hash for last checkpoint.
    115 
    116     Returns:
    117       Exit code for the script.
    118     """
    119     current = self.git.GetCurrentBranchHash()
    120 
    121     PrintWithTime('Incremental run, current is %s, last is %s'
    122                   % (current, last_revision_covered))
    123 
    124     if not os.path.exists(self.context.run_output_dir):
    125       os.makedirs(self.context.run_output_dir)
    126 
    127     if current == last_revision_covered:
    128       PrintWithTime('No changes seen, finishing job')
    129       output_info = {
    130           'metadata': self._BuildRunMetadata(last_revision_covered,
    131                                              current,
    132                                              False)}
    133       self._WriteRawJson(output_info)
    134       return 0
    135 
    136     # Run compare
    137     cmd = ['testing/tools/safetynet_compare.py',
    138            '--this-repo',
    139            '--machine-readable',
    140            '--branch-before=%s' % last_revision_covered,
    141            '--output-dir=%s' % self.context.run_output_dir]
    142     cmd.extend(self.args.input_paths)
    143 
    144     json_output = RunCommandPropagateErr(cmd)
    145 
    146     if json_output is None:
    147       return 1
    148 
    149     output_info = json.loads(json_output)
    150 
    151     run_metadata = self._BuildRunMetadata(last_revision_covered,
    152                                           current,
    153                                           True)
    154     output_info.setdefault('metadata', {}).update(run_metadata)
    155     self._WriteRawJson(output_info)
    156 
    157     PrintConclusionsDictHumanReadable(output_info,
    158                                       colored=(not self.args.output_to_log
    159                                                and not self.args.no_color),
    160                                       key='after')
    161 
    162     status = 0
    163 
    164     if output_info['summary']['improvement']:
    165       PrintWithTime('Improvement detected.')
    166       status = 4
    167 
    168     if output_info['summary']['regression']:
    169       PrintWithTime('Regression detected.')
    170       status = 3
    171 
    172     if status == 0:
    173       PrintWithTime('Nothing detected.')
    174 
    175     self._WriteCheckpoint(current)
    176 
    177     return status
    178 
    179   def _WriteRawJson(self, output_info):
    180     json_output_file = os.path.join(self.context.run_output_dir, 'raw.json')
    181     with open(json_output_file, 'w') as f:
    182       json.dump(output_info, f)
    183 
    184   def _BuildRunMetadata(self, revision_before, revision_after,
    185                         comparison_performed):
    186     return {
    187         'datetime': self.context.datetime,
    188         'revision_before': revision_before,
    189         'revision_after': revision_after,
    190         'comparison_performed': comparison_performed,
    191     }
    192 
    193   def _WriteCheckpoint(self, checkpoint):
    194     if not self.args.no_checkpoint:
    195       with open(self.context.last_revision_covered_file, 'w') as f:
    196         f.write(checkpoint + '\n')
    197 
    198 
    199 def main():
    200   parser = argparse.ArgumentParser()
    201   parser.add_argument('results_dir',
    202                       help='where to write the job results')
    203   parser.add_argument('input_paths', nargs='+',
    204                       help='pdf files or directories to search for pdf files '
    205                            'to run as test cases')
    206   parser.add_argument('--no-checkout', action='store_true',
    207                       help='whether to skip checking out origin/master. Use '
    208                            'for script debugging.')
    209   parser.add_argument('--no-checkpoint', action='store_true',
    210                       help='whether to skip writing the new checkpoint. Use '
    211                            'for script debugging.')
    212   parser.add_argument('--no-color', action='store_true',
    213                       help='whether to write output without color escape '
    214                            'codes.')
    215   parser.add_argument('--output-to-log', action='store_true',
    216                       help='whether to write output to a log file')
    217   args = parser.parse_args()
    218 
    219   job_context = JobContext(args)
    220 
    221   if args.output_to_log:
    222     log_file = open(job_context.run_output_log_file, 'w')
    223     sys.stdout = log_file
    224     sys.stderr = log_file
    225 
    226   run = JobRun(args, job_context)
    227   result = run.Run()
    228 
    229   if args.output_to_log:
    230     log_file.close()
    231 
    232   return result
    233 
    234 
    235 if __name__ == '__main__':
    236   sys.exit(main())
    237 
    238