1 # Copyright (c) 2011 The Chromium Authors. All rights reserved. 2 # Use of this source code is governed by a BSD-style license that can be 3 # found in the LICENSE file. 4 5 """Component for automatically creating masks of changing areas of a website. 6 7 Works by repeated invokation of a browser and scraping of the resulting page. 8 Areas that differ will be added to the auto-generated mask. The mask generator 9 considers the mask complete when further scrapes fail to produce any differences 10 in the mask. 11 """ 12 13 import os # Functions for walking the directory tree 14 import tempfile # Get a temporary directory to hold intermediates 15 import time # Used for sleep() and naming masks by time 16 17 import command_line 18 import drivers 19 from PIL import Image 20 from PIL import ImageChops 21 import scrapers 22 23 24 def CreateCommand(cmdline): 25 """Inserts the command and arguments into a command line for parsing.""" 26 cmd = cmdline.AddCommand( 27 ["maskmaker"], 28 "Automatically generates a mask from a list of URLs", 29 ValidateMaskmaker, 30 ExecuteMaskmaker) 31 32 cmd.AddArgument( 33 ["-bp", "--browserpath"], "Full path to browser's executable", 34 type="readfile", metaname="PATH") 35 cmd.AddArgument( 36 ["-b", "--browser"], "Which browser to use", type="string", 37 default="chrome") 38 cmd.AddArgument( 39 ["-bv", "--browserver"], "Version of the browser", metaname="VERSION") 40 cmd.AddArgument( 41 ["-o", "--outdir"], "Directory to store generated masks", metaname="DIR", 42 required=True) 43 cmd.AddArgument( 44 ["-u", "--url"], "URL to compare") 45 cmd.AddArgument( 46 ["-l", "--list"], "List of URLs to compare", type="readfile") 47 cmd.AddMutualExclusion(["--url", "--list"]) 48 cmd.AddArgument( 49 ["-s", "--startline"], "First line of URL list", type="int") 50 cmd.AddArgument( 51 ["-e", "--endline"], "Last line of URL list (exclusive)", type="int") 52 cmd.AddArgument( 53 ["-c", "--count"], "Number of lines of URL file to use", type="int") 54 cmd.AddDependency("--startline", "--list") 55 cmd.AddRequiredGroup(["--url", "--list"]) 56 cmd.AddDependency("--endline", "--list") 57 cmd.AddDependency("--count", "--list") 58 cmd.AddMutualExclusion(["--count", "--endline"]) 59 cmd.AddDependency("--count", "--startline") 60 cmd.AddArgument( 61 ["-t", "--timeout"], "Amount of time (seconds) to wait for browser to " 62 "finish loading", 63 type="int", default=60) 64 cmd.AddArgument( 65 ["-w", "--wait"], 66 "Amount of time (in seconds) to wait between successive scrapes", 67 type="int", default=60) 68 cmd.AddArgument( 69 ["-sc", "--scrapes"], 70 "Number of successive scrapes which must result in no change to a mask " 71 "before mask creation is considered complete", type="int", default=10) 72 cmd.AddArgument( 73 ["-sz", "--size"], "Browser window size", default=(800, 600), type="coords") 74 cmd.AddArgument(["-sd", "--scrapedir"], "Directory to store scrapes") 75 cmd.AddArgument( 76 ["-gu", "--giveup"], 77 "Number of times to scrape before giving up", type="int", default=50) 78 cmd.AddArgument( 79 ["-th", "--threshhold"], 80 "Percentage of different pixels (0-100) above which the scrape will be" 81 "discarded and the mask not updated.", type="int", default=100) 82 cmd.AddArgument( 83 ["--er", "--errors"], 84 "Number of times a scrape can fail before giving up on the URL.", 85 type="int", default=1) 86 87 88 def ValidateMaskmaker(command): 89 """Validate the arguments to maskmaker. Raises ParseError if failed.""" 90 executables = [".exe", ".com", ".bat"] 91 if command["--browserpath"]: 92 if os.path.splitext(command["--browserpath"])[1].lower() not in executables: 93 raise command_line.ParseError("Browser filename must be an executable") 94 95 96 def ExecuteMaskmaker(command): 97 """Performs automatic mask generation.""" 98 99 # Get the list of URLs to generate masks for 100 class MaskmakerURL(object): 101 """Helper class for holding information about a URL passed to maskmaker.""" 102 __slots__ = ['url', 'consecutive_successes', 'errors'] 103 def __init__(self, url): 104 self.url = url 105 self.consecutive_successes = 0 106 self.errors = 0 107 108 if command["--url"]: 109 url_list = [MaskmakerURL(command["--url"])] 110 else: 111 startline = command["--startline"] 112 if command["--count"]: 113 endline = startline+command["--count"] 114 else: 115 endline = command["--endline"] 116 url_list = [MaskmakerURL(url.strip()) for url in 117 open(command["--list"], "r").readlines()[startline:endline]] 118 119 complete_list = [] 120 error_list = [] 121 122 outdir = command["--outdir"] 123 scrapes = command["--scrapes"] 124 errors = command["--errors"] 125 size = command["--size"] 126 scrape_pass = 0 127 128 scrapedir = command["--scrapedir"] 129 if not scrapedir: scrapedir = tempfile.gettempdir() 130 131 # Get the scraper 132 scraper = scrapers.GetScraper((command["--browser"], command["--browserver"])) 133 134 # Repeatedly iterate through the list of URLs until either every URL has 135 # a successful mask or too many errors, or we've exceeded the giveup limit 136 while url_list and scrape_pass < command["--giveup"]: 137 # Scrape each URL 138 for url in url_list: 139 print "Processing %r..." % url.url 140 mask_filename = drivers.windowing.URLtoFilename(url.url, outdir, ".bmp") 141 142 # Load the existing mask. This is in a loop so we can try to recover 143 # from error conditions 144 while True: 145 try: 146 mask = Image.open(mask_filename) 147 if mask.size != size: 148 print " %r already exists and is the wrong size! (%r vs %r)" % ( 149 mask_filename, mask.size, size) 150 mask_filename = "%s_%r%s" % ( 151 mask_filename[:-4], size, mask_filename[-4:]) 152 print " Trying again as %r..." % mask_filename 153 continue 154 break 155 except IOError: 156 print " %r does not exist, creating" % mask_filename 157 mask = Image.new("1", size, 1) 158 mask.save(mask_filename) 159 160 # Find the stored scrape path 161 mask_scrape_dir = os.path.join( 162 scrapedir, os.path.splitext(os.path.basename(mask_filename))[0]) 163 drivers.windowing.PreparePath(mask_scrape_dir) 164 165 # Find the baseline image 166 mask_scrapes = os.listdir(mask_scrape_dir) 167 mask_scrapes.sort() 168 169 if not mask_scrapes: 170 print " No baseline image found, mask will not be updated" 171 baseline = None 172 else: 173 baseline = Image.open(os.path.join(mask_scrape_dir, mask_scrapes[0])) 174 175 mask_scrape_filename = os.path.join(mask_scrape_dir, 176 time.strftime("%y%m%d-%H%M%S.bmp")) 177 178 # Do the scrape 179 result = scraper.Scrape( 180 [url.url], mask_scrape_dir, size, (0, 0), 181 command["--timeout"], path=command["--browserpath"], 182 filename=mask_scrape_filename) 183 184 if result: 185 # Return value other than None means an error 186 print " Scrape failed with error '%r'" % result 187 url.errors += 1 188 if url.errors >= errors: 189 print " ** Exceeded maximum error count for this URL, giving up" 190 continue 191 192 # Load the new scrape 193 scrape = Image.open(mask_scrape_filename) 194 195 # Calculate the difference between the new scrape and the baseline, 196 # subject to the current mask 197 if baseline: 198 diff = ImageChops.multiply(ImageChops.difference(scrape, baseline), 199 mask.convert(scrape.mode)) 200 201 # If the difference is none, there's nothing to update 202 if max(diff.getextrema()) == (0, 0): 203 print " Scrape identical to baseline, no change in mask" 204 url.consecutive_successes += 1 205 if url.consecutive_successes >= scrapes: 206 print " ** No change for %r scrapes, done!" % scrapes 207 else: 208 # convert the difference to black and white, then change all 209 # black pixels (where the scrape and the baseline were identical) 210 # to white, all others (where the scrape and the baseline differed) 211 # to black. 212 # 213 # Since the below command is a little unclear, here's how it works. 214 # 1. convert("L") converts the RGB image to grayscale 215 # 2. point() maps grayscale values (or the individual channels) 216 # of an RGB image) to different ones. Because it operates on 217 # individual channels, the grayscale conversion from step 1 218 # is necessary. 219 # 3. The "1" second parameter to point() outputs the result as 220 # a monochrome bitmap. If the original RGB image were converted 221 # directly to monochrome, PIL would dither it. 222 diff = diff.convert("L").point([255]+[0]*255, "1") 223 224 # count the number of different pixels 225 diff_pixels = diff.getcolors()[0][0] 226 227 # is this too much? 228 diff_pixel_percent = diff_pixels * 100.0 / (mask.size[0]*mask.size[1]) 229 if diff_pixel_percent > command["--threshhold"]: 230 print (" Scrape differed from baseline by %.2f percent, ignoring" 231 % diff_pixel_percent) 232 else: 233 print " Scrape differed in %d pixels, updating mask" % diff_pixels 234 mask = ImageChops.multiply(mask, diff) 235 mask.save(mask_filename) 236 237 # reset the number of consecutive "good" scrapes 238 url.consecutive_successes = 0 239 240 # Remove URLs whose mask is deemed done 241 complete_list.extend( 242 [url for url in url_list if url.consecutive_successes >= scrapes]) 243 error_list.extend( 244 [url for url in url_list if url.errors >= errors]) 245 url_list = [ 246 url for url in url_list if 247 url.consecutive_successes < scrapes and 248 url.errors < errors] 249 250 scrape_pass += 1 251 print "**Done with scrape pass %d\n" % scrape_pass 252 253 if scrape_pass >= command["--giveup"]: 254 print "**Exceeded giveup threshhold. Giving up." 255 else: 256 print "Waiting %d seconds..." % command["--wait"] 257 time.sleep(command["--wait"]) 258 259 print 260 print "*** MASKMAKER COMPLETE ***" 261 print "Summary report:" 262 print " %d masks successfully generated" % len(complete_list) 263 for url in complete_list: 264 print " ", url.url 265 print " %d masks failed with too many errors" % len(error_list) 266 for url in error_list: 267 print " ", url.url 268 if scrape_pass >= command["--giveup"]: 269 print (" %d masks were not completed before " 270 "reaching the giveup threshhold" % len(url_list)) 271 for url in url_list: 272 print " ", url.url 273
