1 #!/usr/bin/env python 2 # Copyright 2012, 2014 Kodi Arfer 3 # 4 # Permission is hereby granted, free of charge, to any person obtaining a 5 # copy of this software and associated documentation files (the 6 # "Software"), to deal in the Software without restriction, including 7 # without limitation the rights to use, copy, modify, merge, publish, dis- 8 # tribute, sublicense, and/or sell copies of the Software, and to permit 9 # persons to whom the Software is furnished to do so, subject to the fol- 10 # lowing conditions: 11 # 12 # The above copyright notice and this permission notice shall be included 13 # in all copies or substantial portions of the Software. 14 # 15 # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS 16 # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- 17 # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT 18 # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, 19 # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 21 22 import argparse # Hence, Python 2.7 is required. 23 import sys 24 import os.path 25 import string 26 import inspect 27 import datetime, calendar 28 import boto.mturk.connection, boto.mturk.price, boto.mturk.question, boto.mturk.qualification 29 from boto.compat import json 30 31 # -------------------------------------------------- 32 # Globals 33 # ------------------------------------------------- 34 35 interactive = False 36 con = None 37 mturk_website = None 38 39 default_nicknames_path = os.path.expanduser('~/.boto_mturkcli_hit_nicknames') 40 nicknames = {} 41 nickname_pool = set(string.ascii_lowercase) 42 43 get_assignments_page_size = 100 44 45 time_units = dict( 46 s = 1, 47 min = 60, 48 h = 60 * 60, 49 d = 24 * 60 * 60) 50 51 qual_requirements = dict( 52 Adult = '00000000000000000060', 53 Locale = '00000000000000000071', 54 NumberHITsApproved = '00000000000000000040', 55 PercentAssignmentsSubmitted = '00000000000000000000', 56 PercentAssignmentsAbandoned = '00000000000000000070', 57 PercentAssignmentsReturned = '000000000000000000E0', 58 PercentAssignmentsApproved = '000000000000000000L0', 59 PercentAssignmentsRejected = '000000000000000000S0') 60 61 qual_comparators = {v : k for k, v in dict( 62 LessThan = '<', LessThanOrEqualTo = '<=', 63 GreaterThan = '>', GreaterThanOrEqualTo = '>=', 64 EqualTo = '==', NotEqualTo = '!=', 65 Exists = 'exists').items()} 66 67 example_config_file = '''Example configuration file: 68 69 { 70 "title": "Pick your favorite color", 71 "description": "In this task, you are asked to pick your favorite color.", 72 "reward": 0.50, 73 "assignments": 10, 74 "duration": "20 min", 75 "keywords": ["color", "favorites", "survey"], 76 "lifetime": "7 d", 77 "approval_delay": "14 d", 78 "qualifications": [ 79 "PercentAssignmentsApproved > 90", 80 "Locale == US", 81 "2ARFPLSP75KLA8M8DH1HTEQVJT3SY6 exists" 82 ], 83 "question_url": "http://example.com/myhit", 84 "question_frame_height": 450 85 }''' 86 87 # -------------------------------------------------- 88 # Subroutines 89 # -------------------------------------------------- 90 91 def unjson(path): 92 with open(path) as o: 93 return json.load(o) 94 95 def add_argparse_arguments(parser): 96 parser.add_argument('-P', '--production', 97 dest = 'sandbox', action = 'store_false', default = True, 98 help = 'use the production site (default: use the sandbox)') 99 parser.add_argument('--nicknames', 100 dest = 'nicknames_path', metavar = 'PATH', 101 default = default_nicknames_path, 102 help = 'where to store HIT nicknames (default: {})'.format( 103 default_nicknames_path)) 104 105 def init_by_args(args): 106 init(args.sandbox, args.nicknames_path) 107 108 def init(sandbox = False, nicknames_path = default_nicknames_path): 109 global con, mturk_website, nicknames, original_nicknames 110 111 mturk_website = 'workersandbox.mturk.com' if sandbox else 'www.mturk.com' 112 con = boto.mturk.connection.MTurkConnection( 113 host = 'mechanicalturk.sandbox.amazonaws.com' if sandbox else 'mechanicalturk.amazonaws.com') 114 115 try: 116 nicknames = unjson(nicknames_path) 117 except IOError: 118 nicknames = {} 119 original_nicknames = nicknames.copy() 120 121 def save_nicknames(nicknames_path = default_nicknames_path): 122 if nicknames != original_nicknames: 123 with open(nicknames_path, 'w') as o: 124 json.dump(nicknames, o, sort_keys = True, indent = 4) 125 print >>o 126 127 def parse_duration(s): 128 '''Parses durations like "2 d", "48 h", "2880 min", 129 "172800 s", or "172800".''' 130 x = s.split() 131 return int(x[0]) * time_units['s' if len(x) == 1 else x[1]] 132 def display_duration(n): 133 for unit, m in sorted(time_units.items(), key = lambda x: -x[1]): 134 if n % m == 0: 135 return '{} {}'.format(n / m, unit) 136 137 def parse_qualification(inp): 138 '''Parses qualifications like "PercentAssignmentsApproved > 90", 139 "Locale == US", and "2ARFPLSP75KLA8M8DH1HTEQVJT3SY6 exists".''' 140 inp = inp.split() 141 name, comparator, value = inp.pop(0), inp.pop(0), (inp[0] if len(inp) else None) 142 qtid = qual_requirements.get(name) 143 if qtid is None: 144 # Treat "name" as a Qualification Type ID. 145 qtid = name 146 if qtid == qual_requirements['Locale']: 147 return boto.mturk.qualification.LocaleRequirement( 148 qual_comparators[comparator], 149 value, 150 required_to_preview = False) 151 return boto.mturk.qualification.Requirement( 152 qtid, 153 qual_comparators[comparator], 154 value, 155 required_to_preview = qtid == qual_requirements['Adult']) 156 # Thus required_to_preview is true only for the 157 # Worker_Adult requirement. 158 159 def preview_url(hit): 160 return 'https://{}/mturk/preview?groupId={}'.format( 161 mturk_website, hit.HITTypeId) 162 163 def parse_timestamp(s): 164 '''Takes a timestamp like "2012-11-24T16:34:41Z". 165 166 Returns a datetime object in the local time zone.''' 167 return datetime.datetime.fromtimestamp( 168 calendar.timegm( 169 datetime.datetime.strptime(s, '%Y-%m-%dT%H:%M:%SZ').timetuple())) 170 171 def get_hitid(nickname_or_hitid): 172 return nicknames.get(nickname_or_hitid) or nickname_or_hitid 173 174 def get_nickname(hitid): 175 for k, v in nicknames.items(): 176 if v == hitid: 177 return k 178 return None 179 180 def display_datetime(dt): 181 return dt.strftime('%e %b %Y, %l:%M %P') 182 183 def display_hit(hit, verbose = False): 184 et = parse_timestamp(hit.Expiration) 185 return '\n'.join([ 186 '{} - {} ({}, {}, {})'.format( 187 get_nickname(hit.HITId), 188 hit.Title, 189 hit.FormattedPrice, 190 display_duration(int(hit.AssignmentDurationInSeconds)), 191 hit.HITStatus), 192 'HIT ID: ' + hit.HITId, 193 'Type ID: ' + hit.HITTypeId, 194 'Group ID: ' + hit.HITGroupId, 195 'Preview: ' + preview_url(hit), 196 'Created {} {}'.format( 197 display_datetime(parse_timestamp(hit.CreationTime)), 198 'Expired' if et <= datetime.datetime.now() else 199 'Expires ' + display_datetime(et)), 200 'Assignments: {} -- {} avail, {} pending, {} reviewable, {} reviewed'.format( 201 hit.MaxAssignments, 202 hit.NumberOfAssignmentsAvailable, 203 hit.NumberOfAssignmentsPending, 204 int(hit.MaxAssignments) - (int(hit.NumberOfAssignmentsAvailable) + int(hit.NumberOfAssignmentsPending) + int(hit.NumberOfAssignmentsCompleted)), 205 hit.NumberOfAssignmentsCompleted) 206 if hasattr(hit, 'NumberOfAssignmentsAvailable') 207 else 'Assignments: {} total'.format(hit.MaxAssignments), 208 # For some reason, SearchHITs includes the 209 # NumberOfAssignmentsFoobar fields but GetHIT doesn't. 210 ] + ([] if not verbose else [ 211 '\nDescription: ' + hit.Description, 212 '\nKeywords: ' + hit.Keywords 213 ])) + '\n' 214 215 def digest_assignment(a): 216 return dict( 217 answers = {str(x.qid): str(x.fields[0]) for x in a.answers[0]}, 218 **{k: str(getattr(a, k)) for k in ( 219 'AcceptTime', 'SubmitTime', 220 'HITId', 'AssignmentId', 'WorkerId', 221 'AssignmentStatus')}) 222 223 # -------------------------------------------------- 224 # Commands 225 # -------------------------------------------------- 226 227 def get_balance(): 228 return con.get_account_balance() 229 230 def show_hit(hit): 231 return display_hit(con.get_hit(hit)[0], verbose = True) 232 233 def list_hits(): 234 'Lists your 10 most recently created HITs, with the most recent last.' 235 return '\n'.join(reversed(map(display_hit, con.search_hits( 236 sort_by = 'CreationTime', 237 sort_direction = 'Descending', 238 page_size = 10)))) 239 240 def make_hit(title, description, keywords, reward, question_url, question_frame_height, duration, assignments, approval_delay, lifetime, qualifications = []): 241 r = con.create_hit( 242 title = title, 243 description = description, 244 keywords = con.get_keywords_as_string(keywords), 245 reward = con.get_price_as_price(reward), 246 question = boto.mturk.question.ExternalQuestion( 247 question_url, 248 question_frame_height), 249 duration = parse_duration(duration), 250 qualifications = boto.mturk.qualification.Qualifications( 251 map(parse_qualification, qualifications)), 252 max_assignments = assignments, 253 approval_delay = parse_duration(approval_delay), 254 lifetime = parse_duration(lifetime)) 255 nick = None 256 available_nicks = nickname_pool - set(nicknames.keys()) 257 if available_nicks: 258 nick = min(available_nicks) 259 nicknames[nick] = r[0].HITId 260 if interactive: 261 print 'Nickname:', nick 262 print 'HIT ID:', r[0].HITId 263 print 'Preview:', preview_url(r[0]) 264 else: 265 return r[0] 266 267 def extend_hit(hit, assignments_increment = None, expiration_increment = None): 268 con.extend_hit(hit, assignments_increment, expiration_increment) 269 270 def expire_hit(hit): 271 con.expire_hit(hit) 272 273 def delete_hit(hit): 274 '''Deletes a HIT using DisableHIT. 275 276 Unreviewed assignments get automatically approved. Unsubmitted 277 assignments get automatically approved upon submission. 278 279 The API docs say DisableHIT doesn't work with Reviewable HITs, 280 but apparently, it does.''' 281 con.disable_hit(hit) 282 global nicknames 283 nicknames = {k: v for k, v in nicknames.items() if v != hit} 284 285 def list_assignments(hit, only_reviewable = False): 286 # Accumulate all relevant assignments, one page of results at 287 # a time. 288 assignments = [] 289 page = 1 290 while True: 291 rs = con.get_assignments( 292 hit_id = hit, 293 page_size = get_assignments_page_size, 294 page_number = page, 295 status = 'Submitted' if only_reviewable else None) 296 assignments += map(digest_assignment, rs) 297 if len(assignments) >= int(rs.TotalNumResults): 298 break 299 page += 1 300 if interactive: 301 print json.dumps(assignments, sort_keys = True, indent = 4) 302 print ' '.join([a['AssignmentId'] for a in assignments]) 303 print ' '.join([a['WorkerId'] + ',' + a['AssignmentId'] for a in assignments]) 304 else: 305 return assignments 306 307 def grant_bonus(message, amount, pairs): 308 for worker, assignment in pairs: 309 con.grant_bonus(worker, assignment, con.get_price_as_price(amount), message) 310 if interactive: print 'Bonused', worker 311 312 def approve_assignments(message, assignments): 313 for a in assignments: 314 con.approve_assignment(a, message) 315 if interactive: print 'Approved', a 316 317 def reject_assignments(message, assignments): 318 for a in assignments: 319 con.reject_assignment(a, message) 320 if interactive: print 'Rejected', a 321 322 def unreject_assignments(message, assignments): 323 for a in assignments: 324 con.approve_rejected_assignment(a, message) 325 if interactive: print 'Unrejected', a 326 327 def notify_workers(subject, text, workers): 328 con.notify_workers(workers, subject, text) 329 330 def give_qualification(qualification, workers, value = 1, notify = True): 331 for w in workers: 332 con.assign_qualification(qualification, w, value, notify) 333 if interactive: print 'Gave to', w 334 335 def revoke_qualification(qualification, workers, message = None): 336 for w in workers: 337 con.revoke_qualification(w, qualification, message) 338 if interactive: print 'Revoked from', w 339 340 # -------------------------------------------------- 341 # Mainline code 342 # -------------------------------------------------- 343 344 if __name__ == '__main__': 345 interactive = True 346 347 parser = argparse.ArgumentParser() 348 add_argparse_arguments(parser) 349 subs = parser.add_subparsers() 350 351 sub = subs.add_parser('bal', 352 help = 'display your prepaid balance') 353 sub.set_defaults(f = get_balance, a = lambda: []) 354 355 sub = subs.add_parser('hit', 356 help = 'get information about a HIT') 357 sub.add_argument('HIT', 358 help = 'nickname or ID of the HIT to show') 359 sub.set_defaults(f = show_hit, a = lambda: 360 [get_hitid(args.HIT)]) 361 362 sub = subs.add_parser('hits', 363 help = 'list all your HITs') 364 sub.set_defaults(f = list_hits, a = lambda: []) 365 366 sub = subs.add_parser('new', 367 help = 'create a new HIT (external questions only)', 368 epilog = example_config_file, 369 formatter_class = argparse.RawDescriptionHelpFormatter) 370 sub.add_argument('JSON_PATH', 371 help = 'path to JSON configuration file for the HIT') 372 sub.add_argument('-u', '--question-url', dest = 'question_url', 373 metavar = 'URL', 374 help = 'URL for the external question') 375 sub.add_argument('-a', '--assignments', dest = 'assignments', 376 type = int, metavar = 'N', 377 help = 'number of assignments') 378 sub.add_argument('-r', '--reward', dest = 'reward', 379 type = float, metavar = 'PRICE', 380 help = 'reward amount, in USD') 381 sub.set_defaults(f = make_hit, a = lambda: dict( 382 unjson(args.JSON_PATH).items() + [(k, getattr(args, k)) 383 for k in ('question_url', 'assignments', 'reward') 384 if getattr(args, k) is not None])) 385 386 sub = subs.add_parser('extend', 387 help = 'add assignments or time to a HIT') 388 sub.add_argument('HIT', 389 help = 'nickname or ID of the HIT to extend') 390 sub.add_argument('-a', '--assignments', dest = 'assignments', 391 metavar = 'N', type = int, 392 help = 'number of assignments to add') 393 sub.add_argument('-t', '--time', dest = 'time', 394 metavar = 'T', 395 help = 'amount of time to add to the expiration date') 396 sub.set_defaults(f = extend_hit, a = lambda: 397 [get_hitid(args.HIT), args.assignments, 398 args.time and parse_duration(args.time)]) 399 400 sub = subs.add_parser('expire', 401 help = 'force a HIT to expire without deleting it') 402 sub.add_argument('HIT', 403 help = 'nickname or ID of the HIT to expire') 404 sub.set_defaults(f = expire_hit, a = lambda: 405 [get_hitid(args.HIT)]) 406 407 sub = subs.add_parser('rm', 408 help = 'delete a HIT') 409 sub.add_argument('HIT', 410 help = 'nickname or ID of the HIT to delete') 411 sub.set_defaults(f = delete_hit, a = lambda: 412 [get_hitid(args.HIT)]) 413 414 sub = subs.add_parser('as', 415 help = "list a HIT's submitted assignments") 416 sub.add_argument('HIT', 417 help = 'nickname or ID of the HIT to get assignments for') 418 sub.add_argument('-r', '--reviewable', dest = 'only_reviewable', 419 action = 'store_true', 420 help = 'show only unreviewed assignments') 421 sub.set_defaults(f = list_assignments, a = lambda: 422 [get_hitid(args.HIT), args.only_reviewable]) 423 424 for command, fun, helpmsg in [ 425 ('approve', approve_assignments, 'approve assignments'), 426 ('reject', reject_assignments, 'reject assignments'), 427 ('unreject', unreject_assignments, 'approve previously rejected assignments')]: 428 sub = subs.add_parser(command, help = helpmsg) 429 sub.add_argument('ASSIGNMENT', nargs = '+', 430 help = 'ID of an assignment') 431 sub.add_argument('-m', '--message', dest = 'message', 432 metavar = 'TEXT', 433 help = 'feedback message shown to workers') 434 sub.set_defaults(f = fun, a = lambda: 435 [args.message, args.ASSIGNMENT]) 436 437 sub = subs.add_parser('bonus', 438 help = 'give some workers a bonus') 439 sub.add_argument('AMOUNT', type = float, 440 help = 'bonus amount, in USD') 441 sub.add_argument('MESSAGE', 442 help = 'the reason for the bonus (shown to workers in an email sent by MTurk)') 443 sub.add_argument('WIDAID', nargs = '+', 444 help = 'a WORKER_ID,ASSIGNMENT_ID pair') 445 sub.set_defaults(f = grant_bonus, a = lambda: 446 [args.MESSAGE, args.AMOUNT, 447 [p.split(',') for p in args.WIDAID]]) 448 449 sub = subs.add_parser('notify', 450 help = 'send a message to some workers') 451 sub.add_argument('SUBJECT', 452 help = 'subject of the message') 453 sub.add_argument('MESSAGE', 454 help = 'text of the message') 455 sub.add_argument('WORKER', nargs = '+', 456 help = 'ID of a worker') 457 sub.set_defaults(f = notify_workers, a = lambda: 458 [args.SUBJECT, args.MESSAGE, args.WORKER]) 459 460 sub = subs.add_parser('give-qual', 461 help = 'give a qualification to some workers') 462 sub.add_argument('QUAL', 463 help = 'ID of the qualification') 464 sub.add_argument('WORKER', nargs = '+', 465 help = 'ID of a worker') 466 sub.add_argument('-v', '--value', dest = 'value', 467 metavar = 'N', type = int, default = 1, 468 help = 'value of the qualification') 469 sub.add_argument('--dontnotify', dest = 'notify', 470 action = 'store_false', default = True, 471 help = "don't notify workers") 472 sub.set_defaults(f = give_qualification, a = lambda: 473 [args.QUAL, args.WORKER, args.value, args.notify]) 474 475 sub = subs.add_parser('revoke-qual', 476 help = 'revoke a qualification from some workers') 477 sub.add_argument('QUAL', 478 help = 'ID of the qualification') 479 sub.add_argument('WORKER', nargs = '+', 480 help = 'ID of a worker') 481 sub.add_argument('-m', '--message', dest = 'message', 482 metavar = 'TEXT', 483 help = 'the reason the qualification was revoked (shown to workers in an email sent by MTurk)') 484 sub.set_defaults(f = revoke_qualification, a = lambda: 485 [args.QUAL, args.WORKER, args.message]) 486 487 args = parser.parse_args() 488 489 init_by_args(args) 490 491 f = args.f 492 a = args.a() 493 if isinstance(a, dict): 494 # We do some introspective gymnastics so we can produce a 495 # less incomprehensible error message if some arguments 496 # are missing. 497 spec = inspect.getargspec(f) 498 missing = set(spec.args[: len(spec.args) - len(spec.defaults)]) - set(a.keys()) 499 if missing: 500 raise ValueError('Missing arguments: ' + ', '.join(missing)) 501 doit = lambda: f(**a) 502 else: 503 doit = lambda: f(*a) 504 505 try: 506 x = doit() 507 except boto.mturk.connection.MTurkRequestError as e: 508 print 'MTurk error:', e.error_message 509 sys.exit(1) 510 511 if x is not None: 512 print x 513 514 save_nicknames() 515
