1 # Copyright (c) 2012 The Chromium OS Authors. All rights reserved. 2 # Use of this source code is governed by a BSD-style license that can be 3 # found in the LICENSE file. 4 5 import logging 6 import random 7 import signal 8 import sys 9 import threading 10 import time 11 12 from autotest_lib.client.common_lib import error 13 14 15 def handler(signum, frame): 16 """ 17 Register a handler for the timeout. 18 """ 19 raise error.TimeoutException('Call is timed out.') 20 21 22 def install_sigalarm_handler(new_handler): 23 """ 24 Try installing a sigalarm handler. 25 26 In order to protect apache, wsgi intercepts any attempt to install a 27 sigalarm handler, so our function will feel the full force of a sigalarm 28 even if we try to install a pacifying signal handler. To avoid this we 29 need to confirm that the handler we tried to install really was installed. 30 31 @param new_handler: The new handler to install. This must be a callable 32 object, or signal.SIG_IGN/SIG_DFL which correspond to 33 the numbers 1,0 respectively. 34 @return: True if the installation of new_handler succeeded, False otherwise. 35 """ 36 if (new_handler is None or 37 (not callable(new_handler) and 38 new_handler != signal.SIG_IGN and 39 new_handler != signal.SIG_DFL)): 40 logging.warning('Trying to install an invalid sigalarm handler.') 41 return False 42 43 signal.signal(signal.SIGALRM, new_handler) 44 installed_handler = signal.getsignal(signal.SIGALRM) 45 return installed_handler == new_handler 46 47 48 def set_sigalarm_timeout(timeout_secs, default_timeout=60): 49 """ 50 Set the sigalarm timeout. 51 52 This methods treats any timeout <= 0 as a possible error and falls back to 53 using it's default timeout, since negative timeouts can have 'alarming' 54 effects. Though 0 is a valid timeout, it is often used to cancel signals; in 55 order to set a sigalarm of 0 please call signal.alarm directly as there are 56 many situations where a 0 timeout is considered invalid. 57 58 @param timeout_secs: The new timeout, in seconds. 59 @param default_timeout: The default timeout to use, if timeout <= 0. 60 @return: The old sigalarm timeout 61 """ 62 timeout_sec_n = int(timeout_secs) 63 if timeout_sec_n <= 0: 64 timeout_sec_n = int(default_timeout) 65 return signal.alarm(timeout_sec_n) 66 67 68 def timeout(func, args=(), kwargs={}, timeout_sec=60.0, default_result=None): 69 """ 70 This function run the given function using the args, kwargs and 71 return the given default value if the timeout_sec is exceeded. 72 73 @param func: function to be called. 74 @param args: arguments for function to be called. 75 @param kwargs: keyword arguments for function to be called. 76 @param timeout_sec: timeout setting for call to exit, in seconds. 77 @param default_result: default return value for the function call. 78 79 @return 1: is_timeout 2: result of the function call. If 80 is_timeout is True, the call is timed out. If the 81 value is False, the call is finished on time. 82 """ 83 old_alarm_sec = 0 84 old_handler = signal.getsignal(signal.SIGALRM) 85 installed_handler = install_sigalarm_handler(handler) 86 if installed_handler: 87 old_alarm_sec = set_sigalarm_timeout(timeout_sec, default_timeout=60) 88 89 # If old_timeout_time = 0 we either didn't install a handler, or sigalrm 90 # had a signal.SIG_DFL handler with 0 timeout. In the latter case we still 91 # need to restore the handler/timeout. 92 old_timeout_time = (time.time() + old_alarm_sec) if old_alarm_sec > 0 else 0 93 94 try: 95 default_result = func(*args, **kwargs) 96 return False, default_result 97 except error.TimeoutException: 98 return True, default_result 99 finally: 100 # If we installed a sigalarm handler, cancel it since our function 101 # returned on time. If we can successfully restore the old handler, 102 # reset the old timeout, or, if the old timeout's deadline has passed, 103 # set the sigalarm to fire in one second. If the old_timeout_time is 0 104 # we don't need to set the sigalarm timeout since we have already set it 105 # as a byproduct of cancelling the current signal. 106 if installed_handler: 107 signal.alarm(0) 108 if install_sigalarm_handler(old_handler) and old_timeout_time: 109 set_sigalarm_timeout(int(old_timeout_time - time.time()), 110 default_timeout=1) 111 112 113 114 def retry(ExceptionToCheck, timeout_min=1.0, delay_sec=3, blacklist=None): 115 """Retry calling the decorated function using a delay with jitter. 116 117 Will raise RPC ValidationError exceptions from the decorated 118 function without retrying; a malformed RPC isn't going to 119 magically become good. Will raise exceptions in blacklist as well. 120 121 If the retry is done in a child thread, timeout may not be enforced as 122 signal only works in main thread. Therefore, the retry inside a child 123 thread may run longer than timeout or even hang. 124 125 original from: 126 http://www.saltycrane.com/blog/2009/11/trying-out-retry-decorator-python/ 127 128 @param ExceptionToCheck: the exception to check. May be a tuple of 129 exceptions to check. 130 @param timeout_min: timeout in minutes until giving up. 131 @param delay_sec: pre-jittered delay between retries in seconds. Actual 132 delays will be centered around this value, ranging up to 133 50% off this midpoint. 134 @param blacklist: a list of exceptions that will be raised without retrying 135 """ 136 def deco_retry(func): 137 random.seed() 138 139 140 def delay(): 141 """ 142 'Jitter' the delay, up to 50% in either direction. 143 """ 144 random_delay = random.uniform(.5 * delay_sec, 1.5 * delay_sec) 145 logging.warning('Retrying in %f seconds...', random_delay) 146 time.sleep(random_delay) 147 148 149 def func_retry(*args, **kwargs): 150 # Used to cache exception to be raised later. 151 exc_info = None 152 delayed_enabled = False 153 exception_tuple = () if blacklist is None else tuple(blacklist) 154 start_time = time.time() 155 remaining_time = timeout_min * 60 156 is_main_thread = isinstance(threading.current_thread(), 157 threading._MainThread) 158 while remaining_time > 0: 159 if delayed_enabled: 160 delay() 161 else: 162 delayed_enabled = True 163 try: 164 # Clear the cache 165 exc_info = None 166 if is_main_thread: 167 is_timeout, result = timeout(func, args, kwargs, 168 remaining_time) 169 if not is_timeout: 170 return result 171 else: 172 return func(*args, **kwargs) 173 except exception_tuple: 174 raise 175 except error.CrosDynamicSuiteException: 176 raise 177 except ExceptionToCheck as e: 178 logging.warning('%s(%s)', e.__class__, e) 179 # Cache the exception to be raised later. 180 exc_info = sys.exc_info() 181 182 remaining_time = int(timeout_min*60 - 183 (time.time() - start_time)) 184 185 # The call must have timed out or raised ExceptionToCheck. 186 if not exc_info: 187 raise error.TimeoutException('Call is timed out.') 188 # Raise the cached exception with original backtrace. 189 raise exc_info[0], exc_info[1], exc_info[2] 190 191 192 return func_retry # true decorator 193 return deco_retry 194
