Fix corner-case in handling of remaining retry time
[ganeti-github.git] / lib / utils / retry.py
1 #
2 #
3
4 # Copyright (C) 2006, 2007, 2010, 2011 Google Inc.
5 #
6 # This program is free software; you can redistribute it and/or modify
7 # it under the terms of the GNU General Public License as published by
8 # the Free Software Foundation; either version 2 of the License, or
9 # (at your option) any later version.
10 #
11 # This program is distributed in the hope that it will be useful, but
12 # WITHOUT ANY WARRANTY; without even the implied warranty of
13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 # General Public License for more details.
15 #
16 # You should have received a copy of the GNU General Public License
17 # along with this program; if not, write to the Free Software
18 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
19 # 02110-1301, USA.
20
21 """Utility functions for retrying function calls with a timeout.
22
23 """
24
25
26 import time
27
28 from ganeti import errors
29
30
31 #: Special delay to specify whole remaining timeout
32 RETRY_REMAINING_TIME = object()
33
34
35 class RetryTimeout(Exception):
36 """Retry loop timed out.
37
38 Any arguments which was passed by the retried function to RetryAgain will be
39 preserved in RetryTimeout, if it is raised. If such argument was an exception
40 the RaiseInner helper method will reraise it.
41
42 """
43 def RaiseInner(self):
44 if self.args and isinstance(self.args[0], Exception):
45 raise self.args[0]
46 else:
47 raise RetryTimeout(*self.args)
48
49
50 class RetryAgain(Exception):
51 """Retry again.
52
53 Any arguments passed to RetryAgain will be preserved, if a timeout occurs, as
54 arguments to RetryTimeout. If an exception is passed, the RaiseInner() method
55 of the RetryTimeout() method can be used to reraise it.
56
57 """
58
59
60 class _RetryDelayCalculator(object):
61 """Calculator for increasing delays.
62
63 """
64 __slots__ = [
65 "_factor",
66 "_limit",
67 "_next",
68 "_start",
69 ]
70
71 def __init__(self, start, factor, limit):
72 """Initializes this class.
73
74 @type start: float
75 @param start: Initial delay
76 @type factor: float
77 @param factor: Factor for delay increase
78 @type limit: float or None
79 @param limit: Upper limit for delay or None for no limit
80
81 """
82 assert start > 0.0
83 assert factor >= 1.0
84 assert limit is None or limit >= 0.0
85
86 self._start = start
87 self._factor = factor
88 self._limit = limit
89
90 self._next = start
91
92 def __call__(self):
93 """Returns current delay and calculates the next one.
94
95 """
96 current = self._next
97
98 # Update for next run
99 if self._limit is None or self._next < self._limit:
100 self._next = min(self._limit, self._next * self._factor)
101
102 return current
103
104
105 def Retry(fn, delay, timeout, args=None, wait_fn=time.sleep,
106 _time_fn=time.time):
107 """Call a function repeatedly until it succeeds.
108
109 The function C{fn} is called repeatedly until it doesn't throw L{RetryAgain}
110 anymore. Between calls a delay, specified by C{delay}, is inserted. After a
111 total of C{timeout} seconds, this function throws L{RetryTimeout}.
112
113 C{delay} can be one of the following:
114 - callable returning the delay length as a float
115 - Tuple of (start, factor, limit)
116 - L{RETRY_REMAINING_TIME} to sleep until the timeout expires (this is
117 useful when overriding L{wait_fn} to wait for an external event)
118 - A static delay as a number (int or float)
119
120 @type fn: callable
121 @param fn: Function to be called
122 @param delay: Either a callable (returning the delay), a tuple of (start,
123 factor, limit) (see L{_RetryDelayCalculator}),
124 L{RETRY_REMAINING_TIME} or a number (int or float)
125 @type timeout: float
126 @param timeout: Total timeout
127 @type wait_fn: callable
128 @param wait_fn: Waiting function
129 @return: Return value of function
130
131 """
132 assert callable(fn)
133 assert callable(wait_fn)
134 assert callable(_time_fn)
135
136 if args is None:
137 args = []
138
139 end_time = _time_fn() + timeout
140
141 if callable(delay):
142 # External function to calculate delay
143 calc_delay = delay
144
145 elif isinstance(delay, (tuple, list)):
146 # Increasing delay with optional upper boundary
147 (start, factor, limit) = delay
148 calc_delay = _RetryDelayCalculator(start, factor, limit)
149
150 elif delay is RETRY_REMAINING_TIME:
151 # Always use the remaining time
152 calc_delay = None
153
154 else:
155 # Static delay
156 calc_delay = lambda: delay
157
158 assert calc_delay is None or callable(calc_delay)
159
160 while True:
161 retry_args = []
162 try:
163 # pylint: disable=W0142
164 return fn(*args)
165 except RetryAgain, err:
166 retry_args = err.args
167 except RetryTimeout:
168 raise errors.ProgrammerError("Nested retry loop detected that didn't"
169 " handle RetryTimeout")
170
171 remaining_time = end_time - _time_fn()
172
173 if remaining_time <= 0.0:
174 # pylint: disable=W0142
175 raise RetryTimeout(*retry_args)
176
177 assert remaining_time > 0.0
178
179 if calc_delay is None:
180 wait_fn(remaining_time)
181 else:
182 current_delay = calc_delay()
183 if current_delay > 0.0:
184 wait_fn(current_delay)
185
186
187 def SimpleRetry(expected, fn, delay, timeout, args=None, wait_fn=time.sleep,
188 _time_fn=time.time):
189 """A wrapper over L{Retry} implementing a simpler interface.
190
191 All the parameters are the same as for L{Retry}, except it has one
192 extra argument: expected, which can be either a value (will be
193 compared with the result of the function, or a callable (which will
194 get the result passed and has to return a boolean). If the test is
195 false, we will retry until either the timeout has passed or the
196 tests succeeds. In both cases, the last result from calling the
197 function will be returned.
198
199 Note that this function is not expected to raise any retry-related
200 exceptions, always simply returning values. As such, the function is
201 designed to allow easy wrapping of code that doesn't use retry at
202 all (e.g. "if fn(args)" replaced with "if SimpleRetry(True, fn,
203 ...)".
204
205 @see: L{Retry}
206
207 """
208 rdict = {}
209
210 def helper(*innerargs):
211 # pylint: disable=W0142
212 result = rdict["result"] = fn(*innerargs)
213 if not ((callable(expected) and expected(result)) or result == expected):
214 raise RetryAgain()
215 return result
216
217 try:
218 result = Retry(helper, delay, timeout, args=args,
219 wait_fn=wait_fn, _time_fn=_time_fn)
220 except RetryTimeout:
221 assert "result" in rdict
222 result = rdict["result"]
223 return result