如何在Python Requests库中实现重试机制？
我想给Python的Requests库添加一个重试机制，这样使用这个库的脚本在遇到一些非致命错误时就能自动重试。
目前我认为有三种错误是可以恢复的：
- HTTP返回码502、503和504
- 找不到主机（现在不太重要）
- 请求超时
在第一阶段，我希望每分钟重试指定的5xx错误请求。
我希望能够透明地添加这个功能，也就是说，不需要手动为每一个HTTP调用实现恢复机制，这些调用都是在使用Python Requests的脚本或库中进行的。
6 个回答
10
在编程中，有时候我们会遇到一些问题，尤其是在使用某些工具或者库的时候。这些问题可能会让我们感到困惑，不知道该怎么解决。比如，有人可能在使用某个特定的功能时，发现它并没有按照预期工作。这种情况很常见，尤其是对于刚开始学习编程的人来说。
通常，解决这些问题的方法是仔细检查代码，看看是否有拼写错误、逻辑错误或者是使用方法不当的地方。还有一种常见的做法是查阅相关的文档或者在网上搜索一下，看看有没有其他人遇到过类似的问题，以及他们是怎么解决的。
总之，遇到问题时不要慌张，保持冷静，逐步排查，通常都能找到解决办法。
from requests.adapters import HTTPAdapter
from urllib3.util.retry import Retry
MAX_RETRY = 2
MAX_RETRY_FOR_SESSION = 2
BACK_OFF_FACTOR = 0.3
TIME_BETWEEN_RETRIES = 1000
ERROR_CODES = (500, 502, 504)
def requests_retry_session(retries=MAX_RETRY_FOR_SESSION,
back_off_factor=BACK_OFF_FACTOR,
status_force_list=ERROR_CODES,
session=None):
session = session
retry = Retry(total=retries, read=retries, connect=retries,
backoff_factor=back_off_factor,
status_forcelist=status_force_list,
method_whitelist=frozenset(['GET', 'POST']))
adapter = HTTPAdapter(max_retries=retry)
session.mount('http://', adapter)
session.mount('https://', adapter)
return session
class ConfigService:
def __init__(self):
self.session = requests_retry_session(session=requests.Session())
def call_to_api():
config_url = 'http://localhost:8080/predict/'
headers = {
"Content-Type": "application/json",
"x-api-key": self.x_api_key
}
response = self.session.get(config_url, headers=headers)
return response
13
可以尝试使用 retrying 这个包 来解决问题。
from retrying import retry
import requests
def retry_if_connection_error(exception):
""" Specify an exception you need. or just True"""
#return True
return isinstance(exception, ConnectionError)
# if exception retry with 2 second wait
@retry(retry_on_exception=retry_if_connection_error, wait_fixed=2000)
def safe_request(url, **kwargs):
return requests.get(url, **kwargs)
response = safe_request('test.com')
30
这是我用来重试通过urllib2发出的请求的一段代码。也许你可以用它来满足你的需求：
retries = 1
success = False
while not success:
try:
response = urllib2.urlopen(request)
success = True
except Exception as e:
wait = retries * 30;
print 'Error! Waiting %s secs and re-trying...' % wait
sys.stdout.flush()
time.sleep(wait)
retries += 1
等待的时间会逐渐增加，以避免被服务器封禁。
261
这段代码的作用是让同一个会话中的所有HTTP请求最多重试5次。在每次重试之间，会有一个逐渐增加的等待时间，具体是：第一次重试立刻进行，第二次等2秒，第三次等4秒，第四次等8秒，第五次等16秒。它会在一些基本的连接问题（比如DNS查找失败）和HTTP状态码为502、503和504的情况下进行重试。
import logging
import requests
from requests.adapters import HTTPAdapter, Retry
logging.basicConfig(level=logging.DEBUG)
s = requests.Session()
retries = Retry(total=5, backoff_factor=1, status_forcelist=[ 502, 503, 504 ])
s.mount('http://', HTTPAdapter(max_retries=retries))
s.get("http://httpstat.us/503")
想了解更多细节，可以查看Retry类。
-1
我通过扩展 requests.Session 类，成功达到了我想要的可靠性。
编辑 这段代码是：
from requests import Session
from requests.exceptions import ConnectionError
import logging
import time
class ResilientSession(Session):
"""
This class is supposed to retry requests that do return temporary errors.
At this moment it supports: 502, 503, 504
"""
def __recoverable(self, error, url, request, counter=1):
if hasattr(error,'status_code'):
if error.status_code in [502, 503, 504]:
error = "HTTP %s" % error.status_code
else:
return False
DELAY = 10 * counter
logging.warn("Got recoverable error [%s] from %s %s, retry #%s in %ss" % (error, request, url, counter, DELAY))
time.sleep(DELAY)
return True
def get(self, url, **kwargs):
counter = 0
while True:
counter += 1
try:
r = super(ResilientSession, self).get(url, **kwargs)
except ConnectionError as e:
r = e.message
if self.__recoverable(r, url, 'GET', counter):
continue
return r
def post(self, url, **kwargs):
counter = 0
while True:
counter += 1
try:
r = super(ResilientSession, self).post(url, **kwargs)
except ConnectionError as e:
r = e.message
if self.__recoverable(r, url, 'POST', counter):
continue
return r
def delete(self, url, **kwargs):
counter = 0
while True:
counter += 1
try:
r = super(ResilientSession, self).delete(url, **kwargs)
except ConnectionError as e:
r = e.message
if self.__recoverable(r, url, 'DELETE', counter):
continue
return r
def put(self, url, **kwargs):
counter = 0
while True:
counter += 1
try:
r = super(ResilientSession, self).put(url, **kwargs)
except ConnectionError as e:
r = e.message
if self.__recoverable(r, url, 'PUT', counter):
continue
return r
def head(self, url, **kwargs):
counter = 0
while True:
counter += 1
try:
r = super(ResilientSession, self).head(url, **kwargs)
except ConnectionError as e:
r = e.message
if self.__recoverable(r, url, 'HEAD', counter):
continue
return r
def patch(self, url, **kwargs):
counter = 0
while True:
counter += 1
try:
r = super(ResilientSession, self).patch(url, **kwargs)
except ConnectionError as e:
r = e.message
if self.__recoverable(r, url, 'PATCH', counter):
continue
return r
def options(self, url, **kwargs):
counter = 0
while True:
counter += 1
try:
r = super(ResilientSession, self).options(url, **kwargs)
except ConnectionError as e:
r = e.message
if self.__recoverable(r, url, 'OPTIONS', counter):
continue
return r