Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Retry on upload connection errors #625

Open
wants to merge 2 commits into
base: master
Choose a base branch
from
Open

Retry on upload connection errors #625

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
59c6d56
first pass at conn error fix in upload
Jan 25, 2024
9e3c479
second pass
Feb 2, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion internetarchive/__version__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1 +1 @@
__version__ = '3.6.0'
__version__ = '3.7.0.dev1'
124 changes: 63 additions & 61 deletions internetarchive/item.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@
from xml.parsers.expat import ExpatError

from requests import Request, Response
from requests.exceptions import HTTPError
from requests.exceptions import RequestException
from tqdm import tqdm

from internetarchive import catalog
Expand Down Expand Up @@ -1054,73 +1054,75 @@ def _build_request():
body.close()
return prepared_request
else:
try:
while True:
error_msg = ('s3 is overloaded, sleeping for '
f'{retries_sleep} seconds and retrying. '
f'{retries} retries left.')
if retries > 0:
if self.session.s3_is_overloaded(access_key=access_key):
sleep(retries_sleep)
log.info(error_msg)
if verbose:
print(f' warning: {error_msg}', file=sys.stderr)
retries -= 1
continue
request = _build_request()
prepared_request = request.prepare()

# chunked transfer-encoding is NOT supported by IA-S3.
# It should NEVER be set. Requests adds it in certain
# scenarios (e.g. if content-length is 0). Stop it.
if prepared_request.headers.get('transfer-encoding') == 'chunked':
del prepared_request.headers['transfer-encoding']

response = self.session.send(prepared_request,
stream=True,
**request_kwargs)
if (response.status_code == 503) and (retries > 0):
if b'appears to be spam' in response.content:
log.info('detected as spam, upload failed')
break
while True:
error_msg = ('s3 is overloaded, sleeping for '
f'{retries_sleep} seconds and retrying. '
f'{retries} retries left.')
if retries > 0:
if self.session.s3_is_overloaded(access_key=access_key):
sleep(retries_sleep)
log.info(error_msg)
if verbose:
print(f' warning: {error_msg}', file=sys.stderr)
sleep(retries_sleep)
retries -= 1
continue
else:
if response.status_code == 503:
log.info('maximum retries exceeded, upload failed.')
break
response.raise_for_status()
log.info(f'uploaded {key} to {url}')
if delete and response.status_code == 200:
log.info(
f'{key} successfully uploaded to '
f'https://archive.org/download/{self.identifier}/{key} and verified, '
'deleting local copy')
body.close()
os.remove(filename)
response.close()
return response
except HTTPError as exc:
request = _build_request()
prepared_request = request.prepare()

# chunked transfer-encoding is NOT supported by IA-S3.
# It should NEVER be set. Requests adds it in certain
# scenarios (e.g. if content-length is 0). Stop it.
if prepared_request.headers.get('transfer-encoding') == 'chunked':
del prepared_request.headers['transfer-encoding']

try:
msg = get_s3_xml_text(exc.response.content) # type: ignore
except ExpatError: # probably HTTP 500 error and response is invalid XML
msg = ('IA S3 returned invalid XML ' # type: ignore
f'(HTTP status code {exc.response.status_code}). '
'This is a server side error which is either temporary, '
'or requires the intervention of IA admins.')

error_msg = f' error uploading {key} to {self.identifier}, {msg}'
log.error(error_msg)
if verbose:
print(f' error uploading {key}: {msg}', file=sys.stderr)
# Raise HTTPError with error message.
raise type(exc)(error_msg, response=exc.response, request=exc.request)
finally:
response = self.session.send(prepared_request,
stream=True,
**request_kwargs)
response.raise_for_status()
except RequestException as exc:
try:
msg = get_s3_xml_text(exc.response.content) # type: ignore
except ExpatError: # probably HTTP 500 error and response is invalid XML
msg = ('IA S3 returned invalid XML ' # type: ignore
f'(HTTP status code {exc.response.status_code}). '
'This is a server side error which is either temporary, '
'or requires the intervention of IA admins.')

error_msg = f' error uploading {key} to {self.identifier}, {msg}'
log.error(error_msg)
if verbose:
print(f' error uploading {key}: {msg}', file=sys.stderr)
# Raise RequestException with error message.
if retries <= 0:
body.close()
raise type(exc)(error_msg, response=exc.response, request=exc.request)

if (response.status_code == 503) and (retries > 0):
if b'appears to be spam' in response.content:
log.info('detected as spam, upload failed')
break
log.info(error_msg)
if verbose:
print(f' warning: {error_msg}', file=sys.stderr)
sleep(retries_sleep)
retries -= 1
continue
else:
if response.status_code == 503:
log.info('maximum retries exceeded, upload failed.')
break
log.info(f'uploaded {key} to {url}')
if delete and response.status_code == 200:
log.info(
f'{key} successfully uploaded to '
f'https://archive.org/download/{self.identifier}/{key} and verified, '
'deleting local copy')
body.close()
os.remove(filename)
response.close()
body.close()
return response

def upload(self, files,
metadata: Mapping | None = None,
Expand Down
Loading