1600359 - update_bugscache sometimes fails

Status: NEW

(Tree Management :: Treeherder: Infrastructure, defect, P3)

Description

[Armen [:armenzg]]

Attachment: [screenshot] 3 hours span for update_bugscache on stage

This doesn't happen often but I wanted to track it.

Traceback (most recent call last):
File "/app/.heroku/python/lib/python3.7/site-packages/urllib3/connectionpool.py", line 387, in _make_request
six.raise_from(e, None)
File "<string>", line 2, in raise_from
File "/app/.heroku/python/lib/python3.7/site-packages/urllib3/connectionpool.py", line 383, in _make_request
httplib_response = conn.getresponse()
File "/app/.heroku/python/lib/python3.7/site-packages/newrelic/hooks/external_httplib.py", line 70, in httplib_getresponse_wrapper
response = wrapped(*args, **kwargs)
File "/app/.heroku/python/lib/python3.7/http/client.py", line 1321, in getresponse
response.begin()
File "/app/.heroku/python/lib/python3.7/http/client.py", line 296, in begin
version, status, reason = self._read_status()
File "/app/.heroku/python/lib/python3.7/http/client.py", line 257, in _read_status
line = str(self.fp.readline(_MAXLINE + 1), "iso-8859-1")
File "/app/.heroku/python/lib/python3.7/socket.py", line 589, in readinto
return self._sock.recv_into(b)
File "/app/.heroku/python/lib/python3.7/ssl.py", line 1052, in recv_into
return self.read(nbytes, buffer)
File "/app/.heroku/python/lib/python3.7/ssl.py", line 911, in read
return self._sslobj.read(len, buffer)
socket.timeout: The read operation timed out
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/app/.heroku/python/lib/python3.7/site-packages/requests/adapters.py", line 449, in send
timeout=timeout
File "/app/.heroku/python/lib/python3.7/site-packages/urllib3/connectionpool.py", line 641, in urlopen
_stacktrace=sys.exc_info()[2])
File "/app/.heroku/python/lib/python3.7/site-packages/urllib3/util/retry.py", line 368, in increment
raise six.reraise(type(error), error, _stacktrace)
File "/app/.heroku/python/lib/python3.7/site-packages/urllib3/packages/six.py", line 686, in reraise
raise value
File "/app/.heroku/python/lib/python3.7/site-packages/urllib3/connectionpool.py", line 603, in urlopen
chunked=chunked)
File "/app/.heroku/python/lib/python3.7/site-packages/newrelic/hooks/external_urllib3.py", line 18, in _nr_wrapper_make_request_
return wrapped(*args, **kwargs)
File "/app/.heroku/python/lib/python3.7/site-packages/newrelic/hooks/external_urllib3.py", line 18, in _nr_wrapper_make_request_
return wrapped(*args, **kwargs)
File "/app/.heroku/python/lib/python3.7/site-packages/urllib3/connectionpool.py", line 389, in _make_request
self._raise_timeout(err=e, url=url, timeout_value=read_timeout)
File "/app/.heroku/python/lib/python3.7/site-packages/urllib3/connectionpool.py", line 307, in _raise_timeout
raise ReadTimeoutError(self, url, "Read timed out. (read timeout=%s)" % timeout_value)
urllib3.exceptions.ReadTimeoutError: HTTPSConnectionPool(host='bugzilla.mozilla.org', port=443): Read timed out. (read timeout=30)
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "./manage.py", line 34, in <module>
execute_from_command_line(sys.argv)
File "/app/.heroku/python/lib/python3.7/site-packages/django/core/management/__init__.py", line 381, in execute_from_command_line
utility.execute()
File "/app/.heroku/python/lib/python3.7/site-packages/django/core/management/__init__.py", line 375, in execute
self.fetch_command(subcommand).run_from_argv(self.argv)
File "/app/.heroku/python/lib/python3.7/site-packages/newrelic/hooks/framework_django.py", line 978, in _nr_wrapper_BaseCommand_run_from_argv_
return wrapped(*args, **kwargs)
File "/app/.heroku/python/lib/python3.7/site-packages/django/core/management/base.py", line 323, in run_from_argv
self.execute(*args, **cmd_options)
File "/app/.heroku/python/lib/python3.7/site-packages/django/core/management/base.py", line 364, in execute
output = self.handle(*args, **options)
File "/app/.heroku/python/lib/python3.7/site-packages/newrelic/api/function_trace.py", line 150, in literal_wrapper
return wrapped(*args, **kwargs)
  File "/app/treeherder/etl/management/commands/update_bugscache.py", line 11, in handle 
process.run()
File "/app/treeherder/etl/bugzilla.py", line 38, in run
bug_results_chunk = fetch_intermittent_bugs(offset, limit)
File "/app/treeherder/etl/bugzilla.py", line 22, in fetch_intermittent_bugs
response = fetch_json(url, params=params)
File "/app/treeherder/etl/common.py", line 33, in fetch_json
headers={'Accept': 'application/json'})
File "/app/treeherder/etl/common.py", line 17, in make_request
**kwargs)
File "/app/.heroku/python/lib/python3.7/site-packages/newrelic/api/external_trace.py", line 83, in dynamic_wrapper
return wrapped(*args, **kwargs)
File "/app/.heroku/python/lib/python3.7/site-packages/requests/api.py", line 60, in request
return session.request(method=method, url=url, **kwargs)
File "/app/.heroku/python/lib/python3.7/site-packages/requests/sessions.py", line 533, in request
resp = self.send(prep, **send_kwargs)
File "/app/.heroku/python/lib/python3.7/site-packages/newrelic/api/external_trace.py", line 83, in dynamic_wrapper
return wrapped(*args, **kwargs)
File "/app/.heroku/python/lib/python3.7/site-packages/requests/sessions.py", line 646, in send
r = adapter.send(request, **kwargs)
File "/app/.heroku/python/lib/python3.7/site-packages/requests/adapters.py", line 529, in send
raise ReadTimeout(e, request=request)
requests.exceptions.ReadTimeout: HTTPSConnectionPool(host='bugzilla.mozilla.org', port=443): Read timed out. (read timeout=30)