diff --git a/script/lib/github.py b/script/lib/github.py index ae6c630b642..56f5b0ae99f 100644 --- a/script/lib/github.py +++ b/script/lib/github.py @@ -6,6 +6,10 @@ import re import requests import sys import base64 +import logging +from requests.adapters import HTTPAdapter +from urllib3.util.retry import Retry +from urllib3.exceptions import MaxRetryError try: from util import execute, scoped_cwd except ImportError: @@ -31,6 +35,22 @@ class GitHub: def __getattr__(self, attr): return _Callable(self, '/%s' % attr) + def retry_session(retries=5, session=None, backoff_factor=0.3, status_forcelist=(500, 502, 503, 504), + raise_on_status=True): + session = session or requests.Session() + retry = Retry( + total=retries, + read=retries, + connect=retries, + backoff_factor=backoff_factor, + status_forcelist=status_forcelist, + raise_on_status=True, + ) + adapter = HTTPAdapter(max_retries=retry) + session.mount('http://', adapter) + session.mount('https://', adapter) + return session + def send(self, method, path, **kw): if 'headers' not in kw: kw['headers'] = dict() @@ -48,10 +68,17 @@ class GitHub: kw['data'] = json.dumps(kw['data']) try: - r = getattr(requests, method)(url, **kw).json() + if method == "post": + logging.debug("Using session to override urllib3 Retry function for POST actions") + session = self.retry_session() + r = getattr(session, method)(url, **kw).json() + else: + r = getattr(requests, method)(url, **kw).json() except ValueError: # Returned response may be empty in some cases r = {} + except MaxRetryError: + raise MaxRetryError if 'message' in r: raise Exception(json.dumps(r, indent=2, separators=(',', ': '))) return r diff --git a/script/lib/helpers.py b/script/lib/helpers.py index c5c1367c55e..d6fa2d5784e 100644 --- a/script/lib/helpers.py +++ b/script/lib/helpers.py @@ -6,7 +6,12 @@ import os import json import requests -from .config import get_raw_version, get_env_var +import sys + +dirname = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(os.path.join(dirname, '..')) + +from config import get_raw_version, get_env_var # noqa: E402 BRAVE_REPO = "brave/brave-browser" BRAVE_CORE_REPO = "brave/brave-core" diff --git a/script/test/test_upload.py b/script/test/test_upload.py index 5ed53002161..112fa8cd4ae 100644 --- a/script/test/test_upload.py +++ b/script/test/test_upload.py @@ -7,12 +7,16 @@ import sys import unittest import os -import upload from githubmock import Repo, Release, Asset -from mock import MagicMock +from mock import (Mock, MagicMock, patch) dirname = os.path.dirname(os.path.realpath(__file__)) sys.path.append(os.path.join(dirname, '..')) +sys.path.append(os.path.join(dirname, '..', 'lib')) + +from github import * # noqa: E402 +import upload # noqa: E402 +from urllib3.exceptions import MaxRetryError # noqa: E402 # get an existing release (in draft status) from GitHub given a tag name @@ -21,18 +25,17 @@ class TestGetDraft(unittest.TestCase): self.repo = Repo() def test_returns_existing_draft(self): - self.repo.releases._releases = [{'tag_name': 'test', 'draft': True}] - self.assertEquals(upload.get_release(self.repo, - 'test')['tag_name'], 'test', False) + self.repo.releases._releases = [{'tag_name': 'test', 'allow_published_release_updates': False}] + self.assertEquals(upload.get_release(self.repo, 'test', False), None) def test_fails_on_existing_release(self): - self.repo.releases._releases = [{'tag_name': 'test', 'draft': False, 'allow_published_release_updates': False}] + self.repo.releases._releases = [{'tag_name': 'test', 'allow_published_release_updates': False}] self.assertRaises(UserWarning, upload.get_release, self.repo, 'test', False) def test_returns_none_on_new_draft(self): - self.repo.releases._releases = [{'tag_name': 'old', 'draft': False}] + self.repo.releases._releases = [{'tag_name': 'old', 'allow_published_release_updates': False}] upload.get_release(self.repo, 'new', False) - self.assertEquals(upload.get_release(self.repo, 'test'), None) + self.assertEquals(upload.get_release(self.repo, 'test', False), None) class TestGetBravePackages(unittest.TestCase): @@ -290,6 +293,17 @@ class TestUploadBrave(unittest.TestCase): upload.upload_sha256_checksum.assert_called_with( self.release.tag_name, self.file_path) + @patch('github.GitHub.retry_session') + def test_retry_session(self, github_retry): + urlpart1 = "https://uploads.github.com:443/repos/brave/brave-browser/" + urlpart2 = "releases/17325413/assets?name=brave-browser-0.61.51-1.x86_64.rpm" + github_retry.side_effect = MaxRetryError("retry_session_pool", urlpart1 + urlpart2, + MaxRetryError) + try: + github_retry() + except MaxRetryError as mre: + print("Caught MaxRetryError: {}".format(mre)) + # TODO: test `armv7l` code path