Add urllib3 retry logic for upload POST action

This should initiate 5 retries when we receive
502 Bad Gateway errors uploading to GitHub.

Fixes https://github.com/brave/devops/issues/943

script/lib/github.py

@@ -6,6 +6,10 @@ import re
 import requests
 import sys
 import base64
+import logging
+from requests.adapters import HTTPAdapter
+from urllib3.util.retry import Retry
+from urllib3.exceptions import MaxRetryError
 try:
     from util import execute, scoped_cwd
 except ImportError:
@@ -31,6 +35,22 @@ class GitHub:
     def __getattr__(self, attr):
         return _Callable(self, '/%s' % attr)
 
+    def retry_session(retries=5, session=None, backoff_factor=0.3, status_forcelist=(500, 502, 503, 504),
+                      raise_on_status=True):
+        session = session or requests.Session()
+        retry = Retry(
+            total=retries,
+            read=retries,
+            connect=retries,
+            backoff_factor=backoff_factor,
+            status_forcelist=status_forcelist,
+            raise_on_status=True,
+        )
+        adapter = HTTPAdapter(max_retries=retry)
+        session.mount('http://', adapter)
+        session.mount('https://', adapter)
+        return session
+
     def send(self, method, path, **kw):
         if 'headers' not in kw:
             kw['headers'] = dict()
@@ -48,10 +68,17 @@ class GitHub:
                 kw['data'] = json.dumps(kw['data'])
 
         try:
-            r = getattr(requests, method)(url, **kw).json()
+            if method == "post":
+                logging.debug("Using session to override urllib3 Retry function for POST actions")
+                session = self.retry_session()
+                r = getattr(session, method)(url, **kw).json()
+            else:
+                r = getattr(requests, method)(url, **kw).json()
         except ValueError:
             # Returned response may be empty in some cases
             r = {}
+        except MaxRetryError:
+            raise MaxRetryError
         if 'message' in r:
             raise Exception(json.dumps(r, indent=2, separators=(',', ': ')))
         return r

script/lib/helpers.py

@@ -6,7 +6,12 @@
 import os
 import json
 import requests
-from .config import get_raw_version, get_env_var
+import sys
+
+dirname = os.path.dirname(os.path.realpath(__file__))
+sys.path.append(os.path.join(dirname, '..'))
+
+from config import get_raw_version, get_env_var  # noqa: E402
 
 BRAVE_REPO = "brave/brave-browser"
 BRAVE_CORE_REPO = "brave/brave-core"

script/test/test_upload.py

@@ -7,12 +7,16 @@
 import sys
 import unittest
 import os
-import upload
 from githubmock import Repo, Release, Asset
-from mock import MagicMock
+from mock import (Mock, MagicMock, patch)
 
 dirname = os.path.dirname(os.path.realpath(__file__))
 sys.path.append(os.path.join(dirname, '..'))
+sys.path.append(os.path.join(dirname, '..', 'lib'))
+
+from github import *                            # noqa: E402
+import upload                                   # noqa: E402
+from urllib3.exceptions import MaxRetryError    # noqa: E402
 
 
 # get an existing release (in draft status) from GitHub given a tag name
@@ -21,18 +25,17 @@ class TestGetDraft(unittest.TestCase):
         self.repo = Repo()
 
     def test_returns_existing_draft(self):
-        self.repo.releases._releases = [{'tag_name': 'test', 'draft': True}]
-        self.assertEquals(upload.get_release(self.repo,
-                                             'test')['tag_name'], 'test', False)
+        self.repo.releases._releases = [{'tag_name': 'test', 'allow_published_release_updates': False}]
+        self.assertEquals(upload.get_release(self.repo, 'test', False), None)
 
     def test_fails_on_existing_release(self):
-        self.repo.releases._releases = [{'tag_name': 'test', 'draft': False, 'allow_published_release_updates': False}]
+        self.repo.releases._releases = [{'tag_name': 'test', 'allow_published_release_updates': False}]
         self.assertRaises(UserWarning, upload.get_release, self.repo, 'test', False)
 
     def test_returns_none_on_new_draft(self):
-        self.repo.releases._releases = [{'tag_name': 'old', 'draft': False}]
+        self.repo.releases._releases = [{'tag_name': 'old', 'allow_published_release_updates': False}]
         upload.get_release(self.repo, 'new', False)
-        self.assertEquals(upload.get_release(self.repo, 'test'), None)
+        self.assertEquals(upload.get_release(self.repo, 'test', False), None)
 
 
 class TestGetBravePackages(unittest.TestCase):
@@ -290,6 +293,17 @@ class TestUploadBrave(unittest.TestCase):
         upload.upload_sha256_checksum.assert_called_with(
             self.release.tag_name, self.file_path)
 
+    @patch('github.GitHub.retry_session')
+    def test_retry_session(self, github_retry):
+        urlpart1 = "https://uploads.github.com:443/repos/brave/brave-browser/"
+        urlpart2 = "releases/17325413/assets?name=brave-browser-0.61.51-1.x86_64.rpm"
+        github_retry.side_effect = MaxRetryError("retry_session_pool", urlpart1 + urlpart2,
+                                                 MaxRetryError)
+        try:
+            github_retry()
+        except MaxRetryError as mre:
+            print("Caught MaxRetryError: {}".format(mre))
+
     # TODO: test `armv7l` code path