Provide two ways to select platforms to upload to omaha_server

Replacement for bad rebase on PR https://github.com/brave/brave-core/pull/2168

Fixes https://github.com/brave/devops/issues/744

script/lib/helpers.py

@@ -5,7 +5,8 @@
 
 import os
 import json
-from .config import get_raw_version
+import requests
+from .config import get_raw_version, get_env_var
 
 BRAVE_REPO = "brave/brave-browser"
 BRAVE_CORE_REPO = "brave/brave-core"
@@ -27,13 +28,48 @@ def get_channel_display_name():
     return d[release_channel()]
 
 
+def call_github_api(url, headers):
+    try:
+        r = requests.get(url, headers=headers)
+    except requests.exceptions.ConnectionError as e:
+        print("Error: Received requests.exceptions.ConnectionError, Exiting...")
+        exit(1)
+    except Exception as e:
+        raise Exception(e)
+
+    if r.status_code is 200:
+        return r
+
+
 def get_releases_by_tag(repo, tag_name, include_drafts=False):
-    if include_drafts:
-        return [r for r in repo.releases.get() if
-                r['tag_name'] == tag_name]
-    else:
-        return [r for r in repo.releases.get() if
-                r['tag_name'] == tag_name and not r['draft']]
+
+    GITHUB_URL = 'https://api.github.com'
+
+    next_request = ""
+    headers = {'Accept': 'application/vnd.github+json'}
+    release_url = GITHUB_URL + "/repos/" + BRAVE_REPO + "/releases" + '?access_token=' + \
+        get_env_var('GITHUB_TOKEN') + '&page=1&per_page=100'
+    r = call_github_api(release_url, headers=headers)
+    next_request = ""
+    # The GitHub API returns paginated results of 100 items maximum per
+    # response. We will loop until there is no next link header returned
+    # in the response header. This is documented here:
+    # https://developer.github.com/v3/#pagination
+    while next_request is not None:
+        for item in r.json():
+            # print("DEBUG: release: {}".format(item['name']))
+            if include_drafts:
+                if item['tag_name'] == tag_name:
+                    return [item]
+            else:
+                if item['tag_name'] == tag_name and not item['draft']:
+                    return [item]
+        if r.links.get("next"):
+            next_request = r.links["next"]["url"]
+            r = call_github_api(next_request, headers=headers)
+        else:
+            next_request = None
+    return []
 
 
 def get_release(repo, tag, allow_published_release_updates=False):

script/omaha-upload.py

@@ -56,51 +56,92 @@ def download_from_github(args, logging):
         if len(releases) > 1:
             exit("Error: More than 1 release exists with the tag: \'{}\'".format(tag_name))
         release = releases[0]
+    else:
+        exit("Error: Did not get the release \'{}\' from Github.".format(tag_name))
+
+    found_assets_in_github_release = {}
 
     for asset in release['assets']:
-        if re.match(r'.*\.dmg$', asset['name']) \
-                or re.match(r'brave_installer.*\.exe$', asset['name']):
-            filename = asset['name']
-            asset_url = asset['url']
-            if args.debug:
-                logging.debug("GitHub asset_url: {}".format(
-                    asset_url + '/' + filename))
+        if re.match(r'.*\.dmg$', asset['name']):
+            if args.uploaded:
+                if not args.platform:
+                    args.platform = []
+                args.platform.append('darwin')
+            found_assets_in_github_release['darwin'] = {}
+            found_assets_in_github_release['darwin']['name'] = asset['name']
+            found_assets_in_github_release['darwin']['url'] = asset['url']
+        elif re.match(r'brave_installer-ia32\.exe$', asset['name']):
+            if args.uploaded:
+                if not args.platform:
+                    args.platform = []
+                args.platform.append('win32')
+            found_assets_in_github_release['win32'] = {}
+            found_assets_in_github_release['win32']['name'] = asset['name']
+            found_assets_in_github_release['win32']['url'] = asset['url']
+        elif re.match(r'brave_installer-x64\.exe$', asset['name']):
+            if args.uploaded:
+                if not args.platform:
+                    args.platform = []
+                args.platform.append('win64')
+            found_assets_in_github_release['win64'] = {}
+            found_assets_in_github_release['win64']['name'] = asset['name']
+            found_assets_in_github_release['win64']['url'] = asset['url']
 
-            # Instantiate new requests session, versus reusing the repo session above.
-            # Headers was likely being reused in that session, and not allowing us
-            # to set the Accept header to the below.
-            headers = {'Accept': 'application/octet-stream'}
+    logging.debug("Found assets in github release: {}".format(
+                  found_assets_in_github_release))
 
-            asset_auth_url = asset_url + '?access_token=' + \
-                os.environ.get('BRAVE_GITHUB_TOKEN')
+    for requested_platform in args.platform:
+        logging.debug("Verifying platform \'{}\' exists in GitHub release".
+                      format(requested_platform))
+        if requested_platform not in found_assets_in_github_release.keys():
+            logging.error("Platform \'{}\' does not exist in GitHub release".
+                          format(requested_platform))
+            exit(1)
 
-            if args.debug:
-                # disable urllib3 logging for this session to avoid showing
-                # access_token in logs
-                logging.getLogger("urllib3").setLevel(logging.WARNING)
+    for platform in args.platform:
+        if args.debug:
+            logging.debug("GitHub asset_url: {}".format(
+                          found_assets_in_github_release[platform]['url'] + '/'
+                          + found_assets_in_github_release[platform]['name']))
 
-            r = requests.get(asset_auth_url, headers=headers, stream=True)
+        # Instantiate new requests session, versus reusing the repo session above.
+        # Headers was likely being reused in that session, and not allowing us
+        # to set the Accept header to the below.
+        headers = {'Accept': 'application/octet-stream'}
 
-            if args.debug:
-                logging.getLogger("urllib3").setLevel(logging.DEBUG)
+        asset_auth_url = found_assets_in_github_release[platform]['url'] + \
+            '?access_token=' + os.environ.get('BRAVE_GITHUB_TOKEN')
 
-            with open(filename, 'wb') as f:
-                for chunk in r.iter_content(chunk_size=1024):
-                    if chunk:
-                        f.write(chunk)
+        if args.debug:
+            # disable urllib3 logging for this session to avoid showing
+            # access_token in logs
+            logging.getLogger("urllib3").setLevel(logging.WARNING)
 
+        r = requests.get(asset_auth_url, headers=headers, stream=True)
+
+        if args.debug:
+            logging.getLogger("urllib3").setLevel(logging.DEBUG)
+
+        logging.debug("Writing GitHub download to file: {}".format(
+                      found_assets_in_github_release[platform]['name']))
+
+        with open(found_assets_in_github_release[platform]['name'], 'wb') as f:
+            for chunk in r.iter_content(chunk_size=1024):
+                if chunk:
+                    f.write(chunk)
+
+        logging.debug(
+            "Requests Response status_code: {}".format(r.status_code))
+        if r.status_code == 200:
+            file_list.append('./' + found_assets_in_github_release[platform]['name'])
+        else:
             logging.debug(
-                "Requests Response status_code: {}".format(r.status_code))
-            if r.status_code == 200:
-                file_list.append('./' + filename)
-            else:
-                logging.debug(
-                    "Requests Response status_code != 200: {}".format(r.status_code))
+                "Requests Response status_code != 200: {}".format(r.status_code))
 
-    if len(file_list) < 3:
-        logging.error(
-            "Cannot get all 3 install files from Github! (\'*.dmg\', \'brave_installer-x64.exe\',"
-            " \'brave-installer-ia32.exe\')")
+    if len(file_list) < len(args.platform):
+        for item in args.platform:
+            logging.error(
+                "Cannot get requested file from Github! {}".format(found_assets_in_github_release[item]['name']))
         remove_github_downloaded_files(file_list, logging)
         exit(1)
 
@@ -163,8 +204,11 @@ def parse_args():
                         ' from Github before uploading to Omaha (cannot be combined with --file)')
     parser.add_argument('-p', '--preview', action='store_true', help='Preview channels for testing'
                         ' omaha/sparkle uploads by QA before production release')
-    parser.add_argument(
-        '-t', '--tag', help='Version tag to download from Github')
+    parser.add_argument('--platform', help='Platform(s) to upload to Omaha (separated by spaces)',
+                        nargs='*', choices=['win32', 'win64', 'darwin'])
+    parser.add_argument('--uploaded', help='Upload all the platform(s) that are already in the GitHub release',
+                        action='store_true')
+    parser.add_argument('-t', '--tag', help='Version tag to download from Github')
     return parser.parse_args()
 
 
@@ -175,6 +219,16 @@ def main():
         logging.basicConfig(stream=sys.stderr, level=logging.DEBUG)
         logging.debug('brave_version: {}'.format(get_upload_version()))
 
+    if args.uploaded and args.platform:
+        exit("Error: --platform and --uploaded are mutually exclusive, only one allowed")
+
+    # Default to requiring all 3 platforms
+    if not args.uploaded and not args.platform:
+        args.platform = ['win32', 'win64', 'darwin']
+
+    if args.debug and args.platform:
+        logging.debug("args.platform: {}".format(args.platform))
+
     if args.file and args.github:
         exit("Error: --file and --github are mutually exclusive, only one allowed")