diff --git a/.travis.yml b/.travis.yml index bf21ad54b7c..09af836aad8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,6 +3,7 @@ node_js: - "node" dist: trusty +sudo: required branches: only: @@ -18,10 +19,13 @@ env: - TEST_SUITE=lint - TEST_SUITE=test-security - TEST_SUITE=test-unit + - TEST_SUITE=pep8 before_install: - npm i -g npm - npm --version + - sudo pip install --upgrade pip + - sudo pip install pycodestyle install: - npm i diff --git a/package.json b/package.json index 56c585e3be7..0e70862f34b 100644 --- a/package.json +++ b/package.json @@ -6,6 +6,7 @@ "scripts": { "test": "echo \"Error: no test specified\" && exit 1", "lint": "tslint --project tsconfig.json 'components/**/*.{ts,tsx}'", + "pep8": "pycodestyle script/**/*.py", "web-ui": "webpack --config components/webpack/prod.config.js --progress --profile --colors", "web-ui-dev": "webpack --config components/webpack/dev.config.js --progress --profile --colors", "test-unit": "jest -t", diff --git a/script/lib/config.py b/script/lib/config.py index 99192e0043c..e18f33fba70 100644 --- a/script/lib/config.py +++ b/script/lib/config.py @@ -8,112 +8,129 @@ import re import sys PLATFORM = { - 'cygwin': 'win32', - 'darwin': 'darwin', - 'linux2': 'linux', - 'win32': 'win32', + 'cygwin': 'win32', + 'darwin': 'darwin', + 'linux2': 'linux', + 'win32': 'win32', }[sys.platform] -SOURCE_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..')) -CHROMIUM_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..', '..')) +SOURCE_ROOT = os.path.abspath( + os.path.join(os.path.dirname(__file__), '..', '..')) +CHROMIUM_ROOT = os.path.abspath( + os.path.join(os.path.dirname(__file__), '..', '..', '..')) DIST_URL = 'https://brave-brave-binaries.s3.amazonaws.com/releases/' -BRAVE_CORE_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..')) -BRAVE_BROWSER_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..', '..', '..')) - +BRAVE_CORE_ROOT = os.path.abspath( + os.path.join(os.path.dirname(__file__), '..', '..')) +BRAVE_BROWSER_ROOT = os.path.abspath( + os.path.join(os.path.dirname(__file__), '..', '..', '..', '..')) verbose_mode = False def dist_dir(): - return os.path.join(output_dir(), 'dist') + return os.path.join(output_dir(), 'dist') + def output_dir(): - if get_target_arch() == 'x64': - return os.path.join(CHROMIUM_ROOT, 'out', 'Release') - return os.path.join(CHROMIUM_ROOT, 'out_x86', 'Release') + if get_target_arch() == 'x64': + return os.path.join(CHROMIUM_ROOT, 'out', 'Release') + return os.path.join(CHROMIUM_ROOT, 'out_x86', 'Release') + # Use brave-browser/package.json version for canonical version definition def brave_browser_package(): - pjson = os.path.join(BRAVE_BROWSER_ROOT, 'package.json') - with open(pjson) as f: - obj = json.load(f); - return obj; + pjson = os.path.join(BRAVE_BROWSER_ROOT, 'package.json') + with open(pjson) as f: + obj = json.load(f) + return obj + def brave_core_package(): - pjson = os.path.join(BRAVE_CORE_ROOT, 'package.json') - with open(pjson) as f: - obj = json.load(f); - return obj; + pjson = os.path.join(BRAVE_CORE_ROOT, 'package.json') + with open(pjson) as f: + obj = json.load(f) + return obj + def product_name(): - return os.environ.get('npm_config_brave_product_name') or brave_core_package()['name'].split('-')[0] + return (os.environ.get('npm_config_brave_product_name') or + brave_core_package()['name'].split('-')[0]) def project_name(): - return os.environ.get('npm_config_brave_project_name') or brave_core_package()['name'].split('-')[0] + return (os.environ.get('npm_config_brave_project_name') or + brave_core_package()['name'].split('-')[0]) def get_chrome_version(): - version = os.environ.get('npm_config_brave_version') or brave_browser_package()['version'] - return version.split('+')[1] + version = (os.environ.get('npm_config_brave_version') or + brave_browser_package()['version']) + return version.split('+')[1] + def get_brave_version(): - return 'v' + get_raw_version() + return 'v' + get_raw_version() + def get_raw_version(): - return os.environ.get('npm_config_brave_version') or brave_browser_package()['version'] + return (os.environ.get('npm_config_brave_version') or + brave_browser_package()['version']) + def get_platform_key(): - if os.environ.has_key('MAS_BUILD'): - return 'mas' - else: - return PLATFORM + if 'MAS_BUILD' in os.environ: + return 'mas' + else: + return PLATFORM def get_target_arch(): - return os.environ['TARGET_ARCH'] if os.environ.has_key('TARGET_ARCH') else 'x64' + return (os.environ['TARGET_ARCH'] if 'TARGET_ARCH' in os.environ + else 'x64') def get_chromedriver_version(): - pattern = "^chromedriver_version = \"([0-9]\.[0-9]+)\"" - build_gn_path = os.path.join(BRAVE_CORE_ROOT, 'BUILD.gn') - with open(build_gn_path, 'r') as build_gn_file: - for line in build_gn_file: - match = re.search(pattern,line) - if match: - version = match.group(1) - return 'v' + version + pattern = "^chromedriver_version = \"([0-9]\\.[0-9]+)\"" + build_gn_path = os.path.join(BRAVE_CORE_ROOT, 'BUILD.gn') + with open(build_gn_path, 'r') as build_gn_file: + for line in build_gn_file: + match = re.search(pattern, line) + if match: + version = match.group(1) + return 'v' + version def get_env_var(name): - return os.environ.get('BRAVE_' + name) or os.environ.get('npm_config_BRAVE_' + name, '') + return (os.environ.get('BRAVE_' + name) or + os.environ.get('npm_config_BRAVE_' + name, '')) def s3_config(): - config = (get_env_var('S3_BUCKET'), - get_env_var('S3_ACCESS_KEY'), - get_env_var('S3_SECRET_KEY')) - message = ('Error: Please set the $BRAVE_S3_BUCKET, ' - '$BRAVE_S3_ACCESS_KEY, and ' - '$BRAVE_S3_SECRET_KEY environment variables') - assert all(len(c) for c in config), message - return config + config = (get_env_var('S3_BUCKET'), + get_env_var('S3_ACCESS_KEY'), + get_env_var('S3_SECRET_KEY')) + message = ('Error: Please set the $BRAVE_S3_BUCKET, ' + '$BRAVE_S3_ACCESS_KEY, and ' + '$BRAVE_S3_SECRET_KEY environment variables') + assert all(len(c) for c in config), message + return config def enable_verbose_mode(): - print 'Running in verbose mode' - global verbose_mode - verbose_mode = True + print 'Running in verbose mode' + global verbose_mode + verbose_mode = True def is_verbose_mode(): - return verbose_mode + return verbose_mode def get_zip_name(name, version, suffix=''): - arch = get_target_arch() - if arch == 'arm': - arch += 'v7l' - zip_name = '{0}-{1}-{2}-{3}'.format(name, version, get_platform_key(), arch) - if suffix: - zip_name += '-' + suffix - return zip_name + '.zip' + arch = get_target_arch() + if arch == 'arm': + arch += 'v7l' + zip_name = '{0}-{1}-{2}-{3}'.format(name, version, get_platform_key(), + arch) + if suffix: + zip_name += '-' + suffix + return zip_name + '.zip' diff --git a/script/lib/env_util.py b/script/lib/env_util.py index df759241fe8..facf59651f6 100644 --- a/script/lib/env_util.py +++ b/script/lib/env_util.py @@ -8,64 +8,70 @@ import sys def validate_pair(ob): - if not (len(ob) == 2): - print("Unexpected result:", ob, file=sys.stderr) - return False - else: - return True + if not (len(ob) == 2): + print("Unexpected result:", ob, file=sys.stderr) + return False + else: + return True def consume(iter): - try: - while True: next(iter) - except StopIteration: - pass + try: + while True: + next(iter) + except StopIteration: + pass + + +# Define a way to handle each KEY=VALUE line. +def handle_line(l): + return l.rstrip().split('=', 1) def get_environment_from_batch_command(env_cmd, initial=None): - """ - Take a command (either a single command or list of arguments) - and return the environment created after running that command. - Note that if the command must be a batch file or .cmd file, or the - changes to the environment will not be captured. + """ + Take a command (either a single command or list of arguments) + and return the environment created after running that command. + Note that if the command must be a batch file or .cmd file, or the + changes to the environment will not be captured. - If initial is supplied, it is used as the initial environment passed - to the child process. - """ - if not isinstance(env_cmd, (list, tuple)): - env_cmd = [env_cmd] - # Construct the command that will alter the environment. - env_cmd = subprocess.list2cmdline(env_cmd) - # Create a tag so we can tell in the output when the proc is done. - tag = 'END OF BATCH COMMAND' - # Construct a cmd.exe command to do accomplish this. - cmd = 'cmd.exe /s /c "{env_cmd} && echo "{tag}" && set"'.format(**vars()) - # Launch the process. - proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, env=initial) - # Parse the output sent to stdout. - lines = proc.stdout - # Consume whatever output occurs until the tag is reached. - consume(itertools.takewhile(lambda l: tag not in l, lines)) - # Define a way to handle each KEY=VALUE line. - handle_line = lambda l: l.rstrip().split('=',1) - # Parse key/values into pairs. - pairs = map(handle_line, lines) - # Make sure the pairs are valid. - valid_pairs = filter(validate_pair, pairs) - # Construct a dictionary of the pairs. - result = dict(valid_pairs) - # Let the process finish. - proc.communicate() - return result + If initial is supplied, it is used as the initial environment passed + to the child process. + """ + if not isinstance(env_cmd, (list, tuple)): + env_cmd = [env_cmd] + # Construct the command that will alter the environment. + env_cmd = subprocess.list2cmdline(env_cmd) + # Create a tag so we can tell in the output when the proc is done. + tag = 'END OF BATCH COMMAND' + # Construct a cmd.exe command to do accomplish this. + cmd = 'cmd.exe /s /c "{env_cmd} && echo "{tag}" && set"'.format(**vars()) + # Launch the process. + proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, env=initial) + # Parse the output sent to stdout. + lines = proc.stdout + # Consume whatever output occurs until the tag is reached. + consume(itertools.takewhile(lambda l: tag not in l, lines)) + # Parse key/values into pairs. + pairs = map(handle_line, lines) + # Make sure the pairs are valid. + valid_pairs = filter(validate_pair, pairs) + # Construct a dictionary of the pairs. + result = dict(valid_pairs) + # Let the process finish. + proc.communicate() + return result def get_vs_env(vs_version, arch): - """ - Returns the env object for VS building environment. + """ + Returns the env object for VS building environment. - The vs_version can be strings like "12.0" (e.g. VS2013), the arch has to - be one of "x86", "amd64", "arm", "x86_amd64", "x86_arm", "amd64_x86", - "amd64_arm", e.g. the args passed to vcvarsall.bat. - """ - vsvarsall = "C:\\Program Files (x86)\\Microsoft Visual Studio {0}\\VC\\vcvarsall.bat".format(vs_version) - return get_environment_from_batch_command([vsvarsall, arch]) + The vs_version can be strings like "12.0" (e.g. VS2013), the arch has to + be one of "x86", "amd64", "arm", "x86_amd64", "x86_arm", "amd64_x86", + "amd64_arm", e.g. the args passed to vcvarsall.bat. + """ + vsvarsall = "C:\\Program Files (x86)\\Microsoft Visual Studio {0}" + "\\VC\\vcvarsall.bat".format(vs_version) + + return get_environment_from_batch_command([vsvarsall, arch]) diff --git a/script/lib/github.py b/script/lib/github.py index 6e7024837af..aedcd0808f6 100644 --- a/script/lib/github.py +++ b/script/lib/github.py @@ -3,78 +3,80 @@ import json import os import re +import requests import sys REQUESTS_DIR = os.path.abspath(os.path.join(__file__, '..', '..', '..', 'vendor', 'requests')) sys.path.append(os.path.join(REQUESTS_DIR, 'build', 'lib')) sys.path.append(os.path.join(REQUESTS_DIR, 'build', 'lib.linux-x86_64-2.7')) -import requests GITHUB_URL = 'https://api.github.com' GITHUB_UPLOAD_ASSET_URL = 'https://uploads.github.com' + class GitHub: - def __init__(self, access_token): - self._authorization = 'token %s' % access_token + def __init__(self, access_token): + self._authorization = 'token %s' % access_token - pattern = '^/repos/{0}/{0}/releases/{1}/assets$'.format('[^/]+', '[0-9]+') - self._releases_upload_api_pattern = re.compile(pattern) + pattern = '^/repos/{0}/{0}/releases/{1}/assets$'.format( + '[^/]+', '[0-9]+') + self._releases_upload_api_pattern = re.compile(pattern) - def __getattr__(self, attr): - return _Callable(self, '/%s' % attr) + def __getattr__(self, attr): + return _Callable(self, '/%s' % attr) - def send(self, method, path, **kw): - if not 'headers' in kw: - kw['headers'] = dict() - headers = kw['headers'] - headers['Authorization'] = self._authorization - headers['Accept'] = 'application/vnd.github.manifold-preview' + def send(self, method, path, **kw): + if 'headers' not in kw: + kw['headers'] = dict() + headers = kw['headers'] + headers['Authorization'] = self._authorization + headers['Accept'] = 'application/vnd.github.manifold-preview' - # Switch to a different domain for the releases uploading API. - if self._releases_upload_api_pattern.match(path): - url = '%s%s' % (GITHUB_UPLOAD_ASSET_URL, path) - else: - url = '%s%s' % (GITHUB_URL, path) - # Data are sent in JSON format. - if 'data' in kw: - kw['data'] = json.dumps(kw['data']) + # Switch to a different domain for the releases uploading API. + if self._releases_upload_api_pattern.match(path): + url = '%s%s' % (GITHUB_UPLOAD_ASSET_URL, path) + else: + url = '%s%s' % (GITHUB_URL, path) + # Data are sent in JSON format. + if 'data' in kw: + kw['data'] = json.dumps(kw['data']) - try: - r = getattr(requests, method)(url, **kw).json() - except ValueError: - # Returned response may be empty in some cases - r = {} - if 'message' in r: - raise Exception(json.dumps(r, indent=2, separators=(',', ': '))) - return r + try: + r = getattr(requests, method)(url, **kw).json() + except ValueError: + # Returned response may be empty in some cases + r = {} + if 'message' in r: + raise Exception(json.dumps(r, indent=2, separators=(',', ': '))) + return r class _Executable: - def __init__(self, gh, method, path): - self._gh = gh - self._method = method - self._path = path + def __init__(self, gh, method, path): + self._gh = gh + self._method = method + self._path = path - def __call__(self, **kw): - return self._gh.send(self._method, self._path, **kw) + def __call__(self, **kw): + return self._gh.send(self._method, self._path, **kw) class _Callable(object): - def __init__(self, gh, name): - self._gh = gh - self._name = name + def __init__(self, gh, name): + self._gh = gh + self._name = name - def __call__(self, *args): - if len(args) == 0: - return self + def __call__(self, *args): + if len(args) == 0: + return self - name = '%s/%s' % (self._name, '/'.join([str(arg) for arg in args])) - return _Callable(self._gh, name) + name = '%s/%s' % (self._name, '/'.join([str(arg) for arg in args])) + return _Callable(self._gh, name) - def __getattr__(self, attr): - if attr in ['get', 'put', 'post', 'patch', 'delete']: - return _Executable(self._gh, attr, self._name) + def __getattr__(self, attr): + if attr in ['get', 'put', 'post', 'patch', 'delete']: + return _Executable(self._gh, attr, self._name) - name = '%s/%s' % (self._name, attr) - return _Callable(self._gh, name) + name = '%s/%s' % (self._name, attr) + return _Callable(self._gh, name) diff --git a/script/lib/helpers.py b/script/lib/helpers.py index 951cd195c15..fb9784823ef 100644 --- a/script/lib/helpers.py +++ b/script/lib/helpers.py @@ -9,45 +9,56 @@ from .config import get_raw_version BRAVE_REPO = "brave/brave-browser" + def get_channel_display_name(): - d = {'beta': 'Beta', 'canary': 'Canary', 'dev': 'Developer', 'release': 'Release'} - return d[release_channel()] + d = {'beta': 'Beta', 'canary': 'Canary', 'dev': 'Developer', + 'release': 'Release'} + return d[release_channel()] + def get_releases_by_tag(repo, tag_name, include_drafts=False): - if include_drafts: - return [r for r in repo.releases.get() if r['tag_name'] == tag_name] - else: - - return [r for r in repo.releases.get() if r['tag_name'] == tag_name and not r['draft']] + if include_drafts: + return [r for r in repo.releases.get() if + r['tag_name'] == tag_name] + else: + return [r for r in repo.releases.get() if + r['tag_name'] == tag_name and not r['draft']] + def release_channel(): - channel = os.environ['CHANNEL'] - message = ('Error: Please set the $CHANNEL ' - 'environment variable, which is your release channel') - assert channel, message - return channel + channel = os.environ['CHANNEL'] + message = ('Error: Please set the $CHANNEL ' + 'environment variable, which is your release channel') + assert channel, message + return channel + def get_tag(): - return 'v' + get_raw_version() + release_channel() + return 'v' + get_raw_version() + release_channel() + def release_name(): - return '{0} Channel'.format(get_channel_display_name()) + return '{0} Channel'.format(get_channel_display_name()) + def get_releases_by_tag(repo, tag_name, include_drafts=False): - if include_drafts: - return [r for r in repo.releases.get() if r['tag_name'] == tag_name] - else: - return [r for r in repo.releases.get() if r['tag_name'] == tag_name and not r['draft']] + if include_drafts: + return [r for r in repo.releases.get() if r['tag_name'] == tag_name] + else: + return [r for r in repo.releases.get() if + r['tag_name'] == tag_name and not r['draft']] + def retry_func(try_func, catch, retries, catch_func=None): - for count in range(0, retries + 1): - try: - ret = try_func(count) - break - except catch as e: - print('[ERROR] Caught exception {}, {} retries left. {}'.format(catch, count, e.message)) - if catch_func: - catch_func(count) - if count >= retries: - raise e - return ret + for count in range(0, retries + 1): + try: + ret = try_func(count) + break + except catch as e: + print('[ERROR] Caught exception {}, {} retries left. {}'.format( + catch, count, e.message)) + if catch_func: + catch_func(count) + if count >= retries: + raise e + return ret diff --git a/script/lib/transifex.py b/script/lib/transifex.py index ba42de5c64b..69245e38aca 100644 --- a/script/lib/transifex.py +++ b/script/lib/transifex.py @@ -14,463 +14,560 @@ import FP transifex_project_name = 'brave' base_url = 'https://www.transifex.com/api/2/' + def transifex_name_from_filename(source_file_path, filename): - ext = os.path.splitext(source_file_path)[1] - if 'brave_components_strings' in source_file_path: - return 'brave_components_resources' - elif ext == '.grd': - return filename - elif 'brave_rewards' in source_file_path: - return 'rewards_extension' - elif 'brave-extension' in source_file_path: - return 'brave_extension' - assert False, 'JSON files should be mapped explicitly, this one is not: ' + source_file_path + ext = os.path.splitext(source_file_path)[1] + if 'brave_components_strings' in source_file_path: + return 'brave_components_resources' + elif ext == '.grd': + return filename + elif 'brave_rewards' in source_file_path: + return 'rewards_extension' + elif 'brave-extension' in source_file_path: + return 'brave_extension' + assert False, ('JSON files should be mapped explicitly, this ' + 'one is not: ' + source_file_path) + def create_xtb_format_translationbundle_tag(lang): - """Creates the root XTB XML element""" - translationbundle_tag = lxml.etree.Element('translationbundle') - # The lang code "iw" is the old code for Hebrew, the GRDs are updated to use "he". - # But Chromium still uses "iw" inside the XTB, and it causes a compiling error on Windows otherwise. - # So we need to force it back to "iw" here for minimal impact. - translationbundle_tag.set('lang',lang.replace('_', '-').replace('he', 'iw')) - # Adds a newline so the first translation isn't glued to the translationbundle element for us weak humans. - translationbundle_tag.text = '\n' - return translationbundle_tag + """Creates the root XTB XML element""" + translationbundle_tag = lxml.etree.Element('translationbundle') + # The lang code "iw" is the old code for Hebrew, the GRDs are updated to + # use "he". + # But Chromium still uses "iw" inside the XTB, and it causes a compiling + # error on Windows otherwise. So we need to force it back to "iw" here + # for minimal impact. + translationbundle_tag.set( + 'lang', lang.replace('_', '-').replace('he', 'iw')) + # Adds a newline so the first translation isn't glued to the + # translationbundle element for us weak humans. + translationbundle_tag.text = '\n' + return translationbundle_tag def create_xtb_format_translation_tag(fingerprint, string_value): - """Creates child XTB elements for each translation tag""" - string_tag = lxml.etree.Element('translation') - string_tag.set('id', str(fingerprint)) - if string_value.count('<') != string_value.count('>'): - assert False, 'Warning: Unmatched < character, consider fixing on Trasifex, force encoding the following string:' + string_value - string_tag.text = string_value - string_tag.tail = '\n' - return string_tag + """Creates child XTB elements for each translation tag""" + string_tag = lxml.etree.Element('translation') + string_tag.set('id', str(fingerprint)) + if string_value.count('<') != string_value.count('>'): + assert False, 'Warning: Unmatched < character, consider fixing on ' + ' Trasifex, force encoding the following string:' + string_value + string_tag.text = string_value + string_tag.tail = '\n' + return string_tag def create_android_format_resources_tag(): - """Creates intermediate Android format root tag""" - return lxml.etree.Element('resources') + """Creates intermediate Android format root tag""" + return lxml.etree.Element('resources') def create_android_format_string_tag(string_name, string_value): - """Creates intermediate Android format child tag for each translation string""" - string_tag = lxml.etree.Element('string') - string_tag.set('name', string_name) - string_tag.text = string_value - string_tag.tail = '\n' - return string_tag + """Creates intermediate Android format child tag for + each translation string""" + string_tag = lxml.etree.Element('string') + string_tag.set('name', string_name) + string_tag.text = string_value + string_tag.tail = '\n' + return string_tag def get_auth(): - """Creates an HTTPBasicAuth object given the Transifex information""" - username = get_env_var('TRANSIFEX_USERNAME') - password = get_env_var('TRANSIFEX_PASSWORD') - transifex_api_key = get_env_var('TRANSIFEX_API_KEY') - auth = None - if transifex_api_key: - api_key_username = "api:" + transifex_api_key - auth = requests.auth.HTTPBasicAuth(api_key_username, '') - else: - auth = requests.auth.HTTPBasicAuth(username, password) - return auth + """Creates an HTTPBasicAuth object given the Transifex information""" + username = get_env_var('TRANSIFEX_USERNAME') + password = get_env_var('TRANSIFEX_PASSWORD') + transifex_api_key = get_env_var('TRANSIFEX_API_KEY') + auth = None + if transifex_api_key: + api_key_username = "api:" + transifex_api_key + auth = requests.auth.HTTPBasicAuth(api_key_username, '') + else: + auth = requests.auth.HTTPBasicAuth(username, password) + return auth def get_transifex_languages(grd_file_path): - """Extracts the list of locales supported by the passed in GRD file""" - xtb_files = get_xtb_files(grd_file_path) - return set([lang for (lang, xtb_rel_path) in xtb_files]) + """Extracts the list of locales supported by the passed in GRD file""" + xtb_files = get_xtb_files(grd_file_path) + return set([lang for (lang, xtb_rel_path) in xtb_files]) -def get_transifex_translation_file_content(source_file_path, filename, lang_code): - """Obtains a translation Android xml format and returns the string""" - lang_code = lang_code.replace('-', '_') - url_part = 'project/%s/resource/%s/translation/%s' % (transifex_project_name, transifex_name_from_filename(source_file_path, filename), lang_code) - url = base_url + url_part - r = requests.get(url, auth=get_auth()) - assert r.status_code >= 200 and r.status_code <= 299, 'Aborting. Status code %d: %s' % (r.status_code, r.content) - content = r.json()['content'].encode('utf-8') - ext = os.path.splitext(source_file_path)[1] - # For .grd files, for some reason Transifex puts a \\" and \' - if ext == '.grd': - return content.replace('\\"', '"').replace("\\'", "'") - return content +def get_transifex_translation_file_content(source_file_path, filename, + lang_code): + """Obtains a translation Android xml format and returns the string""" + lang_code = lang_code.replace('-', '_') + url_part = 'project/%s/resource/%s/translation/%s' % ( + transifex_project_name, + transifex_name_from_filename(source_file_path, filename), lang_code) + url = base_url + url_part + r = requests.get(url, auth=get_auth()) + assert r.status_code >= 200 and r.status_code <= 299, ( + 'Aborting. Status code %d: %s' % (r.status_code, r.content)) + content = r.json()['content'].encode('utf-8') + ext = os.path.splitext(source_file_path)[1] + # For .grd files, for some reason Transifex puts a \\" and \' + if ext == '.grd': + return content.replace('\\"', '"').replace("\\'", "'") + return content + def get_strings_dict_from_xml_content(xml_content): - """Obtains a dictionary mapping the string name to text from Android xml content""" - strings = lxml.etree.fromstring(xml_content).findall('string') - return { string_tag.get('name'): textify(string_tag) for string_tag in strings } + """Obtains a dictionary mapping the string name to text from Android xml + content""" + strings = lxml.etree.fromstring(xml_content).findall('string') + return {string_tag.get('name'): textify(string_tag) + for string_tag in strings} def get_strings_dict_from_xtb_file(xtb_file_path): - """Obtains a dictionary mapping the string fingerprint to its value for an xtb file""" - # No file exists yet, so just returna an empty dict - if not os.path.isfile(xtb_file_path): - return {} - translation_tags = lxml.etree.parse(xtb_file_path).findall('.//translation') - return { translation_tag.get('id'): textify(translation_tag) for translation_tag in translation_tags } + """Obtains a dictionary mapping the string fingerprint to its value for + an xtb file""" + # No file exists yet, so just returna an empty dict + if not os.path.isfile(xtb_file_path): + return {} + translation_tags = lxml.etree.parse( + xtb_file_path).findall('.//translation') + return {translation_tag.get('id'): textify(translation_tag) + for translation_tag in translation_tags} -def update_source_string_file_to_transifex(source_file_path, filename, content): - """Uploads the specified source string file to transifex""" - print 'Updating existing known resource for filename %s' % filename - url_part = 'project/%s/resource/%s/content' % (transifex_project_name, transifex_name_from_filename(source_file_path, filename)) - url = base_url + url_part - payload = { - 'content': content - } - headers = { 'Content-Type': 'application/json' } - r = requests.put(url, json=payload, auth=get_auth(), headers=headers) - assert r.status_code >= 200 and r.status_code <= 299, 'Aborting. Status code %d: %s' % (r.status_code, r.content) - return True +def update_source_string_file_to_transifex(source_file_path, filename, + content): + """Uploads the specified source string file to transifex""" + print 'Updating existing known resource for filename %s' % filename + url_part = 'project/%s/resource/%s/content' % ( + transifex_project_name, transifex_name_from_filename( + source_file_path, filename)) + url = base_url + url_part + payload = {'content': content} + headers = {'Content-Type': 'application/json'} + r = requests.put(url, json=payload, auth=get_auth(), headers=headers) + assert r.status_code >= 200 and r.status_code <= 299, ( + 'Aborting. Status code %d: %s' % (r.status_code, r.content)) + return True -def upload_source_string_file_to_transifex(source_file_path, filename, xml_content, i18n_type): - """Uploads the specified source string file to transifex""" - url_part = 'project/%s/resources/' % transifex_project_name - url = base_url + url_part - payload = { - 'name': transifex_name_from_filename(source_file_path, filename), - 'slug': transifex_name_from_filename(source_file_path, filename), - 'content': xml_content, - 'i18n_type': i18n_type - } - headers = { 'Content-Type': 'application/json' } - #r = requests.post(url, json=payload, auth=get_auth(), headers=headers) - r = requests.post(url, json=payload, auth=get_auth(), headers=headers) - if r.status_code < 200 or r.status_code > 299: - if r.content.find('Resource with this Slug and Project already exists.') != -1: - return update_source_string_file_to_transifex(source_file_path, filename, xml_content) - else: - assert False, ('Aborting. Status code %d: %s' % (r.status_code, r.content)) - return True +def upload_source_string_file_to_transifex(source_file_path, filename, + xml_content, i18n_type): + """Uploads the specified source string file to transifex""" + url_part = 'project/%s/resources/' % transifex_project_name + url = base_url + url_part + payload = { + 'name': transifex_name_from_filename(source_file_path, filename), + 'slug': transifex_name_from_filename(source_file_path, filename), + 'content': xml_content, + 'i18n_type': i18n_type + } + headers = {'Content-Type': 'application/json'} + # r = requests.post(url, json=payload, auth=get_auth(), headers=headers) + r = requests.post(url, json=payload, auth=get_auth(), headers=headers) + if r.status_code < 200 or r.status_code > 299: + if r.content.find( + 'Resource with this Slug and Project already exists.') != -1: + return update_source_string_file_to_transifex(source_file_path, + filename, + xml_content) + else: + assert False, ('Aborting. Status code %d: %s' % ( + r.status_code, r.content)) + return True def clean_triple_quoted_string(val): - """Grit parses out first 3 and last 3 isngle quote chars if they exist.""" - val = val.strip() - if val.startswith("'''"): - val = val[3:] - if val.endswith("'''"): - val = val[:-3] - return val.strip() + """Grit parses out first 3 and last 3 isngle quote chars if they exist.""" + val = val.strip() + if val.startswith("'''"): + val = val[3:] + if val.endswith("'''"): + val = val[:-3] + return val.strip() def textify(t): - """Returns the text of a node to be translated""" - val = lxml.etree.tostring(t, method='xml', encoding='unicode') - val = val[val.index('>')+1:val.rindex('<')] - val = clean_triple_quoted_string(val) - return val + """Returns the text of a node to be translated""" + val = lxml.etree.tostring(t, method='xml', encoding='unicode') + val = val[val.index('>')+1:val.rindex('<')] + val = clean_triple_quoted_string(val) + return val def get_grd_message_string_tags(grd_file_path): - """Obtains all message tags of the specified GRD file""" - output_elements = [] - if grd_file_path.endswith('.grdp'): - elements = lxml.etree.parse(grd_file_path).findall('./*') - else: - elements = lxml.etree.parse(grd_file_path).findall('.//messages/*') - for element in elements: - if element.tag == 'message': - output_elements.append(element) - elif element.tag == 'if': - expr = element.get('expr') - if expr not in ['chromeos', 'use_titlecase']: - continue - children = list(element) - children = [child for child in children if child.tag == 'message'] - for child in children: - output_elements.append(child) - elif element.tag == 'part': # will be handled below - continue + """Obtains all message tags of the specified GRD file""" + output_elements = [] + if grd_file_path.endswith('.grdp'): + elements = lxml.etree.parse(grd_file_path).findall('./*') else: - assert False, ('Unexpected tag name %s' % element.tag) + elements = lxml.etree.parse(grd_file_path).findall('.//messages/*') + for element in elements: + if element.tag == 'message': + output_elements.append(element) + elif element.tag == 'if': + expr = element.get('expr') + if expr not in ['chromeos', 'use_titlecase']: + continue + children = list(element) + children = [child for child in children if child.tag == 'message'] + for child in children: + output_elements.append(child) + elif element.tag == 'part': # will be handled below + continue + else: + assert False, ('Unexpected tag name %s' % element.tag) - elements = lxml.etree.parse(grd_file_path).findall('.//part') - for element in elements: - grd_base_path = os.path.dirname(grd_file_path) - grd_part_filename = element.get('file') - if grd_part_filename in ['chromeos_strings.grdp', 'media_router_resources.grdp']: - continue - grd_part_path = os.path.join(grd_base_path, grd_part_filename) - part_output_elements = get_grd_message_string_tags(grd_part_path) - output_elements.extend(part_output_elements) + elements = lxml.etree.parse(grd_file_path).findall('.//part') + for element in elements: + grd_base_path = os.path.dirname(grd_file_path) + grd_part_filename = element.get('file') + if grd_part_filename in ['chromeos_strings.grdp', + 'media_router_resources.grdp']: + continue + grd_part_path = os.path.join(grd_base_path, grd_part_filename) + part_output_elements = get_grd_message_string_tags(grd_part_path) + output_elements.extend(part_output_elements) - return output_elements + return output_elements def get_fingerprint_for_xtb(message_tag): - """Obtains the fingerprint meant for xtb files from a message tag.""" - string_to_hash = message_tag.text - string_phs = message_tag.findall('ph') - for string_ph in string_phs: - string_to_hash = string_to_hash + string_ph.get('name').upper() + string_ph.tail - string_to_hash = (string_to_hash or '').strip().encode('utf-8') - string_to_hash = clean_triple_quoted_string(string_to_hash) - return FP.FingerPrint(string_to_hash) & 0x7fffffffffffffffL + """Obtains the fingerprint meant for xtb files from a message tag.""" + string_to_hash = message_tag.text + string_phs = message_tag.findall('ph') + for string_ph in string_phs: + string_to_hash = ( + string_to_hash + string_ph.get('name').upper() + string_ph.tail) + string_to_hash = (string_to_hash or '').strip().encode('utf-8') + string_to_hash = clean_triple_quoted_string(string_to_hash) + return FP.FingerPrint(string_to_hash) & 0x7fffffffffffffffL def get_grd_strings(grd_file_path): - """Obtains a tubple of (name, value, FP) for each string in a GRD file""" - strings = [] - all_message_tags = get_grd_message_string_tags(grd_file_path) - for message_tag in all_message_tags: - message_name = message_tag.get('name') - message_value = textify(message_tag) - translateable = message_tag.get('translateable') - if translateable == 'false': - continue - assert not not message_name, 'Message name is empty' - assert message_name.startswith('IDS_'), ('Invalid message ID: %s' % message_name) - string_name = message_name[4:].lower() - string_fp = get_fingerprint_for_xtb(message_tag) - string_tuple = (string_name, message_value, string_fp) - strings.append(string_tuple) - return strings + """Obtains a tubple of (name, value, FP) for each string in a GRD file""" + strings = [] + all_message_tags = get_grd_message_string_tags(grd_file_path) + for message_tag in all_message_tags: + message_name = message_tag.get('name') + message_value = textify(message_tag) + translateable = message_tag.get('translateable') + if translateable == 'false': + continue + assert not not message_name, 'Message name is empty' + assert message_name.startswith('IDS_'), ( + 'Invalid message ID: %s' % message_name) + string_name = message_name[4:].lower() + string_fp = get_fingerprint_for_xtb(message_tag) + string_tuple = (string_name, message_value, string_fp) + strings.append(string_tuple) + return strings -def generate_source_strings_xml_from_grd(output_xml_file_handle, grd_file_path): - """Generates a source string xml file from a GRD file""" - resources_tag = create_android_format_resources_tag() - all_strings = get_grd_strings(grd_file_path) - for (string_name, string_value, fp) in all_strings: - resources_tag.append(create_android_format_string_tag(string_name, string_value)) - print 'Generating %d strings for GRD: %s' % (len(all_strings), grd_file_path) - xml_string = lxml.etree.tostring(resources_tag) - os.write(output_xml_file_handle, xml_string.encode('utf-8')) - return xml_string +def generate_source_strings_xml_from_grd(output_xml_file_handle, + grd_file_path): + """Generates a source string xml file from a GRD file""" + resources_tag = create_android_format_resources_tag() + all_strings = get_grd_strings(grd_file_path) + for (string_name, string_value, fp) in all_strings: + resources_tag.append( + create_android_format_string_tag(string_name, string_value)) + print 'Generating %d strings for GRD: %s' % ( + len(all_strings), grd_file_path) + xml_string = lxml.etree.tostring(resources_tag) + os.write(output_xml_file_handle, xml_string.encode('utf-8')) + return xml_string def generate_xtb_content(lang_code, grd_strings, translations): - """Generates an XTB file from a set of translations and GRD strings""" - # Used to make sure duplicate fingerprint stringsa re not made - # XTB only contains 1 entry even if multiple string names are different but have the same value. - all_string_fps = set() - translationbundle_tag = create_xtb_format_translationbundle_tag(lang_code) - for string in grd_strings: - if string[0] in translations: - fingerprint = string[2] - if fingerprint in all_string_fps: - continue - all_string_fps.add(fingerprint) - translation = translations[string[0]] - if len(translation) != 0: - translationbundle_tag.append(create_xtb_format_translation_tag(fingerprint, translation)) + """Generates an XTB file from a set of translations and GRD strings""" + # Used to make sure duplicate fingerprint stringsa re not made + # XTB only contains 1 entry even if multiple string names are + # different but have the same value. + all_string_fps = set() + translationbundle_tag = create_xtb_format_translationbundle_tag(lang_code) + for string in grd_strings: + if string[0] in translations: + fingerprint = string[2] + if fingerprint in all_string_fps: + continue + all_string_fps.add(fingerprint) + translation = translations[string[0]] + if len(translation) != 0: + translationbundle_tag.append( + create_xtb_format_translation_tag( + fingerprint, translation)) - xml_string = lxml.etree.tostring(translationbundle_tag) - xml_string = HTMLParser.HTMLParser().unescape(xml_string.encode('utf-8')).encode('utf-8') - xml_string = '\n\n' + xml_string - return xml_string + xml_string = lxml.etree.tostring(translationbundle_tag) + xml_string = HTMLParser.HTMLParser().unescape( + xml_string.encode('utf-8')).encode('utf-8') + xml_string = ( + '\n\n' + xml_string) + return xml_string def get_xtb_files(grd_file_path): - """Obtains all the XTB filesi from the the specified GRD""" - all_xtb_file_tags = lxml.etree.parse(grd_file_path).findall('.//translations/file') - xtb_files = [] - for xtb_file_tag in all_xtb_file_tags: - lang = xtb_file_tag.get('lang') - path = xtb_file_tag.get('path') - pair = (lang, path) - xtb_files.append(pair) - return xtb_files + """Obtains all the XTB filesi from the the specified GRD""" + all_xtb_file_tags = ( + lxml.etree.parse(grd_file_path).findall('.//translations/file')) + xtb_files = [] + for xtb_file_tag in all_xtb_file_tags: + lang = xtb_file_tag.get('lang') + path = xtb_file_tag.get('path') + pair = (lang, path) + xtb_files.append(pair) + return xtb_files def get_original_grd(src_root, grd_file_path): - """Obtains the Chromium GRD file for a specified Brave GRD file.""" - grd_file_name = os.path.basename(grd_file_path) - if grd_file_name == 'components_brave_strings.grd': - return os.path.join(src_root, 'components', 'components_chromium_strings.grd') - elif grd_file_name == 'brave_strings.grd': - return os.path.join(src_root, 'chrome', 'app', 'chromium_strings.grd') - elif grd_file_name == 'generated_resources.grd': - return os.path.join(src_root, 'chrome', 'app', 'generated_resources.grd') + """Obtains the Chromium GRD file for a specified Brave GRD file.""" + grd_file_name = os.path.basename(grd_file_path) + if grd_file_name == 'components_brave_strings.grd': + return os.path.join(src_root, 'components', + 'components_chromium_strings.grd') + elif grd_file_name == 'brave_strings.grd': + return os.path.join(src_root, 'chrome', 'app', 'chromium_strings.grd') + elif grd_file_name == 'generated_resources.grd': + return os.path.join(src_root, 'chrome', 'app', + 'generated_resources.grd') def check_for_chromium_upgrade_extra_langs(src_root, grd_file_path): - """Checks the Brave GRD file vs the Chromium GRD file for extra languages.""" - chromium_grd_file_path = get_original_grd(src_root, grd_file_path) - if not chromium_grd_file_path: - return - brave_langs = get_transifex_languages(grd_file_path) - chromium_langs = get_transifex_languages(chromium_grd_file_path) - x_brave_extra_langs = brave_langs - chromium_langs - assert len(x_brave_extra_langs) == 0, ('Brave GRD %s has extra languages %s over Chromium GRD %s' % - (grd_file_path, chromium_grd_file_path, list(x_brave_extra_langs))) - x_chromium_extra_langs = chromium_langs - brave_langs - assert len(x_chromium_extra_langs) == 0, ('Chromium GRD %s has extra languages %s over Brave GRD %s' % - (chromium_grd_file_path, grd_file_path, list(x_chromium_extra_langs))) + """Checks the Brave GRD file vs the Chromium GRD file for extra + languages.""" + chromium_grd_file_path = get_original_grd(src_root, grd_file_path) + if not chromium_grd_file_path: + return + brave_langs = get_transifex_languages(grd_file_path) + chromium_langs = get_transifex_languages(chromium_grd_file_path) + x_brave_extra_langs = brave_langs - chromium_langs + assert len(x_brave_extra_langs) == 0, ( + 'Brave GRD %s has extra languages %s over Chromium GRD %s' % ( + grd_file_path, chromium_grd_file_path, + list(x_brave_extra_langs))) + x_chromium_extra_langs = chromium_langs - brave_langs + assert len(x_chromium_extra_langs) == 0, ( + 'Chromium GRD %s has extra languages %s over Brave GRD %s' % ( + chromium_grd_file_path, grd_file_path, + list(x_chromium_extra_langs))) def check_for_chromium_missing_grd_strings(src_root, grd_file_path): - """Checks to make sure Brave GRD file vs the Chromium GRD has the same amount of strings.""" - chromium_grd_file_path = get_original_grd(src_root, grd_file_path) - if not chromium_grd_file_path: - return - grd_strings = get_grd_strings(grd_file_path) - chromium_grd_strings = get_grd_strings(chromium_grd_file_path) - brave_strings = { string_name for (string_name, message_value, string_fp) in grd_strings } - chromium_strings = { string_name for (string_name, message_value, string_fp) in chromium_grd_strings } - x_brave_extra_strings = brave_strings - chromium_strings - assert len(x_brave_extra_strings) == 0, ('Brave GRD %s has extra strings %s over Chromium GRD %s' % - (grd_file_path, chromium_grd_file_path, list(x_brave_extra_strings))) - x_chromium_extra_strings = chromium_strings - brave_strings - assert len(x_chromium_extra_strings) == 0, ('Chromium GRD %s has extra strings %s over Brave GRD %s' % - (chromium_grd_file_path, grd_file_path, list(x_chromium_extra_strings))) + """Checks to make sure Brave GRD file vs the Chromium GRD has the same + amount of strings.""" + chromium_grd_file_path = get_original_grd(src_root, grd_file_path) + if not chromium_grd_file_path: + return + grd_strings = get_grd_strings(grd_file_path) + chromium_grd_strings = get_grd_strings(chromium_grd_file_path) + brave_strings = {string_name for ( + string_name, message_value, string_fp) in grd_strings} + chromium_strings = {string_name for ( + string_name, message_value, string_fp) in chromium_grd_strings} + x_brave_extra_strings = brave_strings - chromium_strings + assert len(x_brave_extra_strings) == 0, ( + 'Brave GRD %s has extra strings %s over Chromium GRD %s' % ( + grd_file_path, chromium_grd_file_path, + list(x_brave_extra_strings))) + x_chromium_extra_strings = chromium_strings - brave_strings + assert len(x_chromium_extra_strings) == 0, ( + 'Chromium GRD %s has extra strings %s over Brave GRD %s' % ( + chromium_grd_file_path, grd_file_path, + list(x_chromium_extra_strings))) def get_transifex_string_hash(string_name): - """Obains transifex string hash for the passed string.""" - key = string_name.encode('utf-8') - return str(md5(':'.join([key,''])).hexdigest()) + """Obains transifex string hash for the passed string.""" + key = string_name.encode('utf-8') + return str(md5(':'.join([key, ''])).hexdigest()) def braveify(string_value): - """Replace Chromium branded strings with Brave beranded strings.""" - return (string_value.replace('Chrome', 'Brave') - .replace('Chromium', 'Brave') - .replace('Google', 'Brave Software')) + """Replace Chromium branded strings with Brave beranded strings.""" + return (string_value.replace('Chrome', 'Brave') + .replace('Chromium', 'Brave') + .replace('Google', 'Brave Software')) -def upload_missing_translation_to_transifex(source_string_path, lang_code, filename, string_name, string_value, translated_value): - """Uploads the specified string to the specified language code.""" - url_part = 'project/%s/resource/%s/translation/%s/string/%s/' % (transifex_project_name, transifex_name_from_filename(source_string_path, filename), lang_code, get_transifex_string_hash(string_name)) - url = base_url + url_part - translated_value = braveify(translated_value) - payload = { - 'translation': translated_value, - # Assume Chromium provided strings are reviewed and proofread - 'reviewed': True, - 'proofread': True, - 'user': 'bbondy' - } - headers = { 'Content-Type': 'application/json' } - r = requests.put(url, json=payload, auth=get_auth(), headers=headers) - assert r.status_code >= 200 and r.status_code <= 299, 'Aborting. Status code %d: %s' % (r.status_code, r.content) - print 'Uploaded %s string: %s -- %s...' % (lang_code, string_name, translated_value[:12].encode('utf-8')) - return True +def upload_missing_translation_to_transifex(source_string_path, lang_code, + filename, string_name, + string_value, translated_value): + """Uploads the specified string to the specified language code.""" + url_part = 'project/%s/resource/%s/translation/%s/string/%s/' % ( + transifex_project_name, transifex_name_from_filename( + source_string_path, filename), lang_code, + get_transifex_string_hash(string_name)) + url = base_url + url_part + translated_value = braveify(translated_value) + payload = { + 'translation': translated_value, + # Assume Chromium provided strings are reviewed and proofread + 'reviewed': True, + 'proofread': True, + 'user': 'bbondy' + } + headers = {'Content-Type': 'application/json'} + r = requests.put(url, json=payload, auth=get_auth(), headers=headers) + assert r.status_code >= 200 and r.status_code <= 299, ( + 'Aborting. Status code %d: %s' % (r.status_code, r.content)) + print 'Uploaded %s string: %s -- %s...' % ( + lang_code, string_name, translated_value[:12].encode('utf-8')) + return True -def upload_missing_translations_to_transifex(source_string_path, lang_code, filename, grd_strings, chromium_grd_strings, xtb_strings, chromium_xtb_strings): - """For each chromium translation that we don't know about, upload it.""" - lang_code = lang_code.replace('-', '_') - for idx, (string_name, string_value, string_fp) in enumerate(grd_strings): - string_fp = str(string_fp) - chromium_string_fp = str(chromium_grd_strings[idx][2]) - if chromium_string_fp in chromium_xtb_strings and string_fp not in xtb_strings: - #print 'Uploading for locale %s for missing string ID: %s' % (lang_code, string_name) - upload_missing_translation_to_transifex(source_string_path, lang_code, filename, string_name, string_value, chromium_xtb_strings[chromium_string_fp]) +def upload_missing_translations_to_transifex(source_string_path, lang_code, + filename, grd_strings, + chromium_grd_strings, xtb_strings, + chromium_xtb_strings): + """For each chromium translation that we don't know about, upload it.""" + lang_code = lang_code.replace('-', '_') + for idx, (string_name, string_value, string_fp) in enumerate(grd_strings): + string_fp = str(string_fp) + chromium_string_fp = str(chromium_grd_strings[idx][2]) + if chromium_string_fp in chromium_xtb_strings and ( + string_fp not in xtb_strings): + # print 'Uploading for locale %s for missing ' + # 'string ID: %s' % (lang_code, string_name) + upload_missing_translation_to_transifex( + source_string_path, lang_code, filename, string_name, + string_value, chromium_xtb_strings[chromium_string_fp]) -def fix_missing_xtb_strings_from_chromium_xtb_strings(src_root, grd_file_path): - """Checks to make sure Brave GRD file vs the Chromium GRD has the same amount of strings.""" - chromium_grd_file_path = get_original_grd(src_root, grd_file_path) - if not chromium_grd_file_path: - return +def fix_missing_xtb_strings_from_chromium_xtb_strings( + src_root, grd_file_path): + """Checks to make sure Brave GRD file vs the Chromium GRD has the same + amount of strings.""" + chromium_grd_file_path = get_original_grd(src_root, grd_file_path) + if not chromium_grd_file_path: + return - grd_base_path = os.path.dirname(grd_file_path) - chromium_grd_base_path = os.path.dirname(chromium_grd_file_path) + grd_base_path = os.path.dirname(grd_file_path) + chromium_grd_base_path = os.path.dirname(chromium_grd_file_path) - filename = os.path.basename(grd_file_path).split('.')[0] - grd_strings = get_grd_strings(grd_file_path) - chromium_grd_strings = get_grd_strings(chromium_grd_file_path) + filename = os.path.basename(grd_file_path).split('.')[0] + grd_strings = get_grd_strings(grd_file_path) + chromium_grd_strings = get_grd_strings(chromium_grd_file_path) - xtb_files = get_xtb_files(grd_file_path) - chromium_xtb_files = get_xtb_files(chromium_grd_file_path) - xtb_file_paths = [os.path.join(grd_base_path, path) for (lang, path) in xtb_files] - chromium_xtb_file_paths = [os.path.join(chromium_grd_base_path, path) for (lang, path) in chromium_xtb_files] - langs = [lang for (lang, path) in xtb_files] + xtb_files = get_xtb_files(grd_file_path) + chromium_xtb_files = get_xtb_files(chromium_grd_file_path) + xtb_file_paths = [os.path.join( + grd_base_path, path) for (lang, path) in xtb_files] + chromium_xtb_file_paths = [ + os.path.join(chromium_grd_base_path, path) for + (lang, path) in chromium_xtb_files] + langs = [lang for (lang, path) in xtb_files] - for idx, xtb_file in enumerate(xtb_file_paths): - chromium_xtb_file = chromium_xtb_file_paths[idx] - lang_code = langs[idx] - xtb_strings = get_strings_dict_from_xtb_file(xtb_file) - chromium_xtb_strings = get_strings_dict_from_xtb_file(chromium_xtb_file) - assert(len(grd_strings) == len(chromium_grd_strings)) - upload_missing_translations_to_transifex(grd_file_path, lang_code, filename, grd_strings, chromium_grd_strings, xtb_strings, chromium_xtb_strings) + for idx, xtb_file in enumerate(xtb_file_paths): + chromium_xtb_file = chromium_xtb_file_paths[idx] + lang_code = langs[idx] + xtb_strings = get_strings_dict_from_xtb_file(xtb_file) + chromium_xtb_strings = get_strings_dict_from_xtb_file( + chromium_xtb_file) + assert(len(grd_strings) == len(chromium_grd_strings)) + upload_missing_translations_to_transifex( + grd_file_path, lang_code, filename, grd_strings, + chromium_grd_strings, xtb_strings, chromium_xtb_strings) def check_for_chromium_upgrade(src_root, grd_file_path): - """Performs various checks and changes as needed for when Chromium source files change.""" - check_for_chromium_upgrade_extra_langs(src_root, grd_file_path) - check_for_chromium_missing_grd_strings(src_root, grd_file_path) - fix_missing_xtb_strings_from_chromium_xtb_strings(src_root, grd_file_path) + """Performs various checks and changes as needed for when Chromium source + files change.""" + check_for_chromium_upgrade_extra_langs(src_root, grd_file_path) + check_for_chromium_missing_grd_strings(src_root, grd_file_path) + fix_missing_xtb_strings_from_chromium_xtb_strings(src_root, grd_file_path) def get_transifex_source_resource_strings(grd_file_path): - """Obtains the list of strings from Transifex""" - filename = os.path.basename(grd_file_path).split('.')[0] - url_part = 'project/%s/resource/%s/content/' % (transifex_project_name, transifex_name_from_filename(grd_file_path, filename)) - url = base_url + url_part - r = requests.get(url, auth=get_auth()) - assert r.status_code >= 200 and r.status_code <= 299, 'Aborting. Status code %d: %s' % (r.status_code, r.content) - return get_strings_dict_from_xml_content(r.json()['content'].encode('utf-8')) + """Obtains the list of strings from Transifex""" + filename = os.path.basename(grd_file_path).split('.')[0] + url_part = ( + 'project/%s/resource/%s/content/' % ( + transifex_project_name, + transifex_name_from_filename(grd_file_path, filename))) + url = base_url + url_part + r = requests.get(url, auth=get_auth()) + assert r.status_code >= 200 and r.status_code <= 299, ( + 'Aborting. Status code %d: %s' % (r.status_code, r.content)) + return get_strings_dict_from_xml_content( + r.json()['content'].encode('utf-8')) def check_missing_source_grd_strings_to_transifex(grd_file_path): - """Compares the GRD strings to the strings on Transifex and uploads any missing strings.""" - source_grd_strings = get_grd_strings(grd_file_path) - if len(source_grd_strings) == 0: - return - strings_dict = get_transifex_source_resource_strings(grd_file_path) - transifex_string_ids = set(strings_dict.keys()) - grd_strings_tuple = get_grd_strings(grd_file_path) - grd_string_names = { string_name for (string_name, message_value, string_fp) in grd_strings_tuple } - filename = os.path.basename(grd_file_path).split('.')[0] - x_grd_extra_strings = grd_string_names - transifex_string_ids - assert len(x_grd_extra_strings) == 0, ('GRD has extra strings over Transifex %' % - list(x_grd_extra_strings)) - x_transifex_extra_strings = transifex_string_ids - grd_string_names - assert len(x_transifex_extra_strings) == 0, ('Transifex has extra strings over GRD %s' % - list(x_transifex_extra_strings)) + """Compares the GRD strings to the strings on Transifex and uploads any + missing strings.""" + source_grd_strings = get_grd_strings(grd_file_path) + if len(source_grd_strings) == 0: + return + strings_dict = get_transifex_source_resource_strings(grd_file_path) + transifex_string_ids = set(strings_dict.keys()) + grd_strings_tuple = get_grd_strings(grd_file_path) + grd_string_names = {string_name for (string_name, message_value, + string_fp) in grd_strings_tuple} + filename = os.path.basename(grd_file_path).split('.')[0] + x_grd_extra_strings = grd_string_names - transifex_string_ids + assert len(x_grd_extra_strings) == 0, ( + 'GRD has extra strings over Transifex %' % + list(x_grd_extra_strings)) + x_transifex_extra_strings = transifex_string_ids - grd_string_names + assert len(x_transifex_extra_strings) == 0, ( + 'Transifex has extra strings over GRD %s' % + list(x_transifex_extra_strings)) def upload_source_files_to_transifex(source_file_path, filename): - uploaded = False - i18n_type = '' - content = '' - ext = os.path.splitext(source_file_path)[1] - if ext == '.grd': - # Generate the intermediate Transifex format for the source translations - output_xml_file_handle, output_xml_path = tempfile.mkstemp('.xml') - content = generate_source_strings_xml_from_grd(output_xml_file_handle, source_file_path) - os.close(output_xml_file_handle) - i18n_type = 'ANDROID' - elif ext == '.json': - i18n_type = 'CHROME' - with io.open(source_file_path, mode='r', encoding='utf-8') as json_file: - content = json_file.read() - else: - assert False, 'Unsupported source file ext %s: %s' % (ext, source_file_path) + uploaded = False + i18n_type = '' + content = '' + ext = os.path.splitext(source_file_path)[1] + if ext == '.grd': + # Generate the intermediate Transifex format for the source + # translations. + output_xml_file_handle, output_xml_path = tempfile.mkstemp('.xml') + content = generate_source_strings_xml_from_grd(output_xml_file_handle, + source_file_path) + os.close(output_xml_file_handle) + i18n_type = 'ANDROID' + elif ext == '.json': + i18n_type = 'CHROME' + with io.open(source_file_path, mode='r', + encoding='utf-8') as json_file: + content = json_file.read() + else: + assert False, 'Unsupported source file ext %s: %s' % ( + ext, source_file_path) - uploaded = upload_source_string_file_to_transifex(source_file_path, filename, content, i18n_type) - assert uploaded, 'Could not upload xml file' + uploaded = upload_source_string_file_to_transifex(source_file_path, + filename, content, + i18n_type) + assert uploaded, 'Could not upload xml file' def pull_source_files_from_transifex(source_file_path, filename): - ext = os.path.splitext(source_file_path)[1] - if ext == '.grd': - # Generate the intermediate Transifex format - xtb_files = get_xtb_files(source_file_path) - base_path = os.path.dirname(source_file_path) - grd_strings = get_grd_strings(source_file_path) - for (lang_code, xtb_rel_path) in xtb_files: - xtb_file_path = os.path.join(base_path, xtb_rel_path) - print 'Updating: ', xtb_file_path, lang_code - xml_content = get_transifex_translation_file_content(source_file_path, filename, lang_code) - translations = get_strings_dict_from_xml_content(xml_content) - xtb_content = generate_xtb_content(lang_code, grd_strings, translations) - with open(xtb_file_path, mode='w') as f: - f.write(xtb_content) - elif ext == '.json': - langs_dir_path = os.path.dirname(os.path.dirname(source_file_path)) - lang_codes = set(os.listdir(langs_dir_path)) - lang_codes.discard('en_US') - lang_codes.discard('.DS_Store') - for lang_code in lang_codes: - print 'getting filename %s for lang_code %s' % (filename, lang_code) - content = get_transifex_translation_file_content(source_file_path, filename, lang_code) - localized_translation_path = os.path.join(langs_dir_path, lang_code, 'messages.json') - with open(localized_translation_path, mode='w') as f: - f.write(content) + ext = os.path.splitext(source_file_path)[1] + if ext == '.grd': + # Generate the intermediate Transifex format + xtb_files = get_xtb_files(source_file_path) + base_path = os.path.dirname(source_file_path) + grd_strings = get_grd_strings(source_file_path) + for (lang_code, xtb_rel_path) in xtb_files: + xtb_file_path = os.path.join(base_path, xtb_rel_path) + print 'Updating: ', xtb_file_path, lang_code + xml_content = get_transifex_translation_file_content( + source_file_path, filename, lang_code) + translations = get_strings_dict_from_xml_content(xml_content) + xtb_content = generate_xtb_content(lang_code, grd_strings, + translations) + with open(xtb_file_path, mode='w') as f: + f.write(xtb_content) + elif ext == '.json': + langs_dir_path = os.path.dirname(os.path.dirname(source_file_path)) + lang_codes = set(os.listdir(langs_dir_path)) + lang_codes.discard('en_US') + lang_codes.discard('.DS_Store') + for lang_code in lang_codes: + print 'getting filename %s for lang_code %s' % (filename, + lang_code) + content = get_transifex_translation_file_content(source_file_path, + filename, + lang_code) + localized_translation_path = os.path.join(langs_dir_path, + lang_code, + 'messages.json') + with open(localized_translation_path, mode='w') as f: + f.write(content) diff --git a/script/lib/util.py b/script/lib/util.py index 1231e86d98b..bfcc01475dd 100644 --- a/script/lib/util.py +++ b/script/lib/util.py @@ -23,214 +23,218 @@ BOTO_DIR = os.path.abspath(os.path.join(__file__, '..', '..', '..', 'vendor', def get_host_arch(): - """Returns the host architecture with a predictable string.""" - host_arch = platform.machine() + """Returns the host architecture with a predictable string.""" + host_arch = platform.machine() - # Convert machine type to format recognized by gyp. - if re.match(r'i.86', host_arch) or host_arch == 'i86pc': - host_arch = 'ia32' - elif host_arch in ['x86_64', 'amd64']: - host_arch = 'x64' - elif host_arch.startswith('arm'): - host_arch = 'arm' + # Convert machine type to format recognized by gyp. + if re.match(r'i.86', host_arch) or host_arch == 'i86pc': + host_arch = 'ia32' + elif host_arch in ['x86_64', 'amd64']: + host_arch = 'x64' + elif host_arch.startswith('arm'): + host_arch = 'arm' - # platform.machine is based on running kernel. It's possible to use 64-bit - # kernel with 32-bit userland, e.g. to give linker slightly more memory. - # Distinguish between different userland bitness by querying - # the python binary. - if host_arch == 'x64' and platform.architecture()[0] == '32bit': - host_arch = 'ia32' + # platform.machine is based on running kernel. It's possible to use 64-bit + # kernel with 32-bit userland, e.g. to give linker slightly more memory. + # Distinguish between different userland bitness by querying + # the python binary. + if host_arch == 'x64' and platform.architecture()[0] == '32bit': + host_arch = 'ia32' - return host_arch + return host_arch def tempdir(prefix=''): - directory = tempfile.mkdtemp(prefix=prefix) - atexit.register(shutil.rmtree, directory) - return directory + directory = tempfile.mkdtemp(prefix=prefix) + atexit.register(shutil.rmtree, directory) + return directory @contextlib.contextmanager def scoped_cwd(path): - cwd = os.getcwd() - os.chdir(path) - try: - yield - finally: - os.chdir(cwd) + cwd = os.getcwd() + os.chdir(path) + try: + yield + finally: + os.chdir(cwd) @contextlib.contextmanager def scoped_env(key, value): - origin = '' - if key in os.environ: - origin = os.environ[key] - os.environ[key] = value - try: - yield - finally: - os.environ[key] = origin + origin = '' + if key in os.environ: + origin = os.environ[key] + os.environ[key] = value + try: + yield + finally: + os.environ[key] = origin def download(text, url, path): - safe_mkdir(os.path.dirname(path)) - with open(path, 'wb') as local_file: - if hasattr(ssl, '_create_unverified_context'): - ssl._create_default_https_context = ssl._create_unverified_context + safe_mkdir(os.path.dirname(path)) + with open(path, 'wb') as local_file: + if hasattr(ssl, '_create_unverified_context'): + ssl._create_default_https_context = ssl._create_unverified_context - web_file = urllib2.urlopen(url) - file_size = int(web_file.info().getheaders("Content-Length")[0]) - downloaded_size = 0 - block_size = 128 + web_file = urllib2.urlopen(url) + file_size = int(web_file.info().getheaders("Content-Length")[0]) + downloaded_size = 0 + block_size = 128 - ci = os.environ.get('CI') == '1' + ci = os.environ.get('CI') == '1' - while True: - buf = web_file.read(block_size) - if not buf: - break + while True: + buf = web_file.read(block_size) + if not buf: + break - downloaded_size += len(buf) - local_file.write(buf) + downloaded_size += len(buf) + local_file.write(buf) - if not ci: - percent = downloaded_size * 100. / file_size - status = "\r%s %10d [%3.1f%%]" % (text, downloaded_size, percent) - print status, + if not ci: + percent = downloaded_size * 100. / file_size + status = "\r%s %10d [%3.1f%%]" % ( + text, downloaded_size, percent) + print status, - if ci: - print "%s done." % (text) - else: - print - return path + if ci: + print "%s done." % (text) + else: + print + return path def extract_tarball(tarball_path, member, destination): - with tarfile.open(tarball_path) as tarball: - tarball.extract(member, destination) + with tarfile.open(tarball_path) as tarball: + tarball.extract(member, destination) def extract_zip(zip_path, destination): - if sys.platform == 'darwin': - # Use unzip command on Mac to keep symbol links in zip file work. - execute(['unzip', zip_path, '-d', destination]) - else: - with zipfile.ZipFile(zip_path) as z: - z.extractall(destination) + if sys.platform == 'darwin': + # Use unzip command on Mac to keep symbol links in zip file work. + execute(['unzip', zip_path, '-d', destination]) + else: + with zipfile.ZipFile(zip_path) as z: + z.extractall(destination) + def make_zip(zip_file_path, files, dirs): - safe_unlink(zip_file_path) - if sys.platform == 'darwin': - files += dirs - execute(['zip', '-r', '-y', zip_file_path] + files) - else: - zip_file = zipfile.ZipFile(zip_file_path, "w", zipfile.ZIP_DEFLATED, allowZip64 = True) - for filename in files: - zip_file.write(filename, filename) - for dirname in dirs: - for root, _, filenames in os.walk(dirname): - for f in filenames: - zip_file.write(os.path.join(root, f)) - zip_file.close() + safe_unlink(zip_file_path) + if sys.platform == 'darwin': + files += dirs + execute(['zip', '-r', '-y', zip_file_path] + files) + else: + zip_file = zipfile.ZipFile(zip_file_path, "w", zipfile.ZIP_DEFLATED, + allowZip64=True) + for filename in files: + zip_file.write(filename, filename) + for dirname in dirs: + for root, _, filenames in os.walk(dirname): + for f in filenames: + zip_file.write(os.path.join(root, f)) + zip_file.close() def rm_rf(path): - try: - shutil.rmtree(path) - except OSError: - pass + try: + shutil.rmtree(path) + except OSError: + pass def safe_unlink(path): - try: - os.unlink(path) - except OSError as e: - if e.errno != errno.ENOENT: - raise + try: + os.unlink(path) + except OSError as e: + if e.errno != errno.ENOENT: + raise def safe_mkdir(path): - try: - os.makedirs(path) - except OSError as e: - if e.errno != errno.EEXIST: - raise + try: + os.makedirs(path) + except OSError as e: + if e.errno != errno.EEXIST: + raise def execute(argv, env=os.environ): - if is_verbose_mode(): - print ' '.join(argv) - try: - output = subprocess.check_output(argv, stderr=subprocess.STDOUT, env=env) if is_verbose_mode(): - print output - return output - except subprocess.CalledProcessError as e: - print e.output - raise e + print ' '.join(argv) + try: + output = subprocess.check_output(argv, stderr=subprocess.STDOUT, + env=env) + if is_verbose_mode(): + print output + return output + except subprocess.CalledProcessError as e: + print e.output + raise e def execute_stdout(argv, env=os.environ): - if is_verbose_mode(): - print ' '.join(argv) - try: - subprocess.check_call(argv, env=env) - except subprocess.CalledProcessError as e: - print e.output - raise e - else: - execute(argv, env) + if is_verbose_mode(): + print ' '.join(argv) + try: + subprocess.check_call(argv, env=env) + except subprocess.CalledProcessError as e: + print e.output + raise e + else: + execute(argv, env) def parse_version(version): - if version[0] == 'v': - version = version[1:] + if version[0] == 'v': + version = version[1:] - vs = version.split('+')[0] - vs = vs.split('.') + vs = version.split('+')[0] + vs = vs.split('.') - if len(version.split('+')) == 2: - vs = vs + [version.split('+')[1]] + if len(version.split('+')) == 2: + vs = vs + [version.split('+')[1]] - return vs + return vs def boto_path_dirs(): - return [ - os.path.join(BOTO_DIR, 'build', 'lib'), - os.path.join(BOTO_DIR, 'build', 'lib.linux-x86_64-2.7') - ] + return [ + os.path.join(BOTO_DIR, 'build', 'lib'), + os.path.join(BOTO_DIR, 'build', 'lib.linux-x86_64-2.7') + ] def run_boto_script(access_key, secret_key, script_name, *args): - env = os.environ.copy() - env['AWS_ACCESS_KEY_ID'] = access_key - env['AWS_SECRET_ACCESS_KEY'] = secret_key - env['PYTHONPATH'] = os.path.pathsep.join( - [env.get('PYTHONPATH', '')] + boto_path_dirs()) + env = os.environ.copy() + env['AWS_ACCESS_KEY_ID'] = access_key + env['AWS_SECRET_ACCESS_KEY'] = secret_key + env['PYTHONPATH'] = os.path.pathsep.join( + [env.get('PYTHONPATH', '')] + boto_path_dirs()) - boto = os.path.join(BOTO_DIR, 'bin', script_name) - execute([sys.executable, boto] + list(args), env) + boto = os.path.join(BOTO_DIR, 'bin', script_name) + execute([sys.executable, boto] + list(args), env) def s3put(bucket, access_key, secret_key, prefix, key_prefix, files): - args = [ - '--bucket', bucket, - '--prefix', prefix, - '--key_prefix', key_prefix, - '--grant', 'public-read' - ] + files + args = [ + '--bucket', bucket, + '--prefix', prefix, + '--key_prefix', key_prefix, + '--grant', 'public-read' + ] + files - run_boto_script(access_key, secret_key, 's3put', *args) + run_boto_script(access_key, secret_key, 's3put', *args) def import_vs_env(target_arch): - if sys.platform != 'win32': - return + if sys.platform != 'win32': + return - if target_arch == 'ia32': - vs_arch = 'amd64_x86' - else: - vs_arch = 'x86_amd64' - env = get_vs_env('14.0', vs_arch) - os.environ.update(env) + if target_arch == 'ia32': + vs_arch = 'amd64_x86' + else: + vs_arch = 'x86_amd64' + env = get_vs_env('14.0', vs_arch) + os.environ.update(env) diff --git a/script/test/mock.py b/script/test/mock.py index ab46e0ee25f..8898a03663e 100644 --- a/script/test/mock.py +++ b/script/test/mock.py @@ -3,16 +3,19 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this file, # You can obtain one at http://mozilla.org/MPL/2.0/. + class MockImport(object): - def __import__(self): - pass + def __import__(self): + pass + class Repo(): - def __init__(self): - self.releases = self.Releases() - - class Releases(): def __init__(self): - self._releases = [] - def get(self): - return self._releases + self.releases = self.Releases() + + class Releases(): + def __init__(self): + self._releases = [] + + def get(self): + return self._releases diff --git a/script/test/test_helpers.py b/script/test/test_helpers.py index a97579e55dd..848920c10b9 100644 --- a/script/test/test_helpers.py +++ b/script/test/test_helpers.py @@ -8,72 +8,77 @@ import sys import unittest import os -dirname = os.path.dirname(os.path.realpath(__file__)) -sys.path.append(os.path.join(dirname, '..')) from lib.helpers import * + +dirname = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(os.path.join(dirname, '..')) + + class RetryFunc(): - def __init__(self): - self.ran = 0 - self.calls = [] - self.err = UserWarning + def __init__(self): + self.ran = 0 + self.calls = [] + self.err = UserWarning - def succeed(self, count): - self.ran = self.ran + 1 - self.calls.append(count) + def succeed(self, count): + self.ran = self.ran + 1 + self.calls.append(count) + + def fail(self, count): + self.ran = self.ran + 1 + self.calls.append(count) + raise self.err - def fail(self, count): - self.ran = self.ran + 1 - self.calls.append(count) - raise self.err class TestRetryFunc(unittest.TestCase): - def setUp(self): - self.retry_func = RetryFunc() - self.catch_func = RetryFunc() + def setUp(self): + self.retry_func = RetryFunc() + self.catch_func = RetryFunc() - def test_passes_retry_count(self): - self.assertRaises( - self.retry_func.err, - retry_func, - self.retry_func.fail, - catch=UserWarning, retries=3 - ) - self.assertEqual(self.retry_func.calls, [0, 1, 2, 3]) + def test_passes_retry_count(self): + self.assertRaises( + self.retry_func.err, + retry_func, + self.retry_func.fail, + catch=UserWarning, retries=3 + ) + self.assertEqual(self.retry_func.calls, [0, 1, 2, 3]) - def test_retries_on_fail(self): - self.assertRaises( - self.retry_func.err, - retry_func, - self.retry_func.fail, - catch=UserWarning, retries=3 - ) - self.assertEqual(self.retry_func.ran, 4) + def test_retries_on_fail(self): + self.assertRaises( + self.retry_func.err, + retry_func, + self.retry_func.fail, + catch=UserWarning, retries=3 + ) + self.assertEqual(self.retry_func.ran, 4) - def test_run_catch_func_on_fail(self): - self.assertRaises( - self.retry_func.err, - retry_func, - self.retry_func.fail, - catch_func=self.catch_func.succeed, - catch=UserWarning, retries=3 - ) - self.assertEqual(self.catch_func.ran, 4) + def test_run_catch_func_on_fail(self): + self.assertRaises( + self.retry_func.err, + retry_func, + self.retry_func.fail, + catch_func=self.catch_func.succeed, + catch=UserWarning, retries=3 + ) + self.assertEqual(self.catch_func.ran, 4) - def test_no_retry_on_success(self): - retry_func( - self.retry_func.succeed, - catch=UserWarning, retries=3 - ) - self.assertEqual(self.retry_func.ran, 1) + def test_no_retry_on_success(self): + retry_func( + self.retry_func.succeed, + catch=UserWarning, retries=3 + ) + self.assertEqual(self.retry_func.ran, 1) + + def test_no_run_catch_func_on_success(self): + retry_func( + self.retry_func.succeed, + catch_func=self.catch_func.succeed, + catch=UserWarning, retries=3 + ) + self.assertEqual(self.catch_func.ran, 0) - def test_no_run_catch_func_on_success(self): - retry_func( - self.retry_func.succeed, - catch_func=self.catch_func.succeed, - catch=UserWarning, retries=3 - ) - self.assertEqual(self.catch_func.ran, 0) if __name__ == '__main__': - print unittest.main() + print unittest.main() diff --git a/script/test/test_publish.py b/script/test/test_publish.py index 818d1dac6b7..eabe0ee6b3d 100644 --- a/script/test/test_publish.py +++ b/script/test/test_publish.py @@ -3,9 +3,10 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this file, # You can obtain one at http://mozilla.org/MPL/2.0/. +import os +import publish_release import sys import unittest -import os from mock import MockImport, Repo @@ -15,24 +16,25 @@ sys.modules['requests'] = MockImport dirname = os.path.dirname(os.path.realpath(__file__)) sys.path.append(os.path.join(dirname, '..')) -import publish_release - class TestPublishGetDraft(unittest.TestCase): - def setUp(self): - self.repo = Repo() + def setUp(self): + self.repo = Repo() - def test_fails_on_existing_release(self): - self.repo.releases._releases = [{'tag_name': 'test', 'draft': False}] - self.assertRaises(UserWarning, publish_release.get_draft, self.repo, 'test') + def test_fails_on_existing_release(self): + self.repo.releases._releases = [{'tag_name': 'test', 'draft': False}] + self.assertRaises(UserWarning, publish_release.get_draft, self.repo, + 'test') - def test_fails_on_no_draft(self): - self.repo.releases._releases = [{'tag_name': 'old', 'draft': False}] - self.assertRaises(UserWarning, publish_release.get_draft, self.repo, 'new') + def test_fails_on_no_draft(self): + self.repo.releases._releases = [{'tag_name': 'old', 'draft': False}] + self.assertRaises(UserWarning, publish_release.get_draft, self.repo, + 'new') + + def test_succeeds_on_existing_draft(self): + self.repo.releases._releases = [{'tag_name': 'test', 'draft': True}] + publish_release.get_draft(self.repo, 'test') - def test_succeeds_on_existing_draft(self): - self.repo.releases._releases = [{'tag_name': 'test', 'draft': True}] - publish_release.get_draft(self.repo, 'test') if __name__ == '__main__': - print unittest.main() + print unittest.main() diff --git a/script/test/test_upload.py b/script/test/test_upload.py index 84dcd34e417..89a6ccb2af3 100644 --- a/script/test/test_upload.py +++ b/script/test/test_upload.py @@ -7,117 +7,159 @@ import sys import unittest import os +import upload +from mock import Repo dirname = os.path.dirname(os.path.realpath(__file__)) sys.path.append(os.path.join(dirname, '..')) -import upload - -from mock import Repo class TestGetDraft(unittest.TestCase): - def setUp(self): - self.repo = Repo() + def setUp(self): + self.repo = Repo() - def test_returns_existing_draft(self): - self.repo.releases._releases = [{'tag_name': 'test', 'draft': True}] - self.assertEquals(upload.get_draft(self.repo, 'test')['tag_name'], 'test') + def test_returns_existing_draft(self): + self.repo.releases._releases = [{'tag_name': 'test', 'draft': True}] + self.assertEquals(upload.get_draft(self.repo, + 'test')['tag_name'], 'test') - def test_fails_on_existing_release(self): - self.repo.releases._releases = [{'tag_name': 'test', 'draft': False}] - self.assertRaises(UserWarning, upload.get_draft, self.repo, 'test') + def test_fails_on_existing_release(self): + self.repo.releases._releases = [{'tag_name': 'test', 'draft': False}] + self.assertRaises(UserWarning, upload.get_draft, self.repo, 'test') + + def test_returns_none_on_new_draft(self): + self.repo.releases._releases = [{'tag_name': 'old', 'draft': False}] + upload.get_draft(self.repo, 'new') + self.assertEquals(upload.get_draft(self.repo, 'test'), None) - def test_returns_none_on_new_draft(self): - self.repo.releases._releases = [{'tag_name': 'old', 'draft': False}] - upload.get_draft(self.repo, 'new') - self.assertEquals(upload.get_draft(self.repo, 'test'), None) class TestGetBravePackages(unittest.TestCase): - get_pkgs_dir = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'test_get_pkgs') - _is_setup = False + get_pkgs_dir = os.path.join(os.path.dirname( + os.path.realpath(__file__)), 'test_get_pkgs') + _is_setup = False - def setUp(self): - if not self._is_setup: - for chan in ['Release', 'Dev', 'Beta']: - if chan not in 'Release': - for mode in ['Stub', 'Standalone']: - name = 'BraveBrowser{}{}Setup_70_0_56_8.exe'.format(mode if mode not in 'Stub' else '', chan) - name32 = 'BraveBrowser{}{}Setup32_70_0_56_8.exe'.format(mode if mode not in 'Stub' else '', chan) - with open(os.path.join(self.get_pkgs_dir, 'win32', name), 'w') as f: - f.write(name) - with open(os.path.join(self.get_pkgs_dir, 'win32', name32), 'w') as f: - f.write(name32) - else: - for mode in ['Stub', 'Standalone']: - name = 'BraveBrowser{}Setup_70_0_56_8.exe'.format(mode if mode not in 'Stub' else '') - name32 = 'BraveBrowser{}Setup32_70_0_56_8.exe'.format(mode if mode not in 'Stub' else '') - with open(os.path.join(self.get_pkgs_dir, 'win32', name), 'w') as f: - f.write(name) - with open(os.path.join(self.get_pkgs_dir, 'win32', name32), 'w') as f: - f.write(name32) - self.__class__._is_setup = True + def setUp(self): + if not self._is_setup: + for chan in ['Release', 'Dev', 'Beta']: + if chan not in 'Release': + for mode in ['Stub', 'Standalone']: + name = 'BraveBrowser{}{}Setup_70_0_56_8.exe'.format( + mode if mode not in 'Stub' else '', chan) + name32 = ('BraveBrowser{}{}Setup32_70_0_56_8.exe' + .format(mode if mode not in 'Stub' + else '', chan)) + with open(os.path.join(self.get_pkgs_dir, + 'win32', name), 'w') as f: + f.write(name) + with open(os.path.join(self.get_pkgs_dir, + 'win32', name32), 'w') as f: + f.write(name32) + else: + for mode in ['Stub', 'Standalone']: + name = 'BraveBrowser{}Setup_70_0_56_8.exe'.format( + mode if mode not in 'Stub' else '') + name32 = 'BraveBrowser{}Setup32_70_0_56_8.exe'.format( + mode if mode not in 'Stub' else '') + with open(os.path.join( + self.get_pkgs_dir, 'win32', name), 'w') as f: + f.write(name) + with open(os.path.join( + self.get_pkgs_dir, 'win32', name32), 'w') as f: + f.write(name32) + self.__class__._is_setup = True - def test_only_returns_dev_darwin_package(self): - upload.PLATFORM = 'darwin' - pkgs = list(upload.get_brave_packages(os.path.join(self.get_pkgs_dir, upload.PLATFORM), 'dev', '0.50.8')) - self.assertEquals(pkgs, ['Brave-Browser-Dev.dmg']) + def test_only_returns_dev_darwin_package(self): + upload.PLATFORM = 'darwin' + pkgs = list(upload.get_brave_packages(os.path.join( + self.get_pkgs_dir, upload.PLATFORM), 'dev', '0.50.8')) + self.assertEquals(pkgs, ['Brave-Browser-Dev.dmg']) - def test_only_returns_beta_darwin_package(self): - upload.PLATFORM = 'darwin' - pkgs = list(upload.get_brave_packages(os.path.join(self.get_pkgs_dir, upload.PLATFORM), 'beta', '0.50.8')) - self.assertEquals(pkgs, ['Brave-Browser-Beta.dmg']) + def test_only_returns_beta_darwin_package(self): + upload.PLATFORM = 'darwin' + pkgs = list(upload.get_brave_packages( + os.path.join(self.get_pkgs_dir, upload.PLATFORM), + 'beta', '0.50.8')) + self.assertEquals(pkgs, ['Brave-Browser-Beta.dmg']) - def test_only_returns_release_darwin_package(self): - upload.PLATFORM = 'darwin' - pkgs = list(upload.get_brave_packages(os.path.join(self.get_pkgs_dir, upload.PLATFORM), 'release', '0.50.8')) - self.assertEquals(pkgs, ['Brave-Browser.dmg', 'Brave-Browser.pkg']) + def test_only_returns_release_darwin_package(self): + upload.PLATFORM = 'darwin' + pkgs = list(upload.get_brave_packages( + os.path.join(self.get_pkgs_dir, upload.PLATFORM), + 'release', '0.50.8')) + self.assertEquals(pkgs, ['Brave-Browser.dmg', + 'Brave-Browser.pkg']) - def test_only_returns_dev_linux_packages(self): - upload.PLATFORM = 'linux' - pkgs = list(upload.get_brave_packages(os.path.join(self.get_pkgs_dir, upload.PLATFORM), 'dev', '0.50.8')) - self.assertEquals(sorted(pkgs), sorted(['brave-browser-dev-0.50.8-1.x86_64.rpm', 'brave-browser-dev_0.50.8_amd64.deb'])) + def test_only_returns_dev_linux_packages(self): + upload.PLATFORM = 'linux' + pkgs = list(upload.get_brave_packages(os.path.join(self.get_pkgs_dir, + upload.PLATFORM), + 'dev', '0.50.8')) + self.assertEquals(sorted(pkgs), + sorted(['brave-browser-dev-0.50.8-1.x86_64.rpm', + 'brave-browser-dev_0.50.8_amd64.deb'])) - def test_only_returns_release_linux_packages(self): - upload.PLATFORM = 'linux' - pkgs = list(upload.get_brave_packages(os.path.join(self.get_pkgs_dir, upload.PLATFORM), 'release', '0.50.8')) - self.assertEquals(sorted(pkgs), sorted(['brave-browser-0.50.8-1.x86_64.rpm', 'brave-browser_0.50.8_amd64.deb'])) + def test_only_returns_release_linux_packages(self): + upload.PLATFORM = 'linux' + pkgs = list(upload.get_brave_packages(os.path.join(self.get_pkgs_dir, + upload.PLATFORM), + 'release', '0.50.8')) + self.assertEquals(sorted(pkgs), + sorted(['brave-browser-0.50.8-1.x86_64.rpm', + 'brave-browser_0.50.8_amd64.deb'])) - def test_only_returns_dev_win_x64_package(self): - upload.PLATFORM = 'win32' - os.environ['TARGET_ARCH'] = 'x64' - pkgs = list(upload.get_brave_packages(os.path.join(self.get_pkgs_dir, upload.PLATFORM), 'dev', '0.56.8')) - self.assertEquals(pkgs, ['BraveBrowserStandaloneDevSetup.exe', 'BraveBrowserDevSetup.exe']) + def test_only_returns_dev_win_x64_package(self): + upload.PLATFORM = 'win32' + os.environ['TARGET_ARCH'] = 'x64' + pkgs = list(upload.get_brave_packages(os.path.join( + self.get_pkgs_dir, upload.PLATFORM), 'dev', '0.56.8')) + self.assertEquals(pkgs, ['BraveBrowserStandaloneDevSetup.exe', + 'BraveBrowserDevSetup.exe']) - def test_only_returns_dev_win_ia32_package(self): - upload.PLATFORM = 'win32' - os.environ['TARGET_ARCH'] = 'ia32' - pkgs = list(upload.get_brave_packages(os.path.join(self.get_pkgs_dir, upload.PLATFORM), 'dev', '0.56.8')) - self.assertEquals(pkgs, ['BraveBrowserStandaloneDevSetup32.exe', 'BraveBrowserDevSetup32.exe']) + def test_only_returns_dev_win_ia32_package(self): + upload.PLATFORM = 'win32' + os.environ['TARGET_ARCH'] = 'ia32' + pkgs = list(upload.get_brave_packages( + os.path.join(self.get_pkgs_dir, upload.PLATFORM), 'dev', '0.56.8')) + self.assertEquals(pkgs, ['BraveBrowserStandaloneDevSetup32.exe', + 'BraveBrowserDevSetup32.exe']) - def test_only_returns_beta_win_x64_package(self): - upload.PLATFORM = 'win32' - os.environ['TARGET_ARCH'] = 'x64' - pkgs = list(upload.get_brave_packages(os.path.join(self.get_pkgs_dir, upload.PLATFORM), 'beta', '0.56.8')) - self.assertEquals(pkgs, ['BraveBrowserStandaloneBetaSetup.exe', 'BraveBrowserBetaSetup.exe']) + def test_only_returns_beta_win_x64_package(self): + upload.PLATFORM = 'win32' + os.environ['TARGET_ARCH'] = 'x64' + pkgs = list(upload.get_brave_packages( + os.path.join(self.get_pkgs_dir, upload.PLATFORM), + 'beta', '0.56.8')) + self.assertEquals(pkgs, ['BraveBrowserStandaloneBetaSetup.exe', + 'BraveBrowserBetaSetup.exe']) - def test_only_returns_beta_win_ia32_package(self): - upload.PLATFORM = 'win32' - os.environ['TARGET_ARCH'] = 'ia32' - pkgs = list(upload.get_brave_packages(os.path.join(self.get_pkgs_dir, upload.PLATFORM), 'beta', '0.56.8')) - self.assertEquals(pkgs, ['BraveBrowserStandaloneBetaSetup32.exe', 'BraveBrowserBetaSetup32.exe']) + def test_only_returns_beta_win_ia32_package(self): + upload.PLATFORM = 'win32' + os.environ['TARGET_ARCH'] = 'ia32' + pkgs = list(upload.get_brave_packages( + os.path.join(self.get_pkgs_dir, upload.PLATFORM), + 'beta', '0.56.8')) + self.assertEquals(pkgs, ['BraveBrowserStandaloneBetaSetup32.exe', + 'BraveBrowserBetaSetup32.exe']) - def test_only_returns_release_win_x64_package(self): - upload.PLATFORM = 'win32' - os.environ['TARGET_ARCH'] = 'x64' - pkgs = list(upload.get_brave_packages(os.path.join(self.get_pkgs_dir, upload.PLATFORM), 'release', '0.56.8')) - self.assertEquals(pkgs, ['BraveBrowserSetup.exe', 'BraveBrowserStandaloneSetup.exe']) + def test_only_returns_release_win_x64_package(self): + upload.PLATFORM = 'win32' + os.environ['TARGET_ARCH'] = 'x64' + pkgs = list(upload.get_brave_packages( + os.path.join(self.get_pkgs_dir, upload.PLATFORM), + 'release', '0.56.8')) + self.assertEquals(pkgs, ['BraveBrowserSetup.exe', + 'BraveBrowserStandaloneSetup.exe']) + + def test_only_returns_release_win_ia32_package(self): + upload.PLATFORM = 'win32' + os.environ['TARGET_ARCH'] = 'ia32' + pkgs = list(upload.get_brave_packages( + os.path.join(self.get_pkgs_dir, upload.PLATFORM), + 'release', '0.56.8')) + self.assertEquals(pkgs, ['BraveBrowserStandaloneSetup32.exe', + 'BraveBrowserSetup32.exe']) - def test_only_returns_release_win_ia32_package(self): - upload.PLATFORM = 'win32' - os.environ['TARGET_ARCH'] = 'ia32' - pkgs = list(upload.get_brave_packages(os.path.join(self.get_pkgs_dir, upload.PLATFORM), 'release', '0.56.8')) - self.assertEquals(pkgs, ['BraveBrowserStandaloneSetup32.exe', 'BraveBrowserSetup32.exe']) if __name__ == '__main__': - print unittest.main() + print unittest.main()