From 95f9e90b24fcf4aba04b59e7f7c3fb6ca319c747 Mon Sep 17 00:00:00 2001 From: wchen342 Date: Thu, 4 Nov 2021 23:47:36 +0200 Subject: [PATCH 1/2] Android XML pre-processor --- android/brave_java_resources.gni | 4 + android/brave_xml_preprocessor.gni | 58 +++++++++++++ android/xml_processor.py | 132 +++++++++++++++++++++++++++++ android/xml_processor.pydeps | 31 +++++++ build/android/BUILD.gn | 6 ++ build/android/config.gni | 6 +- 6 files changed, 235 insertions(+), 2 deletions(-) create mode 100644 android/brave_xml_preprocessor.gni create mode 100644 android/xml_processor.py create mode 100644 android/xml_processor.pydeps diff --git a/android/brave_java_resources.gni b/android/brave_java_resources.gni index 96e7d8bce42..7f4a798fdc4 100644 --- a/android/brave_java_resources.gni +++ b/android/brave_java_resources.gni @@ -898,3 +898,7 @@ brave_java_resources = [ "java/res/xml/unstoppable_domains_preferences.xml", "java/res/xml/use_custom_tabs_brave_preference.xml", ] + +# XML preprocessing +brave_java_preprocess_xml_sources = [] +brave_java_preprocess_module_sources = [] diff --git a/android/brave_xml_preprocessor.gni b/android/brave_xml_preprocessor.gni new file mode 100644 index 00000000000..ca237298750 --- /dev/null +++ b/android/brave_xml_preprocessor.gni @@ -0,0 +1,58 @@ +# Copyright 2021 The Brave Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +import("//build/config/android/rules.gni") +import("//build/config/python.gni") + +template("brave_xml_preprocessor") { + forward_variables_from(invoker, [ "testonly" ]) + + _preprocess_target_name = "${target_name}__preprocess" + + _xml_sources_path = "${target_gen_dir}/${target_name}.xml.sources" + _module_sources_path = "${target_gen_dir}/${target_name}.py.sources" + _xml_output_zip = "${target_out_dir}/${target_name}.resources.zip" + + action_with_pydeps(_preprocess_target_name) { + script = "//brave/android/xml_processor.py" + + # Input files + sources = invoker.sources + modules = invoker.modules + inputs = modules # so gn will rebuild upon changes + + outputs = [ _xml_output_zip ] + + # Rebase path + _rebased_xml_source_files = rebase_path(sources, root_build_dir) + _rebased_module_source_files = rebase_path(modules, root_build_dir) + + # Write sources to file + write_file(_xml_sources_path, _rebased_xml_source_files) + write_file(_module_sources_path, _rebased_module_source_files) + + # Rebase file paths for python + _rebased_xml_sources_path = + rebase_path(_xml_sources_path, root_build_dir) + _rebased_module_sources_path = + rebase_path(_module_sources_path, root_build_dir) + + args = [ + "--outputs-zip", + rebase_path(_xml_output_zip, root_build_dir), + "--xml-sources-path=${_rebased_xml_sources_path}", + "--module-sources-path=${_rebased_module_sources_path}", + ] + } + + android_generated_resources(target_name) { + forward_variables_from(invoker, + TESTONLY_AND_VISIBILITY + [ + "deps", + "resource_overlay", + ]) + generating_target = ":$_preprocess_target_name" + generated_resources_zip = _xml_output_zip + } +} diff --git a/android/xml_processor.py b/android/xml_processor.py new file mode 100644 index 00000000000..ba01975cbda --- /dev/null +++ b/android/xml_processor.py @@ -0,0 +1,132 @@ +#!/usr/bin/env python + +# Copyright 2021 The Brave Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""XML processor""" + +from pathlib import Path +import xml.etree.ElementTree as ET + +import codecs +import argparse +import importlib.util +import os +import sys + +sys.path.append(os.path.join(os.path.dirname(__file__), + os.pardir, os.pardir, + 'build', 'android', 'gyp')) +from util import build_utils +from util import resource_utils + + +def _UnderJavaRes(source): + """ + Check from left whether java/res is part of input path, returns relative path to java/res. + Input path shall be absolute. + """ + source_path = Path(source) + source_path_parts = source_path.parts + for i in range(1, len(source_path_parts)): + if source_path_parts[i - 1] == 'java' and source_path_parts[i] == 'res': + parent_path = source_path.parents[len(source_path_parts) - i - 2] + try: + rel_path = str(source_path.relative_to(parent_path)) + return rel_path + except ValueError as e: + print(e) + return None + + +def _AddBravePrefix(relpath): + dirname, filename = os.path.split(relpath) + return os.path.join(dirname, 'brave_' + filename) + + +def _ImportModuleByPath(module_path): + """Imports a module by its source file.""" + sys.path[0] = os.path.dirname(module_path) + + module_name = os.path.splitext(os.path.basename(module_path))[0] + spec = importlib.util.spec_from_file_location(module_name, module_path) + module = importlib.util.module_from_spec(spec) + sys.modules[module_name] = module + spec.loader.exec_module(module) + + return module + + +def _ProcessFile(output_filename, output): + if os.path.isfile(output_filename): + with codecs.open(output_filename, 'r', 'utf-8') as f: + if f.read() == output: + return + + with open(output_filename, 'wb') as output_file: + output_file.write(output) + + +def _XMLTransform(source_pairs, outputs_zip): + """ + Apply module codes to imput sources. + """ + with build_utils.TempDir() as temp_dir: + path_info = resource_utils.ResourceInfoFile() + for source, module in source_pairs: + with codecs.open(source, 'r', 'utf-8') as f: + xml_content = f.read() + loaded_module = _ImportModuleByPath(module) + + if not xml_content or not loaded_module: + continue + + root = ET.XML(xml_content) + result = loaded_module._ProcessXML(root) + output = ET.tostring(result, encoding='utf-8', xml_declaration=True) + + # Parse output path + # For simplicity, we assume input path will always has java/res in it + if not (relpath := _UnderJavaRes(os.path.abspath(source))): + raise Exception('input file %s is not under java/res' % source) + + # resource_overlay doesn't seem to work from android_generated_resources + relpath = _AddBravePrefix(relpath) + output_filename = os.path.join(temp_dir, relpath) + parent_dir = os.path.dirname(output_filename) + build_utils.MakeDirectory(parent_dir) + _ProcessFile(output_filename, output) + path_info.AddMapping(relpath, source) + + path_info.Write(outputs_zip + '.info') + build_utils.ZipDir(outputs_zip, temp_dir) + + +def main(args): + parser = argparse.ArgumentParser() + parser.add_argument('--xml-sources-path', + required=True, + help='Path to a list of input xml sources for this target.') + parser.add_argument('--module-sources-path', + required=True, + help='Path to a list of input python sources for this target. ' + 'Each python file shall have a function named _ProcessXML.') + parser.add_argument('--outputs-zip', + required=True, + help='Path to a list of expected outputs for this target.') + + options = parser.parse_args(args) + + with open(options.xml_sources_path) as f: + options.sources = f.read().splitlines() + with open(options.module_sources_path) as f: + options.modules = f.read().splitlines() + + assert len(options.sources) == len(options.modules) + + _XMLTransform(list(zip(options.sources, options.modules)), options.outputs_zip) + + +if __name__ == '__main__': + main(sys.argv[1:]) diff --git a/android/xml_processor.pydeps b/android/xml_processor.pydeps new file mode 100644 index 00000000000..5c263e397dd --- /dev/null +++ b/android/xml_processor.pydeps @@ -0,0 +1,31 @@ +# Generated by running: +# build/print_python_deps.py --root brave/android --output brave/android/xml_processor.pydeps brave/android/xml_processor.py +../../build/android/gyp/util/__init__.py +../../build/android/gyp/util/build_utils.py +../../build/android/gyp/util/resource_utils.py +../../build/gn_helpers.py +../../third_party/jinja2/__init__.py +../../third_party/jinja2/_compat.py +../../third_party/jinja2/_identifier.py +../../third_party/jinja2/asyncfilters.py +../../third_party/jinja2/asyncsupport.py +../../third_party/jinja2/bccache.py +../../third_party/jinja2/compiler.py +../../third_party/jinja2/defaults.py +../../third_party/jinja2/environment.py +../../third_party/jinja2/exceptions.py +../../third_party/jinja2/filters.py +../../third_party/jinja2/idtracking.py +../../third_party/jinja2/lexer.py +../../third_party/jinja2/loaders.py +../../third_party/jinja2/nodes.py +../../third_party/jinja2/optimizer.py +../../third_party/jinja2/parser.py +../../third_party/jinja2/runtime.py +../../third_party/jinja2/tests.py +../../third_party/jinja2/utils.py +../../third_party/jinja2/visitor.py +../../third_party/markupsafe/__init__.py +../../third_party/markupsafe/_compat.py +../../third_party/markupsafe/_native.py +xml_processor.py diff --git a/build/android/BUILD.gn b/build/android/BUILD.gn index 518a63630b4..16e6611caaf 100755 --- a/build/android/BUILD.gn +++ b/build/android/BUILD.gn @@ -1,3 +1,4 @@ +import("//brave/android/brave_xml_preprocessor.gni") import("//brave/build/config.gni") import("//build/config/android/rules.gni") import("//tools/grit/grit_rule.gni") @@ -168,6 +169,11 @@ java_strings_grd("android_brave_strings_grd") { ] } +brave_xml_preprocessor("brave_java_xml_preprocess_resources") { + sources = brave_java_preprocess_xml_sources + modules = brave_java_preprocess_module_sources +} + generate_jni("jni_headers") { sources = [ "//brave/android/java/org/chromium/chrome/browser/BraveFeatureList.java", diff --git a/build/android/config.gni b/build/android/config.gni index 748e0e899c3..692eb57dff6 100644 --- a/build/android/config.gni +++ b/build/android/config.gni @@ -43,8 +43,10 @@ brave_chrome_java_srcjar_deps = [ "//brave/browser/android/preferences:java_pref_names_srcjar", ] -brave_chrome_app_java_resources_deps = - [ "//brave/build/android:android_brave_strings_grd" ] +brave_chrome_app_java_resources_deps = [ + "//brave/build/android:android_brave_strings_grd", + "//brave/build/android:brave_java_xml_preprocess_resources" +] brave_resources_exclusion_exceptions = [ "*com_google_android_material*design_bottom_*" ] From 145b707730e53ca73fb6fff6dd41431af59e6e7a Mon Sep 17 00:00:00 2001 From: wchen342 Date: Fri, 5 Nov 2021 21:51:41 +0200 Subject: [PATCH 2/2] Format --- android/brave_xml_preprocessor.gni | 7 +++---- android/xml_processor.py | 2 +- build/android/config.gni | 2 +- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/android/brave_xml_preprocessor.gni b/android/brave_xml_preprocessor.gni index ca237298750..70cb7f5e2b6 100644 --- a/android/brave_xml_preprocessor.gni +++ b/android/brave_xml_preprocessor.gni @@ -20,7 +20,7 @@ template("brave_xml_preprocessor") { # Input files sources = invoker.sources modules = invoker.modules - inputs = modules # so gn will rebuild upon changes + inputs = modules # so gn will rebuild upon changes outputs = [ _xml_output_zip ] @@ -33,10 +33,9 @@ template("brave_xml_preprocessor") { write_file(_module_sources_path, _rebased_module_source_files) # Rebase file paths for python - _rebased_xml_sources_path = - rebase_path(_xml_sources_path, root_build_dir) + _rebased_xml_sources_path = rebase_path(_xml_sources_path, root_build_dir) _rebased_module_sources_path = - rebase_path(_module_sources_path, root_build_dir) + rebase_path(_module_sources_path, root_build_dir) args = [ "--outputs-zip", diff --git a/android/xml_processor.py b/android/xml_processor.py index ba01975cbda..757a8f309a9 100644 --- a/android/xml_processor.py +++ b/android/xml_processor.py @@ -70,7 +70,7 @@ def _ProcessFile(output_filename, output): def _XMLTransform(source_pairs, outputs_zip): """ - Apply module codes to imput sources. + Apply module codes to input sources. """ with build_utils.TempDir() as temp_dir: path_info = resource_utils.ResourceInfoFile() diff --git a/build/android/config.gni b/build/android/config.gni index 692eb57dff6..0a30d064d97 100644 --- a/build/android/config.gni +++ b/build/android/config.gni @@ -45,7 +45,7 @@ brave_chrome_java_srcjar_deps = [ brave_chrome_app_java_resources_deps = [ "//brave/build/android:android_brave_strings_grd", - "//brave/build/android:brave_java_xml_preprocess_resources" + "//brave/build/android:brave_java_xml_preprocess_resources", ] brave_resources_exclusion_exceptions =