diff --git a/.pylint2rc b/.pylint2rc new file mode 100644 index 00000000000..716f31d2f1e --- /dev/null +++ b/.pylint2rc @@ -0,0 +1,379 @@ +[MASTER] + +# Specify a configuration file. +#rcfile= + +# Python code to execute, usually for sys.path manipulation such as +# pygtk.require(). +#init-hook= + +# Add files or directories to the blacklist. They should be base names, not +# paths. +ignore=CVS + +# Pickle collected data for later comparisons. +persistent=yes + +# List of plugins (as comma separated values of python modules names) to load, +# usually to register additional checkers. +load-plugins= + +# Use multiple processes to speed up Pylint. +jobs=0 + +# Allow loading of arbitrary C extensions. Extensions are imported into the +# active Python interpreter and may run arbitrary code. +unsafe-load-any-extension=no + +# A comma-separated list of package or module names from where C extensions may +# be loaded. Extensions are loading into the active Python interpreter and may +# run arbitrary code +extension-pkg-whitelist= + +# Allow optimization of some AST trees. This will activate a peephole AST +# optimizer, which will apply various small optimizations. For instance, it can +# be used to obtain the result of joining multiple strings with the addition +# operator. Joining a lot of strings can lead to a maximum recursion error in +# Pylint and this flag can prevent that. It has one side effect, the resulting +# AST will be different than the one from reality. +optimize-ast=no + + +[MESSAGES CONTROL] + +# Only show warnings with the listed confidence levels. Leave empty to show +# all. Valid levels: HIGH, INFERENCE, INFERENCE_FAILURE, UNDEFINED +confidence= + +# Enable the message, report, category or checker with the given id(s). You can +# either give multiple identifier separated by comma (,) or put this option +# multiple time (only on the command line, not in the configuration file where +# it should appear only once). See also the "--disable" option for examples. +#enable= + +# Disable the message, report, category or checker with the given id(s). You +# can either give multiple identifiers separated by comma (,) or put this +# option multiple times (only on the command line, not in the configuration +# file where it should appear only once).You can also use "--disable=all" to +# disable everything first and then reenable specific checks. For example, if +# you want to run only the similarities checker, you can use "--disable=all +# --enable=similarities". If you want to run only the classes checker, but have +# no Warning level messages displayed, use"--disable=all --enable=classes +# --disable=W" +disable=import-star-module-level,old-octal-literal,oct-method,print-statement,unpacking-in-except,parameter-unpacking,backtick,old-raise-syntax,old-ne-operator,long-suffix,dict-view-method,dict-iter-method,metaclass-assignment,next-method-called,raising-string,indexing-exception,raw_input-builtin,long-builtin,file-builtin,execfile-builtin,coerce-builtin,cmp-builtin,buffer-builtin,basestring-builtin,apply-builtin,filter-builtin-not-iterating,using-cmp-argument,useless-suppression,raw-checker-failed,range-builtin-not-iterating,suppressed-message,invalid-name,no-absolute-import,old-division,cmp-method,bad-inline-option,reload-builtin,zip-builtin-not-iterating,intern-builtin,unichr-builtin,reduce-builtin,standarderror-builtin,unicode-builtin,xrange-builtin,coerce-method,delslice-method,getslice-method,setslice-method,input-builtin,round-builtin,file-ignored,locally-disabled,hex-method,nonzero-method,map-builtin-not-iterating,deprecated-pragma,missing-docstring,missing-module-docstring,raise-missing-from,invalid-name + + +[REPORTS] + +# Set the output format. Available formats are text, parseable, colorized, msvs +# (visual studio) and html. You can also give a reporter class, eg +# mypackage.mymodule.MyReporterClass. +output-format=text + +# Put messages in a separate file for each module / package specified on the +# command line instead of printing them on stdout. Reports (if any) will be +# written in a file name "pylint_global.[txt|html]". +files-output=no + +# Tells whether to display a full report or only the messages +reports=no + +# Python expression which should return a note less than 10 (10 is the highest +# note). You have access to the variables errors warning, statement which +# respectively contain the number of errors / warnings messages and the total +# number of statements analyzed. This is used by the global evaluation report +# (RP0004). +evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10) + +# Template used to display messages. This is a python new-style format string +# used to format the message information. See doc for all details +#msg-template= + + +[FORMAT] + +# Maximum number of characters on a single line. +max-line-length=100 + +# Regexp for a line that is allowed to be longer than the limit. +ignore-long-lines=^\s*(# )??$ + +# Allow the body of an if to be on the same line as the test if there is no +# else. +single-line-if-stmt=no + +# List of optional constructs for which whitespace checking is disabled. `dict- +# separator` is used to allow tabulation in dicts, etc.: {1 : 1,\n222: 2}. +# `trailing-comma` allows a space between comma and closing bracket: (a, ). +# `empty-line` allows space-only lines. +no-space-check=trailing-comma,dict-separator + +# Maximum number of lines in a module +max-module-lines=1000 + +# String used as indentation unit. This is usually " " (4 spaces) or "\t" (1 +# tab). +indent-string=' ' + +# Number of spaces of indent required inside a hanging or continued line. +indent-after-paren=4 + +# Expected format of line ending, e.g. empty (any line ending), LF or CRLF. +expected-line-ending-format= + + +[MISCELLANEOUS] + +# List of note tags to take in consideration, separated by a comma. +notes=FIXME,XXX,TODO + + +[SPELLING] + +# Spelling dictionary name. Available dictionaries: none. To make it working +# install python-enchant package. +spelling-dict= + +# List of comma separated words that should not be checked. +spelling-ignore-words= + +# A path to a file that contains private dictionary; one word per line. +spelling-private-dict-file= + +# Tells whether to store unknown words to indicated private dictionary in +# --spelling-private-dict-file option instead of raising a message. +spelling-store-unknown-words=no + + +[TYPECHECK] + +# Tells whether missing members accessed in mixin class should be ignored. A +# mixin class is detected if its name ends with "mixin" (case insensitive). +ignore-mixin-members=yes + +# List of module names for which member attributes should not be checked +# (useful for modules/projects where namespaces are manipulated during runtime +# and thus existing member attributes cannot be deduced by static analysis. It +# supports qualified module names, as well as Unix pattern matching. +ignored-modules= + +# List of classes names for which member attributes should not be checked +# (useful for classes with attributes dynamically set). This supports can work +# with qualified names. +ignored-classes=optparse.Values,thread._local,_thread._local + +# List of members which are set dynamically and missed by pylint inference +# system, and so shouldn't trigger E1101 when accessed. Python regular +# expressions are accepted. +generated-members= + + +[BASIC] + +# List of builtins function names that should not be used, separated by a comma +bad-functions=map,filter,input + +# Good variable names which should always be accepted, separated by a comma +good-names=i,j,k,ex,Run,_ + +# Bad variable names which should always be refused, separated by a comma +bad-names=foo,bar,baz,toto,tutu,tata + +# Colon-delimited sets of names that determine each other's naming style when +# the name regexes allow several styles. +name-group= + +# Include a hint for the correct naming format with invalid-name +include-naming-hint=no + +# Regular expression matching correct function names +function-rgx=[a-z_][a-z0-9_]{2,30}$ + +# Naming hint for function names +function-name-hint=[a-z_][a-z0-9_]{2,30}$ + +# Regular expression matching correct variable names +variable-rgx=[a-z_][a-z0-9_]{2,30}$ + +# Naming hint for variable names +variable-name-hint=[a-z_][a-z0-9_]{2,30}$ + +# Regular expression matching correct constant names +const-rgx=(([A-Z_][A-Z0-9_]*)|(__.*__))$ + +# Naming hint for constant names +const-name-hint=(([A-Z_][A-Z0-9_]*)|(__.*__))$ + +# Regular expression matching correct attribute names +attr-rgx=[a-z_][a-z0-9_]{2,30}$ + +# Naming hint for attribute names +attr-name-hint=[a-z_][a-z0-9_]{2,30}$ + +# Regular expression matching correct argument names +argument-rgx=[a-z_][a-z0-9_]{2,30}$ + +# Naming hint for argument names +argument-name-hint=[a-z_][a-z0-9_]{2,30}$ + +# Regular expression matching correct class attribute names +class-attribute-rgx=([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$ + +# Naming hint for class attribute names +class-attribute-name-hint=([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$ + +# Regular expression matching correct inline iteration names +inlinevar-rgx=[A-Za-z_][A-Za-z0-9_]*$ + +# Naming hint for inline iteration names +inlinevar-name-hint=[A-Za-z_][A-Za-z0-9_]*$ + +# Regular expression matching correct class names +class-rgx=[A-Z_][a-zA-Z0-9]+$ + +# Naming hint for class names +class-name-hint=[A-Z_][a-zA-Z0-9]+$ + +# Regular expression matching correct module names +module-rgx=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$ + +# Naming hint for module names +module-name-hint=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$ + +# Regular expression matching correct method names +method-rgx=[a-z_][a-z0-9_]{2,30}$ + +# Naming hint for method names +method-name-hint=[a-z_][a-z0-9_]{2,30}$ + +# Regular expression which should only match function or class names that do +# not require a docstring. +no-docstring-rgx=^_ + +# Minimum line length for functions/classes that require docstrings, shorter +# ones are exempt. +docstring-min-length=-1 + + +[ELIF] + +# Maximum number of nested blocks for function / method body +max-nested-blocks=5 + + +[SIMILARITIES] + +# Minimum lines number of a similarity. +min-similarity-lines=4 + +# Ignore comments when computing similarities. +ignore-comments=yes + +# Ignore docstrings when computing similarities. +ignore-docstrings=yes + +# Ignore imports when computing similarities. +ignore-imports=no + + +[LOGGING] + +# Logging modules to check that the string format arguments are in logging +# function parameter format +logging-modules=logging + + +[VARIABLES] + +# Tells whether we should check for unused import in __init__ files. +init-import=no + +# A regular expression matching the name of dummy variables (i.e. expectedly +# not used). +dummy-variables-rgx=_+$|(_[a-zA-Z0-9_]*[a-zA-Z0-9]+?$)|dummy|^ignored_|^unused_ + +# List of additional names supposed to be defined in builtins. Remember that +# you should avoid to define new builtins when possible. +additional-builtins= + +# List of strings which can identify a callback function by name. A callback +# name must start or end with one of those strings. +callbacks=cb_,_cb + + +[CLASSES] + +# List of method names used to declare (i.e. assign) instance attributes. +defining-attr-methods=__init__,__new__,setUp,__post_init__ + +# List of valid names for the first argument in a class method. +valid-classmethod-first-arg=cls + +# List of valid names for the first argument in a metaclass class method. +valid-metaclass-classmethod-first-arg=cls + +# List of member names, which should be excluded from the protected access +# warning. +exclude-protected=_asdict,_fields,_replace,_source,_make + + +[IMPORTS] + +# Deprecated modules which should not be used, separated by a comma +deprecated-modules=optparse,tkinter.tix + +# Create a graph of every (i.e. internal and external) dependencies in the +# given file (report RP0402 must not be disabled) +import-graph= + +# Create a graph of external dependencies in the given file (report RP0402 must +# not be disabled) +ext-import-graph= + +# Create a graph of internal dependencies in the given file (report RP0402 must +# not be disabled) +int-import-graph= + + +[DESIGN] + +# Maximum number of arguments for function / method +max-args=8 + +# Argument names that match this expression will be ignored. Default to name +# with leading underscore +ignored-argument-names=_.*|^ignored_|^unused_ + +# Maximum number of locals for function / method body +max-locals=15 + +# Maximum number of return / yield for function / method body +max-returns=6 + +# Maximum number of branch for function / method body +max-branches=12 + +# Maximum number of statements in function / method body +max-statements=50 + +# Maximum number of parents for a class (see R0901). +max-parents=7 + +# Maximum number of attributes for a class (see R0902). +max-attributes=7 + +# Minimum number of public methods for a class (see R0903). +min-public-methods=2 + +# Maximum number of public methods for a class (see R0904). +max-public-methods=20 + +# Maximum number of boolean expressions in a if statement +max-bool-expr=5 + + +[EXCEPTIONS] + +# Exceptions that will emit a warning when being caught. Defaults to +# "Exception" +overgeneral-exceptions=BaseException,Exception diff --git a/build/commands/lib/config.js b/build/commands/lib/config.js index b411af47ce2..716d59f3ae8 100755 --- a/build/commands/lib/config.js +++ b/build/commands/lib/config.js @@ -739,6 +739,8 @@ Config.prototype.getCachePath = function () { Object.defineProperty(Config.prototype, 'defaultOptions', { get: function () { let env = Object.assign({}, process.env) + env = this.addPathToEnv(env, path.join(this.depotToolsDir, 'python3-bin'), true) + env = this.addPathToEnv(env, path.join(this.depotToolsDir, 'python2-bin'), true) env = this.addPathToEnv(env, this.depotToolsDir, true) env = this.addPythonPathToEnv(env, path.join(this.srcDir, 'brave', 'chromium_src', 'python_modules')) env = this.addPythonPathToEnv(env, path.join(this.srcDir, 'brave', 'script')) diff --git a/build/commands/lib/pylint.js b/build/commands/lib/pylint.js new file mode 100644 index 00000000000..f9461aa707a --- /dev/null +++ b/build/commands/lib/pylint.js @@ -0,0 +1,213 @@ +// Copyright (c) 2021 The Brave Authors. All rights reserved. +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// you can obtain one at http://mozilla.org/MPL/2.0/. + +const path = require('path') +const fs = require('fs') +const config = require('../lib/config') +const util = require('../lib/util') +const {EOL} = require('os'); + +const deleteFile = (path) => { + if (fs.existsSync(path)) { + try { + fs.unlinkSync(path) + } catch(err) { + console.error('Unable to delete file: ' + path + ' error: ', err) + process.exit(1) + } + } +} + +const ensureDepotToolsInPath = (options) => { + const depotToolsPath = path.resolve(__dirname, '../../../vendor/depot_tools') + if (!fs.existsSync(depotToolsPath)) { + console.warn('Depot Tools path [' + depotToolsPath + '] doesn\'t exist') + return + } + let newPath = options.env.path + if (!newPath) { + process.env.Path && (newPath = process.env.Path) + process.env.PATH && (newPath = process.env.PATH) + } + newPath = newPath.split(path.delimiter) + if (!newPath.includes(depotToolsPath)) { + console.info('Prepending [' + depotToolsPath + '] to the path') + newPath.unshift(depotToolsPath) + } + options.env.path = newPath.join(path.delimiter) +} + +const getDefaultOptions = () => { + let options = Object.assign({}, config.defaultOptions) + options.cwd = path.resolve(__dirname, '../../..') + ensureDepotToolsInPath(options) + return options +} + +const getPylintInfo = () => { + const cmd_options = getDefaultOptions() + // Use runProcess here because `which` fails silently (so `run` would not + // print anything) and we want to show the error. + const prog = util.runProcess(process.platform !== 'win32' ? 'which' : 'where', + ['pylint'], cmd_options) + if (prog.status !== 0) { + console.error('pylint could not be found in path') + process.exit(1) + } + + // Get pylint version, and parse it to get the associated python version so + // that we know which pylintrc config file to use. Can't seem to get this via + // stdout so ended up redirecting to a file and then reading from it. + const pylintVersionFile = 'pylint-version.txt' + deleteFile(pylintVersionFile) + util.run('pylint', ['--version', '>' + pylintVersionFile], cmd_options) + let data + try { + data = fs.readFileSync(pylintVersionFile, 'utf8'); + } catch(err) { + console.error('Unable to read file: ' + pylintVersionFile + 'error: ', err); + process.exit(1) + } + deleteFile(pylintVersionFile) + console.log(data) + const python = 'Python ' + return data.substr(data.indexOf(python) + python.length, 1) +} + +const runPylint = (args, continueOnFail = true) => { + let cmd_options = getDefaultOptions() + cmd_options.continueOnFail = continueOnFail + const prog = util.run('pylint', args, cmd_options) + return (prog.status === 0) +} + +const runGit = (gitArgs) => { + // Have to use shell because otherwise git gets confused with node's escaping + // of the args. + let prog = util.run('git', gitArgs, { shell: true }) + const output = prog.stdout.toString() + console.log(output) + return output.trim() +} + +const formatFolders = (folders, sep = '/*.py') => { + return folders.join(sep + ' ') + sep +} + +const createEmptyReportFile = (path) => { + try { + fs.writeFileSync(path, EOL) + } catch(err) { + console.error('Unable to write to file: ' + path + ' error: ', err) + process.exit(1) + } +} + +const getAllFiles = (check_folders) => { + const files = runGit(['ls-files', '--', formatFolders(check_folders)]) + if (!files) { + return [] + } + return files.split('\n') +} + +const getChangedFiles = (base_branch, check_folders) => { + if (!base_branch) { + return getAllFiles(check_folders) + } + + const merge_base = runGit(['merge-base', 'origin/' + base_branch, 'HEAD']) + if (!merge_base) { + console.error('Could not determine merge-base for branch ' + base_branch) + process.exit(1) + } + const changed_files = runGit(['diff', '--name-only', '--diff-filter', 'drt', + merge_base, '--', formatFolders(check_folders)]) + if (!changed_files) { + return [] + } + return changed_files.split('\n') +} + +const getDescription = (base, check_folders) => { + let description = 'pylint findings' + if (base) { + description += ' in scripts changed relative to ' + base + } + description += ' in: ' + formatFolders(check_folders, '/') + return description +} + +const pylint = (options = {}) => { + const check_folders = ['build', 'components', 'installer', 'script', 'tools'] + const report_file = 'pylint-report.txt' + + const description = getDescription(options.base, check_folders) + + // Print out which pylint is going to be used and its version + const python_version = getPylintInfo() + + // Get changed or all python files + const paths = getChangedFiles(options.base, check_folders) + if (!paths.length) { + console.log('No ' + description) + if (options.report) { + createEmptyReportFile(report_file) + } + return + } + + // Prepare pylint args + const rcfile = python_version === '2' ? '.pylint2rc' : '.pylintrc' + let args = ['-j0', '-rn', '--rcfile=' + rcfile] + if (options.report) { + args.push('-fparseable') + } + + // On Windows, command line limit is 8192 chars, so may have to make multiple + // calls to pylint. If that happens we will be adding to the same report file, + // so make sure the file is clean. Leave some slack for initial args and + // report file redirect + const maxCmdLineLength = 8000 + if (options.report) { + deleteFile(report_file) + } + + console.log('Checking for ' + description) + + // Convenience funcion + const doPylint = (loop_args) => { + if (options.report) { + loop_args.push('>>' + report_file) + } + return runPylint(loop_args) + } + + let result = true + let currentLen = 0 + let loop_args = [...args] + // Run pylint for each max command line length + for (const pyPath of paths) { + if (currentLen + pyPath.length > maxCmdLineLength) { + result &= doPylint(loop_args) + currentLen = 0 + loop_args = [...args] + } + loop_args.push(pyPath) + currentLen += pyPath.length + } + + if (currentLen > 0) { + result &= doPylint(loop_args) + } + + // When the report option is present we don't want to exit with an error so + // that CI can continue and parse the report + if (!result && !options.report) { + process.exit(1) + } +} + +module.exports = pylint diff --git a/build/commands/scripts/commands.js b/build/commands/scripts/commands.js index 90df7dbe4b2..e3e8cb903e1 100755 --- a/build/commands/scripts/commands.js +++ b/build/commands/scripts/commands.js @@ -19,6 +19,7 @@ const l10nDeleteTranslations = require('../lib/l10nDeleteTranslations') const createDist = require('../lib/createDist') const test = require('../lib/test') const gnCheck = require('../lib/gnCheck') +const pylint = require('../lib/pylint') const collect = (value, accumulator) => { accumulator.push(value) @@ -239,6 +240,12 @@ program .option('--base ', 'set the destination branch for the PR') .action(util.lint) +program + .command('pylint') + .option('--base ', 'only analyse files changed relative to base_branch') + .option('--report', 'produce a parseable report file') + .action(pylint) + program .command('format') .option('--full', 'format all lines in changed files instead of only the changed lines') diff --git a/package.json b/package.json index f375f6fed2f..e1b64f08645 100644 --- a/package.json +++ b/package.json @@ -30,9 +30,7 @@ "test-security": "npm run check_security && npm run audit_deps && node ./build/commands/scripts/commands.js start --enable_brave_update --network_log --user_data_dir_name=brave-network-test", "tslint": "tslint --project tsconfig-lint.json \"components/**/*.{ts,tsx}\"", "pep8": "pycodestyle --max-line-length 120 -r script", - "pylint": "./script/pylint.sh", - "pylint:help": "./script/pylint.sh --help", - "pylint:report": "./script/pylint.sh --report", + "pylint": "node ./build/commands/scripts/commands.js pylint", "web-ui-gen-grd": "node components/webpack/gen-webpack-grd", "web-ui": "webpack --config components/webpack/webpack.config.js --progress --colors", "build-storybook": "build-storybook -c .storybook -o .storybook-out", diff --git a/script/pylint.sh b/script/pylint.sh deleted file mode 100755 index 2b1fc6b92e3..00000000000 --- a/script/pylint.sh +++ /dev/null @@ -1,102 +0,0 @@ -#!/usr/bin/env bash -# Requirements: bash, coreutils, git, pylint. -# The python3 version of pylint should be used (`pip3 install pylint'). -# This is called via `npm run pylint' with the top of the brave-core repo as $PWD. - -# Fail on any error. -shopt -s inherit_errexit -set -eEo pipefail - -# Defaults. Overwritten in parse_cmdline() and set readonly. -base_branch="" -check_folders=(build components installer script tools) -pylint_options=(-j0 -rn --rcfile=.pylintrc) -report=0 -report_file=pylint-report.txt - -# Print a help message and exit with 1 by default. -# $@ := [ exit_status ] -help() { - cat <<-EOF - USAGE: pylint.sh --help - pylint.sh [--compare-with ] [--report] - - Run pylint on versioned python scripts in: - ${check_folders[*]/%/\/} - - --compare-with : only analyse files changed relative to base_branch - --report: produce a parseable report file - - The .pylintrc file at the top of this repo defines pylint options. - - This script should be called via \`npm run', e.g.: - npm run pylint - npm run pylint -- --compare-with master - EOF - exit "${1:-1}" -} - -# Print an error message, followed by the help message and exit. -# $@ := message -error() { printf "%s\n\n" "ERROR: $*" >&2; help; } - -# Parse command line arguments. -# $@ := script_arguments -parse_cmdline() { - for ((i=1; i <= $#; ++i)); do - case "${@:$i:1}" in - --compare-with) - ((++i <= $#)) || error "The --compare-with option requires an argument"; - base_branch="${*:$i:1}" - git merge-base "origin/${base_branch:?}" HEAD >/dev/null \ - || error "Invalid base_branch";; - --report) report=1;; - --help) help 0;; - *) error "Invalid arguments passed: ${*:$i}";; - esac - done - readonly base_branch check_folders pylint_options report report_file -} - -# Print a tab-delimited list of python scripts changed relative to $base_branch. -# $@ := "" -get_changed_files() { - git diff --name-only --diff-filter drt \ - "$(git merge-base "origin/${base_branch:?}" HEAD)" -- \ - "${check_folders[@]/%/\/\*.py}"|tr '\n' '\t' -} - -# Execute pylint (if necessary) and print info messages. -# $@ := description files_to_check -check() { - local description="${1:?} in: ${check_folders[*]/%/\/}" - local -a check_files=("${@:2}") - - if (("${#check_files[@]}" > 0)); then - echo "Checking for $description" >&2 - case "${report:?}" in - 1) exec pylint "${pylint_options[@]}" -fparseable --exit-zero \ - "${check_files[@]}" >"${report_file:?}";; - *) exec pylint "${pylint_options[@]}" "${check_files[@]}";; - esac - else - echo "No $description" >&2 - [[ "$report" == 0 ]] || echo >"$report_file" - fi -} - -# $@ := script_arguments -main() { - local -a check_files - - parse_cmdline "$@" - - case "$base_branch" in - "") check "pylint findings" "${check_folders[@]}";; - *) read -ra check_files <<<"$(get_changed_files)" - check "pylint findings in scripts changed relative to $base_branch" \ - "${check_files[@]}";; - esac -} - -main "$@"