494 lines
20 KiB
Python
Executable file
494 lines
20 KiB
Python
Executable file
#!/usr/bin/env python3
|
|
#
|
|
# Copyright (c) 2013 The Chromium Authors. All rights reserved.
|
|
# Use of this source code is governed by a BSD-style license that can be
|
|
# found in the LICENSE file.
|
|
"""Runs Android's lint tool."""
|
|
|
|
from __future__ import print_function
|
|
|
|
import argparse
|
|
import functools
|
|
import logging
|
|
import os
|
|
import re
|
|
import shutil
|
|
import sys
|
|
import time
|
|
import traceback
|
|
from xml.dom import minidom
|
|
from xml.etree import ElementTree
|
|
|
|
from util import build_utils
|
|
from util import manifest_utils
|
|
from util import server_utils
|
|
|
|
_LINT_MD_URL = 'https://chromium.googlesource.com/chromium/src/+/main/build/android/docs/lint.md' # pylint: disable=line-too-long
|
|
|
|
# These checks are not useful for chromium.
|
|
_DISABLED_ALWAYS = [
|
|
"AppCompatResource", # Lint does not correctly detect our appcompat lib.
|
|
"Assert", # R8 --force-enable-assertions is used to enable java asserts.
|
|
"InflateParams", # Null is ok when inflating views for dialogs.
|
|
"InlinedApi", # Constants are copied so they are always available.
|
|
"LintBaseline", # Don't warn about using baseline.xml files.
|
|
"MissingApplicationIcon", # False positive for non-production targets.
|
|
"SwitchIntDef", # Many C++ enums are not used at all in java.
|
|
"UniqueConstants", # Chromium enums allow aliases.
|
|
"UnusedAttribute", # Chromium apks have various minSdkVersion values.
|
|
"ObsoleteLintCustomCheck", # We have no control over custom lint checks.
|
|
]
|
|
|
|
# These checks are not useful for test targets and adds an unnecessary burden
|
|
# to suppress them.
|
|
_DISABLED_FOR_TESTS = [
|
|
# We should not require test strings.xml files to explicitly add
|
|
# translatable=false since they are not translated and not used in
|
|
# production.
|
|
"MissingTranslation",
|
|
# Test strings.xml files often have simple names and are not translatable,
|
|
# so it may conflict with a production string and cause this error.
|
|
"Untranslatable",
|
|
# Test targets often use the same strings target and resources target as the
|
|
# production targets but may not use all of them.
|
|
"UnusedResources",
|
|
# TODO(wnwen): Turn this back on since to crash it would require running on
|
|
# a device with all the various minSdkVersions.
|
|
# Real NewApi violations crash the app, so the only ones that lint catches
|
|
# but tests still succeed are false positives.
|
|
"NewApi",
|
|
# Tests should be allowed to access these methods/classes.
|
|
"VisibleForTests",
|
|
]
|
|
|
|
_RES_ZIP_DIR = 'RESZIPS'
|
|
_SRCJAR_DIR = 'SRCJARS'
|
|
_AAR_DIR = 'AARS'
|
|
|
|
|
|
def _SrcRelative(path):
|
|
"""Returns relative path to top-level src dir."""
|
|
return os.path.relpath(path, build_utils.DIR_SOURCE_ROOT)
|
|
|
|
|
|
def _GenerateProjectFile(android_manifest,
|
|
android_sdk_root,
|
|
cache_dir,
|
|
sources=None,
|
|
classpath=None,
|
|
srcjar_sources=None,
|
|
resource_sources=None,
|
|
custom_lint_jars=None,
|
|
custom_annotation_zips=None,
|
|
android_sdk_version=None):
|
|
project = ElementTree.Element('project')
|
|
root = ElementTree.SubElement(project, 'root')
|
|
# Run lint from output directory: crbug.com/1115594
|
|
root.set('dir', os.getcwd())
|
|
sdk = ElementTree.SubElement(project, 'sdk')
|
|
# Lint requires that the sdk path be an absolute path.
|
|
sdk.set('dir', os.path.abspath(android_sdk_root))
|
|
cache = ElementTree.SubElement(project, 'cache')
|
|
cache.set('dir', cache_dir)
|
|
main_module = ElementTree.SubElement(project, 'module')
|
|
main_module.set('name', 'main')
|
|
main_module.set('android', 'true')
|
|
main_module.set('library', 'false')
|
|
if android_sdk_version:
|
|
main_module.set('compile_sdk_version', android_sdk_version)
|
|
manifest = ElementTree.SubElement(main_module, 'manifest')
|
|
manifest.set('file', android_manifest)
|
|
if srcjar_sources:
|
|
for srcjar_file in srcjar_sources:
|
|
src = ElementTree.SubElement(main_module, 'src')
|
|
src.set('file', srcjar_file)
|
|
if sources:
|
|
for source in sources:
|
|
src = ElementTree.SubElement(main_module, 'src')
|
|
src.set('file', source)
|
|
if classpath:
|
|
for file_path in classpath:
|
|
classpath_element = ElementTree.SubElement(main_module, 'classpath')
|
|
classpath_element.set('file', file_path)
|
|
if resource_sources:
|
|
for resource_file in resource_sources:
|
|
resource = ElementTree.SubElement(main_module, 'resource')
|
|
resource.set('file', resource_file)
|
|
if custom_lint_jars:
|
|
for lint_jar in custom_lint_jars:
|
|
lint = ElementTree.SubElement(main_module, 'lint-checks')
|
|
lint.set('file', lint_jar)
|
|
if custom_annotation_zips:
|
|
for annotation_zip in custom_annotation_zips:
|
|
annotation = ElementTree.SubElement(main_module, 'annotations')
|
|
annotation.set('file', annotation_zip)
|
|
return project
|
|
|
|
|
|
def _RetrieveBackportedMethods(backported_methods_path):
|
|
with open(backported_methods_path) as f:
|
|
methods = f.read().splitlines()
|
|
# Methods look like:
|
|
# java/util/Set#of(Ljava/lang/Object;)Ljava/util/Set;
|
|
# But error message looks like:
|
|
# Call requires API level R (current min is 21): java.util.Set#of [NewApi]
|
|
methods = (m.replace('/', '\\.') for m in methods)
|
|
methods = (m[:m.index('(')] for m in methods)
|
|
return sorted(set(methods))
|
|
|
|
|
|
def _GenerateConfigXmlTree(orig_config_path, backported_methods):
|
|
if orig_config_path:
|
|
root_node = ElementTree.parse(orig_config_path).getroot()
|
|
else:
|
|
root_node = ElementTree.fromstring('<lint/>')
|
|
|
|
issue_node = ElementTree.SubElement(root_node, 'issue')
|
|
issue_node.attrib['id'] = 'NewApi'
|
|
ignore_node = ElementTree.SubElement(issue_node, 'ignore')
|
|
ignore_node.attrib['regexp'] = '|'.join(backported_methods)
|
|
return root_node
|
|
|
|
|
|
def _GenerateAndroidManifest(original_manifest_path, extra_manifest_paths,
|
|
min_sdk_version, android_sdk_version):
|
|
# Set minSdkVersion in the manifest to the correct value.
|
|
doc, manifest, app_node = manifest_utils.ParseManifest(original_manifest_path)
|
|
|
|
# TODO(crbug.com/1126301): Should this be done using manifest merging?
|
|
# Add anything in the application node of the extra manifests to the main
|
|
# manifest to prevent unused resource errors.
|
|
for path in extra_manifest_paths:
|
|
_, _, extra_app_node = manifest_utils.ParseManifest(path)
|
|
for node in extra_app_node:
|
|
app_node.append(node)
|
|
|
|
if app_node.find(
|
|
'{%s}allowBackup' % manifest_utils.ANDROID_NAMESPACE) is None:
|
|
# Assume no backup is intended, appeases AllowBackup lint check and keeping
|
|
# it working for manifests that do define android:allowBackup.
|
|
app_node.set('{%s}allowBackup' % manifest_utils.ANDROID_NAMESPACE, 'false')
|
|
|
|
uses_sdk = manifest.find('./uses-sdk')
|
|
if uses_sdk is None:
|
|
uses_sdk = ElementTree.Element('uses-sdk')
|
|
manifest.insert(0, uses_sdk)
|
|
uses_sdk.set('{%s}minSdkVersion' % manifest_utils.ANDROID_NAMESPACE,
|
|
min_sdk_version)
|
|
uses_sdk.set('{%s}targetSdkVersion' % manifest_utils.ANDROID_NAMESPACE,
|
|
android_sdk_version)
|
|
return doc
|
|
|
|
|
|
def _WriteXmlFile(root, path):
|
|
logging.info('Writing xml file %s', path)
|
|
build_utils.MakeDirectory(os.path.dirname(path))
|
|
with build_utils.AtomicOutput(path) as f:
|
|
# Although we can write it just with ElementTree.tostring, using minidom
|
|
# makes it a lot easier to read as a human (also on code search).
|
|
f.write(
|
|
minidom.parseString(ElementTree.tostring(
|
|
root, encoding='utf-8')).toprettyxml(indent=' ').encode('utf-8'))
|
|
|
|
|
|
def _RunLint(lint_binary_path,
|
|
backported_methods_path,
|
|
config_path,
|
|
manifest_path,
|
|
extra_manifest_paths,
|
|
sources,
|
|
classpath,
|
|
cache_dir,
|
|
android_sdk_version,
|
|
aars,
|
|
srcjars,
|
|
min_sdk_version,
|
|
resource_sources,
|
|
resource_zips,
|
|
android_sdk_root,
|
|
lint_gen_dir,
|
|
baseline,
|
|
testonly_target=False,
|
|
warnings_as_errors=False):
|
|
logging.info('Lint starting')
|
|
|
|
cmd = [
|
|
lint_binary_path,
|
|
'--quiet', # Silences lint's "." progress updates.
|
|
'--disable',
|
|
','.join(_DISABLED_ALWAYS),
|
|
]
|
|
|
|
if baseline:
|
|
cmd.extend(['--baseline', baseline])
|
|
if testonly_target:
|
|
cmd.extend(['--disable', ','.join(_DISABLED_FOR_TESTS)])
|
|
|
|
if not manifest_path:
|
|
manifest_path = os.path.join(build_utils.DIR_SOURCE_ROOT, 'build',
|
|
'android', 'AndroidManifest.xml')
|
|
|
|
logging.info('Generating config.xml')
|
|
backported_methods = _RetrieveBackportedMethods(backported_methods_path)
|
|
config_xml_node = _GenerateConfigXmlTree(config_path, backported_methods)
|
|
generated_config_path = os.path.join(lint_gen_dir, 'config.xml')
|
|
_WriteXmlFile(config_xml_node, generated_config_path)
|
|
cmd.extend(['--config', generated_config_path])
|
|
|
|
logging.info('Generating Android manifest file')
|
|
android_manifest_tree = _GenerateAndroidManifest(manifest_path,
|
|
extra_manifest_paths,
|
|
min_sdk_version,
|
|
android_sdk_version)
|
|
# Include the rebased manifest_path in the lint generated path so that it is
|
|
# clear in error messages where the original AndroidManifest.xml came from.
|
|
lint_android_manifest_path = os.path.join(lint_gen_dir, manifest_path)
|
|
_WriteXmlFile(android_manifest_tree.getroot(), lint_android_manifest_path)
|
|
|
|
resource_root_dir = os.path.join(lint_gen_dir, _RES_ZIP_DIR)
|
|
# These are zip files with generated resources (e. g. strings from GRD).
|
|
logging.info('Extracting resource zips')
|
|
for resource_zip in resource_zips:
|
|
# Use a consistent root and name rather than a temporary file so that
|
|
# suppressions can be local to the lint target and the resource target.
|
|
resource_dir = os.path.join(resource_root_dir, resource_zip)
|
|
shutil.rmtree(resource_dir, True)
|
|
os.makedirs(resource_dir)
|
|
resource_sources.extend(
|
|
build_utils.ExtractAll(resource_zip, path=resource_dir))
|
|
|
|
logging.info('Extracting aars')
|
|
aar_root_dir = os.path.join(lint_gen_dir, _AAR_DIR)
|
|
custom_lint_jars = []
|
|
custom_annotation_zips = []
|
|
if aars:
|
|
for aar in aars:
|
|
# androidx custom lint checks require a newer version of lint. Disable
|
|
# until we update see https://crbug.com/1225326
|
|
if 'androidx' in aar:
|
|
continue
|
|
# Use relative source for aar files since they are not generated.
|
|
aar_dir = os.path.join(aar_root_dir,
|
|
os.path.splitext(_SrcRelative(aar))[0])
|
|
shutil.rmtree(aar_dir, True)
|
|
os.makedirs(aar_dir)
|
|
aar_files = build_utils.ExtractAll(aar, path=aar_dir)
|
|
for f in aar_files:
|
|
if f.endswith('lint.jar'):
|
|
custom_lint_jars.append(f)
|
|
elif f.endswith('annotations.zip'):
|
|
custom_annotation_zips.append(f)
|
|
|
|
logging.info('Extracting srcjars')
|
|
srcjar_root_dir = os.path.join(lint_gen_dir, _SRCJAR_DIR)
|
|
srcjar_sources = []
|
|
if srcjars:
|
|
for srcjar in srcjars:
|
|
# Use path without extensions since otherwise the file name includes
|
|
# .srcjar and lint treats it as a srcjar.
|
|
srcjar_dir = os.path.join(srcjar_root_dir, os.path.splitext(srcjar)[0])
|
|
shutil.rmtree(srcjar_dir, True)
|
|
os.makedirs(srcjar_dir)
|
|
# Sadly lint's srcjar support is broken since it only considers the first
|
|
# srcjar. Until we roll a lint version with that fixed, we need to extract
|
|
# it ourselves.
|
|
srcjar_sources.extend(build_utils.ExtractAll(srcjar, path=srcjar_dir))
|
|
|
|
logging.info('Generating project file')
|
|
project_file_root = _GenerateProjectFile(lint_android_manifest_path,
|
|
android_sdk_root, cache_dir, sources,
|
|
classpath, srcjar_sources,
|
|
resource_sources, custom_lint_jars,
|
|
custom_annotation_zips,
|
|
android_sdk_version)
|
|
|
|
project_xml_path = os.path.join(lint_gen_dir, 'project.xml')
|
|
_WriteXmlFile(project_file_root, project_xml_path)
|
|
cmd += ['--project', project_xml_path]
|
|
|
|
logging.info('Preparing environment variables')
|
|
env = os.environ.copy()
|
|
# It is important that lint uses the checked-in JDK11 as it is almost 50%
|
|
# faster than JDK8.
|
|
env['JAVA_HOME'] = build_utils.JAVA_HOME
|
|
# This is necessary so that lint errors print stack traces in stdout.
|
|
env['LINT_PRINT_STACKTRACE'] = 'true'
|
|
if baseline and not os.path.exists(baseline):
|
|
# Generating new baselines is only done locally, and requires more memory to
|
|
# avoid OOMs.
|
|
env['LINT_OPTS'] = '-Xmx4g'
|
|
else:
|
|
# The default set in the wrapper script is 1g, but it seems not enough :(
|
|
env['LINT_OPTS'] = '-Xmx2g'
|
|
|
|
# This filter is necessary for JDK11.
|
|
stderr_filter = build_utils.FilterReflectiveAccessJavaWarnings
|
|
stdout_filter = lambda x: build_utils.FilterLines(x, 'No issues found')
|
|
|
|
start = time.time()
|
|
logging.debug('Lint command %s', ' '.join(cmd))
|
|
failed = True
|
|
try:
|
|
failed = bool(
|
|
build_utils.CheckOutput(cmd,
|
|
env=env,
|
|
print_stdout=True,
|
|
stdout_filter=stdout_filter,
|
|
stderr_filter=stderr_filter,
|
|
fail_on_output=warnings_as_errors))
|
|
finally:
|
|
# When not treating warnings as errors, display the extra footer.
|
|
is_debug = os.environ.get('LINT_DEBUG', '0') != '0'
|
|
|
|
if failed:
|
|
print('- For more help with lint in Chrome:', _LINT_MD_URL)
|
|
if is_debug:
|
|
print('- DEBUG MODE: Here is the project.xml: {}'.format(
|
|
_SrcRelative(project_xml_path)))
|
|
else:
|
|
print('- Run with LINT_DEBUG=1 to enable lint configuration debugging')
|
|
|
|
end = time.time() - start
|
|
logging.info('Lint command took %ss', end)
|
|
if not is_debug:
|
|
shutil.rmtree(aar_root_dir, ignore_errors=True)
|
|
shutil.rmtree(resource_root_dir, ignore_errors=True)
|
|
shutil.rmtree(srcjar_root_dir, ignore_errors=True)
|
|
os.unlink(project_xml_path)
|
|
|
|
logging.info('Lint completed')
|
|
|
|
|
|
def _ParseArgs(argv):
|
|
parser = argparse.ArgumentParser()
|
|
build_utils.AddDepfileOption(parser)
|
|
parser.add_argument('--target-name', help='Fully qualified GN target name.')
|
|
parser.add_argument('--skip-build-server',
|
|
action='store_true',
|
|
help='Avoid using the build server.')
|
|
parser.add_argument('--lint-binary-path',
|
|
required=True,
|
|
help='Path to lint executable.')
|
|
parser.add_argument('--backported-methods',
|
|
help='Path to backported methods file created by R8.')
|
|
parser.add_argument('--cache-dir',
|
|
required=True,
|
|
help='Path to the directory in which the android cache '
|
|
'directory tree should be stored.')
|
|
parser.add_argument('--config-path', help='Path to lint suppressions file.')
|
|
parser.add_argument('--lint-gen-dir',
|
|
required=True,
|
|
help='Path to store generated xml files.')
|
|
parser.add_argument('--stamp', help='Path to stamp upon success.')
|
|
parser.add_argument('--android-sdk-version',
|
|
help='Version (API level) of the Android SDK used for '
|
|
'building.')
|
|
parser.add_argument('--min-sdk-version',
|
|
required=True,
|
|
help='Minimal SDK version to lint against.')
|
|
parser.add_argument('--android-sdk-root',
|
|
required=True,
|
|
help='Lint needs an explicit path to the android sdk.')
|
|
parser.add_argument('--testonly',
|
|
action='store_true',
|
|
help='If set, some checks like UnusedResources will be '
|
|
'disabled since they are not helpful for test '
|
|
'targets.')
|
|
parser.add_argument('--create-cache',
|
|
action='store_true',
|
|
help='Whether this invocation is just warming the cache.')
|
|
parser.add_argument('--warnings-as-errors',
|
|
action='store_true',
|
|
help='Treat all warnings as errors.')
|
|
parser.add_argument('--java-sources',
|
|
help='File containing a list of java sources files.')
|
|
parser.add_argument('--aars', help='GN list of included aars.')
|
|
parser.add_argument('--srcjars', help='GN list of included srcjars.')
|
|
parser.add_argument('--manifest-path',
|
|
help='Path to original AndroidManifest.xml')
|
|
parser.add_argument('--extra-manifest-paths',
|
|
action='append',
|
|
help='GYP-list of manifest paths to merge into the '
|
|
'original AndroidManifest.xml')
|
|
parser.add_argument('--resource-sources',
|
|
default=[],
|
|
action='append',
|
|
help='GYP-list of resource sources files, similar to '
|
|
'java sources files, but for resource files.')
|
|
parser.add_argument('--resource-zips',
|
|
default=[],
|
|
action='append',
|
|
help='GYP-list of resource zips, zip files of generated '
|
|
'resource files.')
|
|
parser.add_argument('--classpath',
|
|
help='List of jars to add to the classpath.')
|
|
parser.add_argument('--baseline',
|
|
help='Baseline file to ignore existing errors and fail '
|
|
'on new errors.')
|
|
|
|
args = parser.parse_args(build_utils.ExpandFileArgs(argv))
|
|
args.java_sources = build_utils.ParseGnList(args.java_sources)
|
|
args.aars = build_utils.ParseGnList(args.aars)
|
|
args.srcjars = build_utils.ParseGnList(args.srcjars)
|
|
args.resource_sources = build_utils.ParseGnList(args.resource_sources)
|
|
args.extra_manifest_paths = build_utils.ParseGnList(args.extra_manifest_paths)
|
|
args.resource_zips = build_utils.ParseGnList(args.resource_zips)
|
|
args.classpath = build_utils.ParseGnList(args.classpath)
|
|
return args
|
|
|
|
|
|
def main():
|
|
build_utils.InitLogging('LINT_DEBUG')
|
|
args = _ParseArgs(sys.argv[1:])
|
|
|
|
# TODO(wnwen): Consider removing lint cache now that there are only two lint
|
|
# invocations.
|
|
# Avoid parallelizing cache creation since lint runs without the cache defeat
|
|
# the purpose of creating the cache in the first place.
|
|
if (not args.create_cache and not args.skip_build_server
|
|
and server_utils.MaybeRunCommand(
|
|
name=args.target_name, argv=sys.argv, stamp_file=args.stamp)):
|
|
return
|
|
|
|
sources = []
|
|
for java_sources_file in args.java_sources:
|
|
sources.extend(build_utils.ReadSourcesList(java_sources_file))
|
|
resource_sources = []
|
|
for resource_sources_file in args.resource_sources:
|
|
resource_sources.extend(build_utils.ReadSourcesList(resource_sources_file))
|
|
|
|
possible_depfile_deps = (args.srcjars + args.resource_zips + sources +
|
|
resource_sources + [
|
|
args.baseline,
|
|
args.manifest_path,
|
|
])
|
|
depfile_deps = [p for p in possible_depfile_deps if p]
|
|
|
|
_RunLint(args.lint_binary_path,
|
|
args.backported_methods,
|
|
args.config_path,
|
|
args.manifest_path,
|
|
args.extra_manifest_paths,
|
|
sources,
|
|
args.classpath,
|
|
args.cache_dir,
|
|
args.android_sdk_version,
|
|
args.aars,
|
|
args.srcjars,
|
|
args.min_sdk_version,
|
|
resource_sources,
|
|
args.resource_zips,
|
|
args.android_sdk_root,
|
|
args.lint_gen_dir,
|
|
args.baseline,
|
|
testonly_target=args.testonly,
|
|
warnings_as_errors=args.warnings_as_errors)
|
|
logging.info('Creating stamp file')
|
|
build_utils.Touch(args.stamp)
|
|
|
|
if args.depfile:
|
|
build_utils.WriteDepfile(args.depfile, args.stamp, depfile_deps)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
sys.exit(main())
|