Source code
Revision control
Copy as Markdown
Other Tools
#!/usr/bin/python2.7
# Copyright 2016 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""Prints all non-system dependencies for the given module.
The primary use-case for this script is to generate the list of python modules
required for .isolate files.
This script should be compatible with Python 2 and Python 3.
"""
import argparse
import fnmatch
import os
import pipes
import sys
# Don't use any helper modules, or else they will end up in the results.
_SRC_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir))
def ComputePythonDependencies():
"""Gets the paths of imported non-system python modules.
A path is assumed to be a "system" import if it is outside of chromium's
src/. The paths will be relative to the current directory.
"""
module_paths = (m.__file__ for m in sys.modules.values()
if m and hasattr(m, '__file__') and m.__file__)
src_paths = set()
for path in module_paths:
if path == __file__:
continue
path = os.path.abspath(path)
if not path.startswith(_SRC_ROOT):
continue
if (path.endswith('.pyc')
or (path.endswith('c') and not os.path.splitext(path)[1])):
path = path[:-1]
src_paths.add(path)
return src_paths
def _NormalizeCommandLine(options):
"""Returns a string that when run from SRC_ROOT replicates the command."""
args = ['build/print_python_deps.py']
root = os.path.relpath(options.root, _SRC_ROOT)
if root != '.':
args.extend(('--root', root))
if options.output:
args.extend(('--output', os.path.relpath(options.output, _SRC_ROOT)))
if options.gn_paths:
args.extend(('--gn-paths',))
for allowlist in sorted(options.allowlists):
args.extend(('--allowlist', os.path.relpath(allowlist, _SRC_ROOT)))
args.append(os.path.relpath(options.module, _SRC_ROOT))
return ' '.join(pipes.quote(x) for x in args)
def _FindPythonInDirectory(directory, allow_test):
"""Returns an iterable of all non-test python files in the given directory."""
files = []
for root, _dirnames, filenames in os.walk(directory):
for filename in filenames:
if filename.endswith('.py') and (allow_test
or not filename.endswith('_test.py')):
yield os.path.join(root, filename)
def _GetTargetPythonVersion(module):
"""Heuristically determines the target module's Python version."""
with open(module) as f:
shebang = f.readline().strip()
default_version = 2
if shebang.startswith('#!'):
# Examples:
# '#!/usr/bin/python'
# '#!/usr/bin/python2.7'
# '#!/usr/bin/python3'
# '#!/usr/bin/env python3'
# '#!/usr/bin/env vpython'
# '#!/usr/bin/env vpython3'
exec_name = os.path.basename(shebang[2:].split(' ')[-1])
for python_prefix in ['python', 'vpython']:
if exec_name.startswith(python_prefix):
version_string = exec_name[len(python_prefix):]
break
else:
raise ValueError('Invalid shebang: ' + shebang)
if version_string:
return int(float(version_string))
return default_version
def _ImportModuleByPath(module_path):
"""Imports a module by its source file."""
# Replace the path entry for print_python_deps.py with the one for the given
# module.
sys.path[0] = os.path.dirname(module_path)
if sys.version_info[0] == 2:
import imp # Python 2 only, since it's deprecated in Python 3.
imp.load_source('NAME', module_path)
else:
module_name = os.path.splitext(os.path.basename(module_path))[0]
import importlib.util # Python 3 only, since it's unavailable in Python 2.
spec = importlib.util.spec_from_file_location(module_name, module_path)
module = importlib.util.module_from_spec(spec)
sys.modules[module_name] = module
spec.loader.exec_module(module)
def main():
parser = argparse.ArgumentParser(
description='Prints all non-system dependencies for the given module.')
parser.add_argument('module',
help='The python module to analyze.')
parser.add_argument('--root', default='.',
help='Directory to make paths relative to.')
parser.add_argument('--output',
help='Write output to a file rather than stdout.')
parser.add_argument('--inplace', action='store_true',
help='Write output to a file with the same path as the '
'module, but with a .pydeps extension. Also sets the '
'root to the module\'s directory.')
parser.add_argument('--no-header', action='store_true',
help='Do not write the "# Generated by" header.')
parser.add_argument('--gn-paths', action='store_true',
help='Write paths as //foo/bar/baz.py')
parser.add_argument('--did-relaunch', action='store_true',
help=argparse.SUPPRESS)
parser.add_argument('--allowlist',
default=[],
action='append',
dest='allowlists',
help='Recursively include all non-test python files '
'within this directory. May be specified multiple times.')
options = parser.parse_args()
if options.inplace:
if options.output:
parser.error('Cannot use --inplace and --output at the same time!')
if not options.module.endswith('.py'):
parser.error('Input module path should end with .py suffix!')
options.output = options.module + 'deps'
options.root = os.path.dirname(options.module)
modules = [options.module]
if os.path.isdir(options.module):
modules = list(_FindPythonInDirectory(options.module, allow_test=True))
if not modules:
parser.error('Input directory does not contain any python files!')
target_versions = [_GetTargetPythonVersion(m) for m in modules]
target_version = target_versions[0]
assert target_version in [2, 3]
assert all(v == target_version for v in target_versions)
current_version = sys.version_info[0]
# Trybots run with vpython as default Python, but with a different config
# from //.vpython. To make the is_vpython test work, and to match the behavior
# of dev machines, the shebang line must be run with python2.7.
#
# E.g. $HOME/.vpython-root/dd50d3/bin/python
# E.g. /b/s/w/ir/cache/vpython/ab5c79/bin/python
is_vpython = 'vpython' in sys.executable
if not is_vpython or target_version != current_version:
# Prevent infinite relaunch if something goes awry.
assert not options.did_relaunch
# Re-launch using vpython will cause us to pick up modules specified in
# //.vpython, but does not cause it to pick up modules defined inline via
# [VPYTHON:BEGIN] ... [VPYTHON:END] comments.
# TODO(agrieve): Add support for this if the need ever arises.
vpython_to_use = {2: 'vpython', 3: 'vpython3'}[target_version]
os.execvp(vpython_to_use, [vpython_to_use] + sys.argv + ['--did-relaunch'])
if current_version == 3:
# Work-around for protobuf library not being loadable via importlib
# This is needed due to compile_resources.py.
import importlib._bootstrap_external
importlib._bootstrap_external._NamespacePath.sort = lambda self, **_: 0
paths_set = set()
try:
for module in modules:
_ImportModuleByPath(module)
paths_set.update(ComputePythonDependencies())
except Exception:
# Output extra diagnostics when loading the script fails.
sys.stderr.write('Error running print_python_deps.py.\n')
sys.stderr.write('is_vpython={}\n'.format(is_vpython))
sys.stderr.write('did_relanuch={}\n'.format(options.did_relaunch))
sys.stderr.write('python={}\n'.format(sys.executable))
raise
for path in options.allowlists:
paths_set.update(
os.path.abspath(p)
for p in _FindPythonInDirectory(path, allow_test=False))
paths = [os.path.relpath(p, options.root) for p in paths_set]
normalized_cmdline = _NormalizeCommandLine(options)
out = open(options.output, 'w') if options.output else sys.stdout
with out:
if not options.no_header:
out.write('# Generated by running:\n')
out.write('# %s\n' % normalized_cmdline)
prefix = '//' if options.gn_paths else ''
for path in sorted(paths):
out.write(prefix + path + '\n')
if __name__ == '__main__':
sys.exit(main())