Source code
Revision control
Copy as Markdown
Other Tools
#!/usr/bin/env vpython
# Copyright 2019 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""Helper script used to manage locale-related files in Chromium.
This script is used to check, and potentially fix, many locale-related files
in your Chromium workspace, such as:
- GRIT input files (.grd) and the corresponding translations (.xtb).
- BUILD.gn files listing Android localized resource string resource .xml
generated by GRIT for all supported Chrome locales. These correspond to
<output> elements that use the type="android" attribute.
The --scan-dir <dir> option can be used to check for all files under a specific
directory, and the --fix-inplace option can be used to try fixing any file
that doesn't pass the check.
This can be very handy to avoid tedious and repetitive work when adding new
translations / locales to the Chrome code base, since this script can update
said input files for you.
Important note: checks and fix may fail on some input files. For example
remoting/resources/remoting_strings.grd contains an in-line comment element
inside its <outputs> section that breaks the script. The check will fail, and
trying to fix it too, but at least the file will not be modified.
"""
from __future__ import print_function
import argparse
import json
import os
import re
import shutil
import subprocess
import sys
import unittest
# Assume this script is under build/
_SCRIPT_DIR = os.path.dirname(__file__)
_SCRIPT_NAME = os.path.join(_SCRIPT_DIR, os.path.basename(__file__))
_TOP_SRC_DIR = os.path.join(_SCRIPT_DIR, '..')
# Need to import android/gyp/util/resource_utils.py here.
sys.path.insert(0, os.path.join(_SCRIPT_DIR, 'android/gyp'))
from util import build_utils
from util import resource_utils
# This locale is the default and doesn't have translations.
_DEFAULT_LOCALE = 'en-US'
# Misc terminal codes to provide human friendly progress output.
_CONSOLE_CODE_MOVE_CURSOR_TO_COLUMN_0 = '\x1b[0G'
_CONSOLE_CODE_ERASE_LINE = '\x1b[K'
_CONSOLE_START_LINE = (
_CONSOLE_CODE_MOVE_CURSOR_TO_COLUMN_0 + _CONSOLE_CODE_ERASE_LINE)
##########################################################################
##########################################################################
#####
##### G E N E R I C H E L P E R F U N C T I O N S
#####
##########################################################################
##########################################################################
def _FixChromiumLangAttribute(lang):
"""Map XML "lang" attribute values to Chromium locale names."""
_CHROMIUM_LANG_FIXES = {
'en': 'en-US', # For now, Chromium doesn't have an 'en' locale.
'iw': 'he', # 'iw' is the obsolete form of ISO 639-1 for Hebrew
'no': 'nb', # 'no' is used by the Translation Console for Norwegian (nb).
}
return _CHROMIUM_LANG_FIXES.get(lang, lang)
def _FixTranslationConsoleLocaleName(locale):
_FIXES = {
'nb': 'no', # Norwegian.
'he': 'iw', # Hebrew
}
return _FIXES.get(locale, locale)
def _CompareLocaleLists(list_a, list_expected, list_name):
"""Compare two lists of locale names. Print errors if they differ.
Args:
list_a: First list of locales.
list_expected: Second list of locales, as expected.
list_name: Name of list printed in error messages.
Returns:
On success, return False. On error, print error messages and return True.
"""
errors = []
missing_locales = sorted(set(list_a) - set(list_expected))
if missing_locales:
errors.append('Missing locales: %s' % missing_locales)
extra_locales = sorted(set(list_expected) - set(list_a))
if extra_locales:
errors.append('Unexpected locales: %s' % extra_locales)
if errors:
print('Errors in %s definition:' % list_name)
for error in errors:
print(' %s\n' % error)
return True
return False
def _BuildIntervalList(input_list, predicate):
"""Find ranges of contiguous list items that pass a given predicate.
Args:
input_list: An input list of items of any type.
predicate: A function that takes a list item and return True if it
passes a given test.
Returns:
A list of (start_pos, end_pos) tuples, where all items in
[start_pos, end_pos) pass the predicate.
"""
result = []
size = len(input_list)
start = 0
while True:
# Find first item in list that passes the predicate.
while start < size and not predicate(input_list[start]):
start += 1
if start >= size:
return result
# Find first item in the rest of the list that does not pass the
# predicate.
end = start + 1
while end < size and predicate(input_list[end]):
end += 1
result.append((start, end))
start = end + 1
def _SortListSubRange(input_list, start, end, key_func):
"""Sort an input list's sub-range according to a specific key function.
Args:
input_list: An input list.
start: Sub-range starting position in list.
end: Sub-range limit position in list.
key_func: A function that extracts a sort key from a line.
Returns:
A copy of |input_list|, with all items in [|start|, |end|) sorted
according to |key_func|.
"""
result = input_list[:start]
inputs = []
for pos in xrange(start, end):
line = input_list[pos]
key = key_func(line)
inputs.append((key, line))
for _, line in sorted(inputs):
result.append(line)
result += input_list[end:]
return result
def _SortElementsRanges(lines, element_predicate, element_key):
"""Sort all elements of a given type in a list of lines by a given key.
Args:
lines: input lines.
element_predicate: predicate function to select elements to sort.
element_key: lambda returning a comparison key for each element that
passes the predicate.
Returns:
A new list of input lines, with lines [start..end) sorted.
"""
intervals = _BuildIntervalList(lines, element_predicate)
for start, end in intervals:
lines = _SortListSubRange(lines, start, end, element_key)
return lines
def _ProcessFile(input_file, locales, check_func, fix_func):
"""Process a given input file, potentially fixing it.
Args:
input_file: Input file path.
locales: List of Chrome locales to consider / expect.
check_func: A lambda called to check the input file lines with
(input_lines, locales) argument. It must return an list of error
messages, or None on success.
fix_func: None, or a lambda called to fix the input file lines with
(input_lines, locales). It must return the new list of lines for
the input file, and may raise an Exception in case of error.
Returns:
True at the moment.
"""
print('%sProcessing %s...' % (_CONSOLE_START_LINE, input_file), end=' ')
sys.stdout.flush()
with open(input_file) as f:
input_lines = f.readlines()
errors = check_func(input_file, input_lines, locales)
if errors:
print('\n%s%s' % (_CONSOLE_START_LINE, '\n'.join(errors)))
if fix_func:
try:
input_lines = fix_func(input_file, input_lines, locales)
output = ''.join(input_lines)
with open(input_file, 'wt') as f:
f.write(output)
print('Fixed %s.' % input_file)
except Exception as e: # pylint: disable=broad-except
print('Skipped %s: %s' % (input_file, e))
return True
def _ScanDirectoriesForFiles(scan_dirs, file_predicate):
"""Scan a directory for files that match a given predicate.
Args:
scan_dir: A list of top-level directories to start scan in.
file_predicate: lambda function which is passed the file's base name
and returns True if its full path, relative to |scan_dir|, should be
passed in the result.
Returns:
A list of file full paths.
"""
result = []
for src_dir in scan_dirs:
for root, _, files in os.walk(src_dir):
result.extend(os.path.join(root, f) for f in files if file_predicate(f))
return result
def _WriteFile(file_path, file_data):
"""Write |file_data| to |file_path|."""
with open(file_path, 'w') as f:
f.write(file_data)
def _FindGnExecutable():
"""Locate the real GN executable used by this Chromium checkout.
This is needed because the depot_tools 'gn' wrapper script will look
for .gclient and other things we really don't need here.
Returns:
Path of real host GN executable from current Chromium src/ checkout.
"""
# Simply scan buildtools/*/gn and return the first one found so we don't
# have to guess the platform-specific sub-directory name (e.g. 'linux64'
# for 64-bit Linux machines).
buildtools_dir = os.path.join(_TOP_SRC_DIR, 'buildtools')
for subdir in os.listdir(buildtools_dir):
subdir_path = os.path.join(buildtools_dir, subdir)
if not os.path.isdir(subdir_path):
continue
gn_path = os.path.join(subdir_path, 'gn')
if os.path.exists(gn_path):
return gn_path
return None
def _PrettyPrintListAsLines(input_list, available_width, trailing_comma=False):
result = []
input_str = ', '.join(input_list)
while len(input_str) > available_width:
pos = input_str.rfind(',', 0, available_width)
result.append(input_str[:pos + 1])
input_str = input_str[pos + 1:].lstrip()
if trailing_comma and input_str:
input_str += ','
result.append(input_str)
return result
class _PrettyPrintListAsLinesTest(unittest.TestCase):
def test_empty_list(self):
self.assertListEqual([''], _PrettyPrintListAsLines([], 10))
def test_wrapping(self):
input_list = ['foo', 'bar', 'zoo', 'tool']
self.assertListEqual(
_PrettyPrintListAsLines(input_list, 8),
['foo,', 'bar,', 'zoo,', 'tool'])
self.assertListEqual(
_PrettyPrintListAsLines(input_list, 12), ['foo, bar,', 'zoo, tool'])
self.assertListEqual(
_PrettyPrintListAsLines(input_list, 79), ['foo, bar, zoo, tool'])
def test_trailing_comma(self):
input_list = ['foo', 'bar', 'zoo', 'tool']
self.assertListEqual(
_PrettyPrintListAsLines(input_list, 8, trailing_comma=True),
['foo,', 'bar,', 'zoo,', 'tool,'])
self.assertListEqual(
_PrettyPrintListAsLines(input_list, 12, trailing_comma=True),
['foo, bar,', 'zoo, tool,'])
self.assertListEqual(
_PrettyPrintListAsLines(input_list, 79, trailing_comma=True),
['foo, bar, zoo, tool,'])
##########################################################################
##########################################################################
#####
##### L O C A L E S L I S T S
#####
##########################################################################
##########################################################################
# Various list of locales that will be extracted from build/config/locales.gni
# Do not use these directly, use ChromeLocales(), and IosUnsupportedLocales()
# instead to access these lists.
_INTERNAL_CHROME_LOCALES = []
_INTERNAL_IOS_UNSUPPORTED_LOCALES = []
def ChromeLocales():
"""Return the list of all locales supported by Chrome."""
if not _INTERNAL_CHROME_LOCALES:
_ExtractAllChromeLocalesLists()
return _INTERNAL_CHROME_LOCALES
def IosUnsupportedLocales():
"""Return the list of locales that are unsupported on iOS."""
if not _INTERNAL_IOS_UNSUPPORTED_LOCALES:
_ExtractAllChromeLocalesLists()
return _INTERNAL_IOS_UNSUPPORTED_LOCALES
def _PrepareTinyGnWorkspace(work_dir, out_subdir_name='out'):
"""Populate an empty directory with a tiny set of working GN config files.
This allows us to run 'gn gen <out> --root <work_dir>' as fast as possible
to generate files containing the locales list. This takes about 300ms on
a decent machine, instead of more than 5 seconds when running the equivalent
commands from a real Chromium workspace, which requires regenerating more
than 23k targets.
Args:
work_dir: target working directory.
out_subdir_name: Name of output sub-directory.
Returns:
Full path of output directory created inside |work_dir|.
"""
# Create top-level .gn file that must point to the BUILDCONFIG.gn.
_WriteFile(os.path.join(work_dir, '.gn'),
'buildconfig = "//BUILDCONFIG.gn"\n')
# Create BUILDCONFIG.gn which must set a default toolchain. Also add
# all variables that may be used in locales.gni in a declare_args() block.
_WriteFile(
os.path.join(work_dir, 'BUILDCONFIG.gn'),
r'''set_default_toolchain("toolchain")
declare_args () {
is_ios = false
is_android = true
}
''')
# Create fake toolchain required by BUILDCONFIG.gn.
os.mkdir(os.path.join(work_dir, 'toolchain'))
_WriteFile(os.path.join(work_dir, 'toolchain', 'BUILD.gn'),
r'''toolchain("toolchain") {
tool("stamp") {
command = "touch {{output}}" # Required by action()
}
}
''')
# Create top-level BUILD.gn, GN requires at least one target to build so do
# that with a fake action which will never be invoked. Also write the locales
# to misc files in the output directory.
_WriteFile(
os.path.join(work_dir, 'BUILD.gn'), r'''import("//locales.gni")
action("create_foo") { # fake action to avoid GN complaints.
script = "//build/create_foo.py"
inputs = []
outputs = [ "$target_out_dir/$target_name" ]
}
# Write the locales lists to files in the output directory.
_filename = root_build_dir + "/foo"
write_file(_filename + ".locales", locales, "json")
write_file(_filename + ".ios_unsupported_locales",
ios_unsupported_locales,
"json")
''')
# Copy build/config/locales.gni to the workspace, as required by BUILD.gn.
shutil.copyfile(os.path.join(_TOP_SRC_DIR, 'build', 'config', 'locales.gni'),
os.path.join(work_dir, 'locales.gni'))
# Create output directory.
out_path = os.path.join(work_dir, out_subdir_name)
os.mkdir(out_path)
# And ... we're good.
return out_path
# Set this global variable to the path of a given temporary directory
# before calling _ExtractAllChromeLocalesLists() if you want to debug
# the locales list extraction process.
_DEBUG_LOCALES_WORK_DIR = None
def _ReadJsonList(file_path):
"""Read a JSON file that must contain a list, and return it."""
with open(file_path) as f:
data = json.load(f)
assert isinstance(data, list), "JSON file %s is not a list!" % file_path
return [item.encode('utf8') for item in data]
def _ExtractAllChromeLocalesLists():
with build_utils.TempDir() as tmp_path:
if _DEBUG_LOCALES_WORK_DIR:
tmp_path = _DEBUG_LOCALES_WORK_DIR
build_utils.DeleteDirectory(tmp_path)
build_utils.MakeDirectory(tmp_path)
out_path = _PrepareTinyGnWorkspace(tmp_path, 'out')
# NOTE: The file suffixes used here should be kept in sync with
# build/config/locales.gni
gn_executable = _FindGnExecutable()
try:
subprocess.check_output(
[gn_executable, 'gen', out_path, '--root=' + tmp_path])
except subprocess.CalledProcessError as e:
print(e.output)
raise e
global _INTERNAL_CHROME_LOCALES
_INTERNAL_CHROME_LOCALES = _ReadJsonList(
os.path.join(out_path, 'foo.locales'))
global _INTERNAL_IOS_UNSUPPORTED_LOCALES
_INTERNAL_IOS_UNSUPPORTED_LOCALES = _ReadJsonList(
os.path.join(out_path, 'foo.ios_unsupported_locales'))
##########################################################################
##########################################################################
#####
##### G R D H E L P E R F U N C T I O N S
#####
##########################################################################
##########################################################################
# Technical note:
#
# Even though .grd files are XML, an xml parser library is not used in order
# to preserve the original file's structure after modification. ElementTree
# tends to re-order attributes in each element when re-writing an XML
# document tree, which is undesirable here.
#
# Thus simple line-based regular expression matching is used instead.
#
# Misc regular expressions used to match elements and their attributes.
_RE_OUTPUT_ELEMENT = re.compile(r'<output (.*)\s*/>')
_RE_TRANSLATION_ELEMENT = re.compile(r'<file( | .* )path="(.*\.xtb)".*/>')
_RE_FILENAME_ATTRIBUTE = re.compile(r'filename="([^"]*)"')
_RE_LANG_ATTRIBUTE = re.compile(r'lang="([^"]*)"')
_RE_PATH_ATTRIBUTE = re.compile(r'path="([^"]*)"')
_RE_TYPE_ANDROID_ATTRIBUTE = re.compile(r'type="android"')
def _IsGritInputFile(input_file):
"""Returns True iff this is a GRIT input file."""
return input_file.endswith('.grd')
def _GetXmlLangAttribute(xml_line):
"""Extract the lang attribute value from an XML input line."""
m = _RE_LANG_ATTRIBUTE.search(xml_line)
if not m:
return None
return m.group(1)
class _GetXmlLangAttributeTest(unittest.TestCase):
TEST_DATA = {
'': None,
'foo': None,
'lang=foo': None,
'lang="foo"': 'foo',
'<something lang="foo bar" />': 'foo bar',
'<file lang="fr-CA" path="path/to/strings_fr-CA.xtb" />': 'fr-CA',
}
def test_GetXmlLangAttribute(self):
for test_line, expected in self.TEST_DATA.iteritems():
self.assertEquals(_GetXmlLangAttribute(test_line), expected)
def _SortGrdElementsRanges(grd_lines, element_predicate):
"""Sort all .grd elements of a given type by their lang attribute."""
return _SortElementsRanges(grd_lines, element_predicate, _GetXmlLangAttribute)
def _CheckGrdElementRangeLang(grd_lines, start, end, wanted_locales):
"""Check the element 'lang' attributes in specific .grd lines range.
This really checks the following:
- Each item has a correct 'lang' attribute.
- There are no duplicated lines for the same 'lang' attribute.
- That there are no extra locales that Chromium doesn't want.
- That no wanted locale is missing.
Args:
grd_lines: Input .grd lines.
start: Sub-range start position in input line list.
end: Sub-range limit position in input line list.
wanted_locales: Set of wanted Chromium locale names.
Returns:
List of error message strings for this input. Empty on success.
"""
errors = []
locales = set()
for pos in xrange(start, end):
line = grd_lines[pos]
lang = _GetXmlLangAttribute(line)
if not lang:
errors.append('%d: Missing "lang" attribute in <output> element' % pos +
1)
continue
cr_locale = _FixChromiumLangAttribute(lang)
if cr_locale in locales:
errors.append(
'%d: Redefinition of <output> for "%s" locale' % (pos + 1, lang))
locales.add(cr_locale)
extra_locales = locales.difference(wanted_locales)
if extra_locales:
errors.append('%d-%d: Extra locales found: %s' % (start + 1, end + 1,
sorted(extra_locales)))
missing_locales = wanted_locales.difference(locales)
if missing_locales:
errors.append('%d-%d: Missing locales: %s' % (start + 1, end + 1,
sorted(missing_locales)))
return errors
##########################################################################
##########################################################################
#####
##### G R D A N D R O I D O U T P U T S
#####
##########################################################################
##########################################################################
def _IsGrdAndroidOutputLine(line):
"""Returns True iff this is an Android-specific <output> line."""
m = _RE_OUTPUT_ELEMENT.search(line)
if m:
return 'type="android"' in m.group(1)
return False
assert _IsGrdAndroidOutputLine(' <output type="android"/>')
# Many of the functions below have unused arguments due to genericity.
# pylint: disable=unused-argument
def _CheckGrdElementRangeAndroidOutputFilename(grd_lines, start, end,
wanted_locales):
"""Check all <output> elements in specific input .grd lines range.
This really checks the following:
- Filenames exist for each listed locale.
- Filenames are well-formed.
Args:
grd_lines: Input .grd lines.
start: Sub-range start position in input line list.
end: Sub-range limit position in input line list.
wanted_locales: Set of wanted Chromium locale names.
Returns:
List of error message strings for this input. Empty on success.
"""
errors = []
for pos in xrange(start, end):
line = grd_lines[pos]
lang = _GetXmlLangAttribute(line)
if not lang:
continue
cr_locale = _FixChromiumLangAttribute(lang)
m = _RE_FILENAME_ATTRIBUTE.search(line)
if not m:
errors.append('%d: Missing filename attribute in <output> element' % pos +
1)
else:
filename = m.group(1)
if not filename.endswith('.xml'):
errors.append(
'%d: Filename should end with ".xml": %s' % (pos + 1, filename))
dirname = os.path.basename(os.path.dirname(filename))
prefix = ('values-%s' % resource_utils.ToAndroidLocaleName(cr_locale)
if cr_locale != _DEFAULT_LOCALE else 'values')
if dirname != prefix:
errors.append(
'%s: Directory name should be %s: %s' % (pos + 1, prefix, filename))
return errors
def _CheckGrdAndroidOutputElements(grd_file, grd_lines, wanted_locales):
"""Check all <output> elements related to Android.
Args:
grd_file: Input .grd file path.
grd_lines: List of input .grd lines.
wanted_locales: set of wanted Chromium locale names.
Returns:
List of error message strings. Empty on success.
"""
intervals = _BuildIntervalList(grd_lines, _IsGrdAndroidOutputLine)
errors = []
for start, end in intervals:
errors += _CheckGrdElementRangeLang(grd_lines, start, end, wanted_locales)
errors += _CheckGrdElementRangeAndroidOutputFilename(grd_lines, start, end,
wanted_locales)
return errors
def _AddMissingLocalesInGrdAndroidOutputs(grd_file, grd_lines, wanted_locales):
"""Fix an input .grd line by adding missing Android outputs.
Args:
grd_file: Input .grd file path.
grd_lines: Input .grd line list.
wanted_locales: set of Chromium locale names.
Returns:
A new list of .grd lines, containing new <output> elements when needed
for locales from |wanted_locales| that were not part of the input.
"""
intervals = _BuildIntervalList(grd_lines, _IsGrdAndroidOutputLine)
for start, end in reversed(intervals):
locales = set()
for pos in xrange(start, end):
lang = _GetXmlLangAttribute(grd_lines[pos])
locale = _FixChromiumLangAttribute(lang)
locales.add(locale)
missing_locales = wanted_locales.difference(locales)
if not missing_locales:
continue
src_locale = 'bg'
src_lang_attribute = 'lang="%s"' % src_locale
src_line = None
for pos in xrange(start, end):
if src_lang_attribute in grd_lines[pos]:
src_line = grd_lines[pos]
break
if not src_line:
raise Exception(
'Cannot find <output> element with "%s" lang attribute' % src_locale)
line_count = end - 1
for locale in missing_locales:
android_locale = resource_utils.ToAndroidLocaleName(locale)
dst_line = src_line.replace(
'lang="%s"' % src_locale, 'lang="%s"' % locale).replace(
'values-%s/' % src_locale, 'values-%s/' % android_locale)
grd_lines.insert(line_count, dst_line)
line_count += 1
# Sort the new <output> elements.
return _SortGrdElementsRanges(grd_lines, _IsGrdAndroidOutputLine)
##########################################################################
##########################################################################
#####
##### G R D T R A N S L A T I O N S
#####
##########################################################################
##########################################################################
def _IsTranslationGrdOutputLine(line):
"""Returns True iff this is an output .xtb <file> element."""
m = _RE_TRANSLATION_ELEMENT.search(line)
return m is not None
class _IsTranslationGrdOutputLineTest(unittest.TestCase):
def test_GrdTranslationOutputLines(self):
_VALID_INPUT_LINES = [
'<file path="foo/bar.xtb" />',
'<file path="foo/bar.xtb"/>',
'<file lang="fr-CA" path="translations/aw_strings_fr-CA.xtb"/>',
'<file lang="fr-CA" path="translations/aw_strings_fr-CA.xtb" />',
' <file path="translations/aw_strings_ar.xtb" lang="ar" />',
]
_INVALID_INPUT_LINES = ['<file path="foo/bar.xml" />']
for line in _VALID_INPUT_LINES:
self.assertTrue(
_IsTranslationGrdOutputLine(line),
'_IsTranslationGrdOutputLine() returned False for [%s]' % line)
for line in _INVALID_INPUT_LINES:
self.assertFalse(
_IsTranslationGrdOutputLine(line),
'_IsTranslationGrdOutputLine() returned True for [%s]' % line)
def _CheckGrdTranslationElementRange(grd_lines, start, end,
wanted_locales):
"""Check all <translations> sub-elements in specific input .grd lines range.
This really checks the following:
- Each item has a 'path' attribute.
- Each such path value ends up with '.xtb'.
Args:
grd_lines: Input .grd lines.
start: Sub-range start position in input line list.
end: Sub-range limit position in input line list.
wanted_locales: Set of wanted Chromium locale names.
Returns:
List of error message strings for this input. Empty on success.
"""
errors = []
for pos in xrange(start, end):
line = grd_lines[pos]
lang = _GetXmlLangAttribute(line)
if not lang:
continue
m = _RE_PATH_ATTRIBUTE.search(line)
if not m:
errors.append('%d: Missing path attribute in <file> element' % pos +
1)
else:
filename = m.group(1)
if not filename.endswith('.xtb'):
errors.append(
'%d: Path should end with ".xtb": %s' % (pos + 1, filename))
return errors
def _CheckGrdTranslations(grd_file, grd_lines, wanted_locales):
"""Check all <file> elements that correspond to an .xtb output file.
Args:
grd_file: Input .grd file path.
grd_lines: List of input .grd lines.
wanted_locales: set of wanted Chromium locale names.
Returns:
List of error message strings. Empty on success.
"""
wanted_locales = wanted_locales - set([_DEFAULT_LOCALE])
intervals = _BuildIntervalList(grd_lines, _IsTranslationGrdOutputLine)
errors = []
for start, end in intervals:
errors += _CheckGrdElementRangeLang(grd_lines, start, end, wanted_locales)
errors += _CheckGrdTranslationElementRange(grd_lines, start, end,
wanted_locales)
return errors
# Regular expression used to replace the lang attribute inside .xtb files.
_RE_TRANSLATIONBUNDLE = re.compile('<translationbundle lang="(.*)">')
def _CreateFakeXtbFileFrom(src_xtb_path, dst_xtb_path, dst_locale):
"""Create a fake .xtb file.
Args:
src_xtb_path: Path to source .xtb file to copy from.
dst_xtb_path: Path to destination .xtb file to write to.
dst_locale: Destination locale, the lang attribute in the source file
will be substituted with this value before its lines are written
to the destination file.
"""
with open(src_xtb_path) as f:
src_xtb_lines = f.readlines()
def replace_xtb_lang_attribute(line):
m = _RE_TRANSLATIONBUNDLE.search(line)
if not m:
return line
return line[:m.start(1)] + dst_locale + line[m.end(1):]
dst_xtb_lines = [replace_xtb_lang_attribute(line) for line in src_xtb_lines]
with build_utils.AtomicOutput(dst_xtb_path) as tmp:
tmp.writelines(dst_xtb_lines)
def _AddMissingLocalesInGrdTranslations(grd_file, grd_lines, wanted_locales):
"""Fix an input .grd line by adding missing Android outputs.
This also creates fake .xtb files from the one provided for 'en-GB'.
Args:
grd_file: Input .grd file path.
grd_lines: Input .grd line list.
wanted_locales: set of Chromium locale names.
Returns:
A new list of .grd lines, containing new <output> elements when needed
for locales from |wanted_locales| that were not part of the input.
"""
wanted_locales = wanted_locales - set([_DEFAULT_LOCALE])
intervals = _BuildIntervalList(grd_lines, _IsTranslationGrdOutputLine)
for start, end in reversed(intervals):
locales = set()
for pos in xrange(start, end):
lang = _GetXmlLangAttribute(grd_lines[pos])
locale = _FixChromiumLangAttribute(lang)
locales.add(locale)
missing_locales = wanted_locales.difference(locales)
if not missing_locales:
continue
src_locale = 'en-GB'
src_lang_attribute = 'lang="%s"' % src_locale
src_line = None
for pos in xrange(start, end):
if src_lang_attribute in grd_lines[pos]:
src_line = grd_lines[pos]
break
if not src_line:
raise Exception(
'Cannot find <file> element with "%s" lang attribute' % src_locale)
src_path = os.path.join(
os.path.dirname(grd_file),
_RE_PATH_ATTRIBUTE.search(src_line).group(1))
line_count = end - 1
for locale in missing_locales:
dst_line = src_line.replace(
'lang="%s"' % src_locale, 'lang="%s"' % locale).replace(
'_%s.xtb' % src_locale, '_%s.xtb' % locale)
grd_lines.insert(line_count, dst_line)
line_count += 1
dst_path = src_path.replace('_%s.xtb' % src_locale, '_%s.xtb' % locale)
_CreateFakeXtbFileFrom(src_path, dst_path, locale)
# Sort the new <output> elements.
return _SortGrdElementsRanges(grd_lines, _IsTranslationGrdOutputLine)
##########################################################################
##########################################################################
#####
##### G N A N D R O I D O U T P U T S
#####
##########################################################################
##########################################################################
_RE_GN_VALUES_LIST_LINE = re.compile(
r'^\s*".*values(\-([A-Za-z0-9-]+))?/.*\.xml",\s*$')
def _IsBuildGnInputFile(input_file):
"""Returns True iff this is a BUILD.gn file."""
return os.path.basename(input_file) == 'BUILD.gn'
def _GetAndroidGnOutputLocale(line):
"""Check a GN list, and return its Android locale if it is an output .xml"""
m = _RE_GN_VALUES_LIST_LINE.match(line)
if not m:
return None
if m.group(1): # First group is optional and contains group 2.
return m.group(2)
return resource_utils.ToAndroidLocaleName(_DEFAULT_LOCALE)
def _IsAndroidGnOutputLine(line):
"""Returns True iff this is an Android-specific localized .xml output."""
return _GetAndroidGnOutputLocale(line) != None
def _CheckGnOutputsRangeForLocalizedStrings(gn_lines, start, end):
"""Check that a range of GN lines corresponds to localized strings.
Special case: Some BUILD.gn files list several non-localized .xml files
that should be ignored by this function, e.g. in
components/cronet/android/BUILD.gn, the following appears:
inputs = [
...
"sample/res/layout/activity_main.xml",
"sample/res/layout/dialog_url.xml",
"sample/res/values/dimens.xml",
"sample/res/values/strings.xml",
...
]
These are non-localized strings, and should be ignored. This function is
used to detect them quickly.
"""
for pos in xrange(start, end):
if not 'values/' in gn_lines[pos]:
return True
return False
def _CheckGnOutputsRange(gn_lines, start, end, wanted_locales):
if not _CheckGnOutputsRangeForLocalizedStrings(gn_lines, start, end):
return []
errors = []
locales = set()
for pos in xrange(start, end):
line = gn_lines[pos]
android_locale = _GetAndroidGnOutputLocale(line)
assert android_locale != None
cr_locale = resource_utils.ToChromiumLocaleName(android_locale)
if cr_locale in locales:
errors.append('%s: Redefinition of output for "%s" locale' %
(pos + 1, android_locale))
locales.add(cr_locale)
extra_locales = locales.difference(wanted_locales)
if extra_locales:
errors.append('%d-%d: Extra locales: %s' % (start + 1, end + 1,
sorted(extra_locales)))
missing_locales = wanted_locales.difference(locales)
if missing_locales:
errors.append('%d-%d: Missing locales: %s' % (start + 1, end + 1,
sorted(missing_locales)))
return errors
def _CheckGnAndroidOutputs(gn_file, gn_lines, wanted_locales):
intervals = _BuildIntervalList(gn_lines, _IsAndroidGnOutputLine)
errors = []
for start, end in intervals:
errors += _CheckGnOutputsRange(gn_lines, start, end, wanted_locales)
return errors
def _AddMissingLocalesInGnAndroidOutputs(gn_file, gn_lines, wanted_locales):
intervals = _BuildIntervalList(gn_lines, _IsAndroidGnOutputLine)
# NOTE: Since this may insert new lines to each interval, process the
# list in reverse order to maintain valid (start,end) positions during
# the iteration.
for start, end in reversed(intervals):
if not _CheckGnOutputsRangeForLocalizedStrings(gn_lines, start, end):
continue
locales = set()
for pos in xrange(start, end):
lang = _GetAndroidGnOutputLocale(gn_lines[pos])
locale = resource_utils.ToChromiumLocaleName(lang)
locales.add(locale)
missing_locales = wanted_locales.difference(locales)
if not missing_locales:
continue
src_locale = 'bg'
src_values = 'values-%s/' % resource_utils.ToAndroidLocaleName(src_locale)
src_line = None
for pos in xrange(start, end):
if src_values in gn_lines[pos]:
src_line = gn_lines[pos]
break
if not src_line:
raise Exception(
'Cannot find output list item with "%s" locale' % src_locale)
line_count = end - 1
for locale in missing_locales:
if locale == _DEFAULT_LOCALE:
dst_line = src_line.replace('values-%s/' % src_locale, 'values/')
else:
dst_line = src_line.replace(
'values-%s/' % src_locale,
'values-%s/' % resource_utils.ToAndroidLocaleName(locale))
gn_lines.insert(line_count, dst_line)
line_count += 1
gn_lines = _SortListSubRange(
gn_lines, start, line_count,
lambda line: _RE_GN_VALUES_LIST_LINE.match(line).group(1))
return gn_lines
##########################################################################
##########################################################################
#####
##### T R A N S L A T I O N E X P E C T A T I O N S
#####
##########################################################################
##########################################################################
_EXPECTATIONS_FILENAME = 'translation_expectations.pyl'
# Technical note: the format of translation_expectations.pyl
# is a 'Python literal', which defines a python dictionary, so should
# be easy to parse. However, when modifying it, care should be taken
# to respect the line comments and the order of keys within the text
# file.
def _ReadPythonLiteralFile(pyl_path):
"""Read a .pyl file into a Python data structure."""
with open(pyl_path) as f:
pyl_content = f.read()
# Evaluate as a Python data structure, use an empty global
# and local dictionary.
return eval(pyl_content, dict(), dict())
def _UpdateLocalesInExpectationLines(pyl_lines,
wanted_locales,
available_width=79):
"""Update the locales list(s) found in an expectations file.
Args:
pyl_lines: Iterable of input lines from the file.
wanted_locales: Set or list of new locale names.
available_width: Optional, number of character colums used
to word-wrap the new list items.
Returns:
New list of updated lines.
"""
locales_list = ['"%s"' % loc for loc in sorted(wanted_locales)]
result = []
line_count = len(pyl_lines)
line_num = 0
DICT_START = '"languages": ['
while line_num < line_count:
line = pyl_lines[line_num]
line_num += 1
result.append(line)
# Look for start of "languages" dictionary.
pos = line.find(DICT_START)
if pos < 0:
continue
start_margin = pos
start_line = line_num
# Skip over all lines from the list.
while (line_num < line_count and
not pyl_lines[line_num].rstrip().endswith('],')):
line_num += 1
continue
if line_num == line_count:
raise Exception('%d: Missing list termination!' % start_line)
# Format the new list according to the new margin.
locale_width = available_width - (start_margin + 2)
locale_lines = _PrettyPrintListAsLines(
locales_list, locale_width, trailing_comma=True)
for locale_line in locale_lines:
result.append(' ' * (start_margin + 2) + locale_line)
result.append(' ' * start_margin + '],')
line_num += 1
return result
class _UpdateLocalesInExpectationLinesTest(unittest.TestCase):
def test_simple(self):
self.maxDiff = 1000
input_text = r'''
# This comment should be preserved
# 23456789012345678901234567890123456789
{
"android_grd": {
"languages": [
"aa", "bb", "cc", "dd", "ee",
"ff", "gg", "hh", "ii", "jj",
"kk"],
},
# Example with bad indentation in input.
"another_grd": {
"languages": [
"aa", "bb", "cc", "dd", "ee", "ff", "gg", "hh", "ii", "jj", "kk",
],
},
}
'''
expected_text = r'''
# This comment should be preserved
# 23456789012345678901234567890123456789
{
"android_grd": {
"languages": [
"A2", "AA", "BB", "CC", "DD",
"E2", "EE", "FF", "GG", "HH",
"I2", "II", "JJ", "KK",
],
},
# Example with bad indentation in input.
"another_grd": {
"languages": [
"A2", "AA", "BB", "CC", "DD",
"E2", "EE", "FF", "GG", "HH",
"I2", "II", "JJ", "KK",
],
},
}
'''
input_lines = input_text.splitlines()
test_locales = ([
'AA', 'BB', 'CC', 'DD', 'EE', 'FF', 'GG', 'HH', 'II', 'JJ', 'KK', 'A2',
'E2', 'I2'
])
expected_lines = expected_text.splitlines()
self.assertListEqual(
_UpdateLocalesInExpectationLines(input_lines, test_locales, 40),
expected_lines)
def test_missing_list_termination(self):
input_lines = r'''
"languages": ['
"aa", "bb", "cc", "dd"
'''.splitlines()
with self.assertRaises(Exception) as cm:
_UpdateLocalesInExpectationLines(input_lines, ['a', 'b'], 40)
self.assertEqual(str(cm.exception), '2: Missing list termination!')
def _UpdateLocalesInExpectationFile(pyl_path, wanted_locales):
"""Update all locales listed in a given expectations file.
Args:
pyl_path: Path to .pyl file to update.
wanted_locales: List of locales that need to be written to
the file.
"""
tc_locales = {
_FixTranslationConsoleLocaleName(locale)
for locale in set(wanted_locales) - set([_DEFAULT_LOCALE])
}
with open(pyl_path) as f:
input_lines = [l.rstrip() for l in f.readlines()]
updated_lines = _UpdateLocalesInExpectationLines(input_lines, tc_locales)
with build_utils.AtomicOutput(pyl_path) as f:
f.writelines('\n'.join(updated_lines) + '\n')
##########################################################################
##########################################################################
#####
##### C H E C K E V E R Y T H I N G
#####
##########################################################################
##########################################################################
# pylint: enable=unused-argument
def _IsAllInputFile(input_file):
return _IsGritInputFile(input_file) or _IsBuildGnInputFile(input_file)
def _CheckAllFiles(input_file, input_lines, wanted_locales):
errors = []
if _IsGritInputFile(input_file):
errors += _CheckGrdTranslations(input_file, input_lines, wanted_locales)
errors += _CheckGrdAndroidOutputElements(
input_file, input_lines, wanted_locales)
elif _IsBuildGnInputFile(input_file):
errors += _CheckGnAndroidOutputs(input_file, input_lines, wanted_locales)
return errors
def _AddMissingLocalesInAllFiles(input_file, input_lines, wanted_locales):
if _IsGritInputFile(input_file):
lines = _AddMissingLocalesInGrdTranslations(
input_file, input_lines, wanted_locales)
lines = _AddMissingLocalesInGrdAndroidOutputs(
input_file, lines, wanted_locales)
elif _IsBuildGnInputFile(input_file):
lines = _AddMissingLocalesInGnAndroidOutputs(
input_file, input_lines, wanted_locales)
return lines
##########################################################################
##########################################################################
#####
##### C O M M A N D H A N D L I N G
#####
##########################################################################
##########################################################################
class _Command(object):
"""A base class for all commands recognized by this script.
Usage is the following:
1) Derived classes must re-define the following class-based fields:
- name: Command name (e.g. 'list-locales')
- description: Command short description.
- long_description: Optional. Command long description.
NOTE: As a convenience, if the first character is a newline,
it will be omitted in the help output.
2) Derived classes for commands that take arguments should override
RegisterExtraArgs(), which receives a corresponding argparse
sub-parser as argument.
3) Derived classes should implement a Run() command, which can read
the current arguments from self.args.
"""
name = None
description = None
long_description = None
def __init__(self):
self._parser = None
self.args = None
def RegisterExtraArgs(self, subparser):
pass
def RegisterArgs(self, parser):
subp = parser.add_parser(
self.name, help=self.description,
description=self.long_description or self.description,
formatter_class=argparse.RawDescriptionHelpFormatter)
self._parser = subp
subp.set_defaults(command=self)
group = subp.add_argument_group('%s arguments' % self.name)
self.RegisterExtraArgs(group)
def ProcessArgs(self, args):
self.args = args
class _ListLocalesCommand(_Command):
"""Implement the 'list-locales' command to list locale lists of interest."""
name = 'list-locales'
description = 'List supported Chrome locales'
long_description = r'''
List locales of interest, by default this prints all locales supported by
Chrome, but `--type=ios_unsupported` can be used for the list of locales
unsupported on iOS.
These values are extracted directly from build/config/locales.gni.
Additionally, use the --as-json argument to print the list as a JSON list,
instead of the default format (which is a space-separated list of locale names).
'''
# Maps type argument to a function returning the corresponding locales list.
TYPE_MAP = {
'all': ChromeLocales,
'ios_unsupported': IosUnsupportedLocales,
}
def RegisterExtraArgs(self, group):
group.add_argument(
'--as-json',
action='store_true',
help='Output as JSON list.')
group.add_argument(
'--type',
choices=tuple(self.TYPE_MAP.viewkeys()),
default='all',
help='Select type of locale list to print.')
def Run(self):
locale_list = self.TYPE_MAP[self.args.type]()
if self.args.as_json:
print('[%s]' % ", ".join("'%s'" % loc for loc in locale_list))
else:
print(' '.join(locale_list))
class _CheckInputFileBaseCommand(_Command):
"""Used as a base for other _Command subclasses that check input files.
Subclasses should also define the following class-level variables:
- select_file_func:
A predicate that receives a file name (not path) and return True if it
should be selected for inspection. Used when scanning directories with
'--scan-dir <dir>'.
- check_func:
- fix_func:
Two functions passed as parameters to _ProcessFile(), see relevant
documentation in this function's definition.
"""
select_file_func = None
check_func = None
fix_func = None
def RegisterExtraArgs(self, group):
group.add_argument(
'--scan-dir',
action='append',
help='Optional directory to scan for input files recursively.')
group.add_argument(
'input',
nargs='*',
help='Input file(s) to check.')
group.add_argument(
'--fix-inplace',
action='store_true',
help='Try to fix the files in-place too.')
group.add_argument(
'--add-locales',
help='Space-separated list of additional locales to use')
def Run(self):
args = self.args
input_files = []
if args.input:
input_files = args.input
if args.scan_dir:
input_files.extend(_ScanDirectoriesForFiles(
args.scan_dir, self.select_file_func.__func__))
locales = ChromeLocales()
if args.add_locales:
locales.extend(args.add_locales.split(' '))
locales = set(locales)
for input_file in input_files:
_ProcessFile(input_file,
locales,
self.check_func.__func__,
self.fix_func.__func__ if args.fix_inplace else None)
print('%sDone.' % (_CONSOLE_START_LINE))
class _CheckGrdAndroidOutputsCommand(_CheckInputFileBaseCommand):
name = 'check-grd-android-outputs'
description = (
'Check the Android resource (.xml) files outputs in GRIT input files.')
long_description = r'''
Check the Android .xml files outputs in one or more input GRIT (.grd) files
for the following conditions:
- Each item has a correct 'lang' attribute.
- There are no duplicated lines for the same 'lang' attribute.
- That there are no extra locales that Chromium doesn't want.
- That no wanted locale is missing.
- Filenames exist for each listed locale.
- Filenames are well-formed.
'''
select_file_func = _IsGritInputFile
check_func = _CheckGrdAndroidOutputElements
fix_func = _AddMissingLocalesInGrdAndroidOutputs
class _CheckGrdTranslationsCommand(_CheckInputFileBaseCommand):
name = 'check-grd-translations'
description = (
'Check the translation (.xtb) files outputted by .grd input files.')
long_description = r'''
Check the translation (.xtb) file outputs in one or more input GRIT (.grd) files
for the following conditions:
- Each item has a correct 'lang' attribute.
- There are no duplicated lines for the same 'lang' attribute.
- That there are no extra locales that Chromium doesn't want.
- That no wanted locale is missing.
- Each item has a 'path' attribute.
- Each such path value ends up with '.xtb'.
'''
select_file_func = _IsGritInputFile
check_func = _CheckGrdTranslations
fix_func = _AddMissingLocalesInGrdTranslations
class _CheckGnAndroidOutputsCommand(_CheckInputFileBaseCommand):
name = 'check-gn-android-outputs'
description = 'Check the Android .xml file lists in GN build files.'
long_description = r'''
Check one or more BUILD.gn file, looking for lists of Android resource .xml
files, and checking that:
- There are no duplicated output files in the list.
- Each output file belongs to a wanted Chromium locale.
- There are no output files for unwanted Chromium locales.
'''
select_file_func = _IsBuildGnInputFile
check_func = _CheckGnAndroidOutputs
fix_func = _AddMissingLocalesInGnAndroidOutputs
class _CheckAllCommand(_CheckInputFileBaseCommand):
name = 'check-all'
description = 'Check everything.'
long_description = 'Equivalent to calling all other check-xxx commands.'
select_file_func = _IsAllInputFile
check_func = _CheckAllFiles
fix_func = _AddMissingLocalesInAllFiles
class _UpdateExpectationsCommand(_Command):
name = 'update-expectations'
description = 'Update translation expectations file.'
long_description = r'''
Update %s files to match the current list of locales supported by Chromium.
This is especially useful to add new locales before updating any GRIT or GN
input file with the --add-locales option.
''' % _EXPECTATIONS_FILENAME
def RegisterExtraArgs(self, group):
group.add_argument(
'--add-locales',
help='Space-separated list of additional locales to use.')
def Run(self):
locales = ChromeLocales()
add_locales = self.args.add_locales
if add_locales:
locales.extend(add_locales.split(' '))
expectation_paths = [
'tools/gritsettings/translation_expectations.pyl',
'clank/tools/translation_expectations.pyl',
]
missing_expectation_files = []
for path in enumerate(expectation_paths):
file_path = os.path.join(_TOP_SRC_DIR, path)
if not os.path.exists(file_path):
missing_expectation_files.append(file_path)
continue
_UpdateLocalesInExpectationFile(file_path, locales)
if missing_expectation_files:
sys.stderr.write('WARNING: Missing file(s): %s\n' %
(', '.join(missing_expectation_files)))
class _UnitTestsCommand(_Command):
name = 'unit-tests'
description = 'Run internal unit-tests for this script'
def RegisterExtraArgs(self, group):
group.add_argument(
'-v', '--verbose', action='count', help='Increase test verbosity.')
group.add_argument('args', nargs=argparse.REMAINDER)
def Run(self):
argv = [_SCRIPT_NAME] + self.args.args
unittest.main(argv=argv, verbosity=self.args.verbose)
# List of all commands supported by this script.
_COMMANDS = [
_ListLocalesCommand,
_CheckGrdAndroidOutputsCommand,
_CheckGrdTranslationsCommand,
_CheckGnAndroidOutputsCommand,
_CheckAllCommand,
_UpdateExpectationsCommand,
_UnitTestsCommand,
]
def main(argv):
parser = argparse.ArgumentParser(
description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
subparsers = parser.add_subparsers()
commands = [clazz() for clazz in _COMMANDS]
for command in commands:
command.RegisterArgs(subparsers)
if not argv:
argv = ['--help']
args = parser.parse_args(argv)
args.command.ProcessArgs(args)
args.command.Run()
if __name__ == "__main__":
main(sys.argv[1:])