Index: source/tools/i18n/extractors/extractors.py =================================================================== --- source/tools/i18n/extractors/extractors.py +++ source/tools/i18n/extractors/extractors.py @@ -1,5 +1,3 @@ -# -*- coding:utf-8 -*- -# # Copyright (C) 2016 Wildfire Games. # All rights reserved. # @@ -20,8 +18,6 @@ # HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR # OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -from __future__ import absolute_import, division, print_function, unicode_literals - import codecs, re, os, sys import json as jsonParser @@ -39,7 +35,7 @@ """ s = re.split(r"([*][*]?)", mask) p = "" - for i in xrange(len(s)): + for i in range(len(s)): if i % 2 != 0: p = p + "[^/]+" if len(s[i]) == 2: @@ -327,7 +323,7 @@ def extractFromFile(self, filepath): with codecs.open(filepath, "r", 'utf-8') as fileObject: for message, breadcrumbs in self.extractFromString(fileObject.read()): - yield message, None, self.context, self.formatBreadcrumbs(breadcrumbs), -1, self.comments + yield message, None, self.context, self.formatBreadcrumbs(breadcrumbs), None, self.comments def extractFromString(self, string): self.breadcrumbs = [] @@ -360,7 +356,7 @@ for keyword in dictionary: self.breadcrumbs.append(keyword) if keyword in self.keywords: - if isinstance(dictionary[keyword], unicode): + if isinstance(dictionary[keyword], str): yield dictionary[keyword], self.breadcrumbs elif isinstance(dictionary[keyword], list): for message, breadcrumbs in self.extractList(dictionary[keyword]): @@ -380,7 +376,7 @@ index = 0 for listItem in itemsList: self.breadcrumbs.append(index) - if isinstance(listItem, unicode): + if isinstance(listItem, str): yield listItem, self.breadcrumbs del self.breadcrumbs[-1] index += 1 @@ -388,7 +384,7 @@ def extractDictionary(self, dictionary): for keyword in dictionary: self.breadcrumbs.append(keyword) - if isinstance(dictionary[keyword], unicode): + if isinstance(dictionary[keyword], str): yield dictionary[keyword], self.breadcrumbs del self.breadcrumbs[-1] @@ -429,7 +425,7 @@ attributes = [element.get(attribute) for attribute in self.keywords[keyword]["locationAttributes"] if attribute in element.attrib] breadcrumb = "({attributes})".format(attributes=", ".join(attributes)) if "context" in element.attrib: - context = unicode(element.get("context")) + context = str(element.get("context")) elif "tagAsContext" in self.keywords[keyword]: context = keyword elif "customContext" in self.keywords[keyword]: @@ -442,9 +438,9 @@ for splitText in element.text.split(): # split on whitespace is used for token lists, there, a leading '-' means the token has to be removed, so it's not to be processed here either if splitText[0] != "-": - yield unicode(splitText), None, context, breadcrumb, position, comments + yield str(splitText), None, context, breadcrumb, position, comments else: - yield unicode(element.text), None, context, breadcrumb, position, comments + yield str(element.text), None, context, breadcrumb, position, comments # Hack from http://stackoverflow.com/a/2819788 Index: source/tools/i18n/extractors/jslexer.py =================================================================== --- source/tools/i18n/extractors/jslexer.py +++ source/tools/i18n/extractors/jslexer.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- -# # Copyright (C) 2008-2011 Edgewall Software # Copyright (C) 2013-2014 Wildfire Games. # All rights reserved. @@ -31,8 +29,6 @@ extractor. """ -from __future__ import absolute_import, division, print_function, unicode_literals - from operator import itemgetter import re @@ -128,7 +124,7 @@ escaped_value = escaped.group() if len(escaped_value) == 4: try: - add(unichr(int(escaped_value, 16))) + add(chr(int(escaped_value, 16))) except ValueError: pass else: Index: source/tools/i18n/requirements.txt =================================================================== --- /dev/null +++ source/tools/i18n/requirements.txt @@ -0,0 +1,2 @@ +babel~=2.6 +lxml~=4.5 Index: source/tools/i18n/updateTemplates.py =================================================================== --- source/tools/i18n/updateTemplates.py +++ source/tools/i18n/updateTemplates.py @@ -1,5 +1,4 @@ -#!/usr/bin/env python2 -# -*- coding: utf-8 -*- +#!/usr/bin/env python3 # # Copyright (C) 2018 Wildfire Games. # This file is part of 0 A.D. @@ -21,18 +20,38 @@ import codecs, datetime, json, os, string, textwrap -from pology.catalog import Catalog -from pology.message import Message -from pology.monitored import Monpair, Monlist +from babel.messages.catalog import Catalog as BabelCatalog +from babel.messages.pofile import write_po from lxml import etree +import multiprocessing l10nToolsDirectory = os.path.dirname(os.path.realpath(__file__)) projectRootDirectory = os.path.abspath(os.path.join(l10nToolsDirectory, os.pardir, os.pardir, os.pardir)) l10nFolderName = "l10n" messagesFilename = "messages.json" +class Catalog(BabelCatalog): + """""" + def __init__(self, *args, project, **kwargs): + super().__init__(*args, **kwargs) + self._project = project + + @BabelCatalog.mime_headers.getter + def mime_headers(self): + headers = [] + for name, value in super().mime_headers: + if name in { + "PO-Revision-Date", + "POT-Creation-Date", + "MIME-Version", + "Content-Type", + "Content-Transfer-Encoding", + "Plural-Forms"}: + headers.append((name, value)) + + return [('Project-Id-Version', self._project)] + headers def warnAboutUntouchedMods(): """ @@ -41,7 +60,7 @@ modsRootFolder = os.path.join(projectRootDirectory, "binaries", "data", "mods") untouchedMods = {} for modFolder in os.listdir(modsRootFolder): - if modFolder[0] != "_": + if modFolder[0] != "_" and modFolder[0] != '.': if not os.path.exists(os.path.join(modsRootFolder, modFolder, l10nFolderName)): untouchedMods[modFolder] = "There is no '{folderName}' folder in the root folder of this mod.".format(folderName=l10nFolderName) elif not os.path.exists(os.path.join(modsRootFolder, modFolder, l10nFolderName, messagesFilename)): @@ -60,64 +79,79 @@ """.format(folderName=l10nFolderName, filename=messagesFilename) )) +def generatePOT(templateSettings, rootPath): + if "skip" in templateSettings and templateSettings["skip"] == "yes": + return + + print(f'Generating {templateSettings["project"]}') + + inputRootPath = rootPath + if "inputRoot" in templateSettings: + inputRootPath = os.path.join(rootPath, templateSettings["inputRoot"]) + + template = Catalog( + header_comment=( +f""" +# Translation template for {templateSettings["project"]}. +# Copyright (C) {datetime.datetime.now().year} {templateSettings["copyrightHolder"]} +# This file is distributed under the same license as the {templateSettings["project"]} project. +"""), + charset="utf-8", + fuzzy=False, + creation_date=datetime.datetime.now(), + revision_date=datetime.datetime.now(), + locale='en', + project=templateSettings["project"] + ) + + for rule in templateSettings["rules"]: + if "skip" in rule and rule["skip"] == "yes": + return + + options = rule.get("options", {}) + extractorClass = getattr(__import__("extractors.extractors", {}, {}, [rule["extractor"]]), rule["extractor"]) + extractor = extractorClass(inputRootPath, rule["filemasks"], options) + formatFlag = None + if "format" in options: + formatFlag = options["format"] + for message, plural, context, location, comments in extractor.run(): + message_id = (message, plural) if plural else message + + saved_message = template.get(message_id, context) or template.add( + id=message_id, + context=context, + auto_comments=comments, + flags=[formatFlag] if formatFlag and message.find("%") != -1 else [] + ) + saved_message.locations.append(location) + saved_message.flags.discard('python-format') + + write_po( + fileobj=open(os.path.join(rootPath, templateSettings["output"]), "wb+"), + catalog=template, + sort_by_file=True, + ) + print(u"Generated \"{}\" with {} messages.".format(templateSettings["output"], len(template))) def generateTemplatesForMessagesFile(messagesFilePath): with open(messagesFilePath, 'r') as fileObject: settings = json.load(fileObject) - rootPath = os.path.dirname(messagesFilePath) - for templateSettings in settings: - if "skip" in templateSettings and templateSettings["skip"] == "yes": - continue - - inputRootPath = rootPath - if "inputRoot" in templateSettings: - inputRootPath = os.path.join(rootPath, templateSettings["inputRoot"]) - - template = Catalog(os.path.join(rootPath, templateSettings["output"]), create=True, truncate=True) - h = template.update_header( - templateSettings["project"], - "Translation template for %project.", - "Copyright (C) {year} {holder}".format( - year=datetime.datetime.now().year, - holder=templateSettings["copyrightHolder"] - ), - "This file is distributed under the same license as the %project project.", - plforms="nplurals=2; plural=(n != 1);" - ) - h.remove_field("Report-Msgid-Bugs-To") - h.remove_field("Last-Translator") - h.remove_field("Language-Team") - h.remove_field("Language") - h.author = Monlist() - - for rule in templateSettings["rules"]: - if "skip" in rule and rule["skip"] == "yes": - continue - - options = rule.get("options", {}) - extractorClass = getattr(__import__("extractors.extractors", {}, {}, [rule["extractor"]]), rule["extractor"]) - extractor = extractorClass(inputRootPath, rule["filemasks"], options) - formatFlag = None - if "format" in options: - formatFlag = options["format"] - for message, plural, context, location, comments in extractor.run(): - msg = Message({"msgid": message, "msgid_plural": plural, "msgctxt": context, "auto_comment": comments, "flag": [formatFlag] if formatFlag and string.find(message, "%") != -1 else None, "source": [location]}) - if template.get(msg): - template.get(msg).source.append(Monpair(location)) - else: - template.add(msg) - - template.set_encoding("utf-8") - template.sync(fitplural=True) - print(u"Generated \"{}\" with {} messages.".format(templateSettings["output"], len(template))) + multiprocessing.Process( + target=generatePOT, + args=(templateSettings, os.path.dirname(messagesFilePath)) + ).start() def main(): - - for root, folders, filenames in os.walk(projectRootDirectory): + import argparse + parser = argparse.ArgumentParser() + parser.add_argument("--scandir", help="Directory to start scanning for l10n folders in." + "Type '.' for current working directory") + args = parser.parse_args() + for root, folders, filenames in os.walk(args.scandir or projectRootDirectory): for folder in folders: if folder == l10nFolderName: messagesFilePath = os.path.join(root, folder, messagesFilename)