Index: ps/trunk/source/tools/entity/entvalidate.py =================================================================== --- ps/trunk/source/tools/entity/entvalidate.py (revision 27160) +++ ps/trunk/source/tools/entity/entvalidate.py (revision 27161) @@ -1,44 +1,116 @@ #!/usr/bin/env python3 -from os import chdir +import argparse +import logging from pathlib import Path +import shutil from subprocess import run, CalledProcessError -from sys import exit +import sys +from typing import Sequence + from xml.etree import ElementTree -from scriptlib import warn, SimulTemplateEntity, find_files +from scriptlib import SimulTemplateEntity, find_files + +SIMUL_TEMPLATES_PATH = Path("simulation/templates") +ENTITY_RELAXNG_FNAME = "entity.rng" +RELAXNG_SCHEMA_ERROR_MSG = """Relax NG schema non existant. +Please create the file: {} +You can do that by running 'pyrogenesis -dumpSchema' in the 'system' directory +""" +XMLLINT_ERROR_MSG = ("xmllint not found in your PATH, please install it " + "(usually in libxml2 package)") + +class SingleLevelFilter(logging.Filter): + def __init__(self, passlevel, reject): + self.passlevel = passlevel + self.reject = reject + + def filter(self, record): + if self.reject: + return (record.levelno != self.passlevel) + else: + return (record.levelno == self.passlevel) + +logger = logging.getLogger(__name__) +logger.setLevel(logging.INFO) +# create a console handler, seems nicer to Windows and for future uses +ch = logging.StreamHandler(sys.stdout) +ch.setLevel(logging.INFO) +ch.setFormatter(logging.Formatter('%(levelname)s - %(message)s')) +f1 = SingleLevelFilter(logging.INFO, False) +ch.addFilter(f1) +logger.addHandler(ch) +errorch =logging. StreamHandler(sys.stderr) +errorch.setLevel(logging.WARNING) +errorch.setFormatter(logging.Formatter('%(levelname)s - %(message)s')) +logger.addHandler(errorch) + +def main(argv: Sequence[str] | None = None) -> int: + parser = argparse.ArgumentParser(description="Validate templates") + parser.add_argument("-m", "--mod-name", required=True, + help="The name of the mod to validate.") + parser.add_argument("-r", "--root", dest="vfs_root", default=Path(), + type=Path, help="The path to mod's root location.") + parser.add_argument("-s", "--relaxng-schema", + default=Path() / ENTITY_RELAXNG_FNAME, type=Path, + help="The path to mod's root location.") + parser.add_argument("-t", "--templates", nargs="*", + help="Optionally, a list of templates to validate.") + parser.add_argument("-v", "--verbose", + help="Be verbose about the output.", default=False) + + args = parser.parse_args(argv) + + if not args.relaxng_schema.exists(): + logging.error(RELAXNG_SCHEMA_ERROR_MSG.format(args.relaxng_schema)) + return 1 -def main(): - root = Path(__file__).resolve().parents[3] - relaxng_schema = root / 'binaries' / 'system' / 'entity.rng' - if not relaxng_schema.exists(): - warn(f"""Relax NG schema non existant. -Please create the file {relaxng_schema.relative_to(root)} -You can do that by running 'pyrogenesis -dumpSchema' in the 'system' directory""") - exit(1) - if run(['xmllint', '--version'], capture_output=True).returncode != 0: - warn("xmllint not found in your PATH, please install it (usually in libxml2 package)") - exit(2) - vfs_root = root / 'binaries' / 'data' / 'mods' - simul_templates_path = Path('simulation/templates') - simul_template_entity = SimulTemplateEntity(vfs_root) - count = 0 - failed = 0 - for fp, _ in sorted(find_files(vfs_root, ['public'], 'simulation/templates', 'xml')): - if fp.stem.startswith('template_'): + if not shutil.which("xmllint"): + logging.error(XMLLINT_ERROR_MSG) + return 2 + + if args.templates: + templates = sorted([(Path(t), None) for t in args.templates]) + else: + templates = sorted(find_files(args.vfs_root, [args.mod_name], + SIMUL_TEMPLATES_PATH.as_posix(), "xml")) + + simul_template_entity = SimulTemplateEntity(args.vfs_root, logger) + count, failed = 0, 0 + for fp, _ in templates: + if fp.stem.startswith("template_"): + continue + + path = fp.as_posix() + if (path.startswith(f"{SIMUL_TEMPLATES_PATH.as_posix()}/mixins/") + or path.startswith( + f"{SIMUL_TEMPLATES_PATH.as_posix()}/special/")): continue - print(f"# {fp}...") + + if (args.verbose): + logger.info(f"Parsing {fp}...") count += 1 - entity = simul_template_entity.load_inherited(simul_templates_path, str(fp.relative_to(simul_templates_path)), ['public']) - xmlcontent = ElementTree.tostring(entity, encoding='unicode') + entity = simul_template_entity.load_inherited( + SIMUL_TEMPLATES_PATH, + str(fp.relative_to(SIMUL_TEMPLATES_PATH)), + [args.mod_name] + ) + xmlcontent = ElementTree.tostring(entity, encoding="unicode") try: - run(['xmllint', '--relaxng', str(relaxng_schema.resolve()), '-'], input=xmlcontent, capture_output=True, text=True, check=True) + run(["xmllint", "--relaxng", + str(args.relaxng_schema.resolve()), "-"], + input=xmlcontent, encoding="utf-8", capture_output=True, text=True, check=True) except CalledProcessError as e: failed += 1 - print(e.stderr) - print(e.stdout) - print(f"\nTotal: {count}; failed: {failed}") + if (e.stderr): + logger.error(e.stderr) + if (e.stdout): + logger.info(e.stdout) + + logger.info(f"Total: {count}; failed: {failed}") + + return 0 -if __name__ == '__main__': - chdir(Path(__file__).resolve().parent) - main() +if __name__ == "__main__": + raise SystemExit(main()) Index: ps/trunk/source/tools/entity/scriptlib/__init__.py =================================================================== --- ps/trunk/source/tools/entity/scriptlib/__init__.py (revision 27160) +++ ps/trunk/source/tools/entity/scriptlib/__init__.py (revision 27161) @@ -1,128 +1,138 @@ from collections import Counter from decimal import Decimal from re import split -from sys import stderr from xml.etree import ElementTree from os.path import exists class SimulTemplateEntity: def __init__(self, vfs_root, logger): self.vfs_root = vfs_root self.logger = logger def get_file(self, base_path, vfs_path, mod): default_path = self.vfs_root / mod / base_path - file = (default_path/ "special" / "filter" / vfs_path).with_suffix('.xml') + file = (default_path / "special" / "filter" / vfs_path).with_suffix('.xml') if not exists(file): file = (default_path / "mixins" / vfs_path).with_suffix('.xml') if not exists(file): file = (default_path / vfs_path).with_suffix('.xml') return file def get_main_mod(self, base_path, vfs_path, mods): for mod in mods: fp = self.get_file(base_path, vfs_path, mod) if fp.exists(): main_mod = mod break else: # default to first mod # it should then not exist # it will raise an exception when trying to read it main_mod = mods[0] return main_mod def apply_layer(self, base_tag, tag): """ apply tag layer to base_tag """ if tag.get('datatype') == 'tokens': base_tokens = split(r'\s+', base_tag.text or '') tokens = split(r'\s+', tag.text or '') final_tokens = base_tokens.copy() for token in tokens: if token.startswith('-'): token_to_remove = token[1:] if token_to_remove in final_tokens: final_tokens.remove(token_to_remove) elif token not in final_tokens: final_tokens.append(token) base_tag.text = ' '.join(final_tokens) + base_tag.set("datatype", "tokens") elif tag.get('op'): op = tag.get('op') op1 = Decimal(base_tag.text or '0') op2 = Decimal(tag.text or '0') + # Try converting to integers if possible, to pass validation. if op == 'add': - base_tag.text = str(op1 + op2) + base_tag.text = str(int(op1 + op2) if int(op1 + op2) == op1 + op2 else op1 + op2) elif op == 'mul': - base_tag.text = str(op1 * op2) + base_tag.text = str(int(op1 * op2) if int(op1 * op2) == op1 * op2 else op1 * op2) elif op == 'mul_round': base_tag.text = str(round(op1 * op2)) else: raise ValueError(f"Invalid operator '{op}'") else: base_tag.text = tag.text + for prop in tag.attrib: + if prop not in ('disable', 'replace', 'parent', 'merge'): + base_tag.set(prop, tag.get(prop)) for child in tag: base_child = base_tag.find(child.tag) if 'disable' in child.attrib: if base_child is not None: base_tag.remove(base_child) - else: + elif ('merge' not in child.attrib) or (base_child is not None): if 'replace' in child.attrib and base_child is not None: base_tag.remove(base_child) + base_child = None if base_child is None: base_child = ElementTree.Element(child.tag) base_tag.append(base_child) self.apply_layer(base_child, child) if 'replace' in base_child.attrib: del base_child.attrib['replace'] - def load_inherited(self, base_path, vfs_path, mods, base = None): + def load_inherited(self, base_path, vfs_path, mods): + entity = self._load_inherited(base_path, vfs_path, mods) + entity[:] = sorted(entity[:], key=lambda x: x.tag) + return entity + + def _load_inherited(self, base_path, vfs_path, mods, base=None): """ vfs_path should be relative to base_path in a mod """ if '|' in vfs_path: - paths = vfs_path.split("|", 2) - base = self.load_inherited(base_path, paths[1], mods, base); - base = self.load_inherited(base_path, paths[0], mods, base); + paths = vfs_path.split("|", 1) + base = self._load_inherited(base_path, paths[1], mods, base) + base = self._load_inherited(base_path, paths[0], mods, base) return base main_mod = self.get_main_mod(base_path, vfs_path, mods) fp = self.get_file(base_path, vfs_path, main_mod) layer = ElementTree.parse(fp).getroot() for el in layer.iter(): children = [x.tag for x in el] duplicates = [x for x, c in Counter(children).items() if c > 1] if duplicates: for dup in duplicates: self.logger.warning(f"Duplicate child node '{dup}' in tag {el.tag} of {fp}") if layer.get('parent'): - parent = self.load_inherited(base_path, layer.get('parent'), mods) + parent = self._load_inherited(base_path, layer.get('parent'), mods, base) self.apply_layer(parent, layer) return parent else: if not base: return layer else: self.apply_layer(base, layer) return base def find_files(vfs_root, mods, vfs_path, *ext_list): """ returns a list of 2-size tuple with: - Path relative to the mod base - full Path """ full_exts = ['.' + ext for ext in ext_list] def find_recursive(dp, base): """(relative Path, full Path) generator""" if dp.is_dir(): if dp.name != '.svn' and dp.name != '.git' and not dp.name.endswith('~'): for fp in dp.iterdir(): yield from find_recursive(fp, base) elif dp.suffix in full_exts: relative_file_path = dp.relative_to(base) yield (relative_file_path, dp.resolve()) return [(rp, fp) for mod in mods for (rp, fp) in find_recursive(vfs_root / mod / vfs_path, vfs_root / mod)]