Index: source/tools/entity/Entity.pm =================================================================== --- source/tools/entity/Entity.pm +++ source/tools/entity/Entity.pm @@ -11,15 +11,15 @@ sub get_filename { - my ($vfspath) = @_; - my $fn = "$vfsroot/public/simulation/templates/$vfspath.xml"; + my ($vfspath, $mod) = @_; + my $fn = "$vfsroot/$mod/simulation/templates/$vfspath.xml"; return $fn; } sub get_file { - my ($vfspath) = @_; - my $fn = get_filename($vfspath); + my ($vfspath, $mod) = @_; + my $fn = get_filename($vfspath, $mod); open my $f, $fn or die "Error loading $fn: $!"; local $/; return <$f>; @@ -113,13 +113,35 @@ } } +sub get_main_mod +{ + my ($vfspath, $mods) = @_; + my @mods_list = split(/\|/, $mods); + my $main_mod = $mods_list[0]; + my $fn = "$vfsroot/$main_mod/simulation/templates/$vfspath.xml"; + if (not -e $fn) + { + for my $dep (@mods_list) + { + $fn = "$vfsroot/$dep/simulation/templates/$vfspath.xml"; + if (-e $fn) + { + $main_mod = $dep; + last; + } + } + } + return $main_mod; +} + sub load_inherited { - my ($vfspath) = @_; - my $layer = load_xml($vfspath, get_file($vfspath)); + my ($vfspath, $mods) = @_; + my $main_mod = get_main_mod($vfspath, $mods); + my $layer = load_xml($vfspath, get_file($vfspath, $main_mod)); if ($layer->{Entity}{'@parent'}) { - my $parent = load_inherited($layer->{Entity}{'@parent'}{' content'}); + my $parent = load_inherited($layer->{Entity}{'@parent'}{' content'}, $mods); apply_layer($parent->{Entity}, $layer->{Entity}); return $parent; } else { @@ -129,17 +151,18 @@ sub find_entities { + my ($modName) = @_; my @files; my $find_process = sub { return $File::Find::prune = 1 if $_ eq '.svn'; my $n = $File::Find::name; return if /~$/; return unless -f $_; - $n =~ s~\Q$vfsroot\E/public/simulation/templates/~~; + $n =~ s~\Q$vfsroot\E/$modName/simulation/templates/~~; $n =~ s/\.xml$//; push @files, $n; }; - find({ wanted => $find_process }, "$vfsroot/public/simulation/templates"); + find({ wanted => $find_process }, "$vfsroot/$modName/simulation/templates"); return @files; } Index: source/tools/entity/checkrefs.pl =================================================================== --- source/tools/entity/checkrefs.pl +++ source/tools/entity/checkrefs.pl @@ -4,39 +4,85 @@ use File::Find; use XML::Simple; use JSON; +use Getopt::Long qw(GetOptions); +use lib "."; use Entity; -use constant CHECK_MAPS_XML => 0; -use constant ROOT_ACTORS => 1; +GetOptions ( + '--check-unused' => \(my $checkUnused = 0), + '--check-map-xml' => \(my $checkMapXml = 0), + '--validate-templates' => \(my $validateTemplates = 0), + '--mod-to-check=s' => \(my $modToCheck = "public") +); my @files; my @roots; my @deps; +# Force and checkMapXml if checkUnused is enabled to avoid false positives. +$checkMapXml |= $checkUnused; my $vfsroot = '../../../binaries/data/mods'; +my $supportedTextureFormats = 'dds|png'; +my $mods = get_mod_dependencies_string($modToCheck); +my $mod_list_string = $modToCheck; +if ($mods ne "") +{ + $mod_list_string = $mod_list_string."|$mods"; +} +$mod_list_string = $mod_list_string."|mod"; +print("Checking $modToCheck\'s integrity. \n"); +print("The following mod(s) will be loaded: $mod_list_string. \n"); +my @mods_list = split(/\|/, "$mod_list_string"); -sub vfs_to_physical +sub get_mod_dependencies { - my ($vfspath) = @_; - my $fn = "$vfsroot/public/$vfspath"; - if (not -e $fn) + my ($mod) = @_; + my $modjson = parse_json_file_full_path("$vfsroot/$mod/mod.json"); + my $modjsondeps = $modjson->{'dependencies'}; + for my $dep (@{$modjsondeps}) { - $fn = "$vfsroot/mod/$vfspath"; + # 0ad's folder isn't named like the mod. + if(index($dep, "0ad") != -1) + { + $dep = "public"; + } } - return $fn; + + return $modjsondeps; } +sub get_mod_dependencies_string +{ + my ($mod) = @_; + return join( '|',@{get_mod_dependencies($mod)}); +} + +sub vfs_to_physical +{ + my ($vfsPath) = @_; + my $fn = vfs_to_relative_to_mods($vfsPath); + return "$vfsroot/$fn"; +} + sub vfs_to_relative_to_mods { - my ($vfspath) = @_; - my $fn = "public/$vfspath"; - return $fn; + my ($vfsPath) = @_; + + for my $dep (@mods_list) + { + my $fn = "$dep/$vfsPath"; + + if (-e "$vfsroot/$fn") + { + return $fn; + } + } } sub find_files { - my ($vfspath, $extn) = @_; + my ($vfsPath, $extn) = @_; my @files; my $find_process = sub { return $File::Find::prune = 1 if $_ eq '.svn'; @@ -44,24 +90,33 @@ return if /~$/; return unless -f $_; return unless /\.($extn)$/; - $n =~ s~\Q$vfsroot\E/(public|mod)/~~; + $n =~ s~\Q$vfsroot\E/($mod_list_string)/~~; push @files, $n; }; - find({ wanted => $find_process }, "$vfsroot/public/$vfspath"); - find({ wanted => $find_process }, "$vfsroot/mod/$vfspath") if -d "$vfsroot/mod/$vfspath"; + for my $dep (@mods_list) + { + find({ wanted => $find_process },"$vfsroot/$dep/$vfsPath") if -d "$vfsroot/$dep/$vfsPath"; + } + return @files; } -sub parse_json_file +sub parse_json_file_full_path { my ($vfspath) = @_; - open my $fh, vfs_to_physical($vfspath) or die "Failed to open '$vfspath': $!"; + open my $fh, $vfspath or die "Failed to open '$vfspath': $!"; # decode_json expects a UTF-8 string and doesn't handle BOMs, so we strip those # (see http://trac.wildfiregames.com/ticket/1556) return decode_json(do { local $/; my $file = <$fh>; $file =~ s/^\xEF\xBB\xBF//; $file }); } +sub parse_json_file +{ + my ($vfspath) = @_; + return parse_json_file_full_path(vfs_to_physical($vfspath)) +} + sub add_entities { print "Loading entities...\n"; @@ -73,7 +128,7 @@ { my $path = "simulation/templates/$f.xml"; push @files, $path; - my $ent = Entity::load_inherited($f); + my $ent = Entity::load_inherited($f, "$mod_list_string"); push @deps, [ $path, "simulation/templates/" . $ent->{Entity}{'@parent'}{' content'} . ".xml" ] if $ent->{Entity}{'@parent'}; @@ -103,12 +158,37 @@ if ($ent->{Entity}{Identity}) { - push @deps, [ $path, "art/textures/ui/session/portraits/" . $ent->{Entity}{Identity}{Icon}{' content'} ] if $ent->{Entity}{Identity}{Icon}; + push @deps, [ $path, "art/textures/ui/session/portraits/" . $ent->{Entity}{Identity}{Icon}{' content'} ] if $ent->{Entity}{Identity}{Icon} and $ent->{Entity}{Identity}{Icon}{' content'} ne ''; } + + if ($ent->{Entity}{Formation}) + { + push @deps, [ $path, "art/textures/ui/session/icons/" . $ent->{Entity}{Formation}{Icon}{' content'} ] if $ent->{Entity}{Formation}{Icon} and $ent->{Entity}{Formation}{Icon}{' content'} ne ''; + } } } } +sub push_variant_dependencies +{ + my ($variant, $f) = @_; + push @deps, [ $f, "art/variants/$variant->{file}" ] if $variant->{file}; + push @deps, [ $f, "art/meshes/$variant->{mesh}" ] if $variant->{mesh}; + push @deps, [ $f, "art/particles/$variant->{particles}{file}" ] if $variant->{particles}{file}; + for my $tex (@{$variant->{textures}{texture}}) + { + push @deps, [ $f, "art/textures/skins/$tex->{file}" ] if $tex->{file}; + } + for my $prop (@{$variant->{props}{prop}}) + { + push @deps, [ $f, "art/actors/$prop->{actor}" ] if $prop->{actor}; + } + for my $anim (@{$variant->{animations}{animation}}) + { + push @deps, [ $f, "art/animation/$anim->{file}" ] if $anim->{file}; + } +} + sub add_actors { print "Loading actors...\n"; @@ -117,29 +197,15 @@ for my $f (sort @actorfiles) { push @files, $f; + push @roots, $f; - push @roots, $f if ROOT_ACTORS; - my $actor = XMLin(vfs_to_physical($f), ForceArray => [qw(group variant texture prop animation)], KeyAttr => []) or die "Failed to parse '$f': $!"; for my $group (@{$actor->{group}}) { for my $variant (@{$group->{variant}}) { - push @deps, [ $f, "art/meshes/$variant->{mesh}" ] if $variant->{mesh}; - push @deps, [ $f, "art/particles/$variant->{particles}{file}" ] if $variant->{particles}{file}; - for my $tex (@{$variant->{textures}{texture}}) - { - push @deps, [ $f, "art/textures/skins/$tex->{file}" ] if $tex->{file}; - } - for my $prop (@{$variant->{props}{prop}}) - { - push @deps, [ $f, "art/actors/$prop->{actor}" ] if $prop->{actor}; - } - for my $anim (@{$variant->{animations}{animation}}) - { - push @deps, [ $f, "art/animation/$anim->{file}" ] if $anim->{file}; - } + push_variant_dependencies($variant, $f); } } @@ -147,12 +213,27 @@ } } + +sub add_variants +{ + print "Loading variants...\n"; + my @variantfiles = find_files('art/variants', 'xml'); + + for my $f (sort @variantfiles) + { + push @files, $f; + push @roots, $f; + my $variant = XMLin(vfs_to_physical($f), ForceArray => [qw(texture prop animation)], KeyAttr => []) or die "Failed to parse '$f': $!"; + push_variant_dependencies($variant, $f); + } +} + sub add_art { print "Loading art files...\n"; - push @files, find_files('art/textures/particles', 'dds|png|jpg|tga'); - push @files, find_files('art/textures/terrain', 'dds|png|jpg|tga'); - push @files, find_files('art/textures/skins', 'dds|png|jpg|tga'); + push @files, find_files('art/textures/particles', $supportedTextureFormats); + push @files, find_files('art/textures/terrain', $supportedTextureFormats); + push @files, find_files('art/textures/skins', $supportedTextureFormats); push @files, find_files('art/meshes', 'pmd|dae'); push @files, find_files('art/animation', 'psa|dae'); } @@ -191,12 +272,10 @@ print "Loading maps XML...\n"; my @mapfiles = find_files('maps/scenarios', 'xml'); push @mapfiles, find_files('maps/skirmishes', 'xml'); + push @mapfiles, find_files('maps/tutorials', 'xml'); for my $f (sort @mapfiles) { - print " $f\n"; - push @files, $f; - push @roots, $f; my $map = XMLin(vfs_to_physical($f), ForceArray => [qw(Entity)], KeyAttr => []) or die "Failed to parse '$f': $!"; @@ -301,6 +380,7 @@ for my $f (sort @soundfiles) { push @files, $f; + push @roots, $f; my $sound = XMLin(vfs_to_physical($f), ForceArray => [qw(Sound)], KeyAttr => []) or die "Failed to parse '$f': $!"; @@ -404,7 +484,8 @@ { print "Loading GUI data...\n"; push @files, find_files('gui', 'js'); - push @files, find_files('art/textures/ui', 'dds|png|jpg|tga'); + push @files, find_files('art/textures/ui', $supportedTextureFormats); + push @files, find_files('art/textures/selection', $supportedTextureFormats); } sub add_civs @@ -420,7 +501,7 @@ my $civ = parse_json_file($f); - push @deps, [ $f, "art/textures/ui/" . $civ->{Emblem} ]; + push @deps, [ $f, "art/textures/ui/" . $civ->{Emblem} ] if $civ->{Emblem}; push @deps, [ $f, "audio/music/" . $_->{File} ] for @{$civ->{Music}}; } @@ -435,14 +516,15 @@ for my $f (sort @rmsdefs) { + next if $f =~ /^maps\/random\/rmbiome/; + push @files, $f; push @roots, $f; my $rms = parse_json_file($f); + push @deps, [ $f, "maps/random/" . $rms->{settings}{Script} ] if $rms->{settings}{Script}; - push @deps, [ $f, "maps/random/" . $rms->{settings}{Script} ]; - # Map previews push @deps, [ $f, "art/textures/ui/session/icons/mappreview/" . $rms->{settings}{Preview} ] if $rms->{settings}{Preview}; } @@ -465,6 +547,28 @@ } } +sub add_auras +{ + print "Loading auras...\n"; + + my @aurafiles = find_files('simulation/data/auras', 'json'); + for my $f (sort @aurafiles) + { + push @files, $f; + push @roots, $f; + + my $aura = parse_json_file($f); + + push @deps, [ $f, $aura->{overlayIcon} ] if $aura->{overlayIcon}; + + if($aura->{rangeOverlay}) + { + push @deps, [ $f, "art/textures/selection/" . $aura->{rangeOverlay}{lineTexture} ] if $aura->{rangeOverlay}{lineTexture}; + push @deps, [ $f, "art/textures/selection/" . $aura->{rangeOverlay}{lineTextureMask} ] if $aura->{rangeOverlay}{lineTextureMask}; + } + } +} + sub add_terrains { print "Loading terrains...\n"; @@ -476,6 +580,7 @@ if ($f !~ /terrains.xml$/) { push @files, $f; + push @roots, $f; my $terrain = XMLin(vfs_to_physical($f), ForceArray => [qw(texture)], KeyAttr => []) or die "Failed to parse '$f': $!"; @@ -539,43 +644,34 @@ for my $f (sort @files) { - next if exists $reachable{$f}; + next if exists $reachable{$f} + || index($f, "art/terrains/") != -1 + || index($f, "maps/random/") != -1 + || index($f, "art/materials/") != -1; warn "Unused file '" . vfs_to_relative_to_mods($f) . "'\n"; } } -add_maps_xml() if CHECK_MAPS_XML; - +add_maps_xml() if $checkMapXml; add_maps_pmp(); - add_entities(); - add_actors(); - +add_variants(); add_art(); - add_materials(); - add_particles(); - add_soundgroups(); add_audio(); - add_gui_xml(); add_gui_data(); - add_civs(); - add_rms(); - add_techs(); - add_terrains(); +add_auras(); -# TODO: add non-skin textures, and all the references to them - -print "\n"; check_deps(); -print "\n"; -check_unused(); +check_unused() if $checkUnused; +print "\n" if $checkUnused; +system("perl ../xmlvalidator/validate.pl") if $validateTemplates; Index: source/tools/entity/creationgraph.pl =================================================================== --- source/tools/entity/creationgraph.pl +++ source/tools/entity/creationgraph.pl @@ -1,9 +1,10 @@ use strict; use warnings; +use lib "."; use Entity; -my @files = Entity::find_entities(); +my @files = Entity::find_entities("public"); my %files = map +($_ => 1), @files; @@ -13,7 +14,7 @@ for my $f (sort @files) { next if $f =~ /^template_/; print "# $f...\n"; - my $ent = Entity::load_inherited($f); + my $ent = Entity::load_inherited($f, "public"); if ($ent->{Entity}{Builder}) { my $ents = $ent->{Entity}{Builder}{Entities}{' content'}; Index: source/tools/entity/entvalidate.pl =================================================================== --- source/tools/entity/entvalidate.pl +++ source/tools/entity/entvalidate.pl @@ -3,6 +3,7 @@ use XML::LibXML; +use lib "."; use Entity; my $rngschema = XML::LibXML::RelaxNG->new(location => '../../../binaries/system/entity.rng'); @@ -49,7 +50,7 @@ sub check_all { - my @files = Entity::find_entities(); + my @files = Entity::find_entities("public"); my $count = 0; my $failed = 0; Index: source/tools/entity/readme.md =================================================================== --- source/tools/entity/readme.md +++ source/tools/entity/readme.md @@ -0,0 +1,29 @@ +# Checkrefs.pl + +## Description + +This script checks the game files for missing dependencies, unused files, and for file integrity. If mods are specified, all their dependencies are also checked recursively. This script is particularly useful to detect broken actors or templates. + +## Requirements + +- Perl interpreter installed +- Dependencies: + - XML::Parser + - XML::Simple + - Getopt::Long + - File::Find + - Data::Dumper + - JSON + +## Usage + +- cd in source/tools/entity and run the script. + +``` +Usage: perl checkrefs.pl [OPTION]... +Checks the game files for missing dependencies, unused files, and for file integrity. + --check-unused check for all the unused files in the given mods and their dependencies. Implies --check-map-xml. Currently yields a lot of false positives. + --check-map-xml check maps for missing actor and templates. + --validate-templates run the validate.pl script to check if the xml files match their (.rng) grammar file. This currently only works for the public mod. + --mod-to-check=mods specify which mods to check. 'mods' should be a list of mods separated by '|'. Default value: 'public|mod'. +```