Differential D1381 Diff 7503 source/tools/entity/checkrefs.pl

Changeset View

Standalone View

source/tools/entity/checkrefs.pl

use strict;		use strict;
use warnings;		use warnings;
use Data::Dumper;		use Data::Dumper;
use File::Find;		use File::Find;
use XML::Simple;		use XML::Simple;
use JSON;		use JSON;
		use Getopt::Long qw(GetOptions);

		use lib ".";
use Entity;		use Entity;

use constant CHECK_MAPS_XML => 0;		GetOptions (
use constant ROOT_ACTORS => 1;		'--check-unused' => \(my $checkUnused = 0),
		'--check-map-xml' => \(my $checkMapXml = 0),
		'--validate-templates' => \(my $validateTemplates = 0),
		'--mod-to-check=s' => \(my $modToCheck = "public")
		);

my @files;		my @files;
my @roots;		my @roots;
my @deps;		my @deps;

		# Force and checkMapXml if checkUnused is enabled to avoid false positives.
		$checkMapXml \|= $checkUnused;
my $vfsroot = '../../../binaries/data/mods';		my $vfsroot = '../../../binaries/data/mods';
		my $supportedTextureFormats = 'dds\|png';
		my $mods = get_mod_dependencies_string($modToCheck);
		my $mod_list_string = $modToCheck;
		if ($mods ne "")
		{
		$mod_list_string = $mod_list_string."\|$mods";
		}
		$mod_list_string = $mod_list_string."\|mod";
		print("Checking $modToCheck\'s integrity. \n");
		print("The following mod(s) will be loaded: $mod_list_string. \n");
		my @mods_list = split(/\\|/, "$mod_list_string");

sub vfs_to_physical		sub get_mod_dependencies
{		{
my ($vfspath) = @_;		my ($mod) = @_;
my $fn = "$vfsroot/public/$vfspath";		my $modjson = parse_json_file_full_path("$vfsroot/$mod/mod.json");
if (not -e $fn)		my $modjsondeps = $modjson->{'dependencies'};
		for my $dep (@{$modjsondeps})
{		{
$fn = "$vfsroot/mod/$vfspath";		# 0ad's folder isn't named like the mod.
		if(index($dep, "0ad") != -1)
		{
		$dep = "public";
}		}
return $fn;		}

		return $modjsondeps;
		}

		sub get_mod_dependencies_string
		{
		my ($mod) = @_;
		return join( '\|',@{get_mod_dependencies($mod)});
		}

		sub vfs_to_physical
		{
		my ($vfsPath) = @_;
		my $fn = vfs_to_relative_to_mods($vfsPath);
		return "$vfsroot/$fn";
}		}

sub vfs_to_relative_to_mods		sub vfs_to_relative_to_mods
{		{
my ($vfspath) = @_;		my ($vfsPath) = @_;
my $fn = "public/$vfspath";
		for my $dep (@mods_list)
		{
		my $fn = "$dep/$vfsPath";

		if (-e "$vfsroot/$fn")
		{
return $fn;		return $fn;
}		}
		}
		}

sub find_files		sub find_files
{		{
my ($vfspath, $extn) = @_;		my ($vfsPath, $extn) = @_;
my @files;		my @files;
my $find_process = sub {		my $find_process = sub {
return $File::Find::prune = 1 if $_ eq '.svn';		return $File::Find::prune = 1 if $_ eq '.svn';
my $n = $File::Find::name;		my $n = $File::Find::name;
return if /~$/;		return if /~$/;
return unless -f $_;		return unless -f $_;
return unless /\.($extn)$/;		return unless /\.($extn)$/;
$n =~ s~\Q$vfsroot\E/(public\|mod)/~~;		$n =~ s~\Q$vfsroot\E/($mod_list_string)/~~;
push @files, $n;		push @files, $n;
};		};
find({ wanted => $find_process }, "$vfsroot/public/$vfspath");
find({ wanted => $find_process }, "$vfsroot/mod/$vfspath") if -d "$vfsroot/mod/$vfspath";		for my $dep (@mods_list)
		{
		find({ wanted => $find_process },"$vfsroot/$dep/$vfsPath") if -d "$vfsroot/$dep/$vfsPath";
		}

return @files;		return @files;
}		}

sub parse_json_file		sub parse_json_file_full_path
{		{
my ($vfspath) = @_;		my ($vfspath) = @_;
open my $fh, vfs_to_physical($vfspath) or die "Failed to open '$vfspath': $!";		open my $fh, $vfspath or die "Failed to open '$vfspath': $!";
# decode_json expects a UTF-8 string and doesn't handle BOMs, so we strip those		# decode_json expects a UTF-8 string and doesn't handle BOMs, so we strip those
# (see http://trac.wildfiregames.com/ticket/1556)		# (see http://trac.wildfiregames.com/ticket/1556)
return decode_json(do { local $/; my $file = <$fh>; $file =~ s/^\xEF\xBB\xBF//; $file });		return decode_json(do { local $/; my $file = <$fh>; $file =~ s/^\xEF\xBB\xBF//; $file });
}		}

		sub parse_json_file
		{
		my ($vfspath) = @_;
		return parse_json_file_full_path(vfs_to_physical($vfspath))
		}

sub add_entities		sub add_entities
{		{
print "Loading entities...\n";		print "Loading entities...\n";

my @entfiles = find_files('simulation/templates', 'xml');		my @entfiles = find_files('simulation/templates', 'xml');
s~^simulation/templates/(.*)\.xml$~$1~ for @entfiles;		s~^simulation/templates/(.*)\.xml$~$1~ for @entfiles;

for my $f (sort @entfiles)		for my $f (sort @entfiles)
{		{
my $path = "simulation/templates/$f.xml";		my $path = "simulation/templates/$f.xml";
push @files, $path;		push @files, $path;
my $ent = Entity::load_inherited($f);		my $ent = Entity::load_inherited($f, "$mod_list_string");

push @deps, [ $path, "simulation/templates/" . $ent->{Entity}{'@parent'}{' content'} . ".xml" ] if $ent->{Entity}{'@parent'};		push @deps, [ $path, "simulation/templates/" . $ent->{Entity}{'@parent'}{' content'} . ".xml" ] if $ent->{Entity}{'@parent'};

if ($f !~ /^template_/)		if ($f !~ /^template_/)
{		{
push @roots, $path;		push @roots, $path;
if ($ent->{Entity}{VisualActor})		if ($ent->{Entity}{VisualActor})
{		{
Show All 13 Lines	for my $f (sort @entfiles)
$soundPath =~ s/{gender}/$gender/g;		$soundPath =~ s/{gender}/$gender/g;
$soundPath =~ s/{lang}/$lang/g;		$soundPath =~ s/{lang}/$lang/g;
push @deps, [ $path, "audio/" . $soundPath ];		push @deps, [ $path, "audio/" . $soundPath ];
}		}
}		}

if ($ent->{Entity}{Identity})		if ($ent->{Entity}{Identity})
{		{
push @deps, [ $path, "art/textures/ui/session/portraits/" . $ent->{Entity}{Identity}{Icon}{' content'} ] if $ent->{Entity}{Identity}{Icon};		push @deps, [ $path, "art/textures/ui/session/portraits/" . $ent->{Entity}{Identity}{Icon}{' content'} ] if $ent->{Entity}{Identity}{Icon} and $ent->{Entity}{Identity}{Icon}{' content'} ne '';
		}

		if ($ent->{Entity}{Formation})
		{
		push @deps, [ $path, "art/textures/ui/session/icons/" . $ent->{Entity}{Formation}{Icon}{' content'} ] if $ent->{Entity}{Formation}{Icon} and $ent->{Entity}{Formation}{Icon}{' content'} ne '';
		}
		}
		}
		}

		sub push_variant_dependencies
		{
		my ($variant, $f) = @_;
		push @deps, [ $f, "art/variants/$variant->{file}" ] if $variant->{file};
		push @deps, [ $f, "art/meshes/$variant->{mesh}" ] if $variant->{mesh};
		push @deps, [ $f, "art/particles/$variant->{particles}{file}" ] if $variant->{particles}{file};
		for my $tex (@{$variant->{textures}{texture}})
		{
		push @deps, [ $f, "art/textures/skins/$tex->{file}" ] if $tex->{file};
}		}
		for my $prop (@{$variant->{props}{prop}})
		{
		push @deps, [ $f, "art/actors/$prop->{actor}" ] if $prop->{actor};
}		}
		for my $anim (@{$variant->{animations}{animation}})
		{
		push @deps, [ $f, "art/animation/$anim->{file}" ] if $anim->{file};
}		}
}		}

sub add_actors		sub add_actors
{		{
print "Loading actors...\n";		print "Loading actors...\n";

my @actorfiles = find_files('art/actors', 'xml');		my @actorfiles = find_files('art/actors', 'xml');
for my $f (sort @actorfiles)		for my $f (sort @actorfiles)
{		{
push @files, $f;		push @files, $f;
		push @roots, $f;
push @roots, $f if ROOT_ACTORS;

my $actor = XMLin(vfs_to_physical($f), ForceArray => [qw(group variant texture prop animation)], KeyAttr => []) or die "Failed to parse '$f': $!";		my $actor = XMLin(vfs_to_physical($f), ForceArray => [qw(group variant texture prop animation)], KeyAttr => []) or die "Failed to parse '$f': $!";

for my $group (@{$actor->{group}})		for my $group (@{$actor->{group}})
{		{
for my $variant (@{$group->{variant}})		for my $variant (@{$group->{variant}})
{		{
push @deps, [ $f, "art/meshes/$variant->{mesh}" ] if $variant->{mesh};		push_variant_dependencies($variant, $f);
push @deps, [ $f, "art/particles/$variant->{particles}{file}" ] if $variant->{particles}{file};
for my $tex (@{$variant->{textures}{texture}})
{
push @deps, [ $f, "art/textures/skins/$tex->{file}" ] if $tex->{file};
}
for my $prop (@{$variant->{props}{prop}})
{
push @deps, [ $f, "art/actors/$prop->{actor}" ] if $prop->{actor};
}		}
for my $anim (@{$variant->{animations}{animation}})
{
push @deps, [ $f, "art/animation/$anim->{file}" ] if $anim->{file};
}		}

		push @deps, [ $f, "art/materials/$actor->{material}" ] if $actor->{material};
}		}
}		}

push @deps, [ $f, "art/materials/$actor->{material}" ] if $actor->{material};
		sub add_variants
		{
		print "Loading variants...\n";
		my @variantfiles = find_files('art/variants', 'xml');

		for my $f (sort @variantfiles)
		{
		push @files, $f;
		push @roots, $f;
		my $variant = XMLin(vfs_to_physical($f), ForceArray => [qw(texture prop animation)], KeyAttr => []) or die "Failed to parse '$f': $!";
		push_variant_dependencies($variant, $f);
}		}
}		}

sub add_art		sub add_art
{		{
print "Loading art files...\n";		print "Loading art files...\n";
push @files, find_files('art/textures/particles', 'dds\|png\|jpg\|tga');		push @files, find_files('art/textures/particles', $supportedTextureFormats);
push @files, find_files('art/textures/terrain', 'dds\|png\|jpg\|tga');		push @files, find_files('art/textures/terrain', $supportedTextureFormats);
push @files, find_files('art/textures/skins', 'dds\|png\|jpg\|tga');		push @files, find_files('art/textures/skins', $supportedTextureFormats);
push @files, find_files('art/meshes', 'pmd\|dae');		push @files, find_files('art/meshes', 'pmd\|dae');
push @files, find_files('art/animation', 'psa\|dae');		push @files, find_files('art/animation', 'psa\|dae');
}		}

sub add_materials		sub add_materials
{		{
		elexisUnsubmitted Not Done Inline Actions L180-L193 is copypasta, better put it into a function, so maintenance and extension is half as expensive elexis: L180-L193 is copypasta, better put it into a function, so maintenance and extension is half as…
print "Loading materials...\n";		print "Loading materials...\n";
my @materialfiles = find_files('art/materials', 'xml');		my @materialfiles = find_files('art/materials', 'xml');
for my $f (sort @materialfiles)		for my $f (sort @materialfiles)
{		{
push @files, $f;		push @files, $f;

my $material = XMLin(vfs_to_physical($f), ForceArray => [qw(alternative)], KeyAttr => []);		my $material = XMLin(vfs_to_physical($f), ForceArray => [qw(alternative)], KeyAttr => []);
for my $alternative (@{$material->{alternative}})		for my $alternative (@{$material->{alternative}})
Show All 16 Lines	sub add_particles
}		}
}		}

sub add_maps_xml		sub add_maps_xml
{		{
print "Loading maps XML...\n";		print "Loading maps XML...\n";
my @mapfiles = find_files('maps/scenarios', 'xml');		my @mapfiles = find_files('maps/scenarios', 'xml');
push @mapfiles, find_files('maps/skirmishes', 'xml');		push @mapfiles, find_files('maps/skirmishes', 'xml');
		push @mapfiles, find_files('maps/tutorials', 'xml');
for my $f (sort @mapfiles)		for my $f (sort @mapfiles)
{		{
print " $f\n";

push @files, $f;		push @files, $f;

push @roots, $f;		push @roots, $f;

my $map = XMLin(vfs_to_physical($f), ForceArray => [qw(Entity)], KeyAttr => []) or die "Failed to parse '$f': $!";		my $map = XMLin(vfs_to_physical($f), ForceArray => [qw(Entity)], KeyAttr => []) or die "Failed to parse '$f': $!";

my %used;		my %used;
for my $entity (@{$map->{Entities}{Entity}})		for my $entity (@{$map->{Entities}{Entity}})
{		{
$used{$entity->{Template}} = 1;		$used{$entity->{Template}} = 1;
▲ Show 20 Lines • Show All 88 Lines • ▼ Show 20 Lines

sub add_soundgroups		sub add_soundgroups
{		{
print "Loading sound groups...\n";		print "Loading sound groups...\n";
my @soundfiles = find_files('audio', 'xml');		my @soundfiles = find_files('audio', 'xml');
for my $f (sort @soundfiles)		for my $f (sort @soundfiles)
{		{
push @files, $f;		push @files, $f;
		push @roots, $f;

my $sound = XMLin(vfs_to_physical($f), ForceArray => [qw(Sound)], KeyAttr => []) or die "Failed to parse '$f': $!";		my $sound = XMLin(vfs_to_physical($f), ForceArray => [qw(Sound)], KeyAttr => []) or die "Failed to parse '$f': $!";

my $path = $sound->{Path};		my $path = $sound->{Path};
$path =~ s/\/$//; # strip optional trailing slash		$path =~ s/\/$//; # strip optional trailing slash

for (@{$sound->{Sound}})		for (@{$sound->{Sound}})
{		{
▲ Show 20 Lines • Show All 87 Lines • ▼ Show 20 Lines	for my $f (sort @guifiles)
}		}
}		}
}		}

sub add_gui_data		sub add_gui_data
{		{
print "Loading GUI data...\n";		print "Loading GUI data...\n";
push @files, find_files('gui', 'js');		push @files, find_files('gui', 'js');
push @files, find_files('art/textures/ui', 'dds\|png\|jpg\|tga');		push @files, find_files('art/textures/ui', $supportedTextureFormats);
		push @files, find_files('art/textures/selection', $supportedTextureFormats);
}		}

sub add_civs		sub add_civs
{		{
print "Loading civs...\n";		print "Loading civs...\n";

my @civfiles = find_files('simulation/data/civs', 'json');		my @civfiles = find_files('simulation/data/civs', 'json');
for my $f (sort @civfiles)		for my $f (sort @civfiles)
{		{
push @files, $f;		push @files, $f;

push @roots, $f;		push @roots, $f;

my $civ = parse_json_file($f);		my $civ = parse_json_file($f);

push @deps, [ $f, "art/textures/ui/" . $civ->{Emblem} ];		push @deps, [ $f, "art/textures/ui/" . $civ->{Emblem} ] if $civ->{Emblem};

push @deps, [ $f, "audio/music/" . $_->{File} ] for @{$civ->{Music}};		push @deps, [ $f, "audio/music/" . $_->{File} ] for @{$civ->{Music}};
}		}
}		}

sub add_rms		sub add_rms
{		{
print "Loading random maps...\n";		print "Loading random maps...\n";

push @files, find_files('maps/random', 'js');		push @files, find_files('maps/random', 'js');
my @rmsdefs = find_files('maps/random', 'json');		my @rmsdefs = find_files('maps/random', 'json');

for my $f (sort @rmsdefs)		for my $f (sort @rmsdefs)
{		{
		next if $f =~ /^maps\/random\/rmbiome/;

push @files, $f;		push @files, $f;

push @roots, $f;		push @roots, $f;

my $rms = parse_json_file($f);		my $rms = parse_json_file($f);
		push @deps, [ $f, "maps/random/" . $rms->{settings}{Script} ] if $rms->{settings}{Script};
push @deps, [ $f, "maps/random/" . $rms->{settings}{Script} ];

# Map previews		# Map previews
push @deps, [ $f, "art/textures/ui/session/icons/mappreview/" . $rms->{settings}{Preview} ] if $rms->{settings}{Preview};		push @deps, [ $f, "art/textures/ui/session/icons/mappreview/" . $rms->{settings}{Preview} ] if $rms->{settings}{Preview};
}		}
}		}

sub add_techs		sub add_techs
{		{
print "Loading techs...\n";		print "Loading techs...\n";

my @techfiles = find_files('simulation/data/technologies', 'json');		my @techfiles = find_files('simulation/data/technologies', 'json');
for my $f (sort @techfiles)		for my $f (sort @techfiles)
{		{
push @files, $f;		push @files, $f;
push @roots, $f;		push @roots, $f;

my $tech = parse_json_file($f);		my $tech = parse_json_file($f);

push @deps, [ $f, "art/textures/ui/session/portraits/technologies/" . $tech->{icon} ] if $tech->{icon};		push @deps, [ $f, "art/textures/ui/session/portraits/technologies/" . $tech->{icon} ] if $tech->{icon};
push @deps, [ $f, "simulation/data/technologies/" . $tech->{supersedes} . ".json" ] if $tech->{supersedes};		push @deps, [ $f, "simulation/data/technologies/" . $tech->{supersedes} . ".json" ] if $tech->{supersedes};
}		}
}		}

		sub add_auras
		{
		print "Loading auras...\n";

		my @aurafiles = find_files('simulation/data/auras', 'json');
		for my $f (sort @aurafiles)
		{
		push @files, $f;
		push @roots, $f;

		my $aura = parse_json_file($f);

		push @deps, [ $f, $aura->{overlayIcon} ] if $aura->{overlayIcon};

		if($aura->{rangeOverlay})
		{
		push @deps, [ $f, "art/textures/selection/" . $aura->{rangeOverlay}{lineTexture} ] if $aura->{rangeOverlay}{lineTexture};
		push @deps, [ $f, "art/textures/selection/" . $aura->{rangeOverlay}{lineTextureMask} ] if $aura->{rangeOverlay}{lineTextureMask};
		}
		}
		}

sub add_terrains		sub add_terrains
{		{
print "Loading terrains...\n";		print "Loading terrains...\n";

my @terrains = find_files('art/terrains', 'xml');		my @terrains = find_files('art/terrains', 'xml');
for my $f (sort @terrains)		for my $f (sort @terrains)
{		{
# ignore terrains.xml		# ignore terrains.xml
if ($f !~ /terrains.xml$/)		if ($f !~ /terrains.xml$/)
{		{
push @files, $f;		push @files, $f;
		push @roots, $f;

my $terrain = XMLin(vfs_to_physical($f), ForceArray => [qw(texture)], KeyAttr => []) or die "Failed to parse '$f': $!";		my $terrain = XMLin(vfs_to_physical($f), ForceArray => [qw(texture)], KeyAttr => []) or die "Failed to parse '$f': $!";

for my $texture (@{$terrain->{textures}{texture}})		for my $texture (@{$terrain->{textures}{texture}})
{		{
push @deps, [ $f, "art/textures/terrain/$texture->{file}" ] if $texture->{file};		push @deps, [ $f, "art/textures/terrain/$texture->{file}" ] if $texture->{file};
}		}
push @deps, [ $f, "art/materials/$terrain->{material}" ] if $terrain->{material};		push @deps, [ $f, "art/materials/$terrain->{material}" ] if $terrain->{material};
▲ Show 20 Lines • Show All 47 Lines • ▼ Show 20 Lines	while (1)
push @newreachable, grep { not exists $reachable{$_} } @{$deps{$r}};		push @newreachable, grep { not exists $reachable{$_} } @{$deps{$r}};
}		}
last if @newreachable == 0;		last if @newreachable == 0;
@reachable{@newreachable} = ();		@reachable{@newreachable} = ();
}		}

for my $f (sort @files)		for my $f (sort @files)
{		{
next if exists $reachable{$f};		next if exists $reachable{$f}
		\|\| index($f, "art/terrains/") != -1
		\|\| index($f, "maps/random/") != -1
		\|\| index($f, "art/materials/") != -1;
warn "Unused file '" . vfs_to_relative_to_mods($f) . "'\n";		warn "Unused file '" . vfs_to_relative_to_mods($f) . "'\n";
}		}
}		}


add_maps_xml() if CHECK_MAPS_XML;		add_maps_xml() if $checkMapXml;

add_maps_pmp();		add_maps_pmp();

add_entities();		add_entities();

add_actors();		add_actors();
		add_variants();
add_art();		add_art();

add_materials();		add_materials();

add_particles();		add_particles();

add_soundgroups();		add_soundgroups();
add_audio();		add_audio();

add_gui_xml();		add_gui_xml();
add_gui_data();		add_gui_data();

add_civs();		add_civs();

add_rms();		add_rms();

add_techs();		add_techs();

add_terrains();		add_terrains();
		add_auras();

# TODO: add non-skin textures, and all the references to them

print "\n";
check_deps();		check_deps();
print "\n";		check_unused() if $checkUnused;
check_unused();		print "\n" if $checkUnused;
		system("perl ../xmlvalidator/validate.pl") if $validateTemplates;