Index: ps/trunk/source/lib/file/archive/archive_zip.cpp =================================================================== --- ps/trunk/source/lib/file/archive/archive_zip.cpp (revision 8081) +++ ps/trunk/source/lib/file/archive/archive_zip.cpp (revision 8082) @@ -1,667 +1,672 @@ /* Copyright (c) 2010 Wildfire Games * * Permission is hereby granted, free of charge, to any person obtaining * a copy of this software and associated documentation files (the * "Software"), to deal in the Software without restriction, including * without limitation the rights to use, copy, modify, merge, publish, * distribute, sublicense, and/or sell copies of the Software, and to * permit persons to whom the Software is furnished to do so, subject to * the following conditions: * * The above copyright notice and this permission notice shall be included * in all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ /* * archive backend for Zip files. */ #include "precompiled.h" #include "lib/file/archive/archive_zip.h" #include #include #include "lib/bits.h" #include "lib/byte_order.h" #include "lib/utf8.h" // wstring_from_utf8 #include "lib/fat_time.h" #include "lib/path_util.h" #include "lib/allocators/pool.h" #include "lib/sysdep/cpu.h" // cpu_memcpy #include "lib/file/archive/archive.h" #include "lib/file/archive/codec_zlib.h" #include "lib/file/archive/stream.h" #include "lib/file/file.h" #include "lib/file/io/io.h" #include "lib/file/io/io_align.h" // BLOCK_SIZE #include "lib/file/io/write_buffer.h" //----------------------------------------------------------------------------- // Zip archive definitions //----------------------------------------------------------------------------- static const u32 cdfh_magic = FOURCC_LE('P','K','\1','\2'); static const u32 lfh_magic = FOURCC_LE('P','K','\3','\4'); static const u32 ecdr_magic = FOURCC_LE('P','K','\5','\6'); enum ZipMethod { ZIP_METHOD_NONE = 0, ZIP_METHOD_DEFLATE = 8 }; #pragma pack(push, 1) class LFH { public: void Init(const FileInfo& fileInfo, off_t csize, ZipMethod method, u32 checksum, const fs::wpath& pathname) { const fs::path pathname_c = path_from_wpath(pathname); const size_t pathnameLength = pathname_c.string().length(); m_magic = lfh_magic; m_x1 = to_le16(0); m_flags = to_le16(0); m_method = to_le16(u16_from_larger(method)); m_fat_mtime = to_le32(FAT_from_time_t(fileInfo.MTime())); m_crc = to_le32(checksum); m_csize = to_le32(u32_from_larger(csize)); m_usize = to_le32(u32_from_larger(fileInfo.Size())); m_fn_len = to_le16(u16_from_larger(pathnameLength)); m_e_len = to_le16(0); cpu_memcpy((char*)this + sizeof(LFH), pathname_c.string().c_str(), pathnameLength); } size_t Size() const { debug_assert(m_magic == lfh_magic); size_t size = sizeof(LFH); size += read_le16(&m_fn_len); size += read_le16(&m_e_len); // note: LFH doesn't have a comment field! return size; } private: u32 m_magic; u16 m_x1; // version needed u16 m_flags; u16 m_method; u32 m_fat_mtime; // last modified time (DOS FAT format) u32 m_crc; u32 m_csize; u32 m_usize; u16 m_fn_len; u16 m_e_len; }; cassert(sizeof(LFH) == 30); class CDFH { public: void Init(const FileInfo& fileInfo, off_t ofs, off_t csize, ZipMethod method, u32 checksum, const fs::wpath& pathname, size_t slack) { const fs::path pathname_c = path_from_wpath(pathname); const size_t pathnameLength = pathname_c.string().length(); m_magic = cdfh_magic; m_x1 = to_le32(0); m_flags = to_le16(0); m_method = to_le16(u16_from_larger(method)); m_fat_mtime = to_le32(FAT_from_time_t(fileInfo.MTime())); m_crc = to_le32(checksum); m_csize = to_le32(u32_from_larger(csize)); m_usize = to_le32(u32_from_larger(fileInfo.Size())); m_fn_len = to_le16(u16_from_larger(pathnameLength)); m_e_len = to_le16(0); m_c_len = to_le16(u16_from_larger((size_t)slack)); m_x2 = to_le32(0); m_x3 = to_le32(0); m_lfh_ofs = to_le32(u32_from_larger(ofs)); cpu_memcpy((char*)this + sizeof(CDFH), pathname_c.string().c_str(), pathnameLength); } fs::wpath Pathname() const { const size_t length = (size_t)read_le16(&m_fn_len); const char* pathname = (const char*)this + sizeof(CDFH); // not 0-terminated! return wstring_from_utf8(std::string(pathname, length)); } off_t HeaderOffset() const { return read_le32(&m_lfh_ofs); } off_t USize() const { return (off_t)read_le32(&m_usize); } off_t CSize() const { return (off_t)read_le32(&m_csize); } ZipMethod Method() const { return (ZipMethod)read_le16(&m_method); } u32 Checksum() const { return read_le32(&m_crc); } time_t MTime() const { const u32 fat_mtime = read_le32(&m_fat_mtime); return time_t_from_FAT(fat_mtime); } size_t Size() const { size_t size = sizeof(CDFH); size += read_le16(&m_fn_len); size += read_le16(&m_e_len); size += read_le16(&m_c_len); return size; } private: u32 m_magic; u32 m_x1; // versions u16 m_flags; u16 m_method; u32 m_fat_mtime; // last modified time (DOS FAT format) u32 m_crc; u32 m_csize; u32 m_usize; u16 m_fn_len; u16 m_e_len; u16 m_c_len; u32 m_x2; // spanning u32 m_x3; // attributes u32 m_lfh_ofs; }; cassert(sizeof(CDFH) == 46); class ECDR { public: void Init(size_t cd_numEntries, off_t cd_ofs, size_t cd_size) { m_magic = ecdr_magic; m_diskNum = to_le16(0); m_cd_diskNum = to_le16(0); m_cd_numEntriesOnDisk = to_le16(u16_from_larger(cd_numEntries)); m_cd_numEntries = m_cd_numEntriesOnDisk; m_cd_size = to_le32(u32_from_larger(cd_size)); m_cd_ofs = to_le32(u32_from_larger(cd_ofs)); m_comment_len = to_le16(0); } void Decompose(size_t& cd_numEntries, off_t& cd_ofs, size_t& cd_size) const { cd_numEntries = (size_t)read_le16(&m_cd_numEntries); cd_ofs = (off_t)read_le32(&m_cd_ofs); cd_size = (size_t)read_le32(&m_cd_size); } private: u32 m_magic; u16 m_diskNum; u16 m_cd_diskNum; u16 m_cd_numEntriesOnDisk; u16 m_cd_numEntries; u32 m_cd_size; u32 m_cd_ofs; u16 m_comment_len; }; cassert(sizeof(ECDR) == 22); #pragma pack(pop) //----------------------------------------------------------------------------- // ArchiveFile_Zip //----------------------------------------------------------------------------- class ArchiveFile_Zip : public IArchiveFile { public: ArchiveFile_Zip(const PFile& file, off_t ofs, off_t csize, u32 checksum, ZipMethod method) : m_file(file), m_ofs(ofs) , m_csize(csize), m_checksum(checksum), m_method((u16)method) , m_flags(NeedsFixup) { } virtual size_t Precedence() const { return 2u; } virtual wchar_t LocationCode() const { return 'A'; } + virtual fs::wpath Path() const + { + return m_file->Pathname(); + } + virtual LibError Load(const std::wstring& UNUSED(name), const shared_ptr& buf, size_t size) const { AdjustOffset(); PICodec codec; switch(m_method) { case ZIP_METHOD_NONE: codec = CreateCodec_ZLibNone(); break; case ZIP_METHOD_DEFLATE: codec = CreateDecompressor_ZLibDeflate(); break; default: WARN_RETURN(ERR::ARCHIVE_UNKNOWN_METHOD); } Stream stream(codec); stream.SetOutputBuffer(buf.get(), size); RETURN_ERR(io_Scan(m_file, m_ofs, m_csize, FeedStream, (uintptr_t)&stream)); RETURN_ERR(stream.Finish()); #if CODEC_COMPUTE_CHECKSUM debug_assert(m_checksum == stream.Checksum()); #endif return INFO::OK; } private: enum Flags { // indicates m_ofs points to a "local file header" instead of // the file data. a fixup routine is called when reading the file; // it skips past the LFH and clears this flag. // this is somewhat of a hack, but vital to archive open performance. // without it, we'd have to scan through the entire archive file, // which can take *seconds*. // (we cannot use the information in CDFH, because its 'extra' field // has been observed to differ from that of the LFH) // since we read the LFH right before the rest of the file, the block // cache will absorb the IO cost. NeedsFixup = 1 }; struct LFH_Copier { u8* lfh_dst; size_t lfh_bytes_remaining; }; // this code grabs an LFH struct from file block(s) that are // passed to the callback. usually, one call copies the whole thing, // but the LFH may straddle a block boundary. // // rationale: this allows using temp buffers for zip_fixup_lfh, // which avoids involving the file buffer manager and thus // avoids cluttering the trace and cache contents. static LibError lfh_copier_cb(uintptr_t cbData, const u8* block, size_t size) { LFH_Copier* p = (LFH_Copier*)cbData; debug_assert(size <= p->lfh_bytes_remaining); cpu_memcpy(p->lfh_dst, block, size); p->lfh_dst += size; p->lfh_bytes_remaining -= size; return INFO::CB_CONTINUE; } /** * fix up m_ofs (adjust it to point to cdata instead of the LFH). * * note: we cannot use CDFH filename and extra field lengths to skip * past LFH since that may not mirror CDFH (has happened). * * this is called at file-open time instead of while mounting to * reduce seeks: since reading the file will typically follow, the * block cache entirely absorbs the IO cost. **/ void AdjustOffset() const { if(!(m_flags & NeedsFixup)) return; m_flags &= ~NeedsFixup; // performance note: this ends up reading one file block, which is // only in the block cache if the file starts in the same block as a // previously read file (i.e. both are small). LFH lfh; LFH_Copier params = { (u8*)&lfh, sizeof(LFH) }; if(io_Scan(m_file, m_ofs, sizeof(LFH), lfh_copier_cb, (uintptr_t)¶ms) == INFO::OK) m_ofs += (off_t)lfh.Size(); } PFile m_file; // all relevant LFH/CDFH fields not covered by FileInfo mutable off_t m_ofs; off_t m_csize; u32 m_checksum; u16 m_method; mutable u16 m_flags; }; //----------------------------------------------------------------------------- // ArchiveReader_Zip //----------------------------------------------------------------------------- class ArchiveReader_Zip : public IArchiveReader { public: ArchiveReader_Zip(const fs::wpath& pathname) : m_file(new File(pathname, 'r')) { FileInfo fileInfo; GetFileInfo(pathname, &fileInfo); m_fileSize = fileInfo.Size(); const size_t minFileSize = sizeof(LFH)+sizeof(CDFH)+sizeof(ECDR); debug_assert(m_fileSize >= off_t(minFileSize)); } virtual LibError ReadEntries(ArchiveEntryCallback cb, uintptr_t cbData) { // locate and read Central Directory off_t cd_ofs; size_t cd_numEntries; size_t cd_size; RETURN_ERR(LocateCentralDirectory(m_file, m_fileSize, cd_ofs, cd_numEntries, cd_size)); shared_ptr buf = io_Allocate(cd_size, cd_ofs); u8* cd; RETURN_ERR(io_Read(m_file, cd_ofs, buf.get(), cd_size, cd)); // iterate over Central Directory const u8* pos = cd; for(size_t i = 0; i < cd_numEntries; i++) { // scan for next CDFH CDFH* cdfh = (CDFH*)FindRecord(cd, cd_size, pos, cdfh_magic, sizeof(CDFH)); if(!cdfh) WARN_RETURN(ERR::CORRUPTED); const VfsPath relativePathname(cdfh->Pathname().string()); // convert from fs::wpath const std::wstring name = relativePathname.leaf(); if(name != L".") // ignore directories (i.e. paths ending in slash) { FileInfo fileInfo(name, cdfh->USize(), cdfh->MTime()); shared_ptr archiveFile(new ArchiveFile_Zip(m_file, cdfh->HeaderOffset(), cdfh->CSize(), cdfh->Checksum(), cdfh->Method())); cb(relativePathname, fileInfo, archiveFile, cbData); } pos += cdfh->Size(); } return INFO::OK; } private: /** * Scan buffer for a Zip file record. * * @param buf * @param size * @param start position within buffer * @param magic signature of record * @param recordSize size of record (including signature) * @return pointer to record within buffer or 0 if not found. **/ static const u8* FindRecord(const u8* buf, size_t size, const u8* start, u32 magic, size_t recordSize) { // (don't use as the counter - otherwise we can't tell if // scanning within the buffer was necessary.) for(const u8* p = start; p <= buf+size-recordSize; p++) { // found it if(*(u32*)p == magic) { debug_assert(p == start); // otherwise, the archive is a bit broken return p; } } // passed EOF, didn't find it. // note: do not warn - this happens in the initial ECDR search at // EOF if the archive contains a comment field. return 0; } // search for ECDR in the last bytes of the file. // if found, fill with a copy of the (little-endian) ECDR and // return INFO::OK, otherwise IO error or ERR::CORRUPTED. static LibError ScanForEcdr(const PFile& file, off_t fileSize, u8* buf, size_t maxScanSize, size_t& cd_numEntries, off_t& cd_ofs, size_t& cd_size) { // don't scan more than the entire file const size_t scanSize = std::min(maxScanSize, size_t(fileSize)); // read desired chunk of file into memory const off_t ofs = fileSize - off_t(scanSize); u8* data; RETURN_ERR(io_Read(file, ofs, buf, scanSize, data)); // look for ECDR in buffer const ECDR* ecdr = (const ECDR*)FindRecord(data, scanSize, data, ecdr_magic, sizeof(ECDR)); if(!ecdr) return INFO::CANNOT_HANDLE; ecdr->Decompose(cd_numEntries, cd_ofs, cd_size); return INFO::OK; } static LibError LocateCentralDirectory(const PFile& file, off_t fileSize, off_t& cd_ofs, size_t& cd_numEntries, size_t& cd_size) { const size_t maxScanSize = 66000u; // see below shared_ptr buf = io_Allocate(maxScanSize, BLOCK_SIZE-1); // assume worst-case for alignment // expected case: ECDR at EOF; no file comment LibError ret = ScanForEcdr(file, fileSize, const_cast(buf.get()), sizeof(ECDR), cd_numEntries, cd_ofs, cd_size); if(ret == INFO::OK) return INFO::OK; // worst case: ECDR precedes 64 KiB of file comment ret = ScanForEcdr(file, fileSize, const_cast(buf.get()), maxScanSize, cd_numEntries, cd_ofs, cd_size); if(ret == INFO::OK) return INFO::OK; // both ECDR scans failed - this is not a valid Zip file. RETURN_ERR(io_ReadAligned(file, 0, const_cast(buf.get()), sizeof(LFH))); // the Zip file has an LFH but lacks an ECDR. this can happen if // the user hard-exits while an archive is being written. // notes: // - return ERR::CORRUPTED so VFS will not include this file. // - we could work around this by scanning all LFHs, but won't bother // because it'd be slow. // - do not warn - the corrupt archive will be deleted on next // successful archive builder run anyway. if(FindRecord(buf.get(), sizeof(LFH), buf.get(), lfh_magic, sizeof(LFH))) return ERR::CORRUPTED; // NOWARN // totally bogus else WARN_RETURN(ERR::ARCHIVE_UNKNOWN_FORMAT); } PFile m_file; off_t m_fileSize; }; PIArchiveReader CreateArchiveReader_Zip(const fs::wpath& archivePathname) { return PIArchiveReader(new ArchiveReader_Zip(archivePathname)); } //----------------------------------------------------------------------------- // ArchiveWriter_Zip //----------------------------------------------------------------------------- class ArchiveWriter_Zip : public IArchiveWriter { public: ArchiveWriter_Zip(const fs::wpath& archivePathname) : m_file(new File(archivePathname, 'w')), m_fileSize(0) , m_unalignedWriter(new UnalignedWriter(m_file, 0)) , m_numEntries(0) { THROW_ERR(pool_create(&m_cdfhPool, 10*MiB, 0)); } ~ArchiveWriter_Zip() { // append an ECDR to the CDFH list (this allows us to // write out both to the archive file in one burst) const size_t cd_size = m_cdfhPool.da.pos; ECDR* ecdr = (ECDR*)pool_alloc(&m_cdfhPool, sizeof(ECDR)); if(!ecdr) throw std::bad_alloc(); const off_t cd_ofs = m_fileSize; ecdr->Init(m_numEntries, cd_ofs, cd_size); m_unalignedWriter->Append(m_cdfhPool.da.base, cd_size+sizeof(ECDR)); m_unalignedWriter->Flush(); m_unalignedWriter.reset(); (void)pool_destroy(&m_cdfhPool); const fs::wpath pathname = m_file->Pathname(); // for truncate() m_file.reset(); m_fileSize += off_t(cd_size+sizeof(ECDR)); // remove padding added by UnalignedWriter wtruncate(pathname.string().c_str(), m_fileSize); } LibError AddFile(const fs::wpath& pathname) { FileInfo fileInfo; RETURN_ERR(GetFileInfo(pathname, &fileInfo)); const off_t usize = fileInfo.Size(); // skip 0-length files. // rationale: zip.cpp needs to determine whether a CDFH entry is // a file or directory (the latter are written by some programs but // not needed - they'd only pollute the file table). // it looks like checking for usize=csize=0 is the safest way - // relying on file attributes (which are system-dependent!) is // even less safe. // we thus skip 0-length files to avoid confusing them with directories. if(!usize) return INFO::SKIPPED; PFile file(new File); RETURN_ERR(file->Open(pathname, 'r')); const size_t pathnameLength = pathname.string().length(); // choose method and the corresponding codec ZipMethod method; PICodec codec; if(IsFileTypeIncompressible(pathname)) { method = ZIP_METHOD_NONE; codec = CreateCodec_ZLibNone(); } else { method = ZIP_METHOD_DEFLATE; codec = CreateCompressor_ZLibDeflate(); } // allocate memory const size_t csizeMax = codec->MaxOutputSize(size_t(usize)); shared_ptr buf = io_Allocate(sizeof(LFH) + pathnameLength + csizeMax); // read and compress file contents size_t csize; u32 checksum; { u8* cdata = (u8*)buf.get() + sizeof(LFH) + pathnameLength; Stream stream(codec); stream.SetOutputBuffer(cdata, csizeMax); RETURN_ERR(io_Scan(file, 0, usize, FeedStream, (uintptr_t)&stream)); RETURN_ERR(stream.Finish()); csize = stream.OutSize(); checksum = stream.Checksum(); } // build LFH { LFH* lfh = (LFH*)buf.get(); lfh->Init(fileInfo, (off_t)csize, method, checksum, pathname); } // append a CDFH to the central directory (in memory) const off_t ofs = m_fileSize; const size_t prev_pos = m_cdfhPool.da.pos; // (required to determine padding size) const size_t cdfhSize = sizeof(CDFH) + pathnameLength; CDFH* cdfh = (CDFH*)pool_alloc(&m_cdfhPool, cdfhSize); if(!cdfh) WARN_RETURN(ERR::NO_MEM); const size_t slack = m_cdfhPool.da.pos - prev_pos - cdfhSize; cdfh->Init(fileInfo, ofs, (off_t)csize, method, checksum, pathname, slack); m_numEntries++; // write LFH, pathname and cdata to file const size_t packageSize = sizeof(LFH) + pathnameLength + csize; RETURN_ERR(m_unalignedWriter->Append(buf.get(), packageSize)); m_fileSize += (off_t)packageSize; return INFO::OK; } private: static bool IsFileTypeIncompressible(const fs::wpath& pathname) { const std::wstring extension = fs::extension(pathname); // file extensions that we don't want to compress static const wchar_t* incompressibleExtensions[] = { L".zip", L".rar", L".jpg", L".jpeg", L".png", L".ogg", L".mp3" }; for(size_t i = 0; i < ARRAY_SIZE(incompressibleExtensions); i++) { if(!wcscasecmp(extension.c_str(), incompressibleExtensions[i])) return true; } return false; } PFile m_file; off_t m_fileSize; PUnalignedWriter m_unalignedWriter; Pool m_cdfhPool; size_t m_numEntries; }; PIArchiveWriter CreateArchiveWriter_Zip(const fs::wpath& archivePathname) { return PIArchiveWriter(new ArchiveWriter_Zip(archivePathname)); } Index: ps/trunk/source/lib/file/common/file_loader.h =================================================================== --- ps/trunk/source/lib/file/common/file_loader.h (revision 8081) +++ ps/trunk/source/lib/file/common/file_loader.h (revision 8082) @@ -1,38 +1,39 @@ /* Copyright (c) 2010 Wildfire Games * * Permission is hereby granted, free of charge, to any person obtaining * a copy of this software and associated documentation files (the * "Software"), to deal in the Software without restriction, including * without limitation the rights to use, copy, modify, merge, publish, * distribute, sublicense, and/or sell copies of the Software, and to * permit persons to whom the Software is furnished to do so, subject to * the following conditions: * * The above copyright notice and this permission notice shall be included * in all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ #ifndef INCLUDED_FILE_LOADER #define INCLUDED_FILE_LOADER struct IFileLoader { virtual ~IFileLoader(); virtual size_t Precedence() const = 0; virtual wchar_t LocationCode() const = 0; + virtual fs::wpath Path() const = 0; virtual LibError Load(const std::wstring& name, const shared_ptr& buf, size_t size) const = 0; }; typedef shared_ptr PIFileLoader; #endif // #ifndef INCLUDED_FILE_LOADER Index: ps/trunk/source/lib/file/common/real_directory.h =================================================================== --- ps/trunk/source/lib/file/common/real_directory.h (revision 8081) +++ ps/trunk/source/lib/file/common/real_directory.h (revision 8082) @@ -1,78 +1,77 @@ /* Copyright (c) 2010 Wildfire Games * * Permission is hereby granted, free of charge, to any person obtaining * a copy of this software and associated documentation files (the * "Software"), to deal in the Software without restriction, including * without limitation the rights to use, copy, modify, merge, publish, * distribute, sublicense, and/or sell copies of the Software, and to * permit persons to whom the Software is furnished to do so, subject to * the following conditions: * * The above copyright notice and this permission notice shall be included * in all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ #ifndef INCLUDED_REAL_DIRECTORY #define INCLUDED_REAL_DIRECTORY #include "lib/file/common/file_loader.h" #include "lib/sysdep/dir_watch.h" class RealDirectory : public IFileLoader { NONCOPYABLE(RealDirectory); public: RealDirectory(const fs::wpath& path, size_t priority, size_t flags); - const fs::wpath& Path() const - { - return m_path; - } - size_t Priority() const { return m_priority; } size_t Flags() const { return m_flags; } // IFileLoader virtual size_t Precedence() const; virtual wchar_t LocationCode() const; + virtual fs::wpath Path() const + { + return m_path; + } virtual LibError Load(const std::wstring& name, const shared_ptr& buf, size_t size) const; LibError Store(const std::wstring& name, const shared_ptr& fileContents, size_t size); void Watch(); private: // note: paths are relative to the root directory, so storing the // entire path instead of just the portion relative to the mount point // is not all too wasteful. const fs::wpath m_path; const size_t m_priority; const size_t m_flags; // note: watches are needed in each directory because some APIs // (e.g. FAM) cannot watch entire trees with one call. PDirWatch m_watch; }; typedef shared_ptr PRealDirectory; extern PRealDirectory CreateRealSubdirectory(const PRealDirectory& realDirectory, const std::wstring& subdirectoryName); #endif // #ifndef INCLUDED_REAL_DIRECTORY Index: ps/trunk/source/lib/file/vfs/vfs.cpp =================================================================== --- ps/trunk/source/lib/file/vfs/vfs.cpp (revision 8081) +++ ps/trunk/source/lib/file/vfs/vfs.cpp (revision 8082) @@ -1,239 +1,239 @@ /* Copyright (c) 2010 Wildfire Games * * Permission is hereby granted, free of charge, to any person obtaining * a copy of this software and associated documentation files (the * "Software"), to deal in the Software without restriction, including * without limitation the rights to use, copy, modify, merge, publish, * distribute, sublicense, and/or sell copies of the Software, and to * permit persons to whom the Software is furnished to do so, subject to * the following conditions: * * The above copyright notice and this permission notice shall be included * in all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ #include "precompiled.h" #include "lib/file/vfs/vfs.h" #include "lib/allocators/shared_ptr.h" #include "lib/path_util.h" #include "lib/file/common/file_stats.h" #include "lib/file/common/trace.h" #include "lib/file/archive/archive.h" #include "lib/file/io/io.h" #include "lib/file/vfs/vfs_tree.h" #include "lib/file/vfs/vfs_lookup.h" #include "lib/file/vfs/vfs_populate.h" #include "lib/file/vfs/file_cache.h" ERROR_ASSOCIATE(ERR::VFS_DIR_NOT_FOUND, L"VFS directory not found", -1); ERROR_ASSOCIATE(ERR::VFS_FILE_NOT_FOUND, L"VFS file not found", -1); ERROR_ASSOCIATE(ERR::VFS_ALREADY_MOUNTED, L"VFS path already mounted", -1); class VFS : public IVFS { public: VFS(size_t cacheSize) : m_cacheSize(cacheSize), m_fileCache(m_cacheSize) , m_trace(CreateTrace(4*MiB)) { } virtual LibError Mount(const VfsPath& mountPoint, const fs::wpath& path, size_t flags /* = 0 */, size_t priority /* = 0 */) { if(!fs::exists(path)) { if(flags & VFS_MOUNT_MUST_EXIST) return ERR::VFS_DIR_NOT_FOUND; // NOWARN else RETURN_ERR(CreateDirectories(path, 0700)); } VfsDirectory* directory; CHECK_ERR(vfs_Lookup(mountPoint, &m_rootDirectory, directory, 0, VFS_LOOKUP_ADD)); PRealDirectory realDirectory(new RealDirectory(path, priority, flags)); RETURN_ERR(vfs_Attach(directory, realDirectory)); return INFO::OK; } virtual LibError GetFileInfo(const VfsPath& pathname, FileInfo* pfileInfo) const { VfsDirectory* directory; VfsFile* file; LibError ret = vfs_Lookup(pathname, &m_rootDirectory, directory, &file); if(!pfileInfo) // just indicate if the file exists without raising warnings. return ret; CHECK_ERR(ret); *pfileInfo = FileInfo(file->Name(), file->Size(), file->MTime()); return INFO::OK; } virtual LibError GetDirectoryEntries(const VfsPath& path, FileInfos* fileInfos, DirectoryNames* subdirectoryNames) const { VfsDirectory* directory; CHECK_ERR(vfs_Lookup(path, &m_rootDirectory, directory, 0)); if(fileInfos) { const VfsDirectory::VfsFiles& files = directory->Files(); fileInfos->clear(); fileInfos->reserve(files.size()); for(VfsDirectory::VfsFiles::const_iterator it = files.begin(); it != files.end(); ++it) { const VfsFile& file = it->second; fileInfos->push_back(FileInfo(file.Name(), file.Size(), file.MTime())); } } if(subdirectoryNames) { const VfsDirectory::VfsSubdirectories& subdirectories = directory->Subdirectories(); subdirectoryNames->clear(); subdirectoryNames->reserve(subdirectories.size()); for(VfsDirectory::VfsSubdirectories::const_iterator it = subdirectories.begin(); it != subdirectories.end(); ++it) subdirectoryNames->push_back(it->first); } return INFO::OK; } virtual LibError CreateFile(const VfsPath& pathname, const shared_ptr& fileContents, size_t size) { VfsDirectory* directory; CHECK_ERR(vfs_Lookup(pathname, &m_rootDirectory, directory, 0, VFS_LOOKUP_ADD|VFS_LOOKUP_CREATE)); const PRealDirectory& realDirectory = directory->AssociatedDirectory(); const std::wstring& name = pathname.leaf(); RETURN_ERR(realDirectory->Store(name, fileContents, size)); // wipe out any cached blocks. this is necessary to cover the (rare) case // of file cache contents predating the file write. m_fileCache.Remove(pathname); const VfsFile file(name, size, time(0), realDirectory->Priority(), realDirectory); directory->AddFile(file); m_trace->NotifyStore(pathname.string().c_str(), size); return INFO::OK; } virtual LibError LoadFile(const VfsPath& pathname, shared_ptr& fileContents, size_t& size) { const bool isCacheHit = m_fileCache.Retrieve(pathname, fileContents, size); if(!isCacheHit) { VfsDirectory* directory; VfsFile* file; // per 2010-05-01 meeting, this shouldn't raise 'scary error // dialogs', which often fail to display the culprit pathname // (debug_DumpStack doesn't correctly analyze fs::[w]path). // instead, callers should log the error, including pathname. RETURN_ERR(vfs_Lookup(pathname, &m_rootDirectory, directory, &file)); size = file->Size(); // safely handle zero-length files if(!size) fileContents = DummySharedPtr((u8*)0); else if(size > m_cacheSize) { fileContents = io_Allocate(size); - RETURN_ERR(file->Load(fileContents)); + RETURN_ERR(file->Loader()->Load(file->Name(), fileContents, file->Size())); } else { fileContents = m_fileCache.Reserve(size); - RETURN_ERR(file->Load(fileContents)); + RETURN_ERR(file->Loader()->Load(file->Name(), fileContents, file->Size())); m_fileCache.Add(pathname, fileContents, size); } } stats_io_user_request(size); stats_cache(isCacheHit? CR_HIT : CR_MISS, size); m_trace->NotifyLoad(pathname.string().c_str(), size); return INFO::OK; } virtual std::wstring TextRepresentation() const { std::wstring textRepresentation; textRepresentation.reserve(100*KiB); DirectoryDescriptionR(textRepresentation, m_rootDirectory, 0); return textRepresentation; } virtual LibError GetRealPath(const VfsPath& pathname, fs::wpath& realPathname) { - VfsDirectory* directory; - CHECK_ERR(vfs_Lookup(pathname, &m_rootDirectory, directory, 0)); - realPathname = directory->AssociatedDirectory()->Path() / pathname.leaf(); + VfsDirectory* directory; VfsFile* file; + CHECK_ERR(vfs_Lookup(pathname, &m_rootDirectory, directory, &file)); + realPathname = file->Loader()->Path() / pathname.leaf(); return INFO::OK; } virtual LibError GetVirtualPath(const fs::wpath& realPathname, VfsPath& pathname) { const fs::wpath realPath = AddSlash(realPathname.branch_path()); VfsPath path; RETURN_ERR(FindRealPathR(realPath, m_rootDirectory, L"", path)); pathname = path / realPathname.leaf(); return INFO::OK; } virtual LibError Invalidate(const VfsPath& pathname) { m_fileCache.Remove(pathname); VfsDirectory* directory; RETURN_ERR(vfs_Lookup(pathname, &m_rootDirectory, directory, 0)); const std::wstring name = pathname.leaf(); directory->Invalidate(name); return INFO::OK; } virtual void Clear() { m_rootDirectory.Clear(); } private: LibError FindRealPathR(const fs::wpath& realPath, const VfsDirectory& directory, const VfsPath& curPath, VfsPath& path) { PRealDirectory realDirectory = directory.AssociatedDirectory(); if(realDirectory && realDirectory->Path() == realPath) { path = curPath; return INFO::OK; } const VfsDirectory::VfsSubdirectories& subdirectories = directory.Subdirectories(); for(VfsDirectory::VfsSubdirectories::const_iterator it = subdirectories.begin(); it != subdirectories.end(); ++it) { const std::wstring& subdirectoryName = it->first; const VfsDirectory& subdirectory = it->second; LibError ret = FindRealPathR(realPath, subdirectory, AddSlash(curPath/subdirectoryName), path); if(ret == INFO::OK) return INFO::OK; } return ERR::PATH_NOT_FOUND; // NOWARN } size_t m_cacheSize; FileCache m_fileCache; PITrace m_trace; mutable VfsDirectory m_rootDirectory; }; //----------------------------------------------------------------------------- PIVFS CreateVfs(size_t cacheSize) { return PIVFS(new VFS(cacheSize)); } Index: ps/trunk/source/lib/file/vfs/vfs_tree.h =================================================================== --- ps/trunk/source/lib/file/vfs/vfs_tree.h (revision 8081) +++ ps/trunk/source/lib/file/vfs/vfs_tree.h (revision 8082) @@ -1,171 +1,172 @@ /* Copyright (c) 2010 Wildfire Games * * Permission is hereby granted, free of charge, to any person obtaining * a copy of this software and associated documentation files (the * "Software"), to deal in the Software without restriction, including * without limitation the rights to use, copy, modify, merge, publish, * distribute, sublicense, and/or sell copies of the Software, and to * permit persons to whom the Software is furnished to do so, subject to * the following conditions: * * The above copyright notice and this permission notice shall be included * in all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ /* * 'tree' of VFS directories and files */ #ifndef INCLUDED_VFS_TREE #define INCLUDED_VFS_TREE #include #include "lib/file/file_system.h" // FileInfo #include "lib/file/common/file_loader.h" // PIFileLoader #include "lib/file/common/real_directory.h" // PRealDirectory class VfsFile { public: VfsFile(const std::wstring& name, size_t size, time_t mtime, size_t priority, const PIFileLoader& provider); const std::wstring& Name() const { return m_name; } - wchar_t LocationCode() const - { - return m_loader->LocationCode(); - } - size_t Size() const { return m_size; } time_t MTime() const { return m_mtime; } - bool IsSupersededBy(const VfsFile& file) const; + size_t Priority() const + { + return m_priority; + } - LibError Load(const shared_ptr& buf) const; + const PIFileLoader& Loader() const + { + return m_loader; + } private: std::wstring m_name; size_t m_size; time_t m_mtime; size_t m_priority; PIFileLoader m_loader; }; class VfsDirectory { public: typedef std::map VfsFiles; typedef std::map VfsSubdirectories; VfsDirectory(); /** * @return address of existing or newly inserted file. **/ VfsFile* AddFile(const VfsFile& file); /** * @return address of existing or newly inserted subdirectory. **/ VfsDirectory* AddSubdirectory(const std::wstring& name); /** * @return file with the given name. * (note: non-const to allow changes to the file) **/ VfsFile* GetFile(const std::wstring& name); /** * @return subdirectory with the given name. * (note: non-const to allow changes to the subdirectory) **/ VfsDirectory* GetSubdirectory(const std::wstring& name); // note: exposing only iterators wouldn't enable callers to reserve space. const VfsFiles& Files() const { return m_files; } const VfsSubdirectories& Subdirectories() const { return m_subdirectories; } /** * side effect: the next ShouldPopulate() will return true. **/ void SetAssociatedDirectory(const PRealDirectory& realDirectory); const PRealDirectory& AssociatedDirectory() const { return m_realDirectory; } /** * @return whether this directory should be populated from its * AssociatedDirectory(). note that calling this is a promise to * do so if true is returned -- the flag is reset immediately. **/ bool ShouldPopulate(); /** * indicate that a file has changed; ensure its new version supersedes * the old by removing it and marking the directory for re-population. **/ void Invalidate(const std::wstring& name); /** * empty file and subdirectory lists (e.g. when rebuilding VFS). * CAUTION: this invalidates all previously returned pointers. **/ void Clear(); private: VfsFiles m_files; VfsSubdirectories m_subdirectories; PRealDirectory m_realDirectory; volatile intptr_t m_shouldPopulate; // (cpu_CAS can't be used on bool) }; /** * @return a string containing file attributes (location, size, timestamp) and name. **/ extern std::wstring FileDescription(const VfsFile& file); /** * @return a string holding each files' description (one per line). **/ extern std::wstring FileDescriptions(const VfsDirectory& directory, size_t indentLevel); /** * append each directory's files' description to the given string. **/ void DirectoryDescriptionR(std::wstring& descriptions, const VfsDirectory& directory, size_t indentLevel); #endif // #ifndef INCLUDED_VFS_TREE Index: ps/trunk/source/lib/file/vfs/vfs_tree.cpp =================================================================== --- ps/trunk/source/lib/file/vfs/vfs_tree.cpp (revision 8081) +++ ps/trunk/source/lib/file/vfs/vfs_tree.cpp (revision 8082) @@ -1,209 +1,204 @@ /* Copyright (c) 2010 Wildfire Games * * Permission is hereby granted, free of charge, to any person obtaining * a copy of this software and associated documentation files (the * "Software"), to deal in the Software without restriction, including * without limitation the rights to use, copy, modify, merge, publish, * distribute, sublicense, and/or sell copies of the Software, and to * permit persons to whom the Software is furnished to do so, subject to * the following conditions: * * The above copyright notice and this permission notice shall be included * in all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ /* * 'tree' of VFS directories and files */ #include "precompiled.h" #include "lib/file/vfs/vfs_tree.h" #include #include "lib/file/common/file_stats.h" #include "lib/sysdep/cpu.h" //----------------------------------------------------------------------------- VfsFile::VfsFile(const std::wstring& name, size_t size, time_t mtime, size_t priority, const PIFileLoader& loader) : m_name(name), m_size(size), m_mtime(mtime), m_priority(priority), m_loader(loader) { } -bool VfsFile::IsSupersededBy(const VfsFile& file) const + +//----------------------------------------------------------------------------- + +VfsDirectory::VfsDirectory() + : m_shouldPopulate(0) +{ +} + + +static bool ShouldReplaceWith(const VfsFile& previousFile, const VfsFile& newFile) { // 1) priority (override mods) - if(file.m_priority < m_priority) // lower priority + if(newFile.Priority() < previousFile.Priority()) return false; // 2) timestamp { - const double howMuchNewer = difftime(file.MTime(), MTime()); - const double threshold = 2.0; // [seconds]; resolution provided by FAT - if(howMuchNewer > threshold) // newer timestamp + const double howMuchNewer = difftime(newFile.MTime(), previousFile.MTime()); + const double threshold = 2.0; // FAT timestamp resolution [seconds] + if(howMuchNewer > threshold) // newer return true; - if(howMuchNewer < -threshold) // older timestamp + if(howMuchNewer < -threshold) // older return false; // else: "equal" (tolerating small differences due to FAT's low // mtime resolution) } // 3) precedence (efficiency of file provider) - if(file.m_loader->Precedence() < m_loader->Precedence()) // less efficient + if(newFile.Loader()->Precedence() < previousFile.Loader()->Precedence()) return false; return true; } -LibError VfsFile::Load(const shared_ptr& buf) const -{ - return m_loader->Load(Name(), buf, Size()); -} - - -//----------------------------------------------------------------------------- - -VfsDirectory::VfsDirectory() - : m_shouldPopulate(0) -{ -} - - VfsFile* VfsDirectory::AddFile(const VfsFile& file) { std::pair value = std::make_pair(file.Name(), file); std::pair ret = m_files.insert(value); if(!ret.second) // already existed { VfsFile& previousFile = ret.first->second; const VfsFile& newFile = value.second; - if(previousFile.IsSupersededBy(newFile)) + if(ShouldReplaceWith(previousFile, newFile)) previousFile = newFile; } else stats_vfs_file_add(file.Size()); return &(*ret.first).second; } // rationale: passing in a pre-constructed VfsDirectory and copying that into // our map would be slower and less convenient for the caller. VfsDirectory* VfsDirectory::AddSubdirectory(const std::wstring& name) { std::pair value = std::make_pair(name, VfsDirectory()); std::pair ret = m_subdirectories.insert(value); return &(*ret.first).second; } VfsFile* VfsDirectory::GetFile(const std::wstring& name) { VfsFiles::iterator it = m_files.find(name); if(it == m_files.end()) return 0; return &it->second; } VfsDirectory* VfsDirectory::GetSubdirectory(const std::wstring& name) { VfsSubdirectories::iterator it = m_subdirectories.find(name); if(it == m_subdirectories.end()) return 0; return &it->second; } void VfsDirectory::SetAssociatedDirectory(const PRealDirectory& realDirectory) { if(!cpu_CAS(&m_shouldPopulate, 0, 1)) debug_assert(0); // caller didn't check ShouldPopulate m_realDirectory = realDirectory; } bool VfsDirectory::ShouldPopulate() { return cpu_CAS(&m_shouldPopulate, 1, 0); // test and reset } void VfsDirectory::Invalidate(const std::wstring& name) { m_files.erase(name); m_shouldPopulate = 1; } void VfsDirectory::Clear() { m_files.clear(); m_subdirectories.clear(); m_realDirectory.reset(); m_shouldPopulate = 0; } //----------------------------------------------------------------------------- std::wstring FileDescription(const VfsFile& file) { wchar_t timestamp[25]; const time_t mtime = file.MTime(); wcsftime(timestamp, ARRAY_SIZE(timestamp), L"%a %b %d %H:%M:%S %Y", localtime(&mtime)); wchar_t buf[200]; - swprintf_s(buf, ARRAY_SIZE(buf), L"(%c; %6lu; %ls) %ls", file.LocationCode(), (unsigned long)file.Size(), timestamp, file.Name().c_str()); + swprintf_s(buf, ARRAY_SIZE(buf), L"(%c; %6lu; %ls) %ls", file.Loader()->LocationCode(), (unsigned long)file.Size(), timestamp, file.Name().c_str()); return buf; } std::wstring FileDescriptions(const VfsDirectory& directory, size_t indentLevel) { VfsDirectory::VfsFiles files = directory.Files(); std::wstring descriptions; descriptions.reserve(100*files.size()); const std::wstring indentation(4*indentLevel, ' '); for(VfsDirectory::VfsFiles::const_iterator it = files.begin(); it != files.end(); ++it) { const VfsFile& file = it->second; descriptions += indentation; descriptions += FileDescription(file); descriptions += L"\n"; } return descriptions; } void DirectoryDescriptionR(std::wstring& descriptions, const VfsDirectory& directory, size_t indentLevel) { const std::wstring indentation(4*indentLevel, ' '); const VfsDirectory::VfsSubdirectories& subdirectories = directory.Subdirectories(); for(VfsDirectory::VfsSubdirectories::const_iterator it = subdirectories.begin(); it != subdirectories.end(); ++it) { const std::wstring& name = it->first; const VfsDirectory& subdirectory = it->second; descriptions += indentation; descriptions += std::wstring(L"[") + name + L"]\n"; descriptions += FileDescriptions(subdirectory, indentLevel+1); DirectoryDescriptionR(descriptions, subdirectory, indentLevel+1); } }