source-engine/vpklib/packedstore.cpp

//========= Copyright Valve Corporation, All rights reserved. ============//
//
// Purpose: 
//
//===========================================================================//

#include "vpklib/packedstore.h"
#include "packedstore_internal.h"
#include "tier1/utlintrusivelist.h"
#include "tier1/generichash.h"
#include "tier1/checksum_crc.h"
#include "tier1/checksum_md5.h"
#include "tier1/utldict.h"
#include "tier2/fileutils.h"
#include "tier1/utlbuffer.h"

#ifdef VPK_ENABLE_SIGNING
	#include "crypto.h"
#endif


#ifdef IS_WINDOWS_PC
#include <windows.h>
#endif

// memdbgon must be the last include file in a .cpp file!!!
#include "tier0/memdbgon.h"


typedef uint16 PackFileIndex_t;
#define PACKFILEINDEX_END 0xffff
const uint16 packedfileindex_end = 0xffff;

#pragma pack(1)
struct CFilePartDescr
{
	PackFileIndex_t m_nFileNumber;
	uint32 m_nFileDataOffset;
	uint32 m_nFileDataSize;
};


struct CFileHeaderFixedData
{
	uint32 m_nFileCRC;
	uint16 m_nMetaDataSize;
	CFilePartDescr m_PartDescriptors[1];					// variable length

	FORCEINLINE const void *MetaData( void ) const;
	
	FORCEINLINE const CFilePartDescr *FileData( int nPart = 0 ) const;

	uint32 TotalDataSize( void ) const
	{
		return m_nMetaDataSize + m_PartDescriptors[0].m_nFileDataSize;
	}

	size_t HeaderSizeIncludingMetaData( void ) const
	{
		size_t nRet = sizeof( *this ) - sizeof( m_PartDescriptors ) + m_nMetaDataSize;
		// see how many parts we have and count the size of their descriptors
		CFilePartDescr const *pPart = m_PartDescriptors;
		while( pPart->m_nFileNumber != PACKFILEINDEX_END )
		{
			nRet += sizeof( CFilePartDescr );
			pPart++;
		}
		nRet += sizeof( PackFileIndex_t );					// count terminator
		return nRet;
	}

};
#pragma pack()

#define PACKEDFILE_DIR_HASH_SIZE 43

static int s_FileHeaderSize( char const *pName, int nNumDataParts, int nNumMetaDataBytes )
{
	return 1 + strlen( pName ) + 							// name plus nul
		sizeof( uint32 ) +									// file crc
		sizeof( uint16 ) + 									// meta data size
		nNumMetaDataBytes +									// metadata
		nNumDataParts * sizeof( CFilePartDescr ) +			// part data
		sizeof( PackFileIndex_t );							// part data 0xff end marker
}

class CFileDirectoryData
{
public:
	CFileDirectoryData *m_pNext;
	char const *m_Name;

};



// hash chain for accelerating file lookups. We can find an extension by hash, and find the
// directories containing files with this extension by another hash
class CFileExtensionData
{
public:
	CFileExtensionData *m_pNext;							// next one that has the same hash
	char const *m_Name;						// points at extension string within the directory data
	// nodes for each directory containing a file of this type
	CUtlIntrusiveList<CFileDirectoryData> m_pDirectoryHashTable[PACKEDFILE_DIR_HASH_SIZE];

	~CFileExtensionData( void )
	{
		for( int i = 0; i < ARRAYSIZE( m_pDirectoryHashTable ); i++ )
		{
			m_pDirectoryHashTable[i].Purge();
		}
	}

};


static int SkipFile( char const * &pData )					// returns highest file index
{
	int nHighestChunkIndex = -1;
	pData += 1 + V_strlen( pData );
	pData += sizeof( uint32 );

	uint16 nMetaDataSize;
	Q_memcpy( &nMetaDataSize, pData, sizeof( uint16 ) );

	pData += sizeof( uint16 );
	while ( Q_memcmp( pData, &packedfileindex_end, sizeof( packedfileindex_end ) ) != 0 )
	{
		int nIdx = reinterpret_cast<CFilePartDescr const *>(pData)->m_nFileNumber;

		if ( nIdx != VPKFILENUMBER_EMBEDDED_IN_DIR_FILE )
			nHighestChunkIndex = MAX( nHighestChunkIndex, nIdx  );
		pData += sizeof( CFilePartDescr );
	}
	pData += sizeof( PackFileIndex_t );
	pData += nMetaDataSize;
	return nHighestChunkIndex;
}

static inline int SkipAllFilesInDir( char const * & pData )
{
	int nHighestChunkIndex = -1;
	pData += 1 + strlen( pData );					// skip dir name
	// now, march through all the files
	while( *pData )									// until we're out of files to look at
	{
		int nSkipIndex = SkipFile( pData );
		nHighestChunkIndex = MAX( nHighestChunkIndex, nSkipIndex );
	}
	pData++;												// skip end marker
	return nHighestChunkIndex;
}


CFileHeaderFixedData *CPackedStore::FindFileEntry( char const *pDirname, char const *pBaseName, char const *pExtension, uint8 **pExtBaseOut , uint8 **pNameBaseOut )
{
	if ( pExtBaseOut )
		*pExtBaseOut = NULL;
	if ( pNameBaseOut )
		*pNameBaseOut = NULL;

	int nExtensionHash = HashString( pExtension ) % PACKEDFILE_EXT_HASH_SIZE;
	CFileExtensionData const *pExt = m_pExtensionData[nExtensionHash].FindNamedNodeCaseSensitive( pExtension );
	if ( pExt )
	{
		int nDirHash = HashString( pDirname ) % PACKEDFILE_DIR_HASH_SIZE;
		CFileDirectoryData const *pDir = pExt->m_pDirectoryHashTable[nDirHash].FindNamedNodeCaseSensitive( pDirname );
		if ( pDir )
		{
			if ( pExtBaseOut )
				*pExtBaseOut = (uint8 *) pDir;
			// we found the right directory. now, sequential search. data is heavily packed, so
			// this is a little awkward. See fileformat.txt
			char const *pData = pDir->m_Name;
			pData += 1 + strlen( pData );					// skip dir name
			// now, march through all the files
			while( *pData )									// until we're out of files to look at
			{
				if ( !V_strcmp( pData, pBaseName ) )		// found it?
				{
					if ( pNameBaseOut )
						*pNameBaseOut = (uint8 *) pData;

					return ( CFileHeaderFixedData * )( pData + 1 + V_strlen( pData ) ); // return header
				}
				// this isn't it - skip over it
				SkipFile( pData );
			}
		}
	}
	return NULL;
}


const void *CFileHeaderFixedData::MetaData( void ) const
{
	if ( ! m_nMetaDataSize )
		return NULL;
	const CFilePartDescr *ret = &( m_PartDescriptors[0] );
	while( ret->m_nFileNumber != PACKFILEINDEX_END )
		ret++;
	return reinterpret_cast<uint8 const *>( ret ) + sizeof( PackFileIndex_t );
}
	
CFilePartDescr const *CFileHeaderFixedData::FileData( int nPart ) const
{
	return m_PartDescriptors + nPart;
}

void CPackedStore::Init( void )
{
	m_nHighestChunkFileIndex = -1;
	m_bUseDirFile = false;
	m_pszFileBaseName[0] = 0;
	m_pszFullPathName[0] = 0;
	memset( m_pExtensionData, 0, sizeof( m_pExtensionData ) );
	m_nDirectoryDataSize = 0;
	m_nWriteChunkSize = k_nVPKDefaultChunkSize;

	m_nSizeOfSignedData = 0;
	m_Signature.Purge();
	m_SignaturePrivateKey.Purge();
	m_SignaturePublicKey.Purge();
}
   
void CPackedStore::BuildHashTables( void )
{
	m_nHighestChunkFileIndex = -1;
	for( int i = 0; i < ARRAYSIZE( m_pExtensionData ) ; i++ )
	{
		m_pExtensionData[i].Purge();
	}
	char const *pData = reinterpret_cast< char const *>( DirectoryData() );
	while( *pData )
	{
		// for each extension
		int nExtensionHash = HashString( pData ) % PACKEDFILE_EXT_HASH_SIZE;
		CFileExtensionData *pNewExt = new CFileExtensionData;
		pNewExt->m_Name = pData;
		m_pExtensionData[nExtensionHash].AddToHead( pNewExt );
		// now, iterate over all directories associated with this extension
		pData += 1 + strlen( pData );
		while( *pData )
		{
			int nDirHash = HashString( pData ) % PACKEDFILE_DIR_HASH_SIZE;
			CFileDirectoryData *pNewDir = new CFileDirectoryData;
			pNewDir->m_Name = pData;
			pNewExt->m_pDirectoryHashTable[nDirHash].AddToHead( pNewDir );
			int nDirChunk = SkipAllFilesInDir( pData );
			m_nHighestChunkFileIndex = MAX( m_nHighestChunkFileIndex, nDirChunk );
		}
		// step past \0
		pData++;
	}
}


bool CPackedStore::IsEmpty( void ) const
{
	return ( m_DirectoryData.Count() <= 1 );
}

static void StripTrailingString( char *pszBuf, const char *pszStrip )
{
	int lBuf = V_strlen( pszBuf );
	int lStrip = V_strlen( pszStrip );
	if ( lBuf < lStrip )
		return;
	char *pExpectedPos = pszBuf + lBuf - lStrip;
	if ( V_stricmp( pExpectedPos, pszStrip ) == 0 )
		*pExpectedPos = '\0';
}

CPackedStore::CPackedStore( char const *pFileBasename, char *pszFName, IBaseFileSystem *pFS, bool bOpenForWrite ):m_PackedStoreReadCache( pFS )
{
	Init();
	m_pFileSystem = pFS;
	m_PackedStoreReadCache.m_pPackedStore = this;
	m_DirectoryData.AddToTail( 0 );

	if ( pFileBasename )
	{
		V_strcpy( m_pszFileBaseName, pFileBasename );
		StripTrailingString( m_pszFileBaseName, ".vpk" );
		StripTrailingString( m_pszFileBaseName, "_dir" );
		sprintf( pszFName, "%s_dir.vpk", m_pszFileBaseName );
#ifdef _WIN32
		Q_strlower( pszFName );
#endif
		CInputFile dirFile( pszFName );

		// Try to load the VPK as a standalone (probably an addon) even if the standard _dir name is not present
		if ( dirFile.IsOk() )
		{
			m_bUseDirFile = true;
		}
		else
		{
			m_bUseDirFile = false;
			sprintf( pszFName, "%s.vpk", m_pszFileBaseName );
			dirFile.Open( pszFName );
		}

		bool bNewFileFormat = false;
		if ( dirFile.IsOk() )
		{
			// first, check if it is the new versioned variant
			VPKDirHeader_t dirHeader;
			// try to read the header.
			if ( 
				( dirFile.Read( &dirHeader, sizeof( dirHeader ) ) == sizeof( dirHeader ) ) &&
				( dirHeader.m_nHeaderMarker == VPK_HEADER_MARKER ) )
			{
				if ( dirHeader.m_nVersion == VPK_PREVIOUS_VERSION ) 
				{
					// fill in the fields of the new header.
					dirHeader.m_nEmbeddedChunkSize = dirFile.Size() - dirHeader.m_nDirectorySize - sizeof( VPKDirHeaderOld_t );
					dirHeader.m_nChunkHashesSize = 0;
					dirHeader.m_nSelfHashesSize = 0;
					dirHeader.m_nSignatureSize = 0;
					// pretend we didnt read the extra header
					dirFile.Seek( sizeof( VPKDirHeaderOld_t ) );
				}
				else if ( dirHeader.m_nVersion != VPK_CURRENT_VERSION )
				{
					Error( "Unknown version %d for vpk %s", dirHeader.m_nVersion, pFileBasename );
				}
				bNewFileFormat = true;
			}
			else											// its an old file
			{
				dirFile.Seek( 0 );
				// fill in a fake header, zero out garbage we read
				dirHeader.m_nDirectorySize = dirFile.Size();
				dirHeader.m_nEmbeddedChunkSize = 0;
				dirHeader.m_nChunkHashesSize = 0;
				dirHeader.m_nSelfHashesSize = 0;
				dirHeader.m_nSignatureSize = 0;
			}
			uint32 nSizeOfHeader = dirFile.Tell();
			int nSize = dirHeader.m_nDirectorySize;
			m_nDirectoryDataSize = dirHeader.m_nDirectorySize;
			m_DirectoryData.SetCount( nSize );
			dirFile.MustRead( DirectoryData(), nSize );
			// now, if we are opening for write, read the entire contents of the embedded data chunk in the dir into ram
			if ( bOpenForWrite && bNewFileFormat )
			{
				if ( dirHeader.m_nEmbeddedChunkSize )
				{
					CUtlVector<uint8> readBuffer;
					int nRemainingSize = dirHeader.m_nEmbeddedChunkSize;

					m_EmbeddedChunkData.EnsureCapacity( dirHeader.m_nEmbeddedChunkSize );

					// We'll allocate around half a meg of contiguous memory for the read. Any more and the SDK's VPK 
					// utility has a higher chance of choking on low-end machines.
					readBuffer.SetCount( 524288 );

					while ( nRemainingSize > 0 ) 
					{
						int nReadSize = MIN( nRemainingSize , 524288 );

						dirFile.MustRead( readBuffer.Base(), nReadSize );

						for ( int i = 0; i < nReadSize; i++ )
						{
							m_EmbeddedChunkData.AddToTail( readBuffer[i] );
						}
						nRemainingSize -= nReadSize;
					}
				}
			}
			int cbVecHashes = dirHeader.m_nChunkHashesSize;
			int ctHashes = cbVecHashes/sizeof(m_vecChunkHashFraction[0]);
			m_vecChunkHashFraction.EnsureCount( ctHashes );
			dirFile.MustRead( m_vecChunkHashFraction.Base(), cbVecHashes );
			FOR_EACH_VEC( m_vecChunkHashFraction, i )
			{
				int idxFound = m_vecChunkHashFraction.Find( m_vecChunkHashFraction[i] );
				Assert ( idxFound == i ); idxFound;
			}

			// now read the self hashes
			V_memset( m_DirectoryMD5.bits, 0, sizeof(m_DirectoryMD5.bits) );
			V_memset( m_ChunkHashesMD5.bits, 0, sizeof(m_ChunkHashesMD5.bits) );
			V_memset( m_TotalFileMD5.bits, 0, sizeof(m_TotalFileMD5.bits) );
			if ( dirHeader.m_nSelfHashesSize == 3*sizeof(m_DirectoryMD5.bits) )
			{
				// first is an MD5 of directory data
				dirFile.MustRead( m_DirectoryMD5.bits, sizeof(m_DirectoryMD5.bits) );
				// next is an MD5 of 
				dirFile.MustRead( m_ChunkHashesMD5.bits, sizeof(m_ChunkHashesMD5.bits) );
				// at this point the filesystem has calculated an MD5 of everything in the file up to this point.
				// we could ask it for a snapshot of that MD5 value and then be able to compare it to m_TotalFileMD5
				// but we would have to do it *before* we read it
				dirFile.MustRead( m_TotalFileMD5.bits, sizeof(m_TotalFileMD5.bits) );
			}

			// Is there a signature?
			m_nSizeOfSignedData = 0;
			if ( dirHeader.m_nSignatureSize != 0 )
			{

				// Everything immediately proceeding it should have been signed.
				m_nSizeOfSignedData = dirFile.Tell();
				uint32 nExpectedSignedSize = nSizeOfHeader + dirHeader.ComputeSizeofSignedDataAfterHeader();
				if ( m_nSizeOfSignedData != nExpectedSignedSize )
				{
					Error( "Size mismatch determining size of signed data block (%d vs %d)", m_nSizeOfSignedData, nExpectedSignedSize );
				}

				// Read the public key
				uint32 cubPublicKey = 0;
				dirFile.MustRead( &cubPublicKey, sizeof(cubPublicKey) );
				m_SignaturePublicKey.SetCount( cubPublicKey );
				dirFile.MustRead( m_SignaturePublicKey.Base(), cubPublicKey );

				// Read the private key
				uint32 cubSignature = 0;
				dirFile.MustRead( &cubSignature, sizeof(cubSignature) );
				m_Signature.SetCount( cubSignature );
				dirFile.MustRead( m_Signature.Base(), cubSignature );
			}
		}
		Q_MakeAbsolutePath( m_pszFullPathName, sizeof( m_pszFullPathName ), m_pszFileBaseName );
		V_strcat_safe( m_pszFullPathName, ".vpk" );
		//Q_strlower( m_pszFullPathName ); // NO!  this screws up linux.
		Q_FixSlashes( m_pszFullPathName );
	}
	BuildHashTables();
}


void CPackedStore::GetDataFileName( char *pchFileNameOut, int cchFileNameOut, int nFileNumber ) const
{
	if ( nFileNumber == VPKFILENUMBER_EMBEDDED_IN_DIR_FILE )
	{
		if ( m_bUseDirFile )
		{
			V_snprintf( pchFileNameOut, cchFileNameOut, "%s_dir.vpk", m_pszFileBaseName );
		}
		else
		{
			V_snprintf( pchFileNameOut, cchFileNameOut, "%s.vpk", m_pszFileBaseName );
		}
	}
	else
	{
		V_snprintf( pchFileNameOut, cchFileNameOut, "%s_%03d.vpk", m_pszFileBaseName, nFileNumber );
	}

}

CPackedStore::~CPackedStore( void )
{
	for( int i = 0; i < ARRAYSIZE( m_pExtensionData ) ; i++ )
	{
		m_pExtensionData[i].Purge();
	}

	for (int i = 0; i < ARRAYSIZE( m_FileHandles ); i++ )
	{
		if ( m_FileHandles[i].m_nFileNumber != -1 )
		{
#ifdef IS_WINDOWS_PC
			CloseHandle( m_FileHandles[i].m_hFileHandle );
#else
			m_pFileSystem->Close( m_FileHandles[i].m_hFileHandle );
#endif

		}
	}

	// Free the FindFirst cache data
	m_directoryList.PurgeAndDeleteElementsArray();

	FOR_EACH_MAP( m_dirContents, i )
	{
		m_dirContents[i]->PurgeAndDeleteElementsArray();
		delete m_dirContents[i];
	}
}

void SplitFileComponents( char const *pFileName, char *pDirOut, char *pBaseOut, char *pExtOut )
{
	char pTmpDirOut[MAX_PATH];
	V_ExtractFilePath( pFileName, pTmpDirOut, MAX_PATH );
	// now, pTmpDirOut to pDirOut, except when we find more then one '\' in a row, only output one
	char *pOutDirPtr = pDirOut;
	for( char *pDirInPtr = pTmpDirOut; *pDirInPtr; pDirInPtr++ )
	{
		char c = *( pDirInPtr );
		*( pOutDirPtr++ ) = c;
		// if we copied a \, skip all subsequent slashes
		while( ( c == '\\'  ) && ( pDirInPtr[1] == c ) )
		{
			pDirInPtr++;
		}
	}
	*( pOutDirPtr ) = 0;									// null terminate

	if ( !pDirOut[0] )
		strcpy( pDirOut, " " );								// blank dir name
	V_strncpy( pBaseOut, V_UnqualifiedFileName( pFileName ), MAX_PATH );
	char *pDot = strrchr( pBaseOut, '.' );
	if ( pDot )
	{
		*pDot = 0;
		V_strncpy( pExtOut, pDot+1, MAX_PATH );
	}
	else
	{
		pExtOut[0]=' ';
		pExtOut[1]=0;
	}

	V_FixSlashes( pDirOut, '/' );
	V_strlower( pDirOut );

	// the game sometimes asks for paths like dir1/../dir2/ we will replace this with dir2/. This
	// one line of perl code sucks in c++.
	for(;;)
	{
		char *pDotDot = V_strstr( pDirOut + 1, "/../" );		// start at second char. we don't want a beginning /
		if (! pDotDot )
		{
			break;
		}
		// search backwards from the /.. for the previous directory part
		char *pPrevSlash = pDotDot - 1;
		while( ( pPrevSlash > pDirOut ) && ( pPrevSlash[0] != '/' ) )
		{
			pPrevSlash--;
		}
		// if our path was dir0/dir1/../dir2, we are now pointing at "/dir1".
		// is strmove in all compilers? that would be better than this loop
		char *pStrIn = pDotDot + 3;
		for(;;)
		{
			*pPrevSlash = *pStrIn;
			if ( pStrIn[0] )
			{
				++pPrevSlash;
				++pStrIn;
			}
			else
			{
				break;
			}
		}
	}

	
	char *pLastDirChar = pDirOut + strlen( pDirOut ) - 1;
	if ( ( pLastDirChar[0] == '/' ) || ( pLastDirChar[0] == '\\' ) )
		*pLastDirChar = 0;									// kill trailing slash
	V_strlower( pBaseOut );
	V_strlower( pExtOut );
}

CPackedStoreFileHandle CPackedStore::OpenFile( char const *pFileName )
{
	char dirName[MAX_PATH];
	char baseName[MAX_PATH];
	char extName[MAX_PATH];

	// Fix up the filename first
	char tempFileName[MAX_PATH];

	V_strncpy( tempFileName, pFileName, sizeof( tempFileName ) );
	V_FixSlashes( tempFileName, CORRECT_PATH_SEPARATOR );
//	V_RemoveDotSlashes( tempFileName, CORRECT_PATH_SEPARATOR, true );
	V_FixDoubleSlashes( tempFileName );
	if ( !V_IsAbsolutePath( tempFileName ) )
	{
		V_strlower( tempFileName );
	}
	
	SplitFileComponents( tempFileName, dirName, baseName, extName );

	CPackedStoreFileHandle ret;

	CFileHeaderFixedData *pHeader = FindFileEntry( dirName, baseName, extName, NULL, &( ret.m_pDirFileNamePtr ) );
	
	if ( pHeader )
	{
		ret.m_nFileNumber = pHeader->m_PartDescriptors[0].m_nFileNumber;
		ret.m_nFileOffset = pHeader->m_PartDescriptors[0].m_nFileDataOffset;
		ret.m_nFileSize = pHeader->m_PartDescriptors[0].m_nFileDataSize + pHeader->m_nMetaDataSize;
		ret.m_nCurrentFileOffset = 0;
		ret.m_pMetaData = pHeader->MetaData();
		ret.m_nMetaDataSize = pHeader->m_nMetaDataSize;
  		ret.m_pHeaderData = pHeader;
		ret.m_pOwner = this;
	}
	else
	{
		ret.m_nFileNumber = -1;
		ret.m_pOwner = NULL;
	}
	return ret;

}

CPackedStoreFileHandle CPackedStore::GetHandleForHashingFiles()
{
	CPackedStoreFileHandle ret;
	ret.m_nFileNumber = 0;
	ret.m_nFileOffset = 0;
	ret.m_nFileSize = 0;
	ret.m_nMetaDataSize = 0;
	ret.m_nCurrentFileOffset = 0;
	ret.m_pDirFileNamePtr = NULL;
	ret.m_pHeaderData = NULL;
	ret.m_pMetaData = NULL;
	ret.m_pOwner = this;
	return ret;
}

void CPackedStore::Write( void )
{
	// !KLUDGE!
	// Write the whole header into a buffer in memory.
	// We do this so we can easily sign it.
	CUtlBuffer bufDirFile;

	VPKDirHeader_t headerOut;
	headerOut.m_nDirectorySize = m_DirectoryData.Count();
	headerOut.m_nEmbeddedChunkSize = m_EmbeddedChunkData.Count();
	headerOut.m_nChunkHashesSize = m_vecChunkHashFraction.Count()*sizeof(m_vecChunkHashFraction[0]);
	headerOut.m_nSelfHashesSize = 3*sizeof(m_DirectoryMD5.bits);
	headerOut.m_nSignatureSize = 0;

	// Do we plan on signing this thing and writing a signature?
	m_Signature.Purge();
	if ( m_SignaturePrivateKey.Count() > 0 && m_SignaturePublicKey.Count() > 0 )
	{
		#ifdef VPK_ENABLE_SIGNING
			uint32 nExpectedSignatureSize = k_cubRSASignature;
			headerOut.m_nSignatureSize = sizeof(uint32) + m_SignaturePublicKey.Count() + sizeof(uint32) + nExpectedSignatureSize;
		#else
			Error( "VPK signing not implemented" );
		#endif
	}

	bufDirFile.Put( &headerOut, sizeof( headerOut ) );
	bufDirFile.Put( DirectoryData(), m_DirectoryData.Count() );

	if ( m_EmbeddedChunkData.Count() )
	{
		int nRemainingSize = m_EmbeddedChunkData.Count();
		CUtlVector<uint8> writeBuffer;
		
		writeBuffer.SetCount( 524288 );
		int nChunkOffset = 0;

		while ( nRemainingSize > 0 )
		{
			// We'll write around half a meg of contiguous memory at once. Any more and the SDK's VPK 
			// utility has a higher chance of choking on low-end machines.
			int nWriteSize = MIN( nRemainingSize, 524288 );
			
			for ( int i = 0; i < nWriteSize; i++ )
			{
				writeBuffer[i] = m_EmbeddedChunkData[nChunkOffset++];
			}

			bufDirFile.Put( writeBuffer.Base(), nWriteSize );
			nRemainingSize -= nWriteSize;
		}
	}

	// write the chunk hashes out
	bufDirFile.Put( m_vecChunkHashFraction.Base(), m_vecChunkHashFraction.Count()*sizeof(m_vecChunkHashFraction[0]) );

	// write out the MD5s of the 2 main pieces of data
	bufDirFile.Put( m_DirectoryMD5.bits, sizeof( m_DirectoryMD5.bits ) );
	bufDirFile.Put( m_ChunkHashesMD5.bits, sizeof( m_ChunkHashesMD5.bits ) );

	// compute the final MD5 ( of everything in the file up to this point )
	MD5_ProcessSingleBuffer( bufDirFile.Base(), bufDirFile.TellPut(), m_TotalFileMD5 );
	bufDirFile.Put( m_TotalFileMD5.bits, sizeof( m_TotalFileMD5.bits ) );

	// Should we sign all this stuff?
	m_nSizeOfSignedData = 0;
	#ifdef VPK_ENABLE_SIGNING
		if ( headerOut.m_nSignatureSize > 0 )
		{

			m_nSizeOfSignedData = bufDirFile.TellPut();
			uint32 nExpectedSignedSize = sizeof(headerOut) + headerOut.ComputeSizeofSignedDataAfterHeader();
			if ( m_nSizeOfSignedData != nExpectedSignedSize )
			{
				Error( "Size mismatch determining size of signed data block (%d vs %d)", m_nSizeOfSignedData, nExpectedSignedSize );
			}

			// Allocate more than enough space to hold the signature
			m_Signature.SetCount( nExpectedSignatureSize + 1024 );

			// Calcuate the signature
			uint32 cubSignature = m_Signature.Count();
			if ( !CCrypto::RSASignSHA256( (const uint8 *)bufDirFile.Base(), bufDirFile.TellPut(), 
				(uint8 *)m_Signature.Base(), &cubSignature, 
				(const uint8 *)m_SignaturePrivateKey.Base(), m_SignaturePrivateKey.Count() ) )
			{
				Error( "VPK signing failed.  Private key may be corrupt or invalid" );
			}

			// Confirm that the size was what we expected
			if ( cubSignature != nExpectedSignatureSize )
			{
				Error( "VPK signing produced %d byte signature.  Expected size was %d bytes", cubSignature, nExpectedSignatureSize );
			}

			// Shrink signature to fit
			m_Signature.SetCountNonDestructively( cubSignature );

			// Now re-check the signature, using the public key that we are about
			// to burn into the file, to make sure there's no mismatch.
			if ( !CCrypto::RSAVerifySignatureSHA256( (const uint8 *)bufDirFile.Base(), bufDirFile.TellPut(), 
				(const uint8 *)m_Signature.Base(), cubSignature, 
				(const uint8 *)m_SignaturePublicKey.Base(), m_SignaturePublicKey.Count() ) )
			{
				Error( "VPK signature verification failed immediately after signing.  The public key might be invalid, or might not match the private key used to generate the signature." );
			}

			// Write public key which should be used
			uint32 cubPublicKey = m_SignaturePublicKey.Count();
			bufDirFile.Put( &cubPublicKey, sizeof(cubPublicKey) );
			bufDirFile.Put( m_SignaturePublicKey.Base(), cubPublicKey );

			// Write signature
			bufDirFile.Put( &cubSignature, sizeof(cubSignature) );
			bufDirFile.Put( m_Signature.Base(), cubSignature );
		}
	#endif

	char szOutFileName[MAX_PATH];

	// Delete any existing header file, either the standalone kind,
	// or the _dir kind.
	V_sprintf_safe( szOutFileName, "%s.vpk", m_pszFileBaseName );
	if ( g_pFullFileSystem->FileExists( szOutFileName ) )
		g_pFullFileSystem->RemoveFile( szOutFileName );
	V_sprintf_safe( szOutFileName, "%s_dir.vpk", m_pszFileBaseName );
	if ( g_pFullFileSystem->FileExists( szOutFileName ) )
		g_pFullFileSystem->RemoveFile( szOutFileName );

	// Force on multi-chunk mode if we have any files in a chunk
	if ( m_nHighestChunkFileIndex >= 0 )
		m_bUseDirFile = true;


	// Fetch actual name to write
	GetDataFileName( szOutFileName, sizeof(szOutFileName), VPKFILENUMBER_EMBEDDED_IN_DIR_FILE );

	// Now actually write the data to disk
	COutputFile dirFile( szOutFileName );
	dirFile.Write( bufDirFile.Base(), bufDirFile.TellPut() );
	dirFile.Close();
}

#ifdef VPK_ENABLE_SIGNING

void CPackedStore::SetKeysForSigning( int nPrivateKeySize, const void *pPrivateKeyData, int nPublicKeySize, const void *pPublicKeyData )
{
	m_SignaturePrivateKey.SetSize( nPrivateKeySize );
	V_memcpy( m_SignaturePrivateKey.Base(), pPrivateKeyData, nPrivateKeySize );

	m_SignaturePublicKey.SetSize( nPublicKeySize );
	V_memcpy( m_SignaturePublicKey.Base(), pPublicKeyData, nPublicKeySize );

	// Discard any existing signature
	m_Signature.Purge();
}

CPackedStore::ESignatureCheckResult CPackedStore::CheckSignature( int nSignatureSize, const void *pSignature ) const
{
	if ( m_Signature.Count() == 0 )
		return eSignatureCheckResult_NotSigned;

	Assert( m_nSizeOfSignedData > 0 );

	// Confirm correct public key, if they specified one.
	if ( nSignatureSize > 0 && pSignature != NULL )
	{
		if ( m_SignaturePublicKey.Count() != nSignatureSize || V_memcmp( pSignature, m_SignaturePublicKey.Base(), nSignatureSize ) != 0 )
		{
			return eSignatureCheckResult_WrongKey;
		}
	}

	char szFilename[ MAX_PATH ];
	GetDataFileName( szFilename, sizeof( szFilename ), VPKFILENUMBER_EMBEDDED_IN_DIR_FILE );

	// Read the data
	CUtlBuffer bufSignedData;
	if ( !g_pFullFileSystem->ReadFile( szFilename, NULL, bufSignedData, m_nSizeOfSignedData ) )
		return eSignatureCheckResult_Failed;
	if ( bufSignedData.TellPut() < (int)m_nSizeOfSignedData )
	{
		Assert( false ); // ?
		return eSignatureCheckResult_Failed;
	}

	// Check the signature
	if ( !CCrypto::RSAVerifySignatureSHA256( (const uint8 *)bufSignedData.Base(), m_nSizeOfSignedData, 
		(const uint8 *)m_Signature.Base(), m_Signature.Count(), 
		(const uint8 *)m_SignaturePublicKey.Base(), m_SignaturePublicKey.Count() ) )
	{
		return eSignatureCheckResult_InvalidSignature;
	}

	return eSignatureCheckResult_ValidSignature;
}

#endif

CPackedStoreReadCache::CPackedStoreReadCache( IBaseFileSystem *pFS ):m_treeCachedVPKRead( CachedVPKRead_t::Less )
{
	m_pPackedStore = NULL;
	m_cItemsInCache = 0;
	m_pFileSystem = pFS;
	m_cubReadFromCache = 0;
	m_cReadFromCache = 0;
	m_cDiscardsFromCache = 0;
	m_cAddedToCache = 0;
	m_cCacheMiss = 0;
	m_cubCacheMiss = 0;
	m_cFileErrors = 0;
	m_cFileErrorsCorrected = 0;
	m_cFileResultsDifferent = 0;
	m_pFileTracker = NULL;
}

// check if the read request can be satisfied from the read cache we have in 1MB chunks
bool CPackedStoreReadCache::BCanSatisfyFromReadCache( uint8 *pOutData, CPackedStoreFileHandle &handle, FileHandleTracker_t &fHandle, int nDesiredPos, int nNumBytes, int &nRead )
{
	nRead = 0;
	int nFileFraction = nDesiredPos & k_nCacheBufferMask;
	int nOffset = nDesiredPos - nFileFraction;
	int cubReadChunk = nOffset + nNumBytes;
	if ( cubReadChunk > k_cubCacheBufferSize )
		cubReadChunk = ( k_nCacheBufferMask - nOffset ) & (k_cubCacheBufferSize-1);
	else
		cubReadChunk = nNumBytes;
	// the request might straddle multiple chunks - we make sure we have all of the data, if we are missing any, we fail
	while ( nNumBytes )
	{
		int nReadChunk = 0;
		if ( !BCanSatisfyFromReadCacheInternal( pOutData, handle, fHandle, nDesiredPos, cubReadChunk, nReadChunk ) )
		{
			return false;
		}
		nNumBytes -= cubReadChunk;
		pOutData += cubReadChunk;
		nDesiredPos += cubReadChunk;
		nRead += nReadChunk;
		nFileFraction += k_cubCacheBufferSize;
		cubReadChunk = nNumBytes;
		if ( cubReadChunk > k_cubCacheBufferSize )
			cubReadChunk = k_cubCacheBufferSize;
	}
	return true;
}


// read a single line into the cache
bool CPackedStoreReadCache::ReadCacheLine( FileHandleTracker_t &fHandle, CachedVPKRead_t &cachedVPKRead )
{
	cachedVPKRead.m_cubBuffer = 0;
#ifdef IS_WINDOWS_PC
	if ( cachedVPKRead.m_nFileFraction != fHandle.m_nCurOfs )
		SetFilePointer ( fHandle.m_hFileHandle, cachedVPKRead.m_nFileFraction, NULL,  FILE_BEGIN); 
	ReadFile( fHandle.m_hFileHandle, cachedVPKRead.m_pubBuffer, k_cubCacheBufferSize, (LPDWORD) &cachedVPKRead.m_cubBuffer, NULL );
	SetFilePointer ( fHandle.m_hFileHandle, fHandle.m_nCurOfs, NULL,  FILE_BEGIN); 
#else
	m_pFileSystem->Seek( fHandle.m_hFileHandle, cachedVPKRead.m_nFileFraction, FILESYSTEM_SEEK_HEAD );
	cachedVPKRead.m_cubBuffer = m_pFileSystem->Read( cachedVPKRead.m_pubBuffer, k_cubCacheBufferSize, fHandle.m_hFileHandle );
	m_pFileSystem->Seek( fHandle.m_hFileHandle, fHandle.m_nCurOfs, FILESYSTEM_SEEK_HEAD );
#endif
	Assert( cachedVPKRead.m_hMD5RequestHandle == 0 );
	if ( m_pFileTracker ) // file tracker doesn't exist in the VPK command line tool
	{
		cachedVPKRead.m_hMD5RequestHandle = m_pFileTracker->SubmitThreadedMD5Request( cachedVPKRead.m_pubBuffer, cachedVPKRead.m_cubBuffer, m_pPackedStore->m_PackFileID, cachedVPKRead.m_nPackFileNumber, cachedVPKRead.m_nFileFraction );
	}
	return cachedVPKRead.m_cubBuffer > 0;
}


// check if the MD5 matches
bool CPackedStoreReadCache::CheckMd5Result( CachedVPKRead_t &cachedVPKRead )
{
	ChunkHashFraction_t chunkHashFraction;
	if ( !m_pPackedStore->FindFileHashFraction( cachedVPKRead.m_nPackFileNumber, cachedVPKRead.m_nFileFraction, chunkHashFraction ) )
		return true;

	if ( Q_memcmp( &cachedVPKRead.m_md5Value, &chunkHashFraction.m_md5contents, sizeof( MD5Value_t ) ) != 0	)
	{
		char szFilename[ 512 ];
		m_pPackedStore->GetDataFileName( szFilename, sizeof(szFilename), cachedVPKRead.m_nPackFileNumber );

		char szCalculated[ MD5_DIGEST_LENGTH*2 + 4 ];
		char szExpected[ MD5_DIGEST_LENGTH*2 + 4 ];
		V_binarytohex( cachedVPKRead.m_md5Value.bits, MD5_DIGEST_LENGTH, szCalculated, sizeof(szCalculated) );
		V_binarytohex( chunkHashFraction.m_md5contents.bits, MD5_DIGEST_LENGTH, szExpected, sizeof(szExpected) );

		Warning(
			"Corruption detected in %s\n"
			"\n"
			"Try verifying the integrity of your game cache.\n"
			"https://support.steampowered.com/kb_article.php?ref=2037-QEUH-3335"
			"\n"
			"Offset %d, expected %s, got %s\n",
			szFilename,
			cachedVPKRead.m_nFileFraction, szExpected, szCalculated
		);

		// we got an error reading this chunk, record the error
		m_cFileErrors++;
		cachedVPKRead.m_cFailedHashes++;
		// give a copy to the fail whale
		//m_queueCachedVPKReadsRetry.PushItem( cachedVPKRead );
		return false;
	}
	if ( cachedVPKRead.m_cFailedHashes > 0 )
	{
		m_cFileErrorsCorrected++;
	}
	return true;
}


int CPackedStoreReadCache::FindBufferToUse()
{
	int idxLRU = 0;
	int idxToRemove = m_treeCachedVPKRead.InvalidIndex();
	uint32 uTimeLowest = (uint32)~0; // MAXINT
	// find the oldest item, reuse its buffer
	for ( int i = 0; i < m_cItemsInCache; i++ )
	{
		if ( m_rgLastUsedTime[i] < uTimeLowest )
		{
			uTimeLowest = m_rgLastUsedTime[i];
			idxToRemove = m_rgCurrentCacheIndex[i];
			idxLRU = i;
		}
		int idxCurrent = m_rgCurrentCacheIndex[i];
		// while we are here check if the MD5 is done
		if ( m_treeCachedVPKRead[idxCurrent].m_hMD5RequestHandle )
		{
			CachedVPKRead_t &cachedVPKRead = m_treeCachedVPKRead[idxCurrent];
			if ( m_pFileTracker->IsMD5RequestComplete( cachedVPKRead.m_hMD5RequestHandle, &cachedVPKRead.m_md5Value ) )
			{
				// if it is done, check the results
				cachedVPKRead.m_hMD5RequestHandle = 0;
				// if we got bad data - stop looking, just use this one
				if ( !CheckMd5Result( cachedVPKRead ) )
					return i;
			}
		}
	}

	// if we submitted its MD5 for processing, then wait until that is done
	if ( m_treeCachedVPKRead[idxToRemove].m_hMD5RequestHandle )
	{
		CachedVPKRead_t &cachedVPKRead = m_treeCachedVPKRead[idxToRemove];
		m_pFileTracker->BlockUntilMD5RequestComplete( cachedVPKRead.m_hMD5RequestHandle, &cachedVPKRead.m_md5Value );
		m_treeCachedVPKRead[idxToRemove].m_hMD5RequestHandle = 0;
		// make sure it matches what it is supposed to match
		CheckMd5Result( cachedVPKRead );
	}
	return idxLRU;
}


// manage the cache
bool CPackedStoreReadCache::BCanSatisfyFromReadCacheInternal( uint8 *pOutData, CPackedStoreFileHandle &handle, FileHandleTracker_t &fHandle, int nDesiredPos, int nNumBytes, int &nRead )
{
	m_rwlock.LockForRead();
	bool bLockedForWrite = false;

	CachedVPKRead_t key;
	key.m_nPackFileNumber = handle.m_nFileNumber;
	key.m_nFileFraction = nDesiredPos & k_nCacheBufferMask;
	int idxTrackedVPKFile = m_treeCachedVPKRead.Find( key );
	if ( idxTrackedVPKFile == m_treeCachedVPKRead.InvalidIndex() || m_treeCachedVPKRead[idxTrackedVPKFile].m_pubBuffer == NULL )
	{
		m_rwlock.UnlockRead();
		m_rwlock.LockForWrite();
		bLockedForWrite = true;
		// if we didnt find it, we had to grab the write lock, it may have been added while we waited
		idxTrackedVPKFile = m_treeCachedVPKRead.Find( key );
	}
	if ( idxTrackedVPKFile == m_treeCachedVPKRead.InvalidIndex() )
	{
		idxTrackedVPKFile = m_treeCachedVPKRead.Insert( key );
	}

	CachedVPKRead_t &cachedVPKRead = m_treeCachedVPKRead[idxTrackedVPKFile];

	// Cache hit?
	if ( cachedVPKRead.m_pubBuffer == NULL )
	{

		// We need to have it locked for write, because we're about to muck with these structures.
		if ( !bLockedForWrite )
		{
			Assert( bLockedForWrite );
			return false;
		}

		Assert( cachedVPKRead.m_idxLRU < 0 );

		// Can we add another line to the cache, or should we reuse an existing one?
		int idxLRU = -1;
		if ( m_cItemsInCache >= k_nCacheBuffersToKeep )
		{
			// Need to kick out the LRU.
			idxLRU = FindBufferToUse();
			int idxToRemove = m_rgCurrentCacheIndex[idxLRU];
			Assert( m_treeCachedVPKRead[idxToRemove].m_idxLRU == idxLRU );
			Assert( m_treeCachedVPKRead[idxToRemove].m_pubBuffer != NULL );

			// Transfer ownership of the buffer
			cachedVPKRead.m_pubBuffer = m_treeCachedVPKRead[idxToRemove].m_pubBuffer;
			m_treeCachedVPKRead[idxToRemove].m_pubBuffer = NULL;
			m_treeCachedVPKRead[idxToRemove].m_cubBuffer = 0;
			m_treeCachedVPKRead[idxToRemove].m_idxLRU = -1;
			m_cDiscardsFromCache++;
		}
		else
		{

			// We can add a new one
			idxLRU = m_cItemsInCache;
			m_cItemsInCache++;
			Assert( cachedVPKRead.m_pubBuffer == NULL );
		}
		m_rgCurrentCacheIndex[idxLRU] = idxTrackedVPKFile;
		cachedVPKRead.m_idxLRU = idxLRU;

		if ( cachedVPKRead.m_pubBuffer == NULL )
		{
			cachedVPKRead.m_pubBuffer = (uint8 *)malloc( k_cubCacheBufferSize );
			if ( cachedVPKRead.m_pubBuffer == NULL )
				Error( "Out of memory" );
		}
		ReadCacheLine( fHandle, cachedVPKRead );
		m_cAddedToCache++;
	}
	else
	{
		Assert( cachedVPKRead.m_idxLRU >= 0 );
		Assert( m_rgCurrentCacheIndex[cachedVPKRead.m_idxLRU] == idxTrackedVPKFile );
	}

	// Assume no bytes will be satisfied from cache
	bool bSuccess = false;
	nRead = 0;

	// Can we read at least one byte?
	Assert( cachedVPKRead.m_nFileFraction <= nDesiredPos );
	int nBufferEnd = cachedVPKRead.m_cubBuffer + cachedVPKRead.m_nFileFraction;
	if ( cachedVPKRead.m_pubBuffer != NULL && nBufferEnd > nDesiredPos )
	{
		nRead = Min( nBufferEnd - nDesiredPos, nNumBytes );
		int nOffset = nDesiredPos - cachedVPKRead.m_nFileFraction;
		Assert( nOffset >= 0 );
		memcpy( pOutData, (uint8 *)&cachedVPKRead.m_pubBuffer[nOffset], nRead );
		m_cubReadFromCache += nRead;
		m_cReadFromCache ++;
		bSuccess = true;
		m_rgLastUsedTime[m_treeCachedVPKRead[idxTrackedVPKFile].m_idxLRU] = Plat_MSTime();
	}
	if ( bLockedForWrite )
		m_rwlock.UnlockWrite();
	else
		m_rwlock.UnlockRead();

	return bSuccess;
}


void CPackedStoreReadCache::RetryBadCacheLine( CachedVPKRead_t &cachedVPKRead )
{
	ChunkHashFraction_t chunkHashFraction;
	m_pPackedStore->FindFileHashFraction( cachedVPKRead.m_nPackFileNumber, cachedVPKRead.m_nFileFraction, chunkHashFraction );

	cachedVPKRead.m_pubBuffer = (uint8 *)malloc( k_cubCacheBufferSize );
	FileHandleTracker_t &fHandle = m_pPackedStore->GetFileHandle( cachedVPKRead.m_nPackFileNumber );
	fHandle.m_Mutex.Lock();
	ReadCacheLine( fHandle, cachedVPKRead );
	fHandle.m_Mutex.Unlock();
	m_pFileTracker->BlockUntilMD5RequestComplete( cachedVPKRead.m_hMD5RequestHandle, &cachedVPKRead.m_md5Value );
	cachedVPKRead.m_hMD5RequestHandle = 0;
	CheckMd5Result( cachedVPKRead );
	cachedVPKRead.m_pubBuffer = NULL;
}


// try reloading anything that failed its md5 check
// this is currently only for gathering information, doesnt do anything to repair the cache
void CPackedStoreReadCache::RetryAllBadCacheLines()
{
//	while( m_queueCachedVPKReadsRetry.Count() )
//	{
//		CachedVPKRead_t cachedVPKRead;
//		m_rwlock.LockForWrite();
//		if ( m_queueCachedVPKReadsRetry.PopItem( &cachedVPKRead ) )
//		{
//			// retry anything that didnt match one time
//			RetryBadCacheLine( cachedVPKRead );
//			m_listCachedVPKReadsFailed.AddToTail( cachedVPKRead );
//			// m_listCachedVPKReadsFailed contains all the data about failed reads - for error or OGS reporting
//		}
//		m_rwlock.UnlockWrite();
//	}
}

void CPackedStore::GetPackFileLoadErrorSummary( CUtlString &sErrors )
{
	FOR_EACH_LL( m_PackedStoreReadCache.m_listCachedVPKReadsFailed, i )
	{
		char szDataFileName[MAX_PATH];
		CPackedStoreFileHandle fhandle = GetHandleForHashingFiles();
		fhandle.m_nFileNumber = m_PackedStoreReadCache.m_listCachedVPKReadsFailed[i].m_nPackFileNumber;
		fhandle.GetPackFileName( szDataFileName, sizeof(szDataFileName) );
		const char *pszFileName = V_GetFileName( szDataFileName );

		CUtlString sTemp;
		sTemp.Format( "Pack File %s at offset %x length %x errorcount = %d \n", 
			pszFileName,
			m_PackedStoreReadCache.m_listCachedVPKReadsFailed[i].m_nFileFraction, 
			m_PackedStoreReadCache.m_listCachedVPKReadsFailed[i].m_cubBuffer,
			m_PackedStoreReadCache.m_listCachedVPKReadsFailed[i].m_cFailedHashes );
		sErrors += sTemp ;
		
		char hex[sizeof(MD5Value_t)*2 + 1 ];
		Q_binarytohex( m_PackedStoreReadCache.m_listCachedVPKReadsFailed[i].m_md5Value.bits, 
					sizeof(MD5Value_t),	hex, sizeof( hex ) );

		ChunkHashFraction_t chunkHashFraction;
		FindFileHashFraction( m_PackedStoreReadCache.m_listCachedVPKReadsFailed[i].m_nPackFileNumber, m_PackedStoreReadCache.m_listCachedVPKReadsFailed[i].m_nFileFraction, chunkHashFraction );

		char hex2[sizeof(MD5Value_t)*2 + 1 ];
		Q_binarytohex( chunkHashFraction.m_md5contents.bits, 
			sizeof(MD5Value_t),	hex2, sizeof( hex2 ) );
		sTemp.Format( "Last Md5 Value %s Should be %s \n", hex, hex2 );

		sErrors += sTemp ;
	}

}

int CPackedStore::ReadData( CPackedStoreFileHandle &handle, void *pOutData, int nNumBytes )
{
	int nRet = 0;

	// clamp read size to file size
	nNumBytes = MIN( nNumBytes, handle.m_nFileSize - handle.m_nCurrentFileOffset );
	if ( nNumBytes > 0 )
	{
		// first satisfy from the metadata, if we can
		int nNumMetaDataBytes = MIN( nNumBytes, handle.m_nMetaDataSize - handle.m_nCurrentFileOffset );
		if ( nNumMetaDataBytes > 0 )
		{
			memcpy( pOutData, reinterpret_cast<uint8 const *>( handle.m_pMetaData )
					+ handle.m_nCurrentFileOffset, nNumMetaDataBytes );
			nRet += nNumMetaDataBytes;
			pOutData = reinterpret_cast<uint8 *>( pOutData ) + nNumMetaDataBytes;
			handle.m_nCurrentFileOffset += nNumMetaDataBytes;
			nNumBytes -= nNumMetaDataBytes;
		}
		// satisfy remaining bytes from file
		if ( nNumBytes > 0 )
		{
			FileHandleTracker_t &fHandle = GetFileHandle( handle.m_nFileNumber );
			int nDesiredPos = handle.m_nFileOffset + handle.m_nCurrentFileOffset - handle.m_nMetaDataSize;
			int nRead;
			fHandle.m_Mutex.Lock();
			if ( handle.m_nFileNumber == VPKFILENUMBER_EMBEDDED_IN_DIR_FILE )
			{
				// for file data in the directory header, all offsets are relative to the size of the dir header.
				nDesiredPos += m_nDirectoryDataSize + sizeof( VPKDirHeader_t );
			}

			if ( m_PackedStoreReadCache.BCanSatisfyFromReadCache( (uint8 *)pOutData, handle, fHandle, nDesiredPos, nNumBytes, nRead ) )
			{
				handle.m_nCurrentFileOffset += nRead;
			}
			else
			{
#ifdef IS_WINDOWS_PC
				if ( nDesiredPos != fHandle.m_nCurOfs )
					SetFilePointer ( fHandle.m_hFileHandle, nDesiredPos, NULL,  FILE_BEGIN); 
				ReadFile( fHandle.m_hFileHandle, pOutData, nNumBytes, (LPDWORD) &nRead, NULL );
#else
				m_pFileSystem->Seek( fHandle.m_hFileHandle, nDesiredPos, FILESYSTEM_SEEK_HEAD );
				nRead = m_pFileSystem->Read( pOutData, nNumBytes, fHandle.m_hFileHandle );
#endif
				handle.m_nCurrentFileOffset += nRead;
				fHandle.m_nCurOfs = nRead + nDesiredPos;
			}
			Assert( nRead == nNumBytes );
			nRet += nRead;
			fHandle.m_Mutex.Unlock();
		}
	}
	m_PackedStoreReadCache.RetryAllBadCacheLines();
	return nRet;
}

bool CPackedStore::HashEntirePackFile( CPackedStoreFileHandle &handle, int64 &nFileSize, int nFileFraction, int nFractionSize, FileHash_t &fileHash )
{
#define	CRC_CHUNK_SIZE	(32*1024)
	unsigned char tempBuf[CRC_CHUNK_SIZE];

#ifdef COMPUTE_HASH_TIMES
	CFastTimer timer;
	timer.Start();
#endif

	FileHandleTracker_t &fHandle = GetFileHandle( handle.m_nFileNumber );
	fHandle.m_Mutex.Lock();
	
#ifdef IS_WINDOWS_PC
	unsigned int fileSizeHigh;
	unsigned int fileLength = GetFileSize( fHandle.m_hFileHandle, (LPDWORD) &fileSizeHigh );
#else
	unsigned int fileLength = m_pFileSystem->Size( fHandle.m_hFileHandle );
#endif
	nFileSize = fileLength;
	MD5Context_t ctx;
	memset(&ctx, 0, sizeof(MD5Context_t));
	MD5Init(&ctx);

	int nDesiredPos = nFileFraction;
#ifdef IS_WINDOWS_PC
	if ( nDesiredPos != fHandle.m_nCurOfs )
		SetFilePointer ( fHandle.m_hFileHandle, nDesiredPos, NULL,  FILE_BEGIN); 
#else
	m_pFileSystem->Seek( fHandle.m_hFileHandle, nDesiredPos, FILESYSTEM_SEEK_HEAD );
#endif

	int nFractionLength = ( fileLength - nFileFraction );
	if ( nFractionLength > nFractionSize )
		nFractionLength = nFractionSize;
	int nChunks = nFractionLength / CRC_CHUNK_SIZE + 1;
	unsigned int curStartByte = 0;
	for ( int iChunk=0; iChunk < nChunks; iChunk++ )
	{
		int curEndByte = MIN( curStartByte + CRC_CHUNK_SIZE, (uint)nFractionLength );
		int chunkLen = curEndByte - curStartByte;
		if ( chunkLen == 0 )
			break;

		int nRead;
#ifdef IS_WINDOWS_PC
		ReadFile( fHandle.m_hFileHandle, tempBuf, chunkLen, (LPDWORD) &nRead, NULL );
#else
		nRead = m_pFileSystem->Read( tempBuf, chunkLen, fHandle.m_hFileHandle );
#endif
		MD5Update(&ctx, tempBuf, nRead);

		curStartByte += CRC_CHUNK_SIZE;
	}
	MD5Final( fileHash.m_md5contents.bits, &ctx);
	fileHash.m_crcIOSequence = nFractionLength;
	fileHash.m_cbFileLen = nFractionLength;
	fileHash.m_eFileHashType = FileHash_t::k_EFileHashTypeEntireFile;
	fileHash.m_nPackFileNumber = handle.m_nFileNumber;
	fileHash.m_PackFileID = handle.m_pOwner->m_PackFileID;

	// seek back to where it was
#ifdef IS_WINDOWS_PC
	SetFilePointer ( fHandle.m_hFileHandle, fHandle.m_nCurOfs, NULL,  FILE_BEGIN); 
#else
	m_pFileSystem->Seek( fHandle.m_hFileHandle, fHandle.m_nCurOfs, FILESYSTEM_SEEK_HEAD );
#endif
	fHandle.m_Mutex.Unlock();

#ifdef COMPUTE_HASH_TIMES
	timer.End();
	int nMicroSec = timer.GetDuration().GetMicroseconds();
	char rgch[256];
	Q_snprintf( rgch, 256, "MD5 Pack File %d %d \n", handle.m_nFileNumber, nMicroSec );
	Plat_DebugString( rgch );
#endif
	return true;
}

void CPackedStore::DiscardChunkHashes( int iChunkFileIndex )
{
	// Wow, this could be a LOT faster because the list is
	// sorted.  Probably not worth optimizing
	FOR_EACH_VEC_BACK( m_vecChunkHashFraction, i )
	{
		if ( m_vecChunkHashFraction[i].m_nPackFileNumber == iChunkFileIndex )
			m_vecChunkHashFraction.Remove( i );
	}
}

void CPackedStore::HashChunkFile( int iChunkFileIndex )
{
	AUTO_LOCK( m_Mutex );
	static const int k_nFileFractionSize = 0x00100000; // 1 MB

	// Purge any hashes we already have for this chunk.
	DiscardChunkHashes( iChunkFileIndex );

	CPackedStoreFileHandle VPKHandle = GetHandleForHashingFiles();
	VPKHandle.m_nFileNumber = iChunkFileIndex;

	int nFileFraction = 0;
	while ( 1 )
	{
		FileHash_t filehash;

		// VPKHandle.m_nFileNumber;
		// nFileFraction;
		int64 fileSize = 0;
		// if we have never hashed this before - do it now
		HashEntirePackFile( VPKHandle, fileSize, nFileFraction, k_nFileFractionSize, filehash );
		ChunkHashFraction_t fileHashFraction;
		fileHashFraction.m_cbChunkLen = filehash.m_cbFileLen;
		fileHashFraction.m_nPackFileNumber = VPKHandle.m_nFileNumber;
		fileHashFraction.m_nFileFraction = nFileFraction;
		Q_memcpy( fileHashFraction.m_md5contents.bits, filehash.m_md5contents.bits, sizeof(fileHashFraction.m_md5contents) );
		m_vecChunkHashFraction.Insert( fileHashFraction );
		// move to next section
		nFileFraction += k_nFileFractionSize;
		// if we are at EOF we are done
		if ( nFileFraction > fileSize )
			break;
	}
}


void CPackedStore::HashAllChunkFiles()
{
	// Rebuild the directory hash tables.  The main reason to do this is
	// so that the highest chunk number is correct, in case chunks have
	// been removed.
	BuildHashTables();

	// make brand new hashes
	m_vecChunkHashFraction.Purge();
	for ( int iChunkFileIndex = 0 ; iChunkFileIndex <= GetHighestChunkFileIndex() ; ++iChunkFileIndex )
		HashChunkFile( iChunkFileIndex );
}

void CPackedStore::ComputeDirectoryHash( MD5Value_t &md5Directory )
{
	MD5Context_t ctx;
	memset(&ctx, 0, sizeof(MD5Context_t));
	MD5Init(&ctx);
	MD5Update(&ctx, m_DirectoryData.Base(), m_DirectoryData.Count() );
	MD5Final( md5Directory.bits, &ctx);
}


void CPackedStore::ComputeChunkHash( MD5Value_t &md5ChunkHashes )
{
	MD5Context_t ctx;
	memset(&ctx, 0, sizeof(MD5Context_t));
	MD5Init(&ctx);
	MD5Update(&ctx, (uint8 *)m_vecChunkHashFraction.Base(), m_vecChunkHashFraction.Count()*sizeof(m_vecChunkHashFraction[0]) );
	MD5Final( md5ChunkHashes.bits, &ctx);
}


bool CPackedStore::BTestDirectoryHash()
{
	if ( !BFileContainedHashes() )
		return true;
	MD5Value_t md5Directory;
	ComputeDirectoryHash( md5Directory );
	return Q_memcmp( m_DirectoryMD5.bits, md5Directory.bits, sizeof( md5Directory.bits ) ) == 0;
}


bool CPackedStore::BTestMasterChunkHash()
{
	if ( !BFileContainedHashes() )
		return true;
	MD5Value_t md5ChunkHashes;
	ComputeChunkHash( md5ChunkHashes );
	return Q_memcmp( m_ChunkHashesMD5.bits, md5ChunkHashes.bits, sizeof( md5ChunkHashes.bits ) ) == 0;
}


void CPackedStore::HashEverything()
{
	HashAllChunkFiles();
	HashMetadata();
}

void CPackedStore::HashMetadata()
{
	ComputeDirectoryHash( m_DirectoryMD5 );
	ComputeChunkHash( m_ChunkHashesMD5 );
}


bool CPackedStore::FindFileHashFraction( int nPackFileNumber, int nFileFraction, ChunkHashFraction_t &fileHashFraction )
{
	ChunkHashFraction_t fileHashFractionFind;
	fileHashFractionFind.m_nFileFraction = nFileFraction;
	fileHashFractionFind.m_nPackFileNumber = nPackFileNumber;

	int idx = m_vecChunkHashFraction.Find( fileHashFractionFind );
	if ( idx == m_vecChunkHashFraction.InvalidIndex() )
	{
		Assert( false );
		return false;
	}
	fileHashFraction = m_vecChunkHashFraction[idx];
	return true;
}

void CPackedStore::GetPackFileName( CPackedStoreFileHandle &handle, char *pchFileNameOut, int cchFileNameOut ) const
{
	GetDataFileName( pchFileNameOut, cchFileNameOut, handle.m_nFileNumber );
}

FileHandleTracker_t & CPackedStore::GetFileHandle( int nFileNumber )
{
	AUTO_LOCK( m_Mutex );
	int nFileHandleIdx = nFileNumber % ARRAYSIZE( m_FileHandles );

	if ( m_FileHandles[nFileHandleIdx].m_nFileNumber == nFileNumber )
	{
		return m_FileHandles[nFileHandleIdx];
	}
	else if ( m_FileHandles[nFileHandleIdx].m_nFileNumber == -1 )
	{
		// no luck finding the handle - need a new one
		char pszDataFileName[MAX_PATH];
		GetDataFileName( pszDataFileName, sizeof(pszDataFileName), nFileNumber );
		m_FileHandles[nFileHandleIdx].m_nCurOfs = 0;
#ifdef IS_WINDOWS_PC
		m_FileHandles[nFileHandleIdx].m_hFileHandle = 
			CreateFile( pszDataFileName,               // file to open
						GENERIC_READ,          // open for reading
						FILE_SHARE_READ,       // share for reading
						NULL,                  // default security
						OPEN_EXISTING,         // existing file only
						FILE_ATTRIBUTE_NORMAL, // normal file
						NULL);                 // no attr. template
			
		if ( m_FileHandles[nFileHandleIdx].m_hFileHandle != INVALID_HANDLE_VALUE )
		{
			m_FileHandles[nFileHandleIdx].m_nFileNumber = nFileNumber;
		}
#else
		m_FileHandles[nFileHandleIdx].m_hFileHandle = m_pFileSystem->Open( pszDataFileName, "rb" );
		if ( m_FileHandles[nFileHandleIdx].m_hFileHandle != FILESYSTEM_INVALID_HANDLE )
		{
			m_FileHandles[nFileHandleIdx].m_nFileNumber = nFileNumber;
		}
#endif
		return m_FileHandles[nFileHandleIdx];
	}
	Error( "Exceeded limit of number of vpk files supported (%d)!\n", MAX_ARCHIVE_FILES_TO_KEEP_OPEN_AT_ONCE );
	static FileHandleTracker_t invalid;
#ifdef IS_WINDOWS_PC
	invalid.m_hFileHandle = INVALID_HANDLE_VALUE;
#else
	invalid.m_hFileHandle = FILESYSTEM_INVALID_HANDLE;
#endif
	return invalid;
}

bool CPackedStore::RemoveFileFromDirectory( const char *pszName )
{
	// Remove it without building hash tables
	if ( !InternalRemoveFileFromDirectory( pszName ) )
		return false;

	// We removed it, we need to rebuild hash tables
	BuildHashTables();
	return true;
}

bool CPackedStore::InternalRemoveFileFromDirectory( const char *pszName )
{
	CPackedStoreFileHandle pData = OpenFile( pszName );
	if ( !pData )
		return false;

	CFileHeaderFixedData *pHeader = pData.m_pHeaderData;
	// delete the old header so we can insert a new one with updated contents
	int nBytesToRemove = ( int )( V_strlen( ( char * ) pData.m_pDirFileNamePtr ) + 1 + pHeader->HeaderSizeIncludingMetaData() );
	m_DirectoryData.RemoveMultiple( pData.m_pDirFileNamePtr - m_DirectoryData.Base(), nBytesToRemove );
	return true;
}

void CPackedStore::AddFileToDirectory( const VPKContentFileInfo_t &info )
{
	// this method is fairly complicated because it has to do inserts into the packed directory
	// data Our strategy is to build out the whole ext _ dir _ file record. if none of this is
	// already present, we will just insert it in the head of the file. If the extension is
	// present, we'll insert the dir+file part. If the extension + dir is present, we just insert
	// the file part at the right place. If everything is present, we just need to return the
	// current record

	// First, remove it if it's already there,
	// without rebuilding the hash tables
	InternalRemoveFileFromDirectory( info.m_sName );

	// let's build out a header
	char pszExt[MAX_PATH];
	char pszBase[MAX_PATH];
	char pszDir[MAX_PATH];
	SplitFileComponents( info.m_sName, pszDir, pszBase, pszExt );
	int nNumDataParts = 1;
	int nFileDataSize = s_FileHeaderSize( pszBase, nNumDataParts, info.m_iPreloadSize );
	int nTotalHeaderSize = ( int )( nFileDataSize + ( 2 + strlen( pszExt ) ) + ( 2 + strlen( pszDir ) ) );
	char *pBuf = ( char * ) stackalloc( nTotalHeaderSize );
	char *pOut = pBuf;
	strcpy( pOut, pszExt );
	pOut += strlen( pszExt );
	*( pOut++ ) = 0;										// null on ext name
	strcpy( pOut, pszDir );
	pOut += strlen( pszDir );
	*( pOut++ ) = 0;									// null at end of dir name
	strcpy( pOut, pszBase );
	pOut += strlen( pszBase );
	*( pOut++ ) = 0;
	uint32 nCRC = info.m_crc;
	memcpy( pOut, &nCRC, sizeof( nCRC ) );
	pOut += sizeof( int );
	if ( info.m_iPreloadSize > 0xffff )
		Error( "Preload size for '%s' is too big", info.m_sName.String() );
	uint16 nMetaDataSize = (uint16)info.m_iPreloadSize;
	memcpy( pOut, &nMetaDataSize, sizeof( uint16 ) );
	pOut += sizeof( uint16 );

	// now, build file parts.
	CFilePartDescr newPart;
	newPart.m_nFileDataSize = info.GetSizeInChunkFile();
	newPart.m_nFileNumber = ( info.m_idxChunk < 0 ) ? VPKFILENUMBER_EMBEDDED_IN_DIR_FILE : info.m_idxChunk;
	newPart.m_nFileDataOffset = info.m_iOffsetInChunk;

	memcpy( pOut, &newPart, sizeof( newPart ) );
	pOut += sizeof( newPart );

	PackFileIndex_t endOfPartMarker = PACKFILEINDEX_END;
	memcpy( pOut, &endOfPartMarker, sizeof( endOfPartMarker ) );
	pOut += sizeof( PackFileIndex_t );
	if ( nMetaDataSize )
	{
		Assert( info.m_pPreloadData );
		memcpy( pOut, info.m_pPreloadData, nMetaDataSize );
		pOut += nMetaDataSize;
	}
	*( pOut++ ) = 0;									// mark no more files in dir
	*( pOut++ ) = 0;									// mark no more dirs in extension
	Assert( pOut - pBuf == nTotalHeaderSize );

	// now, we need to insert our header, figuring out how many of the fields are already there
	int nExtensionHash = HashString( pszExt ) % PACKEDFILE_EXT_HASH_SIZE;
	int nInsertOffset = 0;
	CFileExtensionData const *pExt = m_pExtensionData[nExtensionHash].FindNamedNodeCaseSensitive( pszExt );
	char *pHeaderInsertPtr = pBuf;

	if ( pExt )
	{
		// this is not a new extension. we should not insert the extension record
		nTotalHeaderSize -= 2 + strlen( pszExt );		// null + end of dir list marker
		pHeaderInsertPtr += 1 + strlen( pszExt );					// don't insert the name + null
		// now, look for the directory
		int nDirHash = HashString( pszDir ) % PACKEDFILE_DIR_HASH_SIZE;
		CFileDirectoryData const *pDir = pExt->m_pDirectoryHashTable[nDirHash].FindNamedNodeCaseSensitive( pszDir );
		if ( pDir )
		{
			// dir and extension found. all we need to do is insert the file data itself
			nTotalHeaderSize -= 2 + strlen( pszDir );	// null + end of file list marker
			pHeaderInsertPtr += 1 + strlen( pszDir );
			char const *pStartOfDirFileData = pDir->m_Name + 1 + strlen( pDir->m_Name );
			nInsertOffset = pStartOfDirFileData - ( char const * ) ( m_DirectoryData.Base() );
		}
		else
		{
			char const *pStartOfExtFileData = pExt->m_Name + 1 + strlen( pExt->m_Name );
			nInsertOffset = pStartOfExtFileData - ( char const * ) ( m_DirectoryData.Base() );
		}
	}
	m_DirectoryData.InsertMultipleBefore( nInsertOffset, nTotalHeaderSize );
	memcpy( &m_DirectoryData[nInsertOffset], pHeaderInsertPtr, nTotalHeaderSize );
	BuildHashTables();
}

ePackedStoreAddResultCode CPackedStore::AddFile( char const *pFile, uint16 nMetaDataSize, const void *pFileData, uint32 nFileTotalSize, bool bMultiChunk, uint32 const *pCrcValue )
{

	// Calculate CRC if they didn't provide one
	uint32 nCRC;
	if ( pCrcValue )
	{
		nCRC = *pCrcValue;
	}
	else
	{
		nCRC = CRC32_ProcessSingleBuffer( pFileData, nFileTotalSize );
	}

	// Check if it is already here with the same contents
	CPackedStoreFileHandle pData = OpenFile( pFile );
	ePackedStoreAddResultCode nRslt = EPADD_NEWFILE;
	if ( pData )											// already in pack
	{
		CFileHeaderFixedData *pHeader = pData.m_pHeaderData;
		if ( ( nFileTotalSize == pHeader->TotalDataSize() ) && ( pHeader->m_nFileCRC == nCRC ) && ( nMetaDataSize == pHeader->m_nMetaDataSize  ) ) // file unchanged?
		{
			return EPADD_ADDSAMEFILE;
		}
		nRslt = EPADD_UPDATEFILE;
	}

	// Build up the directory info into an interface structure
	VPKContentFileInfo_t dirEntry;
	dirEntry.m_sName = pFile;
	dirEntry.m_iTotalSize = nFileTotalSize;
	dirEntry.m_iPreloadSize = Min( (uint32)nMetaDataSize, (uint32)nFileTotalSize ) ;
	dirEntry.m_pPreloadData = ( dirEntry.m_iPreloadSize > 0 ) ? pFileData : NULL;
	dirEntry.m_crc = nCRC;
	uint32 nBytesInChunk = dirEntry.GetSizeInChunkFile();
	const unsigned char *pDataStart = (const unsigned char *)pFileData + dirEntry.m_iPreloadSize;

	if ( bMultiChunk && nBytesInChunk > 0 )
	{

		// Check if we need to start a new chunk
		char szDataFileName[MAX_PATH];
		if ( m_nHighestChunkFileIndex < 0 )
		{
			dirEntry.m_idxChunk = 0;
			dirEntry.m_iOffsetInChunk = 0;
		}
		else
		{
			dirEntry.m_idxChunk = m_nHighestChunkFileIndex;

			// Append to most recent chunk
			GetDataFileName( szDataFileName, sizeof(szDataFileName), m_nHighestChunkFileIndex );
			dirEntry.m_iOffsetInChunk = g_pFullFileSystem->Size( szDataFileName );
			if ( (int)dirEntry.m_iOffsetInChunk <= 0 ) // technical wrong, but we shouldn't have 2GB chunks.  (Sort of defeats the whole purpose.)
			{
				// Note, there is one possible failure case.  if we have a file whose data
				// is actually all in the preload section, but it is marked as being
				// in a chunk, then we might have a zero byte "chunk."  We really should
				// not be assigning any files to "chunks" if they are entirely in the preload
				// area.
				Error( "Error querying %s for file size\n", szDataFileName );
			}

			// Check if we need to start a new chunk
			if ( (int)dirEntry.m_iOffsetInChunk >= m_nWriteChunkSize )
			{
				++dirEntry.m_idxChunk;
				dirEntry.m_iOffsetInChunk = 0;
			}
		}

		m_nHighestChunkFileIndex = MAX( m_nHighestChunkFileIndex, dirEntry.m_idxChunk );

		// write the actual data
		GetDataFileName( szDataFileName, sizeof(szDataFileName), dirEntry.m_idxChunk );
		FileHandle_t fHandle = m_pFileSystem->Open( szDataFileName, "rb+" );
		if ( !fHandle && dirEntry.m_iOffsetInChunk == 0 )
			fHandle = m_pFileSystem->Open( szDataFileName, "wb" );
		if ( !fHandle )
			Error( "Cannot open %s for writing", szDataFileName );

		m_pFileSystem->Seek( fHandle, dirEntry.m_iOffsetInChunk, FILESYSTEM_SEEK_HEAD );
		m_pFileSystem->Write( pDataStart, nBytesInChunk, fHandle );
		m_pFileSystem->Close( fHandle );

		// Force on the use of the "dir" file
		m_bUseDirFile = true;
	}
	else
	{
		// append to the dir data.
		dirEntry.m_idxChunk = VPKFILENUMBER_EMBEDDED_IN_DIR_FILE;
		dirEntry.m_iOffsetInChunk = m_EmbeddedChunkData.Count();
		m_EmbeddedChunkData.AddMultipleToTail( nBytesInChunk, pDataStart );
	}

	// Update the directory
	AddFileToDirectory( dirEntry );

	return nRslt;
}

int CPackedStore::GetFileList( CUtlStringList &outFilenames, bool bFormattedOutput, bool bSortedOutput )
{
	return GetFileList( NULL, outFilenames, bFormattedOutput, bSortedOutput );
}

int CPackedStore::GetFileList( const char *pWildCard, CUtlStringList &outFilenames, bool bFormattedOutput, bool bSortedOutput )
{
	// Separate the wildcard base from the extension
	char szWildCardPath[MAX_PATH];
	char szWildCardBase[64];
	char szWildCardExt[20];
	bool bNoBaseWildcard = false;
	bool bNoExtWildcard = false;

	szWildCardPath[0] = szWildCardExt[0] = szWildCardBase[0] = NULL;

	// Parse the wildcard string into a base and extension used for string comparisons
	if ( pWildCard )
	{
		V_ExtractFilePath( pWildCard, szWildCardPath, sizeof( szWildCardPath ) );
		V_FixSlashes( szWildCardPath, '/' );

		V_FileBase( pWildCard, szWildCardBase, sizeof( szWildCardBase ) );
		V_ExtractFileExtension( pWildCard, szWildCardExt, sizeof( szWildCardExt ) );

		// Remove '*' from the base and extension strings so that the string comparison calls will match
		char *pcStar = strchr( szWildCardBase, '*' );
		pcStar ? *pcStar = NULL : bNoBaseWildcard = true;
		pcStar = strchr( szWildCardExt, '*' );
		pcStar ? *pcStar = NULL : bNoExtWildcard = true;
	}

	char const *pData = reinterpret_cast< char const *>( DirectoryData() );
	while( *pData )
	{
		// for each extension
		char pszCurExtension[MAX_PATH];
		if ( pData[0] != ' ' )
			sprintf( pszCurExtension, ".%s", pData );
		else
			pszCurExtension[0] = 0;
		// now, iterate over all directories associated with this extension
		pData += 1 + strlen( pData );
		while( *pData )
		{
			char pszCurDir[MAX_PATH];
			if ( pData[0] != ' ' )
				sprintf( pszCurDir, "%s/", pData );
			else
				pszCurDir[0] = 0;
			pData += 1 + strlen( pData );					// skip dir name
			// now, march through all the files
			while( *pData )									// until we're out of files to look at
			{
				char pszFNameOut[MAX_PATH*2];
				if ( bFormattedOutput )
				{
					CFileHeaderFixedData const *pHeader = reinterpret_cast< CFileHeaderFixedData const *>( pData + 1 + strlen( pData ) );
					sprintf( pszFNameOut, "%s%s%s crc=0x%x metadatasz=%d", pszCurDir, pData, pszCurExtension, pHeader->m_nFileCRC, pHeader->m_nMetaDataSize );
					CFilePartDescr const *pPart = &( pHeader->m_PartDescriptors[0] );
					while( pPart->m_nFileNumber != PACKFILEINDEX_END )
					{
						sprintf( pszFNameOut + strlen( pszFNameOut )," fnumber=%d ofs=0x%x sz=%d",
								 pPart->m_nFileNumber, pPart->m_nFileDataOffset, pPart->m_nFileDataSize );
						pPart++;
					}
				}
				else
				{
 					V_strncpy( pszFNameOut, pszCurDir, sizeof( pszFNameOut ) );
 					V_strncat( pszFNameOut, pData, sizeof( pszFNameOut ) );
 					V_strncat( pszFNameOut, pszCurExtension, sizeof( pszFNameOut ) );
				}
				SkipFile( pData );

				bool matches = true;

				if ( pWildCard )
				{
					// See if the filename matches the wildcards
					char szFNameOutPath[MAX_PATH];
					char szFNameOutBase[64];
					char szFNameOutExt[20];

					V_ExtractFilePath( pszFNameOut, szFNameOutPath, sizeof( szFNameOutPath ) );
					V_FileBase( pszFNameOut, szFNameOutBase, sizeof( szFNameOutBase ) );
					V_ExtractFileExtension( pszFNameOut, szFNameOutExt, sizeof( szFNameOutExt ) );

					matches =  !V_strnicmp( szFNameOutPath, szWildCardPath, sizeof( szWildCardPath ) );
					matches = matches && ( !V_strlen( szWildCardExt ) || bNoExtWildcard ? 0 == V_strnicmp( szFNameOutExt, szWildCardExt, strlen( szWildCardExt ) ) : 0 != V_stristr(szFNameOutExt, szWildCardExt ) );
					matches = matches && ( !V_strlen( szWildCardBase ) || bNoBaseWildcard ? 0 == V_strnicmp( szFNameOutBase, szWildCardBase, strlen( szWildCardBase ) ) : 0 != V_stristr(szFNameOutBase, szWildCardBase ) );
				}

				// Add the file to the output list
				if ( matches )
				{
					char *pFName = new char[1 + strlen( pszFNameOut ) ];
					strcpy( pFName, pszFNameOut );
					outFilenames.AddToTail( pFName );
				}
			}
			pData++;												// skip end marker
		}
		pData++;												// skip end marker
	}

	if ( bSortedOutput )
	{
		outFilenames.Sort( &CUtlStringList::SortFunc );
	}
	
	return outFilenames.Count();
}

void CPackedStore::GetFileList( const char *pWildcard, CUtlVector<VPKContentFileInfo_t> &outVecResults )
{

	// !KLUDGE! Get the filenames first, and then "find" them again.
	CUtlStringList vecFilenames;
	GetFileList( vecFilenames, false, false );

	FOR_EACH_VEC( vecFilenames, i )
	{
		// Locate where it is in the existing file
		CPackedStoreFileHandle h = OpenFile( vecFilenames[i] );
		if ( !h )
			Error( "File '%s' was returned by GetFileList, but OpenFile() fails?!", vecFilenames[i] );

		// Convert to output structure
		VPKContentFileInfo_t &f = outVecResults[ outVecResults.AddToTail() ];
		f.m_sName = vecFilenames[i];
		f.m_idxChunk = ( h.m_nFileNumber == VPKFILENUMBER_EMBEDDED_IN_DIR_FILE ) ? -1 : h.m_nFileNumber;
		f.m_iTotalSize = h.m_nFileSize;
		f.m_iOffsetInChunk = h.m_nFileOffset;
		f.m_iPreloadSize = h.m_nMetaDataSize;
		f.m_crc = h.m_pHeaderData->m_nFileCRC;
		f.m_pPreloadData = h.m_pHeaderData->MetaData();
	}
}

int CPackedStore::GetFileAndDirLists( CUtlStringList &outDirnames, CUtlStringList &outFilenames, bool bSortedOutput )
{
	return GetFileAndDirLists( NULL, outDirnames, outFilenames, bSortedOutput );
}

void CPackedStore::BuildFindFirstCache()
{
	CUtlStringList allVPKFiles;
	char szLastDirFound[MAX_PATH];

	// Init
	V_strncpy( szLastDirFound, "$$$$$$$HighlyUnlikelyPathForInitializationPurposes#######", sizeof( szLastDirFound ) );
	m_dirContents.SetLessFunc( DefLessFunc( int ) );

	// Get all files in the VPK
	GetFileList( allVPKFiles, false, true );

	// Add directories to directory list and files into map
	FOR_EACH_VEC( allVPKFiles, i )
	{
		char szFilePath[MAX_PATH];

		V_ExtractFilePath( allVPKFiles[i], szFilePath, sizeof( szFilePath ) );
		Q_StripTrailingSlash( szFilePath );
		
		// New directory
		if ( V_strnicmp( szFilePath, szLastDirFound, sizeof( szLastDirFound ) ) )
		{
			// Mark the new one as the last one encountered
			V_strncpy( szLastDirFound, szFilePath, sizeof( szFilePath ) );

			// Add it
			m_directoryList.CopyAndAddToTail( szFilePath );
			m_dirContents.Insert( m_directoryList.Count(), new CUtlStringList() ); // Freed in destructor
		}
		
		unsigned short nIndex = m_dirContents.Find( m_directoryList.Count() );
		CUtlStringList *pList = m_dirContents.Element( nIndex );

		pList->CopyAndAddToTail( V_UnqualifiedFileName( allVPKFiles[i] ) );
	}
}

int CPackedStore::GetFileAndDirLists( const char *pWildCard, CUtlStringList &outDirnames, CUtlStringList &outFilenames, bool bSortedOutput )
{


	// If this is the first time we've called FindFirst on this CPackedStore then let's build the caches
	if ( !m_directoryList.Count() )
	{
		BuildFindFirstCache();
#ifdef NEVER
		printf("CPackedStore::GetFileAndDirLists - list of directories in VPK files\n");
		FOR_EACH_VEC( m_directoryList, i )
		{
			printf("\t%d : %s\n", i, m_directoryList[i] );
		}
#endif // NEVER
	}

	// printf("CPackedStore::GetFileAndDirLists - Searching for %s\n", pWildCard? pWildCard: "NULL");

	if ( pWildCard )
	{
		CUtlDict<int,int> AddedDirectories; // Used to remove duplicate paths

		char szWildCardPath[MAX_PATH];
		char szWildCardBase[64];
		char szWildCardExt[20];
		
		int nLenWildcardPath = 0;
		int nLenWildcardBase = 0;
		int nLenWildcardExt = 0;

		bool bBaseWildcard = true;
		bool bExtWildcard = true;

		szWildCardPath[0] = szWildCardExt[0] = szWildCardBase[0] = '\0';
		//
		// Parse the wildcard string into a base and extension used for string comparisons
		//
		V_ExtractFilePath( pWildCard, szWildCardPath, sizeof( szWildCardPath ) );
		V_FixSlashes( szWildCardPath, '/' );

		V_FileBase( pWildCard, szWildCardBase, sizeof( szWildCardBase ) );
		V_ExtractFileExtension( pWildCard, szWildCardExt, sizeof( szWildCardExt ) );

		// From the pattern, we now have the directory path up to the file pattern, the filename base, and the filename 
		// extension.

		// Remove '*' from the base and extension strings so that the string comparison calls will match
		char *pcStar = strchr( szWildCardBase, '*' );
		pcStar ? *pcStar = NULL : bBaseWildcard = false;

		pcStar = strchr( szWildCardExt, '*' );
		pcStar ? *pcStar = NULL : bExtWildcard = false;

		nLenWildcardPath = V_strlen( szWildCardPath );
		nLenWildcardBase = V_strlen( szWildCardBase );
		nLenWildcardExt = V_strlen( szWildCardExt );

		// Generate the list of directories and files that match the wildcard
		//

		//
		// Directories first
		//
		FOR_EACH_VEC( m_directoryList, i )
		{
			// Does this file's path match the wildcard path?
			if  (  (  nLenWildcardPath && ( 0 == V_strnicmp( m_directoryList[i], szWildCardPath, nLenWildcardPath ) ) ) 
				|| ( !nLenWildcardPath && ( 0 == V_strlen( m_directoryList[i] ) ) ) )
			{
				// Extract the sub-directory name if there is one
				char szSubDir[64];
				char *szSubDirExtension = NULL; // this is anything after a '.' in szSubDir
				bool bBaseMatch = false;
				bool bExtMatch = false;

				// Copy everything to the right of the root directory
				V_strncpy( szSubDir, &m_directoryList[i][nLenWildcardPath], sizeof( szSubDir ) );

				// Set the next / to NULL and we have our subdirectory
				char *pSlash = strchr( szSubDir, '/' );
				pSlash ? *pSlash = NULL : NULL;

				szSubDirExtension = strchr( szSubDir, '.' );
				if ( szSubDirExtension )
				{
					// Null out the . and move the szSubDirExtension to point to the extension
					*szSubDirExtension = '\0';
					szSubDirExtension++;
				}

				// If we have a base dir name, and we have a szWildCardBase to match against
				if ( bBaseWildcard )
					bBaseMatch = true;  // The base is the wildCard ("*"), so whatever we have as the base matches
				else 
					bBaseMatch = ( 0 == V_strnicmp( szSubDir, szWildCardBase, nLenWildcardBase ) );

				// If we have an extension and we have a szWildCardExtension to mach against
				if ( bExtWildcard )
					bExtMatch = true; // The extension is the wildcard ("*"), so whatever we have as the extension matches
				else 
					bExtMatch = ( NULL == szSubDirExtension && '\0' == *szWildCardExt ) || (( NULL != szSubDirExtension ) && ( 0 == V_strnicmp( szSubDirExtension, szWildCardExt, nLenWildcardExt ) ));

				// If both parts match, then add it to the list of directories that match
				if ( bBaseMatch && bExtMatch )
				{
					char szFullPathToDir[ MAX_PATH ];

					V_strncpy( szFullPathToDir, szWildCardPath, nLenWildcardPath );
					V_strcat_safe( szFullPathToDir, "/" );
					V_strcat_safe( szFullPathToDir, szSubDir );
		
					// Add the subdirectory to the list if it isn't already there
					if ( -1 == AddedDirectories.Find( szFullPathToDir ) )
					{
						char *pDName = new char[1 + strlen( szFullPathToDir )];
						V_strncpy( pDName, szFullPathToDir, 1 + strlen( szFullPathToDir ) );
						outDirnames.AddToTail( pDName );
						AddedDirectories.Insert( pDName, 0 );
					}
				}
			}
		}

		//
		// Files
		//
		FOR_EACH_VEC( m_directoryList, i )
		{
			// We no longer want the trailing slash
			Q_StripTrailingSlash( szWildCardPath );

			// Find the directory that matches the wildcard path
			if ( !V_strnicmp( szWildCardPath, m_directoryList[i], sizeof( szWildCardPath ) ) )
			{
				CUtlStringList &filesInDirectory = *(m_dirContents.Element( i ));

				// Use the cached list of files in this directory
				FOR_EACH_VEC( filesInDirectory, iFile )
				{
					bool matches = true;


					// See if the filename matches the wildcards
					char szFNameOutBase[64];
					char szFNameOutExt[20];

					V_FileBase( filesInDirectory[iFile], szFNameOutBase, sizeof( szFNameOutBase ) );
					V_ExtractFileExtension( filesInDirectory[iFile], szFNameOutExt, sizeof( szFNameOutExt ) );

					// Since we have a sorted list we can optimize using the return code of the compare
					int c = V_strnicmp( szWildCardBase, szFNameOutBase, nLenWildcardBase );
					if ( c < 0 )
						break;
					if ( c > 0 )
						continue;
					matches = ( (nLenWildcardExt <= 0) || bBaseWildcard ? 0 == V_strnicmp( szFNameOutExt, szWildCardExt, nLenWildcardExt ) : V_stristr( szFNameOutExt, szWildCardExt ) != NULL );

					// Add the file to the output list
					if ( matches )
					{	
						bool bFound = false;

						FOR_EACH_VEC( outFilenames, j )
						{
							if ( !V_strncmp( outFilenames[j], filesInDirectory[iFile], V_strlen( filesInDirectory[iFile] ) ) )
							{
								bFound = true;
								break;
							}
						}

						if ( !bFound )
						{
							outFilenames.CopyAndAddToTail( filesInDirectory[iFile] );
						}
					}
				}
			}
		}
	}
	else // Otherwise, simply return the base data
	{
		// Add all the files as well
		FOR_EACH_VEC( m_directoryList, i )
		{
			// Add all directories
			outDirnames.CopyAndAddToTail( m_directoryList[i] );

			// Now add all files
			CUtlStringList &filesInDirectory = *(m_dirContents.Element( i ));
			FOR_EACH_VEC( filesInDirectory, j )
			{
				outFilenames.CopyAndAddToTail( filesInDirectory[j] );
			}
		}
	}

	// Sort the output if requested
	if ( bSortedOutput )
	{
		outDirnames.Sort( &CUtlStringList::SortFunc );
		outFilenames.Sort( &CUtlStringList::SortFunc );
	}

	return outDirnames.Count();
}