source-engine/public/tier1/utlsymbollarge.h

500 lines
15 KiB
C
Raw Normal View History

2020-04-22 16:56:21 +00:00
//========= Copyright Valve Corporation, All rights reserved. ============//
//
// Purpose: Defines a large symbol table (intp sized handles, can store more than 64k strings)
//
// $Header: $
// $NoKeywords: $
//===========================================================================//
#ifndef UTLSYMBOLLARGE_H
#define UTLSYMBOLLARGE_H
#ifdef _WIN32
#pragma once
#endif
#include "tier0/threadtools.h"
#include "tier1/utltshash.h"
#include "tier1/stringpool.h"
#include "tier0/vprof.h"
#include "tier1/utltshash.h"
//-----------------------------------------------------------------------------
// CUtlSymbolTableLarge:
// description:
// This class defines a symbol table, which allows us to perform mappings
// of strings to symbols and back.
//
// This class stores the strings in a series of string pools. The returned CUtlSymbolLarge is just a pointer
// to the string data, the hash precedes it in memory and is used to speed up searching, etc.
//-----------------------------------------------------------------------------
typedef intp UtlSymLargeId_t;
#define UTL_INVAL_SYMBOL_LARGE ((UtlSymLargeId_t)~0)
class CUtlSymbolLarge
{
public:
// constructor, destructor
CUtlSymbolLarge()
{
u.m_Id = UTL_INVAL_SYMBOL_LARGE;
}
CUtlSymbolLarge( UtlSymLargeId_t id )
{
u.m_Id = id;
}
CUtlSymbolLarge( CUtlSymbolLarge const& sym )
{
u.m_Id = sym.u.m_Id;
}
// operator=
CUtlSymbolLarge& operator=( CUtlSymbolLarge const& src )
{
u.m_Id = src.u.m_Id;
return *this;
}
// operator==
bool operator==( CUtlSymbolLarge const& src ) const
{
return u.m_Id == src.u.m_Id;
}
// operator==
bool operator==( UtlSymLargeId_t const& src ) const
{
return u.m_Id == src;
}
// operator==
bool operator!=( CUtlSymbolLarge const& src ) const
{
return u.m_Id != src.u.m_Id;
}
// operator==
bool operator!=( UtlSymLargeId_t const& src ) const
{
return u.m_Id != src;
}
// Gets at the symbol
operator UtlSymLargeId_t const() const
{
return u.m_Id;
}
// Gets the string associated with the symbol
inline const char* String( ) const
{
if ( u.m_Id == UTL_INVAL_SYMBOL_LARGE )
return "";
return u.m_pAsString;
}
inline bool IsValid() const
{
return u.m_Id != UTL_INVAL_SYMBOL_LARGE ? true : false;
}
private:
// Disallowed
CUtlSymbolLarge( const char* pStr ); // they need to go through the table to assign the ptr
bool operator==( const char* pStr ) const; // disallow since we don't know if the table this is from was case sensitive or not... maybe we don't care
union
{
UtlSymLargeId_t m_Id;
char const *m_pAsString;
} u;
};
#define MIN_STRING_POOL_SIZE 2048
inline uint32 CUtlSymbolLarge_Hash( bool CASEINSENSITIVE, const char *pString, int len )
{
return ( CASEINSENSITIVE ? HashStringCaseless( pString ) : HashString( pString ) );
}
typedef uint32 LargeSymbolTableHashDecoration_t;
// The structure consists of the hash immediately followed by the string data
struct CUtlSymbolTableLargeBaseTreeEntry_t
{
LargeSymbolTableHashDecoration_t m_Hash;
// Variable length string data
char m_String[1];
bool IsEmpty() const
{
return ( ( m_Hash == 0 ) && ( 0 == m_String[0] ) );
}
char const *String() const
{
return (const char *)&m_String[ 0 ];
}
CUtlSymbolLarge ToSymbol() const
{
return reinterpret_cast< UtlSymLargeId_t >( String() );
}
LargeSymbolTableHashDecoration_t HashValue() const
{
return m_Hash;
}
};
template< class TreeType, bool CASEINSENSITIVE >
class CTreeEntryLess
{
public:
CTreeEntryLess( int ignored = 0 ) {} // permits default initialization to NULL in CUtlRBTree
bool operator!() const { return false; }
bool operator()( CUtlSymbolTableLargeBaseTreeEntry_t * const &left, CUtlSymbolTableLargeBaseTreeEntry_t * const &right ) const
{
// compare the hashes
if ( left->m_Hash == right->m_Hash )
{
// if the hashes match compare the strings
if ( !CASEINSENSITIVE )
return strcmp( left->String(), right->String() ) < 0;
else
return V_stricmp( left->String(), right->String() ) < 0;
}
else
{
return left->m_Hash < right->m_Hash;
}
}
};
// For non-threaded versions, simply index into CUtlRBTree
template< bool CASEINSENSITIVE >
class CNonThreadsafeTree : public CUtlRBTree<CUtlSymbolTableLargeBaseTreeEntry_t *, intp, CTreeEntryLess< CNonThreadsafeTree< CASEINSENSITIVE >, CASEINSENSITIVE > >
{
public:
typedef CUtlRBTree<CUtlSymbolTableLargeBaseTreeEntry_t *, intp, CTreeEntryLess< CNonThreadsafeTree, CASEINSENSITIVE > > CNonThreadsafeTreeType;
CNonThreadsafeTree() :
CNonThreadsafeTreeType( 0, 16 )
{
}
inline void Commit()
{
// Nothing, only matters for thread-safe tables
}
inline int Insert( CUtlSymbolTableLargeBaseTreeEntry_t *entry )
{
return CNonThreadsafeTreeType::Insert( entry );
}
inline int Find( CUtlSymbolTableLargeBaseTreeEntry_t *entry ) const
{
return CNonThreadsafeTreeType::Find( entry );
}
inline int InvalidIndex() const
{
return CNonThreadsafeTreeType::InvalidIndex();
}
inline int GetElements( int nFirstElement, int nCount, CUtlSymbolLarge *pElements ) const
{
CUtlVector< CUtlSymbolTableLargeBaseTreeEntry_t * > list;
list.EnsureCount( nCount );
for ( int i = 0; i < nCount; ++i )
{
pElements[ i ] = CNonThreadsafeTreeType::Element( i )->ToSymbol();
}
return nCount;
}
};
// Since CUtlSymbolTableLargeBaseTreeEntry_t already has the hash
// contained inside of it, don't need to recompute a hash here
template < int BUCKET_COUNT, class KEYTYPE, bool CASEINSENSITIVE >
class CCThreadsafeTreeHashMethod
{
public:
static int Hash( const KEYTYPE &key, int nBucketMask )
{
uint32 nHash = key->HashValue();
return ( nHash & nBucketMask );
}
static bool Compare( CUtlSymbolTableLargeBaseTreeEntry_t * const &lhs, CUtlSymbolTableLargeBaseTreeEntry_t * const &rhs )
{
if ( lhs->m_Hash != rhs->m_Hash )
return false;
if ( !CASEINSENSITIVE )
{
return ( !Q_strcmp( lhs->String(), rhs->String() ) ? true : false );
}
return ( !Q_stricmp( lhs->String(), rhs->String() ) ? true : false );
}
};
/*
NOTE: So the only crappy thing about using a CUtlTSHash here is that the KEYTYPE is a CUtlSymbolTableLargeBaseTreeEntry_t ptr which has both the
hash and the string since with strings there is a good chance of hash collision after you have a fair number of strings so we have to implement
a Compare method (above) which falls back to strcmp/stricmp if the hashes are equal. This means that all of the data is in the KEYTYPE of the hash and the
payload doesn't matter. So I made the payload also be a pointer to a CUtlSymbolTableLargeBaseTreeEntry_t since that makes using the API more convenient
TODO: If we have a CUtlTSHash that was all about the existence of the KEYTYPE and didn't require a payload (or template on 'void') then we could eliminate
50% of the pointer overhead used for this data structure.
*/
// Thread safe version is based on the
template < bool CASEINSENSITIVE >
class CThreadsafeTree : public CUtlTSHash< CUtlSymbolTableLargeBaseTreeEntry_t *, 2048, CUtlSymbolTableLargeBaseTreeEntry_t *, CCThreadsafeTreeHashMethod< 2048, CUtlSymbolTableLargeBaseTreeEntry_t *, CASEINSENSITIVE > >
{
public:
typedef CUtlTSHash< CUtlSymbolTableLargeBaseTreeEntry_t *, 2048, CUtlSymbolTableLargeBaseTreeEntry_t *, CCThreadsafeTreeHashMethod< 2048, CUtlSymbolTableLargeBaseTreeEntry_t *, CASEINSENSITIVE > > CThreadsafeTreeType;
CThreadsafeTree() :
CThreadsafeTreeType( 32 )
{
}
inline void Commit()
{
CThreadsafeTreeType::Commit();
}
inline int Insert( CUtlSymbolTableLargeBaseTreeEntry_t *entry )
{
return CThreadsafeTreeType::Insert( entry, entry );
}
inline int Find( CUtlSymbolTableLargeBaseTreeEntry_t *entry )
{
return CThreadsafeTreeType::Find( entry );
}
inline int InvalidIndex() const
{
return CThreadsafeTreeType::InvalidHandle();
}
inline int GetElements( int nFirstElement, int nCount, CUtlSymbolLarge *pElements ) const
{
CUtlVector< UtlTSHashHandle_t > list;
list.EnsureCount( nCount );
int c = CThreadsafeTreeType::GetElements( nFirstElement, nCount, list.Base() );
for ( int i = 0; i < c; ++i )
{
pElements[ i ] = CThreadsafeTreeType::Element( list[ i ] )->ToSymbol();
}
return c;
}
};
// Base Class for threaded and non-threaded types
template < class TreeType, bool CASEINSENSITIVE, size_t POOL_SIZE = MIN_STRING_POOL_SIZE >
class CUtlSymbolTableLargeBase
{
public:
// constructor, destructor
CUtlSymbolTableLargeBase();
~CUtlSymbolTableLargeBase();
// Finds and/or creates a symbol based on the string
CUtlSymbolLarge AddString( const char* pString );
// Finds the symbol for pString
CUtlSymbolLarge Find( const char* pString ) const;
// Remove all symbols in the table.
void RemoveAll();
int GetNumStrings( void ) const
{
return m_Lookup.Count();
}
void Commit()
{
m_Lookup.Commit();
}
// Returns elements in the table
int GetElements( int nFirstElement, int nCount, CUtlSymbolLarge *pElements ) const
{
return m_Lookup.GetElements( nFirstElement, nCount, pElements );
}
uint64 GetMemoryUsage() const
{
uint64 unBytesUsed = 0u;
for ( int i=0; i < m_StringPools.Count(); i++ )
{
StringPool_t *pPool = m_StringPools[i];
unBytesUsed += (uint64)pPool->m_TotalLen;
}
return unBytesUsed;
}
protected:
struct StringPool_t
{
int m_TotalLen; // How large is
int m_SpaceUsed;
char m_Data[1];
};
TreeType m_Lookup;
// stores the string data
CUtlVector< StringPool_t * > m_StringPools;
private:
int FindPoolWithSpace( int len ) const;
};
//-----------------------------------------------------------------------------
// constructor, destructor
//-----------------------------------------------------------------------------
template < class TreeType, bool CASEINSENSITIVE, size_t POOL_SIZE >
inline CUtlSymbolTableLargeBase<TreeType, CASEINSENSITIVE, POOL_SIZE >::CUtlSymbolTableLargeBase() :
m_StringPools( 8 )
{
}
template < class TreeType, bool CASEINSENSITIVE, size_t POOL_SIZE >
inline CUtlSymbolTableLargeBase<TreeType, CASEINSENSITIVE, POOL_SIZE>::~CUtlSymbolTableLargeBase()
{
// Release the stringpool string data
RemoveAll();
}
template < class TreeType, bool CASEINSENSITIVE, size_t POOL_SIZE >
inline CUtlSymbolLarge CUtlSymbolTableLargeBase<TreeType, CASEINSENSITIVE, POOL_SIZE>::Find( const char* pString ) const
{
VPROF( "CUtlSymbolLarge::Find" );
if (!pString)
return CUtlSymbolLarge();
// Passing this special invalid symbol makes the comparison function
// use the string passed in the context
int len = Q_strlen( pString ) + 1;
CUtlSymbolTableLargeBaseTreeEntry_t *search = (CUtlSymbolTableLargeBaseTreeEntry_t *)_alloca( len + sizeof( LargeSymbolTableHashDecoration_t ) );
search->m_Hash = CUtlSymbolLarge_Hash( CASEINSENSITIVE, pString, len );
Q_memcpy( (char *)&search->m_String[ 0 ], pString, len );
int idx = const_cast< TreeType & >(m_Lookup).Find( search );
if ( idx == m_Lookup.InvalidIndex() )
return UTL_INVAL_SYMBOL_LARGE;
const CUtlSymbolTableLargeBaseTreeEntry_t *entry = m_Lookup[ idx ];
return entry->ToSymbol();
}
template < class TreeType, bool CASEINSENSITIVE, size_t POOL_SIZE >
inline int CUtlSymbolTableLargeBase<TreeType, CASEINSENSITIVE, POOL_SIZE>::FindPoolWithSpace( int len ) const
{
for ( int i=0; i < m_StringPools.Count(); i++ )
{
StringPool_t *pPool = m_StringPools[i];
if ( (pPool->m_TotalLen - pPool->m_SpaceUsed) >= len )
{
return i;
}
}
return -1;
}
//-----------------------------------------------------------------------------
// Finds and/or creates a symbol based on the string
//-----------------------------------------------------------------------------
template < class TreeType, bool CASEINSENSITIVE, size_t POOL_SIZE >
inline CUtlSymbolLarge CUtlSymbolTableLargeBase<TreeType, CASEINSENSITIVE, POOL_SIZE>::AddString( const char* pString )
{
VPROF("CUtlSymbolLarge::AddString");
if (!pString)
return UTL_INVAL_SYMBOL_LARGE;
CUtlSymbolLarge id = Find( pString );
if ( id != UTL_INVAL_SYMBOL_LARGE )
return id;
int lenString = Q_strlen(pString) + 1; // length of just the string
int lenDecorated = lenString + sizeof(LargeSymbolTableHashDecoration_t); // and with its hash decoration
// make sure that all strings are aligned on 2-byte boundaries so the hashes will read correctly
// This assert seems to be invalid because LargeSymbolTableHashDecoration_t is always
// a uint32, by design.
//COMPILE_TIME_ASSERT(sizeof(LargeSymbolTableHashDecoration_t) == sizeof(intp));
lenDecorated = ALIGN_VALUE(lenDecorated, sizeof( intp ) );
// Find a pool with space for this string, or allocate a new one.
int iPool = FindPoolWithSpace( lenDecorated );
if ( iPool == -1 )
{
// Add a new pool.
int newPoolSize = MAX( lenDecorated + sizeof( StringPool_t ), POOL_SIZE );
StringPool_t *pPool = (StringPool_t*)malloc( newPoolSize );
pPool->m_TotalLen = newPoolSize - sizeof( StringPool_t );
pPool->m_SpaceUsed = 0;
iPool = m_StringPools.AddToTail( pPool );
}
// Compute a hash
LargeSymbolTableHashDecoration_t hash = CUtlSymbolLarge_Hash( CASEINSENSITIVE, pString, lenString );
// Copy the string in.
StringPool_t *pPool = m_StringPools[iPool];
// Assert( pPool->m_SpaceUsed < 0xFFFF ); // Pool could be bigger than 2k
// This should never happen, because if we had a string > 64k, it
// would have been given its entire own pool.
CUtlSymbolTableLargeBaseTreeEntry_t *entry = ( CUtlSymbolTableLargeBaseTreeEntry_t * )&pPool->m_Data[ pPool->m_SpaceUsed ];
pPool->m_SpaceUsed += lenDecorated;
entry->m_Hash = hash;
char *pText = (char *)&entry->m_String [ 0 ];
Q_memcpy( pText, pString, lenString );
// insert the string into the database
MEM_ALLOC_CREDIT();
int idx = m_Lookup.Insert( entry );
return m_Lookup.Element( idx )->ToSymbol();
}
//-----------------------------------------------------------------------------
// Remove all symbols in the table.
//-----------------------------------------------------------------------------
template < class TreeType, bool CASEINSENSITIVE, size_t POOL_SIZE >
inline void CUtlSymbolTableLargeBase<TreeType, CASEINSENSITIVE, POOL_SIZE>::RemoveAll()
{
m_Lookup.Purge();
for ( int i=0; i < m_StringPools.Count(); i++ )
{
StringPool_t * pString = m_StringPools[i];
free( pString );
}
m_StringPools.RemoveAll();
}
// Case-sensitive
typedef CUtlSymbolTableLargeBase< CNonThreadsafeTree< false >, false > CUtlSymbolTableLarge;
// Case-insensitive
typedef CUtlSymbolTableLargeBase< CNonThreadsafeTree< true >, true > CUtlSymbolTableLarge_CI;
// Multi-threaded case-sensitive
typedef CUtlSymbolTableLargeBase< CThreadsafeTree< false >, false > CUtlSymbolTableLargeMT;
// Multi-threaded case-insensitive
typedef CUtlSymbolTableLargeBase< CThreadsafeTree< true >, true > CUtlSymbolTableLargeMT_CI;
#endif // UTLSYMBOLLARGE_H