1

utils/shadercompile/cfgprocessor.h

@@ -0,0 +1,76 @@
+//========= Copyright Valve Corporation, All rights reserved. ============//
+//
+// Purpose: 
+//
+// $NoKeywords: $
+//
+//=============================================================================//
+
+#ifndef CFGPROCESSOR_H
+#define CFGPROCESSOR_H
+#ifdef _WIN32
+#pragma once
+#endif
+
+#include "tier1/smartptr.h"
+
+
+/*
+
+Layout of the internal structures is as follows:
+
+|-------- shader1.fxc ---------||--- shader2.fxc ---||--------- shader3.fxc -----||-...
+| 0 s s 3 s s s s 8 s 10 s s s || s s 2 3 4 s s s 8 || 0 s s s 4 s s s 8 9 s s s ||-...
+| 0 1 2 3 4 5 6 7 8 9 10 * * *   14 * * * * *20 * *   23 * * *27 * * * * * * *35    * * *
+
+GetSection( 10 ) -> shader1.fxc
+GetSection( 27 ) -> shader3.fxc
+
+GetNextCombo(  3,  3, 14 ) -> shader1.fxc : ( riCommandNumber =  8, rhCombo =  "8" )
+GetNextCombo( 10, 10, 14 ) ->   NULL      : ( riCommandNumber = 14, rhCombo = NULL )
+GetNextCombo( 22,  8, 36 ) -> shader3.fxc : ( riCommandNumber = 23, rhCombo =  "0" )
+GetNextCombo( 29, -1, 36 ) -> shader3.fxc : ( riCommandNumber = 31, rhCombo =  "8" )
+
+*/
+
+class CUtlInplaceBuffer;
+
+namespace CfgProcessor
+{
+
+// Working with configuration
+void ReadConfiguration( FILE *fInputStream );
+void ReadConfiguration( CUtlInplaceBuffer *fInputStream );
+
+struct CfgEntryInfo
+{
+	char const *m_szName;				// Name of the shader, e.g. "shader_ps20b"
+	char const *m_szShaderFileName;		// Name of the src file, e.g. "shader_psxx.fxc"
+	uint64 m_numCombos;					// Total possible num of combos, e.g. 1024
+	uint64 m_numDynamicCombos;			// Num of dynamic combos, e.g. 4
+	uint64 m_numStaticCombos;			// Num of static combos, e.g. 256
+	uint64 m_iCommandStart;				// Start command, e.g. 0
+	uint64 m_iCommandEnd;				// End command, e.g. 1024
+};
+
+void DescribeConfiguration( CArrayAutoPtr < CfgEntryInfo > &rarrEntries );
+
+
+// Working with combos
+typedef struct {} * ComboHandle;
+
+ComboHandle Combo_GetCombo( uint64 iCommandNumber );
+ComboHandle Combo_GetNext( uint64 &riCommandNumber, ComboHandle &rhCombo, uint64 iCommandEnd );
+void Combo_FormatCommand( ComboHandle hCombo, char *pchBuffer );
+uint64 Combo_GetCommandNum( ComboHandle hCombo );
+uint64 Combo_GetComboNum( ComboHandle hCombo );
+CfgEntryInfo const *Combo_GetEntryInfo( ComboHandle hCombo );
+
+ComboHandle Combo_Alloc( ComboHandle hComboCopyFrom );
+void Combo_Assign( ComboHandle hComboDst, ComboHandle hComboSrc );
+void Combo_Free( ComboHandle &rhComboFree );
+
+}; // namespace CfgProcessor
+
+
+#endif // #ifndef CFGPROCESSOR_H

utils/shadercompile/cmdsink.cpp

@@ -0,0 +1,114 @@
+//========= Copyright Valve Corporation, All rights reserved. ============//
+//
+// Purpose: Command sink interface implementation.
+//
+// $NoKeywords: $
+//
+//=============================================================================//
+
+#include "cmdsink.h"
+
+
+namespace CmdSink
+{
+
+// ------ implementation of CResponseFiles --------------
+
+CResponseFiles::CResponseFiles( char const *szFileResult, char const *szFileListing ) :
+	m_fResult(NULL),
+	m_fListing(NULL),
+	m_lenResult(0),
+	m_dataResult(NULL),
+	m_dataListing(NULL)
+{
+	sprintf( m_szFileResult, szFileResult );
+	sprintf( m_szFileListing, szFileListing );
+}
+
+CResponseFiles::~CResponseFiles( void )
+{
+	if ( m_fResult )
+		fclose( m_fResult );
+
+	if ( m_fListing )
+		fclose( m_fListing );
+}
+
+bool CResponseFiles::Succeeded( void )
+{
+	OpenResultFile();
+	return ( m_fResult != NULL );
+}
+
+size_t CResponseFiles::GetResultBufferLen( void )
+{
+	ReadResultFile();
+	return m_lenResult;
+}
+
+const void * CResponseFiles::GetResultBuffer( void )
+{
+	ReadResultFile();
+	return m_dataResult;
+}
+
+const char * CResponseFiles::GetListing( void )
+{
+	ReadListingFile();
+	return ( ( m_dataListing && *m_dataListing ) ? m_dataListing : NULL );
+}
+
+void CResponseFiles::OpenResultFile( void )
+{
+	if ( !m_fResult )
+	{
+		m_fResult = fopen( m_szFileResult, "rb" );
+	}
+}
+
+void CResponseFiles::ReadResultFile( void )
+{
+	if ( !m_dataResult )
+	{
+		OpenResultFile();
+
+		if ( m_fResult )
+		{
+			fseek( m_fResult, 0, SEEK_END );
+			m_lenResult = (size_t) ftell( m_fResult );
+
+			if ( m_lenResult != size_t(-1) )
+			{
+				m_bufResult.EnsureCapacity( m_lenResult );
+				fseek( m_fResult, 0, SEEK_SET );
+				fread( m_bufResult.Base(), 1, m_lenResult, m_fResult );
+				m_dataResult = m_bufResult.Base();
+			}
+		}
+	}
+}
+
+void CResponseFiles::ReadListingFile( void )
+{
+	if ( !m_dataListing )
+	{
+		if ( !m_fListing )
+			m_fListing = fopen( m_szFileListing, "rb" );
+
+		if ( m_fListing )
+		{
+			fseek( m_fListing, 0, SEEK_END );
+			size_t len = (size_t) ftell( m_fListing );
+
+			if ( len != size_t(-1) )
+			{
+				m_bufListing.EnsureCapacity( len );
+				fseek( m_fListing, 0, SEEK_SET );
+				fread( m_bufListing.Base(), 1, len, m_fListing );
+				m_dataListing = (const char *) m_bufListing.Base();
+			}
+		}
+	}
+}
+
+}; // namespace CmdSink

utils/shadercompile/cmdsink.h

@@ -0,0 +1,117 @@
+//========= Copyright Valve Corporation, All rights reserved. ============//
+//
+// Purpose: Command sink interface implementation.
+//
+// $NoKeywords: $
+//
+//=============================================================================//
+
+#ifndef CMDSINK_H
+#define CMDSINK_H
+#ifdef _WIN32
+#pragma once
+#endif
+
+#include <stdio.h>
+#include <tier1/utlbuffer.h>
+
+
+namespace CmdSink
+{
+
+/*
+
+struct IResponse
+
+Interface to give back command execution results.
+
+*/
+struct IResponse
+{
+	virtual ~IResponse( void ) {}
+	virtual void Release( void ) { delete this; }
+
+	// Returns whether the command succeeded
+	virtual bool Succeeded( void ) = 0;
+
+	// If the command succeeded returns the result buffer length, otherwise zero
+	virtual size_t GetResultBufferLen( void ) = 0;
+	// If the command succeeded returns the result buffer base pointer, otherwise NULL
+	virtual const void * GetResultBuffer( void ) = 0;
+
+	// Returns a zero-terminated string of messages reported during command execution, or NULL if nothing was reported
+	virtual const char * GetListing( void ) = 0;
+};
+
+
+
+
+/*
+
+Response implementation when the result should appear in
+one file and the listing should appear in another file.
+
+*/
+class CResponseFiles : public IResponse
+{
+public:
+	explicit CResponseFiles( char const *szFileResult, char const *szFileListing );
+	~CResponseFiles( void );
+
+public:
+	// Returns whether the command succeeded
+	virtual bool Succeeded( void );
+
+	// If the command succeeded returns the result buffer length, otherwise zero
+	virtual size_t GetResultBufferLen( void );
+	// If the command succeeded returns the result buffer base pointer, otherwise NULL
+	virtual const void * GetResultBuffer( void );
+
+	// Returns a zero-terminated string of messages reported during command execution
+	virtual const char * GetListing( void );
+
+protected:
+	void OpenResultFile( void );		//!< Opens the result file if not open yet
+	void ReadResultFile( void );		//!< Reads the result buffer if not read yet
+	void ReadListingFile( void );		//!< Reads the listing buffer if not read yet
+
+protected:
+	char m_szFileResult[MAX_PATH];		//!< Name of the result file
+	char m_szFileListing[MAX_PATH];		//!< Name of the listing file
+
+	FILE *m_fResult;					//!< Result file (NULL if not open)
+	FILE *m_fListing;					//!< Listing file (NULL if not open)
+
+	CUtlBuffer m_bufResult;				//!< Buffer holding the result data
+	size_t m_lenResult;					//!< Result data length (0 if result not read yet)
+	const void *m_dataResult;			//!< Data buffer pointer (NULL if result not read yet)
+
+	CUtlBuffer m_bufListing;			//!< Buffer holding the listing
+	const char *m_dataListing;			//!< Listing buffer pointer (NULL if listing not read yet)
+};
+
+/*
+
+Response implementation when the result is a generic error.
+
+*/
+class CResponseError : public IResponse
+{
+public:
+	explicit CResponseError( void ) {}
+	~CResponseError( void ) {}
+
+public:
+	virtual bool Succeeded( void ) { return false; }
+
+	virtual size_t GetResultBufferLen( void ) { return 0; }
+	virtual const void * GetResultBuffer( void ) { return NULL; }
+
+	virtual const char * GetListing( void ) { return NULL; }
+};
+
+
+}; // namespace CmdSink
+
+
+#endif // #ifndef CMDSINK_H

utils/shadercompile/d3dxfxc.cpp

@@ -0,0 +1,259 @@
+//========= Copyright Valve Corporation, All rights reserved. ============//
+//
+// Purpose: D3DX command implementation.
+//
+// $NoKeywords: $
+//
+//=============================================================================//
+
+#include "shadercompile.h"
+
+#include "d3dxfxc.h"
+#include "cmdsink.h"
+
+// Required to compile using D3DX* routines in the same process
+#include <d3dx9shader.h>
+#include "dx_proxy/dx_proxy.h"
+
+#include <tier0/icommandline.h>
+#include <tier1/strtools.h>
+
+#define D3DXSHADER_MICROCODE_BACKEND_OLD_DEPRECATED ( 1 << 25 )
+
+namespace InterceptFxc
+{
+
+	// The command that is intercepted by this namespace routines
+	static const char *s_pszCommand = "fxc.exe ";
+	static size_t s_uCommandLen = strlen( s_pszCommand );
+
+	namespace Private
+	{
+		//
+		// Response implementation
+		//
+		class CResponse : public CmdSink::IResponse
+		{
+		public:
+			explicit CResponse( LPD3DXBUFFER pShader, LPD3DXBUFFER pListing, HRESULT hr );
+			~CResponse( void );
+
+		public:
+			virtual bool Succeeded( void ) { return m_pShader && (m_hr == D3D_OK); }
+			virtual size_t GetResultBufferLen( void ) { return ( Succeeded() ? m_pShader->GetBufferSize() : 0 ); }
+			virtual const void * GetResultBuffer( void ) { return ( Succeeded() ? m_pShader->GetBufferPointer() : NULL ); }
+			virtual const char * GetListing( void ) { return (const char *) ( m_pListing ? m_pListing->GetBufferPointer() : NULL ); }
+
+		protected:
+			LPD3DXBUFFER m_pShader;
+			LPD3DXBUFFER m_pListing;
+			HRESULT m_hr;
+		};
+
+		CResponse::CResponse( LPD3DXBUFFER pShader, LPD3DXBUFFER pListing, HRESULT hr ) :
+			m_pShader(pShader),
+			m_pListing(pListing),
+			m_hr(hr)
+		{
+			NULL;
+		}
+
+		CResponse::~CResponse( void )
+		{
+			if ( m_pShader )
+				m_pShader->Release();
+
+			if ( m_pListing )
+				m_pListing->Release();
+		}
+
+		//
+		// Perform a fast shader file compilation.
+		//		TODO: avoid writing "shader.o" and "output.txt" files to avoid extra filesystem access.
+		//
+		// @param pszFilename		the filename to compile (e.g. "debugdrawenvmapmask_vs20.fxc")
+		// @param pMacros			null-terminated array of macro-defines
+		// @param pszModel			shader model for compilation
+		//
+		void FastShaderCompile( const char *pszFilename, const D3DXMACRO *pMacros, const char *pszModel, CmdSink::IResponse **ppResponse )
+		{
+			LPD3DXBUFFER pShader = NULL; // NOTE: Must release the COM interface later
+			LPD3DXBUFFER pErrorMessages = NULL; // NOTE: Must release COM interface later
+			
+			// DxProxyModule
+			static DxProxyModule s_dxModule;
+
+			// X360TEMP: This needs to be moved to an external semantic (or fixed)
+			bool bIsX360 = false;
+			for ( int i=0; ;i++ )
+			{
+				if ( !pMacros[i].Name )
+				{
+					break;
+				}
+				if ( V_stristr( pMacros[i].Name, "_X360" ) && atoi( pMacros[i].Definition ) )
+				{
+					bIsX360 = true;
+					break;
+				}
+			}
+			
+			HRESULT hr = s_dxModule.D3DXCompileShaderFromFile( pszFilename, pMacros, NULL /* LPD3DXINCLUDE */,
+				"main",	pszModel, 0, &pShader, &pErrorMessages,
+				NULL /* LPD3DXCONSTANTTABLE *ppConstantTable */ );
+
+			if ( ppResponse )
+			{
+				*ppResponse = new CResponse( pShader, pErrorMessages, hr );
+			}
+			else
+			{
+				if ( pShader )
+				{
+					pShader->Release();
+				}
+
+				if ( pErrorMessages )
+				{
+					pErrorMessages->Release();
+				}
+			}
+		}
+
+	}; // namespace Private
+
+	//
+	// Completely mimic the behaviour of "fxc.exe" in the specific cases related
+	// to shader compilations.
+	//
+	// @param pCommand       the command in form
+	//		"fxc.exe /DSHADERCOMBO=1 /DTOTALSHADERCOMBOS=4 /DCENTROIDMASK=0 /DNUMDYNAMICCOMBOS=4 /DFLAGS=0x0 /DNUM_BONES=1 /Dmain=main /Emain /Tvs_2_0 /DSHADER_MODEL_VS_2_0=1 /D_X360=1 /nologo /Foshader.o debugdrawenvmapmask_vs20.fxc>output.txt 2>&1"
+	//
+	void ExecuteCommand( const char *pCommand, CmdSink::IResponse **ppResponse )
+	{
+		// Expect that the command passed is exactly "fxc.exe"
+		Assert( !strncmp( pCommand, s_pszCommand, s_uCommandLen ) );
+		pCommand += s_uCommandLen;
+
+		// A duplicate portion of memory for modifications
+		void *bufEditableCommand = alloca( strlen( pCommand ) + 1 );
+		char *pEditableCommand = strcpy( (char *) bufEditableCommand, pCommand );
+
+		// Macros to be defined for D3DX
+		CUtlVector<D3DXMACRO> macros;
+
+		// Shader model (determined when parsing "/D" flags)
+		const char *pszShaderModel = NULL;
+
+		// Iterate over the command line and find all "/D...=..." settings
+		for ( char *pszFlag = pEditableCommand;
+			( pszFlag = strstr( pszFlag, "/D" ) ) != NULL;
+			/* advance inside */ )
+		{
+			// Make sure this is a command-line flag (basic check for preceding space)
+			if ( pszFlag > pEditableCommand &&
+				pszFlag[-1] &&
+				' ' != pszFlag[-1] )
+			{
+				++ pszFlag;
+				continue;
+			}
+
+			// Name is immediately after "/D"
+			char *pszFlagName = pszFlag + 2; // 2 = length of "/D"
+			// Value will be determined later
+			char *pszValue = "";
+
+			if ( char *pchEq = strchr( pszFlag, '=' ) )
+			{
+				// Value is after '=' sign
+				*pchEq = 0;
+				pszValue = pchEq + 1;
+				pszFlag = pszValue;
+			}
+
+			if ( char *pchSpace = strchr( pszFlag, ' ' ) )
+			{
+				// Space is designating the end of the flag
+				*pchSpace = 0;
+				pszFlag = pchSpace + 1;
+			}
+			else
+			{
+				// Reached end of command line
+				pszFlag = "";
+			}
+
+			// Shader model extraction
+			if ( !strncmp(pszFlagName, "SHADER_MODEL_", 13) )
+			{
+				pszShaderModel = pszFlagName + 13;
+			}
+
+			// Add the macro definition to the macros array
+			int iMacroIdx = macros.AddToTail();
+			D3DXMACRO &m = macros[iMacroIdx];
+
+			// Fill the macro data
+			m.Name = pszFlagName;
+			m.Definition = pszValue;
+		}
+
+		// Add a NULL-terminator
+		{
+			D3DXMACRO nullTerminatorMacro = { NULL, NULL };
+			macros.AddToTail( nullTerminatorMacro );
+		}
+
+		// Convert shader model to lowercase
+		char chShaderModel[20] = {0};
+		if(pszShaderModel)
+		{
+			Q_strncpy( chShaderModel, pszShaderModel, sizeof(chShaderModel) - 1 );
+		}
+		Q_strlower( chShaderModel );
+
+		// Determine the file name (at the end of the command line before redirection)
+		char const *pszFilename = "";
+		if ( const char *pchCmdRedirect = strstr( pCommand, ">output.txt " ) )
+		{
+			size_t uCmdEndOffset = ( pchCmdRedirect - pCommand );
+
+			pEditableCommand[uCmdEndOffset] = 0;
+			pszFilename = &pEditableCommand[uCmdEndOffset];
+
+			while ( pszFilename > pEditableCommand &&
+				pszFilename[-1] &&
+				' ' != pszFilename[-1] )
+			{
+				-- pszFilename;
+			}
+		}
+
+		// Compile the stuff
+		Private::FastShaderCompile( pszFilename, macros.Base(), chShaderModel, ppResponse );
+	}
+
+	bool TryExecuteCommand( const char *pCommand, CmdSink::IResponse **ppResponse )
+	{
+		{
+			static bool s_bNoIntercept = ( CommandLine()->FindParm("-nointercept") != 0 );
+			static int s_dummy = ( Msg( s_bNoIntercept ?
+				"[shadercompile] Using old slow technique - runs 'fxc.exe'.\n" :
+				"[shadercompile] Using new faster Vitaliy's implementation.\n" ), 1 );
+			if ( !s_bNoIntercept && !strncmp(pCommand, InterceptFxc::s_pszCommand, InterceptFxc::s_uCommandLen) )
+			{
+				// Trap "fxc.exe" so that we did not spawn extra process every time
+				InterceptFxc::ExecuteCommand( pCommand, ppResponse );
+				return true;
+			}
+		}
+
+		return false;
+	}
+
+}; // namespace InterceptFxc
+
+
+
+

utils/shadercompile/d3dxfxc.h

@@ -0,0 +1,24 @@
+//========= Copyright Valve Corporation, All rights reserved. ============//
+//
+// Purpose: D3DX command implementation.
+//
+// $NoKeywords: $
+//
+//=============================================================================//
+
+#ifndef D3DXFXC_H
+#define D3DXFXC_H
+#ifdef _WIN32
+#pragma once
+#endif
+
+#include "cmdsink.h"
+
+namespace InterceptFxc
+{
+
+	bool TryExecuteCommand( const char *pCommand, CmdSink::IResponse **ppResponse );
+
+}; // namespace InterceptFxc
+
+#endif // #ifndef D3DXFXC_H

utils/shadercompile/shadercompile.h

@@ -0,0 +1,11 @@
+
+//========= Copyright Valve Corporation, All rights reserved. ============//
+//
+// Purpose: Module prototypes.
+//
+// $NoKeywords: $
+//
+//=============================================================================//
+
+void DebugOut( const char *pMsg, ... );
+void DebugSafeWaitPoint( bool bForceWait = false );

utils/shadercompile/shadercompile_dll.vpc

@@ -0,0 +1,60 @@
+//-----------------------------------------------------------------------------
+//	SHADERCOMPILE_DLL.VPC
+//
+//	Project Script
+//-----------------------------------------------------------------------------
+
+$Macro SRCDIR		"..\.."
+$Macro OUTBINDIR	"$SRCDIR\..\game\bin"
+
+$Include "$SRCDIR\vpc_scripts\source_dll_base.vpc"
+
+$Configuration
+{
+	$Compiler
+	{
+		$AdditionalIncludeDirectories		"$BASE,..\common;..\vmpi;$SRCDIR\dx9sdk\include"
+		$PreprocessorDefinitions			"$BASE;SHADERCOMPILE_EXPORTS;MPI"
+	}
+
+	$Linker
+	{
+		$AdditionalDependencies				"$BASE ws2_32.lib odbc32.lib odbccp32.lib"
+	}
+}
+
+$Project "Shadercompile_dll"
+{
+	$Folder	"Source Files"
+	{
+		$File	"..\common\cmdlib.cpp"
+		$File	"cmdsink.cpp"
+		$File	"d3dxfxc.cpp"
+		$File	"$SRCDIR\public\filesystem_helpers.cpp"
+		$File	"..\common\pacifier.cpp"
+		$File	"shadercompile.cpp"
+		$File	"subprocess.cpp"
+		$File	"cfgprocessor.cpp"
+		$File	"..\common\threads.cpp"
+		$File	"..\common\vmpi_tools_shared.cpp"
+		$File	"..\common\tools_minidump.cpp"
+	}
+
+	$Folder	"Header Files"
+	{
+		$File	"cmdsink.h"
+		$File	"d3dxfxc.h"
+		$File	"$SRCDIR\public\ishadercompiledll.h"
+		$File	"shadercompile.h"
+		$File	"utlnodehash.h"
+		$File	"cfgprocessor.h"
+		$File	"$SRCDIR\public\tier1\UtlStringMap.h"
+	}
+
+	$Folder	"Link Libraries"
+	{
+		$Lib tier2
+		$Lib vmpi
+		$Lib $LIBCOMMON\lzma
+	}
+}

utils/shadercompile/subprocess.cpp

@@ -0,0 +1,303 @@
+//========= Copyright Valve Corporation, All rights reserved. ============//
+//
+// Purpose: 
+//
+// $NoKeywords: $
+//
+//=============================================================================//
+
+#include <windows.h>
+
+#include "cmdsink.h"
+
+#include "subprocess.h"
+
+#include "d3dxfxc.h"
+
+#include "tools_minidump.h"
+
+//////////////////////////////////////////////////////////////////////////
+//
+// Base implementation of the shaderd kernel objects
+//
+//////////////////////////////////////////////////////////////////////////
+
+SubProcessKernelObjects::SubProcessKernelObjects( void ) :
+	m_hMemorySection( NULL ),
+	m_hMutex( NULL )
+{
+	ZeroMemory( m_hEvent, sizeof( m_hEvent ) );
+}
+
+SubProcessKernelObjects::~SubProcessKernelObjects( void )
+{
+	Close();
+}
+
+BOOL SubProcessKernelObjects::Create( char const *szBaseName )
+{
+	char chBufferName[0x100] = { 0 };
+	
+	sprintf( chBufferName, "%s_msec", szBaseName );
+	m_hMemorySection = CreateFileMapping( INVALID_HANDLE_VALUE, NULL,
+		PAGE_READWRITE, 0, 4 * 1024 * 1024, chBufferName ); // 4Mb for a piece
+	if ( NULL != m_hMemorySection )
+	{
+		if ( ERROR_ALREADY_EXISTS == GetLastError() )
+		{
+			CloseHandle( m_hMemorySection );
+			m_hMemorySection = NULL;
+
+			Assert( 0 && "CreateFileMapping - already exists!\n" );
+		}
+	}
+
+	sprintf( chBufferName, "%s_mtx", szBaseName );
+	m_hMutex = CreateMutex( NULL, FALSE, chBufferName );
+
+	for ( int k = 0; k < 2; ++ k )
+	{
+		sprintf( chBufferName, "%s_evt%d", szBaseName, k );
+		m_hEvent[k] = CreateEvent( NULL, FALSE, ( k ? TRUE /* = master */ : FALSE ), chBufferName );
+	}
+
+	return IsValid();
+}
+
+BOOL SubProcessKernelObjects::Open( char const *szBaseName )
+{
+	char chBufferName[0x100] = { 0 };
+
+	sprintf( chBufferName, "%s_msec", szBaseName );
+	m_hMemorySection = OpenFileMapping( FILE_MAP_ALL_ACCESS, FALSE, chBufferName );
+
+	sprintf( chBufferName, "%s_mtx", szBaseName );
+	m_hMutex = OpenMutex( MUTEX_ALL_ACCESS, FALSE, chBufferName );
+
+	for ( int k = 0; k < 2; ++ k )
+	{
+		sprintf( chBufferName, "%s_evt%d", szBaseName, k );
+		m_hEvent[k] = OpenEvent( EVENT_ALL_ACCESS, FALSE, chBufferName );
+	}
+
+	return IsValid();
+}
+
+BOOL SubProcessKernelObjects::IsValid( void ) const
+{
+	return m_hMemorySection && m_hMutex && m_hEvent;
+}
+
+void SubProcessKernelObjects::Close( void )
+{
+	if ( m_hMemorySection )
+		CloseHandle( m_hMemorySection );
+
+	if ( m_hMutex )
+		CloseHandle( m_hMutex );
+
+	for ( int k = 0; k < 2; ++ k )
+		if ( m_hEvent[k] )
+			CloseHandle( m_hEvent[k] );
+}
+
+//////////////////////////////////////////////////////////////////////////
+//
+// Helper class to send data back and forth
+//
+//////////////////////////////////////////////////////////////////////////
+
+void * SubProcessKernelObjects_Memory::Lock( void )
+{
+	// Wait for our turn to act
+	for ( unsigned iWaitAttempt = 0; iWaitAttempt < 13u; ++ iWaitAttempt )
+	{
+		DWORD dwWait = ::WaitForSingleObject( m_pObjs->m_hEvent[ m_pObjs->m_dwCookie ], 10000 );
+		switch ( dwWait )
+		{
+		case WAIT_OBJECT_0:
+			{
+				m_pLockData = MapViewOfFile( m_pObjs->m_hMemorySection, FILE_MAP_ALL_ACCESS, 0, 0, 0 );
+				
+				if ( * ( const DWORD * ) m_pLockData != m_pObjs->m_dwCookie )
+				{
+					// Yes, this is our turn, set our cookie in that memory segment
+					* ( DWORD * ) m_pLockData = m_pObjs->m_dwCookie;
+					m_pMemory = ( ( byte * ) m_pLockData ) + 2 * sizeof( DWORD );
+					
+					return m_pMemory;
+				}
+				else
+				{
+					// We just acted, still waiting for result
+					UnmapViewOfFile( m_pLockData );
+					m_pLockData = NULL;
+					
+					SetEvent( m_pObjs->m_hEvent[ !m_pObjs->m_dwCookie ] );
+					Sleep( 1 );
+					
+					continue;
+				}
+			}
+			break;
+		
+		case WAIT_TIMEOUT:
+			{
+				char chMsg[0x100];
+				sprintf( chMsg, "th%08X> WAIT_TIMEOUT in Memory::Lock (attempt %d).\n", GetCurrentThreadId(), iWaitAttempt );
+				OutputDebugString( chMsg );
+			}
+			continue; // retry
+
+		default:
+			OutputDebugString( "WAIT failure in Memory::Lock\n" );
+			SetLastError( ERROR_BAD_UNIT );
+			return NULL;
+		}
+	}
+	
+	OutputDebugString( "Ran out of wait attempts in Memory::Lock\n" );
+	SetLastError( ERROR_NOT_READY );
+	return NULL;
+}
+
+BOOL SubProcessKernelObjects_Memory::Unlock( void )
+{
+	if ( m_pLockData )
+	{
+		// Assert that the memory hasn't been spoiled
+		Assert( m_pObjs->m_dwCookie == * ( const DWORD * ) m_pLockData );
+		
+		UnmapViewOfFile( m_pLockData );
+		m_pMemory = NULL;
+		m_pLockData = NULL;
+		
+		SetEvent( m_pObjs->m_hEvent[ !m_pObjs->m_dwCookie ] );
+		Sleep( 1 );
+
+		return TRUE;
+	}
+
+	return FALSE;
+}
+
+
+//////////////////////////////////////////////////////////////////////////
+//
+// Implementation of the command subprocess:
+//
+// MASTER      ---- command ------->     SUB
+//				string		-	zero terminated command string.
+//
+//
+// MASTER     <---- result --------      SUB
+//				dword		-	1 if succeeded, 0 if failed
+//				dword		-	result buffer length, 0 if failed
+//				<bytes>		-	result buffer data, none if result buffer length is 0
+//				string		-	zero-terminated listing string
+//
+//////////////////////////////////////////////////////////////////////////
+
+
+CSubProcessResponse::CSubProcessResponse( void const *pvMemory ) :
+	m_pvMemory( pvMemory )
+{
+	byte const *pBytes = ( byte const * ) pvMemory;
+	
+	m_dwResult = * ( DWORD const * ) pBytes;
+	pBytes += sizeof( DWORD );
+
+	m_dwResultBufferLength = * ( DWORD const * ) pBytes;
+	pBytes += sizeof( DWORD );
+
+	m_pvResultBuffer = pBytes;
+	pBytes += m_dwResultBufferLength;
+
+	m_szListing = ( char const * ) ( *pBytes ? pBytes : NULL );
+}
+
+
+void ShaderCompile_Subprocess_ExceptionHandler( unsigned long exceptionCode, void *pvExceptionInfo )
+{
+	// Subprocesses just silently die in our case, then this case will be detected by the worker process and an error code will be passed to the master
+	Assert( !"ShaderCompile_Subprocess_ExceptionHandler" );
+	::TerminateProcess( ::GetCurrentProcess(), exceptionCode );
+}
+
+
+int ShaderCompile_Subprocess_Main( char const *szSubProcessData )
+{
+	// Set our crash handler
+	SetupToolsMinidumpHandler( ShaderCompile_Subprocess_ExceptionHandler );
+
+	// Get our kernel objects
+	SubProcessKernelObjects_Open objs( szSubProcessData );
+
+	if ( !objs.IsValid() )
+		return -1;
+
+	// Enter the command pumping loop
+	SubProcessKernelObjects_Memory shrmem( &objs );
+	for (
+		void *pvMemory = NULL;
+		NULL != ( pvMemory = shrmem.Lock() );
+		shrmem.Unlock()
+		)
+	{
+		// The memory is actually a command
+		char const *szCommand = ( char const * ) pvMemory;
+		
+		if ( !stricmp( "keepalive", szCommand ) )
+		{
+			ZeroMemory( pvMemory, 4 * sizeof( DWORD ) );
+			continue;
+		}
+
+		if ( !stricmp( "quit", szCommand ) )
+		{
+			ZeroMemory( pvMemory, 4 * sizeof( DWORD ) );
+			return 0;
+		}
+
+		CmdSink::IResponse *pResponse = NULL;
+		if ( InterceptFxc::TryExecuteCommand( szCommand, &pResponse ) )
+		{
+			byte *pBytes = ( byte * ) pvMemory;
+			
+			// Result
+			DWORD dwSucceededResult = pResponse->Succeeded() ? 1 : 0;
+			* ( DWORD * ) pBytes = dwSucceededResult;
+			pBytes += sizeof( DWORD );
+
+			// Result buffer len
+			DWORD dwBufferLength = pResponse->GetResultBufferLen();
+			* ( DWORD * ) pBytes = dwBufferLength;
+			pBytes += sizeof( DWORD );
+
+			// Result buffer
+			const void *pvResultBuffer = pResponse->GetResultBuffer();
+			memcpy( pBytes, pvResultBuffer, dwBufferLength );
+			pBytes += dwBufferLength;
+
+			// Listing - copy string
+			const char *szListing = pResponse->GetListing();
+			if ( szListing )
+			{
+				while ( 0 != ( * ( pBytes ++ ) = * ( szListing ++ ) ) )
+				{
+					NULL;
+				}
+			}
+			else
+			{
+				* ( pBytes ++ ) = 0;
+			}
+		}
+		else
+		{
+			ZeroMemory( pvMemory, 4 * sizeof( DWORD ) );
+		}
+	}
+
+	return -2;
+}

utils/shadercompile/subprocess.h

@@ -0,0 +1,104 @@
+//========= Copyright Valve Corporation, All rights reserved. ============//
+//
+// Purpose: 
+//
+// $NoKeywords: $
+//
+//=============================================================================//
+
+#ifndef SUBPROCESS_H
+#define SUBPROCESS_H
+#ifdef _WIN32
+#pragma once
+#endif
+
+class SubProcessKernelObjects
+{
+	friend class SubProcessKernelObjects_Memory;
+
+public:
+	SubProcessKernelObjects( void );
+	~SubProcessKernelObjects( void );
+
+private:
+	SubProcessKernelObjects( SubProcessKernelObjects const & );
+	SubProcessKernelObjects & operator =( SubProcessKernelObjects const & );
+
+protected:
+	BOOL Create( char const *szBaseName );
+	BOOL Open( char const *szBaseName );
+
+public:
+	BOOL IsValid( void ) const;
+	void Close( void );
+
+protected:
+	HANDLE m_hMemorySection;
+	HANDLE m_hMutex;
+	HANDLE m_hEvent[2];
+	DWORD m_dwCookie;
+};
+
+class SubProcessKernelObjects_Create : public SubProcessKernelObjects
+{
+public:
+	SubProcessKernelObjects_Create( char const *szBaseName ) { Create( szBaseName ), m_dwCookie = 1; }
+};
+
+class SubProcessKernelObjects_Open : public SubProcessKernelObjects
+{
+public:
+	SubProcessKernelObjects_Open( char const *szBaseName ) { Open( szBaseName ), m_dwCookie = 0; }
+};
+
+class SubProcessKernelObjects_Memory
+{
+public:
+	SubProcessKernelObjects_Memory( SubProcessKernelObjects *p ) : m_pObjs( p ), m_pLockData( NULL ), m_pMemory( NULL ) {  }
+	~SubProcessKernelObjects_Memory() { Unlock(); }
+
+public:
+	void * Lock( void );
+	BOOL Unlock( void );
+
+public:
+	BOOL IsValid( void ) const { return m_pLockData != NULL; }
+	void * GetMemory( void ) const { return m_pMemory; }
+
+protected:
+	void *m_pMemory;
+
+private:
+	SubProcessKernelObjects *m_pObjs;
+	void *m_pLockData;
+};
+
+
+//
+// Response implementation
+//
+class CSubProcessResponse : public CmdSink::IResponse
+{
+public:
+	explicit CSubProcessResponse( void const *pvMemory );
+	~CSubProcessResponse( void ) { }
+
+public:
+	virtual bool Succeeded( void ) { return ( 1 == m_dwResult ); }
+	virtual size_t GetResultBufferLen( void ) { return ( Succeeded() ? m_dwResultBufferLength : 0 ); }
+	virtual const void * GetResultBuffer( void ) { return ( Succeeded() ? m_pvResultBuffer : NULL ); }
+	virtual const char * GetListing( void ) { return (const char *) ( ( m_szListing && * m_szListing ) ? m_szListing : NULL ); }
+
+protected:
+	void const *m_pvMemory;
+	DWORD m_dwResult;
+	DWORD m_dwResultBufferLength;
+	void const *m_pvResultBuffer;
+	char const *m_szListing;
+};
+
+
+int ShaderCompile_Subprocess_Main( char const *szSubProcessData );
+
+
+#endif // #ifndef SUBPROCESS_H

utils/shadercompile/utlnodehash.h

@@ -0,0 +1,93 @@
+//========= Copyright Valve Corporation, All rights reserved. ============//
+//
+// Purpose: hashed intrusive linked list.
+//
+// $NoKeywords: $
+//
+// Serialization/unserialization buffer
+//=============================================================================//
+
+#ifndef UTLNODEHASH_H
+#define UTLNODEHASH_H
+
+#ifdef _WIN32
+#pragma once
+#endif
+
+#include "tier1/utlmemory.h"
+#include "tier1/byteswap.h"
+#include "tier1/utlintrusivelist.h"
+
+#include <stdarg.h>
+
+// to use this class, your list node class must have a Key() function defined which returns an
+// integer type. May add this class to main utl tier when i'm happy w/ it.
+template<class T, int HASHSIZE = 7907, class K = int > class CUtlNodeHash
+{
+
+	int m_nNumNodes;
+
+public:
+
+	CUtlIntrusiveDList<T> m_HashChains[HASHSIZE];
+
+	CUtlNodeHash( void )
+	{
+		m_nNumNodes = 0;
+	}
+
+
+	T *FindByKey(K nMatchKey, int *pChainNumber = NULL)
+	{
+		unsigned int nChain=(unsigned int) nMatchKey ;
+		nChain %= HASHSIZE;
+		if ( pChainNumber )
+			*( pChainNumber ) = nChain;
+		for( T * pNode = m_HashChains[ nChain ].m_pHead; pNode; pNode = pNode->m_pNext )
+			if ( pNode->Key() == nMatchKey )
+				return pNode;
+		return NULL;
+	}
+
+	void Add( T * pNode )
+	{
+		unsigned int nChain=(unsigned int) pNode->Key();
+		nChain %= HASHSIZE;
+		m_HashChains[ nChain ].AddToHead( pNode );
+		m_nNumNodes++;
+	}
+
+
+	void Purge( void )
+	{
+		m_nNumNodes = 0;
+		// delete all nodes
+		for( int i=0; i < HASHSIZE; i++)
+			m_HashChains[i].Purge();
+	}
+
+	int Count( void ) const
+	{
+		return m_nNumNodes;
+	}
+
+	void DeleteByKey( K nMatchKey )
+	{
+		int nChain;
+		T *pSearch = FindByKey( nMatchKey, &nChain );
+		if ( pSearch )
+		{
+			m_HashChains[ nChain ].RemoveNode( pSearch );
+			m_nNumNodes--;
+		}
+	}
+
+	~CUtlNodeHash( void )
+	{
+		// delete all lists
+		Purge();
+	}
+};
+
+
+#endif