source-engine/utils/common/vmpi_tools_shared.cpp

375 lines
11 KiB
C++
Raw Permalink Normal View History

2020-04-22 16:56:21 +00:00
//========= Copyright Valve Corporation, All rights reserved. ============//
//
// Purpose:
//
//=============================================================================//
#include <windows.h>
#include <dbghelp.h>
#include "vmpi.h"
#include "cmdlib.h"
#include "vmpi_tools_shared.h"
#include "tier1/strtools.h"
#include "mpi_stats.h"
#include "iphelpers.h"
#include "tier0/minidump.h"
// ----------------------------------------------------------------------------- //
// Globals.
// ----------------------------------------------------------------------------- //
static bool g_bReceivedDirectoryInfo = false; // Have we gotten the qdir info yet?
static bool g_bReceivedDBInfo = false;
static CDBInfo g_DBInfo;
static unsigned long g_JobPrimaryID;
static int g_nDisconnects = 0; // Tracks how many remote processes have disconnected ungracefully.
// ----------------------------------------------------------------------------- //
// Shared dispatch code.
// ----------------------------------------------------------------------------- //
bool SharedDispatch( MessageBuffer *pBuf, int iSource, int iPacketID )
{
char *pInPos = &pBuf->data[2];
switch ( pBuf->data[1] )
{
case VMPI_SUBPACKETID_DIRECTORIES:
{
Q_strncpy( gamedir, pInPos, sizeof( gamedir ) );
pInPos += strlen( pInPos ) + 1;
Q_strncpy( qdir, pInPos, sizeof( qdir ) );
g_bReceivedDirectoryInfo = true;
}
return true;
case VMPI_SUBPACKETID_DBINFO:
{
g_DBInfo = *((CDBInfo*)pInPos);
pInPos += sizeof( CDBInfo );
g_JobPrimaryID = *((unsigned long*)pInPos);
g_bReceivedDBInfo = true;
}
return true;
case VMPI_SUBPACKETID_CRASH:
{
char const chCrashInfoType = *pInPos;
pInPos += 2;
switch ( chCrashInfoType )
{
case 't':
Warning( "\nWorker '%s' dead: %s\n", VMPI_GetMachineName( iSource ), pInPos );
break;
case 'f':
{
int iFileSize = * reinterpret_cast< int const * >( pInPos );
pInPos += sizeof( iFileSize );
// Temp folder
char const *szFolder = NULL;
if ( !szFolder ) szFolder = getenv( "TEMP" );
if ( !szFolder ) szFolder = getenv( "TMP" );
if ( !szFolder ) szFolder = "c:";
// Base module name
char chModuleName[_MAX_PATH], *pModuleName = chModuleName;
::GetModuleFileName( NULL, chModuleName, sizeof( chModuleName ) / sizeof( chModuleName[0] ) );
if ( char *pch = strrchr( chModuleName, '.' ) )
*pch = 0;
if ( char *pch = strrchr( chModuleName, '\\' ) )
*pch = 0, pModuleName = pch + 1;
// Current time
time_t currTime = ::time( NULL );
struct tm * pTime = ::localtime( &currTime );
// Number of minidumps this run
static int s_numMiniDumps = 0;
++ s_numMiniDumps;
// Prepare the filename
char chSaveFileName[ 2 * _MAX_PATH ] = { 0 };
sprintf( chSaveFileName, "%s\\vmpi_%s_on_%s_%d%.2d%2d%.2d%.2d%.2d_%d.mdmp",
szFolder,
pModuleName,
VMPI_GetMachineName( iSource ),
pTime->tm_year + 1900, /* Year less 2000 */
pTime->tm_mon + 1, /* month (0 - 11 : 0 = January) */
pTime->tm_mday, /* day of month (1 - 31) */
pTime->tm_hour, /* hour (0 - 23) */
pTime->tm_min, /* minutes (0 - 59) */
pTime->tm_sec, /* seconds (0 - 59) */
s_numMiniDumps
);
if ( FILE *fDump = fopen( chSaveFileName, "wb" ) )
{
fwrite( pInPos, 1, iFileSize, fDump );
fclose( fDump );
Warning( "\nSaved worker crash minidump '%s', size %d byte(s).\n",
chSaveFileName, iFileSize );
}
else
{
Warning( "\nReceived worker crash minidump size %d byte(s), failed to save.\n", iFileSize );
}
}
break;
}
}
return true;
}
return false;
}
CDispatchReg g_SharedDispatchReg( VMPI_SHARED_PACKET_ID, SharedDispatch );
// ----------------------------------------------------------------------------- //
// Module interfaces.
// ----------------------------------------------------------------------------- //
void SendQDirInfo()
{
char cPacketID[2] = { VMPI_SHARED_PACKET_ID, VMPI_SUBPACKETID_DIRECTORIES };
MessageBuffer mb;
mb.write( cPacketID, 2 );
mb.write( gamedir, strlen( gamedir ) + 1 );
mb.write( qdir, strlen( qdir ) + 1 );
VMPI_SendData( mb.data, mb.getLen(), VMPI_PERSISTENT );
}
void RecvQDirInfo()
{
while ( !g_bReceivedDirectoryInfo )
VMPI_DispatchNextMessage();
}
void SendDBInfo( const CDBInfo *pInfo, unsigned long jobPrimaryID )
{
char cPacketInfo[2] = { VMPI_SHARED_PACKET_ID, VMPI_SUBPACKETID_DBINFO };
const void *pChunks[] = { cPacketInfo, pInfo, &jobPrimaryID };
int chunkLengths[] = { 2, sizeof( CDBInfo ), sizeof( jobPrimaryID ) };
VMPI_SendChunks( pChunks, chunkLengths, ARRAYSIZE( pChunks ), VMPI_PERSISTENT );
}
void RecvDBInfo( CDBInfo *pInfo, unsigned long *pJobPrimaryID )
{
while ( !g_bReceivedDBInfo )
VMPI_DispatchNextMessage();
*pInfo = g_DBInfo;
*pJobPrimaryID = g_JobPrimaryID;
}
// If the file is successfully opened, read and sent returns the size of the file in bytes
// otherwise returns 0 and nothing is sent
int VMPI_SendFileChunk( const void *pvChunkPrefix, int lenPrefix, tchar const *ptchFileName )
{
HANDLE hFile = NULL;
HANDLE hMapping = NULL;
void const *pvMappedData = NULL;
int iResult = 0;
hFile = ::CreateFile( ptchFileName, GENERIC_READ, 0, NULL, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL );
if ( !hFile || ( hFile == INVALID_HANDLE_VALUE ) )
goto done;
hMapping = ::CreateFileMapping( hFile, NULL, PAGE_READONLY, 0, 0, NULL );
if ( !hMapping || ( hMapping == INVALID_HANDLE_VALUE ) )
goto done;
pvMappedData = ::MapViewOfFile( hMapping, FILE_MAP_READ, 0, 0, 0 );
if ( !pvMappedData )
goto done;
int iMappedFileSize = ::GetFileSize( hFile, NULL );
if ( INVALID_FILE_SIZE == iMappedFileSize )
goto done;
// Send the data over VMPI
if ( VMPI_Send3Chunks(
pvChunkPrefix, lenPrefix,
&iMappedFileSize, sizeof( iMappedFileSize ),
pvMappedData, iMappedFileSize,
VMPI_MASTER_ID ) )
iResult = iMappedFileSize;
// Fall-through for cleanup code to execute
done:
if ( pvMappedData )
::UnmapViewOfFile( pvMappedData );
if ( hMapping && ( hMapping != INVALID_HANDLE_VALUE ) )
::CloseHandle( hMapping );
if ( hFile && ( hFile != INVALID_HANDLE_VALUE ) )
::CloseHandle( hFile );
return iResult;
}
void VMPI_HandleCrash( const char *pMessage, void *pvExceptionInfo, bool bAssert )
{
static LONG crashHandlerCount = 0;
if ( InterlockedIncrement( &crashHandlerCount ) == 1 )
{
Msg( "\nFAILURE: '%s' (assert: %d)\n", pMessage, bAssert );
// Send a message to the master.
char crashMsg[4] = { VMPI_SHARED_PACKET_ID, VMPI_SUBPACKETID_CRASH, 't', ':' };
VMPI_Send2Chunks(
crashMsg,
sizeof( crashMsg ),
pMessage,
strlen( pMessage ) + 1,
VMPI_MASTER_ID );
// Now attempt to create a minidump with the given exception information
if ( pvExceptionInfo )
{
struct _EXCEPTION_POINTERS *pvExPointers = ( struct _EXCEPTION_POINTERS * ) pvExceptionInfo;
tchar tchMinidumpFileName[_MAX_PATH] = { 0 };
bool bSucceededWritingMinidump = WriteMiniDumpUsingExceptionInfo(
pvExPointers->ExceptionRecord->ExceptionCode,
pvExPointers,
( MINIDUMP_TYPE )( MiniDumpWithDataSegs | MiniDumpWithIndirectlyReferencedMemory | MiniDumpWithProcessThreadData ),
// ( MINIDUMP_TYPE )( MiniDumpWithDataSegs | MiniDumpWithFullMemory | MiniDumpWithHandleData | MiniDumpWithUnloadedModules | MiniDumpWithIndirectlyReferencedMemory | MiniDumpWithProcessThreadData | MiniDumpWithPrivateReadWriteMemory ),
// ( MINIDUMP_TYPE )( MiniDumpNormal ),
tchMinidumpFileName );
if ( bSucceededWritingMinidump )
{
crashMsg[2] = 'f';
VMPI_SendFileChunk( crashMsg, sizeof( crashMsg ), tchMinidumpFileName );
::DeleteFile( tchMinidumpFileName );
}
}
// Let the messages go out.
Sleep( 500 );
}
InterlockedDecrement( &crashHandlerCount );
}
// This is called if we crash inside our crash handler. It just terminates the process immediately.
LONG __stdcall VMPI_SecondExceptionFilter( struct _EXCEPTION_POINTERS *ExceptionInfo )
{
TerminateProcess( GetCurrentProcess(), 2 );
return EXCEPTION_EXECUTE_HANDLER; // (never gets here anyway)
}
void VMPI_ExceptionFilter( unsigned long uCode, void *pvExceptionInfo )
{
// This is called if we crash inside our crash handler. It just terminates the process immediately.
SetUnhandledExceptionFilter( VMPI_SecondExceptionFilter );
//DWORD code = ExceptionInfo->ExceptionRecord->ExceptionCode;
#define ERR_RECORD( name ) { name, #name }
struct
{
int code;
char *pReason;
} errors[] =
{
ERR_RECORD( EXCEPTION_ACCESS_VIOLATION ),
ERR_RECORD( EXCEPTION_ARRAY_BOUNDS_EXCEEDED ),
ERR_RECORD( EXCEPTION_BREAKPOINT ),
ERR_RECORD( EXCEPTION_DATATYPE_MISALIGNMENT ),
ERR_RECORD( EXCEPTION_FLT_DENORMAL_OPERAND ),
ERR_RECORD( EXCEPTION_FLT_DIVIDE_BY_ZERO ),
ERR_RECORD( EXCEPTION_FLT_INEXACT_RESULT ),
ERR_RECORD( EXCEPTION_FLT_INVALID_OPERATION ),
ERR_RECORD( EXCEPTION_FLT_OVERFLOW ),
ERR_RECORD( EXCEPTION_FLT_STACK_CHECK ),
ERR_RECORD( EXCEPTION_FLT_UNDERFLOW ),
ERR_RECORD( EXCEPTION_ILLEGAL_INSTRUCTION ),
ERR_RECORD( EXCEPTION_IN_PAGE_ERROR ),
ERR_RECORD( EXCEPTION_INT_DIVIDE_BY_ZERO ),
ERR_RECORD( EXCEPTION_INT_OVERFLOW ),
ERR_RECORD( EXCEPTION_INVALID_DISPOSITION ),
ERR_RECORD( EXCEPTION_NONCONTINUABLE_EXCEPTION ),
ERR_RECORD( EXCEPTION_PRIV_INSTRUCTION ),
ERR_RECORD( EXCEPTION_SINGLE_STEP ),
ERR_RECORD( EXCEPTION_STACK_OVERFLOW ),
ERR_RECORD( EXCEPTION_ACCESS_VIOLATION ),
};
int nErrors = sizeof( errors ) / sizeof( errors[0] );
int i=0;
char *pchReason = NULL;
char chUnknownBuffer[32];
for ( i; ( i < nErrors ) && !pchReason; i++ )
{
if ( errors[i].code == uCode )
pchReason = errors[i].pReason;
}
if ( i == nErrors )
{
sprintf( chUnknownBuffer, "Error code 0x%08X", uCode );
pchReason = chUnknownBuffer;
}
VMPI_HandleCrash( pchReason, pvExceptionInfo, true );
TerminateProcess( GetCurrentProcess(), 1 );
}
void HandleMPIDisconnect( int procID, const char *pReason )
{
int nLiveWorkers = VMPI_GetCurrentNumberOfConnections() - g_nDisconnects - 1;
// We ran into the size limit before and it wasn't readily apparent that the size limit had
// been breached, so make sure to show errors about invalid packet sizes..
bool bOldSuppress = g_bSuppressPrintfOutput;
g_bSuppressPrintfOutput = ( Q_stristr( pReason, "invalid packet size" ) == 0 );
Warning( "\n\n--- WARNING: lost connection to '%s' (%s).\n", VMPI_GetMachineName( procID ), pReason );
if ( g_bMPIMaster )
{
Warning( "%d workers remain.\n\n", nLiveWorkers );
++g_nDisconnects;
/*
if ( VMPI_GetCurrentNumberOfConnections() - g_nDisconnects <= 1 )
{
Error( "All machines disconnected!" );
}
*/
}
else
{
VMPI_HandleAutoRestart();
Error( "Worker quitting." );
}
g_bSuppressPrintfOutput = bOldSuppress;
}