??承接上一篇【渲染逆向】Hook D3D API.

DX11的Hook流程

??程序的入口點(diǎn)(DllMain)在win32_libentry.cpp中

BOOL APIENTRY DllMain(HMODULE hModule, DWORD ul_reason_for_call, LPVOID lpReserved)
{
  if(ul_reason_for_call == DLL_PROCESS_ATTACH)
  {
    BOOL ret = add_hooks();
    SetLastError(0);
    return ret;
  }

  return TRUE;
}

??在函數(shù)add_hooks的結(jié)尾會(huì)調(diào)用LibraryHooks::RegisterHooks()

void LibraryHooks::RegisterHooks()
{
  BeginHookRegistration();

  for(LibraryHook *lib : LibList())
    lib->RegisterHooks();

  EndHookRegistration();
}

??LibList函數(shù)返回LibraryHook對(duì)象的列表，而向列表中加入對(duì)象，是LibraryHook調(diào)用構(gòu)造函數(shù)的時(shí)候。

LibraryHook::LibraryHook()
{
  LibList().push_back(this);
}

??顯然LibraryHook是一個(gè)基類(lèi)，會(huì)有超級(jí)多需要Hook的庫(kù)來(lái)繼承這一類(lèi)，而我們現(xiàn)在關(guān)心的是DX11的庫(kù)。

class D3D11Hook : LibraryHook
{
public:
  void RegisterHooks();
private:
  static D3D11Hook d3d11hooks;
//something....
}

??可以發(fā)現(xiàn)D3D11Hooks有一個(gè)靜態(tài)成員，當(dāng)這個(gè)類(lèi)型初始化時(shí)，會(huì)自動(dòng)生成一個(gè)D3D11Hook對(duì)象，并調(diào)用父類(lèi)LibraryHook的構(gòu)造函數(shù)，將自身加入Lib列表中。
??當(dāng)函數(shù)add_hooks遍歷列表，調(diào)用RegisterHooks時(shí)，會(huì)調(diào)用D3D11Hook::RegisterHooks。

void RegisterHooks()
{
  //something....

  CreateDevice.Register("d3d11.dll", "D3D11CreateDevice", D3D11CreateDevice_hook);
  CreateDeviceAndSwapChain.Register("d3d11.dll", "D3D11CreateDeviceAndSwapChain",
                                    D3D11CreateDeviceAndSwapChain_hook);
}

??這里會(huì)調(diào)用HookedFunction::Register，然后調(diào)用LibraryHooks::RegisterFunctionHook，將Hook所需的數(shù)據(jù)（dll庫(kù)名稱(chēng)、函數(shù)名稱(chēng)、原始函數(shù)地址、hook函數(shù)地址）記錄到數(shù)據(jù)結(jié)構(gòu)中，之后統(tǒng)一用IAT hook進(jìn)程函數(shù)hook，如果有需要，可以在這里修改為其他hook類(lèi)型，例如：

void LibraryHooks::RegisterFunctionHook(const char *libraryName, const FunctionHook &hook)
{
  //something...

  if(!_stricmp(libraryName, "d3d11.dll") || !_stricmp(libraryName, "dxgi.dll"))
  {
    std::cout << "inline hook " << libraryName
              << "\t-\t" << hook.function.c_str() << std::endl;
    *hook.orig = GetProcAddress(GetModuleHandleA(libraryName), hook.function.c_str());

     DetourTransactionBegin();
     DetourUpdateThread(GetCurrentThread());
     DetourAttach((PVOID *)hook.orig, hook.hook);

     DetourTransactionCommit();
  }
  else
  {
    s_HookData->DllHooks[strlower(rdcstr(libraryName))].FunctionHooks.push_back(hook);
  }
}

??這里我就修改了一下，假如是d3d11.dll或dxgi.dll的函數(shù)，則進(jìn)行inline hook，而其他庫(kù)函數(shù)則保持原樣。
??dxgi.dll的Hook同理:

class DXGIHook : LibraryHook
{
public:
  void RegisterHooks()
  {
    LibraryHooks::RegisterLibraryHook("dxgi.dll", NULL);

    CreateDXGIFactory.Register("dxgi.dll", "CreateDXGIFactory", CreateDXGIFactory_hook);
    CreateDXGIFactory1.Register("dxgi.dll", "CreateDXGIFactory1", CreateDXGIFactory1_hook);
    CreateDXGIFactory2.Register("dxgi.dll", "CreateDXGIFactory2", CreateDXGIFactory2_hook);
    GetDebugInterface.Register("dxgi.dll", "DXGIGetDebugInterface", DXGIGetDebugInterface_hook);
    GetDebugInterface1.Register("dxgi.dll", "DXGIGetDebugInterface1", DXGIGetDebugInterface1_hook);
  }

??這些都是Hook需要查詢(xún)地址的函數(shù)，而類(lèi)似SetRenderTargets之類(lèi)的虛表函數(shù)的Hook方式，放到下一節(jié)來(lái)找。

DX11截幀流程

??RenderDoc按鍵或調(diào)用提供的api截幀都是調(diào)用core.h中的RenderDoc::TriggerCapture(uint32_t numFrames)函數(shù)記錄需要截幾幀。

void TriggerCapture(uint32_t numFrames) { m_Cap = numFrames; }

??同文件下又提供了另一個(gè)方法bool RenderDoc::ShouldTriggerCapture(uint32_t frameNumber)。

bool RenderDoc::ShouldTriggerCapture(uint32_t frameNumber)
{
  bool ret = m_Cap > 0;

  if(m_Cap > 0)
    m_Cap--;
//something....
  return ret;

??繼續(xù)查找引用可以找到一個(gè)相對(duì)有用的函數(shù)——WrappedID3D11Device::Present，這個(gè)函數(shù)在結(jié)尾調(diào)用的ShouldTriggerCapture，并且這個(gè)函數(shù)中還可以調(diào)整OverlayText(RenderDoc左上角的覆蓋層)。

//在WrappedID3D11Device::Present中
if(IsBackgroundCapturing(m_State) && RenderDoc::Inst().ShouldTriggerCapture(m_FrameCounter))
{
  RenderDoc::Inst().StartFrameCapture((ID3D11Device *)this, swapper->GetHWND());

  m_AppControlledCapture = false;
  m_CapturedFrames.back().frameNumber = m_FrameCounter;
}

void RenderDoc::StartFrameCapture(void *dev, void *wnd)
{
  IFrameCapturer *frameCap = MatchFrameCapturer(dev, wnd);
  if(frameCap)
  {
    frameCap->StartFrameCapture(dev, wnd);
    m_CapturesActive++;
  }
}

??frameCap是IFrameCapturer的指針，IFrameCapturer是個(gè)接口，而實(shí)現(xiàn)這個(gè)接口的是WrappedID3D11Device。

void WrappedID3D11Device::StartFrameCapture(void *dev, void *wnd)
{
//something...
  m_State = CaptureState::ActiveCapturing;
//something...
}

??在這里激活了當(dāng)前抓取狀態(tài)，core.h中封裝了一層函數(shù)：

constexpr inline bool IsActiveCapturing(CaptureState state)
{
  return state == CaptureState::ActiveCapturing;
}

??當(dāng)程序調(diào)用渲染api時(shí)，會(huì)進(jìn)入到RenderDoc定義的Hook函數(shù)中，在調(diào)用原有函數(shù)后，會(huì)檢查截幀狀態(tài)，假如處于截幀狀態(tài)，則記錄API，這就是抓幀的實(shí)現(xiàn)，貼一個(gè)例子，下面是一個(gè)Wrap(包裝)好的api：

void WrappedID3D11DeviceContext::IASetPrimitiveTopology(D3D11_PRIMITIVE_TOPOLOGY Topology)
{
  SCOPED_LOCK_OPTIONAL(m_pDevice->D3DLock(), m_pDevice->D3DThreadSafe());

  DrainAnnotationQueue();

  m_EmptyCommandList = false;
  //調(diào)用原有函數(shù)
  SERIALISE_TIME_CALL(m_pRealContext->IASetPrimitiveTopology(Topology));
  //如果處于截幀狀態(tài)，則序列化API和參數(shù)
  if(IsActiveCapturing(m_State))
  {
    USE_SCRATCH_SERIALISER();
    SCOPED_SERIALISE_CHUNK(D3D11Chunk::IASetPrimitiveTopology);
    SERIALISE_ELEMENT(m_ResourceID).Named("Context"_lit).TypedAs("ID3D11DeviceContext *"_lit);
    Serialise_IASetPrimitiveTopology(GET_SERIALISER, Topology);

    m_ContextRecord->AddChunk(scope.Get());
  }

  m_CurrentPipelineState->Change(m_CurrentPipelineState->IA.Topo, Topology);
  VerifyState();
}

??借此我們對(duì)這些Wrap的函數(shù)產(chǎn)生興趣，例如WrappedID3D11Device，查看聲明：

class WrappedID3D11Device : public IFrameCapturer, public ID3DDevice, public ID3D11Device5

??這個(gè)類(lèi)竟然實(shí)現(xiàn)了ID3DDevice這個(gè)DX接口。
??斷點(diǎn)跟蹤下這個(gè)類(lèi)的構(gòu)造函數(shù)調(diào)用堆棧

-D3D11Hook::D3D11CreateDevice_hook
-D3D11Hook::D3D11CreateDeviceAndSwapChain_hook
-D3D11Hook::Create_Internal
-WrappedID3D11Device::WrappedID3D11Device

??第一個(gè)函數(shù)D3D11Hook::D3D11CreateDevice_hook是RenderDoc的hook函數(shù)，本文的第一節(jié)已經(jīng)提到了；在D3D11Hook::Create_Internal中，調(diào)用了原始的D3D11CreateDeviceAndSwapChain方法，然后用返回的device構(gòu)造wrap device，最后將wrap返回，由于wrap device同樣實(shí)現(xiàn)了ID3D11Device類(lèi)的接口，因此在使用上和DirectX的Device沒(méi)有區(qū)別：

HRESULT Create_Internal(RealD3D11CreateFunction real, __in_opt IDXGIAdapter *pAdapter,
                        D3D_DRIVER_TYPE DriverType, HMODULE Software, UINT Flags,
                        __in_ecount_opt(FeatureLevels) CONST D3D_FEATURE_LEVEL *pFeatureLevels,
                        UINT FeatureLevels, UINT SDKVersion,
                        __in_opt CONST DXGI_SWAP_CHAIN_DESC *pSwapChainDesc,
                        __out_opt IDXGISwapChain **ppSwapChain, __out_opt ID3D11Device **ppDevice,
                        __out_opt D3D_FEATURE_LEVEL *pFeatureLevel,
                        __out_opt ID3D11DeviceContext **ppImmediateContext)
{
//...
//調(diào)用原有D3D11CreateDeviceAndSwapChain方法
  HRESULT ret = real(pAdapter, DriverType, Software, Flags, pFeatureLevels, FeatureLevels,
                     SDKVersion, pUsedSwapDesc, ppSwapChain, ppDevice, pFeatureLevel, NULL);
//構(gòu)造wrap device
  WrappedID3D11Device *wrap = new WrappedID3D11Device(*ppDevice, params);
//將wrap device的地址通過(guò)指針?lè)祷?  *ppDevice = wrap;

//...
}

??其他DX組件，例如IDXGIFactory、ID3D11DeviceContext的hook方式與此類(lèi)似。

坑

??報(bào)個(gè)坑，Nvidia 驅(qū)動(dòng)安裝程序Geforce Experience的游戲內(nèi)覆蓋如果開(kāi)啟，有幾率會(huì)彈NVAPI_NO_IMPLEMENT錯(cuò)誤。