Windows and Linux kernels exception handling and stack unwinding

The interesting difference between Windows and Linux kernels is in Windows mechanism to unwind a call stack, aka Frame Unwind. Windows 64 bit and Linux kernels use the table based exception processing to locate a handler for an instruction that caused an exception. Windows kernel can unwind a call stack to locate a caller's handler while Linux requires to have a table entry for each executable address range that can cause an exception.

You can look at pseudo-code for Windows 64 bit RtlUnwind here StackWalk64.cpp .

Some resources on Windows 64 bit SEH implementation.

1. Exceptional behavior: the Windows 8.1 X64 SEH Implementation  http://blog.talosintel.com/2014/06/exceptional-behavior-windows-81-x64-seh.html

2. Exceptional Behavior - x64 Structured Exception Handling - OSR Online. http://www.osronline.com/article.cfm?article=469

3. Johnson, Ken. " Programming against the x64 exception handling support ."  http://www.nynaeve.net/?p=113

The code was borrowed from http://www.nynaeve.net/Code/StackWalk64.cpp

__declspec(noinline)
VOID
StackTrace64(
 VOID
 )
{
 CONTEXT                       Context;
 KNONVOLATILE_CONTEXT_POINTERS NvContext;
 UNWIND_HISTORY_TABLE          UnwindHistoryTable;
 PRUNTIME_FUNCTION             RuntimeFunction;
 PVOID                         HandlerData;
 ULONG64                       EstablisherFrame;
 ULONG64                       ImageBase;

 DbgPrint("StackTrace64: Executing stack trace...\n");

 //
 // First, we'll get the caller's context.
 //

 RtlCaptureContext(&Context);

 //
 // Initialize the (optional) unwind history table.
 //

 RtlZeroMemory(
  &UnwindHistoryTable,
  sizeof(UNWIND_HISTORY_TABLE));

 UnwindHistoryTable.Unwind = TRUE;

 //
 // This unwind loop intentionally skips the first call frame, as it shall
 // correspond to the call to StackTrace64, which we aren't interested in.
 //

 for (ULONG Frame = 0;
   ;
   Frame++)
 {
  //
  // Try to look up unwind metadata for the current function.
  //

  RuntimeFunction = RtlLookupFunctionEntry(
   Context.Rip,
   &ImageBase,
   &UnwindHistoryTable
   );

  RtlZeroMemory(
   &NvContext,
   sizeof(KNONVOLATILE_CONTEXT_POINTERS));

  if (!RuntimeFunction)
  {
   //
   // If we don't have a RUNTIME_FUNCTION, then we've encountered
   // a leaf function.  Adjust the stack approprately.
   //

   Context.Rip  = (ULONG64)(*(PULONG64)Context.Rsp);
   Context.Rsp += 8;
  }
  else
  {
   //
   // Otherwise, call upon RtlVirtualUnwind to execute the unwind for
   // us.
   //

   RtlVirtualUnwind(
    UNW_FLAG_NHANDLER,
    ImageBase,
    Context.Rip,
    RuntimeFunction,
    &Context,
    &HandlerData,
    &EstablisherFrame,
    &NvContext);
  }

  //
  // If we reach an RIP of zero, this means that we've walked off the end
  // of the call stack and are done.
  //

  if (!Context.Rip)
   break;

  //
  // Display the context.  Note that we don't bother showing the XMM
  // context, although we have the nonvolatile portion of it.
  //

  DbgPrint(
   "FRAME %02x: Rip=%p Rsp=%p Rbp=%p\n",
   Frame,
   Context.Rip,
   Context.Rsp,
   Context.Rsp);
  DbgPrint(
   "r12=%p r13=%p r14=%p\n"
   "rdi=%p rsi=%p rbx=%p\n"
   "rbp=%p rsp=%p\n",
   Context.R12,
   Context.R13,
   Context.R14,
   Context.Rdi,
   Context.Rsi,
   Context.Rbx,
   Context.Rbp,
   Context.Rsp
   );

  static const CHAR* RegNames[ 16 ] =
  { "Rax", "Rcx", "Rdx", "Rbx", "Rsp", "Rbp", "Rsi", "Rdi", "R8", "R9",
    "R10", "R11", "R12", "R13", "R14", "R15" };

  //
  // If we have stack-based register stores, then display them here.
  //

  for (ULONG i = 0;
    i < 16;
    i++)
  {
   if (NvContext.IntegerContext[ i ])
   {
    DbgPrint(
     " -> Saved register '%s' on stack at %p (=> %p)\n",
     RegNames[ i ],
     NvContext.IntegerContext[ i ],
     *NvContext.IntegerContext[ i ]);
   }
  }

  DbgPrint("\n");
 }

 DbgBreakPoint();

 return;
}