Кроссплатформенный краш-репорт. Обзор возможностей google breakpad.

Докладчик: Демчук ДмитрийРуководитель С++ отдела

Google Breakpad. ЗадачаПроблема Программы содержат дефекты; Дефекты проявляются на окружении

пользователя/тестировщика; Сборки с дефектами уходят в релиз;

Цель: Возможность исправлять дефекты по отчетам от

пользователей

Google Breakpad. Задача

Google BreakpadPlatforms: Windows Linux Mac OS X Solaris iOS Android

API индивидуальное под каждую платформу.

Report format: Minidump files (Microsoft)

Breakpad. СборкаWindows:1. GYP & Python2. MSVC

Linux:3. autotools4. g++

Mac OS5. xCode, autotools6. clang, g++

Exception handling

App.exe

1. In - process

App.exe

2. out-of-process

crash.dmp crash.dmp

generates

CrashDumper.exeIPC

generates

+ простая интеграция- низкая надежность

- сложная интеграция+ высокая надежность

Breakpad. Windows библиотеки

common.lib

exception_handler.lib

crash_generation_client.lib

crash_generation_server.lib

App.exe

common.lib

CrashDumper.exe

IPC

Windows. In-process-example …#include "exception_handler.h“

…

int main(int argc, char * argv[]) { using namespace google_breakpad; const std::wstring dumpPath(L"./");

ExceptionHandler handler(dumpPath, &filterCallback, &minidumpCallback, NULL, ExceptionHandler::HANDLER_ALL);

/// Application code … return 0; }

Windows. In-process-example …#include "exception_handler.h“

bool filterCallback(void * context, EXCEPTION_POINTERS * exinfo, MDRawAssertionInfo * assertion) { return true;}

bool minidumpCallback(const wchar_t * dump_path, const wchar_t * minidump_id, void * context, EXCEPTION_POINTERS* exinfo, MDRawAssertionInfo* assertion, bool succeeded){ return true;}

Windows. In-process-example Application breakpad

new ExceptionHandler

set handlers

Worker thread

waitFor…

...

kernel32.dll

callback is invoked

start thread

wake upbool filterCallback(…)

WriteDump…bool minidumpCallback(…)

Windows. Out-of-process Client#include "exception_handler.h“

…int main(int argc, char * argv[]){ const std::wstring pipeName(L"\\\\.\\pipe\\BreakpadCrashService");

const std::wstring dumpPath(L"./");

ExceptionHandler handler(dumpPath, &filterCallback, &minidumpCallback, NULL, ExceptionHandler::HANDLER_ALL, MiniDumpNormal, pipeName.c_str(), NULL); /// Application code … return 0; }

Windows. Out-of-process Server #include "client/windows/crash_generation/crash_generation_server.h" #include "client/windows/crash_generation/client_info.h"

void onClientConnectedCallback(void * context, const ClientInfo * client_info) { std::cout << "Client is connected pid= " << client_info->pid() << std::endl; }

void onClientDumpRequestCallback(void * context, const ClientInfo * client_info, const std::wstring * file_path){ std::cout << "pid = " << client_info->pid() << std::endl;}

…

Windows. Out-of-process Server #include "client/windows/crash_generation/crash_generation_server.h" #include "client/windows/crash_generation/client_info.h"

…void onClientExitedCallback(void * context, const google_breakpad::ClientInfo * client_info) { std::cout << "Client is exited pid = " << client_info->pid() << std::endl; }

void onClientUploadRequestCallback(void* context, const DWORD crash_id){ std::cout << "Client upload request callback crashID = " << crash_id << std::endl; }

…

Windows. Out-of-process Server …int main(int argc, char * argv[]){ using namespace google_breakpad;

const std::wstring pipeName(L"\\\\.\\pipe\\BreakpadCrashService"); const std::wstring dumpPath(L"./");

CrashGenerationServer server(pipeName, NULL, onClientConnectedCallback, NULL, onClientDumpRequestCallback, NULL, onClientExitedCallback, NULL, onClientUploadRequestCallback, NULL, true, &dumpPath);

if (!server.Start()) return -1; Sleep(INFINITE); return 0; }

Breakpad. Out-of-process-example

app.exe breakpad

Worker thread

waitFor…

...

filterCallback

WriteDump…waitFor…minidumpCallback

breakpad CrashService.exe

onClientConnectedCallback

onClientDumpRequestCallback

onClientExitedCallback

Breakpad. Capture dump

…int main(int argc, char * argv[]) { using namespace google_breakpad; const std::wstring dumpPath(L"./"); ExceptionHandler handler(dumpPath, &filterCallback, &minidumpCallback, NULL, ExceptionHandler::HANDLER_ALL);

/// Application code … handler.WriteMinidump(); … return 0; }

Windows. РеализацияОбработчики

SetUnhandledExceptionFilter(…);

__set_invalid_parameter_handler(…);

__set_purecall_handler(…);

Снятие дампа

DbgHelp.dll

BOOL MiniDumpWriteDump(HANDLE  hProcess,

DWORD ProcessId, HANDLE hFile … );

Открытие *dmp для Windowsapplication.exelib1.dll...libN.dll

application.pdblib1. pdb...libN. pdb

source code

crash.dmp

Breakpad. Linux библиотеки

breakpad_client

breakpad_client

App

CrashDumper

IPC

Linux. In-process example #include "exception_handler.h“

bool filterCallback(void * context){ return true; }

bool minidumpCallback(const MinidumpDescriptor & descriptor, void * context, bool succeeded) { return true; }

…

Linux. In-process example …int main(int argc, char * argv[]){ using namespace google_breakpad;

MinidumpDescriptor descriptor("./"); ExceptionHandler handler(descriptor, &filterCallback, &minidumpCallback, NULL, true, -1);

/// Application code … return 0; }

Linux. In-process example app.exe breakpad

...

filterCallback

minidumpCallback

sigaction

app.exebreakpadsys_clone

Create *dmp

new ExceptionHandler

Linux. Out-of-process Client#include "exception_handler.h" #include "examplesCommon/common.h" using namespace google_breakpad;

bool filterCallback(void * context)… bool minidumpCallback(const MinidumpDescriptor & descriptor, void * context, bool succeeded)…

int main(int argc, char * argv[]) { int serverFD(0); MinidumpDescriptor descriptor("./"); ExceptionHandler handler(descriptor, &filterCallback, &minidumpCallback, NULL, true, serverFD); /// Application code … return 0; }

Linux. Out-of-process Server#include "client/linux/crash_generation/crash_generation_server.h“#include "client/linux/crash_generation/client_info.h" using namespace google_breakpad;

void onClientDumpRequestCallback(void * context, const ClientInfo * client_info, const std::string * file_path){

}

void OnClientExitingCallback(void * context, const ClientInfo * client_info) {

}

Linux. Out-of-process Server#include "client/linux/crash_generation/crash_generation_server.h“#include "client/linux/crash_generation/client_info.h" using namespace google_breakpad;

…int main(int argc, char * argv[]){ const std::string dumpPath("./"); int serverFD(0), clientFD(0);

if (!CrashGenerationServer::CreateReportChannel(&serverFD, &clientFD)) return -1;

CrashGenerationServer server(serverFD, &onClientDumpRequestCallback, NULL, &OnClientExitingCallback, NULL, true, &dumpPath);

server.Start(); sleep(100000000); …}

Linux. Реализация Обработчикиint sigaction(int signum, const struct sigaction *act, struct sigaction *oldact);

SIGSEGV, SIGABRT, SIGFPE, SIGILL, SIGBUS

Снятие дампа

long ptrace(enum __ptrace_request request, pid_t pid, void *addr, void *data);

Linux. Инструменты работы с dmp

5d8cb….2a102.dmp

dump_symsapplication

my_lib.so

application.sym

my_lib.sym

symbols

… …

minidump_stackwalk stack traceThread 0 (crashed) 0 inProcessExample!handleArgs(int, char**, google_breakpad::ExceptionHandler*) [common.cpp : 86 + 0x3] eip = 0x0804e015 esp = 0xbf916db0 ebp = 0xbf916dd8 ebx = 0x002ebff4 esi = 0x00000000 edi = 0x00000000 eax = 0x00000000 ecx = 0x002ed8b8 edx = 0x00000000 efl = 0x00210286 Found by: given as instruction pointer in context 1 inProcessExample!main [main.cpp : 29 + 0x1a] eip = 0x0804db77 esp = 0xbf916de0 ebp = 0xbf916e88 ebx = 0x002ebff4 esi = 0x00000000 edi = 0x00000000 Found by: call frame info 2 libc-2.15.so + 0x194d3 eip = 0x0015f4d3 esp = 0xbf916e90 ebp = 0x00000000 ebx = 0x002ebff4 esi = 0x00000000 edi = 0x00000000 Found by: call frame info 3 libc-2.15.so + 0x1a5ff4 eip = 0x002ebff4 esp = 0xbf916eb8 ebp = 0x00000000 Found by: stack scanning 4 inProcessExample + 0x5928 eip = 0x0804d928 esp = 0xbf916ee0 ebp = 0x00000000

minidump-2-core *.core Linux core file

Linux. Утилита dump_syms

dump_syms

inProcessExample

…

MODULE Linux x86 62F39E85D30A5F150EBD5B34E085C7AC0 inProcessExampleFILE 0 /home/user/GoogleBreakPad/breakpad/./src/client/linux/crash_generation/crash_generation_client.hFILE 1 /home/user/GoogleBreakPad/breakpad/./src/client/linux/handler/microdump_extra_info.hFILE 2 /home/use...FUNC 5660 39 0 MinidumpWriter::MinidumpWriter5660 f 124 27566f 2a 142 27FUNC 569a 1d9 0 MinidumpWriter::WriteFile569a 1 1118 27...

symbols └── inProcessExample └── 62F39E85D30A5F150EBD5B34E085C7AC0 └── inProcessExample.sym

Linux. Stack traceOperating system: Linux 0.0.0 Linux 3.2.0-70-generic #105-Ubuntu SMP Wed Sep 24 19:49:46 UTC 2014 i686CPU: x86 GenuineIntel family 6 model 42 stepping 7 1 CPU

Crash reason: SIGSEGVCrash address: 0x0Process uptime: not available

Thread 0 (crashed) 0 inProcessExample!handleArgs(int, char**, google_breakpad::ExceptionHandler*) [common.cpp : 86 + 0x3] eip = 0x0804e015 esp = 0xbf916db0 ebp = 0xbf916dd8 ebx = 0x002ebff4 esi = 0x00000000 edi = 0x00000000 eax = 0x00000000 ecx = 0x002ed8b8 edx = 0x00000000 efl = 0x00210286 Found by: given as instruction pointer in context

Linux. Stack trace…0x00000000 ecx = 0x002ed8b8 edx = 0x00000000 efl = 0x00210286 Found by: given as instruction pointer in context 1 inProcessExample!main [main.cpp : 29 + 0x1a] eip = 0x0804db77 esp = 0xbf916de0 ebp = 0xbf916e88 ebx = 0x002ebff4 esi = 0x00000000 edi = 0x00000000 Found by: call frame info 2 libc-2.15.so + 0x194d3 eip = 0x0015f4d3 esp = 0xbf916e90 ebp = 0x00000000 ebx = 0x002ebff4 esi = 0x00000000 edi = 0x00000000 Found by: call frame info…

Linux. Core - file> ./minidump-2-core 5d8cbd57-7164-867f-416fe783-2a02a102.dmp > 5d8cbd57-7164-867f-416fe783-2a02a102.core

> gdb 5d8cbd57-7164-867f-416fe783-2a02a102.core inProcessExample

Program terminated with signal 11, Segmentation fault.#0 0x0804e015 in handleArgs (argc=2, argv=0xbf916f24, handler=0xbf916e04) at /home/user/GoogleBreakPad/samples/source/examplesCommon/common.cpp:8686 *i = 12;(gdb) bt

…

Linux. Core - file(gdb) bt#0 0x0804e015 in handleArgs (argc=2, argv=0xbf916f24, handler=0xbf916e04) at /home/user/GoogleBreakPad/samples/source/examplesCommon/common.cpp:86#1 0x0804db77 in main (argc=2, argv=0xbf916f24) at /home/user/GoogleBreakPad/samples/source/linux/inProcessExample/main.cpp:29(gdb)

Breakpad. Mac OS. Библиотеки

Breakpad.Framework

Breakpad.Framework

App

CrashDumper

IPC

Mac OS. In-process example #include "exception_handler.h“

bool filterCallback(void *context){ return true;}

bool minidumpCallback(const char * dump_dir, const char * minidump_id, void * context, bool succeeded){ return true;}

…

Mac OS. In-process example int main(int argc, char * argv[]){ using namespace google_breakpad;

ExceptionHandler handler("./", &filterCallback, &minidumpCallback, NULL, true, NULL);

/// Application code

…

return 0; }

Mac OS. Out-of-process Client#include "exception_handler.h“

…int main(int argc, char * argv[]){ using namespace google_breakpad; const char * serverPort = "crash_generation_server"; ExceptionHandler handler("./", &filterCallback, &minidumpCallback, NULL, true, serverPort);

/// Application code … return 0; }

Mac OS. Out-of-process Server#include "client/mac/crash_generation/crash_generation_server.h“#include "client/mac/crash_generation/client_info.h" using namespace google_breakpad;

bool filterCallback(void *context){ std::cout << "filterCallback is invoked" << std::endl; return true;}void onClientDumpRequestCallback(void * context, const ClientInfo & client_info, const std::string & file_path){ std::cout << "Client dump request callback threadId = " << client_info.pid() << std::endl;}void OnClientExitingCallback(void * context, const ClientInfo & client_info) { std::cout << "Client is exited threadId = " << client_info.pid() << std::endl; }

Mac OS. Out-of-process Serverint main(int argc, char * argv[]) { const std::string dumpPath("./crashes/");

const char * serverPort = "crash_generation_server"; CrashGenerationServer server(serverPort, &filterCallback, NULL, &onClientDumpRequestCallback, NULL, &OnClientExitingCallback, NULL, true, dumpPath);

if (!server.Start()) return -1;

sleep(10000000); return 0;}

Report server?

breakpad

App.exe

CrashDumper.exe

breakpad

IPC

21a7ccc…. 7067.dmp

generates

Internet

Unix Server

minidump_stackwalk

*.dmp *.sym

stack trace

Report server Soccoro (Mozilla)

mini-breakpad-server

Общие рекомендации1. Сборка Релизные сборки с отладочной информацией RelWithDebInfo

(/Zi) Система версионирования сборок/бинарников Зашивать номер версии в ресурсы бинарников Хранить все артефакты от публичных релизов (source + binary

+ pdb)

2. Реализация out-of-processs способа Когда и как стартует/останавливается вспомогательный

процесс Реализация посылки краш-репорта на сервер

Спасибо за внимание!

Докладчик: Демчук Дмитрий