tdlight/memprof/memprof.cpp

319 lines
8.2 KiB
C++
Raw Normal View History

//
// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2019
//
// Distributed under the Boost Software License, Version 1.0. (See accompanying
// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
//
#include "memprof/memprof.h"
#include "td/utils/port/platform.h"
#if (TD_DARWIN || TD_LINUX) && defined(USE_MEMPROF)
#include <algorithm>
#include <atomic>
#include <cassert>
#include <cstddef>
#include <cstdint>
#include <cstdlib>
#include <cstring>
#include <functional>
#include <new>
#include <utility>
#include <vector>
#include <dlfcn.h>
#include <execinfo.h>
bool is_memprof_on() {
return true;
}
#if USE_MEMPROF_SAFE
double get_fast_backtrace_success_rate() {
return 0;
}
#else
#if TD_LINUX
extern void *__libc_stack_end;
#endif
static void *get_bp() {
void *bp;
#if defined(__i386__)
__asm__ volatile("movl %%ebp, %[r]" : [r] "=r"(bp));
#elif defined(__x86_64__)
__asm__ volatile("movq %%rbp, %[r]" : [r] "=r"(bp));
#endif
return bp;
}
static int fast_backtrace(void **buffer, int size) {
struct stack_frame {
stack_frame *bp;
void *ip;
};
stack_frame *bp = reinterpret_cast<stack_frame *>(get_bp());
int i = 0;
while (i < size &&
#if TD_LINUX
static_cast<void *>(bp) <= __libc_stack_end &&
#endif
!(reinterpret_cast<std::uintptr_t>(static_cast<void *>(bp)) & (sizeof(void *) - 1))) {
void *ip = bp->ip;
buffer[i++] = ip;
stack_frame *p = bp->bp;
if (p <= bp) {
break;
}
bp = p;
}
return i;
}
static std::atomic<std::size_t> fast_backtrace_failed_cnt;
static std::atomic<std::size_t> backtrace_total_cnt;
double get_fast_backtrace_success_rate() {
return 1 - static_cast<double>(fast_backtrace_failed_cnt.load(std::memory_order_relaxed)) /
static_cast<double>(std::max(std::size_t(1), backtrace_total_cnt.load(std::memory_order_relaxed)));
}
#endif
static Backtrace get_backtrace() {
static __thread bool in_backtrace; // static zero-initialized
Backtrace res{{nullptr}};
if (in_backtrace) {
return res;
}
in_backtrace = true;
std::array<void *, res.size() + BACKTRACE_SHIFT + 10> tmp{{nullptr}};
std::size_t n;
#if USE_MEMPROF_SAFE
n = backtrace(tmp.data(), static_cast<int>(tmp.size()));
#else
n = fast_backtrace(tmp.data(), static_cast<int>(tmp.size()));
auto from_shared = [](void *ptr) {
return reinterpret_cast<std::uintptr_t>(ptr) > static_cast<std::uintptr_t>(0x700000000000ull);
};
#if !USE_MEMPROF_FAST
auto end = tmp.begin() + std::min(res.size() + BACKTRACE_SHIFT, n);
if (std::find_if(tmp.begin(), end, from_shared) != end) {
fast_backtrace_failed_cnt.fetch_add(1, std::memory_order_relaxed);
n = backtrace(tmp.data(), static_cast<int>(tmp.size()));
}
backtrace_total_cnt.fetch_add(1, std::memory_order_relaxed);
#endif
n = std::remove_if(tmp.begin(), tmp.begin() + n, from_shared) - tmp.begin();
#endif
n = std::min(res.size() + BACKTRACE_SHIFT, n);
for (std::size_t i = BACKTRACE_SHIFT; i < n; i++) {
res[i - BACKTRACE_SHIFT] = tmp[i];
}
in_backtrace = false;
return res;
}
static constexpr std::size_t reserved = 16;
static constexpr std::int32_t malloc_info_magic = 0x27138373;
struct malloc_info {
std::int32_t magic;
std::int32_t size;
std::int32_t ht_pos;
};
static std::uint64_t get_hash(const Backtrace &bt) {
std::uint64_t h = 7;
for (std::size_t i = 0; i < bt.size() && i < BACKTRACE_HASHED_LENGTH; i++) {
h = h * 0x4372897893428797lu + reinterpret_cast<std::uintptr_t>(bt[i]);
}
return h;
}
struct HashtableNode {
std::atomic<std::uint64_t> hash;
Backtrace backtrace;
std::atomic<std::size_t> size;
};
static constexpr std::size_t ht_max_size = 1000000;
static std::atomic<std::size_t> ht_size{0};
static std::array<HashtableNode, ht_max_size> ht;
std::size_t get_ht_size() {
return ht_size.load();
}
std::int32_t get_ht_pos(const Backtrace &bt, bool force = false) {
auto hash = get_hash(bt);
std::int32_t pos = static_cast<std::int32_t>(hash % ht.size());
bool was_overflow = false;
while (true) {
auto pos_hash = ht[pos].hash.load();
if (pos_hash == 0) {
if (ht_size > ht_max_size / 2) {
if (force) {
assert(ht_size * 10 < ht_max_size * 7);
} else {
Backtrace unknown_bt{{nullptr}};
unknown_bt[0] = reinterpret_cast<void *>(1);
return get_ht_pos(unknown_bt, true);
}
}
std::uint64_t expected = 0;
if (ht[pos].hash.compare_exchange_strong(expected, hash)) {
ht[pos].backtrace = bt;
++ht_size;
return pos;
}
} else if (pos_hash == hash) {
return pos;
} else {
pos++;
if (pos == static_cast<std::int32_t>(ht.size())) {
pos = 0;
if (was_overflow) {
// unreachable
std::abort();
}
was_overflow = true;
}
}
}
}
void dump_alloc(const std::function<void(const AllocInfo &)> &func) {
for (auto &node : ht) {
if (node.size == 0) {
continue;
}
func(AllocInfo{node.backtrace, node.size.load()});
}
}
void register_xalloc(malloc_info *info, std::int32_t diff) {
if (diff > 0) {
ht[info->ht_pos].size += info->size;
} else {
ht[info->ht_pos].size -= info->size;
}
}
extern "C" {
static void *malloc_with_frame(std::size_t size, const Backtrace &frame) {
static_assert(reserved % alignof(std::max_align_t) == 0, "fail");
static_assert(reserved >= sizeof(malloc_info), "fail");
#if TD_DARWIN
static void *malloc_void = dlsym(RTLD_NEXT, "malloc");
static auto malloc_old = *reinterpret_cast<decltype(malloc) **>(&malloc_void);
#else
extern decltype(malloc) __libc_malloc;
static auto malloc_old = __libc_malloc;
#endif
auto *info = static_cast<malloc_info *>(malloc_old(size + reserved));
auto *buf = reinterpret_cast<char *>(info);
info->magic = malloc_info_magic;
info->size = static_cast<std::int32_t>(size);
info->ht_pos = get_ht_pos(frame);
register_xalloc(info, +1);
void *data = buf + reserved;
return data;
}
static malloc_info *get_info(void *data_void) {
char *data = static_cast<char *>(data_void);
auto *buf = data - reserved;
auto *info = reinterpret_cast<malloc_info *>(buf);
assert(info->magic == malloc_info_magic);
return info;
}
void *malloc(std::size_t size) {
return malloc_with_frame(size, get_backtrace());
}
void free(void *data_void) {
if (data_void == nullptr) {
return;
}
auto *info = get_info(data_void);
register_xalloc(info, -1);
#if TD_DARWIN
static void *free_void = dlsym(RTLD_NEXT, "free");
static auto free_old = *reinterpret_cast<decltype(free) **>(&free_void);
#else
extern decltype(free) __libc_free;
static auto free_old = __libc_free;
#endif
return free_old(info);
}
void *calloc(std::size_t size_a, std::size_t size_b) {
auto size = size_a * size_b;
void *res = malloc_with_frame(size, get_backtrace());
std::memset(res, 0, size);
return res;
}
void *realloc(void *ptr, std::size_t size) {
if (ptr == nullptr) {
return malloc_with_frame(size, get_backtrace());
}
auto *info = get_info(ptr);
auto *new_ptr = malloc_with_frame(size, get_backtrace());
auto to_copy = std::min(static_cast<std::int32_t>(size), info->size);
std::memcpy(new_ptr, ptr, to_copy);
free(ptr);
return new_ptr;
}
void *memalign(std::size_t aligment, std::size_t size) {
assert(false && "Memalign is unsupported");
return nullptr;
}
}
// c++14 guarantees that it is enough to override these two operators.
void *operator new(std::size_t count) {
return malloc_with_frame(count, get_backtrace());
}
void operator delete(void *ptr) noexcept(true) {
free(ptr);
}
// because of gcc warning: the program should also define 'void operator delete(void*, std::size_t)'
void operator delete(void *ptr, std::size_t) noexcept(true) {
free(ptr);
}
// c++17
// void *operator new(std::size_t count, std::align_val_t al);
// void operator delete(void *ptr, std::align_val_t al);
#else
bool is_memprof_on() {
return false;
}
void dump_alloc(const std::function<void(const AllocInfo &)> &func) {
}
double get_fast_backtrace_success_rate() {
return 0;
}
std::size_t get_ht_size() {
return 0;
}
#endif
std::size_t get_used_memory_size() {
std::size_t res = 0;
dump_alloc([&](const auto info) { res += info.size; });
return res;
}