319 lines
8.2 KiB
C++
319 lines
8.2 KiB
C++
//
|
|
// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2020
|
|
//
|
|
// Distributed under the Boost Software License, Version 1.0. (See accompanying
|
|
// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
|
|
//
|
|
#include "memprof/memprof.h"
|
|
|
|
#include "td/utils/port/platform.h"
|
|
|
|
#if (TD_DARWIN || TD_LINUX) && defined(USE_MEMPROF)
|
|
#include <algorithm>
|
|
#include <atomic>
|
|
#include <cassert>
|
|
#include <cstddef>
|
|
#include <cstdint>
|
|
#include <cstdlib>
|
|
#include <cstring>
|
|
#include <functional>
|
|
#include <new>
|
|
#include <utility>
|
|
#include <vector>
|
|
|
|
#include <dlfcn.h>
|
|
#include <execinfo.h>
|
|
|
|
bool is_memprof_on() {
|
|
return true;
|
|
}
|
|
|
|
#if USE_MEMPROF_SAFE
|
|
double get_fast_backtrace_success_rate() {
|
|
return 0;
|
|
}
|
|
#else
|
|
|
|
#if TD_LINUX
|
|
extern void *__libc_stack_end;
|
|
#endif
|
|
|
|
static void *get_bp() {
|
|
void *bp;
|
|
#if defined(__i386__)
|
|
__asm__ volatile("movl %%ebp, %[r]" : [ r ] "=r"(bp));
|
|
#elif defined(__x86_64__)
|
|
__asm__ volatile("movq %%rbp, %[r]" : [ r ] "=r"(bp));
|
|
#endif
|
|
return bp;
|
|
}
|
|
|
|
static int fast_backtrace(void **buffer, int size) {
|
|
struct stack_frame {
|
|
stack_frame *bp;
|
|
void *ip;
|
|
};
|
|
|
|
stack_frame *bp = reinterpret_cast<stack_frame *>(get_bp());
|
|
int i = 0;
|
|
while (i < size &&
|
|
#if TD_LINUX
|
|
static_cast<void *>(bp) <= __libc_stack_end &&
|
|
#endif
|
|
!(reinterpret_cast<std::uintptr_t>(static_cast<void *>(bp)) & (sizeof(void *) - 1))) {
|
|
void *ip = bp->ip;
|
|
buffer[i++] = ip;
|
|
stack_frame *p = bp->bp;
|
|
if (p <= bp) {
|
|
break;
|
|
}
|
|
bp = p;
|
|
}
|
|
return i;
|
|
}
|
|
|
|
static std::atomic<std::size_t> fast_backtrace_failed_cnt;
|
|
static std::atomic<std::size_t> backtrace_total_cnt;
|
|
double get_fast_backtrace_success_rate() {
|
|
return 1 - static_cast<double>(fast_backtrace_failed_cnt.load(std::memory_order_relaxed)) /
|
|
static_cast<double>(std::max(std::size_t(1), backtrace_total_cnt.load(std::memory_order_relaxed)));
|
|
}
|
|
|
|
#endif
|
|
|
|
static Backtrace get_backtrace() {
|
|
static __thread bool in_backtrace; // static zero-initialized
|
|
Backtrace res{{nullptr}};
|
|
if (in_backtrace) {
|
|
return res;
|
|
}
|
|
in_backtrace = true;
|
|
std::array<void *, res.size() + BACKTRACE_SHIFT + 10> tmp{{nullptr}};
|
|
std::size_t n;
|
|
#if USE_MEMPROF_SAFE
|
|
n = backtrace(tmp.data(), static_cast<int>(tmp.size()));
|
|
#else
|
|
n = fast_backtrace(tmp.data(), static_cast<int>(tmp.size()));
|
|
auto from_shared = [](void *ptr) {
|
|
return reinterpret_cast<std::uintptr_t>(ptr) > static_cast<std::uintptr_t>(0x700000000000ull);
|
|
};
|
|
|
|
#if !USE_MEMPROF_FAST
|
|
auto end = tmp.begin() + std::min(res.size() + BACKTRACE_SHIFT, n);
|
|
if (std::find_if(tmp.begin(), end, from_shared) != end) {
|
|
fast_backtrace_failed_cnt.fetch_add(1, std::memory_order_relaxed);
|
|
n = backtrace(tmp.data(), static_cast<int>(tmp.size()));
|
|
}
|
|
backtrace_total_cnt.fetch_add(1, std::memory_order_relaxed);
|
|
#endif
|
|
n = std::remove_if(tmp.begin(), tmp.begin() + n, from_shared) - tmp.begin();
|
|
#endif
|
|
n = std::min(res.size() + BACKTRACE_SHIFT, n);
|
|
|
|
for (std::size_t i = BACKTRACE_SHIFT; i < n; i++) {
|
|
res[i - BACKTRACE_SHIFT] = tmp[i];
|
|
}
|
|
in_backtrace = false;
|
|
return res;
|
|
}
|
|
|
|
static constexpr std::size_t reserved = 16;
|
|
static constexpr std::int32_t malloc_info_magic = 0x27138373;
|
|
struct malloc_info {
|
|
std::int32_t magic;
|
|
std::int32_t size;
|
|
std::int32_t ht_pos;
|
|
};
|
|
|
|
static std::uint64_t get_hash(const Backtrace &bt) {
|
|
std::uint64_t h = 7;
|
|
for (std::size_t i = 0; i < bt.size() && i < BACKTRACE_HASHED_LENGTH; i++) {
|
|
h = h * 0x4372897893428797lu + reinterpret_cast<std::uintptr_t>(bt[i]);
|
|
}
|
|
return h;
|
|
}
|
|
|
|
struct HashtableNode {
|
|
std::atomic<std::uint64_t> hash;
|
|
Backtrace backtrace;
|
|
std::atomic<std::size_t> size;
|
|
};
|
|
|
|
static constexpr std::size_t ht_max_size = 1000000;
|
|
static std::atomic<std::size_t> ht_size{0};
|
|
static std::array<HashtableNode, ht_max_size> ht;
|
|
|
|
std::size_t get_ht_size() {
|
|
return ht_size.load();
|
|
}
|
|
|
|
std::int32_t get_ht_pos(const Backtrace &bt, bool force = false) {
|
|
auto hash = get_hash(bt);
|
|
std::int32_t pos = static_cast<std::int32_t>(hash % ht.size());
|
|
bool was_overflow = false;
|
|
while (true) {
|
|
auto pos_hash = ht[pos].hash.load();
|
|
if (pos_hash == 0) {
|
|
if (ht_size > ht_max_size / 2) {
|
|
if (force) {
|
|
assert(ht_size * 10 < ht_max_size * 7);
|
|
} else {
|
|
Backtrace unknown_bt{{nullptr}};
|
|
unknown_bt[0] = reinterpret_cast<void *>(1);
|
|
return get_ht_pos(unknown_bt, true);
|
|
}
|
|
}
|
|
|
|
std::uint64_t expected = 0;
|
|
if (ht[pos].hash.compare_exchange_strong(expected, hash)) {
|
|
ht[pos].backtrace = bt;
|
|
++ht_size;
|
|
return pos;
|
|
}
|
|
} else if (pos_hash == hash) {
|
|
return pos;
|
|
} else {
|
|
pos++;
|
|
if (pos == static_cast<std::int32_t>(ht.size())) {
|
|
pos = 0;
|
|
if (was_overflow) {
|
|
// unreachable
|
|
std::abort();
|
|
}
|
|
was_overflow = true;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
void dump_alloc(const std::function<void(const AllocInfo &)> &func) {
|
|
for (auto &node : ht) {
|
|
if (node.size == 0) {
|
|
continue;
|
|
}
|
|
func(AllocInfo{node.backtrace, node.size.load()});
|
|
}
|
|
}
|
|
|
|
void register_xalloc(malloc_info *info, std::int32_t diff) {
|
|
if (diff > 0) {
|
|
ht[info->ht_pos].size += info->size;
|
|
} else {
|
|
ht[info->ht_pos].size -= info->size;
|
|
}
|
|
}
|
|
|
|
extern "C" {
|
|
|
|
static void *malloc_with_frame(std::size_t size, const Backtrace &frame) {
|
|
static_assert(reserved % alignof(std::max_align_t) == 0, "fail");
|
|
static_assert(reserved >= sizeof(malloc_info), "fail");
|
|
#if TD_DARWIN
|
|
static void *malloc_void = dlsym(RTLD_NEXT, "malloc");
|
|
static auto malloc_old = *reinterpret_cast<decltype(malloc) **>(&malloc_void);
|
|
#else
|
|
extern decltype(malloc) __libc_malloc;
|
|
static auto malloc_old = __libc_malloc;
|
|
#endif
|
|
auto *info = static_cast<malloc_info *>(malloc_old(size + reserved));
|
|
auto *buf = reinterpret_cast<char *>(info);
|
|
|
|
info->magic = malloc_info_magic;
|
|
info->size = static_cast<std::int32_t>(size);
|
|
info->ht_pos = get_ht_pos(frame);
|
|
|
|
register_xalloc(info, +1);
|
|
|
|
void *data = buf + reserved;
|
|
|
|
return data;
|
|
}
|
|
|
|
static malloc_info *get_info(void *data_void) {
|
|
char *data = static_cast<char *>(data_void);
|
|
auto *buf = data - reserved;
|
|
|
|
auto *info = reinterpret_cast<malloc_info *>(buf);
|
|
assert(info->magic == malloc_info_magic);
|
|
return info;
|
|
}
|
|
|
|
void *malloc(std::size_t size) {
|
|
return malloc_with_frame(size, get_backtrace());
|
|
}
|
|
|
|
void free(void *data_void) {
|
|
if (data_void == nullptr) {
|
|
return;
|
|
}
|
|
auto *info = get_info(data_void);
|
|
register_xalloc(info, -1);
|
|
|
|
#if TD_DARWIN
|
|
static void *free_void = dlsym(RTLD_NEXT, "free");
|
|
static auto free_old = *reinterpret_cast<decltype(free) **>(&free_void);
|
|
#else
|
|
extern decltype(free) __libc_free;
|
|
static auto free_old = __libc_free;
|
|
#endif
|
|
return free_old(info);
|
|
}
|
|
void *calloc(std::size_t size_a, std::size_t size_b) {
|
|
auto size = size_a * size_b;
|
|
void *res = malloc_with_frame(size, get_backtrace());
|
|
std::memset(res, 0, size);
|
|
return res;
|
|
}
|
|
void *realloc(void *ptr, std::size_t size) {
|
|
if (ptr == nullptr) {
|
|
return malloc_with_frame(size, get_backtrace());
|
|
}
|
|
auto *info = get_info(ptr);
|
|
auto *new_ptr = malloc_with_frame(size, get_backtrace());
|
|
auto to_copy = std::min(static_cast<std::int32_t>(size), info->size);
|
|
std::memcpy(new_ptr, ptr, to_copy);
|
|
free(ptr);
|
|
return new_ptr;
|
|
}
|
|
void *memalign(std::size_t aligment, std::size_t size) {
|
|
assert(false && "Memalign is unsupported");
|
|
return nullptr;
|
|
}
|
|
}
|
|
|
|
// c++14 guarantees that it is enough to override these two operators.
|
|
void *operator new(std::size_t count) {
|
|
return malloc_with_frame(count, get_backtrace());
|
|
}
|
|
void operator delete(void *ptr) noexcept(true) {
|
|
free(ptr);
|
|
}
|
|
// because of gcc warning: the program should also define 'void operator delete(void*, std::size_t)'
|
|
void operator delete(void *ptr, std::size_t) noexcept(true) {
|
|
free(ptr);
|
|
}
|
|
|
|
// c++17
|
|
// void *operator new(std::size_t count, std::align_val_t al);
|
|
// void operator delete(void *ptr, std::align_val_t al);
|
|
|
|
#else
|
|
bool is_memprof_on() {
|
|
return false;
|
|
}
|
|
void dump_alloc(const std::function<void(const AllocInfo &)> &func) {
|
|
}
|
|
double get_fast_backtrace_success_rate() {
|
|
return 0;
|
|
}
|
|
std::size_t get_ht_size() {
|
|
return 0;
|
|
}
|
|
#endif
|
|
|
|
std::size_t get_used_memory_size() {
|
|
std::size_t res = 0;
|
|
dump_alloc([&](const auto info) { res += info.size; });
|
|
return res;
|
|
}
|