Aggressively inlining the short functions in coding.cc
Summary:
This diff takes an even more aggressive way to inline the functions. A decent rule that I followed is "not inline a function if it is more than 10 lines long."
Normally optimizing code by inline is ugly and hard to control, but since one of our usecase has significant amount of CPU used in functions from coding.cc, I'd like to try this diff out.
Test Plan:
1. the size for some .o file increased a little bit, but most less than 1%. So I think the negative impact of inline is negligible.
2. As the regression test shows (ran for 10 times and I calculated the average number)
Metrics Befor After
========================================================================
rocksdb.build.fillseq.qps 426595 444515 (+4.6%)
rocksdb.build.memtablefillrandom.qps 121739 123110
rocksdb.build.memtablereadrandom.qps 1285103 1280520
rocksdb.build.overwrite.qps 125816 135570 (+9%)
rocksdb.build.readrandom_fillunique_random.qps 285995 296863
rocksdb.build.readrandom_memtable_sst.qps 1027132 1027279
rocksdb.build.readrandom.qps 1041427 1054665
rocksdb.build.readrandom_smallblockcache.qps 1028631 1038433
rocksdb.build.readwhilewriting.qps 918352 914629
Reviewers: haobo, sdong, igor
CC: leveldb
Differential Revision: https://reviews.facebook.net/D15291
This commit is contained in:
Kai Liu
parent
7dea558e6d
commit
bb19b530ca
165
util/coding.cc
165
util/coding.cc
@ -8,49 +8,10 @@
|
||||
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
||||
|
||||
#include "util/coding.h"
|
||||
|
||||
#include <algorithm>
|
||||
|
||||
namespace rocksdb {
|
||||
|
||||
void EncodeFixed32(char* buf, uint32_t value) {
|
||||
#if __BYTE_ORDER == __LITTLE_ENDIAN
|
||||
memcpy(buf, &value, sizeof(value));
|
||||
#else
|
||||
buf[0] = value & 0xff;
|
||||
buf[1] = (value >> 8) & 0xff;
|
||||
buf[2] = (value >> 16) & 0xff;
|
||||
buf[3] = (value >> 24) & 0xff;
|
||||
#endif
|
||||
}
|
||||
|
||||
void EncodeFixed64(char* buf, uint64_t value) {
|
||||
#if __BYTE_ORDER == __LITTLE_ENDIAN
|
||||
memcpy(buf, &value, sizeof(value));
|
||||
#else
|
||||
buf[0] = value & 0xff;
|
||||
buf[1] = (value >> 8) & 0xff;
|
||||
buf[2] = (value >> 16) & 0xff;
|
||||
buf[3] = (value >> 24) & 0xff;
|
||||
buf[4] = (value >> 32) & 0xff;
|
||||
buf[5] = (value >> 40) & 0xff;
|
||||
buf[6] = (value >> 48) & 0xff;
|
||||
buf[7] = (value >> 56) & 0xff;
|
||||
#endif
|
||||
}
|
||||
|
||||
void PutFixed32(std::string* dst, uint32_t value) {
|
||||
char buf[sizeof(value)];
|
||||
EncodeFixed32(buf, value);
|
||||
dst->append(buf, sizeof(buf));
|
||||
}
|
||||
|
||||
void PutFixed64(std::string* dst, uint64_t value) {
|
||||
char buf[sizeof(value)];
|
||||
EncodeFixed64(buf, value);
|
||||
dst->append(buf, sizeof(buf));
|
||||
}
|
||||
|
||||
char* EncodeVarint32(char* dst, uint32_t v) {
|
||||
// Operate on characters as unsigneds
|
||||
unsigned char* ptr = reinterpret_cast<unsigned char*>(dst);
|
||||
@ -79,57 +40,7 @@ char* EncodeVarint32(char* dst, uint32_t v) {
|
||||
return reinterpret_cast<char*>(ptr);
|
||||
}
|
||||
|
||||
void PutVarint32(std::string* dst, uint32_t v) {
|
||||
char buf[5];
|
||||
char* ptr = EncodeVarint32(buf, v);
|
||||
dst->append(buf, ptr - buf);
|
||||
}
|
||||
|
||||
char* EncodeVarint64(char* dst, uint64_t v) {
|
||||
static const unsigned int B = 128;
|
||||
unsigned char* ptr = reinterpret_cast<unsigned char*>(dst);
|
||||
while (v >= B) {
|
||||
*(ptr++) = (v & (B-1)) | B;
|
||||
v >>= 7;
|
||||
}
|
||||
*(ptr++) = static_cast<unsigned char>(v);
|
||||
return reinterpret_cast<char*>(ptr);
|
||||
}
|
||||
|
||||
void PutVarint64(std::string* dst, uint64_t v) {
|
||||
char buf[10];
|
||||
char* ptr = EncodeVarint64(buf, v);
|
||||
dst->append(buf, ptr - buf);
|
||||
}
|
||||
|
||||
void PutLengthPrefixedSlice(std::string* dst, const Slice& value) {
|
||||
PutVarint32(dst, value.size());
|
||||
dst->append(value.data(), value.size());
|
||||
}
|
||||
|
||||
void PutLengthPrefixedSliceParts(std::string* dst,
|
||||
const SliceParts& slice_parts) {
|
||||
uint32_t total_bytes = 0;
|
||||
for (int i = 0; i < slice_parts.num_parts; ++i) {
|
||||
total_bytes += slice_parts.parts[i].size();
|
||||
}
|
||||
PutVarint32(dst, total_bytes);
|
||||
for (int i = 0; i < slice_parts.num_parts; ++i) {
|
||||
dst->append(slice_parts.parts[i].data(), slice_parts.parts[i].size());
|
||||
}
|
||||
}
|
||||
|
||||
int VarintLength(uint64_t v) {
|
||||
int len = 1;
|
||||
while (v >= 128) {
|
||||
v >>= 7;
|
||||
len++;
|
||||
}
|
||||
return len;
|
||||
}
|
||||
|
||||
const char* GetVarint32PtrFallback(const char* p,
|
||||
const char* limit,
|
||||
const char* GetVarint32PtrFallback(const char* p, const char* limit,
|
||||
uint32_t* value) {
|
||||
uint32_t result = 0;
|
||||
for (uint32_t shift = 0; shift <= 28 && p < limit; shift += 7) {
|
||||
@ -147,18 +58,6 @@ const char* GetVarint32PtrFallback(const char* p,
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
bool GetVarint32(Slice* input, uint32_t* value) {
|
||||
const char* p = input->data();
|
||||
const char* limit = p + input->size();
|
||||
const char* q = GetVarint32Ptr(p, limit, value);
|
||||
if (q == nullptr) {
|
||||
return false;
|
||||
} else {
|
||||
*input = Slice(q, limit - q);
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
const char* GetVarint64Ptr(const char* p, const char* limit, uint64_t* value) {
|
||||
uint64_t result = 0;
|
||||
for (uint32_t shift = 0; shift <= 63 && p < limit; shift += 7) {
|
||||
@ -176,58 +75,6 @@ const char* GetVarint64Ptr(const char* p, const char* limit, uint64_t* value) {
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
bool GetVarint64(Slice* input, uint64_t* value) {
|
||||
const char* p = input->data();
|
||||
const char* limit = p + input->size();
|
||||
const char* q = GetVarint64Ptr(p, limit, value);
|
||||
if (q == nullptr) {
|
||||
return false;
|
||||
} else {
|
||||
*input = Slice(q, limit - q);
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
const char* GetLengthPrefixedSlice(const char* p, const char* limit,
|
||||
Slice* result) {
|
||||
uint32_t len;
|
||||
p = GetVarint32Ptr(p, limit, &len);
|
||||
if (p == nullptr) return nullptr;
|
||||
if (p + len > limit) return nullptr;
|
||||
*result = Slice(p, len);
|
||||
return p + len;
|
||||
}
|
||||
|
||||
bool GetLengthPrefixedSlice(Slice* input, Slice* result) {
|
||||
uint32_t len;
|
||||
if (GetVarint32(input, &len) &&
|
||||
input->size() >= len) {
|
||||
*result = Slice(input->data(), len);
|
||||
input->remove_prefix(len);
|
||||
return true;
|
||||
} else {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
Slice GetLengthPrefixedSlice(const char* data) {
|
||||
uint32_t len;
|
||||
const char* p = data;
|
||||
p = GetVarint32Ptr(p, p + 5, &len); // +5: we assume "p" is not corrupted
|
||||
return Slice(p, len);
|
||||
}
|
||||
|
||||
Slice GetSliceUntil(Slice* slice, char delimiter) {
|
||||
uint32_t len;
|
||||
for (len = 0; len < slice->size() && slice->data()[len] != delimiter; ++len) {
|
||||
// nothing
|
||||
}
|
||||
|
||||
Slice ret(slice->data(), len);
|
||||
slice->remove_prefix(len + ((len < slice->size()) ? 1 : 0));
|
||||
return ret;
|
||||
}
|
||||
|
||||
void BitStreamPutInt(char* dst, size_t dstlen, size_t offset,
|
||||
uint32_t bits, uint64_t value) {
|
||||
assert((offset + bits + 7)/8 <= dstlen);
|
||||
@ -316,14 +163,4 @@ void BitStreamPutInt(std::string* dst, size_t offset, uint32_t bits,
|
||||
BitStreamGetInt(dst, offset, bits));
|
||||
}
|
||||
|
||||
uint64_t BitStreamGetInt(const std::string* src, size_t offset,
|
||||
uint32_t bits) {
|
||||
return BitStreamGetInt(src->data(), src->size(), offset, bits);
|
||||
}
|
||||
|
||||
uint64_t BitStreamGetInt(const Slice* src, size_t offset,
|
||||
uint32_t bits) {
|
||||
return BitStreamGetInt(src->data(), src->size(), offset, bits);
|
||||
}
|
||||
|
||||
} // namespace rocksdb
|
||||
|
||||
162
util/coding.h
162
util/coding.h
@ -13,6 +13,7 @@
|
||||
// * Strings are encoded prefixed by their length in varint format
|
||||
|
||||
#pragma once
|
||||
#include <algorithm>
|
||||
#include <stdint.h>
|
||||
#include <string.h>
|
||||
#include <string>
|
||||
@ -136,4 +137,165 @@ extern uint64_t BitStreamGetInt(const std::string* src, size_t offset,
|
||||
extern uint64_t BitStreamGetInt(const Slice* src, size_t offset,
|
||||
uint32_t bits);
|
||||
|
||||
// -- Implementation of the functions declared above
|
||||
inline void EncodeFixed32(char* buf, uint32_t value) {
|
||||
#if __BYTE_ORDER == __LITTLE_ENDIAN
|
||||
memcpy(buf, &value, sizeof(value));
|
||||
#else
|
||||
buf[0] = value & 0xff;
|
||||
buf[1] = (value >> 8) & 0xff;
|
||||
buf[2] = (value >> 16) & 0xff;
|
||||
buf[3] = (value >> 24) & 0xff;
|
||||
#endif
|
||||
}
|
||||
|
||||
inline void EncodeFixed64(char* buf, uint64_t value) {
|
||||
#if __BYTE_ORDER == __LITTLE_ENDIAN
|
||||
memcpy(buf, &value, sizeof(value));
|
||||
#else
|
||||
buf[0] = value & 0xff;
|
||||
buf[1] = (value >> 8) & 0xff;
|
||||
buf[2] = (value >> 16) & 0xff;
|
||||
buf[3] = (value >> 24) & 0xff;
|
||||
buf[4] = (value >> 32) & 0xff;
|
||||
buf[5] = (value >> 40) & 0xff;
|
||||
buf[6] = (value >> 48) & 0xff;
|
||||
buf[7] = (value >> 56) & 0xff;
|
||||
#endif
|
||||
}
|
||||
|
||||
inline void PutFixed32(std::string* dst, uint32_t value) {
|
||||
char buf[sizeof(value)];
|
||||
EncodeFixed32(buf, value);
|
||||
dst->append(buf, sizeof(buf));
|
||||
}
|
||||
|
||||
inline void PutFixed64(std::string* dst, uint64_t value) {
|
||||
char buf[sizeof(value)];
|
||||
EncodeFixed64(buf, value);
|
||||
dst->append(buf, sizeof(buf));
|
||||
}
|
||||
|
||||
inline void PutVarint32(std::string* dst, uint32_t v) {
|
||||
char buf[5];
|
||||
char* ptr = EncodeVarint32(buf, v);
|
||||
dst->append(buf, ptr - buf);
|
||||
}
|
||||
|
||||
inline char* EncodeVarint64(char* dst, uint64_t v) {
|
||||
static const unsigned int B = 128;
|
||||
unsigned char* ptr = reinterpret_cast<unsigned char*>(dst);
|
||||
while (v >= B) {
|
||||
*(ptr++) = (v & (B - 1)) | B;
|
||||
v >>= 7;
|
||||
}
|
||||
*(ptr++) = static_cast<unsigned char>(v);
|
||||
return reinterpret_cast<char*>(ptr);
|
||||
}
|
||||
|
||||
inline void PutVarint64(std::string* dst, uint64_t v) {
|
||||
char buf[10];
|
||||
char* ptr = EncodeVarint64(buf, v);
|
||||
dst->append(buf, ptr - buf);
|
||||
}
|
||||
|
||||
inline void PutLengthPrefixedSlice(std::string* dst, const Slice& value) {
|
||||
PutVarint32(dst, value.size());
|
||||
dst->append(value.data(), value.size());
|
||||
}
|
||||
|
||||
inline void PutLengthPrefixedSliceParts(std::string* dst,
|
||||
const SliceParts& slice_parts) {
|
||||
uint32_t total_bytes = 0;
|
||||
for (int i = 0; i < slice_parts.num_parts; ++i) {
|
||||
total_bytes += slice_parts.parts[i].size();
|
||||
}
|
||||
PutVarint32(dst, total_bytes);
|
||||
for (int i = 0; i < slice_parts.num_parts; ++i) {
|
||||
dst->append(slice_parts.parts[i].data(), slice_parts.parts[i].size());
|
||||
}
|
||||
}
|
||||
|
||||
inline int VarintLength(uint64_t v) {
|
||||
int len = 1;
|
||||
while (v >= 128) {
|
||||
v >>= 7;
|
||||
len++;
|
||||
}
|
||||
return len;
|
||||
}
|
||||
|
||||
inline bool GetVarint32(Slice* input, uint32_t* value) {
|
||||
const char* p = input->data();
|
||||
const char* limit = p + input->size();
|
||||
const char* q = GetVarint32Ptr(p, limit, value);
|
||||
if (q == nullptr) {
|
||||
return false;
|
||||
} else {
|
||||
*input = Slice(q, limit - q);
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
inline bool GetVarint64(Slice* input, uint64_t* value) {
|
||||
const char* p = input->data();
|
||||
const char* limit = p + input->size();
|
||||
const char* q = GetVarint64Ptr(p, limit, value);
|
||||
if (q == nullptr) {
|
||||
return false;
|
||||
} else {
|
||||
*input = Slice(q, limit - q);
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
inline const char* GetLengthPrefixedSlice(const char* p, const char* limit,
|
||||
Slice* result) {
|
||||
uint32_t len = 0;
|
||||
p = GetVarint32Ptr(p, limit, &len);
|
||||
if (p == nullptr) return nullptr;
|
||||
if (p + len > limit) return nullptr;
|
||||
*result = Slice(p, len);
|
||||
return p + len;
|
||||
}
|
||||
|
||||
inline bool GetLengthPrefixedSlice(Slice* input, Slice* result) {
|
||||
uint32_t len = 0;
|
||||
if (GetVarint32(input, &len) && input->size() >= len) {
|
||||
*result = Slice(input->data(), len);
|
||||
input->remove_prefix(len);
|
||||
return true;
|
||||
} else {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
inline Slice GetLengthPrefixedSlice(const char* data) {
|
||||
uint32_t len = 0;
|
||||
const char* p = data;
|
||||
p = GetVarint32Ptr(p, p + 5, &len); // +5: we assume "p" is not corrupted
|
||||
return Slice(p, len);
|
||||
}
|
||||
|
||||
inline Slice GetSliceUntil(Slice* slice, char delimiter) {
|
||||
uint32_t len = 0;
|
||||
for (len = 0; len < slice->size() && slice->data()[len] != delimiter; ++len) {
|
||||
// nothing
|
||||
}
|
||||
|
||||
Slice ret(slice->data(), len);
|
||||
slice->remove_prefix(len + ((len < slice->size()) ? 1 : 0));
|
||||
return ret;
|
||||
}
|
||||
|
||||
inline uint64_t BitStreamGetInt(const std::string* src, size_t offset,
|
||||
uint32_t bits) {
|
||||
return BitStreamGetInt(src->data(), src->size(), offset, bits);
|
||||
}
|
||||
|
||||
inline uint64_t BitStreamGetInt(const Slice* src, size_t offset,
|
||||
uint32_t bits) {
|
||||
return BitStreamGetInt(src->data(), src->size(), offset, bits);
|
||||
}
|
||||
|
||||
} // namespace rocksdb
|
||||
|
||||
Loading…
Reference in New Issue
Block a user