tdlight-team/tdlight
tdlight-team/tdlight
1
1
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Improve TQueue.

Browse Source 
GitOrigin-RevId: 3146441d9035be886a616d8de504024df1602116
This commit is contained in:
levlam 2020-06-11 23:54:56 +03:00
parent 44155da2d8
commit bee9b67e3c
3 changed files with 55 additions and 56 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

75
tddb/td/db/TQueue.cpp
View File

@ -11,6 +11,7 @@
#include "td/db/binlog/BinlogInterface.h" #include "td/db/binlog/BinlogInterface.h"
#include "td/utils/format.h" #include "td/utils/format.h"
#include "td/utils/misc.h"
#include "td/utils/port/Clocks.h" #include "td/utils/port/Clocks.h"
#include "td/utils/Random.h" #include "td/utils/Random.h"
#include "td/utils/StorerBase.h" #include "td/utils/StorerBase.h"
@ -35,14 +36,18 @@ EventId::EventId() {
} }
Result<EventId> EventId::from_int32(int32 id) { Result<EventId> EventId::from_int32(int32 id) {
if (!is_valid(id)) { if (!is_valid_id(id)) {
return Status::Error("Invalid ID"); return Status::Error("Invalid ID");
} }
return EventId(id); return EventId(id);
} }
EventId EventId::create_random() { EventId EventId::create_random() {
return from_int32(Random::fast_uint32() % (MAX_ID / 2) + 10 + MAX_QUEUE_EVENTS).move_as_ok(); return from_int32(Random::fast(MAX_QUEUE_EVENTS + 1, MAX_ID / 2)).move_as_ok();
}
bool EventId::is_valid() const {
return !empty() && is_valid_id(id_);
} }
int32 EventId::value() const { int32 EventId::value() const {
@ -74,34 +79,31 @@ bool EventId::operator<(const EventId &other) const {
return id_ < other.id_; return id_ < other.id_;
} }
StringBuilder &operator<<(StringBuilder &sb, const EventId id) { StringBuilder &operator<<(StringBuilder &string_builder, const EventId id) {
return sb << "EventId{" << id.value() << "}"; return string_builder << "EventId{" << id.value() << "}";
} }
EventId::EventId(int32 id) : id_(id) { EventId::EventId(int32 id) : id_(id) {
CHECK(is_valid(id)); CHECK(is_valid_id(id));
} }
bool EventId::is_valid(int32 id) { bool EventId::is_valid_id(int32 id) {
return 0 <= id && id < MAX_ID; return 0 <= id && id < MAX_ID;
} }
class TQueueImpl : public TQueue { class TQueueImpl : public TQueue {
public: public:
void set_callback(unique_ptr<Callback> callback) override { void set_callback(unique_ptr<StorageCallback> callback) override {
callback_ = std::move(callback); callback_ = std::move(callback);
} }
unique_ptr<Callback> extract_callback() override { unique_ptr<StorageCallback> extract_callback() override {
return std::move(callback_); return std::move(callback_);
} }
void emulate_restart() override {
}
void do_push(QueueId queue_id, RawEvent &&raw_event) override { void do_push(QueueId queue_id, RawEvent &&raw_event) override {
//LOG(ERROR) << "Push " << queue_id << " " << raw_event.event_id; // LOG(ERROR) << "Push to queue " << queue_id << " " << raw_event.event_id;
CHECK(!raw_event.event_id.empty()); CHECK(raw_event.event_id.is_valid());
if (raw_event.logevent_id == 0 && callback_) { if (raw_event.logevent_id == 0 && callback_ != nullptr) {
raw_event.logevent_id = callback_->push(queue_id, raw_event); raw_event.logevent_id = callback_->push(queue_id, raw_event);
} }
auto &q = queues_[queue_id]; auto &q = queues_[queue_id];
@ -128,11 +130,12 @@ class TQueueImpl : public TQueue {
if (event_id.next().is_ok()) { if (event_id.next().is_ok()) {
break; break;
} }
for (auto &e : q.events.as_mutable_span()) { for (auto &event : q.events.as_mutable_span()) {
try_pop(queue_id, e, EventId{}, EventId{}, 0, true); pop(queue_id, event, {});
} }
q.tail_id = {}; q.tail_id = EventId();
q.events = {}; q.events = {};
CHECK(new_id.next().is_ok());
} }
RawEvent raw_event; RawEvent raw_event;
@ -177,12 +180,12 @@ class TQueueImpl : public TQueue {
if (it == from_events.end() || !(it->event_id == event_id)) { if (it == from_events.end() || !(it->event_id == event_id)) {
return; return;
} }
try_pop(queue_id, *it, {}, q.tail_id, 0, true /*force*/); pop(queue_id, *it, q.tail_id);
} }
Result<size_t> get(QueueId queue_id, EventId from_id, bool forget_previous, double now, Result<size_t> get(QueueId queue_id, EventId from_id, bool forget_previous, double now,
MutableSpan<Event> &result_events) override { MutableSpan<Event> &result_events) override {
//LOG(ERROR) << "Get " << queue_id << " " << from_id; // LOG(ERROR) << "Get " << queue_id << " " << from_id;
auto it = queues_.find(queue_id); auto it = queues_.find(queue_id);
if (it == queues_.end()) { if (it == queues_.end()) {
result_events.truncate(0); result_events.truncate(0);
@ -210,7 +213,7 @@ class TQueueImpl : public TQueue {
[](auto &event, EventId event_id) { return event.event_id < event_id; }) - [](auto &event, EventId event_id) { return event.event_id < event_id; }) -
from_events.begin(); from_events.begin();
} }
//LOG(ERROR) << tag("first_i", first_i) << tag("size", from_events.size()); // LOG(ERROR) << tag("first_i", first_i) << tag("size", from_events.size());
for (i = first_i; i < from_events.size(); i++) { for (i = first_i; i < from_events.size(); i++) {
auto &from = from_events[i]; auto &from = from_events[i];
try_pop(queue_id, from, forget_previous ? from_id : EventId{}, q.tail_id, now); try_pop(queue_id, from, forget_previous ? from_id : EventId{}, q.tail_id, now);
@ -265,7 +268,7 @@ class TQueueImpl : public TQueue {
}; };
std::unordered_map<QueueId, Queue> queues_; std::unordered_map<QueueId, Queue> queues_;
unique_ptr<Callback> callback_; unique_ptr<StorageCallback> callback_;
void compactify(VectorQueue<RawEvent> &events, size_t prefix) { void compactify(VectorQueue<RawEvent> &events, size_t prefix) {
auto processed = events.as_mutable_span().substr(0, prefix); auto processed = events.as_mutable_span().substr(0, prefix);
@ -274,24 +277,23 @@ class TQueueImpl : public TQueue {
events.pop_n(removed_n); events.pop_n(removed_n);
} }
void try_pop(QueueId queue_id, RawEvent &event, EventId from_id, EventId tail_id, double now, bool force = false) { void try_pop(QueueId queue_id, RawEvent &event, EventId from_id, EventId tail_id, double now) {
//LOG(ERROR) << event.expires_at << " < " << now << " = " << (event.expires_at < now) << " " // LOG(ERROR) << event.expires_at << " < " << now << " = " << (event.expires_at < now) << " "
//<< (event.event_id.value() < from_id.value()) << " " << force << " " << event.data.empty(); // << (event.event_id < from_id) << " " << event.data.empty();
bool should_drop = if (event.expires_at < now || event.event_id < from_id || event.data.empty()) {
event.expires_at < now || event.event_id.value() < from_id.value() || force || event.data.empty(); pop(queue_id, event, tail_id);
if (!callback_ || event.logevent_id == 0) { }
if (should_drop) { }
event.data = {};
} void pop(QueueId queue_id, RawEvent &event, EventId tail_id) {
if (callback_ == nullptr || event.logevent_id == 0) {
event.logevent_id = 0;
event.data = {};
return; return;
} }
if (!should_drop) { // LOG(ERROR) << "Drop " << queue_id << " " << event.event_id;
return; if (event.event_id.next().ok() == tail_id) {
}
//LOG(ERROR) << "Drop " << queue_id << " " << event.event_id;
if (event.event_id.value() + 1 == tail_id.value()) {
if (!event.data.empty()) { if (!event.data.empty()) {
event.data = {}; event.data = {};
callback_->push(queue_id, event); callback_->push(queue_id, event);
@ -309,7 +311,6 @@ unique_ptr<TQueue> TQueue::create() {
} }
struct TQueueLogEvent : public Storer { struct TQueueLogEvent : public Storer {
TQueueLogEvent() = default;
int64 queue_id; int64 queue_id;
int32 event_id; int32 event_id;
int32 expires_at; int32 expires_at;

30
tddb/td/db/TQueue.h
View File

@ -30,6 +30,8 @@ class TQueue {
static EventId create_random(); static EventId create_random();
bool is_valid() const;
int32 value() const; int32 value() const;
Result<EventId> next() const; Result<EventId> next() const;
@ -47,7 +49,7 @@ class TQueue {
explicit EventId(int32 id); explicit EventId(int32 id);
static bool is_valid(int32 id); static bool is_valid_id(int32 id);
}; };
struct Event { struct Event {
@ -67,17 +69,17 @@ class TQueue {
using QueueId = int64; using QueueId = int64;
class Callback { class StorageCallback {
public: public:
using QueueId = TQueue::QueueId; using QueueId = TQueue::QueueId;
using RawEvent = TQueue::RawEvent; using RawEvent = TQueue::RawEvent;
Callback() = default; StorageCallback() = default;
Callback(const Callback &) = delete; StorageCallback(const StorageCallback &) = delete;
Callback &operator=(const Callback &) = delete; StorageCallback &operator=(const StorageCallback &) = delete;
Callback(Callback &&) = delete; StorageCallback(StorageCallback &&) = delete;
Callback &operator=(Callback &&) = delete; StorageCallback &operator=(StorageCallback &&) = delete;
virtual ~Callback() = default; virtual ~StorageCallback() = default;
virtual uint64 push(QueueId queue_id, const RawEvent &event) = 0; virtual uint64 push(QueueId queue_id, const RawEvent &event) = 0;
virtual void pop(uint64 logevent_id) = 0; virtual void pop(uint64 logevent_id) = 0;
@ -93,10 +95,8 @@ class TQueue {
virtual ~TQueue() = default; virtual ~TQueue() = default;
virtual void set_callback(unique_ptr<Callback> callback) = 0; virtual void set_callback(unique_ptr<StorageCallback> callback) = 0;
virtual unique_ptr<Callback> extract_callback() = 0; virtual unique_ptr<StorageCallback> extract_callback() = 0;
virtual void emulate_restart() = 0; // for testing only
virtual void do_push(QueueId queue_id, RawEvent &&raw_event) = 0; virtual void do_push(QueueId queue_id, RawEvent &&raw_event) = 0;
@ -114,12 +114,12 @@ class TQueue {
virtual void run_gc(double now) = 0; virtual void run_gc(double now) = 0;
}; };
StringBuilder &operator<<(StringBuilder &sb, const TQueue::EventId id); StringBuilder &operator<<(StringBuilder &string_builder, const TQueue::EventId id);
struct BinlogEvent; struct BinlogEvent;
template <class BinlogT> template <class BinlogT>
class TQueueBinlog : public TQueue::Callback { class TQueueBinlog : public TQueue::StorageCallback {
public: public:
TQueueBinlog(); TQueueBinlog();
@ -137,7 +137,7 @@ class TQueueBinlog : public TQueue::Callback {
double diff_{0}; double diff_{0};
}; };
class TQueueMemoryStorage : public TQueue::Callback { class TQueueMemoryStorage : public TQueue::StorageCallback {
public: public:
uint64 push(QueueId queue_id, const RawEvent &event) override; uint64 push(QueueId queue_id, const RawEvent &event) override;
void pop(uint64 logevent_id) override; void pop(uint64 logevent_id) override;

6
test/tqueue.cpp
View File

@ -60,7 +60,6 @@ class TestTQueue {
} }
void restart(Random::Xorshift128plus &rnd, double now) { void restart(Random::Xorshift128plus &rnd, double now) {
baseline_->emulate_restart();
if (rnd.fast(0, 10) == 0) { if (rnd.fast(0, 10) == 0) {
baseline_->run_gc(now); baseline_->run_gc(now);
} }
@ -75,11 +74,10 @@ class TestTQueue {
} }
if (rnd.fast(0, 100) != 0) { if (rnd.fast(0, 100) != 0) {
binlog_->emulate_restart();
return; return;
} }
LOG(ERROR) << "RESTART BINLOG"; LOG(INFO) << "Restart binlog";
binlog_ = TQueue::create(); binlog_ = TQueue::create();
auto tqueue_binlog = make_unique<TQueueBinlog<Binlog>>(); auto tqueue_binlog = make_unique<TQueueBinlog<Binlog>>();
auto binlog = std::make_shared<Binlog>(); auto binlog = std::make_shared<Binlog>();
@ -179,7 +177,7 @@ TEST(TQueue, random) {
q.check_get(next_qid(), rnd, now); q.check_get(next_qid(), rnd, now);
}; };
RandomSteps steps({{push_event, 100}, {check_head_tail, 10}, {get, 40}, {inc_now, 5}, {restart, 1}}); RandomSteps steps({{push_event, 100}, {check_head_tail, 10}, {get, 40}, {inc_now, 5}, {restart, 1}});
for (int i = 0; i < 1000000; i++) { for (int i = 0; i < 100000; i++) {
steps.step(rnd); steps.step(rnd);
} }
} }