2014-10-28 19:54:33 +01:00
|
|
|
// Copyright (c) 2013, Facebook, Inc. All rights reserved.
|
|
|
|
// This source code is licensed under the BSD-style license found in the
|
|
|
|
// LICENSE file in the root directory of this source tree. An additional grant
|
|
|
|
// of patent rights can be found in the PATENTS file in the same directory.
|
|
|
|
//
|
|
|
|
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
|
|
|
|
// Use of this source code is governed by a BSD-style license that can be
|
|
|
|
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
|
|
|
#pragma once
|
|
|
|
|
|
|
|
#include <atomic>
|
|
|
|
#include <deque>
|
|
|
|
#include <limits>
|
|
|
|
#include <set>
|
|
|
|
#include <utility>
|
|
|
|
#include <vector>
|
|
|
|
#include <string>
|
|
|
|
|
|
|
|
#include "db/dbformat.h"
|
|
|
|
#include "db/log_writer.h"
|
|
|
|
#include "db/snapshot.h"
|
|
|
|
#include "db/column_family.h"
|
|
|
|
#include "db/version_edit.h"
|
|
|
|
#include "db/memtable_list.h"
|
|
|
|
#include "port/port.h"
|
|
|
|
#include "rocksdb/db.h"
|
|
|
|
#include "rocksdb/env.h"
|
|
|
|
#include "rocksdb/memtablerep.h"
|
|
|
|
#include "rocksdb/transaction_log.h"
|
|
|
|
#include "util/autovector.h"
|
EventLogger
Summary:
Here's my proposal for making our LOGs easier to read by machines.
The idea is to dump all events as JSON objects. JSON is easy to read by humans, but more importantly, it's easy to read by machines. That way, we can parse this, load into SQLite/mongo and then query or visualize.
I started with table_create and table_delete events, but if everybody agrees, I'll continue by adding more events (flush/compaction/etc etc)
Test Plan:
Ran db_bench. Observed:
2015/01/15-14:13:25.788019 1105ef000 EVENT_LOG_v1 {"time_micros": 1421360005788015, "event": "table_file_creation", "file_number": 12, "file_size": 1909699}
2015/01/15-14:13:25.956500 110740000 EVENT_LOG_v1 {"time_micros": 1421360005956498, "event": "table_file_deletion", "file_number": 12}
Reviewers: yhchiang, rven, dhruba, MarkCallaghan, lgalanis, sdong
Reviewed By: sdong
Subscribers: dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D31647
2015-03-13 18:15:54 +01:00
|
|
|
#include "util/event_logger.h"
|
2015-02-05 06:39:45 +01:00
|
|
|
#include "util/instrumented_mutex.h"
|
2014-10-28 19:54:33 +01:00
|
|
|
#include "util/stop_watch.h"
|
|
|
|
#include "util/thread_local.h"
|
|
|
|
#include "util/scoped_arena_iterator.h"
|
|
|
|
#include "db/internal_stats.h"
|
|
|
|
#include "db/write_controller.h"
|
|
|
|
#include "db/flush_scheduler.h"
|
|
|
|
#include "db/write_thread.h"
|
|
|
|
#include "db/job_context.h"
|
|
|
|
|
|
|
|
namespace rocksdb {
|
|
|
|
|
|
|
|
class MemTable;
|
|
|
|
class TableCache;
|
|
|
|
class Version;
|
|
|
|
class VersionEdit;
|
|
|
|
class VersionSet;
|
|
|
|
class Arena;
|
|
|
|
|
|
|
|
class FlushJob {
|
|
|
|
public:
|
|
|
|
// TODO(icanadi) make effort to reduce number of parameters here
|
|
|
|
// IMPORTANT: mutable_cf_options needs to be alive while FlushJob is alive
|
|
|
|
FlushJob(const std::string& dbname, ColumnFamilyData* cfd,
|
|
|
|
const DBOptions& db_options,
|
|
|
|
const MutableCFOptions& mutable_cf_options,
|
|
|
|
const EnvOptions& env_options, VersionSet* versions,
|
2015-02-05 06:39:45 +01:00
|
|
|
InstrumentedMutex* db_mutex, std::atomic<bool>* shutting_down,
|
2014-11-07 20:50:34 +01:00
|
|
|
SequenceNumber newest_snapshot, JobContext* job_context,
|
|
|
|
LogBuffer* log_buffer, Directory* db_directory,
|
2015-01-26 22:59:38 +01:00
|
|
|
Directory* output_file_directory, CompressionType output_compression,
|
EventLogger
Summary:
Here's my proposal for making our LOGs easier to read by machines.
The idea is to dump all events as JSON objects. JSON is easy to read by humans, but more importantly, it's easy to read by machines. That way, we can parse this, load into SQLite/mongo and then query or visualize.
I started with table_create and table_delete events, but if everybody agrees, I'll continue by adding more events (flush/compaction/etc etc)
Test Plan:
Ran db_bench. Observed:
2015/01/15-14:13:25.788019 1105ef000 EVENT_LOG_v1 {"time_micros": 1421360005788015, "event": "table_file_creation", "file_number": 12, "file_size": 1909699}
2015/01/15-14:13:25.956500 110740000 EVENT_LOG_v1 {"time_micros": 1421360005956498, "event": "table_file_deletion", "file_number": 12}
Reviewers: yhchiang, rven, dhruba, MarkCallaghan, lgalanis, sdong
Reviewed By: sdong
Subscribers: dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D31647
2015-03-13 18:15:54 +01:00
|
|
|
Statistics* stats, EventLogger* event_logger);
|
2015-03-13 18:45:40 +01:00
|
|
|
|
|
|
|
~FlushJob();
|
2014-10-28 19:54:33 +01:00
|
|
|
|
CompactFiles, EventListener and GetDatabaseMetaData
Summary:
This diff adds three sets of APIs to RocksDB.
= GetColumnFamilyMetaData =
* This APIs allow users to obtain the current state of a RocksDB instance on one column family.
* See GetColumnFamilyMetaData in include/rocksdb/db.h
= EventListener =
* A virtual class that allows users to implement a set of
call-back functions which will be called when specific
events of a RocksDB instance happens.
* To register EventListener, simply insert an EventListener to ColumnFamilyOptions::listeners
= CompactFiles =
* CompactFiles API inputs a set of file numbers and an output level, and RocksDB
will try to compact those files into the specified level.
= Example =
* Example code can be found in example/compact_files_example.cc, which implements
a simple external compactor using EventListener, GetColumnFamilyMetaData, and
CompactFiles API.
Test Plan:
listener_test
compactor_test
example/compact_files_example
export ROCKSDB_TESTS=CompactFiles
db_test
export ROCKSDB_TESTS=MetaData
db_test
Reviewers: ljin, igor, rven, sdong
Reviewed By: sdong
Subscribers: MarkCallaghan, dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D24705
2014-11-07 23:45:18 +01:00
|
|
|
Status Run(uint64_t* file_number = nullptr);
|
2014-10-28 19:54:33 +01:00
|
|
|
|
|
|
|
private:
|
2015-05-16 08:22:22 +02:00
|
|
|
void ReportStartedFlush();
|
|
|
|
void ReportFlushInputSize(const autovector<MemTable*>& mems);
|
|
|
|
void RecordFlushIOStats();
|
2014-10-28 19:54:33 +01:00
|
|
|
Status WriteLevel0Table(const autovector<MemTable*>& mems, VersionEdit* edit,
|
|
|
|
uint64_t* filenumber);
|
|
|
|
const std::string& dbname_;
|
|
|
|
ColumnFamilyData* cfd_;
|
|
|
|
const DBOptions& db_options_;
|
|
|
|
const MutableCFOptions& mutable_cf_options_;
|
|
|
|
const EnvOptions& env_options_;
|
|
|
|
VersionSet* versions_;
|
2015-02-05 06:39:45 +01:00
|
|
|
InstrumentedMutex* db_mutex_;
|
2014-10-28 19:54:33 +01:00
|
|
|
std::atomic<bool>* shutting_down_;
|
|
|
|
SequenceNumber newest_snapshot_;
|
|
|
|
JobContext* job_context_;
|
|
|
|
LogBuffer* log_buffer_;
|
|
|
|
Directory* db_directory_;
|
2015-01-26 22:59:38 +01:00
|
|
|
Directory* output_file_directory_;
|
2014-10-28 19:54:33 +01:00
|
|
|
CompressionType output_compression_;
|
|
|
|
Statistics* stats_;
|
EventLogger
Summary:
Here's my proposal for making our LOGs easier to read by machines.
The idea is to dump all events as JSON objects. JSON is easy to read by humans, but more importantly, it's easy to read by machines. That way, we can parse this, load into SQLite/mongo and then query or visualize.
I started with table_create and table_delete events, but if everybody agrees, I'll continue by adding more events (flush/compaction/etc etc)
Test Plan:
Ran db_bench. Observed:
2015/01/15-14:13:25.788019 1105ef000 EVENT_LOG_v1 {"time_micros": 1421360005788015, "event": "table_file_creation", "file_number": 12, "file_size": 1909699}
2015/01/15-14:13:25.956500 110740000 EVENT_LOG_v1 {"time_micros": 1421360005956498, "event": "table_file_deletion", "file_number": 12}
Reviewers: yhchiang, rven, dhruba, MarkCallaghan, lgalanis, sdong
Reviewed By: sdong
Subscribers: dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D31647
2015-03-13 18:15:54 +01:00
|
|
|
EventLogger* event_logger_;
|
2014-10-28 19:54:33 +01:00
|
|
|
};
|
|
|
|
|
|
|
|
} // namespace rocksdb
|