2016-02-10 00:12:00 +01:00
|
|
|
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
|
2013-10-16 23:59:46 +02:00
|
|
|
// This source code is licensed under the BSD-style license found in the
|
|
|
|
// LICENSE file in the root directory of this source tree. An additional grant
|
|
|
|
// of patent rights can be found in the PATENTS file in the same directory.
|
|
|
|
//
|
2011-03-18 23:37:00 +01:00
|
|
|
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
|
|
|
|
// Use of this source code is governed by a BSD-style license that can be
|
|
|
|
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
|
|
|
|
2013-10-05 07:32:05 +02:00
|
|
|
#pragma once
|
2011-03-18 23:37:00 +01:00
|
|
|
#include <stdio.h>
|
2014-07-23 21:31:11 +02:00
|
|
|
#include <string>
|
2013-08-23 17:38:13 +02:00
|
|
|
#include "rocksdb/comparator.h"
|
|
|
|
#include "rocksdb/db.h"
|
|
|
|
#include "rocksdb/filter_policy.h"
|
|
|
|
#include "rocksdb/slice.h"
|
2014-04-10 23:19:43 +02:00
|
|
|
#include "rocksdb/slice_transform.h"
|
2013-10-29 01:54:09 +01:00
|
|
|
#include "rocksdb/table.h"
|
2013-08-23 17:38:13 +02:00
|
|
|
#include "rocksdb/types.h"
|
2011-03-18 23:37:00 +01:00
|
|
|
#include "util/coding.h"
|
|
|
|
#include "util/logging.h"
|
|
|
|
|
2013-10-04 06:49:15 +02:00
|
|
|
namespace rocksdb {
|
2011-03-18 23:37:00 +01:00
|
|
|
|
|
|
|
class InternalKey;
|
|
|
|
|
|
|
|
// Value types encoded as the last component of internal keys.
|
|
|
|
// DO NOT CHANGE THESE ENUM VALUES: they are embedded in the on-disk
|
|
|
|
// data structures.
|
2014-01-27 22:53:22 +01:00
|
|
|
// The highest bit of the value type needs to be reserved to SST tables
|
|
|
|
// for them to do more flexible encoding.
|
|
|
|
enum ValueType : unsigned char {
|
2011-03-18 23:37:00 +01:00
|
|
|
kTypeDeletion = 0x0,
|
2013-03-21 23:59:47 +01:00
|
|
|
kTypeValue = 0x1,
|
2013-08-15 01:32:46 +02:00
|
|
|
kTypeMerge = 0x2,
|
Support for SingleDelete()
Summary:
This patch fixes #7460559. It introduces SingleDelete as a new database
operation. This operation can be used to delete keys that were never
overwritten (no put following another put of the same key). If an overwritten
key is single deleted the behavior is undefined. Single deletion of a
non-existent key has no effect but multiple consecutive single deletions are
not allowed (see limitations).
In contrast to the conventional Delete() operation, the deletion entry is
removed along with the value when the two are lined up in a compaction. Note:
The semantics are similar to @igor's prototype that allowed to have this
behavior on the granularity of a column family (
https://reviews.facebook.net/D42093 ). This new patch, however, is more
aggressive when it comes to removing tombstones: It removes the SingleDelete
together with the value whenever there is no snapshot between them while the
older patch only did this when the sequence number of the deletion was older
than the earliest snapshot.
Most of the complex additions are in the Compaction Iterator, all other changes
should be relatively straightforward. The patch also includes basic support for
single deletions in db_stress and db_bench.
Limitations:
- Not compatible with cuckoo hash tables
- Single deletions cannot be used in combination with merges and normal
deletions on the same key (other keys are not affected by this)
- Consecutive single deletions are currently not allowed (and older version of
this patch supported this so it could be resurrected if needed)
Test Plan: make all check
Reviewers: yhchiang, sdong, rven, anthony, yoshinorim, igor
Reviewed By: igor
Subscribers: maykov, dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D43179
2015-09-17 20:42:56 +02:00
|
|
|
kTypeLogData = 0x3, // WAL only.
|
|
|
|
kTypeColumnFamilyDeletion = 0x4, // WAL only.
|
|
|
|
kTypeColumnFamilyValue = 0x5, // WAL only.
|
|
|
|
kTypeColumnFamilyMerge = 0x6, // WAL only.
|
|
|
|
kTypeSingleDeletion = 0x7,
|
|
|
|
kTypeColumnFamilySingleDeletion = 0x8, // WAL only.
|
Modification of WriteBatch to support two phase commit
Summary: Adds three new WriteBatch data types: Prepare(xid), Commit(xid), Rollback(xid). Prepare(xid) should precede the (single) operation to which is applies. There can obviously be multiple Prepare(xid) markers. There should only be one Rollback(xid) or Commit(xid) marker yet not both. None of this logic is currently enforced and will most likely be implemented further up such as in the memtableinserter. All three markers are similar to PutLogData in that they are writebatch meta-data, ie stored but not counted. All three markers differ from PutLogData in that they will actually be written to disk. As for WriteBatchWithIndex, Prepare, Commit, Rollback are all implemented just as PutLogData and none are tested just as PutLogData.
Test Plan: single unit test in write_batch_test.
Reviewers: hermanlee4, sdong, anthony
Subscribers: andrewkr, vasilep, dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D54093
2016-04-08 08:35:51 +02:00
|
|
|
kTypeBeginPrepareXID = 0x9, // WAL only.
|
|
|
|
kTypeEndPrepareXID = 0xA, // WAL only.
|
|
|
|
kTypeCommitXID = 0xB, // WAL only.
|
|
|
|
kTypeRollbackXID = 0xC, // WAL only.
|
|
|
|
kTypeNoop = 0xD, // WAL only.
|
Support for SingleDelete()
Summary:
This patch fixes #7460559. It introduces SingleDelete as a new database
operation. This operation can be used to delete keys that were never
overwritten (no put following another put of the same key). If an overwritten
key is single deleted the behavior is undefined. Single deletion of a
non-existent key has no effect but multiple consecutive single deletions are
not allowed (see limitations).
In contrast to the conventional Delete() operation, the deletion entry is
removed along with the value when the two are lined up in a compaction. Note:
The semantics are similar to @igor's prototype that allowed to have this
behavior on the granularity of a column family (
https://reviews.facebook.net/D42093 ). This new patch, however, is more
aggressive when it comes to removing tombstones: It removes the SingleDelete
together with the value whenever there is no snapshot between them while the
older patch only did this when the sequence number of the deletion was older
than the earliest snapshot.
Most of the complex additions are in the Compaction Iterator, all other changes
should be relatively straightforward. The patch also includes basic support for
single deletions in db_stress and db_bench.
Limitations:
- Not compatible with cuckoo hash tables
- Single deletions cannot be used in combination with merges and normal
deletions on the same key (other keys are not affected by this)
- Consecutive single deletions are currently not allowed (and older version of
this patch supported this so it could be resurrected if needed)
Test Plan: make all check
Reviewers: yhchiang, sdong, rven, anthony, yoshinorim, igor
Reviewed By: igor
Subscribers: maykov, dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D43179
2015-09-17 20:42:56 +02:00
|
|
|
kMaxValue = 0x7F // Not used for storing records.
|
2011-03-18 23:37:00 +01:00
|
|
|
};
|
2014-01-27 22:53:22 +01:00
|
|
|
|
2011-03-18 23:37:00 +01:00
|
|
|
// kValueTypeForSeek defines the ValueType that should be passed when
|
|
|
|
// constructing a ParsedInternalKey object for seeking to a particular
|
|
|
|
// sequence number (since we sort sequence numbers in decreasing order
|
|
|
|
// and the value type is embedded as the low 8 bits in the sequence
|
|
|
|
// number in internal keys, we need to use the highest-numbered
|
|
|
|
// ValueType, not the lowest).
|
Support for SingleDelete()
Summary:
This patch fixes #7460559. It introduces SingleDelete as a new database
operation. This operation can be used to delete keys that were never
overwritten (no put following another put of the same key). If an overwritten
key is single deleted the behavior is undefined. Single deletion of a
non-existent key has no effect but multiple consecutive single deletions are
not allowed (see limitations).
In contrast to the conventional Delete() operation, the deletion entry is
removed along with the value when the two are lined up in a compaction. Note:
The semantics are similar to @igor's prototype that allowed to have this
behavior on the granularity of a column family (
https://reviews.facebook.net/D42093 ). This new patch, however, is more
aggressive when it comes to removing tombstones: It removes the SingleDelete
together with the value whenever there is no snapshot between them while the
older patch only did this when the sequence number of the deletion was older
than the earliest snapshot.
Most of the complex additions are in the Compaction Iterator, all other changes
should be relatively straightforward. The patch also includes basic support for
single deletions in db_stress and db_bench.
Limitations:
- Not compatible with cuckoo hash tables
- Single deletions cannot be used in combination with merges and normal
deletions on the same key (other keys are not affected by this)
- Consecutive single deletions are currently not allowed (and older version of
this patch supported this so it could be resurrected if needed)
Test Plan: make all check
Reviewers: yhchiang, sdong, rven, anthony, yoshinorim, igor
Reviewed By: igor
Subscribers: maykov, dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D43179
2015-09-17 20:42:56 +02:00
|
|
|
static const ValueType kValueTypeForSeek = kTypeSingleDeletion;
|
|
|
|
|
|
|
|
// Checks whether a type is a value type (i.e. a type used in memtables and sst
|
|
|
|
// files).
|
|
|
|
inline bool IsValueType(ValueType t) {
|
|
|
|
return t <= kTypeMerge || t == kTypeSingleDeletion;
|
|
|
|
}
|
2011-03-18 23:37:00 +01:00
|
|
|
|
|
|
|
// We leave eight bits empty at the bottom so a type and sequence#
|
|
|
|
// can be packed together into 64-bits.
|
|
|
|
static const SequenceNumber kMaxSequenceNumber =
|
|
|
|
((0x1ull << 56) - 1);
|
|
|
|
|
|
|
|
struct ParsedInternalKey {
|
|
|
|
Slice user_key;
|
|
|
|
SequenceNumber sequence;
|
|
|
|
ValueType type;
|
|
|
|
|
|
|
|
ParsedInternalKey() { } // Intentionally left uninitialized (for speed)
|
|
|
|
ParsedInternalKey(const Slice& u, const SequenceNumber& seq, ValueType t)
|
|
|
|
: user_key(u), sequence(seq), type(t) { }
|
2012-12-16 03:28:36 +01:00
|
|
|
std::string DebugString(bool hex = false) const;
|
2011-03-18 23:37:00 +01:00
|
|
|
};
|
|
|
|
|
|
|
|
// Return the length of the encoding of "key".
|
|
|
|
inline size_t InternalKeyEncodingLength(const ParsedInternalKey& key) {
|
|
|
|
return key.user_key.size() + 8;
|
|
|
|
}
|
|
|
|
|
2015-05-29 23:36:35 +02:00
|
|
|
// Pack a sequence number and a ValueType into a uint64_t
|
2014-04-01 23:45:30 +02:00
|
|
|
extern uint64_t PackSequenceAndType(uint64_t seq, ValueType t);
|
|
|
|
|
2015-05-29 23:36:35 +02:00
|
|
|
// Given the result of PackSequenceAndType, store the sequence number in *seq
|
|
|
|
// and the ValueType in *t.
|
|
|
|
extern void UnPackSequenceAndType(uint64_t packed, uint64_t* seq, ValueType* t);
|
|
|
|
|
2011-03-18 23:37:00 +01:00
|
|
|
// Append the serialization of "key" to *result.
|
|
|
|
extern void AppendInternalKey(std::string* result,
|
|
|
|
const ParsedInternalKey& key);
|
|
|
|
|
|
|
|
// Attempt to parse an internal key from "internal_key". On success,
|
|
|
|
// stores the parsed data in "*result", and returns true.
|
|
|
|
//
|
|
|
|
// On error, returns false, leaves "*result" in an undefined state.
|
|
|
|
extern bool ParseInternalKey(const Slice& internal_key,
|
|
|
|
ParsedInternalKey* result);
|
|
|
|
|
|
|
|
// Returns the user key portion of an internal key.
|
|
|
|
inline Slice ExtractUserKey(const Slice& internal_key) {
|
|
|
|
assert(internal_key.size() >= 8);
|
|
|
|
return Slice(internal_key.data(), internal_key.size() - 8);
|
|
|
|
}
|
|
|
|
|
|
|
|
inline ValueType ExtractValueType(const Slice& internal_key) {
|
|
|
|
assert(internal_key.size() >= 8);
|
|
|
|
const size_t n = internal_key.size();
|
|
|
|
uint64_t num = DecodeFixed64(internal_key.data() + n - 8);
|
|
|
|
unsigned char c = num & 0xff;
|
|
|
|
return static_cast<ValueType>(c);
|
|
|
|
}
|
|
|
|
|
|
|
|
// A comparator for internal keys that uses a specified comparator for
|
|
|
|
// the user key portion and breaks ties by decreasing sequence number.
|
|
|
|
class InternalKeyComparator : public Comparator {
|
|
|
|
private:
|
|
|
|
const Comparator* user_comparator_;
|
2013-06-10 22:28:58 +02:00
|
|
|
std::string name_;
|
2011-03-18 23:37:00 +01:00
|
|
|
public:
|
2013-06-10 22:28:58 +02:00
|
|
|
explicit InternalKeyComparator(const Comparator* c) : user_comparator_(c),
|
2013-10-05 07:32:05 +02:00
|
|
|
name_("rocksdb.InternalKeyComparator:" +
|
2013-06-10 22:28:58 +02:00
|
|
|
std::string(user_comparator_->Name())) {
|
|
|
|
}
|
2014-01-27 22:53:22 +01:00
|
|
|
virtual ~InternalKeyComparator() {}
|
2013-06-10 22:28:58 +02:00
|
|
|
|
2015-02-26 20:28:41 +01:00
|
|
|
virtual const char* Name() const override;
|
|
|
|
virtual int Compare(const Slice& a, const Slice& b) const override;
|
|
|
|
virtual void FindShortestSeparator(std::string* start,
|
|
|
|
const Slice& limit) const override;
|
|
|
|
virtual void FindShortSuccessor(std::string* key) const override;
|
2011-03-18 23:37:00 +01:00
|
|
|
|
|
|
|
const Comparator* user_comparator() const { return user_comparator_; }
|
|
|
|
|
|
|
|
int Compare(const InternalKey& a, const InternalKey& b) const;
|
2014-01-27 22:53:22 +01:00
|
|
|
int Compare(const ParsedInternalKey& a, const ParsedInternalKey& b) const;
|
2011-03-18 23:37:00 +01:00
|
|
|
};
|
|
|
|
|
|
|
|
// Modules in this directory should keep internal keys wrapped inside
|
|
|
|
// the following class instead of plain strings so that we do not
|
|
|
|
// incorrectly use string comparisons instead of an InternalKeyComparator.
|
|
|
|
class InternalKey {
|
|
|
|
private:
|
|
|
|
std::string rep_;
|
|
|
|
public:
|
|
|
|
InternalKey() { } // Leave rep_ as empty to indicate it is invalid
|
2014-11-06 20:14:28 +01:00
|
|
|
InternalKey(const Slice& _user_key, SequenceNumber s, ValueType t) {
|
|
|
|
AppendInternalKey(&rep_, ParsedInternalKey(_user_key, s, t));
|
2011-03-18 23:37:00 +01:00
|
|
|
}
|
|
|
|
|
2015-04-24 03:08:37 +02:00
|
|
|
// sets the internal key to be bigger or equal to all internal keys with this
|
|
|
|
// user key
|
|
|
|
void SetMaxPossibleForUserKey(const Slice& _user_key) {
|
|
|
|
AppendInternalKey(&rep_, ParsedInternalKey(_user_key, kMaxSequenceNumber,
|
|
|
|
kValueTypeForSeek));
|
|
|
|
}
|
|
|
|
|
|
|
|
// sets the internal key to be smaller or equal to all internal keys with this
|
|
|
|
// user key
|
|
|
|
void SetMinPossibleForUserKey(const Slice& _user_key) {
|
|
|
|
AppendInternalKey(
|
|
|
|
&rep_, ParsedInternalKey(_user_key, 0, static_cast<ValueType>(0)));
|
|
|
|
}
|
|
|
|
|
[fix] SIGSEGV when VersionEdit in MANIFEST is corrupted
Summary:
This was reported by our customers in task #4295529.
Cause:
* MANIFEST file contains a VersionEdit, which contains file entries whose 'smallest' and 'largest' internal keys are empty. String with zero characters. Root cause of corruption was not investigated. We should report corruption when this happens. However, we currently SIGSEGV.
Here's what happens:
* VersionEdit encodes zero-strings happily and stores them in smallest and largest InternalKeys. InternalKey::Encode() does assert when `rep_.empty()`, but we don't assert in production environemnts. Also, we should never assert as a result of DB corruption.
* As part of our ConsistencyCheck, we call GetLiveFilesMetaData()
* GetLiveFilesMetadata() calls `file->largest.user_key().ToString()`
* user_key() function does: 1. assert(size > 8) (ooops, no assert), 2. returns `Slice(internal_key.data(), internal_key.size() - 8)`
* since `internal_key.size()` is unsigned int, this call translates to `Slice(whatever, 1298471928561892576182756)`. Bazinga.
Fix:
* VersionEdit checks if InternalKey is valid in `VersionEdit::GetInternalKey()`. If it's invalid, returns corruption.
Lessons learned:
* Always keep in mind that even if you `assert()`, production code will continue execution even if assert fails.
* Never `assert` based on DB corruption. Assert only if the code should guarantee that assert can't fail.
Test Plan: dumped offending manifest. Before: assert. Now: corruption
Reviewers: dhruba, haobo, sdong
Reviewed By: dhruba
CC: leveldb
Differential Revision: https://reviews.facebook.net/D18507
2014-05-08 01:52:12 +02:00
|
|
|
bool Valid() const {
|
|
|
|
ParsedInternalKey parsed;
|
|
|
|
return ParseInternalKey(Slice(rep_), &parsed);
|
|
|
|
}
|
|
|
|
|
2011-03-18 23:37:00 +01:00
|
|
|
void DecodeFrom(const Slice& s) { rep_.assign(s.data(), s.size()); }
|
|
|
|
Slice Encode() const {
|
|
|
|
assert(!rep_.empty());
|
|
|
|
return rep_;
|
|
|
|
}
|
|
|
|
|
|
|
|
Slice user_key() const { return ExtractUserKey(rep_); }
|
2015-09-10 23:35:25 +02:00
|
|
|
size_t size() { return rep_.size(); }
|
2011-03-18 23:37:00 +01:00
|
|
|
|
|
|
|
void SetFrom(const ParsedInternalKey& p) {
|
|
|
|
rep_.clear();
|
|
|
|
AppendInternalKey(&rep_, p);
|
|
|
|
}
|
|
|
|
|
|
|
|
void Clear() { rep_.clear(); }
|
2011-10-06 01:30:28 +02:00
|
|
|
|
2012-12-16 03:28:36 +01:00
|
|
|
std::string DebugString(bool hex = false) const;
|
2011-03-18 23:37:00 +01:00
|
|
|
};
|
|
|
|
|
|
|
|
inline int InternalKeyComparator::Compare(
|
|
|
|
const InternalKey& a, const InternalKey& b) const {
|
|
|
|
return Compare(a.Encode(), b.Encode());
|
|
|
|
}
|
|
|
|
|
|
|
|
inline bool ParseInternalKey(const Slice& internal_key,
|
|
|
|
ParsedInternalKey* result) {
|
|
|
|
const size_t n = internal_key.size();
|
|
|
|
if (n < 8) return false;
|
|
|
|
uint64_t num = DecodeFixed64(internal_key.data() + n - 8);
|
|
|
|
unsigned char c = num & 0xff;
|
|
|
|
result->sequence = num >> 8;
|
|
|
|
result->type = static_cast<ValueType>(c);
|
2014-01-27 22:53:22 +01:00
|
|
|
assert(result->type <= ValueType::kMaxValue);
|
2011-03-18 23:37:00 +01:00
|
|
|
result->user_key = Slice(internal_key.data(), n - 8);
|
Support for SingleDelete()
Summary:
This patch fixes #7460559. It introduces SingleDelete as a new database
operation. This operation can be used to delete keys that were never
overwritten (no put following another put of the same key). If an overwritten
key is single deleted the behavior is undefined. Single deletion of a
non-existent key has no effect but multiple consecutive single deletions are
not allowed (see limitations).
In contrast to the conventional Delete() operation, the deletion entry is
removed along with the value when the two are lined up in a compaction. Note:
The semantics are similar to @igor's prototype that allowed to have this
behavior on the granularity of a column family (
https://reviews.facebook.net/D42093 ). This new patch, however, is more
aggressive when it comes to removing tombstones: It removes the SingleDelete
together with the value whenever there is no snapshot between them while the
older patch only did this when the sequence number of the deletion was older
than the earliest snapshot.
Most of the complex additions are in the Compaction Iterator, all other changes
should be relatively straightforward. The patch also includes basic support for
single deletions in db_stress and db_bench.
Limitations:
- Not compatible with cuckoo hash tables
- Single deletions cannot be used in combination with merges and normal
deletions on the same key (other keys are not affected by this)
- Consecutive single deletions are currently not allowed (and older version of
this patch supported this so it could be resurrected if needed)
Test Plan: make all check
Reviewers: yhchiang, sdong, rven, anthony, yoshinorim, igor
Reviewed By: igor
Subscribers: maykov, dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D43179
2015-09-17 20:42:56 +02:00
|
|
|
return IsValueType(result->type);
|
2011-03-18 23:37:00 +01:00
|
|
|
}
|
|
|
|
|
2015-07-14 09:21:41 +02:00
|
|
|
// Update the sequence number in the internal key.
|
|
|
|
// Guarantees not to invalidate ikey.data().
|
Support for SingleDelete()
Summary:
This patch fixes #7460559. It introduces SingleDelete as a new database
operation. This operation can be used to delete keys that were never
overwritten (no put following another put of the same key). If an overwritten
key is single deleted the behavior is undefined. Single deletion of a
non-existent key has no effect but multiple consecutive single deletions are
not allowed (see limitations).
In contrast to the conventional Delete() operation, the deletion entry is
removed along with the value when the two are lined up in a compaction. Note:
The semantics are similar to @igor's prototype that allowed to have this
behavior on the granularity of a column family (
https://reviews.facebook.net/D42093 ). This new patch, however, is more
aggressive when it comes to removing tombstones: It removes the SingleDelete
together with the value whenever there is no snapshot between them while the
older patch only did this when the sequence number of the deletion was older
than the earliest snapshot.
Most of the complex additions are in the Compaction Iterator, all other changes
should be relatively straightforward. The patch also includes basic support for
single deletions in db_stress and db_bench.
Limitations:
- Not compatible with cuckoo hash tables
- Single deletions cannot be used in combination with merges and normal
deletions on the same key (other keys are not affected by this)
- Consecutive single deletions are currently not allowed (and older version of
this patch supported this so it could be resurrected if needed)
Test Plan: make all check
Reviewers: yhchiang, sdong, rven, anthony, yoshinorim, igor
Reviewed By: igor
Subscribers: maykov, dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D43179
2015-09-17 20:42:56 +02:00
|
|
|
inline void UpdateInternalKey(std::string* ikey, uint64_t seq, ValueType t) {
|
2015-07-14 09:21:41 +02:00
|
|
|
size_t ikey_sz = ikey->size();
|
|
|
|
assert(ikey_sz >= 8);
|
2013-02-15 23:31:24 +01:00
|
|
|
uint64_t newval = (seq << 8) | t;
|
2015-07-14 09:21:41 +02:00
|
|
|
|
|
|
|
// Note: Since C++11, strings are guaranteed to be stored contiguously and
|
|
|
|
// string::operator[]() is guaranteed not to change ikey.data().
|
|
|
|
EncodeFixed64(&(*ikey)[ikey_sz - 8], newval);
|
2013-02-15 23:31:24 +01:00
|
|
|
}
|
|
|
|
|
2013-06-14 07:09:08 +02:00
|
|
|
// Get the sequence number from the internal key
|
|
|
|
inline uint64_t GetInternalKeySeqno(const Slice& internal_key) {
|
|
|
|
const size_t n = internal_key.size();
|
|
|
|
assert(n >= 8);
|
|
|
|
uint64_t num = DecodeFixed64(internal_key.data() + n - 8);
|
|
|
|
return num >> 8;
|
|
|
|
}
|
|
|
|
|
|
|
|
|
2011-06-22 04:36:45 +02:00
|
|
|
// A helper class useful for DBImpl::Get()
|
|
|
|
class LookupKey {
|
|
|
|
public:
|
|
|
|
// Initialize *this for looking up user_key at a snapshot with
|
|
|
|
// the specified sequence number.
|
2014-11-06 20:14:28 +01:00
|
|
|
LookupKey(const Slice& _user_key, SequenceNumber sequence);
|
2011-06-22 04:36:45 +02:00
|
|
|
|
|
|
|
~LookupKey();
|
|
|
|
|
|
|
|
// Return a key suitable for lookup in a MemTable.
|
2014-11-11 22:47:22 +01:00
|
|
|
Slice memtable_key() const {
|
|
|
|
return Slice(start_, static_cast<size_t>(end_ - start_));
|
|
|
|
}
|
2011-06-22 04:36:45 +02:00
|
|
|
|
|
|
|
// Return an internal key (suitable for passing to an internal iterator)
|
2014-11-11 22:47:22 +01:00
|
|
|
Slice internal_key() const {
|
|
|
|
return Slice(kstart_, static_cast<size_t>(end_ - kstart_));
|
|
|
|
}
|
2011-06-22 04:36:45 +02:00
|
|
|
|
|
|
|
// Return the user key
|
2014-11-11 22:47:22 +01:00
|
|
|
Slice user_key() const {
|
|
|
|
return Slice(kstart_, static_cast<size_t>(end_ - kstart_ - 8));
|
|
|
|
}
|
2011-06-22 04:36:45 +02:00
|
|
|
|
|
|
|
private:
|
|
|
|
// We construct a char array of the form:
|
|
|
|
// klength varint32 <-- start_
|
|
|
|
// userkey char[klength] <-- kstart_
|
|
|
|
// tag uint64
|
|
|
|
// <-- end_
|
|
|
|
// The array is a suitable MemTable key.
|
|
|
|
// The suffix starting with "userkey" can be used as an InternalKey.
|
|
|
|
const char* start_;
|
|
|
|
const char* kstart_;
|
|
|
|
const char* end_;
|
|
|
|
char space_[200]; // Avoid allocation for short keys
|
|
|
|
|
|
|
|
// No copying allowed
|
|
|
|
LookupKey(const LookupKey&);
|
|
|
|
void operator=(const LookupKey&);
|
|
|
|
};
|
|
|
|
|
|
|
|
inline LookupKey::~LookupKey() {
|
|
|
|
if (start_ != space_) delete[] start_;
|
|
|
|
}
|
|
|
|
|
2014-04-08 01:56:26 +02:00
|
|
|
class IterKey {
|
|
|
|
public:
|
2015-12-16 21:08:30 +01:00
|
|
|
IterKey()
|
|
|
|
: buf_(space_), buf_size_(sizeof(space_)), key_(buf_), key_size_(0) {}
|
2014-04-08 01:56:26 +02:00
|
|
|
|
2014-04-09 02:30:45 +02:00
|
|
|
~IterKey() { ResetBuffer(); }
|
2014-04-08 01:56:26 +02:00
|
|
|
|
2014-04-09 02:30:45 +02:00
|
|
|
Slice GetKey() const { return Slice(key_, key_size_); }
|
2014-04-08 01:56:26 +02:00
|
|
|
|
Support for SingleDelete()
Summary:
This patch fixes #7460559. It introduces SingleDelete as a new database
operation. This operation can be used to delete keys that were never
overwritten (no put following another put of the same key). If an overwritten
key is single deleted the behavior is undefined. Single deletion of a
non-existent key has no effect but multiple consecutive single deletions are
not allowed (see limitations).
In contrast to the conventional Delete() operation, the deletion entry is
removed along with the value when the two are lined up in a compaction. Note:
The semantics are similar to @igor's prototype that allowed to have this
behavior on the granularity of a column family (
https://reviews.facebook.net/D42093 ). This new patch, however, is more
aggressive when it comes to removing tombstones: It removes the SingleDelete
together with the value whenever there is no snapshot between them while the
older patch only did this when the sequence number of the deletion was older
than the earliest snapshot.
Most of the complex additions are in the Compaction Iterator, all other changes
should be relatively straightforward. The patch also includes basic support for
single deletions in db_stress and db_bench.
Limitations:
- Not compatible with cuckoo hash tables
- Single deletions cannot be used in combination with merges and normal
deletions on the same key (other keys are not affected by this)
- Consecutive single deletions are currently not allowed (and older version of
this patch supported this so it could be resurrected if needed)
Test Plan: make all check
Reviewers: yhchiang, sdong, rven, anthony, yoshinorim, igor
Reviewed By: igor
Subscribers: maykov, dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D43179
2015-09-17 20:42:56 +02:00
|
|
|
Slice GetUserKey() const {
|
|
|
|
assert(key_size_ >= 8);
|
|
|
|
return Slice(key_, key_size_ - 8);
|
|
|
|
}
|
|
|
|
|
2015-09-18 20:10:00 +02:00
|
|
|
size_t Size() const { return key_size_; }
|
2014-07-23 21:31:11 +02:00
|
|
|
|
2014-04-09 02:30:45 +02:00
|
|
|
void Clear() { key_size_ = 0; }
|
2014-04-08 01:56:26 +02:00
|
|
|
|
2014-07-23 21:31:11 +02:00
|
|
|
// Append "non_shared_data" to its back, from "shared_len"
|
|
|
|
// This function is used in Block::Iter::ParseNextKey
|
|
|
|
// shared_len: bytes in [0, shard_len-1] would be remained
|
|
|
|
// non_shared_data: data to be append, its length must be >= non_shared_len
|
|
|
|
void TrimAppend(const size_t shared_len, const char* non_shared_data,
|
|
|
|
const size_t non_shared_len) {
|
|
|
|
assert(shared_len <= key_size_);
|
|
|
|
size_t total_size = shared_len + non_shared_len;
|
2015-12-16 21:08:30 +01:00
|
|
|
|
|
|
|
if (IsKeyPinned() /* key is not in buf_ */) {
|
|
|
|
// Copy the key from external memory to buf_ (copy shared_len bytes)
|
|
|
|
EnlargeBufferIfNeeded(total_size);
|
|
|
|
memcpy(buf_, key_, shared_len);
|
|
|
|
} else if (total_size > buf_size_) {
|
2014-07-23 21:31:11 +02:00
|
|
|
// Need to allocate space, delete previous space
|
|
|
|
char* p = new char[total_size];
|
|
|
|
memcpy(p, key_, shared_len);
|
|
|
|
|
2015-12-16 21:08:30 +01:00
|
|
|
if (buf_ != space_) {
|
|
|
|
delete[] buf_;
|
2014-07-23 21:31:11 +02:00
|
|
|
}
|
|
|
|
|
2015-12-16 21:08:30 +01:00
|
|
|
buf_ = p;
|
2014-07-23 21:31:11 +02:00
|
|
|
buf_size_ = total_size;
|
|
|
|
}
|
|
|
|
|
2015-12-16 21:08:30 +01:00
|
|
|
memcpy(buf_ + shared_len, non_shared_data, non_shared_len);
|
|
|
|
key_ = buf_;
|
|
|
|
key_size_ = total_size;
|
2014-07-23 21:31:11 +02:00
|
|
|
}
|
|
|
|
|
2015-12-16 21:08:30 +01:00
|
|
|
Slice SetKey(const Slice& key, bool copy = true) {
|
2014-05-30 23:31:55 +02:00
|
|
|
size_t size = key.size();
|
2015-12-16 21:08:30 +01:00
|
|
|
if (copy) {
|
|
|
|
// Copy key to buf_
|
|
|
|
EnlargeBufferIfNeeded(size);
|
|
|
|
memcpy(buf_, key.data(), size);
|
|
|
|
key_ = buf_;
|
|
|
|
} else {
|
|
|
|
// Update key_ to point to external memory
|
|
|
|
key_ = key.data();
|
|
|
|
}
|
2014-04-09 02:30:45 +02:00
|
|
|
key_size_ = size;
|
Support for SingleDelete()
Summary:
This patch fixes #7460559. It introduces SingleDelete as a new database
operation. This operation can be used to delete keys that were never
overwritten (no put following another put of the same key). If an overwritten
key is single deleted the behavior is undefined. Single deletion of a
non-existent key has no effect but multiple consecutive single deletions are
not allowed (see limitations).
In contrast to the conventional Delete() operation, the deletion entry is
removed along with the value when the two are lined up in a compaction. Note:
The semantics are similar to @igor's prototype that allowed to have this
behavior on the granularity of a column family (
https://reviews.facebook.net/D42093 ). This new patch, however, is more
aggressive when it comes to removing tombstones: It removes the SingleDelete
together with the value whenever there is no snapshot between them while the
older patch only did this when the sequence number of the deletion was older
than the earliest snapshot.
Most of the complex additions are in the Compaction Iterator, all other changes
should be relatively straightforward. The patch also includes basic support for
single deletions in db_stress and db_bench.
Limitations:
- Not compatible with cuckoo hash tables
- Single deletions cannot be used in combination with merges and normal
deletions on the same key (other keys are not affected by this)
- Consecutive single deletions are currently not allowed (and older version of
this patch supported this so it could be resurrected if needed)
Test Plan: make all check
Reviewers: yhchiang, sdong, rven, anthony, yoshinorim, igor
Reviewed By: igor
Subscribers: maykov, dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D43179
2015-09-17 20:42:56 +02:00
|
|
|
return Slice(key_, key_size_);
|
|
|
|
}
|
|
|
|
|
|
|
|
// Copies the content of key, updates the reference to the user key in ikey
|
|
|
|
// and returns a Slice referencing the new copy.
|
|
|
|
Slice SetKey(const Slice& key, ParsedInternalKey* ikey) {
|
|
|
|
size_t key_n = key.size();
|
|
|
|
assert(key_n >= 8);
|
|
|
|
SetKey(key);
|
|
|
|
ikey->user_key = Slice(key_, key_n - 8);
|
|
|
|
return Slice(key_, key_n);
|
|
|
|
}
|
|
|
|
|
|
|
|
// Update the sequence number in the internal key. Guarantees not to
|
|
|
|
// invalidate slices to the key (and the user key).
|
|
|
|
void UpdateInternalKey(uint64_t seq, ValueType t) {
|
2015-12-16 21:08:30 +01:00
|
|
|
assert(!IsKeyPinned());
|
Support for SingleDelete()
Summary:
This patch fixes #7460559. It introduces SingleDelete as a new database
operation. This operation can be used to delete keys that were never
overwritten (no put following another put of the same key). If an overwritten
key is single deleted the behavior is undefined. Single deletion of a
non-existent key has no effect but multiple consecutive single deletions are
not allowed (see limitations).
In contrast to the conventional Delete() operation, the deletion entry is
removed along with the value when the two are lined up in a compaction. Note:
The semantics are similar to @igor's prototype that allowed to have this
behavior on the granularity of a column family (
https://reviews.facebook.net/D42093 ). This new patch, however, is more
aggressive when it comes to removing tombstones: It removes the SingleDelete
together with the value whenever there is no snapshot between them while the
older patch only did this when the sequence number of the deletion was older
than the earliest snapshot.
Most of the complex additions are in the Compaction Iterator, all other changes
should be relatively straightforward. The patch also includes basic support for
single deletions in db_stress and db_bench.
Limitations:
- Not compatible with cuckoo hash tables
- Single deletions cannot be used in combination with merges and normal
deletions on the same key (other keys are not affected by this)
- Consecutive single deletions are currently not allowed (and older version of
this patch supported this so it could be resurrected if needed)
Test Plan: make all check
Reviewers: yhchiang, sdong, rven, anthony, yoshinorim, igor
Reviewed By: igor
Subscribers: maykov, dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D43179
2015-09-17 20:42:56 +02:00
|
|
|
assert(key_size_ >= 8);
|
|
|
|
uint64_t newval = (seq << 8) | t;
|
2015-12-16 21:08:30 +01:00
|
|
|
EncodeFixed64(&buf_[key_size_ - 8], newval);
|
2014-04-08 01:56:26 +02:00
|
|
|
}
|
|
|
|
|
2015-12-16 21:08:30 +01:00
|
|
|
bool IsKeyPinned() const { return (key_ != buf_); }
|
|
|
|
|
2014-06-19 01:36:48 +02:00
|
|
|
void SetInternalKey(const Slice& key_prefix, const Slice& user_key,
|
|
|
|
SequenceNumber s,
|
2014-04-08 01:56:26 +02:00
|
|
|
ValueType value_type = kValueTypeForSeek) {
|
2014-06-19 01:36:48 +02:00
|
|
|
size_t psize = key_prefix.size();
|
2014-04-08 01:56:26 +02:00
|
|
|
size_t usize = user_key.size();
|
2014-06-19 01:36:48 +02:00
|
|
|
EnlargeBufferIfNeeded(psize + usize + sizeof(uint64_t));
|
|
|
|
if (psize > 0) {
|
2015-12-16 21:08:30 +01:00
|
|
|
memcpy(buf_, key_prefix.data(), psize);
|
2014-06-19 01:36:48 +02:00
|
|
|
}
|
2015-12-16 21:08:30 +01:00
|
|
|
memcpy(buf_ + psize, user_key.data(), usize);
|
|
|
|
EncodeFixed64(buf_ + usize + psize, PackSequenceAndType(s, value_type));
|
|
|
|
|
|
|
|
key_ = buf_;
|
2014-06-19 01:36:48 +02:00
|
|
|
key_size_ = psize + usize + sizeof(uint64_t);
|
|
|
|
}
|
|
|
|
|
|
|
|
void SetInternalKey(const Slice& user_key, SequenceNumber s,
|
|
|
|
ValueType value_type = kValueTypeForSeek) {
|
|
|
|
SetInternalKey(Slice(), user_key, s, value_type);
|
|
|
|
}
|
|
|
|
|
|
|
|
void Reserve(size_t size) {
|
|
|
|
EnlargeBufferIfNeeded(size);
|
|
|
|
key_size_ = size;
|
2014-04-08 01:56:26 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
void SetInternalKey(const ParsedInternalKey& parsed_key) {
|
2014-06-19 01:36:48 +02:00
|
|
|
SetInternalKey(Slice(), parsed_key);
|
|
|
|
}
|
|
|
|
|
|
|
|
void SetInternalKey(const Slice& key_prefix,
|
|
|
|
const ParsedInternalKey& parsed_key_suffix) {
|
|
|
|
SetInternalKey(key_prefix, parsed_key_suffix.user_key,
|
|
|
|
parsed_key_suffix.sequence, parsed_key_suffix.type);
|
2014-04-08 01:56:26 +02:00
|
|
|
}
|
|
|
|
|
2014-07-01 20:05:05 +02:00
|
|
|
void EncodeLengthPrefixedKey(const Slice& key) {
|
|
|
|
auto size = key.size();
|
2014-11-11 22:47:22 +01:00
|
|
|
EnlargeBufferIfNeeded(size + static_cast<size_t>(VarintLength(size)));
|
2015-12-16 21:08:30 +01:00
|
|
|
char* ptr = EncodeVarint32(buf_, static_cast<uint32_t>(size));
|
2014-07-01 20:05:05 +02:00
|
|
|
memcpy(ptr, key.data(), size);
|
2015-12-16 21:08:30 +01:00
|
|
|
key_ = buf_;
|
2014-07-01 20:05:05 +02:00
|
|
|
}
|
|
|
|
|
2014-04-08 01:56:26 +02:00
|
|
|
private:
|
2015-12-16 21:08:30 +01:00
|
|
|
char* buf_;
|
2014-04-08 01:56:26 +02:00
|
|
|
size_t buf_size_;
|
2015-12-16 21:08:30 +01:00
|
|
|
const char* key_;
|
2014-04-08 01:56:26 +02:00
|
|
|
size_t key_size_;
|
|
|
|
char space_[32]; // Avoid allocation for short keys
|
|
|
|
|
2014-04-09 02:30:45 +02:00
|
|
|
void ResetBuffer() {
|
2015-12-16 21:08:30 +01:00
|
|
|
if (buf_ != space_) {
|
|
|
|
delete[] buf_;
|
|
|
|
buf_ = space_;
|
2014-04-09 02:30:45 +02:00
|
|
|
}
|
2014-04-24 02:51:16 +02:00
|
|
|
buf_size_ = sizeof(space_);
|
2014-04-09 02:30:45 +02:00
|
|
|
key_size_ = 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
// Enlarge the buffer size if needed based on key_size.
|
|
|
|
// By default, static allocated buffer is used. Once there is a key
|
|
|
|
// larger than the static allocated buffer, another buffer is dynamically
|
|
|
|
// allocated, until a larger key buffer is requested. In that case, we
|
|
|
|
// reallocate buffer and delete the old one.
|
|
|
|
void EnlargeBufferIfNeeded(size_t key_size) {
|
|
|
|
// If size is smaller than buffer size, continue using current buffer,
|
|
|
|
// or the static allocated one, as default
|
|
|
|
if (key_size > buf_size_) {
|
|
|
|
// Need to enlarge the buffer.
|
|
|
|
ResetBuffer();
|
2015-12-16 21:08:30 +01:00
|
|
|
buf_ = new char[key_size];
|
2014-04-09 02:30:45 +02:00
|
|
|
buf_size_ = key_size;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2014-04-08 01:56:26 +02:00
|
|
|
// No copying allowed
|
|
|
|
IterKey(const IterKey&) = delete;
|
|
|
|
void operator=(const IterKey&) = delete;
|
|
|
|
};
|
|
|
|
|
2014-04-10 23:19:43 +02:00
|
|
|
class InternalKeySliceTransform : public SliceTransform {
|
|
|
|
public:
|
|
|
|
explicit InternalKeySliceTransform(const SliceTransform* transform)
|
|
|
|
: transform_(transform) {}
|
|
|
|
|
2015-02-26 20:28:41 +01:00
|
|
|
virtual const char* Name() const override { return transform_->Name(); }
|
2014-04-10 23:19:43 +02:00
|
|
|
|
2015-02-26 20:28:41 +01:00
|
|
|
virtual Slice Transform(const Slice& src) const override {
|
2014-04-10 23:19:43 +02:00
|
|
|
auto user_key = ExtractUserKey(src);
|
|
|
|
return transform_->Transform(user_key);
|
|
|
|
}
|
|
|
|
|
2015-02-26 20:28:41 +01:00
|
|
|
virtual bool InDomain(const Slice& src) const override {
|
2014-04-10 23:19:43 +02:00
|
|
|
auto user_key = ExtractUserKey(src);
|
|
|
|
return transform_->InDomain(user_key);
|
|
|
|
}
|
|
|
|
|
2015-02-26 20:28:41 +01:00
|
|
|
virtual bool InRange(const Slice& dst) const override {
|
2014-04-10 23:19:43 +02:00
|
|
|
auto user_key = ExtractUserKey(dst);
|
|
|
|
return transform_->InRange(user_key);
|
|
|
|
}
|
|
|
|
|
|
|
|
const SliceTransform* user_prefix_extractor() const { return transform_; }
|
|
|
|
|
|
|
|
private:
|
|
|
|
// Like comparator, InternalKeySliceTransform will not take care of the
|
|
|
|
// deletion of transform_
|
|
|
|
const SliceTransform* const transform_;
|
|
|
|
};
|
|
|
|
|
2016-04-02 00:23:46 +02:00
|
|
|
// Read the key of a record from a write batch.
|
|
|
|
// if this record represent the default column family then cf_record
|
|
|
|
// must be passed as false, otherwise it must be passed as true.
|
|
|
|
extern bool ReadKeyFromWriteBatchEntry(Slice* input, Slice* key,
|
|
|
|
bool cf_record);
|
|
|
|
|
2014-08-19 00:19:17 +02:00
|
|
|
// Read record from a write batch piece from input.
|
|
|
|
// tag, column_family, key, value and blob are return values. Callers own the
|
|
|
|
// Slice they point to.
|
|
|
|
// Tag is defined as ValueType.
|
|
|
|
// input will be advanced to after the record.
|
|
|
|
extern Status ReadRecordFromWriteBatch(Slice* input, char* tag,
|
|
|
|
uint32_t* column_family, Slice* key,
|
Modification of WriteBatch to support two phase commit
Summary: Adds three new WriteBatch data types: Prepare(xid), Commit(xid), Rollback(xid). Prepare(xid) should precede the (single) operation to which is applies. There can obviously be multiple Prepare(xid) markers. There should only be one Rollback(xid) or Commit(xid) marker yet not both. None of this logic is currently enforced and will most likely be implemented further up such as in the memtableinserter. All three markers are similar to PutLogData in that they are writebatch meta-data, ie stored but not counted. All three markers differ from PutLogData in that they will actually be written to disk. As for WriteBatchWithIndex, Prepare, Commit, Rollback are all implemented just as PutLogData and none are tested just as PutLogData.
Test Plan: single unit test in write_batch_test.
Reviewers: hermanlee4, sdong, anthony
Subscribers: andrewkr, vasilep, dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D54093
2016-04-08 08:35:51 +02:00
|
|
|
Slice* value, Slice* blob, Slice* xid);
|
2013-10-04 06:49:15 +02:00
|
|
|
} // namespace rocksdb
|