2016-02-10 00:12:00 +01:00
|
|
|
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
|
2017-07-16 01:03:42 +02:00
|
|
|
// This source code is licensed under both the GPLv2 (found in the
|
|
|
|
// COPYING file in the root directory) and Apache 2.0 License
|
|
|
|
// (found in the LICENSE.Apache file in the root directory).
|
Pessimistic Transactions
Summary:
Initial implementation of Pessimistic Transactions. This diff contains the api changes discussed in D38913. This diff is pretty large, so let me know if people would prefer to meet up to discuss it.
MyRocks folks: please take a look at the API in include/rocksdb/utilities/transaction[_db].h and let me know if you have any issues.
Also, you'll notice a couple of TODOs in the implementation of RollbackToSavePoint(). After chatting with Siying, I'm going to send out a separate diff for an alternate implementation of this feature that implements the rollback inside of WriteBatch/WriteBatchWithIndex. We can then decide which route is preferable.
Next, I'm planning on doing some perf testing and then integrating this diff into MongoRocks for further testing.
Test Plan: Unit tests, db_bench parallel testing.
Reviewers: igor, rven, sdong, yhchiang, yoshinorim
Reviewed By: sdong
Subscribers: hermanlee4, maykov, spetrunia, leveldb, dhruba
Differential Revision: https://reviews.facebook.net/D40869
2015-05-26 02:37:33 +02:00
|
|
|
|
|
|
|
#ifndef ROCKSDB_LITE
|
|
|
|
|
2017-08-08 01:07:40 +02:00
|
|
|
#include "utilities/transactions/pessimistic_transaction.h"
|
Pessimistic Transactions
Summary:
Initial implementation of Pessimistic Transactions. This diff contains the api changes discussed in D38913. This diff is pretty large, so let me know if people would prefer to meet up to discuss it.
MyRocks folks: please take a look at the API in include/rocksdb/utilities/transaction[_db].h and let me know if you have any issues.
Also, you'll notice a couple of TODOs in the implementation of RollbackToSavePoint(). After chatting with Siying, I'm going to send out a separate diff for an alternate implementation of this feature that implements the rollback inside of WriteBatch/WriteBatchWithIndex. We can then decide which route is preferable.
Next, I'm planning on doing some perf testing and then integrating this diff into MongoRocks for further testing.
Test Plan: Unit tests, db_bench parallel testing.
Reviewers: igor, rven, sdong, yhchiang, yoshinorim
Reviewed By: sdong
Subscribers: hermanlee4, maykov, spetrunia, leveldb, dhruba
Differential Revision: https://reviews.facebook.net/D40869
2015-05-26 02:37:33 +02:00
|
|
|
|
|
|
|
#include <map>
|
|
|
|
#include <set>
|
|
|
|
#include <string>
|
|
|
|
#include <vector>
|
|
|
|
|
|
|
|
#include "db/column_family.h"
|
2019-05-31 20:52:59 +02:00
|
|
|
#include "db/db_impl/db_impl.h"
|
Pessimistic Transactions
Summary:
Initial implementation of Pessimistic Transactions. This diff contains the api changes discussed in D38913. This diff is pretty large, so let me know if people would prefer to meet up to discuss it.
MyRocks folks: please take a look at the API in include/rocksdb/utilities/transaction[_db].h and let me know if you have any issues.
Also, you'll notice a couple of TODOs in the implementation of RollbackToSavePoint(). After chatting with Siying, I'm going to send out a separate diff for an alternate implementation of this feature that implements the rollback inside of WriteBatch/WriteBatchWithIndex. We can then decide which route is preferable.
Next, I'm planning on doing some perf testing and then integrating this diff into MongoRocks for further testing.
Test Plan: Unit tests, db_bench parallel testing.
Reviewers: igor, rven, sdong, yhchiang, yoshinorim
Reviewed By: sdong
Subscribers: hermanlee4, maykov, spetrunia, leveldb, dhruba
Differential Revision: https://reviews.facebook.net/D40869
2015-05-26 02:37:33 +02:00
|
|
|
#include "rocksdb/comparator.h"
|
|
|
|
#include "rocksdb/db.h"
|
2015-08-01 00:18:27 +02:00
|
|
|
#include "rocksdb/snapshot.h"
|
Pessimistic Transactions
Summary:
Initial implementation of Pessimistic Transactions. This diff contains the api changes discussed in D38913. This diff is pretty large, so let me know if people would prefer to meet up to discuss it.
MyRocks folks: please take a look at the API in include/rocksdb/utilities/transaction[_db].h and let me know if you have any issues.
Also, you'll notice a couple of TODOs in the implementation of RollbackToSavePoint(). After chatting with Siying, I'm going to send out a separate diff for an alternate implementation of this feature that implements the rollback inside of WriteBatch/WriteBatchWithIndex. We can then decide which route is preferable.
Next, I'm planning on doing some perf testing and then integrating this diff into MongoRocks for further testing.
Test Plan: Unit tests, db_bench parallel testing.
Reviewers: igor, rven, sdong, yhchiang, yoshinorim
Reviewed By: sdong
Subscribers: hermanlee4, maykov, spetrunia, leveldb, dhruba
Differential Revision: https://reviews.facebook.net/D40869
2015-05-26 02:37:33 +02:00
|
|
|
#include "rocksdb/status.h"
|
|
|
|
#include "rocksdb/utilities/transaction_db.h"
|
2019-05-31 02:39:43 +02:00
|
|
|
#include "test_util/sync_point.h"
|
2017-07-29 01:23:50 +02:00
|
|
|
#include "util/cast_util.h"
|
Pessimistic Transactions
Summary:
Initial implementation of Pessimistic Transactions. This diff contains the api changes discussed in D38913. This diff is pretty large, so let me know if people would prefer to meet up to discuss it.
MyRocks folks: please take a look at the API in include/rocksdb/utilities/transaction[_db].h and let me know if you have any issues.
Also, you'll notice a couple of TODOs in the implementation of RollbackToSavePoint(). After chatting with Siying, I'm going to send out a separate diff for an alternate implementation of this feature that implements the rollback inside of WriteBatch/WriteBatchWithIndex. We can then decide which route is preferable.
Next, I'm planning on doing some perf testing and then integrating this diff into MongoRocks for further testing.
Test Plan: Unit tests, db_bench parallel testing.
Reviewers: igor, rven, sdong, yhchiang, yoshinorim
Reviewed By: sdong
Subscribers: hermanlee4, maykov, spetrunia, leveldb, dhruba
Differential Revision: https://reviews.facebook.net/D40869
2015-05-26 02:37:33 +02:00
|
|
|
#include "util/string_util.h"
|
2017-08-06 02:17:48 +02:00
|
|
|
#include "utilities/transactions/pessimistic_transaction_db.h"
|
Pessimistic Transactions
Summary:
Initial implementation of Pessimistic Transactions. This diff contains the api changes discussed in D38913. This diff is pretty large, so let me know if people would prefer to meet up to discuss it.
MyRocks folks: please take a look at the API in include/rocksdb/utilities/transaction[_db].h and let me know if you have any issues.
Also, you'll notice a couple of TODOs in the implementation of RollbackToSavePoint(). After chatting with Siying, I'm going to send out a separate diff for an alternate implementation of this feature that implements the rollback inside of WriteBatch/WriteBatchWithIndex. We can then decide which route is preferable.
Next, I'm planning on doing some perf testing and then integrating this diff into MongoRocks for further testing.
Test Plan: Unit tests, db_bench parallel testing.
Reviewers: igor, rven, sdong, yhchiang, yoshinorim
Reviewed By: sdong
Subscribers: hermanlee4, maykov, spetrunia, leveldb, dhruba
Differential Revision: https://reviews.facebook.net/D40869
2015-05-26 02:37:33 +02:00
|
|
|
#include "utilities/transactions/transaction_util.h"
|
|
|
|
|
2020-02-20 21:07:53 +01:00
|
|
|
namespace ROCKSDB_NAMESPACE {
|
Pessimistic Transactions
Summary:
Initial implementation of Pessimistic Transactions. This diff contains the api changes discussed in D38913. This diff is pretty large, so let me know if people would prefer to meet up to discuss it.
MyRocks folks: please take a look at the API in include/rocksdb/utilities/transaction[_db].h and let me know if you have any issues.
Also, you'll notice a couple of TODOs in the implementation of RollbackToSavePoint(). After chatting with Siying, I'm going to send out a separate diff for an alternate implementation of this feature that implements the rollback inside of WriteBatch/WriteBatchWithIndex. We can then decide which route is preferable.
Next, I'm planning on doing some perf testing and then integrating this diff into MongoRocks for further testing.
Test Plan: Unit tests, db_bench parallel testing.
Reviewers: igor, rven, sdong, yhchiang, yoshinorim
Reviewed By: sdong
Subscribers: hermanlee4, maykov, spetrunia, leveldb, dhruba
Differential Revision: https://reviews.facebook.net/D40869
2015-05-26 02:37:33 +02:00
|
|
|
|
|
|
|
struct WriteOptions;
|
|
|
|
|
2017-08-08 01:07:40 +02:00
|
|
|
std::atomic<TransactionID> PessimisticTransaction::txn_id_counter_(1);
|
Pessimistic Transactions
Summary:
Initial implementation of Pessimistic Transactions. This diff contains the api changes discussed in D38913. This diff is pretty large, so let me know if people would prefer to meet up to discuss it.
MyRocks folks: please take a look at the API in include/rocksdb/utilities/transaction[_db].h and let me know if you have any issues.
Also, you'll notice a couple of TODOs in the implementation of RollbackToSavePoint(). After chatting with Siying, I'm going to send out a separate diff for an alternate implementation of this feature that implements the rollback inside of WriteBatch/WriteBatchWithIndex. We can then decide which route is preferable.
Next, I'm planning on doing some perf testing and then integrating this diff into MongoRocks for further testing.
Test Plan: Unit tests, db_bench parallel testing.
Reviewers: igor, rven, sdong, yhchiang, yoshinorim
Reviewed By: sdong
Subscribers: hermanlee4, maykov, spetrunia, leveldb, dhruba
Differential Revision: https://reviews.facebook.net/D40869
2015-05-26 02:37:33 +02:00
|
|
|
|
2017-08-08 01:07:40 +02:00
|
|
|
TransactionID PessimisticTransaction::GenTxnID() {
|
Pessimistic Transactions
Summary:
Initial implementation of Pessimistic Transactions. This diff contains the api changes discussed in D38913. This diff is pretty large, so let me know if people would prefer to meet up to discuss it.
MyRocks folks: please take a look at the API in include/rocksdb/utilities/transaction[_db].h and let me know if you have any issues.
Also, you'll notice a couple of TODOs in the implementation of RollbackToSavePoint(). After chatting with Siying, I'm going to send out a separate diff for an alternate implementation of this feature that implements the rollback inside of WriteBatch/WriteBatchWithIndex. We can then decide which route is preferable.
Next, I'm planning on doing some perf testing and then integrating this diff into MongoRocks for further testing.
Test Plan: Unit tests, db_bench parallel testing.
Reviewers: igor, rven, sdong, yhchiang, yoshinorim
Reviewed By: sdong
Subscribers: hermanlee4, maykov, spetrunia, leveldb, dhruba
Differential Revision: https://reviews.facebook.net/D40869
2015-05-26 02:37:33 +02:00
|
|
|
return txn_id_counter_.fetch_add(1);
|
|
|
|
}
|
|
|
|
|
2017-08-17 01:49:11 +02:00
|
|
|
PessimisticTransaction::PessimisticTransaction(
|
|
|
|
TransactionDB* txn_db, const WriteOptions& write_options,
|
2019-01-16 03:07:50 +01:00
|
|
|
const TransactionOptions& txn_options, const bool init)
|
2020-10-19 19:12:53 +02:00
|
|
|
: TransactionBaseImpl(
|
|
|
|
txn_db->GetRootDB(), write_options,
|
|
|
|
static_cast_with_check<PessimisticTransactionDB>(txn_db)
|
|
|
|
->GetLockTrackerFactory()),
|
Pessimistic Transactions
Summary:
Initial implementation of Pessimistic Transactions. This diff contains the api changes discussed in D38913. This diff is pretty large, so let me know if people would prefer to meet up to discuss it.
MyRocks folks: please take a look at the API in include/rocksdb/utilities/transaction[_db].h and let me know if you have any issues.
Also, you'll notice a couple of TODOs in the implementation of RollbackToSavePoint(). After chatting with Siying, I'm going to send out a separate diff for an alternate implementation of this feature that implements the rollback inside of WriteBatch/WriteBatchWithIndex. We can then decide which route is preferable.
Next, I'm planning on doing some perf testing and then integrating this diff into MongoRocks for further testing.
Test Plan: Unit tests, db_bench parallel testing.
Reviewers: igor, rven, sdong, yhchiang, yoshinorim
Reviewed By: sdong
Subscribers: hermanlee4, maykov, spetrunia, leveldb, dhruba
Differential Revision: https://reviews.facebook.net/D40869
2015-05-26 02:37:33 +02:00
|
|
|
txn_db_impl_(nullptr),
|
2017-08-03 17:46:47 +02:00
|
|
|
expiration_time_(0),
|
2016-02-03 04:19:17 +01:00
|
|
|
txn_id_(0),
|
2016-09-28 02:43:06 +02:00
|
|
|
waiting_cf_id_(0),
|
|
|
|
waiting_key_(nullptr),
|
2016-09-28 10:23:33 +02:00
|
|
|
lock_timeout_(0),
|
|
|
|
deadlock_detect_(false),
|
2018-09-11 01:45:59 +02:00
|
|
|
deadlock_detect_depth_(0),
|
|
|
|
skip_concurrency_control_(false) {
|
2020-04-29 22:06:27 +02:00
|
|
|
txn_db_impl_ = static_cast_with_check<PessimisticTransactionDB>(txn_db);
|
|
|
|
db_impl_ = static_cast_with_check<DBImpl>(db_);
|
2019-01-16 03:07:50 +01:00
|
|
|
if (init) {
|
|
|
|
Initialize(txn_options);
|
|
|
|
}
|
2016-02-03 04:19:17 +01:00
|
|
|
}
|
|
|
|
|
2017-08-08 01:07:40 +02:00
|
|
|
void PessimisticTransaction::Initialize(const TransactionOptions& txn_options) {
|
2016-02-03 04:19:17 +01:00
|
|
|
txn_id_ = GenTxnID();
|
|
|
|
|
2016-10-05 22:39:00 +02:00
|
|
|
txn_state_ = STARTED;
|
2016-02-03 04:19:17 +01:00
|
|
|
|
2016-09-28 10:23:33 +02:00
|
|
|
deadlock_detect_ = txn_options.deadlock_detect;
|
|
|
|
deadlock_detect_depth_ = txn_options.deadlock_detect_depth;
|
2017-04-11 00:38:34 +02:00
|
|
|
write_batch_.SetMaxBytes(txn_options.max_write_batch_size);
|
2018-09-11 01:45:59 +02:00
|
|
|
skip_concurrency_control_ = txn_options.skip_concurrency_control;
|
2016-09-28 10:23:33 +02:00
|
|
|
|
2016-02-03 04:19:17 +01:00
|
|
|
lock_timeout_ = txn_options.lock_timeout * 1000;
|
Pessimistic Transactions
Summary:
Initial implementation of Pessimistic Transactions. This diff contains the api changes discussed in D38913. This diff is pretty large, so let me know if people would prefer to meet up to discuss it.
MyRocks folks: please take a look at the API in include/rocksdb/utilities/transaction[_db].h and let me know if you have any issues.
Also, you'll notice a couple of TODOs in the implementation of RollbackToSavePoint(). After chatting with Siying, I'm going to send out a separate diff for an alternate implementation of this feature that implements the rollback inside of WriteBatch/WriteBatchWithIndex. We can then decide which route is preferable.
Next, I'm planning on doing some perf testing and then integrating this diff into MongoRocks for further testing.
Test Plan: Unit tests, db_bench parallel testing.
Reviewers: igor, rven, sdong, yhchiang, yoshinorim
Reviewed By: sdong
Subscribers: hermanlee4, maykov, spetrunia, leveldb, dhruba
Differential Revision: https://reviews.facebook.net/D40869
2015-05-26 02:37:33 +02:00
|
|
|
if (lock_timeout_ < 0) {
|
|
|
|
// Lock timeout not set, use default
|
2015-09-08 21:36:48 +02:00
|
|
|
lock_timeout_ =
|
|
|
|
txn_db_impl_->GetTxnDBOptions().transaction_lock_timeout * 1000;
|
Pessimistic Transactions
Summary:
Initial implementation of Pessimistic Transactions. This diff contains the api changes discussed in D38913. This diff is pretty large, so let me know if people would prefer to meet up to discuss it.
MyRocks folks: please take a look at the API in include/rocksdb/utilities/transaction[_db].h and let me know if you have any issues.
Also, you'll notice a couple of TODOs in the implementation of RollbackToSavePoint(). After chatting with Siying, I'm going to send out a separate diff for an alternate implementation of this feature that implements the rollback inside of WriteBatch/WriteBatchWithIndex. We can then decide which route is preferable.
Next, I'm planning on doing some perf testing and then integrating this diff into MongoRocks for further testing.
Test Plan: Unit tests, db_bench parallel testing.
Reviewers: igor, rven, sdong, yhchiang, yoshinorim
Reviewed By: sdong
Subscribers: hermanlee4, maykov, spetrunia, leveldb, dhruba
Differential Revision: https://reviews.facebook.net/D40869
2015-05-26 02:37:33 +02:00
|
|
|
}
|
|
|
|
|
2016-02-03 04:19:17 +01:00
|
|
|
if (txn_options.expiration >= 0) {
|
|
|
|
expiration_time_ = start_time_ + txn_options.expiration * 1000;
|
|
|
|
} else {
|
|
|
|
expiration_time_ = 0;
|
|
|
|
}
|
|
|
|
|
Pessimistic Transactions
Summary:
Initial implementation of Pessimistic Transactions. This diff contains the api changes discussed in D38913. This diff is pretty large, so let me know if people would prefer to meet up to discuss it.
MyRocks folks: please take a look at the API in include/rocksdb/utilities/transaction[_db].h and let me know if you have any issues.
Also, you'll notice a couple of TODOs in the implementation of RollbackToSavePoint(). After chatting with Siying, I'm going to send out a separate diff for an alternate implementation of this feature that implements the rollback inside of WriteBatch/WriteBatchWithIndex. We can then decide which route is preferable.
Next, I'm planning on doing some perf testing and then integrating this diff into MongoRocks for further testing.
Test Plan: Unit tests, db_bench parallel testing.
Reviewers: igor, rven, sdong, yhchiang, yoshinorim
Reviewed By: sdong
Subscribers: hermanlee4, maykov, spetrunia, leveldb, dhruba
Differential Revision: https://reviews.facebook.net/D40869
2015-05-26 02:37:33 +02:00
|
|
|
if (txn_options.set_snapshot) {
|
|
|
|
SetSnapshot();
|
|
|
|
}
|
2016-03-04 00:36:26 +01:00
|
|
|
|
2016-02-02 02:07:05 +01:00
|
|
|
if (expiration_time_ > 0) {
|
|
|
|
txn_db_impl_->InsertExpirableTransaction(txn_id_, this);
|
|
|
|
}
|
2017-11-02 01:23:52 +01:00
|
|
|
use_only_the_last_commit_time_batch_for_recovery_ =
|
|
|
|
txn_options.use_only_the_last_commit_time_batch_for_recovery;
|
2020-05-01 01:21:05 +02:00
|
|
|
skip_prepare_ = txn_options.skip_prepare;
|
Pessimistic Transactions
Summary:
Initial implementation of Pessimistic Transactions. This diff contains the api changes discussed in D38913. This diff is pretty large, so let me know if people would prefer to meet up to discuss it.
MyRocks folks: please take a look at the API in include/rocksdb/utilities/transaction[_db].h and let me know if you have any issues.
Also, you'll notice a couple of TODOs in the implementation of RollbackToSavePoint(). After chatting with Siying, I'm going to send out a separate diff for an alternate implementation of this feature that implements the rollback inside of WriteBatch/WriteBatchWithIndex. We can then decide which route is preferable.
Next, I'm planning on doing some perf testing and then integrating this diff into MongoRocks for further testing.
Test Plan: Unit tests, db_bench parallel testing.
Reviewers: igor, rven, sdong, yhchiang, yoshinorim
Reviewed By: sdong
Subscribers: hermanlee4, maykov, spetrunia, leveldb, dhruba
Differential Revision: https://reviews.facebook.net/D40869
2015-05-26 02:37:33 +02:00
|
|
|
}
|
|
|
|
|
2017-08-08 01:07:40 +02:00
|
|
|
PessimisticTransaction::~PessimisticTransaction() {
|
Replace tracked_keys with a new LockTracker interface in TransactionDB (#7013)
Summary:
We're going to support more locking protocols such as range lock in transaction.
However, in current design, `TransactionBase` has a member `tracked_keys` which assumes that point lock (lock a single key) is used, and is used in snapshot checking (isolation protocol). When using range lock, we may use read committed instead of snapshot checking as the isolation protocol.
The most significant usage scenarios of `tracked_keys` are:
1. pessimistic transaction uses it to track the locked keys, and unlock these keys when commit or rollback.
2. optimistic transaction does not lock keys upfront, it only tracks the lock intentions in tracked_keys, and do write conflict checking when commit.
3. each `SavePoint` tracks the keys that are locked since the `SavePoint`, `RollbackToSavePoint` or `PopSavePoint` relies on both the tracked keys in `SavePoint`s and `tracked_keys`.
Based on these scenarios, if we can abstract out a `LockTracker` interface to hold a set of tracked locks (can be keys or key ranges), and have methods that can be composed together to implement the scenarios, then `tracked_keys` can be an internal data structure of one implementation of `LockTracker`. See `utilities/transactions/lock/lock_tracker.h` for the detailed interface design, and `utilities/transactions/lock/point_lock_tracker.cc` for the implementation.
In the future, a `RangeLockTracker` can be implemented to track range locks without affecting other components.
After this PR, a clean interface for lock manager should be possible, and then ideally, we can have pluggable locking protocols.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/7013
Test Plan: Run `transaction_test` and `optimistic_transaction_test`.
Reviewed By: ajkr
Differential Revision: D22163706
Pulled By: cheng-chang
fbshipit-source-id: f2860577b5334e31dd2994f5bc6d7c40d502b1b4
2020-08-06 21:36:48 +02:00
|
|
|
txn_db_impl_->UnLock(this, *tracked_locks_);
|
2016-02-02 02:07:05 +01:00
|
|
|
if (expiration_time_ > 0) {
|
|
|
|
txn_db_impl_->RemoveExpirableTransaction(txn_id_);
|
|
|
|
}
|
2020-03-06 21:29:30 +01:00
|
|
|
if (!name_.empty() && txn_state_ != COMMITTED) {
|
2016-04-18 20:15:50 +02:00
|
|
|
txn_db_impl_->UnregisterTransaction(this);
|
|
|
|
}
|
Pessimistic Transactions
Summary:
Initial implementation of Pessimistic Transactions. This diff contains the api changes discussed in D38913. This diff is pretty large, so let me know if people would prefer to meet up to discuss it.
MyRocks folks: please take a look at the API in include/rocksdb/utilities/transaction[_db].h and let me know if you have any issues.
Also, you'll notice a couple of TODOs in the implementation of RollbackToSavePoint(). After chatting with Siying, I'm going to send out a separate diff for an alternate implementation of this feature that implements the rollback inside of WriteBatch/WriteBatchWithIndex. We can then decide which route is preferable.
Next, I'm planning on doing some perf testing and then integrating this diff into MongoRocks for further testing.
Test Plan: Unit tests, db_bench parallel testing.
Reviewers: igor, rven, sdong, yhchiang, yoshinorim
Reviewed By: sdong
Subscribers: hermanlee4, maykov, spetrunia, leveldb, dhruba
Differential Revision: https://reviews.facebook.net/D40869
2015-05-26 02:37:33 +02:00
|
|
|
}
|
|
|
|
|
2017-08-08 01:07:40 +02:00
|
|
|
void PessimisticTransaction::Clear() {
|
Replace tracked_keys with a new LockTracker interface in TransactionDB (#7013)
Summary:
We're going to support more locking protocols such as range lock in transaction.
However, in current design, `TransactionBase` has a member `tracked_keys` which assumes that point lock (lock a single key) is used, and is used in snapshot checking (isolation protocol). When using range lock, we may use read committed instead of snapshot checking as the isolation protocol.
The most significant usage scenarios of `tracked_keys` are:
1. pessimistic transaction uses it to track the locked keys, and unlock these keys when commit or rollback.
2. optimistic transaction does not lock keys upfront, it only tracks the lock intentions in tracked_keys, and do write conflict checking when commit.
3. each `SavePoint` tracks the keys that are locked since the `SavePoint`, `RollbackToSavePoint` or `PopSavePoint` relies on both the tracked keys in `SavePoint`s and `tracked_keys`.
Based on these scenarios, if we can abstract out a `LockTracker` interface to hold a set of tracked locks (can be keys or key ranges), and have methods that can be composed together to implement the scenarios, then `tracked_keys` can be an internal data structure of one implementation of `LockTracker`. See `utilities/transactions/lock/lock_tracker.h` for the detailed interface design, and `utilities/transactions/lock/point_lock_tracker.cc` for the implementation.
In the future, a `RangeLockTracker` can be implemented to track range locks without affecting other components.
After this PR, a clean interface for lock manager should be possible, and then ideally, we can have pluggable locking protocols.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/7013
Test Plan: Run `transaction_test` and `optimistic_transaction_test`.
Reviewed By: ajkr
Differential Revision: D22163706
Pulled By: cheng-chang
fbshipit-source-id: f2860577b5334e31dd2994f5bc6d7c40d502b1b4
2020-08-06 21:36:48 +02:00
|
|
|
txn_db_impl_->UnLock(this, *tracked_locks_);
|
2015-08-25 04:13:18 +02:00
|
|
|
TransactionBaseImpl::Clear();
|
Pessimistic Transactions
Summary:
Initial implementation of Pessimistic Transactions. This diff contains the api changes discussed in D38913. This diff is pretty large, so let me know if people would prefer to meet up to discuss it.
MyRocks folks: please take a look at the API in include/rocksdb/utilities/transaction[_db].h and let me know if you have any issues.
Also, you'll notice a couple of TODOs in the implementation of RollbackToSavePoint(). After chatting with Siying, I'm going to send out a separate diff for an alternate implementation of this feature that implements the rollback inside of WriteBatch/WriteBatchWithIndex. We can then decide which route is preferable.
Next, I'm planning on doing some perf testing and then integrating this diff into MongoRocks for further testing.
Test Plan: Unit tests, db_bench parallel testing.
Reviewers: igor, rven, sdong, yhchiang, yoshinorim
Reviewed By: sdong
Subscribers: hermanlee4, maykov, spetrunia, leveldb, dhruba
Differential Revision: https://reviews.facebook.net/D40869
2015-05-26 02:37:33 +02:00
|
|
|
}
|
|
|
|
|
2017-08-17 01:49:11 +02:00
|
|
|
void PessimisticTransaction::Reinitialize(
|
|
|
|
TransactionDB* txn_db, const WriteOptions& write_options,
|
|
|
|
const TransactionOptions& txn_options) {
|
2020-03-06 21:29:30 +01:00
|
|
|
if (!name_.empty() && txn_state_ != COMMITTED) {
|
2016-04-18 20:15:50 +02:00
|
|
|
txn_db_impl_->UnregisterTransaction(this);
|
|
|
|
}
|
2016-08-11 23:19:33 +02:00
|
|
|
TransactionBaseImpl::Reinitialize(txn_db->GetRootDB(), write_options);
|
2016-02-03 04:19:17 +01:00
|
|
|
Initialize(txn_options);
|
|
|
|
}
|
|
|
|
|
2017-08-08 01:07:40 +02:00
|
|
|
bool PessimisticTransaction::IsExpired() const {
|
Pessimistic Transactions
Summary:
Initial implementation of Pessimistic Transactions. This diff contains the api changes discussed in D38913. This diff is pretty large, so let me know if people would prefer to meet up to discuss it.
MyRocks folks: please take a look at the API in include/rocksdb/utilities/transaction[_db].h and let me know if you have any issues.
Also, you'll notice a couple of TODOs in the implementation of RollbackToSavePoint(). After chatting with Siying, I'm going to send out a separate diff for an alternate implementation of this feature that implements the rollback inside of WriteBatch/WriteBatchWithIndex. We can then decide which route is preferable.
Next, I'm planning on doing some perf testing and then integrating this diff into MongoRocks for further testing.
Test Plan: Unit tests, db_bench parallel testing.
Reviewers: igor, rven, sdong, yhchiang, yoshinorim
Reviewed By: sdong
Subscribers: hermanlee4, maykov, spetrunia, leveldb, dhruba
Differential Revision: https://reviews.facebook.net/D40869
2015-05-26 02:37:33 +02:00
|
|
|
if (expiration_time_ > 0) {
|
2015-09-08 21:36:48 +02:00
|
|
|
if (db_->GetEnv()->NowMicros() >= expiration_time_) {
|
Pessimistic Transactions
Summary:
Initial implementation of Pessimistic Transactions. This diff contains the api changes discussed in D38913. This diff is pretty large, so let me know if people would prefer to meet up to discuss it.
MyRocks folks: please take a look at the API in include/rocksdb/utilities/transaction[_db].h and let me know if you have any issues.
Also, you'll notice a couple of TODOs in the implementation of RollbackToSavePoint(). After chatting with Siying, I'm going to send out a separate diff for an alternate implementation of this feature that implements the rollback inside of WriteBatch/WriteBatchWithIndex. We can then decide which route is preferable.
Next, I'm planning on doing some perf testing and then integrating this diff into MongoRocks for further testing.
Test Plan: Unit tests, db_bench parallel testing.
Reviewers: igor, rven, sdong, yhchiang, yoshinorim
Reviewed By: sdong
Subscribers: hermanlee4, maykov, spetrunia, leveldb, dhruba
Differential Revision: https://reviews.facebook.net/D40869
2015-05-26 02:37:33 +02:00
|
|
|
// Transaction is expired.
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
|
2017-08-17 01:49:11 +02:00
|
|
|
WriteCommittedTxn::WriteCommittedTxn(TransactionDB* txn_db,
|
|
|
|
const WriteOptions& write_options,
|
|
|
|
const TransactionOptions& txn_options)
|
2017-08-08 01:07:40 +02:00
|
|
|
: PessimisticTransaction(txn_db, write_options, txn_options){};
|
2017-08-03 17:46:47 +02:00
|
|
|
|
2017-09-13 20:56:27 +02:00
|
|
|
Status PessimisticTransaction::CommitBatch(WriteBatch* batch) {
|
2020-10-19 19:12:53 +02:00
|
|
|
std::unique_ptr<LockTracker> keys_to_unlock(lock_tracker_factory_.Create());
|
Replace tracked_keys with a new LockTracker interface in TransactionDB (#7013)
Summary:
We're going to support more locking protocols such as range lock in transaction.
However, in current design, `TransactionBase` has a member `tracked_keys` which assumes that point lock (lock a single key) is used, and is used in snapshot checking (isolation protocol). When using range lock, we may use read committed instead of snapshot checking as the isolation protocol.
The most significant usage scenarios of `tracked_keys` are:
1. pessimistic transaction uses it to track the locked keys, and unlock these keys when commit or rollback.
2. optimistic transaction does not lock keys upfront, it only tracks the lock intentions in tracked_keys, and do write conflict checking when commit.
3. each `SavePoint` tracks the keys that are locked since the `SavePoint`, `RollbackToSavePoint` or `PopSavePoint` relies on both the tracked keys in `SavePoint`s and `tracked_keys`.
Based on these scenarios, if we can abstract out a `LockTracker` interface to hold a set of tracked locks (can be keys or key ranges), and have methods that can be composed together to implement the scenarios, then `tracked_keys` can be an internal data structure of one implementation of `LockTracker`. See `utilities/transactions/lock/lock_tracker.h` for the detailed interface design, and `utilities/transactions/lock/point_lock_tracker.cc` for the implementation.
In the future, a `RangeLockTracker` can be implemented to track range locks without affecting other components.
After this PR, a clean interface for lock manager should be possible, and then ideally, we can have pluggable locking protocols.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/7013
Test Plan: Run `transaction_test` and `optimistic_transaction_test`.
Reviewed By: ajkr
Differential Revision: D22163706
Pulled By: cheng-chang
fbshipit-source-id: f2860577b5334e31dd2994f5bc6d7c40d502b1b4
2020-08-06 21:36:48 +02:00
|
|
|
Status s = LockBatch(batch, keys_to_unlock.get());
|
Pessimistic Transactions
Summary:
Initial implementation of Pessimistic Transactions. This diff contains the api changes discussed in D38913. This diff is pretty large, so let me know if people would prefer to meet up to discuss it.
MyRocks folks: please take a look at the API in include/rocksdb/utilities/transaction[_db].h and let me know if you have any issues.
Also, you'll notice a couple of TODOs in the implementation of RollbackToSavePoint(). After chatting with Siying, I'm going to send out a separate diff for an alternate implementation of this feature that implements the rollback inside of WriteBatch/WriteBatchWithIndex. We can then decide which route is preferable.
Next, I'm planning on doing some perf testing and then integrating this diff into MongoRocks for further testing.
Test Plan: Unit tests, db_bench parallel testing.
Reviewers: igor, rven, sdong, yhchiang, yoshinorim
Reviewed By: sdong
Subscribers: hermanlee4, maykov, spetrunia, leveldb, dhruba
Differential Revision: https://reviews.facebook.net/D40869
2015-05-26 02:37:33 +02:00
|
|
|
|
2016-04-18 20:15:50 +02:00
|
|
|
if (!s.ok()) {
|
|
|
|
return s;
|
|
|
|
}
|
Pessimistic Transactions
Summary:
Initial implementation of Pessimistic Transactions. This diff contains the api changes discussed in D38913. This diff is pretty large, so let me know if people would prefer to meet up to discuss it.
MyRocks folks: please take a look at the API in include/rocksdb/utilities/transaction[_db].h and let me know if you have any issues.
Also, you'll notice a couple of TODOs in the implementation of RollbackToSavePoint(). After chatting with Siying, I'm going to send out a separate diff for an alternate implementation of this feature that implements the rollback inside of WriteBatch/WriteBatchWithIndex. We can then decide which route is preferable.
Next, I'm planning on doing some perf testing and then integrating this diff into MongoRocks for further testing.
Test Plan: Unit tests, db_bench parallel testing.
Reviewers: igor, rven, sdong, yhchiang, yoshinorim
Reviewed By: sdong
Subscribers: hermanlee4, maykov, spetrunia, leveldb, dhruba
Differential Revision: https://reviews.facebook.net/D40869
2015-05-26 02:37:33 +02:00
|
|
|
|
2016-04-18 20:15:50 +02:00
|
|
|
bool can_commit = false;
|
|
|
|
|
|
|
|
if (IsExpired()) {
|
|
|
|
s = Status::Expired();
|
|
|
|
} else if (expiration_time_ > 0) {
|
2016-10-05 22:39:00 +02:00
|
|
|
TransactionState expected = STARTED;
|
|
|
|
can_commit = std::atomic_compare_exchange_strong(&txn_state_, &expected,
|
2016-04-18 20:15:50 +02:00
|
|
|
AWAITING_COMMIT);
|
2016-10-05 22:39:00 +02:00
|
|
|
} else if (txn_state_ == STARTED) {
|
2016-04-18 20:15:50 +02:00
|
|
|
// lock stealing is not a concern
|
|
|
|
can_commit = true;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (can_commit) {
|
2016-10-05 22:39:00 +02:00
|
|
|
txn_state_.store(AWAITING_COMMIT);
|
2017-09-09 00:53:51 +02:00
|
|
|
s = CommitBatchInternal(batch);
|
2016-04-18 20:15:50 +02:00
|
|
|
if (s.ok()) {
|
2020-03-06 21:29:30 +01:00
|
|
|
txn_state_.store(COMMITTED);
|
2016-04-18 20:15:50 +02:00
|
|
|
}
|
2016-10-05 22:39:00 +02:00
|
|
|
} else if (txn_state_ == LOCKS_STOLEN) {
|
2016-04-18 20:15:50 +02:00
|
|
|
s = Status::Expired();
|
|
|
|
} else {
|
|
|
|
s = Status::InvalidArgument("Transaction is not in state for commit.");
|
Pessimistic Transactions
Summary:
Initial implementation of Pessimistic Transactions. This diff contains the api changes discussed in D38913. This diff is pretty large, so let me know if people would prefer to meet up to discuss it.
MyRocks folks: please take a look at the API in include/rocksdb/utilities/transaction[_db].h and let me know if you have any issues.
Also, you'll notice a couple of TODOs in the implementation of RollbackToSavePoint(). After chatting with Siying, I'm going to send out a separate diff for an alternate implementation of this feature that implements the rollback inside of WriteBatch/WriteBatchWithIndex. We can then decide which route is preferable.
Next, I'm planning on doing some perf testing and then integrating this diff into MongoRocks for further testing.
Test Plan: Unit tests, db_bench parallel testing.
Reviewers: igor, rven, sdong, yhchiang, yoshinorim
Reviewed By: sdong
Subscribers: hermanlee4, maykov, spetrunia, leveldb, dhruba
Differential Revision: https://reviews.facebook.net/D40869
2015-05-26 02:37:33 +02:00
|
|
|
}
|
|
|
|
|
Replace tracked_keys with a new LockTracker interface in TransactionDB (#7013)
Summary:
We're going to support more locking protocols such as range lock in transaction.
However, in current design, `TransactionBase` has a member `tracked_keys` which assumes that point lock (lock a single key) is used, and is used in snapshot checking (isolation protocol). When using range lock, we may use read committed instead of snapshot checking as the isolation protocol.
The most significant usage scenarios of `tracked_keys` are:
1. pessimistic transaction uses it to track the locked keys, and unlock these keys when commit or rollback.
2. optimistic transaction does not lock keys upfront, it only tracks the lock intentions in tracked_keys, and do write conflict checking when commit.
3. each `SavePoint` tracks the keys that are locked since the `SavePoint`, `RollbackToSavePoint` or `PopSavePoint` relies on both the tracked keys in `SavePoint`s and `tracked_keys`.
Based on these scenarios, if we can abstract out a `LockTracker` interface to hold a set of tracked locks (can be keys or key ranges), and have methods that can be composed together to implement the scenarios, then `tracked_keys` can be an internal data structure of one implementation of `LockTracker`. See `utilities/transactions/lock/lock_tracker.h` for the detailed interface design, and `utilities/transactions/lock/point_lock_tracker.cc` for the implementation.
In the future, a `RangeLockTracker` can be implemented to track range locks without affecting other components.
After this PR, a clean interface for lock manager should be possible, and then ideally, we can have pluggable locking protocols.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/7013
Test Plan: Run `transaction_test` and `optimistic_transaction_test`.
Reviewed By: ajkr
Differential Revision: D22163706
Pulled By: cheng-chang
fbshipit-source-id: f2860577b5334e31dd2994f5bc6d7c40d502b1b4
2020-08-06 21:36:48 +02:00
|
|
|
txn_db_impl_->UnLock(this, *keys_to_unlock);
|
2016-04-18 20:15:50 +02:00
|
|
|
|
Pessimistic Transactions
Summary:
Initial implementation of Pessimistic Transactions. This diff contains the api changes discussed in D38913. This diff is pretty large, so let me know if people would prefer to meet up to discuss it.
MyRocks folks: please take a look at the API in include/rocksdb/utilities/transaction[_db].h and let me know if you have any issues.
Also, you'll notice a couple of TODOs in the implementation of RollbackToSavePoint(). After chatting with Siying, I'm going to send out a separate diff for an alternate implementation of this feature that implements the rollback inside of WriteBatch/WriteBatchWithIndex. We can then decide which route is preferable.
Next, I'm planning on doing some perf testing and then integrating this diff into MongoRocks for further testing.
Test Plan: Unit tests, db_bench parallel testing.
Reviewers: igor, rven, sdong, yhchiang, yoshinorim
Reviewed By: sdong
Subscribers: hermanlee4, maykov, spetrunia, leveldb, dhruba
Differential Revision: https://reviews.facebook.net/D40869
2015-05-26 02:37:33 +02:00
|
|
|
return s;
|
|
|
|
}
|
|
|
|
|
2017-08-08 01:07:40 +02:00
|
|
|
Status PessimisticTransaction::Prepare() {
|
2016-04-18 20:15:50 +02:00
|
|
|
|
|
|
|
if (name_.empty()) {
|
|
|
|
return Status::InvalidArgument(
|
|
|
|
"Cannot prepare a transaction that has not been named.");
|
|
|
|
}
|
Pessimistic Transactions
Summary:
Initial implementation of Pessimistic Transactions. This diff contains the api changes discussed in D38913. This diff is pretty large, so let me know if people would prefer to meet up to discuss it.
MyRocks folks: please take a look at the API in include/rocksdb/utilities/transaction[_db].h and let me know if you have any issues.
Also, you'll notice a couple of TODOs in the implementation of RollbackToSavePoint(). After chatting with Siying, I'm going to send out a separate diff for an alternate implementation of this feature that implements the rollback inside of WriteBatch/WriteBatchWithIndex. We can then decide which route is preferable.
Next, I'm planning on doing some perf testing and then integrating this diff into MongoRocks for further testing.
Test Plan: Unit tests, db_bench parallel testing.
Reviewers: igor, rven, sdong, yhchiang, yoshinorim
Reviewed By: sdong
Subscribers: hermanlee4, maykov, spetrunia, leveldb, dhruba
Differential Revision: https://reviews.facebook.net/D40869
2015-05-26 02:37:33 +02:00
|
|
|
|
2016-04-18 20:15:50 +02:00
|
|
|
if (IsExpired()) {
|
|
|
|
return Status::Expired();
|
|
|
|
}
|
|
|
|
|
2020-10-21 23:02:00 +02:00
|
|
|
Status s;
|
2016-04-18 20:15:50 +02:00
|
|
|
bool can_prepare = false;
|
|
|
|
|
|
|
|
if (expiration_time_ > 0) {
|
|
|
|
// must concern ourselves with expiraton and/or lock stealing
|
|
|
|
// need to compare/exchange bc locks could be stolen under us here
|
2016-10-05 22:39:00 +02:00
|
|
|
TransactionState expected = STARTED;
|
|
|
|
can_prepare = std::atomic_compare_exchange_strong(&txn_state_, &expected,
|
2016-04-18 20:15:50 +02:00
|
|
|
AWAITING_PREPARE);
|
2016-10-05 22:39:00 +02:00
|
|
|
} else if (txn_state_ == STARTED) {
|
2016-04-18 20:15:50 +02:00
|
|
|
// expiration and lock stealing is not possible
|
2020-05-02 02:35:41 +02:00
|
|
|
txn_state_.store(AWAITING_PREPARE);
|
2016-04-18 20:15:50 +02:00
|
|
|
can_prepare = true;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (can_prepare) {
|
|
|
|
// transaction can't expire after preparation
|
|
|
|
expiration_time_ = 0;
|
2019-04-03 00:14:41 +02:00
|
|
|
assert(log_number_ == 0 ||
|
|
|
|
txn_db_impl_->GetTxnDBOptions().write_policy == WRITE_UNPREPARED);
|
2018-07-24 09:09:18 +02:00
|
|
|
|
2017-08-08 01:07:40 +02:00
|
|
|
s = PrepareInternal();
|
2016-04-18 20:15:50 +02:00
|
|
|
if (s.ok()) {
|
2016-10-05 22:39:00 +02:00
|
|
|
txn_state_.store(PREPARED);
|
2016-04-18 20:15:50 +02:00
|
|
|
}
|
2016-10-05 22:39:00 +02:00
|
|
|
} else if (txn_state_ == LOCKS_STOLEN) {
|
2016-04-18 20:15:50 +02:00
|
|
|
s = Status::Expired();
|
2016-10-05 22:39:00 +02:00
|
|
|
} else if (txn_state_ == PREPARED) {
|
2016-04-18 20:15:50 +02:00
|
|
|
s = Status::InvalidArgument("Transaction has already been prepared.");
|
2020-03-06 21:29:30 +01:00
|
|
|
} else if (txn_state_ == COMMITTED) {
|
2016-04-18 20:15:50 +02:00
|
|
|
s = Status::InvalidArgument("Transaction has already been committed.");
|
2016-10-05 22:39:00 +02:00
|
|
|
} else if (txn_state_ == ROLLEDBACK) {
|
2016-04-18 20:15:50 +02:00
|
|
|
s = Status::InvalidArgument("Transaction has already been rolledback.");
|
|
|
|
} else {
|
|
|
|
s = Status::InvalidArgument("Transaction is not in state for commit.");
|
|
|
|
}
|
Pessimistic Transactions
Summary:
Initial implementation of Pessimistic Transactions. This diff contains the api changes discussed in D38913. This diff is pretty large, so let me know if people would prefer to meet up to discuss it.
MyRocks folks: please take a look at the API in include/rocksdb/utilities/transaction[_db].h and let me know if you have any issues.
Also, you'll notice a couple of TODOs in the implementation of RollbackToSavePoint(). After chatting with Siying, I'm going to send out a separate diff for an alternate implementation of this feature that implements the rollback inside of WriteBatch/WriteBatchWithIndex. We can then decide which route is preferable.
Next, I'm planning on doing some perf testing and then integrating this diff into MongoRocks for further testing.
Test Plan: Unit tests, db_bench parallel testing.
Reviewers: igor, rven, sdong, yhchiang, yoshinorim
Reviewed By: sdong
Subscribers: hermanlee4, maykov, spetrunia, leveldb, dhruba
Differential Revision: https://reviews.facebook.net/D40869
2015-05-26 02:37:33 +02:00
|
|
|
|
|
|
|
return s;
|
|
|
|
}
|
|
|
|
|
2017-08-08 01:07:40 +02:00
|
|
|
Status WriteCommittedTxn::PrepareInternal() {
|
|
|
|
WriteOptions write_options = write_options_;
|
|
|
|
write_options.disableWAL = false;
|
2020-10-21 23:02:00 +02:00
|
|
|
auto s = WriteBatchInternal::MarkEndPrepare(GetWriteBatch()->GetWriteBatch(),
|
|
|
|
name_);
|
|
|
|
assert(s.ok());
|
2019-04-03 00:14:41 +02:00
|
|
|
class MarkLogCallback : public PreReleaseCallback {
|
|
|
|
public:
|
|
|
|
MarkLogCallback(DBImpl* db, bool two_write_queues)
|
|
|
|
: db_(db), two_write_queues_(two_write_queues) {
|
|
|
|
(void)two_write_queues_; // to silence unused private field warning
|
|
|
|
}
|
|
|
|
virtual Status Callback(SequenceNumber, bool is_mem_disabled,
|
2019-06-10 20:47:16 +02:00
|
|
|
uint64_t log_number, size_t /*index*/,
|
|
|
|
size_t /*total*/) override {
|
2019-04-03 00:14:41 +02:00
|
|
|
#ifdef NDEBUG
|
|
|
|
(void)is_mem_disabled;
|
|
|
|
#endif
|
|
|
|
assert(log_number != 0);
|
|
|
|
assert(!two_write_queues_ || is_mem_disabled); // implies the 2nd queue
|
|
|
|
db_->logs_with_prep_tracker()->MarkLogAsContainingPrepSection(log_number);
|
|
|
|
return Status::OK();
|
|
|
|
}
|
|
|
|
|
|
|
|
private:
|
|
|
|
DBImpl* db_;
|
|
|
|
bool two_write_queues_;
|
|
|
|
} mark_log_callback(db_impl_,
|
|
|
|
db_impl_->immutable_db_options().two_write_queues);
|
|
|
|
|
|
|
|
WriteCallback* const kNoWriteCallback = nullptr;
|
|
|
|
const uint64_t kRefNoLog = 0;
|
|
|
|
const bool kDisableMemtable = true;
|
|
|
|
SequenceNumber* const KIgnoreSeqUsed = nullptr;
|
|
|
|
const size_t kNoBatchCount = 0;
|
2020-10-21 23:02:00 +02:00
|
|
|
s = db_impl_->WriteImpl(write_options, GetWriteBatch()->GetWriteBatch(),
|
|
|
|
kNoWriteCallback, &log_number_, kRefNoLog,
|
|
|
|
kDisableMemtable, KIgnoreSeqUsed, kNoBatchCount,
|
|
|
|
&mark_log_callback);
|
2017-08-08 01:07:40 +02:00
|
|
|
return s;
|
|
|
|
}
|
|
|
|
|
|
|
|
Status PessimisticTransaction::Commit() {
|
|
|
|
bool commit_without_prepare = false;
|
2016-04-18 20:15:50 +02:00
|
|
|
bool commit_prepared = false;
|
Pessimistic Transactions
Summary:
Initial implementation of Pessimistic Transactions. This diff contains the api changes discussed in D38913. This diff is pretty large, so let me know if people would prefer to meet up to discuss it.
MyRocks folks: please take a look at the API in include/rocksdb/utilities/transaction[_db].h and let me know if you have any issues.
Also, you'll notice a couple of TODOs in the implementation of RollbackToSavePoint(). After chatting with Siying, I'm going to send out a separate diff for an alternate implementation of this feature that implements the rollback inside of WriteBatch/WriteBatchWithIndex. We can then decide which route is preferable.
Next, I'm planning on doing some perf testing and then integrating this diff into MongoRocks for further testing.
Test Plan: Unit tests, db_bench parallel testing.
Reviewers: igor, rven, sdong, yhchiang, yoshinorim
Reviewed By: sdong
Subscribers: hermanlee4, maykov, spetrunia, leveldb, dhruba
Differential Revision: https://reviews.facebook.net/D40869
2015-05-26 02:37:33 +02:00
|
|
|
|
2016-04-18 20:15:50 +02:00
|
|
|
if (IsExpired()) {
|
|
|
|
return Status::Expired();
|
|
|
|
}
|
2016-02-02 02:07:05 +01:00
|
|
|
|
2016-04-18 20:15:50 +02:00
|
|
|
if (expiration_time_ > 0) {
|
|
|
|
// we must atomicaly compare and exchange the state here because at
|
|
|
|
// this state in the transaction it is possible for another thread
|
|
|
|
// to change our state out from under us in the even that we expire and have
|
|
|
|
// our locks stolen. In this case the only valid state is STARTED because
|
|
|
|
// a state of PREPARED would have a cleared expiration_time_.
|
2016-10-05 22:39:00 +02:00
|
|
|
TransactionState expected = STARTED;
|
2017-08-08 01:07:40 +02:00
|
|
|
commit_without_prepare = std::atomic_compare_exchange_strong(
|
|
|
|
&txn_state_, &expected, AWAITING_COMMIT);
|
2016-02-02 02:07:05 +01:00
|
|
|
TEST_SYNC_POINT("TransactionTest::ExpirableTransactionDataRace:1");
|
2016-10-05 22:39:00 +02:00
|
|
|
} else if (txn_state_ == PREPARED) {
|
2016-04-18 20:15:50 +02:00
|
|
|
// expiration and lock stealing is not a concern
|
|
|
|
commit_prepared = true;
|
2016-10-05 22:39:00 +02:00
|
|
|
} else if (txn_state_ == STARTED) {
|
2016-04-18 20:15:50 +02:00
|
|
|
// expiration and lock stealing is not a concern
|
2020-05-01 01:21:05 +02:00
|
|
|
if (skip_prepare_) {
|
|
|
|
commit_without_prepare = true;
|
|
|
|
} else {
|
|
|
|
return Status::TxnNotPrepared();
|
|
|
|
}
|
2016-04-18 20:15:50 +02:00
|
|
|
}
|
2016-02-02 02:07:05 +01:00
|
|
|
|
2020-10-21 23:02:00 +02:00
|
|
|
Status s;
|
2017-08-08 01:07:40 +02:00
|
|
|
if (commit_without_prepare) {
|
2016-04-18 20:15:50 +02:00
|
|
|
assert(!commit_prepared);
|
|
|
|
if (WriteBatchInternal::Count(GetCommitTimeWriteBatch()) > 0) {
|
|
|
|
s = Status::InvalidArgument(
|
|
|
|
"Commit-time batch contains values that will not be committed.");
|
2016-02-02 02:07:05 +01:00
|
|
|
} else {
|
2016-10-05 22:39:00 +02:00
|
|
|
txn_state_.store(AWAITING_COMMIT);
|
2018-07-24 09:09:18 +02:00
|
|
|
if (log_number_ > 0) {
|
|
|
|
dbimpl_->logs_with_prep_tracker()->MarkLogAsHavingPrepSectionFlushed(
|
|
|
|
log_number_);
|
|
|
|
}
|
2017-08-08 01:07:40 +02:00
|
|
|
s = CommitWithoutPrepareInternal();
|
2018-07-24 09:09:18 +02:00
|
|
|
if (!name_.empty()) {
|
|
|
|
txn_db_impl_->UnregisterTransaction(this);
|
|
|
|
}
|
2016-04-18 20:15:50 +02:00
|
|
|
Clear();
|
|
|
|
if (s.ok()) {
|
2020-03-06 21:29:30 +01:00
|
|
|
txn_state_.store(COMMITTED);
|
2016-04-18 20:15:50 +02:00
|
|
|
}
|
2016-02-02 02:07:05 +01:00
|
|
|
}
|
2016-04-18 20:15:50 +02:00
|
|
|
} else if (commit_prepared) {
|
2016-10-05 22:39:00 +02:00
|
|
|
txn_state_.store(AWAITING_COMMIT);
|
2016-04-18 20:15:50 +02:00
|
|
|
|
2017-08-08 01:07:40 +02:00
|
|
|
s = CommitInternal();
|
Add facility to write only a portion of WriteBatch to WAL
Summary:
When constructing a write batch a client may now call MarkWalTerminationPoint() on that batch. No batch operations after this call will be added written to the WAL but will still be inserted into the Memtable. This facility is used to remove one of the three WriteImpl calls in 2PC transactions. This produces a ~1% perf improvement.
```
RocksDB - unoptimized 2pc, sync_binlog=1, disable_2pc=off
INFO 2016-08-31 14:30:38,814 [main]: REQUEST PHASE COMPLETED. 75000000 requests done in 2619 seconds. Requests/second = 28628
RocksDB - optimized 2pc , sync_binlog=1, disable_2pc=off
INFO 2016-08-31 16:26:59,442 [main]: REQUEST PHASE COMPLETED. 75000000 requests done in 2581 seconds. Requests/second = 29054
```
Test Plan: Two unit tests added.
Reviewers: sdong, yiwu, IslamAbdelRahman
Reviewed By: yiwu
Subscribers: hermanlee4, dhruba, andrewkr
Differential Revision: https://reviews.facebook.net/D64599
2016-10-07 20:31:26 +02:00
|
|
|
|
2016-04-18 20:15:50 +02:00
|
|
|
if (!s.ok()) {
|
2017-06-24 23:06:43 +02:00
|
|
|
ROCKS_LOG_WARN(db_impl_->immutable_db_options().info_log,
|
|
|
|
"Commit write failed");
|
2016-04-18 20:15:50 +02:00
|
|
|
return s;
|
|
|
|
}
|
|
|
|
|
|
|
|
// FindObsoleteFiles must now look to the memtables
|
|
|
|
// to determine what prep logs must be kept around,
|
|
|
|
// not the prep section heap.
|
|
|
|
assert(log_number_ > 0);
|
Skip deleted WALs during recovery
Summary:
This patch record min log number to keep to the manifest while flushing SST files to ignore them and any WAL older than them during recovery. This is to avoid scenarios when we have a gap between the WAL files are fed to the recovery procedure. The gap could happen by for example out-of-order WAL deletion. Such gap could cause problems in 2PC recovery where the prepared and commit entry are placed into two separate WAL and gap in the WALs could result into not processing the WAL with the commit entry and hence breaking the 2PC recovery logic.
Before the commit, for 2PC case, we determined which log number to keep in FindObsoleteFiles(). We looked at the earliest logs with outstanding prepare entries, or prepare entries whose respective commit or abort are in memtable. With the commit, the same calculation is done while we apply the SST flush. Just before installing the flush file, we precompute the earliest log file to keep after the flush finishes using the same logic (but skipping the memtables just flushed), record this information to the manifest entry for this new flushed SST file. This pre-computed value is also remembered in memory, and will later be used to determine whether a log file can be deleted. This value is unlikely to change until next flush because the commit entry will stay in memtable. (In WritePrepared, we could have removed the older log files as soon as all prepared entries are committed. It's not yet done anyway. Even if we do it, the only thing we loss with this new approach is earlier log deletion between two flushes, which does not guarantee to happen anyway because the obsolete file clean-up function is only executed after flush or compaction)
This min log number to keep is stored in the manifest using the safely-ignore customized field of AddFile entry, in order to guarantee that the DB generated using newer release can be opened by previous releases no older than 4.2.
Closes https://github.com/facebook/rocksdb/pull/3765
Differential Revision: D7747618
Pulled By: siying
fbshipit-source-id: d00c92105b4f83852e9754a1b70d6b64cb590729
2018-05-04 00:35:11 +02:00
|
|
|
dbimpl_->logs_with_prep_tracker()->MarkLogAsHavingPrepSectionFlushed(
|
|
|
|
log_number_);
|
2016-04-18 20:15:50 +02:00
|
|
|
txn_db_impl_->UnregisterTransaction(this);
|
|
|
|
|
|
|
|
Clear();
|
2020-03-06 21:29:30 +01:00
|
|
|
txn_state_.store(COMMITTED);
|
2016-10-05 22:39:00 +02:00
|
|
|
} else if (txn_state_ == LOCKS_STOLEN) {
|
2016-04-18 20:15:50 +02:00
|
|
|
s = Status::Expired();
|
2020-03-06 21:29:30 +01:00
|
|
|
} else if (txn_state_ == COMMITTED) {
|
2016-04-18 20:15:50 +02:00
|
|
|
s = Status::InvalidArgument("Transaction has already been committed.");
|
2016-10-05 22:39:00 +02:00
|
|
|
} else if (txn_state_ == ROLLEDBACK) {
|
2016-04-18 20:15:50 +02:00
|
|
|
s = Status::InvalidArgument("Transaction has already been rolledback.");
|
Pessimistic Transactions
Summary:
Initial implementation of Pessimistic Transactions. This diff contains the api changes discussed in D38913. This diff is pretty large, so let me know if people would prefer to meet up to discuss it.
MyRocks folks: please take a look at the API in include/rocksdb/utilities/transaction[_db].h and let me know if you have any issues.
Also, you'll notice a couple of TODOs in the implementation of RollbackToSavePoint(). After chatting with Siying, I'm going to send out a separate diff for an alternate implementation of this feature that implements the rollback inside of WriteBatch/WriteBatchWithIndex. We can then decide which route is preferable.
Next, I'm planning on doing some perf testing and then integrating this diff into MongoRocks for further testing.
Test Plan: Unit tests, db_bench parallel testing.
Reviewers: igor, rven, sdong, yhchiang, yoshinorim
Reviewed By: sdong
Subscribers: hermanlee4, maykov, spetrunia, leveldb, dhruba
Differential Revision: https://reviews.facebook.net/D40869
2015-05-26 02:37:33 +02:00
|
|
|
} else {
|
2016-04-18 20:15:50 +02:00
|
|
|
s = Status::InvalidArgument("Transaction is not in state for commit.");
|
Pessimistic Transactions
Summary:
Initial implementation of Pessimistic Transactions. This diff contains the api changes discussed in D38913. This diff is pretty large, so let me know if people would prefer to meet up to discuss it.
MyRocks folks: please take a look at the API in include/rocksdb/utilities/transaction[_db].h and let me know if you have any issues.
Also, you'll notice a couple of TODOs in the implementation of RollbackToSavePoint(). After chatting with Siying, I'm going to send out a separate diff for an alternate implementation of this feature that implements the rollback inside of WriteBatch/WriteBatchWithIndex. We can then decide which route is preferable.
Next, I'm planning on doing some perf testing and then integrating this diff into MongoRocks for further testing.
Test Plan: Unit tests, db_bench parallel testing.
Reviewers: igor, rven, sdong, yhchiang, yoshinorim
Reviewed By: sdong
Subscribers: hermanlee4, maykov, spetrunia, leveldb, dhruba
Differential Revision: https://reviews.facebook.net/D40869
2015-05-26 02:37:33 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
return s;
|
|
|
|
}
|
|
|
|
|
2017-08-08 01:07:40 +02:00
|
|
|
Status WriteCommittedTxn::CommitWithoutPrepareInternal() {
|
2018-10-24 21:06:19 +02:00
|
|
|
uint64_t seq_used = kMaxSequenceNumber;
|
|
|
|
auto s =
|
|
|
|
db_impl_->WriteImpl(write_options_, GetWriteBatch()->GetWriteBatch(),
|
|
|
|
/*callback*/ nullptr, /*log_used*/ nullptr,
|
|
|
|
/*log_ref*/ 0, /*disable_memtable*/ false, &seq_used);
|
|
|
|
assert(!s.ok() || seq_used != kMaxSequenceNumber);
|
|
|
|
if (s.ok()) {
|
|
|
|
SetId(seq_used);
|
|
|
|
}
|
2017-08-08 01:07:40 +02:00
|
|
|
return s;
|
|
|
|
}
|
|
|
|
|
2018-02-06 03:32:54 +01:00
|
|
|
Status WriteCommittedTxn::CommitBatchInternal(WriteBatch* batch, size_t) {
|
2018-10-24 21:06:19 +02:00
|
|
|
uint64_t seq_used = kMaxSequenceNumber;
|
|
|
|
auto s = db_impl_->WriteImpl(write_options_, batch, /*callback*/ nullptr,
|
|
|
|
/*log_used*/ nullptr, /*log_ref*/ 0,
|
|
|
|
/*disable_memtable*/ false, &seq_used);
|
|
|
|
assert(!s.ok() || seq_used != kMaxSequenceNumber);
|
|
|
|
if (s.ok()) {
|
|
|
|
SetId(seq_used);
|
|
|
|
}
|
2017-09-09 00:53:51 +02:00
|
|
|
return s;
|
|
|
|
}
|
|
|
|
|
2017-08-08 01:07:40 +02:00
|
|
|
Status WriteCommittedTxn::CommitInternal() {
|
|
|
|
// We take the commit-time batch and append the Commit marker.
|
|
|
|
// The Memtable will ignore the Commit marker in non-recovery mode
|
|
|
|
WriteBatch* working_batch = GetCommitTimeWriteBatch();
|
2020-10-21 23:02:00 +02:00
|
|
|
auto s = WriteBatchInternal::MarkCommit(working_batch, name_);
|
|
|
|
assert(s.ok());
|
2017-08-08 01:07:40 +02:00
|
|
|
|
|
|
|
// any operations appended to this working_batch will be ignored from WAL
|
|
|
|
working_batch->MarkWalTerminationPoint();
|
|
|
|
|
|
|
|
// insert prepared batch into Memtable only skipping WAL.
|
|
|
|
// Memtable will ignore BeginPrepare/EndPrepare markers
|
|
|
|
// in non recovery mode and simply insert the values
|
2020-10-21 23:02:00 +02:00
|
|
|
s = WriteBatchInternal::Append(working_batch,
|
|
|
|
GetWriteBatch()->GetWriteBatch());
|
|
|
|
assert(s.ok());
|
2017-08-08 01:07:40 +02:00
|
|
|
|
2018-10-24 21:06:19 +02:00
|
|
|
uint64_t seq_used = kMaxSequenceNumber;
|
2020-10-21 23:02:00 +02:00
|
|
|
s = db_impl_->WriteImpl(write_options_, working_batch, /*callback*/ nullptr,
|
2018-10-24 21:06:19 +02:00
|
|
|
/*log_used*/ nullptr, /*log_ref*/ log_number_,
|
2020-10-21 23:02:00 +02:00
|
|
|
/*disable_memtable*/ false, &seq_used);
|
2018-10-24 21:06:19 +02:00
|
|
|
assert(!s.ok() || seq_used != kMaxSequenceNumber);
|
|
|
|
if (s.ok()) {
|
|
|
|
SetId(seq_used);
|
|
|
|
}
|
2017-08-08 01:07:40 +02:00
|
|
|
return s;
|
|
|
|
}
|
|
|
|
|
2017-10-03 04:46:42 +02:00
|
|
|
Status PessimisticTransaction::Rollback() {
|
2016-04-18 20:15:50 +02:00
|
|
|
Status s;
|
2016-10-05 22:39:00 +02:00
|
|
|
if (txn_state_ == PREPARED) {
|
|
|
|
txn_state_.store(AWAITING_ROLLBACK);
|
2017-10-03 04:46:42 +02:00
|
|
|
|
|
|
|
s = RollbackInternal();
|
|
|
|
|
2016-04-18 20:15:50 +02:00
|
|
|
if (s.ok()) {
|
|
|
|
// we do not need to keep our prepared section around
|
|
|
|
assert(log_number_ > 0);
|
Skip deleted WALs during recovery
Summary:
This patch record min log number to keep to the manifest while flushing SST files to ignore them and any WAL older than them during recovery. This is to avoid scenarios when we have a gap between the WAL files are fed to the recovery procedure. The gap could happen by for example out-of-order WAL deletion. Such gap could cause problems in 2PC recovery where the prepared and commit entry are placed into two separate WAL and gap in the WALs could result into not processing the WAL with the commit entry and hence breaking the 2PC recovery logic.
Before the commit, for 2PC case, we determined which log number to keep in FindObsoleteFiles(). We looked at the earliest logs with outstanding prepare entries, or prepare entries whose respective commit or abort are in memtable. With the commit, the same calculation is done while we apply the SST flush. Just before installing the flush file, we precompute the earliest log file to keep after the flush finishes using the same logic (but skipping the memtables just flushed), record this information to the manifest entry for this new flushed SST file. This pre-computed value is also remembered in memory, and will later be used to determine whether a log file can be deleted. This value is unlikely to change until next flush because the commit entry will stay in memtable. (In WritePrepared, we could have removed the older log files as soon as all prepared entries are committed. It's not yet done anyway. Even if we do it, the only thing we loss with this new approach is earlier log deletion between two flushes, which does not guarantee to happen anyway because the obsolete file clean-up function is only executed after flush or compaction)
This min log number to keep is stored in the manifest using the safely-ignore customized field of AddFile entry, in order to guarantee that the DB generated using newer release can be opened by previous releases no older than 4.2.
Closes https://github.com/facebook/rocksdb/pull/3765
Differential Revision: D7747618
Pulled By: siying
fbshipit-source-id: d00c92105b4f83852e9754a1b70d6b64cb590729
2018-05-04 00:35:11 +02:00
|
|
|
dbimpl_->logs_with_prep_tracker()->MarkLogAsHavingPrepSectionFlushed(
|
|
|
|
log_number_);
|
2016-04-18 20:15:50 +02:00
|
|
|
Clear();
|
2016-10-05 22:39:00 +02:00
|
|
|
txn_state_.store(ROLLEDBACK);
|
2016-04-18 20:15:50 +02:00
|
|
|
}
|
2016-10-05 22:39:00 +02:00
|
|
|
} else if (txn_state_ == STARTED) {
|
2018-07-24 09:09:18 +02:00
|
|
|
if (log_number_ > 0) {
|
|
|
|
assert(txn_db_impl_->GetTxnDBOptions().write_policy == WRITE_UNPREPARED);
|
|
|
|
assert(GetId() > 0);
|
|
|
|
s = RollbackInternal();
|
|
|
|
|
|
|
|
if (s.ok()) {
|
|
|
|
dbimpl_->logs_with_prep_tracker()->MarkLogAsHavingPrepSectionFlushed(
|
|
|
|
log_number_);
|
|
|
|
}
|
|
|
|
}
|
2016-04-18 20:15:50 +02:00
|
|
|
// prepare couldn't have taken place
|
|
|
|
Clear();
|
2020-03-06 21:29:30 +01:00
|
|
|
} else if (txn_state_ == COMMITTED) {
|
2016-04-18 20:15:50 +02:00
|
|
|
s = Status::InvalidArgument("This transaction has already been committed.");
|
|
|
|
} else {
|
|
|
|
s = Status::InvalidArgument(
|
|
|
|
"Two phase transaction is not in state for rollback.");
|
|
|
|
}
|
|
|
|
|
|
|
|
return s;
|
|
|
|
}
|
2017-10-03 04:46:42 +02:00
|
|
|
|
|
|
|
Status WriteCommittedTxn::RollbackInternal() {
|
|
|
|
WriteBatch rollback_marker;
|
2020-10-21 23:02:00 +02:00
|
|
|
auto s = WriteBatchInternal::MarkRollback(&rollback_marker, name_);
|
|
|
|
assert(s.ok());
|
|
|
|
s = db_impl_->WriteImpl(write_options_, &rollback_marker);
|
2017-10-03 04:46:42 +02:00
|
|
|
return s;
|
|
|
|
}
|
Pessimistic Transactions
Summary:
Initial implementation of Pessimistic Transactions. This diff contains the api changes discussed in D38913. This diff is pretty large, so let me know if people would prefer to meet up to discuss it.
MyRocks folks: please take a look at the API in include/rocksdb/utilities/transaction[_db].h and let me know if you have any issues.
Also, you'll notice a couple of TODOs in the implementation of RollbackToSavePoint(). After chatting with Siying, I'm going to send out a separate diff for an alternate implementation of this feature that implements the rollback inside of WriteBatch/WriteBatchWithIndex. We can then decide which route is preferable.
Next, I'm planning on doing some perf testing and then integrating this diff into MongoRocks for further testing.
Test Plan: Unit tests, db_bench parallel testing.
Reviewers: igor, rven, sdong, yhchiang, yoshinorim
Reviewed By: sdong
Subscribers: hermanlee4, maykov, spetrunia, leveldb, dhruba
Differential Revision: https://reviews.facebook.net/D40869
2015-05-26 02:37:33 +02:00
|
|
|
|
2017-08-08 01:07:40 +02:00
|
|
|
Status PessimisticTransaction::RollbackToSavePoint() {
|
2016-10-05 22:39:00 +02:00
|
|
|
if (txn_state_ != STARTED) {
|
2016-04-18 20:15:50 +02:00
|
|
|
return Status::InvalidArgument("Transaction is beyond state for rollback.");
|
|
|
|
}
|
|
|
|
|
Replace tracked_keys with a new LockTracker interface in TransactionDB (#7013)
Summary:
We're going to support more locking protocols such as range lock in transaction.
However, in current design, `TransactionBase` has a member `tracked_keys` which assumes that point lock (lock a single key) is used, and is used in snapshot checking (isolation protocol). When using range lock, we may use read committed instead of snapshot checking as the isolation protocol.
The most significant usage scenarios of `tracked_keys` are:
1. pessimistic transaction uses it to track the locked keys, and unlock these keys when commit or rollback.
2. optimistic transaction does not lock keys upfront, it only tracks the lock intentions in tracked_keys, and do write conflict checking when commit.
3. each `SavePoint` tracks the keys that are locked since the `SavePoint`, `RollbackToSavePoint` or `PopSavePoint` relies on both the tracked keys in `SavePoint`s and `tracked_keys`.
Based on these scenarios, if we can abstract out a `LockTracker` interface to hold a set of tracked locks (can be keys or key ranges), and have methods that can be composed together to implement the scenarios, then `tracked_keys` can be an internal data structure of one implementation of `LockTracker`. See `utilities/transactions/lock/lock_tracker.h` for the detailed interface design, and `utilities/transactions/lock/point_lock_tracker.cc` for the implementation.
In the future, a `RangeLockTracker` can be implemented to track range locks without affecting other components.
After this PR, a clean interface for lock manager should be possible, and then ideally, we can have pluggable locking protocols.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/7013
Test Plan: Run `transaction_test` and `optimistic_transaction_test`.
Reviewed By: ajkr
Differential Revision: D22163706
Pulled By: cheng-chang
fbshipit-source-id: f2860577b5334e31dd2994f5bc6d7c40d502b1b4
2020-08-06 21:36:48 +02:00
|
|
|
if (save_points_ != nullptr && !save_points_->empty()) {
|
|
|
|
// Unlock any keys locked since last transaction
|
|
|
|
auto& save_point_tracker = *save_points_->top().new_locks_;
|
|
|
|
std::unique_ptr<LockTracker> t(
|
|
|
|
tracked_locks_->GetTrackedLocksSinceSavePoint(save_point_tracker));
|
|
|
|
if (t) {
|
|
|
|
txn_db_impl_->UnLock(this, *t);
|
|
|
|
}
|
2015-09-12 03:10:50 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
return TransactionBaseImpl::RollbackToSavePoint();
|
|
|
|
}
|
|
|
|
|
Pessimistic Transactions
Summary:
Initial implementation of Pessimistic Transactions. This diff contains the api changes discussed in D38913. This diff is pretty large, so let me know if people would prefer to meet up to discuss it.
MyRocks folks: please take a look at the API in include/rocksdb/utilities/transaction[_db].h and let me know if you have any issues.
Also, you'll notice a couple of TODOs in the implementation of RollbackToSavePoint(). After chatting with Siying, I'm going to send out a separate diff for an alternate implementation of this feature that implements the rollback inside of WriteBatch/WriteBatchWithIndex. We can then decide which route is preferable.
Next, I'm planning on doing some perf testing and then integrating this diff into MongoRocks for further testing.
Test Plan: Unit tests, db_bench parallel testing.
Reviewers: igor, rven, sdong, yhchiang, yoshinorim
Reviewed By: sdong
Subscribers: hermanlee4, maykov, spetrunia, leveldb, dhruba
Differential Revision: https://reviews.facebook.net/D40869
2015-05-26 02:37:33 +02:00
|
|
|
// Lock all keys in this batch.
|
|
|
|
// On success, caller should unlock keys_to_unlock
|
2017-08-08 01:07:40 +02:00
|
|
|
Status PessimisticTransaction::LockBatch(WriteBatch* batch,
|
Replace tracked_keys with a new LockTracker interface in TransactionDB (#7013)
Summary:
We're going to support more locking protocols such as range lock in transaction.
However, in current design, `TransactionBase` has a member `tracked_keys` which assumes that point lock (lock a single key) is used, and is used in snapshot checking (isolation protocol). When using range lock, we may use read committed instead of snapshot checking as the isolation protocol.
The most significant usage scenarios of `tracked_keys` are:
1. pessimistic transaction uses it to track the locked keys, and unlock these keys when commit or rollback.
2. optimistic transaction does not lock keys upfront, it only tracks the lock intentions in tracked_keys, and do write conflict checking when commit.
3. each `SavePoint` tracks the keys that are locked since the `SavePoint`, `RollbackToSavePoint` or `PopSavePoint` relies on both the tracked keys in `SavePoint`s and `tracked_keys`.
Based on these scenarios, if we can abstract out a `LockTracker` interface to hold a set of tracked locks (can be keys or key ranges), and have methods that can be composed together to implement the scenarios, then `tracked_keys` can be an internal data structure of one implementation of `LockTracker`. See `utilities/transactions/lock/lock_tracker.h` for the detailed interface design, and `utilities/transactions/lock/point_lock_tracker.cc` for the implementation.
In the future, a `RangeLockTracker` can be implemented to track range locks without affecting other components.
After this PR, a clean interface for lock manager should be possible, and then ideally, we can have pluggable locking protocols.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/7013
Test Plan: Run `transaction_test` and `optimistic_transaction_test`.
Reviewed By: ajkr
Differential Revision: D22163706
Pulled By: cheng-chang
fbshipit-source-id: f2860577b5334e31dd2994f5bc6d7c40d502b1b4
2020-08-06 21:36:48 +02:00
|
|
|
LockTracker* keys_to_unlock) {
|
Pessimistic Transactions
Summary:
Initial implementation of Pessimistic Transactions. This diff contains the api changes discussed in D38913. This diff is pretty large, so let me know if people would prefer to meet up to discuss it.
MyRocks folks: please take a look at the API in include/rocksdb/utilities/transaction[_db].h and let me know if you have any issues.
Also, you'll notice a couple of TODOs in the implementation of RollbackToSavePoint(). After chatting with Siying, I'm going to send out a separate diff for an alternate implementation of this feature that implements the rollback inside of WriteBatch/WriteBatchWithIndex. We can then decide which route is preferable.
Next, I'm planning on doing some perf testing and then integrating this diff into MongoRocks for further testing.
Test Plan: Unit tests, db_bench parallel testing.
Reviewers: igor, rven, sdong, yhchiang, yoshinorim
Reviewed By: sdong
Subscribers: hermanlee4, maykov, spetrunia, leveldb, dhruba
Differential Revision: https://reviews.facebook.net/D40869
2015-05-26 02:37:33 +02:00
|
|
|
class Handler : public WriteBatch::Handler {
|
|
|
|
public:
|
|
|
|
// Sorted map of column_family_id to sorted set of keys.
|
|
|
|
// Since LockBatch() always locks keys in sorted order, it cannot deadlock
|
|
|
|
// with itself. We're not using a comparator here since it doesn't matter
|
|
|
|
// what the sorting is as long as it's consistent.
|
|
|
|
std::map<uint32_t, std::set<std::string>> keys_;
|
|
|
|
|
|
|
|
Handler() {}
|
|
|
|
|
|
|
|
void RecordKey(uint32_t column_family_id, const Slice& key) {
|
|
|
|
std::string key_str = key.ToString();
|
|
|
|
|
2019-10-07 21:21:39 +02:00
|
|
|
auto& cfh_keys = keys_[column_family_id];
|
|
|
|
auto iter = cfh_keys.find(key_str);
|
|
|
|
if (iter == cfh_keys.end()) {
|
Pessimistic Transactions
Summary:
Initial implementation of Pessimistic Transactions. This diff contains the api changes discussed in D38913. This diff is pretty large, so let me know if people would prefer to meet up to discuss it.
MyRocks folks: please take a look at the API in include/rocksdb/utilities/transaction[_db].h and let me know if you have any issues.
Also, you'll notice a couple of TODOs in the implementation of RollbackToSavePoint(). After chatting with Siying, I'm going to send out a separate diff for an alternate implementation of this feature that implements the rollback inside of WriteBatch/WriteBatchWithIndex. We can then decide which route is preferable.
Next, I'm planning on doing some perf testing and then integrating this diff into MongoRocks for further testing.
Test Plan: Unit tests, db_bench parallel testing.
Reviewers: igor, rven, sdong, yhchiang, yoshinorim
Reviewed By: sdong
Subscribers: hermanlee4, maykov, spetrunia, leveldb, dhruba
Differential Revision: https://reviews.facebook.net/D40869
2015-05-26 02:37:33 +02:00
|
|
|
// key not yet seen, store it.
|
2019-10-07 21:21:39 +02:00
|
|
|
cfh_keys.insert({std::move(key_str)});
|
Pessimistic Transactions
Summary:
Initial implementation of Pessimistic Transactions. This diff contains the api changes discussed in D38913. This diff is pretty large, so let me know if people would prefer to meet up to discuss it.
MyRocks folks: please take a look at the API in include/rocksdb/utilities/transaction[_db].h and let me know if you have any issues.
Also, you'll notice a couple of TODOs in the implementation of RollbackToSavePoint(). After chatting with Siying, I'm going to send out a separate diff for an alternate implementation of this feature that implements the rollback inside of WriteBatch/WriteBatchWithIndex. We can then decide which route is preferable.
Next, I'm planning on doing some perf testing and then integrating this diff into MongoRocks for further testing.
Test Plan: Unit tests, db_bench parallel testing.
Reviewers: igor, rven, sdong, yhchiang, yoshinorim
Reviewed By: sdong
Subscribers: hermanlee4, maykov, spetrunia, leveldb, dhruba
Differential Revision: https://reviews.facebook.net/D40869
2015-05-26 02:37:33 +02:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2019-02-14 22:52:47 +01:00
|
|
|
Status PutCF(uint32_t column_family_id, const Slice& key,
|
|
|
|
const Slice& /* unused */) override {
|
Pessimistic Transactions
Summary:
Initial implementation of Pessimistic Transactions. This diff contains the api changes discussed in D38913. This diff is pretty large, so let me know if people would prefer to meet up to discuss it.
MyRocks folks: please take a look at the API in include/rocksdb/utilities/transaction[_db].h and let me know if you have any issues.
Also, you'll notice a couple of TODOs in the implementation of RollbackToSavePoint(). After chatting with Siying, I'm going to send out a separate diff for an alternate implementation of this feature that implements the rollback inside of WriteBatch/WriteBatchWithIndex. We can then decide which route is preferable.
Next, I'm planning on doing some perf testing and then integrating this diff into MongoRocks for further testing.
Test Plan: Unit tests, db_bench parallel testing.
Reviewers: igor, rven, sdong, yhchiang, yoshinorim
Reviewed By: sdong
Subscribers: hermanlee4, maykov, spetrunia, leveldb, dhruba
Differential Revision: https://reviews.facebook.net/D40869
2015-05-26 02:37:33 +02:00
|
|
|
RecordKey(column_family_id, key);
|
|
|
|
return Status::OK();
|
|
|
|
}
|
2019-02-14 22:52:47 +01:00
|
|
|
Status MergeCF(uint32_t column_family_id, const Slice& key,
|
|
|
|
const Slice& /* unused */) override {
|
Pessimistic Transactions
Summary:
Initial implementation of Pessimistic Transactions. This diff contains the api changes discussed in D38913. This diff is pretty large, so let me know if people would prefer to meet up to discuss it.
MyRocks folks: please take a look at the API in include/rocksdb/utilities/transaction[_db].h and let me know if you have any issues.
Also, you'll notice a couple of TODOs in the implementation of RollbackToSavePoint(). After chatting with Siying, I'm going to send out a separate diff for an alternate implementation of this feature that implements the rollback inside of WriteBatch/WriteBatchWithIndex. We can then decide which route is preferable.
Next, I'm planning on doing some perf testing and then integrating this diff into MongoRocks for further testing.
Test Plan: Unit tests, db_bench parallel testing.
Reviewers: igor, rven, sdong, yhchiang, yoshinorim
Reviewed By: sdong
Subscribers: hermanlee4, maykov, spetrunia, leveldb, dhruba
Differential Revision: https://reviews.facebook.net/D40869
2015-05-26 02:37:33 +02:00
|
|
|
RecordKey(column_family_id, key);
|
|
|
|
return Status::OK();
|
|
|
|
}
|
2019-02-14 22:52:47 +01:00
|
|
|
Status DeleteCF(uint32_t column_family_id, const Slice& key) override {
|
Pessimistic Transactions
Summary:
Initial implementation of Pessimistic Transactions. This diff contains the api changes discussed in D38913. This diff is pretty large, so let me know if people would prefer to meet up to discuss it.
MyRocks folks: please take a look at the API in include/rocksdb/utilities/transaction[_db].h and let me know if you have any issues.
Also, you'll notice a couple of TODOs in the implementation of RollbackToSavePoint(). After chatting with Siying, I'm going to send out a separate diff for an alternate implementation of this feature that implements the rollback inside of WriteBatch/WriteBatchWithIndex. We can then decide which route is preferable.
Next, I'm planning on doing some perf testing and then integrating this diff into MongoRocks for further testing.
Test Plan: Unit tests, db_bench parallel testing.
Reviewers: igor, rven, sdong, yhchiang, yoshinorim
Reviewed By: sdong
Subscribers: hermanlee4, maykov, spetrunia, leveldb, dhruba
Differential Revision: https://reviews.facebook.net/D40869
2015-05-26 02:37:33 +02:00
|
|
|
RecordKey(column_family_id, key);
|
|
|
|
return Status::OK();
|
|
|
|
}
|
|
|
|
};
|
|
|
|
|
|
|
|
// Iterating on this handler will add all keys in this batch into keys
|
|
|
|
Handler handler;
|
2020-10-21 23:02:00 +02:00
|
|
|
Status s = batch->Iterate(&handler);
|
|
|
|
if (!s.ok()) {
|
|
|
|
return s;
|
|
|
|
}
|
Pessimistic Transactions
Summary:
Initial implementation of Pessimistic Transactions. This diff contains the api changes discussed in D38913. This diff is pretty large, so let me know if people would prefer to meet up to discuss it.
MyRocks folks: please take a look at the API in include/rocksdb/utilities/transaction[_db].h and let me know if you have any issues.
Also, you'll notice a couple of TODOs in the implementation of RollbackToSavePoint(). After chatting with Siying, I'm going to send out a separate diff for an alternate implementation of this feature that implements the rollback inside of WriteBatch/WriteBatchWithIndex. We can then decide which route is preferable.
Next, I'm planning on doing some perf testing and then integrating this diff into MongoRocks for further testing.
Test Plan: Unit tests, db_bench parallel testing.
Reviewers: igor, rven, sdong, yhchiang, yoshinorim
Reviewed By: sdong
Subscribers: hermanlee4, maykov, spetrunia, leveldb, dhruba
Differential Revision: https://reviews.facebook.net/D40869
2015-05-26 02:37:33 +02:00
|
|
|
|
|
|
|
// Attempt to lock all keys
|
|
|
|
for (const auto& cf_iter : handler.keys_) {
|
|
|
|
uint32_t cfh_id = cf_iter.first;
|
|
|
|
auto& cfh_keys = cf_iter.second;
|
|
|
|
|
|
|
|
for (const auto& key_iter : cfh_keys) {
|
|
|
|
const std::string& key = key_iter;
|
|
|
|
|
2016-12-06 02:18:14 +01:00
|
|
|
s = txn_db_impl_->TryLock(this, cfh_id, key, true /* exclusive */);
|
Pessimistic Transactions
Summary:
Initial implementation of Pessimistic Transactions. This diff contains the api changes discussed in D38913. This diff is pretty large, so let me know if people would prefer to meet up to discuss it.
MyRocks folks: please take a look at the API in include/rocksdb/utilities/transaction[_db].h and let me know if you have any issues.
Also, you'll notice a couple of TODOs in the implementation of RollbackToSavePoint(). After chatting with Siying, I'm going to send out a separate diff for an alternate implementation of this feature that implements the rollback inside of WriteBatch/WriteBatchWithIndex. We can then decide which route is preferable.
Next, I'm planning on doing some perf testing and then integrating this diff into MongoRocks for further testing.
Test Plan: Unit tests, db_bench parallel testing.
Reviewers: igor, rven, sdong, yhchiang, yoshinorim
Reviewed By: sdong
Subscribers: hermanlee4, maykov, spetrunia, leveldb, dhruba
Differential Revision: https://reviews.facebook.net/D40869
2015-05-26 02:37:33 +02:00
|
|
|
if (!s.ok()) {
|
|
|
|
break;
|
|
|
|
}
|
Replace tracked_keys with a new LockTracker interface in TransactionDB (#7013)
Summary:
We're going to support more locking protocols such as range lock in transaction.
However, in current design, `TransactionBase` has a member `tracked_keys` which assumes that point lock (lock a single key) is used, and is used in snapshot checking (isolation protocol). When using range lock, we may use read committed instead of snapshot checking as the isolation protocol.
The most significant usage scenarios of `tracked_keys` are:
1. pessimistic transaction uses it to track the locked keys, and unlock these keys when commit or rollback.
2. optimistic transaction does not lock keys upfront, it only tracks the lock intentions in tracked_keys, and do write conflict checking when commit.
3. each `SavePoint` tracks the keys that are locked since the `SavePoint`, `RollbackToSavePoint` or `PopSavePoint` relies on both the tracked keys in `SavePoint`s and `tracked_keys`.
Based on these scenarios, if we can abstract out a `LockTracker` interface to hold a set of tracked locks (can be keys or key ranges), and have methods that can be composed together to implement the scenarios, then `tracked_keys` can be an internal data structure of one implementation of `LockTracker`. See `utilities/transactions/lock/lock_tracker.h` for the detailed interface design, and `utilities/transactions/lock/point_lock_tracker.cc` for the implementation.
In the future, a `RangeLockTracker` can be implemented to track range locks without affecting other components.
After this PR, a clean interface for lock manager should be possible, and then ideally, we can have pluggable locking protocols.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/7013
Test Plan: Run `transaction_test` and `optimistic_transaction_test`.
Reviewed By: ajkr
Differential Revision: D22163706
Pulled By: cheng-chang
fbshipit-source-id: f2860577b5334e31dd2994f5bc6d7c40d502b1b4
2020-08-06 21:36:48 +02:00
|
|
|
PointLockRequest r;
|
|
|
|
r.column_family_id = cfh_id;
|
|
|
|
r.key = key;
|
|
|
|
r.seq = kMaxSequenceNumber;
|
|
|
|
r.read_only = false;
|
|
|
|
r.exclusive = true;
|
|
|
|
keys_to_unlock->Track(r);
|
Pessimistic Transactions
Summary:
Initial implementation of Pessimistic Transactions. This diff contains the api changes discussed in D38913. This diff is pretty large, so let me know if people would prefer to meet up to discuss it.
MyRocks folks: please take a look at the API in include/rocksdb/utilities/transaction[_db].h and let me know if you have any issues.
Also, you'll notice a couple of TODOs in the implementation of RollbackToSavePoint(). After chatting with Siying, I'm going to send out a separate diff for an alternate implementation of this feature that implements the rollback inside of WriteBatch/WriteBatchWithIndex. We can then decide which route is preferable.
Next, I'm planning on doing some perf testing and then integrating this diff into MongoRocks for further testing.
Test Plan: Unit tests, db_bench parallel testing.
Reviewers: igor, rven, sdong, yhchiang, yoshinorim
Reviewed By: sdong
Subscribers: hermanlee4, maykov, spetrunia, leveldb, dhruba
Differential Revision: https://reviews.facebook.net/D40869
2015-05-26 02:37:33 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
if (!s.ok()) {
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
if (!s.ok()) {
|
Replace tracked_keys with a new LockTracker interface in TransactionDB (#7013)
Summary:
We're going to support more locking protocols such as range lock in transaction.
However, in current design, `TransactionBase` has a member `tracked_keys` which assumes that point lock (lock a single key) is used, and is used in snapshot checking (isolation protocol). When using range lock, we may use read committed instead of snapshot checking as the isolation protocol.
The most significant usage scenarios of `tracked_keys` are:
1. pessimistic transaction uses it to track the locked keys, and unlock these keys when commit or rollback.
2. optimistic transaction does not lock keys upfront, it only tracks the lock intentions in tracked_keys, and do write conflict checking when commit.
3. each `SavePoint` tracks the keys that are locked since the `SavePoint`, `RollbackToSavePoint` or `PopSavePoint` relies on both the tracked keys in `SavePoint`s and `tracked_keys`.
Based on these scenarios, if we can abstract out a `LockTracker` interface to hold a set of tracked locks (can be keys or key ranges), and have methods that can be composed together to implement the scenarios, then `tracked_keys` can be an internal data structure of one implementation of `LockTracker`. See `utilities/transactions/lock/lock_tracker.h` for the detailed interface design, and `utilities/transactions/lock/point_lock_tracker.cc` for the implementation.
In the future, a `RangeLockTracker` can be implemented to track range locks without affecting other components.
After this PR, a clean interface for lock manager should be possible, and then ideally, we can have pluggable locking protocols.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/7013
Test Plan: Run `transaction_test` and `optimistic_transaction_test`.
Reviewed By: ajkr
Differential Revision: D22163706
Pulled By: cheng-chang
fbshipit-source-id: f2860577b5334e31dd2994f5bc6d7c40d502b1b4
2020-08-06 21:36:48 +02:00
|
|
|
txn_db_impl_->UnLock(this, *keys_to_unlock);
|
Pessimistic Transactions
Summary:
Initial implementation of Pessimistic Transactions. This diff contains the api changes discussed in D38913. This diff is pretty large, so let me know if people would prefer to meet up to discuss it.
MyRocks folks: please take a look at the API in include/rocksdb/utilities/transaction[_db].h and let me know if you have any issues.
Also, you'll notice a couple of TODOs in the implementation of RollbackToSavePoint(). After chatting with Siying, I'm going to send out a separate diff for an alternate implementation of this feature that implements the rollback inside of WriteBatch/WriteBatchWithIndex. We can then decide which route is preferable.
Next, I'm planning on doing some perf testing and then integrating this diff into MongoRocks for further testing.
Test Plan: Unit tests, db_bench parallel testing.
Reviewers: igor, rven, sdong, yhchiang, yoshinorim
Reviewed By: sdong
Subscribers: hermanlee4, maykov, spetrunia, leveldb, dhruba
Differential Revision: https://reviews.facebook.net/D40869
2015-05-26 02:37:33 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
return s;
|
|
|
|
}
|
|
|
|
|
|
|
|
// Attempt to lock this key.
|
|
|
|
// Returns OK if the key has been successfully locked. Non-ok, otherwise.
|
|
|
|
// If check_shapshot is true and this transaction has a snapshot set,
|
|
|
|
// this key will only be locked if there have been no writes to this key since
|
|
|
|
// the snapshot time.
|
2017-08-08 01:07:40 +02:00
|
|
|
Status PessimisticTransaction::TryLock(ColumnFamilyHandle* column_family,
|
2017-08-17 01:49:11 +02:00
|
|
|
const Slice& key, bool read_only,
|
2018-12-07 02:46:57 +01:00
|
|
|
bool exclusive, const bool do_validate,
|
|
|
|
const bool assume_tracked) {
|
|
|
|
assert(!assume_tracked || !do_validate);
|
2018-09-11 01:45:59 +02:00
|
|
|
Status s;
|
|
|
|
if (UNLIKELY(skip_concurrency_control_)) {
|
|
|
|
return s;
|
|
|
|
}
|
Pessimistic Transactions
Summary:
Initial implementation of Pessimistic Transactions. This diff contains the api changes discussed in D38913. This diff is pretty large, so let me know if people would prefer to meet up to discuss it.
MyRocks folks: please take a look at the API in include/rocksdb/utilities/transaction[_db].h and let me know if you have any issues.
Also, you'll notice a couple of TODOs in the implementation of RollbackToSavePoint(). After chatting with Siying, I'm going to send out a separate diff for an alternate implementation of this feature that implements the rollback inside of WriteBatch/WriteBatchWithIndex. We can then decide which route is preferable.
Next, I'm planning on doing some perf testing and then integrating this diff into MongoRocks for further testing.
Test Plan: Unit tests, db_bench parallel testing.
Reviewers: igor, rven, sdong, yhchiang, yoshinorim
Reviewed By: sdong
Subscribers: hermanlee4, maykov, spetrunia, leveldb, dhruba
Differential Revision: https://reviews.facebook.net/D40869
2015-05-26 02:37:33 +02:00
|
|
|
uint32_t cfh_id = GetColumnFamilyID(column_family);
|
|
|
|
std::string key_str = key.ToString();
|
2020-12-08 05:16:18 +01:00
|
|
|
|
|
|
|
PointLockStatus status;
|
|
|
|
bool lock_upgrade;
|
|
|
|
bool previously_locked;
|
|
|
|
if (tracked_locks_->IsPointLockSupported()) {
|
|
|
|
status = tracked_locks_->GetPointLockStatus(cfh_id, key_str);
|
|
|
|
previously_locked = status.locked;
|
|
|
|
lock_upgrade = previously_locked && exclusive && !status.exclusive;
|
|
|
|
} else {
|
|
|
|
// If the record is tracked, we can assume it was locked, too.
|
|
|
|
previously_locked = assume_tracked;
|
|
|
|
status.locked = false;
|
|
|
|
lock_upgrade = false;
|
|
|
|
}
|
Pessimistic Transactions
Summary:
Initial implementation of Pessimistic Transactions. This diff contains the api changes discussed in D38913. This diff is pretty large, so let me know if people would prefer to meet up to discuss it.
MyRocks folks: please take a look at the API in include/rocksdb/utilities/transaction[_db].h and let me know if you have any issues.
Also, you'll notice a couple of TODOs in the implementation of RollbackToSavePoint(). After chatting with Siying, I'm going to send out a separate diff for an alternate implementation of this feature that implements the rollback inside of WriteBatch/WriteBatchWithIndex. We can then decide which route is preferable.
Next, I'm planning on doing some perf testing and then integrating this diff into MongoRocks for further testing.
Test Plan: Unit tests, db_bench parallel testing.
Reviewers: igor, rven, sdong, yhchiang, yoshinorim
Reviewed By: sdong
Subscribers: hermanlee4, maykov, spetrunia, leveldb, dhruba
Differential Revision: https://reviews.facebook.net/D40869
2015-05-26 02:37:33 +02:00
|
|
|
|
2017-04-11 00:47:20 +02:00
|
|
|
// Lock this key if this transactions hasn't already locked it or we require
|
|
|
|
// an upgrade.
|
|
|
|
if (!previously_locked || lock_upgrade) {
|
2016-12-06 02:18:14 +01:00
|
|
|
s = txn_db_impl_->TryLock(this, cfh_id, key_str, exclusive);
|
Pessimistic Transactions
Summary:
Initial implementation of Pessimistic Transactions. This diff contains the api changes discussed in D38913. This diff is pretty large, so let me know if people would prefer to meet up to discuss it.
MyRocks folks: please take a look at the API in include/rocksdb/utilities/transaction[_db].h and let me know if you have any issues.
Also, you'll notice a couple of TODOs in the implementation of RollbackToSavePoint(). After chatting with Siying, I'm going to send out a separate diff for an alternate implementation of this feature that implements the rollback inside of WriteBatch/WriteBatchWithIndex. We can then decide which route is preferable.
Next, I'm planning on doing some perf testing and then integrating this diff into MongoRocks for further testing.
Test Plan: Unit tests, db_bench parallel testing.
Reviewers: igor, rven, sdong, yhchiang, yoshinorim
Reviewed By: sdong
Subscribers: hermanlee4, maykov, spetrunia, leveldb, dhruba
Differential Revision: https://reviews.facebook.net/D40869
2015-05-26 02:37:33 +02:00
|
|
|
}
|
|
|
|
|
2015-09-28 21:12:17 +02:00
|
|
|
SetSnapshotIfNeeded();
|
|
|
|
|
|
|
|
// Even though we do not care about doing conflict checking for this write,
|
|
|
|
// we still need to take a lock to make sure we do not cause a conflict with
|
|
|
|
// some other write. However, we do not need to check if there have been
|
|
|
|
// any writes since this transaction's snapshot.
|
|
|
|
// TODO(agiardullo): could optimize by supporting shared txn locks in the
|
|
|
|
// future
|
Replace tracked_keys with a new LockTracker interface in TransactionDB (#7013)
Summary:
We're going to support more locking protocols such as range lock in transaction.
However, in current design, `TransactionBase` has a member `tracked_keys` which assumes that point lock (lock a single key) is used, and is used in snapshot checking (isolation protocol). When using range lock, we may use read committed instead of snapshot checking as the isolation protocol.
The most significant usage scenarios of `tracked_keys` are:
1. pessimistic transaction uses it to track the locked keys, and unlock these keys when commit or rollback.
2. optimistic transaction does not lock keys upfront, it only tracks the lock intentions in tracked_keys, and do write conflict checking when commit.
3. each `SavePoint` tracks the keys that are locked since the `SavePoint`, `RollbackToSavePoint` or `PopSavePoint` relies on both the tracked keys in `SavePoint`s and `tracked_keys`.
Based on these scenarios, if we can abstract out a `LockTracker` interface to hold a set of tracked locks (can be keys or key ranges), and have methods that can be composed together to implement the scenarios, then `tracked_keys` can be an internal data structure of one implementation of `LockTracker`. See `utilities/transactions/lock/lock_tracker.h` for the detailed interface design, and `utilities/transactions/lock/point_lock_tracker.cc` for the implementation.
In the future, a `RangeLockTracker` can be implemented to track range locks without affecting other components.
After this PR, a clean interface for lock manager should be possible, and then ideally, we can have pluggable locking protocols.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/7013
Test Plan: Run `transaction_test` and `optimistic_transaction_test`.
Reviewed By: ajkr
Differential Revision: D22163706
Pulled By: cheng-chang
fbshipit-source-id: f2860577b5334e31dd2994f5bc6d7c40d502b1b4
2020-08-06 21:36:48 +02:00
|
|
|
SequenceNumber tracked_at_seq =
|
|
|
|
status.locked ? status.seq : kMaxSequenceNumber;
|
2018-12-07 02:46:57 +01:00
|
|
|
if (!do_validate || snapshot_ == nullptr) {
|
2020-12-08 05:16:18 +01:00
|
|
|
if (assume_tracked && !previously_locked &&
|
|
|
|
tracked_locks_->IsPointLockSupported()) {
|
2018-12-07 02:46:57 +01:00
|
|
|
s = Status::InvalidArgument(
|
|
|
|
"assume_tracked is set but it is not tracked yet");
|
|
|
|
}
|
2015-09-28 21:12:17 +02:00
|
|
|
// Need to remember the earliest sequence number that we know that this
|
|
|
|
// key has not been modified after. This is useful if this same
|
|
|
|
// transaction
|
|
|
|
// later tries to lock this key again.
|
2017-11-11 22:08:22 +01:00
|
|
|
if (tracked_at_seq == kMaxSequenceNumber) {
|
2015-09-28 21:12:17 +02:00
|
|
|
// Since we haven't checked a snapshot, we only know this key has not
|
|
|
|
// been modified since after we locked it.
|
2017-12-01 08:39:56 +01:00
|
|
|
// Note: when last_seq_same_as_publish_seq_==false this is less than the
|
2017-11-11 22:08:22 +01:00
|
|
|
// latest allocated seq but it is ok since i) this is just a heuristic
|
|
|
|
// used only as a hint to avoid actual check for conflicts, ii) this would
|
|
|
|
// cause a false positive only if the snapthot is taken right after the
|
|
|
|
// lock, which would be an unusual sequence.
|
|
|
|
tracked_at_seq = db_->GetLatestSequenceNumber();
|
2015-09-28 21:12:17 +02:00
|
|
|
}
|
|
|
|
} else {
|
Pessimistic Transactions
Summary:
Initial implementation of Pessimistic Transactions. This diff contains the api changes discussed in D38913. This diff is pretty large, so let me know if people would prefer to meet up to discuss it.
MyRocks folks: please take a look at the API in include/rocksdb/utilities/transaction[_db].h and let me know if you have any issues.
Also, you'll notice a couple of TODOs in the implementation of RollbackToSavePoint(). After chatting with Siying, I'm going to send out a separate diff for an alternate implementation of this feature that implements the rollback inside of WriteBatch/WriteBatchWithIndex. We can then decide which route is preferable.
Next, I'm planning on doing some perf testing and then integrating this diff into MongoRocks for further testing.
Test Plan: Unit tests, db_bench parallel testing.
Reviewers: igor, rven, sdong, yhchiang, yoshinorim
Reviewed By: sdong
Subscribers: hermanlee4, maykov, spetrunia, leveldb, dhruba
Differential Revision: https://reviews.facebook.net/D40869
2015-05-26 02:37:33 +02:00
|
|
|
// If a snapshot is set, we need to make sure the key hasn't been modified
|
|
|
|
// since the snapshot. This must be done after we locked the key.
|
2017-11-11 22:08:22 +01:00
|
|
|
// If we already have validated an earilier snapshot it must has been
|
|
|
|
// reflected in tracked_at_seq and ValidateSnapshot will return OK.
|
2015-09-28 21:12:17 +02:00
|
|
|
if (s.ok()) {
|
2017-11-11 22:08:22 +01:00
|
|
|
s = ValidateSnapshot(column_family, key, &tracked_at_seq);
|
2015-09-28 21:12:17 +02:00
|
|
|
|
|
|
|
if (!s.ok()) {
|
|
|
|
// Failed to validate key
|
Replace tracked_keys with a new LockTracker interface in TransactionDB (#7013)
Summary:
We're going to support more locking protocols such as range lock in transaction.
However, in current design, `TransactionBase` has a member `tracked_keys` which assumes that point lock (lock a single key) is used, and is used in snapshot checking (isolation protocol). When using range lock, we may use read committed instead of snapshot checking as the isolation protocol.
The most significant usage scenarios of `tracked_keys` are:
1. pessimistic transaction uses it to track the locked keys, and unlock these keys when commit or rollback.
2. optimistic transaction does not lock keys upfront, it only tracks the lock intentions in tracked_keys, and do write conflict checking when commit.
3. each `SavePoint` tracks the keys that are locked since the `SavePoint`, `RollbackToSavePoint` or `PopSavePoint` relies on both the tracked keys in `SavePoint`s and `tracked_keys`.
Based on these scenarios, if we can abstract out a `LockTracker` interface to hold a set of tracked locks (can be keys or key ranges), and have methods that can be composed together to implement the scenarios, then `tracked_keys` can be an internal data structure of one implementation of `LockTracker`. See `utilities/transactions/lock/lock_tracker.h` for the detailed interface design, and `utilities/transactions/lock/point_lock_tracker.cc` for the implementation.
In the future, a `RangeLockTracker` can be implemented to track range locks without affecting other components.
After this PR, a clean interface for lock manager should be possible, and then ideally, we can have pluggable locking protocols.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/7013
Test Plan: Run `transaction_test` and `optimistic_transaction_test`.
Reviewed By: ajkr
Differential Revision: D22163706
Pulled By: cheng-chang
fbshipit-source-id: f2860577b5334e31dd2994f5bc6d7c40d502b1b4
2020-08-06 21:36:48 +02:00
|
|
|
// Unlock key we just locked
|
|
|
|
if (lock_upgrade) {
|
|
|
|
s = txn_db_impl_->TryLock(this, cfh_id, key_str,
|
|
|
|
false /* exclusive */);
|
|
|
|
assert(s.ok());
|
|
|
|
} else if (!previously_locked) {
|
|
|
|
txn_db_impl_->UnLock(this, cfh_id, key.ToString());
|
Pessimistic Transactions
Summary:
Initial implementation of Pessimistic Transactions. This diff contains the api changes discussed in D38913. This diff is pretty large, so let me know if people would prefer to meet up to discuss it.
MyRocks folks: please take a look at the API in include/rocksdb/utilities/transaction[_db].h and let me know if you have any issues.
Also, you'll notice a couple of TODOs in the implementation of RollbackToSavePoint(). After chatting with Siying, I'm going to send out a separate diff for an alternate implementation of this feature that implements the rollback inside of WriteBatch/WriteBatchWithIndex. We can then decide which route is preferable.
Next, I'm planning on doing some perf testing and then integrating this diff into MongoRocks for further testing.
Test Plan: Unit tests, db_bench parallel testing.
Reviewers: igor, rven, sdong, yhchiang, yoshinorim
Reviewed By: sdong
Subscribers: hermanlee4, maykov, spetrunia, leveldb, dhruba
Differential Revision: https://reviews.facebook.net/D40869
2015-05-26 02:37:33 +02:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2015-09-12 03:10:50 +02:00
|
|
|
if (s.ok()) {
|
2017-11-11 22:08:22 +01:00
|
|
|
// We must track all the locked keys so that we can unlock them later. If
|
|
|
|
// the key is already locked, this func will update some stats on the
|
2019-04-13 01:34:20 +02:00
|
|
|
// tracked key. It could also update the tracked_at_seq if it is lower
|
|
|
|
// than the existing tracked key seq. These stats are necessary for
|
|
|
|
// RollbackToSavePoint to determine whether a key can be safely removed
|
|
|
|
// from tracked_keys_. Removal can only be done if a key was only locked
|
|
|
|
// during the current savepoint.
|
|
|
|
//
|
|
|
|
// Recall that if assume_tracked is true, we assume that TrackKey has been
|
|
|
|
// called previously since the last savepoint, with the same exclusive
|
|
|
|
// setting, and at a lower sequence number, so skipping here should be
|
|
|
|
// safe.
|
|
|
|
if (!assume_tracked) {
|
|
|
|
TrackKey(cfh_id, key_str, tracked_at_seq, read_only, exclusive);
|
|
|
|
} else {
|
|
|
|
#ifndef NDEBUG
|
2020-12-08 05:16:18 +01:00
|
|
|
if (tracked_locks_->IsPointLockSupported()) {
|
|
|
|
PointLockStatus lock_status =
|
|
|
|
tracked_locks_->GetPointLockStatus(cfh_id, key_str);
|
|
|
|
assert(lock_status.locked);
|
|
|
|
assert(lock_status.seq <= tracked_at_seq);
|
|
|
|
assert(lock_status.exclusive == exclusive);
|
|
|
|
}
|
2019-04-13 01:34:20 +02:00
|
|
|
#endif
|
|
|
|
}
|
2015-09-12 03:10:50 +02:00
|
|
|
}
|
|
|
|
|
Pessimistic Transactions
Summary:
Initial implementation of Pessimistic Transactions. This diff contains the api changes discussed in D38913. This diff is pretty large, so let me know if people would prefer to meet up to discuss it.
MyRocks folks: please take a look at the API in include/rocksdb/utilities/transaction[_db].h and let me know if you have any issues.
Also, you'll notice a couple of TODOs in the implementation of RollbackToSavePoint(). After chatting with Siying, I'm going to send out a separate diff for an alternate implementation of this feature that implements the rollback inside of WriteBatch/WriteBatchWithIndex. We can then decide which route is preferable.
Next, I'm planning on doing some perf testing and then integrating this diff into MongoRocks for further testing.
Test Plan: Unit tests, db_bench parallel testing.
Reviewers: igor, rven, sdong, yhchiang, yoshinorim
Reviewed By: sdong
Subscribers: hermanlee4, maykov, spetrunia, leveldb, dhruba
Differential Revision: https://reviews.facebook.net/D40869
2015-05-26 02:37:33 +02:00
|
|
|
return s;
|
|
|
|
}
|
|
|
|
|
2020-12-08 05:16:18 +01:00
|
|
|
Status PessimisticTransaction::GetRangeLock(ColumnFamilyHandle* column_family,
|
|
|
|
const Endpoint& start_endp,
|
|
|
|
const Endpoint& end_endp) {
|
|
|
|
ColumnFamilyHandle* cfh =
|
|
|
|
column_family ? column_family : db_impl_->DefaultColumnFamily();
|
|
|
|
uint32_t cfh_id = GetColumnFamilyID(cfh);
|
|
|
|
|
|
|
|
Status s = txn_db_impl_->TryRangeLock(this, cfh_id, start_endp, end_endp);
|
|
|
|
|
|
|
|
if (s.ok()) {
|
|
|
|
RangeLockRequest req{cfh_id, start_endp, end_endp};
|
|
|
|
tracked_locks_->Track(req);
|
|
|
|
}
|
|
|
|
return s;
|
|
|
|
}
|
|
|
|
|
Pessimistic Transactions
Summary:
Initial implementation of Pessimistic Transactions. This diff contains the api changes discussed in D38913. This diff is pretty large, so let me know if people would prefer to meet up to discuss it.
MyRocks folks: please take a look at the API in include/rocksdb/utilities/transaction[_db].h and let me know if you have any issues.
Also, you'll notice a couple of TODOs in the implementation of RollbackToSavePoint(). After chatting with Siying, I'm going to send out a separate diff for an alternate implementation of this feature that implements the rollback inside of WriteBatch/WriteBatchWithIndex. We can then decide which route is preferable.
Next, I'm planning on doing some perf testing and then integrating this diff into MongoRocks for further testing.
Test Plan: Unit tests, db_bench parallel testing.
Reviewers: igor, rven, sdong, yhchiang, yoshinorim
Reviewed By: sdong
Subscribers: hermanlee4, maykov, spetrunia, leveldb, dhruba
Differential Revision: https://reviews.facebook.net/D40869
2015-05-26 02:37:33 +02:00
|
|
|
// Return OK() if this key has not been modified more recently than the
|
|
|
|
// transaction snapshot_.
|
2017-11-11 22:08:22 +01:00
|
|
|
// tracked_at_seq is the global seq at which we either locked the key or already
|
|
|
|
// have done ValidateSnapshot.
|
2017-08-17 01:49:11 +02:00
|
|
|
Status PessimisticTransaction::ValidateSnapshot(
|
|
|
|
ColumnFamilyHandle* column_family, const Slice& key,
|
2017-11-11 22:08:22 +01:00
|
|
|
SequenceNumber* tracked_at_seq) {
|
2015-09-28 21:12:17 +02:00
|
|
|
assert(snapshot_);
|
|
|
|
|
2017-11-11 22:08:22 +01:00
|
|
|
SequenceNumber snap_seq = snapshot_->GetSequenceNumber();
|
|
|
|
if (*tracked_at_seq <= snap_seq) {
|
|
|
|
// If the key has been previous validated (or locked) at a sequence number
|
|
|
|
// earlier than the current snapshot's sequence number, we already know it
|
|
|
|
// has not been modified aftter snap_seq either.
|
2015-09-28 21:12:17 +02:00
|
|
|
return Status::OK();
|
|
|
|
}
|
2017-11-11 22:08:22 +01:00
|
|
|
// Otherwise we have either
|
|
|
|
// 1: tracked_at_seq == kMaxSequenceNumber, i.e., first time tracking the key
|
|
|
|
// 2: snap_seq < tracked_at_seq: last time we lock the key was via
|
2018-12-07 02:46:57 +01:00
|
|
|
// do_validate=false which means we had skipped ValidateSnapshot. In both
|
2017-11-11 22:08:22 +01:00
|
|
|
// cases we should do ValidateSnapshot now.
|
Pessimistic Transactions
Summary:
Initial implementation of Pessimistic Transactions. This diff contains the api changes discussed in D38913. This diff is pretty large, so let me know if people would prefer to meet up to discuss it.
MyRocks folks: please take a look at the API in include/rocksdb/utilities/transaction[_db].h and let me know if you have any issues.
Also, you'll notice a couple of TODOs in the implementation of RollbackToSavePoint(). After chatting with Siying, I'm going to send out a separate diff for an alternate implementation of this feature that implements the rollback inside of WriteBatch/WriteBatchWithIndex. We can then decide which route is preferable.
Next, I'm planning on doing some perf testing and then integrating this diff into MongoRocks for further testing.
Test Plan: Unit tests, db_bench parallel testing.
Reviewers: igor, rven, sdong, yhchiang, yoshinorim
Reviewed By: sdong
Subscribers: hermanlee4, maykov, spetrunia, leveldb, dhruba
Differential Revision: https://reviews.facebook.net/D40869
2015-05-26 02:37:33 +02:00
|
|
|
|
2017-11-11 22:08:22 +01:00
|
|
|
*tracked_at_seq = snap_seq;
|
Pessimistic Transactions
Summary:
Initial implementation of Pessimistic Transactions. This diff contains the api changes discussed in D38913. This diff is pretty large, so let me know if people would prefer to meet up to discuss it.
MyRocks folks: please take a look at the API in include/rocksdb/utilities/transaction[_db].h and let me know if you have any issues.
Also, you'll notice a couple of TODOs in the implementation of RollbackToSavePoint(). After chatting with Siying, I'm going to send out a separate diff for an alternate implementation of this feature that implements the rollback inside of WriteBatch/WriteBatchWithIndex. We can then decide which route is preferable.
Next, I'm planning on doing some perf testing and then integrating this diff into MongoRocks for further testing.
Test Plan: Unit tests, db_bench parallel testing.
Reviewers: igor, rven, sdong, yhchiang, yoshinorim
Reviewed By: sdong
Subscribers: hermanlee4, maykov, spetrunia, leveldb, dhruba
Differential Revision: https://reviews.facebook.net/D40869
2015-05-26 02:37:33 +02:00
|
|
|
|
2015-09-28 21:12:17 +02:00
|
|
|
ColumnFamilyHandle* cfh =
|
2017-08-03 17:46:47 +02:00
|
|
|
column_family ? column_family : db_impl_->DefaultColumnFamily();
|
Pessimistic Transactions
Summary:
Initial implementation of Pessimistic Transactions. This diff contains the api changes discussed in D38913. This diff is pretty large, so let me know if people would prefer to meet up to discuss it.
MyRocks folks: please take a look at the API in include/rocksdb/utilities/transaction[_db].h and let me know if you have any issues.
Also, you'll notice a couple of TODOs in the implementation of RollbackToSavePoint(). After chatting with Siying, I'm going to send out a separate diff for an alternate implementation of this feature that implements the rollback inside of WriteBatch/WriteBatchWithIndex. We can then decide which route is preferable.
Next, I'm planning on doing some perf testing and then integrating this diff into MongoRocks for further testing.
Test Plan: Unit tests, db_bench parallel testing.
Reviewers: igor, rven, sdong, yhchiang, yoshinorim
Reviewed By: sdong
Subscribers: hermanlee4, maykov, spetrunia, leveldb, dhruba
Differential Revision: https://reviews.facebook.net/D40869
2015-05-26 02:37:33 +02:00
|
|
|
|
2017-11-11 22:08:22 +01:00
|
|
|
return TransactionUtil::CheckKeyForConflicts(
|
|
|
|
db_impl_, cfh, key.ToString(), snap_seq, false /* cache_only */);
|
Pessimistic Transactions
Summary:
Initial implementation of Pessimistic Transactions. This diff contains the api changes discussed in D38913. This diff is pretty large, so let me know if people would prefer to meet up to discuss it.
MyRocks folks: please take a look at the API in include/rocksdb/utilities/transaction[_db].h and let me know if you have any issues.
Also, you'll notice a couple of TODOs in the implementation of RollbackToSavePoint(). After chatting with Siying, I'm going to send out a separate diff for an alternate implementation of this feature that implements the rollback inside of WriteBatch/WriteBatchWithIndex. We can then decide which route is preferable.
Next, I'm planning on doing some perf testing and then integrating this diff into MongoRocks for further testing.
Test Plan: Unit tests, db_bench parallel testing.
Reviewers: igor, rven, sdong, yhchiang, yoshinorim
Reviewed By: sdong
Subscribers: hermanlee4, maykov, spetrunia, leveldb, dhruba
Differential Revision: https://reviews.facebook.net/D40869
2015-05-26 02:37:33 +02:00
|
|
|
}
|
|
|
|
|
2017-08-08 01:07:40 +02:00
|
|
|
bool PessimisticTransaction::TryStealingLocks() {
|
2016-02-02 02:07:05 +01:00
|
|
|
assert(IsExpired());
|
2016-10-05 22:39:00 +02:00
|
|
|
TransactionState expected = STARTED;
|
|
|
|
return std::atomic_compare_exchange_strong(&txn_state_, &expected,
|
2016-02-02 02:07:05 +01:00
|
|
|
LOCKS_STOLEN);
|
|
|
|
}
|
|
|
|
|
2017-08-17 01:49:11 +02:00
|
|
|
void PessimisticTransaction::UnlockGetForUpdate(
|
|
|
|
ColumnFamilyHandle* column_family, const Slice& key) {
|
2015-09-15 02:11:52 +02:00
|
|
|
txn_db_impl_->UnLock(this, GetColumnFamilyID(column_family), key.ToString());
|
|
|
|
}
|
|
|
|
|
2017-08-08 01:07:40 +02:00
|
|
|
Status PessimisticTransaction::SetName(const TransactionName& name) {
|
2016-04-18 20:15:50 +02:00
|
|
|
Status s;
|
2016-10-05 22:39:00 +02:00
|
|
|
if (txn_state_ == STARTED) {
|
2016-04-18 20:15:50 +02:00
|
|
|
if (name_.length()) {
|
|
|
|
s = Status::InvalidArgument("Transaction has already been named.");
|
|
|
|
} else if (txn_db_impl_->GetTransactionByName(name) != nullptr) {
|
|
|
|
s = Status::InvalidArgument("Transaction name must be unique.");
|
|
|
|
} else if (name.length() < 1 || name.length() > 512) {
|
|
|
|
s = Status::InvalidArgument(
|
|
|
|
"Transaction name length must be between 1 and 512 chars.");
|
|
|
|
} else {
|
|
|
|
name_ = name;
|
|
|
|
txn_db_impl_->RegisterTransaction(this);
|
|
|
|
}
|
|
|
|
} else {
|
|
|
|
s = Status::InvalidArgument("Transaction is beyond state for naming.");
|
|
|
|
}
|
|
|
|
return s;
|
|
|
|
}
|
|
|
|
|
2020-02-20 21:07:53 +01:00
|
|
|
} // namespace ROCKSDB_NAMESPACE
|
Pessimistic Transactions
Summary:
Initial implementation of Pessimistic Transactions. This diff contains the api changes discussed in D38913. This diff is pretty large, so let me know if people would prefer to meet up to discuss it.
MyRocks folks: please take a look at the API in include/rocksdb/utilities/transaction[_db].h and let me know if you have any issues.
Also, you'll notice a couple of TODOs in the implementation of RollbackToSavePoint(). After chatting with Siying, I'm going to send out a separate diff for an alternate implementation of this feature that implements the rollback inside of WriteBatch/WriteBatchWithIndex. We can then decide which route is preferable.
Next, I'm planning on doing some perf testing and then integrating this diff into MongoRocks for further testing.
Test Plan: Unit tests, db_bench parallel testing.
Reviewers: igor, rven, sdong, yhchiang, yoshinorim
Reviewed By: sdong
Subscribers: hermanlee4, maykov, spetrunia, leveldb, dhruba
Differential Revision: https://reviews.facebook.net/D40869
2015-05-26 02:37:33 +02:00
|
|
|
|
|
|
|
#endif // ROCKSDB_LITE
|