[directory includes cleanup] Move cross-function test points

Summary:
I split the db-specific test points out into a separate file under db/
directory. There were also a few bugs to fix in xfunc.{h,cc} that prevented it
from compiling previously; see https://reviews.facebook.net/D36825.

Test Plan:
compilation works now, below command works, will also run "make xfunc".

  $ make check ROCKSDB_XFUNC_TEST='managed_new' tests-regexp='DBTest' -j32

Reviewers: sdong, yhchiang

Subscribers: dhruba

Differential Revision: https://reviews.facebook.net/D53343

CMakeLists.txt

@@ -145,6 +145,7 @@ set(SOURCES
         db/write_batch_base.cc
         db/write_controller.cc
         db/write_thread.cc
+        db/xfunc_test_points.cc
         memtable/hash_cuckoo_rep.cc
         memtable/hash_linklist_rep.cc
         memtable/hash_skiplist_rep.cc

db/db_impl.cc

@@ -54,6 +54,7 @@
 #include "db/write_batch_internal.h"
 #include "db/write_callback.h"
 #include "db/writebuffer.h"
+#include "db/xfunc_test_points.h"
 #include "memtable/hash_linklist_rep.h"
 #include "memtable/hash_skiplist_rep.h"
 #include "port/likely.h"

db/managed_iterator.cc

@@ -5,6 +5,8 @@
 
 #ifndef ROCKSDB_LITE
 
+#include "db/managed_iterator.h"
+
 #include <limits>
 #include <string>
 #include <utility>
@@ -13,7 +15,7 @@
 #include "db/db_impl.h"
 #include "db/db_iter.h"
 #include "db/dbformat.h"
-#include "db/managed_iterator.h"
+#include "db/xfunc_test_points.h"
 #include "rocksdb/env.h"
 #include "rocksdb/slice.h"
 #include "rocksdb/slice_transform.h"

db/xfunc_test_points.cc

@@ -0,0 +1,145 @@
+//  Copyright (c) 2016, Facebook, Inc.  All rights reserved.
+//  This source code is licensed under the BSD-style license found in the
+//  LICENSE file in the root directory of this source tree. An additional grant
+//  of patent rights can be found in the PATENTS file in the same directory.
+
+#include "db/xfunc_test_points.h"
+#include "util/xfunc.h"
+
+namespace rocksdb {
+
+#ifdef XFUNC
+
+void xf_manage_release(ManagedIterator* iter) {
+  if (!(XFuncPoint::GetSkip() & kSkipNoPrefix)) {
+    iter->ReleaseIter(false);
+  }
+}
+
+void xf_manage_create(ManagedIterator* iter) { iter->SetDropOld(false); }
+
+void xf_manage_new(DBImpl* db, ReadOptions* read_options,
+                   bool is_snapshot_supported) {
+  if ((!XFuncPoint::Check("managed_xftest_dropold") &&
+       (!XFuncPoint::Check("managed_xftest_release"))) ||
+      (!read_options->managed)) {
+    return;
+  }
+  if ((!read_options->tailing) && (read_options->snapshot == nullptr) &&
+      (!is_snapshot_supported)) {
+    read_options->managed = false;
+    return;
+  }
+  if (db->GetOptions().prefix_extractor != nullptr) {
+    if (strcmp(db->GetOptions().table_factory.get()->Name(), "PlainTable")) {
+      if (!(XFuncPoint::GetSkip() & kSkipNoPrefix)) {
+        read_options->total_order_seek = true;
+      }
+    } else {
+      read_options->managed = false;
+    }
+  }
+}
+
+class XFTransactionWriteHandler : public WriteBatch::Handler {
+ public:
+  Transaction* txn_;
+  DBImpl* db_impl_;
+
+  XFTransactionWriteHandler(Transaction* txn, DBImpl* db_impl)
+      : txn_(txn), db_impl_(db_impl) {}
+
+  virtual Status PutCF(uint32_t column_family_id, const Slice& key,
+                       const Slice& value) override {
+    InstrumentedMutexLock l(&db_impl_->mutex_);
+
+    ColumnFamilyHandle* cfh = db_impl_->GetColumnFamilyHandle(column_family_id);
+    if (cfh == nullptr) {
+      return Status::InvalidArgument(
+          "XFUNC test could not find column family "
+          "handle for id ",
+          ToString(column_family_id));
+    }
+
+    txn_->Put(cfh, key, value);
+
+    return Status::OK();
+  }
+
+  virtual Status MergeCF(uint32_t column_family_id, const Slice& key,
+                         const Slice& value) override {
+    InstrumentedMutexLock l(&db_impl_->mutex_);
+
+    ColumnFamilyHandle* cfh = db_impl_->GetColumnFamilyHandle(column_family_id);
+    if (cfh == nullptr) {
+      return Status::InvalidArgument(
+          "XFUNC test could not find column family "
+          "handle for id ",
+          ToString(column_family_id));
+    }
+
+    txn_->Merge(cfh, key, value);
+
+    return Status::OK();
+  }
+
+  virtual Status DeleteCF(uint32_t column_family_id,
+                          const Slice& key) override {
+    InstrumentedMutexLock l(&db_impl_->mutex_);
+
+    ColumnFamilyHandle* cfh = db_impl_->GetColumnFamilyHandle(column_family_id);
+    if (cfh == nullptr) {
+      return Status::InvalidArgument(
+          "XFUNC test could not find column family "
+          "handle for id ",
+          ToString(column_family_id));
+    }
+
+    txn_->Delete(cfh, key);
+
+    return Status::OK();
+  }
+
+  virtual void LogData(const Slice& blob) override { txn_->PutLogData(blob); }
+};
+
+// Whenever DBImpl::Write is called, create a transaction and do the write via
+// the transaction.
+void xf_transaction_write(const WriteOptions& write_options,
+                          const DBOptions& db_options, WriteBatch* my_batch,
+                          WriteCallback* callback, DBImpl* db_impl, Status* s,
+                          bool* write_attempted) {
+  if (callback != nullptr) {
+    // We may already be in a transaction, don't force a transaction
+    *write_attempted = false;
+    return;
+  }
+
+  OptimisticTransactionDB* txn_db = new OptimisticTransactionDB(db_impl);
+  Transaction* txn = Transaction::BeginTransaction(txn_db, write_options);
+
+  XFTransactionWriteHandler handler(txn, db_impl);
+  *s = my_batch->Iterate(&handler);
+
+  if (!s->ok()) {
+    Log(InfoLogLevel::ERROR_LEVEL, db_options.info_log,
+        "XFUNC test could not iterate batch.  status: $s\n",
+        s->ToString().c_str());
+  }
+
+  *s = txn->Commit();
+
+  if (!s->ok()) {
+    Log(InfoLogLevel::ERROR_LEVEL, db_options.info_log,
+        "XFUNC test could not commit transaction.  status: $s\n",
+        s->ToString().c_str());
+  }
+
+  *write_attempted = true;
+  delete txn;
+  delete txn_db;
+}
+
+#endif  // XFUNC
+
+}  // namespace rocksdb

db/xfunc_test_points.h

@@ -0,0 +1,33 @@
+//  Copyright (c) 2016, Facebook, Inc.  All rights reserved.
+//  This source code is licensed under the BSD-style license found in the
+//  LICENSE file in the root directory of this source tree. An additional grant
+//  of patent rights can be found in the PATENTS file in the same directory.
+
+#pragma once
+
+#include "db/db_impl.h"
+#include "db/managed_iterator.h"
+#include "db/write_callback.h"
+#include "rocksdb/options.h"
+#include "rocksdb/write_batch.h"
+#include "util/xfunc.h"
+
+namespace rocksdb {
+
+#ifdef XFUNC
+
+// DB-specific test points for the cross-functional test framework (see
+// util/xfunc.h).
+void xf_manage_release(ManagedIterator* iter);
+void xf_manage_create(ManagedIterator* iter);
+void xf_manage_new(DBImpl* db, ReadOptions* readoptions,
+                   bool is_snapshot_supported);
+void xf_transaction_write(const WriteOptions& write_options,
+                          const DBOptions& db_options,
+                          class WriteBatch* my_batch,
+                          class WriteCallback* callback, DBImpl* db_impl,
+                          Status* success, bool* write_attempted);
+
+#endif  // XFUNC
+
+}  // namespace rocksdb

src.mk

@@ -46,6 +46,7 @@ LIB_SOURCES =                                                   \
   db/write_batch_base.cc                                        \
   db/write_controller.cc                                        \
   db/write_thread.cc                                            \
+  db/xfunc_test_points.cc                                       \
   memtable/hash_cuckoo_rep.cc                                   \
   memtable/hash_linklist_rep.cc                                 \
   memtable/hash_skiplist_rep.cc                                 \

util/xfunc.cc

@@ -4,16 +4,14 @@
 //  of patent rights can be found in the PATENTS file in the same directory.
 
 #ifdef XFUNC
+#include "util/xfunc.h"
+
 #include <string>
-#include "db/db_impl.h"
+
-#include "db/managed_iterator.h"
-#include "db/write_callback.h"
 #include "rocksdb/db.h"
 #include "rocksdb/options.h"
 #include "rocksdb/utilities/optimistic_transaction_db.h"
 #include "rocksdb/write_batch.h"
-#include "util/xfunc.h"
-
 
 namespace rocksdb {
 
@@ -29,12 +27,6 @@ void GetXFTestOptions(Options* options, int skip_policy) {
   }
 }
 
-void xf_manage_release(ManagedIterator* iter) {
-  if (!(XFuncPoint::GetSkip() & kSkipNoPrefix)) {
-    iter->ReleaseIter(false);
-  }
-}
-
 void xf_manage_options(ReadOptions* read_options) {
   if (!XFuncPoint::Check("managed_xftest_dropold") &&
       (!XFuncPoint::Check("managed_xftest_release"))) {
@@ -43,31 +35,6 @@ void xf_manage_options(ReadOptions* read_options) {
   read_options->managed = true;
 }
 
-void xf_manage_new(DBImpl* db, ReadOptions* read_options,
-                   bool is_snapshot_supported) {
-  if ((!XFuncPoint::Check("managed_xftest_dropold") &&
-       (!XFuncPoint::Check("managed_xftest_release"))) ||
-      (!read_options->managed)) {
-    return;
-  }
-  if ((!read_options->tailing) && (read_options->snapshot == nullptr) &&
-      (!is_snapshot_supported)) {
-    read_options->managed = false;
-    return;
-  }
-  if (db->GetOptions().prefix_extractor != nullptr) {
-    if (strcmp(db->GetOptions().table_factory.get()->Name(), "PlainTable")) {
-      if (!(XFuncPoint::GetSkip() & kSkipNoPrefix)) {
-        read_options->total_order_seek = true;
-      }
-    } else {
-      read_options->managed = false;
-    }
-  }
-}
-
-void xf_manage_create(ManagedIterator* iter) { iter->SetDropOld(false); }
-
 void xf_transaction_set_memtable_history(
     int32_t* max_write_buffer_number_to_maintain) {
   *max_write_buffer_number_to_maintain = 10;
@@ -78,106 +45,6 @@ void xf_transaction_clear_memtable_history(
   *max_write_buffer_number_to_maintain = 0;
 }
 
-class XFTransactionWriteHandler : public WriteBatch::Handler {
- public:
-  OptimisticTransaction* txn_;
-  DBImpl* db_impl_;
-
-  XFTransactionWriteHandler(OptimisticTransaction* txn, DBImpl* db_impl)
-      : txn_(txn), db_impl_(db_impl) {}
-
-  virtual Status PutCF(uint32_t column_family_id, const Slice& key,
-                       const Slice& value) override {
-    InstrumentedMutexLock l(&db_impl_->mutex_);
-
-    ColumnFamilyHandle* cfh = db_impl_->GetColumnFamilyHandle(column_family_id);
-    if (cfh == nullptr) {
-      return Status::InvalidArgument(
-          "XFUNC test could not find column family "
-          "handle for id ",
-          ToString(column_family_id));
-    }
-
-    txn_->Put(cfh, key, value);
-
-    return Status::OK();
-  }
-
-  virtual Status MergeCF(uint32_t column_family_id, const Slice& key,
-                         const Slice& value) override {
-    InstrumentedMutexLock l(&db_impl_->mutex_);
-
-    ColumnFamilyHandle* cfh = db_impl_->GetColumnFamilyHandle(column_family_id);
-    if (cfh == nullptr) {
-      return Status::InvalidArgument(
-          "XFUNC test could not find column family "
-          "handle for id ",
-          ToString(column_family_id));
-    }
-
-    txn_->Merge(cfh, key, value);
-
-    return Status::OK();
-  }
-
-  virtual Status DeleteCF(uint32_t column_family_id,
-                          const Slice& key) override {
-    InstrumentedMutexLock l(&db_impl_->mutex_);
-
-    ColumnFamilyHandle* cfh = db_impl_->GetColumnFamilyHandle(column_family_id);
-    if (cfh == nullptr) {
-      return Status::InvalidArgument(
-          "XFUNC test could not find column family "
-          "handle for id ",
-          ToString(column_family_id));
-    }
-
-    txn_->Delete(cfh, key);
-
-    return Status::OK();
-  }
-
-  virtual void LogData(const Slice& blob) override { txn_->PutLogData(blob); }
-};
-
-// Whenever DBImpl::Write is called, create a transaction and do the write via
-// the transaction.
-void xf_transaction_write(const WriteOptions& write_options,
-                          const DBOptions& db_options, WriteBatch* my_batch,
-                          WriteCallback* callback, DBImpl* db_impl, Status* s,
-                          bool* write_attempted) {
-  if (callback != nullptr) {
-    // We may already be in a transaction, don't force a transaction
-    *write_attempted = false;
-    return;
-  }
-
-  OptimisticTransactionDB* txn_db = new OptimisticTransactionDB(db_impl);
-  OptimisticTransaction* txn =
-      OptimisticTransaction::BeginTransaction(txn_db, write_options);
-
-  XFTransactionWriteHandler handler(txn, db_impl);
-  *s = my_batch->Iterate(&handler);
-
-  if (!s->ok()) {
-    Log(InfoLogLevel::ERROR_LEVEL, db_options.info_log,
-        "XFUNC test could not iterate batch.  status: $s\n",
-        s->ToString().c_str());
-  }
-
-  *s = txn->Commit();
-
-  if (!s->ok()) {
-    Log(InfoLogLevel::ERROR_LEVEL, db_options.info_log,
-        "XFUNC test could not commit transaction.  status: $s\n",
-        s->ToString().c_str());
-  }
-
-  *write_attempted = true;
-  delete txn;
-  delete txn_db;
-}
-
 }  // namespace rocksdb
 
 #endif  // XFUNC

util/xfunc.h

@@ -7,6 +7,8 @@
 #include <cstdlib>
 #include <string>
 
+#include "rocksdb/options.h"
+
 namespace rocksdb {
 
 /*
@@ -31,25 +33,12 @@ namespace rocksdb {
 #define XFUNC_TEST(condition, location, lfname, fname, ...)
 #else
 
-struct Options;
-struct WriteOptions;
-class ManagedIterator;
-class DBImpl;
 void GetXFTestOptions(Options* options, int skip_policy);
-void xf_manage_release(ManagedIterator* iter);
-void xf_manage_new(DBImpl* db, ReadOptions* readoptions,
-                   bool is_snapshot_supported);
-void xf_manage_create(ManagedIterator* iter);
 void xf_manage_options(ReadOptions* read_options);
 void xf_transaction_set_memtable_history(
     int32_t* max_write_buffer_number_to_maintain);
 void xf_transaction_clear_memtable_history(
     int32_t* max_write_buffer_number_to_maintain);
-void xf_transaction_write(const WriteOptions& write_options,
-                          const DBOptions& db_options,
-                          class WriteBatch* my_batch,
-                          class WriteCallback* callback, DBImpl* db_impl,
-                          Status* success, bool* write_attempted);
 
 // This class provides the facility to run custom code to test a specific
 // feature typically with all existing unit tests.