put log in a seperate dir

Summary: added a new option db_log_dir, which points the log dir. Inside that dir, in order to make log names unique, the log file name is prefixed with the leveldb data dir absolute path.

Test Plan: db_test

Reviewers: dhruba

Reviewed By: dhruba

Differential Revision: https://reviews.facebook.net/D5205

db/c.cc

@@ -502,6 +502,26 @@ void leveldb_options_set_compression(leveldb_options_t* opt, int t) {
   opt->rep.compression = static_cast<CompressionType>(t);
 }
 
+void leveldb_options_set_disable_data_sync(
+    leveldb_options_t* opt, bool disable_data_sync) {
+  opt->rep.disableDataSync = disable_data_sync;
+}
+
+void leveldb_options_set_use_fsync(
+    leveldb_options_t* opt, bool use_fsync) {
+  opt->rep.use_fsync = use_fsync;
+}
+
+void leveldb_options_set_db_stats_log_interval(
+    leveldb_options_t* opt, int db_stats_log_interval) {
+  opt->rep.db_stats_log_interval = db_stats_log_interval;
+}
+
+void leveldb_options_set_db_log_dir(
+    leveldb_options_t* opt, const char* db_log_dir) {
+  opt->rep.db_log_dir = db_log_dir;
+}
+
 leveldb_comparator_t* leveldb_comparator_create(
     void* state,
     void (*destructor)(void*),

db/db_impl.cc

@@ -98,12 +98,16 @@ Options SanitizeOptions(const std::string& dbname,
   ClipToRange(&result.max_open_files,            20,     50000);
   ClipToRange(&result.write_buffer_size,         64<<10, 1<<30);
   ClipToRange(&result.block_size,                1<<10,  4<<20);
+  std::string db_absolute_path;
+  src.env->GetAbsolutePath(dbname, &db_absolute_path);
   if (result.info_log == NULL) {
     // Open a log file in the same directory as the db
     src.env->CreateDir(dbname);  // In case it does not exist
-    src.env->RenameFile(InfoLogFileName(dbname),
+    src.env->RenameFile(InfoLogFileName(dbname, db_absolute_path,
-        OldInfoLogFileName(dbname, src.env->NowMicros()));
+        result.db_log_dir), OldInfoLogFileName(dbname,src.env->NowMicros(),
-    Status s = src.env->NewLogger(InfoLogFileName(dbname), &result.info_log);
+            db_absolute_path, result.db_log_dir));
+    Status s = src.env->NewLogger(InfoLogFileName(dbname, db_absolute_path,
+        result.db_log_dir), &result.info_log);
     if (!s.ok()) {
       // No place suitable for logging
       result.info_log = NULL;
@@ -140,6 +144,7 @@ DBImpl::DBImpl(const Options& options, const std::string& dbname)
   mem_->Ref();
   has_imm_.Release_Store(NULL);
 
+  env_->GetAbsolutePath(dbname, &db_absolute_path_);
   stats_ = new CompactionStats[options.num_levels];
   // Reserve ten files or so for other uses and give the rest to TableCache.
   const int table_cache_size = options_.max_open_files - 10;
@@ -247,7 +252,7 @@ void DBImpl::DeleteObsoleteFiles() {
   env_->GetChildren(dbname_, &filenames); // Ignoring errors on purpose
   uint64_t number;
   FileType type;
-  std::vector<uint64_t> old_log_files_ts;
+  std::vector<std::string> old_log_files;
   for (size_t i = 0; i < filenames.size(); i++) {
     if (ParseFileName(filenames[i], &number, &type)) {
       bool keep = true;
@@ -272,7 +277,7 @@ void DBImpl::DeleteObsoleteFiles() {
         case kInfoLogFile:
           keep = true;
           if (number != 0) {
-            old_log_files_ts.push_back(number);
+            old_log_files.push_back(filenames[i]);
           }
           break;
         case kCurrentFile:
@@ -299,15 +304,14 @@ void DBImpl::DeleteObsoleteFiles() {
   }
 
   // Delete old log files.
-  int old_log_file_count = old_log_files_ts.size();
+  int old_log_file_count = old_log_files.size();
-  if (old_log_file_count >= KEEP_LOG_FILE_NUM) {
+  if (old_log_file_count >= KEEP_LOG_FILE_NUM &&
-    std::sort(old_log_files_ts.begin(), old_log_files_ts.end());
+      !options_.db_log_dir.empty()) {
+    std::sort(old_log_files.begin(), old_log_files.end());
     for (int i = 0; i >= (old_log_file_count - KEEP_LOG_FILE_NUM); i++) {
-      uint64_t ts = old_log_files_ts.at(i);
+      std::string& to_delete = old_log_files.at(i);
-      std::string to_delete = OldInfoLogFileName(dbname_, ts);
+      Log(options_.info_log, "Delete type=%d %s\n",
-      Log(options_.info_log, "Delete type=%d #%lld\n",
+          int(kInfoLogFile), to_delete.c_str());
-          int(kInfoLogFile),
-          static_cast<unsigned long long>(ts));
       env_->DeleteFile(dbname_ + "/" + to_delete);
     }
   }

db/db_impl.h

@@ -210,6 +210,7 @@ class DBImpl : public DB {
   CompactionStats* stats_;
 
   static const int KEEP_LOG_FILE_NUM = 1000;
+  std::string db_absolute_path_;
 
   // No copying allowed
   DBImpl(const DBImpl&);

db/db_stats_logger.cc

@@ -74,11 +74,9 @@ void DBImpl::LogDBDeployStats() {
 
   int64_t unix_ts;
   env_->GetCurrentTime(&unix_ts);
-  std::string data_dir;
-  env_->GetAbsolutePath(dbname_, &data_dir);
 
   logger_->Log_Deploy_Stats(version_info, host_name_,
-      data_dir, file_total_size, file_total_num, file_num_per_level,
+      db_absolute_path_, file_total_size, file_total_num, file_num_per_level,
       data_size_per_level, unix_ts);
 
   mutex_.Lock();

db/db_test.cc

@@ -154,6 +154,7 @@ class DBTest {
     kFilter,
     kUncompressed,
     kNumLevel_3,
+    kDBLogDir,
     kEnd
   };
   int option_config_;
@@ -201,11 +202,14 @@ class DBTest {
         options.filter_policy = filter_policy_;
         break;
       case kUncompressed:
-	options.compression = kNoCompression;
+        options.compression = kNoCompression;
-	break;
+        break;
       case kNumLevel_3:
-	options.num_levels = 3;
+        options.num_levels = 3;
-	break;
+        break;
+      case kDBLogDir:
+        options.db_log_dir = test::TmpDir();
+        break;
       default:
         break;
     }

db/filename.cc

@@ -8,9 +8,37 @@
 #include "db/dbformat.h"
 #include "leveldb/env.h"
 #include "util/logging.h"
+#include <iostream>
 
 namespace leveldb {
 
+// Given a path, flatten the path name by replacing all chars not in
+// {[0-9,a-z,A-Z,-,_,.]} with _. And append '\0' at the end.
+// Return the number of chars stored in dest not including the trailing '\0'.
+static int FlattenPath(const std::string& path, char* dest, int len) {
+  int write_idx = 0;
+  int i = 0;
+  int src_len = path.size();
+
+  while (i < src_len && write_idx < len - 1) {
+    if ((path[i] >= 'a' && path[i] <= 'z') ||
+        (path[i] >= '0' && path[i] <= '9') ||
+        (path[i] >= 'A' && path[i] <= 'Z') ||
+        path[i] == '-' ||
+        path[i] == '.' ||
+        path[i] == '_'){
+      dest[write_idx++] = path[i];
+    } else {
+      if (i > 0)
+        dest[write_idx++] = '_';
+    }
+    i++;
+  }
+
+  dest[write_idx] = '\0';
+  return write_idx;
+}
+
 // A utility routine: write "data" to the named file and Sync() it.
 extern Status WriteStringToFileSync(Env* env, const Slice& data,
                                     const std::string& fname);
@@ -55,15 +83,28 @@ std::string TempFileName(const std::string& dbname, uint64_t number) {
   return MakeFileName(dbname, number, "dbtmp");
 }
 
-std::string InfoLogFileName(const std::string& dbname) {
+std::string InfoLogFileName(const std::string& dbname,
-  return dbname + "/LOG";
+    const std::string& db_path, const std::string& log_dir) {
+  if (log_dir.empty())
+    return dbname + "/LOG";
+
+  char flatten_db_path[256];
+  FlattenPath(db_path, flatten_db_path, 256);
+  return log_dir + "/" + flatten_db_path + "_LOG";
 }
 
 // Return the name of the old info log file for "dbname".
-std::string OldInfoLogFileName(const std::string& dbname, uint64_t ts) {
+std::string OldInfoLogFileName(const std::string& dbname, uint64_t ts,
+    const std::string& db_path, const std::string& log_dir) {
   char buf[50];
   snprintf(buf, sizeof(buf), "%llu", static_cast<unsigned long long>(ts));
-  return dbname + "/LOG.old." + buf;
+
+  if (log_dir.empty())
+    return dbname + "/LOG.old." + buf;
+
+  char flatten_db_path[256];
+  FlattenPath(db_path, flatten_db_path, 256);
+  return log_dir + "/" + flatten_db_path + "_LOG.old." + buf;
 }
 
 

db/filename.h

@@ -57,10 +57,12 @@ extern std::string LockFileName(const std::string& dbname);
 extern std::string TempFileName(const std::string& dbname, uint64_t number);
 
 // Return the name of the info log file for "dbname".
-extern std::string InfoLogFileName(const std::string& dbname);
+extern std::string InfoLogFileName(const std::string& dbname,
+    const std::string& db_path="", const std::string& log_dir="");
 
 // Return the name of the old info log file for "dbname".
-extern std::string OldInfoLogFileName(const std::string& dbname, uint64_t ts);
+extern std::string OldInfoLogFileName(const std::string& dbname, uint64_t ts,
+    const std::string& db_path="", const std::string& log_dir="");
 
 // If filename is a leveldb file, store the type of the file in *type.
 // The number encoded in the filename is stored in *number.  If the

include/leveldb/options.h

@@ -6,6 +6,7 @@
 #define STORAGE_LEVELDB_INCLUDE_OPTIONS_H_
 
 #include <stddef.h>
+#include <string>
 
 namespace leveldb {
 
@@ -212,6 +213,13 @@ struct Options {
   // Default value is 1800 (half an hour).
   int db_stats_log_interval;
 
+  // This specifies the log dir.
+  // If it is empty, the log files will be in the same dir as data.
+  // If it is non empty, the log files will be in the specified dir,
+  // and the db data dir's absolute path will be used as the log file
+  // name's prefix.
+  std::string db_log_dir;
+
   // Create an Options object with default values for all fields.
   Options();
 

port/port_posix.h

@@ -320,7 +320,7 @@ inline bool BZip2_Compress(const char* input, size_t length,
   return true;
   return output;
 #endif
-  return NULL;
+  return false;
 }
 
 inline char*  BZip2_Uncompress(const char* input_data, size_t input_length,

util/options.cc

@@ -38,7 +38,8 @@ Options::Options()
       statistics(NULL),
       disableDataSync(false),
       use_fsync(false),
-      db_stats_log_interval(1800) {
+      db_stats_log_interval(1800),
+      db_log_dir("") {
 }
 
 void
@@ -84,6 +85,8 @@ Options::Dump(
         expanded_compaction_factor);
     Log(log,"         Options.max_grandparent_overlap_factor: %d",
         max_grandparent_overlap_factor);
+    Log(log,"         Options.db_log_dir: %s",
+        db_log_dir.c_str());
 
 }   // Options::Dump