From 213a68eb68ece93a8915ef18a8eba920d5a924a4 Mon Sep 17 00:00:00 2001 From: Hans Wennborg Date: Mon, 12 Sep 2011 10:21:10 +0100 Subject: [PATCH] Sync with upstream @23860137. Fix GCC -Wshadow warnings in LevelDB's public header files, reported by Dustin. Add in-memory Env implementation (helpers/memenv/*). This enables users to create LevelDB databases in-memory. Initialize ShardedLRUCache::last_id_ to zero. This fixes a Valgrind warning. (Also delete port/sha1_* which were removed upstream some time ago.) --- Makefile | 12 +- helpers/memenv/memenv.cc | 374 ++++++++++++++++++++++++++++++++++ helpers/memenv/memenv.h | 20 ++ helpers/memenv/memenv_test.cc | 232 +++++++++++++++++++++ include/leveldb/env.h | 4 +- include/leveldb/slice.h | 4 +- port/sha1_portable.cc | 298 --------------------------- port/sha1_portable.h | 25 --- port/sha1_test.cc | 39 ---- util/cache.cc | 3 +- 10 files changed, 642 insertions(+), 369 deletions(-) create mode 100644 helpers/memenv/memenv.cc create mode 100644 helpers/memenv/memenv.h create mode 100644 helpers/memenv/memenv_test.cc delete mode 100644 port/sha1_portable.cc delete mode 100644 port/sha1_portable.h delete mode 100644 port/sha1_test.cc diff --git a/Makefile b/Makefile index 86e24e8cb..18386f28b 100644 --- a/Makefile +++ b/Makefile @@ -92,6 +92,7 @@ TESTS = \ env_test \ filename_test \ log_test \ + memenv_test \ skiplist_test \ table_test \ version_edit_test \ @@ -102,6 +103,7 @@ PROGRAMS = db_bench $(TESTS) BENCHMARKS = db_bench_sqlite3 db_bench_tree_db LIBRARY = libleveldb.a +MEMENVLIBRARY = libmemenv.a all: $(LIBRARY) @@ -109,7 +111,7 @@ check: $(PROGRAMS) $(TESTS) for t in $(TESTS); do echo "***** Running $$t"; ./$$t || exit 1; done clean: - -rm -f $(PROGRAMS) $(BENCHMARKS) $(LIBRARY) */*.o */*/*.o ios-x86/*/*.o ios-arm/*/*.o + -rm -f $(PROGRAMS) $(BENCHMARKS) $(LIBRARY) $(MEMENVLIBRARY) */*.o */*/*.o ios-x86/*/*.o ios-arm/*/*.o -rm -rf ios-x86/* ios-arm/* -rm build_config.mk @@ -174,6 +176,13 @@ version_set_test: db/version_set_test.o $(LIBOBJECTS) $(TESTHARNESS) write_batch_test: db/write_batch_test.o $(LIBOBJECTS) $(TESTHARNESS) $(CC) $(LDFLAGS) db/write_batch_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ +$(MEMENVLIBRARY) : helpers/memenv/memenv.o + rm -f $@ + $(AR) -rs $@ helpers/memenv/memenv.o + +memenv_test : helpers/memenv/memenv_test.o $(MEMENVLIBRARY) $(LIBRARY) $(TESTHARNESS) + $(CC) $(LDFLAGS) helpers/memenv/memenv_test.o $(MEMENVLIBRARY) $(LIBRARY) $(TESTHARNESS) -o $@ + ifeq ($(PLATFORM), IOS) # For iOS, create universal object files to be used on both the simulator and # a device. @@ -202,4 +211,3 @@ else .c.o: $(CC) $(CFLAGS) $< -o $@ endif - diff --git a/helpers/memenv/memenv.cc b/helpers/memenv/memenv.cc new file mode 100644 index 000000000..dab80fe8f --- /dev/null +++ b/helpers/memenv/memenv.cc @@ -0,0 +1,374 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "helpers/memenv/memenv.h" + +#include "leveldb/env.h" +#include "leveldb/status.h" +#include "port/port.h" +#include "util/mutexlock.h" +#include +#include +#include +#include + +namespace leveldb { + +namespace { + +class FileState { + public: + // FileStates are reference counted. The initial reference count is zero + // and the caller must call Ref() at least once. + FileState() : refs_(0), size_(0) {} + + // Increase the reference count. + void Ref() { + MutexLock lock(&refs_mutex_); + ++refs_; + } + + // Decrease the reference count. Delete if this is the last reference. + void Unref() { + bool do_delete = false; + + { + MutexLock lock(&refs_mutex_); + --refs_; + assert(refs_ >= 0); + if (refs_ <= 0) { + do_delete = true; + } + } + + if (do_delete) { + delete this; + } + } + + uint64_t Size() const { return size_; } + + Status Read(uint64_t offset, size_t n, Slice* result, char* scratch) const { + if (offset > size_) { + return Status::IOError("Offset greater than file size."); + } + const uint64_t available = size_ - offset; + if (n > available) { + n = available; + } + if (n == 0) { + *result = Slice(); + return Status::OK(); + } + + size_t block = offset / kBlockSize; + size_t block_offset = offset % kBlockSize; + + if (n <= kBlockSize - block_offset) { + // The requested bytes are all in the first block. + *result = Slice(blocks_[block] + block_offset, n); + return Status::OK(); + } + + size_t bytes_to_copy = n; + char* dst = scratch; + + while (bytes_to_copy > 0) { + size_t avail = kBlockSize - block_offset; + if (avail > bytes_to_copy) { + avail = bytes_to_copy; + } + memcpy(dst, blocks_[block] + block_offset, avail); + + bytes_to_copy -= avail; + dst += avail; + block++; + block_offset = 0; + } + + *result = Slice(scratch, n); + return Status::OK(); + } + + Status Append(const Slice& data) { + const char* src = data.data(); + size_t src_len = data.size(); + + while (src_len > 0) { + size_t avail; + size_t offset = size_ % kBlockSize; + + if (offset != 0) { + // There is some room in the last block. + avail = kBlockSize - offset; + } else { + // No room in the last block; push new one. + blocks_.push_back(new char[kBlockSize]); + avail = kBlockSize; + } + + if (avail > src_len) { + avail = src_len; + } + memcpy(blocks_.back() + offset, src, avail); + src_len -= avail; + src += avail; + size_ += avail; + } + + return Status::OK(); + } + + private: + // Private since only Unref() should be used to delete it. + ~FileState() { + for (std::vector::iterator i = blocks_.begin(); i != blocks_.end(); + ++i) { + delete [] *i; + } + } + + // No copying allowed. + FileState(const FileState&); + void operator=(const FileState&); + + port::Mutex refs_mutex_; + int refs_; // Protected by refs_mutex_; + + // The following fields are not protected by any mutex. They are only mutable + // while the file is being written, and concurrent access is not allowed + // to writable files. + std::vector blocks_; + uint64_t size_; + + enum { kBlockSize = 8 * 1024 }; +}; + +class SequentialFileImpl : public SequentialFile { + public: + explicit SequentialFileImpl(FileState* file) : file_(file), pos_(0) { + file_->Ref(); + } + + ~SequentialFileImpl() { + file_->Unref(); + } + + virtual Status Read(size_t n, Slice* result, char* scratch) { + Status s = file_->Read(pos_, n, result, scratch); + if (s.ok()) { + pos_ += result->size(); + } + return s; + } + + virtual Status Skip(uint64_t n) { + if (pos_ > file_->Size()) { + return Status::IOError("pos_ > file_->Size()"); + } + const size_t available = file_->Size() - pos_; + if (n > available) { + n = available; + } + pos_ += n; + return Status::OK(); + } + + private: + FileState* file_; + size_t pos_; +}; + +class RandomAccessFileImpl : public RandomAccessFile { + public: + explicit RandomAccessFileImpl(FileState* file) : file_(file) { + file_->Ref(); + } + + ~RandomAccessFileImpl() { + file_->Unref(); + } + + virtual Status Read(uint64_t offset, size_t n, Slice* result, + char* scratch) const { + return file_->Read(offset, n, result, scratch); + } + + private: + FileState* file_; +}; + +class WritableFileImpl : public WritableFile { + public: + WritableFileImpl(FileState* file) : file_(file) { + file_->Ref(); + } + + ~WritableFileImpl() { + file_->Unref(); + } + + virtual Status Append(const Slice& data) { + return file_->Append(data); + } + + virtual Status Close() { return Status::OK(); } + virtual Status Flush() { return Status::OK(); } + virtual Status Sync() { return Status::OK(); } + + private: + FileState* file_; +}; + +class InMemoryEnv : public EnvWrapper { + public: + explicit InMemoryEnv(Env* base_env) : EnvWrapper(base_env) { } + + virtual ~InMemoryEnv() { + for (FileSystem::iterator i = file_map_.begin(); i != file_map_.end(); ++i){ + i->second->Unref(); + } + } + + // Partial implementation of the Env interface. + virtual Status NewSequentialFile(const std::string& fname, + SequentialFile** result) { + MutexLock lock(&mutex_); + if (file_map_.find(fname) == file_map_.end()) { + *result = NULL; + return Status::IOError(fname, "File not found"); + } + + *result = new SequentialFileImpl(file_map_[fname]); + return Status::OK(); + } + + virtual Status NewRandomAccessFile(const std::string& fname, + RandomAccessFile** result) { + MutexLock lock(&mutex_); + if (file_map_.find(fname) == file_map_.end()) { + *result = NULL; + return Status::IOError(fname, "File not found"); + } + + *result = new RandomAccessFileImpl(file_map_[fname]); + return Status::OK(); + } + + virtual Status NewWritableFile(const std::string& fname, + WritableFile** result) { + MutexLock lock(&mutex_); + if (file_map_.find(fname) != file_map_.end()) { + DeleteFileInternal(fname); + } + + FileState* file = new FileState(); + file->Ref(); + file_map_[fname] = file; + + *result = new WritableFileImpl(file); + return Status::OK(); + } + + virtual bool FileExists(const std::string& fname) { + MutexLock lock(&mutex_); + return file_map_.find(fname) != file_map_.end(); + } + + virtual Status GetChildren(const std::string& dir, + std::vector* result) { + MutexLock lock(&mutex_); + result->clear(); + + for (FileSystem::iterator i = file_map_.begin(); i != file_map_.end(); ++i){ + const std::string& filename = i->first; + + if (filename.size() >= dir.size() + 1 && filename[dir.size()] == '/' && + Slice(filename).starts_with(Slice(dir))) { + result->push_back(filename.substr(dir.size() + 1)); + } + } + + return Status::OK(); + } + + void DeleteFileInternal(const std::string& fname) { + if (file_map_.find(fname) == file_map_.end()) { + return; + } + + file_map_[fname]->Unref(); + file_map_.erase(fname); + } + + virtual Status DeleteFile(const std::string& fname) { + MutexLock lock(&mutex_); + if (file_map_.find(fname) == file_map_.end()) { + return Status::IOError(fname, "File not found"); + } + + DeleteFileInternal(fname); + return Status::OK(); + } + + virtual Status CreateDir(const std::string& dirname) { + return Status::OK(); + } + + virtual Status DeleteDir(const std::string& dirname) { + return Status::OK(); + } + + virtual Status GetFileSize(const std::string& fname, uint64_t* file_size) { + MutexLock lock(&mutex_); + if (file_map_.find(fname) == file_map_.end()) { + return Status::IOError(fname, "File not found"); + } + + *file_size = file_map_[fname]->Size(); + return Status::OK(); + } + + virtual Status RenameFile(const std::string& src, + const std::string& target) { + MutexLock lock(&mutex_); + if (file_map_.find(src) == file_map_.end()) { + return Status::IOError(src, "File not found"); + } + + DeleteFileInternal(target); + file_map_[target] = file_map_[src]; + file_map_.erase(src); + return Status::OK(); + } + + virtual Status LockFile(const std::string& fname, FileLock** lock) { + *lock = new FileLock; + return Status::OK(); + } + + virtual Status UnlockFile(FileLock* lock) { + delete lock; + return Status::OK(); + } + + virtual Status GetTestDirectory(std::string* path) { + *path = "/test"; + return Status::OK(); + } + + private: + // Map from filenames to FileState objects, representing a simple file system. + typedef std::map FileSystem; + port::Mutex mutex_; + FileSystem file_map_; // Protected by mutex_. +}; + +} + +Env* NewMemEnv(Env* base_env) { + return new InMemoryEnv(base_env); +} + +} diff --git a/helpers/memenv/memenv.h b/helpers/memenv/memenv.h new file mode 100644 index 000000000..835b944f4 --- /dev/null +++ b/helpers/memenv/memenv.h @@ -0,0 +1,20 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#ifndef STORAGE_LEVELDB_HELPERS_MEMENV_MEMENV_H_ +#define STORAGE_LEVELDB_HELPERS_MEMENV_MEMENV_H_ + +namespace leveldb { + +class Env; + +// Returns a new environment that stores its data in memory and delegates +// all non-file-storage tasks to base_env. The caller must delete the result +// when it is no longer needed. +// *base_env must remain live while the result is in use. +Env* NewMemEnv(Env* base_env); + +} + +#endif // STORAGE_LEVELDB_HELPERS_MEMENV_MEMENV_H_ diff --git a/helpers/memenv/memenv_test.cc b/helpers/memenv/memenv_test.cc new file mode 100644 index 000000000..30b0bb058 --- /dev/null +++ b/helpers/memenv/memenv_test.cc @@ -0,0 +1,232 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "helpers/memenv/memenv.h" + +#include "db/db_impl.h" +#include "leveldb/db.h" +#include "leveldb/env.h" +#include "util/testharness.h" +#include +#include + +namespace leveldb { + +class MemEnvTest { + public: + Env* env_; + + MemEnvTest() + : env_(NewMemEnv(Env::Default())) { + } + ~MemEnvTest() { + delete env_; + } +}; + +TEST(MemEnvTest, Basics) { + size_t file_size; + WritableFile* writable_file; + std::vector children; + + ASSERT_OK(env_->CreateDir("/dir")); + + // Check that the directory is empty. + ASSERT_TRUE(!env_->FileExists("/dir/non_existent")); + ASSERT_TRUE(!env_->GetFileSize("/dir/non_existent", &file_size).ok()); + ASSERT_OK(env_->GetChildren("/dir", &children)); + ASSERT_EQ(0, children.size()); + + // Create a file. + ASSERT_OK(env_->NewWritableFile("/dir/f", &writable_file)); + delete writable_file; + + // Check that the file exists. + ASSERT_TRUE(env_->FileExists("/dir/f")); + ASSERT_OK(env_->GetFileSize("/dir/f", &file_size)); + ASSERT_EQ(0, file_size); + ASSERT_OK(env_->GetChildren("/dir", &children)); + ASSERT_EQ(1, children.size()); + ASSERT_EQ("f", children[0]); + + // Write to the file. + ASSERT_OK(env_->NewWritableFile("/dir/f", &writable_file)); + ASSERT_OK(writable_file->Append("abc")); + delete writable_file; + + // Check for expected size. + ASSERT_OK(env_->GetFileSize("/dir/f", &file_size)); + ASSERT_EQ(3, file_size); + + // Check that renaming works. + ASSERT_TRUE(!env_->RenameFile("/dir/non_existent", "/dir/g").ok()); + ASSERT_OK(env_->RenameFile("/dir/f", "/dir/g")); + ASSERT_TRUE(!env_->FileExists("/dir/f")); + ASSERT_TRUE(env_->FileExists("/dir/g")); + ASSERT_OK(env_->GetFileSize("/dir/g", &file_size)); + ASSERT_EQ(3, file_size); + + // Check that opening non-existent file fails. + SequentialFile* seq_file; + RandomAccessFile* rand_file; + ASSERT_TRUE(!env_->NewSequentialFile("/dir/non_existent", &seq_file).ok()); + ASSERT_TRUE(!seq_file); + ASSERT_TRUE(!env_->NewRandomAccessFile("/dir/non_existent", &rand_file).ok()); + ASSERT_TRUE(!rand_file); + + // Check that deleting works. + ASSERT_TRUE(!env_->DeleteFile("/dir/non_existent").ok()); + ASSERT_OK(env_->DeleteFile("/dir/g")); + ASSERT_TRUE(!env_->FileExists("/dir/g")); + ASSERT_OK(env_->GetChildren("/dir", &children)); + ASSERT_EQ(0, children.size()); + ASSERT_OK(env_->DeleteDir("/dir")); +} + +TEST(MemEnvTest, ReadWrite) { + WritableFile* writable_file; + SequentialFile* seq_file; + RandomAccessFile* rand_file; + Slice result; + char scratch[100]; + + ASSERT_OK(env_->CreateDir("/dir")); + + ASSERT_OK(env_->NewWritableFile("/dir/f", &writable_file)); + ASSERT_OK(writable_file->Append("hello ")); + ASSERT_OK(writable_file->Append("world")); + delete writable_file; + + // Read sequentially. + ASSERT_OK(env_->NewSequentialFile("/dir/f", &seq_file)); + ASSERT_OK(seq_file->Read(5, &result, scratch)); // Read "hello". + ASSERT_EQ(0, result.compare("hello")); + ASSERT_OK(seq_file->Skip(1)); + ASSERT_OK(seq_file->Read(1000, &result, scratch)); // Read "world". + ASSERT_EQ(0, result.compare("world")); + ASSERT_OK(seq_file->Read(1000, &result, scratch)); // Try reading past EOF. + ASSERT_EQ(0, result.size()); + ASSERT_OK(seq_file->Skip(100)); // Try to skip past end of file. + ASSERT_OK(seq_file->Read(1000, &result, scratch)); + ASSERT_EQ(0, result.size()); + delete seq_file; + + // Random reads. + ASSERT_OK(env_->NewRandomAccessFile("/dir/f", &rand_file)); + ASSERT_OK(rand_file->Read(6, 5, &result, scratch)); // Read "world". + ASSERT_EQ(0, result.compare("world")); + ASSERT_OK(rand_file->Read(0, 5, &result, scratch)); // Read "hello". + ASSERT_EQ(0, result.compare("hello")); + ASSERT_OK(rand_file->Read(10, 100, &result, scratch)); // Read "d". + ASSERT_EQ(0, result.compare("d")); + + // Too high offset. + ASSERT_TRUE(!rand_file->Read(1000, 5, &result, scratch).ok()); + delete rand_file; +} + +TEST(MemEnvTest, Locks) { + FileLock* lock; + + // These are no-ops, but we test they return success. + ASSERT_OK(env_->LockFile("some file", &lock)); + ASSERT_OK(env_->UnlockFile(lock)); +} + +TEST(MemEnvTest, Misc) { + std::string test_dir; + ASSERT_OK(env_->GetTestDirectory(&test_dir)); + ASSERT_TRUE(!test_dir.empty()); + + WritableFile* writable_file; + ASSERT_OK(env_->NewWritableFile("/a/b", &writable_file)); + + // These are no-ops, but we test they return success. + ASSERT_OK(writable_file->Sync()); + ASSERT_OK(writable_file->Flush()); + ASSERT_OK(writable_file->Close()); + delete writable_file; +} + +TEST(MemEnvTest, LargeWrite) { + const size_t kWriteSize = 300 * 1024; + char* scratch = new char[kWriteSize * 2]; + + std::string write_data; + for (size_t i = 0; i < kWriteSize; ++i) { + write_data.append(1, static_cast(i)); + } + + WritableFile* writable_file; + ASSERT_OK(env_->NewWritableFile("/dir/f", &writable_file)); + ASSERT_OK(writable_file->Append("foo")); + ASSERT_OK(writable_file->Append(write_data)); + delete writable_file; + + SequentialFile* seq_file; + Slice result; + ASSERT_OK(env_->NewSequentialFile("/dir/f", &seq_file)); + ASSERT_OK(seq_file->Read(3, &result, scratch)); // Read "foo". + ASSERT_EQ(0, result.compare("foo")); + + size_t read = 0; + std::string read_data; + while (read < kWriteSize) { + ASSERT_OK(seq_file->Read(kWriteSize - read, &result, scratch)); + read_data.append(result.data(), result.size()); + read += result.size(); + } + ASSERT_TRUE(write_data == read_data); + delete seq_file; + delete [] scratch; +} + +TEST(MemEnvTest, DBTest) { + Options options; + options.create_if_missing = true; + options.env = env_; + DB* db; + + const Slice keys[] = {Slice("aaa"), Slice("bbb"), Slice("ccc")}; + const Slice vals[] = {Slice("foo"), Slice("bar"), Slice("baz")}; + + ASSERT_OK(DB::Open(options, "/dir/db", &db)); + for (size_t i = 0; i < 3; ++i) { + ASSERT_OK(db->Put(WriteOptions(), keys[i], vals[i])); + } + + for (size_t i = 0; i < 3; ++i) { + std::string res; + ASSERT_OK(db->Get(ReadOptions(), keys[i], &res)); + ASSERT_TRUE(res == vals[i]); + } + + Iterator* iterator = db->NewIterator(ReadOptions()); + iterator->SeekToFirst(); + for (size_t i = 0; i < 3; ++i) { + ASSERT_TRUE(iterator->Valid()); + ASSERT_TRUE(keys[i] == iterator->key()); + ASSERT_TRUE(vals[i] == iterator->value()); + iterator->Next(); + } + ASSERT_TRUE(!iterator->Valid()); + delete iterator; + + DBImpl* dbi = reinterpret_cast(db); + ASSERT_OK(dbi->TEST_CompactMemTable()); + + for (size_t i = 0; i < 3; ++i) { + std::string res; + ASSERT_OK(db->Get(ReadOptions(), keys[i], &res)); + ASSERT_TRUE(res == vals[i]); + } + + delete db; +} + +} + +int main(int argc, char** argv) { + return leveldb::test::RunAllTests(); +} diff --git a/include/leveldb/env.h b/include/leveldb/env.h index bf5100893..1a8ff6bc3 100644 --- a/include/leveldb/env.h +++ b/include/leveldb/env.h @@ -258,8 +258,8 @@ extern Status ReadFileToString(Env* env, const std::string& fname, // functionality of another Env. class EnvWrapper : public Env { public: - // Initialize an EnvWrapper that delegates all calls to *target - explicit EnvWrapper(Env* target) : target_(target) { } + // Initialize an EnvWrapper that delegates all calls to *t + explicit EnvWrapper(Env* t) : target_(t) { } virtual ~EnvWrapper(); // Return the target to which this Env forwards all calls diff --git a/include/leveldb/slice.h b/include/leveldb/slice.h index 3c000b814..7c66d1b85 100644 --- a/include/leveldb/slice.h +++ b/include/leveldb/slice.h @@ -27,8 +27,8 @@ class Slice { // Create an empty slice. Slice() : data_(""), size_(0) { } - // Create a slice that refers to data[0,n-1]. - Slice(const char* data, size_t n) : data_(data), size_(n) { } + // Create a slice that refers to d[0,n-1]. + Slice(const char* d, size_t n) : data_(d), size_(n) { } // Create a slice that refers to the contents of "s" Slice(const std::string& s) : data_(s.data()), size_(s.size()) { } diff --git a/port/sha1_portable.cc b/port/sha1_portable.cc deleted file mode 100644 index 8fa727744..000000000 --- a/port/sha1_portable.cc +++ /dev/null @@ -1,298 +0,0 @@ -// Portions copyright (c) 2011 The LevelDB Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. See the AUTHORS file for names of contributors. -// -// This module provides a slow but portable implementation of -// the SHA1 hash function. -// -// It is adapted from free code written by Paul E. Jones -// . See http://www.packetizer.com/security/sha1/ -// -// The license for the original code is: -/* - Copyright (C) 1998, 2009 - Paul E. Jones - - Freeware Public License (FPL) - - This software is licensed as "freeware." Permission to distribute - this software in source and binary forms, including incorporation - into other products, is hereby granted without a fee. THIS SOFTWARE - IS PROVIDED 'AS IS' AND WITHOUT ANY EXPRESSED OR IMPLIED WARRANTIES, - INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY - AND FITNESS FOR A PARTICULAR PURPOSE. THE AUTHOR SHALL NOT BE HELD - LIABLE FOR ANY DAMAGES RESULTING FROM THE USE OF THIS SOFTWARE, EITHER - DIRECTLY OR INDIRECTLY, INCLUDING, BUT NOT LIMITED TO, LOSS OF DATA - OR DATA BEING RENDERED INACCURATE. -*/ - -#include "port/sha1_portable.h" -#include -#include -#include - -namespace leveldb { -namespace port { - -/* - * Description: - * This class implements the Secure Hashing Standard as defined - * in FIPS PUB 180-1 published April 17, 1995. - */ - -/* - * This structure will hold context information for the hashing - * operation - */ -typedef struct SHA1Context { - unsigned Message_Digest[5]; /* Message Digest (output) */ - - unsigned Length_Low; /* Message length in bits */ - unsigned Length_High; /* Message length in bits */ - - unsigned char Message_Block[64]; /* 512-bit message blocks */ - int Message_Block_Index; /* Index into message block array */ - - bool Computed; /* Is the digest computed? */ - bool Corrupted; /* Is the message digest corruped? */ -} SHA1Context; - -/* - * Portability Issues: - * SHA-1 is defined in terms of 32-bit "words". This code was - * written with the expectation that the processor has at least - * a 32-bit machine word size. If the machine word size is larger, - * the code should still function properly. One caveat to that - * is that the input functions taking characters and character - * arrays assume that only 8 bits of information are stored in each - * character. - */ - -/* - * Define the circular shift macro - */ -#define SHA1CircularShift(bits,word) \ - ((((word) << (bits)) & 0xFFFFFFFF) | \ - ((word) >> (32-(bits)))) - -/* Function prototypes */ -static void SHA1ProcessMessageBlock(SHA1Context *); -static void SHA1PadMessage(SHA1Context *); - -// Initialize the SHA1Context in preparation for computing a new -// message digest. -static void SHA1Reset(SHA1Context* context) { - context->Length_Low = 0; - context->Length_High = 0; - context->Message_Block_Index = 0; - - context->Message_Digest[0] = 0x67452301; - context->Message_Digest[1] = 0xEFCDAB89; - context->Message_Digest[2] = 0x98BADCFE; - context->Message_Digest[3] = 0x10325476; - context->Message_Digest[4] = 0xC3D2E1F0; - - context->Computed = false; - context->Corrupted = false; -} - -// This function will return the 160-bit message digest into the -// Message_Digest array within the SHA1Context provided -static bool SHA1Result(SHA1Context *context) { - if (context->Corrupted) { - return false; - } - - if (!context->Computed) { - SHA1PadMessage(context); - context->Computed = true; - } - return true; -} - -// This function accepts an array of bytes as the next portion of -// the message. -static void SHA1Input(SHA1Context *context, - const unsigned char *message_array, - unsigned length) { - if (!length) return; - - if (context->Computed || context->Corrupted) { - context->Corrupted = true; - return; - } - - while(length-- && !context->Corrupted) { - context->Message_Block[context->Message_Block_Index++] = - (*message_array & 0xFF); - - context->Length_Low += 8; - /* Force it to 32 bits */ - context->Length_Low &= 0xFFFFFFFF; - if (context->Length_Low == 0) { - context->Length_High++; - /* Force it to 32 bits */ - context->Length_High &= 0xFFFFFFFF; - if (context->Length_High == 0) - { - /* Message is too long */ - context->Corrupted = true; - } - } - - if (context->Message_Block_Index == 64) - { - SHA1ProcessMessageBlock(context); - } - - message_array++; - } -} - -// This function will process the next 512 bits of the message stored -// in the Message_Block array. -static void SHA1ProcessMessageBlock(SHA1Context *context) { - const unsigned K[] = // Constants defined in SHA-1 - { - 0x5A827999, - 0x6ED9EBA1, - 0x8F1BBCDC, - 0xCA62C1D6 - }; - int t; // Loop counter - unsigned temp; // Temporary word value - unsigned W[80]; // Word sequence - unsigned A, B, C, D, E; // Word buffers - - // Initialize the first 16 words in the array W - for(t = 0; t < 16; t++) { - W[t] = ((unsigned) context->Message_Block[t * 4]) << 24; - W[t] |= ((unsigned) context->Message_Block[t * 4 + 1]) << 16; - W[t] |= ((unsigned) context->Message_Block[t * 4 + 2]) << 8; - W[t] |= ((unsigned) context->Message_Block[t * 4 + 3]); - } - - for(t = 16; t < 80; t++) { - W[t] = SHA1CircularShift(1,W[t-3] ^ W[t-8] ^ W[t-14] ^ W[t-16]); - } - - A = context->Message_Digest[0]; - B = context->Message_Digest[1]; - C = context->Message_Digest[2]; - D = context->Message_Digest[3]; - E = context->Message_Digest[4]; - - for(t = 0; t < 20; t++) { - temp = SHA1CircularShift(5,A) + - ((B & C) | ((~B) & D)) + E + W[t] + K[0]; - temp &= 0xFFFFFFFF; - E = D; - D = C; - C = SHA1CircularShift(30,B); - B = A; - A = temp; - } - - for(t = 20; t < 40; t++) { - temp = SHA1CircularShift(5,A) + (B ^ C ^ D) + E + W[t] + K[1]; - temp &= 0xFFFFFFFF; - E = D; - D = C; - C = SHA1CircularShift(30,B); - B = A; - A = temp; - } - - for(t = 40; t < 60; t++) { - temp = SHA1CircularShift(5,A) + - ((B & C) | (B & D) | (C & D)) + E + W[t] + K[2]; - temp &= 0xFFFFFFFF; - E = D; - D = C; - C = SHA1CircularShift(30,B); - B = A; - A = temp; - } - - for(t = 60; t < 80; t++) { - temp = SHA1CircularShift(5,A) + (B ^ C ^ D) + E + W[t] + K[3]; - temp &= 0xFFFFFFFF; - E = D; - D = C; - C = SHA1CircularShift(30,B); - B = A; - A = temp; - } - - context->Message_Digest[0] = (context->Message_Digest[0] + A) & 0xFFFFFFFF; - context->Message_Digest[1] = (context->Message_Digest[1] + B) & 0xFFFFFFFF; - context->Message_Digest[2] = (context->Message_Digest[2] + C) & 0xFFFFFFFF; - context->Message_Digest[3] = (context->Message_Digest[3] + D) & 0xFFFFFFFF; - context->Message_Digest[4] = (context->Message_Digest[4] + E) & 0xFFFFFFFF; - - context->Message_Block_Index = 0; -} - -// According to the standard, the message must be padded to an even -// 512 bits. The first padding bit must be a '1'. The last 64 bits -// represent the length of the original message. All bits in between -// should be 0. This function will pad the message according to those -// rules by filling the Message_Block array accordingly. It will also -// call SHA1ProcessMessageBlock() appropriately. When it returns, it -// can be assumed that the message digest has been computed. -static void SHA1PadMessage(SHA1Context *context) { - // Check to see if the current message block is too small to hold - // the initial padding bits and length. If so, we will pad the - // block, process it, and then continue padding into a second block. - if (context->Message_Block_Index > 55) { - context->Message_Block[context->Message_Block_Index++] = 0x80; - while(context->Message_Block_Index < 64) { - context->Message_Block[context->Message_Block_Index++] = 0; - } - - SHA1ProcessMessageBlock(context); - - while(context->Message_Block_Index < 56) { - context->Message_Block[context->Message_Block_Index++] = 0; - } - } else { - context->Message_Block[context->Message_Block_Index++] = 0x80; - while(context->Message_Block_Index < 56) { - context->Message_Block[context->Message_Block_Index++] = 0; - } - } - - // Store the message length as the last 8 octets - context->Message_Block[56] = (context->Length_High >> 24) & 0xFF; - context->Message_Block[57] = (context->Length_High >> 16) & 0xFF; - context->Message_Block[58] = (context->Length_High >> 8) & 0xFF; - context->Message_Block[59] = (context->Length_High) & 0xFF; - context->Message_Block[60] = (context->Length_Low >> 24) & 0xFF; - context->Message_Block[61] = (context->Length_Low >> 16) & 0xFF; - context->Message_Block[62] = (context->Length_Low >> 8) & 0xFF; - context->Message_Block[63] = (context->Length_Low) & 0xFF; - - SHA1ProcessMessageBlock(context); -} - - -void SHA1_Hash_Portable(const char* data, size_t len, char* hash_array) { - SHA1Context context; - SHA1Reset(&context); - SHA1Input(&context, reinterpret_cast(data), len); - bool ok = SHA1Result(&context); - if (!ok) { - fprintf(stderr, "Unexpected error in SHA1_Hash_Portable code\n"); - exit(1); - } - for (int i = 0; i < 5; i++) { - uint32_t value = context.Message_Digest[i]; - hash_array[i*4 + 0] = (value >> 24) & 0xff; - hash_array[i*4 + 1] = (value >> 16) & 0xff; - hash_array[i*4 + 2] = (value >> 8) & 0xff; - hash_array[i*4 + 3] = value & 0xff; - } -} - -} -} diff --git a/port/sha1_portable.h b/port/sha1_portable.h deleted file mode 100644 index 31db30528..000000000 --- a/port/sha1_portable.h +++ /dev/null @@ -1,25 +0,0 @@ -// Copyright (c) 2011 The LevelDB Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. See the AUTHORS file for names of contributors. - -#ifndef STORAGE_LEVELDB_PORT_SHA1_PORTABLE_H_ -#define STORAGE_LEVELDB_PORT_SHA1_PORTABLE_H_ - -#include - -namespace leveldb { -namespace port { - -// Compute the SHA1 hash value of "data[0..len-1]" and store it in -// "hash_array[0..19]". hash_array must have 20 bytes of space available. -// -// This function is portable but may not be as fast as a version -// optimized for your platform. It is provided as a default method -// that can be used when porting leveldb to a new platform if no -// better SHA1 hash implementation is available. -void SHA1_Hash_Portable(const char* data, size_t len, char* hash_array); - -} -} - -#endif // STORAGE_LEVELDB_PORT_SHA1_PORTABLE_H_ diff --git a/port/sha1_test.cc b/port/sha1_test.cc deleted file mode 100644 index b182e6757..000000000 --- a/port/sha1_test.cc +++ /dev/null @@ -1,39 +0,0 @@ -// Copyright (c) 2011 The LevelDB Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. See the AUTHORS file for names of contributors. - -#include "port/port.h" -#include "util/testharness.h" - -namespace leveldb { -namespace port { - -class SHA1 { }; - -static std::string TestSHA1(const char* data, size_t len) { - char hash_val[20]; - SHA1_Hash(data, len, hash_val); - char buf[41]; - for (int i = 0; i < 20; i++) { - snprintf(buf + i * 2, 41 - i * 2, - "%02x", - static_cast(static_cast( - hash_val[i]))); - } - return std::string(buf, 40); -} - -TEST(SHA1, Simple) { - ASSERT_EQ("da39a3ee5e6b4b0d3255bfef95601890afd80709", TestSHA1("", 0)); - ASSERT_EQ("aaf4c61ddcc5e8a2dabede0f3b482cd9aea9434d", TestSHA1("hello", 5)); - std::string x(10000, 'x'); - ASSERT_EQ("f8c5cde791c5056cf515881e701c8a9ecb439a75", - TestSHA1(x.data(), x.size())); -} - -} -} - -int main(int argc, char** argv) { - return leveldb::test::RunAllTests(); -} diff --git a/util/cache.cc b/util/cache.cc index ce99f08c2..7d1ebc12b 100644 --- a/util/cache.cc +++ b/util/cache.cc @@ -285,7 +285,8 @@ class ShardedLRUCache : public Cache { } public: - explicit ShardedLRUCache(size_t capacity) { + explicit ShardedLRUCache(size_t capacity) + : last_id_(0) { const size_t per_shard = (capacity + (kNumShards - 1)) / kNumShards; for (int s = 0; s < kNumShards; s++) { shard_[s].SetCapacity(per_shard);