2013-10-16 23:59:46 +02:00
|
|
|
// Copyright (c) 2013, Facebook, Inc. All rights reserved.
|
|
|
|
// This source code is licensed under the BSD-style license found in the
|
|
|
|
// LICENSE file in the root directory of this source tree. An additional grant
|
|
|
|
// of patent rights can be found in the PATENTS file in the same directory.
|
|
|
|
//
|
2011-03-18 23:37:00 +01:00
|
|
|
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
|
|
|
|
// Use of this source code is governed by a BSD-style license that can be
|
|
|
|
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
|
|
|
|
|
|
|
#include <assert.h>
|
2011-06-29 02:30:50 +02:00
|
|
|
#include <stdio.h>
|
|
|
|
#include <stdlib.h>
|
2011-03-18 23:37:00 +01:00
|
|
|
|
2013-08-23 17:38:13 +02:00
|
|
|
#include "rocksdb/cache.h"
|
2011-03-18 23:37:00 +01:00
|
|
|
#include "port/port.h"
|
2014-01-02 20:26:57 +01:00
|
|
|
#include "util/autovector.h"
|
2011-03-18 23:37:00 +01:00
|
|
|
#include "util/hash.h"
|
|
|
|
#include "util/mutexlock.h"
|
|
|
|
|
2013-10-04 06:49:15 +02:00
|
|
|
namespace rocksdb {
|
2011-03-18 23:37:00 +01:00
|
|
|
|
|
|
|
Cache::~Cache() {
|
|
|
|
}
|
|
|
|
|
|
|
|
namespace {
|
|
|
|
|
|
|
|
// LRU cache implementation
|
|
|
|
|
Modifed the LRU cache eviction code so that it doesn't evict blocks which have exteranl references
Summary:
Currently, blocks which have more than one reference (ie referenced by something other than cache itself) are evicted from cache. This doesn't make much sense:
- blocks are still in RAM, so the RAM usage reported by the cache is incorrect
- if the same block is needed by another iterator, it will be loaded and decompressed again
This diff changes the reference counting scheme a bit. Previously, if the cache contained the block, this was accounted for in its refcount. After this change, the refcount is only used to track external references. There is a boolean flag which indicates whether or not the block is contained in the cache.
This diff also changes how LRU list is used. Previously, both hashtable and the LRU list contained all blocks. After this change, the LRU list contains blocks with the refcount==0, ie those which can be evicted from the cache.
Note that this change still allows for cache to grow beyond its capacity. This happens when all blocks are pinned (ie refcount>0). This is consistent with the current behavior. The cache's insert function never fails. I spent lots of time trying to make table_reader and other places work with the insert which might failed. It turned out to be pretty hard. It might really destabilize some customers, so finally, I decided against doing this.
table_cache_remove_scan_count_limit option will be unneeded after this change, but I will remove it in the following diff, if this one gets approved
Test Plan: Ran tests, made sure they pass
Reviewers: sdong, ljin
Differential Revision: https://reviews.facebook.net/D25503
2014-10-21 20:49:13 +02:00
|
|
|
// An entry is a variable length heap-allocated structure.
|
|
|
|
// Entries are referenced by cache and/or by any external entity.
|
|
|
|
// The cache keeps all its entries in table. Some elements
|
|
|
|
// are also stored on LRU list.
|
|
|
|
//
|
|
|
|
// LRUHandle can be in these states:
|
|
|
|
// 1. Referenced externally AND in hash table.
|
|
|
|
// In that case the entry is *not* in the LRU. (refs > 1 && in_cache == true)
|
|
|
|
// 2. Not referenced externally and in hash table. In that case the entry is
|
|
|
|
// in the LRU and can be freed. (refs == 1 && in_cache == true)
|
|
|
|
// 3. Referenced externally and not in hash table. In that case the entry is
|
|
|
|
// in not on LRU and not in table. (refs >= 1 && in_cache == false)
|
|
|
|
//
|
|
|
|
// All newly created LRUHandles are in state 1. If you call LRUCache::Release
|
|
|
|
// on entry in state 1, it will go into state 2. To move from state 1 to
|
|
|
|
// state 3, either call LRUCache::Erase or LRUCache::Insert with the same key.
|
|
|
|
// To move from state 2 to state 1, use LRUCache::Lookup.
|
|
|
|
// Before destruction, make sure that no handles are in state 1. This means
|
|
|
|
// that any successful LRUCache::Lookup/LRUCache::Insert have a matching
|
|
|
|
// RUCache::Release (to move into state 2) or LRUCache::Erase (for state 3)
|
|
|
|
|
2011-03-18 23:37:00 +01:00
|
|
|
struct LRUHandle {
|
|
|
|
void* value;
|
|
|
|
void (*deleter)(const Slice&, void* value);
|
2011-06-29 02:30:50 +02:00
|
|
|
LRUHandle* next_hash;
|
2011-03-18 23:37:00 +01:00
|
|
|
LRUHandle* next;
|
|
|
|
LRUHandle* prev;
|
|
|
|
size_t charge; // TODO(opt): Only allow uint32_t?
|
|
|
|
size_t key_length;
|
Modifed the LRU cache eviction code so that it doesn't evict blocks which have exteranl references
Summary:
Currently, blocks which have more than one reference (ie referenced by something other than cache itself) are evicted from cache. This doesn't make much sense:
- blocks are still in RAM, so the RAM usage reported by the cache is incorrect
- if the same block is needed by another iterator, it will be loaded and decompressed again
This diff changes the reference counting scheme a bit. Previously, if the cache contained the block, this was accounted for in its refcount. After this change, the refcount is only used to track external references. There is a boolean flag which indicates whether or not the block is contained in the cache.
This diff also changes how LRU list is used. Previously, both hashtable and the LRU list contained all blocks. After this change, the LRU list contains blocks with the refcount==0, ie those which can be evicted from the cache.
Note that this change still allows for cache to grow beyond its capacity. This happens when all blocks are pinned (ie refcount>0). This is consistent with the current behavior. The cache's insert function never fails. I spent lots of time trying to make table_reader and other places work with the insert which might failed. It turned out to be pretty hard. It might really destabilize some customers, so finally, I decided against doing this.
table_cache_remove_scan_count_limit option will be unneeded after this change, but I will remove it in the following diff, if this one gets approved
Test Plan: Ran tests, made sure they pass
Reviewers: sdong, ljin
Differential Revision: https://reviews.facebook.net/D25503
2014-10-21 20:49:13 +02:00
|
|
|
uint32_t refs; // a number of refs to this entry
|
|
|
|
// cache itself is counted as 1
|
|
|
|
bool in_cache; // true, if this entry is referenced by the hash table
|
2011-08-22 23:08:51 +02:00
|
|
|
uint32_t hash; // Hash of key(); used for fast sharding and comparisons
|
2011-03-18 23:37:00 +01:00
|
|
|
char key_data[1]; // Beginning of key
|
|
|
|
|
|
|
|
Slice key() const {
|
|
|
|
// For cheaper lookups, we allow a temporary Handle object
|
|
|
|
// to store a pointer to a key in "value".
|
|
|
|
if (next == this) {
|
|
|
|
return *(reinterpret_cast<Slice*>(value));
|
|
|
|
} else {
|
|
|
|
return Slice(key_data, key_length);
|
|
|
|
}
|
|
|
|
}
|
Modifed the LRU cache eviction code so that it doesn't evict blocks which have exteranl references
Summary:
Currently, blocks which have more than one reference (ie referenced by something other than cache itself) are evicted from cache. This doesn't make much sense:
- blocks are still in RAM, so the RAM usage reported by the cache is incorrect
- if the same block is needed by another iterator, it will be loaded and decompressed again
This diff changes the reference counting scheme a bit. Previously, if the cache contained the block, this was accounted for in its refcount. After this change, the refcount is only used to track external references. There is a boolean flag which indicates whether or not the block is contained in the cache.
This diff also changes how LRU list is used. Previously, both hashtable and the LRU list contained all blocks. After this change, the LRU list contains blocks with the refcount==0, ie those which can be evicted from the cache.
Note that this change still allows for cache to grow beyond its capacity. This happens when all blocks are pinned (ie refcount>0). This is consistent with the current behavior. The cache's insert function never fails. I spent lots of time trying to make table_reader and other places work with the insert which might failed. It turned out to be pretty hard. It might really destabilize some customers, so finally, I decided against doing this.
table_cache_remove_scan_count_limit option will be unneeded after this change, but I will remove it in the following diff, if this one gets approved
Test Plan: Ran tests, made sure they pass
Reviewers: sdong, ljin
Differential Revision: https://reviews.facebook.net/D25503
2014-10-21 20:49:13 +02:00
|
|
|
|
|
|
|
void Free() {
|
|
|
|
assert((refs == 1 && in_cache) || (refs == 0 && !in_cache));
|
|
|
|
(*deleter)(key(), value);
|
|
|
|
free(this);
|
|
|
|
}
|
2011-03-18 23:37:00 +01:00
|
|
|
};
|
|
|
|
|
2011-06-29 02:30:50 +02:00
|
|
|
// We provide our own simple hash table since it removes a whole bunch
|
|
|
|
// of porting hacks and is also faster than some of the built-in hash
|
|
|
|
// table implementations in some of the compiler/runtime combinations
|
|
|
|
// we have tested. E.g., readrandom speeds up by ~5% over the g++
|
|
|
|
// 4.4.3's builtin hashtable.
|
|
|
|
class HandleTable {
|
|
|
|
public:
|
2013-03-01 03:04:58 +01:00
|
|
|
HandleTable() : length_(0), elems_(0), list_(nullptr) { Resize(); }
|
Modifed the LRU cache eviction code so that it doesn't evict blocks which have exteranl references
Summary:
Currently, blocks which have more than one reference (ie referenced by something other than cache itself) are evicted from cache. This doesn't make much sense:
- blocks are still in RAM, so the RAM usage reported by the cache is incorrect
- if the same block is needed by another iterator, it will be loaded and decompressed again
This diff changes the reference counting scheme a bit. Previously, if the cache contained the block, this was accounted for in its refcount. After this change, the refcount is only used to track external references. There is a boolean flag which indicates whether or not the block is contained in the cache.
This diff also changes how LRU list is used. Previously, both hashtable and the LRU list contained all blocks. After this change, the LRU list contains blocks with the refcount==0, ie those which can be evicted from the cache.
Note that this change still allows for cache to grow beyond its capacity. This happens when all blocks are pinned (ie refcount>0). This is consistent with the current behavior. The cache's insert function never fails. I spent lots of time trying to make table_reader and other places work with the insert which might failed. It turned out to be pretty hard. It might really destabilize some customers, so finally, I decided against doing this.
table_cache_remove_scan_count_limit option will be unneeded after this change, but I will remove it in the following diff, if this one gets approved
Test Plan: Ran tests, made sure they pass
Reviewers: sdong, ljin
Differential Revision: https://reviews.facebook.net/D25503
2014-10-21 20:49:13 +02:00
|
|
|
|
|
|
|
template <typename T>
|
|
|
|
void ApplyToAllCacheEntries(T func) {
|
|
|
|
for (uint32_t i = 0; i < length_; i++) {
|
|
|
|
LRUHandle* h = list_[i];
|
|
|
|
while (h != nullptr) {
|
|
|
|
auto n = h->next_hash;
|
|
|
|
assert(h->in_cache);
|
|
|
|
func(h);
|
|
|
|
h = n;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
~HandleTable() {
|
|
|
|
ApplyToAllCacheEntries([](LRUHandle* h) {
|
|
|
|
if (h->refs == 1) {
|
|
|
|
h->Free();
|
|
|
|
}
|
|
|
|
});
|
|
|
|
delete[] list_;
|
|
|
|
}
|
2011-06-29 02:30:50 +02:00
|
|
|
|
2011-08-22 23:08:51 +02:00
|
|
|
LRUHandle* Lookup(const Slice& key, uint32_t hash) {
|
|
|
|
return *FindPointer(key, hash);
|
2011-06-29 02:30:50 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
LRUHandle* Insert(LRUHandle* h) {
|
2011-08-22 23:08:51 +02:00
|
|
|
LRUHandle** ptr = FindPointer(h->key(), h->hash);
|
2011-06-29 02:30:50 +02:00
|
|
|
LRUHandle* old = *ptr;
|
2013-03-01 03:04:58 +01:00
|
|
|
h->next_hash = (old == nullptr ? nullptr : old->next_hash);
|
2011-06-29 02:30:50 +02:00
|
|
|
*ptr = h;
|
2013-03-01 03:04:58 +01:00
|
|
|
if (old == nullptr) {
|
2011-06-29 02:30:50 +02:00
|
|
|
++elems_;
|
|
|
|
if (elems_ > length_) {
|
|
|
|
// Since each cache entry is fairly large, we aim for a small
|
|
|
|
// average linked list length (<= 1).
|
|
|
|
Resize();
|
|
|
|
}
|
2011-03-18 23:37:00 +01:00
|
|
|
}
|
2011-06-29 02:30:50 +02:00
|
|
|
return old;
|
|
|
|
}
|
|
|
|
|
2011-08-22 23:08:51 +02:00
|
|
|
LRUHandle* Remove(const Slice& key, uint32_t hash) {
|
|
|
|
LRUHandle** ptr = FindPointer(key, hash);
|
2011-06-29 02:30:50 +02:00
|
|
|
LRUHandle* result = *ptr;
|
2013-03-01 03:04:58 +01:00
|
|
|
if (result != nullptr) {
|
2011-06-29 02:30:50 +02:00
|
|
|
*ptr = result->next_hash;
|
|
|
|
--elems_;
|
2011-03-18 23:37:00 +01:00
|
|
|
}
|
2011-06-29 02:30:50 +02:00
|
|
|
return result;
|
|
|
|
}
|
|
|
|
|
|
|
|
private:
|
|
|
|
// The table consists of an array of buckets where each bucket is
|
|
|
|
// a linked list of cache entries that hash into the bucket.
|
|
|
|
uint32_t length_;
|
|
|
|
uint32_t elems_;
|
|
|
|
LRUHandle** list_;
|
|
|
|
|
|
|
|
// Return a pointer to slot that points to a cache entry that
|
2011-08-22 23:08:51 +02:00
|
|
|
// matches key/hash. If there is no such cache entry, return a
|
|
|
|
// pointer to the trailing slot in the corresponding linked list.
|
|
|
|
LRUHandle** FindPointer(const Slice& key, uint32_t hash) {
|
2011-06-29 02:30:50 +02:00
|
|
|
LRUHandle** ptr = &list_[hash & (length_ - 1)];
|
2013-03-01 03:04:58 +01:00
|
|
|
while (*ptr != nullptr &&
|
2011-08-22 23:08:51 +02:00
|
|
|
((*ptr)->hash != hash || key != (*ptr)->key())) {
|
2011-06-29 02:30:50 +02:00
|
|
|
ptr = &(*ptr)->next_hash;
|
2011-03-18 23:37:00 +01:00
|
|
|
}
|
2011-06-29 02:30:50 +02:00
|
|
|
return ptr;
|
|
|
|
}
|
2011-03-18 23:37:00 +01:00
|
|
|
|
2011-06-29 02:30:50 +02:00
|
|
|
void Resize() {
|
2013-12-11 17:33:29 +01:00
|
|
|
uint32_t new_length = 16;
|
|
|
|
while (new_length < elems_ * 1.5) {
|
2011-06-29 02:30:50 +02:00
|
|
|
new_length *= 2;
|
|
|
|
}
|
|
|
|
LRUHandle** new_list = new LRUHandle*[new_length];
|
|
|
|
memset(new_list, 0, sizeof(new_list[0]) * new_length);
|
|
|
|
uint32_t count = 0;
|
2011-08-06 02:19:37 +02:00
|
|
|
for (uint32_t i = 0; i < length_; i++) {
|
2011-06-29 02:30:50 +02:00
|
|
|
LRUHandle* h = list_[i];
|
2013-03-01 03:04:58 +01:00
|
|
|
while (h != nullptr) {
|
2011-06-29 02:30:50 +02:00
|
|
|
LRUHandle* next = h->next_hash;
|
2011-08-22 23:08:51 +02:00
|
|
|
uint32_t hash = h->hash;
|
2011-06-29 02:30:50 +02:00
|
|
|
LRUHandle** ptr = &new_list[hash & (new_length - 1)];
|
|
|
|
h->next_hash = *ptr;
|
|
|
|
*ptr = h;
|
|
|
|
h = next;
|
|
|
|
count++;
|
|
|
|
}
|
2011-03-18 23:37:00 +01:00
|
|
|
}
|
2011-06-29 02:30:50 +02:00
|
|
|
assert(elems_ == count);
|
|
|
|
delete[] list_;
|
|
|
|
list_ = new_list;
|
|
|
|
length_ = new_length;
|
|
|
|
}
|
|
|
|
};
|
2011-03-18 23:37:00 +01:00
|
|
|
|
2011-08-22 23:08:51 +02:00
|
|
|
// A single shard of sharded cache.
|
|
|
|
class LRUCache {
|
2011-03-18 23:37:00 +01:00
|
|
|
public:
|
2011-08-22 23:08:51 +02:00
|
|
|
LRUCache();
|
|
|
|
~LRUCache();
|
2011-03-18 23:37:00 +01:00
|
|
|
|
2011-08-22 23:08:51 +02:00
|
|
|
// Separate from constructor so caller can easily make an array of LRUCache
|
2015-04-24 23:12:58 +02:00
|
|
|
// if current usage is more than new capacity, the function will attempt to
|
|
|
|
// free the needed space
|
|
|
|
void SetCapacity(size_t capacity);
|
2011-08-22 23:08:51 +02:00
|
|
|
|
|
|
|
// Like Cache methods, but with an extra "hash" parameter.
|
|
|
|
Cache::Handle* Insert(const Slice& key, uint32_t hash,
|
|
|
|
void* value, size_t charge,
|
|
|
|
void (*deleter)(const Slice& key, void* value));
|
|
|
|
Cache::Handle* Lookup(const Slice& key, uint32_t hash);
|
|
|
|
void Release(Cache::Handle* handle);
|
|
|
|
void Erase(const Slice& key, uint32_t hash);
|
2013-12-14 00:43:05 +01:00
|
|
|
// Although in some platforms the update of size_t is atomic, to make sure
|
|
|
|
// GetUsage() works correctly under any platforms, we'll protect this
|
|
|
|
// function with mutex.
|
|
|
|
size_t GetUsage() const {
|
|
|
|
MutexLock l(&mutex_);
|
|
|
|
return usage_;
|
|
|
|
}
|
2011-03-18 23:37:00 +01:00
|
|
|
|
2014-05-02 22:24:04 +02:00
|
|
|
void ApplyToAllCacheEntries(void (*callback)(void*, size_t),
|
|
|
|
bool thread_safe);
|
|
|
|
|
2011-03-18 23:37:00 +01:00
|
|
|
private:
|
|
|
|
void LRU_Remove(LRUHandle* e);
|
|
|
|
void LRU_Append(LRUHandle* e);
|
2013-10-08 00:37:40 +02:00
|
|
|
// Just reduce the reference count by 1.
|
|
|
|
// Return true if last reference
|
|
|
|
bool Unref(LRUHandle* e);
|
2011-03-18 23:37:00 +01:00
|
|
|
|
2015-04-24 23:12:58 +02:00
|
|
|
// Free some space following strict LRU policy until enough space
|
|
|
|
// to hold (usage_ + charge) is freed or the lru list is empty
|
|
|
|
// This function is not thread safe - it needs to be executed while
|
|
|
|
// holding the mutex_
|
|
|
|
void EvictFromLRU(size_t charge,
|
|
|
|
autovector<LRUHandle*>* deleted);
|
|
|
|
|
2011-08-22 23:08:51 +02:00
|
|
|
// Initialized before use.
|
|
|
|
size_t capacity_;
|
2011-03-18 23:37:00 +01:00
|
|
|
|
|
|
|
// mutex_ protects the following state.
|
2013-12-14 00:43:05 +01:00
|
|
|
// We don't count mutex_ as the cache's internal state so semantically we
|
|
|
|
// don't mind mutex_ invoking the non-const actions.
|
|
|
|
mutable port::Mutex mutex_;
|
|
|
|
size_t usage_;
|
2011-03-18 23:37:00 +01:00
|
|
|
|
|
|
|
// Dummy head of LRU list.
|
|
|
|
// lru.prev is newest entry, lru.next is oldest entry.
|
Modifed the LRU cache eviction code so that it doesn't evict blocks which have exteranl references
Summary:
Currently, blocks which have more than one reference (ie referenced by something other than cache itself) are evicted from cache. This doesn't make much sense:
- blocks are still in RAM, so the RAM usage reported by the cache is incorrect
- if the same block is needed by another iterator, it will be loaded and decompressed again
This diff changes the reference counting scheme a bit. Previously, if the cache contained the block, this was accounted for in its refcount. After this change, the refcount is only used to track external references. There is a boolean flag which indicates whether or not the block is contained in the cache.
This diff also changes how LRU list is used. Previously, both hashtable and the LRU list contained all blocks. After this change, the LRU list contains blocks with the refcount==0, ie those which can be evicted from the cache.
Note that this change still allows for cache to grow beyond its capacity. This happens when all blocks are pinned (ie refcount>0). This is consistent with the current behavior. The cache's insert function never fails. I spent lots of time trying to make table_reader and other places work with the insert which might failed. It turned out to be pretty hard. It might really destabilize some customers, so finally, I decided against doing this.
table_cache_remove_scan_count_limit option will be unneeded after this change, but I will remove it in the following diff, if this one gets approved
Test Plan: Ran tests, made sure they pass
Reviewers: sdong, ljin
Differential Revision: https://reviews.facebook.net/D25503
2014-10-21 20:49:13 +02:00
|
|
|
// LRU contains items which can be evicted, ie reference only by cache
|
2011-03-18 23:37:00 +01:00
|
|
|
LRUHandle lru_;
|
|
|
|
|
|
|
|
HandleTable table_;
|
|
|
|
};
|
|
|
|
|
2011-08-22 23:08:51 +02:00
|
|
|
LRUCache::LRUCache()
|
2013-11-13 05:05:28 +01:00
|
|
|
: usage_(0) {
|
2011-03-18 23:37:00 +01:00
|
|
|
// Make empty circular linked list
|
|
|
|
lru_.next = &lru_;
|
|
|
|
lru_.prev = &lru_;
|
|
|
|
}
|
|
|
|
|
Modifed the LRU cache eviction code so that it doesn't evict blocks which have exteranl references
Summary:
Currently, blocks which have more than one reference (ie referenced by something other than cache itself) are evicted from cache. This doesn't make much sense:
- blocks are still in RAM, so the RAM usage reported by the cache is incorrect
- if the same block is needed by another iterator, it will be loaded and decompressed again
This diff changes the reference counting scheme a bit. Previously, if the cache contained the block, this was accounted for in its refcount. After this change, the refcount is only used to track external references. There is a boolean flag which indicates whether or not the block is contained in the cache.
This diff also changes how LRU list is used. Previously, both hashtable and the LRU list contained all blocks. After this change, the LRU list contains blocks with the refcount==0, ie those which can be evicted from the cache.
Note that this change still allows for cache to grow beyond its capacity. This happens when all blocks are pinned (ie refcount>0). This is consistent with the current behavior. The cache's insert function never fails. I spent lots of time trying to make table_reader and other places work with the insert which might failed. It turned out to be pretty hard. It might really destabilize some customers, so finally, I decided against doing this.
table_cache_remove_scan_count_limit option will be unneeded after this change, but I will remove it in the following diff, if this one gets approved
Test Plan: Ran tests, made sure they pass
Reviewers: sdong, ljin
Differential Revision: https://reviews.facebook.net/D25503
2014-10-21 20:49:13 +02:00
|
|
|
LRUCache::~LRUCache() {}
|
2011-03-18 23:37:00 +01:00
|
|
|
|
2013-10-08 00:37:40 +02:00
|
|
|
bool LRUCache::Unref(LRUHandle* e) {
|
2011-03-18 23:37:00 +01:00
|
|
|
assert(e->refs > 0);
|
|
|
|
e->refs--;
|
2013-10-08 00:37:40 +02:00
|
|
|
return e->refs == 0;
|
|
|
|
}
|
2013-04-04 03:53:42 +02:00
|
|
|
|
Modifed the LRU cache eviction code so that it doesn't evict blocks which have exteranl references
Summary:
Currently, blocks which have more than one reference (ie referenced by something other than cache itself) are evicted from cache. This doesn't make much sense:
- blocks are still in RAM, so the RAM usage reported by the cache is incorrect
- if the same block is needed by another iterator, it will be loaded and decompressed again
This diff changes the reference counting scheme a bit. Previously, if the cache contained the block, this was accounted for in its refcount. After this change, the refcount is only used to track external references. There is a boolean flag which indicates whether or not the block is contained in the cache.
This diff also changes how LRU list is used. Previously, both hashtable and the LRU list contained all blocks. After this change, the LRU list contains blocks with the refcount==0, ie those which can be evicted from the cache.
Note that this change still allows for cache to grow beyond its capacity. This happens when all blocks are pinned (ie refcount>0). This is consistent with the current behavior. The cache's insert function never fails. I spent lots of time trying to make table_reader and other places work with the insert which might failed. It turned out to be pretty hard. It might really destabilize some customers, so finally, I decided against doing this.
table_cache_remove_scan_count_limit option will be unneeded after this change, but I will remove it in the following diff, if this one gets approved
Test Plan: Ran tests, made sure they pass
Reviewers: sdong, ljin
Differential Revision: https://reviews.facebook.net/D25503
2014-10-21 20:49:13 +02:00
|
|
|
// Call deleter and free
|
2011-03-18 23:37:00 +01:00
|
|
|
|
2014-05-02 22:24:04 +02:00
|
|
|
void LRUCache::ApplyToAllCacheEntries(void (*callback)(void*, size_t),
|
|
|
|
bool thread_safe) {
|
|
|
|
if (thread_safe) {
|
|
|
|
mutex_.Lock();
|
|
|
|
}
|
Modifed the LRU cache eviction code so that it doesn't evict blocks which have exteranl references
Summary:
Currently, blocks which have more than one reference (ie referenced by something other than cache itself) are evicted from cache. This doesn't make much sense:
- blocks are still in RAM, so the RAM usage reported by the cache is incorrect
- if the same block is needed by another iterator, it will be loaded and decompressed again
This diff changes the reference counting scheme a bit. Previously, if the cache contained the block, this was accounted for in its refcount. After this change, the refcount is only used to track external references. There is a boolean flag which indicates whether or not the block is contained in the cache.
This diff also changes how LRU list is used. Previously, both hashtable and the LRU list contained all blocks. After this change, the LRU list contains blocks with the refcount==0, ie those which can be evicted from the cache.
Note that this change still allows for cache to grow beyond its capacity. This happens when all blocks are pinned (ie refcount>0). This is consistent with the current behavior. The cache's insert function never fails. I spent lots of time trying to make table_reader and other places work with the insert which might failed. It turned out to be pretty hard. It might really destabilize some customers, so finally, I decided against doing this.
table_cache_remove_scan_count_limit option will be unneeded after this change, but I will remove it in the following diff, if this one gets approved
Test Plan: Ran tests, made sure they pass
Reviewers: sdong, ljin
Differential Revision: https://reviews.facebook.net/D25503
2014-10-21 20:49:13 +02:00
|
|
|
table_.ApplyToAllCacheEntries([callback](LRUHandle* h) {
|
|
|
|
callback(h->value, h->charge);
|
|
|
|
});
|
2014-05-02 22:24:04 +02:00
|
|
|
if (thread_safe) {
|
|
|
|
mutex_.Unlock();
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2011-03-18 23:37:00 +01:00
|
|
|
void LRUCache::LRU_Remove(LRUHandle* e) {
|
Modifed the LRU cache eviction code so that it doesn't evict blocks which have exteranl references
Summary:
Currently, blocks which have more than one reference (ie referenced by something other than cache itself) are evicted from cache. This doesn't make much sense:
- blocks are still in RAM, so the RAM usage reported by the cache is incorrect
- if the same block is needed by another iterator, it will be loaded and decompressed again
This diff changes the reference counting scheme a bit. Previously, if the cache contained the block, this was accounted for in its refcount. After this change, the refcount is only used to track external references. There is a boolean flag which indicates whether or not the block is contained in the cache.
This diff also changes how LRU list is used. Previously, both hashtable and the LRU list contained all blocks. After this change, the LRU list contains blocks with the refcount==0, ie those which can be evicted from the cache.
Note that this change still allows for cache to grow beyond its capacity. This happens when all blocks are pinned (ie refcount>0). This is consistent with the current behavior. The cache's insert function never fails. I spent lots of time trying to make table_reader and other places work with the insert which might failed. It turned out to be pretty hard. It might really destabilize some customers, so finally, I decided against doing this.
table_cache_remove_scan_count_limit option will be unneeded after this change, but I will remove it in the following diff, if this one gets approved
Test Plan: Ran tests, made sure they pass
Reviewers: sdong, ljin
Differential Revision: https://reviews.facebook.net/D25503
2014-10-21 20:49:13 +02:00
|
|
|
assert(e->next != nullptr);
|
|
|
|
assert(e->prev != nullptr);
|
2011-03-18 23:37:00 +01:00
|
|
|
e->next->prev = e->prev;
|
|
|
|
e->prev->next = e->next;
|
Modifed the LRU cache eviction code so that it doesn't evict blocks which have exteranl references
Summary:
Currently, blocks which have more than one reference (ie referenced by something other than cache itself) are evicted from cache. This doesn't make much sense:
- blocks are still in RAM, so the RAM usage reported by the cache is incorrect
- if the same block is needed by another iterator, it will be loaded and decompressed again
This diff changes the reference counting scheme a bit. Previously, if the cache contained the block, this was accounted for in its refcount. After this change, the refcount is only used to track external references. There is a boolean flag which indicates whether or not the block is contained in the cache.
This diff also changes how LRU list is used. Previously, both hashtable and the LRU list contained all blocks. After this change, the LRU list contains blocks with the refcount==0, ie those which can be evicted from the cache.
Note that this change still allows for cache to grow beyond its capacity. This happens when all blocks are pinned (ie refcount>0). This is consistent with the current behavior. The cache's insert function never fails. I spent lots of time trying to make table_reader and other places work with the insert which might failed. It turned out to be pretty hard. It might really destabilize some customers, so finally, I decided against doing this.
table_cache_remove_scan_count_limit option will be unneeded after this change, but I will remove it in the following diff, if this one gets approved
Test Plan: Ran tests, made sure they pass
Reviewers: sdong, ljin
Differential Revision: https://reviews.facebook.net/D25503
2014-10-21 20:49:13 +02:00
|
|
|
e->prev = e->next = nullptr;
|
2011-03-18 23:37:00 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
void LRUCache::LRU_Append(LRUHandle* e) {
|
|
|
|
// Make "e" newest entry by inserting just before lru_
|
Modifed the LRU cache eviction code so that it doesn't evict blocks which have exteranl references
Summary:
Currently, blocks which have more than one reference (ie referenced by something other than cache itself) are evicted from cache. This doesn't make much sense:
- blocks are still in RAM, so the RAM usage reported by the cache is incorrect
- if the same block is needed by another iterator, it will be loaded and decompressed again
This diff changes the reference counting scheme a bit. Previously, if the cache contained the block, this was accounted for in its refcount. After this change, the refcount is only used to track external references. There is a boolean flag which indicates whether or not the block is contained in the cache.
This diff also changes how LRU list is used. Previously, both hashtable and the LRU list contained all blocks. After this change, the LRU list contains blocks with the refcount==0, ie those which can be evicted from the cache.
Note that this change still allows for cache to grow beyond its capacity. This happens when all blocks are pinned (ie refcount>0). This is consistent with the current behavior. The cache's insert function never fails. I spent lots of time trying to make table_reader and other places work with the insert which might failed. It turned out to be pretty hard. It might really destabilize some customers, so finally, I decided against doing this.
table_cache_remove_scan_count_limit option will be unneeded after this change, but I will remove it in the following diff, if this one gets approved
Test Plan: Ran tests, made sure they pass
Reviewers: sdong, ljin
Differential Revision: https://reviews.facebook.net/D25503
2014-10-21 20:49:13 +02:00
|
|
|
assert(e->next == nullptr);
|
|
|
|
assert(e->prev == nullptr);
|
2011-03-18 23:37:00 +01:00
|
|
|
e->next = &lru_;
|
|
|
|
e->prev = lru_.prev;
|
|
|
|
e->prev->next = e;
|
|
|
|
e->next->prev = e;
|
|
|
|
}
|
|
|
|
|
2015-04-24 23:12:58 +02:00
|
|
|
void LRUCache::EvictFromLRU(size_t charge,
|
|
|
|
autovector<LRUHandle*>* deleted) {
|
|
|
|
while (usage_ + charge > capacity_ && lru_.next != &lru_) {
|
|
|
|
LRUHandle* old = lru_.next;
|
|
|
|
assert(old->in_cache);
|
|
|
|
assert(old->refs == 1); // LRU list contains elements which may be evicted
|
|
|
|
LRU_Remove(old);
|
|
|
|
table_.Remove(old->key(), old->hash);
|
|
|
|
old->in_cache = false;
|
|
|
|
Unref(old);
|
|
|
|
usage_ -= old->charge;
|
|
|
|
deleted->push_back(old);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
void LRUCache::SetCapacity(size_t capacity) {
|
|
|
|
autovector<LRUHandle*> last_reference_list;
|
|
|
|
{
|
|
|
|
MutexLock l(&mutex_);
|
|
|
|
capacity_ = capacity;
|
|
|
|
EvictFromLRU(0, &last_reference_list);
|
|
|
|
}
|
|
|
|
// we free the entries here outside of mutex for
|
|
|
|
// performance reasons
|
|
|
|
for (auto entry : last_reference_list) {
|
|
|
|
entry->Free();
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2011-08-22 23:08:51 +02:00
|
|
|
Cache::Handle* LRUCache::Lookup(const Slice& key, uint32_t hash) {
|
2011-03-18 23:37:00 +01:00
|
|
|
MutexLock l(&mutex_);
|
2011-08-22 23:08:51 +02:00
|
|
|
LRUHandle* e = table_.Lookup(key, hash);
|
2013-03-01 03:04:58 +01:00
|
|
|
if (e != nullptr) {
|
Modifed the LRU cache eviction code so that it doesn't evict blocks which have exteranl references
Summary:
Currently, blocks which have more than one reference (ie referenced by something other than cache itself) are evicted from cache. This doesn't make much sense:
- blocks are still in RAM, so the RAM usage reported by the cache is incorrect
- if the same block is needed by another iterator, it will be loaded and decompressed again
This diff changes the reference counting scheme a bit. Previously, if the cache contained the block, this was accounted for in its refcount. After this change, the refcount is only used to track external references. There is a boolean flag which indicates whether or not the block is contained in the cache.
This diff also changes how LRU list is used. Previously, both hashtable and the LRU list contained all blocks. After this change, the LRU list contains blocks with the refcount==0, ie those which can be evicted from the cache.
Note that this change still allows for cache to grow beyond its capacity. This happens when all blocks are pinned (ie refcount>0). This is consistent with the current behavior. The cache's insert function never fails. I spent lots of time trying to make table_reader and other places work with the insert which might failed. It turned out to be pretty hard. It might really destabilize some customers, so finally, I decided against doing this.
table_cache_remove_scan_count_limit option will be unneeded after this change, but I will remove it in the following diff, if this one gets approved
Test Plan: Ran tests, made sure they pass
Reviewers: sdong, ljin
Differential Revision: https://reviews.facebook.net/D25503
2014-10-21 20:49:13 +02:00
|
|
|
assert(e->in_cache);
|
|
|
|
if (e->refs == 1) {
|
|
|
|
LRU_Remove(e);
|
|
|
|
}
|
2011-03-18 23:37:00 +01:00
|
|
|
e->refs++;
|
|
|
|
}
|
2011-08-22 23:08:51 +02:00
|
|
|
return reinterpret_cast<Cache::Handle*>(e);
|
2011-03-18 23:37:00 +01:00
|
|
|
}
|
|
|
|
|
2011-08-22 23:08:51 +02:00
|
|
|
void LRUCache::Release(Cache::Handle* handle) {
|
2013-10-08 00:37:40 +02:00
|
|
|
LRUHandle* e = reinterpret_cast<LRUHandle*>(handle);
|
|
|
|
bool last_reference = false;
|
|
|
|
{
|
|
|
|
MutexLock l(&mutex_);
|
|
|
|
last_reference = Unref(e);
|
Modifed the LRU cache eviction code so that it doesn't evict blocks which have exteranl references
Summary:
Currently, blocks which have more than one reference (ie referenced by something other than cache itself) are evicted from cache. This doesn't make much sense:
- blocks are still in RAM, so the RAM usage reported by the cache is incorrect
- if the same block is needed by another iterator, it will be loaded and decompressed again
This diff changes the reference counting scheme a bit. Previously, if the cache contained the block, this was accounted for in its refcount. After this change, the refcount is only used to track external references. There is a boolean flag which indicates whether or not the block is contained in the cache.
This diff also changes how LRU list is used. Previously, both hashtable and the LRU list contained all blocks. After this change, the LRU list contains blocks with the refcount==0, ie those which can be evicted from the cache.
Note that this change still allows for cache to grow beyond its capacity. This happens when all blocks are pinned (ie refcount>0). This is consistent with the current behavior. The cache's insert function never fails. I spent lots of time trying to make table_reader and other places work with the insert which might failed. It turned out to be pretty hard. It might really destabilize some customers, so finally, I decided against doing this.
table_cache_remove_scan_count_limit option will be unneeded after this change, but I will remove it in the following diff, if this one gets approved
Test Plan: Ran tests, made sure they pass
Reviewers: sdong, ljin
Differential Revision: https://reviews.facebook.net/D25503
2014-10-21 20:49:13 +02:00
|
|
|
if (last_reference) {
|
|
|
|
usage_ -= e->charge;
|
|
|
|
}
|
|
|
|
if (e->refs == 1 && e->in_cache) {
|
|
|
|
// The item is still in cache, and nobody else holds a reference to it
|
|
|
|
if (usage_ > capacity_) {
|
|
|
|
// the cache is full
|
|
|
|
// The LRU list must be empty since the cache is full
|
|
|
|
assert(lru_.next == &lru_);
|
|
|
|
// take this opportunity and remove the item
|
|
|
|
table_.Remove(e->key(), e->hash);
|
|
|
|
e->in_cache = false;
|
|
|
|
Unref(e);
|
|
|
|
usage_ -= e->charge;
|
|
|
|
last_reference = true;
|
|
|
|
} else {
|
|
|
|
// put the item on the list to be potentially freed
|
|
|
|
LRU_Append(e);
|
|
|
|
}
|
|
|
|
}
|
2013-10-08 00:37:40 +02:00
|
|
|
}
|
Modifed the LRU cache eviction code so that it doesn't evict blocks which have exteranl references
Summary:
Currently, blocks which have more than one reference (ie referenced by something other than cache itself) are evicted from cache. This doesn't make much sense:
- blocks are still in RAM, so the RAM usage reported by the cache is incorrect
- if the same block is needed by another iterator, it will be loaded and decompressed again
This diff changes the reference counting scheme a bit. Previously, if the cache contained the block, this was accounted for in its refcount. After this change, the refcount is only used to track external references. There is a boolean flag which indicates whether or not the block is contained in the cache.
This diff also changes how LRU list is used. Previously, both hashtable and the LRU list contained all blocks. After this change, the LRU list contains blocks with the refcount==0, ie those which can be evicted from the cache.
Note that this change still allows for cache to grow beyond its capacity. This happens when all blocks are pinned (ie refcount>0). This is consistent with the current behavior. The cache's insert function never fails. I spent lots of time trying to make table_reader and other places work with the insert which might failed. It turned out to be pretty hard. It might really destabilize some customers, so finally, I decided against doing this.
table_cache_remove_scan_count_limit option will be unneeded after this change, but I will remove it in the following diff, if this one gets approved
Test Plan: Ran tests, made sure they pass
Reviewers: sdong, ljin
Differential Revision: https://reviews.facebook.net/D25503
2014-10-21 20:49:13 +02:00
|
|
|
|
|
|
|
// free outside of mutex
|
2013-10-08 00:37:40 +02:00
|
|
|
if (last_reference) {
|
Modifed the LRU cache eviction code so that it doesn't evict blocks which have exteranl references
Summary:
Currently, blocks which have more than one reference (ie referenced by something other than cache itself) are evicted from cache. This doesn't make much sense:
- blocks are still in RAM, so the RAM usage reported by the cache is incorrect
- if the same block is needed by another iterator, it will be loaded and decompressed again
This diff changes the reference counting scheme a bit. Previously, if the cache contained the block, this was accounted for in its refcount. After this change, the refcount is only used to track external references. There is a boolean flag which indicates whether or not the block is contained in the cache.
This diff also changes how LRU list is used. Previously, both hashtable and the LRU list contained all blocks. After this change, the LRU list contains blocks with the refcount==0, ie those which can be evicted from the cache.
Note that this change still allows for cache to grow beyond its capacity. This happens when all blocks are pinned (ie refcount>0). This is consistent with the current behavior. The cache's insert function never fails. I spent lots of time trying to make table_reader and other places work with the insert which might failed. It turned out to be pretty hard. It might really destabilize some customers, so finally, I decided against doing this.
table_cache_remove_scan_count_limit option will be unneeded after this change, but I will remove it in the following diff, if this one gets approved
Test Plan: Ran tests, made sure they pass
Reviewers: sdong, ljin
Differential Revision: https://reviews.facebook.net/D25503
2014-10-21 20:49:13 +02:00
|
|
|
e->Free();
|
2013-10-08 00:37:40 +02:00
|
|
|
}
|
2011-03-18 23:37:00 +01:00
|
|
|
}
|
|
|
|
|
2011-08-22 23:08:51 +02:00
|
|
|
Cache::Handle* LRUCache::Insert(
|
|
|
|
const Slice& key, uint32_t hash, void* value, size_t charge,
|
|
|
|
void (*deleter)(const Slice& key, void* value)) {
|
2011-03-18 23:37:00 +01:00
|
|
|
|
Modifed the LRU cache eviction code so that it doesn't evict blocks which have exteranl references
Summary:
Currently, blocks which have more than one reference (ie referenced by something other than cache itself) are evicted from cache. This doesn't make much sense:
- blocks are still in RAM, so the RAM usage reported by the cache is incorrect
- if the same block is needed by another iterator, it will be loaded and decompressed again
This diff changes the reference counting scheme a bit. Previously, if the cache contained the block, this was accounted for in its refcount. After this change, the refcount is only used to track external references. There is a boolean flag which indicates whether or not the block is contained in the cache.
This diff also changes how LRU list is used. Previously, both hashtable and the LRU list contained all blocks. After this change, the LRU list contains blocks with the refcount==0, ie those which can be evicted from the cache.
Note that this change still allows for cache to grow beyond its capacity. This happens when all blocks are pinned (ie refcount>0). This is consistent with the current behavior. The cache's insert function never fails. I spent lots of time trying to make table_reader and other places work with the insert which might failed. It turned out to be pretty hard. It might really destabilize some customers, so finally, I decided against doing this.
table_cache_remove_scan_count_limit option will be unneeded after this change, but I will remove it in the following diff, if this one gets approved
Test Plan: Ran tests, made sure they pass
Reviewers: sdong, ljin
Differential Revision: https://reviews.facebook.net/D25503
2014-10-21 20:49:13 +02:00
|
|
|
// Allocate the memory here outside of the mutex
|
|
|
|
// If the cache is full, we'll have to release it
|
|
|
|
// It shouldn't happen very often though.
|
|
|
|
LRUHandle* e =
|
|
|
|
reinterpret_cast<LRUHandle*>(malloc(sizeof(LRUHandle) - 1 + key.size()));
|
2014-01-02 20:26:57 +01:00
|
|
|
autovector<LRUHandle*> last_reference_list;
|
2013-12-11 17:33:29 +01:00
|
|
|
|
|
|
|
e->value = value;
|
|
|
|
e->deleter = deleter;
|
|
|
|
e->charge = charge;
|
|
|
|
e->key_length = key.size();
|
|
|
|
e->hash = hash;
|
|
|
|
e->refs = 2; // One from LRUCache, one for the returned handle
|
Modifed the LRU cache eviction code so that it doesn't evict blocks which have exteranl references
Summary:
Currently, blocks which have more than one reference (ie referenced by something other than cache itself) are evicted from cache. This doesn't make much sense:
- blocks are still in RAM, so the RAM usage reported by the cache is incorrect
- if the same block is needed by another iterator, it will be loaded and decompressed again
This diff changes the reference counting scheme a bit. Previously, if the cache contained the block, this was accounted for in its refcount. After this change, the refcount is only used to track external references. There is a boolean flag which indicates whether or not the block is contained in the cache.
This diff also changes how LRU list is used. Previously, both hashtable and the LRU list contained all blocks. After this change, the LRU list contains blocks with the refcount==0, ie those which can be evicted from the cache.
Note that this change still allows for cache to grow beyond its capacity. This happens when all blocks are pinned (ie refcount>0). This is consistent with the current behavior. The cache's insert function never fails. I spent lots of time trying to make table_reader and other places work with the insert which might failed. It turned out to be pretty hard. It might really destabilize some customers, so finally, I decided against doing this.
table_cache_remove_scan_count_limit option will be unneeded after this change, but I will remove it in the following diff, if this one gets approved
Test Plan: Ran tests, made sure they pass
Reviewers: sdong, ljin
Differential Revision: https://reviews.facebook.net/D25503
2014-10-21 20:49:13 +02:00
|
|
|
e->next = e->prev = nullptr;
|
|
|
|
e->in_cache = true;
|
2013-12-11 17:33:29 +01:00
|
|
|
memcpy(e->key_data, key.data(), key.size());
|
2013-10-08 00:37:40 +02:00
|
|
|
|
|
|
|
{
|
|
|
|
MutexLock l(&mutex_);
|
|
|
|
|
2013-10-10 02:04:40 +02:00
|
|
|
// Free the space following strict LRU policy until enough space
|
Modifed the LRU cache eviction code so that it doesn't evict blocks which have exteranl references
Summary:
Currently, blocks which have more than one reference (ie referenced by something other than cache itself) are evicted from cache. This doesn't make much sense:
- blocks are still in RAM, so the RAM usage reported by the cache is incorrect
- if the same block is needed by another iterator, it will be loaded and decompressed again
This diff changes the reference counting scheme a bit. Previously, if the cache contained the block, this was accounted for in its refcount. After this change, the refcount is only used to track external references. There is a boolean flag which indicates whether or not the block is contained in the cache.
This diff also changes how LRU list is used. Previously, both hashtable and the LRU list contained all blocks. After this change, the LRU list contains blocks with the refcount==0, ie those which can be evicted from the cache.
Note that this change still allows for cache to grow beyond its capacity. This happens when all blocks are pinned (ie refcount>0). This is consistent with the current behavior. The cache's insert function never fails. I spent lots of time trying to make table_reader and other places work with the insert which might failed. It turned out to be pretty hard. It might really destabilize some customers, so finally, I decided against doing this.
table_cache_remove_scan_count_limit option will be unneeded after this change, but I will remove it in the following diff, if this one gets approved
Test Plan: Ran tests, made sure they pass
Reviewers: sdong, ljin
Differential Revision: https://reviews.facebook.net/D25503
2014-10-21 20:49:13 +02:00
|
|
|
// is freed or the lru list is empty
|
2015-04-24 23:12:58 +02:00
|
|
|
EvictFromLRU(charge, &last_reference_list);
|
Modifed the LRU cache eviction code so that it doesn't evict blocks which have exteranl references
Summary:
Currently, blocks which have more than one reference (ie referenced by something other than cache itself) are evicted from cache. This doesn't make much sense:
- blocks are still in RAM, so the RAM usage reported by the cache is incorrect
- if the same block is needed by another iterator, it will be loaded and decompressed again
This diff changes the reference counting scheme a bit. Previously, if the cache contained the block, this was accounted for in its refcount. After this change, the refcount is only used to track external references. There is a boolean flag which indicates whether or not the block is contained in the cache.
This diff also changes how LRU list is used. Previously, both hashtable and the LRU list contained all blocks. After this change, the LRU list contains blocks with the refcount==0, ie those which can be evicted from the cache.
Note that this change still allows for cache to grow beyond its capacity. This happens when all blocks are pinned (ie refcount>0). This is consistent with the current behavior. The cache's insert function never fails. I spent lots of time trying to make table_reader and other places work with the insert which might failed. It turned out to be pretty hard. It might really destabilize some customers, so finally, I decided against doing this.
table_cache_remove_scan_count_limit option will be unneeded after this change, but I will remove it in the following diff, if this one gets approved
Test Plan: Ran tests, made sure they pass
Reviewers: sdong, ljin
Differential Revision: https://reviews.facebook.net/D25503
2014-10-21 20:49:13 +02:00
|
|
|
|
|
|
|
// insert into the cache
|
|
|
|
// note that the cache might get larger than its capacity if not enough
|
|
|
|
// space was freed
|
|
|
|
LRUHandle* old = table_.Insert(e);
|
|
|
|
usage_ += e->charge;
|
|
|
|
if (old != nullptr) {
|
|
|
|
old->in_cache = false;
|
2013-10-08 00:37:40 +02:00
|
|
|
if (Unref(old)) {
|
Modifed the LRU cache eviction code so that it doesn't evict blocks which have exteranl references
Summary:
Currently, blocks which have more than one reference (ie referenced by something other than cache itself) are evicted from cache. This doesn't make much sense:
- blocks are still in RAM, so the RAM usage reported by the cache is incorrect
- if the same block is needed by another iterator, it will be loaded and decompressed again
This diff changes the reference counting scheme a bit. Previously, if the cache contained the block, this was accounted for in its refcount. After this change, the refcount is only used to track external references. There is a boolean flag which indicates whether or not the block is contained in the cache.
This diff also changes how LRU list is used. Previously, both hashtable and the LRU list contained all blocks. After this change, the LRU list contains blocks with the refcount==0, ie those which can be evicted from the cache.
Note that this change still allows for cache to grow beyond its capacity. This happens when all blocks are pinned (ie refcount>0). This is consistent with the current behavior. The cache's insert function never fails. I spent lots of time trying to make table_reader and other places work with the insert which might failed. It turned out to be pretty hard. It might really destabilize some customers, so finally, I decided against doing this.
table_cache_remove_scan_count_limit option will be unneeded after this change, but I will remove it in the following diff, if this one gets approved
Test Plan: Ran tests, made sure they pass
Reviewers: sdong, ljin
Differential Revision: https://reviews.facebook.net/D25503
2014-10-21 20:49:13 +02:00
|
|
|
usage_ -= old->charge;
|
|
|
|
// old is on LRU because it's in cache and its reference count
|
|
|
|
// was just 1 (Unref returned 0)
|
|
|
|
LRU_Remove(old);
|
2013-10-08 00:37:40 +02:00
|
|
|
last_reference_list.push_back(old);
|
|
|
|
}
|
|
|
|
}
|
2011-03-18 23:37:00 +01:00
|
|
|
}
|
|
|
|
|
2013-10-08 00:37:40 +02:00
|
|
|
// we free the entries here outside of mutex for
|
|
|
|
// performance reasons
|
|
|
|
for (auto entry : last_reference_list) {
|
Modifed the LRU cache eviction code so that it doesn't evict blocks which have exteranl references
Summary:
Currently, blocks which have more than one reference (ie referenced by something other than cache itself) are evicted from cache. This doesn't make much sense:
- blocks are still in RAM, so the RAM usage reported by the cache is incorrect
- if the same block is needed by another iterator, it will be loaded and decompressed again
This diff changes the reference counting scheme a bit. Previously, if the cache contained the block, this was accounted for in its refcount. After this change, the refcount is only used to track external references. There is a boolean flag which indicates whether or not the block is contained in the cache.
This diff also changes how LRU list is used. Previously, both hashtable and the LRU list contained all blocks. After this change, the LRU list contains blocks with the refcount==0, ie those which can be evicted from the cache.
Note that this change still allows for cache to grow beyond its capacity. This happens when all blocks are pinned (ie refcount>0). This is consistent with the current behavior. The cache's insert function never fails. I spent lots of time trying to make table_reader and other places work with the insert which might failed. It turned out to be pretty hard. It might really destabilize some customers, so finally, I decided against doing this.
table_cache_remove_scan_count_limit option will be unneeded after this change, but I will remove it in the following diff, if this one gets approved
Test Plan: Ran tests, made sure they pass
Reviewers: sdong, ljin
Differential Revision: https://reviews.facebook.net/D25503
2014-10-21 20:49:13 +02:00
|
|
|
entry->Free();
|
2011-03-18 23:37:00 +01:00
|
|
|
}
|
|
|
|
|
2011-08-22 23:08:51 +02:00
|
|
|
return reinterpret_cast<Cache::Handle*>(e);
|
2011-03-18 23:37:00 +01:00
|
|
|
}
|
|
|
|
|
2011-08-22 23:08:51 +02:00
|
|
|
void LRUCache::Erase(const Slice& key, uint32_t hash) {
|
2013-10-08 00:37:40 +02:00
|
|
|
LRUHandle* e;
|
|
|
|
bool last_reference = false;
|
|
|
|
{
|
|
|
|
MutexLock l(&mutex_);
|
|
|
|
e = table_.Remove(key, hash);
|
|
|
|
if (e != nullptr) {
|
|
|
|
last_reference = Unref(e);
|
Modifed the LRU cache eviction code so that it doesn't evict blocks which have exteranl references
Summary:
Currently, blocks which have more than one reference (ie referenced by something other than cache itself) are evicted from cache. This doesn't make much sense:
- blocks are still in RAM, so the RAM usage reported by the cache is incorrect
- if the same block is needed by another iterator, it will be loaded and decompressed again
This diff changes the reference counting scheme a bit. Previously, if the cache contained the block, this was accounted for in its refcount. After this change, the refcount is only used to track external references. There is a boolean flag which indicates whether or not the block is contained in the cache.
This diff also changes how LRU list is used. Previously, both hashtable and the LRU list contained all blocks. After this change, the LRU list contains blocks with the refcount==0, ie those which can be evicted from the cache.
Note that this change still allows for cache to grow beyond its capacity. This happens when all blocks are pinned (ie refcount>0). This is consistent with the current behavior. The cache's insert function never fails. I spent lots of time trying to make table_reader and other places work with the insert which might failed. It turned out to be pretty hard. It might really destabilize some customers, so finally, I decided against doing this.
table_cache_remove_scan_count_limit option will be unneeded after this change, but I will remove it in the following diff, if this one gets approved
Test Plan: Ran tests, made sure they pass
Reviewers: sdong, ljin
Differential Revision: https://reviews.facebook.net/D25503
2014-10-21 20:49:13 +02:00
|
|
|
if (last_reference) {
|
|
|
|
usage_ -= e->charge;
|
|
|
|
}
|
|
|
|
if (last_reference && e->in_cache) {
|
|
|
|
LRU_Remove(e);
|
|
|
|
}
|
|
|
|
e->in_cache = false;
|
2013-10-08 00:37:40 +02:00
|
|
|
}
|
|
|
|
}
|
Modifed the LRU cache eviction code so that it doesn't evict blocks which have exteranl references
Summary:
Currently, blocks which have more than one reference (ie referenced by something other than cache itself) are evicted from cache. This doesn't make much sense:
- blocks are still in RAM, so the RAM usage reported by the cache is incorrect
- if the same block is needed by another iterator, it will be loaded and decompressed again
This diff changes the reference counting scheme a bit. Previously, if the cache contained the block, this was accounted for in its refcount. After this change, the refcount is only used to track external references. There is a boolean flag which indicates whether or not the block is contained in the cache.
This diff also changes how LRU list is used. Previously, both hashtable and the LRU list contained all blocks. After this change, the LRU list contains blocks with the refcount==0, ie those which can be evicted from the cache.
Note that this change still allows for cache to grow beyond its capacity. This happens when all blocks are pinned (ie refcount>0). This is consistent with the current behavior. The cache's insert function never fails. I spent lots of time trying to make table_reader and other places work with the insert which might failed. It turned out to be pretty hard. It might really destabilize some customers, so finally, I decided against doing this.
table_cache_remove_scan_count_limit option will be unneeded after this change, but I will remove it in the following diff, if this one gets approved
Test Plan: Ran tests, made sure they pass
Reviewers: sdong, ljin
Differential Revision: https://reviews.facebook.net/D25503
2014-10-21 20:49:13 +02:00
|
|
|
|
2013-10-08 00:37:40 +02:00
|
|
|
// mutex not held here
|
|
|
|
// last_reference will only be true if e != nullptr
|
|
|
|
if (last_reference) {
|
Modifed the LRU cache eviction code so that it doesn't evict blocks which have exteranl references
Summary:
Currently, blocks which have more than one reference (ie referenced by something other than cache itself) are evicted from cache. This doesn't make much sense:
- blocks are still in RAM, so the RAM usage reported by the cache is incorrect
- if the same block is needed by another iterator, it will be loaded and decompressed again
This diff changes the reference counting scheme a bit. Previously, if the cache contained the block, this was accounted for in its refcount. After this change, the refcount is only used to track external references. There is a boolean flag which indicates whether or not the block is contained in the cache.
This diff also changes how LRU list is used. Previously, both hashtable and the LRU list contained all blocks. After this change, the LRU list contains blocks with the refcount==0, ie those which can be evicted from the cache.
Note that this change still allows for cache to grow beyond its capacity. This happens when all blocks are pinned (ie refcount>0). This is consistent with the current behavior. The cache's insert function never fails. I spent lots of time trying to make table_reader and other places work with the insert which might failed. It turned out to be pretty hard. It might really destabilize some customers, so finally, I decided against doing this.
table_cache_remove_scan_count_limit option will be unneeded after this change, but I will remove it in the following diff, if this one gets approved
Test Plan: Ran tests, made sure they pass
Reviewers: sdong, ljin
Differential Revision: https://reviews.facebook.net/D25503
2014-10-21 20:49:13 +02:00
|
|
|
e->Free();
|
2011-03-18 23:37:00 +01:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2013-10-10 02:04:40 +02:00
|
|
|
static int kNumShardBits = 4; // default values, can be overridden
|
2011-08-22 23:08:51 +02:00
|
|
|
|
|
|
|
class ShardedLRUCache : public Cache {
|
|
|
|
private:
|
2013-12-11 01:21:49 +01:00
|
|
|
LRUCache* shards_;
|
2011-08-22 23:08:51 +02:00
|
|
|
port::Mutex id_mutex_;
|
2015-04-24 23:12:58 +02:00
|
|
|
port::Mutex capacity_mutex_;
|
2011-08-22 23:08:51 +02:00
|
|
|
uint64_t last_id_;
|
2013-12-11 01:21:49 +01:00
|
|
|
int num_shard_bits_;
|
2012-09-30 03:02:02 +02:00
|
|
|
size_t capacity_;
|
2011-08-22 23:08:51 +02:00
|
|
|
|
|
|
|
static inline uint32_t HashSlice(const Slice& s) {
|
|
|
|
return Hash(s.data(), s.size(), 0);
|
|
|
|
}
|
|
|
|
|
2012-08-29 18:47:53 +02:00
|
|
|
uint32_t Shard(uint32_t hash) {
|
2013-04-04 03:53:42 +02:00
|
|
|
// Note, hash >> 32 yields hash in gcc, not the zero we expect!
|
2013-12-11 01:21:49 +01:00
|
|
|
return (num_shard_bits_ > 0) ? (hash >> (32 - num_shard_bits_)) : 0;
|
2011-08-22 23:08:51 +02:00
|
|
|
}
|
|
|
|
|
2015-03-17 23:04:37 +01:00
|
|
|
public:
|
|
|
|
ShardedLRUCache(size_t capacity, int num_shard_bits)
|
|
|
|
: last_id_(0), num_shard_bits_(num_shard_bits), capacity_(capacity) {
|
2013-12-11 01:21:49 +01:00
|
|
|
int num_shards = 1 << num_shard_bits_;
|
|
|
|
shards_ = new LRUCache[num_shards];
|
|
|
|
const size_t per_shard = (capacity + (num_shards - 1)) / num_shards;
|
|
|
|
for (int s = 0; s < num_shards; s++) {
|
|
|
|
shards_[s].SetCapacity(per_shard);
|
2011-08-22 23:08:51 +02:00
|
|
|
}
|
|
|
|
}
|
2013-01-08 20:24:15 +01:00
|
|
|
virtual ~ShardedLRUCache() {
|
2013-12-11 01:21:49 +01:00
|
|
|
delete[] shards_;
|
2013-01-08 20:24:15 +01:00
|
|
|
}
|
2015-04-24 23:12:58 +02:00
|
|
|
virtual void SetCapacity(size_t capacity) {
|
|
|
|
int num_shards = 1 << num_shard_bits_;
|
|
|
|
const size_t per_shard = (capacity + (num_shards - 1)) / num_shards;
|
|
|
|
MutexLock l(&capacity_mutex_);
|
|
|
|
for (int s = 0; s < num_shards; s++) {
|
|
|
|
shards_[s].SetCapacity(per_shard);
|
|
|
|
}
|
|
|
|
capacity_ = capacity;
|
|
|
|
}
|
2011-08-22 23:08:51 +02:00
|
|
|
virtual Handle* Insert(const Slice& key, void* value, size_t charge,
|
2015-02-26 20:28:41 +01:00
|
|
|
void (*deleter)(const Slice& key,
|
|
|
|
void* value)) override {
|
2011-08-22 23:08:51 +02:00
|
|
|
const uint32_t hash = HashSlice(key);
|
2013-12-11 01:21:49 +01:00
|
|
|
return shards_[Shard(hash)].Insert(key, hash, value, charge, deleter);
|
2011-08-22 23:08:51 +02:00
|
|
|
}
|
2015-02-26 20:28:41 +01:00
|
|
|
virtual Handle* Lookup(const Slice& key) override {
|
2011-08-22 23:08:51 +02:00
|
|
|
const uint32_t hash = HashSlice(key);
|
2013-12-11 01:21:49 +01:00
|
|
|
return shards_[Shard(hash)].Lookup(key, hash);
|
2011-08-22 23:08:51 +02:00
|
|
|
}
|
2015-02-26 20:28:41 +01:00
|
|
|
virtual void Release(Handle* handle) override {
|
2011-08-22 23:08:51 +02:00
|
|
|
LRUHandle* h = reinterpret_cast<LRUHandle*>(handle);
|
2013-12-11 01:21:49 +01:00
|
|
|
shards_[Shard(h->hash)].Release(handle);
|
2011-08-22 23:08:51 +02:00
|
|
|
}
|
2015-02-26 20:28:41 +01:00
|
|
|
virtual void Erase(const Slice& key) override {
|
2011-08-22 23:08:51 +02:00
|
|
|
const uint32_t hash = HashSlice(key);
|
2013-12-11 01:21:49 +01:00
|
|
|
shards_[Shard(hash)].Erase(key, hash);
|
2011-08-22 23:08:51 +02:00
|
|
|
}
|
2015-02-26 20:28:41 +01:00
|
|
|
virtual void* Value(Handle* handle) override {
|
2011-08-22 23:08:51 +02:00
|
|
|
return reinterpret_cast<LRUHandle*>(handle)->value;
|
|
|
|
}
|
2015-02-26 20:28:41 +01:00
|
|
|
virtual uint64_t NewId() override {
|
2011-08-22 23:08:51 +02:00
|
|
|
MutexLock l(&id_mutex_);
|
|
|
|
return ++(last_id_);
|
|
|
|
}
|
2015-02-26 20:28:41 +01:00
|
|
|
virtual size_t GetCapacity() const override { return capacity_; }
|
2014-01-28 19:35:48 +01:00
|
|
|
|
2015-02-26 20:28:41 +01:00
|
|
|
virtual size_t GetUsage() const override {
|
2013-12-11 01:21:49 +01:00
|
|
|
// We will not lock the cache when getting the usage from shards.
|
|
|
|
// for (size_t i = 0; i < num_shard_bits_; ++i)
|
|
|
|
int num_shards = 1 << num_shard_bits_;
|
|
|
|
size_t usage = 0;
|
|
|
|
for (int s = 0; s < num_shards; s++) {
|
|
|
|
usage += shards_[s].GetUsage();
|
|
|
|
}
|
|
|
|
return usage;
|
|
|
|
}
|
2014-01-28 19:35:48 +01:00
|
|
|
|
2015-02-26 20:28:41 +01:00
|
|
|
virtual void DisownData() override { shards_ = nullptr; }
|
2014-05-02 22:24:04 +02:00
|
|
|
|
|
|
|
virtual void ApplyToAllCacheEntries(void (*callback)(void*, size_t),
|
|
|
|
bool thread_safe) override {
|
|
|
|
int num_shards = 1 << num_shard_bits_;
|
|
|
|
for (int s = 0; s < num_shards; s++) {
|
|
|
|
shards_[s].ApplyToAllCacheEntries(callback, thread_safe);
|
|
|
|
}
|
|
|
|
}
|
2011-08-22 23:08:51 +02:00
|
|
|
};
|
2011-03-18 23:37:00 +01:00
|
|
|
|
|
|
|
} // end anonymous namespace
|
|
|
|
|
2013-01-20 11:07:13 +01:00
|
|
|
shared_ptr<Cache> NewLRUCache(size_t capacity) {
|
2013-10-10 02:04:40 +02:00
|
|
|
return NewLRUCache(capacity, kNumShardBits);
|
2011-03-18 23:37:00 +01:00
|
|
|
}
|
|
|
|
|
2013-12-11 01:21:49 +01:00
|
|
|
shared_ptr<Cache> NewLRUCache(size_t capacity, int num_shard_bits) {
|
|
|
|
if (num_shard_bits >= 20) {
|
2013-03-01 03:04:58 +01:00
|
|
|
return nullptr; // the cache cannot be sharded into too many fine pieces
|
2012-08-29 18:47:53 +02:00
|
|
|
}
|
2015-03-17 23:04:37 +01:00
|
|
|
return std::make_shared<ShardedLRUCache>(capacity, num_shard_bits);
|
2012-05-17 02:22:33 +02:00
|
|
|
}
|
|
|
|
|
2013-10-04 06:49:15 +02:00
|
|
|
} // namespace rocksdb
|