aboutsummaryrefslogtreecommitdiff
path: root/src/leveldb/helpers/memenv
diff options
context:
space:
mode:
Diffstat (limited to 'src/leveldb/helpers/memenv')
-rw-r--r--src/leveldb/helpers/memenv/memenv.cc394
-rw-r--r--src/leveldb/helpers/memenv/memenv.h22
-rw-r--r--src/leveldb/helpers/memenv/memenv_test.cc259
3 files changed, 675 insertions, 0 deletions
diff --git a/src/leveldb/helpers/memenv/memenv.cc b/src/leveldb/helpers/memenv/memenv.cc
new file mode 100644
index 0000000000..47e4481f7c
--- /dev/null
+++ b/src/leveldb/helpers/memenv/memenv.cc
@@ -0,0 +1,394 @@
+// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file. See the AUTHORS file for names of contributors.
+
+#include "helpers/memenv/memenv.h"
+
+#include <string.h>
+
+#include <limits>
+#include <map>
+#include <string>
+#include <vector>
+
+#include "leveldb/env.h"
+#include "leveldb/status.h"
+#include "port/port.h"
+#include "port/thread_annotations.h"
+#include "util/mutexlock.h"
+
+namespace leveldb {
+
+namespace {
+
+class FileState {
+ public:
+ // FileStates are reference counted. The initial reference count is zero
+ // and the caller must call Ref() at least once.
+ FileState() : refs_(0), size_(0) {}
+
+ // No copying allowed.
+ FileState(const FileState&) = delete;
+ FileState& operator=(const FileState&) = delete;
+
+ // Increase the reference count.
+ void Ref() {
+ MutexLock lock(&refs_mutex_);
+ ++refs_;
+ }
+
+ // Decrease the reference count. Delete if this is the last reference.
+ void Unref() {
+ bool do_delete = false;
+
+ {
+ MutexLock lock(&refs_mutex_);
+ --refs_;
+ assert(refs_ >= 0);
+ if (refs_ <= 0) {
+ do_delete = true;
+ }
+ }
+
+ if (do_delete) {
+ delete this;
+ }
+ }
+
+ uint64_t Size() const {
+ MutexLock lock(&blocks_mutex_);
+ return size_;
+ }
+
+ void Truncate() {
+ MutexLock lock(&blocks_mutex_);
+ for (char*& block : blocks_) {
+ delete[] block;
+ }
+ blocks_.clear();
+ size_ = 0;
+ }
+
+ Status Read(uint64_t offset, size_t n, Slice* result, char* scratch) const {
+ MutexLock lock(&blocks_mutex_);
+ if (offset > size_) {
+ return Status::IOError("Offset greater than file size.");
+ }
+ const uint64_t available = size_ - offset;
+ if (n > available) {
+ n = static_cast<size_t>(available);
+ }
+ if (n == 0) {
+ *result = Slice();
+ return Status::OK();
+ }
+
+ assert(offset / kBlockSize <= std::numeric_limits<size_t>::max());
+ size_t block = static_cast<size_t>(offset / kBlockSize);
+ size_t block_offset = offset % kBlockSize;
+ size_t bytes_to_copy = n;
+ char* dst = scratch;
+
+ while (bytes_to_copy > 0) {
+ size_t avail = kBlockSize - block_offset;
+ if (avail > bytes_to_copy) {
+ avail = bytes_to_copy;
+ }
+ memcpy(dst, blocks_[block] + block_offset, avail);
+
+ bytes_to_copy -= avail;
+ dst += avail;
+ block++;
+ block_offset = 0;
+ }
+
+ *result = Slice(scratch, n);
+ return Status::OK();
+ }
+
+ Status Append(const Slice& data) {
+ const char* src = data.data();
+ size_t src_len = data.size();
+
+ MutexLock lock(&blocks_mutex_);
+ while (src_len > 0) {
+ size_t avail;
+ size_t offset = size_ % kBlockSize;
+
+ if (offset != 0) {
+ // There is some room in the last block.
+ avail = kBlockSize - offset;
+ } else {
+ // No room in the last block; push new one.
+ blocks_.push_back(new char[kBlockSize]);
+ avail = kBlockSize;
+ }
+
+ if (avail > src_len) {
+ avail = src_len;
+ }
+ memcpy(blocks_.back() + offset, src, avail);
+ src_len -= avail;
+ src += avail;
+ size_ += avail;
+ }
+
+ return Status::OK();
+ }
+
+ private:
+ enum { kBlockSize = 8 * 1024 };
+
+ // Private since only Unref() should be used to delete it.
+ ~FileState() { Truncate(); }
+
+ port::Mutex refs_mutex_;
+ int refs_ GUARDED_BY(refs_mutex_);
+
+ mutable port::Mutex blocks_mutex_;
+ std::vector<char*> blocks_ GUARDED_BY(blocks_mutex_);
+ uint64_t size_ GUARDED_BY(blocks_mutex_);
+};
+
+class SequentialFileImpl : public SequentialFile {
+ public:
+ explicit SequentialFileImpl(FileState* file) : file_(file), pos_(0) {
+ file_->Ref();
+ }
+
+ ~SequentialFileImpl() override { file_->Unref(); }
+
+ Status Read(size_t n, Slice* result, char* scratch) override {
+ Status s = file_->Read(pos_, n, result, scratch);
+ if (s.ok()) {
+ pos_ += result->size();
+ }
+ return s;
+ }
+
+ Status Skip(uint64_t n) override {
+ if (pos_ > file_->Size()) {
+ return Status::IOError("pos_ > file_->Size()");
+ }
+ const uint64_t available = file_->Size() - pos_;
+ if (n > available) {
+ n = available;
+ }
+ pos_ += n;
+ return Status::OK();
+ }
+
+ virtual std::string GetName() const override { return "[memenv]"; }
+ private:
+ FileState* file_;
+ uint64_t pos_;
+};
+
+class RandomAccessFileImpl : public RandomAccessFile {
+ public:
+ explicit RandomAccessFileImpl(FileState* file) : file_(file) { file_->Ref(); }
+
+ ~RandomAccessFileImpl() override { file_->Unref(); }
+
+ Status Read(uint64_t offset, size_t n, Slice* result,
+ char* scratch) const override {
+ return file_->Read(offset, n, result, scratch);
+ }
+
+ virtual std::string GetName() const override { return "[memenv]"; }
+ private:
+ FileState* file_;
+};
+
+class WritableFileImpl : public WritableFile {
+ public:
+ WritableFileImpl(FileState* file) : file_(file) { file_->Ref(); }
+
+ ~WritableFileImpl() override { file_->Unref(); }
+
+ Status Append(const Slice& data) override { return file_->Append(data); }
+
+ Status Close() override { return Status::OK(); }
+ Status Flush() override { return Status::OK(); }
+ Status Sync() override { return Status::OK(); }
+
+ virtual std::string GetName() const override { return "[memenv]"; }
+ private:
+ FileState* file_;
+};
+
+class NoOpLogger : public Logger {
+ public:
+ void Logv(const char* format, va_list ap) override {}
+};
+
+class InMemoryEnv : public EnvWrapper {
+ public:
+ explicit InMemoryEnv(Env* base_env) : EnvWrapper(base_env) {}
+
+ ~InMemoryEnv() override {
+ for (const auto& kvp : file_map_) {
+ kvp.second->Unref();
+ }
+ }
+
+ // Partial implementation of the Env interface.
+ Status NewSequentialFile(const std::string& fname,
+ SequentialFile** result) override {
+ MutexLock lock(&mutex_);
+ if (file_map_.find(fname) == file_map_.end()) {
+ *result = nullptr;
+ return Status::IOError(fname, "File not found");
+ }
+
+ *result = new SequentialFileImpl(file_map_[fname]);
+ return Status::OK();
+ }
+
+ Status NewRandomAccessFile(const std::string& fname,
+ RandomAccessFile** result) override {
+ MutexLock lock(&mutex_);
+ if (file_map_.find(fname) == file_map_.end()) {
+ *result = nullptr;
+ return Status::IOError(fname, "File not found");
+ }
+
+ *result = new RandomAccessFileImpl(file_map_[fname]);
+ return Status::OK();
+ }
+
+ Status NewWritableFile(const std::string& fname,
+ WritableFile** result) override {
+ MutexLock lock(&mutex_);
+ FileSystem::iterator it = file_map_.find(fname);
+
+ FileState* file;
+ if (it == file_map_.end()) {
+ // File is not currently open.
+ file = new FileState();
+ file->Ref();
+ file_map_[fname] = file;
+ } else {
+ file = it->second;
+ file->Truncate();
+ }
+
+ *result = new WritableFileImpl(file);
+ return Status::OK();
+ }
+
+ Status NewAppendableFile(const std::string& fname,
+ WritableFile** result) override {
+ MutexLock lock(&mutex_);
+ FileState** sptr = &file_map_[fname];
+ FileState* file = *sptr;
+ if (file == nullptr) {
+ file = new FileState();
+ file->Ref();
+ }
+ *result = new WritableFileImpl(file);
+ return Status::OK();
+ }
+
+ bool FileExists(const std::string& fname) override {
+ MutexLock lock(&mutex_);
+ return file_map_.find(fname) != file_map_.end();
+ }
+
+ Status GetChildren(const std::string& dir,
+ std::vector<std::string>* result) override {
+ MutexLock lock(&mutex_);
+ result->clear();
+
+ for (const auto& kvp : file_map_) {
+ const std::string& filename = kvp.first;
+
+ if (filename.size() >= dir.size() + 1 && filename[dir.size()] == '/' &&
+ Slice(filename).starts_with(Slice(dir))) {
+ result->push_back(filename.substr(dir.size() + 1));
+ }
+ }
+
+ return Status::OK();
+ }
+
+ void DeleteFileInternal(const std::string& fname)
+ EXCLUSIVE_LOCKS_REQUIRED(mutex_) {
+ if (file_map_.find(fname) == file_map_.end()) {
+ return;
+ }
+
+ file_map_[fname]->Unref();
+ file_map_.erase(fname);
+ }
+
+ Status DeleteFile(const std::string& fname) override {
+ MutexLock lock(&mutex_);
+ if (file_map_.find(fname) == file_map_.end()) {
+ return Status::IOError(fname, "File not found");
+ }
+
+ DeleteFileInternal(fname);
+ return Status::OK();
+ }
+
+ Status CreateDir(const std::string& dirname) override { return Status::OK(); }
+
+ Status DeleteDir(const std::string& dirname) override { return Status::OK(); }
+
+ Status GetFileSize(const std::string& fname, uint64_t* file_size) override {
+ MutexLock lock(&mutex_);
+ if (file_map_.find(fname) == file_map_.end()) {
+ return Status::IOError(fname, "File not found");
+ }
+
+ *file_size = file_map_[fname]->Size();
+ return Status::OK();
+ }
+
+ Status RenameFile(const std::string& src,
+ const std::string& target) override {
+ MutexLock lock(&mutex_);
+ if (file_map_.find(src) == file_map_.end()) {
+ return Status::IOError(src, "File not found");
+ }
+
+ DeleteFileInternal(target);
+ file_map_[target] = file_map_[src];
+ file_map_.erase(src);
+ return Status::OK();
+ }
+
+ Status LockFile(const std::string& fname, FileLock** lock) override {
+ *lock = new FileLock;
+ return Status::OK();
+ }
+
+ Status UnlockFile(FileLock* lock) override {
+ delete lock;
+ return Status::OK();
+ }
+
+ Status GetTestDirectory(std::string* path) override {
+ *path = "/test";
+ return Status::OK();
+ }
+
+ Status NewLogger(const std::string& fname, Logger** result) override {
+ *result = new NoOpLogger;
+ return Status::OK();
+ }
+
+ private:
+ // Map from filenames to FileState objects, representing a simple file system.
+ typedef std::map<std::string, FileState*> FileSystem;
+
+ port::Mutex mutex_;
+ FileSystem file_map_ GUARDED_BY(mutex_);
+};
+
+} // namespace
+
+Env* NewMemEnv(Env* base_env) { return new InMemoryEnv(base_env); }
+
+} // namespace leveldb
diff --git a/src/leveldb/helpers/memenv/memenv.h b/src/leveldb/helpers/memenv/memenv.h
new file mode 100644
index 0000000000..3d929e4c4e
--- /dev/null
+++ b/src/leveldb/helpers/memenv/memenv.h
@@ -0,0 +1,22 @@
+// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file. See the AUTHORS file for names of contributors.
+
+#ifndef STORAGE_LEVELDB_HELPERS_MEMENV_MEMENV_H_
+#define STORAGE_LEVELDB_HELPERS_MEMENV_MEMENV_H_
+
+#include "leveldb/export.h"
+
+namespace leveldb {
+
+class Env;
+
+// Returns a new environment that stores its data in memory and delegates
+// all non-file-storage tasks to base_env. The caller must delete the result
+// when it is no longer needed.
+// *base_env must remain live while the result is in use.
+LEVELDB_EXPORT Env* NewMemEnv(Env* base_env);
+
+} // namespace leveldb
+
+#endif // STORAGE_LEVELDB_HELPERS_MEMENV_MEMENV_H_
diff --git a/src/leveldb/helpers/memenv/memenv_test.cc b/src/leveldb/helpers/memenv/memenv_test.cc
new file mode 100644
index 0000000000..94ad06be68
--- /dev/null
+++ b/src/leveldb/helpers/memenv/memenv_test.cc
@@ -0,0 +1,259 @@
+// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file. See the AUTHORS file for names of contributors.
+
+#include "helpers/memenv/memenv.h"
+
+#include <string>
+#include <vector>
+
+#include "db/db_impl.h"
+#include "leveldb/db.h"
+#include "leveldb/env.h"
+#include "util/testharness.h"
+
+namespace leveldb {
+
+class MemEnvTest {
+ public:
+ MemEnvTest() : env_(NewMemEnv(Env::Default())) {}
+ ~MemEnvTest() { delete env_; }
+
+ Env* env_;
+};
+
+TEST(MemEnvTest, Basics) {
+ uint64_t file_size;
+ WritableFile* writable_file;
+ std::vector<std::string> children;
+
+ ASSERT_OK(env_->CreateDir("/dir"));
+
+ // Check that the directory is empty.
+ ASSERT_TRUE(!env_->FileExists("/dir/non_existent"));
+ ASSERT_TRUE(!env_->GetFileSize("/dir/non_existent", &file_size).ok());
+ ASSERT_OK(env_->GetChildren("/dir", &children));
+ ASSERT_EQ(0, children.size());
+
+ // Create a file.
+ ASSERT_OK(env_->NewWritableFile("/dir/f", &writable_file));
+ ASSERT_OK(env_->GetFileSize("/dir/f", &file_size));
+ ASSERT_EQ(0, file_size);
+ delete writable_file;
+
+ // Check that the file exists.
+ ASSERT_TRUE(env_->FileExists("/dir/f"));
+ ASSERT_OK(env_->GetFileSize("/dir/f", &file_size));
+ ASSERT_EQ(0, file_size);
+ ASSERT_OK(env_->GetChildren("/dir", &children));
+ ASSERT_EQ(1, children.size());
+ ASSERT_EQ("f", children[0]);
+
+ // Write to the file.
+ ASSERT_OK(env_->NewWritableFile("/dir/f", &writable_file));
+ ASSERT_OK(writable_file->Append("abc"));
+ delete writable_file;
+
+ // Check that append works.
+ ASSERT_OK(env_->NewAppendableFile("/dir/f", &writable_file));
+ ASSERT_OK(env_->GetFileSize("/dir/f", &file_size));
+ ASSERT_EQ(3, file_size);
+ ASSERT_OK(writable_file->Append("hello"));
+ delete writable_file;
+
+ // Check for expected size.
+ ASSERT_OK(env_->GetFileSize("/dir/f", &file_size));
+ ASSERT_EQ(8, file_size);
+
+ // Check that renaming works.
+ ASSERT_TRUE(!env_->RenameFile("/dir/non_existent", "/dir/g").ok());
+ ASSERT_OK(env_->RenameFile("/dir/f", "/dir/g"));
+ ASSERT_TRUE(!env_->FileExists("/dir/f"));
+ ASSERT_TRUE(env_->FileExists("/dir/g"));
+ ASSERT_OK(env_->GetFileSize("/dir/g", &file_size));
+ ASSERT_EQ(8, file_size);
+
+ // Check that opening non-existent file fails.
+ SequentialFile* seq_file;
+ RandomAccessFile* rand_file;
+ ASSERT_TRUE(!env_->NewSequentialFile("/dir/non_existent", &seq_file).ok());
+ ASSERT_TRUE(!seq_file);
+ ASSERT_TRUE(!env_->NewRandomAccessFile("/dir/non_existent", &rand_file).ok());
+ ASSERT_TRUE(!rand_file);
+
+ // Check that deleting works.
+ ASSERT_TRUE(!env_->DeleteFile("/dir/non_existent").ok());
+ ASSERT_OK(env_->DeleteFile("/dir/g"));
+ ASSERT_TRUE(!env_->FileExists("/dir/g"));
+ ASSERT_OK(env_->GetChildren("/dir", &children));
+ ASSERT_EQ(0, children.size());
+ ASSERT_OK(env_->DeleteDir("/dir"));
+}
+
+TEST(MemEnvTest, ReadWrite) {
+ WritableFile* writable_file;
+ SequentialFile* seq_file;
+ RandomAccessFile* rand_file;
+ Slice result;
+ char scratch[100];
+
+ ASSERT_OK(env_->CreateDir("/dir"));
+
+ ASSERT_OK(env_->NewWritableFile("/dir/f", &writable_file));
+ ASSERT_OK(writable_file->Append("hello "));
+ ASSERT_OK(writable_file->Append("world"));
+ delete writable_file;
+
+ // Read sequentially.
+ ASSERT_OK(env_->NewSequentialFile("/dir/f", &seq_file));
+ ASSERT_OK(seq_file->Read(5, &result, scratch)); // Read "hello".
+ ASSERT_EQ(0, result.compare("hello"));
+ ASSERT_OK(seq_file->Skip(1));
+ ASSERT_OK(seq_file->Read(1000, &result, scratch)); // Read "world".
+ ASSERT_EQ(0, result.compare("world"));
+ ASSERT_OK(seq_file->Read(1000, &result, scratch)); // Try reading past EOF.
+ ASSERT_EQ(0, result.size());
+ ASSERT_OK(seq_file->Skip(100)); // Try to skip past end of file.
+ ASSERT_OK(seq_file->Read(1000, &result, scratch));
+ ASSERT_EQ(0, result.size());
+ delete seq_file;
+
+ // Random reads.
+ ASSERT_OK(env_->NewRandomAccessFile("/dir/f", &rand_file));
+ ASSERT_OK(rand_file->Read(6, 5, &result, scratch)); // Read "world".
+ ASSERT_EQ(0, result.compare("world"));
+ ASSERT_OK(rand_file->Read(0, 5, &result, scratch)); // Read "hello".
+ ASSERT_EQ(0, result.compare("hello"));
+ ASSERT_OK(rand_file->Read(10, 100, &result, scratch)); // Read "d".
+ ASSERT_EQ(0, result.compare("d"));
+
+ // Too high offset.
+ ASSERT_TRUE(!rand_file->Read(1000, 5, &result, scratch).ok());
+ delete rand_file;
+}
+
+TEST(MemEnvTest, Locks) {
+ FileLock* lock;
+
+ // These are no-ops, but we test they return success.
+ ASSERT_OK(env_->LockFile("some file", &lock));
+ ASSERT_OK(env_->UnlockFile(lock));
+}
+
+TEST(MemEnvTest, Misc) {
+ std::string test_dir;
+ ASSERT_OK(env_->GetTestDirectory(&test_dir));
+ ASSERT_TRUE(!test_dir.empty());
+
+ WritableFile* writable_file;
+ ASSERT_OK(env_->NewWritableFile("/a/b", &writable_file));
+
+ // These are no-ops, but we test they return success.
+ ASSERT_OK(writable_file->Sync());
+ ASSERT_OK(writable_file->Flush());
+ ASSERT_OK(writable_file->Close());
+ delete writable_file;
+}
+
+TEST(MemEnvTest, LargeWrite) {
+ const size_t kWriteSize = 300 * 1024;
+ char* scratch = new char[kWriteSize * 2];
+
+ std::string write_data;
+ for (size_t i = 0; i < kWriteSize; ++i) {
+ write_data.append(1, static_cast<char>(i));
+ }
+
+ WritableFile* writable_file;
+ ASSERT_OK(env_->NewWritableFile("/dir/f", &writable_file));
+ ASSERT_OK(writable_file->Append("foo"));
+ ASSERT_OK(writable_file->Append(write_data));
+ delete writable_file;
+
+ SequentialFile* seq_file;
+ Slice result;
+ ASSERT_OK(env_->NewSequentialFile("/dir/f", &seq_file));
+ ASSERT_OK(seq_file->Read(3, &result, scratch)); // Read "foo".
+ ASSERT_EQ(0, result.compare("foo"));
+
+ size_t read = 0;
+ std::string read_data;
+ while (read < kWriteSize) {
+ ASSERT_OK(seq_file->Read(kWriteSize - read, &result, scratch));
+ read_data.append(result.data(), result.size());
+ read += result.size();
+ }
+ ASSERT_TRUE(write_data == read_data);
+ delete seq_file;
+ delete[] scratch;
+}
+
+TEST(MemEnvTest, OverwriteOpenFile) {
+ const char kWrite1Data[] = "Write #1 data";
+ const size_t kFileDataLen = sizeof(kWrite1Data) - 1;
+ const std::string kTestFileName = test::TmpDir() + "/leveldb-TestFile.dat";
+
+ ASSERT_OK(WriteStringToFile(env_, kWrite1Data, kTestFileName));
+
+ RandomAccessFile* rand_file;
+ ASSERT_OK(env_->NewRandomAccessFile(kTestFileName, &rand_file));
+
+ const char kWrite2Data[] = "Write #2 data";
+ ASSERT_OK(WriteStringToFile(env_, kWrite2Data, kTestFileName));
+
+ // Verify that overwriting an open file will result in the new file data
+ // being read from files opened before the write.
+ Slice result;
+ char scratch[kFileDataLen];
+ ASSERT_OK(rand_file->Read(0, kFileDataLen, &result, scratch));
+ ASSERT_EQ(0, result.compare(kWrite2Data));
+
+ delete rand_file;
+}
+
+TEST(MemEnvTest, DBTest) {
+ Options options;
+ options.create_if_missing = true;
+ options.env = env_;
+ DB* db;
+
+ const Slice keys[] = {Slice("aaa"), Slice("bbb"), Slice("ccc")};
+ const Slice vals[] = {Slice("foo"), Slice("bar"), Slice("baz")};
+
+ ASSERT_OK(DB::Open(options, "/dir/db", &db));
+ for (size_t i = 0; i < 3; ++i) {
+ ASSERT_OK(db->Put(WriteOptions(), keys[i], vals[i]));
+ }
+
+ for (size_t i = 0; i < 3; ++i) {
+ std::string res;
+ ASSERT_OK(db->Get(ReadOptions(), keys[i], &res));
+ ASSERT_TRUE(res == vals[i]);
+ }
+
+ Iterator* iterator = db->NewIterator(ReadOptions());
+ iterator->SeekToFirst();
+ for (size_t i = 0; i < 3; ++i) {
+ ASSERT_TRUE(iterator->Valid());
+ ASSERT_TRUE(keys[i] == iterator->key());
+ ASSERT_TRUE(vals[i] == iterator->value());
+ iterator->Next();
+ }
+ ASSERT_TRUE(!iterator->Valid());
+ delete iterator;
+
+ DBImpl* dbi = reinterpret_cast<DBImpl*>(db);
+ ASSERT_OK(dbi->TEST_CompactMemTable());
+
+ for (size_t i = 0; i < 3; ++i) {
+ std::string res;
+ ASSERT_OK(db->Get(ReadOptions(), keys[i], &res));
+ ASSERT_TRUE(res == vals[i]);
+ }
+
+ delete db;
+}
+
+} // namespace leveldb
+
+int main(int argc, char** argv) { return leveldb::test::RunAllTests(); }
generated by cgit v1.2.3 (git 2.26.2) at 2024-09-03 13:08:36 +0000