diff options
Diffstat (limited to 'src/leveldb/db')
44 files changed, 2515 insertions, 3323 deletions
diff --git a/src/leveldb/db/autocompact_test.cc b/src/leveldb/db/autocompact_test.cc index d20a2362c3..e6c97a05a6 100644 --- a/src/leveldb/db/autocompact_test.cc +++ b/src/leveldb/db/autocompact_test.cc @@ -2,9 +2,9 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. See the AUTHORS file for names of contributors. -#include "leveldb/db.h" #include "db/db_impl.h" #include "leveldb/cache.h" +#include "leveldb/db.h" #include "util/testharness.h" #include "util/testutil.h" @@ -12,11 +12,6 @@ namespace leveldb { class AutoCompactTest { public: - std::string dbname_; - Cache* tiny_cache_; - Options options_; - DB* db_; - AutoCompactTest() { dbname_ = test::TmpDir() + "/autocompact_test"; tiny_cache_ = NewLRUCache(100); @@ -47,6 +42,12 @@ class AutoCompactTest { } void DoReads(int n); + + private: + std::string dbname_; + Cache* tiny_cache_; + Options options_; + DB* db_; }; static const int kValueSize = 200 * 1024; @@ -81,17 +82,16 @@ void AutoCompactTest::DoReads(int n) { ASSERT_LT(read, 100) << "Taking too long to compact"; Iterator* iter = db_->NewIterator(ReadOptions()); for (iter->SeekToFirst(); - iter->Valid() && iter->key().ToString() < limit_key; - iter->Next()) { + iter->Valid() && iter->key().ToString() < limit_key; iter->Next()) { // Drop data } delete iter; // Wait a little bit to allow any triggered compactions to complete. Env::Default()->SleepForMicroseconds(1000000); uint64_t size = Size(Key(0), Key(n)); - fprintf(stderr, "iter %3d => %7.3f MB [other %7.3f MB]\n", - read+1, size/1048576.0, Size(Key(n), Key(kCount))/1048576.0); - if (size <= initial_size/10) { + fprintf(stderr, "iter %3d => %7.3f MB [other %7.3f MB]\n", read + 1, + size / 1048576.0, Size(Key(n), Key(kCount)) / 1048576.0); + if (size <= initial_size / 10) { break; } } @@ -100,19 +100,13 @@ void AutoCompactTest::DoReads(int n) { // is pretty much unchanged. const int64_t final_other_size = Size(Key(n), Key(kCount)); ASSERT_LE(final_other_size, initial_other_size + 1048576); - ASSERT_GE(final_other_size, initial_other_size/5 - 1048576); + ASSERT_GE(final_other_size, initial_other_size / 5 - 1048576); } -TEST(AutoCompactTest, ReadAll) { - DoReads(kCount); -} +TEST(AutoCompactTest, ReadAll) { DoReads(kCount); } -TEST(AutoCompactTest, ReadHalf) { - DoReads(kCount/2); -} +TEST(AutoCompactTest, ReadHalf) { DoReads(kCount / 2); } } // namespace leveldb -int main(int argc, char** argv) { - return leveldb::test::RunAllTests(); -} +int main(int argc, char** argv) { return leveldb::test::RunAllTests(); } diff --git a/src/leveldb/db/builder.cc b/src/leveldb/db/builder.cc index f419882197..9520ee4535 100644 --- a/src/leveldb/db/builder.cc +++ b/src/leveldb/db/builder.cc @@ -4,8 +4,8 @@ #include "db/builder.h" -#include "db/filename.h" #include "db/dbformat.h" +#include "db/filename.h" #include "db/table_cache.h" #include "db/version_edit.h" #include "leveldb/db.h" @@ -14,12 +14,8 @@ namespace leveldb { -Status BuildTable(const std::string& dbname, - Env* env, - const Options& options, - TableCache* table_cache, - Iterator* iter, - FileMetaData* meta) { +Status BuildTable(const std::string& dbname, Env* env, const Options& options, + TableCache* table_cache, Iterator* iter, FileMetaData* meta) { Status s; meta->file_size = 0; iter->SeekToFirst(); @@ -41,14 +37,10 @@ Status BuildTable(const std::string& dbname, } // Finish and check for builder errors + s = builder->Finish(); if (s.ok()) { - s = builder->Finish(); - if (s.ok()) { - meta->file_size = builder->FileSize(); - assert(meta->file_size > 0); - } - } else { - builder->Abandon(); + meta->file_size = builder->FileSize(); + assert(meta->file_size > 0); } delete builder; @@ -60,12 +52,11 @@ Status BuildTable(const std::string& dbname, s = file->Close(); } delete file; - file = NULL; + file = nullptr; if (s.ok()) { // Verify that the table is usable - Iterator* it = table_cache->NewIterator(ReadOptions(), - meta->number, + Iterator* it = table_cache->NewIterator(ReadOptions(), meta->number, meta->file_size); s = it->status(); delete it; diff --git a/src/leveldb/db/builder.h b/src/leveldb/db/builder.h index 62431fcf44..7bd0b8049b 100644 --- a/src/leveldb/db/builder.h +++ b/src/leveldb/db/builder.h @@ -22,12 +22,8 @@ class VersionEdit; // *meta will be filled with metadata about the generated table. // If no data is present in *iter, meta->file_size will be set to // zero, and no Table file will be produced. -extern Status BuildTable(const std::string& dbname, - Env* env, - const Options& options, - TableCache* table_cache, - Iterator* iter, - FileMetaData* meta); +Status BuildTable(const std::string& dbname, Env* env, const Options& options, + TableCache* table_cache, Iterator* iter, FileMetaData* meta); } // namespace leveldb diff --git a/src/leveldb/db/c.cc b/src/leveldb/db/c.cc index b23e3dcc9d..3a492f9ac5 100644 --- a/src/leveldb/db/c.cc +++ b/src/leveldb/db/c.cc @@ -4,10 +4,9 @@ #include "leveldb/c.h" -#include <stdlib.h> -#ifndef WIN32 -#include <unistd.h> -#endif +#include <cstdint> +#include <cstdlib> + #include "leveldb/cache.h" #include "leveldb/comparator.h" #include "leveldb/db.h" @@ -45,69 +44,72 @@ using leveldb::WriteOptions; extern "C" { -struct leveldb_t { DB* rep; }; -struct leveldb_iterator_t { Iterator* rep; }; -struct leveldb_writebatch_t { WriteBatch rep; }; -struct leveldb_snapshot_t { const Snapshot* rep; }; -struct leveldb_readoptions_t { ReadOptions rep; }; -struct leveldb_writeoptions_t { WriteOptions rep; }; -struct leveldb_options_t { Options rep; }; -struct leveldb_cache_t { Cache* rep; }; -struct leveldb_seqfile_t { SequentialFile* rep; }; -struct leveldb_randomfile_t { RandomAccessFile* rep; }; -struct leveldb_writablefile_t { WritableFile* rep; }; -struct leveldb_logger_t { Logger* rep; }; -struct leveldb_filelock_t { FileLock* rep; }; +struct leveldb_t { + DB* rep; +}; +struct leveldb_iterator_t { + Iterator* rep; +}; +struct leveldb_writebatch_t { + WriteBatch rep; +}; +struct leveldb_snapshot_t { + const Snapshot* rep; +}; +struct leveldb_readoptions_t { + ReadOptions rep; +}; +struct leveldb_writeoptions_t { + WriteOptions rep; +}; +struct leveldb_options_t { + Options rep; +}; +struct leveldb_cache_t { + Cache* rep; +}; +struct leveldb_seqfile_t { + SequentialFile* rep; +}; +struct leveldb_randomfile_t { + RandomAccessFile* rep; +}; +struct leveldb_writablefile_t { + WritableFile* rep; +}; +struct leveldb_logger_t { + Logger* rep; +}; +struct leveldb_filelock_t { + FileLock* rep; +}; struct leveldb_comparator_t : public Comparator { - void* state_; - void (*destructor_)(void*); - int (*compare_)( - void*, - const char* a, size_t alen, - const char* b, size_t blen); - const char* (*name_)(void*); + ~leveldb_comparator_t() override { (*destructor_)(state_); } - virtual ~leveldb_comparator_t() { - (*destructor_)(state_); - } - - virtual int Compare(const Slice& a, const Slice& b) const { + int Compare(const Slice& a, const Slice& b) const override { return (*compare_)(state_, a.data(), a.size(), b.data(), b.size()); } - virtual const char* Name() const { - return (*name_)(state_); - } + const char* Name() const override { return (*name_)(state_); } // No-ops since the C binding does not support key shortening methods. - virtual void FindShortestSeparator(std::string*, const Slice&) const { } - virtual void FindShortSuccessor(std::string* key) const { } -}; + void FindShortestSeparator(std::string*, const Slice&) const override {} + void FindShortSuccessor(std::string* key) const override {} -struct leveldb_filterpolicy_t : public FilterPolicy { void* state_; void (*destructor_)(void*); + int (*compare_)(void*, const char* a, size_t alen, const char* b, + size_t blen); const char* (*name_)(void*); - char* (*create_)( - void*, - const char* const* key_array, const size_t* key_length_array, - int num_keys, - size_t* filter_length); - unsigned char (*key_match_)( - void*, - const char* key, size_t length, - const char* filter, size_t filter_length); - - virtual ~leveldb_filterpolicy_t() { - (*destructor_)(state_); - } +}; - virtual const char* Name() const { - return (*name_)(state_); - } +struct leveldb_filterpolicy_t : public FilterPolicy { + ~leveldb_filterpolicy_t() override { (*destructor_)(state_); } - virtual void CreateFilter(const Slice* keys, int n, std::string* dst) const { + const char* Name() const override { return (*name_)(state_); } + + void CreateFilter(const Slice* keys, int n, std::string* dst) const override { std::vector<const char*> key_pointers(n); std::vector<size_t> key_sizes(n); for (int i = 0; i < n; i++) { @@ -120,10 +122,19 @@ struct leveldb_filterpolicy_t : public FilterPolicy { free(filter); } - virtual bool KeyMayMatch(const Slice& key, const Slice& filter) const { - return (*key_match_)(state_, key.data(), key.size(), - filter.data(), filter.size()); + bool KeyMayMatch(const Slice& key, const Slice& filter) const override { + return (*key_match_)(state_, key.data(), key.size(), filter.data(), + filter.size()); } + + void* state_; + void (*destructor_)(void*); + const char* (*name_)(void*); + char* (*create_)(void*, const char* const* key_array, + const size_t* key_length_array, int num_keys, + size_t* filter_length); + uint8_t (*key_match_)(void*, const char* key, size_t length, + const char* filter, size_t filter_length); }; struct leveldb_env_t { @@ -132,10 +143,10 @@ struct leveldb_env_t { }; static bool SaveError(char** errptr, const Status& s) { - assert(errptr != NULL); + assert(errptr != nullptr); if (s.ok()) { return false; - } else if (*errptr == NULL) { + } else if (*errptr == nullptr) { *errptr = strdup(s.ToString().c_str()); } else { // TODO(sanjay): Merge with existing error? @@ -151,13 +162,11 @@ static char* CopyString(const std::string& str) { return result; } -leveldb_t* leveldb_open( - const leveldb_options_t* options, - const char* name, - char** errptr) { +leveldb_t* leveldb_open(const leveldb_options_t* options, const char* name, + char** errptr) { DB* db; if (SaveError(errptr, DB::Open(options->rep, std::string(name), &db))) { - return NULL; + return nullptr; } leveldb_t* result = new leveldb_t; result->rep = db; @@ -169,40 +178,27 @@ void leveldb_close(leveldb_t* db) { delete db; } -void leveldb_put( - leveldb_t* db, - const leveldb_writeoptions_t* options, - const char* key, size_t keylen, - const char* val, size_t vallen, - char** errptr) { +void leveldb_put(leveldb_t* db, const leveldb_writeoptions_t* options, + const char* key, size_t keylen, const char* val, size_t vallen, + char** errptr) { SaveError(errptr, db->rep->Put(options->rep, Slice(key, keylen), Slice(val, vallen))); } -void leveldb_delete( - leveldb_t* db, - const leveldb_writeoptions_t* options, - const char* key, size_t keylen, - char** errptr) { +void leveldb_delete(leveldb_t* db, const leveldb_writeoptions_t* options, + const char* key, size_t keylen, char** errptr) { SaveError(errptr, db->rep->Delete(options->rep, Slice(key, keylen))); } - -void leveldb_write( - leveldb_t* db, - const leveldb_writeoptions_t* options, - leveldb_writebatch_t* batch, - char** errptr) { +void leveldb_write(leveldb_t* db, const leveldb_writeoptions_t* options, + leveldb_writebatch_t* batch, char** errptr) { SaveError(errptr, db->rep->Write(options->rep, &batch->rep)); } -char* leveldb_get( - leveldb_t* db, - const leveldb_readoptions_t* options, - const char* key, size_t keylen, - size_t* vallen, - char** errptr) { - char* result = NULL; +char* leveldb_get(leveldb_t* db, const leveldb_readoptions_t* options, + const char* key, size_t keylen, size_t* vallen, + char** errptr) { + char* result = nullptr; std::string tmp; Status s = db->rep->Get(options->rep, Slice(key, keylen), &tmp); if (s.ok()) { @@ -218,45 +214,40 @@ char* leveldb_get( } leveldb_iterator_t* leveldb_create_iterator( - leveldb_t* db, - const leveldb_readoptions_t* options) { + leveldb_t* db, const leveldb_readoptions_t* options) { leveldb_iterator_t* result = new leveldb_iterator_t; result->rep = db->rep->NewIterator(options->rep); return result; } -const leveldb_snapshot_t* leveldb_create_snapshot( - leveldb_t* db) { +const leveldb_snapshot_t* leveldb_create_snapshot(leveldb_t* db) { leveldb_snapshot_t* result = new leveldb_snapshot_t; result->rep = db->rep->GetSnapshot(); return result; } -void leveldb_release_snapshot( - leveldb_t* db, - const leveldb_snapshot_t* snapshot) { +void leveldb_release_snapshot(leveldb_t* db, + const leveldb_snapshot_t* snapshot) { db->rep->ReleaseSnapshot(snapshot->rep); delete snapshot; } -char* leveldb_property_value( - leveldb_t* db, - const char* propname) { +char* leveldb_property_value(leveldb_t* db, const char* propname) { std::string tmp; if (db->rep->GetProperty(Slice(propname), &tmp)) { // We use strdup() since we expect human readable output. return strdup(tmp.c_str()); } else { - return NULL; + return nullptr; } } -void leveldb_approximate_sizes( - leveldb_t* db, - int num_ranges, - const char* const* range_start_key, const size_t* range_start_key_len, - const char* const* range_limit_key, const size_t* range_limit_key_len, - uint64_t* sizes) { +void leveldb_approximate_sizes(leveldb_t* db, int num_ranges, + const char* const* range_start_key, + const size_t* range_start_key_len, + const char* const* range_limit_key, + const size_t* range_limit_key_len, + uint64_t* sizes) { Range* ranges = new Range[num_ranges]; for (int i = 0; i < num_ranges; i++) { ranges[i].start = Slice(range_start_key[i], range_start_key_len[i]); @@ -266,28 +257,23 @@ void leveldb_approximate_sizes( delete[] ranges; } -void leveldb_compact_range( - leveldb_t* db, - const char* start_key, size_t start_key_len, - const char* limit_key, size_t limit_key_len) { +void leveldb_compact_range(leveldb_t* db, const char* start_key, + size_t start_key_len, const char* limit_key, + size_t limit_key_len) { Slice a, b; db->rep->CompactRange( - // Pass NULL Slice if corresponding "const char*" is NULL - (start_key ? (a = Slice(start_key, start_key_len), &a) : NULL), - (limit_key ? (b = Slice(limit_key, limit_key_len), &b) : NULL)); + // Pass null Slice if corresponding "const char*" is null + (start_key ? (a = Slice(start_key, start_key_len), &a) : nullptr), + (limit_key ? (b = Slice(limit_key, limit_key_len), &b) : nullptr)); } -void leveldb_destroy_db( - const leveldb_options_t* options, - const char* name, - char** errptr) { +void leveldb_destroy_db(const leveldb_options_t* options, const char* name, + char** errptr) { SaveError(errptr, DestroyDB(name, options->rep)); } -void leveldb_repair_db( - const leveldb_options_t* options, - const char* name, - char** errptr) { +void leveldb_repair_db(const leveldb_options_t* options, const char* name, + char** errptr) { SaveError(errptr, RepairDB(name, options->rep)); } @@ -296,7 +282,7 @@ void leveldb_iter_destroy(leveldb_iterator_t* iter) { delete iter; } -unsigned char leveldb_iter_valid(const leveldb_iterator_t* iter) { +uint8_t leveldb_iter_valid(const leveldb_iterator_t* iter) { return iter->rep->Valid(); } @@ -312,13 +298,9 @@ void leveldb_iter_seek(leveldb_iterator_t* iter, const char* k, size_t klen) { iter->rep->Seek(Slice(k, klen)); } -void leveldb_iter_next(leveldb_iterator_t* iter) { - iter->rep->Next(); -} +void leveldb_iter_next(leveldb_iterator_t* iter) { iter->rep->Next(); } -void leveldb_iter_prev(leveldb_iterator_t* iter) { - iter->rep->Prev(); -} +void leveldb_iter_prev(leveldb_iterator_t* iter) { iter->rep->Prev(); } const char* leveldb_iter_key(const leveldb_iterator_t* iter, size_t* klen) { Slice s = iter->rep->key(); @@ -340,41 +322,34 @@ leveldb_writebatch_t* leveldb_writebatch_create() { return new leveldb_writebatch_t; } -void leveldb_writebatch_destroy(leveldb_writebatch_t* b) { - delete b; -} +void leveldb_writebatch_destroy(leveldb_writebatch_t* b) { delete b; } -void leveldb_writebatch_clear(leveldb_writebatch_t* b) { - b->rep.Clear(); -} +void leveldb_writebatch_clear(leveldb_writebatch_t* b) { b->rep.Clear(); } -void leveldb_writebatch_put( - leveldb_writebatch_t* b, - const char* key, size_t klen, - const char* val, size_t vlen) { +void leveldb_writebatch_put(leveldb_writebatch_t* b, const char* key, + size_t klen, const char* val, size_t vlen) { b->rep.Put(Slice(key, klen), Slice(val, vlen)); } -void leveldb_writebatch_delete( - leveldb_writebatch_t* b, - const char* key, size_t klen) { +void leveldb_writebatch_delete(leveldb_writebatch_t* b, const char* key, + size_t klen) { b->rep.Delete(Slice(key, klen)); } -void leveldb_writebatch_iterate( - leveldb_writebatch_t* b, - void* state, - void (*put)(void*, const char* k, size_t klen, const char* v, size_t vlen), - void (*deleted)(void*, const char* k, size_t klen)) { +void leveldb_writebatch_iterate(const leveldb_writebatch_t* b, void* state, + void (*put)(void*, const char* k, size_t klen, + const char* v, size_t vlen), + void (*deleted)(void*, const char* k, + size_t klen)) { class H : public WriteBatch::Handler { public: void* state_; void (*put_)(void*, const char* k, size_t klen, const char* v, size_t vlen); void (*deleted_)(void*, const char* k, size_t klen); - virtual void Put(const Slice& key, const Slice& value) { + void Put(const Slice& key, const Slice& value) override { (*put_)(state_, key.data(), key.size(), value.data(), value.size()); } - virtual void Delete(const Slice& key) { + void Delete(const Slice& key) override { (*deleted_)(state_, key.data(), key.size()); } }; @@ -385,47 +360,43 @@ void leveldb_writebatch_iterate( b->rep.Iterate(&handler); } -leveldb_options_t* leveldb_options_create() { - return new leveldb_options_t; +void leveldb_writebatch_append(leveldb_writebatch_t* destination, + const leveldb_writebatch_t* source) { + destination->rep.Append(source->rep); } -void leveldb_options_destroy(leveldb_options_t* options) { - delete options; -} +leveldb_options_t* leveldb_options_create() { return new leveldb_options_t; } + +void leveldb_options_destroy(leveldb_options_t* options) { delete options; } -void leveldb_options_set_comparator( - leveldb_options_t* opt, - leveldb_comparator_t* cmp) { +void leveldb_options_set_comparator(leveldb_options_t* opt, + leveldb_comparator_t* cmp) { opt->rep.comparator = cmp; } -void leveldb_options_set_filter_policy( - leveldb_options_t* opt, - leveldb_filterpolicy_t* policy) { +void leveldb_options_set_filter_policy(leveldb_options_t* opt, + leveldb_filterpolicy_t* policy) { opt->rep.filter_policy = policy; } -void leveldb_options_set_create_if_missing( - leveldb_options_t* opt, unsigned char v) { +void leveldb_options_set_create_if_missing(leveldb_options_t* opt, uint8_t v) { opt->rep.create_if_missing = v; } -void leveldb_options_set_error_if_exists( - leveldb_options_t* opt, unsigned char v) { +void leveldb_options_set_error_if_exists(leveldb_options_t* opt, uint8_t v) { opt->rep.error_if_exists = v; } -void leveldb_options_set_paranoid_checks( - leveldb_options_t* opt, unsigned char v) { +void leveldb_options_set_paranoid_checks(leveldb_options_t* opt, uint8_t v) { opt->rep.paranoid_checks = v; } void leveldb_options_set_env(leveldb_options_t* opt, leveldb_env_t* env) { - opt->rep.env = (env ? env->rep : NULL); + opt->rep.env = (env ? env->rep : nullptr); } void leveldb_options_set_info_log(leveldb_options_t* opt, leveldb_logger_t* l) { - opt->rep.info_log = (l ? l->rep : NULL); + opt->rep.info_log = (l ? l->rep : nullptr); } void leveldb_options_set_write_buffer_size(leveldb_options_t* opt, size_t s) { @@ -448,17 +419,18 @@ void leveldb_options_set_block_restart_interval(leveldb_options_t* opt, int n) { opt->rep.block_restart_interval = n; } +void leveldb_options_set_max_file_size(leveldb_options_t* opt, size_t s) { + opt->rep.max_file_size = s; +} + void leveldb_options_set_compression(leveldb_options_t* opt, int t) { opt->rep.compression = static_cast<CompressionType>(t); } leveldb_comparator_t* leveldb_comparator_create( - void* state, - void (*destructor)(void*), - int (*compare)( - void*, - const char* a, size_t alen, - const char* b, size_t blen), + void* state, void (*destructor)(void*), + int (*compare)(void*, const char* a, size_t alen, const char* b, + size_t blen), const char* (*name)(void*)) { leveldb_comparator_t* result = new leveldb_comparator_t; result->state_ = state; @@ -468,22 +440,15 @@ leveldb_comparator_t* leveldb_comparator_create( return result; } -void leveldb_comparator_destroy(leveldb_comparator_t* cmp) { - delete cmp; -} +void leveldb_comparator_destroy(leveldb_comparator_t* cmp) { delete cmp; } leveldb_filterpolicy_t* leveldb_filterpolicy_create( - void* state, - void (*destructor)(void*), - char* (*create_filter)( - void*, - const char* const* key_array, const size_t* key_length_array, - int num_keys, - size_t* filter_length), - unsigned char (*key_may_match)( - void*, - const char* key, size_t length, - const char* filter, size_t filter_length), + void* state, void (*destructor)(void*), + char* (*create_filter)(void*, const char* const* key_array, + const size_t* key_length_array, int num_keys, + size_t* filter_length), + uint8_t (*key_may_match)(void*, const char* key, size_t length, + const char* filter, size_t filter_length), const char* (*name)(void*)) { leveldb_filterpolicy_t* result = new leveldb_filterpolicy_t; result->state_ = state; @@ -503,7 +468,8 @@ leveldb_filterpolicy_t* leveldb_filterpolicy_create_bloom(int bits_per_key) { // they delegate to a NewBloomFilterPolicy() instead of user // supplied C functions. struct Wrapper : public leveldb_filterpolicy_t { - const FilterPolicy* rep_; + static void DoNothing(void*) {} + ~Wrapper() { delete rep_; } const char* Name() const { return rep_->Name(); } void CreateFilter(const Slice* keys, int n, std::string* dst) const { @@ -512,11 +478,12 @@ leveldb_filterpolicy_t* leveldb_filterpolicy_create_bloom(int bits_per_key) { bool KeyMayMatch(const Slice& key, const Slice& filter) const { return rep_->KeyMayMatch(key, filter); } - static void DoNothing(void*) { } + + const FilterPolicy* rep_; }; Wrapper* wrapper = new Wrapper; wrapper->rep_ = NewBloomFilterPolicy(bits_per_key); - wrapper->state_ = NULL; + wrapper->state_ = nullptr; wrapper->destructor_ = &Wrapper::DoNothing; return wrapper; } @@ -525,37 +492,29 @@ leveldb_readoptions_t* leveldb_readoptions_create() { return new leveldb_readoptions_t; } -void leveldb_readoptions_destroy(leveldb_readoptions_t* opt) { - delete opt; -} +void leveldb_readoptions_destroy(leveldb_readoptions_t* opt) { delete opt; } -void leveldb_readoptions_set_verify_checksums( - leveldb_readoptions_t* opt, - unsigned char v) { +void leveldb_readoptions_set_verify_checksums(leveldb_readoptions_t* opt, + uint8_t v) { opt->rep.verify_checksums = v; } -void leveldb_readoptions_set_fill_cache( - leveldb_readoptions_t* opt, unsigned char v) { +void leveldb_readoptions_set_fill_cache(leveldb_readoptions_t* opt, uint8_t v) { opt->rep.fill_cache = v; } -void leveldb_readoptions_set_snapshot( - leveldb_readoptions_t* opt, - const leveldb_snapshot_t* snap) { - opt->rep.snapshot = (snap ? snap->rep : NULL); +void leveldb_readoptions_set_snapshot(leveldb_readoptions_t* opt, + const leveldb_snapshot_t* snap) { + opt->rep.snapshot = (snap ? snap->rep : nullptr); } leveldb_writeoptions_t* leveldb_writeoptions_create() { return new leveldb_writeoptions_t; } -void leveldb_writeoptions_destroy(leveldb_writeoptions_t* opt) { - delete opt; -} +void leveldb_writeoptions_destroy(leveldb_writeoptions_t* opt) { delete opt; } -void leveldb_writeoptions_set_sync( - leveldb_writeoptions_t* opt, unsigned char v) { +void leveldb_writeoptions_set_sync(leveldb_writeoptions_t* opt, uint8_t v) { opt->rep.sync = v; } @@ -582,16 +541,22 @@ void leveldb_env_destroy(leveldb_env_t* env) { delete env; } -void leveldb_free(void* ptr) { - free(ptr); -} +char* leveldb_env_get_test_directory(leveldb_env_t* env) { + std::string result; + if (!env->rep->GetTestDirectory(&result).ok()) { + return nullptr; + } -int leveldb_major_version() { - return kMajorVersion; + char* buffer = static_cast<char*>(malloc(result.size() + 1)); + memcpy(buffer, result.data(), result.size()); + buffer[result.size()] = '\0'; + return buffer; } -int leveldb_minor_version() { - return kMinorVersion; -} +void leveldb_free(void* ptr) { free(ptr); } + +int leveldb_major_version() { return kMajorVersion; } + +int leveldb_minor_version() { return kMinorVersion; } } // end extern "C" diff --git a/src/leveldb/db/c_test.c b/src/leveldb/db/c_test.c index 7cd5ee0207..16c77eed6a 100644 --- a/src/leveldb/db/c_test.c +++ b/src/leveldb/db/c_test.c @@ -8,24 +8,14 @@ #include <stdio.h> #include <stdlib.h> #include <string.h> -#include <sys/types.h> -#include <unistd.h> const char* phase = ""; -static char dbname[200]; static void StartPhase(const char* name) { fprintf(stderr, "=== Test %s\n", name); phase = name; } -static const char* GetTempDir(void) { - const char* ret = getenv("TEST_TMPDIR"); - if (ret == NULL || ret[0] == '\0') - ret = "/tmp"; - return ret; -} - #define CheckNoError(err) \ if ((err) != NULL) { \ fprintf(stderr, "%s:%d: %s: %s\n", __FILE__, __LINE__, phase, (err)); \ @@ -130,7 +120,7 @@ static const char* CmpName(void* arg) { } // Custom filter policy -static unsigned char fake_filter_result = 1; +static uint8_t fake_filter_result = 1; static void FilterDestroy(void* arg) { } static const char* FilterName(void* arg) { return "TestFilter"; @@ -145,10 +135,8 @@ static char* FilterCreate( memcpy(result, "fake", 4); return result; } -unsigned char FilterKeyMatch( - void* arg, - const char* key, size_t length, - const char* filter, size_t filter_length) { +uint8_t FilterKeyMatch(void* arg, const char* key, size_t length, + const char* filter, size_t filter_length) { CheckCondition(filter_length == 4); CheckCondition(memcmp(filter, "fake", 4) == 0); return fake_filter_result; @@ -162,21 +150,19 @@ int main(int argc, char** argv) { leveldb_options_t* options; leveldb_readoptions_t* roptions; leveldb_writeoptions_t* woptions; + char* dbname; char* err = NULL; int run = -1; CheckCondition(leveldb_major_version() >= 1); CheckCondition(leveldb_minor_version() >= 1); - snprintf(dbname, sizeof(dbname), - "%s/leveldb_c_test-%d", - GetTempDir(), - ((int) geteuid())); - StartPhase("create_objects"); cmp = leveldb_comparator_create(NULL, CmpDestroy, CmpCompare, CmpName); env = leveldb_create_default_env(); cache = leveldb_cache_create_lru(100000); + dbname = leveldb_env_get_test_directory(env); + CheckCondition(dbname != NULL); options = leveldb_options_create(); leveldb_options_set_comparator(options, cmp); @@ -189,6 +175,7 @@ int main(int argc, char** argv) { leveldb_options_set_max_open_files(options, 10); leveldb_options_set_block_size(options, 1024); leveldb_options_set_block_restart_interval(options, 8); + leveldb_options_set_max_file_size(options, 3 << 20); leveldb_options_set_compression(options, leveldb_no_compression); roptions = leveldb_readoptions_create(); @@ -239,12 +226,18 @@ int main(int argc, char** argv) { leveldb_writebatch_clear(wb); leveldb_writebatch_put(wb, "bar", 3, "b", 1); leveldb_writebatch_put(wb, "box", 3, "c", 1); - leveldb_writebatch_delete(wb, "bar", 3); + + leveldb_writebatch_t* wb2 = leveldb_writebatch_create(); + leveldb_writebatch_delete(wb2, "bar", 3); + leveldb_writebatch_append(wb, wb2); + leveldb_writebatch_destroy(wb2); + leveldb_write(db, woptions, wb, &err); CheckNoError(err); CheckGet(db, roptions, "foo", "hello"); CheckGet(db, roptions, "bar", NULL); CheckGet(db, roptions, "box", "c"); + int pos = 0; leveldb_writebatch_iterate(wb, &pos, CheckPut, CheckDel); CheckCondition(pos == 3); @@ -381,6 +374,7 @@ int main(int argc, char** argv) { leveldb_options_destroy(options); leveldb_readoptions_destroy(roptions); leveldb_writeoptions_destroy(woptions); + leveldb_free(dbname); leveldb_cache_destroy(cache); leveldb_comparator_destroy(cmp); leveldb_env_destroy(env); diff --git a/src/leveldb/db/corruption_test.cc b/src/leveldb/db/corruption_test.cc index 37a484d25f..42f5237c65 100644 --- a/src/leveldb/db/corruption_test.cc +++ b/src/leveldb/db/corruption_test.cc @@ -2,20 +2,16 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. See the AUTHORS file for names of contributors. -#include "leveldb/db.h" - -#include <errno.h> -#include <fcntl.h> -#include <sys/stat.h> #include <sys/types.h> -#include "leveldb/cache.h" -#include "leveldb/env.h" -#include "leveldb/table.h" -#include "leveldb/write_batch.h" + #include "db/db_impl.h" #include "db/filename.h" #include "db/log_format.h" #include "db/version_set.h" +#include "leveldb/cache.h" +#include "leveldb/db.h" +#include "leveldb/table.h" +#include "leveldb/write_batch.h" #include "util/logging.h" #include "util/testharness.h" #include "util/testutil.h" @@ -26,44 +22,35 @@ static const int kValueSize = 1000; class CorruptionTest { public: - test::ErrorEnv env_; - std::string dbname_; - Cache* tiny_cache_; - Options options_; - DB* db_; - - CorruptionTest() { - tiny_cache_ = NewLRUCache(100); + CorruptionTest() + : db_(nullptr), + dbname_("/memenv/corruption_test"), + tiny_cache_(NewLRUCache(100)) { options_.env = &env_; options_.block_cache = tiny_cache_; - dbname_ = test::TmpDir() + "/corruption_test"; DestroyDB(dbname_, options_); - db_ = NULL; options_.create_if_missing = true; Reopen(); options_.create_if_missing = false; } ~CorruptionTest() { - delete db_; - DestroyDB(dbname_, Options()); - delete tiny_cache_; + delete db_; + delete tiny_cache_; } Status TryReopen() { delete db_; - db_ = NULL; + db_ = nullptr; return DB::Open(options_, dbname_, &db_); } - void Reopen() { - ASSERT_OK(TryReopen()); - } + void Reopen() { ASSERT_OK(TryReopen()); } void RepairDB() { delete db_; - db_ = NULL; + db_ = nullptr; ASSERT_OK(::leveldb::RepairDB(dbname_, options_)); } @@ -71,7 +58,7 @@ class CorruptionTest { std::string key_space, value_space; WriteBatch batch; for (int i = 0; i < n; i++) { - //if ((i % 100) == 0) fprintf(stderr, "@ %d of %d\n", i, n); + // if ((i % 100) == 0) fprintf(stderr, "@ %d of %d\n", i, n); Slice key = Key(i, &key_space); batch.Clear(); batch.Put(key, Value(i, &value_space)); @@ -100,8 +87,7 @@ class CorruptionTest { // Ignore boundary keys. continue; } - if (!ConsumeDecimalNumber(&in, &key) || - !in.empty() || + if (!ConsumeDecimalNumber(&in, &key) || !in.empty() || key < next_expected) { bad_keys++; continue; @@ -126,14 +112,13 @@ class CorruptionTest { void Corrupt(FileType filetype, int offset, int bytes_to_corrupt) { // Pick file to corrupt std::vector<std::string> filenames; - ASSERT_OK(env_.GetChildren(dbname_, &filenames)); + ASSERT_OK(env_.target()->GetChildren(dbname_, &filenames)); uint64_t number; FileType type; std::string fname; int picked_number = -1; for (size_t i = 0; i < filenames.size(); i++) { - if (ParseFileName(filenames[i], &number, &type) && - type == filetype && + if (ParseFileName(filenames[i], &number, &type) && type == filetype && int(number) > picked_number) { // Pick latest file fname = dbname_ + "/" + filenames[i]; picked_number = number; @@ -141,35 +126,32 @@ class CorruptionTest { } ASSERT_TRUE(!fname.empty()) << filetype; - struct stat sbuf; - if (stat(fname.c_str(), &sbuf) != 0) { - const char* msg = strerror(errno); - ASSERT_TRUE(false) << fname << ": " << msg; - } + uint64_t file_size; + ASSERT_OK(env_.target()->GetFileSize(fname, &file_size)); if (offset < 0) { // Relative to end of file; make it absolute - if (-offset > sbuf.st_size) { + if (-offset > file_size) { offset = 0; } else { - offset = sbuf.st_size + offset; + offset = file_size + offset; } } - if (offset > sbuf.st_size) { - offset = sbuf.st_size; + if (offset > file_size) { + offset = file_size; } - if (offset + bytes_to_corrupt > sbuf.st_size) { - bytes_to_corrupt = sbuf.st_size - offset; + if (offset + bytes_to_corrupt > file_size) { + bytes_to_corrupt = file_size - offset; } // Do it std::string contents; - Status s = ReadFileToString(Env::Default(), fname, &contents); + Status s = ReadFileToString(env_.target(), fname, &contents); ASSERT_TRUE(s.ok()) << s.ToString(); for (int i = 0; i < bytes_to_corrupt; i++) { contents[i + offset] ^= 0x80; } - s = WriteStringToFile(Env::Default(), contents, fname); + s = WriteStringToFile(env_.target(), contents, fname); ASSERT_TRUE(s.ok()) << s.ToString(); } @@ -197,12 +179,20 @@ class CorruptionTest { Random r(k); return test::RandomString(&r, kValueSize, storage); } + + test::ErrorEnv env_; + Options options_; + DB* db_; + + private: + std::string dbname_; + Cache* tiny_cache_; }; TEST(CorruptionTest, Recovery) { Build(100); Check(100, 100); - Corrupt(kLogFile, 19, 1); // WriteBatch tag for first record + Corrupt(kLogFile, 19, 1); // WriteBatch tag for first record Corrupt(kLogFile, log::kBlockSize + 1000, 1); // Somewhere in second block Reopen(); @@ -237,8 +227,8 @@ TEST(CorruptionTest, TableFile) { Build(100); DBImpl* dbi = reinterpret_cast<DBImpl*>(db_); dbi->TEST_CompactMemTable(); - dbi->TEST_CompactRange(0, NULL, NULL); - dbi->TEST_CompactRange(1, NULL, NULL); + dbi->TEST_CompactRange(0, nullptr, nullptr); + dbi->TEST_CompactRange(1, nullptr, nullptr); Corrupt(kTableFile, 100, 1); Check(90, 99); @@ -251,8 +241,8 @@ TEST(CorruptionTest, TableFileRepair) { Build(100); DBImpl* dbi = reinterpret_cast<DBImpl*>(db_); dbi->TEST_CompactMemTable(); - dbi->TEST_CompactRange(0, NULL, NULL); - dbi->TEST_CompactRange(1, NULL, NULL); + dbi->TEST_CompactRange(0, nullptr, nullptr); + dbi->TEST_CompactRange(1, nullptr, nullptr); Corrupt(kTableFile, 100, 1); RepairDB(); @@ -302,7 +292,7 @@ TEST(CorruptionTest, CorruptedDescriptor) { ASSERT_OK(db_->Put(WriteOptions(), "foo", "hello")); DBImpl* dbi = reinterpret_cast<DBImpl*>(db_); dbi->TEST_CompactMemTable(); - dbi->TEST_CompactRange(0, NULL, NULL); + dbi->TEST_CompactRange(0, nullptr, nullptr); Corrupt(kDescriptorFile, 0, 1000); Status s = TryReopen(); @@ -343,7 +333,7 @@ TEST(CorruptionTest, CompactionInputErrorParanoid) { Corrupt(kTableFile, 100, 1); env_.SleepForMicroseconds(100000); } - dbi->CompactRange(NULL, NULL); + dbi->CompactRange(nullptr, nullptr); // Write must fail because of corrupted table std::string tmp1, tmp2; @@ -369,6 +359,4 @@ TEST(CorruptionTest, UnrelatedKeys) { } // namespace leveldb -int main(int argc, char** argv) { - return leveldb::test::RunAllTests(); -} +int main(int argc, char** argv) { return leveldb::test::RunAllTests(); } diff --git a/src/leveldb/db/db_bench.cc b/src/leveldb/db/db_bench.cc deleted file mode 100644 index 3ad19a512b..0000000000 --- a/src/leveldb/db/db_bench.cc +++ /dev/null @@ -1,1020 +0,0 @@ -// Copyright (c) 2011 The LevelDB Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. See the AUTHORS file for names of contributors. - -#include <sys/types.h> -#include <stdio.h> -#include <stdlib.h> -#include "db/db_impl.h" -#include "db/version_set.h" -#include "leveldb/cache.h" -#include "leveldb/db.h" -#include "leveldb/env.h" -#include "leveldb/write_batch.h" -#include "port/port.h" -#include "util/crc32c.h" -#include "util/histogram.h" -#include "util/mutexlock.h" -#include "util/random.h" -#include "util/testutil.h" - -// Comma-separated list of operations to run in the specified order -// Actual benchmarks: -// fillseq -- write N values in sequential key order in async mode -// fillrandom -- write N values in random key order in async mode -// overwrite -- overwrite N values in random key order in async mode -// fillsync -- write N/100 values in random key order in sync mode -// fill100K -- write N/1000 100K values in random order in async mode -// deleteseq -- delete N keys in sequential order -// deleterandom -- delete N keys in random order -// readseq -- read N times sequentially -// readreverse -- read N times in reverse order -// readrandom -- read N times in random order -// readmissing -- read N missing keys in random order -// readhot -- read N times in random order from 1% section of DB -// seekrandom -- N random seeks -// open -- cost of opening a DB -// crc32c -- repeated crc32c of 4K of data -// acquireload -- load N*1000 times -// Meta operations: -// compact -- Compact the entire DB -// stats -- Print DB stats -// sstables -- Print sstable info -// heapprofile -- Dump a heap profile (if supported by this port) -static const char* FLAGS_benchmarks = - "fillseq," - "fillsync," - "fillrandom," - "overwrite," - "readrandom," - "readrandom," // Extra run to allow previous compactions to quiesce - "readseq," - "readreverse," - "compact," - "readrandom," - "readseq," - "readreverse," - "fill100K," - "crc32c," - "snappycomp," - "snappyuncomp," - "acquireload," - ; - -// Number of key/values to place in database -static int FLAGS_num = 1000000; - -// Number of read operations to do. If negative, do FLAGS_num reads. -static int FLAGS_reads = -1; - -// Number of concurrent threads to run. -static int FLAGS_threads = 1; - -// Size of each value -static int FLAGS_value_size = 100; - -// Arrange to generate values that shrink to this fraction of -// their original size after compression -static double FLAGS_compression_ratio = 0.5; - -// Print histogram of operation timings -static bool FLAGS_histogram = false; - -// Number of bytes to buffer in memtable before compacting -// (initialized to default value by "main") -static int FLAGS_write_buffer_size = 0; - -// Number of bytes written to each file. -// (initialized to default value by "main") -static int FLAGS_max_file_size = 0; - -// Approximate size of user data packed per block (before compression. -// (initialized to default value by "main") -static int FLAGS_block_size = 0; - -// Number of bytes to use as a cache of uncompressed data. -// Negative means use default settings. -static int FLAGS_cache_size = -1; - -// Maximum number of files to keep open at the same time (use default if == 0) -static int FLAGS_open_files = 0; - -// Bloom filter bits per key. -// Negative means use default settings. -static int FLAGS_bloom_bits = -1; - -// If true, do not destroy the existing database. If you set this -// flag and also specify a benchmark that wants a fresh database, that -// benchmark will fail. -static bool FLAGS_use_existing_db = false; - -// If true, reuse existing log/MANIFEST files when re-opening a database. -static bool FLAGS_reuse_logs = false; - -// Use the db with the following name. -static const char* FLAGS_db = NULL; - -namespace leveldb { - -namespace { -leveldb::Env* g_env = NULL; - -// Helper for quickly generating random data. -class RandomGenerator { - private: - std::string data_; - int pos_; - - public: - RandomGenerator() { - // We use a limited amount of data over and over again and ensure - // that it is larger than the compression window (32KB), and also - // large enough to serve all typical value sizes we want to write. - Random rnd(301); - std::string piece; - while (data_.size() < 1048576) { - // Add a short fragment that is as compressible as specified - // by FLAGS_compression_ratio. - test::CompressibleString(&rnd, FLAGS_compression_ratio, 100, &piece); - data_.append(piece); - } - pos_ = 0; - } - - Slice Generate(size_t len) { - if (pos_ + len > data_.size()) { - pos_ = 0; - assert(len < data_.size()); - } - pos_ += len; - return Slice(data_.data() + pos_ - len, len); - } -}; - -#if defined(__linux) -static Slice TrimSpace(Slice s) { - size_t start = 0; - while (start < s.size() && isspace(s[start])) { - start++; - } - size_t limit = s.size(); - while (limit > start && isspace(s[limit-1])) { - limit--; - } - return Slice(s.data() + start, limit - start); -} -#endif - -static void AppendWithSpace(std::string* str, Slice msg) { - if (msg.empty()) return; - if (!str->empty()) { - str->push_back(' '); - } - str->append(msg.data(), msg.size()); -} - -class Stats { - private: - double start_; - double finish_; - double seconds_; - int done_; - int next_report_; - int64_t bytes_; - double last_op_finish_; - Histogram hist_; - std::string message_; - - public: - Stats() { Start(); } - - void Start() { - next_report_ = 100; - last_op_finish_ = start_; - hist_.Clear(); - done_ = 0; - bytes_ = 0; - seconds_ = 0; - start_ = g_env->NowMicros(); - finish_ = start_; - message_.clear(); - } - - void Merge(const Stats& other) { - hist_.Merge(other.hist_); - done_ += other.done_; - bytes_ += other.bytes_; - seconds_ += other.seconds_; - if (other.start_ < start_) start_ = other.start_; - if (other.finish_ > finish_) finish_ = other.finish_; - - // Just keep the messages from one thread - if (message_.empty()) message_ = other.message_; - } - - void Stop() { - finish_ = g_env->NowMicros(); - seconds_ = (finish_ - start_) * 1e-6; - } - - void AddMessage(Slice msg) { - AppendWithSpace(&message_, msg); - } - - void FinishedSingleOp() { - if (FLAGS_histogram) { - double now = g_env->NowMicros(); - double micros = now - last_op_finish_; - hist_.Add(micros); - if (micros > 20000) { - fprintf(stderr, "long op: %.1f micros%30s\r", micros, ""); - fflush(stderr); - } - last_op_finish_ = now; - } - - done_++; - if (done_ >= next_report_) { - if (next_report_ < 1000) next_report_ += 100; - else if (next_report_ < 5000) next_report_ += 500; - else if (next_report_ < 10000) next_report_ += 1000; - else if (next_report_ < 50000) next_report_ += 5000; - else if (next_report_ < 100000) next_report_ += 10000; - else if (next_report_ < 500000) next_report_ += 50000; - else next_report_ += 100000; - fprintf(stderr, "... finished %d ops%30s\r", done_, ""); - fflush(stderr); - } - } - - void AddBytes(int64_t n) { - bytes_ += n; - } - - void Report(const Slice& name) { - // Pretend at least one op was done in case we are running a benchmark - // that does not call FinishedSingleOp(). - if (done_ < 1) done_ = 1; - - std::string extra; - if (bytes_ > 0) { - // Rate is computed on actual elapsed time, not the sum of per-thread - // elapsed times. - double elapsed = (finish_ - start_) * 1e-6; - char rate[100]; - snprintf(rate, sizeof(rate), "%6.1f MB/s", - (bytes_ / 1048576.0) / elapsed); - extra = rate; - } - AppendWithSpace(&extra, message_); - - fprintf(stdout, "%-12s : %11.3f micros/op;%s%s\n", - name.ToString().c_str(), - seconds_ * 1e6 / done_, - (extra.empty() ? "" : " "), - extra.c_str()); - if (FLAGS_histogram) { - fprintf(stdout, "Microseconds per op:\n%s\n", hist_.ToString().c_str()); - } - fflush(stdout); - } -}; - -// State shared by all concurrent executions of the same benchmark. -struct SharedState { - port::Mutex mu; - port::CondVar cv; - int total; - - // Each thread goes through the following states: - // (1) initializing - // (2) waiting for others to be initialized - // (3) running - // (4) done - - int num_initialized; - int num_done; - bool start; - - SharedState() : cv(&mu) { } -}; - -// Per-thread state for concurrent executions of the same benchmark. -struct ThreadState { - int tid; // 0..n-1 when running in n threads - Random rand; // Has different seeds for different threads - Stats stats; - SharedState* shared; - - ThreadState(int index) - : tid(index), - rand(1000 + index) { - } -}; - -} // namespace - -class Benchmark { - private: - Cache* cache_; - const FilterPolicy* filter_policy_; - DB* db_; - int num_; - int value_size_; - int entries_per_batch_; - WriteOptions write_options_; - int reads_; - int heap_counter_; - - void PrintHeader() { - const int kKeySize = 16; - PrintEnvironment(); - fprintf(stdout, "Keys: %d bytes each\n", kKeySize); - fprintf(stdout, "Values: %d bytes each (%d bytes after compression)\n", - FLAGS_value_size, - static_cast<int>(FLAGS_value_size * FLAGS_compression_ratio + 0.5)); - fprintf(stdout, "Entries: %d\n", num_); - fprintf(stdout, "RawSize: %.1f MB (estimated)\n", - ((static_cast<int64_t>(kKeySize + FLAGS_value_size) * num_) - / 1048576.0)); - fprintf(stdout, "FileSize: %.1f MB (estimated)\n", - (((kKeySize + FLAGS_value_size * FLAGS_compression_ratio) * num_) - / 1048576.0)); - PrintWarnings(); - fprintf(stdout, "------------------------------------------------\n"); - } - - void PrintWarnings() { -#if defined(__GNUC__) && !defined(__OPTIMIZE__) - fprintf(stdout, - "WARNING: Optimization is disabled: benchmarks unnecessarily slow\n" - ); -#endif -#ifndef NDEBUG - fprintf(stdout, - "WARNING: Assertions are enabled; benchmarks unnecessarily slow\n"); -#endif - - // See if snappy is working by attempting to compress a compressible string - const char text[] = "yyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyy"; - std::string compressed; - if (!port::Snappy_Compress(text, sizeof(text), &compressed)) { - fprintf(stdout, "WARNING: Snappy compression is not enabled\n"); - } else if (compressed.size() >= sizeof(text)) { - fprintf(stdout, "WARNING: Snappy compression is not effective\n"); - } - } - - void PrintEnvironment() { - fprintf(stderr, "LevelDB: version %d.%d\n", - kMajorVersion, kMinorVersion); - -#if defined(__linux) - time_t now = time(NULL); - fprintf(stderr, "Date: %s", ctime(&now)); // ctime() adds newline - - FILE* cpuinfo = fopen("/proc/cpuinfo", "r"); - if (cpuinfo != NULL) { - char line[1000]; - int num_cpus = 0; - std::string cpu_type; - std::string cache_size; - while (fgets(line, sizeof(line), cpuinfo) != NULL) { - const char* sep = strchr(line, ':'); - if (sep == NULL) { - continue; - } - Slice key = TrimSpace(Slice(line, sep - 1 - line)); - Slice val = TrimSpace(Slice(sep + 1)); - if (key == "model name") { - ++num_cpus; - cpu_type = val.ToString(); - } else if (key == "cache size") { - cache_size = val.ToString(); - } - } - fclose(cpuinfo); - fprintf(stderr, "CPU: %d * %s\n", num_cpus, cpu_type.c_str()); - fprintf(stderr, "CPUCache: %s\n", cache_size.c_str()); - } -#endif - } - - public: - Benchmark() - : cache_(FLAGS_cache_size >= 0 ? NewLRUCache(FLAGS_cache_size) : NULL), - filter_policy_(FLAGS_bloom_bits >= 0 - ? NewBloomFilterPolicy(FLAGS_bloom_bits) - : NULL), - db_(NULL), - num_(FLAGS_num), - value_size_(FLAGS_value_size), - entries_per_batch_(1), - reads_(FLAGS_reads < 0 ? FLAGS_num : FLAGS_reads), - heap_counter_(0) { - std::vector<std::string> files; - g_env->GetChildren(FLAGS_db, &files); - for (size_t i = 0; i < files.size(); i++) { - if (Slice(files[i]).starts_with("heap-")) { - g_env->DeleteFile(std::string(FLAGS_db) + "/" + files[i]); - } - } - if (!FLAGS_use_existing_db) { - DestroyDB(FLAGS_db, Options()); - } - } - - ~Benchmark() { - delete db_; - delete cache_; - delete filter_policy_; - } - - void Run() { - PrintHeader(); - Open(); - - const char* benchmarks = FLAGS_benchmarks; - while (benchmarks != NULL) { - const char* sep = strchr(benchmarks, ','); - Slice name; - if (sep == NULL) { - name = benchmarks; - benchmarks = NULL; - } else { - name = Slice(benchmarks, sep - benchmarks); - benchmarks = sep + 1; - } - - // Reset parameters that may be overridden below - num_ = FLAGS_num; - reads_ = (FLAGS_reads < 0 ? FLAGS_num : FLAGS_reads); - value_size_ = FLAGS_value_size; - entries_per_batch_ = 1; - write_options_ = WriteOptions(); - - void (Benchmark::*method)(ThreadState*) = NULL; - bool fresh_db = false; - int num_threads = FLAGS_threads; - - if (name == Slice("open")) { - method = &Benchmark::OpenBench; - num_ /= 10000; - if (num_ < 1) num_ = 1; - } else if (name == Slice("fillseq")) { - fresh_db = true; - method = &Benchmark::WriteSeq; - } else if (name == Slice("fillbatch")) { - fresh_db = true; - entries_per_batch_ = 1000; - method = &Benchmark::WriteSeq; - } else if (name == Slice("fillrandom")) { - fresh_db = true; - method = &Benchmark::WriteRandom; - } else if (name == Slice("overwrite")) { - fresh_db = false; - method = &Benchmark::WriteRandom; - } else if (name == Slice("fillsync")) { - fresh_db = true; - num_ /= 1000; - write_options_.sync = true; - method = &Benchmark::WriteRandom; - } else if (name == Slice("fill100K")) { - fresh_db = true; - num_ /= 1000; - value_size_ = 100 * 1000; - method = &Benchmark::WriteRandom; - } else if (name == Slice("readseq")) { - method = &Benchmark::ReadSequential; - } else if (name == Slice("readreverse")) { - method = &Benchmark::ReadReverse; - } else if (name == Slice("readrandom")) { - method = &Benchmark::ReadRandom; - } else if (name == Slice("readmissing")) { - method = &Benchmark::ReadMissing; - } else if (name == Slice("seekrandom")) { - method = &Benchmark::SeekRandom; - } else if (name == Slice("readhot")) { - method = &Benchmark::ReadHot; - } else if (name == Slice("readrandomsmall")) { - reads_ /= 1000; - method = &Benchmark::ReadRandom; - } else if (name == Slice("deleteseq")) { - method = &Benchmark::DeleteSeq; - } else if (name == Slice("deleterandom")) { - method = &Benchmark::DeleteRandom; - } else if (name == Slice("readwhilewriting")) { - num_threads++; // Add extra thread for writing - method = &Benchmark::ReadWhileWriting; - } else if (name == Slice("compact")) { - method = &Benchmark::Compact; - } else if (name == Slice("crc32c")) { - method = &Benchmark::Crc32c; - } else if (name == Slice("acquireload")) { - method = &Benchmark::AcquireLoad; - } else if (name == Slice("snappycomp")) { - method = &Benchmark::SnappyCompress; - } else if (name == Slice("snappyuncomp")) { - method = &Benchmark::SnappyUncompress; - } else if (name == Slice("heapprofile")) { - HeapProfile(); - } else if (name == Slice("stats")) { - PrintStats("leveldb.stats"); - } else if (name == Slice("sstables")) { - PrintStats("leveldb.sstables"); - } else { - if (name != Slice()) { // No error message for empty name - fprintf(stderr, "unknown benchmark '%s'\n", name.ToString().c_str()); - } - } - - if (fresh_db) { - if (FLAGS_use_existing_db) { - fprintf(stdout, "%-12s : skipped (--use_existing_db is true)\n", - name.ToString().c_str()); - method = NULL; - } else { - delete db_; - db_ = NULL; - DestroyDB(FLAGS_db, Options()); - Open(); - } - } - - if (method != NULL) { - RunBenchmark(num_threads, name, method); - } - } - } - - private: - struct ThreadArg { - Benchmark* bm; - SharedState* shared; - ThreadState* thread; - void (Benchmark::*method)(ThreadState*); - }; - - static void ThreadBody(void* v) { - ThreadArg* arg = reinterpret_cast<ThreadArg*>(v); - SharedState* shared = arg->shared; - ThreadState* thread = arg->thread; - { - MutexLock l(&shared->mu); - shared->num_initialized++; - if (shared->num_initialized >= shared->total) { - shared->cv.SignalAll(); - } - while (!shared->start) { - shared->cv.Wait(); - } - } - - thread->stats.Start(); - (arg->bm->*(arg->method))(thread); - thread->stats.Stop(); - - { - MutexLock l(&shared->mu); - shared->num_done++; - if (shared->num_done >= shared->total) { - shared->cv.SignalAll(); - } - } - } - - void RunBenchmark(int n, Slice name, - void (Benchmark::*method)(ThreadState*)) { - SharedState shared; - shared.total = n; - shared.num_initialized = 0; - shared.num_done = 0; - shared.start = false; - - ThreadArg* arg = new ThreadArg[n]; - for (int i = 0; i < n; i++) { - arg[i].bm = this; - arg[i].method = method; - arg[i].shared = &shared; - arg[i].thread = new ThreadState(i); - arg[i].thread->shared = &shared; - g_env->StartThread(ThreadBody, &arg[i]); - } - - shared.mu.Lock(); - while (shared.num_initialized < n) { - shared.cv.Wait(); - } - - shared.start = true; - shared.cv.SignalAll(); - while (shared.num_done < n) { - shared.cv.Wait(); - } - shared.mu.Unlock(); - - for (int i = 1; i < n; i++) { - arg[0].thread->stats.Merge(arg[i].thread->stats); - } - arg[0].thread->stats.Report(name); - - for (int i = 0; i < n; i++) { - delete arg[i].thread; - } - delete[] arg; - } - - void Crc32c(ThreadState* thread) { - // Checksum about 500MB of data total - const int size = 4096; - const char* label = "(4K per op)"; - std::string data(size, 'x'); - int64_t bytes = 0; - uint32_t crc = 0; - while (bytes < 500 * 1048576) { - crc = crc32c::Value(data.data(), size); - thread->stats.FinishedSingleOp(); - bytes += size; - } - // Print so result is not dead - fprintf(stderr, "... crc=0x%x\r", static_cast<unsigned int>(crc)); - - thread->stats.AddBytes(bytes); - thread->stats.AddMessage(label); - } - - void AcquireLoad(ThreadState* thread) { - int dummy; - port::AtomicPointer ap(&dummy); - int count = 0; - void *ptr = NULL; - thread->stats.AddMessage("(each op is 1000 loads)"); - while (count < 100000) { - for (int i = 0; i < 1000; i++) { - ptr = ap.Acquire_Load(); - } - count++; - thread->stats.FinishedSingleOp(); - } - if (ptr == NULL) exit(1); // Disable unused variable warning. - } - - void SnappyCompress(ThreadState* thread) { - RandomGenerator gen; - Slice input = gen.Generate(Options().block_size); - int64_t bytes = 0; - int64_t produced = 0; - bool ok = true; - std::string compressed; - while (ok && bytes < 1024 * 1048576) { // Compress 1G - ok = port::Snappy_Compress(input.data(), input.size(), &compressed); - produced += compressed.size(); - bytes += input.size(); - thread->stats.FinishedSingleOp(); - } - - if (!ok) { - thread->stats.AddMessage("(snappy failure)"); - } else { - char buf[100]; - snprintf(buf, sizeof(buf), "(output: %.1f%%)", - (produced * 100.0) / bytes); - thread->stats.AddMessage(buf); - thread->stats.AddBytes(bytes); - } - } - - void SnappyUncompress(ThreadState* thread) { - RandomGenerator gen; - Slice input = gen.Generate(Options().block_size); - std::string compressed; - bool ok = port::Snappy_Compress(input.data(), input.size(), &compressed); - int64_t bytes = 0; - char* uncompressed = new char[input.size()]; - while (ok && bytes < 1024 * 1048576) { // Compress 1G - ok = port::Snappy_Uncompress(compressed.data(), compressed.size(), - uncompressed); - bytes += input.size(); - thread->stats.FinishedSingleOp(); - } - delete[] uncompressed; - - if (!ok) { - thread->stats.AddMessage("(snappy failure)"); - } else { - thread->stats.AddBytes(bytes); - } - } - - void Open() { - assert(db_ == NULL); - Options options; - options.env = g_env; - options.create_if_missing = !FLAGS_use_existing_db; - options.block_cache = cache_; - options.write_buffer_size = FLAGS_write_buffer_size; - options.max_file_size = FLAGS_max_file_size; - options.block_size = FLAGS_block_size; - options.max_open_files = FLAGS_open_files; - options.filter_policy = filter_policy_; - options.reuse_logs = FLAGS_reuse_logs; - Status s = DB::Open(options, FLAGS_db, &db_); - if (!s.ok()) { - fprintf(stderr, "open error: %s\n", s.ToString().c_str()); - exit(1); - } - } - - void OpenBench(ThreadState* thread) { - for (int i = 0; i < num_; i++) { - delete db_; - Open(); - thread->stats.FinishedSingleOp(); - } - } - - void WriteSeq(ThreadState* thread) { - DoWrite(thread, true); - } - - void WriteRandom(ThreadState* thread) { - DoWrite(thread, false); - } - - void DoWrite(ThreadState* thread, bool seq) { - if (num_ != FLAGS_num) { - char msg[100]; - snprintf(msg, sizeof(msg), "(%d ops)", num_); - thread->stats.AddMessage(msg); - } - - RandomGenerator gen; - WriteBatch batch; - Status s; - int64_t bytes = 0; - for (int i = 0; i < num_; i += entries_per_batch_) { - batch.Clear(); - for (int j = 0; j < entries_per_batch_; j++) { - const int k = seq ? i+j : (thread->rand.Next() % FLAGS_num); - char key[100]; - snprintf(key, sizeof(key), "%016d", k); - batch.Put(key, gen.Generate(value_size_)); - bytes += value_size_ + strlen(key); - thread->stats.FinishedSingleOp(); - } - s = db_->Write(write_options_, &batch); - if (!s.ok()) { - fprintf(stderr, "put error: %s\n", s.ToString().c_str()); - exit(1); - } - } - thread->stats.AddBytes(bytes); - } - - void ReadSequential(ThreadState* thread) { - Iterator* iter = db_->NewIterator(ReadOptions()); - int i = 0; - int64_t bytes = 0; - for (iter->SeekToFirst(); i < reads_ && iter->Valid(); iter->Next()) { - bytes += iter->key().size() + iter->value().size(); - thread->stats.FinishedSingleOp(); - ++i; - } - delete iter; - thread->stats.AddBytes(bytes); - } - - void ReadReverse(ThreadState* thread) { - Iterator* iter = db_->NewIterator(ReadOptions()); - int i = 0; - int64_t bytes = 0; - for (iter->SeekToLast(); i < reads_ && iter->Valid(); iter->Prev()) { - bytes += iter->key().size() + iter->value().size(); - thread->stats.FinishedSingleOp(); - ++i; - } - delete iter; - thread->stats.AddBytes(bytes); - } - - void ReadRandom(ThreadState* thread) { - ReadOptions options; - std::string value; - int found = 0; - for (int i = 0; i < reads_; i++) { - char key[100]; - const int k = thread->rand.Next() % FLAGS_num; - snprintf(key, sizeof(key), "%016d", k); - if (db_->Get(options, key, &value).ok()) { - found++; - } - thread->stats.FinishedSingleOp(); - } - char msg[100]; - snprintf(msg, sizeof(msg), "(%d of %d found)", found, num_); - thread->stats.AddMessage(msg); - } - - void ReadMissing(ThreadState* thread) { - ReadOptions options; - std::string value; - for (int i = 0; i < reads_; i++) { - char key[100]; - const int k = thread->rand.Next() % FLAGS_num; - snprintf(key, sizeof(key), "%016d.", k); - db_->Get(options, key, &value); - thread->stats.FinishedSingleOp(); - } - } - - void ReadHot(ThreadState* thread) { - ReadOptions options; - std::string value; - const int range = (FLAGS_num + 99) / 100; - for (int i = 0; i < reads_; i++) { - char key[100]; - const int k = thread->rand.Next() % range; - snprintf(key, sizeof(key), "%016d", k); - db_->Get(options, key, &value); - thread->stats.FinishedSingleOp(); - } - } - - void SeekRandom(ThreadState* thread) { - ReadOptions options; - int found = 0; - for (int i = 0; i < reads_; i++) { - Iterator* iter = db_->NewIterator(options); - char key[100]; - const int k = thread->rand.Next() % FLAGS_num; - snprintf(key, sizeof(key), "%016d", k); - iter->Seek(key); - if (iter->Valid() && iter->key() == key) found++; - delete iter; - thread->stats.FinishedSingleOp(); - } - char msg[100]; - snprintf(msg, sizeof(msg), "(%d of %d found)", found, num_); - thread->stats.AddMessage(msg); - } - - void DoDelete(ThreadState* thread, bool seq) { - RandomGenerator gen; - WriteBatch batch; - Status s; - for (int i = 0; i < num_; i += entries_per_batch_) { - batch.Clear(); - for (int j = 0; j < entries_per_batch_; j++) { - const int k = seq ? i+j : (thread->rand.Next() % FLAGS_num); - char key[100]; - snprintf(key, sizeof(key), "%016d", k); - batch.Delete(key); - thread->stats.FinishedSingleOp(); - } - s = db_->Write(write_options_, &batch); - if (!s.ok()) { - fprintf(stderr, "del error: %s\n", s.ToString().c_str()); - exit(1); - } - } - } - - void DeleteSeq(ThreadState* thread) { - DoDelete(thread, true); - } - - void DeleteRandom(ThreadState* thread) { - DoDelete(thread, false); - } - - void ReadWhileWriting(ThreadState* thread) { - if (thread->tid > 0) { - ReadRandom(thread); - } else { - // Special thread that keeps writing until other threads are done. - RandomGenerator gen; - while (true) { - { - MutexLock l(&thread->shared->mu); - if (thread->shared->num_done + 1 >= thread->shared->num_initialized) { - // Other threads have finished - break; - } - } - - const int k = thread->rand.Next() % FLAGS_num; - char key[100]; - snprintf(key, sizeof(key), "%016d", k); - Status s = db_->Put(write_options_, key, gen.Generate(value_size_)); - if (!s.ok()) { - fprintf(stderr, "put error: %s\n", s.ToString().c_str()); - exit(1); - } - } - - // Do not count any of the preceding work/delay in stats. - thread->stats.Start(); - } - } - - void Compact(ThreadState* thread) { - db_->CompactRange(NULL, NULL); - } - - void PrintStats(const char* key) { - std::string stats; - if (!db_->GetProperty(key, &stats)) { - stats = "(failed)"; - } - fprintf(stdout, "\n%s\n", stats.c_str()); - } - - static void WriteToFile(void* arg, const char* buf, int n) { - reinterpret_cast<WritableFile*>(arg)->Append(Slice(buf, n)); - } - - void HeapProfile() { - char fname[100]; - snprintf(fname, sizeof(fname), "%s/heap-%04d", FLAGS_db, ++heap_counter_); - WritableFile* file; - Status s = g_env->NewWritableFile(fname, &file); - if (!s.ok()) { - fprintf(stderr, "%s\n", s.ToString().c_str()); - return; - } - bool ok = port::GetHeapProfile(WriteToFile, file); - delete file; - if (!ok) { - fprintf(stderr, "heap profiling not supported\n"); - g_env->DeleteFile(fname); - } - } -}; - -} // namespace leveldb - -int main(int argc, char** argv) { - FLAGS_write_buffer_size = leveldb::Options().write_buffer_size; - FLAGS_max_file_size = leveldb::Options().max_file_size; - FLAGS_block_size = leveldb::Options().block_size; - FLAGS_open_files = leveldb::Options().max_open_files; - std::string default_db_path; - - for (int i = 1; i < argc; i++) { - double d; - int n; - char junk; - if (leveldb::Slice(argv[i]).starts_with("--benchmarks=")) { - FLAGS_benchmarks = argv[i] + strlen("--benchmarks="); - } else if (sscanf(argv[i], "--compression_ratio=%lf%c", &d, &junk) == 1) { - FLAGS_compression_ratio = d; - } else if (sscanf(argv[i], "--histogram=%d%c", &n, &junk) == 1 && - (n == 0 || n == 1)) { - FLAGS_histogram = n; - } else if (sscanf(argv[i], "--use_existing_db=%d%c", &n, &junk) == 1 && - (n == 0 || n == 1)) { - FLAGS_use_existing_db = n; - } else if (sscanf(argv[i], "--reuse_logs=%d%c", &n, &junk) == 1 && - (n == 0 || n == 1)) { - FLAGS_reuse_logs = n; - } else if (sscanf(argv[i], "--num=%d%c", &n, &junk) == 1) { - FLAGS_num = n; - } else if (sscanf(argv[i], "--reads=%d%c", &n, &junk) == 1) { - FLAGS_reads = n; - } else if (sscanf(argv[i], "--threads=%d%c", &n, &junk) == 1) { - FLAGS_threads = n; - } else if (sscanf(argv[i], "--value_size=%d%c", &n, &junk) == 1) { - FLAGS_value_size = n; - } else if (sscanf(argv[i], "--write_buffer_size=%d%c", &n, &junk) == 1) { - FLAGS_write_buffer_size = n; - } else if (sscanf(argv[i], "--max_file_size=%d%c", &n, &junk) == 1) { - FLAGS_max_file_size = n; - } else if (sscanf(argv[i], "--block_size=%d%c", &n, &junk) == 1) { - FLAGS_block_size = n; - } else if (sscanf(argv[i], "--cache_size=%d%c", &n, &junk) == 1) { - FLAGS_cache_size = n; - } else if (sscanf(argv[i], "--bloom_bits=%d%c", &n, &junk) == 1) { - FLAGS_bloom_bits = n; - } else if (sscanf(argv[i], "--open_files=%d%c", &n, &junk) == 1) { - FLAGS_open_files = n; - } else if (strncmp(argv[i], "--db=", 5) == 0) { - FLAGS_db = argv[i] + 5; - } else { - fprintf(stderr, "Invalid flag '%s'\n", argv[i]); - exit(1); - } - } - - leveldb::g_env = leveldb::Env::Default(); - - // Choose a location for the test database if none given with --db=<path> - if (FLAGS_db == NULL) { - leveldb::g_env->GetTestDirectory(&default_db_path); - default_db_path += "/dbbench"; - FLAGS_db = default_db_path.c_str(); - } - - leveldb::Benchmark benchmark; - benchmark.Run(); - return 0; -} diff --git a/src/leveldb/db/db_impl.cc b/src/leveldb/db/db_impl.cc index 3bb58e560a..65e31724bc 100644 --- a/src/leveldb/db/db_impl.cc +++ b/src/leveldb/db/db_impl.cc @@ -4,12 +4,15 @@ #include "db/db_impl.h" +#include <stdint.h> +#include <stdio.h> + #include <algorithm> +#include <atomic> #include <set> #include <string> -#include <stdint.h> -#include <stdio.h> #include <vector> + #include "db/builder.h" #include "db/db_iter.h" #include "db/dbformat.h" @@ -39,16 +42,33 @@ const int kNumNonTableCacheFiles = 10; // Information kept for every waiting writer struct DBImpl::Writer { + explicit Writer(port::Mutex* mu) + : batch(nullptr), sync(false), done(false), cv(mu) {} + Status status; WriteBatch* batch; bool sync; bool done; port::CondVar cv; - - explicit Writer(port::Mutex* mu) : cv(mu) { } }; struct DBImpl::CompactionState { + // Files produced by compaction + struct Output { + uint64_t number; + uint64_t file_size; + InternalKey smallest, largest; + }; + + Output* current_output() { return &outputs[outputs.size() - 1]; } + + explicit CompactionState(Compaction* c) + : compaction(c), + smallest_snapshot(0), + outfile(nullptr), + builder(nullptr), + total_bytes(0) {} + Compaction* const compaction; // Sequence numbers < smallest_snapshot are not significant since we @@ -57,12 +77,6 @@ struct DBImpl::CompactionState { // we can drop all entries for the same key with sequence numbers < S. SequenceNumber smallest_snapshot; - // Files produced by compaction - struct Output { - uint64_t number; - uint64_t file_size; - InternalKey smallest, largest; - }; std::vector<Output> outputs; // State kept for output being generated @@ -70,19 +84,10 @@ struct DBImpl::CompactionState { TableBuilder* builder; uint64_t total_bytes; - - Output* current_output() { return &outputs[outputs.size()-1]; } - - explicit CompactionState(Compaction* c) - : compaction(c), - outfile(NULL), - builder(NULL), - total_bytes(0) { - } }; // Fix user-supplied options to be reasonable -template <class T,class V> +template <class T, class V> static void ClipToRange(T* ptr, V minvalue, V maxvalue) { if (static_cast<V>(*ptr) > maxvalue) *ptr = maxvalue; if (static_cast<V>(*ptr) < minvalue) *ptr = minvalue; @@ -93,27 +98,32 @@ Options SanitizeOptions(const std::string& dbname, const Options& src) { Options result = src; result.comparator = icmp; - result.filter_policy = (src.filter_policy != NULL) ? ipolicy : NULL; - ClipToRange(&result.max_open_files, 64 + kNumNonTableCacheFiles, 50000); - ClipToRange(&result.write_buffer_size, 64<<10, 1<<30); - ClipToRange(&result.max_file_size, 1<<20, 1<<30); - ClipToRange(&result.block_size, 1<<10, 4<<20); - if (result.info_log == NULL) { + result.filter_policy = (src.filter_policy != nullptr) ? ipolicy : nullptr; + ClipToRange(&result.max_open_files, 64 + kNumNonTableCacheFiles, 50000); + ClipToRange(&result.write_buffer_size, 64 << 10, 1 << 30); + ClipToRange(&result.max_file_size, 1 << 20, 1 << 30); + ClipToRange(&result.block_size, 1 << 10, 4 << 20); + if (result.info_log == nullptr) { // Open a log file in the same directory as the db src.env->CreateDir(dbname); // In case it does not exist src.env->RenameFile(InfoLogFileName(dbname), OldInfoLogFileName(dbname)); Status s = src.env->NewLogger(InfoLogFileName(dbname), &result.info_log); if (!s.ok()) { // No place suitable for logging - result.info_log = NULL; + result.info_log = nullptr; } } - if (result.block_cache == NULL) { + if (result.block_cache == nullptr) { result.block_cache = NewLRUCache(8 << 20); } return result; } +static int TableCacheSize(const Options& sanitized_options) { + // Reserve ten files or so for other uses and give the rest to TableCache. + return sanitized_options.max_open_files - kNumNonTableCacheFiles; +} + DBImpl::DBImpl(const Options& raw_options, const std::string& dbname) : env_(raw_options.env), internal_comparator_(raw_options.comparator), @@ -123,44 +133,39 @@ DBImpl::DBImpl(const Options& raw_options, const std::string& dbname) owns_info_log_(options_.info_log != raw_options.info_log), owns_cache_(options_.block_cache != raw_options.block_cache), dbname_(dbname), - db_lock_(NULL), - shutting_down_(NULL), - bg_cv_(&mutex_), - mem_(NULL), - imm_(NULL), - logfile_(NULL), + table_cache_(new TableCache(dbname_, options_, TableCacheSize(options_))), + db_lock_(nullptr), + shutting_down_(false), + background_work_finished_signal_(&mutex_), + mem_(nullptr), + imm_(nullptr), + has_imm_(false), + logfile_(nullptr), logfile_number_(0), - log_(NULL), + log_(nullptr), seed_(0), tmp_batch_(new WriteBatch), - bg_compaction_scheduled_(false), - manual_compaction_(NULL) { - has_imm_.Release_Store(NULL); - - // Reserve ten files or so for other uses and give the rest to TableCache. - const int table_cache_size = options_.max_open_files - kNumNonTableCacheFiles; - table_cache_ = new TableCache(dbname_, &options_, table_cache_size); - - versions_ = new VersionSet(dbname_, &options_, table_cache_, - &internal_comparator_); -} + background_compaction_scheduled_(false), + manual_compaction_(nullptr), + versions_(new VersionSet(dbname_, &options_, table_cache_, + &internal_comparator_)) {} DBImpl::~DBImpl() { - // Wait for background work to finish + // Wait for background work to finish. mutex_.Lock(); - shutting_down_.Release_Store(this); // Any non-NULL value is ok - while (bg_compaction_scheduled_) { - bg_cv_.Wait(); + shutting_down_.store(true, std::memory_order_release); + while (background_compaction_scheduled_) { + background_work_finished_signal_.Wait(); } mutex_.Unlock(); - if (db_lock_ != NULL) { + if (db_lock_ != nullptr) { env_->UnlockFile(db_lock_); } delete versions_; - if (mem_ != NULL) mem_->Unref(); - if (imm_ != NULL) imm_->Unref(); + if (mem_ != nullptr) mem_->Unref(); + if (imm_ != nullptr) imm_->Unref(); delete tmp_batch_; delete log_; delete logfile_; @@ -216,6 +221,8 @@ void DBImpl::MaybeIgnoreError(Status* s) const { } void DBImpl::DeleteObsoleteFiles() { + mutex_.AssertHeld(); + if (!bg_error_.ok()) { // After a background error, we don't know whether a new version may // or may not have been committed, so we cannot safely garbage collect. @@ -227,11 +234,12 @@ void DBImpl::DeleteObsoleteFiles() { versions_->AddLiveFiles(&live); std::vector<std::string> filenames; - env_->GetChildren(dbname_, &filenames); // Ignoring errors on purpose + env_->GetChildren(dbname_, &filenames); // Ignoring errors on purpose uint64_t number; FileType type; - for (size_t i = 0; i < filenames.size(); i++) { - if (ParseFileName(filenames[i], &number, &type)) { + std::vector<std::string> files_to_delete; + for (std::string& filename : filenames) { + if (ParseFileName(filename, &number, &type)) { bool keep = true; switch (type) { case kLogFile: @@ -259,26 +267,34 @@ void DBImpl::DeleteObsoleteFiles() { } if (!keep) { + files_to_delete.push_back(std::move(filename)); if (type == kTableFile) { table_cache_->Evict(number); } - Log(options_.info_log, "Delete type=%d #%lld\n", - int(type), + Log(options_.info_log, "Delete type=%d #%lld\n", static_cast<int>(type), static_cast<unsigned long long>(number)); - env_->DeleteFile(dbname_ + "/" + filenames[i]); } } } + + // While deleting all files unblock other threads. All files being deleted + // have unique names which will not collide with newly created files and + // are therefore safe to delete while allowing other threads to proceed. + mutex_.Unlock(); + for (const std::string& filename : files_to_delete) { + env_->DeleteFile(dbname_ + "/" + filename); + } + mutex_.Lock(); } -Status DBImpl::Recover(VersionEdit* edit, bool *save_manifest) { +Status DBImpl::Recover(VersionEdit* edit, bool* save_manifest) { mutex_.AssertHeld(); // Ignore error from CreateDir since the creation of the DB is // committed only when the descriptor is created, and this directory // may already exist from a previous failed creation attempt. env_->CreateDir(dbname_); - assert(db_lock_ == NULL); + assert(db_lock_ == nullptr); Status s = env_->LockFile(LockFileName(dbname_), &db_lock_); if (!s.ok()) { return s; @@ -296,8 +312,8 @@ Status DBImpl::Recover(VersionEdit* edit, bool *save_manifest) { } } else { if (options_.error_if_exists) { - return Status::InvalidArgument( - dbname_, "exists (error_if_exists is true)"); + return Status::InvalidArgument(dbname_, + "exists (error_if_exists is true)"); } } @@ -369,12 +385,12 @@ Status DBImpl::RecoverLogFile(uint64_t log_number, bool last_log, Env* env; Logger* info_log; const char* fname; - Status* status; // NULL if options_.paranoid_checks==false - virtual void Corruption(size_t bytes, const Status& s) { + Status* status; // null if options_.paranoid_checks==false + void Corruption(size_t bytes, const Status& s) override { Log(info_log, "%s%s: dropping %d bytes; %s", - (this->status == NULL ? "(ignoring error) " : ""), - fname, static_cast<int>(bytes), s.ToString().c_str()); - if (this->status != NULL && this->status->ok()) *this->status = s; + (this->status == nullptr ? "(ignoring error) " : ""), fname, + static_cast<int>(bytes), s.ToString().c_str()); + if (this->status != nullptr && this->status->ok()) *this->status = s; } }; @@ -394,32 +410,30 @@ Status DBImpl::RecoverLogFile(uint64_t log_number, bool last_log, reporter.env = env_; reporter.info_log = options_.info_log; reporter.fname = fname.c_str(); - reporter.status = (options_.paranoid_checks ? &status : NULL); + reporter.status = (options_.paranoid_checks ? &status : nullptr); // We intentionally make log::Reader do checksumming even if // paranoid_checks==false so that corruptions cause entire commits // to be skipped instead of propagating bad information (like overly // large sequence numbers). - log::Reader reader(file, &reporter, true/*checksum*/, - 0/*initial_offset*/); + log::Reader reader(file, &reporter, true /*checksum*/, 0 /*initial_offset*/); Log(options_.info_log, "Recovering log #%llu", - (unsigned long long) log_number); + (unsigned long long)log_number); // Read all the records and add to a memtable std::string scratch; Slice record; WriteBatch batch; int compactions = 0; - MemTable* mem = NULL; - while (reader.ReadRecord(&record, &scratch) && - status.ok()) { + MemTable* mem = nullptr; + while (reader.ReadRecord(&record, &scratch) && status.ok()) { if (record.size() < 12) { - reporter.Corruption( - record.size(), Status::Corruption("log record too small", fname)); + reporter.Corruption(record.size(), + Status::Corruption("log record too small", fname)); continue; } WriteBatchInternal::SetContents(&batch, record); - if (mem == NULL) { + if (mem == nullptr) { mem = new MemTable(internal_comparator_); mem->Ref(); } @@ -428,9 +442,8 @@ Status DBImpl::RecoverLogFile(uint64_t log_number, bool last_log, if (!status.ok()) { break; } - const SequenceNumber last_seq = - WriteBatchInternal::Sequence(&batch) + - WriteBatchInternal::Count(&batch) - 1; + const SequenceNumber last_seq = WriteBatchInternal::Sequence(&batch) + + WriteBatchInternal::Count(&batch) - 1; if (last_seq > *max_sequence) { *max_sequence = last_seq; } @@ -438,9 +451,9 @@ Status DBImpl::RecoverLogFile(uint64_t log_number, bool last_log, if (mem->ApproximateMemoryUsage() > options_.write_buffer_size) { compactions++; *save_manifest = true; - status = WriteLevel0Table(mem, edit, NULL); + status = WriteLevel0Table(mem, edit, nullptr); mem->Unref(); - mem = NULL; + mem = nullptr; if (!status.ok()) { // Reflect errors immediately so that conditions like full // file-systems cause the DB::Open() to fail. @@ -453,31 +466,31 @@ Status DBImpl::RecoverLogFile(uint64_t log_number, bool last_log, // See if we should keep reusing the last log file. if (status.ok() && options_.reuse_logs && last_log && compactions == 0) { - assert(logfile_ == NULL); - assert(log_ == NULL); - assert(mem_ == NULL); + assert(logfile_ == nullptr); + assert(log_ == nullptr); + assert(mem_ == nullptr); uint64_t lfile_size; if (env_->GetFileSize(fname, &lfile_size).ok() && env_->NewAppendableFile(fname, &logfile_).ok()) { Log(options_.info_log, "Reusing old log %s \n", fname.c_str()); log_ = new log::Writer(logfile_, lfile_size); logfile_number_ = log_number; - if (mem != NULL) { + if (mem != nullptr) { mem_ = mem; - mem = NULL; + mem = nullptr; } else { - // mem can be NULL if lognum exists but was empty. + // mem can be nullptr if lognum exists but was empty. mem_ = new MemTable(internal_comparator_); mem_->Ref(); } } } - if (mem != NULL) { + if (mem != nullptr) { // mem did not get reused; compact it. if (status.ok()) { *save_manifest = true; - status = WriteLevel0Table(mem, edit, NULL); + status = WriteLevel0Table(mem, edit, nullptr); } mem->Unref(); } @@ -494,7 +507,7 @@ Status DBImpl::WriteLevel0Table(MemTable* mem, VersionEdit* edit, pending_outputs_.insert(meta.number); Iterator* iter = mem->NewIterator(); Log(options_.info_log, "Level-0 table #%llu: started", - (unsigned long long) meta.number); + (unsigned long long)meta.number); Status s; { @@ -504,24 +517,22 @@ Status DBImpl::WriteLevel0Table(MemTable* mem, VersionEdit* edit, } Log(options_.info_log, "Level-0 table #%llu: %lld bytes %s", - (unsigned long long) meta.number, - (unsigned long long) meta.file_size, + (unsigned long long)meta.number, (unsigned long long)meta.file_size, s.ToString().c_str()); delete iter; pending_outputs_.erase(meta.number); - // Note that if file_size is zero, the file has been deleted and // should not be added to the manifest. int level = 0; if (s.ok() && meta.file_size > 0) { const Slice min_user_key = meta.smallest.user_key(); const Slice max_user_key = meta.largest.user_key(); - if (base != NULL) { + if (base != nullptr) { level = base->PickLevelForMemTableOutput(min_user_key, max_user_key); } - edit->AddFile(level, meta.number, meta.file_size, - meta.smallest, meta.largest); + edit->AddFile(level, meta.number, meta.file_size, meta.smallest, + meta.largest); } CompactionStats stats; @@ -533,7 +544,7 @@ Status DBImpl::WriteLevel0Table(MemTable* mem, VersionEdit* edit, void DBImpl::CompactMemTable() { mutex_.AssertHeld(); - assert(imm_ != NULL); + assert(imm_ != nullptr); // Save the contents of the memtable as a new Table VersionEdit edit; @@ -542,7 +553,7 @@ void DBImpl::CompactMemTable() { Status s = WriteLevel0Table(imm_, &edit, base); base->Unref(); - if (s.ok() && shutting_down_.Acquire_Load()) { + if (s.ok() && shutting_down_.load(std::memory_order_acquire)) { s = Status::IOError("Deleting DB during memtable compaction"); } @@ -556,8 +567,8 @@ void DBImpl::CompactMemTable() { if (s.ok()) { // Commit to the new state imm_->Unref(); - imm_ = NULL; - has_imm_.Release_Store(NULL); + imm_ = nullptr; + has_imm_.store(false, std::memory_order_release); DeleteObsoleteFiles(); } else { RecordBackgroundError(s); @@ -575,13 +586,14 @@ void DBImpl::CompactRange(const Slice* begin, const Slice* end) { } } } - TEST_CompactMemTable(); // TODO(sanjay): Skip if memtable does not overlap + TEST_CompactMemTable(); // TODO(sanjay): Skip if memtable does not overlap for (int level = 0; level < max_level_with_files; level++) { TEST_CompactRange(level, begin, end); } } -void DBImpl::TEST_CompactRange(int level, const Slice* begin,const Slice* end) { +void DBImpl::TEST_CompactRange(int level, const Slice* begin, + const Slice* end) { assert(level >= 0); assert(level + 1 < config::kNumLevels); @@ -590,44 +602,45 @@ void DBImpl::TEST_CompactRange(int level, const Slice* begin,const Slice* end) { ManualCompaction manual; manual.level = level; manual.done = false; - if (begin == NULL) { - manual.begin = NULL; + if (begin == nullptr) { + manual.begin = nullptr; } else { begin_storage = InternalKey(*begin, kMaxSequenceNumber, kValueTypeForSeek); manual.begin = &begin_storage; } - if (end == NULL) { - manual.end = NULL; + if (end == nullptr) { + manual.end = nullptr; } else { end_storage = InternalKey(*end, 0, static_cast<ValueType>(0)); manual.end = &end_storage; } MutexLock l(&mutex_); - while (!manual.done && !shutting_down_.Acquire_Load() && bg_error_.ok()) { - if (manual_compaction_ == NULL) { // Idle + while (!manual.done && !shutting_down_.load(std::memory_order_acquire) && + bg_error_.ok()) { + if (manual_compaction_ == nullptr) { // Idle manual_compaction_ = &manual; MaybeScheduleCompaction(); } else { // Running either my compaction or another compaction. - bg_cv_.Wait(); + background_work_finished_signal_.Wait(); } } if (manual_compaction_ == &manual) { // Cancel my manual compaction since we aborted early for some reason. - manual_compaction_ = NULL; + manual_compaction_ = nullptr; } } Status DBImpl::TEST_CompactMemTable() { - // NULL batch means just wait for earlier writes to be done - Status s = Write(WriteOptions(), NULL); + // nullptr batch means just wait for earlier writes to be done + Status s = Write(WriteOptions(), nullptr); if (s.ok()) { // Wait until the compaction completes MutexLock l(&mutex_); - while (imm_ != NULL && bg_error_.ok()) { - bg_cv_.Wait(); + while (imm_ != nullptr && bg_error_.ok()) { + background_work_finished_signal_.Wait(); } - if (imm_ != NULL) { + if (imm_ != nullptr) { s = bg_error_; } } @@ -638,24 +651,23 @@ void DBImpl::RecordBackgroundError(const Status& s) { mutex_.AssertHeld(); if (bg_error_.ok()) { bg_error_ = s; - bg_cv_.SignalAll(); + background_work_finished_signal_.SignalAll(); } } void DBImpl::MaybeScheduleCompaction() { mutex_.AssertHeld(); - if (bg_compaction_scheduled_) { + if (background_compaction_scheduled_) { // Already scheduled - } else if (shutting_down_.Acquire_Load()) { + } else if (shutting_down_.load(std::memory_order_acquire)) { // DB is being deleted; no more background compactions } else if (!bg_error_.ok()) { // Already got an error; no more changes - } else if (imm_ == NULL && - manual_compaction_ == NULL && + } else if (imm_ == nullptr && manual_compaction_ == nullptr && !versions_->NeedsCompaction()) { // No work to be done } else { - bg_compaction_scheduled_ = true; + background_compaction_scheduled_ = true; env_->Schedule(&DBImpl::BGWork, this); } } @@ -666,8 +678,8 @@ void DBImpl::BGWork(void* db) { void DBImpl::BackgroundCall() { MutexLock l(&mutex_); - assert(bg_compaction_scheduled_); - if (shutting_down_.Acquire_Load()) { + assert(background_compaction_scheduled_); + if (shutting_down_.load(std::memory_order_acquire)) { // No more background work when shutting down. } else if (!bg_error_.ok()) { // No more background work after a background error. @@ -675,36 +687,35 @@ void DBImpl::BackgroundCall() { BackgroundCompaction(); } - bg_compaction_scheduled_ = false; + background_compaction_scheduled_ = false; // Previous compaction may have produced too many files in a level, // so reschedule another compaction if needed. MaybeScheduleCompaction(); - bg_cv_.SignalAll(); + background_work_finished_signal_.SignalAll(); } void DBImpl::BackgroundCompaction() { mutex_.AssertHeld(); - if (imm_ != NULL) { + if (imm_ != nullptr) { CompactMemTable(); return; } Compaction* c; - bool is_manual = (manual_compaction_ != NULL); + bool is_manual = (manual_compaction_ != nullptr); InternalKey manual_end; if (is_manual) { ManualCompaction* m = manual_compaction_; c = versions_->CompactRange(m->level, m->begin, m->end); - m->done = (c == NULL); - if (c != NULL) { + m->done = (c == nullptr); + if (c != nullptr) { manual_end = c->input(0, c->num_input_files(0) - 1)->largest; } Log(options_.info_log, "Manual compaction at level-%d from %s .. %s; will stop at %s\n", - m->level, - (m->begin ? m->begin->DebugString().c_str() : "(begin)"), + m->level, (m->begin ? m->begin->DebugString().c_str() : "(begin)"), (m->end ? m->end->DebugString().c_str() : "(end)"), (m->done ? "(end)" : manual_end.DebugString().c_str())); } else { @@ -712,26 +723,24 @@ void DBImpl::BackgroundCompaction() { } Status status; - if (c == NULL) { + if (c == nullptr) { // Nothing to do } else if (!is_manual && c->IsTrivialMove()) { // Move file to next level assert(c->num_input_files(0) == 1); FileMetaData* f = c->input(0, 0); c->edit()->DeleteFile(c->level(), f->number); - c->edit()->AddFile(c->level() + 1, f->number, f->file_size, - f->smallest, f->largest); + c->edit()->AddFile(c->level() + 1, f->number, f->file_size, f->smallest, + f->largest); status = versions_->LogAndApply(c->edit(), &mutex_); if (!status.ok()) { RecordBackgroundError(status); } VersionSet::LevelSummaryStorage tmp; Log(options_.info_log, "Moved #%lld to level-%d %lld bytes %s: %s\n", - static_cast<unsigned long long>(f->number), - c->level() + 1, + static_cast<unsigned long long>(f->number), c->level() + 1, static_cast<unsigned long long>(f->file_size), - status.ToString().c_str(), - versions_->LevelSummary(&tmp)); + status.ToString().c_str(), versions_->LevelSummary(&tmp)); } else { CompactionState* compact = new CompactionState(c); status = DoCompactionWork(compact); @@ -746,11 +755,10 @@ void DBImpl::BackgroundCompaction() { if (status.ok()) { // Done - } else if (shutting_down_.Acquire_Load()) { + } else if (shutting_down_.load(std::memory_order_acquire)) { // Ignore compaction errors found during shutting down } else { - Log(options_.info_log, - "Compaction error: %s", status.ToString().c_str()); + Log(options_.info_log, "Compaction error: %s", status.ToString().c_str()); } if (is_manual) { @@ -764,18 +772,18 @@ void DBImpl::BackgroundCompaction() { m->tmp_storage = manual_end; m->begin = &m->tmp_storage; } - manual_compaction_ = NULL; + manual_compaction_ = nullptr; } } void DBImpl::CleanupCompaction(CompactionState* compact) { mutex_.AssertHeld(); - if (compact->builder != NULL) { + if (compact->builder != nullptr) { // May happen if we get a shutdown call in the middle of compaction compact->builder->Abandon(); delete compact->builder; } else { - assert(compact->outfile == NULL); + assert(compact->outfile == nullptr); } delete compact->outfile; for (size_t i = 0; i < compact->outputs.size(); i++) { @@ -786,8 +794,8 @@ void DBImpl::CleanupCompaction(CompactionState* compact) { } Status DBImpl::OpenCompactionOutputFile(CompactionState* compact) { - assert(compact != NULL); - assert(compact->builder == NULL); + assert(compact != nullptr); + assert(compact->builder == nullptr); uint64_t file_number; { mutex_.Lock(); @@ -812,9 +820,9 @@ Status DBImpl::OpenCompactionOutputFile(CompactionState* compact) { Status DBImpl::FinishCompactionOutputFile(CompactionState* compact, Iterator* input) { - assert(compact != NULL); - assert(compact->outfile != NULL); - assert(compact->builder != NULL); + assert(compact != nullptr); + assert(compact->outfile != nullptr); + assert(compact->builder != nullptr); const uint64_t output_number = compact->current_output()->number; assert(output_number != 0); @@ -831,7 +839,7 @@ Status DBImpl::FinishCompactionOutputFile(CompactionState* compact, compact->current_output()->file_size = current_bytes; compact->total_bytes += current_bytes; delete compact->builder; - compact->builder = NULL; + compact->builder = nullptr; // Finish and check for file errors if (s.ok()) { @@ -841,35 +849,29 @@ Status DBImpl::FinishCompactionOutputFile(CompactionState* compact, s = compact->outfile->Close(); } delete compact->outfile; - compact->outfile = NULL; + compact->outfile = nullptr; if (s.ok() && current_entries > 0) { // Verify that the table is usable - Iterator* iter = table_cache_->NewIterator(ReadOptions(), - output_number, - current_bytes); + Iterator* iter = + table_cache_->NewIterator(ReadOptions(), output_number, current_bytes); s = iter->status(); delete iter; if (s.ok()) { - Log(options_.info_log, - "Generated table #%llu@%d: %lld keys, %lld bytes", - (unsigned long long) output_number, - compact->compaction->level(), - (unsigned long long) current_entries, - (unsigned long long) current_bytes); + Log(options_.info_log, "Generated table #%llu@%d: %lld keys, %lld bytes", + (unsigned long long)output_number, compact->compaction->level(), + (unsigned long long)current_entries, + (unsigned long long)current_bytes); } } return s; } - Status DBImpl::InstallCompactionResults(CompactionState* compact) { mutex_.AssertHeld(); - Log(options_.info_log, "Compacted %d@%d + %d@%d files => %lld bytes", - compact->compaction->num_input_files(0), - compact->compaction->level(), - compact->compaction->num_input_files(1), - compact->compaction->level() + 1, + Log(options_.info_log, "Compacted %d@%d + %d@%d files => %lld bytes", + compact->compaction->num_input_files(0), compact->compaction->level(), + compact->compaction->num_input_files(1), compact->compaction->level() + 1, static_cast<long long>(compact->total_bytes)); // Add compaction outputs @@ -877,9 +879,8 @@ Status DBImpl::InstallCompactionResults(CompactionState* compact) { const int level = compact->compaction->level(); for (size_t i = 0; i < compact->outputs.size(); i++) { const CompactionState::Output& out = compact->outputs[i]; - compact->compaction->edit()->AddFile( - level + 1, - out.number, out.file_size, out.smallest, out.largest); + compact->compaction->edit()->AddFile(level + 1, out.number, out.file_size, + out.smallest, out.largest); } return versions_->LogAndApply(compact->compaction->edit(), &mutex_); } @@ -888,39 +889,40 @@ Status DBImpl::DoCompactionWork(CompactionState* compact) { const uint64_t start_micros = env_->NowMicros(); int64_t imm_micros = 0; // Micros spent doing imm_ compactions - Log(options_.info_log, "Compacting %d@%d + %d@%d files", - compact->compaction->num_input_files(0), - compact->compaction->level(), + Log(options_.info_log, "Compacting %d@%d + %d@%d files", + compact->compaction->num_input_files(0), compact->compaction->level(), compact->compaction->num_input_files(1), compact->compaction->level() + 1); assert(versions_->NumLevelFiles(compact->compaction->level()) > 0); - assert(compact->builder == NULL); - assert(compact->outfile == NULL); + assert(compact->builder == nullptr); + assert(compact->outfile == nullptr); if (snapshots_.empty()) { compact->smallest_snapshot = versions_->LastSequence(); } else { - compact->smallest_snapshot = snapshots_.oldest()->number_; + compact->smallest_snapshot = snapshots_.oldest()->sequence_number(); } + Iterator* input = versions_->MakeInputIterator(compact->compaction); + // Release mutex while we're actually doing the compaction work mutex_.Unlock(); - Iterator* input = versions_->MakeInputIterator(compact->compaction); input->SeekToFirst(); Status status; ParsedInternalKey ikey; std::string current_user_key; bool has_current_user_key = false; SequenceNumber last_sequence_for_key = kMaxSequenceNumber; - for (; input->Valid() && !shutting_down_.Acquire_Load(); ) { + while (input->Valid() && !shutting_down_.load(std::memory_order_acquire)) { // Prioritize immutable compaction work - if (has_imm_.NoBarrier_Load() != NULL) { + if (has_imm_.load(std::memory_order_relaxed)) { const uint64_t imm_start = env_->NowMicros(); mutex_.Lock(); - if (imm_ != NULL) { + if (imm_ != nullptr) { CompactMemTable(); - bg_cv_.SignalAll(); // Wakeup MakeRoomForWrite() if necessary + // Wake up MakeRoomForWrite() if necessary. + background_work_finished_signal_.SignalAll(); } mutex_.Unlock(); imm_micros += (env_->NowMicros() - imm_start); @@ -928,7 +930,7 @@ Status DBImpl::DoCompactionWork(CompactionState* compact) { Slice key = input->key(); if (compact->compaction->ShouldStopBefore(key) && - compact->builder != NULL) { + compact->builder != nullptr) { status = FinishCompactionOutputFile(compact, input); if (!status.ok()) { break; @@ -944,8 +946,8 @@ Status DBImpl::DoCompactionWork(CompactionState* compact) { last_sequence_for_key = kMaxSequenceNumber; } else { if (!has_current_user_key || - user_comparator()->Compare(ikey.user_key, - Slice(current_user_key)) != 0) { + user_comparator()->Compare(ikey.user_key, Slice(current_user_key)) != + 0) { // First occurrence of this user key current_user_key.assign(ikey.user_key.data(), ikey.user_key.size()); has_current_user_key = true; @@ -954,7 +956,7 @@ Status DBImpl::DoCompactionWork(CompactionState* compact) { if (last_sequence_for_key <= compact->smallest_snapshot) { // Hidden by an newer entry for same user key - drop = true; // (A) + drop = true; // (A) } else if (ikey.type == kTypeDeletion && ikey.sequence <= compact->smallest_snapshot && compact->compaction->IsBaseLevelForKey(ikey.user_key)) { @@ -982,7 +984,7 @@ Status DBImpl::DoCompactionWork(CompactionState* compact) { if (!drop) { // Open output file if necessary - if (compact->builder == NULL) { + if (compact->builder == nullptr) { status = OpenCompactionOutputFile(compact); if (!status.ok()) { break; @@ -1007,17 +1009,17 @@ Status DBImpl::DoCompactionWork(CompactionState* compact) { input->Next(); } - if (status.ok() && shutting_down_.Acquire_Load()) { + if (status.ok() && shutting_down_.load(std::memory_order_acquire)) { status = Status::IOError("Deleting DB during compaction"); } - if (status.ok() && compact->builder != NULL) { + if (status.ok() && compact->builder != nullptr) { status = FinishCompactionOutputFile(compact, input); } if (status.ok()) { status = input->status(); } delete input; - input = NULL; + input = nullptr; CompactionStats stats; stats.micros = env_->NowMicros() - start_micros - imm_micros; @@ -1040,34 +1042,37 @@ Status DBImpl::DoCompactionWork(CompactionState* compact) { RecordBackgroundError(status); } VersionSet::LevelSummaryStorage tmp; - Log(options_.info_log, - "compacted to: %s", versions_->LevelSummary(&tmp)); + Log(options_.info_log, "compacted to: %s", versions_->LevelSummary(&tmp)); return status; } namespace { + struct IterState { - port::Mutex* mu; - Version* version; - MemTable* mem; - MemTable* imm; + port::Mutex* const mu; + Version* const version GUARDED_BY(mu); + MemTable* const mem GUARDED_BY(mu); + MemTable* const imm GUARDED_BY(mu); + + IterState(port::Mutex* mutex, MemTable* mem, MemTable* imm, Version* version) + : mu(mutex), version(version), mem(mem), imm(imm) {} }; static void CleanupIteratorState(void* arg1, void* arg2) { IterState* state = reinterpret_cast<IterState*>(arg1); state->mu->Lock(); state->mem->Unref(); - if (state->imm != NULL) state->imm->Unref(); + if (state->imm != nullptr) state->imm->Unref(); state->version->Unref(); state->mu->Unlock(); delete state; } -} // namespace + +} // anonymous namespace Iterator* DBImpl::NewInternalIterator(const ReadOptions& options, SequenceNumber* latest_snapshot, uint32_t* seed) { - IterState* cleanup = new IterState; mutex_.Lock(); *latest_snapshot = versions_->LastSequence(); @@ -1075,7 +1080,7 @@ Iterator* DBImpl::NewInternalIterator(const ReadOptions& options, std::vector<Iterator*> list; list.push_back(mem_->NewIterator()); mem_->Ref(); - if (imm_ != NULL) { + if (imm_ != nullptr) { list.push_back(imm_->NewIterator()); imm_->Ref(); } @@ -1084,11 +1089,8 @@ Iterator* DBImpl::NewInternalIterator(const ReadOptions& options, NewMergingIterator(&internal_comparator_, &list[0], list.size()); versions_->current()->Ref(); - cleanup->mu = &mutex_; - cleanup->mem = mem_; - cleanup->imm = imm_; - cleanup->version = versions_->current(); - internal_iter->RegisterCleanup(CleanupIteratorState, cleanup, NULL); + IterState* cleanup = new IterState(&mutex_, mem_, imm_, versions_->current()); + internal_iter->RegisterCleanup(CleanupIteratorState, cleanup, nullptr); *seed = ++seed_; mutex_.Unlock(); @@ -1106,14 +1108,14 @@ int64_t DBImpl::TEST_MaxNextLevelOverlappingBytes() { return versions_->MaxNextLevelOverlappingBytes(); } -Status DBImpl::Get(const ReadOptions& options, - const Slice& key, +Status DBImpl::Get(const ReadOptions& options, const Slice& key, std::string* value) { Status s; MutexLock l(&mutex_); SequenceNumber snapshot; - if (options.snapshot != NULL) { - snapshot = reinterpret_cast<const SnapshotImpl*>(options.snapshot)->number_; + if (options.snapshot != nullptr) { + snapshot = + static_cast<const SnapshotImpl*>(options.snapshot)->sequence_number(); } else { snapshot = versions_->LastSequence(); } @@ -1122,7 +1124,7 @@ Status DBImpl::Get(const ReadOptions& options, MemTable* imm = imm_; Version* current = versions_->current(); mem->Ref(); - if (imm != NULL) imm->Ref(); + if (imm != nullptr) imm->Ref(); current->Ref(); bool have_stat_update = false; @@ -1135,7 +1137,7 @@ Status DBImpl::Get(const ReadOptions& options, LookupKey lkey(key, snapshot); if (mem->Get(lkey, value, &s)) { // Done - } else if (imm != NULL && imm->Get(lkey, value, &s)) { + } else if (imm != nullptr && imm->Get(lkey, value, &s)) { // Done } else { s = current->Get(options, lkey, value, &stats); @@ -1148,7 +1150,7 @@ Status DBImpl::Get(const ReadOptions& options, MaybeScheduleCompaction(); } mem->Unref(); - if (imm != NULL) imm->Unref(); + if (imm != nullptr) imm->Unref(); current->Unref(); return s; } @@ -1157,12 +1159,12 @@ Iterator* DBImpl::NewIterator(const ReadOptions& options) { SequenceNumber latest_snapshot; uint32_t seed; Iterator* iter = NewInternalIterator(options, &latest_snapshot, &seed); - return NewDBIterator( - this, user_comparator(), iter, - (options.snapshot != NULL - ? reinterpret_cast<const SnapshotImpl*>(options.snapshot)->number_ - : latest_snapshot), - seed); + return NewDBIterator(this, user_comparator(), iter, + (options.snapshot != nullptr + ? static_cast<const SnapshotImpl*>(options.snapshot) + ->sequence_number() + : latest_snapshot), + seed); } void DBImpl::RecordReadSample(Slice key) { @@ -1177,9 +1179,9 @@ const Snapshot* DBImpl::GetSnapshot() { return snapshots_.New(versions_->LastSequence()); } -void DBImpl::ReleaseSnapshot(const Snapshot* s) { +void DBImpl::ReleaseSnapshot(const Snapshot* snapshot) { MutexLock l(&mutex_); - snapshots_.Delete(reinterpret_cast<const SnapshotImpl*>(s)); + snapshots_.Delete(static_cast<const SnapshotImpl*>(snapshot)); } // Convenience methods @@ -1191,9 +1193,9 @@ Status DBImpl::Delete(const WriteOptions& options, const Slice& key) { return DB::Delete(options, key); } -Status DBImpl::Write(const WriteOptions& options, WriteBatch* my_batch) { +Status DBImpl::Write(const WriteOptions& options, WriteBatch* updates) { Writer w(&mutex_); - w.batch = my_batch; + w.batch = updates; w.sync = options.sync; w.done = false; @@ -1207,13 +1209,13 @@ Status DBImpl::Write(const WriteOptions& options, WriteBatch* my_batch) { } // May temporarily unlock and wait. - Status status = MakeRoomForWrite(my_batch == NULL); + Status status = MakeRoomForWrite(updates == nullptr); uint64_t last_sequence = versions_->LastSequence(); Writer* last_writer = &w; - if (status.ok() && my_batch != NULL) { // NULL batch is for compactions - WriteBatch* updates = BuildBatchGroup(&last_writer); - WriteBatchInternal::SetSequence(updates, last_sequence + 1); - last_sequence += WriteBatchInternal::Count(updates); + if (status.ok() && updates != nullptr) { // nullptr batch is for compactions + WriteBatch* write_batch = BuildBatchGroup(&last_writer); + WriteBatchInternal::SetSequence(write_batch, last_sequence + 1); + last_sequence += WriteBatchInternal::Count(write_batch); // Add to log and apply to memtable. We can release the lock // during this phase since &w is currently responsible for logging @@ -1221,7 +1223,7 @@ Status DBImpl::Write(const WriteOptions& options, WriteBatch* my_batch) { // into mem_. { mutex_.Unlock(); - status = log_->AddRecord(WriteBatchInternal::Contents(updates)); + status = log_->AddRecord(WriteBatchInternal::Contents(write_batch)); bool sync_error = false; if (status.ok() && options.sync) { status = logfile_->Sync(); @@ -1230,7 +1232,7 @@ Status DBImpl::Write(const WriteOptions& options, WriteBatch* my_batch) { } } if (status.ok()) { - status = WriteBatchInternal::InsertInto(updates, mem_); + status = WriteBatchInternal::InsertInto(write_batch, mem_); } mutex_.Lock(); if (sync_error) { @@ -1240,7 +1242,7 @@ Status DBImpl::Write(const WriteOptions& options, WriteBatch* my_batch) { RecordBackgroundError(status); } } - if (updates == tmp_batch_) tmp_batch_->Clear(); + if (write_batch == tmp_batch_) tmp_batch_->Clear(); versions_->SetLastSequence(last_sequence); } @@ -1265,12 +1267,13 @@ Status DBImpl::Write(const WriteOptions& options, WriteBatch* my_batch) { } // REQUIRES: Writer list must be non-empty -// REQUIRES: First writer must have a non-NULL batch +// REQUIRES: First writer must have a non-null batch WriteBatch* DBImpl::BuildBatchGroup(Writer** last_writer) { + mutex_.AssertHeld(); assert(!writers_.empty()); Writer* first = writers_.front(); WriteBatch* result = first->batch; - assert(result != NULL); + assert(result != nullptr); size_t size = WriteBatchInternal::ByteSize(first->batch); @@ -1278,8 +1281,8 @@ WriteBatch* DBImpl::BuildBatchGroup(Writer** last_writer) { // original write is small, limit the growth so we do not slow // down the small write too much. size_t max_size = 1 << 20; - if (size <= (128<<10)) { - max_size = size + (128<<10); + if (size <= (128 << 10)) { + max_size = size + (128 << 10); } *last_writer = first; @@ -1292,7 +1295,7 @@ WriteBatch* DBImpl::BuildBatchGroup(Writer** last_writer) { break; } - if (w->batch != NULL) { + if (w->batch != nullptr) { size += WriteBatchInternal::ByteSize(w->batch); if (size > max_size) { // Do not make batch too big @@ -1325,9 +1328,8 @@ Status DBImpl::MakeRoomForWrite(bool force) { // Yield previous error s = bg_error_; break; - } else if ( - allow_delay && - versions_->NumLevelFiles(0) >= config::kL0_SlowdownWritesTrigger) { + } else if (allow_delay && versions_->NumLevelFiles(0) >= + config::kL0_SlowdownWritesTrigger) { // We are getting close to hitting a hard limit on the number of // L0 files. Rather than delaying a single write by several // seconds when we hit the hard limit, start delaying each @@ -1342,20 +1344,20 @@ Status DBImpl::MakeRoomForWrite(bool force) { (mem_->ApproximateMemoryUsage() <= options_.write_buffer_size)) { // There is room in current memtable break; - } else if (imm_ != NULL) { + } else if (imm_ != nullptr) { // We have filled up the current memtable, but the previous // one is still being compacted, so we wait. Log(options_.info_log, "Current memtable full; waiting...\n"); - bg_cv_.Wait(); + background_work_finished_signal_.Wait(); } else if (versions_->NumLevelFiles(0) >= config::kL0_StopWritesTrigger) { // There are too many level-0 files. Log(options_.info_log, "Too many L0 files; waiting...\n"); - bg_cv_.Wait(); + background_work_finished_signal_.Wait(); } else { // Attempt to switch to a new memtable and trigger compaction of old assert(versions_->PrevLogNumber() == 0); uint64_t new_log_number = versions_->NewFileNumber(); - WritableFile* lfile = NULL; + WritableFile* lfile = nullptr; s = env_->NewWritableFile(LogFileName(dbname_, new_log_number), &lfile); if (!s.ok()) { // Avoid chewing through file number space in a tight loop. @@ -1368,10 +1370,10 @@ Status DBImpl::MakeRoomForWrite(bool force) { logfile_number_ = new_log_number; log_ = new log::Writer(lfile); imm_ = mem_; - has_imm_.Release_Store(imm_); + has_imm_.store(true, std::memory_order_release); mem_ = new MemTable(internal_comparator_); mem_->Ref(); - force = false; // Do not force another compaction if have room + force = false; // Do not force another compaction if have room MaybeScheduleCompaction(); } } @@ -1405,21 +1407,16 @@ bool DBImpl::GetProperty(const Slice& property, std::string* value) { snprintf(buf, sizeof(buf), " Compactions\n" "Level Files Size(MB) Time(sec) Read(MB) Write(MB)\n" - "--------------------------------------------------\n" - ); + "--------------------------------------------------\n"); value->append(buf); for (int level = 0; level < config::kNumLevels; level++) { int files = versions_->NumLevelFiles(level); if (stats_[level].micros > 0 || files > 0) { - snprintf( - buf, sizeof(buf), - "%3d %8d %8.0f %9.0f %8.0f %9.0f\n", - level, - files, - versions_->NumLevelBytes(level) / 1048576.0, - stats_[level].micros / 1e6, - stats_[level].bytes_read / 1048576.0, - stats_[level].bytes_written / 1048576.0); + snprintf(buf, sizeof(buf), "%3d %8d %8.0f %9.0f %8.0f %9.0f\n", level, + files, versions_->NumLevelBytes(level) / 1048576.0, + stats_[level].micros / 1e6, + stats_[level].bytes_read / 1048576.0, + stats_[level].bytes_written / 1048576.0); value->append(buf); } } @@ -1445,16 +1442,11 @@ bool DBImpl::GetProperty(const Slice& property, std::string* value) { return false; } -void DBImpl::GetApproximateSizes( - const Range* range, int n, - uint64_t* sizes) { +void DBImpl::GetApproximateSizes(const Range* range, int n, uint64_t* sizes) { // TODO(opt): better implementation - Version* v; - { - MutexLock l(&mutex_); - versions_->current()->Ref(); - v = versions_->current(); - } + MutexLock l(&mutex_); + Version* v = versions_->current(); + v->Ref(); for (int i = 0; i < n; i++) { // Convert user_key into a corresponding internal key. @@ -1465,10 +1457,7 @@ void DBImpl::GetApproximateSizes( sizes[i] = (limit >= start ? limit - start : 0); } - { - MutexLock l(&mutex_); - v->Unref(); - } + v->Unref(); } // Default implementations of convenience methods that subclasses of DB @@ -1485,11 +1474,10 @@ Status DB::Delete(const WriteOptions& opt, const Slice& key) { return Write(opt, &batch); } -DB::~DB() { } +DB::~DB() = default; -Status DB::Open(const Options& options, const std::string& dbname, - DB** dbptr) { - *dbptr = NULL; +Status DB::Open(const Options& options, const std::string& dbname, DB** dbptr) { + *dbptr = nullptr; DBImpl* impl = new DBImpl(options, dbname); impl->mutex_.Lock(); @@ -1497,7 +1485,7 @@ Status DB::Open(const Options& options, const std::string& dbname, // Recover handles create_if_missing, error_if_exists bool save_manifest = false; Status s = impl->Recover(&edit, &save_manifest); - if (s.ok() && impl->mem_ == NULL) { + if (s.ok() && impl->mem_ == nullptr) { // Create new log and a corresponding memtable. uint64_t new_log_number = impl->versions_->NewFileNumber(); WritableFile* lfile; @@ -1523,7 +1511,7 @@ Status DB::Open(const Options& options, const std::string& dbname, } impl->mutex_.Unlock(); if (s.ok()) { - assert(impl->mem_ != NULL); + assert(impl->mem_ != nullptr); *dbptr = impl; } else { delete impl; @@ -1531,21 +1519,20 @@ Status DB::Open(const Options& options, const std::string& dbname, return s; } -Snapshot::~Snapshot() { -} +Snapshot::~Snapshot() = default; Status DestroyDB(const std::string& dbname, const Options& options) { Env* env = options.env; std::vector<std::string> filenames; - // Ignore error in case directory does not exist - env->GetChildren(dbname, &filenames); - if (filenames.empty()) { + Status result = env->GetChildren(dbname, &filenames); + if (!result.ok()) { + // Ignore error in case directory does not exist return Status::OK(); } FileLock* lock; const std::string lockname = LockFileName(dbname); - Status result = env->LockFile(lockname, &lock); + result = env->LockFile(lockname, &lock); if (result.ok()) { uint64_t number; FileType type; diff --git a/src/leveldb/db/db_impl.h b/src/leveldb/db/db_impl.h index 8ff323e728..685735c733 100644 --- a/src/leveldb/db/db_impl.h +++ b/src/leveldb/db/db_impl.h @@ -5,8 +5,11 @@ #ifndef STORAGE_LEVELDB_DB_DB_IMPL_H_ #define STORAGE_LEVELDB_DB_DB_IMPL_H_ +#include <atomic> #include <deque> #include <set> +#include <string> + #include "db/dbformat.h" #include "db/log_writer.h" #include "db/snapshot.h" @@ -26,21 +29,25 @@ class VersionSet; class DBImpl : public DB { public: DBImpl(const Options& options, const std::string& dbname); - virtual ~DBImpl(); + + DBImpl(const DBImpl&) = delete; + DBImpl& operator=(const DBImpl&) = delete; + + ~DBImpl() override; // Implementations of the DB interface - virtual Status Put(const WriteOptions&, const Slice& key, const Slice& value); - virtual Status Delete(const WriteOptions&, const Slice& key); - virtual Status Write(const WriteOptions& options, WriteBatch* updates); - virtual Status Get(const ReadOptions& options, - const Slice& key, - std::string* value); - virtual Iterator* NewIterator(const ReadOptions&); - virtual const Snapshot* GetSnapshot(); - virtual void ReleaseSnapshot(const Snapshot* snapshot); - virtual bool GetProperty(const Slice& property, std::string* value); - virtual void GetApproximateSizes(const Range* range, int n, uint64_t* sizes); - virtual void CompactRange(const Slice* begin, const Slice* end); + Status Put(const WriteOptions&, const Slice& key, + const Slice& value) override; + Status Delete(const WriteOptions&, const Slice& key) override; + Status Write(const WriteOptions& options, WriteBatch* updates) override; + Status Get(const ReadOptions& options, const Slice& key, + std::string* value) override; + Iterator* NewIterator(const ReadOptions&) override; + const Snapshot* GetSnapshot() override; + void ReleaseSnapshot(const Snapshot* snapshot) override; + bool GetProperty(const Slice& property, std::string* value) override; + void GetApproximateSizes(const Range* range, int n, uint64_t* sizes) override; + void CompactRange(const Slice* begin, const Slice* end) override; // Extra methods (for testing) that are not in the public DB interface @@ -69,6 +76,31 @@ class DBImpl : public DB { struct CompactionState; struct Writer; + // Information for a manual compaction + struct ManualCompaction { + int level; + bool done; + const InternalKey* begin; // null means beginning of key range + const InternalKey* end; // null means end of key range + InternalKey tmp_storage; // Used to keep track of compaction progress + }; + + // Per level compaction stats. stats_[level] stores the stats for + // compactions that produced data for the specified "level". + struct CompactionStats { + CompactionStats() : micros(0), bytes_read(0), bytes_written(0) {} + + void Add(const CompactionStats& c) { + this->micros += c.micros; + this->bytes_read += c.bytes_read; + this->bytes_written += c.bytes_written; + } + + int64_t micros; + int64_t bytes_read; + int64_t bytes_written; + }; + Iterator* NewInternalIterator(const ReadOptions&, SequenceNumber* latest_snapshot, uint32_t* seed); @@ -84,7 +116,7 @@ class DBImpl : public DB { void MaybeIgnoreError(Status* s) const; // Delete any unneeded files and stale in-memory entries. - void DeleteObsoleteFiles(); + void DeleteObsoleteFiles() EXCLUSIVE_LOCKS_REQUIRED(mutex_); // Compact the in-memory write buffer to disk. Switches to a new // log-file/memtable and writes a new descriptor iff successful. @@ -100,14 +132,15 @@ class DBImpl : public DB { Status MakeRoomForWrite(bool force /* compact even if there is room? */) EXCLUSIVE_LOCKS_REQUIRED(mutex_); - WriteBatch* BuildBatchGroup(Writer** last_writer); + WriteBatch* BuildBatchGroup(Writer** last_writer) + EXCLUSIVE_LOCKS_REQUIRED(mutex_); void RecordBackgroundError(const Status& s); void MaybeScheduleCompaction() EXCLUSIVE_LOCKS_REQUIRED(mutex_); static void BGWork(void* db); void BackgroundCall(); - void BackgroundCompaction() EXCLUSIVE_LOCKS_REQUIRED(mutex_); + void BackgroundCompaction() EXCLUSIVE_LOCKS_REQUIRED(mutex_); void CleanupCompaction(CompactionState* compact) EXCLUSIVE_LOCKS_REQUIRED(mutex_); Status DoCompactionWork(CompactionState* compact) @@ -118,93 +151,66 @@ class DBImpl : public DB { Status InstallCompactionResults(CompactionState* compact) EXCLUSIVE_LOCKS_REQUIRED(mutex_); + const Comparator* user_comparator() const { + return internal_comparator_.user_comparator(); + } + // Constant after construction Env* const env_; const InternalKeyComparator internal_comparator_; const InternalFilterPolicy internal_filter_policy_; const Options options_; // options_.comparator == &internal_comparator_ - bool owns_info_log_; - bool owns_cache_; + const bool owns_info_log_; + const bool owns_cache_; const std::string dbname_; // table_cache_ provides its own synchronization - TableCache* table_cache_; + TableCache* const table_cache_; - // Lock over the persistent DB state. Non-NULL iff successfully acquired. + // Lock over the persistent DB state. Non-null iff successfully acquired. FileLock* db_lock_; // State below is protected by mutex_ port::Mutex mutex_; - port::AtomicPointer shutting_down_; - port::CondVar bg_cv_; // Signalled when background work finishes + std::atomic<bool> shutting_down_; + port::CondVar background_work_finished_signal_ GUARDED_BY(mutex_); MemTable* mem_; - MemTable* imm_; // Memtable being compacted - port::AtomicPointer has_imm_; // So bg thread can detect non-NULL imm_ + MemTable* imm_ GUARDED_BY(mutex_); // Memtable being compacted + std::atomic<bool> has_imm_; // So bg thread can detect non-null imm_ WritableFile* logfile_; - uint64_t logfile_number_; + uint64_t logfile_number_ GUARDED_BY(mutex_); log::Writer* log_; - uint32_t seed_; // For sampling. + uint32_t seed_ GUARDED_BY(mutex_); // For sampling. // Queue of writers. - std::deque<Writer*> writers_; - WriteBatch* tmp_batch_; + std::deque<Writer*> writers_ GUARDED_BY(mutex_); + WriteBatch* tmp_batch_ GUARDED_BY(mutex_); - SnapshotList snapshots_; + SnapshotList snapshots_ GUARDED_BY(mutex_); // Set of table files to protect from deletion because they are // part of ongoing compactions. - std::set<uint64_t> pending_outputs_; + std::set<uint64_t> pending_outputs_ GUARDED_BY(mutex_); // Has a background compaction been scheduled or is running? - bool bg_compaction_scheduled_; + bool background_compaction_scheduled_ GUARDED_BY(mutex_); - // Information for a manual compaction - struct ManualCompaction { - int level; - bool done; - const InternalKey* begin; // NULL means beginning of key range - const InternalKey* end; // NULL means end of key range - InternalKey tmp_storage; // Used to keep track of compaction progress - }; - ManualCompaction* manual_compaction_; + ManualCompaction* manual_compaction_ GUARDED_BY(mutex_); - VersionSet* versions_; + VersionSet* const versions_ GUARDED_BY(mutex_); // Have we encountered a background error in paranoid mode? - Status bg_error_; - - // Per level compaction stats. stats_[level] stores the stats for - // compactions that produced data for the specified "level". - struct CompactionStats { - int64_t micros; - int64_t bytes_read; - int64_t bytes_written; - - CompactionStats() : micros(0), bytes_read(0), bytes_written(0) { } + Status bg_error_ GUARDED_BY(mutex_); - void Add(const CompactionStats& c) { - this->micros += c.micros; - this->bytes_read += c.bytes_read; - this->bytes_written += c.bytes_written; - } - }; - CompactionStats stats_[config::kNumLevels]; - - // No copying allowed - DBImpl(const DBImpl&); - void operator=(const DBImpl&); - - const Comparator* user_comparator() const { - return internal_comparator_.user_comparator(); - } + CompactionStats stats_[config::kNumLevels] GUARDED_BY(mutex_); }; // Sanitize db options. The caller should delete result.info_log if // it is not equal to src.info_log. -extern Options SanitizeOptions(const std::string& db, - const InternalKeyComparator* icmp, - const InternalFilterPolicy* ipolicy, - const Options& src); +Options SanitizeOptions(const std::string& db, + const InternalKeyComparator* icmp, + const InternalFilterPolicy* ipolicy, + const Options& src); } // namespace leveldb diff --git a/src/leveldb/db/db_iter.cc b/src/leveldb/db/db_iter.cc index 3b2035e9e3..98715a9502 100644 --- a/src/leveldb/db/db_iter.cc +++ b/src/leveldb/db/db_iter.cc @@ -4,9 +4,9 @@ #include "db/db_iter.h" -#include "db/filename.h" #include "db/db_impl.h" #include "db/dbformat.h" +#include "db/filename.h" #include "leveldb/env.h" #include "leveldb/iterator.h" #include "port/port.h" @@ -36,17 +36,14 @@ namespace { // combines multiple entries for the same userkey found in the DB // representation into a single entry while accounting for sequence // numbers, deletion markers, overwrites, etc. -class DBIter: public Iterator { +class DBIter : public Iterator { public: // Which direction is the iterator currently moving? // (1) When moving forward, the internal iterator is positioned at // the exact entry that yields this->key(), this->value() // (2) When moving backwards, the internal iterator is positioned // just before all entries whose user key == this->key(). - enum Direction { - kForward, - kReverse - }; + enum Direction { kForward, kReverse }; DBIter(DBImpl* db, const Comparator* cmp, Iterator* iter, SequenceNumber s, uint32_t seed) @@ -57,21 +54,22 @@ class DBIter: public Iterator { direction_(kForward), valid_(false), rnd_(seed), - bytes_counter_(RandomPeriod()) { - } - virtual ~DBIter() { - delete iter_; - } - virtual bool Valid() const { return valid_; } - virtual Slice key() const { + bytes_until_read_sampling_(RandomCompactionPeriod()) {} + + DBIter(const DBIter&) = delete; + DBIter& operator=(const DBIter&) = delete; + + ~DBIter() override { delete iter_; } + bool Valid() const override { return valid_; } + Slice key() const override { assert(valid_); return (direction_ == kForward) ? ExtractUserKey(iter_->key()) : saved_key_; } - virtual Slice value() const { + Slice value() const override { assert(valid_); return (direction_ == kForward) ? iter_->value() : saved_value_; } - virtual Status status() const { + Status status() const override { if (status_.ok()) { return iter_->status(); } else { @@ -79,11 +77,11 @@ class DBIter: public Iterator { } } - virtual void Next(); - virtual void Prev(); - virtual void Seek(const Slice& target); - virtual void SeekToFirst(); - virtual void SeekToLast(); + void Next() override; + void Prev() override; + void Seek(const Slice& target) override; + void SeekToFirst() override; + void SeekToLast() override; private: void FindNextUserEntry(bool skipping, std::string* skip); @@ -103,38 +101,35 @@ class DBIter: public Iterator { } } - // Pick next gap with average value of config::kReadBytesPeriod. - ssize_t RandomPeriod() { - return rnd_.Uniform(2*config::kReadBytesPeriod); + // Picks the number of bytes that can be read until a compaction is scheduled. + size_t RandomCompactionPeriod() { + return rnd_.Uniform(2 * config::kReadBytesPeriod); } DBImpl* db_; const Comparator* const user_comparator_; Iterator* const iter_; SequenceNumber const sequence_; - Status status_; - std::string saved_key_; // == current key when direction_==kReverse - std::string saved_value_; // == current raw value when direction_==kReverse + std::string saved_key_; // == current key when direction_==kReverse + std::string saved_value_; // == current raw value when direction_==kReverse Direction direction_; bool valid_; - Random rnd_; - ssize_t bytes_counter_; - - // No copying allowed - DBIter(const DBIter&); - void operator=(const DBIter&); + size_t bytes_until_read_sampling_; }; inline bool DBIter::ParseKey(ParsedInternalKey* ikey) { Slice k = iter_->key(); - ssize_t n = k.size() + iter_->value().size(); - bytes_counter_ -= n; - while (bytes_counter_ < 0) { - bytes_counter_ += RandomPeriod(); + + size_t bytes_read = k.size() + iter_->value().size(); + while (bytes_until_read_sampling_ < bytes_read) { + bytes_until_read_sampling_ += RandomCompactionPeriod(); db_->RecordReadSample(k); } + assert(bytes_until_read_sampling_ >= bytes_read); + bytes_until_read_sampling_ -= bytes_read; + if (!ParseInternalKey(k, ikey)) { status_ = Status::Corruption("corrupted internal key in DBIter"); return false; @@ -165,6 +160,15 @@ void DBIter::Next() { } else { // Store in saved_key_ the current key so we skip it below. SaveKey(ExtractUserKey(iter_->key()), &saved_key_); + + // iter_ is pointing to current key. We can now safely move to the next to + // avoid checking current key. + iter_->Next(); + if (!iter_->Valid()) { + valid_ = false; + saved_key_.clear(); + return; + } } FindNextUserEntry(true, &saved_key_); @@ -218,8 +222,8 @@ void DBIter::Prev() { ClearSavedValue(); return; } - if (user_comparator_->Compare(ExtractUserKey(iter_->key()), - saved_key_) < 0) { + if (user_comparator_->Compare(ExtractUserKey(iter_->key()), saved_key_) < + 0) { break; } } @@ -275,8 +279,8 @@ void DBIter::Seek(const Slice& target) { direction_ = kForward; ClearSavedValue(); saved_key_.clear(); - AppendInternalKey( - &saved_key_, ParsedInternalKey(target, sequence_, kValueTypeForSeek)); + AppendInternalKey(&saved_key_, + ParsedInternalKey(target, sequence_, kValueTypeForSeek)); iter_->Seek(saved_key_); if (iter_->Valid()) { FindNextUserEntry(false, &saved_key_ /* temporary storage */); @@ -305,12 +309,9 @@ void DBIter::SeekToLast() { } // anonymous namespace -Iterator* NewDBIterator( - DBImpl* db, - const Comparator* user_key_comparator, - Iterator* internal_iter, - SequenceNumber sequence, - uint32_t seed) { +Iterator* NewDBIterator(DBImpl* db, const Comparator* user_key_comparator, + Iterator* internal_iter, SequenceNumber sequence, + uint32_t seed) { return new DBIter(db, user_key_comparator, internal_iter, sequence, seed); } diff --git a/src/leveldb/db/db_iter.h b/src/leveldb/db/db_iter.h index 04927e937b..fd93e912a0 100644 --- a/src/leveldb/db/db_iter.h +++ b/src/leveldb/db/db_iter.h @@ -6,8 +6,9 @@ #define STORAGE_LEVELDB_DB_DB_ITER_H_ #include <stdint.h> -#include "leveldb/db.h" + #include "db/dbformat.h" +#include "leveldb/db.h" namespace leveldb { @@ -16,12 +17,9 @@ class DBImpl; // Return a new iterator that converts internal keys (yielded by // "*internal_iter") that were live at the specified "sequence" number // into appropriate user keys. -extern Iterator* NewDBIterator( - DBImpl* db, - const Comparator* user_key_comparator, - Iterator* internal_iter, - SequenceNumber sequence, - uint32_t seed); +Iterator* NewDBIterator(DBImpl* db, const Comparator* user_key_comparator, + Iterator* internal_iter, SequenceNumber sequence, + uint32_t seed); } // namespace leveldb diff --git a/src/leveldb/db/db_test.cc b/src/leveldb/db/db_test.cc index a0b08bc19c..beb1d3bdef 100644 --- a/src/leveldb/db/db_test.cc +++ b/src/leveldb/db/db_test.cc @@ -3,14 +3,20 @@ // found in the LICENSE file. See the AUTHORS file for names of contributors. #include "leveldb/db.h" -#include "leveldb/filter_policy.h" + +#include <atomic> +#include <string> + #include "db/db_impl.h" #include "db/filename.h" #include "db/version_set.h" #include "db/write_batch_internal.h" #include "leveldb/cache.h" #include "leveldb/env.h" +#include "leveldb/filter_policy.h" #include "leveldb/table.h" +#include "port/port.h" +#include "port/thread_annotations.h" #include "util/hash.h" #include "util/logging.h" #include "util/mutexlock.h" @@ -25,83 +31,116 @@ static std::string RandomString(Random* rnd, int len) { return r; } +static std::string RandomKey(Random* rnd) { + int len = + (rnd->OneIn(3) ? 1 // Short sometimes to encourage collisions + : (rnd->OneIn(100) ? rnd->Skewed(10) : rnd->Uniform(10))); + return test::RandomKey(rnd, len); +} + namespace { class AtomicCounter { - private: - port::Mutex mu_; - int count_; public: - AtomicCounter() : count_(0) { } - void Increment() { - IncrementBy(1); - } - void IncrementBy(int count) { + AtomicCounter() : count_(0) {} + void Increment() { IncrementBy(1); } + void IncrementBy(int count) LOCKS_EXCLUDED(mu_) { MutexLock l(&mu_); count_ += count; } - int Read() { + int Read() LOCKS_EXCLUDED(mu_) { MutexLock l(&mu_); return count_; } - void Reset() { + void Reset() LOCKS_EXCLUDED(mu_) { MutexLock l(&mu_); count_ = 0; } + + private: + port::Mutex mu_; + int count_ GUARDED_BY(mu_); }; void DelayMilliseconds(int millis) { Env::Default()->SleepForMicroseconds(millis * 1000); } -} +} // namespace + +// Test Env to override default Env behavior for testing. +class TestEnv : public EnvWrapper { + public: + explicit TestEnv(Env* base) : EnvWrapper(base), ignore_dot_files_(false) {} + + void SetIgnoreDotFiles(bool ignored) { ignore_dot_files_ = ignored; } + + Status GetChildren(const std::string& dir, + std::vector<std::string>* result) override { + Status s = target()->GetChildren(dir, result); + if (!s.ok() || !ignore_dot_files_) { + return s; + } + + std::vector<std::string>::iterator it = result->begin(); + while (it != result->end()) { + if ((*it == ".") || (*it == "..")) { + it = result->erase(it); + } else { + ++it; + } + } + + return s; + } + + private: + bool ignore_dot_files_; +}; -// Special Env used to delay background operations +// Special Env used to delay background operations. class SpecialEnv : public EnvWrapper { public: - // sstable/log Sync() calls are blocked while this pointer is non-NULL. - port::AtomicPointer delay_data_sync_; + // sstable/log Sync() calls are blocked while this pointer is non-null. + std::atomic<bool> delay_data_sync_; // sstable/log Sync() calls return an error. - port::AtomicPointer data_sync_error_; + std::atomic<bool> data_sync_error_; - // Simulate no-space errors while this pointer is non-NULL. - port::AtomicPointer no_space_; + // Simulate no-space errors while this pointer is non-null. + std::atomic<bool> no_space_; - // Simulate non-writable file system while this pointer is non-NULL - port::AtomicPointer non_writable_; + // Simulate non-writable file system while this pointer is non-null. + std::atomic<bool> non_writable_; - // Force sync of manifest files to fail while this pointer is non-NULL - port::AtomicPointer manifest_sync_error_; + // Force sync of manifest files to fail while this pointer is non-null. + std::atomic<bool> manifest_sync_error_; - // Force write to manifest files to fail while this pointer is non-NULL - port::AtomicPointer manifest_write_error_; + // Force write to manifest files to fail while this pointer is non-null. + std::atomic<bool> manifest_write_error_; bool count_random_reads_; AtomicCounter random_read_counter_; - explicit SpecialEnv(Env* base) : EnvWrapper(base) { - delay_data_sync_.Release_Store(NULL); - data_sync_error_.Release_Store(NULL); - no_space_.Release_Store(NULL); - non_writable_.Release_Store(NULL); - count_random_reads_ = false; - manifest_sync_error_.Release_Store(NULL); - manifest_write_error_.Release_Store(NULL); - } + explicit SpecialEnv(Env* base) + : EnvWrapper(base), + delay_data_sync_(false), + data_sync_error_(false), + no_space_(false), + non_writable_(false), + manifest_sync_error_(false), + manifest_write_error_(false), + count_random_reads_(false) {} Status NewWritableFile(const std::string& f, WritableFile** r) { class DataFile : public WritableFile { private: - SpecialEnv* env_; - WritableFile* base_; + SpecialEnv* const env_; + WritableFile* const base_; public: - DataFile(SpecialEnv* env, WritableFile* base) - : env_(env), - base_(base) { - } + DataFile(SpecialEnv* env, WritableFile* base) : env_(env), base_(base) {} ~DataFile() { delete base_; } Status Append(const Slice& data) { - if (env_->no_space_.Acquire_Load() != NULL) { + if (env_->no_space_.load(std::memory_order_acquire)) { // Drop writes on the floor return Status::OK(); } else { @@ -111,24 +150,26 @@ class SpecialEnv : public EnvWrapper { Status Close() { return base_->Close(); } Status Flush() { return base_->Flush(); } Status Sync() { - if (env_->data_sync_error_.Acquire_Load() != NULL) { + if (env_->data_sync_error_.load(std::memory_order_acquire)) { return Status::IOError("simulated data sync error"); } - while (env_->delay_data_sync_.Acquire_Load() != NULL) { + while (env_->delay_data_sync_.load(std::memory_order_acquire)) { DelayMilliseconds(100); } return base_->Sync(); } + std::string GetName() const override { return ""; } }; class ManifestFile : public WritableFile { private: SpecialEnv* env_; WritableFile* base_; + public: - ManifestFile(SpecialEnv* env, WritableFile* b) : env_(env), base_(b) { } + ManifestFile(SpecialEnv* env, WritableFile* b) : env_(env), base_(b) {} ~ManifestFile() { delete base_; } Status Append(const Slice& data) { - if (env_->manifest_write_error_.Acquire_Load() != NULL) { + if (env_->manifest_write_error_.load(std::memory_order_acquire)) { return Status::IOError("simulated writer error"); } else { return base_->Append(data); @@ -137,24 +178,25 @@ class SpecialEnv : public EnvWrapper { Status Close() { return base_->Close(); } Status Flush() { return base_->Flush(); } Status Sync() { - if (env_->manifest_sync_error_.Acquire_Load() != NULL) { + if (env_->manifest_sync_error_.load(std::memory_order_acquire)) { return Status::IOError("simulated sync error"); } else { return base_->Sync(); } } + std::string GetName() const override { return ""; } }; - if (non_writable_.Acquire_Load() != NULL) { + if (non_writable_.load(std::memory_order_acquire)) { return Status::IOError("simulated write error"); } Status s = target()->NewWritableFile(f, r); if (s.ok()) { - if (strstr(f.c_str(), ".ldb") != NULL || - strstr(f.c_str(), ".log") != NULL) { + if (strstr(f.c_str(), ".ldb") != nullptr || + strstr(f.c_str(), ".log") != nullptr) { *r = new DataFile(this, *r); - } else if (strstr(f.c_str(), "MANIFEST") != NULL) { + } else if (strstr(f.c_str(), "MANIFEST") != nullptr) { *r = new ManifestFile(this, *r); } } @@ -166,16 +208,17 @@ class SpecialEnv : public EnvWrapper { private: RandomAccessFile* target_; AtomicCounter* counter_; + public: CountingFile(RandomAccessFile* target, AtomicCounter* counter) - : target_(target), counter_(counter) { - } - virtual ~CountingFile() { delete target_; } - virtual Status Read(uint64_t offset, size_t n, Slice* result, - char* scratch) const { + : target_(target), counter_(counter) {} + ~CountingFile() override { delete target_; } + Status Read(uint64_t offset, size_t n, Slice* result, + char* scratch) const override { counter_->Increment(); return target_->Read(offset, n, result, scratch); } + std::string GetName() const override { return ""; } }; Status s = target()->NewRandomAccessFile(f, r); @@ -187,19 +230,6 @@ class SpecialEnv : public EnvWrapper { }; class DBTest { - private: - const FilterPolicy* filter_policy_; - - // Sequence of option configurations to try - enum OptionConfig { - kDefault, - kReuse, - kFilter, - kUncompressed, - kEnd - }; - int option_config_; - public: std::string dbname_; SpecialEnv* env_; @@ -207,12 +237,11 @@ class DBTest { Options last_options_; - DBTest() : option_config_(kDefault), - env_(new SpecialEnv(Env::Default())) { + DBTest() : env_(new SpecialEnv(Env::Default())), option_config_(kDefault) { filter_policy_ = NewBloomFilterPolicy(10); dbname_ = test::TmpDir() + "/db_test"; DestroyDB(dbname_, Options()); - db_ = NULL; + db_ = nullptr; Reopen(); } @@ -255,31 +284,27 @@ class DBTest { return options; } - DBImpl* dbfull() { - return reinterpret_cast<DBImpl*>(db_); - } + DBImpl* dbfull() { return reinterpret_cast<DBImpl*>(db_); } - void Reopen(Options* options = NULL) { - ASSERT_OK(TryReopen(options)); - } + void Reopen(Options* options = nullptr) { ASSERT_OK(TryReopen(options)); } void Close() { delete db_; - db_ = NULL; + db_ = nullptr; } - void DestroyAndReopen(Options* options = NULL) { + void DestroyAndReopen(Options* options = nullptr) { delete db_; - db_ = NULL; + db_ = nullptr; DestroyDB(dbname_, Options()); ASSERT_OK(TryReopen(options)); } Status TryReopen(Options* options) { delete db_; - db_ = NULL; + db_ = nullptr; Options opts; - if (options != NULL) { + if (options != nullptr) { opts = *options; } else { opts = CurrentOptions(); @@ -294,11 +319,9 @@ class DBTest { return db_->Put(WriteOptions(), k, v); } - Status Delete(const std::string& k) { - return db_->Delete(WriteOptions(), k); - } + Status Delete(const std::string& k) { return db_->Delete(WriteOptions(), k); } - std::string Get(const std::string& k, const Snapshot* snapshot = NULL) { + std::string Get(const std::string& k, const Snapshot* snapshot = nullptr) { ReadOptions options; options.snapshot = snapshot; std::string result; @@ -382,10 +405,9 @@ class DBTest { int NumTableFilesAtLevel(int level) { std::string property; - ASSERT_TRUE( - db_->GetProperty("leveldb.num-files-at-level" + NumberToString(level), - &property)); - return atoi(property.c_str()); + ASSERT_TRUE(db_->GetProperty( + "leveldb.num-files-at-level" + NumberToString(level), &property)); + return std::stoi(property); } int TotalTableFiles() { @@ -431,11 +453,12 @@ class DBTest { } // Do n memtable compactions, each of which produces an sstable - // covering the range [small,large]. - void MakeTables(int n, const std::string& small, const std::string& large) { + // covering the range [small_key,large_key]. + void MakeTables(int n, const std::string& small_key, + const std::string& large_key) { for (int i = 0; i < n; i++) { - Put(small, "begin"); - Put(large, "end"); + Put(small_key, "begin"); + Put(large_key, "end"); dbfull()->TEST_CompactMemTable(); } } @@ -448,9 +471,9 @@ class DBTest { void DumpFileCounts(const char* label) { fprintf(stderr, "---\n%s:\n", label); - fprintf(stderr, "maxoverlap: %lld\n", - static_cast<long long>( - dbfull()->TEST_MaxNextLevelOverlappingBytes())); + fprintf( + stderr, "maxoverlap: %lld\n", + static_cast<long long>(dbfull()->TEST_MaxNextLevelOverlappingBytes())); for (int level = 0; level < config::kNumLevels; level++) { int num = NumTableFilesAtLevel(level); if (num > 0) { @@ -506,15 +529,42 @@ class DBTest { } return files_renamed; } + + private: + // Sequence of option configurations to try + enum OptionConfig { kDefault, kReuse, kFilter, kUncompressed, kEnd }; + + const FilterPolicy* filter_policy_; + int option_config_; }; TEST(DBTest, Empty) { do { - ASSERT_TRUE(db_ != NULL); + ASSERT_TRUE(db_ != nullptr); ASSERT_EQ("NOT_FOUND", Get("foo")); } while (ChangeOptions()); } +TEST(DBTest, EmptyKey) { + do { + ASSERT_OK(Put("", "v1")); + ASSERT_EQ("v1", Get("")); + ASSERT_OK(Put("", "v2")); + ASSERT_EQ("v2", Get("")); + } while (ChangeOptions()); +} + +TEST(DBTest, EmptyValue) { + do { + ASSERT_OK(Put("key", "v1")); + ASSERT_EQ("v1", Get("key")); + ASSERT_OK(Put("key", "")); + ASSERT_EQ("", Get("key")); + ASSERT_OK(Put("key", "v2")); + ASSERT_EQ("v2", Get("key")); + } while (ChangeOptions()); +} + TEST(DBTest, ReadWrite) { do { ASSERT_OK(Put("foo", "v1")); @@ -547,11 +597,13 @@ TEST(DBTest, GetFromImmutableLayer) { ASSERT_OK(Put("foo", "v1")); ASSERT_EQ("v1", Get("foo")); - env_->delay_data_sync_.Release_Store(env_); // Block sync calls - Put("k1", std::string(100000, 'x')); // Fill memtable - Put("k2", std::string(100000, 'y')); // Trigger compaction + // Block sync calls. + env_->delay_data_sync_.store(true, std::memory_order_release); + Put("k1", std::string(100000, 'x')); // Fill memtable. + Put("k2", std::string(100000, 'y')); // Trigger compaction. ASSERT_EQ("v1", Get("foo")); - env_->delay_data_sync_.Release_Store(NULL); // Release sync calls + // Release sync calls. + env_->delay_data_sync_.store(false, std::memory_order_release); } while (ChangeOptions()); } @@ -568,9 +620,9 @@ TEST(DBTest, GetMemUsage) { ASSERT_OK(Put("foo", "v1")); std::string val; ASSERT_TRUE(db_->GetProperty("leveldb.approximate-memory-usage", &val)); - int mem_usage = atoi(val.c_str()); + int mem_usage = std::stoi(val); ASSERT_GT(mem_usage, 0); - ASSERT_LT(mem_usage, 5*1024*1024); + ASSERT_LT(mem_usage, 5 * 1024 * 1024); } while (ChangeOptions()); } @@ -592,6 +644,55 @@ TEST(DBTest, GetSnapshot) { } while (ChangeOptions()); } +TEST(DBTest, GetIdenticalSnapshots) { + do { + // Try with both a short key and a long key + for (int i = 0; i < 2; i++) { + std::string key = (i == 0) ? std::string("foo") : std::string(200, 'x'); + ASSERT_OK(Put(key, "v1")); + const Snapshot* s1 = db_->GetSnapshot(); + const Snapshot* s2 = db_->GetSnapshot(); + const Snapshot* s3 = db_->GetSnapshot(); + ASSERT_OK(Put(key, "v2")); + ASSERT_EQ("v2", Get(key)); + ASSERT_EQ("v1", Get(key, s1)); + ASSERT_EQ("v1", Get(key, s2)); + ASSERT_EQ("v1", Get(key, s3)); + db_->ReleaseSnapshot(s1); + dbfull()->TEST_CompactMemTable(); + ASSERT_EQ("v2", Get(key)); + ASSERT_EQ("v1", Get(key, s2)); + db_->ReleaseSnapshot(s2); + ASSERT_EQ("v1", Get(key, s3)); + db_->ReleaseSnapshot(s3); + } + } while (ChangeOptions()); +} + +TEST(DBTest, IterateOverEmptySnapshot) { + do { + const Snapshot* snapshot = db_->GetSnapshot(); + ReadOptions read_options; + read_options.snapshot = snapshot; + ASSERT_OK(Put("foo", "v1")); + ASSERT_OK(Put("foo", "v2")); + + Iterator* iterator1 = db_->NewIterator(read_options); + iterator1->SeekToFirst(); + ASSERT_TRUE(!iterator1->Valid()); + delete iterator1; + + dbfull()->TEST_CompactMemTable(); + + Iterator* iterator2 = db_->NewIterator(read_options); + iterator2->SeekToFirst(); + ASSERT_TRUE(!iterator2->Valid()); + delete iterator2; + + db_->ReleaseSnapshot(snapshot); + } while (ChangeOptions()); +} + TEST(DBTest, GetLevel0Ordering) { do { // Check that we process level-0 files in correct order. The code @@ -646,8 +747,7 @@ TEST(DBTest, GetEncountersEmptyLevel) { // Step 1: First place sstables in levels 0 and 2 int compaction_count = 0; - while (NumTableFilesAtLevel(0) == 0 || - NumTableFilesAtLevel(2) == 0) { + while (NumTableFilesAtLevel(0) == 0 || NumTableFilesAtLevel(2) == 0) { ASSERT_LE(compaction_count, 100) << "could not fill levels 0 and 2"; compaction_count++; Put("a", "begin"); @@ -656,7 +756,7 @@ TEST(DBTest, GetEncountersEmptyLevel) { } // Step 2: clear level 1 if necessary. - dbfull()->TEST_CompactRange(1, NULL, NULL); + dbfull()->TEST_CompactRange(1, nullptr, nullptr); ASSERT_EQ(NumTableFilesAtLevel(0), 1); ASSERT_EQ(NumTableFilesAtLevel(1), 0); ASSERT_EQ(NumTableFilesAtLevel(2), 1); @@ -784,10 +884,10 @@ TEST(DBTest, IterMulti) { ASSERT_EQ(IterStatus(iter), "b->vb"); // Make sure iter stays at snapshot - ASSERT_OK(Put("a", "va2")); + ASSERT_OK(Put("a", "va2")); ASSERT_OK(Put("a2", "va3")); - ASSERT_OK(Put("b", "vb2")); - ASSERT_OK(Put("c", "vc2")); + ASSERT_OK(Put("b", "vb2")); + ASSERT_OK(Put("c", "vc2")); ASSERT_OK(Delete("b")); iter->SeekToFirst(); ASSERT_EQ(IterStatus(iter), "a->va"); @@ -978,7 +1078,7 @@ TEST(DBTest, RecoverWithLargeLog) { TEST(DBTest, CompactionsGenerateMultipleFiles) { Options options = CurrentOptions(); - options.write_buffer_size = 100000000; // Large write buffer + options.write_buffer_size = 100000000; // Large write buffer Reopen(&options); Random rnd(301); @@ -993,7 +1093,7 @@ TEST(DBTest, CompactionsGenerateMultipleFiles) { // Reopening moves updates to level-0 Reopen(&options); - dbfull()->TEST_CompactRange(0, NULL, NULL); + dbfull()->TEST_CompactRange(0, nullptr, nullptr); ASSERT_EQ(NumTableFilesAtLevel(0), 0); ASSERT_GT(NumTableFilesAtLevel(1), 1); @@ -1017,7 +1117,7 @@ TEST(DBTest, RepeatedWritesToSameKey) { for (int i = 0; i < 5 * kMaxFiles; i++) { Put("key", value); ASSERT_LE(TotalTableFiles(), kMaxFiles); - fprintf(stderr, "after %d: %d files\n", int(i+1), TotalTableFiles()); + fprintf(stderr, "after %d: %d files\n", i + 1, TotalTableFiles()); } } @@ -1044,29 +1144,28 @@ TEST(DBTest, SparseMerge) { } Put("C", "vc"); dbfull()->TEST_CompactMemTable(); - dbfull()->TEST_CompactRange(0, NULL, NULL); + dbfull()->TEST_CompactRange(0, nullptr, nullptr); // Make sparse update - Put("A", "va2"); + Put("A", "va2"); Put("B100", "bvalue2"); - Put("C", "vc2"); + Put("C", "vc2"); dbfull()->TEST_CompactMemTable(); // Compactions should not cause us to create a situation where // a file overlaps too much data at the next level. - ASSERT_LE(dbfull()->TEST_MaxNextLevelOverlappingBytes(), 20*1048576); - dbfull()->TEST_CompactRange(0, NULL, NULL); - ASSERT_LE(dbfull()->TEST_MaxNextLevelOverlappingBytes(), 20*1048576); - dbfull()->TEST_CompactRange(1, NULL, NULL); - ASSERT_LE(dbfull()->TEST_MaxNextLevelOverlappingBytes(), 20*1048576); + ASSERT_LE(dbfull()->TEST_MaxNextLevelOverlappingBytes(), 20 * 1048576); + dbfull()->TEST_CompactRange(0, nullptr, nullptr); + ASSERT_LE(dbfull()->TEST_MaxNextLevelOverlappingBytes(), 20 * 1048576); + dbfull()->TEST_CompactRange(1, nullptr, nullptr); + ASSERT_LE(dbfull()->TEST_MaxNextLevelOverlappingBytes(), 20 * 1048576); } static bool Between(uint64_t val, uint64_t low, uint64_t high) { bool result = (val >= low) && (val <= high); if (!result) { fprintf(stderr, "Value %llu is not in range [%llu, %llu]\n", - (unsigned long long)(val), - (unsigned long long)(low), + (unsigned long long)(val), (unsigned long long)(low), (unsigned long long)(high)); } return result; @@ -1075,7 +1174,7 @@ static bool Between(uint64_t val, uint64_t low, uint64_t high) { TEST(DBTest, ApproximateSizes) { do { Options options = CurrentOptions(); - options.write_buffer_size = 100000000; // Large write buffer + options.write_buffer_size = 100000000; // Large write buffer options.compression = kNoCompression; DestroyAndReopen(); @@ -1110,12 +1209,13 @@ TEST(DBTest, ApproximateSizes) { for (int compact_start = 0; compact_start < N; compact_start += 10) { for (int i = 0; i < N; i += 10) { - ASSERT_TRUE(Between(Size("", Key(i)), S1*i, S2*i)); - ASSERT_TRUE(Between(Size("", Key(i)+".suffix"), S1*(i+1), S2*(i+1))); - ASSERT_TRUE(Between(Size(Key(i), Key(i+10)), S1*10, S2*10)); + ASSERT_TRUE(Between(Size("", Key(i)), S1 * i, S2 * i)); + ASSERT_TRUE(Between(Size("", Key(i) + ".suffix"), S1 * (i + 1), + S2 * (i + 1))); + ASSERT_TRUE(Between(Size(Key(i), Key(i + 10)), S1 * 10, S2 * 10)); } - ASSERT_TRUE(Between(Size("", Key(50)), S1*50, S2*50)); - ASSERT_TRUE(Between(Size("", Key(50)+".suffix"), S1*50, S2*50)); + ASSERT_TRUE(Between(Size("", Key(50)), S1 * 50, S2 * 50)); + ASSERT_TRUE(Between(Size("", Key(50) + ".suffix"), S1 * 50, S2 * 50)); std::string cstart_str = Key(compact_start); std::string cend_str = Key(compact_start + 9); @@ -1168,7 +1268,7 @@ TEST(DBTest, ApproximateSizes_MixOfSmallAndLarge) { ASSERT_TRUE(Between(Size(Key(3), Key(5)), 110000, 111000)); - dbfull()->TEST_CompactRange(0, NULL, NULL); + dbfull()->TEST_CompactRange(0, nullptr, nullptr); } } while (ChangeOptions()); } @@ -1182,7 +1282,7 @@ TEST(DBTest, IteratorPinsRef) { // Write to force compactions Put("foo", "newvalue1"); for (int i = 0; i < 100; i++) { - ASSERT_OK(Put(Key(i), Key(i) + std::string(100000, 'v'))); // 100K values + ASSERT_OK(Put(Key(i), Key(i) + std::string(100000, 'v'))); // 100K values } Put("foo", "newvalue2"); @@ -1234,7 +1334,7 @@ TEST(DBTest, HiddenValuesAreRemoved) { Put("pastfoo", "v"); const Snapshot* snapshot = db_->GetSnapshot(); Put("foo", "tiny"); - Put("pastfoo2", "v2"); // Advance sequence number one more + Put("pastfoo2", "v2"); // Advance sequence number one more ASSERT_OK(dbfull()->TEST_CompactMemTable()); ASSERT_GT(NumTableFilesAtLevel(0), 0); @@ -1244,11 +1344,11 @@ TEST(DBTest, HiddenValuesAreRemoved) { db_->ReleaseSnapshot(snapshot); ASSERT_EQ(AllEntriesFor("foo"), "[ tiny, " + big + " ]"); Slice x("x"); - dbfull()->TEST_CompactRange(0, NULL, &x); + dbfull()->TEST_CompactRange(0, nullptr, &x); ASSERT_EQ(AllEntriesFor("foo"), "[ tiny ]"); ASSERT_EQ(NumTableFilesAtLevel(0), 0); ASSERT_GE(NumTableFilesAtLevel(1), 1); - dbfull()->TEST_CompactRange(1, NULL, &x); + dbfull()->TEST_CompactRange(1, nullptr, &x); ASSERT_EQ(AllEntriesFor("foo"), "[ tiny ]"); ASSERT_TRUE(Between(Size("", "pastfoo"), 0, 1000)); @@ -1259,14 +1359,14 @@ TEST(DBTest, DeletionMarkers1) { Put("foo", "v1"); ASSERT_OK(dbfull()->TEST_CompactMemTable()); const int last = config::kMaxMemCompactLevel; - ASSERT_EQ(NumTableFilesAtLevel(last), 1); // foo => v1 is now in last level + ASSERT_EQ(NumTableFilesAtLevel(last), 1); // foo => v1 is now in last level // Place a table at level last-1 to prevent merging with preceding mutation Put("a", "begin"); Put("z", "end"); dbfull()->TEST_CompactMemTable(); ASSERT_EQ(NumTableFilesAtLevel(last), 1); - ASSERT_EQ(NumTableFilesAtLevel(last-1), 1); + ASSERT_EQ(NumTableFilesAtLevel(last - 1), 1); Delete("foo"); Put("foo", "v2"); @@ -1274,11 +1374,11 @@ TEST(DBTest, DeletionMarkers1) { ASSERT_OK(dbfull()->TEST_CompactMemTable()); // Moves to level last-2 ASSERT_EQ(AllEntriesFor("foo"), "[ v2, DEL, v1 ]"); Slice z("z"); - dbfull()->TEST_CompactRange(last-2, NULL, &z); + dbfull()->TEST_CompactRange(last - 2, nullptr, &z); // DEL eliminated, but v1 remains because we aren't compacting that level // (DEL can be eliminated because v2 hides v1). ASSERT_EQ(AllEntriesFor("foo"), "[ v2, v1 ]"); - dbfull()->TEST_CompactRange(last-1, NULL, NULL); + dbfull()->TEST_CompactRange(last - 1, nullptr, nullptr); // Merging last-1 w/ last, so we are the base level for "foo", so // DEL is removed. (as is v1). ASSERT_EQ(AllEntriesFor("foo"), "[ v2 ]"); @@ -1288,23 +1388,23 @@ TEST(DBTest, DeletionMarkers2) { Put("foo", "v1"); ASSERT_OK(dbfull()->TEST_CompactMemTable()); const int last = config::kMaxMemCompactLevel; - ASSERT_EQ(NumTableFilesAtLevel(last), 1); // foo => v1 is now in last level + ASSERT_EQ(NumTableFilesAtLevel(last), 1); // foo => v1 is now in last level // Place a table at level last-1 to prevent merging with preceding mutation Put("a", "begin"); Put("z", "end"); dbfull()->TEST_CompactMemTable(); ASSERT_EQ(NumTableFilesAtLevel(last), 1); - ASSERT_EQ(NumTableFilesAtLevel(last-1), 1); + ASSERT_EQ(NumTableFilesAtLevel(last - 1), 1); Delete("foo"); ASSERT_EQ(AllEntriesFor("foo"), "[ DEL, v1 ]"); ASSERT_OK(dbfull()->TEST_CompactMemTable()); // Moves to level last-2 ASSERT_EQ(AllEntriesFor("foo"), "[ DEL, v1 ]"); - dbfull()->TEST_CompactRange(last-2, NULL, NULL); + dbfull()->TEST_CompactRange(last - 2, nullptr, nullptr); // DEL kept: "last" file overlaps ASSERT_EQ(AllEntriesFor("foo"), "[ DEL, v1 ]"); - dbfull()->TEST_CompactRange(last-1, NULL, NULL); + dbfull()->TEST_CompactRange(last - 1, nullptr, nullptr); // Merging last-1 w/ last, so we are the base level for "foo", so // DEL is removed. (as is v1). ASSERT_EQ(AllEntriesFor("foo"), "[ ]"); @@ -1314,7 +1414,8 @@ TEST(DBTest, OverlapInLevel0) { do { ASSERT_EQ(config::kMaxMemCompactLevel, 2) << "Fix test to match config"; - // Fill levels 1 and 2 to disable the pushing of new memtables to levels > 0. + // Fill levels 1 and 2 to disable the pushing of new memtables to levels > + // 0. ASSERT_OK(Put("100", "v100")); ASSERT_OK(Put("999", "v999")); dbfull()->TEST_CompactMemTable(); @@ -1337,8 +1438,8 @@ TEST(DBTest, OverlapInLevel0) { ASSERT_EQ("2,1,1", FilesPerLevel()); // Compact away the placeholder files we created initially - dbfull()->TEST_CompactRange(1, NULL, NULL); - dbfull()->TEST_CompactRange(2, NULL, NULL); + dbfull()->TEST_CompactRange(1, nullptr, nullptr); + dbfull()->TEST_CompactRange(2, nullptr, nullptr); ASSERT_EQ("2", FilesPerLevel()); // Do a memtable compaction. Before bug-fix, the compaction would @@ -1370,21 +1471,21 @@ TEST(DBTest, L0_CompactionBug_Issue44_a) { TEST(DBTest, L0_CompactionBug_Issue44_b) { Reopen(); - Put("",""); + Put("", ""); Reopen(); Delete("e"); - Put("",""); + Put("", ""); Reopen(); Put("c", "cv"); Reopen(); - Put("",""); + Put("", ""); Reopen(); - Put("",""); + Put("", ""); DelayMilliseconds(1000); // Wait for compaction to finish Reopen(); - Put("d","dv"); + Put("d", "dv"); Reopen(); - Put("",""); + Put("", ""); Reopen(); Delete("d"); Delete("b"); @@ -1394,17 +1495,26 @@ TEST(DBTest, L0_CompactionBug_Issue44_b) { ASSERT_EQ("(->)(c->cv)", Contents()); } +TEST(DBTest, Fflush_Issue474) { + static const int kNum = 100000; + Random rnd(test::RandomSeed()); + for (int i = 0; i < kNum; i++) { + fflush(nullptr); + ASSERT_OK(Put(RandomKey(&rnd), RandomString(&rnd, 100))); + } +} + TEST(DBTest, ComparatorCheck) { class NewComparator : public Comparator { public: - virtual const char* Name() const { return "leveldb.NewComparator"; } - virtual int Compare(const Slice& a, const Slice& b) const { + const char* Name() const override { return "leveldb.NewComparator"; } + int Compare(const Slice& a, const Slice& b) const override { return BytewiseComparator()->Compare(a, b); } - virtual void FindShortestSeparator(std::string* s, const Slice& l) const { + void FindShortestSeparator(std::string* s, const Slice& l) const override { BytewiseComparator()->FindShortestSeparator(s, l); } - virtual void FindShortSuccessor(std::string* key) const { + void FindShortSuccessor(std::string* key) const override { BytewiseComparator()->FindShortSuccessor(key); } }; @@ -1420,21 +1530,22 @@ TEST(DBTest, ComparatorCheck) { TEST(DBTest, CustomComparator) { class NumberComparator : public Comparator { public: - virtual const char* Name() const { return "test.NumberComparator"; } - virtual int Compare(const Slice& a, const Slice& b) const { + const char* Name() const override { return "test.NumberComparator"; } + int Compare(const Slice& a, const Slice& b) const override { return ToNumber(a) - ToNumber(b); } - virtual void FindShortestSeparator(std::string* s, const Slice& l) const { - ToNumber(*s); // Check format - ToNumber(l); // Check format + void FindShortestSeparator(std::string* s, const Slice& l) const override { + ToNumber(*s); // Check format + ToNumber(l); // Check format } - virtual void FindShortSuccessor(std::string* key) const { - ToNumber(*key); // Check format + void FindShortSuccessor(std::string* key) const override { + ToNumber(*key); // Check format } + private: static int ToNumber(const Slice& x) { // Check that there are no extra characters. - ASSERT_TRUE(x.size() >= 2 && x[0] == '[' && x[x.size()-1] == ']') + ASSERT_TRUE(x.size() >= 2 && x[0] == '[' && x[x.size() - 1] == ']') << EscapeString(x); int val; char ignored; @@ -1447,7 +1558,7 @@ TEST(DBTest, CustomComparator) { Options new_options = CurrentOptions(); new_options.create_if_missing = true; new_options.comparator = &cmp; - new_options.filter_policy = NULL; // Cannot use bloom filters + new_options.filter_policy = nullptr; // Cannot use bloom filters new_options.write_buffer_size = 1000; // Compact more often DestroyAndReopen(&new_options); ASSERT_OK(Put("[10]", "ten")); @@ -1465,7 +1576,7 @@ TEST(DBTest, CustomComparator) { for (int run = 0; run < 2; run++) { for (int i = 0; i < 1000; i++) { char buf[100]; - snprintf(buf, sizeof(buf), "[%d]", i*10); + snprintf(buf, sizeof(buf), "[%d]", i * 10); ASSERT_OK(Put(buf, buf)); } Compact("[0]", "[1000000]"); @@ -1502,7 +1613,7 @@ TEST(DBTest, ManualCompaction) { // Compact all MakeTables(1, "a", "z"); ASSERT_EQ("0,1,2", FilesPerLevel()); - db_->CompactRange(NULL, NULL); + db_->CompactRange(nullptr, nullptr); ASSERT_EQ("0,0,1", FilesPerLevel()); } @@ -1511,42 +1622,94 @@ TEST(DBTest, DBOpen_Options) { DestroyDB(dbname, Options()); // Does not exist, and create_if_missing == false: error - DB* db = NULL; + DB* db = nullptr; Options opts; opts.create_if_missing = false; Status s = DB::Open(opts, dbname, &db); - ASSERT_TRUE(strstr(s.ToString().c_str(), "does not exist") != NULL); - ASSERT_TRUE(db == NULL); + ASSERT_TRUE(strstr(s.ToString().c_str(), "does not exist") != nullptr); + ASSERT_TRUE(db == nullptr); // Does not exist, and create_if_missing == true: OK opts.create_if_missing = true; s = DB::Open(opts, dbname, &db); ASSERT_OK(s); - ASSERT_TRUE(db != NULL); + ASSERT_TRUE(db != nullptr); delete db; - db = NULL; + db = nullptr; // Does exist, and error_if_exists == true: error opts.create_if_missing = false; opts.error_if_exists = true; s = DB::Open(opts, dbname, &db); - ASSERT_TRUE(strstr(s.ToString().c_str(), "exists") != NULL); - ASSERT_TRUE(db == NULL); + ASSERT_TRUE(strstr(s.ToString().c_str(), "exists") != nullptr); + ASSERT_TRUE(db == nullptr); // Does exist, and error_if_exists == false: OK opts.create_if_missing = true; opts.error_if_exists = false; s = DB::Open(opts, dbname, &db); ASSERT_OK(s); - ASSERT_TRUE(db != NULL); + ASSERT_TRUE(db != nullptr); + + delete db; + db = nullptr; +} + +TEST(DBTest, DestroyEmptyDir) { + std::string dbname = test::TmpDir() + "/db_empty_dir"; + TestEnv env(Env::Default()); + env.DeleteDir(dbname); + ASSERT_TRUE(!env.FileExists(dbname)); + + Options opts; + opts.env = &env; + + ASSERT_OK(env.CreateDir(dbname)); + ASSERT_TRUE(env.FileExists(dbname)); + std::vector<std::string> children; + ASSERT_OK(env.GetChildren(dbname, &children)); + // The stock Env's do not filter out '.' and '..' special files. + ASSERT_EQ(2, children.size()); + ASSERT_OK(DestroyDB(dbname, opts)); + ASSERT_TRUE(!env.FileExists(dbname)); + + // Should also be destroyed if Env is filtering out dot files. + env.SetIgnoreDotFiles(true); + ASSERT_OK(env.CreateDir(dbname)); + ASSERT_TRUE(env.FileExists(dbname)); + ASSERT_OK(env.GetChildren(dbname, &children)); + ASSERT_EQ(0, children.size()); + ASSERT_OK(DestroyDB(dbname, opts)); + ASSERT_TRUE(!env.FileExists(dbname)); +} + +TEST(DBTest, DestroyOpenDB) { + std::string dbname = test::TmpDir() + "/open_db_dir"; + env_->DeleteDir(dbname); + ASSERT_TRUE(!env_->FileExists(dbname)); + + Options opts; + opts.create_if_missing = true; + DB* db = nullptr; + ASSERT_OK(DB::Open(opts, dbname, &db)); + ASSERT_TRUE(db != nullptr); + + // Must fail to destroy an open db. + ASSERT_TRUE(env_->FileExists(dbname)); + ASSERT_TRUE(!DestroyDB(dbname, Options()).ok()); + ASSERT_TRUE(env_->FileExists(dbname)); delete db; - db = NULL; + db = nullptr; + + // Should succeed destroying a closed db. + ASSERT_OK(DestroyDB(dbname, Options())); + ASSERT_TRUE(!env_->FileExists(dbname)); } TEST(DBTest, Locking) { - DB* db2 = NULL; + DB* db2 = nullptr; Status s = DB::Open(CurrentOptions(), dbname_, &db2); ASSERT_TRUE(!s.ok()) << "Locking did not prevent re-opening db"; } @@ -1561,13 +1724,14 @@ TEST(DBTest, NoSpace) { ASSERT_EQ("v1", Get("foo")); Compact("a", "z"); const int num_files = CountFiles(); - env_->no_space_.Release_Store(env_); // Force out-of-space errors + // Force out-of-space errors. + env_->no_space_.store(true, std::memory_order_release); for (int i = 0; i < 10; i++) { - for (int level = 0; level < config::kNumLevels-1; level++) { - dbfull()->TEST_CompactRange(level, NULL, NULL); + for (int level = 0; level < config::kNumLevels - 1; level++) { + dbfull()->TEST_CompactRange(level, nullptr, nullptr); } } - env_->no_space_.Release_Store(NULL); + env_->no_space_.store(false, std::memory_order_release); ASSERT_LT(CountFiles(), num_files + 3); } @@ -1577,7 +1741,8 @@ TEST(DBTest, NonWritableFileSystem) { options.env = env_; Reopen(&options); ASSERT_OK(Put("foo", "v1")); - env_->non_writable_.Release_Store(env_); // Force errors for new files + // Force errors for new files. + env_->non_writable_.store(true, std::memory_order_release); std::string big(100000, 'x'); int errors = 0; for (int i = 0; i < 20; i++) { @@ -1588,7 +1753,7 @@ TEST(DBTest, NonWritableFileSystem) { } } ASSERT_GT(errors, 0); - env_->non_writable_.Release_Store(NULL); + env_->non_writable_.store(false, std::memory_order_release); } TEST(DBTest, WriteSyncError) { @@ -1598,7 +1763,7 @@ TEST(DBTest, WriteSyncError) { Options options = CurrentOptions(); options.env = env_; Reopen(&options); - env_->data_sync_error_.Release_Store(env_); + env_->data_sync_error_.store(true, std::memory_order_release); // (b) Normal write should succeed WriteOptions w; @@ -1612,7 +1777,7 @@ TEST(DBTest, WriteSyncError) { ASSERT_EQ("NOT_FOUND", Get("k2")); // (d) make sync behave normally - env_->data_sync_error_.Release_Store(NULL); + env_->data_sync_error_.store(false, std::memory_order_release); // (e) Do a non-sync write; should fail w.sync = false; @@ -1632,9 +1797,8 @@ TEST(DBTest, ManifestWriteError) { // We iterate twice. In the second iteration, everything is the // same except the log record never makes it to the MANIFEST file. for (int iter = 0; iter < 2; iter++) { - port::AtomicPointer* error_type = (iter == 0) - ? &env_->manifest_sync_error_ - : &env_->manifest_write_error_; + std::atomic<bool>* error_type = (iter == 0) ? &env_->manifest_sync_error_ + : &env_->manifest_write_error_; // Insert foo=>bar mapping Options options = CurrentOptions(); @@ -1649,15 +1813,15 @@ TEST(DBTest, ManifestWriteError) { dbfull()->TEST_CompactMemTable(); ASSERT_EQ("bar", Get("foo")); const int last = config::kMaxMemCompactLevel; - ASSERT_EQ(NumTableFilesAtLevel(last), 1); // foo=>bar is now in last level + ASSERT_EQ(NumTableFilesAtLevel(last), 1); // foo=>bar is now in last level // Merging compaction (will fail) - error_type->Release_Store(env_); - dbfull()->TEST_CompactRange(last, NULL, NULL); // Should fail + error_type->store(true, std::memory_order_release); + dbfull()->TEST_CompactRange(last, nullptr, nullptr); // Should fail ASSERT_EQ("bar", Get("foo")); // Recovery: should not lose data - error_type->Release_Store(NULL); + error_type->store(false, std::memory_order_release); Reopen(&options); ASSERT_EQ("bar", Get("foo")); } @@ -1677,8 +1841,7 @@ TEST(DBTest, MissingSSTFile) { options.paranoid_checks = true; Status s = TryReopen(&options); ASSERT_TRUE(!s.ok()); - ASSERT_TRUE(s.ToString().find("issing") != std::string::npos) - << s.ToString(); + ASSERT_TRUE(s.ToString().find("issing") != std::string::npos) << s.ToString(); } TEST(DBTest, StillReadSST) { @@ -1728,7 +1891,7 @@ TEST(DBTest, BloomFilter) { dbfull()->TEST_CompactMemTable(); // Prevent auto compactions triggered by seeks - env_->delay_data_sync_.Release_Store(env_); + env_->delay_data_sync_.store(true, std::memory_order_release); // Lookup present keys. Should rarely read from small sstable. env_->random_read_counter_.Reset(); @@ -1738,7 +1901,7 @@ TEST(DBTest, BloomFilter) { int reads = env_->random_read_counter_.Read(); fprintf(stderr, "%d present => %d reads\n", N, reads); ASSERT_GE(reads, N); - ASSERT_LE(reads, N + 2*N/100); + ASSERT_LE(reads, N + 2 * N / 100); // Lookup present keys. Should rarely read from either sstable. env_->random_read_counter_.Reset(); @@ -1747,9 +1910,9 @@ TEST(DBTest, BloomFilter) { } reads = env_->random_read_counter_.Read(); fprintf(stderr, "%d missing => %d reads\n", N, reads); - ASSERT_LE(reads, 3*N/100); + ASSERT_LE(reads, 3 * N / 100); - env_->delay_data_sync_.Release_Store(NULL); + env_->delay_data_sync_.store(false, std::memory_order_release); Close(); delete options.block_cache; delete options.filter_policy; @@ -1764,9 +1927,9 @@ static const int kNumKeys = 1000; struct MTState { DBTest* test; - port::AtomicPointer stop; - port::AtomicPointer counter[kNumThreads]; - port::AtomicPointer thread_done[kNumThreads]; + std::atomic<bool> stop; + std::atomic<int> counter[kNumThreads]; + std::atomic<bool> thread_done[kNumThreads]; }; struct MTThread { @@ -1778,13 +1941,13 @@ static void MTThreadBody(void* arg) { MTThread* t = reinterpret_cast<MTThread*>(arg); int id = t->id; DB* db = t->state->test->db_; - uintptr_t counter = 0; + int counter = 0; fprintf(stderr, "... starting thread %d\n", id); Random rnd(1000 + id); std::string value; char valbuf[1500]; - while (t->state->stop.Acquire_Load() == NULL) { - t->state->counter[id].Release_Store(reinterpret_cast<void*>(counter)); + while (!t->state->stop.load(std::memory_order_acquire)) { + t->state->counter[id].store(counter, std::memory_order_release); int key = rnd.Uniform(kNumKeys); char keybuf[20]; @@ -1793,8 +1956,8 @@ static void MTThreadBody(void* arg) { if (rnd.OneIn(2)) { // Write values of the form <key, my id, counter>. // We add some padding for force compactions. - snprintf(valbuf, sizeof(valbuf), "%d.%d.%-1000d", - key, id, static_cast<int>(counter)); + snprintf(valbuf, sizeof(valbuf), "%d.%d.%-1000d", key, id, + static_cast<int>(counter)); ASSERT_OK(db->Put(WriteOptions(), Slice(keybuf), Slice(valbuf))); } else { // Read a value and verify that it matches the pattern written above. @@ -1809,14 +1972,13 @@ static void MTThreadBody(void* arg) { ASSERT_EQ(k, key); ASSERT_GE(w, 0); ASSERT_LT(w, kNumThreads); - ASSERT_LE(static_cast<uintptr_t>(c), reinterpret_cast<uintptr_t>( - t->state->counter[w].Acquire_Load())); + ASSERT_LE(c, t->state->counter[w].load(std::memory_order_acquire)); } } counter++; } - t->state->thread_done[id].Release_Store(t); - fprintf(stderr, "... stopping thread %d after %d ops\n", id, int(counter)); + t->state->thread_done[id].store(true, std::memory_order_release); + fprintf(stderr, "... stopping thread %d after %d ops\n", id, counter); } } // namespace @@ -1826,10 +1988,10 @@ TEST(DBTest, MultiThreaded) { // Initialize state MTState mt; mt.test = this; - mt.stop.Release_Store(0); + mt.stop.store(false, std::memory_order_release); for (int id = 0; id < kNumThreads; id++) { - mt.counter[id].Release_Store(0); - mt.thread_done[id].Release_Store(0); + mt.counter[id].store(false, std::memory_order_release); + mt.thread_done[id].store(false, std::memory_order_release); } // Start threads @@ -1844,9 +2006,9 @@ TEST(DBTest, MultiThreaded) { DelayMilliseconds(kTestSeconds * 1000); // Stop the threads and wait for them to finish - mt.stop.Release_Store(&mt); + mt.stop.store(true, std::memory_order_release); for (int id = 0; id < kNumThreads; id++) { - while (mt.thread_done[id].Acquire_Load() == NULL) { + while (!mt.thread_done[id].load(std::memory_order_acquire)) { DelayMilliseconds(100); } } @@ -1857,28 +2019,28 @@ namespace { typedef std::map<std::string, std::string> KVMap; } -class ModelDB: public DB { +class ModelDB : public DB { public: class ModelSnapshot : public Snapshot { public: KVMap map_; }; - explicit ModelDB(const Options& options): options_(options) { } - ~ModelDB() { } - virtual Status Put(const WriteOptions& o, const Slice& k, const Slice& v) { + explicit ModelDB(const Options& options) : options_(options) {} + ~ModelDB() override = default; + Status Put(const WriteOptions& o, const Slice& k, const Slice& v) override { return DB::Put(o, k, v); } - virtual Status Delete(const WriteOptions& o, const Slice& key) { + Status Delete(const WriteOptions& o, const Slice& key) override { return DB::Delete(o, key); } - virtual Status Get(const ReadOptions& options, - const Slice& key, std::string* value) { - assert(false); // Not implemented + Status Get(const ReadOptions& options, const Slice& key, + std::string* value) override { + assert(false); // Not implemented return Status::NotFound(key); } - virtual Iterator* NewIterator(const ReadOptions& options) { - if (options.snapshot == NULL) { + Iterator* NewIterator(const ReadOptions& options) override { + if (options.snapshot == nullptr) { KVMap* saved = new KVMap; *saved = map_; return new ModelIter(saved, true); @@ -1888,68 +2050,65 @@ class ModelDB: public DB { return new ModelIter(snapshot_state, false); } } - virtual const Snapshot* GetSnapshot() { + const Snapshot* GetSnapshot() override { ModelSnapshot* snapshot = new ModelSnapshot; snapshot->map_ = map_; return snapshot; } - virtual void ReleaseSnapshot(const Snapshot* snapshot) { + void ReleaseSnapshot(const Snapshot* snapshot) override { delete reinterpret_cast<const ModelSnapshot*>(snapshot); } - virtual Status Write(const WriteOptions& options, WriteBatch* batch) { + Status Write(const WriteOptions& options, WriteBatch* batch) override { class Handler : public WriteBatch::Handler { public: KVMap* map_; - virtual void Put(const Slice& key, const Slice& value) { + void Put(const Slice& key, const Slice& value) override { (*map_)[key.ToString()] = value.ToString(); } - virtual void Delete(const Slice& key) { - map_->erase(key.ToString()); - } + void Delete(const Slice& key) override { map_->erase(key.ToString()); } }; Handler handler; handler.map_ = &map_; return batch->Iterate(&handler); } - virtual bool GetProperty(const Slice& property, std::string* value) { + bool GetProperty(const Slice& property, std::string* value) override { return false; } - virtual void GetApproximateSizes(const Range* r, int n, uint64_t* sizes) { + void GetApproximateSizes(const Range* r, int n, uint64_t* sizes) override { for (int i = 0; i < n; i++) { sizes[i] = 0; } } - virtual void CompactRange(const Slice* start, const Slice* end) { - } + void CompactRange(const Slice* start, const Slice* end) override {} private: - class ModelIter: public Iterator { + class ModelIter : public Iterator { public: ModelIter(const KVMap* map, bool owned) - : map_(map), owned_(owned), iter_(map_->end()) { - } - ~ModelIter() { + : map_(map), owned_(owned), iter_(map_->end()) {} + ~ModelIter() override { if (owned_) delete map_; } - virtual bool Valid() const { return iter_ != map_->end(); } - virtual void SeekToFirst() { iter_ = map_->begin(); } - virtual void SeekToLast() { + bool Valid() const override { return iter_ != map_->end(); } + void SeekToFirst() override { iter_ = map_->begin(); } + void SeekToLast() override { if (map_->empty()) { iter_ = map_->end(); } else { iter_ = map_->find(map_->rbegin()->first); } } - virtual void Seek(const Slice& k) { + void Seek(const Slice& k) override { iter_ = map_->lower_bound(k.ToString()); } - virtual void Next() { ++iter_; } - virtual void Prev() { --iter_; } - virtual Slice key() const { return iter_->first; } - virtual Slice value() const { return iter_->second; } - virtual Status status() const { return Status::OK(); } + void Next() override { ++iter_; } + void Prev() override { --iter_; } + Slice key() const override { return iter_->first; } + Slice value() const override { return iter_->second; } + Status status() const override { return Status::OK(); } + private: const KVMap* const map_; const bool owned_; // Do we own map_ @@ -1959,16 +2118,7 @@ class ModelDB: public DB { KVMap map_; }; -static std::string RandomKey(Random* rnd) { - int len = (rnd->OneIn(3) - ? 1 // Short sometimes to encourage collisions - : (rnd->OneIn(100) ? rnd->Skewed(10) : rnd->Uniform(10))); - return test::RandomKey(rnd, len); -} - -static bool CompareIterators(int step, - DB* model, - DB* db, +static bool CompareIterators(int step, DB* model, DB* db, const Snapshot* model_snap, const Snapshot* db_snap) { ReadOptions options; @@ -1979,12 +2129,10 @@ static bool CompareIterators(int step, bool ok = true; int count = 0; for (miter->SeekToFirst(), dbiter->SeekToFirst(); - ok && miter->Valid() && dbiter->Valid(); - miter->Next(), dbiter->Next()) { + ok && miter->Valid() && dbiter->Valid(); miter->Next(), dbiter->Next()) { count++; if (miter->key().compare(dbiter->key()) != 0) { - fprintf(stderr, "step %d: Key mismatch: '%s' vs. '%s'\n", - step, + fprintf(stderr, "step %d: Key mismatch: '%s' vs. '%s'\n", step, EscapeString(miter->key()).c_str(), EscapeString(dbiter->key()).c_str()); ok = false; @@ -1993,8 +2141,7 @@ static bool CompareIterators(int step, if (miter->value().compare(dbiter->value()) != 0) { fprintf(stderr, "step %d: Value mismatch for key '%s': '%s' vs. '%s'\n", - step, - EscapeString(miter->key()).c_str(), + step, EscapeString(miter->key()).c_str(), EscapeString(miter->value()).c_str(), EscapeString(miter->value()).c_str()); ok = false; @@ -2019,8 +2166,8 @@ TEST(DBTest, Randomized) { do { ModelDB model(CurrentOptions()); const int N = 10000; - const Snapshot* model_snap = NULL; - const Snapshot* db_snap = NULL; + const Snapshot* model_snap = nullptr; + const Snapshot* db_snap = nullptr; std::string k, v; for (int step = 0; step < N; step++) { if (step % 100 == 0) { @@ -2028,22 +2175,19 @@ TEST(DBTest, Randomized) { } // TODO(sanjay): Test Get() works int p = rnd.Uniform(100); - if (p < 45) { // Put + if (p < 45) { // Put k = RandomKey(&rnd); - v = RandomString(&rnd, - rnd.OneIn(20) - ? 100 + rnd.Uniform(100) - : rnd.Uniform(8)); + v = RandomString( + &rnd, rnd.OneIn(20) ? 100 + rnd.Uniform(100) : rnd.Uniform(8)); ASSERT_OK(model.Put(WriteOptions(), k, v)); ASSERT_OK(db_->Put(WriteOptions(), k, v)); - } else if (p < 90) { // Delete + } else if (p < 90) { // Delete k = RandomKey(&rnd); ASSERT_OK(model.Delete(WriteOptions(), k)); ASSERT_OK(db_->Delete(WriteOptions(), k)); - - } else { // Multi-element batch + } else { // Multi-element batch WriteBatch b; const int num = rnd.Uniform(8); for (int i = 0; i < num; i++) { @@ -2065,23 +2209,23 @@ TEST(DBTest, Randomized) { } if ((step % 100) == 0) { - ASSERT_TRUE(CompareIterators(step, &model, db_, NULL, NULL)); + ASSERT_TRUE(CompareIterators(step, &model, db_, nullptr, nullptr)); ASSERT_TRUE(CompareIterators(step, &model, db_, model_snap, db_snap)); // Save a snapshot from each DB this time that we'll use next // time we compare things, to make sure the current state is // preserved with the snapshot - if (model_snap != NULL) model.ReleaseSnapshot(model_snap); - if (db_snap != NULL) db_->ReleaseSnapshot(db_snap); + if (model_snap != nullptr) model.ReleaseSnapshot(model_snap); + if (db_snap != nullptr) db_->ReleaseSnapshot(db_snap); Reopen(); - ASSERT_TRUE(CompareIterators(step, &model, db_, NULL, NULL)); + ASSERT_TRUE(CompareIterators(step, &model, db_, nullptr, nullptr)); model_snap = model.GetSnapshot(); db_snap = db_->GetSnapshot(); } } - if (model_snap != NULL) model.ReleaseSnapshot(model_snap); - if (db_snap != NULL) db_->ReleaseSnapshot(db_snap); + if (model_snap != nullptr) model.ReleaseSnapshot(model_snap); + if (db_snap != nullptr) db_->ReleaseSnapshot(db_snap); } while (ChangeOptions()); } @@ -2095,15 +2239,15 @@ void BM_LogAndApply(int iters, int num_base_files) { std::string dbname = test::TmpDir() + "/leveldb_test_benchmark"; DestroyDB(dbname, Options()); - DB* db = NULL; + DB* db = nullptr; Options opts; opts.create_if_missing = true; Status s = DB::Open(opts, dbname, &db); ASSERT_OK(s); - ASSERT_TRUE(db != NULL); + ASSERT_TRUE(db != nullptr); delete db; - db = NULL; + db = nullptr; Env* env = Env::Default(); @@ -2112,14 +2256,14 @@ void BM_LogAndApply(int iters, int num_base_files) { InternalKeyComparator cmp(BytewiseComparator()); Options options; - VersionSet vset(dbname, &options, NULL, &cmp); + VersionSet vset(dbname, &options, nullptr, &cmp); bool save_manifest; ASSERT_OK(vset.Recover(&save_manifest)); VersionEdit vbase; uint64_t fnum = 1; for (int i = 0; i < num_base_files; i++) { - InternalKey start(MakeKey(2*fnum), 1, kTypeValue); - InternalKey limit(MakeKey(2*fnum+1), 1, kTypeDeletion); + InternalKey start(MakeKey(2 * fnum), 1, kTypeValue); + InternalKey limit(MakeKey(2 * fnum + 1), 1, kTypeDeletion); vbase.AddFile(2, fnum++, 1 /* file size */, start, limit); } ASSERT_OK(vset.LogAndApply(&vbase, &mu)); @@ -2129,8 +2273,8 @@ void BM_LogAndApply(int iters, int num_base_files) { for (int i = 0; i < iters; i++) { VersionEdit vedit; vedit.DeleteFile(2, fnum); - InternalKey start(MakeKey(2*fnum), 1, kTypeValue); - InternalKey limit(MakeKey(2*fnum+1), 1, kTypeDeletion); + InternalKey start(MakeKey(2 * fnum), 1, kTypeValue); + InternalKey limit(MakeKey(2 * fnum + 1), 1, kTypeDeletion); vedit.AddFile(2, fnum++, 1 /* file size */, start, limit); vset.LogAndApply(&vedit, &mu); } @@ -2139,8 +2283,8 @@ void BM_LogAndApply(int iters, int num_base_files) { char buf[16]; snprintf(buf, sizeof(buf), "%d", num_base_files); fprintf(stderr, - "BM_LogAndApply/%-6s %8d iters : %9u us (%7.0f us / iter)\n", - buf, iters, us, ((float)us) / iters); + "BM_LogAndApply/%-6s %8d iters : %9u us (%7.0f us / iter)\n", buf, + iters, us, ((float)us) / iters); } } // namespace leveldb diff --git a/src/leveldb/db/dbformat.cc b/src/leveldb/db/dbformat.cc index 20a7ca4462..459eddf5b1 100644 --- a/src/leveldb/db/dbformat.cc +++ b/src/leveldb/db/dbformat.cc @@ -2,8 +2,12 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. See the AUTHORS file for names of contributors. -#include <stdio.h> #include "db/dbformat.h" + +#include <stdio.h> + +#include <sstream> + #include "port/port.h" #include "util/coding.h" @@ -21,26 +25,20 @@ void AppendInternalKey(std::string* result, const ParsedInternalKey& key) { } std::string ParsedInternalKey::DebugString() const { - char buf[50]; - snprintf(buf, sizeof(buf), "' @ %llu : %d", - (unsigned long long) sequence, - int(type)); - std::string result = "'"; - result += EscapeString(user_key.ToString()); - result += buf; - return result; + std::ostringstream ss; + ss << '\'' << EscapeString(user_key.ToString()) << "' @ " << sequence << " : " + << static_cast<int>(type); + return ss.str(); } std::string InternalKey::DebugString() const { - std::string result; ParsedInternalKey parsed; if (ParseInternalKey(rep_, &parsed)) { - result = parsed.DebugString(); - } else { - result = "(bad)"; - result.append(EscapeString(rep_)); + return parsed.DebugString(); } - return result; + std::ostringstream ss; + ss << "(bad)" << EscapeString(rep_); + return ss.str(); } const char* InternalKeyComparator::Name() const { @@ -65,9 +63,8 @@ int InternalKeyComparator::Compare(const Slice& akey, const Slice& bkey) const { return r; } -void InternalKeyComparator::FindShortestSeparator( - std::string* start, - const Slice& limit) const { +void InternalKeyComparator::FindShortestSeparator(std::string* start, + const Slice& limit) const { // Attempt to shorten the user portion of the key Slice user_start = ExtractUserKey(*start); Slice user_limit = ExtractUserKey(limit); @@ -77,7 +74,8 @@ void InternalKeyComparator::FindShortestSeparator( user_comparator_->Compare(user_start, tmp) < 0) { // User key has become shorter physically, but larger logically. // Tack on the earliest possible number to the shortened user key. - PutFixed64(&tmp, PackSequenceAndType(kMaxSequenceNumber,kValueTypeForSeek)); + PutFixed64(&tmp, + PackSequenceAndType(kMaxSequenceNumber, kValueTypeForSeek)); assert(this->Compare(*start, tmp) < 0); assert(this->Compare(tmp, limit) < 0); start->swap(tmp); @@ -92,15 +90,14 @@ void InternalKeyComparator::FindShortSuccessor(std::string* key) const { user_comparator_->Compare(user_key, tmp) < 0) { // User key has become shorter physically, but larger logically. // Tack on the earliest possible number to the shortened user key. - PutFixed64(&tmp, PackSequenceAndType(kMaxSequenceNumber,kValueTypeForSeek)); + PutFixed64(&tmp, + PackSequenceAndType(kMaxSequenceNumber, kValueTypeForSeek)); assert(this->Compare(*key, tmp) < 0); key->swap(tmp); } } -const char* InternalFilterPolicy::Name() const { - return user_policy_->Name(); -} +const char* InternalFilterPolicy::Name() const { return user_policy_->Name(); } void InternalFilterPolicy::CreateFilter(const Slice* keys, int n, std::string* dst) const { diff --git a/src/leveldb/db/dbformat.h b/src/leveldb/db/dbformat.h index ea897b13c0..a1c30ed88c 100644 --- a/src/leveldb/db/dbformat.h +++ b/src/leveldb/db/dbformat.h @@ -5,7 +5,10 @@ #ifndef STORAGE_LEVELDB_DB_DBFORMAT_H_ #define STORAGE_LEVELDB_DB_DBFORMAT_H_ -#include <stdio.h> +#include <cstddef> +#include <cstdint> +#include <string> + #include "leveldb/comparator.h" #include "leveldb/db.h" #include "leveldb/filter_policy.h" @@ -48,10 +51,7 @@ class InternalKey; // Value types encoded as the last component of internal keys. // DO NOT CHANGE THESE ENUM VALUES: they are embedded in the on-disk // data structures. -enum ValueType { - kTypeDeletion = 0x0, - kTypeValue = 0x1 -}; +enum ValueType { kTypeDeletion = 0x0, kTypeValue = 0x1 }; // kValueTypeForSeek defines the ValueType that should be passed when // constructing a ParsedInternalKey object for seeking to a particular // sequence number (since we sort sequence numbers in decreasing order @@ -64,17 +64,16 @@ typedef uint64_t SequenceNumber; // We leave eight bits empty at the bottom so a type and sequence# // can be packed together into 64-bits. -static const SequenceNumber kMaxSequenceNumber = - ((0x1ull << 56) - 1); +static const SequenceNumber kMaxSequenceNumber = ((0x1ull << 56) - 1); struct ParsedInternalKey { Slice user_key; SequenceNumber sequence; ValueType type; - ParsedInternalKey() { } // Intentionally left uninitialized (for speed) + ParsedInternalKey() {} // Intentionally left uninitialized (for speed) ParsedInternalKey(const Slice& u, const SequenceNumber& seq, ValueType t) - : user_key(u), sequence(seq), type(t) { } + : user_key(u), sequence(seq), type(t) {} std::string DebugString() const; }; @@ -84,15 +83,13 @@ inline size_t InternalKeyEncodingLength(const ParsedInternalKey& key) { } // Append the serialization of "key" to *result. -extern void AppendInternalKey(std::string* result, - const ParsedInternalKey& key); +void AppendInternalKey(std::string* result, const ParsedInternalKey& key); // Attempt to parse an internal key from "internal_key". On success, // stores the parsed data in "*result", and returns true. // // On error, returns false, leaves "*result" in an undefined state. -extern bool ParseInternalKey(const Slice& internal_key, - ParsedInternalKey* result); +bool ParseInternalKey(const Slice& internal_key, ParsedInternalKey* result); // Returns the user key portion of an internal key. inline Slice ExtractUserKey(const Slice& internal_key) { @@ -100,27 +97,19 @@ inline Slice ExtractUserKey(const Slice& internal_key) { return Slice(internal_key.data(), internal_key.size() - 8); } -inline ValueType ExtractValueType(const Slice& internal_key) { - assert(internal_key.size() >= 8); - const size_t n = internal_key.size(); - uint64_t num = DecodeFixed64(internal_key.data() + n - 8); - unsigned char c = num & 0xff; - return static_cast<ValueType>(c); -} - // A comparator for internal keys that uses a specified comparator for // the user key portion and breaks ties by decreasing sequence number. class InternalKeyComparator : public Comparator { private: const Comparator* user_comparator_; + public: - explicit InternalKeyComparator(const Comparator* c) : user_comparator_(c) { } - virtual const char* Name() const; - virtual int Compare(const Slice& a, const Slice& b) const; - virtual void FindShortestSeparator( - std::string* start, - const Slice& limit) const; - virtual void FindShortSuccessor(std::string* key) const; + explicit InternalKeyComparator(const Comparator* c) : user_comparator_(c) {} + const char* Name() const override; + int Compare(const Slice& a, const Slice& b) const override; + void FindShortestSeparator(std::string* start, + const Slice& limit) const override; + void FindShortSuccessor(std::string* key) const override; const Comparator* user_comparator() const { return user_comparator_; } @@ -131,11 +120,12 @@ class InternalKeyComparator : public Comparator { class InternalFilterPolicy : public FilterPolicy { private: const FilterPolicy* const user_policy_; + public: - explicit InternalFilterPolicy(const FilterPolicy* p) : user_policy_(p) { } - virtual const char* Name() const; - virtual void CreateFilter(const Slice* keys, int n, std::string* dst) const; - virtual bool KeyMayMatch(const Slice& key, const Slice& filter) const; + explicit InternalFilterPolicy(const FilterPolicy* p) : user_policy_(p) {} + const char* Name() const override; + void CreateFilter(const Slice* keys, int n, std::string* dst) const override; + bool KeyMayMatch(const Slice& key, const Slice& filter) const override; }; // Modules in this directory should keep internal keys wrapped inside @@ -144,13 +134,18 @@ class InternalFilterPolicy : public FilterPolicy { class InternalKey { private: std::string rep_; + public: - InternalKey() { } // Leave rep_ as empty to indicate it is invalid + InternalKey() {} // Leave rep_ as empty to indicate it is invalid InternalKey(const Slice& user_key, SequenceNumber s, ValueType t) { AppendInternalKey(&rep_, ParsedInternalKey(user_key, s, t)); } - void DecodeFrom(const Slice& s) { rep_.assign(s.data(), s.size()); } + bool DecodeFrom(const Slice& s) { + rep_.assign(s.data(), s.size()); + return !rep_.empty(); + } + Slice Encode() const { assert(!rep_.empty()); return rep_; @@ -168,8 +163,8 @@ class InternalKey { std::string DebugString() const; }; -inline int InternalKeyComparator::Compare( - const InternalKey& a, const InternalKey& b) const { +inline int InternalKeyComparator::Compare(const InternalKey& a, + const InternalKey& b) const { return Compare(a.Encode(), b.Encode()); } @@ -178,11 +173,11 @@ inline bool ParseInternalKey(const Slice& internal_key, const size_t n = internal_key.size(); if (n < 8) return false; uint64_t num = DecodeFixed64(internal_key.data() + n - 8); - unsigned char c = num & 0xff; + uint8_t c = num & 0xff; result->sequence = num >> 8; result->type = static_cast<ValueType>(c); result->user_key = Slice(internal_key.data(), n - 8); - return (c <= static_cast<unsigned char>(kTypeValue)); + return (c <= static_cast<uint8_t>(kTypeValue)); } // A helper class useful for DBImpl::Get() @@ -192,6 +187,9 @@ class LookupKey { // the specified sequence number. LookupKey(const Slice& user_key, SequenceNumber sequence); + LookupKey(const LookupKey&) = delete; + LookupKey& operator=(const LookupKey&) = delete; + ~LookupKey(); // Return a key suitable for lookup in a MemTable. @@ -214,11 +212,7 @@ class LookupKey { const char* start_; const char* kstart_; const char* end_; - char space_[200]; // Avoid allocation for short keys - - // No copying allowed - LookupKey(const LookupKey&); - void operator=(const LookupKey&); + char space_[200]; // Avoid allocation for short keys }; inline LookupKey::~LookupKey() { diff --git a/src/leveldb/db/dbformat_test.cc b/src/leveldb/db/dbformat_test.cc index 5d82f5d313..1209369c31 100644 --- a/src/leveldb/db/dbformat_test.cc +++ b/src/leveldb/db/dbformat_test.cc @@ -8,8 +8,7 @@ namespace leveldb { -static std::string IKey(const std::string& user_key, - uint64_t seq, +static std::string IKey(const std::string& user_key, uint64_t seq, ValueType vt) { std::string encoded; AppendInternalKey(&encoded, ParsedInternalKey(user_key, seq, vt)); @@ -28,9 +27,7 @@ static std::string ShortSuccessor(const std::string& s) { return result; } -static void TestKey(const std::string& key, - uint64_t seq, - ValueType vt) { +static void TestKey(const std::string& key, uint64_t seq, ValueType vt) { std::string encoded = IKey(key, seq, vt); Slice in(encoded); @@ -44,16 +41,22 @@ static void TestKey(const std::string& key, ASSERT_TRUE(!ParseInternalKey(Slice("bar"), &decoded)); } -class FormatTest { }; +class FormatTest {}; TEST(FormatTest, InternalKey_EncodeDecode) { - const char* keys[] = { "", "k", "hello", "longggggggggggggggggggggg" }; - const uint64_t seq[] = { - 1, 2, 3, - (1ull << 8) - 1, 1ull << 8, (1ull << 8) + 1, - (1ull << 16) - 1, 1ull << 16, (1ull << 16) + 1, - (1ull << 32) - 1, 1ull << 32, (1ull << 32) + 1 - }; + const char* keys[] = {"", "k", "hello", "longggggggggggggggggggggg"}; + const uint64_t seq[] = {1, + 2, + 3, + (1ull << 8) - 1, + 1ull << 8, + (1ull << 8) + 1, + (1ull << 16) - 1, + 1ull << 16, + (1ull << 16) + 1, + (1ull << 32) - 1, + 1ull << 32, + (1ull << 32) + 1}; for (int k = 0; k < sizeof(keys) / sizeof(keys[0]); k++) { for (int s = 0; s < sizeof(seq) / sizeof(seq[0]); s++) { TestKey(keys[k], seq[s], kTypeValue); @@ -62,40 +65,44 @@ TEST(FormatTest, InternalKey_EncodeDecode) { } } +TEST(FormatTest, InternalKey_DecodeFromEmpty) { + InternalKey internal_key; + + ASSERT_TRUE(!internal_key.DecodeFrom("")); +} + TEST(FormatTest, InternalKeyShortSeparator) { // When user keys are same ASSERT_EQ(IKey("foo", 100, kTypeValue), - Shorten(IKey("foo", 100, kTypeValue), - IKey("foo", 99, kTypeValue))); - ASSERT_EQ(IKey("foo", 100, kTypeValue), - Shorten(IKey("foo", 100, kTypeValue), - IKey("foo", 101, kTypeValue))); - ASSERT_EQ(IKey("foo", 100, kTypeValue), - Shorten(IKey("foo", 100, kTypeValue), - IKey("foo", 100, kTypeValue))); - ASSERT_EQ(IKey("foo", 100, kTypeValue), - Shorten(IKey("foo", 100, kTypeValue), - IKey("foo", 100, kTypeDeletion))); + Shorten(IKey("foo", 100, kTypeValue), IKey("foo", 99, kTypeValue))); + ASSERT_EQ( + IKey("foo", 100, kTypeValue), + Shorten(IKey("foo", 100, kTypeValue), IKey("foo", 101, kTypeValue))); + ASSERT_EQ( + IKey("foo", 100, kTypeValue), + Shorten(IKey("foo", 100, kTypeValue), IKey("foo", 100, kTypeValue))); + ASSERT_EQ( + IKey("foo", 100, kTypeValue), + Shorten(IKey("foo", 100, kTypeValue), IKey("foo", 100, kTypeDeletion))); // When user keys are misordered ASSERT_EQ(IKey("foo", 100, kTypeValue), - Shorten(IKey("foo", 100, kTypeValue), - IKey("bar", 99, kTypeValue))); + Shorten(IKey("foo", 100, kTypeValue), IKey("bar", 99, kTypeValue))); // When user keys are different, but correctly ordered - ASSERT_EQ(IKey("g", kMaxSequenceNumber, kValueTypeForSeek), - Shorten(IKey("foo", 100, kTypeValue), - IKey("hello", 200, kTypeValue))); + ASSERT_EQ( + IKey("g", kMaxSequenceNumber, kValueTypeForSeek), + Shorten(IKey("foo", 100, kTypeValue), IKey("hello", 200, kTypeValue))); // When start user key is prefix of limit user key - ASSERT_EQ(IKey("foo", 100, kTypeValue), - Shorten(IKey("foo", 100, kTypeValue), - IKey("foobar", 200, kTypeValue))); + ASSERT_EQ( + IKey("foo", 100, kTypeValue), + Shorten(IKey("foo", 100, kTypeValue), IKey("foobar", 200, kTypeValue))); // When limit user key is prefix of start user key - ASSERT_EQ(IKey("foobar", 100, kTypeValue), - Shorten(IKey("foobar", 100, kTypeValue), - IKey("foo", 200, kTypeValue))); + ASSERT_EQ( + IKey("foobar", 100, kTypeValue), + Shorten(IKey("foobar", 100, kTypeValue), IKey("foo", 200, kTypeValue))); } TEST(FormatTest, InternalKeyShortestSuccessor) { @@ -105,8 +112,20 @@ TEST(FormatTest, InternalKeyShortestSuccessor) { ShortSuccessor(IKey("\xff\xff", 100, kTypeValue))); } -} // namespace leveldb +TEST(FormatTest, ParsedInternalKeyDebugString) { + ParsedInternalKey key("The \"key\" in 'single quotes'", 42, kTypeValue); + + ASSERT_EQ("'The \"key\" in 'single quotes'' @ 42 : 1", key.DebugString()); +} + +TEST(FormatTest, InternalKeyDebugString) { + InternalKey key("The \"key\" in 'single quotes'", 42, kTypeValue); + ASSERT_EQ("'The \"key\" in 'single quotes'' @ 42 : 1", key.DebugString()); -int main(int argc, char** argv) { - return leveldb::test::RunAllTests(); + InternalKey invalid_key; + ASSERT_EQ("(bad)", invalid_key.DebugString()); } + +} // namespace leveldb + +int main(int argc, char** argv) { return leveldb::test::RunAllTests(); } diff --git a/src/leveldb/db/dumpfile.cc b/src/leveldb/db/dumpfile.cc index 61c47c2ff9..77d59003cf 100644 --- a/src/leveldb/db/dumpfile.cc +++ b/src/leveldb/db/dumpfile.cc @@ -2,7 +2,10 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. See the AUTHORS file for names of contributors. +#include "leveldb/dumpfile.h" + #include <stdio.h> + #include "db/dbformat.h" #include "db/filename.h" #include "db/log_reader.h" @@ -35,8 +38,7 @@ bool GuessType(const std::string& fname, FileType* type) { // Notified when log reader encounters corruption. class CorruptionReporter : public log::Reader::Reporter { public: - WritableFile* dst_; - virtual void Corruption(size_t bytes, const Status& status) { + void Corruption(size_t bytes, const Status& status) override { std::string r = "corruption: "; AppendNumberTo(&r, bytes); r += " bytes; "; @@ -44,6 +46,8 @@ class CorruptionReporter : public log::Reader::Reporter { r.push_back('\n'); dst_->Append(r); } + + WritableFile* dst_; }; // Print contents of a log file. (*func)() is called on every record. @@ -70,8 +74,7 @@ Status PrintLogContents(Env* env, const std::string& fname, // Called on every item found in a WriteBatch. class WriteBatchItemPrinter : public WriteBatch::Handler { public: - WritableFile* dst_; - virtual void Put(const Slice& key, const Slice& value) { + void Put(const Slice& key, const Slice& value) override { std::string r = " put '"; AppendEscapedStringTo(&r, key); r += "' '"; @@ -79,14 +82,15 @@ class WriteBatchItemPrinter : public WriteBatch::Handler { r += "'\n"; dst_->Append(r); } - virtual void Delete(const Slice& key) { + void Delete(const Slice& key) override { std::string r = " del '"; AppendEscapedStringTo(&r, key); r += "'\n"; dst_->Append(r); } -}; + WritableFile* dst_; +}; // Called on every log record (each one of which is a WriteBatch) // found in a kLogFile. @@ -142,8 +146,8 @@ Status DumpDescriptor(Env* env, const std::string& fname, WritableFile* dst) { Status DumpTable(Env* env, const std::string& fname, WritableFile* dst) { uint64_t file_size; - RandomAccessFile* file = NULL; - Table* table = NULL; + RandomAccessFile* file = nullptr; + Table* table = nullptr; Status s = env->GetFileSize(fname, &file_size); if (s.ok()) { s = env->NewRandomAccessFile(fname, &file); @@ -213,9 +217,12 @@ Status DumpFile(Env* env, const std::string& fname, WritableFile* dst) { return Status::InvalidArgument(fname + ": unknown file type"); } switch (ftype) { - case kLogFile: return DumpLog(env, fname, dst); - case kDescriptorFile: return DumpDescriptor(env, fname, dst); - case kTableFile: return DumpTable(env, fname, dst); + case kLogFile: + return DumpLog(env, fname, dst); + case kDescriptorFile: + return DumpDescriptor(env, fname, dst); + case kTableFile: + return DumpTable(env, fname, dst); default: break; } diff --git a/src/leveldb/db/fault_injection_test.cc b/src/leveldb/db/fault_injection_test.cc index 875dfe81ee..bf705cb60f 100644 --- a/src/leveldb/db/fault_injection_test.cc +++ b/src/leveldb/db/fault_injection_test.cc @@ -6,18 +6,20 @@ // the last "sync". It then checks for data loss errors by purposely dropping // file data (or entire files) not protected by a "sync". -#include "leveldb/db.h" - #include <map> #include <set> + #include "db/db_impl.h" #include "db/filename.h" #include "db/log_format.h" #include "db/version_set.h" #include "leveldb/cache.h" +#include "leveldb/db.h" #include "leveldb/env.h" #include "leveldb/table.h" #include "leveldb/write_batch.h" +#include "port/port.h" +#include "port/thread_annotations.h" #include "util/logging.h" #include "util/mutexlock.h" #include "util/testharness.h" @@ -34,7 +36,7 @@ class FaultInjectionTestEnv; namespace { // Assume a filename, and not a directory name like "/foo/bar/" -static std::string GetDirName(const std::string filename) { +static std::string GetDirName(const std::string& filename) { size_t found = filename.find_last_of("/\\"); if (found == std::string::npos) { return ""; @@ -54,8 +56,7 @@ Status Truncate(const std::string& filename, uint64_t length) { SequentialFile* orig_file; Status s = env->NewSequentialFile(filename, &orig_file); - if (!s.ok()) - return s; + if (!s.ok()) return s; char* scratch = new char[length]; leveldb::Slice result; @@ -83,15 +84,15 @@ Status Truncate(const std::string& filename, uint64_t length) { struct FileState { std::string filename_; - ssize_t pos_; - ssize_t pos_at_last_sync_; - ssize_t pos_at_last_flush_; + int64_t pos_; + int64_t pos_at_last_sync_; + int64_t pos_at_last_flush_; FileState(const std::string& filename) : filename_(filename), pos_(-1), pos_at_last_sync_(-1), - pos_at_last_flush_(-1) { } + pos_at_last_flush_(-1) {} FileState() : pos_(-1), pos_at_last_sync_(-1), pos_at_last_flush_(-1) {} @@ -106,14 +107,14 @@ struct FileState { // is written to or sync'ed. class TestWritableFile : public WritableFile { public: - TestWritableFile(const FileState& state, - WritableFile* f, + TestWritableFile(const FileState& state, WritableFile* f, FaultInjectionTestEnv* env); - virtual ~TestWritableFile(); - virtual Status Append(const Slice& data); - virtual Status Close(); - virtual Status Flush(); - virtual Status Sync(); + ~TestWritableFile() override; + Status Append(const Slice& data) override; + Status Close() override; + Status Flush() override; + Status Sync() override; + std::string GetName() const override { return ""; } private: FileState state_; @@ -126,14 +127,15 @@ class TestWritableFile : public WritableFile { class FaultInjectionTestEnv : public EnvWrapper { public: - FaultInjectionTestEnv() : EnvWrapper(Env::Default()), filesystem_active_(true) {} - virtual ~FaultInjectionTestEnv() { } - virtual Status NewWritableFile(const std::string& fname, - WritableFile** result); - virtual Status NewAppendableFile(const std::string& fname, - WritableFile** result); - virtual Status DeleteFile(const std::string& f); - virtual Status RenameFile(const std::string& s, const std::string& t); + FaultInjectionTestEnv() + : EnvWrapper(Env::Default()), filesystem_active_(true) {} + ~FaultInjectionTestEnv() override = default; + Status NewWritableFile(const std::string& fname, + WritableFile** result) override; + Status NewAppendableFile(const std::string& fname, + WritableFile** result) override; + Status DeleteFile(const std::string& f) override; + Status RenameFile(const std::string& s, const std::string& t) override; void WritableFileClosed(const FileState& state); Status DropUnsyncedFileData(); @@ -146,24 +148,26 @@ class FaultInjectionTestEnv : public EnvWrapper { // system reset. Setting to inactive will freeze our saved filesystem state so // that it will stop being recorded. It can then be reset back to the state at // the time of the reset. - bool IsFilesystemActive() const { return filesystem_active_; } - void SetFilesystemActive(bool active) { filesystem_active_ = active; } + bool IsFilesystemActive() LOCKS_EXCLUDED(mutex_) { + MutexLock l(&mutex_); + return filesystem_active_; + } + void SetFilesystemActive(bool active) LOCKS_EXCLUDED(mutex_) { + MutexLock l(&mutex_); + filesystem_active_ = active; + } private: port::Mutex mutex_; - std::map<std::string, FileState> db_file_state_; - std::set<std::string> new_files_since_last_dir_sync_; - bool filesystem_active_; // Record flushes, syncs, writes + std::map<std::string, FileState> db_file_state_ GUARDED_BY(mutex_); + std::set<std::string> new_files_since_last_dir_sync_ GUARDED_BY(mutex_); + bool filesystem_active_ GUARDED_BY(mutex_); // Record flushes, syncs, writes }; -TestWritableFile::TestWritableFile(const FileState& state, - WritableFile* f, +TestWritableFile::TestWritableFile(const FileState& state, WritableFile* f, FaultInjectionTestEnv* env) - : state_(state), - target_(f), - writable_file_opened_(true), - env_(env) { - assert(f != NULL); + : state_(state), target_(f), writable_file_opened_(true), env_(env) { + assert(f != nullptr); } TestWritableFile::~TestWritableFile() { @@ -265,10 +269,11 @@ Status FaultInjectionTestEnv::NewAppendableFile(const std::string& fname, Status FaultInjectionTestEnv::DropUnsyncedFileData() { Status s; MutexLock l(&mutex_); - for (std::map<std::string, FileState>::const_iterator it = - db_file_state_.begin(); - s.ok() && it != db_file_state_.end(); ++it) { - const FileState& state = it->second; + for (const auto& kvp : db_file_state_) { + if (!s.ok()) { + break; + } + const FileState& state = kvp.second; if (!state.IsFullySynced()) { s = state.DropUnsyncedData(); } @@ -328,7 +333,6 @@ void FaultInjectionTestEnv::ResetState() { // Since we are not destroying the database, the existing files // should keep their recorded synced/flushed state. Therefore // we do not reset db_file_state_ and new_files_since_last_dir_sync_. - MutexLock l(&mutex_); SetFilesystemActive(true); } @@ -338,12 +342,14 @@ Status FaultInjectionTestEnv::DeleteFilesCreatedAfterLastDirSync() { std::set<std::string> new_files(new_files_since_last_dir_sync_.begin(), new_files_since_last_dir_sync_.end()); mutex_.Unlock(); - Status s; - std::set<std::string>::const_iterator it; - for (it = new_files.begin(); s.ok() && it != new_files.end(); ++it) { - s = DeleteFile(*it); + Status status; + for (const auto& new_file : new_files) { + Status delete_status = DeleteFile(new_file); + if (!delete_status.ok() && status.ok()) { + status = std::move(delete_status); + } } - return s; + return status; } void FaultInjectionTestEnv::WritableFileClosed(const FileState& state) { @@ -352,7 +358,7 @@ void FaultInjectionTestEnv::WritableFileClosed(const FileState& state) { } Status FileState::DropUnsyncedData() const { - ssize_t sync_pos = pos_at_last_sync_ == -1 ? 0 : pos_at_last_sync_; + int64_t sync_pos = pos_at_last_sync_ == -1 ? 0 : pos_at_last_sync_; return Truncate(filename_, sync_pos); } @@ -370,7 +376,7 @@ class FaultInjectionTest { FaultInjectionTest() : env_(new FaultInjectionTestEnv), tiny_cache_(NewLRUCache(100)), - db_(NULL) { + db_(nullptr) { dbname_ = test::TmpDir() + "/fault_test"; DestroyDB(dbname_, Options()); // Destroy any db from earlier run options_.reuse_logs = true; @@ -387,9 +393,7 @@ class FaultInjectionTest { delete env_; } - void ReuseLogs(bool reuse) { - options_.reuse_logs = reuse; - } + void ReuseLogs(bool reuse) { options_.reuse_logs = reuse; } void Build(int start_idx, int num_vals) { std::string key_space, value_space; @@ -449,19 +453,18 @@ class FaultInjectionTest { Status OpenDB() { delete db_; - db_ = NULL; + db_ = nullptr; env_->ResetState(); return DB::Open(options_, dbname_, &db_); } void CloseDB() { delete db_; - db_ = NULL; + db_ = nullptr; } void DeleteAllData() { Iterator* iter = db_->NewIterator(ReadOptions()); - WriteOptions options; for (iter->SeekToFirst(); iter->Valid(); iter->Next()) { ASSERT_OK(db_->Delete(WriteOptions(), iter->key())); } @@ -485,23 +488,22 @@ class FaultInjectionTest { void PartialCompactTestPreFault(int num_pre_sync, int num_post_sync) { DeleteAllData(); Build(0, num_pre_sync); - db_->CompactRange(NULL, NULL); + db_->CompactRange(nullptr, nullptr); Build(num_pre_sync, num_post_sync); } void PartialCompactTestReopenWithFault(ResetMethod reset_method, - int num_pre_sync, - int num_post_sync) { + int num_pre_sync, int num_post_sync) { env_->SetFilesystemActive(false); CloseDB(); ResetDBState(reset_method); ASSERT_OK(OpenDB()); ASSERT_OK(Verify(0, num_pre_sync, FaultInjectionTest::VAL_EXPECT_NO_ERROR)); - ASSERT_OK(Verify(num_pre_sync, num_post_sync, FaultInjectionTest::VAL_EXPECT_ERROR)); + ASSERT_OK(Verify(num_pre_sync, num_post_sync, + FaultInjectionTest::VAL_EXPECT_ERROR)); } - void NoWriteTestPreFault() { - } + void NoWriteTestPreFault() {} void NoWriteTestReopenWithFault(ResetMethod reset_method) { CloseDB(); @@ -517,8 +519,7 @@ class FaultInjectionTest { int num_post_sync = rnd.Uniform(kMaxNumValues); PartialCompactTestPreFault(num_pre_sync, num_post_sync); - PartialCompactTestReopenWithFault(RESET_DROP_UNSYNCED_DATA, - num_pre_sync, + PartialCompactTestReopenWithFault(RESET_DROP_UNSYNCED_DATA, num_pre_sync, num_post_sync); NoWriteTestPreFault(); @@ -528,8 +529,7 @@ class FaultInjectionTest { // No new files created so we expect all values since no files will be // dropped. PartialCompactTestReopenWithFault(RESET_DELETE_UNSYNCED_FILES, - num_pre_sync + num_post_sync, - 0); + num_pre_sync + num_post_sync, 0); NoWriteTestPreFault(); NoWriteTestReopenWithFault(RESET_DELETE_UNSYNCED_FILES); @@ -549,6 +549,4 @@ TEST(FaultInjectionTest, FaultTestWithLogReuse) { } // namespace leveldb -int main(int argc, char** argv) { - return leveldb::test::RunAllTests(); -} +int main(int argc, char** argv) { return leveldb::test::RunAllTests(); } diff --git a/src/leveldb/db/filename.cc b/src/leveldb/db/filename.cc index da32946d99..85de45c507 100644 --- a/src/leveldb/db/filename.cc +++ b/src/leveldb/db/filename.cc @@ -2,9 +2,11 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. See the AUTHORS file for names of contributors. +#include "db/filename.h" + #include <ctype.h> #include <stdio.h> -#include "db/filename.h" + #include "db/dbformat.h" #include "leveldb/env.h" #include "util/logging.h" @@ -12,31 +14,30 @@ namespace leveldb { // A utility routine: write "data" to the named file and Sync() it. -extern Status WriteStringToFileSync(Env* env, const Slice& data, - const std::string& fname); +Status WriteStringToFileSync(Env* env, const Slice& data, + const std::string& fname); -static std::string MakeFileName(const std::string& name, uint64_t number, +static std::string MakeFileName(const std::string& dbname, uint64_t number, const char* suffix) { char buf[100]; snprintf(buf, sizeof(buf), "/%06llu.%s", - static_cast<unsigned long long>(number), - suffix); - return name + buf; + static_cast<unsigned long long>(number), suffix); + return dbname + buf; } -std::string LogFileName(const std::string& name, uint64_t number) { +std::string LogFileName(const std::string& dbname, uint64_t number) { assert(number > 0); - return MakeFileName(name, number, "log"); + return MakeFileName(dbname, number, "log"); } -std::string TableFileName(const std::string& name, uint64_t number) { +std::string TableFileName(const std::string& dbname, uint64_t number) { assert(number > 0); - return MakeFileName(name, number, "ldb"); + return MakeFileName(dbname, number, "ldb"); } -std::string SSTTableFileName(const std::string& name, uint64_t number) { +std::string SSTTableFileName(const std::string& dbname, uint64_t number) { assert(number > 0); - return MakeFileName(name, number, "sst"); + return MakeFileName(dbname, number, "sst"); } std::string DescriptorFileName(const std::string& dbname, uint64_t number) { @@ -51,9 +52,7 @@ std::string CurrentFileName(const std::string& dbname) { return dbname + "/CURRENT"; } -std::string LockFileName(const std::string& dbname) { - return dbname + "/LOCK"; -} +std::string LockFileName(const std::string& dbname) { return dbname + "/LOCK"; } std::string TempFileName(const std::string& dbname, uint64_t number) { assert(number > 0); @@ -69,7 +68,6 @@ std::string OldInfoLogFileName(const std::string& dbname) { return dbname + "/LOG.old"; } - // Owned filenames have the form: // dbname/CURRENT // dbname/LOCK @@ -77,10 +75,9 @@ std::string OldInfoLogFileName(const std::string& dbname) { // dbname/LOG.old // dbname/MANIFEST-[0-9]+ // dbname/[0-9]+.(log|sst|ldb) -bool ParseFileName(const std::string& fname, - uint64_t* number, +bool ParseFileName(const std::string& filename, uint64_t* number, FileType* type) { - Slice rest(fname); + Slice rest(filename); if (rest == "CURRENT") { *number = 0; *type = kCurrentFile; diff --git a/src/leveldb/db/filename.h b/src/leveldb/db/filename.h index 87a752605d..524e813c06 100644 --- a/src/leveldb/db/filename.h +++ b/src/leveldb/db/filename.h @@ -8,7 +8,9 @@ #define STORAGE_LEVELDB_DB_FILENAME_H_ #include <stdint.h> + #include <string> + #include "leveldb/slice.h" #include "leveldb/status.h" #include "port/port.h" @@ -30,55 +32,52 @@ enum FileType { // Return the name of the log file with the specified number // in the db named by "dbname". The result will be prefixed with // "dbname". -extern std::string LogFileName(const std::string& dbname, uint64_t number); +std::string LogFileName(const std::string& dbname, uint64_t number); // Return the name of the sstable with the specified number // in the db named by "dbname". The result will be prefixed with // "dbname". -extern std::string TableFileName(const std::string& dbname, uint64_t number); +std::string TableFileName(const std::string& dbname, uint64_t number); // Return the legacy file name for an sstable with the specified number // in the db named by "dbname". The result will be prefixed with // "dbname". -extern std::string SSTTableFileName(const std::string& dbname, uint64_t number); +std::string SSTTableFileName(const std::string& dbname, uint64_t number); // Return the name of the descriptor file for the db named by // "dbname" and the specified incarnation number. The result will be // prefixed with "dbname". -extern std::string DescriptorFileName(const std::string& dbname, - uint64_t number); +std::string DescriptorFileName(const std::string& dbname, uint64_t number); // Return the name of the current file. This file contains the name // of the current manifest file. The result will be prefixed with // "dbname". -extern std::string CurrentFileName(const std::string& dbname); +std::string CurrentFileName(const std::string& dbname); // Return the name of the lock file for the db named by // "dbname". The result will be prefixed with "dbname". -extern std::string LockFileName(const std::string& dbname); +std::string LockFileName(const std::string& dbname); // Return the name of a temporary file owned by the db named "dbname". // The result will be prefixed with "dbname". -extern std::string TempFileName(const std::string& dbname, uint64_t number); +std::string TempFileName(const std::string& dbname, uint64_t number); // Return the name of the info log file for "dbname". -extern std::string InfoLogFileName(const std::string& dbname); +std::string InfoLogFileName(const std::string& dbname); // Return the name of the old info log file for "dbname". -extern std::string OldInfoLogFileName(const std::string& dbname); +std::string OldInfoLogFileName(const std::string& dbname); // If filename is a leveldb file, store the type of the file in *type. // The number encoded in the filename is stored in *number. If the // filename was successfully parsed, returns true. Else return false. -extern bool ParseFileName(const std::string& filename, - uint64_t* number, - FileType* type); +bool ParseFileName(const std::string& filename, uint64_t* number, + FileType* type); // Make the CURRENT file point to the descriptor file with the // specified number. -extern Status SetCurrentFile(Env* env, const std::string& dbname, - uint64_t descriptor_number); - +Status SetCurrentFile(Env* env, const std::string& dbname, + uint64_t descriptor_number); } // namespace leveldb diff --git a/src/leveldb/db/filename_test.cc b/src/leveldb/db/filename_test.cc index a32556deaf..952f32008e 100644 --- a/src/leveldb/db/filename_test.cc +++ b/src/leveldb/db/filename_test.cc @@ -11,7 +11,7 @@ namespace leveldb { -class FileNameTest { }; +class FileNameTest {}; TEST(FileNameTest, Parse) { Slice db; @@ -24,17 +24,17 @@ TEST(FileNameTest, Parse) { uint64_t number; FileType type; } cases[] = { - { "100.log", 100, kLogFile }, - { "0.log", 0, kLogFile }, - { "0.sst", 0, kTableFile }, - { "0.ldb", 0, kTableFile }, - { "CURRENT", 0, kCurrentFile }, - { "LOCK", 0, kDBLockFile }, - { "MANIFEST-2", 2, kDescriptorFile }, - { "MANIFEST-7", 7, kDescriptorFile }, - { "LOG", 0, kInfoLogFile }, - { "LOG.old", 0, kInfoLogFile }, - { "18446744073709551615.log", 18446744073709551615ull, kLogFile }, + {"100.log", 100, kLogFile}, + {"0.log", 0, kLogFile}, + {"0.sst", 0, kTableFile}, + {"0.ldb", 0, kTableFile}, + {"CURRENT", 0, kCurrentFile}, + {"LOCK", 0, kDBLockFile}, + {"MANIFEST-2", 2, kDescriptorFile}, + {"MANIFEST-7", 7, kDescriptorFile}, + {"LOG", 0, kInfoLogFile}, + {"LOG.old", 0, kInfoLogFile}, + {"18446744073709551615.log", 18446744073709551615ull, kLogFile}, }; for (int i = 0; i < sizeof(cases) / sizeof(cases[0]); i++) { std::string f = cases[i].fname; @@ -44,30 +44,28 @@ TEST(FileNameTest, Parse) { } // Errors - static const char* errors[] = { - "", - "foo", - "foo-dx-100.log", - ".log", - "", - "manifest", - "CURREN", - "CURRENTX", - "MANIFES", - "MANIFEST", - "MANIFEST-", - "XMANIFEST-3", - "MANIFEST-3x", - "LOC", - "LOCKx", - "LO", - "LOGx", - "18446744073709551616.log", - "184467440737095516150.log", - "100", - "100.", - "100.lop" - }; + static const char* errors[] = {"", + "foo", + "foo-dx-100.log", + ".log", + "", + "manifest", + "CURREN", + "CURRENTX", + "MANIFES", + "MANIFEST", + "MANIFEST-", + "XMANIFEST-3", + "MANIFEST-3x", + "LOC", + "LOCKx", + "LO", + "LOGx", + "18446744073709551616.log", + "184467440737095516150.log", + "100", + "100.", + "100.lop"}; for (int i = 0; i < sizeof(errors) / sizeof(errors[0]); i++) { std::string f = errors[i]; ASSERT_TRUE(!ParseFileName(f, &number, &type)) << f; @@ -114,10 +112,20 @@ TEST(FileNameTest, Construction) { ASSERT_TRUE(ParseFileName(fname.c_str() + 4, &number, &type)); ASSERT_EQ(999, number); ASSERT_EQ(kTempFile, type); + + fname = InfoLogFileName("foo"); + ASSERT_EQ("foo/", std::string(fname.data(), 4)); + ASSERT_TRUE(ParseFileName(fname.c_str() + 4, &number, &type)); + ASSERT_EQ(0, number); + ASSERT_EQ(kInfoLogFile, type); + + fname = OldInfoLogFileName("foo"); + ASSERT_EQ("foo/", std::string(fname.data(), 4)); + ASSERT_TRUE(ParseFileName(fname.c_str() + 4, &number, &type)); + ASSERT_EQ(0, number); + ASSERT_EQ(kInfoLogFile, type); } } // namespace leveldb -int main(int argc, char** argv) { - return leveldb::test::RunAllTests(); -} +int main(int argc, char** argv) { return leveldb::test::RunAllTests(); } diff --git a/src/leveldb/db/leveldbutil.cc b/src/leveldb/db/leveldbutil.cc index d06d64d640..9ed9667d37 100644 --- a/src/leveldb/db/leveldbutil.cc +++ b/src/leveldb/db/leveldbutil.cc @@ -3,6 +3,7 @@ // found in the LICENSE file. See the AUTHORS file for names of contributors. #include <stdio.h> + #include "leveldb/dumpfile.h" #include "leveldb/env.h" #include "leveldb/status.h" @@ -12,14 +13,14 @@ namespace { class StdoutPrinter : public WritableFile { public: - virtual Status Append(const Slice& data) { + Status Append(const Slice& data) override { fwrite(data.data(), 1, data.size(), stdout); return Status::OK(); } - virtual Status Close() { return Status::OK(); } - virtual Status Flush() { return Status::OK(); } - virtual Status Sync() { return Status::OK(); } - virtual std::string GetName() const { return "[stdout]"; } + Status Close() override { return Status::OK(); } + Status Flush() override { return Status::OK(); } + Status Sync() override { return Status::OK(); } + std::string GetName() const override { return "[stdout]"; } }; bool HandleDumpCommand(Env* env, char** files, int num) { @@ -39,11 +40,9 @@ bool HandleDumpCommand(Env* env, char** files, int num) { } // namespace leveldb static void Usage() { - fprintf( - stderr, - "Usage: leveldbutil command...\n" - " dump files... -- dump contents of specified files\n" - ); + fprintf(stderr, + "Usage: leveldbutil command...\n" + " dump files... -- dump contents of specified files\n"); } int main(int argc, char** argv) { @@ -55,7 +54,7 @@ int main(int argc, char** argv) { } else { std::string command = argv[1]; if (command == "dump") { - ok = leveldb::HandleDumpCommand(env, argv+2, argc-2); + ok = leveldb::HandleDumpCommand(env, argv + 2, argc - 2); } else { Usage(); ok = false; diff --git a/src/leveldb/db/log_reader.cc b/src/leveldb/db/log_reader.cc index 8b6ad136d7..1ccfb7b34a 100644 --- a/src/leveldb/db/log_reader.cc +++ b/src/leveldb/db/log_reader.cc @@ -5,6 +5,7 @@ #include "db/log_reader.h" #include <stdio.h> + #include "leveldb/env.h" #include "util/coding.h" #include "util/crc32c.h" @@ -12,8 +13,7 @@ namespace leveldb { namespace log { -Reader::Reporter::~Reporter() { -} +Reader::Reporter::~Reporter() = default; Reader::Reader(SequentialFile* file, Reporter* reporter, bool checksum, uint64_t initial_offset) @@ -26,20 +26,16 @@ Reader::Reader(SequentialFile* file, Reporter* reporter, bool checksum, last_record_offset_(0), end_of_buffer_offset_(0), initial_offset_(initial_offset), - resyncing_(initial_offset > 0) { -} + resyncing_(initial_offset > 0) {} -Reader::~Reader() { - delete[] backing_store_; -} +Reader::~Reader() { delete[] backing_store_; } bool Reader::SkipToInitialBlock() { - size_t offset_in_block = initial_offset_ % kBlockSize; + const size_t offset_in_block = initial_offset_ % kBlockSize; uint64_t block_start_location = initial_offset_ - offset_in_block; // Don't search a block if we'd be in the trailer if (offset_in_block > kBlockSize - 6) { - offset_in_block = 0; block_start_location += kBlockSize; } @@ -99,9 +95,7 @@ bool Reader::ReadRecord(Slice* record, std::string* scratch) { // it could emit an empty kFirstType record at the tail end // of a block followed by a kFullType or kFirstType record // at the beginning of the next block. - if (scratch->empty()) { - in_fragmented_record = false; - } else { + if (!scratch->empty()) { ReportCorruption(scratch->size(), "partial record without end(1)"); } } @@ -117,9 +111,7 @@ bool Reader::ReadRecord(Slice* record, std::string* scratch) { // it could emit an empty kFirstType record at the tail end // of a block followed by a kFullType or kFirstType record // at the beginning of the next block. - if (scratch->empty()) { - in_fragmented_record = false; - } else { + if (!scratch->empty()) { ReportCorruption(scratch->size(), "partial record without end(2)"); } } @@ -181,16 +173,14 @@ bool Reader::ReadRecord(Slice* record, std::string* scratch) { return false; } -uint64_t Reader::LastRecordOffset() { - return last_record_offset_; -} +uint64_t Reader::LastRecordOffset() { return last_record_offset_; } void Reader::ReportCorruption(uint64_t bytes, const char* reason) { ReportDrop(bytes, Status::Corruption(reason, file_->GetName())); } void Reader::ReportDrop(uint64_t bytes, const Status& reason) { - if (reporter_ != NULL && + if (reporter_ != nullptr && end_of_buffer_offset_ - buffer_.size() - bytes >= initial_offset_) { reporter_->Corruption(static_cast<size_t>(bytes), reason); } diff --git a/src/leveldb/db/log_reader.h b/src/leveldb/db/log_reader.h index 8389d61f8f..001da8948a 100644 --- a/src/leveldb/db/log_reader.h +++ b/src/leveldb/db/log_reader.h @@ -32,7 +32,7 @@ class Reader { // Create a reader that will return log records from "*file". // "*file" must remain live while this Reader is in use. // - // If "reporter" is non-NULL, it is notified whenever some data is + // If "reporter" is non-null, it is notified whenever some data is // dropped due to a detected corruption. "*reporter" must remain // live while this Reader is in use. // @@ -43,6 +43,9 @@ class Reader { Reader(SequentialFile* file, Reporter* reporter, bool checksum, uint64_t initial_offset); + Reader(const Reader&) = delete; + Reader& operator=(const Reader&) = delete; + ~Reader(); // Read the next record into *record. Returns true if read @@ -58,26 +61,6 @@ class Reader { uint64_t LastRecordOffset(); private: - SequentialFile* const file_; - Reporter* const reporter_; - bool const checksum_; - char* const backing_store_; - Slice buffer_; - bool eof_; // Last Read() indicated EOF by returning < kBlockSize - - // Offset of the last record returned by ReadRecord. - uint64_t last_record_offset_; - // Offset of the first location past the end of buffer_. - uint64_t end_of_buffer_offset_; - - // Offset at which to start looking for the first record to return - uint64_t const initial_offset_; - - // True if we are resynchronizing after a seek (initial_offset_ > 0). In - // particular, a run of kMiddleType and kLastType records can be silently - // skipped in this mode - bool resyncing_; - // Extend record types with the following special values enum { kEof = kMaxRecordType + 1, @@ -102,9 +85,25 @@ class Reader { void ReportCorruption(uint64_t bytes, const char* reason); void ReportDrop(uint64_t bytes, const Status& reason); - // No copying allowed - Reader(const Reader&); - void operator=(const Reader&); + SequentialFile* const file_; + Reporter* const reporter_; + bool const checksum_; + char* const backing_store_; + Slice buffer_; + bool eof_; // Last Read() indicated EOF by returning < kBlockSize + + // Offset of the last record returned by ReadRecord. + uint64_t last_record_offset_; + // Offset of the first location past the end of buffer_. + uint64_t end_of_buffer_offset_; + + // Offset at which to start looking for the first record to return + uint64_t const initial_offset_; + + // True if we are resynchronizing after a seek (initial_offset_ > 0). In + // particular, a run of kMiddleType and kLastType records can be silently + // skipped in this mode + bool resyncing_; }; } // namespace log diff --git a/src/leveldb/db/log_test.cc b/src/leveldb/db/log_test.cc index 48a5928657..41fc043068 100644 --- a/src/leveldb/db/log_test.cc +++ b/src/leveldb/db/log_test.cc @@ -37,87 +37,12 @@ static std::string RandomSkewedString(int i, Random* rnd) { } class LogTest { - private: - class StringDest : public WritableFile { - public: - std::string contents_; - - virtual Status Close() { return Status::OK(); } - virtual Status Flush() { return Status::OK(); } - virtual Status Sync() { return Status::OK(); } - virtual Status Append(const Slice& slice) { - contents_.append(slice.data(), slice.size()); - return Status::OK(); - } - }; - - class StringSource : public SequentialFile { - public: - Slice contents_; - bool force_error_; - bool returned_partial_; - StringSource() : force_error_(false), returned_partial_(false) { } - - virtual Status Read(size_t n, Slice* result, char* scratch) { - ASSERT_TRUE(!returned_partial_) << "must not Read() after eof/error"; - - if (force_error_) { - force_error_ = false; - returned_partial_ = true; - return Status::Corruption("read error"); - } - - if (contents_.size() < n) { - n = contents_.size(); - returned_partial_ = true; - } - *result = Slice(contents_.data(), n); - contents_.remove_prefix(n); - return Status::OK(); - } - - virtual Status Skip(uint64_t n) { - if (n > contents_.size()) { - contents_.clear(); - return Status::NotFound("in-memory file skipped past end"); - } - - contents_.remove_prefix(n); - - return Status::OK(); - } - }; - - class ReportCollector : public Reader::Reporter { - public: - size_t dropped_bytes_; - std::string message_; - - ReportCollector() : dropped_bytes_(0) { } - virtual void Corruption(size_t bytes, const Status& status) { - dropped_bytes_ += bytes; - message_.append(status.ToString()); - } - }; - - StringDest dest_; - StringSource source_; - ReportCollector report_; - bool reading_; - Writer* writer_; - Reader* reader_; - - // Record metadata for testing initial offset functionality - static size_t initial_offset_record_sizes_[]; - static uint64_t initial_offset_last_record_offsets_[]; - static int num_initial_offset_records_; - public: - LogTest() : reading_(false), - writer_(new Writer(&dest_)), - reader_(new Reader(&source_, &report_, true/*checksum*/, - 0/*initial_offset*/)) { - } + LogTest() + : reading_(false), + writer_(new Writer(&dest_)), + reader_(new Reader(&source_, &report_, true /*checksum*/, + 0 /*initial_offset*/)) {} ~LogTest() { delete writer_; @@ -134,9 +59,7 @@ class LogTest { writer_->AddRecord(Slice(msg)); } - size_t WrittenBytes() const { - return dest_.contents_.size(); - } + size_t WrittenBytes() const { return dest_.contents_.size(); } std::string Read() { if (!reading_) { @@ -166,22 +89,16 @@ class LogTest { void FixChecksum(int header_offset, int len) { // Compute crc of type/len/data - uint32_t crc = crc32c::Value(&dest_.contents_[header_offset+6], 1 + len); + uint32_t crc = crc32c::Value(&dest_.contents_[header_offset + 6], 1 + len); crc = crc32c::Mask(crc); EncodeFixed32(&dest_.contents_[header_offset], crc); } - void ForceError() { - source_.force_error_ = true; - } + void ForceError() { source_.force_error_ = true; } - size_t DroppedBytes() const { - return report_.dropped_bytes_; - } + size_t DroppedBytes() const { return report_.dropped_bytes_; } - std::string ReportMessage() const { - return report_.message_; - } + std::string ReportMessage() const { return report_.message_; } // Returns OK iff recorded error message contains "msg" std::string MatchError(const std::string& msg) const { @@ -202,14 +119,14 @@ class LogTest { void StartReadingAt(uint64_t initial_offset) { delete reader_; - reader_ = new Reader(&source_, &report_, true/*checksum*/, initial_offset); + reader_ = new Reader(&source_, &report_, true /*checksum*/, initial_offset); } void CheckOffsetPastEndReturnsNoRecords(uint64_t offset_past_end) { WriteInitialOffsetLog(); reading_ = true; source_.contents_ = Slice(dest_.contents_); - Reader* offset_reader = new Reader(&source_, &report_, true/*checksum*/, + Reader* offset_reader = new Reader(&source_, &report_, true /*checksum*/, WrittenBytes() + offset_past_end); Slice record; std::string scratch; @@ -222,8 +139,8 @@ class LogTest { WriteInitialOffsetLog(); reading_ = true; source_.contents_ = Slice(dest_.contents_); - Reader* offset_reader = new Reader(&source_, &report_, true/*checksum*/, - initial_offset); + Reader* offset_reader = + new Reader(&source_, &report_, true /*checksum*/, initial_offset); // Read all records from expected_record_offset through the last one. ASSERT_LT(expected_record_offset, num_initial_offset_records_); @@ -240,36 +157,110 @@ class LogTest { } delete offset_reader; } + + private: + class StringDest : public WritableFile { + public: + Status Close() override { return Status::OK(); } + Status Flush() override { return Status::OK(); } + Status Sync() override { return Status::OK(); } + Status Append(const Slice& slice) override { + contents_.append(slice.data(), slice.size()); + return Status::OK(); + } + std::string GetName() const override { return ""; } + + std::string contents_; + }; + + class StringSource : public SequentialFile { + public: + StringSource() : force_error_(false), returned_partial_(false) {} + + Status Read(size_t n, Slice* result, char* scratch) override { + ASSERT_TRUE(!returned_partial_) << "must not Read() after eof/error"; + + if (force_error_) { + force_error_ = false; + returned_partial_ = true; + return Status::Corruption("read error"); + } + + if (contents_.size() < n) { + n = contents_.size(); + returned_partial_ = true; + } + *result = Slice(contents_.data(), n); + contents_.remove_prefix(n); + return Status::OK(); + } + + Status Skip(uint64_t n) override { + if (n > contents_.size()) { + contents_.clear(); + return Status::NotFound("in-memory file skipped past end"); + } + + contents_.remove_prefix(n); + + return Status::OK(); + } + std::string GetName() const { return ""; } + + Slice contents_; + bool force_error_; + bool returned_partial_; + }; + + class ReportCollector : public Reader::Reporter { + public: + ReportCollector() : dropped_bytes_(0) {} + void Corruption(size_t bytes, const Status& status) override { + dropped_bytes_ += bytes; + message_.append(status.ToString()); + } + + size_t dropped_bytes_; + std::string message_; + }; + + // Record metadata for testing initial offset functionality + static size_t initial_offset_record_sizes_[]; + static uint64_t initial_offset_last_record_offsets_[]; + static int num_initial_offset_records_; + + StringDest dest_; + StringSource source_; + ReportCollector report_; + bool reading_; + Writer* writer_; + Reader* reader_; +}; + +size_t LogTest::initial_offset_record_sizes_[] = { + 10000, // Two sizable records in first block + 10000, + 2 * log::kBlockSize - 1000, // Span three blocks + 1, + 13716, // Consume all but two bytes of block 3. + log::kBlockSize - kHeaderSize, // Consume the entirety of block 4. }; -size_t LogTest::initial_offset_record_sizes_[] = - {10000, // Two sizable records in first block - 10000, - 2 * log::kBlockSize - 1000, // Span three blocks - 1, - 13716, // Consume all but two bytes of block 3. - log::kBlockSize - kHeaderSize, // Consume the entirety of block 4. - }; - -uint64_t LogTest::initial_offset_last_record_offsets_[] = - {0, - kHeaderSize + 10000, - 2 * (kHeaderSize + 10000), - 2 * (kHeaderSize + 10000) + - (2 * log::kBlockSize - 1000) + 3 * kHeaderSize, - 2 * (kHeaderSize + 10000) + - (2 * log::kBlockSize - 1000) + 3 * kHeaderSize - + kHeaderSize + 1, - 3 * log::kBlockSize, - }; +uint64_t LogTest::initial_offset_last_record_offsets_[] = { + 0, + kHeaderSize + 10000, + 2 * (kHeaderSize + 10000), + 2 * (kHeaderSize + 10000) + (2 * log::kBlockSize - 1000) + 3 * kHeaderSize, + 2 * (kHeaderSize + 10000) + (2 * log::kBlockSize - 1000) + 3 * kHeaderSize + + kHeaderSize + 1, + 3 * log::kBlockSize, +}; // LogTest::initial_offset_last_record_offsets_ must be defined before this. int LogTest::num_initial_offset_records_ = - sizeof(LogTest::initial_offset_last_record_offsets_)/sizeof(uint64_t); + sizeof(LogTest::initial_offset_last_record_offsets_) / sizeof(uint64_t); -TEST(LogTest, Empty) { - ASSERT_EQ("EOF", Read()); -} +TEST(LogTest, Empty) { ASSERT_EQ("EOF", Read()); } TEST(LogTest, ReadWrite) { Write("foo"); @@ -306,7 +297,7 @@ TEST(LogTest, Fragmentation) { TEST(LogTest, MarginalTrailer) { // Make a trailer that is exactly the same length as an empty record. - const int n = kBlockSize - 2*kHeaderSize; + const int n = kBlockSize - 2 * kHeaderSize; Write(BigString("foo", n)); ASSERT_EQ(kBlockSize - kHeaderSize, WrittenBytes()); Write(""); @@ -319,7 +310,7 @@ TEST(LogTest, MarginalTrailer) { TEST(LogTest, MarginalTrailer2) { // Make a trailer that is exactly the same length as an empty record. - const int n = kBlockSize - 2*kHeaderSize; + const int n = kBlockSize - 2 * kHeaderSize; Write(BigString("foo", n)); ASSERT_EQ(kBlockSize - kHeaderSize, WrittenBytes()); Write("bar"); @@ -331,7 +322,7 @@ TEST(LogTest, MarginalTrailer2) { } TEST(LogTest, ShortTrailer) { - const int n = kBlockSize - 2*kHeaderSize + 4; + const int n = kBlockSize - 2 * kHeaderSize + 4; Write(BigString("foo", n)); ASSERT_EQ(kBlockSize - kHeaderSize + 4, WrittenBytes()); Write(""); @@ -343,7 +334,7 @@ TEST(LogTest, ShortTrailer) { } TEST(LogTest, AlignedEof) { - const int n = kBlockSize - 2*kHeaderSize + 4; + const int n = kBlockSize - 2 * kHeaderSize + 4; Write(BigString("foo", n)); ASSERT_EQ(kBlockSize - kHeaderSize + 4, WrittenBytes()); ASSERT_EQ(BigString("foo", n), Read()); @@ -394,7 +385,7 @@ TEST(LogTest, BadRecordType) { TEST(LogTest, TruncatedTrailingRecordIsIgnored) { Write("foo"); - ShrinkSize(4); // Drop all payload as well as a header byte + ShrinkSize(4); // Drop all payload as well as a header byte ASSERT_EQ("EOF", Read()); // Truncated last record is ignored, not treated as an error. ASSERT_EQ(0, DroppedBytes()); @@ -492,7 +483,7 @@ TEST(LogTest, SkipIntoMultiRecord) { // If initial_offset points to a record after first(R1) but before first(R2) // incomplete fragment errors are not actual errors, and must be suppressed // until a new first or full record is encountered. - Write(BigString("foo", 3*kBlockSize)); + Write(BigString("foo", 3 * kBlockSize)); Write("correct"); StartReadingAt(kBlockSize); @@ -514,44 +505,30 @@ TEST(LogTest, ErrorJoinsRecords) { Write("correct"); // Wipe the middle block - for (int offset = kBlockSize; offset < 2*kBlockSize; offset++) { + for (int offset = kBlockSize; offset < 2 * kBlockSize; offset++) { SetByte(offset, 'x'); } ASSERT_EQ("correct", Read()); ASSERT_EQ("EOF", Read()); const size_t dropped = DroppedBytes(); - ASSERT_LE(dropped, 2*kBlockSize + 100); - ASSERT_GE(dropped, 2*kBlockSize); + ASSERT_LE(dropped, 2 * kBlockSize + 100); + ASSERT_GE(dropped, 2 * kBlockSize); } -TEST(LogTest, ReadStart) { - CheckInitialOffsetRecord(0, 0); -} +TEST(LogTest, ReadStart) { CheckInitialOffsetRecord(0, 0); } -TEST(LogTest, ReadSecondOneOff) { - CheckInitialOffsetRecord(1, 1); -} +TEST(LogTest, ReadSecondOneOff) { CheckInitialOffsetRecord(1, 1); } -TEST(LogTest, ReadSecondTenThousand) { - CheckInitialOffsetRecord(10000, 1); -} +TEST(LogTest, ReadSecondTenThousand) { CheckInitialOffsetRecord(10000, 1); } -TEST(LogTest, ReadSecondStart) { - CheckInitialOffsetRecord(10007, 1); -} +TEST(LogTest, ReadSecondStart) { CheckInitialOffsetRecord(10007, 1); } -TEST(LogTest, ReadThirdOneOff) { - CheckInitialOffsetRecord(10008, 2); -} +TEST(LogTest, ReadThirdOneOff) { CheckInitialOffsetRecord(10008, 2); } -TEST(LogTest, ReadThirdStart) { - CheckInitialOffsetRecord(20014, 2); -} +TEST(LogTest, ReadThirdStart) { CheckInitialOffsetRecord(20014, 2); } -TEST(LogTest, ReadFourthOneOff) { - CheckInitialOffsetRecord(20015, 3); -} +TEST(LogTest, ReadFourthOneOff) { CheckInitialOffsetRecord(20015, 3); } TEST(LogTest, ReadFourthFirstBlockTrailer) { CheckInitialOffsetRecord(log::kBlockSize - 4, 3); @@ -575,17 +552,11 @@ TEST(LogTest, ReadInitialOffsetIntoBlockPadding) { CheckInitialOffsetRecord(3 * log::kBlockSize - 3, 5); } -TEST(LogTest, ReadEnd) { - CheckOffsetPastEndReturnsNoRecords(0); -} +TEST(LogTest, ReadEnd) { CheckOffsetPastEndReturnsNoRecords(0); } -TEST(LogTest, ReadPastEnd) { - CheckOffsetPastEndReturnsNoRecords(5); -} +TEST(LogTest, ReadPastEnd) { CheckOffsetPastEndReturnsNoRecords(5); } } // namespace log } // namespace leveldb -int main(int argc, char** argv) { - return leveldb::test::RunAllTests(); -} +int main(int argc, char** argv) { return leveldb::test::RunAllTests(); } diff --git a/src/leveldb/db/log_writer.cc b/src/leveldb/db/log_writer.cc index 74a03270da..bfb16fb486 100644 --- a/src/leveldb/db/log_writer.cc +++ b/src/leveldb/db/log_writer.cc @@ -5,6 +5,7 @@ #include "db/log_writer.h" #include <stdint.h> + #include "leveldb/env.h" #include "util/coding.h" #include "util/crc32c.h" @@ -19,9 +20,7 @@ static void InitTypeCrc(uint32_t* type_crc) { } } -Writer::Writer(WritableFile* dest) - : dest_(dest), - block_offset_(0) { +Writer::Writer(WritableFile* dest) : dest_(dest), block_offset_(0) { InitTypeCrc(type_crc_); } @@ -30,8 +29,7 @@ Writer::Writer(WritableFile* dest, uint64_t dest_length) InitTypeCrc(type_crc_); } -Writer::~Writer() { -} +Writer::~Writer() = default; Status Writer::AddRecord(const Slice& slice) { const char* ptr = slice.data(); @@ -49,7 +47,7 @@ Status Writer::AddRecord(const Slice& slice) { // Switch to a new block if (leftover > 0) { // Fill the trailer (literal below relies on kHeaderSize being 7) - assert(kHeaderSize == 7); + static_assert(kHeaderSize == 7, ""); dest_->Append(Slice("\x00\x00\x00\x00\x00\x00", leftover)); } block_offset_ = 0; @@ -81,30 +79,31 @@ Status Writer::AddRecord(const Slice& slice) { return s; } -Status Writer::EmitPhysicalRecord(RecordType t, const char* ptr, size_t n) { - assert(n <= 0xffff); // Must fit in two bytes - assert(block_offset_ + kHeaderSize + n <= kBlockSize); +Status Writer::EmitPhysicalRecord(RecordType t, const char* ptr, + size_t length) { + assert(length <= 0xffff); // Must fit in two bytes + assert(block_offset_ + kHeaderSize + length <= kBlockSize); // Format the header char buf[kHeaderSize]; - buf[4] = static_cast<char>(n & 0xff); - buf[5] = static_cast<char>(n >> 8); + buf[4] = static_cast<char>(length & 0xff); + buf[5] = static_cast<char>(length >> 8); buf[6] = static_cast<char>(t); // Compute the crc of the record type and the payload. - uint32_t crc = crc32c::Extend(type_crc_[t], ptr, n); - crc = crc32c::Mask(crc); // Adjust for storage + uint32_t crc = crc32c::Extend(type_crc_[t], ptr, length); + crc = crc32c::Mask(crc); // Adjust for storage EncodeFixed32(buf, crc); // Write the header and the payload Status s = dest_->Append(Slice(buf, kHeaderSize)); if (s.ok()) { - s = dest_->Append(Slice(ptr, n)); + s = dest_->Append(Slice(ptr, length)); if (s.ok()) { s = dest_->Flush(); } } - block_offset_ += kHeaderSize + n; + block_offset_ += kHeaderSize + length; return s; } diff --git a/src/leveldb/db/log_writer.h b/src/leveldb/db/log_writer.h index 9e7cc4705b..c0a21147ee 100644 --- a/src/leveldb/db/log_writer.h +++ b/src/leveldb/db/log_writer.h @@ -6,6 +6,7 @@ #define STORAGE_LEVELDB_DB_LOG_WRITER_H_ #include <stdint.h> + #include "db/log_format.h" #include "leveldb/slice.h" #include "leveldb/status.h" @@ -28,24 +29,23 @@ class Writer { // "*dest" must remain live while this Writer is in use. Writer(WritableFile* dest, uint64_t dest_length); + Writer(const Writer&) = delete; + Writer& operator=(const Writer&) = delete; + ~Writer(); Status AddRecord(const Slice& slice); private: + Status EmitPhysicalRecord(RecordType type, const char* ptr, size_t length); + WritableFile* dest_; - int block_offset_; // Current offset in block + int block_offset_; // Current offset in block // crc32c values for all supported record types. These are // pre-computed to reduce the overhead of computing the crc of the // record type stored in the header. uint32_t type_crc_[kMaxRecordType + 1]; - - Status EmitPhysicalRecord(RecordType type, const char* ptr, size_t length); - - // No copying allowed - Writer(const Writer&); - void operator=(const Writer&); }; } // namespace log diff --git a/src/leveldb/db/memtable.cc b/src/leveldb/db/memtable.cc index 287afdbdcb..00931d4671 100644 --- a/src/leveldb/db/memtable.cc +++ b/src/leveldb/db/memtable.cc @@ -18,20 +18,15 @@ static Slice GetLengthPrefixedSlice(const char* data) { return Slice(p, len); } -MemTable::MemTable(const InternalKeyComparator& cmp) - : comparator_(cmp), - refs_(0), - table_(comparator_, &arena_) { -} +MemTable::MemTable(const InternalKeyComparator& comparator) + : comparator_(comparator), refs_(0), table_(comparator_, &arena_) {} -MemTable::~MemTable() { - assert(refs_ == 0); -} +MemTable::~MemTable() { assert(refs_ == 0); } size_t MemTable::ApproximateMemoryUsage() { return arena_.MemoryUsage(); } -int MemTable::KeyComparator::operator()(const char* aptr, const char* bptr) - const { +int MemTable::KeyComparator::operator()(const char* aptr, + const char* bptr) const { // Internal keys are encoded as length-prefixed strings. Slice a = GetLengthPrefixedSlice(aptr); Slice b = GetLengthPrefixedSlice(bptr); @@ -48,39 +43,37 @@ static const char* EncodeKey(std::string* scratch, const Slice& target) { return scratch->data(); } -class MemTableIterator: public Iterator { +class MemTableIterator : public Iterator { public: - explicit MemTableIterator(MemTable::Table* table) : iter_(table) { } - - virtual bool Valid() const { return iter_.Valid(); } - virtual void Seek(const Slice& k) { iter_.Seek(EncodeKey(&tmp_, k)); } - virtual void SeekToFirst() { iter_.SeekToFirst(); } - virtual void SeekToLast() { iter_.SeekToLast(); } - virtual void Next() { iter_.Next(); } - virtual void Prev() { iter_.Prev(); } - virtual Slice key() const { return GetLengthPrefixedSlice(iter_.key()); } - virtual Slice value() const { + explicit MemTableIterator(MemTable::Table* table) : iter_(table) {} + + MemTableIterator(const MemTableIterator&) = delete; + MemTableIterator& operator=(const MemTableIterator&) = delete; + + ~MemTableIterator() override = default; + + bool Valid() const override { return iter_.Valid(); } + void Seek(const Slice& k) override { iter_.Seek(EncodeKey(&tmp_, k)); } + void SeekToFirst() override { iter_.SeekToFirst(); } + void SeekToLast() override { iter_.SeekToLast(); } + void Next() override { iter_.Next(); } + void Prev() override { iter_.Prev(); } + Slice key() const override { return GetLengthPrefixedSlice(iter_.key()); } + Slice value() const override { Slice key_slice = GetLengthPrefixedSlice(iter_.key()); return GetLengthPrefixedSlice(key_slice.data() + key_slice.size()); } - virtual Status status() const { return Status::OK(); } + Status status() const override { return Status::OK(); } private: MemTable::Table::Iterator iter_; - std::string tmp_; // For passing to EncodeKey - - // No copying allowed - MemTableIterator(const MemTableIterator&); - void operator=(const MemTableIterator&); + std::string tmp_; // For passing to EncodeKey }; -Iterator* MemTable::NewIterator() { - return new MemTableIterator(&table_); -} +Iterator* MemTable::NewIterator() { return new MemTableIterator(&table_); } -void MemTable::Add(SequenceNumber s, ValueType type, - const Slice& key, +void MemTable::Add(SequenceNumber s, ValueType type, const Slice& key, const Slice& value) { // Format of an entry is concatenation of: // key_size : varint32 of internal_key.size() @@ -90,9 +83,9 @@ void MemTable::Add(SequenceNumber s, ValueType type, size_t key_size = key.size(); size_t val_size = value.size(); size_t internal_key_size = key_size + 8; - const size_t encoded_len = - VarintLength(internal_key_size) + internal_key_size + - VarintLength(val_size) + val_size; + const size_t encoded_len = VarintLength(internal_key_size) + + internal_key_size + VarintLength(val_size) + + val_size; char* buf = arena_.Allocate(encoded_len); char* p = EncodeVarint32(buf, internal_key_size); memcpy(p, key.data(), key_size); @@ -121,10 +114,9 @@ bool MemTable::Get(const LookupKey& key, std::string* value, Status* s) { // all entries with overly large sequence numbers. const char* entry = iter.key(); uint32_t key_length; - const char* key_ptr = GetVarint32Ptr(entry, entry+5, &key_length); + const char* key_ptr = GetVarint32Ptr(entry, entry + 5, &key_length); if (comparator_.comparator.user_comparator()->Compare( - Slice(key_ptr, key_length - 8), - key.user_key()) == 0) { + Slice(key_ptr, key_length - 8), key.user_key()) == 0) { // Correct user key const uint64_t tag = DecodeFixed64(key_ptr + key_length - 8); switch (static_cast<ValueType>(tag & 0xff)) { diff --git a/src/leveldb/db/memtable.h b/src/leveldb/db/memtable.h index 9f41567cde..9d986b1070 100644 --- a/src/leveldb/db/memtable.h +++ b/src/leveldb/db/memtable.h @@ -6,15 +6,15 @@ #define STORAGE_LEVELDB_DB_MEMTABLE_H_ #include <string> -#include "leveldb/db.h" + #include "db/dbformat.h" #include "db/skiplist.h" +#include "leveldb/db.h" #include "util/arena.h" namespace leveldb { class InternalKeyComparator; -class Mutex; class MemTableIterator; class MemTable { @@ -23,6 +23,9 @@ class MemTable { // is zero and the caller must call Ref() at least once. explicit MemTable(const InternalKeyComparator& comparator); + MemTable(const MemTable&) = delete; + MemTable& operator=(const MemTable&) = delete; + // Increase reference count. void Ref() { ++refs_; } @@ -50,8 +53,7 @@ class MemTable { // Add an entry into memtable that maps key to value at the // specified sequence number and with the specified type. // Typically value will be empty if type==kTypeDeletion. - void Add(SequenceNumber seq, ValueType type, - const Slice& key, + void Add(SequenceNumber seq, ValueType type, const Slice& key, const Slice& value); // If memtable contains a value for key, store it in *value and return true. @@ -61,26 +63,23 @@ class MemTable { bool Get(const LookupKey& key, std::string* value, Status* s); private: - ~MemTable(); // Private since only Unref() should be used to delete it + friend class MemTableIterator; + friend class MemTableBackwardIterator; struct KeyComparator { const InternalKeyComparator comparator; - explicit KeyComparator(const InternalKeyComparator& c) : comparator(c) { } + explicit KeyComparator(const InternalKeyComparator& c) : comparator(c) {} int operator()(const char* a, const char* b) const; }; - friend class MemTableIterator; - friend class MemTableBackwardIterator; typedef SkipList<const char*, KeyComparator> Table; + ~MemTable(); // Private since only Unref() should be used to delete it + KeyComparator comparator_; int refs_; Arena arena_; Table table_; - - // No copying allowed - MemTable(const MemTable&); - void operator=(const MemTable&); }; } // namespace leveldb diff --git a/src/leveldb/db/recovery_test.cc b/src/leveldb/db/recovery_test.cc index 9596f4288a..547a9591ea 100644 --- a/src/leveldb/db/recovery_test.cc +++ b/src/leveldb/db/recovery_test.cc @@ -17,7 +17,7 @@ namespace leveldb { class RecoveryTest { public: - RecoveryTest() : env_(Env::Default()), db_(NULL) { + RecoveryTest() : env_(Env::Default()), db_(nullptr) { dbname_ = test::TmpDir() + "/recovery_test"; DestroyDB(dbname_, Options()); Open(); @@ -44,22 +44,26 @@ class RecoveryTest { void Close() { delete db_; - db_ = NULL; + db_ = nullptr; } - void Open(Options* options = NULL) { + Status OpenWithStatus(Options* options = nullptr) { Close(); Options opts; - if (options != NULL) { + if (options != nullptr) { opts = *options; } else { opts.reuse_logs = true; // TODO(sanjay): test both ways opts.create_if_missing = true; } - if (opts.env == NULL) { + if (opts.env == nullptr) { opts.env = env_; } - ASSERT_OK(DB::Open(opts, dbname_, &db_)); + return DB::Open(opts, dbname_, &db_); + } + + void Open(Options* options = nullptr) { + ASSERT_OK(OpenWithStatus(options)); ASSERT_EQ(1, NumLogs()); } @@ -67,7 +71,7 @@ class RecoveryTest { return db_->Put(WriteOptions(), k, v); } - std::string Get(const std::string& k, const Snapshot* snapshot = NULL) { + std::string Get(const std::string& k, const Snapshot* snapshot = nullptr) { std::string result; Status s = db_->Get(ReadOptions(), k, &result); if (s.IsNotFound()) { @@ -82,17 +86,18 @@ class RecoveryTest { std::string current; ASSERT_OK(ReadFileToString(env_, CurrentFileName(dbname_), ¤t)); size_t len = current.size(); - if (len > 0 && current[len-1] == '\n') { + if (len > 0 && current[len - 1] == '\n') { current.resize(len - 1); } return dbname_ + "/" + current; } - std::string LogName(uint64_t number) { - return LogFileName(dbname_, number); - } + std::string LogName(uint64_t number) { return LogFileName(dbname_, number); } size_t DeleteLogFiles() { + // Linux allows unlinking open files, but Windows does not. + // Closing the db allows for file deletion. + Close(); std::vector<uint64_t> logs = GetFiles(kLogFile); for (size_t i = 0; i < logs.size(); i++) { ASSERT_OK(env_->DeleteFile(LogName(logs[i]))) << LogName(logs[i]); @@ -100,9 +105,9 @@ class RecoveryTest { return logs.size(); } - uint64_t FirstLogFile() { - return GetFiles(kLogFile)[0]; - } + void DeleteManifestFile() { ASSERT_OK(env_->DeleteFile(ManifestFileName())); } + + uint64_t FirstLogFile() { return GetFiles(kLogFile)[0]; } std::vector<uint64_t> GetFiles(FileType t) { std::vector<std::string> filenames; @@ -118,13 +123,9 @@ class RecoveryTest { return result; } - int NumLogs() { - return GetFiles(kLogFile).size(); - } + int NumLogs() { return GetFiles(kLogFile).size(); } - int NumTables() { - return GetFiles(kTableFile).size(); - } + int NumTables() { return GetFiles(kTableFile).size(); } uint64_t FileSize(const std::string& fname) { uint64_t result; @@ -132,9 +133,7 @@ class RecoveryTest { return result; } - void CompactMemTable() { - dbfull()->TEST_CompactMemTable(); - } + void CompactMemTable() { dbfull()->TEST_CompactMemTable(); } // Directly construct a log file that sets key to val. void MakeLogFile(uint64_t lognum, SequenceNumber seq, Slice key, Slice val) { @@ -186,7 +185,7 @@ TEST(RecoveryTest, LargeManifestCompacted) { uint64_t len = FileSize(old_manifest); WritableFile* file; ASSERT_OK(env()->NewAppendableFile(old_manifest, &file)); - std::string zeroes(3*1048576 - static_cast<size_t>(len), 0); + std::string zeroes(3 * 1048576 - static_cast<size_t>(len), 0); ASSERT_OK(file->Append(zeroes)); ASSERT_OK(file->Flush()); delete file; @@ -259,7 +258,7 @@ TEST(RecoveryTest, MultipleMemTables) { // Force creation of multiple memtables by reducing the write buffer size. Options opt; opt.reuse_logs = true; - opt.write_buffer_size = (kNum*100) / 2; + opt.write_buffer_size = (kNum * 100) / 2; Open(&opt); ASSERT_LE(2, NumTables()); ASSERT_EQ(1, NumLogs()); @@ -278,16 +277,16 @@ TEST(RecoveryTest, MultipleLogFiles) { // Make a bunch of uncompacted log files. uint64_t old_log = FirstLogFile(); - MakeLogFile(old_log+1, 1000, "hello", "world"); - MakeLogFile(old_log+2, 1001, "hi", "there"); - MakeLogFile(old_log+3, 1002, "foo", "bar2"); + MakeLogFile(old_log + 1, 1000, "hello", "world"); + MakeLogFile(old_log + 2, 1001, "hi", "there"); + MakeLogFile(old_log + 3, 1002, "foo", "bar2"); // Recover and check that all log files were processed. Open(); ASSERT_LE(1, NumTables()); ASSERT_EQ(1, NumLogs()); uint64_t new_log = FirstLogFile(); - ASSERT_LE(old_log+3, new_log); + ASSERT_LE(old_log + 3, new_log); ASSERT_EQ("bar2", Get("foo")); ASSERT_EQ("world", Get("hello")); ASSERT_EQ("there", Get("hi")); @@ -305,7 +304,7 @@ TEST(RecoveryTest, MultipleLogFiles) { // Check that introducing an older log file does not cause it to be re-read. Close(); - MakeLogFile(old_log+1, 2000, "hello", "stale write"); + MakeLogFile(old_log + 1, 2000, "hello", "stale write"); Open(); ASSERT_LE(1, NumTables()); ASSERT_EQ(1, NumLogs()); @@ -317,8 +316,15 @@ TEST(RecoveryTest, MultipleLogFiles) { ASSERT_EQ("there", Get("hi")); } -} // namespace leveldb +TEST(RecoveryTest, ManifestMissing) { + ASSERT_OK(Put("foo", "bar")); + Close(); + DeleteManifestFile(); -int main(int argc, char** argv) { - return leveldb::test::RunAllTests(); + Status status = OpenWithStatus(); + ASSERT_TRUE(status.IsCorruption()); } + +} // namespace leveldb + +int main(int argc, char** argv) { return leveldb::test::RunAllTests(); } diff --git a/src/leveldb/db/repair.cc b/src/leveldb/db/repair.cc index 7281e3d345..04847c3bbf 100644 --- a/src/leveldb/db/repair.cc +++ b/src/leveldb/db/repair.cc @@ -54,7 +54,7 @@ class Repairer { owns_cache_(options_.block_cache != options.block_cache), next_file_number_(1) { // TableCache can be small since we expect each table to be opened once. - table_cache_ = new TableCache(dbname_, &options_, 10); + table_cache_ = new TableCache(dbname_, options_, 10); } ~Repairer() { @@ -84,9 +84,7 @@ class Repairer { "recovered %d files; %llu bytes. " "Some data may have been lost. " "****", - dbname_.c_str(), - static_cast<int>(tables_.size()), - bytes); + dbname_.c_str(), static_cast<int>(tables_.size()), bytes); } return status; } @@ -97,22 +95,6 @@ class Repairer { SequenceNumber max_sequence; }; - std::string const dbname_; - Env* const env_; - InternalKeyComparator const icmp_; - InternalFilterPolicy const ipolicy_; - Options const options_; - bool owns_info_log_; - bool owns_cache_; - TableCache* table_cache_; - VersionEdit edit_; - - std::vector<std::string> manifests_; - std::vector<uint64_t> table_numbers_; - std::vector<uint64_t> logs_; - std::vector<TableInfo> tables_; - uint64_t next_file_number_; - Status FindFiles() { std::vector<std::string> filenames; Status status = env_->GetChildren(dbname_, &filenames); @@ -152,8 +134,7 @@ class Repairer { Status status = ConvertLogToTable(logs_[i]); if (!status.ok()) { Log(options_.info_log, "Log #%llu: ignoring conversion error: %s", - (unsigned long long) logs_[i], - status.ToString().c_str()); + (unsigned long long)logs_[i], status.ToString().c_str()); } ArchiveFile(logname); } @@ -164,11 +145,10 @@ class Repairer { Env* env; Logger* info_log; uint64_t lognum; - virtual void Corruption(size_t bytes, const Status& s) { + void Corruption(size_t bytes, const Status& s) override { // We print error messages for corruption, but continue repairing. Log(info_log, "Log #%llu: dropping %d bytes; %s", - (unsigned long long) lognum, - static_cast<int>(bytes), + (unsigned long long)lognum, static_cast<int>(bytes), s.ToString().c_str()); } }; @@ -190,8 +170,8 @@ class Repairer { // corruptions cause entire commits to be skipped instead of // propagating bad information (like overly large sequence // numbers). - log::Reader reader(lfile, &reporter, false/*do not checksum*/, - 0/*initial_offset*/); + log::Reader reader(lfile, &reporter, false /*do not checksum*/, + 0 /*initial_offset*/); // Read all the records and add to a memtable std::string scratch; @@ -202,8 +182,8 @@ class Repairer { int counter = 0; while (reader.ReadRecord(&record, &scratch)) { if (record.size() < 12) { - reporter.Corruption( - record.size(), Status::Corruption("log record too small", logname)); + reporter.Corruption(record.size(), + Status::Corruption("log record too small", logname)); continue; } WriteBatchInternal::SetContents(&batch, record); @@ -212,8 +192,7 @@ class Repairer { counter += WriteBatchInternal::Count(&batch); } else { Log(options_.info_log, "Log #%llu: ignoring %s", - (unsigned long long) log, - status.ToString().c_str()); + (unsigned long long)log, status.ToString().c_str()); status = Status::OK(); // Keep going with rest of file } } @@ -227,16 +206,14 @@ class Repairer { status = BuildTable(dbname_, env_, options_, table_cache_, iter, &meta); delete iter; mem->Unref(); - mem = NULL; + mem = nullptr; if (status.ok()) { if (meta.file_size > 0) { table_numbers_.push_back(meta.number); } } Log(options_.info_log, "Log #%llu: %d ops saved to Table #%llu %s", - (unsigned long long) log, - counter, - (unsigned long long) meta.number, + (unsigned long long)log, counter, (unsigned long long)meta.number, status.ToString().c_str()); return status; } @@ -272,8 +249,7 @@ class Repairer { ArchiveFile(TableFileName(dbname_, number)); ArchiveFile(SSTTableFileName(dbname_, number)); Log(options_.info_log, "Table #%llu: dropped: %s", - (unsigned long long) t.meta.number, - status.ToString().c_str()); + (unsigned long long)t.meta.number, status.ToString().c_str()); return; } @@ -287,8 +263,7 @@ class Repairer { Slice key = iter->key(); if (!ParseInternalKey(key, &parsed)) { Log(options_.info_log, "Table #%llu: unparsable key %s", - (unsigned long long) t.meta.number, - EscapeString(key).c_str()); + (unsigned long long)t.meta.number, EscapeString(key).c_str()); continue; } @@ -307,9 +282,7 @@ class Repairer { } delete iter; Log(options_.info_log, "Table #%llu: %d entries %s", - (unsigned long long) t.meta.number, - counter, - status.ToString().c_str()); + (unsigned long long)t.meta.number, counter, status.ToString().c_str()); if (status.ok()) { tables_.push_back(t); @@ -350,20 +323,20 @@ class Repairer { } } delete builder; - builder = NULL; + builder = nullptr; if (s.ok()) { s = file->Close(); } delete file; - file = NULL; + file = nullptr; if (counter > 0 && s.ok()) { std::string orig = TableFileName(dbname_, t.meta.number); s = env_->RenameFile(copy, orig); if (s.ok()) { Log(options_.info_log, "Table #%llu: %d entries repaired", - (unsigned long long) t.meta.number, counter); + (unsigned long long)t.meta.number, counter); tables_.push_back(t); } } @@ -395,11 +368,11 @@ class Repairer { for (size_t i = 0; i < tables_.size(); i++) { // TODO(opt): separate out into multiple levels const TableInfo& t = tables_[i]; - edit_.AddFile(0, t.meta.number, t.meta.file_size, - t.meta.smallest, t.meta.largest); + edit_.AddFile(0, t.meta.number, t.meta.file_size, t.meta.smallest, + t.meta.largest); } - //fprintf(stderr, "NewDescriptor:\n%s\n", edit_.DebugString().c_str()); + // fprintf(stderr, "NewDescriptor:\n%s\n", edit_.DebugString().c_str()); { log::Writer log(file); std::string record; @@ -410,7 +383,7 @@ class Repairer { status = file->Close(); } delete file; - file = NULL; + file = nullptr; if (!status.ok()) { env_->DeleteFile(tmp); @@ -438,18 +411,34 @@ class Repairer { // dir/lost/foo const char* slash = strrchr(fname.c_str(), '/'); std::string new_dir; - if (slash != NULL) { + if (slash != nullptr) { new_dir.assign(fname.data(), slash - fname.data()); } new_dir.append("/lost"); env_->CreateDir(new_dir); // Ignore error std::string new_file = new_dir; new_file.append("/"); - new_file.append((slash == NULL) ? fname.c_str() : slash + 1); + new_file.append((slash == nullptr) ? fname.c_str() : slash + 1); Status s = env_->RenameFile(fname, new_file); - Log(options_.info_log, "Archiving %s: %s\n", - fname.c_str(), s.ToString().c_str()); + Log(options_.info_log, "Archiving %s: %s\n", fname.c_str(), + s.ToString().c_str()); } + + const std::string dbname_; + Env* const env_; + InternalKeyComparator const icmp_; + InternalFilterPolicy const ipolicy_; + const Options options_; + bool owns_info_log_; + bool owns_cache_; + TableCache* table_cache_; + VersionEdit edit_; + + std::vector<std::string> manifests_; + std::vector<uint64_t> table_numbers_; + std::vector<uint64_t> logs_; + std::vector<TableInfo> tables_; + uint64_t next_file_number_; }; } // namespace diff --git a/src/leveldb/db/skiplist.h b/src/leveldb/db/skiplist.h index 8bd77764d8..a59b45b380 100644 --- a/src/leveldb/db/skiplist.h +++ b/src/leveldb/db/skiplist.h @@ -27,9 +27,10 @@ // // ... prev vs. next pointer ordering ... -#include <assert.h> -#include <stdlib.h> -#include "port/port.h" +#include <atomic> +#include <cassert> +#include <cstdlib> + #include "util/arena.h" #include "util/random.h" @@ -37,7 +38,7 @@ namespace leveldb { class Arena; -template<typename Key, class Comparator> +template <typename Key, class Comparator> class SkipList { private: struct Node; @@ -48,6 +49,9 @@ class SkipList { // must remain allocated for the lifetime of the skiplist object. explicit SkipList(Comparator cmp, Arena* arena); + SkipList(const SkipList&) = delete; + SkipList& operator=(const SkipList&) = delete; + // Insert key into the list. // REQUIRES: nothing that compares equal to key is currently in the list. void Insert(const Key& key); @@ -97,24 +101,10 @@ class SkipList { private: enum { kMaxHeight = 12 }; - // Immutable after construction - Comparator const compare_; - Arena* const arena_; // Arena used for allocations of nodes - - Node* const head_; - - // Modified only by Insert(). Read racily by readers, but stale - // values are ok. - port::AtomicPointer max_height_; // Height of the entire list - inline int GetMaxHeight() const { - return static_cast<int>( - reinterpret_cast<intptr_t>(max_height_.NoBarrier_Load())); + return max_height_.load(std::memory_order_relaxed); } - // Read/written only by Insert(). - Random rnd_; - Node* NewNode(const Key& key, int height); int RandomHeight(); bool Equal(const Key& a, const Key& b) const { return (compare_(a, b) == 0); } @@ -123,9 +113,9 @@ class SkipList { bool KeyIsAfterNode(const Key& key, Node* n) const; // Return the earliest node that comes at or after key. - // Return NULL if there is no such node. + // Return nullptr if there is no such node. // - // If prev is non-NULL, fills prev[level] with pointer to previous + // If prev is non-null, fills prev[level] with pointer to previous // node at "level" for every level in [0..max_height_-1]. Node* FindGreaterOrEqual(const Key& key, Node** prev) const; @@ -137,15 +127,24 @@ class SkipList { // Return head_ if list is empty. Node* FindLast() const; - // No copying allowed - SkipList(const SkipList&); - void operator=(const SkipList&); + // Immutable after construction + Comparator const compare_; + Arena* const arena_; // Arena used for allocations of nodes + + Node* const head_; + + // Modified only by Insert(). Read racily by readers, but stale + // values are ok. + std::atomic<int> max_height_; // Height of the entire list + + // Read/written only by Insert(). + Random rnd_; }; // Implementation details follow -template<typename Key, class Comparator> -struct SkipList<Key,Comparator>::Node { - explicit Node(const Key& k) : key(k) { } +template <typename Key, class Comparator> +struct SkipList<Key, Comparator>::Node { + explicit Node(const Key& k) : key(k) {} Key const key; @@ -155,92 +154,92 @@ struct SkipList<Key,Comparator>::Node { assert(n >= 0); // Use an 'acquire load' so that we observe a fully initialized // version of the returned Node. - return reinterpret_cast<Node*>(next_[n].Acquire_Load()); + return next_[n].load(std::memory_order_acquire); } void SetNext(int n, Node* x) { assert(n >= 0); // Use a 'release store' so that anybody who reads through this // pointer observes a fully initialized version of the inserted node. - next_[n].Release_Store(x); + next_[n].store(x, std::memory_order_release); } // No-barrier variants that can be safely used in a few locations. Node* NoBarrier_Next(int n) { assert(n >= 0); - return reinterpret_cast<Node*>(next_[n].NoBarrier_Load()); + return next_[n].load(std::memory_order_relaxed); } void NoBarrier_SetNext(int n, Node* x) { assert(n >= 0); - next_[n].NoBarrier_Store(x); + next_[n].store(x, std::memory_order_relaxed); } private: // Array of length equal to the node height. next_[0] is lowest level link. - port::AtomicPointer next_[1]; + std::atomic<Node*> next_[1]; }; -template<typename Key, class Comparator> -typename SkipList<Key,Comparator>::Node* -SkipList<Key,Comparator>::NewNode(const Key& key, int height) { - char* mem = arena_->AllocateAligned( - sizeof(Node) + sizeof(port::AtomicPointer) * (height - 1)); - return new (mem) Node(key); +template <typename Key, class Comparator> +typename SkipList<Key, Comparator>::Node* SkipList<Key, Comparator>::NewNode( + const Key& key, int height) { + char* const node_memory = arena_->AllocateAligned( + sizeof(Node) + sizeof(std::atomic<Node*>) * (height - 1)); + return new (node_memory) Node(key); } -template<typename Key, class Comparator> -inline SkipList<Key,Comparator>::Iterator::Iterator(const SkipList* list) { +template <typename Key, class Comparator> +inline SkipList<Key, Comparator>::Iterator::Iterator(const SkipList* list) { list_ = list; - node_ = NULL; + node_ = nullptr; } -template<typename Key, class Comparator> -inline bool SkipList<Key,Comparator>::Iterator::Valid() const { - return node_ != NULL; +template <typename Key, class Comparator> +inline bool SkipList<Key, Comparator>::Iterator::Valid() const { + return node_ != nullptr; } -template<typename Key, class Comparator> -inline const Key& SkipList<Key,Comparator>::Iterator::key() const { +template <typename Key, class Comparator> +inline const Key& SkipList<Key, Comparator>::Iterator::key() const { assert(Valid()); return node_->key; } -template<typename Key, class Comparator> -inline void SkipList<Key,Comparator>::Iterator::Next() { +template <typename Key, class Comparator> +inline void SkipList<Key, Comparator>::Iterator::Next() { assert(Valid()); node_ = node_->Next(0); } -template<typename Key, class Comparator> -inline void SkipList<Key,Comparator>::Iterator::Prev() { +template <typename Key, class Comparator> +inline void SkipList<Key, Comparator>::Iterator::Prev() { // Instead of using explicit "prev" links, we just search for the // last node that falls before key. assert(Valid()); node_ = list_->FindLessThan(node_->key); if (node_ == list_->head_) { - node_ = NULL; + node_ = nullptr; } } -template<typename Key, class Comparator> -inline void SkipList<Key,Comparator>::Iterator::Seek(const Key& target) { - node_ = list_->FindGreaterOrEqual(target, NULL); +template <typename Key, class Comparator> +inline void SkipList<Key, Comparator>::Iterator::Seek(const Key& target) { + node_ = list_->FindGreaterOrEqual(target, nullptr); } -template<typename Key, class Comparator> -inline void SkipList<Key,Comparator>::Iterator::SeekToFirst() { +template <typename Key, class Comparator> +inline void SkipList<Key, Comparator>::Iterator::SeekToFirst() { node_ = list_->head_->Next(0); } -template<typename Key, class Comparator> -inline void SkipList<Key,Comparator>::Iterator::SeekToLast() { +template <typename Key, class Comparator> +inline void SkipList<Key, Comparator>::Iterator::SeekToLast() { node_ = list_->FindLast(); if (node_ == list_->head_) { - node_ = NULL; + node_ = nullptr; } } -template<typename Key, class Comparator> -int SkipList<Key,Comparator>::RandomHeight() { +template <typename Key, class Comparator> +int SkipList<Key, Comparator>::RandomHeight() { // Increase height with probability 1 in kBranching static const unsigned int kBranching = 4; int height = 1; @@ -252,15 +251,16 @@ int SkipList<Key,Comparator>::RandomHeight() { return height; } -template<typename Key, class Comparator> -bool SkipList<Key,Comparator>::KeyIsAfterNode(const Key& key, Node* n) const { - // NULL n is considered infinite - return (n != NULL) && (compare_(n->key, key) < 0); +template <typename Key, class Comparator> +bool SkipList<Key, Comparator>::KeyIsAfterNode(const Key& key, Node* n) const { + // null n is considered infinite + return (n != nullptr) && (compare_(n->key, key) < 0); } -template<typename Key, class Comparator> -typename SkipList<Key,Comparator>::Node* SkipList<Key,Comparator>::FindGreaterOrEqual(const Key& key, Node** prev) - const { +template <typename Key, class Comparator> +typename SkipList<Key, Comparator>::Node* +SkipList<Key, Comparator>::FindGreaterOrEqual(const Key& key, + Node** prev) const { Node* x = head_; int level = GetMaxHeight() - 1; while (true) { @@ -269,7 +269,7 @@ typename SkipList<Key,Comparator>::Node* SkipList<Key,Comparator>::FindGreaterOr // Keep searching in this list x = next; } else { - if (prev != NULL) prev[level] = x; + if (prev != nullptr) prev[level] = x; if (level == 0) { return next; } else { @@ -280,15 +280,15 @@ typename SkipList<Key,Comparator>::Node* SkipList<Key,Comparator>::FindGreaterOr } } -template<typename Key, class Comparator> -typename SkipList<Key,Comparator>::Node* -SkipList<Key,Comparator>::FindLessThan(const Key& key) const { +template <typename Key, class Comparator> +typename SkipList<Key, Comparator>::Node* +SkipList<Key, Comparator>::FindLessThan(const Key& key) const { Node* x = head_; int level = GetMaxHeight() - 1; while (true) { assert(x == head_ || compare_(x->key, key) < 0); Node* next = x->Next(level); - if (next == NULL || compare_(next->key, key) >= 0) { + if (next == nullptr || compare_(next->key, key) >= 0) { if (level == 0) { return x; } else { @@ -301,14 +301,14 @@ SkipList<Key,Comparator>::FindLessThan(const Key& key) const { } } -template<typename Key, class Comparator> -typename SkipList<Key,Comparator>::Node* SkipList<Key,Comparator>::FindLast() +template <typename Key, class Comparator> +typename SkipList<Key, Comparator>::Node* SkipList<Key, Comparator>::FindLast() const { Node* x = head_; int level = GetMaxHeight() - 1; while (true) { Node* next = x->Next(level); - if (next == NULL) { + if (next == nullptr) { if (level == 0) { return x; } else { @@ -321,43 +321,41 @@ typename SkipList<Key,Comparator>::Node* SkipList<Key,Comparator>::FindLast() } } -template<typename Key, class Comparator> -SkipList<Key,Comparator>::SkipList(Comparator cmp, Arena* arena) +template <typename Key, class Comparator> +SkipList<Key, Comparator>::SkipList(Comparator cmp, Arena* arena) : compare_(cmp), arena_(arena), head_(NewNode(0 /* any key will do */, kMaxHeight)), - max_height_(reinterpret_cast<void*>(1)), + max_height_(1), rnd_(0xdeadbeef) { for (int i = 0; i < kMaxHeight; i++) { - head_->SetNext(i, NULL); + head_->SetNext(i, nullptr); } } -template<typename Key, class Comparator> -void SkipList<Key,Comparator>::Insert(const Key& key) { +template <typename Key, class Comparator> +void SkipList<Key, Comparator>::Insert(const Key& key) { // TODO(opt): We can use a barrier-free variant of FindGreaterOrEqual() // here since Insert() is externally synchronized. Node* prev[kMaxHeight]; Node* x = FindGreaterOrEqual(key, prev); // Our data structure does not allow duplicate insertion - assert(x == NULL || !Equal(key, x->key)); + assert(x == nullptr || !Equal(key, x->key)); int height = RandomHeight(); if (height > GetMaxHeight()) { for (int i = GetMaxHeight(); i < height; i++) { prev[i] = head_; } - //fprintf(stderr, "Change height from %d to %d\n", max_height_, height); - // It is ok to mutate max_height_ without any synchronization // with concurrent readers. A concurrent reader that observes // the new value of max_height_ will see either the old value of - // new level pointers from head_ (NULL), or a new value set in + // new level pointers from head_ (nullptr), or a new value set in // the loop below. In the former case the reader will - // immediately drop to the next level since NULL sorts after all + // immediately drop to the next level since nullptr sorts after all // keys. In the latter case the reader will use the new node. - max_height_.NoBarrier_Store(reinterpret_cast<void*>(height)); + max_height_.store(height, std::memory_order_relaxed); } x = NewNode(key, height); @@ -369,10 +367,10 @@ void SkipList<Key,Comparator>::Insert(const Key& key) { } } -template<typename Key, class Comparator> -bool SkipList<Key,Comparator>::Contains(const Key& key) const { - Node* x = FindGreaterOrEqual(key, NULL); - if (x != NULL && Equal(key, x->key)) { +template <typename Key, class Comparator> +bool SkipList<Key, Comparator>::Contains(const Key& key) const { + Node* x = FindGreaterOrEqual(key, nullptr); + if (x != nullptr && Equal(key, x->key)) { return true; } else { return false; diff --git a/src/leveldb/db/skiplist_test.cc b/src/leveldb/db/skiplist_test.cc index aee1461e1b..9fa2d96829 100644 --- a/src/leveldb/db/skiplist_test.cc +++ b/src/leveldb/db/skiplist_test.cc @@ -3,8 +3,13 @@ // found in the LICENSE file. See the AUTHORS file for names of contributors. #include "db/skiplist.h" + +#include <atomic> #include <set> + #include "leveldb/env.h" +#include "port/port.h" +#include "port/thread_annotations.h" #include "util/arena.h" #include "util/hash.h" #include "util/random.h" @@ -26,7 +31,7 @@ struct Comparator { } }; -class SkipTest { }; +class SkipTest {}; TEST(SkipTest, Empty) { Arena arena; @@ -112,8 +117,7 @@ TEST(SkipTest, InsertAndLookup) { // Compare against model iterator for (std::set<Key>::reverse_iterator model_iter = keys.rbegin(); - model_iter != keys.rend(); - ++model_iter) { + model_iter != keys.rend(); ++model_iter) { ASSERT_TRUE(iter.Valid()); ASSERT_EQ(*model_iter, iter.key()); iter.Prev(); @@ -126,7 +130,7 @@ TEST(SkipTest, InsertAndLookup) { // concurrent readers (with no synchronization other than when a // reader's iterator is created), the reader always observes all the // data that was present in the skip list when the iterator was -// constructor. Because insertions are happening concurrently, we may +// constructed. Because insertions are happening concurrently, we may // also observe new values that were inserted since the iterator was // constructed, but we should never miss any values that were present // at iterator construction time. @@ -155,12 +159,12 @@ class ConcurrentTest { static uint64_t hash(Key key) { return key & 0xff; } static uint64_t HashNumbers(uint64_t k, uint64_t g) { - uint64_t data[2] = { k, g }; + uint64_t data[2] = {k, g}; return Hash(reinterpret_cast<char*>(data), sizeof(data), 0); } static Key MakeKey(uint64_t k, uint64_t g) { - assert(sizeof(Key) == sizeof(uint64_t)); + static_assert(sizeof(Key) == sizeof(uint64_t), ""); assert(k <= K); // We sometimes pass K to seek to the end of the skiplist assert(g <= 0xffffffffu); return ((k << 40) | (g << 8) | (HashNumbers(k, g) & 0xff)); @@ -186,13 +190,11 @@ class ConcurrentTest { // Per-key generation struct State { - port::AtomicPointer generation[K]; - void Set(int k, intptr_t v) { - generation[k].Release_Store(reinterpret_cast<void*>(v)); - } - intptr_t Get(int k) { - return reinterpret_cast<intptr_t>(generation[k].Acquire_Load()); + std::atomic<int> generation[K]; + void Set(int k, int v) { + generation[k].store(v, std::memory_order_release); } + int Get(int k) { return generation[k].load(std::memory_order_acquire); } State() { for (int k = 0; k < K; k++) { @@ -211,7 +213,7 @@ class ConcurrentTest { SkipList<Key, Comparator> list_; public: - ConcurrentTest() : list_(Comparator(), &arena_) { } + ConcurrentTest() : list_(Comparator(), &arena_) {} // REQUIRES: External synchronization void WriteStep(Random* rnd) { @@ -250,11 +252,9 @@ class ConcurrentTest { // Note that generation 0 is never inserted, so it is ok if // <*,0,*> is missing. ASSERT_TRUE((gen(pos) == 0) || - (gen(pos) > static_cast<Key>(initial_state.Get(key(pos)))) - ) << "key: " << key(pos) - << "; gen: " << gen(pos) - << "; initgen: " - << initial_state.Get(key(pos)); + (gen(pos) > static_cast<Key>(initial_state.Get(key(pos))))) + << "key: " << key(pos) << "; gen: " << gen(pos) + << "; initgen: " << initial_state.Get(key(pos)); // Advance to next key in the valid key space if (key(pos) < key(current)) { @@ -298,21 +298,14 @@ class TestState { public: ConcurrentTest t_; int seed_; - port::AtomicPointer quit_flag_; + std::atomic<bool> quit_flag_; - enum ReaderState { - STARTING, - RUNNING, - DONE - }; + enum ReaderState { STARTING, RUNNING, DONE }; explicit TestState(int s) - : seed_(s), - quit_flag_(NULL), - state_(STARTING), - state_cv_(&mu_) {} + : seed_(s), quit_flag_(false), state_(STARTING), state_cv_(&mu_) {} - void Wait(ReaderState s) { + void Wait(ReaderState s) LOCKS_EXCLUDED(mu_) { mu_.Lock(); while (state_ != s) { state_cv_.Wait(); @@ -320,7 +313,7 @@ class TestState { mu_.Unlock(); } - void Change(ReaderState s) { + void Change(ReaderState s) LOCKS_EXCLUDED(mu_) { mu_.Lock(); state_ = s; state_cv_.Signal(); @@ -329,8 +322,8 @@ class TestState { private: port::Mutex mu_; - ReaderState state_; - port::CondVar state_cv_; + ReaderState state_ GUARDED_BY(mu_); + port::CondVar state_cv_ GUARDED_BY(mu_); }; static void ConcurrentReader(void* arg) { @@ -338,7 +331,7 @@ static void ConcurrentReader(void* arg) { Random rnd(state->seed_); int64_t reads = 0; state->Change(TestState::RUNNING); - while (!state->quit_flag_.Acquire_Load()) { + while (!state->quit_flag_.load(std::memory_order_acquire)) { state->t_.ReadStep(&rnd); ++reads; } @@ -360,7 +353,7 @@ static void RunConcurrent(int run) { for (int i = 0; i < kSize; i++) { state.t_.WriteStep(&rnd); } - state.quit_flag_.Release_Store(&state); // Any non-NULL arg will do + state.quit_flag_.store(true, std::memory_order_release); state.Wait(TestState::DONE); } } @@ -373,6 +366,4 @@ TEST(SkipTest, Concurrent5) { RunConcurrent(5); } } // namespace leveldb -int main(int argc, char** argv) { - return leveldb::test::RunAllTests(); -} +int main(int argc, char** argv) { return leveldb::test::RunAllTests(); } diff --git a/src/leveldb/db/snapshot.h b/src/leveldb/db/snapshot.h index 6ed413c42d..9f1d66491d 100644 --- a/src/leveldb/db/snapshot.h +++ b/src/leveldb/db/snapshot.h @@ -16,50 +16,78 @@ class SnapshotList; // Each SnapshotImpl corresponds to a particular sequence number. class SnapshotImpl : public Snapshot { public: - SequenceNumber number_; // const after creation + SnapshotImpl(SequenceNumber sequence_number) + : sequence_number_(sequence_number) {} + + SequenceNumber sequence_number() const { return sequence_number_; } private: friend class SnapshotList; - // SnapshotImpl is kept in a doubly-linked circular list + // SnapshotImpl is kept in a doubly-linked circular list. The SnapshotList + // implementation operates on the next/previous fields direcly. SnapshotImpl* prev_; SnapshotImpl* next_; - SnapshotList* list_; // just for sanity checks + const SequenceNumber sequence_number_; + +#if !defined(NDEBUG) + SnapshotList* list_ = nullptr; +#endif // !defined(NDEBUG) }; class SnapshotList { public: - SnapshotList() { - list_.prev_ = &list_; - list_.next_ = &list_; + SnapshotList() : head_(0) { + head_.prev_ = &head_; + head_.next_ = &head_; + } + + bool empty() const { return head_.next_ == &head_; } + SnapshotImpl* oldest() const { + assert(!empty()); + return head_.next_; } + SnapshotImpl* newest() const { + assert(!empty()); + return head_.prev_; + } + + // Creates a SnapshotImpl and appends it to the end of the list. + SnapshotImpl* New(SequenceNumber sequence_number) { + assert(empty() || newest()->sequence_number_ <= sequence_number); + + SnapshotImpl* snapshot = new SnapshotImpl(sequence_number); - bool empty() const { return list_.next_ == &list_; } - SnapshotImpl* oldest() const { assert(!empty()); return list_.next_; } - SnapshotImpl* newest() const { assert(!empty()); return list_.prev_; } - - const SnapshotImpl* New(SequenceNumber seq) { - SnapshotImpl* s = new SnapshotImpl; - s->number_ = seq; - s->list_ = this; - s->next_ = &list_; - s->prev_ = list_.prev_; - s->prev_->next_ = s; - s->next_->prev_ = s; - return s; +#if !defined(NDEBUG) + snapshot->list_ = this; +#endif // !defined(NDEBUG) + snapshot->next_ = &head_; + snapshot->prev_ = head_.prev_; + snapshot->prev_->next_ = snapshot; + snapshot->next_->prev_ = snapshot; + return snapshot; } - void Delete(const SnapshotImpl* s) { - assert(s->list_ == this); - s->prev_->next_ = s->next_; - s->next_->prev_ = s->prev_; - delete s; + // Removes a SnapshotImpl from this list. + // + // The snapshot must have been created by calling New() on this list. + // + // The snapshot pointer should not be const, because its memory is + // deallocated. However, that would force us to change DB::ReleaseSnapshot(), + // which is in the API, and currently takes a const Snapshot. + void Delete(const SnapshotImpl* snapshot) { +#if !defined(NDEBUG) + assert(snapshot->list_ == this); +#endif // !defined(NDEBUG) + snapshot->prev_->next_ = snapshot->next_; + snapshot->next_->prev_ = snapshot->prev_; + delete snapshot; } private: // Dummy head of doubly-linked list of snapshots - SnapshotImpl list_; + SnapshotImpl head_; }; } // namespace leveldb diff --git a/src/leveldb/db/table_cache.cc b/src/leveldb/db/table_cache.cc index e3d82cd3ea..73f05fd7b1 100644 --- a/src/leveldb/db/table_cache.cc +++ b/src/leveldb/db/table_cache.cc @@ -29,18 +29,14 @@ static void UnrefEntry(void* arg1, void* arg2) { cache->Release(h); } -TableCache::TableCache(const std::string& dbname, - const Options* options, +TableCache::TableCache(const std::string& dbname, const Options& options, int entries) - : env_(options->env), + : env_(options.env), dbname_(dbname), options_(options), - cache_(NewLRUCache(entries)) { -} + cache_(NewLRUCache(entries)) {} -TableCache::~TableCache() { - delete cache_; -} +TableCache::~TableCache() { delete cache_; } Status TableCache::FindTable(uint64_t file_number, uint64_t file_size, Cache::Handle** handle) { @@ -49,10 +45,10 @@ Status TableCache::FindTable(uint64_t file_number, uint64_t file_size, EncodeFixed64(buf, file_number); Slice key(buf, sizeof(buf)); *handle = cache_->Lookup(key); - if (*handle == NULL) { + if (*handle == nullptr) { std::string fname = TableFileName(dbname_, file_number); - RandomAccessFile* file = NULL; - Table* table = NULL; + RandomAccessFile* file = nullptr; + Table* table = nullptr; s = env_->NewRandomAccessFile(fname, &file); if (!s.ok()) { std::string old_fname = SSTTableFileName(dbname_, file_number); @@ -61,11 +57,11 @@ Status TableCache::FindTable(uint64_t file_number, uint64_t file_size, } } if (s.ok()) { - s = Table::Open(*options_, file, file_size, &table); + s = Table::Open(options_, file, file_size, &table); } if (!s.ok()) { - assert(table == NULL); + assert(table == nullptr); delete file; // We do not cache error results so that if the error is transient, // or somebody repairs the file, we recover automatically. @@ -80,14 +76,13 @@ Status TableCache::FindTable(uint64_t file_number, uint64_t file_size, } Iterator* TableCache::NewIterator(const ReadOptions& options, - uint64_t file_number, - uint64_t file_size, + uint64_t file_number, uint64_t file_size, Table** tableptr) { - if (tableptr != NULL) { - *tableptr = NULL; + if (tableptr != nullptr) { + *tableptr = nullptr; } - Cache::Handle* handle = NULL; + Cache::Handle* handle = nullptr; Status s = FindTable(file_number, file_size, &handle); if (!s.ok()) { return NewErrorIterator(s); @@ -96,23 +91,21 @@ Iterator* TableCache::NewIterator(const ReadOptions& options, Table* table = reinterpret_cast<TableAndFile*>(cache_->Value(handle))->table; Iterator* result = table->NewIterator(options); result->RegisterCleanup(&UnrefEntry, cache_, handle); - if (tableptr != NULL) { + if (tableptr != nullptr) { *tableptr = table; } return result; } -Status TableCache::Get(const ReadOptions& options, - uint64_t file_number, - uint64_t file_size, - const Slice& k, - void* arg, - void (*saver)(void*, const Slice&, const Slice&)) { - Cache::Handle* handle = NULL; +Status TableCache::Get(const ReadOptions& options, uint64_t file_number, + uint64_t file_size, const Slice& k, void* arg, + void (*handle_result)(void*, const Slice&, + const Slice&)) { + Cache::Handle* handle = nullptr; Status s = FindTable(file_number, file_size, &handle); if (s.ok()) { Table* t = reinterpret_cast<TableAndFile*>(cache_->Value(handle))->table; - s = t->InternalGet(options, k, arg, saver); + s = t->InternalGet(options, k, arg, handle_result); cache_->Release(handle); } return s; diff --git a/src/leveldb/db/table_cache.h b/src/leveldb/db/table_cache.h index 8cf4aaf12d..93069c8844 100644 --- a/src/leveldb/db/table_cache.h +++ b/src/leveldb/db/table_cache.h @@ -7,8 +7,10 @@ #ifndef STORAGE_LEVELDB_DB_TABLE_CACHE_H_ #define STORAGE_LEVELDB_DB_TABLE_CACHE_H_ -#include <string> #include <stdint.h> + +#include <string> + #include "db/dbformat.h" #include "leveldb/cache.h" #include "leveldb/table.h" @@ -20,40 +22,35 @@ class Env; class TableCache { public: - TableCache(const std::string& dbname, const Options* options, int entries); + TableCache(const std::string& dbname, const Options& options, int entries); ~TableCache(); // Return an iterator for the specified file number (the corresponding // file length must be exactly "file_size" bytes). If "tableptr" is - // non-NULL, also sets "*tableptr" to point to the Table object - // underlying the returned iterator, or NULL if no Table object underlies - // the returned iterator. The returned "*tableptr" object is owned by - // the cache and should not be deleted, and is valid for as long as the + // non-null, also sets "*tableptr" to point to the Table object + // underlying the returned iterator, or to nullptr if no Table object + // underlies the returned iterator. The returned "*tableptr" object is owned + // by the cache and should not be deleted, and is valid for as long as the // returned iterator is live. - Iterator* NewIterator(const ReadOptions& options, - uint64_t file_number, - uint64_t file_size, - Table** tableptr = NULL); + Iterator* NewIterator(const ReadOptions& options, uint64_t file_number, + uint64_t file_size, Table** tableptr = nullptr); // If a seek to internal key "k" in specified file finds an entry, // call (*handle_result)(arg, found_key, found_value). - Status Get(const ReadOptions& options, - uint64_t file_number, - uint64_t file_size, - const Slice& k, - void* arg, + Status Get(const ReadOptions& options, uint64_t file_number, + uint64_t file_size, const Slice& k, void* arg, void (*handle_result)(void*, const Slice&, const Slice&)); // Evict any entry for the specified file number void Evict(uint64_t file_number); private: + Status FindTable(uint64_t file_number, uint64_t file_size, Cache::Handle**); + Env* const env_; const std::string dbname_; - const Options* options_; + const Options& options_; Cache* cache_; - - Status FindTable(uint64_t file_number, uint64_t file_size, Cache::Handle**); }; } // namespace leveldb diff --git a/src/leveldb/db/version_edit.cc b/src/leveldb/db/version_edit.cc index f10a2d58b2..cd770ef12d 100644 --- a/src/leveldb/db/version_edit.cc +++ b/src/leveldb/db/version_edit.cc @@ -12,15 +12,15 @@ namespace leveldb { // Tag numbers for serialized VersionEdit. These numbers are written to // disk and should not be changed. enum Tag { - kComparator = 1, - kLogNumber = 2, - kNextFileNumber = 3, - kLastSequence = 4, - kCompactPointer = 5, - kDeletedFile = 6, - kNewFile = 7, + kComparator = 1, + kLogNumber = 2, + kNextFileNumber = 3, + kLastSequence = 4, + kCompactPointer = 5, + kDeletedFile = 6, + kNewFile = 7, // 8 was used for large value refs - kPrevLogNumber = 9 + kPrevLogNumber = 9 }; void VersionEdit::Clear() { @@ -66,12 +66,10 @@ void VersionEdit::EncodeTo(std::string* dst) const { PutLengthPrefixedSlice(dst, compact_pointers_[i].second.Encode()); } - for (DeletedFileSet::const_iterator iter = deleted_files_.begin(); - iter != deleted_files_.end(); - ++iter) { + for (const auto& deleted_file_kvp : deleted_files_) { PutVarint32(dst, kDeletedFile); - PutVarint32(dst, iter->first); // level - PutVarint64(dst, iter->second); // file number + PutVarint32(dst, deleted_file_kvp.first); // level + PutVarint64(dst, deleted_file_kvp.second); // file number } for (size_t i = 0; i < new_files_.size(); i++) { @@ -88,8 +86,7 @@ void VersionEdit::EncodeTo(std::string* dst) const { static bool GetInternalKey(Slice* input, InternalKey* dst) { Slice str; if (GetLengthPrefixedSlice(input, &str)) { - dst->DecodeFrom(str); - return true; + return dst->DecodeFrom(str); } else { return false; } @@ -97,8 +94,7 @@ static bool GetInternalKey(Slice* input, InternalKey* dst) { static bool GetLevel(Slice* input, int* level) { uint32_t v; - if (GetVarint32(input, &v) && - v < config::kNumLevels) { + if (GetVarint32(input, &v) && v < config::kNumLevels) { *level = v; return true; } else { @@ -109,7 +105,7 @@ static bool GetLevel(Slice* input, int* level) { Status VersionEdit::DecodeFrom(const Slice& src) { Clear(); Slice input = src; - const char* msg = NULL; + const char* msg = nullptr; uint32_t tag; // Temporary storage for parsing @@ -119,7 +115,7 @@ Status VersionEdit::DecodeFrom(const Slice& src) { Slice str; InternalKey key; - while (msg == NULL && GetVarint32(&input, &tag)) { + while (msg == nullptr && GetVarint32(&input, &tag)) { switch (tag) { case kComparator: if (GetLengthPrefixedSlice(&input, &str)) { @@ -163,8 +159,7 @@ Status VersionEdit::DecodeFrom(const Slice& src) { break; case kCompactPointer: - if (GetLevel(&input, &level) && - GetInternalKey(&input, &key)) { + if (GetLevel(&input, &level) && GetInternalKey(&input, &key)) { compact_pointers_.push_back(std::make_pair(level, key)); } else { msg = "compaction pointer"; @@ -172,8 +167,7 @@ Status VersionEdit::DecodeFrom(const Slice& src) { break; case kDeletedFile: - if (GetLevel(&input, &level) && - GetVarint64(&input, &number)) { + if (GetLevel(&input, &level) && GetVarint64(&input, &number)) { deleted_files_.insert(std::make_pair(level, number)); } else { msg = "deleted file"; @@ -181,8 +175,7 @@ Status VersionEdit::DecodeFrom(const Slice& src) { break; case kNewFile: - if (GetLevel(&input, &level) && - GetVarint64(&input, &f.number) && + if (GetLevel(&input, &level) && GetVarint64(&input, &f.number) && GetVarint64(&input, &f.file_size) && GetInternalKey(&input, &f.smallest) && GetInternalKey(&input, &f.largest)) { @@ -198,12 +191,12 @@ Status VersionEdit::DecodeFrom(const Slice& src) { } } - if (msg == NULL && !input.empty()) { + if (msg == nullptr && !input.empty()) { msg = "invalid tag"; } Status result; - if (msg != NULL) { + if (msg != nullptr) { result = Status::Corruption("VersionEdit", msg); } return result; @@ -238,13 +231,11 @@ std::string VersionEdit::DebugString() const { r.append(" "); r.append(compact_pointers_[i].second.DebugString()); } - for (DeletedFileSet::const_iterator iter = deleted_files_.begin(); - iter != deleted_files_.end(); - ++iter) { + for (const auto& deleted_files_kvp : deleted_files_) { r.append("\n DeleteFile: "); - AppendNumberTo(&r, iter->first); + AppendNumberTo(&r, deleted_files_kvp.first); r.append(" "); - AppendNumberTo(&r, iter->second); + AppendNumberTo(&r, deleted_files_kvp.second); } for (size_t i = 0; i < new_files_.size(); i++) { const FileMetaData& f = new_files_[i].second; diff --git a/src/leveldb/db/version_edit.h b/src/leveldb/db/version_edit.h index eaef77b327..0de4531773 100644 --- a/src/leveldb/db/version_edit.h +++ b/src/leveldb/db/version_edit.h @@ -8,6 +8,7 @@ #include <set> #include <utility> #include <vector> + #include "db/dbformat.h" namespace leveldb { @@ -15,20 +16,20 @@ namespace leveldb { class VersionSet; struct FileMetaData { + FileMetaData() : refs(0), allowed_seeks(1 << 30), file_size(0) {} + int refs; - int allowed_seeks; // Seeks allowed until compaction + int allowed_seeks; // Seeks allowed until compaction uint64_t number; - uint64_t file_size; // File size in bytes - InternalKey smallest; // Smallest internal key served by table - InternalKey largest; // Largest internal key served by table - - FileMetaData() : refs(0), allowed_seeks(1 << 30), file_size(0) { } + uint64_t file_size; // File size in bytes + InternalKey smallest; // Smallest internal key served by table + InternalKey largest; // Largest internal key served by table }; class VersionEdit { public: VersionEdit() { Clear(); } - ~VersionEdit() { } + ~VersionEdit() = default; void Clear(); @@ -59,10 +60,8 @@ class VersionEdit { // Add the specified file at the specified number. // REQUIRES: This version has not been saved (see VersionSet::SaveTo) // REQUIRES: "smallest" and "largest" are smallest and largest keys in file - void AddFile(int level, uint64_t file, - uint64_t file_size, - const InternalKey& smallest, - const InternalKey& largest) { + void AddFile(int level, uint64_t file, uint64_t file_size, + const InternalKey& smallest, const InternalKey& largest) { FileMetaData f; f.number = file; f.file_size = file_size; @@ -84,7 +83,7 @@ class VersionEdit { private: friend class VersionSet; - typedef std::set< std::pair<int, uint64_t> > DeletedFileSet; + typedef std::set<std::pair<int, uint64_t>> DeletedFileSet; std::string comparator_; uint64_t log_number_; @@ -97,9 +96,9 @@ class VersionEdit { bool has_next_file_number_; bool has_last_sequence_; - std::vector< std::pair<int, InternalKey> > compact_pointers_; + std::vector<std::pair<int, InternalKey>> compact_pointers_; DeletedFileSet deleted_files_; - std::vector< std::pair<int, FileMetaData> > new_files_; + std::vector<std::pair<int, FileMetaData>> new_files_; }; } // namespace leveldb diff --git a/src/leveldb/db/version_edit_test.cc b/src/leveldb/db/version_edit_test.cc index 280310b49d..0b7cda8854 100644 --- a/src/leveldb/db/version_edit_test.cc +++ b/src/leveldb/db/version_edit_test.cc @@ -17,7 +17,7 @@ static void TestEncodeDecode(const VersionEdit& edit) { ASSERT_EQ(encoded, encoded2); } -class VersionEditTest { }; +class VersionEditTest {}; TEST(VersionEditTest, EncodeDecode) { static const uint64_t kBig = 1ull << 50; @@ -41,6 +41,4 @@ TEST(VersionEditTest, EncodeDecode) { } // namespace leveldb -int main(int argc, char** argv) { - return leveldb::test::RunAllTests(); -} +int main(int argc, char** argv) { return leveldb::test::RunAllTests(); } diff --git a/src/leveldb/db/version_set.cc b/src/leveldb/db/version_set.cc index 2cb6d80ed3..cd07346ea8 100644 --- a/src/leveldb/db/version_set.cc +++ b/src/leveldb/db/version_set.cc @@ -4,8 +4,10 @@ #include "db/version_set.h" -#include <algorithm> #include <stdio.h> + +#include <algorithm> + #include "db/filename.h" #include "db/log_reader.h" #include "db/log_writer.h" @@ -84,8 +86,7 @@ Version::~Version() { } int FindFile(const InternalKeyComparator& icmp, - const std::vector<FileMetaData*>& files, - const Slice& key) { + const std::vector<FileMetaData*>& files, const Slice& key) { uint32_t left = 0; uint32_t right = files.size(); while (left < right) { @@ -104,26 +105,25 @@ int FindFile(const InternalKeyComparator& icmp, return right; } -static bool AfterFile(const Comparator* ucmp, - const Slice* user_key, const FileMetaData* f) { - // NULL user_key occurs before all keys and is therefore never after *f - return (user_key != NULL && +static bool AfterFile(const Comparator* ucmp, const Slice* user_key, + const FileMetaData* f) { + // null user_key occurs before all keys and is therefore never after *f + return (user_key != nullptr && ucmp->Compare(*user_key, f->largest.user_key()) > 0); } -static bool BeforeFile(const Comparator* ucmp, - const Slice* user_key, const FileMetaData* f) { - // NULL user_key occurs after all keys and is therefore never before *f - return (user_key != NULL && +static bool BeforeFile(const Comparator* ucmp, const Slice* user_key, + const FileMetaData* f) { + // null user_key occurs after all keys and is therefore never before *f + return (user_key != nullptr && ucmp->Compare(*user_key, f->smallest.user_key()) < 0); } -bool SomeFileOverlapsRange( - const InternalKeyComparator& icmp, - bool disjoint_sorted_files, - const std::vector<FileMetaData*>& files, - const Slice* smallest_user_key, - const Slice* largest_user_key) { +bool SomeFileOverlapsRange(const InternalKeyComparator& icmp, + bool disjoint_sorted_files, + const std::vector<FileMetaData*>& files, + const Slice* smallest_user_key, + const Slice* largest_user_key) { const Comparator* ucmp = icmp.user_comparator(); if (!disjoint_sorted_files) { // Need to check against all files @@ -141,10 +141,11 @@ bool SomeFileOverlapsRange( // Binary search over file list uint32_t index = 0; - if (smallest_user_key != NULL) { + if (smallest_user_key != nullptr) { // Find the earliest possible internal key for smallest_user_key - InternalKey small(*smallest_user_key, kMaxSequenceNumber,kValueTypeForSeek); - index = FindFile(icmp, files, small.Encode()); + InternalKey small_key(*smallest_user_key, kMaxSequenceNumber, + kValueTypeForSeek); + index = FindFile(icmp, files, small_key.Encode()); } if (index >= files.size()) { @@ -164,25 +165,21 @@ class Version::LevelFileNumIterator : public Iterator { public: LevelFileNumIterator(const InternalKeyComparator& icmp, const std::vector<FileMetaData*>* flist) - : icmp_(icmp), - flist_(flist), - index_(flist->size()) { // Marks as invalid - } - virtual bool Valid() const { - return index_ < flist_->size(); + : icmp_(icmp), flist_(flist), index_(flist->size()) { // Marks as invalid } - virtual void Seek(const Slice& target) { + bool Valid() const override { return index_ < flist_->size(); } + void Seek(const Slice& target) override { index_ = FindFile(icmp_, *flist_, target); } - virtual void SeekToFirst() { index_ = 0; } - virtual void SeekToLast() { + void SeekToFirst() override { index_ = 0; } + void SeekToLast() override { index_ = flist_->empty() ? 0 : flist_->size() - 1; } - virtual void Next() { + void Next() override { assert(Valid()); index_++; } - virtual void Prev() { + void Prev() override { assert(Valid()); if (index_ == 0) { index_ = flist_->size(); // Marks as invalid @@ -190,17 +187,18 @@ class Version::LevelFileNumIterator : public Iterator { index_--; } } - Slice key() const { + Slice key() const override { assert(Valid()); return (*flist_)[index_]->largest.Encode(); } - Slice value() const { + Slice value() const override { assert(Valid()); EncodeFixed64(value_buf_, (*flist_)[index_]->number); - EncodeFixed64(value_buf_+8, (*flist_)[index_]->file_size); + EncodeFixed64(value_buf_ + 8, (*flist_)[index_]->file_size); return Slice(value_buf_, sizeof(value_buf_)); } - virtual Status status() const { return Status::OK(); } + Status status() const override { return Status::OK(); } + private: const InternalKeyComparator icmp_; const std::vector<FileMetaData*>* const flist_; @@ -210,16 +208,14 @@ class Version::LevelFileNumIterator : public Iterator { mutable char value_buf_[16]; }; -static Iterator* GetFileIterator(void* arg, - const ReadOptions& options, +static Iterator* GetFileIterator(void* arg, const ReadOptions& options, const Slice& file_value) { TableCache* cache = reinterpret_cast<TableCache*>(arg); if (file_value.size() != 16) { return NewErrorIterator( Status::Corruption("FileReader invoked with unexpected value")); } else { - return cache->NewIterator(options, - DecodeFixed64(file_value.data()), + return cache->NewIterator(options, DecodeFixed64(file_value.data()), DecodeFixed64(file_value.data() + 8)); } } @@ -227,17 +223,16 @@ static Iterator* GetFileIterator(void* arg, Iterator* Version::NewConcatenatingIterator(const ReadOptions& options, int level) const { return NewTwoLevelIterator( - new LevelFileNumIterator(vset_->icmp_, &files_[level]), - &GetFileIterator, vset_->table_cache_, options); + new LevelFileNumIterator(vset_->icmp_, &files_[level]), &GetFileIterator, + vset_->table_cache_, options); } void Version::AddIterators(const ReadOptions& options, std::vector<Iterator*>* iters) { // Merge all level zero files together since they may overlap for (size_t i = 0; i < files_[0].size(); i++) { - iters->push_back( - vset_->table_cache_->NewIterator( - options, files_[0][i]->number, files_[0][i]->file_size)); + iters->push_back(vset_->table_cache_->NewIterator( + options, files_[0][i]->number, files_[0][i]->file_size)); } // For levels > 0, we can use a concatenating iterator that sequentially @@ -264,7 +259,7 @@ struct Saver { Slice user_key; std::string* value; }; -} +} // namespace static void SaveValue(void* arg, const Slice& ikey, const Slice& v) { Saver* s = reinterpret_cast<Saver*>(arg); ParsedInternalKey parsed_key; @@ -284,10 +279,8 @@ static bool NewestFirst(FileMetaData* a, FileMetaData* b) { return a->number > b->number; } -void Version::ForEachOverlapping(Slice user_key, Slice internal_key, - void* arg, +void Version::ForEachOverlapping(Slice user_key, Slice internal_key, void* arg, bool (*func)(void*, int, FileMetaData*)) { - // TODO(sanjay): Change Version::Get() to use this function. const Comparator* ucmp = vset_->icmp_.user_comparator(); // Search level-0 in order from newest to oldest. @@ -329,110 +322,89 @@ void Version::ForEachOverlapping(Slice user_key, Slice internal_key, } } -Status Version::Get(const ReadOptions& options, - const LookupKey& k, - std::string* value, - GetStats* stats) { - Slice ikey = k.internal_key(); - Slice user_key = k.user_key(); - const Comparator* ucmp = vset_->icmp_.user_comparator(); - Status s; - - stats->seek_file = NULL; +Status Version::Get(const ReadOptions& options, const LookupKey& k, + std::string* value, GetStats* stats) { + stats->seek_file = nullptr; stats->seek_file_level = -1; - FileMetaData* last_file_read = NULL; - int last_file_read_level = -1; - // We can search level-by-level since entries never hop across - // levels. Therefore we are guaranteed that if we find data - // in an smaller level, later levels are irrelevant. - std::vector<FileMetaData*> tmp; - FileMetaData* tmp2; - for (int level = 0; level < config::kNumLevels; level++) { - size_t num_files = files_[level].size(); - if (num_files == 0) continue; + struct State { + Saver saver; + GetStats* stats; + const ReadOptions* options; + Slice ikey; + FileMetaData* last_file_read; + int last_file_read_level; - // Get the list of files to search in this level - FileMetaData* const* files = &files_[level][0]; - if (level == 0) { - // Level-0 files may overlap each other. Find all files that - // overlap user_key and process them in order from newest to oldest. - tmp.reserve(num_files); - for (uint32_t i = 0; i < num_files; i++) { - FileMetaData* f = files[i]; - if (ucmp->Compare(user_key, f->smallest.user_key()) >= 0 && - ucmp->Compare(user_key, f->largest.user_key()) <= 0) { - tmp.push_back(f); - } - } - if (tmp.empty()) continue; + VersionSet* vset; + Status s; + bool found; - std::sort(tmp.begin(), tmp.end(), NewestFirst); - files = &tmp[0]; - num_files = tmp.size(); - } else { - // Binary search to find earliest index whose largest key >= ikey. - uint32_t index = FindFile(vset_->icmp_, files_[level], ikey); - if (index >= num_files) { - files = NULL; - num_files = 0; - } else { - tmp2 = files[index]; - if (ucmp->Compare(user_key, tmp2->smallest.user_key()) < 0) { - // All of "tmp2" is past any data for user_key - files = NULL; - num_files = 0; - } else { - files = &tmp2; - num_files = 1; - } - } - } + static bool Match(void* arg, int level, FileMetaData* f) { + State* state = reinterpret_cast<State*>(arg); - for (uint32_t i = 0; i < num_files; ++i) { - if (last_file_read != NULL && stats->seek_file == NULL) { + if (state->stats->seek_file == nullptr && + state->last_file_read != nullptr) { // We have had more than one seek for this read. Charge the 1st file. - stats->seek_file = last_file_read; - stats->seek_file_level = last_file_read_level; + state->stats->seek_file = state->last_file_read; + state->stats->seek_file_level = state->last_file_read_level; } - FileMetaData* f = files[i]; - last_file_read = f; - last_file_read_level = level; - - Saver saver; - saver.state = kNotFound; - saver.ucmp = ucmp; - saver.user_key = user_key; - saver.value = value; - s = vset_->table_cache_->Get(options, f->number, f->file_size, - ikey, &saver, SaveValue); - if (!s.ok()) { - return s; + state->last_file_read = f; + state->last_file_read_level = level; + + state->s = state->vset->table_cache_->Get(*state->options, f->number, + f->file_size, state->ikey, + &state->saver, SaveValue); + if (!state->s.ok()) { + state->found = true; + return false; } - switch (saver.state) { + switch (state->saver.state) { case kNotFound: - break; // Keep searching in other files + return true; // Keep searching in other files case kFound: - return s; + state->found = true; + return false; case kDeleted: - s = Status::NotFound(Slice()); // Use empty error message for speed - return s; + return false; case kCorrupt: - s = Status::Corruption("corrupted key for ", user_key); - return s; + state->s = + Status::Corruption("corrupted key for ", state->saver.user_key); + state->found = true; + return false; } + + // Not reached. Added to avoid false compilation warnings of + // "control reaches end of non-void function". + return false; } - } + }; + + State state; + state.found = false; + state.stats = stats; + state.last_file_read = nullptr; + state.last_file_read_level = -1; - return Status::NotFound(Slice()); // Use an empty error message for speed + state.options = &options; + state.ikey = k.internal_key(); + state.vset = vset_; + + state.saver.state = kNotFound; + state.saver.ucmp = vset_->icmp_.user_comparator(); + state.saver.user_key = k.user_key(); + state.saver.value = value; + + ForEachOverlapping(state.saver.user_key, state.ikey, &state, &State::Match); + + return state.found ? state.s : Status::NotFound(Slice()); } bool Version::UpdateStats(const GetStats& stats) { FileMetaData* f = stats.seek_file; - if (f != NULL) { + if (f != nullptr) { f->allowed_seeks--; - if (f->allowed_seeks <= 0 && file_to_compact_ == NULL) { + if (f->allowed_seeks <= 0 && file_to_compact_ == nullptr) { file_to_compact_ = f; file_to_compact_level_ = stats.seek_file_level; return true; @@ -479,9 +451,7 @@ bool Version::RecordReadSample(Slice internal_key) { return false; } -void Version::Ref() { - ++refs_; -} +void Version::Ref() { ++refs_; } void Version::Unref() { assert(this != &vset_->dummy_versions_); @@ -492,16 +462,14 @@ void Version::Unref() { } } -bool Version::OverlapInLevel(int level, - const Slice* smallest_user_key, +bool Version::OverlapInLevel(int level, const Slice* smallest_user_key, const Slice* largest_user_key) { return SomeFileOverlapsRange(vset_->icmp_, (level > 0), files_[level], smallest_user_key, largest_user_key); } -int Version::PickLevelForMemTableOutput( - const Slice& smallest_user_key, - const Slice& largest_user_key) { +int Version::PickLevelForMemTableOutput(const Slice& smallest_user_key, + const Slice& largest_user_key) { int level = 0; if (!OverlapInLevel(0, &smallest_user_key, &largest_user_key)) { // Push to next level if there is no overlap in next level, @@ -528,40 +496,39 @@ int Version::PickLevelForMemTableOutput( } // Store in "*inputs" all files in "level" that overlap [begin,end] -void Version::GetOverlappingInputs( - int level, - const InternalKey* begin, - const InternalKey* end, - std::vector<FileMetaData*>* inputs) { +void Version::GetOverlappingInputs(int level, const InternalKey* begin, + const InternalKey* end, + std::vector<FileMetaData*>* inputs) { assert(level >= 0); assert(level < config::kNumLevels); inputs->clear(); Slice user_begin, user_end; - if (begin != NULL) { + if (begin != nullptr) { user_begin = begin->user_key(); } - if (end != NULL) { + if (end != nullptr) { user_end = end->user_key(); } const Comparator* user_cmp = vset_->icmp_.user_comparator(); - for (size_t i = 0; i < files_[level].size(); ) { + for (size_t i = 0; i < files_[level].size();) { FileMetaData* f = files_[level][i++]; const Slice file_start = f->smallest.user_key(); const Slice file_limit = f->largest.user_key(); - if (begin != NULL && user_cmp->Compare(file_limit, user_begin) < 0) { + if (begin != nullptr && user_cmp->Compare(file_limit, user_begin) < 0) { // "f" is completely before specified range; skip it - } else if (end != NULL && user_cmp->Compare(file_start, user_end) > 0) { + } else if (end != nullptr && user_cmp->Compare(file_start, user_end) > 0) { // "f" is completely after specified range; skip it } else { inputs->push_back(f); if (level == 0) { // Level-0 files may overlap each other. So check if the newly // added file has expanded the range. If so, restart search. - if (begin != NULL && user_cmp->Compare(file_start, user_begin) < 0) { + if (begin != nullptr && user_cmp->Compare(file_start, user_begin) < 0) { user_begin = file_start; inputs->clear(); i = 0; - } else if (end != NULL && user_cmp->Compare(file_limit, user_end) > 0) { + } else if (end != nullptr && + user_cmp->Compare(file_limit, user_end) > 0) { user_end = file_limit; inputs->clear(); i = 0; @@ -629,9 +596,7 @@ class VersionSet::Builder { public: // Initialize a builder with the files from *base and other info from *vset - Builder(VersionSet* vset, Version* base) - : vset_(vset), - base_(base) { + Builder(VersionSet* vset, Version* base) : vset_(vset), base_(base) { base_->Ref(); BySmallestKey cmp; cmp.internal_comparator = &vset_->icmp_; @@ -645,8 +610,8 @@ class VersionSet::Builder { const FileSet* added = levels_[level].added_files; std::vector<FileMetaData*> to_unref; to_unref.reserve(added->size()); - for (FileSet::const_iterator it = added->begin(); - it != added->end(); ++it) { + for (FileSet::const_iterator it = added->begin(); it != added->end(); + ++it) { to_unref.push_back(*it); } delete added; @@ -671,12 +636,9 @@ class VersionSet::Builder { } // Delete files - const VersionEdit::DeletedFileSet& del = edit->deleted_files_; - for (VersionEdit::DeletedFileSet::const_iterator iter = del.begin(); - iter != del.end(); - ++iter) { - const int level = iter->first; - const uint64_t number = iter->second; + for (const auto& deleted_file_set_kvp : edit->deleted_files_) { + const int level = deleted_file_set_kvp.first; + const uint64_t number = deleted_file_set_kvp.second; levels_[level].deleted_files.insert(number); } @@ -699,7 +661,7 @@ class VersionSet::Builder { // same as the compaction of 40KB of data. We are a little // conservative and allow approximately one seek for every 16KB // of data before triggering a compaction. - f->allowed_seeks = (f->file_size / 16384); + f->allowed_seeks = static_cast<int>((f->file_size / 16384U)); if (f->allowed_seeks < 100) f->allowed_seeks = 100; levels_[level].deleted_files.erase(f->number); @@ -717,20 +679,17 @@ class VersionSet::Builder { const std::vector<FileMetaData*>& base_files = base_->files_[level]; std::vector<FileMetaData*>::const_iterator base_iter = base_files.begin(); std::vector<FileMetaData*>::const_iterator base_end = base_files.end(); - const FileSet* added = levels_[level].added_files; - v->files_[level].reserve(base_files.size() + added->size()); - for (FileSet::const_iterator added_iter = added->begin(); - added_iter != added->end(); - ++added_iter) { + const FileSet* added_files = levels_[level].added_files; + v->files_[level].reserve(base_files.size() + added_files->size()); + for (const auto& added_file : *added_files) { // Add all smaller files listed in base_ - for (std::vector<FileMetaData*>::const_iterator bpos - = std::upper_bound(base_iter, base_end, *added_iter, cmp); - base_iter != bpos; - ++base_iter) { + for (std::vector<FileMetaData*>::const_iterator bpos = + std::upper_bound(base_iter, base_end, added_file, cmp); + base_iter != bpos; ++base_iter) { MaybeAddFile(v, level, *base_iter); } - MaybeAddFile(v, level, *added_iter); + MaybeAddFile(v, level, added_file); } // Add remaining base files @@ -742,7 +701,7 @@ class VersionSet::Builder { // Make sure there is no overlap in levels > 0 if (level > 0) { for (uint32_t i = 1; i < v->files_[level].size(); i++) { - const InternalKey& prev_end = v->files_[level][i-1]->largest; + const InternalKey& prev_end = v->files_[level][i - 1]->largest; const InternalKey& this_begin = v->files_[level][i]->smallest; if (vset_->icmp_.Compare(prev_end, this_begin) >= 0) { fprintf(stderr, "overlapping ranges in same level %s vs. %s\n", @@ -763,7 +722,7 @@ class VersionSet::Builder { std::vector<FileMetaData*>* files = &v->files_[level]; if (level > 0 && !files->empty()) { // Must not overlap - assert(vset_->icmp_.Compare((*files)[files->size()-1]->largest, + assert(vset_->icmp_.Compare((*files)[files->size() - 1]->largest, f->smallest) < 0); } f->refs++; @@ -772,8 +731,7 @@ class VersionSet::Builder { } }; -VersionSet::VersionSet(const std::string& dbname, - const Options* options, +VersionSet::VersionSet(const std::string& dbname, const Options* options, TableCache* table_cache, const InternalKeyComparator* cmp) : env_(options->env), @@ -786,10 +744,10 @@ VersionSet::VersionSet(const std::string& dbname, last_sequence_(0), log_number_(0), prev_log_number_(0), - descriptor_file_(NULL), - descriptor_log_(NULL), + descriptor_file_(nullptr), + descriptor_log_(nullptr), dummy_versions_(this), - current_(NULL) { + current_(nullptr) { AppendVersion(new Version(this)); } @@ -804,7 +762,7 @@ void VersionSet::AppendVersion(Version* v) { // Make "v" current assert(v->refs_ == 0); assert(v != current_); - if (current_ != NULL) { + if (current_ != nullptr) { current_->Unref(); } current_ = v; @@ -844,10 +802,10 @@ Status VersionSet::LogAndApply(VersionEdit* edit, port::Mutex* mu) { // a temporary file that contains a snapshot of the current version. std::string new_manifest_file; Status s; - if (descriptor_log_ == NULL) { + if (descriptor_log_ == nullptr) { // No reason to unlock *mu here since we only hit this path in the // first call to LogAndApply (when opening the database). - assert(descriptor_file_ == NULL); + assert(descriptor_file_ == nullptr); new_manifest_file = DescriptorFileName(dbname_, manifest_file_number_); edit->SetNextFile(next_file_number_); s = env_->NewWritableFile(new_manifest_file, &descriptor_file_); @@ -893,8 +851,8 @@ Status VersionSet::LogAndApply(VersionEdit* edit, port::Mutex* mu) { if (!new_manifest_file.empty()) { delete descriptor_log_; delete descriptor_file_; - descriptor_log_ = NULL; - descriptor_file_ = NULL; + descriptor_log_ = nullptr; + descriptor_file_ = nullptr; env_->DeleteFile(new_manifest_file); } } @@ -902,10 +860,10 @@ Status VersionSet::LogAndApply(VersionEdit* edit, port::Mutex* mu) { return s; } -Status VersionSet::Recover(bool *save_manifest) { +Status VersionSet::Recover(bool* save_manifest) { struct LogReporter : public log::Reader::Reporter { Status* status; - virtual void Corruption(size_t bytes, const Status& s) { + void Corruption(size_t bytes, const Status& s) override { if (this->status->ok()) *this->status = s; } }; @@ -916,7 +874,7 @@ Status VersionSet::Recover(bool *save_manifest) { if (!s.ok()) { return s; } - if (current.empty() || current[current.size()-1] != '\n') { + if (current.empty() || current[current.size() - 1] != '\n') { return Status::Corruption("CURRENT file does not end with newline"); } current.resize(current.size() - 1); @@ -925,6 +883,10 @@ Status VersionSet::Recover(bool *save_manifest) { SequentialFile* file; s = env_->NewSequentialFile(dscname, &file); if (!s.ok()) { + if (s.IsNotFound()) { + return Status::Corruption("CURRENT points to a non-existent file", + s.ToString()); + } return s; } @@ -941,7 +903,8 @@ Status VersionSet::Recover(bool *save_manifest) { { LogReporter reporter; reporter.status = &s; - log::Reader reader(file, &reporter, true/*checksum*/, 0/*initial_offset*/); + log::Reader reader(file, &reporter, true /*checksum*/, + 0 /*initial_offset*/); Slice record; std::string scratch; while (reader.ReadRecord(&record, &scratch) && s.ok()) { @@ -982,7 +945,7 @@ Status VersionSet::Recover(bool *save_manifest) { } } delete file; - file = NULL; + file = nullptr; if (s.ok()) { if (!have_next_file) { @@ -1040,12 +1003,12 @@ bool VersionSet::ReuseManifest(const std::string& dscname, return false; } - assert(descriptor_file_ == NULL); - assert(descriptor_log_ == NULL); + assert(descriptor_file_ == nullptr); + assert(descriptor_log_ == nullptr); Status r = env_->NewAppendableFile(dscname, &descriptor_file_); if (!r.ok()) { Log(options_->info_log, "Reuse MANIFEST: %s\n", r.ToString().c_str()); - assert(descriptor_file_ == NULL); + assert(descriptor_file_ == nullptr); return false; } @@ -1066,7 +1029,7 @@ void VersionSet::Finalize(Version* v) { int best_level = -1; double best_score = -1; - for (int level = 0; level < config::kNumLevels-1; level++) { + for (int level = 0; level < config::kNumLevels - 1; level++) { double score; if (level == 0) { // We treat level-0 specially by bounding the number of files @@ -1081,7 +1044,7 @@ void VersionSet::Finalize(Version* v) { // setting, or very high compression ratios, or lots of // overwrites/deletions). score = v->files_[level].size() / - static_cast<double>(config::kL0_CompactionTrigger); + static_cast<double>(config::kL0_CompactionTrigger); } else { // Compute the ratio of current size to size limit. const uint64_t level_bytes = TotalFileSize(v->files_[level]); @@ -1137,16 +1100,12 @@ int VersionSet::NumLevelFiles(int level) const { const char* VersionSet::LevelSummary(LevelSummaryStorage* scratch) const { // Update code if kNumLevels changes - assert(config::kNumLevels == 7); + static_assert(config::kNumLevels == 7, ""); snprintf(scratch->buffer, sizeof(scratch->buffer), - "files[ %d %d %d %d %d %d %d ]", - int(current_->files_[0].size()), - int(current_->files_[1].size()), - int(current_->files_[2].size()), - int(current_->files_[3].size()), - int(current_->files_[4].size()), - int(current_->files_[5].size()), - int(current_->files_[6].size())); + "files[ %d %d %d %d %d %d %d ]", int(current_->files_[0].size()), + int(current_->files_[1].size()), int(current_->files_[2].size()), + int(current_->files_[3].size()), int(current_->files_[4].size()), + int(current_->files_[5].size()), int(current_->files_[6].size())); return scratch->buffer; } @@ -1172,7 +1131,7 @@ uint64_t VersionSet::ApproximateOffsetOf(Version* v, const InternalKey& ikey) { Table* tableptr; Iterator* iter = table_cache_->NewIterator( ReadOptions(), files[i]->number, files[i]->file_size, &tableptr); - if (tableptr != NULL) { + if (tableptr != nullptr) { result += tableptr->ApproximateOffsetOf(ikey.Encode()); } delete iter; @@ -1183,8 +1142,7 @@ uint64_t VersionSet::ApproximateOffsetOf(Version* v, const InternalKey& ikey) { } void VersionSet::AddLiveFiles(std::set<uint64_t>* live) { - for (Version* v = dummy_versions_.next_; - v != &dummy_versions_; + for (Version* v = dummy_versions_.next_; v != &dummy_versions_; v = v->next_) { for (int level = 0; level < config::kNumLevels; level++) { const std::vector<FileMetaData*>& files = v->files_[level]; @@ -1207,7 +1165,7 @@ int64_t VersionSet::MaxNextLevelOverlappingBytes() { for (int level = 1; level < config::kNumLevels - 1; level++) { for (size_t i = 0; i < current_->files_[level].size(); i++) { const FileMetaData* f = current_->files_[level][i]; - current_->GetOverlappingInputs(level+1, &f->smallest, &f->largest, + current_->GetOverlappingInputs(level + 1, &f->smallest, &f->largest, &overlaps); const int64_t sum = TotalFileSize(overlaps); if (sum > result) { @@ -1222,8 +1180,7 @@ int64_t VersionSet::MaxNextLevelOverlappingBytes() { // *smallest, *largest. // REQUIRES: inputs is not empty void VersionSet::GetRange(const std::vector<FileMetaData*>& inputs, - InternalKey* smallest, - InternalKey* largest) { + InternalKey* smallest, InternalKey* largest) { assert(!inputs.empty()); smallest->Clear(); largest->Clear(); @@ -1248,8 +1205,7 @@ void VersionSet::GetRange(const std::vector<FileMetaData*>& inputs, // REQUIRES: inputs is not empty void VersionSet::GetRange2(const std::vector<FileMetaData*>& inputs1, const std::vector<FileMetaData*>& inputs2, - InternalKey* smallest, - InternalKey* largest) { + InternalKey* smallest, InternalKey* largest) { std::vector<FileMetaData*> all = inputs1; all.insert(all.end(), inputs2.begin(), inputs2.end()); GetRange(all, smallest, largest); @@ -1271,8 +1227,8 @@ Iterator* VersionSet::MakeInputIterator(Compaction* c) { if (c->level() + which == 0) { const std::vector<FileMetaData*>& files = c->inputs_[which]; for (size_t i = 0; i < files.size(); i++) { - list[num++] = table_cache_->NewIterator( - options, files[i]->number, files[i]->file_size); + list[num++] = table_cache_->NewIterator(options, files[i]->number, + files[i]->file_size); } } else { // Create concatenating iterator for the files from this level @@ -1295,11 +1251,11 @@ Compaction* VersionSet::PickCompaction() { // We prefer compactions triggered by too much data in a level over // the compactions triggered by seeks. const bool size_compaction = (current_->compaction_score_ >= 1); - const bool seek_compaction = (current_->file_to_compact_ != NULL); + const bool seek_compaction = (current_->file_to_compact_ != nullptr); if (size_compaction) { level = current_->compaction_level_; assert(level >= 0); - assert(level+1 < config::kNumLevels); + assert(level + 1 < config::kNumLevels); c = new Compaction(options_, level); // Pick the first file that comes after compact_pointer_[level] @@ -1320,7 +1276,7 @@ Compaction* VersionSet::PickCompaction() { c = new Compaction(options_, level); c->inputs_[0].push_back(current_->file_to_compact_); } else { - return NULL; + return nullptr; } c->input_version_ = current_; @@ -1342,12 +1298,94 @@ Compaction* VersionSet::PickCompaction() { return c; } +// Finds the largest key in a vector of files. Returns true if files it not +// empty. +bool FindLargestKey(const InternalKeyComparator& icmp, + const std::vector<FileMetaData*>& files, + InternalKey* largest_key) { + if (files.empty()) { + return false; + } + *largest_key = files[0]->largest; + for (size_t i = 1; i < files.size(); ++i) { + FileMetaData* f = files[i]; + if (icmp.Compare(f->largest, *largest_key) > 0) { + *largest_key = f->largest; + } + } + return true; +} + +// Finds minimum file b2=(l2, u2) in level file for which l2 > u1 and +// user_key(l2) = user_key(u1) +FileMetaData* FindSmallestBoundaryFile( + const InternalKeyComparator& icmp, + const std::vector<FileMetaData*>& level_files, + const InternalKey& largest_key) { + const Comparator* user_cmp = icmp.user_comparator(); + FileMetaData* smallest_boundary_file = nullptr; + for (size_t i = 0; i < level_files.size(); ++i) { + FileMetaData* f = level_files[i]; + if (icmp.Compare(f->smallest, largest_key) > 0 && + user_cmp->Compare(f->smallest.user_key(), largest_key.user_key()) == + 0) { + if (smallest_boundary_file == nullptr || + icmp.Compare(f->smallest, smallest_boundary_file->smallest) < 0) { + smallest_boundary_file = f; + } + } + } + return smallest_boundary_file; +} + +// Extracts the largest file b1 from |compaction_files| and then searches for a +// b2 in |level_files| for which user_key(u1) = user_key(l2). If it finds such a +// file b2 (known as a boundary file) it adds it to |compaction_files| and then +// searches again using this new upper bound. +// +// If there are two blocks, b1=(l1, u1) and b2=(l2, u2) and +// user_key(u1) = user_key(l2), and if we compact b1 but not b2 then a +// subsequent get operation will yield an incorrect result because it will +// return the record from b2 in level i rather than from b1 because it searches +// level by level for records matching the supplied user key. +// +// parameters: +// in level_files: List of files to search for boundary files. +// in/out compaction_files: List of files to extend by adding boundary files. +void AddBoundaryInputs(const InternalKeyComparator& icmp, + const std::vector<FileMetaData*>& level_files, + std::vector<FileMetaData*>* compaction_files) { + InternalKey largest_key; + + // Quick return if compaction_files is empty. + if (!FindLargestKey(icmp, *compaction_files, &largest_key)) { + return; + } + + bool continue_searching = true; + while (continue_searching) { + FileMetaData* smallest_boundary_file = + FindSmallestBoundaryFile(icmp, level_files, largest_key); + + // If a boundary file was found advance largest_key, otherwise we're done. + if (smallest_boundary_file != NULL) { + compaction_files->push_back(smallest_boundary_file); + largest_key = smallest_boundary_file->largest; + } else { + continue_searching = false; + } + } +} + void VersionSet::SetupOtherInputs(Compaction* c) { const int level = c->level(); InternalKey smallest, largest; + + AddBoundaryInputs(icmp_, current_->files_[level], &c->inputs_[0]); GetRange(c->inputs_[0], &smallest, &largest); - current_->GetOverlappingInputs(level+1, &smallest, &largest, &c->inputs_[1]); + current_->GetOverlappingInputs(level + 1, &smallest, &largest, + &c->inputs_[1]); // Get entire range covered by compaction InternalKey all_start, all_limit; @@ -1358,6 +1396,7 @@ void VersionSet::SetupOtherInputs(Compaction* c) { if (!c->inputs_[1].empty()) { std::vector<FileMetaData*> expanded0; current_->GetOverlappingInputs(level, &all_start, &all_limit, &expanded0); + AddBoundaryInputs(icmp_, current_->files_[level], &expanded0); const int64_t inputs0_size = TotalFileSize(c->inputs_[0]); const int64_t inputs1_size = TotalFileSize(c->inputs_[1]); const int64_t expanded0_size = TotalFileSize(expanded0); @@ -1367,18 +1406,14 @@ void VersionSet::SetupOtherInputs(Compaction* c) { InternalKey new_start, new_limit; GetRange(expanded0, &new_start, &new_limit); std::vector<FileMetaData*> expanded1; - current_->GetOverlappingInputs(level+1, &new_start, &new_limit, + current_->GetOverlappingInputs(level + 1, &new_start, &new_limit, &expanded1); if (expanded1.size() == c->inputs_[1].size()) { Log(options_->info_log, "Expanding@%d %d+%d (%ld+%ld bytes) to %d+%d (%ld+%ld bytes)\n", - level, - int(c->inputs_[0].size()), - int(c->inputs_[1].size()), - long(inputs0_size), long(inputs1_size), - int(expanded0.size()), - int(expanded1.size()), - long(expanded0_size), long(inputs1_size)); + level, int(c->inputs_[0].size()), int(c->inputs_[1].size()), + long(inputs0_size), long(inputs1_size), int(expanded0.size()), + int(expanded1.size()), long(expanded0_size), long(inputs1_size)); smallest = new_start; largest = new_limit; c->inputs_[0] = expanded0; @@ -1395,13 +1430,6 @@ void VersionSet::SetupOtherInputs(Compaction* c) { &c->grandparents_); } - if (false) { - Log(options_->info_log, "Compacting %d '%s' .. '%s'", - level, - smallest.DebugString().c_str(), - largest.DebugString().c_str()); - } - // Update the place where we will do the next compaction for this level. // We update this immediately instead of waiting for the VersionEdit // to be applied so that if the compaction fails, we will try a different @@ -1410,14 +1438,12 @@ void VersionSet::SetupOtherInputs(Compaction* c) { c->edit_.SetCompactPointer(level, largest); } -Compaction* VersionSet::CompactRange( - int level, - const InternalKey* begin, - const InternalKey* end) { +Compaction* VersionSet::CompactRange(int level, const InternalKey* begin, + const InternalKey* end) { std::vector<FileMetaData*> inputs; current_->GetOverlappingInputs(level, begin, end, &inputs); if (inputs.empty()) { - return NULL; + return nullptr; } // Avoid compacting too much in one shot in case the range is large. @@ -1448,7 +1474,7 @@ Compaction* VersionSet::CompactRange( Compaction::Compaction(const Options* options, int level) : level_(level), max_output_file_size_(MaxFileSizeForLevel(options, level)), - input_version_(NULL), + input_version_(nullptr), grandparent_index_(0), seen_key_(false), overlapped_bytes_(0) { @@ -1458,7 +1484,7 @@ Compaction::Compaction(const Options* options, int level) } Compaction::~Compaction() { - if (input_version_ != NULL) { + if (input_version_ != nullptr) { input_version_->Unref(); } } @@ -1486,7 +1512,7 @@ bool Compaction::IsBaseLevelForKey(const Slice& user_key) { const Comparator* user_cmp = input_version_->vset_->icmp_.user_comparator(); for (int lvl = level_ + 2; lvl < config::kNumLevels; lvl++) { const std::vector<FileMetaData*>& files = input_version_->files_[lvl]; - for (; level_ptrs_[lvl] < files.size(); ) { + while (level_ptrs_[lvl] < files.size()) { FileMetaData* f = files[level_ptrs_[lvl]]; if (user_cmp->Compare(user_key, f->largest.user_key()) <= 0) { // We've advanced far enough @@ -1507,8 +1533,9 @@ bool Compaction::ShouldStopBefore(const Slice& internal_key) { // Scan to find earliest grandparent file that contains key. const InternalKeyComparator* icmp = &vset->icmp_; while (grandparent_index_ < grandparents_.size() && - icmp->Compare(internal_key, - grandparents_[grandparent_index_]->largest.Encode()) > 0) { + icmp->Compare(internal_key, + grandparents_[grandparent_index_]->largest.Encode()) > + 0) { if (seen_key_) { overlapped_bytes_ += grandparents_[grandparent_index_]->file_size; } @@ -1526,9 +1553,9 @@ bool Compaction::ShouldStopBefore(const Slice& internal_key) { } void Compaction::ReleaseInputs() { - if (input_version_ != NULL) { + if (input_version_ != nullptr) { input_version_->Unref(); - input_version_ = NULL; + input_version_ = nullptr; } } diff --git a/src/leveldb/db/version_set.h b/src/leveldb/db/version_set.h index 7935a965a7..69f3d70133 100644 --- a/src/leveldb/db/version_set.h +++ b/src/leveldb/db/version_set.h @@ -18,6 +18,7 @@ #include <map> #include <set> #include <vector> + #include "db/dbformat.h" #include "db/version_edit.h" #include "port/port.h" @@ -25,7 +26,9 @@ namespace leveldb { -namespace log { class Writer; } +namespace log { +class Writer; +} class Compaction; class Iterator; @@ -39,30 +42,23 @@ class WritableFile; // Return the smallest index i such that files[i]->largest >= key. // Return files.size() if there is no such file. // REQUIRES: "files" contains a sorted list of non-overlapping files. -extern int FindFile(const InternalKeyComparator& icmp, - const std::vector<FileMetaData*>& files, - const Slice& key); +int FindFile(const InternalKeyComparator& icmp, + const std::vector<FileMetaData*>& files, const Slice& key); // Returns true iff some file in "files" overlaps the user key range // [*smallest,*largest]. -// smallest==NULL represents a key smaller than all keys in the DB. -// largest==NULL represents a key largest than all keys in the DB. +// smallest==nullptr represents a key smaller than all keys in the DB. +// largest==nullptr represents a key largest than all keys in the DB. // REQUIRES: If disjoint_sorted_files, files[] contains disjoint ranges // in sorted order. -extern bool SomeFileOverlapsRange( - const InternalKeyComparator& icmp, - bool disjoint_sorted_files, - const std::vector<FileMetaData*>& files, - const Slice* smallest_user_key, - const Slice* largest_user_key); +bool SomeFileOverlapsRange(const InternalKeyComparator& icmp, + bool disjoint_sorted_files, + const std::vector<FileMetaData*>& files, + const Slice* smallest_user_key, + const Slice* largest_user_key); class Version { public: - // Append to *iters a sequence of iterators that will - // yield the contents of this Version when merged together. - // REQUIRES: This version has been saved (see VersionSet::SaveTo) - void AddIterators(const ReadOptions&, std::vector<Iterator*>* iters); - // Lookup the value for key. If found, store it in *val and // return OK. Else return a non-OK status. Fills *stats. // REQUIRES: lock is not held @@ -70,6 +66,12 @@ class Version { FileMetaData* seek_file; int seek_file_level; }; + + // Append to *iters a sequence of iterators that will + // yield the contents of this Version when merged together. + // REQUIRES: This version has been saved (see VersionSet::SaveTo) + void AddIterators(const ReadOptions&, std::vector<Iterator*>* iters); + Status Get(const ReadOptions&, const LookupKey& key, std::string* val, GetStats* stats); @@ -91,16 +93,15 @@ class Version { void GetOverlappingInputs( int level, - const InternalKey* begin, // NULL means before all keys - const InternalKey* end, // NULL means after all keys + const InternalKey* begin, // nullptr means before all keys + const InternalKey* end, // nullptr means after all keys std::vector<FileMetaData*>* inputs); // Returns true iff some file in the specified level overlaps // some part of [*smallest_user_key,*largest_user_key]. - // smallest_user_key==NULL represents a key smaller than all keys in the DB. - // largest_user_key==NULL represents a key largest than all keys in the DB. - bool OverlapInLevel(int level, - const Slice* smallest_user_key, + // smallest_user_key==nullptr represents a key smaller than all the DB's keys. + // largest_user_key==nullptr represents a key largest than all the DB's keys. + bool OverlapInLevel(int level, const Slice* smallest_user_key, const Slice* largest_user_key); // Return the level at which we should place a new memtable compaction @@ -118,6 +119,22 @@ class Version { friend class VersionSet; class LevelFileNumIterator; + + explicit Version(VersionSet* vset) + : vset_(vset), + next_(this), + prev_(this), + refs_(0), + file_to_compact_(nullptr), + file_to_compact_level_(-1), + compaction_score_(-1), + compaction_level_(-1) {} + + Version(const Version&) = delete; + Version& operator=(const Version&) = delete; + + ~Version(); + Iterator* NewConcatenatingIterator(const ReadOptions&, int level) const; // Call func(arg, level, f) for every file that overlaps user_key in @@ -125,14 +142,13 @@ class Version { // false, makes no more calls. // // REQUIRES: user portion of internal_key == user_key. - void ForEachOverlapping(Slice user_key, Slice internal_key, - void* arg, + void ForEachOverlapping(Slice user_key, Slice internal_key, void* arg, bool (*func)(void*, int, FileMetaData*)); - VersionSet* vset_; // VersionSet to which this Version belongs - Version* next_; // Next version in linked list - Version* prev_; // Previous version in linked list - int refs_; // Number of live refs to this version + VersionSet* vset_; // VersionSet to which this Version belongs + Version* next_; // Next version in linked list + Version* prev_; // Previous version in linked list + int refs_; // Number of live refs to this version // List of files per level std::vector<FileMetaData*> files_[config::kNumLevels]; @@ -146,28 +162,15 @@ class Version { // are initialized by Finalize(). double compaction_score_; int compaction_level_; - - explicit Version(VersionSet* vset) - : vset_(vset), next_(this), prev_(this), refs_(0), - file_to_compact_(NULL), - file_to_compact_level_(-1), - compaction_score_(-1), - compaction_level_(-1) { - } - - ~Version(); - - // No copying allowed - Version(const Version&); - void operator=(const Version&); }; class VersionSet { public: - VersionSet(const std::string& dbname, - const Options* options, - TableCache* table_cache, - const InternalKeyComparator*); + VersionSet(const std::string& dbname, const Options* options, + TableCache* table_cache, const InternalKeyComparator*); + VersionSet(const VersionSet&) = delete; + VersionSet& operator=(const VersionSet&) = delete; + ~VersionSet(); // Apply *edit to the current version to form a new descriptor that @@ -179,7 +182,7 @@ class VersionSet { EXCLUSIVE_LOCKS_REQUIRED(mu); // Recover the last saved descriptor from persistent storage. - Status Recover(bool *save_manifest); + Status Recover(bool* save_manifest); // Return the current version. Version* current() const { return current_; } @@ -225,19 +228,17 @@ class VersionSet { uint64_t PrevLogNumber() const { return prev_log_number_; } // Pick level and inputs for a new compaction. - // Returns NULL if there is no compaction to be done. + // Returns nullptr if there is no compaction to be done. // Otherwise returns a pointer to a heap-allocated object that // describes the compaction. Caller should delete the result. Compaction* PickCompaction(); // Return a compaction object for compacting the range [begin,end] in - // the specified level. Returns NULL if there is nothing in that + // the specified level. Returns nullptr if there is nothing in that // level that overlaps the specified range. Caller should delete // the result. - Compaction* CompactRange( - int level, - const InternalKey* begin, - const InternalKey* end); + Compaction* CompactRange(int level, const InternalKey* begin, + const InternalKey* end); // Return the maximum overlapping data (in bytes) at next level for any // file at a level >= 1. @@ -250,7 +251,7 @@ class VersionSet { // Returns true iff some level needs a compaction. bool NeedsCompaction() const { Version* v = current_; - return (v->compaction_score_ >= 1) || (v->file_to_compact_ != NULL); + return (v->compaction_score_ >= 1) || (v->file_to_compact_ != nullptr); } // Add all files listed in any live version to *live. @@ -278,14 +279,12 @@ class VersionSet { void Finalize(Version* v); - void GetRange(const std::vector<FileMetaData*>& inputs, - InternalKey* smallest, + void GetRange(const std::vector<FileMetaData*>& inputs, InternalKey* smallest, InternalKey* largest); void GetRange2(const std::vector<FileMetaData*>& inputs1, const std::vector<FileMetaData*>& inputs2, - InternalKey* smallest, - InternalKey* largest); + InternalKey* smallest, InternalKey* largest); void SetupOtherInputs(Compaction* c); @@ -314,10 +313,6 @@ class VersionSet { // Per-level key at which the next compaction at that level should start. // Either an empty string, or a valid InternalKey. std::string compact_pointer_[config::kNumLevels]; - - // No copying allowed - VersionSet(const VersionSet&); - void operator=(const VersionSet&); }; // A Compaction encapsulates information about a compaction. @@ -374,7 +369,7 @@ class Compaction { VersionEdit edit_; // Each compaction reads inputs from "level_" and "level_+1" - std::vector<FileMetaData*> inputs_[2]; // The two sets of inputs + std::vector<FileMetaData*> inputs_[2]; // The two sets of inputs // State used to check for number of overlapping grandparent files // (parent == level_ + 1, grandparent == level_ + 2) diff --git a/src/leveldb/db/version_set_test.cc b/src/leveldb/db/version_set_test.cc index 501e34d133..c1056a1e7d 100644 --- a/src/leveldb/db/version_set_test.cc +++ b/src/leveldb/db/version_set_test.cc @@ -11,10 +11,7 @@ namespace leveldb { class FindFileTest { public: - std::vector<FileMetaData*> files_; - bool disjoint_sorted_files_; - - FindFileTest() : disjoint_sorted_files_(true) { } + FindFileTest() : disjoint_sorted_files_(true) {} ~FindFileTest() { for (int i = 0; i < files_.size(); i++) { @@ -40,20 +37,25 @@ class FindFileTest { bool Overlaps(const char* smallest, const char* largest) { InternalKeyComparator cmp(BytewiseComparator()); - Slice s(smallest != NULL ? smallest : ""); - Slice l(largest != NULL ? largest : ""); + Slice s(smallest != nullptr ? smallest : ""); + Slice l(largest != nullptr ? largest : ""); return SomeFileOverlapsRange(cmp, disjoint_sorted_files_, files_, - (smallest != NULL ? &s : NULL), - (largest != NULL ? &l : NULL)); + (smallest != nullptr ? &s : nullptr), + (largest != nullptr ? &l : nullptr)); } + + bool disjoint_sorted_files_; + + private: + std::vector<FileMetaData*> files_; }; TEST(FindFileTest, Empty) { ASSERT_EQ(0, Find("foo")); - ASSERT_TRUE(! Overlaps("a", "z")); - ASSERT_TRUE(! Overlaps(NULL, "z")); - ASSERT_TRUE(! Overlaps("a", NULL)); - ASSERT_TRUE(! Overlaps(NULL, NULL)); + ASSERT_TRUE(!Overlaps("a", "z")); + ASSERT_TRUE(!Overlaps(nullptr, "z")); + ASSERT_TRUE(!Overlaps("a", nullptr)); + ASSERT_TRUE(!Overlaps(nullptr, nullptr)); } TEST(FindFileTest, Single) { @@ -65,8 +67,8 @@ TEST(FindFileTest, Single) { ASSERT_EQ(1, Find("q1")); ASSERT_EQ(1, Find("z")); - ASSERT_TRUE(! Overlaps("a", "b")); - ASSERT_TRUE(! Overlaps("z1", "z2")); + ASSERT_TRUE(!Overlaps("a", "b")); + ASSERT_TRUE(!Overlaps("z1", "z2")); ASSERT_TRUE(Overlaps("a", "p")); ASSERT_TRUE(Overlaps("a", "q")); ASSERT_TRUE(Overlaps("a", "z")); @@ -78,15 +80,14 @@ TEST(FindFileTest, Single) { ASSERT_TRUE(Overlaps("q", "q")); ASSERT_TRUE(Overlaps("q", "q1")); - ASSERT_TRUE(! Overlaps(NULL, "j")); - ASSERT_TRUE(! Overlaps("r", NULL)); - ASSERT_TRUE(Overlaps(NULL, "p")); - ASSERT_TRUE(Overlaps(NULL, "p1")); - ASSERT_TRUE(Overlaps("q", NULL)); - ASSERT_TRUE(Overlaps(NULL, NULL)); + ASSERT_TRUE(!Overlaps(nullptr, "j")); + ASSERT_TRUE(!Overlaps("r", nullptr)); + ASSERT_TRUE(Overlaps(nullptr, "p")); + ASSERT_TRUE(Overlaps(nullptr, "p1")); + ASSERT_TRUE(Overlaps("q", nullptr)); + ASSERT_TRUE(Overlaps(nullptr, nullptr)); } - TEST(FindFileTest, Multiple) { Add("150", "200"); Add("200", "250"); @@ -110,10 +111,10 @@ TEST(FindFileTest, Multiple) { ASSERT_EQ(3, Find("450")); ASSERT_EQ(4, Find("451")); - ASSERT_TRUE(! Overlaps("100", "149")); - ASSERT_TRUE(! Overlaps("251", "299")); - ASSERT_TRUE(! Overlaps("451", "500")); - ASSERT_TRUE(! Overlaps("351", "399")); + ASSERT_TRUE(!Overlaps("100", "149")); + ASSERT_TRUE(!Overlaps("251", "299")); + ASSERT_TRUE(!Overlaps("451", "500")); + ASSERT_TRUE(!Overlaps("351", "399")); ASSERT_TRUE(Overlaps("100", "150")); ASSERT_TRUE(Overlaps("100", "200")); @@ -130,25 +131,25 @@ TEST(FindFileTest, MultipleNullBoundaries) { Add("200", "250"); Add("300", "350"); Add("400", "450"); - ASSERT_TRUE(! Overlaps(NULL, "149")); - ASSERT_TRUE(! Overlaps("451", NULL)); - ASSERT_TRUE(Overlaps(NULL, NULL)); - ASSERT_TRUE(Overlaps(NULL, "150")); - ASSERT_TRUE(Overlaps(NULL, "199")); - ASSERT_TRUE(Overlaps(NULL, "200")); - ASSERT_TRUE(Overlaps(NULL, "201")); - ASSERT_TRUE(Overlaps(NULL, "400")); - ASSERT_TRUE(Overlaps(NULL, "800")); - ASSERT_TRUE(Overlaps("100", NULL)); - ASSERT_TRUE(Overlaps("200", NULL)); - ASSERT_TRUE(Overlaps("449", NULL)); - ASSERT_TRUE(Overlaps("450", NULL)); + ASSERT_TRUE(!Overlaps(nullptr, "149")); + ASSERT_TRUE(!Overlaps("451", nullptr)); + ASSERT_TRUE(Overlaps(nullptr, nullptr)); + ASSERT_TRUE(Overlaps(nullptr, "150")); + ASSERT_TRUE(Overlaps(nullptr, "199")); + ASSERT_TRUE(Overlaps(nullptr, "200")); + ASSERT_TRUE(Overlaps(nullptr, "201")); + ASSERT_TRUE(Overlaps(nullptr, "400")); + ASSERT_TRUE(Overlaps(nullptr, "800")); + ASSERT_TRUE(Overlaps("100", nullptr)); + ASSERT_TRUE(Overlaps("200", nullptr)); + ASSERT_TRUE(Overlaps("449", nullptr)); + ASSERT_TRUE(Overlaps("450", nullptr)); } TEST(FindFileTest, OverlapSequenceChecks) { Add("200", "200", 5000, 3000); - ASSERT_TRUE(! Overlaps("199", "199")); - ASSERT_TRUE(! Overlaps("201", "300")); + ASSERT_TRUE(!Overlaps("199", "199")); + ASSERT_TRUE(!Overlaps("201", "300")); ASSERT_TRUE(Overlaps("200", "200")); ASSERT_TRUE(Overlaps("190", "200")); ASSERT_TRUE(Overlaps("200", "210")); @@ -158,8 +159,8 @@ TEST(FindFileTest, OverlappingFiles) { Add("150", "600"); Add("400", "500"); disjoint_sorted_files_ = false; - ASSERT_TRUE(! Overlaps("100", "149")); - ASSERT_TRUE(! Overlaps("601", "700")); + ASSERT_TRUE(!Overlaps("100", "149")); + ASSERT_TRUE(!Overlaps("601", "700")); ASSERT_TRUE(Overlaps("100", "150")); ASSERT_TRUE(Overlaps("100", "200")); ASSERT_TRUE(Overlaps("100", "300")); @@ -172,8 +173,160 @@ TEST(FindFileTest, OverlappingFiles) { ASSERT_TRUE(Overlaps("600", "700")); } -} // namespace leveldb +void AddBoundaryInputs(const InternalKeyComparator& icmp, + const std::vector<FileMetaData*>& level_files, + std::vector<FileMetaData*>* compaction_files); + +class AddBoundaryInputsTest { + public: + std::vector<FileMetaData*> level_files_; + std::vector<FileMetaData*> compaction_files_; + std::vector<FileMetaData*> all_files_; + InternalKeyComparator icmp_; + + AddBoundaryInputsTest() : icmp_(BytewiseComparator()) {} + + ~AddBoundaryInputsTest() { + for (size_t i = 0; i < all_files_.size(); ++i) { + delete all_files_[i]; + } + all_files_.clear(); + } + + FileMetaData* CreateFileMetaData(uint64_t number, InternalKey smallest, + InternalKey largest) { + FileMetaData* f = new FileMetaData(); + f->number = number; + f->smallest = smallest; + f->largest = largest; + all_files_.push_back(f); + return f; + } +}; + +TEST(AddBoundaryInputsTest, TestEmptyFileSets) { + AddBoundaryInputs(icmp_, level_files_, &compaction_files_); + ASSERT_TRUE(compaction_files_.empty()); + ASSERT_TRUE(level_files_.empty()); +} + +TEST(AddBoundaryInputsTest, TestEmptyLevelFiles) { + FileMetaData* f1 = + CreateFileMetaData(1, InternalKey("100", 2, kTypeValue), + InternalKey(InternalKey("100", 1, kTypeValue))); + compaction_files_.push_back(f1); + + AddBoundaryInputs(icmp_, level_files_, &compaction_files_); + ASSERT_EQ(1, compaction_files_.size()); + ASSERT_EQ(f1, compaction_files_[0]); + ASSERT_TRUE(level_files_.empty()); +} + +TEST(AddBoundaryInputsTest, TestEmptyCompactionFiles) { + FileMetaData* f1 = + CreateFileMetaData(1, InternalKey("100", 2, kTypeValue), + InternalKey(InternalKey("100", 1, kTypeValue))); + level_files_.push_back(f1); + + AddBoundaryInputs(icmp_, level_files_, &compaction_files_); + ASSERT_TRUE(compaction_files_.empty()); + ASSERT_EQ(1, level_files_.size()); + ASSERT_EQ(f1, level_files_[0]); +} + +TEST(AddBoundaryInputsTest, TestNoBoundaryFiles) { + FileMetaData* f1 = + CreateFileMetaData(1, InternalKey("100", 2, kTypeValue), + InternalKey(InternalKey("100", 1, kTypeValue))); + FileMetaData* f2 = + CreateFileMetaData(1, InternalKey("200", 2, kTypeValue), + InternalKey(InternalKey("200", 1, kTypeValue))); + FileMetaData* f3 = + CreateFileMetaData(1, InternalKey("300", 2, kTypeValue), + InternalKey(InternalKey("300", 1, kTypeValue))); + + level_files_.push_back(f3); + level_files_.push_back(f2); + level_files_.push_back(f1); + compaction_files_.push_back(f2); + compaction_files_.push_back(f3); + + AddBoundaryInputs(icmp_, level_files_, &compaction_files_); + ASSERT_EQ(2, compaction_files_.size()); +} + +TEST(AddBoundaryInputsTest, TestOneBoundaryFiles) { + FileMetaData* f1 = + CreateFileMetaData(1, InternalKey("100", 3, kTypeValue), + InternalKey(InternalKey("100", 2, kTypeValue))); + FileMetaData* f2 = + CreateFileMetaData(1, InternalKey("100", 1, kTypeValue), + InternalKey(InternalKey("200", 3, kTypeValue))); + FileMetaData* f3 = + CreateFileMetaData(1, InternalKey("300", 2, kTypeValue), + InternalKey(InternalKey("300", 1, kTypeValue))); + + level_files_.push_back(f3); + level_files_.push_back(f2); + level_files_.push_back(f1); + compaction_files_.push_back(f1); + + AddBoundaryInputs(icmp_, level_files_, &compaction_files_); + ASSERT_EQ(2, compaction_files_.size()); + ASSERT_EQ(f1, compaction_files_[0]); + ASSERT_EQ(f2, compaction_files_[1]); +} + +TEST(AddBoundaryInputsTest, TestTwoBoundaryFiles) { + FileMetaData* f1 = + CreateFileMetaData(1, InternalKey("100", 6, kTypeValue), + InternalKey(InternalKey("100", 5, kTypeValue))); + FileMetaData* f2 = + CreateFileMetaData(1, InternalKey("100", 2, kTypeValue), + InternalKey(InternalKey("300", 1, kTypeValue))); + FileMetaData* f3 = + CreateFileMetaData(1, InternalKey("100", 4, kTypeValue), + InternalKey(InternalKey("100", 3, kTypeValue))); + + level_files_.push_back(f2); + level_files_.push_back(f3); + level_files_.push_back(f1); + compaction_files_.push_back(f1); -int main(int argc, char** argv) { - return leveldb::test::RunAllTests(); + AddBoundaryInputs(icmp_, level_files_, &compaction_files_); + ASSERT_EQ(3, compaction_files_.size()); + ASSERT_EQ(f1, compaction_files_[0]); + ASSERT_EQ(f3, compaction_files_[1]); + ASSERT_EQ(f2, compaction_files_[2]); } + +TEST(AddBoundaryInputsTest, TestDisjoinFilePointers) { + FileMetaData* f1 = + CreateFileMetaData(1, InternalKey("100", 6, kTypeValue), + InternalKey(InternalKey("100", 5, kTypeValue))); + FileMetaData* f2 = + CreateFileMetaData(1, InternalKey("100", 6, kTypeValue), + InternalKey(InternalKey("100", 5, kTypeValue))); + FileMetaData* f3 = + CreateFileMetaData(1, InternalKey("100", 2, kTypeValue), + InternalKey(InternalKey("300", 1, kTypeValue))); + FileMetaData* f4 = + CreateFileMetaData(1, InternalKey("100", 4, kTypeValue), + InternalKey(InternalKey("100", 3, kTypeValue))); + + level_files_.push_back(f2); + level_files_.push_back(f3); + level_files_.push_back(f4); + + compaction_files_.push_back(f1); + + AddBoundaryInputs(icmp_, level_files_, &compaction_files_); + ASSERT_EQ(3, compaction_files_.size()); + ASSERT_EQ(f1, compaction_files_[0]); + ASSERT_EQ(f4, compaction_files_[1]); + ASSERT_EQ(f3, compaction_files_[2]); +} + +} // namespace leveldb + +int main(int argc, char** argv) { return leveldb::test::RunAllTests(); } diff --git a/src/leveldb/db/write_batch.cc b/src/leveldb/db/write_batch.cc index 33f4a4257e..b54313c35e 100644 --- a/src/leveldb/db/write_batch.cc +++ b/src/leveldb/db/write_batch.cc @@ -15,10 +15,10 @@ #include "leveldb/write_batch.h" -#include "leveldb/db.h" #include "db/dbformat.h" #include "db/memtable.h" #include "db/write_batch_internal.h" +#include "leveldb/db.h" #include "util/coding.h" namespace leveldb { @@ -26,19 +26,19 @@ namespace leveldb { // WriteBatch header has an 8-byte sequence number followed by a 4-byte count. static const size_t kHeader = 12; -WriteBatch::WriteBatch() { - Clear(); -} +WriteBatch::WriteBatch() { Clear(); } -WriteBatch::~WriteBatch() { } +WriteBatch::~WriteBatch() = default; -WriteBatch::Handler::~Handler() { } +WriteBatch::Handler::~Handler() = default; void WriteBatch::Clear() { rep_.clear(); rep_.resize(kHeader); } +size_t WriteBatch::ApproximateSize() const { return rep_.size(); } + Status WriteBatch::Iterate(Handler* handler) const { Slice input(rep_); if (input.size() < kHeader) { @@ -108,25 +108,28 @@ void WriteBatch::Delete(const Slice& key) { PutLengthPrefixedSlice(&rep_, key); } +void WriteBatch::Append(const WriteBatch& source) { + WriteBatchInternal::Append(this, &source); +} + namespace { class MemTableInserter : public WriteBatch::Handler { public: SequenceNumber sequence_; MemTable* mem_; - virtual void Put(const Slice& key, const Slice& value) { + void Put(const Slice& key, const Slice& value) override { mem_->Add(sequence_, kTypeValue, key, value); sequence_++; } - virtual void Delete(const Slice& key) { + void Delete(const Slice& key) override { mem_->Add(sequence_, kTypeDeletion, key, Slice()); sequence_++; } }; } // namespace -Status WriteBatchInternal::InsertInto(const WriteBatch* b, - MemTable* memtable) { +Status WriteBatchInternal::InsertInto(const WriteBatch* b, MemTable* memtable) { MemTableInserter inserter; inserter.sequence_ = WriteBatchInternal::Sequence(b); inserter.mem_ = memtable; diff --git a/src/leveldb/db/write_batch_internal.h b/src/leveldb/db/write_batch_internal.h index 9448ef7b21..fce86e3f1f 100644 --- a/src/leveldb/db/write_batch_internal.h +++ b/src/leveldb/db/write_batch_internal.h @@ -29,13 +29,9 @@ class WriteBatchInternal { // this batch. static void SetSequence(WriteBatch* batch, SequenceNumber seq); - static Slice Contents(const WriteBatch* batch) { - return Slice(batch->rep_); - } + static Slice Contents(const WriteBatch* batch) { return Slice(batch->rep_); } - static size_t ByteSize(const WriteBatch* batch) { - return batch->rep_.size(); - } + static size_t ByteSize(const WriteBatch* batch) { return batch->rep_.size(); } static void SetContents(WriteBatch* batch, const Slice& contents); @@ -46,5 +42,4 @@ class WriteBatchInternal { } // namespace leveldb - #endif // STORAGE_LEVELDB_DB_WRITE_BATCH_INTERNAL_H_ diff --git a/src/leveldb/db/write_batch_test.cc b/src/leveldb/db/write_batch_test.cc index 9064e3d85e..c32317fb5e 100644 --- a/src/leveldb/db/write_batch_test.cc +++ b/src/leveldb/db/write_batch_test.cc @@ -52,7 +52,7 @@ static std::string PrintContents(WriteBatch* b) { return state; } -class WriteBatchTest { }; +class WriteBatchTest {}; TEST(WriteBatchTest, Empty) { WriteBatch batch; @@ -68,10 +68,11 @@ TEST(WriteBatchTest, Multiple) { WriteBatchInternal::SetSequence(&batch, 100); ASSERT_EQ(100, WriteBatchInternal::Sequence(&batch)); ASSERT_EQ(3, WriteBatchInternal::Count(&batch)); - ASSERT_EQ("Put(baz, boo)@102" - "Delete(box)@101" - "Put(foo, bar)@100", - PrintContents(&batch)); + ASSERT_EQ( + "Put(baz, boo)@102" + "Delete(box)@101" + "Put(foo, bar)@100", + PrintContents(&batch)); } TEST(WriteBatchTest, Corruption) { @@ -81,40 +82,56 @@ TEST(WriteBatchTest, Corruption) { WriteBatchInternal::SetSequence(&batch, 200); Slice contents = WriteBatchInternal::Contents(&batch); WriteBatchInternal::SetContents(&batch, - Slice(contents.data(),contents.size()-1)); - ASSERT_EQ("Put(foo, bar)@200" - "ParseError()", - PrintContents(&batch)); + Slice(contents.data(), contents.size() - 1)); + ASSERT_EQ( + "Put(foo, bar)@200" + "ParseError()", + PrintContents(&batch)); } TEST(WriteBatchTest, Append) { WriteBatch b1, b2; WriteBatchInternal::SetSequence(&b1, 200); WriteBatchInternal::SetSequence(&b2, 300); - WriteBatchInternal::Append(&b1, &b2); - ASSERT_EQ("", - PrintContents(&b1)); + b1.Append(b2); + ASSERT_EQ("", PrintContents(&b1)); b2.Put("a", "va"); - WriteBatchInternal::Append(&b1, &b2); - ASSERT_EQ("Put(a, va)@200", - PrintContents(&b1)); + b1.Append(b2); + ASSERT_EQ("Put(a, va)@200", PrintContents(&b1)); b2.Clear(); b2.Put("b", "vb"); - WriteBatchInternal::Append(&b1, &b2); - ASSERT_EQ("Put(a, va)@200" - "Put(b, vb)@201", - PrintContents(&b1)); + b1.Append(b2); + ASSERT_EQ( + "Put(a, va)@200" + "Put(b, vb)@201", + PrintContents(&b1)); b2.Delete("foo"); - WriteBatchInternal::Append(&b1, &b2); - ASSERT_EQ("Put(a, va)@200" - "Put(b, vb)@202" - "Put(b, vb)@201" - "Delete(foo)@203", - PrintContents(&b1)); + b1.Append(b2); + ASSERT_EQ( + "Put(a, va)@200" + "Put(b, vb)@202" + "Put(b, vb)@201" + "Delete(foo)@203", + PrintContents(&b1)); } -} // namespace leveldb +TEST(WriteBatchTest, ApproximateSize) { + WriteBatch batch; + size_t empty_size = batch.ApproximateSize(); + + batch.Put(Slice("foo"), Slice("bar")); + size_t one_key_size = batch.ApproximateSize(); + ASSERT_LT(empty_size, one_key_size); + + batch.Put(Slice("baz"), Slice("boo")); + size_t two_keys_size = batch.ApproximateSize(); + ASSERT_LT(one_key_size, two_keys_size); -int main(int argc, char** argv) { - return leveldb::test::RunAllTests(); + batch.Delete(Slice("box")); + size_t post_delete_size = batch.ApproximateSize(); + ASSERT_LT(two_keys_size, post_delete_size); } + +} // namespace leveldb + +int main(int argc, char** argv) { return leveldb::test::RunAllTests(); } |