From 4b0e2d75d78036e7e76fd9584e17b0379ba08d24 Mon Sep 17 00:00:00 2001 From: Pieter Wuille Date: Thu, 16 Oct 2014 12:23:50 -0700 Subject: Squashed 'src/leveldb/' changes from 7924331..7d41e6f 7d41e6f Merge upstream LevelDB 1.18 803d692 Release 1.18 git-subtree-dir: src/leveldb git-subtree-split: 7d41e6f89ff04ce9e6a742932924796f69c6e23d --- db/db_bench.cc | 3 +- db/db_impl.cc | 4 +- db/db_test.cc | 2 +- db/dbformat.h | 6 +- db/dumpfile.cc | 225 ++++++++++++++++++++++++++++++++++++++++++++++ db/leveldb_main.cc | 204 ++++------------------------------------- db/log_format.h | 4 +- db/log_reader.cc | 6 +- db/log_reader.h | 4 +- db/log_test.cc | 2 +- db/repair.cc | 2 +- db/skiplist.h | 5 ++ db/write_batch_internal.h | 4 +- 13 files changed, 263 insertions(+), 208 deletions(-) create mode 100644 db/dumpfile.cc (limited to 'db') diff --git a/db/db_bench.cc b/db/db_bench.cc index fc46d89693..705a170aae 100644 --- a/db/db_bench.cc +++ b/db/db_bench.cc @@ -431,7 +431,7 @@ class Benchmark { benchmarks = sep + 1; } - // Reset parameters that may be overriddden bwlow + // Reset parameters that may be overridden below num_ = FLAGS_num; reads_ = (FLAGS_reads < 0 ? FLAGS_num : FLAGS_reads); value_size_ = FLAGS_value_size; @@ -811,7 +811,6 @@ class Benchmark { void SeekRandom(ThreadState* thread) { ReadOptions options; - std::string value; int found = 0; for (int i = 0; i < reads_; i++) { Iterator* iter = db_->NewIterator(options); diff --git a/db/db_impl.cc b/db/db_impl.cc index faf5e7d7ba..49b95953b4 100644 --- a/db/db_impl.cc +++ b/db/db_impl.cc @@ -392,7 +392,7 @@ Status DBImpl::RecoverLogFile(uint64_t log_number, reporter.info_log = options_.info_log; reporter.fname = fname.c_str(); reporter.status = (options_.paranoid_checks ? &status : NULL); - // We intentially make log::Reader do checksumming even if + // We intentionally make log::Reader do checksumming even if // paranoid_checks==false so that corruptions cause entire commits // to be skipped instead of propagating bad information (like overly // large sequence numbers). @@ -1267,7 +1267,7 @@ WriteBatch* DBImpl::BuildBatchGroup(Writer** last_writer) { break; } - // Append to *reuslt + // Append to *result if (result == first->batch) { // Switch to temporary batch instead of disturbing caller's batch result = tmp_batch_; diff --git a/db/db_test.cc b/db/db_test.cc index 280b01c14b..0fed9137d5 100644 --- a/db/db_test.cc +++ b/db/db_test.cc @@ -626,7 +626,7 @@ TEST(DBTest, GetEncountersEmptyLevel) { // * sstable B in level 2 // Then do enough Get() calls to arrange for an automatic compaction // of sstable A. A bug would cause the compaction to be marked as - // occuring at level 1 (instead of the correct level 0). + // occurring at level 1 (instead of the correct level 0). // Step 1: First place sstables in levels 0 and 2 int compaction_count = 0; diff --git a/db/dbformat.h b/db/dbformat.h index 5d8a032bd3..ea897b13c0 100644 --- a/db/dbformat.h +++ b/db/dbformat.h @@ -2,8 +2,8 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. See the AUTHORS file for names of contributors. -#ifndef STORAGE_LEVELDB_DB_FORMAT_H_ -#define STORAGE_LEVELDB_DB_FORMAT_H_ +#ifndef STORAGE_LEVELDB_DB_DBFORMAT_H_ +#define STORAGE_LEVELDB_DB_DBFORMAT_H_ #include #include "leveldb/comparator.h" @@ -227,4 +227,4 @@ inline LookupKey::~LookupKey() { } // namespace leveldb -#endif // STORAGE_LEVELDB_DB_FORMAT_H_ +#endif // STORAGE_LEVELDB_DB_DBFORMAT_H_ diff --git a/db/dumpfile.cc b/db/dumpfile.cc new file mode 100644 index 0000000000..61c47c2ff9 --- /dev/null +++ b/db/dumpfile.cc @@ -0,0 +1,225 @@ +// Copyright (c) 2012 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include +#include "db/dbformat.h" +#include "db/filename.h" +#include "db/log_reader.h" +#include "db/version_edit.h" +#include "db/write_batch_internal.h" +#include "leveldb/env.h" +#include "leveldb/iterator.h" +#include "leveldb/options.h" +#include "leveldb/status.h" +#include "leveldb/table.h" +#include "leveldb/write_batch.h" +#include "util/logging.h" + +namespace leveldb { + +namespace { + +bool GuessType(const std::string& fname, FileType* type) { + size_t pos = fname.rfind('/'); + std::string basename; + if (pos == std::string::npos) { + basename = fname; + } else { + basename = std::string(fname.data() + pos + 1, fname.size() - pos - 1); + } + uint64_t ignored; + return ParseFileName(basename, &ignored, type); +} + +// Notified when log reader encounters corruption. +class CorruptionReporter : public log::Reader::Reporter { + public: + WritableFile* dst_; + virtual void Corruption(size_t bytes, const Status& status) { + std::string r = "corruption: "; + AppendNumberTo(&r, bytes); + r += " bytes; "; + r += status.ToString(); + r.push_back('\n'); + dst_->Append(r); + } +}; + +// Print contents of a log file. (*func)() is called on every record. +Status PrintLogContents(Env* env, const std::string& fname, + void (*func)(uint64_t, Slice, WritableFile*), + WritableFile* dst) { + SequentialFile* file; + Status s = env->NewSequentialFile(fname, &file); + if (!s.ok()) { + return s; + } + CorruptionReporter reporter; + reporter.dst_ = dst; + log::Reader reader(file, &reporter, true, 0); + Slice record; + std::string scratch; + while (reader.ReadRecord(&record, &scratch)) { + (*func)(reader.LastRecordOffset(), record, dst); + } + delete file; + return Status::OK(); +} + +// Called on every item found in a WriteBatch. +class WriteBatchItemPrinter : public WriteBatch::Handler { + public: + WritableFile* dst_; + virtual void Put(const Slice& key, const Slice& value) { + std::string r = " put '"; + AppendEscapedStringTo(&r, key); + r += "' '"; + AppendEscapedStringTo(&r, value); + r += "'\n"; + dst_->Append(r); + } + virtual void Delete(const Slice& key) { + std::string r = " del '"; + AppendEscapedStringTo(&r, key); + r += "'\n"; + dst_->Append(r); + } +}; + + +// Called on every log record (each one of which is a WriteBatch) +// found in a kLogFile. +static void WriteBatchPrinter(uint64_t pos, Slice record, WritableFile* dst) { + std::string r = "--- offset "; + AppendNumberTo(&r, pos); + r += "; "; + if (record.size() < 12) { + r += "log record length "; + AppendNumberTo(&r, record.size()); + r += " is too small\n"; + dst->Append(r); + return; + } + WriteBatch batch; + WriteBatchInternal::SetContents(&batch, record); + r += "sequence "; + AppendNumberTo(&r, WriteBatchInternal::Sequence(&batch)); + r.push_back('\n'); + dst->Append(r); + WriteBatchItemPrinter batch_item_printer; + batch_item_printer.dst_ = dst; + Status s = batch.Iterate(&batch_item_printer); + if (!s.ok()) { + dst->Append(" error: " + s.ToString() + "\n"); + } +} + +Status DumpLog(Env* env, const std::string& fname, WritableFile* dst) { + return PrintLogContents(env, fname, WriteBatchPrinter, dst); +} + +// Called on every log record (each one of which is a WriteBatch) +// found in a kDescriptorFile. +static void VersionEditPrinter(uint64_t pos, Slice record, WritableFile* dst) { + std::string r = "--- offset "; + AppendNumberTo(&r, pos); + r += "; "; + VersionEdit edit; + Status s = edit.DecodeFrom(record); + if (!s.ok()) { + r += s.ToString(); + r.push_back('\n'); + } else { + r += edit.DebugString(); + } + dst->Append(r); +} + +Status DumpDescriptor(Env* env, const std::string& fname, WritableFile* dst) { + return PrintLogContents(env, fname, VersionEditPrinter, dst); +} + +Status DumpTable(Env* env, const std::string& fname, WritableFile* dst) { + uint64_t file_size; + RandomAccessFile* file = NULL; + Table* table = NULL; + Status s = env->GetFileSize(fname, &file_size); + if (s.ok()) { + s = env->NewRandomAccessFile(fname, &file); + } + if (s.ok()) { + // We use the default comparator, which may or may not match the + // comparator used in this database. However this should not cause + // problems since we only use Table operations that do not require + // any comparisons. In particular, we do not call Seek or Prev. + s = Table::Open(Options(), file, file_size, &table); + } + if (!s.ok()) { + delete table; + delete file; + return s; + } + + ReadOptions ro; + ro.fill_cache = false; + Iterator* iter = table->NewIterator(ro); + std::string r; + for (iter->SeekToFirst(); iter->Valid(); iter->Next()) { + r.clear(); + ParsedInternalKey key; + if (!ParseInternalKey(iter->key(), &key)) { + r = "badkey '"; + AppendEscapedStringTo(&r, iter->key()); + r += "' => '"; + AppendEscapedStringTo(&r, iter->value()); + r += "'\n"; + dst->Append(r); + } else { + r = "'"; + AppendEscapedStringTo(&r, key.user_key); + r += "' @ "; + AppendNumberTo(&r, key.sequence); + r += " : "; + if (key.type == kTypeDeletion) { + r += "del"; + } else if (key.type == kTypeValue) { + r += "val"; + } else { + AppendNumberTo(&r, key.type); + } + r += " => '"; + AppendEscapedStringTo(&r, iter->value()); + r += "'\n"; + dst->Append(r); + } + } + s = iter->status(); + if (!s.ok()) { + dst->Append("iterator error: " + s.ToString() + "\n"); + } + + delete iter; + delete table; + delete file; + return Status::OK(); +} + +} // namespace + +Status DumpFile(Env* env, const std::string& fname, WritableFile* dst) { + FileType ftype; + if (!GuessType(fname, &ftype)) { + return Status::InvalidArgument(fname + ": unknown file type"); + } + switch (ftype) { + case kLogFile: return DumpLog(env, fname, dst); + case kDescriptorFile: return DumpDescriptor(env, fname, dst); + case kTableFile: return DumpTable(env, fname, dst); + default: + break; + } + return Status::InvalidArgument(fname + ": not a dump-able file type"); +} + +} // namespace leveldb diff --git a/db/leveldb_main.cc b/db/leveldb_main.cc index 995d76107a..9f4b7dd70c 100644 --- a/db/leveldb_main.cc +++ b/db/leveldb_main.cc @@ -3,212 +3,38 @@ // found in the LICENSE file. See the AUTHORS file for names of contributors. #include -#include "db/dbformat.h" -#include "db/filename.h" -#include "db/log_reader.h" -#include "db/version_edit.h" -#include "db/write_batch_internal.h" +#include "leveldb/dumpfile.h" #include "leveldb/env.h" -#include "leveldb/iterator.h" -#include "leveldb/options.h" #include "leveldb/status.h" -#include "leveldb/table.h" -#include "leveldb/write_batch.h" -#include "util/logging.h" namespace leveldb { - namespace { -bool GuessType(const std::string& fname, FileType* type) { - size_t pos = fname.rfind('/'); - std::string basename; - if (pos == std::string::npos) { - basename = fname; - } else { - basename = std::string(fname.data() + pos + 1, fname.size() - pos - 1); - } - uint64_t ignored; - return ParseFileName(basename, &ignored, type); -} - -// Notified when log reader encounters corruption. -class CorruptionReporter : public log::Reader::Reporter { - public: - virtual void Corruption(size_t bytes, const Status& status) { - printf("corruption: %d bytes; %s\n", - static_cast(bytes), - status.ToString().c_str()); - } -}; - -// Print contents of a log file. (*func)() is called on every record. -bool PrintLogContents(Env* env, const std::string& fname, - void (*func)(Slice)) { - SequentialFile* file; - Status s = env->NewSequentialFile(fname, &file); - if (!s.ok()) { - fprintf(stderr, "%s\n", s.ToString().c_str()); - return false; - } - CorruptionReporter reporter; - log::Reader reader(file, &reporter, true, 0); - Slice record; - std::string scratch; - while (reader.ReadRecord(&record, &scratch)) { - printf("--- offset %llu; ", - static_cast(reader.LastRecordOffset())); - (*func)(record); - } - delete file; - return true; -} - -// Called on every item found in a WriteBatch. -class WriteBatchItemPrinter : public WriteBatch::Handler { +class StdoutPrinter : public WritableFile { public: - uint64_t offset_; - uint64_t sequence_; - - virtual void Put(const Slice& key, const Slice& value) { - printf(" put '%s' '%s'\n", - EscapeString(key).c_str(), - EscapeString(value).c_str()); - } - virtual void Delete(const Slice& key) { - printf(" del '%s'\n", - EscapeString(key).c_str()); + virtual Status Append(const Slice& data) { + fwrite(data.data(), 1, data.size(), stdout); + return Status::OK(); } + virtual Status Close() { return Status::OK(); } + virtual Status Flush() { return Status::OK(); } + virtual Status Sync() { return Status::OK(); } }; - -// Called on every log record (each one of which is a WriteBatch) -// found in a kLogFile. -static void WriteBatchPrinter(Slice record) { - if (record.size() < 12) { - printf("log record length %d is too small\n", - static_cast(record.size())); - return; - } - WriteBatch batch; - WriteBatchInternal::SetContents(&batch, record); - printf("sequence %llu\n", - static_cast(WriteBatchInternal::Sequence(&batch))); - WriteBatchItemPrinter batch_item_printer; - Status s = batch.Iterate(&batch_item_printer); - if (!s.ok()) { - printf(" error: %s\n", s.ToString().c_str()); - } -} - -bool DumpLog(Env* env, const std::string& fname) { - return PrintLogContents(env, fname, WriteBatchPrinter); -} - -// Called on every log record (each one of which is a WriteBatch) -// found in a kDescriptorFile. -static void VersionEditPrinter(Slice record) { - VersionEdit edit; - Status s = edit.DecodeFrom(record); - if (!s.ok()) { - printf("%s\n", s.ToString().c_str()); - return; - } - printf("%s", edit.DebugString().c_str()); -} - -bool DumpDescriptor(Env* env, const std::string& fname) { - return PrintLogContents(env, fname, VersionEditPrinter); -} - -bool DumpTable(Env* env, const std::string& fname) { - uint64_t file_size; - RandomAccessFile* file = NULL; - Table* table = NULL; - Status s = env->GetFileSize(fname, &file_size); - if (s.ok()) { - s = env->NewRandomAccessFile(fname, &file); - } - if (s.ok()) { - // We use the default comparator, which may or may not match the - // comparator used in this database. However this should not cause - // problems since we only use Table operations that do not require - // any comparisons. In particular, we do not call Seek or Prev. - s = Table::Open(Options(), file, file_size, &table); - } - if (!s.ok()) { - fprintf(stderr, "%s\n", s.ToString().c_str()); - delete table; - delete file; - return false; - } - - ReadOptions ro; - ro.fill_cache = false; - Iterator* iter = table->NewIterator(ro); - for (iter->SeekToFirst(); iter->Valid(); iter->Next()) { - ParsedInternalKey key; - if (!ParseInternalKey(iter->key(), &key)) { - printf("badkey '%s' => '%s'\n", - EscapeString(iter->key()).c_str(), - EscapeString(iter->value()).c_str()); - } else { - char kbuf[20]; - const char* type; - if (key.type == kTypeDeletion) { - type = "del"; - } else if (key.type == kTypeValue) { - type = "val"; - } else { - snprintf(kbuf, sizeof(kbuf), "%d", static_cast(key.type)); - type = kbuf; - } - printf("'%s' @ %8llu : %s => '%s'\n", - EscapeString(key.user_key).c_str(), - static_cast(key.sequence), - type, - EscapeString(iter->value()).c_str()); - } - } - s = iter->status(); - if (!s.ok()) { - printf("iterator error: %s\n", s.ToString().c_str()); - } - - delete iter; - delete table; - delete file; - return true; -} - -bool DumpFile(Env* env, const std::string& fname) { - FileType ftype; - if (!GuessType(fname, &ftype)) { - fprintf(stderr, "%s: unknown file type\n", fname.c_str()); - return false; - } - switch (ftype) { - case kLogFile: return DumpLog(env, fname); - case kDescriptorFile: return DumpDescriptor(env, fname); - case kTableFile: return DumpTable(env, fname); - - default: { - fprintf(stderr, "%s: not a dump-able file type\n", fname.c_str()); - break; - } - } - return false; -} - bool HandleDumpCommand(Env* env, char** files, int num) { + StdoutPrinter printer; bool ok = true; for (int i = 0; i < num; i++) { - ok &= DumpFile(env, files[i]); + Status s = DumpFile(env, files[i], &printer); + if (!s.ok()) { + fprintf(stderr, "%s\n", s.ToString().c_str()); + ok = false; + } } return ok; } -} +} // namespace } // namespace leveldb static void Usage() { diff --git a/db/log_format.h b/db/log_format.h index 2690cb9789..a8c06efe18 100644 --- a/db/log_format.h +++ b/db/log_format.h @@ -26,8 +26,8 @@ static const int kMaxRecordType = kLastType; static const int kBlockSize = 32768; -// Header is checksum (4 bytes), type (1 byte), length (2 bytes). -static const int kHeaderSize = 4 + 1 + 2; +// Header is checksum (4 bytes), length (2 bytes), type (1 byte). +static const int kHeaderSize = 4 + 2 + 1; } // namespace log } // namespace leveldb diff --git a/db/log_reader.cc b/db/log_reader.cc index 4919216d04..e44b66c85b 100644 --- a/db/log_reader.cc +++ b/db/log_reader.cc @@ -167,14 +167,14 @@ uint64_t Reader::LastRecordOffset() { return last_record_offset_; } -void Reader::ReportCorruption(size_t bytes, const char* reason) { +void Reader::ReportCorruption(uint64_t bytes, const char* reason) { ReportDrop(bytes, Status::Corruption(reason)); } -void Reader::ReportDrop(size_t bytes, const Status& reason) { +void Reader::ReportDrop(uint64_t bytes, const Status& reason) { if (reporter_ != NULL && end_of_buffer_offset_ - buffer_.size() - bytes >= initial_offset_) { - reporter_->Corruption(bytes, reason); + reporter_->Corruption(static_cast(bytes), reason); } } diff --git a/db/log_reader.h b/db/log_reader.h index 82d4bee68d..6aff791716 100644 --- a/db/log_reader.h +++ b/db/log_reader.h @@ -94,8 +94,8 @@ class Reader { // Reports dropped bytes to the reporter. // buffer_ must be updated to remove the dropped bytes prior to invocation. - void ReportCorruption(size_t bytes, const char* reason); - void ReportDrop(size_t bytes, const Status& reason); + void ReportCorruption(uint64_t bytes, const char* reason); + void ReportDrop(uint64_t bytes, const Status& reason); // No copying allowed Reader(const Reader&); diff --git a/db/log_test.cc b/db/log_test.cc index 91d3caafc3..dcf0562652 100644 --- a/db/log_test.cc +++ b/db/log_test.cc @@ -463,7 +463,7 @@ TEST(LogTest, ErrorJoinsRecords) { ASSERT_EQ("correct", Read()); ASSERT_EQ("EOF", Read()); - const int dropped = DroppedBytes(); + const size_t dropped = DroppedBytes(); ASSERT_LE(dropped, 2*kBlockSize + 100); ASSERT_GE(dropped, 2*kBlockSize); } diff --git a/db/repair.cc b/db/repair.cc index 7727fafc58..4cd4bb047f 100644 --- a/db/repair.cc +++ b/db/repair.cc @@ -186,7 +186,7 @@ class Repairer { reporter.env = env_; reporter.info_log = options_.info_log; reporter.lognum = log; - // We intentially make log::Reader do checksumming so that + // We intentionally make log::Reader do checksumming so that // corruptions cause entire commits to be skipped instead of // propagating bad information (like overly large sequence // numbers). diff --git a/db/skiplist.h b/db/skiplist.h index af85be6d01..ed8b092203 100644 --- a/db/skiplist.h +++ b/db/skiplist.h @@ -1,3 +1,6 @@ +#ifndef STORAGE_LEVELDB_DB_SKIPLIST_H_ +#define STORAGE_LEVELDB_DB_SKIPLIST_H_ + // Copyright (c) 2011 The LevelDB Authors. All rights reserved. // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. See the AUTHORS file for names of contributors. @@ -377,3 +380,5 @@ bool SkipList::Contains(const Key& key) const { } } // namespace leveldb + +#endif // STORAGE_LEVELDB_DB_SKIPLIST_H_ diff --git a/db/write_batch_internal.h b/db/write_batch_internal.h index 4423a7f318..310a3c8912 100644 --- a/db/write_batch_internal.h +++ b/db/write_batch_internal.h @@ -21,10 +21,10 @@ class WriteBatchInternal { // Set the count for the number of entries in the batch. static void SetCount(WriteBatch* batch, int n); - // Return the seqeunce number for the start of this batch. + // Return the sequence number for the start of this batch. static SequenceNumber Sequence(const WriteBatch* batch); - // Store the specified number as the seqeunce number for the start of + // Store the specified number as the sequence number for the start of // this batch. static void SetSequence(WriteBatch* batch, SequenceNumber seq); -- cgit v1.2.3