aboutsummaryrefslogtreecommitdiff
path: root/src/leveldb/util
diff options
context:
space:
mode:
Diffstat (limited to 'src/leveldb/util')
-rw-r--r--src/leveldb/util/crc32c.cc18
-rw-r--r--src/leveldb/util/env_posix.cc195
-rw-r--r--src/leveldb/util/env_posix_test.cc66
-rw-r--r--src/leveldb/util/env_posix_test_helper.h28
-rw-r--r--src/leveldb/util/env_test.cc18
-rw-r--r--src/leveldb/util/env_win.cc12
-rw-r--r--src/leveldb/util/options.cc1
7 files changed, 266 insertions, 72 deletions
diff --git a/src/leveldb/util/crc32c.cc b/src/leveldb/util/crc32c.cc
index 6db9e77077..edd61cfd6f 100644
--- a/src/leveldb/util/crc32c.cc
+++ b/src/leveldb/util/crc32c.cc
@@ -8,6 +8,8 @@
#include "util/crc32c.h"
#include <stdint.h>
+
+#include "port/port.h"
#include "util/coding.h"
namespace leveldb {
@@ -283,7 +285,23 @@ static inline uint32_t LE_LOAD32(const uint8_t *p) {
return DecodeFixed32(reinterpret_cast<const char*>(p));
}
+// Determine if the CPU running this program can accelerate the CRC32C
+// calculation.
+static bool CanAccelerateCRC32C() {
+ // port::AcceleretedCRC32C returns zero when unable to accelerate.
+ static const char kTestCRCBuffer[] = "TestCRCBuffer";
+ static const char kBufSize = sizeof(kTestCRCBuffer) - 1;
+ static const uint32_t kTestCRCValue = 0xdcbc59fa;
+
+ return port::AcceleratedCRC32C(0, kTestCRCBuffer, kBufSize) == kTestCRCValue;
+}
+
uint32_t Extend(uint32_t crc, const char* buf, size_t size) {
+ static bool accelerate = CanAccelerateCRC32C();
+ if (accelerate) {
+ return port::AcceleratedCRC32C(crc, buf, size);
+ }
+
const uint8_t *p = reinterpret_cast<const uint8_t *>(buf);
const uint8_t *e = p + size;
uint32_t l = crc ^ 0xffffffffu;
diff --git a/src/leveldb/util/env_posix.cc b/src/leveldb/util/env_posix.cc
index e0fca52f46..dd852af354 100644
--- a/src/leveldb/util/env_posix.cc
+++ b/src/leveldb/util/env_posix.cc
@@ -11,12 +11,14 @@
#include <stdlib.h>
#include <string.h>
#include <sys/mman.h>
+#include <sys/resource.h>
#include <sys/stat.h>
#include <sys/time.h>
#include <sys/types.h>
#include <time.h>
#include <unistd.h>
#include <deque>
+#include <limits>
#include <set>
#include "leveldb/env.h"
#include "leveldb/slice.h"
@@ -24,15 +26,70 @@
#include "util/logging.h"
#include "util/mutexlock.h"
#include "util/posix_logger.h"
+#include "util/env_posix_test_helper.h"
namespace leveldb {
namespace {
+static int open_read_only_file_limit = -1;
+static int mmap_limit = -1;
+
static Status IOError(const std::string& context, int err_number) {
return Status::IOError(context, strerror(err_number));
}
+// Helper class to limit resource usage to avoid exhaustion.
+// Currently used to limit read-only file descriptors and mmap file usage
+// so that we do not end up running out of file descriptors, virtual memory,
+// or running into kernel performance problems for very large databases.
+class Limiter {
+ public:
+ // Limit maximum number of resources to |n|.
+ Limiter(intptr_t n) {
+ SetAllowed(n);
+ }
+
+ // If another resource is available, acquire it and return true.
+ // Else return false.
+ bool Acquire() {
+ if (GetAllowed() <= 0) {
+ return false;
+ }
+ MutexLock l(&mu_);
+ intptr_t x = GetAllowed();
+ if (x <= 0) {
+ return false;
+ } else {
+ SetAllowed(x - 1);
+ return true;
+ }
+ }
+
+ // Release a resource acquired by a previous call to Acquire() that returned
+ // true.
+ void Release() {
+ MutexLock l(&mu_);
+ SetAllowed(GetAllowed() + 1);
+ }
+
+ private:
+ port::Mutex mu_;
+ port::AtomicPointer allowed_;
+
+ intptr_t GetAllowed() const {
+ return reinterpret_cast<intptr_t>(allowed_.Acquire_Load());
+ }
+
+ // REQUIRES: mu_ must be held
+ void SetAllowed(intptr_t v) {
+ allowed_.Release_Store(reinterpret_cast<void*>(v));
+ }
+
+ Limiter(const Limiter&);
+ void operator=(const Limiter&);
+};
+
class PosixSequentialFile: public SequentialFile {
private:
std::string filename_;
@@ -70,73 +127,51 @@ class PosixSequentialFile: public SequentialFile {
class PosixRandomAccessFile: public RandomAccessFile {
private:
std::string filename_;
+ bool temporary_fd_; // If true, fd_ is -1 and we open on every read.
int fd_;
+ Limiter* limiter_;
public:
- PosixRandomAccessFile(const std::string& fname, int fd)
- : filename_(fname), fd_(fd) { }
- virtual ~PosixRandomAccessFile() { close(fd_); }
+ PosixRandomAccessFile(const std::string& fname, int fd, Limiter* limiter)
+ : filename_(fname), fd_(fd), limiter_(limiter) {
+ temporary_fd_ = !limiter->Acquire();
+ if (temporary_fd_) {
+ // Open file on every access.
+ close(fd_);
+ fd_ = -1;
+ }
+ }
+
+ virtual ~PosixRandomAccessFile() {
+ if (!temporary_fd_) {
+ close(fd_);
+ limiter_->Release();
+ }
+ }
virtual Status Read(uint64_t offset, size_t n, Slice* result,
char* scratch) const {
+ int fd = fd_;
+ if (temporary_fd_) {
+ fd = open(filename_.c_str(), O_RDONLY);
+ if (fd < 0) {
+ return IOError(filename_, errno);
+ }
+ }
+
Status s;
- ssize_t r = pread(fd_, scratch, n, static_cast<off_t>(offset));
+ ssize_t r = pread(fd, scratch, n, static_cast<off_t>(offset));
*result = Slice(scratch, (r < 0) ? 0 : r);
if (r < 0) {
// An error: return a non-ok status
s = IOError(filename_, errno);
}
- return s;
- }
-};
-
-// Helper class to limit mmap file usage so that we do not end up
-// running out virtual memory or running into kernel performance
-// problems for very large databases.
-class MmapLimiter {
- public:
- // Up to 1000 mmaps for 64-bit binaries; none for smaller pointer sizes.
- MmapLimiter() {
- SetAllowed(sizeof(void*) >= 8 ? 1000 : 0);
- }
-
- // If another mmap slot is available, acquire it and return true.
- // Else return false.
- bool Acquire() {
- if (GetAllowed() <= 0) {
- return false;
- }
- MutexLock l(&mu_);
- intptr_t x = GetAllowed();
- if (x <= 0) {
- return false;
- } else {
- SetAllowed(x - 1);
- return true;
+ if (temporary_fd_) {
+ // Close the temporary file descriptor opened earlier.
+ close(fd);
}
+ return s;
}
-
- // Release a slot acquired by a previous call to Acquire() that returned true.
- void Release() {
- MutexLock l(&mu_);
- SetAllowed(GetAllowed() + 1);
- }
-
- private:
- port::Mutex mu_;
- port::AtomicPointer allowed_;
-
- intptr_t GetAllowed() const {
- return reinterpret_cast<intptr_t>(allowed_.Acquire_Load());
- }
-
- // REQUIRES: mu_ must be held
- void SetAllowed(intptr_t v) {
- allowed_.Release_Store(reinterpret_cast<void*>(v));
- }
-
- MmapLimiter(const MmapLimiter&);
- void operator=(const MmapLimiter&);
};
// mmap() based random-access
@@ -145,12 +180,12 @@ class PosixMmapReadableFile: public RandomAccessFile {
std::string filename_;
void* mmapped_region_;
size_t length_;
- MmapLimiter* limiter_;
+ Limiter* limiter_;
public:
// base[0,length-1] contains the mmapped contents of the file.
PosixMmapReadableFile(const std::string& fname, void* base, size_t length,
- MmapLimiter* limiter)
+ Limiter* limiter)
: filename_(fname), mmapped_region_(base), length_(length),
limiter_(limiter) {
}
@@ -231,7 +266,7 @@ class PosixWritableFile : public WritableFile {
if (fd < 0) {
s = IOError(dir, errno);
} else {
- if (fsync(fd) < 0) {
+ if (fsync(fd) < 0 && errno != EINVAL) {
s = IOError(dir, errno);
}
close(fd);
@@ -333,7 +368,7 @@ class PosixEnv : public Env {
mmap_limit_.Release();
}
} else {
- *result = new PosixRandomAccessFile(fname, fd);
+ *result = new PosixRandomAccessFile(fname, fd, &fd_limit_);
}
return s;
}
@@ -533,10 +568,42 @@ class PosixEnv : public Env {
BGQueue queue_;
PosixLockTable locks_;
- MmapLimiter mmap_limit_;
+ Limiter mmap_limit_;
+ Limiter fd_limit_;
};
-PosixEnv::PosixEnv() : started_bgthread_(false) {
+// Return the maximum number of concurrent mmaps.
+static int MaxMmaps() {
+ if (mmap_limit >= 0) {
+ return mmap_limit;
+ }
+ // Up to 1000 mmaps for 64-bit binaries; none for smaller pointer sizes.
+ mmap_limit = sizeof(void*) >= 8 ? 1000 : 0;
+ return mmap_limit;
+}
+
+// Return the maximum number of read-only files to keep open.
+static intptr_t MaxOpenFiles() {
+ if (open_read_only_file_limit >= 0) {
+ return open_read_only_file_limit;
+ }
+ struct rlimit rlim;
+ if (getrlimit(RLIMIT_NOFILE, &rlim)) {
+ // getrlimit failed, fallback to hard-coded default.
+ open_read_only_file_limit = 50;
+ } else if (rlim.rlim_cur == RLIM_INFINITY) {
+ open_read_only_file_limit = std::numeric_limits<int>::max();
+ } else {
+ // Allow use of 20% of available file descriptors for read-only files.
+ open_read_only_file_limit = rlim.rlim_cur / 5;
+ }
+ return open_read_only_file_limit;
+}
+
+PosixEnv::PosixEnv()
+ : started_bgthread_(false),
+ mmap_limit_(MaxMmaps()),
+ fd_limit_(MaxOpenFiles()) {
PthreadCall("mutex_init", pthread_mutex_init(&mu_, NULL));
PthreadCall("cvar_init", pthread_cond_init(&bgsignal_, NULL));
}
@@ -611,6 +678,16 @@ static pthread_once_t once = PTHREAD_ONCE_INIT;
static Env* default_env;
static void InitDefaultEnv() { default_env = new PosixEnv; }
+void EnvPosixTestHelper::SetReadOnlyFDLimit(int limit) {
+ assert(default_env == NULL);
+ open_read_only_file_limit = limit;
+}
+
+void EnvPosixTestHelper::SetReadOnlyMMapLimit(int limit) {
+ assert(default_env == NULL);
+ mmap_limit = limit;
+}
+
Env* Env::Default() {
pthread_once(&once, InitDefaultEnv);
return default_env;
diff --git a/src/leveldb/util/env_posix_test.cc b/src/leveldb/util/env_posix_test.cc
new file mode 100644
index 0000000000..295f8ae440
--- /dev/null
+++ b/src/leveldb/util/env_posix_test.cc
@@ -0,0 +1,66 @@
+// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file. See the AUTHORS file for names of contributors.
+
+#include "leveldb/env.h"
+
+#include "port/port.h"
+#include "util/testharness.h"
+#include "util/env_posix_test_helper.h"
+
+namespace leveldb {
+
+static const int kDelayMicros = 100000;
+static const int kReadOnlyFileLimit = 4;
+static const int kMMapLimit = 4;
+
+class EnvPosixTest {
+ public:
+ Env* env_;
+ EnvPosixTest() : env_(Env::Default()) { }
+
+ static void SetFileLimits(int read_only_file_limit, int mmap_limit) {
+ EnvPosixTestHelper::SetReadOnlyFDLimit(read_only_file_limit);
+ EnvPosixTestHelper::SetReadOnlyMMapLimit(mmap_limit);
+ }
+};
+
+TEST(EnvPosixTest, TestOpenOnRead) {
+ // Write some test data to a single file that will be opened |n| times.
+ std::string test_dir;
+ ASSERT_OK(env_->GetTestDirectory(&test_dir));
+ std::string test_file = test_dir + "/open_on_read.txt";
+
+ FILE* f = fopen(test_file.c_str(), "w");
+ ASSERT_TRUE(f != NULL);
+ const char kFileData[] = "abcdefghijklmnopqrstuvwxyz";
+ fputs(kFileData, f);
+ fclose(f);
+
+ // Open test file some number above the sum of the two limits to force
+ // open-on-read behavior of POSIX Env leveldb::RandomAccessFile.
+ const int kNumFiles = kReadOnlyFileLimit + kMMapLimit + 5;
+ leveldb::RandomAccessFile* files[kNumFiles] = {0};
+ for (int i = 0; i < kNumFiles; i++) {
+ ASSERT_OK(env_->NewRandomAccessFile(test_file, &files[i]));
+ }
+ char scratch;
+ Slice read_result;
+ for (int i = 0; i < kNumFiles; i++) {
+ ASSERT_OK(files[i]->Read(i, 1, &read_result, &scratch));
+ ASSERT_EQ(kFileData[i], read_result[0]);
+ }
+ for (int i = 0; i < kNumFiles; i++) {
+ delete files[i];
+ }
+ ASSERT_OK(env_->DeleteFile(test_file));
+}
+
+} // namespace leveldb
+
+int main(int argc, char** argv) {
+ // All tests currently run with the same read-only file limits.
+ leveldb::EnvPosixTest::SetFileLimits(leveldb::kReadOnlyFileLimit,
+ leveldb::kMMapLimit);
+ return leveldb::test::RunAllTests();
+}
diff --git a/src/leveldb/util/env_posix_test_helper.h b/src/leveldb/util/env_posix_test_helper.h
new file mode 100644
index 0000000000..0386960598
--- /dev/null
+++ b/src/leveldb/util/env_posix_test_helper.h
@@ -0,0 +1,28 @@
+// Copyright 2017 The LevelDB Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file. See the AUTHORS file for names of contributors.
+
+#ifndef STORAGE_LEVELDB_UTIL_ENV_POSIX_TEST_HELPER_H_
+#define STORAGE_LEVELDB_UTIL_ENV_POSIX_TEST_HELPER_H_
+
+namespace leveldb {
+
+class EnvPosixTest;
+
+// A helper for the POSIX Env to facilitate testing.
+class EnvPosixTestHelper {
+ private:
+ friend class EnvPosixTest;
+
+ // Set the maximum number of read-only files that will be opened.
+ // Must be called before creating an Env.
+ static void SetReadOnlyFDLimit(int limit);
+
+ // Set the maximum number of read-only files that will be mapped via mmap.
+ // Must be called before creating an Env.
+ static void SetReadOnlyMMapLimit(int limit);
+};
+
+} // namespace leveldb
+
+#endif // STORAGE_LEVELDB_UTIL_ENV_POSIX_TEST_HELPER_H_
diff --git a/src/leveldb/util/env_test.cc b/src/leveldb/util/env_test.cc
index b72cb44384..839ae56a1a 100644
--- a/src/leveldb/util/env_test.cc
+++ b/src/leveldb/util/env_test.cc
@@ -10,29 +10,31 @@
namespace leveldb {
static const int kDelayMicros = 100000;
+static const int kReadOnlyFileLimit = 4;
+static const int kMMapLimit = 4;
-class EnvPosixTest {
+class EnvTest {
private:
port::Mutex mu_;
std::string events_;
public:
Env* env_;
- EnvPosixTest() : env_(Env::Default()) { }
+ EnvTest() : env_(Env::Default()) { }
};
static void SetBool(void* ptr) {
reinterpret_cast<port::AtomicPointer*>(ptr)->NoBarrier_Store(ptr);
}
-TEST(EnvPosixTest, RunImmediately) {
+TEST(EnvTest, RunImmediately) {
port::AtomicPointer called (NULL);
env_->Schedule(&SetBool, &called);
- Env::Default()->SleepForMicroseconds(kDelayMicros);
+ env_->SleepForMicroseconds(kDelayMicros);
ASSERT_TRUE(called.NoBarrier_Load() != NULL);
}
-TEST(EnvPosixTest, RunMany) {
+TEST(EnvTest, RunMany) {
port::AtomicPointer last_id (NULL);
struct CB {
@@ -59,7 +61,7 @@ TEST(EnvPosixTest, RunMany) {
env_->Schedule(&CB::Run, &cb3);
env_->Schedule(&CB::Run, &cb4);
- Env::Default()->SleepForMicroseconds(kDelayMicros);
+ env_->SleepForMicroseconds(kDelayMicros);
void* cur = last_id.Acquire_Load();
ASSERT_EQ(4, reinterpret_cast<uintptr_t>(cur));
}
@@ -78,7 +80,7 @@ static void ThreadBody(void* arg) {
s->mu.Unlock();
}
-TEST(EnvPosixTest, StartThread) {
+TEST(EnvTest, StartThread) {
State state;
state.val = 0;
state.num_running = 3;
@@ -92,7 +94,7 @@ TEST(EnvPosixTest, StartThread) {
if (num == 0) {
break;
}
- Env::Default()->SleepForMicroseconds(kDelayMicros);
+ env_->SleepForMicroseconds(kDelayMicros);
}
ASSERT_EQ(state.val, 3);
}
diff --git a/src/leveldb/util/env_win.cc b/src/leveldb/util/env_win.cc
index b074b7579e..d32c4e676c 100644
--- a/src/leveldb/util/env_win.cc
+++ b/src/leveldb/util/env_win.cc
@@ -1,7 +1,7 @@
// This file contains source that originates from:
// http://code.google.com/p/leveldbwin/source/browse/trunk/win32_impl_src/env_win32.h
// http://code.google.com/p/leveldbwin/source/browse/trunk/win32_impl_src/port_win32.cc
-// Those files dont' have any explict license headers but the
+// Those files don't have any explicit license headers but the
// project (http://code.google.com/p/leveldbwin/) lists the 'New BSD License'
// as the license.
#if defined(LEVELDB_PLATFORM_WINDOWS)
@@ -355,11 +355,13 @@ BOOL Win32SequentialFile::_Init()
ToWidePath(_filename, path);
_hFile = CreateFileW(path.c_str(),
GENERIC_READ,
- FILE_SHARE_READ,
+ FILE_SHARE_READ | FILE_SHARE_WRITE,
NULL,
OPEN_EXISTING,
- FILE_ATTRIBUTE_NORMAL,
+ FILE_ATTRIBUTE_NORMAL | FILE_FLAG_SEQUENTIAL_SCAN,
NULL);
+ if (_hFile == INVALID_HANDLE_VALUE)
+ _hFile = NULL;
return _hFile ? TRUE : FALSE;
}
@@ -403,7 +405,7 @@ BOOL Win32RandomAccessFile::_Init( LPCWSTR path )
{
BOOL bRet = FALSE;
if(!_hFile)
- _hFile = ::CreateFileW(path,GENERIC_READ,FILE_SHARE_READ,NULL,OPEN_EXISTING,
+ _hFile = ::CreateFileW(path,GENERIC_READ,FILE_SHARE_READ|FILE_SHARE_WRITE,NULL,OPEN_EXISTING,
FILE_ATTRIBUTE_NORMAL | FILE_FLAG_RANDOM_ACCESS,NULL);
if(!_hFile || _hFile == INVALID_HANDLE_VALUE )
_hFile = NULL;
@@ -669,7 +671,7 @@ Status Win32Env::GetFileSize( const std::string& fname, uint64_t* file_size )
ToWidePath(ModifyPath(path), wpath);
HANDLE file = ::CreateFileW(wpath.c_str(),
- GENERIC_READ,FILE_SHARE_READ,NULL,OPEN_EXISTING,FILE_ATTRIBUTE_NORMAL,NULL);
+ GENERIC_READ,FILE_SHARE_READ|FILE_SHARE_WRITE,NULL,OPEN_EXISTING,FILE_ATTRIBUTE_NORMAL,NULL);
LARGE_INTEGER li;
if(::GetFileSizeEx(file,&li)){
*file_size = (uint64_t)li.QuadPart;
diff --git a/src/leveldb/util/options.cc b/src/leveldb/util/options.cc
index 8b618fb1ae..b5e6227613 100644
--- a/src/leveldb/util/options.cc
+++ b/src/leveldb/util/options.cc
@@ -21,6 +21,7 @@ Options::Options()
block_cache(NULL),
block_size(4096),
block_restart_interval(16),
+ max_file_size(2<<20),
compression(kSnappyCompression),
reuse_logs(false),
filter_policy(NULL) {
generated by cgit v1.2.3 (git 2.26.2) at 2024-10-30 22:19:06 +0000