Merge #17398: build: Update leveldb to 1.22+

677fb8e92380d4deb6a3753047c01f7cf7b5af91 test: Add ubsan surpression for crc32c (Wladimir J. van der Laan)
8e68bb1ddeca504bedd40aee8492b5478a88c1e5 build: Disable msvc warning 4722 for leveldb build (Aaron Clauson)
be23949765e1b2e050574c6c2a136658a89dee5d build: MSVC changes for leveldb update (Aaron Clauson)
9ebdf047578f0da7e6578d0c51c32f55e84ac157 build: CRC32C build system integration (Wladimir J. van der Laan)
402252a8081e25f22aa1a5c60708714cf1d84ec4 build: Add LCOV exception for crc32c (Wladimir J. van der Laan)
3a037d0067c2c12a1c2c800fb85613a0a2911253 test: Add crc32c exception to various linters and generation scripts (Wladimir J. van der Laan)
84ff1b2076ef91ce688930d0aa0a7f4078ef3e1d test: Add crc32c to subtree check linter (Wladimir J. van der Laan)
7cf13a513409c18d18dff2f6203b3630937b487d doc: Add crc32c subtree to developer notes (Wladimir J. van der Laan)
24d02a9ac00a82d172b171f73554a882df264c80 build: Update build system for new leveldb (Wladimir J. van der Laan)
2e1819311a59fb5cb26e3ca50a510bfe01358350 Squashed 'src/crc32c/' content from commit 224988680f7673cd7c769963d4035cb315aa3388 (Wladimir J. van der Laan)
66480821b36c839ab7615cb9309850015bceadb0 Squashed 'src/leveldb/' changes from f545dfabff4c2e9836efed094dba99a34fbc6b88..f8ae182c1e5176d12e816fb2217ae33a5472fdd7 (Wladimir J. van der Laan)

Pull request description:

  This updates leveldb to currently newest upstream commit https://github.com/bitcoin-core/leveldb/commit/0c40829872a9f00f38e11dc370ff8adb3e19f25b:

  - CRC32C hardware acceleration is now an external library [crc32c](https://github.com/google/crc32c). This adds acceleration on ARM, and should be faster on x86 because of using prefetch. It also makes it easy to support similar instruction sets on other platforms in the future.
  - Thread handling uses C++11, instead of platform specific code.
  - Native windows environment was added. No need to maintain our own hacky one, anymore.
  - Upstream now builds using CMake. This doesn't mean we need to use that (phew), but internal configuration changed to a a series of checks, instead of OS profiles. This means the blanket error "Cannot build leveldb for $host. Please file a bug report' is removed.

  All changes: https://github.com/google/leveldb/compare/a53934a3ae1244679f812d998a4f16f2c7f309a6...0c40829872a9f00f38e11dc370ff8adb3e19f25b

  Pretty much all our changes have been subsumed by upstream, so we figured it was cleaner to start over with a new branch from upstream with the still-relevant patches applied: https://github.com/bitcoin-core/leveldb/tree/bitcoin-fork-new

  There's quite some testing to be done (see below). See https://github.com/bitcoin-core/leveldb/issues/25 and https://github.com/bitcoin-core/leveldb/pull/26 for more history and context.

  TODO:
  - [x] Subtree `crc32c`
  - [x] Make linters happy about crc32 subtree
  - [x] Integrate `crc32c` library into build system
  - [x] MSVC build system

ACKs for top commit:
  sipa:
    ACK 677fb8e92380d4deb6a3753047c01f7cf7b5af91

Tree-SHA512: 37ee92a750e053e924bc4626b12bb3fd81faa9f8c5ebaa343931fee810c45ba05aa6051fdea82535fa351bf2be7297801b98af9469865fc5ead771650a5d6240

1 files changed, 183 insertions, 52 deletions

diff --git a/src/leveldb/util/env_test.cc b/src/leveldb/util/env_test.cc
index 839ae56a1a..7db03fc11c 100644
--- a/src/leveldb/util/env_test.cc
+++ b/src/leveldb/util/env_test.cc
@@ -4,72 +4,144 @@
 
 #include "leveldb/env.h"
 
+#include <algorithm>
+
 #include "port/port.h"
+#include "port/thread_annotations.h"
+#include "util/mutexlock.h"
 #include "util/testharness.h"
+#include "util/testutil.h"
 
 namespace leveldb {
 
 static const int kDelayMicros = 100000;
-static const int kReadOnlyFileLimit = 4;
-static const int kMMapLimit = 4;
 
 class EnvTest {
- private:
-  port::Mutex mu_;
-  std::string events_;
-
  public:
+  EnvTest() : env_(Env::Default()) {}
+
   Env* env_;
-  EnvTest() : env_(Env::Default()) { }
 };
 
-static void SetBool(void* ptr) {
-  reinterpret_cast<port::AtomicPointer*>(ptr)->NoBarrier_Store(ptr);
+TEST(EnvTest, ReadWrite) {
+  Random rnd(test::RandomSeed());
+
+  // Get file to use for testing.
+  std::string test_dir;
+  ASSERT_OK(env_->GetTestDirectory(&test_dir));
+  std::string test_file_name = test_dir + "/open_on_read.txt";
+  WritableFile* writable_file;
+  ASSERT_OK(env_->NewWritableFile(test_file_name, &writable_file));
+
+  // Fill a file with data generated via a sequence of randomly sized writes.
+  static const size_t kDataSize = 10 * 1048576;
+  std::string data;
+  while (data.size() < kDataSize) {
+    int len = rnd.Skewed(18);  // Up to 2^18 - 1, but typically much smaller
+    std::string r;
+    test::RandomString(&rnd, len, &r);
+    ASSERT_OK(writable_file->Append(r));
+    data += r;
+    if (rnd.OneIn(10)) {
+      ASSERT_OK(writable_file->Flush());
+    }
+  }
+  ASSERT_OK(writable_file->Sync());
+  ASSERT_OK(writable_file->Close());
+  delete writable_file;
+
+  // Read all data using a sequence of randomly sized reads.
+  SequentialFile* sequential_file;
+  ASSERT_OK(env_->NewSequentialFile(test_file_name, &sequential_file));
+  std::string read_result;
+  std::string scratch;
+  while (read_result.size() < data.size()) {
+    int len = std::min<int>(rnd.Skewed(18), data.size() - read_result.size());
+    scratch.resize(std::max(len, 1));  // at least 1 so &scratch[0] is legal
+    Slice read;
+    ASSERT_OK(sequential_file->Read(len, &read, &scratch[0]));
+    if (len > 0) {
+      ASSERT_GT(read.size(), 0);
+    }
+    ASSERT_LE(read.size(), len);
+    read_result.append(read.data(), read.size());
+  }
+  ASSERT_EQ(read_result, data);
+  delete sequential_file;
 }
 
 TEST(EnvTest, RunImmediately) {
-  port::AtomicPointer called (NULL);
-  env_->Schedule(&SetBool, &called);
-  env_->SleepForMicroseconds(kDelayMicros);
-  ASSERT_TRUE(called.NoBarrier_Load() != NULL);
+  struct RunState {
+    port::Mutex mu;
+    port::CondVar cvar{&mu};
+    bool called = false;
+
+    static void Run(void* arg) {
+      RunState* state = reinterpret_cast<RunState*>(arg);
+      MutexLock l(&state->mu);
+      ASSERT_EQ(state->called, false);
+      state->called = true;
+      state->cvar.Signal();
+    }
+  };
+
+  RunState state;
+  env_->Schedule(&RunState::Run, &state);
+
+  MutexLock l(&state.mu);
+  while (!state.called) {
+    state.cvar.Wait();
+  }
 }
 
 TEST(EnvTest, RunMany) {
-  port::AtomicPointer last_id (NULL);
+  struct RunState {
+    port::Mutex mu;
+    port::CondVar cvar{&mu};
+    int last_id = 0;
+  };
+
+  struct Callback {
+    RunState* state_;  // Pointer to shared state.
+    const int id_;  // Order# for the execution of this callback.
 
-  struct CB {
-    port::AtomicPointer* last_id_ptr;   // Pointer to shared slot
-    uintptr_t id;             // Order# for the execution of this callback
+    Callback(RunState* s, int id) : state_(s), id_(id) {}
 
-    CB(port::AtomicPointer* p, int i) : last_id_ptr(p), id(i) { }
+    static void Run(void* arg) {
+      Callback* callback = reinterpret_cast<Callback*>(arg);
+      RunState* state = callback->state_;
 
-    static void Run(void* v) {
-      CB* cb = reinterpret_cast<CB*>(v);
-      void* cur = cb->last_id_ptr->NoBarrier_Load();
-      ASSERT_EQ(cb->id-1, reinterpret_cast<uintptr_t>(cur));
-      cb->last_id_ptr->Release_Store(reinterpret_cast<void*>(cb->id));
+      MutexLock l(&state->mu);
+      ASSERT_EQ(state->last_id, callback->id_ - 1);
+      state->last_id = callback->id_;
+      state->cvar.Signal();
     }
   };
 
-  // Schedule in different order than start time
-  CB cb1(&last_id, 1);
-  CB cb2(&last_id, 2);
-  CB cb3(&last_id, 3);
-  CB cb4(&last_id, 4);
-  env_->Schedule(&CB::Run, &cb1);
-  env_->Schedule(&CB::Run, &cb2);
-  env_->Schedule(&CB::Run, &cb3);
-  env_->Schedule(&CB::Run, &cb4);
-
-  env_->SleepForMicroseconds(kDelayMicros);
-  void* cur = last_id.Acquire_Load();
-  ASSERT_EQ(4, reinterpret_cast<uintptr_t>(cur));
+  RunState state;
+  Callback callback1(&state, 1);
+  Callback callback2(&state, 2);
+  Callback callback3(&state, 3);
+  Callback callback4(&state, 4);
+  env_->Schedule(&Callback::Run, &callback1);
+  env_->Schedule(&Callback::Run, &callback2);
+  env_->Schedule(&Callback::Run, &callback3);
+  env_->Schedule(&Callback::Run, &callback4);
+
+  MutexLock l(&state.mu);
+  while (state.last_id != 4) {
+    state.cvar.Wait();
+  }
 }
 
 struct State {
   port::Mutex mu;
-  int val;
-  int num_running;
+  port::CondVar cvar{&mu};
+
+  int val GUARDED_BY(mu);
+  int num_running GUARDED_BY(mu);
+
+  State(int val, int num_running) : val(val), num_running(num_running) {}
 };
 
 static void ThreadBody(void* arg) {
@@ -77,30 +149,89 @@ static void ThreadBody(void* arg) {
   s->mu.Lock();
   s->val += 1;
   s->num_running -= 1;
+  s->cvar.Signal();
   s->mu.Unlock();
 }
 
 TEST(EnvTest, StartThread) {
-  State state;
-  state.val = 0;
-  state.num_running = 3;
+  State state(0, 3);
   for (int i = 0; i < 3; i++) {
     env_->StartThread(&ThreadBody, &state);
   }
-  while (true) {
-    state.mu.Lock();
-    int num = state.num_running;
-    state.mu.Unlock();
-    if (num == 0) {
-      break;
-    }
-    env_->SleepForMicroseconds(kDelayMicros);
+
+  MutexLock l(&state.mu);
+  while (state.num_running != 0) {
+    state.cvar.Wait();
   }
   ASSERT_EQ(state.val, 3);
 }
 
-}  // namespace leveldb
+TEST(EnvTest, TestOpenNonExistentFile) {
+  // Write some test data to a single file that will be opened |n| times.
+  std::string test_dir;
+  ASSERT_OK(env_->GetTestDirectory(&test_dir));
+
+  std::string non_existent_file = test_dir + "/non_existent_file";
+  ASSERT_TRUE(!env_->FileExists(non_existent_file));
+
+  RandomAccessFile* random_access_file;
+  Status status =
+      env_->NewRandomAccessFile(non_existent_file, &random_access_file);
+  ASSERT_TRUE(status.IsNotFound());
 
-int main(int argc, char** argv) {
-  return leveldb::test::RunAllTests();
+  SequentialFile* sequential_file;
+  status = env_->NewSequentialFile(non_existent_file, &sequential_file);
+  ASSERT_TRUE(status.IsNotFound());
 }
+
+TEST(EnvTest, ReopenWritableFile) {
+  std::string test_dir;
+  ASSERT_OK(env_->GetTestDirectory(&test_dir));
+  std::string test_file_name = test_dir + "/reopen_writable_file.txt";
+  env_->DeleteFile(test_file_name);
+
+  WritableFile* writable_file;
+  ASSERT_OK(env_->NewWritableFile(test_file_name, &writable_file));
+  std::string data("hello world!");
+  ASSERT_OK(writable_file->Append(data));
+  ASSERT_OK(writable_file->Close());
+  delete writable_file;
+
+  ASSERT_OK(env_->NewWritableFile(test_file_name, &writable_file));
+  data = "42";
+  ASSERT_OK(writable_file->Append(data));
+  ASSERT_OK(writable_file->Close());
+  delete writable_file;
+
+  ASSERT_OK(ReadFileToString(env_, test_file_name, &data));
+  ASSERT_EQ(std::string("42"), data);
+  env_->DeleteFile(test_file_name);
+}
+
+TEST(EnvTest, ReopenAppendableFile) {
+  std::string test_dir;
+  ASSERT_OK(env_->GetTestDirectory(&test_dir));
+  std::string test_file_name = test_dir + "/reopen_appendable_file.txt";
+  env_->DeleteFile(test_file_name);
+
+  WritableFile* appendable_file;
+  ASSERT_OK(env_->NewAppendableFile(test_file_name, &appendable_file));
+  std::string data("hello world!");
+  ASSERT_OK(appendable_file->Append(data));
+  ASSERT_OK(appendable_file->Close());
+  delete appendable_file;
+
+  ASSERT_OK(env_->NewAppendableFile(test_file_name, &appendable_file));
+  data = "42";
+  ASSERT_OK(appendable_file->Append(data));
+  ASSERT_OK(appendable_file->Close());
+  delete appendable_file;
+
+  ASSERT_OK(ReadFileToString(env_, test_file_name, &data));
+  ASSERT_EQ(std::string("hello world!42"), data);
+  env_->DeleteFile(test_file_name);
+}
+
+}  // namespace leveldb
+
+int main(int argc, char** argv) { return leveldb::test::RunAllTests(); }