summaryrefslogtreecommitdiffstats
path: root/src/rocksdb/tools/ldb_test.py
diff options
context:
space:
mode:
Diffstat (limited to '')
-rw-r--r--src/rocksdb/tools/ldb_test.py955
1 files changed, 955 insertions, 0 deletions
diff --git a/src/rocksdb/tools/ldb_test.py b/src/rocksdb/tools/ldb_test.py
new file mode 100644
index 000000000..e243d69c0
--- /dev/null
+++ b/src/rocksdb/tools/ldb_test.py
@@ -0,0 +1,955 @@
+#!/usr/bin/env python3
+# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
+from __future__ import absolute_import, division, print_function, unicode_literals
+
+import glob
+
+import os
+import os.path
+import re
+import shutil
+import subprocess
+import tempfile
+import time
+import unittest
+
+
+def my_check_output(*popenargs, **kwargs):
+ """
+ If we had python 2.7, we should simply use subprocess.check_output.
+ This is a stop-gap solution for python 2.6
+ """
+ if "stdout" in kwargs:
+ raise ValueError("stdout argument not allowed, it will be overridden.")
+ process = subprocess.Popen(
+ stderr=subprocess.PIPE, stdout=subprocess.PIPE, *popenargs, **kwargs
+ )
+ output, unused_err = process.communicate()
+ retcode = process.poll()
+ if retcode:
+ cmd = kwargs.get("args")
+ if cmd is None:
+ cmd = popenargs[0]
+ raise Exception("Exit code is not 0. It is %d. Command: %s" % (retcode, cmd))
+ return output.decode("utf-8")
+
+
+def run_err_null(cmd):
+ return os.system(cmd + " 2>/dev/null ")
+
+
+class LDBTestCase(unittest.TestCase):
+ def setUp(self):
+ self.TMP_DIR = tempfile.mkdtemp(prefix="ldb_test_")
+ self.DB_NAME = "testdb"
+
+ def tearDown(self):
+ assert (
+ self.TMP_DIR.strip() != "/"
+ and self.TMP_DIR.strip() != "/tmp"
+ and self.TMP_DIR.strip() != "/tmp/"
+ ) # Just some paranoia
+
+ shutil.rmtree(self.TMP_DIR)
+
+ def dbParam(self, dbName):
+ return "--db=%s" % os.path.join(self.TMP_DIR, dbName)
+
+ def assertRunOKFull(
+ self, params, expectedOutput, unexpected=False, isPattern=False
+ ):
+ """
+ All command-line params must be specified.
+ Allows full flexibility in testing; for example: missing db param.
+ """
+ output = my_check_output(
+ './ldb %s |grep -v "Created bg thread"' % params, shell=True
+ )
+ if not unexpected:
+ if isPattern:
+ self.assertNotEqual(expectedOutput.search(output.strip()), None)
+ else:
+ self.assertEqual(output.strip(), expectedOutput.strip())
+ else:
+ if isPattern:
+ self.assertEqual(expectedOutput.search(output.strip()), None)
+ else:
+ self.assertNotEqual(output.strip(), expectedOutput.strip())
+
+ def assertRunFAILFull(self, params):
+ """
+ All command-line params must be specified.
+ Allows full flexibility in testing; for example: missing db param.
+ """
+ try:
+
+ my_check_output(
+ './ldb %s >/dev/null 2>&1 |grep -v "Created bg \
+ thread"'
+ % params,
+ shell=True,
+ )
+ except Exception:
+ return
+ self.fail(
+ "Exception should have been raised for command with params: %s" % params
+ )
+
+ def assertRunOK(self, params, expectedOutput, unexpected=False):
+ """
+ Uses the default test db.
+ """
+ self.assertRunOKFull(
+ "%s %s" % (self.dbParam(self.DB_NAME), params), expectedOutput, unexpected
+ )
+
+ def assertRunFAIL(self, params):
+ """
+ Uses the default test db.
+ """
+ self.assertRunFAILFull("%s %s" % (self.dbParam(self.DB_NAME), params))
+
+ def testSimpleStringPutGet(self):
+ print("Running testSimpleStringPutGet...")
+ self.assertRunFAIL("put x1 y1")
+ self.assertRunOK("put --create_if_missing x1 y1", "OK")
+ self.assertRunOK("get x1", "y1")
+ self.assertRunFAIL("get x2")
+
+ self.assertRunOK("put x2 y2", "OK")
+ self.assertRunOK("get x1", "y1")
+ self.assertRunOK("get x2", "y2")
+ self.assertRunFAIL("get x3")
+
+ self.assertRunOK("scan --from=x1 --to=z", "x1 : y1\nx2 : y2")
+ self.assertRunOK("put x3 y3", "OK")
+
+ self.assertRunOK("scan --from=x1 --to=z", "x1 : y1\nx2 : y2\nx3 : y3")
+ self.assertRunOK("scan", "x1 : y1\nx2 : y2\nx3 : y3")
+ self.assertRunOK("scan --from=x", "x1 : y1\nx2 : y2\nx3 : y3")
+
+ self.assertRunOK("scan --to=x2", "x1 : y1")
+ self.assertRunOK("scan --from=x1 --to=z --max_keys=1", "x1 : y1")
+ self.assertRunOK("scan --from=x1 --to=z --max_keys=2", "x1 : y1\nx2 : y2")
+
+ self.assertRunOK(
+ "scan --from=x1 --to=z --max_keys=3", "x1 : y1\nx2 : y2\nx3 : y3"
+ )
+ self.assertRunOK(
+ "scan --from=x1 --to=z --max_keys=4", "x1 : y1\nx2 : y2\nx3 : y3"
+ )
+ self.assertRunOK("scan --from=x1 --to=x2", "x1 : y1")
+ self.assertRunOK("scan --from=x2 --to=x4", "x2 : y2\nx3 : y3")
+ self.assertRunFAIL("scan --from=x4 --to=z") # No results => FAIL
+ self.assertRunFAIL("scan --from=x1 --to=z --max_keys=foo")
+
+ self.assertRunOK("scan", "x1 : y1\nx2 : y2\nx3 : y3")
+
+ self.assertRunOK("delete x1", "OK")
+ self.assertRunOK("scan", "x2 : y2\nx3 : y3")
+
+ self.assertRunOK("delete NonExistentKey", "OK")
+ # It is weird that GET and SCAN raise exception for
+ # non-existent key, while delete does not
+
+ self.assertRunOK("checkconsistency", "OK")
+
+ def dumpDb(self, params, dumpFile):
+ return 0 == run_err_null("./ldb dump %s > %s" % (params, dumpFile))
+
+ def loadDb(self, params, dumpFile):
+ return 0 == run_err_null("cat %s | ./ldb load %s" % (dumpFile, params))
+
+ def writeExternSst(self, params, inputDumpFile, outputSst):
+ return 0 == run_err_null(
+ "cat %s | ./ldb write_extern_sst %s %s" % (inputDumpFile, outputSst, params)
+ )
+
+ def ingestExternSst(self, params, inputSst):
+ return 0 == run_err_null("./ldb ingest_extern_sst %s %s" % (inputSst, params))
+
+ def testStringBatchPut(self):
+ print("Running testStringBatchPut...")
+ self.assertRunOK("batchput x1 y1 --create_if_missing", "OK")
+ self.assertRunOK("scan", "x1 : y1")
+ self.assertRunOK('batchput x2 y2 x3 y3 "x4 abc" "y4 xyz"', "OK")
+ self.assertRunOK("scan", "x1 : y1\nx2 : y2\nx3 : y3\nx4 abc : y4 xyz")
+ self.assertRunFAIL("batchput")
+ self.assertRunFAIL("batchput k1")
+ self.assertRunFAIL("batchput k1 v1 k2")
+
+ def testBlobBatchPut(self):
+ print("Running testBlobBatchPut...")
+
+ dbPath = os.path.join(self.TMP_DIR, self.DB_NAME)
+ self.assertRunOK("batchput x1 y1 --create_if_missing --enable_blob_files", "OK")
+ self.assertRunOK("scan", "x1 : y1")
+ self.assertRunOK(
+ 'batchput --enable_blob_files x2 y2 x3 y3 "x4 abc" "y4 xyz"', "OK"
+ )
+ self.assertRunOK("scan", "x1 : y1\nx2 : y2\nx3 : y3\nx4 abc : y4 xyz")
+
+ blob_files = self.getBlobFiles(dbPath)
+ self.assertTrue(len(blob_files) >= 1)
+
+ def testBlobPut(self):
+ print("Running testBlobPut...")
+
+ dbPath = os.path.join(self.TMP_DIR, self.DB_NAME)
+ self.assertRunOK("put --create_if_missing --enable_blob_files x1 y1", "OK")
+ self.assertRunOK("get x1", "y1")
+ self.assertRunOK("put --enable_blob_files x2 y2", "OK")
+ self.assertRunOK("get x1", "y1")
+ self.assertRunOK("get x2", "y2")
+ self.assertRunFAIL("get x3")
+
+ blob_files = self.getBlobFiles(dbPath)
+ self.assertTrue(len(blob_files) >= 1)
+
+ def testBlobStartingLevel(self):
+ print("Running testBlobStartingLevel...")
+
+ dbPath = os.path.join(self.TMP_DIR, self.DB_NAME)
+ self.assertRunOK(
+ "put --create_if_missing --enable_blob_files --blob_file_starting_level=10 x1 y1",
+ "OK",
+ )
+ self.assertRunOK("get x1", "y1")
+
+ blob_files = self.getBlobFiles(dbPath)
+ self.assertTrue(len(blob_files) == 0)
+
+ self.assertRunOK(
+ "put --enable_blob_files --blob_file_starting_level=0 x2 y2", "OK"
+ )
+ self.assertRunOK("get x1", "y1")
+ self.assertRunOK("get x2", "y2")
+ self.assertRunFAIL("get x3")
+
+ blob_files = self.getBlobFiles(dbPath)
+ self.assertTrue(len(blob_files) >= 1)
+
+ def testCountDelimDump(self):
+ print("Running testCountDelimDump...")
+ self.assertRunOK("batchput x.1 x1 --create_if_missing", "OK")
+ self.assertRunOK("batchput y.abc abc y.2 2 z.13c pqr", "OK")
+ self.assertRunOK(
+ "dump --count_delim",
+ "x => count:1\tsize:5\ny => count:2\tsize:12\nz => count:1\tsize:8",
+ )
+ self.assertRunOK(
+ 'dump --count_delim="."',
+ "x => count:1\tsize:5\ny => count:2\tsize:12\nz => count:1\tsize:8",
+ )
+ self.assertRunOK("batchput x,2 x2 x,abc xabc", "OK")
+ self.assertRunOK(
+ 'dump --count_delim=","',
+ "x => count:2\tsize:14\nx.1 => count:1\tsize:5\ny.2 => count:1\tsize:4\ny.abc => count:1\tsize:8\nz.13c => count:1\tsize:8",
+ )
+
+ def testCountDelimIDump(self):
+ print("Running testCountDelimIDump...")
+ self.assertRunOK("batchput x.1 x1 --create_if_missing", "OK")
+ self.assertRunOK("batchput y.abc abc y.2 2 z.13c pqr", "OK")
+ self.assertRunOK(
+ "idump --count_delim",
+ "x => count:1\tsize:5\ny => count:2\tsize:12\nz => count:1\tsize:8",
+ )
+ self.assertRunOK(
+ 'idump --count_delim="."',
+ "x => count:1\tsize:5\ny => count:2\tsize:12\nz => count:1\tsize:8",
+ )
+ self.assertRunOK("batchput x,2 x2 x,abc xabc", "OK")
+ self.assertRunOK(
+ 'idump --count_delim=","',
+ "x => count:2\tsize:14\nx.1 => count:1\tsize:5\ny.2 => count:1\tsize:4\ny.abc => count:1\tsize:8\nz.13c => count:1\tsize:8",
+ )
+
+ def testInvalidCmdLines(self):
+ print("Running testInvalidCmdLines...")
+ # db not specified
+ self.assertRunFAILFull("put 0x6133 0x6233 --hex --create_if_missing")
+ # No param called he
+ self.assertRunFAIL("put 0x6133 0x6233 --he --create_if_missing")
+ # max_keys is not applicable for put
+ self.assertRunFAIL("put 0x6133 0x6233 --max_keys=1 --create_if_missing")
+ # hex has invalid boolean value
+
+ def testHexPutGet(self):
+ print("Running testHexPutGet...")
+ self.assertRunOK("put a1 b1 --create_if_missing", "OK")
+ self.assertRunOK("scan", "a1 : b1")
+ self.assertRunOK("scan --hex", "0x6131 : 0x6231")
+ self.assertRunFAIL("put --hex 6132 6232")
+ self.assertRunOK("put --hex 0x6132 0x6232", "OK")
+ self.assertRunOK("scan --hex", "0x6131 : 0x6231\n0x6132 : 0x6232")
+ self.assertRunOK("scan", "a1 : b1\na2 : b2")
+ self.assertRunOK("get a1", "b1")
+ self.assertRunOK("get --hex 0x6131", "0x6231")
+ self.assertRunOK("get a2", "b2")
+ self.assertRunOK("get --hex 0x6132", "0x6232")
+ self.assertRunOK("get --key_hex 0x6132", "b2")
+ self.assertRunOK("get --key_hex --value_hex 0x6132", "0x6232")
+ self.assertRunOK("get --value_hex a2", "0x6232")
+ self.assertRunOK(
+ "scan --key_hex --value_hex", "0x6131 : 0x6231\n0x6132 : 0x6232"
+ )
+ self.assertRunOK(
+ "scan --hex --from=0x6131 --to=0x6133", "0x6131 : 0x6231\n0x6132 : 0x6232"
+ )
+ self.assertRunOK("scan --hex --from=0x6131 --to=0x6132", "0x6131 : 0x6231")
+ self.assertRunOK("scan --key_hex", "0x6131 : b1\n0x6132 : b2")
+ self.assertRunOK("scan --value_hex", "a1 : 0x6231\na2 : 0x6232")
+ self.assertRunOK("batchput --hex 0x6133 0x6233 0x6134 0x6234", "OK")
+ self.assertRunOK("scan", "a1 : b1\na2 : b2\na3 : b3\na4 : b4")
+ self.assertRunOK("delete --hex 0x6133", "OK")
+ self.assertRunOK("scan", "a1 : b1\na2 : b2\na4 : b4")
+ self.assertRunOK("checkconsistency", "OK")
+
+ def testTtlPutGet(self):
+ print("Running testTtlPutGet...")
+ self.assertRunOK("put a1 b1 --ttl --create_if_missing", "OK")
+ self.assertRunOK("scan --hex", "0x6131 : 0x6231", True)
+ self.assertRunOK("dump --ttl ", "a1 ==> b1", True)
+ self.assertRunOK("dump --hex --ttl ", "0x6131 ==> 0x6231\nKeys in range: 1")
+ self.assertRunOK("scan --hex --ttl", "0x6131 : 0x6231")
+ self.assertRunOK("get --value_hex a1", "0x6231", True)
+ self.assertRunOK("get --ttl a1", "b1")
+ self.assertRunOK("put a3 b3 --create_if_missing", "OK")
+ # fails because timstamp's length is greater than value's
+ self.assertRunFAIL("get --ttl a3")
+ self.assertRunOK("checkconsistency", "OK")
+
+ def testInvalidCmdLines(self): # noqa: F811 T25377293 Grandfathered in
+ print("Running testInvalidCmdLines...")
+ # db not specified
+ self.assertRunFAILFull("put 0x6133 0x6233 --hex --create_if_missing")
+ # No param called he
+ self.assertRunFAIL("put 0x6133 0x6233 --he --create_if_missing")
+ # max_keys is not applicable for put
+ self.assertRunFAIL("put 0x6133 0x6233 --max_keys=1 --create_if_missing")
+ # hex has invalid boolean value
+ self.assertRunFAIL("put 0x6133 0x6233 --hex=Boo --create_if_missing")
+
+ def testDumpLoad(self):
+ print("Running testDumpLoad...")
+ self.assertRunOK("batchput --create_if_missing x1 y1 x2 y2 x3 y3 x4 y4", "OK")
+ self.assertRunOK("scan", "x1 : y1\nx2 : y2\nx3 : y3\nx4 : y4")
+ origDbPath = os.path.join(self.TMP_DIR, self.DB_NAME)
+
+ # Dump and load without any additional params specified
+ dumpFilePath = os.path.join(self.TMP_DIR, "dump1")
+ loadedDbPath = os.path.join(self.TMP_DIR, "loaded_from_dump1")
+ self.assertTrue(self.dumpDb("--db=%s" % origDbPath, dumpFilePath))
+ self.assertTrue(
+ self.loadDb("--db=%s --create_if_missing" % loadedDbPath, dumpFilePath)
+ )
+ self.assertRunOKFull(
+ "scan --db=%s" % loadedDbPath, "x1 : y1\nx2 : y2\nx3 : y3\nx4 : y4"
+ )
+
+ # Dump and load in hex
+ dumpFilePath = os.path.join(self.TMP_DIR, "dump2")
+ loadedDbPath = os.path.join(self.TMP_DIR, "loaded_from_dump2")
+ self.assertTrue(self.dumpDb("--db=%s --hex" % origDbPath, dumpFilePath))
+ self.assertTrue(
+ self.loadDb(
+ "--db=%s --hex --create_if_missing" % loadedDbPath, dumpFilePath
+ )
+ )
+ self.assertRunOKFull(
+ "scan --db=%s" % loadedDbPath, "x1 : y1\nx2 : y2\nx3 : y3\nx4 : y4"
+ )
+
+ # Dump only a portion of the key range
+ dumpFilePath = os.path.join(self.TMP_DIR, "dump3")
+ loadedDbPath = os.path.join(self.TMP_DIR, "loaded_from_dump3")
+ self.assertTrue(
+ self.dumpDb("--db=%s --from=x1 --to=x3" % origDbPath, dumpFilePath)
+ )
+ self.assertTrue(
+ self.loadDb("--db=%s --create_if_missing" % loadedDbPath, dumpFilePath)
+ )
+ self.assertRunOKFull("scan --db=%s" % loadedDbPath, "x1 : y1\nx2 : y2")
+
+ # Dump upto max_keys rows
+ dumpFilePath = os.path.join(self.TMP_DIR, "dump4")
+ loadedDbPath = os.path.join(self.TMP_DIR, "loaded_from_dump4")
+ self.assertTrue(self.dumpDb("--db=%s --max_keys=3" % origDbPath, dumpFilePath))
+ self.assertTrue(
+ self.loadDb("--db=%s --create_if_missing" % loadedDbPath, dumpFilePath)
+ )
+ self.assertRunOKFull("scan --db=%s" % loadedDbPath, "x1 : y1\nx2 : y2\nx3 : y3")
+
+ # Load into an existing db, create_if_missing is not specified
+ self.assertTrue(self.dumpDb("--db=%s" % origDbPath, dumpFilePath))
+ self.assertTrue(self.loadDb("--db=%s" % loadedDbPath, dumpFilePath))
+ self.assertRunOKFull(
+ "scan --db=%s" % loadedDbPath, "x1 : y1\nx2 : y2\nx3 : y3\nx4 : y4"
+ )
+
+ # Dump and load with WAL disabled
+ dumpFilePath = os.path.join(self.TMP_DIR, "dump5")
+ loadedDbPath = os.path.join(self.TMP_DIR, "loaded_from_dump5")
+ self.assertTrue(self.dumpDb("--db=%s" % origDbPath, dumpFilePath))
+ self.assertTrue(
+ self.loadDb(
+ "--db=%s --disable_wal --create_if_missing" % loadedDbPath, dumpFilePath
+ )
+ )
+ self.assertRunOKFull(
+ "scan --db=%s" % loadedDbPath, "x1 : y1\nx2 : y2\nx3 : y3\nx4 : y4"
+ )
+
+ # Dump and load with lots of extra params specified
+ extraParams = " ".join(
+ [
+ "--bloom_bits=14",
+ "--block_size=1024",
+ "--auto_compaction=true",
+ "--write_buffer_size=4194304",
+ "--file_size=2097152",
+ ]
+ )
+ dumpFilePath = os.path.join(self.TMP_DIR, "dump6")
+ loadedDbPath = os.path.join(self.TMP_DIR, "loaded_from_dump6")
+ self.assertTrue(
+ self.dumpDb("--db=%s %s" % (origDbPath, extraParams), dumpFilePath)
+ )
+ self.assertTrue(
+ self.loadDb(
+ "--db=%s %s --create_if_missing" % (loadedDbPath, extraParams),
+ dumpFilePath,
+ )
+ )
+ self.assertRunOKFull(
+ "scan --db=%s" % loadedDbPath, "x1 : y1\nx2 : y2\nx3 : y3\nx4 : y4"
+ )
+
+ # Dump with count_only
+ dumpFilePath = os.path.join(self.TMP_DIR, "dump7")
+ loadedDbPath = os.path.join(self.TMP_DIR, "loaded_from_dump7")
+ self.assertTrue(self.dumpDb("--db=%s --count_only" % origDbPath, dumpFilePath))
+ self.assertTrue(
+ self.loadDb("--db=%s --create_if_missing" % loadedDbPath, dumpFilePath)
+ )
+ # DB should have atleast one value for scan to work
+ self.assertRunOKFull("put --db=%s k1 v1" % loadedDbPath, "OK")
+ self.assertRunOKFull("scan --db=%s" % loadedDbPath, "k1 : v1")
+
+ # Dump command fails because of typo in params
+ dumpFilePath = os.path.join(self.TMP_DIR, "dump8")
+ self.assertFalse(
+ self.dumpDb("--db=%s --create_if_missing" % origDbPath, dumpFilePath)
+ )
+
+ # Dump and load with BlobDB enabled
+ blobParams = " ".join(
+ ["--enable_blob_files", "--min_blob_size=1", "--blob_file_size=2097152"]
+ )
+ dumpFilePath = os.path.join(self.TMP_DIR, "dump9")
+ loadedDbPath = os.path.join(self.TMP_DIR, "loaded_from_dump9")
+ self.assertTrue(self.dumpDb("--db=%s" % (origDbPath), dumpFilePath))
+ self.assertTrue(
+ self.loadDb(
+ "--db=%s %s --create_if_missing --disable_wal"
+ % (loadedDbPath, blobParams),
+ dumpFilePath,
+ )
+ )
+ self.assertRunOKFull(
+ "scan --db=%s" % loadedDbPath, "x1 : y1\nx2 : y2\nx3 : y3\nx4 : y4"
+ )
+ blob_files = self.getBlobFiles(loadedDbPath)
+ self.assertTrue(len(blob_files) >= 1)
+
+ def testIDumpBasics(self):
+ print("Running testIDumpBasics...")
+ self.assertRunOK("put a val --create_if_missing", "OK")
+ self.assertRunOK("put b val", "OK")
+ self.assertRunOK(
+ "idump",
+ "'a' seq:1, type:1 => val\n"
+ "'b' seq:2, type:1 => val\nInternal keys in range: 2",
+ )
+ self.assertRunOK(
+ "idump --input_key_hex --from=%s --to=%s" % (hex(ord("a")), hex(ord("b"))),
+ "'a' seq:1, type:1 => val\nInternal keys in range: 1",
+ )
+
+ def testIDumpDecodeBlobIndex(self):
+ print("Running testIDumpDecodeBlobIndex...")
+ self.assertRunOK("put a val --create_if_missing", "OK")
+ self.assertRunOK("put b val --enable_blob_files", "OK")
+
+ # Pattern to expect from dump with decode_blob_index flag enabled.
+ regex = ".*\[blob ref\].*"
+ expected_pattern = re.compile(regex)
+ cmd = "idump %s --decode_blob_index"
+ self.assertRunOKFull(
+ (cmd) % (self.dbParam(self.DB_NAME)),
+ expected_pattern,
+ unexpected=False,
+ isPattern=True,
+ )
+
+ def testMiscAdminTask(self):
+ print("Running testMiscAdminTask...")
+ # These tests need to be improved; for example with asserts about
+ # whether compaction or level reduction actually took place.
+ self.assertRunOK("batchput --create_if_missing x1 y1 x2 y2 x3 y3 x4 y4", "OK")
+ self.assertRunOK("scan", "x1 : y1\nx2 : y2\nx3 : y3\nx4 : y4")
+ origDbPath = os.path.join(self.TMP_DIR, self.DB_NAME)
+
+ self.assertTrue(0 == run_err_null("./ldb compact --db=%s" % origDbPath))
+ self.assertRunOK("scan", "x1 : y1\nx2 : y2\nx3 : y3\nx4 : y4")
+
+ self.assertTrue(
+ 0 == run_err_null("./ldb reduce_levels --db=%s --new_levels=2" % origDbPath)
+ )
+ self.assertRunOK("scan", "x1 : y1\nx2 : y2\nx3 : y3\nx4 : y4")
+
+ self.assertTrue(
+ 0 == run_err_null("./ldb reduce_levels --db=%s --new_levels=3" % origDbPath)
+ )
+ self.assertRunOK("scan", "x1 : y1\nx2 : y2\nx3 : y3\nx4 : y4")
+
+ self.assertTrue(
+ 0 == run_err_null("./ldb compact --db=%s --from=x1 --to=x3" % origDbPath)
+ )
+ self.assertRunOK("scan", "x1 : y1\nx2 : y2\nx3 : y3\nx4 : y4")
+
+ self.assertTrue(
+ 0
+ == run_err_null(
+ "./ldb compact --db=%s --hex --from=0x6131 --to=0x6134" % origDbPath
+ )
+ )
+ self.assertRunOK("scan", "x1 : y1\nx2 : y2\nx3 : y3\nx4 : y4")
+
+ # TODO(dilip): Not sure what should be passed to WAL.Currently corrupted.
+ self.assertTrue(
+ 0
+ == run_err_null(
+ "./ldb dump_wal --db=%s --walfile=%s --header"
+ % (origDbPath, os.path.join(origDbPath, "LOG"))
+ )
+ )
+ self.assertRunOK("scan", "x1 : y1\nx2 : y2\nx3 : y3\nx4 : y4")
+
+ def testCheckConsistency(self):
+ print("Running testCheckConsistency...")
+
+ dbPath = os.path.join(self.TMP_DIR, self.DB_NAME)
+ self.assertRunOK("put x1 y1 --create_if_missing", "OK")
+ self.assertRunOK("put x2 y2", "OK")
+ self.assertRunOK("get x1", "y1")
+ self.assertRunOK("checkconsistency", "OK")
+
+ sstFilePath = my_check_output(
+ "ls %s" % os.path.join(dbPath, "*.sst"), shell=True
+ )
+
+ # Modify the file
+ my_check_output("echo 'evil' > %s" % sstFilePath, shell=True)
+ self.assertRunFAIL("checkconsistency")
+
+ # Delete the file
+ my_check_output("rm -f %s" % sstFilePath, shell=True)
+ self.assertRunFAIL("checkconsistency")
+
+ def dumpLiveFiles(self, params, dumpFile):
+ return 0 == run_err_null("./ldb dump_live_files %s > %s" % (params, dumpFile))
+
+ def testDumpLiveFiles(self):
+ print("Running testDumpLiveFiles...")
+
+ dbPath = os.path.join(self.TMP_DIR, self.DB_NAME)
+ self.assertRunOK("put x1 y1 --create_if_missing", "OK")
+ self.assertRunOK("put x2 y2 --enable_blob_files", "OK")
+ dumpFilePath = os.path.join(self.TMP_DIR, "dump1")
+ self.assertTrue(self.dumpLiveFiles("--db=%s" % dbPath, dumpFilePath))
+ self.assertRunOK("delete x1", "OK")
+ self.assertRunOK("put x3 y3", "OK")
+ dumpFilePath = os.path.join(self.TMP_DIR, "dump2")
+
+ # Test that if the user provides a db path that ends with
+ # a slash '/', there is no double (or more!) slashes in the
+ # SST and manifest file names.
+
+ # Add a '/' at the end of dbPath (which normally shouldnt contain any)
+ if dbPath[-1] != "/":
+ dbPath += "/"
+
+ # Call the dump_live_files function with the edited dbPath name.
+ self.assertTrue(
+ self.dumpLiveFiles(
+ "--db=%s --decode_blob_index --dump_uncompressed_blobs" % dbPath,
+ dumpFilePath,
+ )
+ )
+
+ # Investigate the output
+ with open(dumpFilePath, "r") as tmp:
+ data = tmp.read()
+
+ # Check that all the SST filenames have a correct full path (no multiple '/').
+ sstFileList = re.findall(r"%s.*\d+.sst" % dbPath, data)
+ self.assertTrue(len(sstFileList) >= 1)
+ for sstFilename in sstFileList:
+ filenumber = re.findall(r"\d+.sst", sstFilename)[0]
+ self.assertEqual(sstFilename, dbPath + filenumber)
+
+ # Check that all the Blob filenames have a correct full path (no multiple '/').
+ blobFileList = re.findall(r"%s.*\d+.blob" % dbPath, data)
+ self.assertTrue(len(blobFileList) >= 1)
+ for blobFilename in blobFileList:
+ filenumber = re.findall(r"\d+.blob", blobFilename)[0]
+ self.assertEqual(blobFilename, dbPath + filenumber)
+
+ # Check that all the manifest filenames
+ # have a correct full path (no multiple '/').
+ manifestFileList = re.findall(r"%s.*MANIFEST-\d+" % dbPath, data)
+ self.assertTrue(len(manifestFileList) >= 1)
+ for manifestFilename in manifestFileList:
+ filenumber = re.findall(r"(?<=MANIFEST-)\d+", manifestFilename)[0]
+ self.assertEqual(manifestFilename, dbPath + "MANIFEST-" + filenumber)
+
+ # Check that the blob file index is decoded.
+ decodedBlobIndex = re.findall(r"\[blob ref\]", data)
+ self.assertTrue(len(decodedBlobIndex) >= 1)
+
+ def listLiveFilesMetadata(self, params, dumpFile):
+ return 0 == run_err_null(
+ "./ldb list_live_files_metadata %s > %s" % (params, dumpFile)
+ )
+
+ def testListLiveFilesMetadata(self):
+ print("Running testListLiveFilesMetadata...")
+
+ dbPath = os.path.join(self.TMP_DIR, self.DB_NAME)
+ self.assertRunOK("put x1 y1 --create_if_missing", "OK")
+ self.assertRunOK("put x2 y2", "OK")
+
+ # Compare the SST filename and the level of list_live_files_metadata
+ # with the data collected from dump_live_files.
+ dumpFilePath1 = os.path.join(self.TMP_DIR, "dump1")
+ self.assertTrue(self.dumpLiveFiles("--db=%s" % dbPath, dumpFilePath1))
+ dumpFilePath2 = os.path.join(self.TMP_DIR, "dump2")
+ self.assertTrue(
+ self.listLiveFilesMetadata(
+ "--sort_by_filename --db=%s" % dbPath, dumpFilePath2
+ )
+ )
+
+ # Collect SST filename and level from dump_live_files
+ with open(dumpFilePath1, "r") as tmp:
+ data = tmp.read()
+ filename1 = re.findall(r".*\d+\.sst", data)[0]
+ level1 = re.findall(r"level:\d+", data)[0].split(":")[1]
+
+ # Collect SST filename and level from list_live_files_metadata
+ with open(dumpFilePath2, "r") as tmp:
+ data = tmp.read()
+ filename2 = re.findall(r".*\d+\.sst", data)[0]
+ level2 = re.findall(r"level \d+", data)[0].split(" ")[1]
+
+ # Assert equality between filenames and levels.
+ self.assertEqual(filename1, filename2)
+ self.assertEqual(level1, level2)
+
+ # Create multiple column families and compare the output
+ # of list_live_files_metadata with dump_live_files once again.
+ # Create new CF, and insert data:
+ self.assertRunOK("create_column_family mycol1", "OK")
+ self.assertRunOK("put --column_family=mycol1 v1 v2", "OK")
+ self.assertRunOK("create_column_family mycol2", "OK")
+ self.assertRunOK("put --column_family=mycol2 h1 h2", "OK")
+ self.assertRunOK("put --column_family=mycol2 h3 h4", "OK")
+
+ # Call dump_live_files and list_live_files_metadata
+ # and pipe the output to compare them later.
+ dumpFilePath3 = os.path.join(self.TMP_DIR, "dump3")
+ self.assertTrue(self.dumpLiveFiles("--db=%s" % dbPath, dumpFilePath3))
+ dumpFilePath4 = os.path.join(self.TMP_DIR, "dump4")
+ self.assertTrue(
+ self.listLiveFilesMetadata(
+ "--sort_by_filename --db=%s" % dbPath, dumpFilePath4
+ )
+ )
+
+ # dump_live_files:
+ # parse the output and create a map:
+ # [key: sstFilename]->[value:[LSM level, Column Family Name]]
+ referenceMap = {}
+ with open(dumpFilePath3, "r") as tmp:
+ data = tmp.read()
+ # Note: the following regex are contingent on what the
+ # dump_live_files outputs.
+ namesAndLevels = re.findall(r"\d+.sst level:\d+", data)
+ cfs = re.findall(r"(?<=column family name=)\w+", data)
+ # re.findall should not reorder the data.
+ # Therefore namesAndLevels[i] matches the data from cfs[i].
+ for count, nameAndLevel in enumerate(namesAndLevels):
+ sstFilename = re.findall(r"\d+.sst", nameAndLevel)[0]
+ sstLevel = re.findall(r"(?<=level:)\d+", nameAndLevel)[0]
+ cf = cfs[count]
+ referenceMap[sstFilename] = [sstLevel, cf]
+
+ # list_live_files_metadata:
+ # parse the output and create a map:
+ # [key: sstFilename]->[value:[LSM level, Column Family Name]]
+ testMap = {}
+ with open(dumpFilePath4, "r") as tmp:
+ data = tmp.read()
+ # Since for each SST file, all the information is contained
+ # on one line, the parsing is easy to perform and relies on
+ # the appearance of an "00xxx.sst" pattern.
+ sstLines = re.findall(r".*\d+.sst.*", data)
+ for line in sstLines:
+ sstFilename = re.findall(r"\d+.sst", line)[0]
+ sstLevel = re.findall(r"(?<=level )\d+", line)[0]
+ cf = re.findall(r"(?<=column family \')\w+(?=\')", line)[0]
+ testMap[sstFilename] = [sstLevel, cf]
+
+ # Compare the map obtained from dump_live_files and the map
+ # obtained from list_live_files_metadata. Everything should match.
+ self.assertEqual(referenceMap, testMap)
+
+ def getManifests(self, directory):
+ return glob.glob(directory + "/MANIFEST-*")
+
+ def getSSTFiles(self, directory):
+ return glob.glob(directory + "/*.sst")
+
+ def getWALFiles(self, directory):
+ return glob.glob(directory + "/*.log")
+
+ def getBlobFiles(self, directory):
+ return glob.glob(directory + "/*.blob")
+
+ def copyManifests(self, src, dest):
+ return 0 == run_err_null("cp " + src + " " + dest)
+
+ def testManifestDump(self):
+ print("Running testManifestDump...")
+ dbPath = os.path.join(self.TMP_DIR, self.DB_NAME)
+ self.assertRunOK("put 1 1 --create_if_missing", "OK")
+ self.assertRunOK("put 2 2", "OK")
+ self.assertRunOK("put 3 3", "OK")
+ # Pattern to expect from manifest_dump.
+ num = "[0-9]+"
+ st = ".*"
+ subpat = st + " seq:" + num + ", type:" + num
+ regex = num + ":" + num + "\[" + subpat + ".." + subpat + "\]"
+ expected_pattern = re.compile(regex)
+ cmd = "manifest_dump --db=%s"
+ manifest_files = self.getManifests(dbPath)
+ self.assertTrue(len(manifest_files) == 1)
+ # Test with the default manifest file in dbPath.
+ self.assertRunOKFull(
+ cmd % dbPath, expected_pattern, unexpected=False, isPattern=True
+ )
+ self.copyManifests(manifest_files[0], manifest_files[0] + "1")
+ manifest_files = self.getManifests(dbPath)
+ self.assertTrue(len(manifest_files) == 2)
+ # Test with multiple manifest files in dbPath.
+ self.assertRunFAILFull(cmd % dbPath)
+ # Running it with the copy we just created should pass.
+ self.assertRunOKFull(
+ (cmd + " --path=%s") % (dbPath, manifest_files[1]),
+ expected_pattern,
+ unexpected=False,
+ isPattern=True,
+ )
+ # Make sure that using the dump with --path will result in identical
+ # output as just using manifest_dump.
+ cmd = "dump --path=%s"
+ self.assertRunOKFull(
+ (cmd) % (manifest_files[1]),
+ expected_pattern,
+ unexpected=False,
+ isPattern=True,
+ )
+
+ # Check if null characters doesn't infer with output format.
+ self.assertRunOK("put a1 b1", "OK")
+ self.assertRunOK("put a2 b2", "OK")
+ self.assertRunOK("put --hex 0x12000DA0 0x80C0000B", "OK")
+ self.assertRunOK("put --hex 0x7200004f 0x80000004", "OK")
+ self.assertRunOK("put --hex 0xa000000a 0xf000000f", "OK")
+ self.assertRunOK("put a3 b3", "OK")
+ self.assertRunOK("put a4 b4", "OK")
+
+ # Verifies that all "levels" are printed out.
+ # There should be 66 mentions of levels.
+ expected_verbose_output = re.compile("matched")
+ # Test manifest_dump verbose and verify that key 0x7200004f
+ # is present. Note that we are forced to use grep here because
+ # an output with a non-terminating null character in it isn't piped
+ # correctly through the Python subprocess object.
+ # Also note that 0x72=r and 0x4f=O, hence the regex \'r.{2}O\'
+ # (we cannot use null character in the subprocess input either,
+ # so we have to use '.{2}')
+ cmd_verbose = (
+ "manifest_dump --verbose --db=%s | grep -aq $''r.{2}O'' && echo 'matched' || echo 'not matched'"
+ % dbPath
+ )
+
+ self.assertRunOKFull(
+ cmd_verbose, expected_verbose_output, unexpected=False, isPattern=True
+ )
+
+ def testGetProperty(self):
+ print("Running testGetProperty...")
+ dbPath = os.path.join(self.TMP_DIR, self.DB_NAME)
+ self.assertRunOK("put 1 1 --create_if_missing", "OK")
+ self.assertRunOK("put 2 2", "OK")
+ # A "string" property
+ cmd = "--db=%s get_property rocksdb.estimate-num-keys"
+ self.assertRunOKFull(cmd % dbPath, "rocksdb.estimate-num-keys: 2")
+ # A "map" property
+ # FIXME: why doesn't this pick up two entries?
+ cmd = "--db=%s get_property rocksdb.aggregated-table-properties"
+ part = "rocksdb.aggregated-table-properties.num_entries: "
+ expected_pattern = re.compile(part)
+ self.assertRunOKFull(
+ cmd % dbPath, expected_pattern, unexpected=False, isPattern=True
+ )
+ # An invalid property
+ cmd = "--db=%s get_property rocksdb.this-property-does-not-exist"
+ self.assertRunFAILFull(cmd % dbPath)
+
+ def testSSTDump(self):
+ print("Running testSSTDump...")
+
+ dbPath = os.path.join(self.TMP_DIR, self.DB_NAME)
+ self.assertRunOK("put sst1 sst1_val --create_if_missing", "OK")
+ self.assertRunOK("put sst2 sst2_val --enable_blob_files", "OK")
+ self.assertRunOK("get sst1", "sst1_val")
+
+ # Pattern to expect from SST dump.
+ regex = ".*Sst file format:.*\n.*\[blob ref\].*"
+ expected_pattern = re.compile(regex)
+
+ sst_files = self.getSSTFiles(dbPath)
+ self.assertTrue(len(sst_files) >= 1)
+ cmd = "dump --path=%s --decode_blob_index"
+ self.assertRunOKFull(
+ (cmd) % (sst_files[0]), expected_pattern, unexpected=False, isPattern=True
+ )
+
+ def testBlobDump(self):
+ print("Running testBlobDump")
+ dbPath = os.path.join(self.TMP_DIR, self.DB_NAME)
+ self.assertRunOK("batchput x1 y1 --create_if_missing --enable_blob_files", "OK")
+ self.assertRunOK(
+ 'batchput --enable_blob_files x2 y2 x3 y3 "x4 abc" "y4 xyz"', "OK"
+ )
+
+ # Pattern to expect from blob file dump.
+ regex = ".*Blob log header[\s\S]*Blob log footer[\s\S]*Read record[\s\S]*Summary" # noqa
+ expected_pattern = re.compile(regex)
+ blob_files = self.getBlobFiles(dbPath)
+ self.assertTrue(len(blob_files) >= 1)
+ cmd = "dump --path=%s --dump_uncompressed_blobs"
+ self.assertRunOKFull(
+ (cmd) % (blob_files[0]), expected_pattern, unexpected=False, isPattern=True
+ )
+
+ def testWALDump(self):
+ print("Running testWALDump...")
+
+ dbPath = os.path.join(self.TMP_DIR, self.DB_NAME)
+ self.assertRunOK("put wal1 wal1_val --create_if_missing", "OK")
+ self.assertRunOK("put wal2 wal2_val", "OK")
+ self.assertRunOK("get wal1", "wal1_val")
+
+ # Pattern to expect from WAL dump.
+ regex = "^Sequence,Count,ByteSize,Physical Offset,Key\(s\).*"
+ expected_pattern = re.compile(regex)
+
+ wal_files = self.getWALFiles(dbPath)
+ self.assertTrue(len(wal_files) >= 1)
+ cmd = "dump --path=%s"
+ self.assertRunOKFull(
+ (cmd) % (wal_files[0]), expected_pattern, unexpected=False, isPattern=True
+ )
+
+ def testListColumnFamilies(self):
+ print("Running testListColumnFamilies...")
+ self.assertRunOK("put x1 y1 --create_if_missing", "OK")
+ cmd = 'list_column_families | grep -v "Column families"'
+ # Test on valid dbPath.
+ self.assertRunOK(cmd, "{default}")
+ # Test on empty path.
+ self.assertRunFAIL(cmd)
+
+ def testColumnFamilies(self):
+ print("Running testColumnFamilies...")
+ _ = os.path.join(self.TMP_DIR, self.DB_NAME)
+ self.assertRunOK("put cf1_1 1 --create_if_missing", "OK")
+ self.assertRunOK("put cf1_2 2 --create_if_missing", "OK")
+ self.assertRunOK("put cf1_3 3 --try_load_options", "OK")
+ # Given non-default column family to single CF DB.
+ self.assertRunFAIL("get cf1_1 --column_family=two")
+ self.assertRunOK("create_column_family two", "OK")
+ self.assertRunOK("put cf2_1 1 --create_if_missing --column_family=two", "OK")
+ self.assertRunOK("put cf2_2 2 --create_if_missing --column_family=two", "OK")
+ self.assertRunOK("delete cf1_2", "OK")
+ self.assertRunOK("create_column_family three", "OK")
+ self.assertRunOK("delete cf2_2 --column_family=two", "OK")
+ self.assertRunOK("put cf3_1 3 --create_if_missing --column_family=three", "OK")
+ self.assertRunOK("get cf1_1 --column_family=default", "1")
+ self.assertRunOK("dump --column_family=two", "cf2_1 ==> 1\nKeys in range: 1")
+ self.assertRunOK(
+ "dump --column_family=two --try_load_options",
+ "cf2_1 ==> 1\nKeys in range: 1",
+ )
+ self.assertRunOK("dump", "cf1_1 ==> 1\ncf1_3 ==> 3\nKeys in range: 2")
+ self.assertRunOK("get cf2_1 --column_family=two", "1")
+ self.assertRunOK("get cf3_1 --column_family=three", "3")
+ self.assertRunOK("drop_column_family three", "OK")
+ # non-existing column family.
+ self.assertRunFAIL("get cf3_1 --column_family=four")
+ self.assertRunFAIL("drop_column_family four")
+
+ def testIngestExternalSst(self):
+ print("Running testIngestExternalSst...")
+
+ # Dump, load, write external sst and ingest it in another db
+ dbPath = os.path.join(self.TMP_DIR, "db1")
+ self.assertRunOK(
+ "batchput --db=%s --create_if_missing x1 y1 x2 y2 x3 y3 x4 y4" % dbPath,
+ "OK",
+ )
+ self.assertRunOK("scan --db=%s" % dbPath, "x1 : y1\nx2 : y2\nx3 : y3\nx4 : y4")
+ dumpFilePath = os.path.join(self.TMP_DIR, "dump1")
+ with open(dumpFilePath, "w") as f:
+ f.write("x1 ==> y10\nx2 ==> y20\nx3 ==> y30\nx4 ==> y40")
+ externSstPath = os.path.join(self.TMP_DIR, "extern_data1.sst")
+ self.assertTrue(
+ self.writeExternSst(
+ "--create_if_missing --db=%s" % dbPath, dumpFilePath, externSstPath
+ )
+ )
+ # cannot ingest if allow_global_seqno is false
+ self.assertFalse(
+ self.ingestExternSst(
+ "--create_if_missing --allow_global_seqno=false --db=%s" % dbPath,
+ externSstPath,
+ )
+ )
+ self.assertTrue(
+ self.ingestExternSst(
+ "--create_if_missing --allow_global_seqno --db=%s" % dbPath,
+ externSstPath,
+ )
+ )
+ self.assertRunOKFull(
+ "scan --db=%s" % dbPath, "x1 : y10\nx2 : y20\nx3 : y30\nx4 : y40"
+ )
+
+
+if __name__ == "__main__":
+ unittest.main()
generated by cgit v1.2.3 (git 2.39.1) at 2024-07-16 10:38:00 +0000