mirror of
				https://source.quilibrium.com/quilibrium/ceremonyclient.git
				synced 2025-11-04 07:57:27 +00:00 
			
		
		
		
	
		
			
				
	
	
		
			628 lines
		
	
	
		
			7.8 KiB
		
	
	
	
		
			Plaintext
		
	
	
	
	
	
			
		
		
	
	
			628 lines
		
	
	
		
			7.8 KiB
		
	
	
	
		
			Plaintext
		
	
	
	
	
	
# We create multiple SSTs, one of which overlaps with the memtable (scheduling a flush).
 | 
						|
# Check that the SSTs get ingested to the lowest levels possible.
 | 
						|
 | 
						|
batch
 | 
						|
set a 0
 | 
						|
----
 | 
						|
 | 
						|
# The SST below overlaps with memtable and thus should be placed in L0
 | 
						|
# post flush.
 | 
						|
build ext1
 | 
						|
set a 1
 | 
						|
----
 | 
						|
 | 
						|
# The SST below overlaps with nothing and thus should be placed in L6 post
 | 
						|
# flush.
 | 
						|
build ext2
 | 
						|
set b 1
 | 
						|
----
 | 
						|
 | 
						|
# The SST below doesn't overlap with any SSTs in the LSM and thus can be placed
 | 
						|
# in L6 post-flush.
 | 
						|
build ext3
 | 
						|
set d 1
 | 
						|
----
 | 
						|
 | 
						|
# We block the flush, so the SSTs should still be in the flushable queue.
 | 
						|
blockFlush
 | 
						|
----
 | 
						|
 | 
						|
ingest ext1 ext2 ext3
 | 
						|
----
 | 
						|
 | 
						|
allowFlush
 | 
						|
----
 | 
						|
 | 
						|
lsm
 | 
						|
----
 | 
						|
 | 
						|
get
 | 
						|
a
 | 
						|
b
 | 
						|
d
 | 
						|
----
 | 
						|
a:1
 | 
						|
b:1
 | 
						|
d:1
 | 
						|
 | 
						|
# We expect 1 WAL for an immutable memtable, 1 file for the ingested ssts,
 | 
						|
# one for the mutable memtable. We also expect 3 ssts corresponding to the
 | 
						|
# ingested files.
 | 
						|
ls
 | 
						|
----
 | 
						|
000002.log
 | 
						|
000004.sst
 | 
						|
000005.sst
 | 
						|
000006.sst
 | 
						|
000007.log
 | 
						|
000008.log
 | 
						|
CURRENT
 | 
						|
LOCK
 | 
						|
MANIFEST-000001
 | 
						|
OPTIONS-000003
 | 
						|
ext
 | 
						|
marker.format-version.000015.016
 | 
						|
marker.manifest.000001.MANIFEST-000001
 | 
						|
 | 
						|
# Test basic WAL replay
 | 
						|
close
 | 
						|
----
 | 
						|
 | 
						|
# In this case only the flushable was holding a reference to the sstables. Even
 | 
						|
# after the DB is closed, those sstables should still be hanging around.
 | 
						|
ls
 | 
						|
----
 | 
						|
000002.log
 | 
						|
000004.sst
 | 
						|
000005.sst
 | 
						|
000006.sst
 | 
						|
000007.log
 | 
						|
000008.log
 | 
						|
CURRENT
 | 
						|
LOCK
 | 
						|
MANIFEST-000001
 | 
						|
OPTIONS-000003
 | 
						|
ext
 | 
						|
marker.format-version.000015.016
 | 
						|
marker.manifest.000001.MANIFEST-000001
 | 
						|
 | 
						|
open
 | 
						|
----
 | 
						|
 | 
						|
# Make sure that the sstables got flushed in the correct order on a WAL replay.
 | 
						|
lsm
 | 
						|
----
 | 
						|
0.1:
 | 
						|
  000004:[a#11,SET-a#11,SET]
 | 
						|
0.0:
 | 
						|
  000009:[a#10,SET-a#10,SET]
 | 
						|
  000005:[b#12,SET-b#12,SET]
 | 
						|
  000006:[d#13,SET-d#13,SET]
 | 
						|
 | 
						|
get
 | 
						|
a
 | 
						|
b
 | 
						|
d
 | 
						|
----
 | 
						|
a:1
 | 
						|
b:1
 | 
						|
d:1
 | 
						|
 | 
						|
reset
 | 
						|
----
 | 
						|
 | 
						|
# Repeat the steps above without closing Pebble. Note that the final lsm state
 | 
						|
# will be different because WAL replay just placed the files in L0.
 | 
						|
batch
 | 
						|
set a 0
 | 
						|
----
 | 
						|
 | 
						|
build ext1
 | 
						|
set a 1
 | 
						|
----
 | 
						|
 | 
						|
build ext2
 | 
						|
set b 1
 | 
						|
----
 | 
						|
 | 
						|
build ext3
 | 
						|
set d 1
 | 
						|
----
 | 
						|
 | 
						|
ingest ext1 ext2 ext3
 | 
						|
----
 | 
						|
 | 
						|
lsm
 | 
						|
----
 | 
						|
0.1:
 | 
						|
  000004:[a#11,SET-a#11,SET]
 | 
						|
0.0:
 | 
						|
  000009:[a#10,SET-a#10,SET]
 | 
						|
6:
 | 
						|
  000005:[b#12,SET-b#12,SET]
 | 
						|
  000006:[d#13,SET-d#13,SET]
 | 
						|
 | 
						|
reset
 | 
						|
----
 | 
						|
 | 
						|
# Test multiple overlapping ingests interleaving batch sets, and then flushing.
 | 
						|
batch
 | 
						|
set a 0
 | 
						|
----
 | 
						|
 | 
						|
build ext4
 | 
						|
set a 1
 | 
						|
----
 | 
						|
 | 
						|
build ext5
 | 
						|
set a 2
 | 
						|
----
 | 
						|
 | 
						|
blockFlush
 | 
						|
----
 | 
						|
 | 
						|
ingest ext4
 | 
						|
----
 | 
						|
 | 
						|
allowFlush
 | 
						|
----
 | 
						|
 | 
						|
get
 | 
						|
a
 | 
						|
----
 | 
						|
a:1
 | 
						|
 | 
						|
batch
 | 
						|
set b 1
 | 
						|
----
 | 
						|
 | 
						|
get
 | 
						|
a
 | 
						|
b
 | 
						|
----
 | 
						|
a:1
 | 
						|
b:1
 | 
						|
 | 
						|
# Should get ingested into L0 above the memtable flush.
 | 
						|
blockFlush
 | 
						|
----
 | 
						|
 | 
						|
ingest ext5
 | 
						|
----
 | 
						|
 | 
						|
allowFlush
 | 
						|
----
 | 
						|
 | 
						|
get
 | 
						|
a
 | 
						|
b
 | 
						|
----
 | 
						|
a:2
 | 
						|
b:1
 | 
						|
 | 
						|
batch
 | 
						|
set c 1
 | 
						|
----
 | 
						|
 | 
						|
flush
 | 
						|
----
 | 
						|
 | 
						|
lsm
 | 
						|
----
 | 
						|
0.2:
 | 
						|
  000007:[a#13,SET-a#13,SET]
 | 
						|
0.1:
 | 
						|
  000004:[a#11,SET-a#11,SET]
 | 
						|
0.0:
 | 
						|
  000010:[a#10,SET-a#10,SET]
 | 
						|
  000011:[b#12,SET-b#12,SET]
 | 
						|
 | 
						|
# Value of a should be the value of a in the second ingested SST.
 | 
						|
get
 | 
						|
a
 | 
						|
b
 | 
						|
c
 | 
						|
----
 | 
						|
a:2
 | 
						|
b:1
 | 
						|
c:1
 | 
						|
 | 
						|
# Test that non-overlapping ingest still works normally.
 | 
						|
reset
 | 
						|
----
 | 
						|
 | 
						|
batch
 | 
						|
set a 0
 | 
						|
----
 | 
						|
 | 
						|
build ext1
 | 
						|
set b 1
 | 
						|
----
 | 
						|
 | 
						|
build ext2
 | 
						|
set d 1
 | 
						|
----
 | 
						|
 | 
						|
ingest ext1 ext2
 | 
						|
----
 | 
						|
 | 
						|
lsm
 | 
						|
----
 | 
						|
6:
 | 
						|
  000004:[b#11,SET-b#11,SET]
 | 
						|
  000005:[d#12,SET-d#12,SET]
 | 
						|
 | 
						|
 | 
						|
# Verify target level of ingestedFlushable.
 | 
						|
reset
 | 
						|
----
 | 
						|
 | 
						|
batch
 | 
						|
set a 0
 | 
						|
----
 | 
						|
 | 
						|
build ext1
 | 
						|
set a 1
 | 
						|
----
 | 
						|
 | 
						|
build ext2
 | 
						|
set b 1
 | 
						|
----
 | 
						|
 | 
						|
build ext3
 | 
						|
set d 1
 | 
						|
----
 | 
						|
 | 
						|
ingest ext1 ext2 ext3
 | 
						|
----
 | 
						|
 | 
						|
lsm
 | 
						|
----
 | 
						|
0.1:
 | 
						|
  000004:[a#11,SET-a#11,SET]
 | 
						|
0.0:
 | 
						|
  000009:[a#10,SET-a#10,SET]
 | 
						|
6:
 | 
						|
  000005:[b#12,SET-b#12,SET]
 | 
						|
  000006:[d#13,SET-d#13,SET]
 | 
						|
 | 
						|
 | 
						|
batch
 | 
						|
set a 3
 | 
						|
----
 | 
						|
 | 
						|
build ext4
 | 
						|
set a 4
 | 
						|
----
 | 
						|
 | 
						|
build ext5
 | 
						|
set b 5
 | 
						|
----
 | 
						|
 | 
						|
ingest ext4 ext5
 | 
						|
----
 | 
						|
 | 
						|
# Looking for the sstable with the key a to go into 0.3, and the sstable with
 | 
						|
# key b to go into 0.0. The sstable doesn't go into L5, because L5 isn't open
 | 
						|
# yet.
 | 
						|
lsm
 | 
						|
----
 | 
						|
0.3:
 | 
						|
  000010:[a#15,SET-a#15,SET]
 | 
						|
0.2:
 | 
						|
  000014:[a#14,SET-a#14,SET]
 | 
						|
0.1:
 | 
						|
  000004:[a#11,SET-a#11,SET]
 | 
						|
0.0:
 | 
						|
  000009:[a#10,SET-a#10,SET]
 | 
						|
  000011:[b#16,SET-b#16,SET]
 | 
						|
6:
 | 
						|
  000005:[b#12,SET-b#12,SET]
 | 
						|
  000006:[d#13,SET-d#13,SET]
 | 
						|
 | 
						|
# Testing whether the new mutable memtable with data is flushed correctly during
 | 
						|
# WAL replay.
 | 
						|
reset
 | 
						|
----
 | 
						|
 | 
						|
batch
 | 
						|
set a 0
 | 
						|
----
 | 
						|
 | 
						|
# The SST below overlaps with memtable and thus should be placed in L0
 | 
						|
# post flush.
 | 
						|
build ext1
 | 
						|
set a 1
 | 
						|
----
 | 
						|
 | 
						|
# The SST below overlaps with nothing and thus should be placed in L6 post
 | 
						|
# flush.
 | 
						|
build ext2
 | 
						|
set b 1
 | 
						|
----
 | 
						|
 | 
						|
# The SST below doesn't overlap with any SSTs in the LSM and thus can be placed
 | 
						|
# in L6 post-flush.
 | 
						|
build ext3
 | 
						|
set d 1
 | 
						|
----
 | 
						|
 | 
						|
# We block the flush, so the SSTs should still be in the flushable queue.
 | 
						|
blockFlush
 | 
						|
----
 | 
						|
 | 
						|
ingest ext1 ext2 ext3
 | 
						|
----
 | 
						|
 | 
						|
# Add another write which should go to the new mutable memtable.
 | 
						|
batch
 | 
						|
set f 1
 | 
						|
----
 | 
						|
 | 
						|
allowFlush
 | 
						|
----
 | 
						|
 | 
						|
lsm
 | 
						|
----
 | 
						|
 | 
						|
get
 | 
						|
a
 | 
						|
b
 | 
						|
d
 | 
						|
f
 | 
						|
----
 | 
						|
a:1
 | 
						|
b:1
 | 
						|
d:1
 | 
						|
f:1
 | 
						|
 | 
						|
# We expect 1 WAL for an immutable memtable, 1 file for the ingested ssts,
 | 
						|
# one for the mutable memtable. We also expect 3 ssts corresponding to the
 | 
						|
# ingested files.
 | 
						|
ls
 | 
						|
----
 | 
						|
000002.log
 | 
						|
000004.sst
 | 
						|
000005.sst
 | 
						|
000006.sst
 | 
						|
000007.log
 | 
						|
000008.log
 | 
						|
CURRENT
 | 
						|
LOCK
 | 
						|
MANIFEST-000001
 | 
						|
OPTIONS-000003
 | 
						|
ext
 | 
						|
marker.format-version.000015.016
 | 
						|
marker.manifest.000001.MANIFEST-000001
 | 
						|
 | 
						|
close
 | 
						|
----
 | 
						|
 | 
						|
# In this case only the memtable was holding a reference to the sstables. Even
 | 
						|
# after the DB is closed, those memtables should still be hanging around.
 | 
						|
ls
 | 
						|
----
 | 
						|
000002.log
 | 
						|
000004.sst
 | 
						|
000005.sst
 | 
						|
000006.sst
 | 
						|
000007.log
 | 
						|
000008.log
 | 
						|
CURRENT
 | 
						|
LOCK
 | 
						|
MANIFEST-000001
 | 
						|
OPTIONS-000003
 | 
						|
ext
 | 
						|
marker.format-version.000015.016
 | 
						|
marker.manifest.000001.MANIFEST-000001
 | 
						|
 | 
						|
open
 | 
						|
----
 | 
						|
 | 
						|
# Make sure that the sstables got flushed in the correct order on a WAL replay.
 | 
						|
lsm
 | 
						|
----
 | 
						|
0.1:
 | 
						|
  000004:[a#11,SET-a#11,SET]
 | 
						|
0.0:
 | 
						|
  000009:[a#10,SET-a#10,SET]
 | 
						|
  000005:[b#12,SET-b#12,SET]
 | 
						|
  000006:[d#13,SET-d#13,SET]
 | 
						|
  000010:[f#14,SET-f#14,SET]
 | 
						|
 | 
						|
# Check if the new mutable memtable is using a new log file, and that the
 | 
						|
# previous log files have been deleted appropriately after the flush.
 | 
						|
ls
 | 
						|
----
 | 
						|
000004.sst
 | 
						|
000005.sst
 | 
						|
000006.sst
 | 
						|
000009.sst
 | 
						|
000010.sst
 | 
						|
000011.log
 | 
						|
CURRENT
 | 
						|
LOCK
 | 
						|
MANIFEST-000001
 | 
						|
MANIFEST-000012
 | 
						|
OPTIONS-000013
 | 
						|
ext
 | 
						|
marker.format-version.000015.016
 | 
						|
marker.manifest.000002.MANIFEST-000012
 | 
						|
 | 
						|
# Make sure that the new mutable memtable can accept writes.
 | 
						|
batch
 | 
						|
set h 2
 | 
						|
----
 | 
						|
 | 
						|
get
 | 
						|
h
 | 
						|
----
 | 
						|
h:2
 | 
						|
 | 
						|
# Test correct WAL replay with read only mode. We essentially want to make sure
 | 
						|
# that once a flushable is added to the queue, we create a new mutable memtable
 | 
						|
# on top of the flushable. Otherwise, we can invert sequence number invariants.
 | 
						|
reset
 | 
						|
----
 | 
						|
 | 
						|
batch
 | 
						|
set a 0
 | 
						|
----
 | 
						|
 | 
						|
# The SST below overlaps with memtable and thus should be placed in L0
 | 
						|
# post flush.
 | 
						|
build ext1
 | 
						|
set a 1
 | 
						|
----
 | 
						|
 | 
						|
# The SST below overlaps with nothing and thus should be placed in L6 post
 | 
						|
# flush.
 | 
						|
build ext2
 | 
						|
set b 1
 | 
						|
----
 | 
						|
 | 
						|
# The SST below doesn't overlap with any SSTs in the LSM and thus can be placed
 | 
						|
# in L6 post-flush.
 | 
						|
build ext3
 | 
						|
set d 1
 | 
						|
----
 | 
						|
 | 
						|
# We block the flush, so the SSTs should still be in the flushable queue.
 | 
						|
blockFlush
 | 
						|
----
 | 
						|
 | 
						|
ingest ext1 ext2 ext3
 | 
						|
----
 | 
						|
 | 
						|
# Add another write which should go to the new mutable memtable.
 | 
						|
batch
 | 
						|
set a 3
 | 
						|
----
 | 
						|
 | 
						|
allowFlush
 | 
						|
----
 | 
						|
 | 
						|
lsm
 | 
						|
----
 | 
						|
 | 
						|
get
 | 
						|
a
 | 
						|
b
 | 
						|
d
 | 
						|
----
 | 
						|
a:3
 | 
						|
b:1
 | 
						|
d:1
 | 
						|
 | 
						|
close
 | 
						|
----
 | 
						|
 | 
						|
open readOnly
 | 
						|
----
 | 
						|
 | 
						|
get
 | 
						|
a
 | 
						|
b
 | 
						|
d
 | 
						|
----
 | 
						|
a:3
 | 
						|
b:1
 | 
						|
d:1
 | 
						|
 | 
						|
# Test with StrictFS
 | 
						|
reset strictMem
 | 
						|
----
 | 
						|
 | 
						|
batch
 | 
						|
set a 1
 | 
						|
set b 1
 | 
						|
----
 | 
						|
 | 
						|
build ext1
 | 
						|
set a 2
 | 
						|
set b 2
 | 
						|
----
 | 
						|
 | 
						|
blockFlush
 | 
						|
----
 | 
						|
 | 
						|
ingest ext1
 | 
						|
----
 | 
						|
 | 
						|
get
 | 
						|
a
 | 
						|
b
 | 
						|
----
 | 
						|
a:2
 | 
						|
b:2
 | 
						|
 | 
						|
ignoreSyncs true
 | 
						|
----
 | 
						|
 | 
						|
lsm
 | 
						|
----
 | 
						|
 | 
						|
allowFlush
 | 
						|
----
 | 
						|
 | 
						|
flush
 | 
						|
----
 | 
						|
 | 
						|
# The ingested file is placed above the sstable generated by memtable flush. The
 | 
						|
# ingested file has a lower file number, but a higher sequence number as
 | 
						|
# expected.
 | 
						|
lsm
 | 
						|
----
 | 
						|
0.1:
 | 
						|
  000004:[a#12,SET-b#12,SET]
 | 
						|
0.0:
 | 
						|
  000007:[a#10,SET-b#11,SET]
 | 
						|
 | 
						|
ls
 | 
						|
----
 | 
						|
000002.log
 | 
						|
000004.sst
 | 
						|
000005.log
 | 
						|
000006.log
 | 
						|
000007.sst
 | 
						|
CURRENT
 | 
						|
LOCK
 | 
						|
MANIFEST-000001
 | 
						|
OPTIONS-000003
 | 
						|
ext
 | 
						|
marker.format-version.000015.016
 | 
						|
marker.manifest.000001.MANIFEST-000001
 | 
						|
 | 
						|
close
 | 
						|
----
 | 
						|
 | 
						|
# At this point, the changes to the manifest should be lost. Note that 7.sst
 | 
						|
# is gone because that file was never synced.
 | 
						|
resetToSynced
 | 
						|
----
 | 
						|
000002.log
 | 
						|
000004.sst
 | 
						|
000005.log
 | 
						|
000006.log
 | 
						|
CURRENT
 | 
						|
LOCK
 | 
						|
MANIFEST-000001
 | 
						|
OPTIONS-000003
 | 
						|
ext
 | 
						|
ext1
 | 
						|
marker.format-version.000015.016
 | 
						|
marker.manifest.000001.MANIFEST-000001
 | 
						|
 | 
						|
ignoreSyncs false
 | 
						|
----
 | 
						|
 | 
						|
open
 | 
						|
----
 | 
						|
 | 
						|
lsm
 | 
						|
----
 | 
						|
0.1:
 | 
						|
  000004:[a#12,SET-b#12,SET]
 | 
						|
0.0:
 | 
						|
  000007:[a#10,SET-b#11,SET]
 |