ceremonyclient/pebble/testdata/flushable_ingest
Cassandra Heart 2e2a1e4789
v1.2.0 (#31)
2024-01-03 01:31:42 -06:00

628 lines
7.8 KiB
Plaintext

# We create multiple SSTs, one of which overlaps with the memtable (scheduling a flush).
# Check that the SSTs get ingested to the lowest levels possible.
batch
set a 0
----
# The SST below overlaps with memtable and thus should be placed in L0
# post flush.
build ext1
set a 1
----
# The SST below overlaps with nothing and thus should be placed in L6 post
# flush.
build ext2
set b 1
----
# The SST below doesn't overlap with any SSTs in the LSM and thus can be placed
# in L6 post-flush.
build ext3
set d 1
----
# We block the flush, so the SSTs should still be in the flushable queue.
blockFlush
----
ingest ext1 ext2 ext3
----
allowFlush
----
lsm
----
get
a
b
d
----
a:1
b:1
d:1
# We expect 1 WAL for an immutable memtable, 1 file for the ingested ssts,
# one for the mutable memtable. We also expect 3 ssts corresponding to the
# ingested files.
ls
----
000002.log
000004.sst
000005.sst
000006.sst
000007.log
000008.log
CURRENT
LOCK
MANIFEST-000001
OPTIONS-000003
ext
marker.format-version.000015.016
marker.manifest.000001.MANIFEST-000001
# Test basic WAL replay
close
----
# In this case only the flushable was holding a reference to the sstables. Even
# after the DB is closed, those sstables should still be hanging around.
ls
----
000002.log
000004.sst
000005.sst
000006.sst
000007.log
000008.log
CURRENT
LOCK
MANIFEST-000001
OPTIONS-000003
ext
marker.format-version.000015.016
marker.manifest.000001.MANIFEST-000001
open
----
# Make sure that the sstables got flushed in the correct order on a WAL replay.
lsm
----
0.1:
000004:[a#11,SET-a#11,SET]
0.0:
000009:[a#10,SET-a#10,SET]
000005:[b#12,SET-b#12,SET]
000006:[d#13,SET-d#13,SET]
get
a
b
d
----
a:1
b:1
d:1
reset
----
# Repeat the steps above without closing Pebble. Note that the final lsm state
# will be different because WAL replay just placed the files in L0.
batch
set a 0
----
build ext1
set a 1
----
build ext2
set b 1
----
build ext3
set d 1
----
ingest ext1 ext2 ext3
----
lsm
----
0.1:
000004:[a#11,SET-a#11,SET]
0.0:
000009:[a#10,SET-a#10,SET]
6:
000005:[b#12,SET-b#12,SET]
000006:[d#13,SET-d#13,SET]
reset
----
# Test multiple overlapping ingests interleaving batch sets, and then flushing.
batch
set a 0
----
build ext4
set a 1
----
build ext5
set a 2
----
blockFlush
----
ingest ext4
----
allowFlush
----
get
a
----
a:1
batch
set b 1
----
get
a
b
----
a:1
b:1
# Should get ingested into L0 above the memtable flush.
blockFlush
----
ingest ext5
----
allowFlush
----
get
a
b
----
a:2
b:1
batch
set c 1
----
flush
----
lsm
----
0.2:
000007:[a#13,SET-a#13,SET]
0.1:
000004:[a#11,SET-a#11,SET]
0.0:
000010:[a#10,SET-a#10,SET]
000011:[b#12,SET-b#12,SET]
# Value of a should be the value of a in the second ingested SST.
get
a
b
c
----
a:2
b:1
c:1
# Test that non-overlapping ingest still works normally.
reset
----
batch
set a 0
----
build ext1
set b 1
----
build ext2
set d 1
----
ingest ext1 ext2
----
lsm
----
6:
000004:[b#11,SET-b#11,SET]
000005:[d#12,SET-d#12,SET]
# Verify target level of ingestedFlushable.
reset
----
batch
set a 0
----
build ext1
set a 1
----
build ext2
set b 1
----
build ext3
set d 1
----
ingest ext1 ext2 ext3
----
lsm
----
0.1:
000004:[a#11,SET-a#11,SET]
0.0:
000009:[a#10,SET-a#10,SET]
6:
000005:[b#12,SET-b#12,SET]
000006:[d#13,SET-d#13,SET]
batch
set a 3
----
build ext4
set a 4
----
build ext5
set b 5
----
ingest ext4 ext5
----
# Looking for the sstable with the key a to go into 0.3, and the sstable with
# key b to go into 0.0. The sstable doesn't go into L5, because L5 isn't open
# yet.
lsm
----
0.3:
000010:[a#15,SET-a#15,SET]
0.2:
000014:[a#14,SET-a#14,SET]
0.1:
000004:[a#11,SET-a#11,SET]
0.0:
000009:[a#10,SET-a#10,SET]
000011:[b#16,SET-b#16,SET]
6:
000005:[b#12,SET-b#12,SET]
000006:[d#13,SET-d#13,SET]
# Testing whether the new mutable memtable with data is flushed correctly during
# WAL replay.
reset
----
batch
set a 0
----
# The SST below overlaps with memtable and thus should be placed in L0
# post flush.
build ext1
set a 1
----
# The SST below overlaps with nothing and thus should be placed in L6 post
# flush.
build ext2
set b 1
----
# The SST below doesn't overlap with any SSTs in the LSM and thus can be placed
# in L6 post-flush.
build ext3
set d 1
----
# We block the flush, so the SSTs should still be in the flushable queue.
blockFlush
----
ingest ext1 ext2 ext3
----
# Add another write which should go to the new mutable memtable.
batch
set f 1
----
allowFlush
----
lsm
----
get
a
b
d
f
----
a:1
b:1
d:1
f:1
# We expect 1 WAL for an immutable memtable, 1 file for the ingested ssts,
# one for the mutable memtable. We also expect 3 ssts corresponding to the
# ingested files.
ls
----
000002.log
000004.sst
000005.sst
000006.sst
000007.log
000008.log
CURRENT
LOCK
MANIFEST-000001
OPTIONS-000003
ext
marker.format-version.000015.016
marker.manifest.000001.MANIFEST-000001
close
----
# In this case only the memtable was holding a reference to the sstables. Even
# after the DB is closed, those memtables should still be hanging around.
ls
----
000002.log
000004.sst
000005.sst
000006.sst
000007.log
000008.log
CURRENT
LOCK
MANIFEST-000001
OPTIONS-000003
ext
marker.format-version.000015.016
marker.manifest.000001.MANIFEST-000001
open
----
# Make sure that the sstables got flushed in the correct order on a WAL replay.
lsm
----
0.1:
000004:[a#11,SET-a#11,SET]
0.0:
000009:[a#10,SET-a#10,SET]
000005:[b#12,SET-b#12,SET]
000006:[d#13,SET-d#13,SET]
000010:[f#14,SET-f#14,SET]
# Check if the new mutable memtable is using a new log file, and that the
# previous log files have been deleted appropriately after the flush.
ls
----
000004.sst
000005.sst
000006.sst
000009.sst
000010.sst
000011.log
CURRENT
LOCK
MANIFEST-000001
MANIFEST-000012
OPTIONS-000013
ext
marker.format-version.000015.016
marker.manifest.000002.MANIFEST-000012
# Make sure that the new mutable memtable can accept writes.
batch
set h 2
----
get
h
----
h:2
# Test correct WAL replay with read only mode. We essentially want to make sure
# that once a flushable is added to the queue, we create a new mutable memtable
# on top of the flushable. Otherwise, we can invert sequence number invariants.
reset
----
batch
set a 0
----
# The SST below overlaps with memtable and thus should be placed in L0
# post flush.
build ext1
set a 1
----
# The SST below overlaps with nothing and thus should be placed in L6 post
# flush.
build ext2
set b 1
----
# The SST below doesn't overlap with any SSTs in the LSM and thus can be placed
# in L6 post-flush.
build ext3
set d 1
----
# We block the flush, so the SSTs should still be in the flushable queue.
blockFlush
----
ingest ext1 ext2 ext3
----
# Add another write which should go to the new mutable memtable.
batch
set a 3
----
allowFlush
----
lsm
----
get
a
b
d
----
a:3
b:1
d:1
close
----
open readOnly
----
get
a
b
d
----
a:3
b:1
d:1
# Test with StrictFS
reset strictMem
----
batch
set a 1
set b 1
----
build ext1
set a 2
set b 2
----
blockFlush
----
ingest ext1
----
get
a
b
----
a:2
b:2
ignoreSyncs true
----
lsm
----
allowFlush
----
flush
----
# The ingested file is placed above the sstable generated by memtable flush. The
# ingested file has a lower file number, but a higher sequence number as
# expected.
lsm
----
0.1:
000004:[a#12,SET-b#12,SET]
0.0:
000007:[a#10,SET-b#11,SET]
ls
----
000002.log
000004.sst
000005.log
000006.log
000007.sst
CURRENT
LOCK
MANIFEST-000001
OPTIONS-000003
ext
marker.format-version.000015.016
marker.manifest.000001.MANIFEST-000001
close
----
# At this point, the changes to the manifest should be lost. Note that 7.sst
# is gone because that file was never synced.
resetToSynced
----
000002.log
000004.sst
000005.log
000006.log
CURRENT
LOCK
MANIFEST-000001
OPTIONS-000003
ext
ext1
marker.format-version.000015.016
marker.manifest.000001.MANIFEST-000001
ignoreSyncs false
----
open
----
lsm
----
0.1:
000004:[a#12,SET-b#12,SET]
0.0:
000007:[a#10,SET-b#11,SET]