batch set a 1 set b 2 ---- compact a-b ---- 6: 000005:[a#10,SET-b#11,SET] batch set c 3 set d 4 ---- compact c-d ---- 6: 000005:[a#10,SET-b#11,SET] 000007:[c#12,SET-d#13,SET] batch set b 5 set c 6 ---- compact a-d ---- 6: 000010:[a#0,SET-d#0,SET] # This also tests flushing a memtable that only contains range # deletions. batch del-range a e ---- compact a-d ---- # Test that a multi-output-file compaction generates non-overlapping files. define target-file-sizes=(100, 1) L0 b.SET.1:v L0 a.SET.2:v ---- 0.0: 000005:[a#2,SET-a#2,SET] 000004:[b#1,SET-b#1,SET] compact a-b ---- 1: 000006:[a#0,SET-a#0,SET] 000007:[b#0,SET-b#0,SET] # A range tombstone extends past the grandparent file boundary used to limit the # size of future compactions. Verify the range tombstone is split at that file # boundary. define target-file-sizes=(1, 1, 1, 1) L1 a.SET.3:v L2 a.RANGEDEL.2:e L3 a.SET.0:v b.SET.0:v L3 c.SET.0:v d.SET.0:v ---- 1: 000004:[a#3,SET-a#3,SET] 2: 000005:[a#2,RANGEDEL-e#inf,RANGEDEL] 3: 000006:[a#0,SET-b#0,SET] 000007:[c#0,SET-d#0,SET] wait-pending-table-stats 000005 ---- num-entries: 1 num-deletions: 1 num-range-key-sets: 0 point-deletions-bytes-estimate: 0 range-deletions-bytes-estimate: 1278 compact a-e L1 ---- 2: 000008:[a#3,SETWITHDEL-c#inf,RANGEDEL] 000009:[c#2,RANGEDEL-e#inf,RANGEDEL] 3: 000006:[a#0,SET-b#0,SET] 000007:[c#0,SET-d#0,SET] wait-pending-table-stats 000008 ---- num-entries: 2 num-deletions: 1 num-range-key-sets: 0 point-deletions-bytes-estimate: 0 range-deletions-bytes-estimate: 639 # Same as above, except range tombstone covers multiple grandparent file boundaries. define target-file-sizes=(1, 1, 1, 1) L1 a.SET.3:v L2 a.RANGEDEL.2:g L3 a.SET.0:v b.SET.0:v L3 c.SET.0:v d.SET.0:v L3 e.SET.0:v f.SET.1:v L3 f.SET.0:v g.SET.0:v ---- 1: 000004:[a#3,SET-a#3,SET] 2: 000005:[a#2,RANGEDEL-g#inf,RANGEDEL] 3: 000006:[a#0,SET-b#0,SET] 000007:[c#0,SET-d#0,SET] 000008:[e#0,SET-f#1,SET] 000009:[f#0,SET-g#0,SET] compact a-e L1 ---- 2: 000010:[a#3,SETWITHDEL-c#inf,RANGEDEL] 000011:[c#2,RANGEDEL-e#inf,RANGEDEL] 000012:[e#2,RANGEDEL-f#inf,RANGEDEL] 000013:[f#2,RANGEDEL-g#inf,RANGEDEL] 3: 000006:[a#0,SET-b#0,SET] 000007:[c#0,SET-d#0,SET] 000008:[e#0,SET-f#1,SET] 000009:[f#0,SET-g#0,SET] # A range tombstone covers multiple grandparent file boundaries between point keys, # rather than after all point keys. define target-file-sizes=(1, 1, 1, 1) L1 a.SET.3:v h.SET.3:v L2 a.RANGEDEL.2:g L3 a.SET.0:v b.SET.0:v L3 c.SET.0:v d.SET.0:v L3 e.SET.0:v f.SET.1:v ---- 1: 000004:[a#3,SET-h#3,SET] 2: 000005:[a#2,RANGEDEL-g#inf,RANGEDEL] 3: 000006:[a#0,SET-b#0,SET] 000007:[c#0,SET-d#0,SET] 000008:[e#0,SET-f#1,SET] compact a-e L1 ---- 2: 000009:[a#3,SETWITHDEL-c#inf,RANGEDEL] 000010:[c#2,RANGEDEL-h#3,SET] 3: 000006:[a#0,SET-b#0,SET] 000007:[c#0,SET-d#0,SET] 000008:[e#0,SET-f#1,SET] # A range tombstone is the first and only item output by a compaction, and it # extends past the grandparent file boundary used to limit the size of future # compactions. Verify the range tombstone is split at that file boundary. define target-file-sizes=(1, 1, 1, 1) L1 a.RANGEDEL.3:e L2 a.SET.2:v L3 a.SET.0:v b.SET.0:v L3 c.SET.0:v d.SET.0:v ---- 1: 000004:[a#3,RANGEDEL-e#inf,RANGEDEL] 2: 000005:[a#2,SET-a#2,SET] 3: 000006:[a#0,SET-b#0,SET] 000007:[c#0,SET-d#0,SET] compact a-e L1 ---- 2: 000008:[a#3,RANGEDEL-c#inf,RANGEDEL] 000009:[c#3,RANGEDEL-e#inf,RANGEDEL] 3: 000006:[a#0,SET-b#0,SET] 000007:[c#0,SET-d#0,SET] # An elided range tombstone is the first item encountered by a compaction, # and the grandparent limit set by it extends to the next item, also a range # tombstone. The first item should be elided, and the second item should # reset the grandparent limit. define target-file-sizes=(100, 100, 100, 100) L1 a.RANGEDEL.4:d L1 grandparent.RANGEDEL.2:z h.SET.3:v L2 grandparent.SET.1:v L3 grandparent.SET.0:v L3 m.SET.0:v ---- 1: 000004:[a#4,RANGEDEL-d#inf,RANGEDEL] 000005:[grandparent#2,RANGEDEL-z#inf,RANGEDEL] 2: 000006:[grandparent#1,SET-grandparent#1,SET] 3: 000007:[grandparent#0,SET-grandparent#0,SET] 000008:[m#0,SET-m#0,SET] compact a-h L1 ---- 2: 000009:[grandparent#2,RANGEDEL-m#inf,RANGEDEL] 000010:[m#2,RANGEDEL-z#inf,RANGEDEL] 3: 000007:[grandparent#0,SET-grandparent#0,SET] 000008:[m#0,SET-m#0,SET] # Setup such that grandparent overlap limit is exceeded multiple times at the same user key ("b"). # Ensures the compaction output files are non-overlapping. define target-file-sizes=(1, 1, 1, 1) L1 a.SET.2:v c.SET.2:v L2 a.RANGEDEL.3:c L3 b.SET.2:v L3 b.SET.1:v L3 b.SET.0:v ---- 1: 000004:[a#2,SET-c#2,SET] 2: 000005:[a#3,RANGEDEL-c#inf,RANGEDEL] 3: 000006:[b#2,SET-b#2,SET] 000007:[b#1,SET-b#1,SET] 000008:[b#0,SET-b#0,SET] compact a-c L1 ---- 2: 000009:[a#3,RANGEDEL-b#inf,RANGEDEL] 000010:[b#3,RANGEDEL-c#2,SET] 3: 000006:[b#2,SET-b#2,SET] 000007:[b#1,SET-b#1,SET] 000008:[b#0,SET-b#0,SET] # Regression test for a bug where compaction would stop process range # tombstones for an input level upon finding an sstable in the input # level with no range tombstones. In the scenario below, sstable 6 # does not contain any range tombstones while sstable 7 does. Both are # compacted together with sstable 5. reset ---- batch set a 1 set b 1 set c 1 set d 1 set z 1 ---- compact a-z ---- 6: 000005:[a#10,SET-z#14,SET] build ext1 set a 2 ---- build ext2 set b 2 del-range c z ---- ingest ext1 ext2 ---- 0.0: 000006:[a#15,SET-a#15,SET] 000007:[b#16,SET-z#inf,RANGEDEL] 6: 000005:[a#10,SET-z#14,SET] iter first next next next ---- a: (2, .) b: (2, .) z: (1, .) . compact a-z ---- 6: 000008:[a#0,SET-z#0,SET] iter first next next next ---- a: (2, .) b: (2, .) z: (1, .) . # Regresion test for a bug in sstable smallest boundary generation # where the smallest key for an sstable was set to a key "larger" than # the start key of the first range tombstone. This in turn fouled up # the processing logic of range tombstones used by mergingIter which # allowed stepping out of an sstable even though it contained a range # tombstone that covered keys in lower levels. define target-file-sizes=(1, 1, 1, 1) L0 c.SET.4:4 L1 a.SET.3:3 L2 a.RANGEDEL.2:e L3 b.SET.1:1 ---- 0.0: 000004:[c#4,SET-c#4,SET] 1: 000005:[a#3,SET-a#3,SET] 2: 000006:[a#2,RANGEDEL-e#inf,RANGEDEL] 3: 000007:[b#1,SET-b#1,SET] compact a-e L1 ---- 0.0: 000004:[c#4,SET-c#4,SET] 2: 000008:[a#3,SETWITHDEL-b#inf,RANGEDEL] 000009:[b#2,RANGEDEL-e#inf,RANGEDEL] 3: 000007:[b#1,SET-b#1,SET] # We should only see a:3 and c:4 at this point. iter first next next ---- a: (3, .) c: (4, .) . # The bug allowed seeing b:1 during reverse iteration. iter last prev prev ---- c: (4, .) a: (3, .) . # This is a similar scenario to the one above. In older versions of Pebble this # case necessitated adjusting the seqnum of the range tombstone to # prev.LargestKey.SeqNum-1. We no longer allow user keys to be split across # sstables, and the seqnum adjustment is no longer necessary. # # Note the target-file-size of 26 is specially tailored to get the # desired compaction output. define target-file-sizes=(26, 26, 26, 26) snapshots=(1, 2, 3) L1 a.SET.4:4 L1 b.SET.2:2 b.RANGEDEL.3:e L3 b.SET.1:1 ---- 1: 000004:[a#4,SET-a#4,SET] 000005:[b#3,RANGEDEL-e#inf,RANGEDEL] 3: 000006:[b#1,SET-b#1,SET] compact a-e L1 ---- 2: 000007:[a#4,SET-a#4,SET] 000008:[b#3,RANGEDEL-e#inf,RANGEDEL] 3: 000006:[b#1,SET-b#1,SET] iter first next last prev ---- a: (4, .) . a: (4, .) . # Similar to the preceding scenario, except the range tombstone has # the same seqnum as the largest key in the preceding file. define target-file-sizes=(26, 26, 26, 26) snapshots=(1, 2, 3) L1 a.SET.4:4 L1 b.SET.3:3 b.RANGEDEL.3:e L3 b.SET.1:1 ---- 1: 000004:[a#4,SET-a#4,SET] 000005:[b#3,RANGEDEL-e#inf,RANGEDEL] 3: 000006:[b#1,SET-b#1,SET] compact a-e L1 ---- 2: 000007:[a#4,SET-a#4,SET] 000008:[b#3,RANGEDEL-e#inf,RANGEDEL] 3: 000006:[b#1,SET-b#1,SET] iter first next next last prev prev ---- a: (4, .) b: (3, .) . b: (3, .) a: (4, .) . # Similar to the preceding scenario, except the range tombstone has # a smaller seqnum than the largest key in the preceding file. define target-file-sizes=(26, 26, 26, 26) snapshots=(1, 2, 3) L1 a.SET.4:4 L1 b.SET.4:4 b.RANGEDEL.2:e L3 b.SET.1:1 ---- 1: 000004:[a#4,SET-a#4,SET] 000005:[b#4,SET-e#inf,RANGEDEL] 3: 000006:[b#1,SET-b#1,SET] compact a-e L1 ---- 2: 000007:[a#4,SET-a#4,SET] 000008:[b#4,SET-e#inf,RANGEDEL] 3: 000006:[b#1,SET-b#1,SET] iter first next next last prev prev ---- a: (4, .) b: (4, .) . b: (4, .) a: (4, .) . # Test a scenario where the last point key in an sstable has a seqnum # of 0. define target-file-sizes=(1, 1, 26) snapshots=(2) L1 a.SET.3:3 b.RANGEDEL.3:e b.SET.0:0 L3 a.RANGEDEL.2:b L3 c.SET.0:0 d.SET.0:0 ---- 1: 000004:[a#3,SET-e#inf,RANGEDEL] 3: 000005:[a#2,RANGEDEL-b#inf,RANGEDEL] 000006:[c#0,SET-d#0,SET] iter last prev ---- a: (3, .) . compact a-e L1 ---- 2: 000007:[a#3,SET-c#inf,RANGEDEL] 000008:[c#3,RANGEDEL-e#inf,RANGEDEL] 3: 000005:[a#2,RANGEDEL-b#inf,RANGEDEL] 000006:[c#0,SET-d#0,SET] iter last prev ---- a: (3, .) . # Test a scenario where the last point key in an sstable before the # grandparent limit is reached has a seqnum of 0. We want to cut the # sstable after the next point key is added, rather than continuing to # add keys indefinitely (or till the size limit is reached). define target-file-sizes=(100, 1, 52) snapshots=(2) L1 a.SET.3:3 b.RANGEDEL.3:e b.SET.0:0 c.SET.3:1 d.SET.1:1 L3 c.RANGEDEL.2:d ---- 1: 000004:[a#3,SET-e#inf,RANGEDEL] 3: 000005:[c#2,RANGEDEL-d#inf,RANGEDEL] compact a-f L1 ---- 2: 000006:[a#3,SET-c#inf,RANGEDEL] 000007:[c#3,RANGEDEL-e#inf,RANGEDEL] 3: 000005:[c#2,RANGEDEL-d#inf,RANGEDEL] # Test a scenario where we the last point key in an sstable has a # seqnum of 0, but there is another range tombstone later in the # compaction. This scenario was previously triggering an assertion due # to the rangedel.Fragmenter being finished prematurely. define target-file-sizes=(1, 1, 1) L1 a.SET.0:0 c.RANGEDEL.1:d L3 b.SET.0:0 ---- 1: 000004:[a#0,SET-d#inf,RANGEDEL] 3: 000005:[b#0,SET-b#0,SET] compact a-e L1 ---- 2: 000006:[a#0,SET-a#0,SET] 3: 000005:[b#0,SET-b#0,SET] define target-file-sizes=(1, 1, 1, 1) L0 b.SET.1:v L0 a.SET.2:v ---- 0.0: 000005:[a#2,SET-a#2,SET] 000004:[b#1,SET-b#1,SET] add-ongoing-compaction startLevel=0 outputLevel=1 start=a end=z ---- async-compact a-b L0 ---- manual compaction blocked until ongoing finished 1: 000006:[a#0,SET-a#0,SET] 000007:[b#0,SET-b#0,SET] compact a-b L1 ---- 2: 000008:[a#0,SET-a#0,SET] 000009:[b#0,SET-b#0,SET] add-ongoing-compaction startLevel=0 outputLevel=1 start=a end=z ---- async-compact a-b L2 ---- manual compaction blocked until ongoing finished 3: 000010:[a#0,SET-a#0,SET] 000011:[b#0,SET-b#0,SET] add-ongoing-compaction startLevel=0 outputLevel=1 start=a end=z ---- set-concurrent-compactions num=2 ---- async-compact a-b L3 ---- manual compaction did not block for ongoing 4: 000012:[a#0,SET-a#0,SET] 000013:[b#0,SET-b#0,SET] remove-ongoing-compaction ---- add-ongoing-compaction startLevel=4 outputLevel=5 start=a end=b ---- async-compact a-b L4 ---- manual compaction blocked until ongoing finished 5: 000014:[a#0,SET-a#0,SET] 000015:[b#0,SET-b#0,SET] # Test of a scenario where consecutive elided range tombstones and grandparent # boundaries could result in an invariant violation in the rangedel fragmenter. define target-file-sizes=(1, 1, 1, 1) L1 a.RANGEDEL.4:b c.RANGEDEL.4:d e.RANGEDEL.4:f L1 g.RANGEDEL.6:h i.RANGEDEL.4:j L1 k.RANGEDEL.5:q m.RANGEDEL.4:q L2 a.SET.2:foo L3 a.SET.1:foo c.SET.1:foo L3 ff.SET.1:v L3 k.SET.1:foo ---- 1: 000004:[a#4,RANGEDEL-f#inf,RANGEDEL] 000005:[g#6,RANGEDEL-j#inf,RANGEDEL] 000006:[k#5,RANGEDEL-q#inf,RANGEDEL] 2: 000007:[a#2,SET-a#2,SET] 3: 000008:[a#1,SET-c#1,SET] 000009:[ff#1,SET-ff#1,SET] 000010:[k#1,SET-k#1,SET] compact a-q L1 ---- 2: 000011:[a#4,RANGEDEL-d#inf,RANGEDEL] 000012:[k#5,RANGEDEL-m#inf,RANGEDEL] 3: 000008:[a#1,SET-c#1,SET] 000009:[ff#1,SET-ff#1,SET] 000010:[k#1,SET-k#1,SET] # Test a case where a new output file is started, there are no previous output # files, there are no additional keys (key = nil) and the rangedel fragmenter # is non-empty. define target-file-sizes=(1, 1, 1) L1 a.RANGEDEL.10:b d.RANGEDEL.9:e q.RANGEDEL.8:r L2 g.RANGEDEL.7:h L3 q.SET.6:6 ---- 1: 000004:[a#10,RANGEDEL-r#inf,RANGEDEL] 2: 000005:[g#7,RANGEDEL-h#inf,RANGEDEL] 3: 000006:[q#6,SET-q#6,SET] compact a-r L1 ---- 2: 000007:[q#8,RANGEDEL-r#inf,RANGEDEL] 3: 000006:[q#6,SET-q#6,SET] define target-file-sizes=(100, 100, 100) L1 a.RANGEDEL.10:b b.SET.0:foo d.RANGEDEL.0:e j.SET.10:foo L2 f.RANGEDEL.7:g L3 c.SET.6:6 L3 c.SET.5:5 L3 c.SET.4:4 L4 a.SET.0:0 f.SET.0:0 ---- 1: 000004:[a#10,RANGEDEL-j#10,SET] 2: 000005:[f#7,RANGEDEL-g#inf,RANGEDEL] 3: 000006:[c#6,SET-c#6,SET] 000007:[c#5,SET-c#5,SET] 000008:[c#4,SET-c#4,SET] 4: 000009:[a#0,SET-f#0,SET] compact a-r L1 ---- 2: 000010:[a#10,RANGEDEL-b#0,SET] 000011:[d#0,RANGEDEL-j#10,SET] 3: 000006:[c#6,SET-c#6,SET] 000007:[c#5,SET-c#5,SET] 000008:[c#4,SET-c#4,SET] 4: 000009:[a#0,SET-f#0,SET] # Test a snapshot that separates a range deletion from all the data that it # deletes. Ensure that we respect the target-file-size and split into multiple # outputs. define target-file-sizes=(1, 1, 1) snapshots=(14) L1 a.RANGEDEL.15:z b.SET.11:foo c.SET.11:foo L2 c.SET.0:foo d.SET.0:foo ---- 1: 000004:[a#15,RANGEDEL-z#inf,RANGEDEL] 2: 000005:[c#0,SET-d#0,SET] compact a-z L1 ---- 2: 000006:[a#15,RANGEDEL-c#inf,RANGEDEL] 000007:[c#15,RANGEDEL-d#inf,RANGEDEL] 000008:[d#15,RANGEDEL-z#inf,RANGEDEL] # Test an interaction between a range deletion that will be elided with # output splitting. Ensure that the output is still split (previous versions # of the code did not, because of intricacies around preventing a zero # sequence number in an output's largest key). define target-file-sizes=(1, 1, 1) L1 a.RANGEDEL.10:z b.SET.11:foo c.SET.11:foo L2 c.SET.0:foo d.SET.0:foo ---- 1: 000004:[a#10,RANGEDEL-z#inf,RANGEDEL] 2: 000005:[c#0,SET-d#0,SET] compact a-z L1 ---- 2: 000006:[b#0,SET-b#0,SET] 000007:[c#0,SET-c#0,SET]