// Copyright 2021 The LevelDB-Go and Pebble Authors. All rights reserved. Use // of this source code is governed by a BSD-style license that can be found in // the LICENSE file. package sstable import ( "bytes" "context" "fmt" "io" "math" "math/rand" "sort" "strconv" "strings" "testing" "github.com/cockroachdb/datadriven" "github.com/cockroachdb/errors" "github.com/cockroachdb/pebble/internal/base" "github.com/cockroachdb/pebble/internal/rangekey" "github.com/cockroachdb/pebble/internal/testkeys" "github.com/stretchr/testify/require" ) func TestIntervalEncodeDecode(t *testing.T) { testCases := []struct { name string lower uint64 upper uint64 len int }{ { name: "empty zero", lower: 0, upper: 0, len: 0, }, { name: "empty non-zero", lower: 5, upper: 5, len: 0, }, { name: "empty lower > upper", lower: math.MaxUint64, upper: math.MaxUint64 - 1, len: 0, }, { name: "small", lower: 50, upper: 61, len: 2, }, { name: "big", lower: 0, upper: math.MaxUint64, len: 11, }, } for _, tc := range testCases { buf := make([]byte, 100) t.Run(tc.name, func(t *testing.T) { i1 := interval{lower: tc.lower, upper: tc.upper} b1 := i1.encode(nil) b2 := i1.encode(buf[:0]) require.True(t, bytes.Equal(b1, b2), "%x != %x", b1, b2) expectedInterval := i1 if expectedInterval.lower >= expectedInterval.upper { expectedInterval = interval{} } // Arbitrary initial value. arbitraryInterval := interval{lower: 1000, upper: 1000} i2 := arbitraryInterval i2.decode(b1) require.Equal(t, expectedInterval, i2) i2 = arbitraryInterval i2.decode(b2) require.Equal(t, expectedInterval, i2) require.Equal(t, tc.len, len(b1)) }) } } func TestIntervalUnionIntersects(t *testing.T) { testCases := []struct { name string i1 interval i2 interval union interval intersects bool }{ { name: "empty and empty", i1: interval{}, i2: interval{}, union: interval{}, intersects: false, }, { name: "empty and empty non-zero", i1: interval{}, i2: interval{100, 99}, union: interval{}, intersects: false, }, { name: "empty and non-empty", i1: interval{}, i2: interval{80, 100}, union: interval{80, 100}, intersects: false, }, { name: "disjoint sets", i1: interval{50, 60}, i2: interval{math.MaxUint64 - 5, math.MaxUint64}, union: interval{50, math.MaxUint64}, intersects: false, }, { name: "adjacent sets", i1: interval{50, 60}, i2: interval{60, 100}, union: interval{50, 100}, intersects: false, }, { name: "overlapping sets", i1: interval{50, 60}, i2: interval{59, 120}, union: interval{50, 120}, intersects: true, }, } isEmpty := func(i interval) bool { return i.lower >= i.upper } // adjustUnionExpectation exists because union does not try to // canonicalize empty sets by turning them into [0, 0), since it is // unnecessary -- the higher level context of the BlockIntervalCollector // will do so when calling interval.encode. adjustUnionExpectation := func(expected interval, i1 interval, i2 interval) interval { if isEmpty(i2) { return i1 } if isEmpty(i1) { return i2 } return expected } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { require.Equal(t, tc.intersects, tc.i1.intersects(tc.i2)) require.Equal(t, tc.intersects, tc.i2.intersects(tc.i1)) require.Equal(t, !isEmpty(tc.i1), tc.i1.intersects(tc.i1)) require.Equal(t, !isEmpty(tc.i2), tc.i2.intersects(tc.i2)) union := tc.i1 union.union(tc.i2) require.Equal(t, adjustUnionExpectation(tc.union, tc.i1, tc.i2), union) union = tc.i2 union.union(tc.i1) require.Equal(t, adjustUnionExpectation(tc.union, tc.i2, tc.i1), union) }) } } type testDataBlockIntervalCollector struct { i interval } func (c *testDataBlockIntervalCollector) Add(key InternalKey, value []byte) error { return nil } func (c *testDataBlockIntervalCollector) FinishDataBlock() (lower uint64, upper uint64, err error) { return c.i.lower, c.i.upper, nil } func TestBlockIntervalCollector(t *testing.T) { var points, ranges testDataBlockIntervalCollector bic := NewBlockIntervalCollector("foo", &points, &ranges) require.Equal(t, "foo", bic.Name()) // Set up the point key collector with an initial (empty) interval. points.i = interval{1, 1} // First data block has empty point key interval. encoded, err := bic.FinishDataBlock(nil) require.NoError(t, err) require.True(t, bytes.Equal(nil, encoded)) bic.AddPrevDataBlockToIndexBlock() // Second data block contains a point and range key interval. The latter // should not contribute to the block interval. points.i = interval{20, 25} ranges.i = interval{5, 150} encoded, err = bic.FinishDataBlock(nil) require.NoError(t, err) var decoded interval require.NoError(t, decoded.decode(encoded)) require.Equal(t, interval{20, 25}, decoded) var encodedIndexBlock []byte // Finish index block before including second data block. encodedIndexBlock, err = bic.FinishIndexBlock(nil) require.NoError(t, err) require.True(t, bytes.Equal(nil, encodedIndexBlock)) bic.AddPrevDataBlockToIndexBlock() // Third data block. points.i = interval{10, 15} encoded, err = bic.FinishDataBlock(nil) require.NoError(t, err) require.NoError(t, decoded.decode(encoded)) require.Equal(t, interval{10, 15}, decoded) bic.AddPrevDataBlockToIndexBlock() // Fourth data block. points.i = interval{100, 105} encoded, err = bic.FinishDataBlock(nil) require.NoError(t, err) require.NoError(t, decoded.decode(encoded)) require.Equal(t, interval{100, 105}, decoded) // Finish index block before including fourth data block. encodedIndexBlock, err = bic.FinishIndexBlock(nil) require.NoError(t, err) require.NoError(t, decoded.decode(encodedIndexBlock)) require.Equal(t, interval{10, 25}, decoded) bic.AddPrevDataBlockToIndexBlock() // Finish index block that contains only fourth data block. encodedIndexBlock, err = bic.FinishIndexBlock(nil) require.NoError(t, err) require.NoError(t, decoded.decode(encodedIndexBlock)) require.Equal(t, interval{100, 105}, decoded) var encodedTable []byte // Finish table. The table interval is the union of the current point key // table interval [10, 105) and the range key interval [5, 150). encodedTable, err = bic.FinishTable(nil) require.NoError(t, err) require.NoError(t, decoded.decode(encodedTable)) require.Equal(t, interval{5, 150}, decoded) } func TestBlockIntervalFilter(t *testing.T) { testCases := []struct { name string filter interval prop interval intersects bool }{ { name: "non-empty and empty", filter: interval{10, 15}, prop: interval{}, intersects: false, }, { name: "does not intersect", filter: interval{10, 15}, prop: interval{15, 20}, intersects: false, }, { name: "intersects", filter: interval{10, 15}, prop: interval{14, 20}, intersects: true, }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { var points testDataBlockIntervalCollector name := "foo" bic := NewBlockIntervalCollector(name, &points, nil) bif := NewBlockIntervalFilter(name, tc.filter.lower, tc.filter.upper) points.i = tc.prop prop, _ := bic.FinishDataBlock(nil) intersects, err := bif.Intersects(prop) require.NoError(t, err) require.Equal(t, tc.intersects, intersects) }) } } func TestBlockPropertiesEncoderDecoder(t *testing.T) { var encoder blockPropertiesEncoder scratch := encoder.getScratchForProp() scratch = append(scratch, []byte("foo")...) encoder.addProp(1, scratch) scratch = encoder.getScratchForProp() require.LessOrEqual(t, 3, cap(scratch)) scratch = append(scratch, []byte("cockroach")...) encoder.addProp(10, scratch) props1 := encoder.props() unsafeProps := encoder.unsafeProps() require.True(t, bytes.Equal(props1, unsafeProps), "%x != %x", props1, unsafeProps) decodeProps1 := func() { decoder := blockPropertiesDecoder{props: props1} require.False(t, decoder.done()) id, prop, err := decoder.next() require.NoError(t, err) require.Equal(t, shortID(1), id) require.Equal(t, string(prop), "foo") require.False(t, decoder.done()) id, prop, err = decoder.next() require.NoError(t, err) require.Equal(t, shortID(10), id) require.Equal(t, string(prop), "cockroach") require.True(t, decoder.done()) } decodeProps1() encoder.resetProps() scratch = encoder.getScratchForProp() require.LessOrEqual(t, 9, cap(scratch)) scratch = append(scratch, []byte("bar")...) encoder.addProp(10, scratch) props2 := encoder.props() unsafeProps = encoder.unsafeProps() require.True(t, bytes.Equal(props2, unsafeProps), "%x != %x", props2, unsafeProps) // Safe props should still decode. decodeProps1() // Decode props2 decoder := blockPropertiesDecoder{props: props2} require.False(t, decoder.done()) id, prop, err := decoder.next() require.NoError(t, err) require.Equal(t, shortID(10), id) require.Equal(t, string(prop), "bar") require.True(t, decoder.done()) } // filterWithTrueForEmptyProp is a wrapper for BlockPropertyFilter that // delegates to it except when the property is empty, in which case it returns // true. type filterWithTrueForEmptyProp struct { BlockPropertyFilter } func (b filterWithTrueForEmptyProp) Intersects(prop []byte) (bool, error) { if len(prop) == 0 { return true, nil } return b.BlockPropertyFilter.Intersects(prop) } func TestBlockPropertiesFilterer_IntersectsUserPropsAndFinishInit(t *testing.T) { // props with id=0, interval [10, 20); id=10, interval [110, 120). var dbic testDataBlockIntervalCollector bic0 := NewBlockIntervalCollector("p0", &dbic, nil) bic0Id := byte(0) bic10 := NewBlockIntervalCollector("p10", &dbic, nil) bic10Id := byte(10) dbic.i = interval{10, 20} prop0 := append([]byte(nil), bic0Id) _, err := bic0.FinishDataBlock(nil) require.NoError(t, err) prop0, err = bic0.FinishTable(prop0) require.NoError(t, err) dbic.i = interval{110, 120} prop10 := append([]byte(nil), bic10Id) _, err = bic10.FinishDataBlock(nil) require.NoError(t, err) prop10, err = bic10.FinishTable(prop10) require.NoError(t, err) prop0Str := string(prop0) prop10Str := string(prop10) type filter struct { name string i interval } testCases := []struct { name string userProps map[string]string filters []filter // Expected results intersects bool shortIDToFiltersIndex []int }{ { name: "no filter, no props", userProps: map[string]string{}, filters: nil, intersects: true, }, { name: "no props", userProps: map[string]string{}, filters: []filter{ {name: "p0", i: interval{20, 30}}, {name: "p10", i: interval{20, 30}}, }, intersects: true, }, { name: "prop0, does not intersect", userProps: map[string]string{"p0": prop0Str}, filters: []filter{ {name: "p0", i: interval{20, 30}}, {name: "p10", i: interval{20, 30}}, }, intersects: false, }, { name: "prop0, intersects", userProps: map[string]string{"p0": prop0Str}, filters: []filter{ {name: "p0", i: interval{11, 21}}, {name: "p10", i: interval{20, 30}}, }, intersects: true, shortIDToFiltersIndex: []int{0}, }, { name: "prop10, does not intersect", userProps: map[string]string{"p10": prop10Str}, filters: []filter{ {name: "p0", i: interval{11, 21}}, {name: "p10", i: interval{20, 30}}, }, intersects: false, }, { name: "prop10, intersects", userProps: map[string]string{"p10": prop10Str}, filters: []filter{ {name: "p0", i: interval{11, 21}}, {name: "p10", i: interval{115, 125}}, }, intersects: true, shortIDToFiltersIndex: []int{-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 1}, }, { name: "prop10, intersects", userProps: map[string]string{"p10": prop10Str}, filters: []filter{ {name: "p10", i: interval{115, 125}}, {name: "p0", i: interval{11, 21}}, }, intersects: true, shortIDToFiltersIndex: []int{-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 0}, }, { name: "prop0 and prop10, does not intersect", userProps: map[string]string{"p0": prop0Str, "p10": prop10Str}, filters: []filter{ {name: "p10", i: interval{115, 125}}, {name: "p0", i: interval{20, 30}}, }, intersects: false, shortIDToFiltersIndex: []int{-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 0}, }, { name: "prop0 and prop10, does not intersect", userProps: map[string]string{"p0": prop0Str, "p10": prop10Str}, filters: []filter{ {name: "p0", i: interval{10, 20}}, {name: "p10", i: interval{125, 135}}, }, intersects: false, shortIDToFiltersIndex: []int{0}, }, { name: "prop0 and prop10, intersects", userProps: map[string]string{"p0": prop0Str, "p10": prop10Str}, filters: []filter{ {name: "p10", i: interval{115, 125}}, {name: "p0", i: interval{10, 20}}, }, intersects: true, shortIDToFiltersIndex: []int{1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 0}, }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { var filters []BlockPropertyFilter for _, f := range tc.filters { filter := NewBlockIntervalFilter(f.name, f.i.lower, f.i.upper) filters = append(filters, filter) } filterer := newBlockPropertiesFilterer(filters, nil) intersects, err := filterer.intersectsUserPropsAndFinishInit(tc.userProps) require.NoError(t, err) require.Equal(t, tc.intersects, intersects) require.Equal(t, tc.shortIDToFiltersIndex, filterer.shortIDToFiltersIndex) }) } } func TestBlockPropertiesFilterer_Intersects(t *testing.T) { // Setup two different properties values to filter against. var emptyProps []byte // props with id=0, interval [10, 20); id=10, interval [110, 120). var encoder blockPropertiesEncoder var dbic testDataBlockIntervalCollector bic0 := NewBlockIntervalCollector("", &dbic, nil) bic0Id := shortID(0) bic10 := NewBlockIntervalCollector("", &dbic, nil) bic10Id := shortID(10) dbic.i = interval{10, 20} prop, err := bic0.FinishDataBlock(encoder.getScratchForProp()) require.NoError(t, err) encoder.addProp(bic0Id, prop) dbic.i = interval{110, 120} prop, err = bic10.FinishDataBlock(encoder.getScratchForProp()) require.NoError(t, err) encoder.addProp(bic10Id, prop) props0And10 := encoder.props() type filter struct { shortID shortID i interval intersectsForEmptyProp bool } testCases := []struct { name string props []byte // filters must be in ascending order of shortID. filters []filter intersects bool }{ { name: "no filter, empty props", props: emptyProps, intersects: true, }, { name: "no filter", props: props0And10, intersects: true, }, { name: "filter 0, empty props, does not intersect", props: emptyProps, filters: []filter{ { shortID: 0, i: interval{5, 15}, }, }, intersects: false, }, { name: "filter 10, empty props, does not intersect", props: emptyProps, filters: []filter{ { shortID: 0, i: interval{105, 111}, }, }, intersects: false, }, { name: "filter 0, intersects", props: props0And10, filters: []filter{ { shortID: 0, i: interval{5, 15}, }, }, intersects: true, }, { name: "filter 0, does not intersect", props: props0And10, filters: []filter{ { shortID: 0, i: interval{20, 25}, }, }, intersects: false, }, { name: "filter 10, intersects", props: props0And10, filters: []filter{ { shortID: 10, i: interval{105, 111}, }, }, intersects: true, }, { name: "filter 10, does not intersect", props: props0And10, filters: []filter{ { shortID: 10, i: interval{105, 110}, }, }, intersects: false, }, { name: "filter 5, does not intersect since no property", props: props0And10, filters: []filter{ { shortID: 5, i: interval{105, 110}, }, }, intersects: false, }, { name: "filter 0 and 5, intersects and not intersects means overall not intersects", props: props0And10, filters: []filter{ { shortID: 0, i: interval{5, 15}, }, { shortID: 5, i: interval{105, 110}, }, }, intersects: false, }, { name: "filter 0, 5, 7, 11, all intersect", props: props0And10, filters: []filter{ { shortID: 0, i: interval{5, 15}, }, { shortID: 5, i: interval{105, 110}, intersectsForEmptyProp: true, }, { shortID: 7, i: interval{105, 110}, intersectsForEmptyProp: true, }, { shortID: 11, i: interval{105, 110}, intersectsForEmptyProp: true, }, }, intersects: true, }, { name: "filter 0, 5, 7, 10, 11, all intersect", props: props0And10, filters: []filter{ { shortID: 0, i: interval{5, 15}, }, { shortID: 5, i: interval{105, 110}, intersectsForEmptyProp: true, }, { shortID: 7, i: interval{105, 110}, intersectsForEmptyProp: true, }, { shortID: 10, i: interval{105, 111}, }, { shortID: 11, i: interval{105, 110}, intersectsForEmptyProp: true, }, }, intersects: true, }, { name: "filter 0, 5, 7, 10, 11, all intersect except for 10", props: props0And10, filters: []filter{ { shortID: 0, i: interval{5, 15}, }, { shortID: 5, i: interval{105, 110}, intersectsForEmptyProp: true, }, { shortID: 7, i: interval{105, 110}, intersectsForEmptyProp: true, }, { shortID: 10, i: interval{105, 110}, }, { shortID: 11, i: interval{105, 110}, intersectsForEmptyProp: true, }, }, intersects: false, }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { var filters []BlockPropertyFilter var shortIDToFiltersIndex []int if len(tc.filters) > 0 { shortIDToFiltersIndex = make([]int, tc.filters[len(tc.filters)-1].shortID+1) for i := range shortIDToFiltersIndex { shortIDToFiltersIndex[i] = -1 } } for _, f := range tc.filters { filter := NewBlockIntervalFilter("", f.i.lower, f.i.upper) bpf := BlockPropertyFilter(filter) if f.intersectsForEmptyProp { bpf = filterWithTrueForEmptyProp{filter} } shortIDToFiltersIndex[f.shortID] = len(filters) filters = append(filters, bpf) } doFiltering := func() { bpFilterer := BlockPropertiesFilterer{ filters: filters, shortIDToFiltersIndex: shortIDToFiltersIndex, boundLimitedShortID: -1, } intersects, err := bpFilterer.intersects(tc.props) require.NoError(t, err) require.Equal(t, tc.intersects, intersects == blockIntersects) } doFiltering() if len(filters) > 1 { // Permute the filters so that the use of // shortIDToFiltersIndex is better tested. permutation := rand.Perm(len(filters)) filterPerm := make([]BlockPropertyFilter, len(filters)) for i := range permutation { filterPerm[i] = filters[permutation[i]] shortIDToFiltersIndex[tc.filters[permutation[i]].shortID] = i } filters = filterPerm doFiltering() } }) } } // valueCharBlockIntervalCollector implements DataBlockIntervalCollector by // maintaining the (inclusive) lower and (exclusive) upper bound of a fixed // character position in the value, when represented as an integer. type valueCharBlockIntervalCollector struct { charIdx int initialized bool lower, upper uint64 } var _ DataBlockIntervalCollector = &valueCharBlockIntervalCollector{} // Add implements DataBlockIntervalCollector by maintaining the lower and upper // bound of a fixed character position in the value. func (c *valueCharBlockIntervalCollector) Add(_ InternalKey, value []byte) error { charIdx := c.charIdx if charIdx == -1 { charIdx = len(value) - 1 } val, err := strconv.Atoi(string(value[charIdx])) if err != nil { return err } uval := uint64(val) if !c.initialized { c.lower, c.upper = uval, uval+1 c.initialized = true return nil } if uval < c.lower { c.lower = uval } if uval >= c.upper { c.upper = uval + 1 } return nil } // Finish implements DataBlockIntervalCollector, returning the lower and upper // bound for the block. The range is reset to zero in anticipation of the next // block. func (c *valueCharBlockIntervalCollector) FinishDataBlock() (lower, upper uint64, err error) { l, u := c.lower, c.upper c.lower, c.upper = 0, 0 c.initialized = false return l, u, nil } // testKeysSuffixIntervalCollector maintains an interval over the timestamps in // MVCC-like suffixes for keys (e.g. foo@123). type suffixIntervalCollector struct { initialized bool lower, upper uint64 } // Add implements DataBlockIntervalCollector by adding the timestamp(s) in the // suffix(es) of this record to the current interval. // // Note that range sets and unsets may have multiple suffixes. Range key deletes // do not have a suffix. All other point keys have a single suffix. func (c *suffixIntervalCollector) Add(key InternalKey, value []byte) error { var bs [][]byte // Range keys have their suffixes encoded into the value. if rangekey.IsRangeKey(key.Kind()) { if key.Kind() == base.InternalKeyKindRangeKeyDelete { return nil } s, err := rangekey.Decode(key, value, nil) if err != nil { return err } for _, k := range s.Keys { if len(k.Suffix) > 0 { bs = append(bs, k.Suffix) } } } else { // All other keys have a single suffix encoded into the value. bs = append(bs, key.UserKey) } for _, b := range bs { i := testkeys.Comparer.Split(b) ts, err := strconv.Atoi(string(b[i+1:])) if err != nil { return err } uts := uint64(ts) if !c.initialized { c.lower, c.upper = uts, uts+1 c.initialized = true continue } if uts < c.lower { c.lower = uts } if uts >= c.upper { c.upper = uts + 1 } } return nil } // FinishDataBlock implements DataBlockIntervalCollector. func (c *suffixIntervalCollector) FinishDataBlock() (lower, upper uint64, err error) { l, u := c.lower, c.upper c.lower, c.upper = 0, 0 c.initialized = false return l, u, nil } func TestBlockProperties(t *testing.T) { var r *Reader defer func() { if r != nil { require.NoError(t, r.Close()) } }() var stats base.InternalIteratorStats datadriven.RunTest(t, "testdata/block_properties", func(t *testing.T, td *datadriven.TestData) string { switch td.Cmd { case "build": if r != nil { _ = r.Close() r = nil } var output string r, output = runBlockPropertiesBuildCmd(td) return output case "collectors": return runCollectorsCmd(r, td) case "table-props": return runTablePropsCmd(r, td) case "block-props": return runBlockPropsCmd(r, td) case "filter": var points, ranges []BlockPropertyFilter for _, cmd := range td.CmdArgs { filter, err := parseIntervalFilter(cmd) if err != nil { return err.Error() } switch cmd.Key { case "point-filter": points = append(points, filter) case "range-filter": ranges = append(ranges, filter) default: return fmt.Sprintf("unknown command: %s", td.Cmd) } } // Point keys filter matches. var buf bytes.Buffer var f *BlockPropertiesFilterer buf.WriteString("points: ") if len(points) > 0 { f = newBlockPropertiesFilterer(points, nil) ok, err := f.intersectsUserPropsAndFinishInit(r.Properties.UserProperties) if err != nil { return err.Error() } buf.WriteString(strconv.FormatBool(ok)) if !ok { f = nil } // Enumerate point key data blocks encoded into the index. if f != nil { indexH, err := r.readIndex(context.Background(), nil, nil) if err != nil { return err.Error() } defer indexH.Release() buf.WriteString(", blocks=[") var blocks []int var i int iter, _ := newBlockIter(r.Compare, indexH.Get()) for key, value := iter.First(); key != nil; key, value = iter.Next() { bh, err := decodeBlockHandleWithProperties(value.InPlaceValue()) if err != nil { return err.Error() } intersects, err := f.intersects(bh.Props) if err != nil { return err.Error() } if intersects == blockIntersects { blocks = append(blocks, i) } i++ } for i, b := range blocks { buf.WriteString(strconv.Itoa(b)) if i < len(blocks)-1 { buf.WriteString(",") } } buf.WriteString("]") } } else { // Without filters, the table matches by default. buf.WriteString("true (no filters provided)") } buf.WriteString("\n") // Range key filter matches. buf.WriteString("ranges: ") if len(ranges) > 0 { f := newBlockPropertiesFilterer(ranges, nil) ok, err := f.intersectsUserPropsAndFinishInit(r.Properties.UserProperties) if err != nil { return err.Error() } buf.WriteString(strconv.FormatBool(ok)) } else { // Without filters, the table matches by default. buf.WriteString("true (no filters provided)") } buf.WriteString("\n") return buf.String() case "iter": var lower, upper []byte var filters []BlockPropertyFilter for _, arg := range td.CmdArgs { switch arg.Key { case "lower": lower = []byte(arg.Vals[0]) case "upper": upper = []byte(arg.Vals[0]) case "point-key-filter": f, err := parseIntervalFilter(arg) if err != nil { return err.Error() } filters = append(filters, f) } } filterer := newBlockPropertiesFilterer(filters, nil) ok, err := filterer.intersectsUserPropsAndFinishInit(r.Properties.UserProperties) if err != nil { return err.Error() } else if !ok { return "filter excludes entire table" } iter, err := r.NewIterWithBlockPropertyFilters( lower, upper, filterer, false /* use (bloom) filter */, &stats, CategoryAndQoS{}, nil, TrivialReaderProvider{Reader: r}) if err != nil { return err.Error() } return runIterCmd(td, iter, false, runIterCmdEveryOpAfter(func(w io.Writer) { // After every op, point the value of MaybeFilteredKeys. fmt.Fprintf(w, " MaybeFilteredKeys()=%t", iter.MaybeFilteredKeys()) })) default: return fmt.Sprintf("unknown command: %s", td.Cmd) } }) } func TestBlockProperties_BoundLimited(t *testing.T) { var r *Reader defer func() { if r != nil { require.NoError(t, r.Close()) } }() var stats base.InternalIteratorStats datadriven.RunTest(t, "testdata/block_properties_boundlimited", func(t *testing.T, td *datadriven.TestData) string { switch td.Cmd { case "build": if r != nil { _ = r.Close() r = nil } var output string r, output = runBlockPropertiesBuildCmd(td) return output case "collectors": return runCollectorsCmd(r, td) case "table-props": return runTablePropsCmd(r, td) case "block-props": return runBlockPropsCmd(r, td) case "iter": var buf bytes.Buffer var lower, upper []byte filter := boundLimitedWrapper{ w: &buf, cmp: testkeys.Comparer.Compare, } for _, arg := range td.CmdArgs { switch arg.Key { case "lower": lower = []byte(arg.Vals[0]) case "upper": upper = []byte(arg.Vals[0]) case "filter": f, err := parseIntervalFilter(arg) if err != nil { return err.Error() } filter.inner = f case "filter-upper": ik := base.MakeInternalKey([]byte(arg.Vals[0]), 0, base.InternalKeyKindSet) filter.upper = &ik case "filter-lower": ik := base.MakeInternalKey([]byte(arg.Vals[0]), 0, base.InternalKeyKindSet) filter.lower = &ik } } if filter.inner == nil { return "missing block property filter" } filterer := newBlockPropertiesFilterer(nil, &filter) ok, err := filterer.intersectsUserPropsAndFinishInit(r.Properties.UserProperties) if err != nil { return err.Error() } else if !ok { return "filter excludes entire table" } iter, err := r.NewIterWithBlockPropertyFilters( lower, upper, filterer, false /* use (bloom) filter */, &stats, CategoryAndQoS{}, nil, TrivialReaderProvider{Reader: r}) if err != nil { return err.Error() } return runIterCmd(td, iter, false, runIterCmdEveryOp(func(w io.Writer) { // Copy the bound-limited-wrapper's accumulated output to the // iterator's writer. This interleaves its output with the // iterator output. io.Copy(w, &buf) buf.Reset() }), runIterCmdEveryOpAfter(func(w io.Writer) { // After every op, point the value of MaybeFilteredKeys. fmt.Fprintf(w, " MaybeFilteredKeys()=%t", iter.MaybeFilteredKeys()) })) default: return fmt.Sprintf("unrecognized command %q", td.Cmd) } }) } type boundLimitedWrapper struct { w io.Writer cmp base.Compare inner BlockPropertyFilter lower *InternalKey upper *InternalKey } func (bl *boundLimitedWrapper) Name() string { return bl.inner.Name() } func (bl *boundLimitedWrapper) Intersects(prop []byte) (bool, error) { propString := fmt.Sprintf("%x", prop) var i interval if err := i.decode(prop); err == nil { // If it decodes as an interval, pretty print it as an interval. propString = fmt.Sprintf("[%d, %d)", i.lower, i.upper) } v, err := bl.inner.Intersects(prop) if bl.w != nil { fmt.Fprintf(bl.w, " filter.Intersects(%s) = (%t, %v)\n", propString, v, err) } return v, err } func (bl *boundLimitedWrapper) KeyIsWithinLowerBound(key []byte) (ret bool) { if bl.lower == nil { ret = true } else { ret = bl.cmp(key, bl.lower.UserKey) >= 0 } if bl.w != nil { fmt.Fprintf(bl.w, " filter.KeyIsWithinLowerBound(%s) = %t\n", key, ret) } return ret } func (bl *boundLimitedWrapper) KeyIsWithinUpperBound(key []byte) (ret bool) { if bl.upper == nil { ret = true } else { ret = bl.cmp(key, bl.upper.UserKey) <= 0 } if bl.w != nil { fmt.Fprintf(bl.w, " filter.KeyIsWithinUpperBound(%s) = %t\n", key, ret) } return ret } func parseIntervalFilter(cmd datadriven.CmdArg) (BlockPropertyFilter, error) { name := cmd.Vals[0] minS, maxS := cmd.Vals[1], cmd.Vals[2] min, err := strconv.ParseUint(minS, 10, 64) if err != nil { return nil, err } max, err := strconv.ParseUint(maxS, 10, 64) if err != nil { return nil, err } return NewBlockIntervalFilter(name, min, max), nil } func runCollectorsCmd(r *Reader, td *datadriven.TestData) string { var lines []string for k, v := range r.Properties.UserProperties { lines = append(lines, fmt.Sprintf("%d: %s", v[0], k)) } linesSorted := sort.StringSlice(lines) linesSorted.Sort() return strings.Join(lines, "\n") } func runTablePropsCmd(r *Reader, td *datadriven.TestData) string { var lines []string for _, val := range r.Properties.UserProperties { id := shortID(val[0]) var i interval if err := i.decode([]byte(val[1:])); err != nil { return err.Error() } lines = append(lines, fmt.Sprintf("%d: [%d, %d)", id, i.lower, i.upper)) } linesSorted := sort.StringSlice(lines) linesSorted.Sort() return strings.Join(lines, "\n") } func runBlockPropertiesBuildCmd(td *datadriven.TestData) (r *Reader, out string) { opts := WriterOptions{ TableFormat: TableFormatPebblev2, IndexBlockSize: math.MaxInt32, // Default to a single level index for simplicity. } for _, cmd := range td.CmdArgs { switch cmd.Key { case "block-size": if len(cmd.Vals) != 1 { return r, fmt.Sprintf("%s: arg %s expects 1 value", td.Cmd, cmd.Key) } var err error opts.BlockSize, err = strconv.Atoi(cmd.Vals[0]) if err != nil { return r, err.Error() } case "collectors": for _, c := range cmd.Vals { var points, ranges DataBlockIntervalCollector switch c { case "value-first": points = &valueCharBlockIntervalCollector{charIdx: 0} case "value-last": points = &valueCharBlockIntervalCollector{charIdx: -1} case "suffix": points, ranges = &suffixIntervalCollector{}, &suffixIntervalCollector{} case "suffix-point-keys-only": points = &suffixIntervalCollector{} case "suffix-range-keys-only": ranges = &suffixIntervalCollector{} case "nil-points-and-ranges": points, ranges = nil, nil default: return r, fmt.Sprintf("unknown collector: %s", c) } name := c opts.BlockPropertyCollectors = append( opts.BlockPropertyCollectors, func() BlockPropertyCollector { return NewBlockIntervalCollector(name, points, ranges) }) } case "index-block-size": var err error opts.IndexBlockSize, err = strconv.Atoi(cmd.Vals[0]) if err != nil { return r, err.Error() } } } var meta *WriterMetadata var err error func() { defer func() { if r := recover(); r != nil { err = errors.Errorf("%v", r) } }() meta, r, err = runBuildCmd(td, &opts, 0) }() if err != nil { return r, err.Error() } return r, fmt.Sprintf("point: [%s,%s]\nrangedel: [%s,%s]\nrangekey: [%s,%s]\nseqnums: [%d,%d]\n", meta.SmallestPoint, meta.LargestPoint, meta.SmallestRangeDel, meta.LargestRangeDel, meta.SmallestRangeKey, meta.LargestRangeKey, meta.SmallestSeqNum, meta.LargestSeqNum) } func runBlockPropsCmd(r *Reader, td *datadriven.TestData) string { bh, err := r.readIndex(context.Background(), nil, nil) if err != nil { return err.Error() } twoLevelIndex := r.Properties.IndexPartitions > 0 i, err := newBlockIter(r.Compare, bh.Get()) if err != nil { return err.Error() } defer bh.Release() var sb strings.Builder decodeProps := func(props []byte, indent string) error { d := blockPropertiesDecoder{props: props} var lines []string for !d.done() { id, prop, err := d.next() if err != nil { return err } var i interval if err := i.decode(prop); err != nil { return err } lines = append(lines, fmt.Sprintf("%s%d: [%d, %d)\n", indent, id, i.lower, i.upper)) } linesSorted := sort.StringSlice(lines) linesSorted.Sort() for _, line := range lines { sb.WriteString(line) } return nil } for key, val := i.First(); key != nil; key, val = i.Next() { sb.WriteString(fmt.Sprintf("%s:\n", key)) bhp, err := decodeBlockHandleWithProperties(val.InPlaceValue()) if err != nil { return err.Error() } if err := decodeProps(bhp.Props, " "); err != nil { return err.Error() } // If the table has a two-level index, also decode the index // block that bhp points to, along with its block properties. if twoLevelIndex { subiter := &blockIter{} subIndex, err := r.readBlock( context.Background(), bhp.BlockHandle, nil, nil, nil, nil, nil) if err != nil { return err.Error() } if err := subiter.init( r.Compare, subIndex.Get(), 0 /* globalSeqNum */, false); err != nil { return err.Error() } for key, value := subiter.First(); key != nil; key, value = subiter.Next() { sb.WriteString(fmt.Sprintf(" %s:\n", key)) dataBH, err := decodeBlockHandleWithProperties(value.InPlaceValue()) if err != nil { return err.Error() } if err := decodeProps(dataBH.Props, " "); err != nil { return err.Error() } } subIndex.Release() } } return sb.String() } type keyCountCollector struct { name string block, index, table int } var _ BlockPropertyCollector = &keyCountCollector{} var _ SuffixReplaceableBlockCollector = &keyCountCollector{} func keyCountCollectorFn(name string) func() BlockPropertyCollector { return func() BlockPropertyCollector { return &keyCountCollector{name: name} } } func (p *keyCountCollector) Name() string { return p.name } func (p *keyCountCollector) Add(k InternalKey, _ []byte) error { if rangekey.IsRangeKey(k.Kind()) { p.table++ } else { p.block++ } return nil } func (p *keyCountCollector) FinishDataBlock(buf []byte) ([]byte, error) { buf = append(buf, []byte(strconv.Itoa(int(p.block)))...) p.table += p.block return buf, nil } func (p *keyCountCollector) AddPrevDataBlockToIndexBlock() { p.index += p.block p.block = 0 } func (p *keyCountCollector) FinishIndexBlock(buf []byte) ([]byte, error) { buf = append(buf, []byte(strconv.Itoa(int(p.index)))...) p.index = 0 return buf, nil } func (p *keyCountCollector) FinishTable(buf []byte) ([]byte, error) { buf = append(buf, []byte(strconv.Itoa(int(p.table)))...) p.table = 0 return buf, nil } func (p *keyCountCollector) UpdateKeySuffixes(old []byte, _, _ []byte) error { n, err := strconv.Atoi(string(old)) if err != nil { return err } p.block = n return nil } // intSuffixCollector is testing prop collector that collects the min and // max value of numeric suffix of keys (interpreting suffixLen bytes as ascii // for conversion with atoi). type intSuffixCollector struct { suffixLen int min, max uint64 // inclusive } func makeIntSuffixCollector(len int) intSuffixCollector { return intSuffixCollector{len, math.MaxUint64, 0} } func (p *intSuffixCollector) setFromSuffix(to []byte) error { if len(to) >= p.suffixLen { parsed, err := strconv.Atoi(string(to[len(to)-p.suffixLen:])) if err != nil { return err } p.min = uint64(parsed) p.max = uint64(parsed) } return nil } type intSuffixTablePropCollector struct { name string intSuffixCollector } var _ TablePropertyCollector = &intSuffixTablePropCollector{} var _ SuffixReplaceableTableCollector = &intSuffixTablePropCollector{} func intSuffixTablePropCollectorFn(name string, len int) func() TablePropertyCollector { return func() TablePropertyCollector { return &intSuffixTablePropCollector{name, makeIntSuffixCollector(len)} } } func (p *intSuffixCollector) Add(key InternalKey, _ []byte) error { if len(key.UserKey) > p.suffixLen { parsed, err := strconv.Atoi(string(key.UserKey[len(key.UserKey)-p.suffixLen:])) if err != nil { return err } v := uint64(parsed) if v > p.max { p.max = v } if v < p.min { p.min = v } } return nil } func (p *intSuffixTablePropCollector) Finish(userProps map[string]string) error { userProps[p.name+".min"] = fmt.Sprint(p.min) userProps[p.name+".max"] = fmt.Sprint(p.max) return nil } func (p *intSuffixTablePropCollector) Name() string { return p.name } func (p *intSuffixTablePropCollector) UpdateKeySuffixes( oldProps map[string]string, from, to []byte, ) error { return p.setFromSuffix(to) } // testIntSuffixIntervalCollector is a wrapper for testIntSuffixCollector that // uses it to implement a block interval collector. type intSuffixIntervalCollector struct { intSuffixCollector } func intSuffixIntervalCollectorFn(name string, length int) func() BlockPropertyCollector { return func() BlockPropertyCollector { return NewBlockIntervalCollector(name, &intSuffixIntervalCollector{makeIntSuffixCollector(length)}, nil) } } var _ DataBlockIntervalCollector = &intSuffixIntervalCollector{} var _ SuffixReplaceableBlockCollector = &intSuffixIntervalCollector{} func (p *intSuffixIntervalCollector) FinishDataBlock() (lower uint64, upper uint64, err error) { return p.min, p.max + 1, nil } func (p *intSuffixIntervalCollector) UpdateKeySuffixes(oldProp []byte, from, to []byte) error { return p.setFromSuffix(to) }