From 7600222879acbf99a14a42d9c413c4e338c73a0d Mon Sep 17 00:00:00 2001 From: Zach Musgrave Date: Mon, 28 Oct 2019 13:59:23 -0700 Subject: [PATCH 01/42] Fixed a bug in index assignments involving a where clause for indexes that cannot be merged together. This partial index usage resulted in pushdown getting an IndexableTable for only one side of the OR expression, returning incorrect results. Signed-off-by: Zach Musgrave --- sql/analyzer/assign_indexes.go | 23 ++-- sql/analyzer/assign_indexes_test.go | 166 +++++++++++++++++++++++----- sql/analyzer/pushdown_test.go | 6 +- 3 files changed, 158 insertions(+), 37 deletions(-) diff --git a/sql/analyzer/assign_indexes.go b/sql/analyzer/assign_indexes.go index b6bcb1327..c5c32a93b 100644 --- a/sql/analyzer/assign_indexes.go +++ b/sql/analyzer/assign_indexes.go @@ -101,12 +101,19 @@ func getIndexes(e sql.Expression, aliases map[string]sql.Expression, a *Analyzer return nil, err } - for table, idx := range leftIndexes { - if idx2, ok := rightIndexes[table]; ok && canMergeIndexes(idx.lookup, idx2.lookup) { - idx.lookup = idx.lookup.(sql.SetOperations).Union(idx2.lookup) - idx.indexes = append(idx.indexes, idx2.indexes...) + for table, leftIdx := range leftIndexes { + if rightIdx, ok := rightIndexes[table]; ok { + if canMergeIndexes(leftIdx.lookup, rightIdx.lookup) { + leftIdx.lookup = leftIdx.lookup.(sql.SetOperations).Union(rightIdx.lookup) + leftIdx.indexes = append(leftIdx.indexes, rightIdx.indexes...) + } else { + // Since we can return one index per table, if we can't merge the second index from this table, return no + // indexes. Returning a single one will lead to incorrect results from e.g. pushdown operations when only one + // side of the OR expression is used to index the table. + return nil, nil + } } - result[table] = idx + result[table] = leftIdx } // Put in the result map the indexes for tables we don't have indexes yet. @@ -161,7 +168,6 @@ func getIndexes(e sql.Expression, aliases map[string]sql.Expression, a *Analyzer lookup, errLookup = nidx.Not(values[0]) } else { lookup, errLookup = idx.Get(values[0]) - } if errLookup != nil { @@ -175,16 +181,15 @@ func getIndexes(e sql.Expression, aliases map[string]sql.Expression, a *Analyzer lookup2, errLookup = nidx.Not(v) } else { lookup2, errLookup = idx.Get(v) - } if errLookup != nil { return nil, err } - // if one of the indexes cannot be merged, return already + // if one of the indexes cannot be merged, return a nil result for this table if !canMergeIndexes(lookup, lookup2) { - return result, nil + return nil, nil } if negate { diff --git a/sql/analyzer/assign_indexes_test.go b/sql/analyzer/assign_indexes_test.go index d8e61f05a..516ff1495 100644 --- a/sql/analyzer/assign_indexes_test.go +++ b/sql/analyzer/assign_indexes_test.go @@ -16,7 +16,7 @@ func TestNegateIndex(t *testing.T) { require := require.New(t) catalog := sql.NewCatalog() - idx1 := &dummyIndex{ + idx1 := &mergableDummyIndex{ "t1", []sql.Expression{ expression.NewGetFieldWithTable(0, sql.Int64, "t1", "foo", false), @@ -61,7 +61,7 @@ func TestAssignIndexes(t *testing.T) { require := require.New(t) catalog := sql.NewCatalog() - idx1 := &dummyIndex{ + idx1 := &mergableDummyIndex{ "t2", []sql.Expression{ expression.NewGetFieldWithTable(0, sql.Int64, "t2", "bar", false), @@ -72,7 +72,7 @@ func TestAssignIndexes(t *testing.T) { close(done) <-ready - idx2 := &dummyIndex{ + idx2 := &mergableDummyIndex{ "t1", []sql.Expression{ expression.NewGetFieldWithTable(0, sql.Int64, "t1", "foo", false), @@ -84,6 +84,18 @@ func TestAssignIndexes(t *testing.T) { close(done) <-ready + idx3 := &unmergableDummyIndex{ + "t1", + []sql.Expression{ + expression.NewGetFieldWithTable(0, sql.Int64, "t1", "bar", false), + }, + } + + done, ready, err = catalog.AddIndex(idx3) + require.NoError(err) + close(done) + <-ready + a := NewDefault(catalog) t1 := memory.NewTable("t1", sql.Schema{ @@ -135,10 +147,50 @@ func TestAssignIndexes(t *testing.T) { mergeable, ok = lookupIdxs.lookup.(*mergeableIndexLookup) require.True(ok) require.True(mergeable.id == "1") + + node = plan.NewProject( + []sql.Expression{}, + plan.NewFilter( + expression.NewOr( + expression.NewEquals( + expression.NewGetFieldWithTable(0, sql.Int64, "t1", "bar", false), + expression.NewLiteral(int64(1), sql.Int64), + ), + expression.NewEquals( + expression.NewGetFieldWithTable(0, sql.Int64, "t1", "bar", false), + expression.NewLiteral(int64(2), sql.Int64), + ), + ), + plan.NewResolvedTable(t1), + ), + ) + + result, err = assignIndexes(a, node) + require.NoError(err) + + lookupIdxs, ok = result["t1"] + require.False(ok) + + node = plan.NewProject( + []sql.Expression{}, + plan.NewFilter( + expression.NewIn( + expression.NewGetFieldWithTable(0, sql.Int64, "t1", "bar", false), + expression.NewTuple(expression.NewLiteral(int64(1), sql.Int64), expression.NewLiteral(int64(2), sql.Int64)), + ), + plan.NewResolvedTable(t1), + ), + ) + + result, err = assignIndexes(a, node) + require.NoError(err) + + lookupIdxs, ok = result["t1"] + require.False(ok) } func TestGetIndexes(t *testing.T) { - indexes := []*dummyIndex{ + indexes := []*mergableDummyIndex{ { "t1", []sql.Expression{ @@ -697,7 +749,7 @@ func TestGetMultiColumnIndexes(t *testing.T) { require := require.New(t) catalog := sql.NewCatalog() - indexes := []*dummyIndex{ + indexes := []*mergableDummyIndex{ { "t1", []sql.Expression{ @@ -902,19 +954,80 @@ func (dummyIndexLookup) Values(sql.Partition) (sql.IndexValueIter, error) { return nil, nil } -type dummyIndex struct { +type mergableDummyIndex struct { table string expr []sql.Expression } -var _ sql.Index = (*dummyIndex)(nil) -var _ sql.AscendIndex = (*dummyIndex)(nil) -var _ sql.DescendIndex = (*dummyIndex)(nil) -var _ sql.NegateIndex = (*dummyIndex)(nil) +type unmergableDummyIndex struct { + table string + expr []sql.Expression +} + +func (u *unmergableDummyIndex) Database() string { return "" } +func (u *unmergableDummyIndex) Driver() string { return "" } +func (u *unmergableDummyIndex) Expressions() []string { + var exprs []string + for _, e := range u.expr { + exprs = append(exprs, e.String()) + } + return exprs +} + +func (u *unmergableDummyIndex) Get(key ...interface{}) (sql.IndexLookup, error) { + if len(key) != 1 { + var parts = make([]string, len(key)) + for i, p := range key { + parts[i] = fmt.Sprint(p) + } + + return &unmergeableIndexLookup{id: strings.Join(parts, ", ")}, nil + } + + return &unmergeableIndexLookup{id: fmt.Sprint(key[0])}, nil +} + +type unmergeableIndexLookup struct { + id string +} -func (dummyIndex) Database() string { return "" } -func (dummyIndex) Driver() string { return "" } -func (i dummyIndex) Expressions() []string { +func (u unmergeableIndexLookup) Values(sql.Partition) (sql.IndexValueIter, error) { + panic("unimplemented") +} + +func (u unmergeableIndexLookup) Indexes() []string { + return []string{u.id} +} + +func (u *unmergableDummyIndex) Has(partition sql.Partition, key ...interface{}) (bool, error) { + panic("unimplemented") +} + +func (u *unmergableDummyIndex) ID() string { + if len(u.expr) == 1 { + return u.expr[0].String() + } + var parts = make([]string, len(u.expr)) + for i, e := range u.expr { + parts[i] = e.String() + } + + return "(" + strings.Join(parts, ", ") + ")" +} + +func (u *unmergableDummyIndex) Table() string { + return u.table +} + +var _ sql.Index = (*mergableDummyIndex)(nil) +var _ sql.Index = (*unmergableDummyIndex)(nil) +var _ sql.AscendIndex = (*mergableDummyIndex)(nil) +var _ sql.DescendIndex = (*mergableDummyIndex)(nil) +var _ sql.NegateIndex = (*mergableDummyIndex)(nil) + +func (mergableDummyIndex) Database() string { return "" } +func (mergableDummyIndex) Driver() string { return "" } +func (i mergableDummyIndex) Expressions() []string { var exprs []string for _, e := range i.expr { exprs = append(exprs, e.String()) @@ -922,31 +1035,31 @@ func (i dummyIndex) Expressions() []string { return exprs } -func (i dummyIndex) AscendGreaterOrEqual(keys ...interface{}) (sql.IndexLookup, error) { +func (i mergableDummyIndex) AscendGreaterOrEqual(keys ...interface{}) (sql.IndexLookup, error) { return &ascendIndexLookup{gte: keys}, nil } -func (i dummyIndex) AscendLessThan(keys ...interface{}) (sql.IndexLookup, error) { +func (i mergableDummyIndex) AscendLessThan(keys ...interface{}) (sql.IndexLookup, error) { return &ascendIndexLookup{lt: keys}, nil } -func (i dummyIndex) AscendRange(greaterOrEqual, lessThan []interface{}) (sql.IndexLookup, error) { +func (i mergableDummyIndex) AscendRange(greaterOrEqual, lessThan []interface{}) (sql.IndexLookup, error) { return &ascendIndexLookup{gte: greaterOrEqual, lt: lessThan}, nil } -func (i dummyIndex) DescendGreater(keys ...interface{}) (sql.IndexLookup, error) { +func (i mergableDummyIndex) DescendGreater(keys ...interface{}) (sql.IndexLookup, error) { return &descendIndexLookup{gt: keys}, nil } -func (i dummyIndex) DescendLessOrEqual(keys ...interface{}) (sql.IndexLookup, error) { +func (i mergableDummyIndex) DescendLessOrEqual(keys ...interface{}) (sql.IndexLookup, error) { return &descendIndexLookup{lte: keys}, nil } -func (i dummyIndex) DescendRange(lessOrEqual, greaterThan []interface{}) (sql.IndexLookup, error) { +func (i mergableDummyIndex) DescendRange(lessOrEqual, greaterThan []interface{}) (sql.IndexLookup, error) { return &descendIndexLookup{gt: greaterThan, lte: lessOrEqual}, nil } -func (i dummyIndex) Not(keys ...interface{}) (sql.IndexLookup, error) { +func (i mergableDummyIndex) Not(keys ...interface{}) (sql.IndexLookup, error) { lookup, err := i.Get(keys...) if err != nil { return nil, err @@ -956,7 +1069,7 @@ func (i dummyIndex) Not(keys ...interface{}) (sql.IndexLookup, error) { return &negateIndexLookup{value: mergeable.id}, nil } -func (i dummyIndex) Get(key ...interface{}) (sql.IndexLookup, error) { +func (i mergableDummyIndex) Get(key ...interface{}) (sql.IndexLookup, error) { if len(key) != 1 { var parts = make([]string, len(key)) for i, p := range key { @@ -968,10 +1081,12 @@ func (i dummyIndex) Get(key ...interface{}) (sql.IndexLookup, error) { return &mergeableIndexLookup{id: fmt.Sprint(key[0])}, nil } -func (i dummyIndex) Has(sql.Partition, ...interface{}) (bool, error) { + +func (i mergableDummyIndex) Has(sql.Partition, ...interface{}) (bool, error) { panic("not implemented") } -func (i dummyIndex) ID() string { + +func (i mergableDummyIndex) ID() string { if len(i.expr) == 1 { return i.expr[0].String() } @@ -982,7 +1097,8 @@ func (i dummyIndex) ID() string { return "(" + strings.Join(parts, ", ") + ")" } -func (i dummyIndex) Table() string { return i.table } + +func (i mergableDummyIndex) Table() string { return i.table } type mergedIndexLookup struct { children []sql.IndexLookup @@ -1123,7 +1239,7 @@ func (descendIndexLookup) Intersection(...sql.IndexLookup) sql.IndexLookup { func TestIndexesIntersection(t *testing.T) { require := require.New(t) - idx1, idx2 := &dummyIndex{table: "bar"}, &dummyIndex{table: "foo"} + idx1, idx2 := &mergableDummyIndex{table: "bar"}, &mergableDummyIndex{table: "foo"} left := map[string]*indexLookup{ "a": &indexLookup{&mergeableIndexLookup{id: "a"}, nil}, diff --git a/sql/analyzer/pushdown_test.go b/sql/analyzer/pushdown_test.go index 818eb40a0..3169d12cf 100644 --- a/sql/analyzer/pushdown_test.go +++ b/sql/analyzer/pushdown_test.go @@ -105,7 +105,7 @@ func TestPushdownIndexable(t *testing.T) { catalog := sql.NewCatalog() catalog.AddDatabase(db) - idx1 := &dummyIndex{ + idx1 := &mergableDummyIndex{ "mytable", []sql.Expression{ expression.NewGetFieldWithTable(0, sql.Int32, "mytable", "i", false), @@ -116,7 +116,7 @@ func TestPushdownIndexable(t *testing.T) { close(done) <-ready - idx2 := &dummyIndex{ + idx2 := &mergableDummyIndex{ "mytable", []sql.Expression{ expression.NewGetFieldWithTable(1, sql.Float64, "mytable", "f", false), @@ -127,7 +127,7 @@ func TestPushdownIndexable(t *testing.T) { close(done) <-ready - idx3 := &dummyIndex{ + idx3 := &mergableDummyIndex{ "mytable2", []sql.Expression{ expression.NewGetFieldWithTable(0, sql.Int32, "mytable2", "i2", false), From 96e4761feb66217a953564f591f1229da81132d2 Mon Sep 17 00:00:00 2001 From: Zach Musgrave Date: Mon, 28 Oct 2019 14:00:38 -0700 Subject: [PATCH 02/42] Fixed misspelling in unmergeable. Signed-off-by: Zach Musgrave --- sql/analyzer/assign_indexes_test.go | 70 ++++++++++++++--------------- sql/analyzer/pushdown_test.go | 6 +-- 2 files changed, 38 insertions(+), 38 deletions(-) diff --git a/sql/analyzer/assign_indexes_test.go b/sql/analyzer/assign_indexes_test.go index 516ff1495..9fd9313e7 100644 --- a/sql/analyzer/assign_indexes_test.go +++ b/sql/analyzer/assign_indexes_test.go @@ -16,7 +16,7 @@ func TestNegateIndex(t *testing.T) { require := require.New(t) catalog := sql.NewCatalog() - idx1 := &mergableDummyIndex{ + idx1 := &mergeableDummyIndex{ "t1", []sql.Expression{ expression.NewGetFieldWithTable(0, sql.Int64, "t1", "foo", false), @@ -61,7 +61,7 @@ func TestAssignIndexes(t *testing.T) { require := require.New(t) catalog := sql.NewCatalog() - idx1 := &mergableDummyIndex{ + idx1 := &mergeableDummyIndex{ "t2", []sql.Expression{ expression.NewGetFieldWithTable(0, sql.Int64, "t2", "bar", false), @@ -72,7 +72,7 @@ func TestAssignIndexes(t *testing.T) { close(done) <-ready - idx2 := &mergableDummyIndex{ + idx2 := &mergeableDummyIndex{ "t1", []sql.Expression{ expression.NewGetFieldWithTable(0, sql.Int64, "t1", "foo", false), @@ -84,7 +84,7 @@ func TestAssignIndexes(t *testing.T) { close(done) <-ready - idx3 := &unmergableDummyIndex{ + idx3 := &unmergeableDummyIndex{ "t1", []sql.Expression{ expression.NewGetFieldWithTable(0, sql.Int64, "t1", "bar", false), @@ -190,7 +190,7 @@ func TestAssignIndexes(t *testing.T) { } func TestGetIndexes(t *testing.T) { - indexes := []*mergableDummyIndex{ + indexes := []*mergeableDummyIndex{ { "t1", []sql.Expression{ @@ -749,7 +749,7 @@ func TestGetMultiColumnIndexes(t *testing.T) { require := require.New(t) catalog := sql.NewCatalog() - indexes := []*mergableDummyIndex{ + indexes := []*mergeableDummyIndex{ { "t1", []sql.Expression{ @@ -954,19 +954,19 @@ func (dummyIndexLookup) Values(sql.Partition) (sql.IndexValueIter, error) { return nil, nil } -type mergableDummyIndex struct { +type mergeableDummyIndex struct { table string expr []sql.Expression } -type unmergableDummyIndex struct { +type unmergeableDummyIndex struct { table string expr []sql.Expression } -func (u *unmergableDummyIndex) Database() string { return "" } -func (u *unmergableDummyIndex) Driver() string { return "" } -func (u *unmergableDummyIndex) Expressions() []string { +func (u *unmergeableDummyIndex) Database() string { return "" } +func (u *unmergeableDummyIndex) Driver() string { return "" } +func (u *unmergeableDummyIndex) Expressions() []string { var exprs []string for _, e := range u.expr { exprs = append(exprs, e.String()) @@ -974,7 +974,7 @@ func (u *unmergableDummyIndex) Expressions() []string { return exprs } -func (u *unmergableDummyIndex) Get(key ...interface{}) (sql.IndexLookup, error) { +func (u *unmergeableDummyIndex) Get(key ...interface{}) (sql.IndexLookup, error) { if len(key) != 1 { var parts = make([]string, len(key)) for i, p := range key { @@ -999,11 +999,11 @@ func (u unmergeableIndexLookup) Indexes() []string { return []string{u.id} } -func (u *unmergableDummyIndex) Has(partition sql.Partition, key ...interface{}) (bool, error) { +func (u *unmergeableDummyIndex) Has(partition sql.Partition, key ...interface{}) (bool, error) { panic("unimplemented") } -func (u *unmergableDummyIndex) ID() string { +func (u *unmergeableDummyIndex) ID() string { if len(u.expr) == 1 { return u.expr[0].String() } @@ -1015,19 +1015,19 @@ func (u *unmergableDummyIndex) ID() string { return "(" + strings.Join(parts, ", ") + ")" } -func (u *unmergableDummyIndex) Table() string { +func (u *unmergeableDummyIndex) Table() string { return u.table } -var _ sql.Index = (*mergableDummyIndex)(nil) -var _ sql.Index = (*unmergableDummyIndex)(nil) -var _ sql.AscendIndex = (*mergableDummyIndex)(nil) -var _ sql.DescendIndex = (*mergableDummyIndex)(nil) -var _ sql.NegateIndex = (*mergableDummyIndex)(nil) +var _ sql.Index = (*mergeableDummyIndex)(nil) +var _ sql.Index = (*unmergeableDummyIndex)(nil) +var _ sql.AscendIndex = (*mergeableDummyIndex)(nil) +var _ sql.DescendIndex = (*mergeableDummyIndex)(nil) +var _ sql.NegateIndex = (*mergeableDummyIndex)(nil) -func (mergableDummyIndex) Database() string { return "" } -func (mergableDummyIndex) Driver() string { return "" } -func (i mergableDummyIndex) Expressions() []string { +func (mergeableDummyIndex) Database() string { return "" } +func (mergeableDummyIndex) Driver() string { return "" } +func (i mergeableDummyIndex) Expressions() []string { var exprs []string for _, e := range i.expr { exprs = append(exprs, e.String()) @@ -1035,31 +1035,31 @@ func (i mergableDummyIndex) Expressions() []string { return exprs } -func (i mergableDummyIndex) AscendGreaterOrEqual(keys ...interface{}) (sql.IndexLookup, error) { +func (i mergeableDummyIndex) AscendGreaterOrEqual(keys ...interface{}) (sql.IndexLookup, error) { return &ascendIndexLookup{gte: keys}, nil } -func (i mergableDummyIndex) AscendLessThan(keys ...interface{}) (sql.IndexLookup, error) { +func (i mergeableDummyIndex) AscendLessThan(keys ...interface{}) (sql.IndexLookup, error) { return &ascendIndexLookup{lt: keys}, nil } -func (i mergableDummyIndex) AscendRange(greaterOrEqual, lessThan []interface{}) (sql.IndexLookup, error) { +func (i mergeableDummyIndex) AscendRange(greaterOrEqual, lessThan []interface{}) (sql.IndexLookup, error) { return &ascendIndexLookup{gte: greaterOrEqual, lt: lessThan}, nil } -func (i mergableDummyIndex) DescendGreater(keys ...interface{}) (sql.IndexLookup, error) { +func (i mergeableDummyIndex) DescendGreater(keys ...interface{}) (sql.IndexLookup, error) { return &descendIndexLookup{gt: keys}, nil } -func (i mergableDummyIndex) DescendLessOrEqual(keys ...interface{}) (sql.IndexLookup, error) { +func (i mergeableDummyIndex) DescendLessOrEqual(keys ...interface{}) (sql.IndexLookup, error) { return &descendIndexLookup{lte: keys}, nil } -func (i mergableDummyIndex) DescendRange(lessOrEqual, greaterThan []interface{}) (sql.IndexLookup, error) { +func (i mergeableDummyIndex) DescendRange(lessOrEqual, greaterThan []interface{}) (sql.IndexLookup, error) { return &descendIndexLookup{gt: greaterThan, lte: lessOrEqual}, nil } -func (i mergableDummyIndex) Not(keys ...interface{}) (sql.IndexLookup, error) { +func (i mergeableDummyIndex) Not(keys ...interface{}) (sql.IndexLookup, error) { lookup, err := i.Get(keys...) if err != nil { return nil, err @@ -1069,7 +1069,7 @@ func (i mergableDummyIndex) Not(keys ...interface{}) (sql.IndexLookup, error) { return &negateIndexLookup{value: mergeable.id}, nil } -func (i mergableDummyIndex) Get(key ...interface{}) (sql.IndexLookup, error) { +func (i mergeableDummyIndex) Get(key ...interface{}) (sql.IndexLookup, error) { if len(key) != 1 { var parts = make([]string, len(key)) for i, p := range key { @@ -1082,11 +1082,11 @@ func (i mergableDummyIndex) Get(key ...interface{}) (sql.IndexLookup, error) { return &mergeableIndexLookup{id: fmt.Sprint(key[0])}, nil } -func (i mergableDummyIndex) Has(sql.Partition, ...interface{}) (bool, error) { +func (i mergeableDummyIndex) Has(sql.Partition, ...interface{}) (bool, error) { panic("not implemented") } -func (i mergableDummyIndex) ID() string { +func (i mergeableDummyIndex) ID() string { if len(i.expr) == 1 { return i.expr[0].String() } @@ -1098,7 +1098,7 @@ func (i mergableDummyIndex) ID() string { return "(" + strings.Join(parts, ", ") + ")" } -func (i mergableDummyIndex) Table() string { return i.table } +func (i mergeableDummyIndex) Table() string { return i.table } type mergedIndexLookup struct { children []sql.IndexLookup @@ -1239,7 +1239,7 @@ func (descendIndexLookup) Intersection(...sql.IndexLookup) sql.IndexLookup { func TestIndexesIntersection(t *testing.T) { require := require.New(t) - idx1, idx2 := &mergableDummyIndex{table: "bar"}, &mergableDummyIndex{table: "foo"} + idx1, idx2 := &mergeableDummyIndex{table: "bar"}, &mergeableDummyIndex{table: "foo"} left := map[string]*indexLookup{ "a": &indexLookup{&mergeableIndexLookup{id: "a"}, nil}, diff --git a/sql/analyzer/pushdown_test.go b/sql/analyzer/pushdown_test.go index 3169d12cf..5137daf78 100644 --- a/sql/analyzer/pushdown_test.go +++ b/sql/analyzer/pushdown_test.go @@ -105,7 +105,7 @@ func TestPushdownIndexable(t *testing.T) { catalog := sql.NewCatalog() catalog.AddDatabase(db) - idx1 := &mergableDummyIndex{ + idx1 := &mergeableDummyIndex{ "mytable", []sql.Expression{ expression.NewGetFieldWithTable(0, sql.Int32, "mytable", "i", false), @@ -116,7 +116,7 @@ func TestPushdownIndexable(t *testing.T) { close(done) <-ready - idx2 := &mergableDummyIndex{ + idx2 := &mergeableDummyIndex{ "mytable", []sql.Expression{ expression.NewGetFieldWithTable(1, sql.Float64, "mytable", "f", false), @@ -127,7 +127,7 @@ func TestPushdownIndexable(t *testing.T) { close(done) <-ready - idx3 := &mergableDummyIndex{ + idx3 := &mergeableDummyIndex{ "mytable2", []sql.Expression{ expression.NewGetFieldWithTable(0, sql.Int32, "mytable2", "i2", false), From 1643ae7bd3f8845c6768ebafd0b39046863a0d3b Mon Sep 17 00:00:00 2001 From: Zach Musgrave Date: Mon, 28 Oct 2019 15:21:36 -0700 Subject: [PATCH 03/42] Moved index types out of the main test package, with the intention of using them in engine_test.go to reproduce index-related bugs in SELECT statements Signed-off-by: Zach Musgrave --- sql/analyzer/assign_indexes_test.go | 515 +++++----------------------- sql/analyzer/pushdown_test.go | 11 +- sql/test_util/AscendIndex.go | 35 ++ sql/test_util/DescendIndex.go | 36 ++ sql/test_util/MergeableIndex.go | 183 ++++++++++ sql/test_util/NegateIndexLookup.go | 49 +++ sql/test_util/UmergeableIndex.go | 68 ++++ 7 files changed, 464 insertions(+), 433 deletions(-) create mode 100755 sql/test_util/AscendIndex.go create mode 100755 sql/test_util/DescendIndex.go create mode 100755 sql/test_util/MergeableIndex.go create mode 100755 sql/test_util/NegateIndexLookup.go create mode 100755 sql/test_util/UmergeableIndex.go diff --git a/sql/analyzer/assign_indexes_test.go b/sql/analyzer/assign_indexes_test.go index 9fd9313e7..256d5ac76 100644 --- a/sql/analyzer/assign_indexes_test.go +++ b/sql/analyzer/assign_indexes_test.go @@ -1,8 +1,7 @@ package analyzer import ( - "fmt" - "strings" + "github.com/src-d/go-mysql-server/sql/test_util" "testing" "github.com/src-d/go-mysql-server/memory" @@ -16,7 +15,7 @@ func TestNegateIndex(t *testing.T) { require := require.New(t) catalog := sql.NewCatalog() - idx1 := &mergeableDummyIndex{ + idx1 := &testutil.MergeableDummyIndex{ "t1", []sql.Expression{ expression.NewGetFieldWithTable(0, sql.Int64, "t1", "foo", false), @@ -52,16 +51,16 @@ func TestNegateIndex(t *testing.T) { lookupIdxs, ok := result["t1"] require.True(ok) - negate, ok := lookupIdxs.lookup.(*negateIndexLookup) + negate, ok := lookupIdxs.lookup.(*testutil.NegateIndexLookup) require.True(ok) - require.True(negate.value == "1") + require.True(negate.Value == "1") } func TestAssignIndexes(t *testing.T) { require := require.New(t) catalog := sql.NewCatalog() - idx1 := &mergeableDummyIndex{ + idx1 := &testutil.MergeableDummyIndex{ "t2", []sql.Expression{ expression.NewGetFieldWithTable(0, sql.Int64, "t2", "bar", false), @@ -72,7 +71,7 @@ func TestAssignIndexes(t *testing.T) { close(done) <-ready - idx2 := &mergeableDummyIndex{ + idx2 := &testutil.MergeableDummyIndex{ "t1", []sql.Expression{ expression.NewGetFieldWithTable(0, sql.Int64, "t1", "foo", false), @@ -84,7 +83,7 @@ func TestAssignIndexes(t *testing.T) { close(done) <-ready - idx3 := &unmergeableDummyIndex{ + idx3 := &testutil.UnmergeableDummyIndex{ "t1", []sql.Expression{ expression.NewGetFieldWithTable(0, sql.Int64, "t1", "bar", false), @@ -137,16 +136,16 @@ func TestAssignIndexes(t *testing.T) { lookupIdxs, ok := result["t1"] require.True(ok) - mergeable, ok := lookupIdxs.lookup.(*mergeableIndexLookup) + mergeable, ok := lookupIdxs.lookup.(*testutil.MergeableIndexLookup) require.True(ok) - require.True(mergeable.id == "2") + require.True(mergeable.Id == "2") lookupIdxs, ok = result["t2"] require.True(ok) - mergeable, ok = lookupIdxs.lookup.(*mergeableIndexLookup) + mergeable, ok = lookupIdxs.lookup.(*testutil.MergeableIndexLookup) require.True(ok) - require.True(mergeable.id == "1") + require.True(mergeable.Id == "1") node = plan.NewProject( []sql.Expression{}, @@ -190,7 +189,7 @@ func TestAssignIndexes(t *testing.T) { } func TestGetIndexes(t *testing.T) { - indexes := []*mergeableDummyIndex{ + indexes := []*testutil.MergeableDummyIndex{ { "t1", []sql.Expression{ @@ -232,7 +231,7 @@ func TestGetIndexes(t *testing.T) { ), map[string]*indexLookup{ "t1": &indexLookup{ - &mergeableIndexLookup{id: "1"}, + &testutil.MergeableIndexLookup{Id: "1"}, []sql.Index{indexes[0]}, }, }, @@ -251,7 +250,7 @@ func TestGetIndexes(t *testing.T) { ), map[string]*indexLookup{ "t1": &indexLookup{ - &mergeableIndexLookup{id: "1", unions: []string{"2"}}, + &testutil.MergeableIndexLookup{Id: "1", Unions: []string{"2"}}, []sql.Index{ indexes[0], indexes[0], @@ -273,7 +272,7 @@ func TestGetIndexes(t *testing.T) { ), map[string]*indexLookup{ "t1": &indexLookup{ - &mergeableIndexLookup{id: "1", intersections: []string{"2"}}, + &testutil.MergeableIndexLookup{Id: "1", Intersections: []string{"2"}}, []sql.Index{ indexes[0], indexes[0], @@ -307,7 +306,7 @@ func TestGetIndexes(t *testing.T) { ), map[string]*indexLookup{ "t1": &indexLookup{ - &mergeableIndexLookup{id: "1", unions: []string{"2", "4"}, intersections: []string{"3"}}, + &testutil.MergeableIndexLookup{Id: "1", Unions: []string{"2", "4"}, Intersections: []string{"3"}}, []sql.Index{ indexes[0], indexes[0], @@ -343,7 +342,7 @@ func TestGetIndexes(t *testing.T) { ), map[string]*indexLookup{ "t1": &indexLookup{ - &mergeableIndexLookup{id: "1", unions: []string{"2", "3", "4"}}, + &testutil.MergeableIndexLookup{Id: "1", Unions: []string{"2", "3", "4"}}, []sql.Index{ indexes[0], indexes[0], @@ -366,7 +365,7 @@ func TestGetIndexes(t *testing.T) { ), map[string]*indexLookup{ "t1": &indexLookup{ - &mergeableIndexLookup{id: "1", unions: []string{"2", "3", "4"}}, + &testutil.MergeableIndexLookup{Id: "1", Unions: []string{"2", "3", "4"}}, []sql.Index{indexes[0]}, }, }, @@ -397,11 +396,11 @@ func TestGetIndexes(t *testing.T) { ), map[string]*indexLookup{ "t1": &indexLookup{ - &mergeableIndexLookup{id: "3"}, + &testutil.MergeableIndexLookup{Id: "3"}, []sql.Index{indexes[0]}, }, "t2": &indexLookup{ - &mergeableIndexLookup{id: "1, 2"}, + &testutil.MergeableIndexLookup{Id: "1, 2"}, []sql.Index{indexes[1]}, }, }, @@ -438,11 +437,11 @@ func TestGetIndexes(t *testing.T) { ), map[string]*indexLookup{ "t1": &indexLookup{ - &mergeableIndexLookup{id: "3"}, + &testutil.MergeableIndexLookup{Id: "3"}, []sql.Index{indexes[0]}, }, "t2": &indexLookup{ - &mergeableIndexLookup{id: "5", unions: []string{"1, 2"}}, + &testutil.MergeableIndexLookup{Id: "5", Unions: []string{"1, 2"}}, []sql.Index{ indexes[2], indexes[1], @@ -458,7 +457,7 @@ func TestGetIndexes(t *testing.T) { ), map[string]*indexLookup{ "t1": &indexLookup{ - &descendIndexLookup{gt: []interface{}{int64(1)}}, + &testutil.DescendIndexLookup{Gt: []interface{}{int64(1)}}, []sql.Index{indexes[0]}, }, }, @@ -471,7 +470,7 @@ func TestGetIndexes(t *testing.T) { ), map[string]*indexLookup{ "t1": &indexLookup{ - &ascendIndexLookup{lt: []interface{}{int64(1)}}, + &testutil.AscendIndexLookup{Lt: []interface{}{int64(1)}}, []sql.Index{indexes[0]}, }, }, @@ -484,7 +483,7 @@ func TestGetIndexes(t *testing.T) { ), map[string]*indexLookup{ "t1": &indexLookup{ - &ascendIndexLookup{gte: []interface{}{int64(1)}}, + &testutil.AscendIndexLookup{Gte: []interface{}{int64(1)}}, []sql.Index{indexes[0]}, }, }, @@ -497,7 +496,7 @@ func TestGetIndexes(t *testing.T) { ), map[string]*indexLookup{ "t1": &indexLookup{ - &descendIndexLookup{lte: []interface{}{int64(1)}}, + &testutil.DescendIndexLookup{Lte: []interface{}{int64(1)}}, []sql.Index{indexes[0]}, }, }, @@ -511,15 +510,15 @@ func TestGetIndexes(t *testing.T) { ), map[string]*indexLookup{ "t1": &indexLookup{ - &mergedIndexLookup{ + &testutil.MergedIndexLookup{ []sql.IndexLookup{ - &ascendIndexLookup{ - gte: []interface{}{int64(1)}, - lt: []interface{}{int64(5)}, + &testutil.AscendIndexLookup{ + Gte: []interface{}{int64(1)}, + Lt: []interface{}{int64(5)}, }, - &descendIndexLookup{ - gt: []interface{}{int64(1)}, - lte: []interface{}{int64(5)}, + &testutil.DescendIndexLookup{ + Gt: []interface{}{int64(1)}, + Lte: []interface{}{int64(5)}, }, }, }, @@ -537,8 +536,8 @@ func TestGetIndexes(t *testing.T) { ), map[string]*indexLookup{ "t1": &indexLookup{ - &negateIndexLookup{ - value: "1", + &testutil.NegateIndexLookup{ + Value: "1", }, []sql.Index{indexes[0]}, }, @@ -555,7 +554,7 @@ func TestGetIndexes(t *testing.T) { ), map[string]*indexLookup{ "t1": &indexLookup{ - &descendIndexLookup{lte: []interface{}{int64(10)}}, + &testutil.DescendIndexLookup{Lte: []interface{}{int64(10)}}, []sql.Index{indexes[0]}, }, }, @@ -571,7 +570,7 @@ func TestGetIndexes(t *testing.T) { ), map[string]*indexLookup{ "t1": &indexLookup{ - &ascendIndexLookup{lt: []interface{}{int64(10)}}, + &testutil.AscendIndexLookup{Lt: []interface{}{int64(10)}}, []sql.Index{indexes[0]}, }, }, @@ -587,7 +586,7 @@ func TestGetIndexes(t *testing.T) { ), map[string]*indexLookup{ "t1": &indexLookup{ - &descendIndexLookup{gt: []interface{}{int64(10)}}, + &testutil.DescendIndexLookup{Gt: []interface{}{int64(10)}}, []sql.Index{indexes[0]}, }, }, @@ -603,7 +602,7 @@ func TestGetIndexes(t *testing.T) { ), map[string]*indexLookup{ "t1": &indexLookup{ - &ascendIndexLookup{gte: []interface{}{int64(10)}}, + &testutil.AscendIndexLookup{Gte: []interface{}{int64(10)}}, []sql.Index{indexes[0]}, }, }, @@ -624,13 +623,13 @@ func TestGetIndexes(t *testing.T) { ), map[string]*indexLookup{ "t1": &indexLookup{ - &mergedIndexLookup{ - children: []sql.IndexLookup{ - &negateIndexLookup{ - value: "10", + &testutil.MergedIndexLookup{ + Children: []sql.IndexLookup{ + &testutil.NegateIndexLookup{ + Value: "10", }, - &negateIndexLookup{ - value: "11", + &testutil.NegateIndexLookup{ + Value: "11", }, }, }, @@ -657,9 +656,9 @@ func TestGetIndexes(t *testing.T) { ), map[string]*indexLookup{ "t1": &indexLookup{ - &mergeableIndexLookup{ - id: "not 10", - intersections: []string{"not 11"}, + &testutil.MergeableIndexLookup{ + Id: "not 10", + Intersections: []string{"not 11"}, }, []sql.Index{ indexes[0], @@ -687,8 +686,8 @@ func TestGetIndexes(t *testing.T) { ), map[string]*indexLookup{ "t2": &indexLookup{ - &negateIndexLookup{ - value: "110", + &testutil.NegateIndexLookup{ + Value: "110", }, []sql.Index{ indexes[2], @@ -709,9 +708,9 @@ func TestGetIndexes(t *testing.T) { ), map[string]*indexLookup{ "t1": &indexLookup{ - &mergeableIndexLookup{ - id: "not 1", - intersections: []string{"not 2", "not 3", "not 4"}, + &testutil.MergeableIndexLookup{ + Id: "not 1", + Intersections: []string{"not 2", "not 3", "not 4"}, }, []sql.Index{indexes[0]}, }, @@ -749,7 +748,7 @@ func TestGetMultiColumnIndexes(t *testing.T) { require := require.New(t) catalog := sql.NewCatalog() - indexes := []*mergeableDummyIndex{ + indexes := []*testutil.MergeableDummyIndex{ { "t1", []sql.Expression{ @@ -836,22 +835,22 @@ func TestGetMultiColumnIndexes(t *testing.T) { expected := map[string]*indexLookup{ "t1": &indexLookup{ - &mergeableIndexLookup{id: "5, 6"}, + &testutil.MergeableIndexLookup{Id: "5, 6"}, []sql.Index{indexes[0]}, }, "t2": &indexLookup{ - &mergeableIndexLookup{id: "1, 2, 3"}, + &testutil.MergeableIndexLookup{Id: "1, 2, 3"}, []sql.Index{indexes[1]}, }, "t4": &indexLookup{ - &mergedIndexLookup{[]sql.IndexLookup{ - &ascendIndexLookup{ - gte: []interface{}{int64(1), int64(2)}, - lt: []interface{}{int64(6), int64(5)}, + &testutil.MergedIndexLookup{[]sql.IndexLookup{ + &testutil.AscendIndexLookup{ + Gte: []interface{}{int64(1), int64(2)}, + Lt: []interface{}{int64(6), int64(5)}, }, - &descendIndexLookup{ - gt: []interface{}{int64(1), int64(2)}, - lte: []interface{}{int64(6), int64(5)}, + &testutil.DescendIndexLookup{ + Gt: []interface{}{int64(1), int64(2)}, + Lte: []interface{}{int64(6), int64(5)}, }, }}, []sql.Index{indexes[4]}, @@ -946,325 +945,49 @@ func TestExpressionSources(t *testing.T) { require.Equal(t, expected, sources) } -type dummyIndexLookup struct{} +type DummyIndexLookup struct{} -func (dummyIndexLookup) Indexes() []string { return nil } +func (DummyIndexLookup) Indexes() []string { return nil } -func (dummyIndexLookup) Values(sql.Partition) (sql.IndexValueIter, error) { +func (DummyIndexLookup) Values(sql.Partition) (sql.IndexValueIter, error) { return nil, nil } -type mergeableDummyIndex struct { - table string - expr []sql.Expression -} - -type unmergeableDummyIndex struct { - table string - expr []sql.Expression -} - -func (u *unmergeableDummyIndex) Database() string { return "" } -func (u *unmergeableDummyIndex) Driver() string { return "" } -func (u *unmergeableDummyIndex) Expressions() []string { - var exprs []string - for _, e := range u.expr { - exprs = append(exprs, e.String()) - } - return exprs -} - -func (u *unmergeableDummyIndex) Get(key ...interface{}) (sql.IndexLookup, error) { - if len(key) != 1 { - var parts = make([]string, len(key)) - for i, p := range key { - parts[i] = fmt.Sprint(p) - } - - return &unmergeableIndexLookup{id: strings.Join(parts, ", ")}, nil - } - - return &unmergeableIndexLookup{id: fmt.Sprint(key[0])}, nil -} - -type unmergeableIndexLookup struct { - id string -} - -func (u unmergeableIndexLookup) Values(sql.Partition) (sql.IndexValueIter, error) { - panic("unimplemented") -} - -func (u unmergeableIndexLookup) Indexes() []string { - return []string{u.id} -} - -func (u *unmergeableDummyIndex) Has(partition sql.Partition, key ...interface{}) (bool, error) { - panic("unimplemented") -} - -func (u *unmergeableDummyIndex) ID() string { - if len(u.expr) == 1 { - return u.expr[0].String() - } - var parts = make([]string, len(u.expr)) - for i, e := range u.expr { - parts[i] = e.String() - } - - return "(" + strings.Join(parts, ", ") + ")" -} - -func (u *unmergeableDummyIndex) Table() string { - return u.table -} - -var _ sql.Index = (*mergeableDummyIndex)(nil) -var _ sql.Index = (*unmergeableDummyIndex)(nil) -var _ sql.AscendIndex = (*mergeableDummyIndex)(nil) -var _ sql.DescendIndex = (*mergeableDummyIndex)(nil) -var _ sql.NegateIndex = (*mergeableDummyIndex)(nil) - -func (mergeableDummyIndex) Database() string { return "" } -func (mergeableDummyIndex) Driver() string { return "" } -func (i mergeableDummyIndex) Expressions() []string { - var exprs []string - for _, e := range i.expr { - exprs = append(exprs, e.String()) - } - return exprs -} - -func (i mergeableDummyIndex) AscendGreaterOrEqual(keys ...interface{}) (sql.IndexLookup, error) { - return &ascendIndexLookup{gte: keys}, nil -} - -func (i mergeableDummyIndex) AscendLessThan(keys ...interface{}) (sql.IndexLookup, error) { - return &ascendIndexLookup{lt: keys}, nil -} - -func (i mergeableDummyIndex) AscendRange(greaterOrEqual, lessThan []interface{}) (sql.IndexLookup, error) { - return &ascendIndexLookup{gte: greaterOrEqual, lt: lessThan}, nil -} - -func (i mergeableDummyIndex) DescendGreater(keys ...interface{}) (sql.IndexLookup, error) { - return &descendIndexLookup{gt: keys}, nil -} - -func (i mergeableDummyIndex) DescendLessOrEqual(keys ...interface{}) (sql.IndexLookup, error) { - return &descendIndexLookup{lte: keys}, nil -} - -func (i mergeableDummyIndex) DescendRange(lessOrEqual, greaterThan []interface{}) (sql.IndexLookup, error) { - return &descendIndexLookup{gt: greaterThan, lte: lessOrEqual}, nil -} - -func (i mergeableDummyIndex) Not(keys ...interface{}) (sql.IndexLookup, error) { - lookup, err := i.Get(keys...) - if err != nil { - return nil, err - } - - mergeable, _ := lookup.(*mergeableIndexLookup) - return &negateIndexLookup{value: mergeable.id}, nil -} - -func (i mergeableDummyIndex) Get(key ...interface{}) (sql.IndexLookup, error) { - if len(key) != 1 { - var parts = make([]string, len(key)) - for i, p := range key { - parts[i] = fmt.Sprint(p) - } - - return &mergeableIndexLookup{id: strings.Join(parts, ", ")}, nil - } - - return &mergeableIndexLookup{id: fmt.Sprint(key[0])}, nil -} - -func (i mergeableDummyIndex) Has(sql.Partition, ...interface{}) (bool, error) { - panic("not implemented") -} - -func (i mergeableDummyIndex) ID() string { - if len(i.expr) == 1 { - return i.expr[0].String() - } - var parts = make([]string, len(i.expr)) - for i, e := range i.expr { - parts[i] = e.String() - } - - return "(" + strings.Join(parts, ", ") + ")" -} - -func (i mergeableDummyIndex) Table() string { return i.table } - -type mergedIndexLookup struct { - children []sql.IndexLookup -} - -func (mergedIndexLookup) Values(sql.Partition) (sql.IndexValueIter, error) { - panic("mergedIndexLookup.Values is a placeholder") -} - -func (i *mergedIndexLookup) Indexes() []string { - var indexes []string - for _, c := range i.children { - indexes = append(indexes, c.Indexes()...) - } - return indexes -} - -func (i *mergedIndexLookup) IsMergeable(sql.IndexLookup) bool { - return true -} - -func (i *mergedIndexLookup) Union(lookups ...sql.IndexLookup) sql.IndexLookup { - return &mergedIndexLookup{append(i.children, lookups...)} -} - -func (mergedIndexLookup) Difference(...sql.IndexLookup) sql.IndexLookup { - panic("mergedIndexLookup.Difference is not implemented") -} - -func (mergedIndexLookup) Intersection(...sql.IndexLookup) sql.IndexLookup { - panic("mergedIndexLookup.Intersection is not implemented") -} - -type negateIndexLookup struct { - value string - intersections []string - unions []string -} - -func (l *negateIndexLookup) ID() string { return "not " + l.value } -func (l *negateIndexLookup) Unions() []string { return l.unions } -func (l *negateIndexLookup) Intersections() []string { return l.intersections } - -func (*negateIndexLookup) Values(sql.Partition) (sql.IndexValueIter, error) { - panic("negateIndexLookup.Values is a placeholder") -} - -func (l *negateIndexLookup) Indexes() []string { - return []string{l.ID()} -} - -func (*negateIndexLookup) IsMergeable(sql.IndexLookup) bool { - return true -} - -func (l *negateIndexLookup) Union(lookups ...sql.IndexLookup) sql.IndexLookup { - return &mergedIndexLookup{append([]sql.IndexLookup{l}, lookups...)} -} - -func (*negateIndexLookup) Difference(...sql.IndexLookup) sql.IndexLookup { - panic("negateIndexLookup.Difference is not implemented") -} - -func (l *negateIndexLookup) Intersection(indexes ...sql.IndexLookup) sql.IndexLookup { - var intersections, unions []string - for _, idx := range indexes { - intersections = append(intersections, idx.(mergeableLookup).ID()) - intersections = append(intersections, idx.(mergeableLookup).Intersections()...) - unions = append(unions, idx.(mergeableLookup).Unions()...) - } - return &mergeableIndexLookup{ - l.ID(), - append(l.unions, unions...), - append(l.intersections, intersections...), - } -} - -type ascendIndexLookup struct { - id string - gte []interface{} - lt []interface{} -} - -func (ascendIndexLookup) Values(sql.Partition) (sql.IndexValueIter, error) { - panic("ascendIndexLookup.Values is a placeholder") -} - -func (l *ascendIndexLookup) Indexes() []string { - return []string{l.id} -} - -func (l *ascendIndexLookup) IsMergeable(sql.IndexLookup) bool { - return true -} - -func (l *ascendIndexLookup) Union(lookups ...sql.IndexLookup) sql.IndexLookup { - return &mergedIndexLookup{append([]sql.IndexLookup{l}, lookups...)} -} - -func (ascendIndexLookup) Difference(...sql.IndexLookup) sql.IndexLookup { - panic("ascendIndexLookup.Difference is not implemented") -} - -func (ascendIndexLookup) Intersection(...sql.IndexLookup) sql.IndexLookup { - panic("ascendIndexLookup.Intersection is not implemented") -} - -type descendIndexLookup struct { - id string - gt []interface{} - lte []interface{} -} - -func (descendIndexLookup) Values(sql.Partition) (sql.IndexValueIter, error) { - panic("descendIndexLookup.Values is a placeholder") -} - -func (l *descendIndexLookup) Indexes() []string { - return []string{l.id} -} - -func (l *descendIndexLookup) IsMergeable(sql.IndexLookup) bool { - return true -} - -func (l *descendIndexLookup) Union(lookups ...sql.IndexLookup) sql.IndexLookup { - return &mergedIndexLookup{append([]sql.IndexLookup{l}, lookups...)} -} - -func (descendIndexLookup) Difference(...sql.IndexLookup) sql.IndexLookup { - panic("descendIndexLookup.Difference is not implemented") -} - -func (descendIndexLookup) Intersection(...sql.IndexLookup) sql.IndexLookup { - panic("descendIndexLookup.Intersection is not implemented") -} +var _ sql.Index = (*testutil.MergeableDummyIndex)(nil) +var _ sql.Index = (*testutil.UnmergeableDummyIndex)(nil) +var _ sql.AscendIndex = (*testutil.MergeableDummyIndex)(nil) +var _ sql.DescendIndex = (*testutil.MergeableDummyIndex)(nil) +var _ sql.NegateIndex = (*testutil.MergeableDummyIndex)(nil) func TestIndexesIntersection(t *testing.T) { require := require.New(t) - idx1, idx2 := &mergeableDummyIndex{table: "bar"}, &mergeableDummyIndex{table: "foo"} + idx1, idx2 := &testutil.MergeableDummyIndex{TableName: "bar"}, &testutil.MergeableDummyIndex{TableName: "foo"} left := map[string]*indexLookup{ - "a": &indexLookup{&mergeableIndexLookup{id: "a"}, nil}, - "b": &indexLookup{&mergeableIndexLookup{id: "b"}, []sql.Index{idx1}}, - "c": &indexLookup{new(dummyIndexLookup), nil}, + "a": &indexLookup{&testutil.MergeableIndexLookup{Id: "a"}, nil}, + "b": &indexLookup{&testutil.MergeableIndexLookup{Id: "b"}, []sql.Index{idx1}}, + "c": &indexLookup{new(DummyIndexLookup), nil}, } right := map[string]*indexLookup{ - "b": &indexLookup{&mergeableIndexLookup{id: "b2"}, []sql.Index{idx2}}, - "c": &indexLookup{&mergeableIndexLookup{id: "c"}, nil}, - "d": &indexLookup{&mergeableIndexLookup{id: "d"}, nil}, + "b": &indexLookup{&testutil.MergeableIndexLookup{Id: "b2"}, []sql.Index{idx2}}, + "c": &indexLookup{&testutil.MergeableIndexLookup{Id: "c"}, nil}, + "d": &indexLookup{&testutil.MergeableIndexLookup{Id: "d"}, nil}, } require.Equal( map[string]*indexLookup{ - "a": &indexLookup{&mergeableIndexLookup{id: "a"}, nil}, + "a": &indexLookup{&testutil.MergeableIndexLookup{Id: "a"}, nil}, "b": &indexLookup{ - &mergeableIndexLookup{ - id: "b", - intersections: []string{"b2"}, + &testutil.MergeableIndexLookup{ + Id: "b", + Intersections: []string{"b2"}, }, []sql.Index{idx1, idx2}, }, - "c": &indexLookup{new(dummyIndexLookup), nil}, - "d": &indexLookup{&mergeableIndexLookup{id: "d"}, nil}, + "c": &indexLookup{new(DummyIndexLookup), nil}, + "d": &indexLookup{&testutil.MergeableIndexLookup{Id: "d"}, nil}, }, indexesIntersection(NewDefault(sql.NewCatalog()), left, right), ) @@ -1273,70 +996,6 @@ func TestIndexesIntersection(t *testing.T) { func TestCanMergeIndexes(t *testing.T) { require := require.New(t) - require.False(canMergeIndexes(new(mergeableIndexLookup), new(dummyIndexLookup))) - require.True(canMergeIndexes(new(mergeableIndexLookup), new(mergeableIndexLookup))) -} - -type mergeableLookup interface { - ID() string - Unions() []string - Intersections() []string -} - -type mergeableIndexLookup struct { - id string - unions []string - intersections []string -} - -var _ sql.Mergeable = (*mergeableIndexLookup)(nil) -var _ sql.SetOperations = (*mergeableIndexLookup)(nil) - -func (i *mergeableIndexLookup) ID() string { return i.id } -func (i *mergeableIndexLookup) Unions() []string { return i.unions } -func (i *mergeableIndexLookup) Intersections() []string { return i.intersections } - -func (i *mergeableIndexLookup) IsMergeable(lookup sql.IndexLookup) bool { - _, ok := lookup.(mergeableLookup) - return ok -} - -func (i *mergeableIndexLookup) Values(sql.Partition) (sql.IndexValueIter, error) { - panic("not implemented") -} - -func (i *mergeableIndexLookup) Indexes() []string { - return []string{i.ID()} -} - -func (i *mergeableIndexLookup) Difference(indexes ...sql.IndexLookup) sql.IndexLookup { - panic("not implemented") -} - -func (i *mergeableIndexLookup) Intersection(indexes ...sql.IndexLookup) sql.IndexLookup { - var intersections, unions []string - for _, idx := range indexes { - intersections = append(intersections, idx.(mergeableLookup).ID()) - intersections = append(intersections, idx.(mergeableLookup).Intersections()...) - unions = append(unions, idx.(mergeableLookup).Unions()...) - } - return &mergeableIndexLookup{ - i.id, - append(i.unions, unions...), - append(i.intersections, intersections...), - } -} - -func (i *mergeableIndexLookup) Union(indexes ...sql.IndexLookup) sql.IndexLookup { - var intersections, unions []string - for _, idx := range indexes { - unions = append(unions, idx.(*mergeableIndexLookup).id) - unions = append(unions, idx.(*mergeableIndexLookup).unions...) - intersections = append(intersections, idx.(*mergeableIndexLookup).intersections...) - } - return &mergeableIndexLookup{ - i.id, - append(i.unions, unions...), - append(i.intersections, intersections...), - } -} + require.False(canMergeIndexes(new(testutil.MergeableIndexLookup), new(DummyIndexLookup))) + require.True(canMergeIndexes(new(testutil.MergeableIndexLookup), new(testutil.MergeableIndexLookup))) +} \ No newline at end of file diff --git a/sql/analyzer/pushdown_test.go b/sql/analyzer/pushdown_test.go index 5137daf78..ef3979c3f 100644 --- a/sql/analyzer/pushdown_test.go +++ b/sql/analyzer/pushdown_test.go @@ -1,6 +1,7 @@ package analyzer import ( + "github.com/src-d/go-mysql-server/sql/test_util" "testing" "github.com/src-d/go-mysql-server/memory" @@ -105,7 +106,7 @@ func TestPushdownIndexable(t *testing.T) { catalog := sql.NewCatalog() catalog.AddDatabase(db) - idx1 := &mergeableDummyIndex{ + idx1 := &testutil.MergeableDummyIndex{ "mytable", []sql.Expression{ expression.NewGetFieldWithTable(0, sql.Int32, "mytable", "i", false), @@ -116,7 +117,7 @@ func TestPushdownIndexable(t *testing.T) { close(done) <-ready - idx2 := &mergeableDummyIndex{ + idx2 := &testutil.MergeableDummyIndex{ "mytable", []sql.Expression{ expression.NewGetFieldWithTable(1, sql.Float64, "mytable", "f", false), @@ -127,7 +128,7 @@ func TestPushdownIndexable(t *testing.T) { close(done) <-ready - idx3 := &mergeableDummyIndex{ + idx3 := &testutil.MergeableDummyIndex{ "mytable2", []sql.Expression{ expression.NewGetFieldWithTable(0, sql.Int32, "mytable2", "i2", false), @@ -189,7 +190,7 @@ func TestPushdownIndexable(t *testing.T) { ), }).(*memory.Table). WithProjection([]string{"i", "f"}).(*memory.Table). - WithIndexLookup(&mergeableIndexLookup{id: "3.14"}), + WithIndexLookup(&testutil.MergeableIndexLookup{Id: "3.14"}), ), plan.NewResolvedTable( table2.WithFilters([]sql.Expression{ @@ -201,7 +202,7 @@ func TestPushdownIndexable(t *testing.T) { ), }).(*memory.Table). WithProjection([]string{"i2"}).(*memory.Table). - WithIndexLookup(&negateIndexLookup{value: "2"}), + WithIndexLookup(&testutil.NegateIndexLookup{Value: "2"}), ), ), ), diff --git a/sql/test_util/AscendIndex.go b/sql/test_util/AscendIndex.go new file mode 100755 index 000000000..fa8c8baee --- /dev/null +++ b/sql/test_util/AscendIndex.go @@ -0,0 +1,35 @@ +package testutil + +import ( + "github.com/src-d/go-mysql-server/sql" +) + +type AscendIndexLookup struct { + id string + Gte []interface{} + Lt []interface{} +} + +func (AscendIndexLookup) Values(sql.Partition) (sql.IndexValueIter, error) { + panic("ascendIndexLookup.Values is a placeholder") +} + +func (l *AscendIndexLookup) Indexes() []string { + return []string{l.id} +} + +func (l *AscendIndexLookup) IsMergeable(sql.IndexLookup) bool { + return true +} + +func (l *AscendIndexLookup) Union(lookups ...sql.IndexLookup) sql.IndexLookup { + return &MergedIndexLookup{append([]sql.IndexLookup{l}, lookups...)} +} + +func (AscendIndexLookup) Difference(...sql.IndexLookup) sql.IndexLookup { + panic("ascendIndexLookup.Difference is not implemented") +} + +func (AscendIndexLookup) Intersection(...sql.IndexLookup) sql.IndexLookup { + panic("ascendIndexLookup.Intersection is not implemented") +} diff --git a/sql/test_util/DescendIndex.go b/sql/test_util/DescendIndex.go new file mode 100755 index 000000000..ac15e26c3 --- /dev/null +++ b/sql/test_util/DescendIndex.go @@ -0,0 +1,36 @@ +package testutil + +import ( + "github.com/src-d/go-mysql-server/sql" +) + +type DescendIndexLookup struct { + id string + Gt []interface{} + Lte []interface{} +} + +func (DescendIndexLookup) Values(sql.Partition) (sql.IndexValueIter, error) { + panic("descendIndexLookup.Values is a placeholder") +} + +func (l *DescendIndexLookup) Indexes() []string { + return []string{l.id} +} + +func (l *DescendIndexLookup) IsMergeable(sql.IndexLookup) bool { + return true +} + +func (l *DescendIndexLookup) Union(lookups ...sql.IndexLookup) sql.IndexLookup { + return &MergedIndexLookup{append([]sql.IndexLookup{l}, lookups...)} +} + +func (DescendIndexLookup) Difference(...sql.IndexLookup) sql.IndexLookup { + panic("descendIndexLookup.Difference is not implemented") +} + +func (DescendIndexLookup) Intersection(...sql.IndexLookup) sql.IndexLookup { + panic("descendIndexLookup.Intersection is not implemented") +} + diff --git a/sql/test_util/MergeableIndex.go b/sql/test_util/MergeableIndex.go new file mode 100755 index 000000000..590149c75 --- /dev/null +++ b/sql/test_util/MergeableIndex.go @@ -0,0 +1,183 @@ +package testutil + +import ( + "fmt" + "github.com/src-d/go-mysql-server/sql" + "strings" +) + +type MergeableLookup interface { + ID() string + GetUnions() []string + GetIntersections() []string +} + +type MergeableIndexLookup struct { + Id string + Unions []string + Intersections []string +} + +var _ sql.Mergeable = (*MergeableIndexLookup)(nil) +var _ sql.SetOperations = (*MergeableIndexLookup)(nil) + +func (i *MergeableIndexLookup) ID() string { return i.Id } +func (i *MergeableIndexLookup) GetUnions() []string { return i.Unions } +func (i *MergeableIndexLookup) GetIntersections() []string { return i.Intersections } + +func (i *MergeableIndexLookup) IsMergeable(lookup sql.IndexLookup) bool { + _, ok := lookup.(MergeableLookup) + return ok +} + +func (i *MergeableIndexLookup) Values(sql.Partition) (sql.IndexValueIter, error) { + panic("not implemented") +} + +func (i *MergeableIndexLookup) Indexes() []string { + return []string{i.ID()} +} + +func (i *MergeableIndexLookup) Difference(indexes ...sql.IndexLookup) sql.IndexLookup { + panic("not implemented") +} + +func (i *MergeableIndexLookup) Intersection(indexes ...sql.IndexLookup) sql.IndexLookup { + var intersections, unions []string + for _, idx := range indexes { + intersections = append(intersections, idx.(MergeableLookup).ID()) + intersections = append(intersections, idx.(MergeableLookup).GetIntersections()...) + unions = append(unions, idx.(MergeableLookup).GetUnions()...) + } + return &MergeableIndexLookup{ + i.Id, + append(i.Unions, unions...), + append(i.Intersections, intersections...), + } +} + +func (i *MergeableIndexLookup) Union(indexes ...sql.IndexLookup) sql.IndexLookup { + var intersections, unions []string + for _, idx := range indexes { + unions = append(unions, idx.(*MergeableIndexLookup).Id) + unions = append(unions, idx.(*MergeableIndexLookup).Unions...) + intersections = append(intersections, idx.(*MergeableIndexLookup).Intersections...) + } + return &MergeableIndexLookup{ + i.Id, + append(i.Unions, unions...), + append(i.Intersections, intersections...), + } +} + +type MergedIndexLookup struct { + Children []sql.IndexLookup +} + +func (MergedIndexLookup) Values(sql.Partition) (sql.IndexValueIter, error) { + panic("mergedIndexLookup.Values is a placeholder") +} + +func (i *MergedIndexLookup) Indexes() []string { + var indexes []string + for _, c := range i.Children { + indexes = append(indexes, c.Indexes()...) + } + return indexes +} + +func (i *MergedIndexLookup) IsMergeable(sql.IndexLookup) bool { + return true +} + +func (i *MergedIndexLookup) Union(lookups ...sql.IndexLookup) sql.IndexLookup { + return &MergedIndexLookup{append(i.Children, lookups...)} +} + +func (MergedIndexLookup) Difference(...sql.IndexLookup) sql.IndexLookup { + panic("mergedIndexLookup.Difference is not implemented") +} + +func (MergedIndexLookup) Intersection(...sql.IndexLookup) sql.IndexLookup { + panic("mergedIndexLookup.Intersection is not implemented") +} + +type MergeableDummyIndex struct { + TableName string + Exprs []sql.Expression +} + +func (MergeableDummyIndex) Database() string { return "" } +func (MergeableDummyIndex) Driver() string { return "" } +func (i MergeableDummyIndex) Expressions() []string { + var exprs []string + for _, e := range i.Exprs { + exprs = append(exprs, e.String()) + } + return exprs +} + +func (i MergeableDummyIndex) AscendGreaterOrEqual(keys ...interface{}) (sql.IndexLookup, error) { + return &AscendIndexLookup{Gte: keys}, nil +} + +func (i MergeableDummyIndex) AscendLessThan(keys ...interface{}) (sql.IndexLookup, error) { + return &AscendIndexLookup{Lt: keys}, nil +} + +func (i MergeableDummyIndex) AscendRange(greaterOrEqual, lessThan []interface{}) (sql.IndexLookup, error) { + return &AscendIndexLookup{Gte: greaterOrEqual, Lt: lessThan}, nil +} + +func (i MergeableDummyIndex) DescendGreater(keys ...interface{}) (sql.IndexLookup, error) { + return &DescendIndexLookup{Gt: keys}, nil +} + +func (i MergeableDummyIndex) DescendLessOrEqual(keys ...interface{}) (sql.IndexLookup, error) { + return &DescendIndexLookup{Lte: keys}, nil +} + +func (i MergeableDummyIndex) DescendRange(lessOrEqual, greaterThan []interface{}) (sql.IndexLookup, error) { + return &DescendIndexLookup{Gt: greaterThan, Lte: lessOrEqual}, nil +} + +func (i MergeableDummyIndex) Not(keys ...interface{}) (sql.IndexLookup, error) { + lookup, err := i.Get(keys...) + if err != nil { + return nil, err + } + + mergeable, _ := lookup.(*MergeableIndexLookup) + return &NegateIndexLookup{Value: mergeable.Id}, nil +} + +func (i MergeableDummyIndex) Get(key ...interface{}) (sql.IndexLookup, error) { + if len(key) != 1 { + var parts = make([]string, len(key)) + for i, p := range key { + parts[i] = fmt.Sprint(p) + } + + return &MergeableIndexLookup{Id: strings.Join(parts, ", ")}, nil + } + + return &MergeableIndexLookup{Id: fmt.Sprint(key[0])}, nil +} + +func (i MergeableDummyIndex) Has(sql.Partition, ...interface{}) (bool, error) { + panic("not implemented") +} + +func (i MergeableDummyIndex) ID() string { + if len(i.Exprs) == 1 { + return i.Exprs[0].String() + } + var parts = make([]string, len(i.Exprs)) + for i, e := range i.Exprs { + parts[i] = e.String() + } + + return "(" + strings.Join(parts, ", ") + ")" +} + +func (i MergeableDummyIndex) Table() string { return i.TableName } \ No newline at end of file diff --git a/sql/test_util/NegateIndexLookup.go b/sql/test_util/NegateIndexLookup.go new file mode 100755 index 000000000..34650cede --- /dev/null +++ b/sql/test_util/NegateIndexLookup.go @@ -0,0 +1,49 @@ +package testutil + +import ( + "github.com/src-d/go-mysql-server/sql" +) + +type NegateIndexLookup struct { + Value string + intersections []string + unions []string +} + +func (l *NegateIndexLookup) ID() string { return "not " + l.Value } +func (l *NegateIndexLookup) GetUnions() []string { return l.unions } +func (l *NegateIndexLookup) GetIntersections() []string { return l.intersections } + +func (*NegateIndexLookup) Values(sql.Partition) (sql.IndexValueIter, error) { + panic("negateIndexLookup.Values is a placeholder") +} + +func (l *NegateIndexLookup) Indexes() []string { + return []string{l.ID()} +} + +func (*NegateIndexLookup) IsMergeable(sql.IndexLookup) bool { + return true +} + +func (l *NegateIndexLookup) Union(lookups ...sql.IndexLookup) sql.IndexLookup { + return &MergedIndexLookup{append([]sql.IndexLookup{l}, lookups...)} +} + +func (*NegateIndexLookup) Difference(...sql.IndexLookup) sql.IndexLookup { + panic("negateIndexLookup.Difference is not implemented") +} + +func (l *NegateIndexLookup) Intersection(indexes ...sql.IndexLookup) sql.IndexLookup { + var intersections, unions []string + for _, idx := range indexes { + intersections = append(intersections, idx.(MergeableLookup).ID()) + intersections = append(intersections, idx.(MergeableLookup).GetIntersections()...) + unions = append(unions, idx.(MergeableLookup).GetUnions()...) + } + return &MergeableIndexLookup{ + l.ID(), + append(l.unions, unions...), + append(l.intersections, intersections...), + } +} \ No newline at end of file diff --git a/sql/test_util/UmergeableIndex.go b/sql/test_util/UmergeableIndex.go new file mode 100755 index 000000000..5ea916e44 --- /dev/null +++ b/sql/test_util/UmergeableIndex.go @@ -0,0 +1,68 @@ +package testutil + +import ( + "fmt" + "github.com/src-d/go-mysql-server/sql" + "strings" +) + +type UnmergeableDummyIndex struct { + TableName string + Exprs []sql.Expression +} + +func (u *UnmergeableDummyIndex) Database() string { return "" } +func (u *UnmergeableDummyIndex) Driver() string { return "" } +func (u *UnmergeableDummyIndex) Expressions() []string { + var exprs []string + for _, e := range u.Exprs { + exprs = append(exprs, e.String()) + } + return exprs +} + +func (u *UnmergeableDummyIndex) Get(key ...interface{}) (sql.IndexLookup, error) { + if len(key) != 1 { + var parts = make([]string, len(key)) + for i, p := range key { + parts[i] = fmt.Sprint(p) + } + + return &UnmergeableIndexLookup{id: strings.Join(parts, ", ")}, nil + } + + return &UnmergeableIndexLookup{id: fmt.Sprint(key[0])}, nil +} + +type UnmergeableIndexLookup struct { + id string +} + +func (u UnmergeableIndexLookup) Values(sql.Partition) (sql.IndexValueIter, error) { + panic("unimplemented") +} + +func (u UnmergeableIndexLookup) Indexes() []string { + return []string{u.id} +} + +func (u *UnmergeableDummyIndex) Has(partition sql.Partition, key ...interface{}) (bool, error) { + panic("unimplemented") +} + +func (u *UnmergeableDummyIndex) ID() string { + if len(u.Exprs) == 1 { + return u.Exprs[0].String() + } + var parts = make([]string, len(u.Exprs)) + for i, e := range u.Exprs { + parts[i] = e.String() + } + + return "(" + strings.Join(parts, ", ") + ")" +} + +func (u *UnmergeableDummyIndex) Table() string { + return u.TableName +} + From 120ddc0f13dd19e9f8a5006c8b0b89267bdd6f9d Mon Sep 17 00:00:00 2001 From: Zach Musgrave Date: Mon, 28 Oct 2019 16:13:13 -0700 Subject: [PATCH 04/42] Added tests for getIndexes for or and in for various mergeable and unmergeable indexes. Signed-off-by: Zach Musgrave --- sql/analyzer/assign_indexes_test.go | 74 ++++++++++++++++++++++++----- sql/analyzer/common_test.go | 8 ++++ 2 files changed, 71 insertions(+), 11 deletions(-) diff --git a/sql/analyzer/assign_indexes_test.go b/sql/analyzer/assign_indexes_test.go index 256d5ac76..36f89ab20 100644 --- a/sql/analyzer/assign_indexes_test.go +++ b/sql/analyzer/assign_indexes_test.go @@ -189,26 +189,32 @@ func TestAssignIndexes(t *testing.T) { } func TestGetIndexes(t *testing.T) { - indexes := []*testutil.MergeableDummyIndex{ - { + indexes := []sql.Index { + &testutil.MergeableDummyIndex{ "t1", []sql.Expression{ col(0, "t1", "bar"), }, }, - { + &testutil.MergeableDummyIndex{ "t2", []sql.Expression{ col(0, "t2", "foo"), col(0, "t2", "bar"), }, }, - { + &testutil.MergeableDummyIndex{ "t2", []sql.Expression{ col(0, "t2", "bar"), }, }, + &testutil.UnmergeableDummyIndex{ + "t3", + []sql.Expression{ + col(0, "t3", "foo"), + }, + }, } testCases := []struct { @@ -259,6 +265,57 @@ func TestGetIndexes(t *testing.T) { }, true, }, + { + or( + eq( + col(0, "t3", "foo"), + lit(1), + ), + eq( + col(0, "t3", "foo"), + lit(2), + ), + ), + nil, + true, + }, + { + or( + eq( + col(0, "t3", "foo"), + lit(1), + ), + eq( + col(0, "t3", "foo"), + lit(2), + ), + ), + nil, + true, + }, + { + in( + col(0, "t3", "foo"), + tuple(lit(1), lit(2)), + ), + nil, + true, + }, + { + in( + col(0, "t1", "bar"), + tuple(lit(1), lit(2)), + ), + map[string]*indexLookup{ + "t1": &indexLookup{ + &testutil.MergeableIndexLookup{Id: "1", Unions: []string{"2"}}, + []sql.Index{ + indexes[0], + }, + }, + }, + true, + }, { and( eq( @@ -354,14 +411,9 @@ func TestGetIndexes(t *testing.T) { true, }, { - expression.NewIn( + in( col(0, "t1", "bar"), - expression.NewTuple( - lit(1), - lit(2), - lit(3), - lit(4), - ), + tuple(lit(1), lit(2), lit(3), lit(4)), ), map[string]*indexLookup{ "t1": &indexLookup{ diff --git a/sql/analyzer/common_test.go b/sql/analyzer/common_test.go index a5beb26c9..377e9d10b 100644 --- a/sql/analyzer/common_test.go +++ b/sql/analyzer/common_test.go @@ -29,6 +29,14 @@ func or(left, right sql.Expression) sql.Expression { return expression.NewOr(left, right) } +func in(col sql.Expression, tuple sql.Expression) sql.Expression { + return expression.NewIn(col, tuple) +} + +func tuple(vals ...sql.Expression) sql.Expression { + return expression.NewTuple(vals...) +} + func and(left, right sql.Expression) sql.Expression { return expression.NewAnd(left, right) } From 1b352950fa6fae330a0a990ae30243fdd19acb35 Mon Sep 17 00:00:00 2001 From: Zach Musgrave Date: Mon, 28 Oct 2019 16:44:20 -0700 Subject: [PATCH 05/42] Apparently forgot how to name go files Signed-off-by: Zach Musgrave --- .../{AscendIndex.go => ascend_index.go} | 0 .../{DescendIndex.go => descend_index.go} | 0 sql/test_util/index_driver.go | 29 +++++++++++++++++++ .../{MergeableIndex.go => mergeable_index.go} | 0 ...NegateIndexLookup.go => negative_index.go} | 0 ...mergeableIndex.go => unmergeable_index.go} | 0 6 files changed, 29 insertions(+) rename sql/test_util/{AscendIndex.go => ascend_index.go} (100%) rename sql/test_util/{DescendIndex.go => descend_index.go} (100%) create mode 100755 sql/test_util/index_driver.go rename sql/test_util/{MergeableIndex.go => mergeable_index.go} (100%) rename sql/test_util/{NegateIndexLookup.go => negative_index.go} (100%) rename sql/test_util/{UmergeableIndex.go => unmergeable_index.go} (100%) diff --git a/sql/test_util/AscendIndex.go b/sql/test_util/ascend_index.go similarity index 100% rename from sql/test_util/AscendIndex.go rename to sql/test_util/ascend_index.go diff --git a/sql/test_util/DescendIndex.go b/sql/test_util/descend_index.go similarity index 100% rename from sql/test_util/DescendIndex.go rename to sql/test_util/descend_index.go diff --git a/sql/test_util/index_driver.go b/sql/test_util/index_driver.go new file mode 100755 index 000000000..340b82d36 --- /dev/null +++ b/sql/test_util/index_driver.go @@ -0,0 +1,29 @@ +package testutil + +import ( + "github.com/src-d/go-mysql-server/sql" +) + +type TestIndexDriver struct { + +} + +func (d *TestIndexDriver) ID() string { + panic("TestIndexDriver") +} + +func (d *TestIndexDriver) LoadAll(db, table string) ([]sql.Index, error) { + panic("implement me") +} + +func (d *TestIndexDriver) Save(*sql.Context, sql.Index, sql.PartitionIndexKeyValueIter) error { + panic("not implemented") +} + +func (d *TestIndexDriver) Delete(sql.Index, sql.PartitionIter) error { + panic("not implemented") +} + +func (d *TestIndexDriver) Create(db, table, id string, expressions []sql.Expression, config map[string]string) (sql.Index, error) { + panic("not implemented") +} \ No newline at end of file diff --git a/sql/test_util/MergeableIndex.go b/sql/test_util/mergeable_index.go similarity index 100% rename from sql/test_util/MergeableIndex.go rename to sql/test_util/mergeable_index.go diff --git a/sql/test_util/NegateIndexLookup.go b/sql/test_util/negative_index.go similarity index 100% rename from sql/test_util/NegateIndexLookup.go rename to sql/test_util/negative_index.go diff --git a/sql/test_util/UmergeableIndex.go b/sql/test_util/unmergeable_index.go similarity index 100% rename from sql/test_util/UmergeableIndex.go rename to sql/test_util/unmergeable_index.go From 50744356ed3c139c6ce6874000024b671a8d2ab9 Mon Sep 17 00:00:00 2001 From: Zach Musgrave Date: Mon, 28 Oct 2019 17:14:46 -0700 Subject: [PATCH 06/42] Closer to adding an index driver to engine test. Factored out a method for DB / table setup. Signed-off-by: Zach Musgrave --- engine_test.go | 59 +++++++++++++++++++---------------- sql/test_util/index_driver.go | 6 +++- 2 files changed, 37 insertions(+), 28 deletions(-) diff --git a/engine_test.go b/engine_test.go index 5070589ee..dea9be3eb 100644 --- a/engine_test.go +++ b/engine_test.go @@ -2804,62 +2804,64 @@ func newEngine(t *testing.T) *sqle.Engine { return newEngineWithParallelism(t, 1) } -func newEngineWithParallelism(t *testing.T, parallelism int) *sqle.Engine { - table := memory.NewPartitionedTable("mytable", sql.Schema{ +func allTestTables(t *testing.T) map[string]*memory.Table { + tables := make(map[string]*memory.Table) + + tables["mytable"] = memory.NewPartitionedTable("mytable", sql.Schema{ {Name: "i", Type: sql.Int64, Source: "mytable"}, {Name: "s", Type: sql.Text, Source: "mytable"}, }, testNumPartitions) insertRows( - t, table, + t, tables["mytable"], sql.NewRow(int64(1), "first row"), sql.NewRow(int64(2), "second row"), sql.NewRow(int64(3), "third row"), ) - table2 := memory.NewPartitionedTable("othertable", sql.Schema{ + tables["othertable"] = memory.NewPartitionedTable("othertable", sql.Schema{ {Name: "s2", Type: sql.Text, Source: "othertable"}, {Name: "i2", Type: sql.Int64, Source: "othertable"}, }, testNumPartitions) insertRows( - t, table2, + t, tables["othertable"], sql.NewRow("first", int64(3)), sql.NewRow("second", int64(2)), sql.NewRow("third", int64(1)), ) - table3 := memory.NewPartitionedTable("tabletest", sql.Schema{ + tables["tabletest"] = memory.NewPartitionedTable("tabletest", sql.Schema{ {Name: "i", Type: sql.Int32, Source: "tabletest"}, {Name: "s", Type: sql.Text, Source: "tabletest"}, }, testNumPartitions) insertRows( - t, table3, + t, tables["tabletest"], sql.NewRow(int64(1), "first row"), sql.NewRow(int64(2), "second row"), sql.NewRow(int64(3), "third row"), ) - table4 := memory.NewPartitionedTable("other_table", sql.Schema{ + tables["other_table"] = memory.NewPartitionedTable("other_table", sql.Schema{ {Name: "text", Type: sql.Text, Source: "tabletest"}, {Name: "number", Type: sql.Int32, Source: "tabletest"}, }, testNumPartitions) insertRows( - t, table4, + t, tables["other_table"], sql.NewRow("a", int32(4)), sql.NewRow("b", int32(2)), sql.NewRow("c", int32(0)), ) - bigtable := memory.NewPartitionedTable("bigtable", sql.Schema{ + tables["bigtable"] = memory.NewPartitionedTable("bigtable", sql.Schema{ {Name: "t", Type: sql.Text, Source: "bigtable"}, {Name: "n", Type: sql.Int64, Source: "bigtable"}, }, testNumPartitions) insertRows( - t, bigtable, + t, tables["bigtable"], sql.NewRow("a", int64(1)), sql.NewRow("s", int64(2)), sql.NewRow("f", int64(3)), @@ -2876,14 +2878,14 @@ func newEngineWithParallelism(t *testing.T, parallelism int) *sqle.Engine { sql.NewRow("b", int64(9)), ) - floatTable := memory.NewPartitionedTable("floattable", sql.Schema{ + tables["floattable"] = memory.NewPartitionedTable("floattable", sql.Schema{ {Name: "i", Type: sql.Int64, Source: "floattable"}, {Name: "f32", Type: sql.Float32, Source: "floattable"}, {Name: "f64", Type: sql.Float64, Source: "floattable"}, }, testNumPartitions) insertRows( - t, floatTable, + t, tables["floattable"], sql.NewRow(int64(1), float32(1.0), float64(1.0)), sql.NewRow(int64(2), float32(1.5), float64(1.5)), sql.NewRow(int64(3), float32(2.0), float64(2.0)), @@ -2892,14 +2894,14 @@ func newEngineWithParallelism(t *testing.T, parallelism int) *sqle.Engine { sql.NewRow(int64(-2), float32(-1.5), float64(-1.5)), ) - nilTable := memory.NewPartitionedTable("niltable", sql.Schema{ + tables["niltable"] = memory.NewPartitionedTable("niltable", sql.Schema{ {Name: "i", Type: sql.Int64, Source: "niltable", Nullable: true}, {Name: "b", Type: sql.Boolean, Source: "niltable", Nullable: true}, {Name: "f", Type: sql.Float64, Source: "niltable", Nullable: true}, }, testNumPartitions) insertRows( - t, nilTable, + t, tables["niltable"], sql.NewRow(int64(1), true, float64(1.0)), sql.NewRow(int64(2), nil, float64(2.0)), sql.NewRow(nil, false, float64(3.0)), @@ -2907,13 +2909,13 @@ func newEngineWithParallelism(t *testing.T, parallelism int) *sqle.Engine { sql.NewRow(nil, nil, nil), ) - newlineTable := memory.NewPartitionedTable("newlinetable", sql.Schema{ + tables["newlinetable"] = memory.NewPartitionedTable("newlinetable", sql.Schema{ {Name: "i", Type: sql.Int64, Source: "newlinetable"}, {Name: "s", Type: sql.Text, Source: "newlinetable"}, }, testNumPartitions) insertRows( - t, newlineTable, + t, tables["newlinetable"], sql.NewRow(int64(1), "\nthere is some text in here"), sql.NewRow(int64(2), "there is some\ntext in here"), sql.NewRow(int64(3), "there is some text\nin here"), @@ -2921,7 +2923,7 @@ func newEngineWithParallelism(t *testing.T, parallelism int) *sqle.Engine { sql.NewRow(int64(5), "there is some text in here"), ) - typestable := memory.NewPartitionedTable("typestable", sql.Schema{ + tables["typestable"] = memory.NewPartitionedTable("typestable", sql.Schema{ {Name: "id", Type: sql.Int64, Source: "typestable"}, {Name: "i8", Type: sql.Int8, Source: "typestable", Nullable: true}, {Name: "i16", Type: sql.Int16, Source: "typestable", Nullable: true}, @@ -2941,18 +2943,21 @@ func newEngineWithParallelism(t *testing.T, parallelism int) *sqle.Engine { {Name: "bl", Type: sql.Blob, Source: "typestable", Nullable: true}, }, testNumPartitions) + return tables +} + +func newEngineWithParallelism(t *testing.T, parallelism int) *sqle.Engine { + tables := allTestTables(t) + db := memory.NewDatabase("mydb") - db.AddTable("mytable", table) - db.AddTable("othertable", table2) - db.AddTable("tabletest", table3) - db.AddTable("bigtable", bigtable) - db.AddTable("floattable", floatTable) - db.AddTable("niltable", nilTable) - db.AddTable("newlinetable", newlineTable) - db.AddTable("typestable", typestable) + for name, table := range tables { + if name != "other_table" { + db.AddTable(name, table) + } + } db2 := memory.NewDatabase("foo") - db2.AddTable("other_table", table4) + db2.AddTable("other_table", tables["other_table"]) catalog := sql.NewCatalog() catalog.AddDatabase(db) diff --git a/sql/test_util/index_driver.go b/sql/test_util/index_driver.go index 340b82d36..f0cad7fde 100755 --- a/sql/test_util/index_driver.go +++ b/sql/test_util/index_driver.go @@ -5,7 +5,11 @@ import ( ) type TestIndexDriver struct { + indexes map[string][]sql.Index +} +func NewIndexDriver(indexes map[string][]sql.Index) *TestIndexDriver { + return &TestIndexDriver{indexes: indexes} } func (d *TestIndexDriver) ID() string { @@ -13,7 +17,7 @@ func (d *TestIndexDriver) ID() string { } func (d *TestIndexDriver) LoadAll(db, table string) ([]sql.Index, error) { - panic("implement me") + return d.indexes[table], nil } func (d *TestIndexDriver) Save(*sql.Context, sql.Index, sql.PartitionIndexKeyValueIter) error { From 5409063fda946ccab2e4cda82a5fed6a3ac9eee9 Mon Sep 17 00:00:00 2001 From: Zach Musgrave Date: Tue, 29 Oct 2019 14:14:59 -0700 Subject: [PATCH 07/42] Checkpoint. Realizing that the in-memory databases and the index implementations necessarily need to be coupled, so starting down that path. Signed-off-by: Zach Musgrave --- engine_test.go | 96 +++++++++++++++++++++++++---- sql/analyzer/assign_indexes.go | 24 +++++--- sql/analyzer/assign_indexes_test.go | 42 +++++-------- sql/test_util/ascend_index.go | 2 +- sql/test_util/descend_index.go | 2 +- sql/test_util/index_driver.go | 12 +++- sql/test_util/mergeable_index.go | 15 +++-- sql/test_util/negative_index.go | 2 +- sql/test_util/unmergeable_index.go | 13 ++-- 9 files changed, 141 insertions(+), 67 deletions(-) diff --git a/engine_test.go b/engine_test.go index dea9be3eb..2c3397130 100644 --- a/engine_test.go +++ b/engine_test.go @@ -2,6 +2,8 @@ package sqle_test import ( "context" + "github.com/src-d/go-mysql-server/sql/expression" + testutil "github.com/src-d/go-mysql-server/sql/test_util" "io" "math" "strings" @@ -58,6 +60,14 @@ var queries = []struct { "SELECT i FROM mytable WHERE i <> 2;", []sql.Row{{int64(1)}, {int64(3)}}, }, + { + "SELECT i FROM mytable WHERE i in (1, 3)", + []sql.Row{{int64(1)}, {int64(3)}}, + }, + { + "SELECT i FROM mytable WHERE i = 1 or i = 3", + []sql.Row{{int64(1)}, {int64(3)}}, + }, { "SELECT f32 FROM floattable WHERE f64 = 2.0;", []sql.Row{{float32(2.0)}}, @@ -1588,17 +1598,54 @@ var queries = []struct { } func TestQueries(t *testing.T) { - e := newEngine(t) - t.Run("sequential", func(t *testing.T) { - for _, tt := range queries { - testQuery(t, e, tt.query, tt.expected) - } + // e := newEngine(t) + // t.Run("sequential", func(t *testing.T) { + // for _, tt := range queries { + // testQuery(t, e, tt.query, tt.expected) + // } + // }) + // + // ep := newEngineWithParallelism(t, 2) + // t.Run("parallel", func(t *testing.T) { + // for _, tt := range queries { + // testQuery(t, ep, tt.query, tt.expected) + // } + // }) + + indexDriver := testutil.NewIndexDriver("mydb", map[string][]sql.Index{ + "mytable": { + &testutil.UnmergeableDummyIndex{ + DB: "mydb", + DriverName: testutil.IndexDriverId, + TableName: "mytable", + Exprs: []sql.Expression{ + expression.NewGetFieldWithTable(0, sql.Int64, "mytable", "i", false), + }, + }, + &testutil.UnmergeableDummyIndex{ + DB: "mydb", + DriverName: testutil.IndexDriverId, + TableName: "mytable", + Exprs: []sql.Expression{ + expression.NewGetFieldWithTable(0, sql.Text, "mytable", "s", false), + }, + }, + &testutil.UnmergeableDummyIndex{ + DB: "mydb", + DriverName: testutil.IndexDriverId, + TableName: "mytable", + Exprs: []sql.Expression{ + expression.NewGetFieldWithTable(0, sql.Int64, "mytable", "i", false), + expression.NewGetFieldWithTable(0, sql.Text, "mytable", "s", false), + }, + }, + }, }) - ep := newEngineWithParallelism(t, 2) - t.Run("parallel", func(t *testing.T) { + engIndexes := newEngineWithIndexes(t, indexDriver) + t.Run("indexed", func(t *testing.T) { for _, tt := range queries { - testQuery(t, ep, tt.query, tt.expected) + testQuery(t, engIndexes, tt.query, tt.expected) } }) } @@ -2800,10 +2847,6 @@ func testQueryWithContext(ctx *sql.Context, t *testing.T, e *sqle.Engine, q stri }) } -func newEngine(t *testing.T) *sqle.Engine { - return newEngineWithParallelism(t, 1) -} - func allTestTables(t *testing.T) map[string]*memory.Table { tables := make(map[string]*memory.Table) @@ -2946,6 +2989,35 @@ func allTestTables(t *testing.T) map[string]*memory.Table { return tables } +func newEngine(t *testing.T) *sqle.Engine { + return newEngineWithParallelism(t, 1) +} + +func newEngineWithIndexes(t *testing.T, driver sql.IndexDriver) *sqle.Engine { + tables := allTestTables(t) + + db := memory.NewDatabase("mydb") + for name, table := range tables { + if name != "other_table" { + db.AddTable(name, table) + } + } + + db2 := memory.NewDatabase("foo") + db2.AddTable("other_table", tables["other_table"]) + + catalog := sql.NewCatalog() + catalog.AddDatabase(db) + catalog.AddDatabase(db2) + catalog.AddDatabase(sql.NewInformationSchemaDatabase(catalog)) + catalog.RegisterIndexDriver(driver) + + a := analyzer.NewDefault(catalog) + engine := sqle.New(catalog, a, new(sqle.Config)) + require.NoError(t, engine.Init()) + return engine +} + func newEngineWithParallelism(t *testing.T, parallelism int) *sqle.Engine { tables := allTestTables(t) diff --git a/sql/analyzer/assign_indexes.go b/sql/analyzer/assign_indexes.go index c5c32a93b..bc70d0130 100644 --- a/sql/analyzer/assign_indexes.go +++ b/sql/analyzer/assign_indexes.go @@ -102,17 +102,21 @@ func getIndexes(e sql.Expression, aliases map[string]sql.Expression, a *Analyzer } for table, leftIdx := range leftIndexes { - if rightIdx, ok := rightIndexes[table]; ok { - if canMergeIndexes(leftIdx.lookup, rightIdx.lookup) { - leftIdx.lookup = leftIdx.lookup.(sql.SetOperations).Union(rightIdx.lookup) - leftIdx.indexes = append(leftIdx.indexes, rightIdx.indexes...) - } else { - // Since we can return one index per table, if we can't merge the second index from this table, return no - // indexes. Returning a single one will lead to incorrect results from e.g. pushdown operations when only one - // side of the OR expression is used to index the table. - return nil, nil - } + if rightIdx, ok := rightIndexes[table]; ok && canMergeIndexes(leftIdx.lookup, rightIdx.lookup) { + leftIdx.lookup = leftIdx.lookup.(sql.SetOperations).Union(rightIdx.lookup) + leftIdx.indexes = append(leftIdx.indexes, rightIdx.indexes...) } + // if rightIdx, ok := rightIndexes[table]; ok { + // if canMergeIndexes(leftIdx.lookup, rightIdx.lookup) { + // leftIdx.lookup = leftIdx.lookup.(sql.SetOperations).Union(rightIdx.lookup) + // leftIdx.indexes = append(leftIdx.indexes, rightIdx.indexes...) + // } else { + // // Since we can return one index per table, if we can't merge the second index from this table, return no + // // indexes. Returning a single one will lead to incorrect results from e.g. pushdown operations when only one + // // side of the OR expression is used to index the table. + // return nil, nil + // } + // } result[table] = leftIdx } diff --git a/sql/analyzer/assign_indexes_test.go b/sql/analyzer/assign_indexes_test.go index 36f89ab20..9f920525a 100644 --- a/sql/analyzer/assign_indexes_test.go +++ b/sql/analyzer/assign_indexes_test.go @@ -61,8 +61,8 @@ func TestAssignIndexes(t *testing.T) { catalog := sql.NewCatalog() idx1 := &testutil.MergeableDummyIndex{ - "t2", - []sql.Expression{ + TableName: "t2", + Exprs: []sql.Expression{ expression.NewGetFieldWithTable(0, sql.Int64, "t2", "bar", false), }, } @@ -72,8 +72,8 @@ func TestAssignIndexes(t *testing.T) { <-ready idx2 := &testutil.MergeableDummyIndex{ - "t1", - []sql.Expression{ + TableName: "t1", + Exprs: []sql.Expression{ expression.NewGetFieldWithTable(0, sql.Int64, "t1", "foo", false), }, } @@ -84,8 +84,8 @@ func TestAssignIndexes(t *testing.T) { <-ready idx3 := &testutil.UnmergeableDummyIndex{ - "t1", - []sql.Expression{ + TableName: "t1", + Exprs: []sql.Expression{ expression.NewGetFieldWithTable(0, sql.Int64, "t1", "bar", false), }, } @@ -191,27 +191,27 @@ func TestAssignIndexes(t *testing.T) { func TestGetIndexes(t *testing.T) { indexes := []sql.Index { &testutil.MergeableDummyIndex{ - "t1", - []sql.Expression{ + TableName: "t1", + Exprs: []sql.Expression{ col(0, "t1", "bar"), }, }, &testutil.MergeableDummyIndex{ - "t2", - []sql.Expression{ + TableName: "t2", + Exprs: []sql.Expression{ col(0, "t2", "foo"), col(0, "t2", "bar"), }, }, &testutil.MergeableDummyIndex{ - "t2", - []sql.Expression{ + TableName: "t2", + Exprs: []sql.Expression{ col(0, "t2", "bar"), }, }, &testutil.UnmergeableDummyIndex{ - "t3", - []sql.Expression{ + TableName: "t3", + Exprs: []sql.Expression{ col(0, "t3", "foo"), }, }, @@ -279,20 +279,6 @@ func TestGetIndexes(t *testing.T) { nil, true, }, - { - or( - eq( - col(0, "t3", "foo"), - lit(1), - ), - eq( - col(0, "t3", "foo"), - lit(2), - ), - ), - nil, - true, - }, { in( col(0, "t3", "foo"), diff --git a/sql/test_util/ascend_index.go b/sql/test_util/ascend_index.go index fa8c8baee..88b122e1a 100755 --- a/sql/test_util/ascend_index.go +++ b/sql/test_util/ascend_index.go @@ -11,7 +11,7 @@ type AscendIndexLookup struct { } func (AscendIndexLookup) Values(sql.Partition) (sql.IndexValueIter, error) { - panic("ascendIndexLookup.Values is a placeholder") + return nil, nil } func (l *AscendIndexLookup) Indexes() []string { diff --git a/sql/test_util/descend_index.go b/sql/test_util/descend_index.go index ac15e26c3..077ed12dd 100755 --- a/sql/test_util/descend_index.go +++ b/sql/test_util/descend_index.go @@ -11,7 +11,7 @@ type DescendIndexLookup struct { } func (DescendIndexLookup) Values(sql.Partition) (sql.IndexValueIter, error) { - panic("descendIndexLookup.Values is a placeholder") + return nil, nil } func (l *DescendIndexLookup) Indexes() []string { diff --git a/sql/test_util/index_driver.go b/sql/test_util/index_driver.go index f0cad7fde..afe9ea8ec 100755 --- a/sql/test_util/index_driver.go +++ b/sql/test_util/index_driver.go @@ -4,19 +4,25 @@ import ( "github.com/src-d/go-mysql-server/sql" ) +const IndexDriverId = "TestIndexDriver" + type TestIndexDriver struct { + db string indexes map[string][]sql.Index } -func NewIndexDriver(indexes map[string][]sql.Index) *TestIndexDriver { - return &TestIndexDriver{indexes: indexes} +func NewIndexDriver(db string, indexes map[string][]sql.Index) *TestIndexDriver { + return &TestIndexDriver{db: db, indexes: indexes} } func (d *TestIndexDriver) ID() string { - panic("TestIndexDriver") + return IndexDriverId } func (d *TestIndexDriver) LoadAll(db, table string) ([]sql.Index, error) { + if d.db != db { + return nil, nil + } return d.indexes[table], nil } diff --git a/sql/test_util/mergeable_index.go b/sql/test_util/mergeable_index.go index 590149c75..134078503 100755 --- a/sql/test_util/mergeable_index.go +++ b/sql/test_util/mergeable_index.go @@ -31,7 +31,7 @@ func (i *MergeableIndexLookup) IsMergeable(lookup sql.IndexLookup) bool { } func (i *MergeableIndexLookup) Values(sql.Partition) (sql.IndexValueIter, error) { - panic("not implemented") + return nil, nil } func (i *MergeableIndexLookup) Indexes() []string { @@ -75,7 +75,7 @@ type MergedIndexLookup struct { } func (MergedIndexLookup) Values(sql.Partition) (sql.IndexValueIter, error) { - panic("mergedIndexLookup.Values is a placeholder") + return nil, nil } func (i *MergedIndexLookup) Indexes() []string { @@ -103,12 +103,15 @@ func (MergedIndexLookup) Intersection(...sql.IndexLookup) sql.IndexLookup { } type MergeableDummyIndex struct { - TableName string - Exprs []sql.Expression + DB string // required for engine tests + DriverName string // required for engine tests + TableName string + Exprs []sql.Expression } -func (MergeableDummyIndex) Database() string { return "" } -func (MergeableDummyIndex) Driver() string { return "" } +func (i MergeableDummyIndex) Database() string { return i.DB } +func (i MergeableDummyIndex) Driver() string { return i.DriverName } + func (i MergeableDummyIndex) Expressions() []string { var exprs []string for _, e := range i.Exprs { diff --git a/sql/test_util/negative_index.go b/sql/test_util/negative_index.go index 34650cede..02bdfce04 100755 --- a/sql/test_util/negative_index.go +++ b/sql/test_util/negative_index.go @@ -15,7 +15,7 @@ func (l *NegateIndexLookup) GetUnions() []string { return l.unions } func (l *NegateIndexLookup) GetIntersections() []string { return l.intersections } func (*NegateIndexLookup) Values(sql.Partition) (sql.IndexValueIter, error) { - panic("negateIndexLookup.Values is a placeholder") + return nil, nil } func (l *NegateIndexLookup) Indexes() []string { diff --git a/sql/test_util/unmergeable_index.go b/sql/test_util/unmergeable_index.go index 5ea916e44..bca6c5c4a 100755 --- a/sql/test_util/unmergeable_index.go +++ b/sql/test_util/unmergeable_index.go @@ -7,12 +7,15 @@ import ( ) type UnmergeableDummyIndex struct { - TableName string - Exprs []sql.Expression + DB string // required for engine tests with driver + DriverName string // required for engine tests with driver + TableName string + Exprs []sql.Expression } -func (u *UnmergeableDummyIndex) Database() string { return "" } -func (u *UnmergeableDummyIndex) Driver() string { return "" } +func (u *UnmergeableDummyIndex) Database() string { return u.DB } +func (u *UnmergeableDummyIndex) Driver() string { return u.DriverName } + func (u *UnmergeableDummyIndex) Expressions() []string { var exprs []string for _, e := range u.Exprs { @@ -39,7 +42,7 @@ type UnmergeableIndexLookup struct { } func (u UnmergeableIndexLookup) Values(sql.Partition) (sql.IndexValueIter, error) { - panic("unimplemented") + return nil, nil } func (u UnmergeableIndexLookup) Indexes() []string { From 1c8fd6fa7a3bb97e56465cdda4aae8a81518106e Mon Sep 17 00:00:00 2001 From: Zach Musgrave Date: Tue, 29 Oct 2019 15:50:05 -0700 Subject: [PATCH 08/42] Moved index implementations to the memory package, made engine_test run queries with every combination of indexed / parallel / partitioned Signed-off-by: Zach Musgrave --- engine_test.go | 114 ++++++------- {sql/test_util => memory}/ascend_index.go | 2 +- {sql/test_util => memory}/descend_index.go | 2 +- {sql/test_util => memory}/index_driver.go | 4 +- {sql/test_util => memory}/mergeable_index.go | 2 +- {sql/test_util => memory}/negative_index.go | 2 +- .../test_util => memory}/unmergeable_index.go | 2 +- sql/analyzer/assign_indexes_test.go | 150 +++++++++--------- sql/analyzer/pushdown_test.go | 23 ++- 9 files changed, 143 insertions(+), 158 deletions(-) rename {sql/test_util => memory}/ascend_index.go (97%) rename {sql/test_util => memory}/descend_index.go (97%) rename {sql/test_util => memory}/index_driver.go (93%) rename {sql/test_util => memory}/mergeable_index.go (99%) rename {sql/test_util => memory}/negative_index.go (98%) rename {sql/test_util => memory}/unmergeable_index.go (98%) diff --git a/engine_test.go b/engine_test.go index 2c3397130..ece771db8 100644 --- a/engine_test.go +++ b/engine_test.go @@ -2,8 +2,8 @@ package sqle_test import ( "context" + "fmt" "github.com/src-d/go-mysql-server/sql/expression" - testutil "github.com/src-d/go-mysql-server/sql/test_util" "io" "math" "strings" @@ -20,7 +20,6 @@ import ( "github.com/src-d/go-mysql-server/sql/plan" "github.com/src-d/go-mysql-server/test" - "github.com/stretchr/testify/require" ) @@ -1598,42 +1597,28 @@ var queries = []struct { } func TestQueries(t *testing.T) { - // e := newEngine(t) - // t.Run("sequential", func(t *testing.T) { - // for _, tt := range queries { - // testQuery(t, e, tt.query, tt.expected) - // } - // }) - // - // ep := newEngineWithParallelism(t, 2) - // t.Run("parallel", func(t *testing.T) { - // for _, tt := range queries { - // testQuery(t, ep, tt.query, tt.expected) - // } - // }) - - indexDriver := testutil.NewIndexDriver("mydb", map[string][]sql.Index{ + testIndexDriver := memory.NewIndexDriver("mydb", map[string][]sql.Index{ "mytable": { - &testutil.UnmergeableDummyIndex{ - DB: "mydb", - DriverName: testutil.IndexDriverId, - TableName: "mytable", + &memory.UnmergeableDummyIndex{ + DB: "mydb", + DriverName: memory.IndexDriverId, + TableName: "mytable", Exprs: []sql.Expression{ expression.NewGetFieldWithTable(0, sql.Int64, "mytable", "i", false), }, }, - &testutil.UnmergeableDummyIndex{ - DB: "mydb", - DriverName: testutil.IndexDriverId, - TableName: "mytable", + &memory.UnmergeableDummyIndex{ + DB: "mydb", + DriverName: memory.IndexDriverId, + TableName: "mytable", Exprs: []sql.Expression{ expression.NewGetFieldWithTable(0, sql.Text, "mytable", "s", false), }, }, - &testutil.UnmergeableDummyIndex{ - DB: "mydb", - DriverName: testutil.IndexDriverId, - TableName: "mytable", + &memory.UnmergeableDummyIndex{ + DB: "mydb", + DriverName: memory.IndexDriverId, + TableName: "mytable", Exprs: []sql.Expression{ expression.NewGetFieldWithTable(0, sql.Int64, "mytable", "i", false), expression.NewGetFieldWithTable(0, sql.Text, "mytable", "s", false), @@ -1642,12 +1627,31 @@ func TestQueries(t *testing.T) { }, }) - engIndexes := newEngineWithIndexes(t, indexDriver) - t.Run("indexed", func(t *testing.T) { - for _, tt := range queries { - testQuery(t, engIndexes, tt.query, tt.expected) + // Test all queries with these combinations, for a total of 8 runs: + // 1) Partitioned tables / non partitioned tables + // 2) Indexes enabled / disabled + // 3) Parallelism on / off + numPartitionsVals := []int{1, testNumPartitions} + useIndexesVals := []bool{false, true} + parallelVals := []int{1, 2} + for _, numPartitions := range numPartitionsVals { + for _, useIndexes := range useIndexesVals { + for _, parallelism := range parallelVals { + var indexDriver sql.IndexDriver + if useIndexes { + indexDriver = testIndexDriver + } + engine := newEngineWithParallelism(t, parallelism, indexDriver) + + testName := fmt.Sprintf("partitions=%d, indexes=%v, parallelism=%v", numPartitions, useIndexes, parallelism) + t.Run(testName, func(t *testing.T) { + for _, tt := range queries { + testQuery(t, engine, tt.query, tt.expected) + } + }) + } } - }) + } } func TestSessionSelectLimit(t *testing.T) { @@ -1798,7 +1802,7 @@ func TestWarnings(t *testing.T) { } e := newEngine(t) - ep := newEngineWithParallelism(t, 2) + ep := newEngineWithParallelism(t, 2, nil) t.Run("sequential", func(t *testing.T) { for _, tt := range queries { @@ -1862,7 +1866,7 @@ func TestClearWarnings(t *testing.T) { func TestDescribe(t *testing.T) { e := newEngine(t) - ep := newEngineWithParallelism(t, 2) + ep := newEngineWithParallelism(t, 2, nil) query := `DESCRIBE FORMAT=TREE SELECT * FROM mytable` expectedSeq := []sql.Row{ @@ -2990,35 +2994,10 @@ func allTestTables(t *testing.T) map[string]*memory.Table { } func newEngine(t *testing.T) *sqle.Engine { - return newEngineWithParallelism(t, 1) -} - -func newEngineWithIndexes(t *testing.T, driver sql.IndexDriver) *sqle.Engine { - tables := allTestTables(t) - - db := memory.NewDatabase("mydb") - for name, table := range tables { - if name != "other_table" { - db.AddTable(name, table) - } - } - - db2 := memory.NewDatabase("foo") - db2.AddTable("other_table", tables["other_table"]) - - catalog := sql.NewCatalog() - catalog.AddDatabase(db) - catalog.AddDatabase(db2) - catalog.AddDatabase(sql.NewInformationSchemaDatabase(catalog)) - catalog.RegisterIndexDriver(driver) - - a := analyzer.NewDefault(catalog) - engine := sqle.New(catalog, a, new(sqle.Config)) - require.NoError(t, engine.Init()) - return engine + return newEngineWithParallelism(t, 1, nil) } -func newEngineWithParallelism(t *testing.T, parallelism int) *sqle.Engine { +func newEngineWithParallelism(t *testing.T, parallelism int, driver sql.IndexDriver) *sqle.Engine { tables := allTestTables(t) db := memory.NewDatabase("mydb") @@ -3043,7 +3022,14 @@ func newEngineWithParallelism(t *testing.T, parallelism int) *sqle.Engine { a = analyzer.NewDefault(catalog) } - return sqle.New(catalog, a, new(sqle.Config)) + if driver != nil { + catalog.RegisterIndexDriver(driver) + } + + engine := sqle.New(catalog, a, new(sqle.Config)) + require.NoError(t, engine.Init()) + + return engine } const expectedTree = `Limit(5) diff --git a/sql/test_util/ascend_index.go b/memory/ascend_index.go similarity index 97% rename from sql/test_util/ascend_index.go rename to memory/ascend_index.go index 88b122e1a..36be10240 100755 --- a/sql/test_util/ascend_index.go +++ b/memory/ascend_index.go @@ -1,4 +1,4 @@ -package testutil +package memory import ( "github.com/src-d/go-mysql-server/sql" diff --git a/sql/test_util/descend_index.go b/memory/descend_index.go similarity index 97% rename from sql/test_util/descend_index.go rename to memory/descend_index.go index 077ed12dd..16e3024ac 100755 --- a/sql/test_util/descend_index.go +++ b/memory/descend_index.go @@ -1,4 +1,4 @@ -package testutil +package memory import ( "github.com/src-d/go-mysql-server/sql" diff --git a/sql/test_util/index_driver.go b/memory/index_driver.go similarity index 93% rename from sql/test_util/index_driver.go rename to memory/index_driver.go index afe9ea8ec..e05c2f48a 100755 --- a/sql/test_util/index_driver.go +++ b/memory/index_driver.go @@ -1,10 +1,10 @@ -package testutil +package memory import ( "github.com/src-d/go-mysql-server/sql" ) -const IndexDriverId = "TestIndexDriver" +const IndexDriverId = "MemoryIndexDriver" type TestIndexDriver struct { db string diff --git a/sql/test_util/mergeable_index.go b/memory/mergeable_index.go similarity index 99% rename from sql/test_util/mergeable_index.go rename to memory/mergeable_index.go index 134078503..4d78ae9b0 100755 --- a/sql/test_util/mergeable_index.go +++ b/memory/mergeable_index.go @@ -1,4 +1,4 @@ -package testutil +package memory import ( "fmt" diff --git a/sql/test_util/negative_index.go b/memory/negative_index.go similarity index 98% rename from sql/test_util/negative_index.go rename to memory/negative_index.go index 02bdfce04..c39292d99 100755 --- a/sql/test_util/negative_index.go +++ b/memory/negative_index.go @@ -1,4 +1,4 @@ -package testutil +package memory import ( "github.com/src-d/go-mysql-server/sql" diff --git a/sql/test_util/unmergeable_index.go b/memory/unmergeable_index.go similarity index 98% rename from sql/test_util/unmergeable_index.go rename to memory/unmergeable_index.go index bca6c5c4a..3da5cc1b1 100755 --- a/sql/test_util/unmergeable_index.go +++ b/memory/unmergeable_index.go @@ -1,4 +1,4 @@ -package testutil +package memory import ( "fmt" diff --git a/sql/analyzer/assign_indexes_test.go b/sql/analyzer/assign_indexes_test.go index 9f920525a..89fe71027 100644 --- a/sql/analyzer/assign_indexes_test.go +++ b/sql/analyzer/assign_indexes_test.go @@ -1,7 +1,6 @@ package analyzer import ( - "github.com/src-d/go-mysql-server/sql/test_util" "testing" "github.com/src-d/go-mysql-server/memory" @@ -15,9 +14,9 @@ func TestNegateIndex(t *testing.T) { require := require.New(t) catalog := sql.NewCatalog() - idx1 := &testutil.MergeableDummyIndex{ - "t1", - []sql.Expression{ + idx1 := &memory.MergeableDummyIndex{ + TableName: "t1", + Exprs: []sql.Expression{ expression.NewGetFieldWithTable(0, sql.Int64, "t1", "foo", false), }, } @@ -51,7 +50,7 @@ func TestNegateIndex(t *testing.T) { lookupIdxs, ok := result["t1"] require.True(ok) - negate, ok := lookupIdxs.lookup.(*testutil.NegateIndexLookup) + negate, ok := lookupIdxs.lookup.(*memory.NegateIndexLookup) require.True(ok) require.True(negate.Value == "1") } @@ -60,7 +59,7 @@ func TestAssignIndexes(t *testing.T) { require := require.New(t) catalog := sql.NewCatalog() - idx1 := &testutil.MergeableDummyIndex{ + idx1 := &memory.MergeableDummyIndex{ TableName: "t2", Exprs: []sql.Expression{ expression.NewGetFieldWithTable(0, sql.Int64, "t2", "bar", false), @@ -71,7 +70,7 @@ func TestAssignIndexes(t *testing.T) { close(done) <-ready - idx2 := &testutil.MergeableDummyIndex{ + idx2 := &memory.MergeableDummyIndex{ TableName: "t1", Exprs: []sql.Expression{ expression.NewGetFieldWithTable(0, sql.Int64, "t1", "foo", false), @@ -83,7 +82,7 @@ func TestAssignIndexes(t *testing.T) { close(done) <-ready - idx3 := &testutil.UnmergeableDummyIndex{ + idx3 := &memory.UnmergeableDummyIndex{ TableName: "t1", Exprs: []sql.Expression{ expression.NewGetFieldWithTable(0, sql.Int64, "t1", "bar", false), @@ -136,14 +135,14 @@ func TestAssignIndexes(t *testing.T) { lookupIdxs, ok := result["t1"] require.True(ok) - mergeable, ok := lookupIdxs.lookup.(*testutil.MergeableIndexLookup) + mergeable, ok := lookupIdxs.lookup.(*memory.MergeableIndexLookup) require.True(ok) require.True(mergeable.Id == "2") lookupIdxs, ok = result["t2"] require.True(ok) - mergeable, ok = lookupIdxs.lookup.(*testutil.MergeableIndexLookup) + mergeable, ok = lookupIdxs.lookup.(*memory.MergeableIndexLookup) require.True(ok) require.True(mergeable.Id == "1") @@ -190,26 +189,26 @@ func TestAssignIndexes(t *testing.T) { func TestGetIndexes(t *testing.T) { indexes := []sql.Index { - &testutil.MergeableDummyIndex{ + &memory.MergeableDummyIndex{ TableName: "t1", Exprs: []sql.Expression{ col(0, "t1", "bar"), }, }, - &testutil.MergeableDummyIndex{ + &memory.MergeableDummyIndex{ TableName: "t2", Exprs: []sql.Expression{ col(0, "t2", "foo"), col(0, "t2", "bar"), }, }, - &testutil.MergeableDummyIndex{ + &memory.MergeableDummyIndex{ TableName: "t2", Exprs: []sql.Expression{ col(0, "t2", "bar"), }, }, - &testutil.UnmergeableDummyIndex{ + &memory.UnmergeableDummyIndex{ TableName: "t3", Exprs: []sql.Expression{ col(0, "t3", "foo"), @@ -237,7 +236,7 @@ func TestGetIndexes(t *testing.T) { ), map[string]*indexLookup{ "t1": &indexLookup{ - &testutil.MergeableIndexLookup{Id: "1"}, + &memory.MergeableIndexLookup{Id: "1"}, []sql.Index{indexes[0]}, }, }, @@ -256,7 +255,7 @@ func TestGetIndexes(t *testing.T) { ), map[string]*indexLookup{ "t1": &indexLookup{ - &testutil.MergeableIndexLookup{Id: "1", Unions: []string{"2"}}, + &memory.MergeableIndexLookup{Id: "1", Unions: []string{"2"}}, []sql.Index{ indexes[0], indexes[0], @@ -294,7 +293,7 @@ func TestGetIndexes(t *testing.T) { ), map[string]*indexLookup{ "t1": &indexLookup{ - &testutil.MergeableIndexLookup{Id: "1", Unions: []string{"2"}}, + &memory.MergeableIndexLookup{Id: "1", Unions: []string{"2"}}, []sql.Index{ indexes[0], }, @@ -315,7 +314,7 @@ func TestGetIndexes(t *testing.T) { ), map[string]*indexLookup{ "t1": &indexLookup{ - &testutil.MergeableIndexLookup{Id: "1", Intersections: []string{"2"}}, + &memory.MergeableIndexLookup{Id: "1", Intersections: []string{"2"}}, []sql.Index{ indexes[0], indexes[0], @@ -349,7 +348,7 @@ func TestGetIndexes(t *testing.T) { ), map[string]*indexLookup{ "t1": &indexLookup{ - &testutil.MergeableIndexLookup{Id: "1", Unions: []string{"2", "4"}, Intersections: []string{"3"}}, + &memory.MergeableIndexLookup{Id: "1", Unions: []string{"2", "4"}, Intersections: []string{"3"}}, []sql.Index{ indexes[0], indexes[0], @@ -385,7 +384,7 @@ func TestGetIndexes(t *testing.T) { ), map[string]*indexLookup{ "t1": &indexLookup{ - &testutil.MergeableIndexLookup{Id: "1", Unions: []string{"2", "3", "4"}}, + &memory.MergeableIndexLookup{Id: "1", Unions: []string{"2", "3", "4"}}, []sql.Index{ indexes[0], indexes[0], @@ -403,7 +402,7 @@ func TestGetIndexes(t *testing.T) { ), map[string]*indexLookup{ "t1": &indexLookup{ - &testutil.MergeableIndexLookup{Id: "1", Unions: []string{"2", "3", "4"}}, + &memory.MergeableIndexLookup{Id: "1", Unions: []string{"2", "3", "4"}}, []sql.Index{indexes[0]}, }, }, @@ -434,11 +433,11 @@ func TestGetIndexes(t *testing.T) { ), map[string]*indexLookup{ "t1": &indexLookup{ - &testutil.MergeableIndexLookup{Id: "3"}, + &memory.MergeableIndexLookup{Id: "3"}, []sql.Index{indexes[0]}, }, "t2": &indexLookup{ - &testutil.MergeableIndexLookup{Id: "1, 2"}, + &memory.MergeableIndexLookup{Id: "1, 2"}, []sql.Index{indexes[1]}, }, }, @@ -475,11 +474,11 @@ func TestGetIndexes(t *testing.T) { ), map[string]*indexLookup{ "t1": &indexLookup{ - &testutil.MergeableIndexLookup{Id: "3"}, + &memory.MergeableIndexLookup{Id: "3"}, []sql.Index{indexes[0]}, }, "t2": &indexLookup{ - &testutil.MergeableIndexLookup{Id: "5", Unions: []string{"1, 2"}}, + &memory.MergeableIndexLookup{Id: "5", Unions: []string{"1, 2"}}, []sql.Index{ indexes[2], indexes[1], @@ -495,7 +494,7 @@ func TestGetIndexes(t *testing.T) { ), map[string]*indexLookup{ "t1": &indexLookup{ - &testutil.DescendIndexLookup{Gt: []interface{}{int64(1)}}, + &memory.DescendIndexLookup{Gt: []interface{}{int64(1)}}, []sql.Index{indexes[0]}, }, }, @@ -508,7 +507,7 @@ func TestGetIndexes(t *testing.T) { ), map[string]*indexLookup{ "t1": &indexLookup{ - &testutil.AscendIndexLookup{Lt: []interface{}{int64(1)}}, + &memory.AscendIndexLookup{Lt: []interface{}{int64(1)}}, []sql.Index{indexes[0]}, }, }, @@ -521,7 +520,7 @@ func TestGetIndexes(t *testing.T) { ), map[string]*indexLookup{ "t1": &indexLookup{ - &testutil.AscendIndexLookup{Gte: []interface{}{int64(1)}}, + &memory.AscendIndexLookup{Gte: []interface{}{int64(1)}}, []sql.Index{indexes[0]}, }, }, @@ -534,7 +533,7 @@ func TestGetIndexes(t *testing.T) { ), map[string]*indexLookup{ "t1": &indexLookup{ - &testutil.DescendIndexLookup{Lte: []interface{}{int64(1)}}, + &memory.DescendIndexLookup{Lte: []interface{}{int64(1)}}, []sql.Index{indexes[0]}, }, }, @@ -548,13 +547,13 @@ func TestGetIndexes(t *testing.T) { ), map[string]*indexLookup{ "t1": &indexLookup{ - &testutil.MergedIndexLookup{ + &memory.MergedIndexLookup{ []sql.IndexLookup{ - &testutil.AscendIndexLookup{ + &memory.AscendIndexLookup{ Gte: []interface{}{int64(1)}, Lt: []interface{}{int64(5)}, }, - &testutil.DescendIndexLookup{ + &memory.DescendIndexLookup{ Gt: []interface{}{int64(1)}, Lte: []interface{}{int64(5)}, }, @@ -574,7 +573,7 @@ func TestGetIndexes(t *testing.T) { ), map[string]*indexLookup{ "t1": &indexLookup{ - &testutil.NegateIndexLookup{ + &memory.NegateIndexLookup{ Value: "1", }, []sql.Index{indexes[0]}, @@ -592,7 +591,7 @@ func TestGetIndexes(t *testing.T) { ), map[string]*indexLookup{ "t1": &indexLookup{ - &testutil.DescendIndexLookup{Lte: []interface{}{int64(10)}}, + &memory.DescendIndexLookup{Lte: []interface{}{int64(10)}}, []sql.Index{indexes[0]}, }, }, @@ -608,7 +607,7 @@ func TestGetIndexes(t *testing.T) { ), map[string]*indexLookup{ "t1": &indexLookup{ - &testutil.AscendIndexLookup{Lt: []interface{}{int64(10)}}, + &memory.AscendIndexLookup{Lt: []interface{}{int64(10)}}, []sql.Index{indexes[0]}, }, }, @@ -624,7 +623,7 @@ func TestGetIndexes(t *testing.T) { ), map[string]*indexLookup{ "t1": &indexLookup{ - &testutil.DescendIndexLookup{Gt: []interface{}{int64(10)}}, + &memory.DescendIndexLookup{Gt: []interface{}{int64(10)}}, []sql.Index{indexes[0]}, }, }, @@ -640,7 +639,7 @@ func TestGetIndexes(t *testing.T) { ), map[string]*indexLookup{ "t1": &indexLookup{ - &testutil.AscendIndexLookup{Gte: []interface{}{int64(10)}}, + &memory.AscendIndexLookup{Gte: []interface{}{int64(10)}}, []sql.Index{indexes[0]}, }, }, @@ -661,12 +660,12 @@ func TestGetIndexes(t *testing.T) { ), map[string]*indexLookup{ "t1": &indexLookup{ - &testutil.MergedIndexLookup{ + &memory.MergedIndexLookup{ Children: []sql.IndexLookup{ - &testutil.NegateIndexLookup{ + &memory.NegateIndexLookup{ Value: "10", }, - &testutil.NegateIndexLookup{ + &memory.NegateIndexLookup{ Value: "11", }, }, @@ -694,7 +693,7 @@ func TestGetIndexes(t *testing.T) { ), map[string]*indexLookup{ "t1": &indexLookup{ - &testutil.MergeableIndexLookup{ + &memory.MergeableIndexLookup{ Id: "not 10", Intersections: []string{"not 11"}, }, @@ -724,7 +723,7 @@ func TestGetIndexes(t *testing.T) { ), map[string]*indexLookup{ "t2": &indexLookup{ - &testutil.NegateIndexLookup{ + &memory.NegateIndexLookup{ Value: "110", }, []sql.Index{ @@ -746,7 +745,7 @@ func TestGetIndexes(t *testing.T) { ), map[string]*indexLookup{ "t1": &indexLookup{ - &testutil.MergeableIndexLookup{ + &memory.MergeableIndexLookup{ Id: "not 1", Intersections: []string{"not 2", "not 3", "not 4"}, }, @@ -786,36 +785,37 @@ func TestGetMultiColumnIndexes(t *testing.T) { require := require.New(t) catalog := sql.NewCatalog() - indexes := []*testutil.MergeableDummyIndex{ + indexes := []*memory.MergeableDummyIndex{ { - "t1", - []sql.Expression{ + TableName: "t1", + Exprs: []sql.Expression{ col(1, "t1", "foo"), col(2, "t1", "bar"), }, }, { - "t2", - []sql.Expression{ + TableName: "t2", + Exprs: []sql.Expression{ col(0, "t2", "foo"), col(1, "t2", "bar"), col(2, "t2", "baz"), }, }, { - "t2", - []sql.Expression{ + TableName: "t2", + Exprs: []sql.Expression{ col(0, "t2", "foo"), col(0, "t2", "bar"), }, }, { - "t3", - []sql.Expression{col(0, "t3", "foo")}, + + TableName: "t3", + Exprs: []sql.Expression{col(0, "t3", "foo")}, }, { - "t4", - []sql.Expression{ + TableName: "t4", + Exprs: []sql.Expression{ col(1, "t4", "foo"), col(2, "t4", "bar"), }, @@ -873,20 +873,20 @@ func TestGetMultiColumnIndexes(t *testing.T) { expected := map[string]*indexLookup{ "t1": &indexLookup{ - &testutil.MergeableIndexLookup{Id: "5, 6"}, + &memory.MergeableIndexLookup{Id: "5, 6"}, []sql.Index{indexes[0]}, }, "t2": &indexLookup{ - &testutil.MergeableIndexLookup{Id: "1, 2, 3"}, + &memory.MergeableIndexLookup{Id: "1, 2, 3"}, []sql.Index{indexes[1]}, }, "t4": &indexLookup{ - &testutil.MergedIndexLookup{[]sql.IndexLookup{ - &testutil.AscendIndexLookup{ + &memory.MergedIndexLookup{[]sql.IndexLookup{ + &memory.AscendIndexLookup{ Gte: []interface{}{int64(1), int64(2)}, Lt: []interface{}{int64(6), int64(5)}, }, - &testutil.DescendIndexLookup{ + &memory.DescendIndexLookup{ Gt: []interface{}{int64(1), int64(2)}, Lte: []interface{}{int64(6), int64(5)}, }, @@ -991,41 +991,41 @@ func (DummyIndexLookup) Values(sql.Partition) (sql.IndexValueIter, error) { return nil, nil } -var _ sql.Index = (*testutil.MergeableDummyIndex)(nil) -var _ sql.Index = (*testutil.UnmergeableDummyIndex)(nil) -var _ sql.AscendIndex = (*testutil.MergeableDummyIndex)(nil) -var _ sql.DescendIndex = (*testutil.MergeableDummyIndex)(nil) -var _ sql.NegateIndex = (*testutil.MergeableDummyIndex)(nil) +var _ sql.Index = (*memory.MergeableDummyIndex)(nil) +var _ sql.Index = (*memory.UnmergeableDummyIndex)(nil) +var _ sql.AscendIndex = (*memory.MergeableDummyIndex)(nil) +var _ sql.DescendIndex = (*memory.MergeableDummyIndex)(nil) +var _ sql.NegateIndex = (*memory.MergeableDummyIndex)(nil) func TestIndexesIntersection(t *testing.T) { require := require.New(t) - idx1, idx2 := &testutil.MergeableDummyIndex{TableName: "bar"}, &testutil.MergeableDummyIndex{TableName: "foo"} + idx1, idx2 := &memory.MergeableDummyIndex{TableName: "bar"}, &memory.MergeableDummyIndex{TableName: "foo"} left := map[string]*indexLookup{ - "a": &indexLookup{&testutil.MergeableIndexLookup{Id: "a"}, nil}, - "b": &indexLookup{&testutil.MergeableIndexLookup{Id: "b"}, []sql.Index{idx1}}, + "a": &indexLookup{&memory.MergeableIndexLookup{Id: "a"}, nil}, + "b": &indexLookup{&memory.MergeableIndexLookup{Id: "b"}, []sql.Index{idx1}}, "c": &indexLookup{new(DummyIndexLookup), nil}, } right := map[string]*indexLookup{ - "b": &indexLookup{&testutil.MergeableIndexLookup{Id: "b2"}, []sql.Index{idx2}}, - "c": &indexLookup{&testutil.MergeableIndexLookup{Id: "c"}, nil}, - "d": &indexLookup{&testutil.MergeableIndexLookup{Id: "d"}, nil}, + "b": &indexLookup{&memory.MergeableIndexLookup{Id: "b2"}, []sql.Index{idx2}}, + "c": &indexLookup{&memory.MergeableIndexLookup{Id: "c"}, nil}, + "d": &indexLookup{&memory.MergeableIndexLookup{Id: "d"}, nil}, } require.Equal( map[string]*indexLookup{ - "a": &indexLookup{&testutil.MergeableIndexLookup{Id: "a"}, nil}, + "a": &indexLookup{&memory.MergeableIndexLookup{Id: "a"}, nil}, "b": &indexLookup{ - &testutil.MergeableIndexLookup{ + &memory.MergeableIndexLookup{ Id: "b", Intersections: []string{"b2"}, }, []sql.Index{idx1, idx2}, }, "c": &indexLookup{new(DummyIndexLookup), nil}, - "d": &indexLookup{&testutil.MergeableIndexLookup{Id: "d"}, nil}, + "d": &indexLookup{&memory.MergeableIndexLookup{Id: "d"}, nil}, }, indexesIntersection(NewDefault(sql.NewCatalog()), left, right), ) @@ -1034,6 +1034,6 @@ func TestIndexesIntersection(t *testing.T) { func TestCanMergeIndexes(t *testing.T) { require := require.New(t) - require.False(canMergeIndexes(new(testutil.MergeableIndexLookup), new(DummyIndexLookup))) - require.True(canMergeIndexes(new(testutil.MergeableIndexLookup), new(testutil.MergeableIndexLookup))) + require.False(canMergeIndexes(new(memory.MergeableIndexLookup), new(DummyIndexLookup))) + require.True(canMergeIndexes(new(memory.MergeableIndexLookup), new(memory.MergeableIndexLookup))) } \ No newline at end of file diff --git a/sql/analyzer/pushdown_test.go b/sql/analyzer/pushdown_test.go index ef3979c3f..9f8758a7b 100644 --- a/sql/analyzer/pushdown_test.go +++ b/sql/analyzer/pushdown_test.go @@ -1,7 +1,6 @@ package analyzer import ( - "github.com/src-d/go-mysql-server/sql/test_util" "testing" "github.com/src-d/go-mysql-server/memory" @@ -106,9 +105,9 @@ func TestPushdownIndexable(t *testing.T) { catalog := sql.NewCatalog() catalog.AddDatabase(db) - idx1 := &testutil.MergeableDummyIndex{ - "mytable", - []sql.Expression{ + idx1 := &memory.MergeableDummyIndex{ + TableName: "mytable", + Exprs: []sql.Expression{ expression.NewGetFieldWithTable(0, sql.Int32, "mytable", "i", false), }, } @@ -117,9 +116,9 @@ func TestPushdownIndexable(t *testing.T) { close(done) <-ready - idx2 := &testutil.MergeableDummyIndex{ - "mytable", - []sql.Expression{ + idx2 := &memory.MergeableDummyIndex{ + TableName: "mytable", + Exprs: []sql.Expression{ expression.NewGetFieldWithTable(1, sql.Float64, "mytable", "f", false), }, } @@ -128,9 +127,9 @@ func TestPushdownIndexable(t *testing.T) { close(done) <-ready - idx3 := &testutil.MergeableDummyIndex{ - "mytable2", - []sql.Expression{ + idx3 := &memory.MergeableDummyIndex{ + TableName: "mytable2", + Exprs: []sql.Expression{ expression.NewGetFieldWithTable(0, sql.Int32, "mytable2", "i2", false), }, } @@ -190,7 +189,7 @@ func TestPushdownIndexable(t *testing.T) { ), }).(*memory.Table). WithProjection([]string{"i", "f"}).(*memory.Table). - WithIndexLookup(&testutil.MergeableIndexLookup{Id: "3.14"}), + WithIndexLookup(&memory.MergeableIndexLookup{Id: "3.14"}), ), plan.NewResolvedTable( table2.WithFilters([]sql.Expression{ @@ -202,7 +201,7 @@ func TestPushdownIndexable(t *testing.T) { ), }).(*memory.Table). WithProjection([]string{"i2"}).(*memory.Table). - WithIndexLookup(&testutil.NegateIndexLookup{Value: "2"}), + WithIndexLookup(&memory.NegateIndexLookup{Value: "2"}), ), ), ), From 79f34fb2379498d19d10f3891cfb160a477f9ea5 Mon Sep 17 00:00:00 2001 From: Zach Musgrave Date: Wed, 30 Oct 2019 14:18:28 -0700 Subject: [PATCH 09/42] First pass at complete index.Values implementation. This just iterates over each in the parent table looking for matches -- not here to make queries faster, just to test if they give correct results when indexes are involved. Signed-off-by: Zach Musgrave --- memory/unmergeable_index.go | 97 ++++++++++++++++++++++++++++++++----- 1 file changed, 85 insertions(+), 12 deletions(-) diff --git a/memory/unmergeable_index.go b/memory/unmergeable_index.go index 3da5cc1b1..58273c71e 100755 --- a/memory/unmergeable_index.go +++ b/memory/unmergeable_index.go @@ -3,12 +3,17 @@ package memory import ( "fmt" "github.com/src-d/go-mysql-server/sql" + "io" "strings" ) +// A very dumb index that iterates over the rows of a table, evaluates its matching expressions against each row, and +// stores those values to be later retrieved. Only here to test the functionality of indexed queries. This kind of index +// cannot be merged with any other index. type UnmergeableDummyIndex struct { DB string // required for engine tests with driver DriverName string // required for engine tests with driver + Tbl *Table // required for engine tests with driver TableName string Exprs []sql.Expression } @@ -25,28 +30,96 @@ func (u *UnmergeableDummyIndex) Expressions() []string { } func (u *UnmergeableDummyIndex) Get(key ...interface{}) (sql.IndexLookup, error) { - if len(key) != 1 { - var parts = make([]string, len(key)) - for i, p := range key { - parts[i] = fmt.Sprint(p) + return &UnmergeableIndexLookup{key: key}, nil +} + +type UnmergeableIndexLookup struct { + key []interface{} + idx UnmergeableDummyIndex +} + +type unmergeableIndexValueIter struct { + tbl *Table + partition sql.Partition + lookup *UnmergeableIndexLookup + values [][]byte + i int +} + +func (u *unmergeableIndexValueIter) Next() ([]byte, error) { + err := u.initValues() + if err != nil { + return nil, err + } + + if u.i < len(u.values) { + valBytes := u.values[u.i] + u.i++ + return valBytes, nil + } + + return nil, io.EOF +} + +func (u *unmergeableIndexValueIter) initValues() error { + if u.values == nil { + rows, ok := u.tbl.partitions[string(u.partition.Key())] + if !ok { + return fmt.Errorf( + "partition not found: %q", u.partition.Key(), + ) } - return &UnmergeableIndexLookup{id: strings.Join(parts, ", ")}, nil + for i, row := range rows { + match := true + for exprI, expr := range u.lookup.idx.Exprs { + colVal, err := expr.Eval(sql.NewEmptyContext(), row) + if colVal != u.lookup.key[exprI] { + match = false + break + } + + if err != nil { + return err + } + } + + if match { + idxVal := &indexValue{ + Key: "", + Pos: i, + } + encoded, err := encodeIndexValue(idxVal) + if err != nil { + return err + } + + u.values = append(u.values, encoded) + } + } } - return &UnmergeableIndexLookup{id: fmt.Sprint(key[0])}, nil + return nil } -type UnmergeableIndexLookup struct { - id string +func (u *unmergeableIndexValueIter) Close() error { + return nil } -func (u UnmergeableIndexLookup) Values(sql.Partition) (sql.IndexValueIter, error) { - return nil, nil +func (u *UnmergeableIndexLookup) Values(p sql.Partition) (sql.IndexValueIter, error) { + return &unmergeableIndexValueIter{ + tbl: u.idx.Tbl, + partition: p, + lookup: u, + }, nil } -func (u UnmergeableIndexLookup) Indexes() []string { - return []string{u.id} +func (u *UnmergeableIndexLookup) Indexes() []string { + var idxes = make([]string, len(u.key)) + for i, e := range u.idx.Exprs { + idxes[i] = fmt.Sprint(e) + } + return idxes } func (u *UnmergeableDummyIndex) Has(partition sql.Partition, key ...interface{}) (bool, error) { From cc69039e287475a5e6c5c7af9fd2f0d10c3a2f6a Mon Sep 17 00:00:00 2001 From: Zach Musgrave Date: Wed, 30 Oct 2019 15:51:17 -0700 Subject: [PATCH 10/42] Finally have a failing test case with new index driver. Signed-off-by: Zach Musgrave --- engine_test.go | 84 ++++++++++++++++++++----------------- memory/unmergeable_index.go | 55 ++++++++++++++++++++---- 2 files changed, 92 insertions(+), 47 deletions(-) diff --git a/engine_test.go b/engine_test.go index ece771db8..11884a4cd 100644 --- a/engine_test.go +++ b/engine_test.go @@ -1597,51 +1597,59 @@ var queries = []struct { } func TestQueries(t *testing.T) { - testIndexDriver := memory.NewIndexDriver("mydb", map[string][]sql.Index{ - "mytable": { - &memory.UnmergeableDummyIndex{ - DB: "mydb", - DriverName: memory.IndexDriverId, - TableName: "mytable", - Exprs: []sql.Expression{ - expression.NewGetFieldWithTable(0, sql.Int64, "mytable", "i", false), - }, - }, - &memory.UnmergeableDummyIndex{ - DB: "mydb", - DriverName: memory.IndexDriverId, - TableName: "mytable", - Exprs: []sql.Expression{ - expression.NewGetFieldWithTable(0, sql.Text, "mytable", "s", false), - }, - }, - &memory.UnmergeableDummyIndex{ - DB: "mydb", - DriverName: memory.IndexDriverId, - TableName: "mytable", - Exprs: []sql.Expression{ - expression.NewGetFieldWithTable(0, sql.Int64, "mytable", "i", false), - expression.NewGetFieldWithTable(0, sql.Text, "mytable", "s", false), - }, - }, - }, - }) // Test all queries with these combinations, for a total of 8 runs: // 1) Partitioned tables / non partitioned tables // 2) Indexes enabled / disabled // 3) Parallelism on / off - numPartitionsVals := []int{1, testNumPartitions} - useIndexesVals := []bool{false, true} - parallelVals := []int{1, 2} + numPartitionsVals := []int{1} + useIndexesVals := []bool{true} + parallelVals := []int{1} + // numPartitionsVals := []int{1, testNumPartitions} + // useIndexesVals := []bool{false, true} + // parallelVals := []int{1, 2} for _, numPartitions := range numPartitionsVals { for _, useIndexes := range useIndexesVals { for _, parallelism := range parallelVals { + tables := allTestTables(t) + testIndexDriver := memory.NewIndexDriver("mydb", map[string][]sql.Index{ + "mytable": { + &memory.UnmergeableDummyIndex{ + DB: "mydb", + DriverName: memory.IndexDriverId, + TableName: "mytable", + Tbl: tables["mytable"], + Exprs: []sql.Expression{ + expression.NewGetFieldWithTable(0, sql.Int64, "mytable", "i", false), + }, + }, + &memory.UnmergeableDummyIndex{ + DB: "mydb", + DriverName: memory.IndexDriverId, + TableName: "mytable", + Tbl: tables["mytable"], + Exprs: []sql.Expression{ + expression.NewGetFieldWithTable(1, sql.Text, "mytable", "s", false), + }, + }, + &memory.UnmergeableDummyIndex{ + DB: "mydb", + DriverName: memory.IndexDriverId, + TableName: "mytable", + Tbl: tables["mytable"], + Exprs: []sql.Expression{ + expression.NewGetFieldWithTable(0, sql.Int64, "mytable", "i", false), + expression.NewGetFieldWithTable(1, sql.Text, "mytable", "s", false), + }, + }, + }, + }) + var indexDriver sql.IndexDriver if useIndexes { indexDriver = testIndexDriver } - engine := newEngineWithParallelism(t, parallelism, indexDriver) + engine := newEngineWithParallelism(t, parallelism, tables, indexDriver) testName := fmt.Sprintf("partitions=%d, indexes=%v, parallelism=%v", numPartitions, useIndexes, parallelism) t.Run(testName, func(t *testing.T) { @@ -1802,7 +1810,7 @@ func TestWarnings(t *testing.T) { } e := newEngine(t) - ep := newEngineWithParallelism(t, 2, nil) + ep := newEngineWithParallelism(t, 2, allTestTables(t), nil) t.Run("sequential", func(t *testing.T) { for _, tt := range queries { @@ -1866,7 +1874,7 @@ func TestClearWarnings(t *testing.T) { func TestDescribe(t *testing.T) { e := newEngine(t) - ep := newEngineWithParallelism(t, 2, nil) + ep := newEngineWithParallelism(t, 2, allTestTables(t), nil) query := `DESCRIBE FORMAT=TREE SELECT * FROM mytable` expectedSeq := []sql.Row{ @@ -2994,12 +3002,10 @@ func allTestTables(t *testing.T) map[string]*memory.Table { } func newEngine(t *testing.T) *sqle.Engine { - return newEngineWithParallelism(t, 1, nil) + return newEngineWithParallelism(t, 1, allTestTables(t), nil) } -func newEngineWithParallelism(t *testing.T, parallelism int, driver sql.IndexDriver) *sqle.Engine { - tables := allTestTables(t) - +func newEngineWithParallelism(t *testing.T, parallelism int, tables map[string]*memory.Table, driver sql.IndexDriver) *sqle.Engine { db := memory.NewDatabase("mydb") for name, table := range tables { if name != "other_table" { diff --git a/memory/unmergeable_index.go b/memory/unmergeable_index.go index 58273c71e..74f38c8b7 100755 --- a/memory/unmergeable_index.go +++ b/memory/unmergeable_index.go @@ -3,6 +3,7 @@ package memory import ( "fmt" "github.com/src-d/go-mysql-server/sql" + "github.com/src-d/go-mysql-server/sql/expression" "io" "strings" ) @@ -30,12 +31,15 @@ func (u *UnmergeableDummyIndex) Expressions() []string { } func (u *UnmergeableDummyIndex) Get(key ...interface{}) (sql.IndexLookup, error) { - return &UnmergeableIndexLookup{key: key}, nil + return &UnmergeableIndexLookup{ + key: key, + idx: u, + }, nil } type UnmergeableIndexLookup struct { key []interface{} - idx UnmergeableDummyIndex + idx *UnmergeableDummyIndex } type unmergeableIndexValueIter struct { @@ -73,20 +77,23 @@ func (u *unmergeableIndexValueIter) initValues() error { for i, row := range rows { match := true for exprI, expr := range u.lookup.idx.Exprs { - colVal, err := expr.Eval(sql.NewEmptyContext(), row) - if colVal != u.lookup.key[exprI] { - match = false - break - } + // colVal, err := expr.Eval(sql.NewEmptyContext(), row) + lit, typ := getType(u.lookup.key[exprI]) + eq := expression.NewEquals(expr, expression.NewLiteral(lit, typ)) + ok, err := sql.EvaluateCondition(sql.NewEmptyContext(), eq, row) if err != nil { return err } + + if !ok { + match = false + break + } } if match { idxVal := &indexValue{ - Key: "", Pos: i, } encoded, err := encodeIndexValue(idxVal) @@ -102,6 +109,38 @@ func (u *unmergeableIndexValueIter) initValues() error { return nil } +func getType(val interface{}) (interface{}, sql.Type) { + switch val := val.(type) { + case int8: + return int64(val), sql.Int64 + case uint8: + return int64(val), sql.Int64 + case int16: + return int64(val), sql.Int64 + case uint16: + return int64(val), sql.Int64 + case int32: + return int64(val), sql.Int64 + case uint32: + return int64(val), sql.Int64 + case int64: + return int64(val), sql.Int64 + case uint64: + return int64(val), sql.Int64 + case float32: + return float64(val), sql.Float64 + case float64: + return float64(val), sql.Float64 + case string: + return val, sql.Text + default:panic(fmt.Sprintf("Unsupported type for %v of type %T", val, val)) + } +} + +func valuesMatch(val1 interface{}, val2 interface{}) bool { + return false +} + func (u *unmergeableIndexValueIter) Close() error { return nil } From 5545932a5495358038b8a5dc8570a3ce1251d906 Mon Sep 17 00:00:00 2001 From: Zach Musgrave Date: Thu, 31 Oct 2019 14:01:23 -0700 Subject: [PATCH 11/42] Reverted failing buggy code in assign_indexes.go after verifying new test catches the bug Signed-off-by: Zach Musgrave --- engine_test.go | 14 +++++++------- sql/analyzer/assign_indexes.go | 26 +++++++++++--------------- 2 files changed, 18 insertions(+), 22 deletions(-) diff --git a/engine_test.go b/engine_test.go index 11884a4cd..bc8b92cbc 100644 --- a/engine_test.go +++ b/engine_test.go @@ -1602,12 +1602,12 @@ func TestQueries(t *testing.T) { // 1) Partitioned tables / non partitioned tables // 2) Indexes enabled / disabled // 3) Parallelism on / off - numPartitionsVals := []int{1} - useIndexesVals := []bool{true} - parallelVals := []int{1} - // numPartitionsVals := []int{1, testNumPartitions} - // useIndexesVals := []bool{false, true} - // parallelVals := []int{1, 2} + // numPartitionsVals := []int{1} + // useIndexesVals := []bool{true} + // parallelVals := []int{1} + numPartitionsVals := []int{1, testNumPartitions} + useIndexesVals := []bool{false, true} + parallelVals := []int{1, 2} for _, numPartitions := range numPartitionsVals { for _, useIndexes := range useIndexesVals { for _, parallelism := range parallelVals { @@ -1651,7 +1651,7 @@ func TestQueries(t *testing.T) { } engine := newEngineWithParallelism(t, parallelism, tables, indexDriver) - testName := fmt.Sprintf("partitions=%d, indexes=%v, parallelism=%v", numPartitions, useIndexes, parallelism) + testName := fmt.Sprintf("partitions=%d,indexes=%v,parallelism=%v", numPartitions, useIndexes, parallelism) t.Run(testName, func(t *testing.T) { for _, tt := range queries { testQuery(t, engine, tt.query, tt.expected) diff --git a/sql/analyzer/assign_indexes.go b/sql/analyzer/assign_indexes.go index bc70d0130..ee135edb7 100644 --- a/sql/analyzer/assign_indexes.go +++ b/sql/analyzer/assign_indexes.go @@ -102,22 +102,18 @@ func getIndexes(e sql.Expression, aliases map[string]sql.Expression, a *Analyzer } for table, leftIdx := range leftIndexes { - if rightIdx, ok := rightIndexes[table]; ok && canMergeIndexes(leftIdx.lookup, rightIdx.lookup) { - leftIdx.lookup = leftIdx.lookup.(sql.SetOperations).Union(rightIdx.lookup) - leftIdx.indexes = append(leftIdx.indexes, rightIdx.indexes...) + if rightIdx, ok := rightIndexes[table]; ok { + if canMergeIndexes(leftIdx.lookup, rightIdx.lookup) { + leftIdx.lookup = leftIdx.lookup.(sql.SetOperations).Union(rightIdx.lookup) + leftIdx.indexes = append(leftIdx.indexes, rightIdx.indexes...) + result[table] = leftIdx + } else { + // Since we can return one index per table, if we can't merge the second index from this table, return no + // indexes. Returning a single one will lead to incorrect results from e.g. pushdown operations when only one + // side of the OR expression is used to index the table. + return nil, nil + } } - // if rightIdx, ok := rightIndexes[table]; ok { - // if canMergeIndexes(leftIdx.lookup, rightIdx.lookup) { - // leftIdx.lookup = leftIdx.lookup.(sql.SetOperations).Union(rightIdx.lookup) - // leftIdx.indexes = append(leftIdx.indexes, rightIdx.indexes...) - // } else { - // // Since we can return one index per table, if we can't merge the second index from this table, return no - // // indexes. Returning a single one will lead to incorrect results from e.g. pushdown operations when only one - // // side of the OR expression is used to index the table. - // return nil, nil - // } - // } - result[table] = leftIdx } // Put in the result map the indexes for tables we don't have indexes yet. From f3cb328375debd00c9315e850dac67278bbee487 Mon Sep 17 00:00:00 2001 From: Zach Musgrave Date: Thu, 31 Oct 2019 14:07:31 -0700 Subject: [PATCH 12/42] Fixed bug in assign_indexes for or statements Signed-off-by: Zach Musgrave --- sql/analyzer/assign_indexes.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/sql/analyzer/assign_indexes.go b/sql/analyzer/assign_indexes.go index ee135edb7..6eb5f686f 100644 --- a/sql/analyzer/assign_indexes.go +++ b/sql/analyzer/assign_indexes.go @@ -101,6 +101,11 @@ func getIndexes(e sql.Expression, aliases map[string]sql.Expression, a *Analyzer return nil, err } + for table, leftIdx := range leftIndexes { + result[table] = leftIdx + } + + // Merge any indexes for the same table on the left and right sides. for table, leftIdx := range leftIndexes { if rightIdx, ok := rightIndexes[table]; ok { if canMergeIndexes(leftIdx.lookup, rightIdx.lookup) { From 39bfdb83a49278d2a91e066784c010e64fa87daa Mon Sep 17 00:00:00 2001 From: Zach Musgrave Date: Thu, 31 Oct 2019 16:15:33 -0700 Subject: [PATCH 13/42] Added some more test indexes, fixed a bug where I wasn't correctly altering the number of partitions for the test setup Signed-off-by: Zach Musgrave --- engine_test.go | 95 +++++++++++++++++++++++++++----------------------- 1 file changed, 52 insertions(+), 43 deletions(-) diff --git a/engine_test.go b/engine_test.go index bc8b92cbc..94d4ba5f1 100644 --- a/engine_test.go +++ b/engine_test.go @@ -1597,7 +1597,6 @@ var queries = []struct { } func TestQueries(t *testing.T) { - // Test all queries with these combinations, for a total of 8 runs: // 1) Partitioned tables / non partitioned tables // 2) Indexes enabled / disabled @@ -1611,37 +1610,37 @@ func TestQueries(t *testing.T) { for _, numPartitions := range numPartitionsVals { for _, useIndexes := range useIndexesVals { for _, parallelism := range parallelVals { - tables := allTestTables(t) + tables := allTestTables(t, numPartitions) testIndexDriver := memory.NewIndexDriver("mydb", map[string][]sql.Index{ "mytable": { - &memory.UnmergeableDummyIndex{ - DB: "mydb", - DriverName: memory.IndexDriverId, - TableName: "mytable", - Tbl: tables["mytable"], - Exprs: []sql.Expression{ - expression.NewGetFieldWithTable(0, sql.Int64, "mytable", "i", false), - }, - }, - &memory.UnmergeableDummyIndex{ - DB: "mydb", - DriverName: memory.IndexDriverId, - TableName: "mytable", - Tbl: tables["mytable"], - Exprs: []sql.Expression{ - expression.NewGetFieldWithTable(1, sql.Text, "mytable", "s", false), - }, - }, - &memory.UnmergeableDummyIndex{ - DB: "mydb", - DriverName: memory.IndexDriverId, - TableName: "mytable", - Tbl: tables["mytable"], - Exprs: []sql.Expression{ - expression.NewGetFieldWithTable(0, sql.Int64, "mytable", "i", false), - expression.NewGetFieldWithTable(1, sql.Text, "mytable", "s", false), - }, - }, + newUnmergableIndex(tables, "mytable", + expression.NewGetFieldWithTable(0, sql.Int64, "mytable", "i", false)), + newUnmergableIndex(tables, "mytable", + expression.NewGetFieldWithTable(1, sql.Text, "mytable", "s", false)), + newUnmergableIndex(tables, "mytable", + expression.NewGetFieldWithTable(0, sql.Int64, "mytable", "i", false), + expression.NewGetFieldWithTable(1, sql.Text, "mytable", "s", false)), + }, + "othertable": { + newUnmergableIndex(tables, "othertable", + expression.NewGetFieldWithTable(0, sql.Text, "othertable", "s2", false)), + newUnmergableIndex(tables, "othertable", + expression.NewGetFieldWithTable(1, sql.Text, "othertable", "i2", false)), + newUnmergableIndex(tables, "othertable", + expression.NewGetFieldWithTable(0, sql.Text, "othertable", "s2", false), + expression.NewGetFieldWithTable(1, sql.Text, "othertable", "i2", false)), + }, + "bigtable": { + newUnmergableIndex(tables, "bigtable", + expression.NewGetFieldWithTable(0, sql.Text, "bigtable", "t", false)), + }, + "floattable": { + newUnmergableIndex(tables, "floattable", + expression.NewGetFieldWithTable(2, sql.Text, "floattable", "f64", false)), + }, + "niltable": { + newUnmergableIndex(tables, "niltable", + expression.NewGetFieldWithTable(0, sql.Int64, "niltable", "i", false)), }, }) @@ -1662,6 +1661,16 @@ func TestQueries(t *testing.T) { } } +func newUnmergableIndex(tables map[string]*memory.Table, tableName string, exprs ...sql.Expression) *memory.UnmergeableDummyIndex { + return &memory.UnmergeableDummyIndex{ + DB: "mydb", + DriverName: memory.IndexDriverId, + TableName: tableName, + Tbl: tables[tableName], + Exprs: exprs, + } +} + func TestSessionSelectLimit(t *testing.T) { ctx := newCtx() ctx.Session.Set("sql_select_limit", sql.Int64, int64(1)) @@ -1810,7 +1819,7 @@ func TestWarnings(t *testing.T) { } e := newEngine(t) - ep := newEngineWithParallelism(t, 2, allTestTables(t), nil) + ep := newEngineWithParallelism(t, 2, allTestTables(t, testNumPartitions), nil) t.Run("sequential", func(t *testing.T) { for _, tt := range queries { @@ -1874,7 +1883,7 @@ func TestClearWarnings(t *testing.T) { func TestDescribe(t *testing.T) { e := newEngine(t) - ep := newEngineWithParallelism(t, 2, allTestTables(t), nil) + ep := newEngineWithParallelism(t, 2, allTestTables(t, testNumPartitions), nil) query := `DESCRIBE FORMAT=TREE SELECT * FROM mytable` expectedSeq := []sql.Row{ @@ -2859,13 +2868,13 @@ func testQueryWithContext(ctx *sql.Context, t *testing.T, e *sqle.Engine, q stri }) } -func allTestTables(t *testing.T) map[string]*memory.Table { +func allTestTables(t *testing.T, numPartitions int) map[string]*memory.Table { tables := make(map[string]*memory.Table) tables["mytable"] = memory.NewPartitionedTable("mytable", sql.Schema{ {Name: "i", Type: sql.Int64, Source: "mytable"}, {Name: "s", Type: sql.Text, Source: "mytable"}, - }, testNumPartitions) + }, numPartitions) insertRows( t, tables["mytable"], @@ -2877,7 +2886,7 @@ func allTestTables(t *testing.T) map[string]*memory.Table { tables["othertable"] = memory.NewPartitionedTable("othertable", sql.Schema{ {Name: "s2", Type: sql.Text, Source: "othertable"}, {Name: "i2", Type: sql.Int64, Source: "othertable"}, - }, testNumPartitions) + }, numPartitions) insertRows( t, tables["othertable"], @@ -2889,7 +2898,7 @@ func allTestTables(t *testing.T) map[string]*memory.Table { tables["tabletest"] = memory.NewPartitionedTable("tabletest", sql.Schema{ {Name: "i", Type: sql.Int32, Source: "tabletest"}, {Name: "s", Type: sql.Text, Source: "tabletest"}, - }, testNumPartitions) + }, numPartitions) insertRows( t, tables["tabletest"], @@ -2901,7 +2910,7 @@ func allTestTables(t *testing.T) map[string]*memory.Table { tables["other_table"] = memory.NewPartitionedTable("other_table", sql.Schema{ {Name: "text", Type: sql.Text, Source: "tabletest"}, {Name: "number", Type: sql.Int32, Source: "tabletest"}, - }, testNumPartitions) + }, numPartitions) insertRows( t, tables["other_table"], @@ -2913,7 +2922,7 @@ func allTestTables(t *testing.T) map[string]*memory.Table { tables["bigtable"] = memory.NewPartitionedTable("bigtable", sql.Schema{ {Name: "t", Type: sql.Text, Source: "bigtable"}, {Name: "n", Type: sql.Int64, Source: "bigtable"}, - }, testNumPartitions) + }, numPartitions) insertRows( t, tables["bigtable"], @@ -2937,7 +2946,7 @@ func allTestTables(t *testing.T) map[string]*memory.Table { {Name: "i", Type: sql.Int64, Source: "floattable"}, {Name: "f32", Type: sql.Float32, Source: "floattable"}, {Name: "f64", Type: sql.Float64, Source: "floattable"}, - }, testNumPartitions) + }, numPartitions) insertRows( t, tables["floattable"], @@ -2953,7 +2962,7 @@ func allTestTables(t *testing.T) map[string]*memory.Table { {Name: "i", Type: sql.Int64, Source: "niltable", Nullable: true}, {Name: "b", Type: sql.Boolean, Source: "niltable", Nullable: true}, {Name: "f", Type: sql.Float64, Source: "niltable", Nullable: true}, - }, testNumPartitions) + }, numPartitions) insertRows( t, tables["niltable"], @@ -2967,7 +2976,7 @@ func allTestTables(t *testing.T) map[string]*memory.Table { tables["newlinetable"] = memory.NewPartitionedTable("newlinetable", sql.Schema{ {Name: "i", Type: sql.Int64, Source: "newlinetable"}, {Name: "s", Type: sql.Text, Source: "newlinetable"}, - }, testNumPartitions) + }, numPartitions) insertRows( t, tables["newlinetable"], @@ -2996,13 +3005,13 @@ func allTestTables(t *testing.T) map[string]*memory.Table { {Name: "bo", Type: sql.Boolean, Source: "typestable", Nullable: true}, {Name: "js", Type: sql.JSON, Source: "typestable", Nullable: true}, {Name: "bl", Type: sql.Blob, Source: "typestable", Nullable: true}, - }, testNumPartitions) + }, numPartitions) return tables } func newEngine(t *testing.T) *sqle.Engine { - return newEngineWithParallelism(t, 1, allTestTables(t), nil) + return newEngineWithParallelism(t, 1, allTestTables(t, testNumPartitions), nil) } func newEngineWithParallelism(t *testing.T, parallelism int, tables map[string]*memory.Table, driver sql.IndexDriver) *sqle.Engine { From dbc915d18c43623f076b8d8210f354849f123637 Mon Sep 17 00:00:00 2001 From: Zach Musgrave Date: Fri, 1 Nov 2019 10:06:16 -0700 Subject: [PATCH 14/42] Better mechanism for sharing dummy index value iter code Signed-off-by: Zach Musgrave --- memory/mergeable_index.go | 8 +++++--- memory/unmergeable_index.go | 25 ++++++++++++------------- 2 files changed, 17 insertions(+), 16 deletions(-) diff --git a/memory/mergeable_index.go b/memory/mergeable_index.go index 4d78ae9b0..086a6fb23 100755 --- a/memory/mergeable_index.go +++ b/memory/mergeable_index.go @@ -74,8 +74,9 @@ type MergedIndexLookup struct { Children []sql.IndexLookup } -func (MergedIndexLookup) Values(sql.Partition) (sql.IndexValueIter, error) { +func (i *MergedIndexLookup) Values(sql.Partition) (sql.IndexValueIter, error) { return nil, nil +// return &dummyIndexValueIter{}, nil } func (i *MergedIndexLookup) Indexes() []string { @@ -103,8 +104,9 @@ func (MergedIndexLookup) Intersection(...sql.IndexLookup) sql.IndexLookup { } type MergeableDummyIndex struct { - DB string // required for engine tests - DriverName string // required for engine tests + DB string // required for engine tests with driver + DriverName string // required for engine tests with driver + Tbl *Table // required for engine tests with driver TableName string Exprs []sql.Expression } diff --git a/memory/unmergeable_index.go b/memory/unmergeable_index.go index 74f38c8b7..1ed2a82a5 100755 --- a/memory/unmergeable_index.go +++ b/memory/unmergeable_index.go @@ -45,7 +45,8 @@ type UnmergeableIndexLookup struct { type unmergeableIndexValueIter struct { tbl *Table partition sql.Partition - lookup *UnmergeableIndexLookup + // Returns a set of expresssions that must match a given row to be included in the value iterator for a lookup + matchExpressions func() []sql.Expression values [][]byte i int } @@ -76,12 +77,8 @@ func (u *unmergeableIndexValueIter) initValues() error { for i, row := range rows { match := true - for exprI, expr := range u.lookup.idx.Exprs { - // colVal, err := expr.Eval(sql.NewEmptyContext(), row) - lit, typ := getType(u.lookup.key[exprI]) - eq := expression.NewEquals(expr, expression.NewLiteral(lit, typ)) - - ok, err := sql.EvaluateCondition(sql.NewEmptyContext(), eq, row) + for _, matchExpr := range u.matchExpressions() { + ok, err := sql.EvaluateCondition(sql.NewEmptyContext(), matchExpr, row) if err != nil { return err } @@ -137,10 +134,6 @@ func getType(val interface{}) (interface{}, sql.Type) { } } -func valuesMatch(val1 interface{}, val2 interface{}) bool { - return false -} - func (u *unmergeableIndexValueIter) Close() error { return nil } @@ -149,8 +142,14 @@ func (u *UnmergeableIndexLookup) Values(p sql.Partition) (sql.IndexValueIter, er return &unmergeableIndexValueIter{ tbl: u.idx.Tbl, partition: p, - lookup: u, - }, nil + matchExpressions: func() []sql.Expression { + var exprs []sql.Expression + for exprI, expr := range u.idx.Exprs { + lit, typ := getType(u.key[exprI]) + exprs = append(exprs, expression.NewEquals(expr, expression.NewLiteral(lit, typ))) + } + return exprs + }}, nil } func (u *UnmergeableIndexLookup) Indexes() []string { From 63090f18e767e6aced0a1c83e4d3e5d578e7af85 Mon Sep 17 00:00:00 2001 From: Zach Musgrave Date: Fri, 1 Nov 2019 10:28:42 -0700 Subject: [PATCH 15/42] Dummy indexValueIter Signed-off-by: Zach Musgrave --- memory/unmergeable_index.go | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/memory/unmergeable_index.go b/memory/unmergeable_index.go index 1ed2a82a5..c405d0afb 100755 --- a/memory/unmergeable_index.go +++ b/memory/unmergeable_index.go @@ -42,7 +42,10 @@ type UnmergeableIndexLookup struct { idx *UnmergeableDummyIndex } -type unmergeableIndexValueIter struct { +// dummyIndexValueIter does a very simple and verifiable iteration over the table values for a given index. It does this +// by iterating over all the table rows for a partition and evaluating each of them for inclusion in the index. This is +// not an efficient way to store an index, and is only suitable for testing the correctness of index code in the engine. +type dummyIndexValueIter struct { tbl *Table partition sql.Partition // Returns a set of expresssions that must match a given row to be included in the value iterator for a lookup @@ -51,7 +54,7 @@ type unmergeableIndexValueIter struct { i int } -func (u *unmergeableIndexValueIter) Next() ([]byte, error) { +func (u *dummyIndexValueIter) Next() ([]byte, error) { err := u.initValues() if err != nil { return nil, err @@ -66,7 +69,7 @@ func (u *unmergeableIndexValueIter) Next() ([]byte, error) { return nil, io.EOF } -func (u *unmergeableIndexValueIter) initValues() error { +func (u *dummyIndexValueIter) initValues() error { if u.values == nil { rows, ok := u.tbl.partitions[string(u.partition.Key())] if !ok { @@ -134,12 +137,12 @@ func getType(val interface{}) (interface{}, sql.Type) { } } -func (u *unmergeableIndexValueIter) Close() error { +func (u *dummyIndexValueIter) Close() error { return nil } func (u *UnmergeableIndexLookup) Values(p sql.Partition) (sql.IndexValueIter, error) { - return &unmergeableIndexValueIter{ + return &dummyIndexValueIter{ tbl: u.idx.Tbl, partition: p, matchExpressions: func() []sql.Expression { From 198699954f5ab13cfed145c802bee1903fc30cfd Mon Sep 17 00:00:00 2001 From: Zach Musgrave Date: Fri, 1 Nov 2019 13:45:18 -0700 Subject: [PATCH 16/42] assign index tests now compiling, trying to get them passing Signed-off-by: Zach Musgrave --- memory/mergeable_index.go | 50 +++++++++++++--------- memory/negative_index.go | 10 ++--- sql/analyzer/assign_indexes_test.go | 66 ++++++++++++++++------------- sql/analyzer/pushdown_test.go | 5 ++- 4 files changed, 73 insertions(+), 58 deletions(-) diff --git a/memory/mergeable_index.go b/memory/mergeable_index.go index 086a6fb23..a749f00d6 100755 --- a/memory/mergeable_index.go +++ b/memory/mergeable_index.go @@ -12,8 +12,14 @@ type MergeableLookup interface { GetIntersections() []string } +// ExpressionsIndex is an index made out of one or more expressions (usually field expressions) +type ExpressionsIndex interface { + ColumnExpressions() []sql.Expression +} + type MergeableIndexLookup struct { - Id string + Key []interface{} + Index ExpressionsIndex Unions []string Intersections []string } @@ -21,7 +27,7 @@ type MergeableIndexLookup struct { var _ sql.Mergeable = (*MergeableIndexLookup)(nil) var _ sql.SetOperations = (*MergeableIndexLookup)(nil) -func (i *MergeableIndexLookup) ID() string { return i.Id } +func (i *MergeableIndexLookup) ID() string { return strings.Join(i.Indexes(), ",") } func (i *MergeableIndexLookup) GetUnions() []string { return i.Unions } func (i *MergeableIndexLookup) GetIntersections() []string { return i.Intersections } @@ -35,7 +41,11 @@ func (i *MergeableIndexLookup) Values(sql.Partition) (sql.IndexValueIter, error) } func (i *MergeableIndexLookup) Indexes() []string { - return []string{i.ID()} + var idxes = make([]string, len(i.Key)) + for i, e := range i.Key { + idxes[i] = fmt.Sprint(e) + } + return idxes } func (i *MergeableIndexLookup) Difference(indexes ...sql.IndexLookup) sql.IndexLookup { @@ -49,24 +59,30 @@ func (i *MergeableIndexLookup) Intersection(indexes ...sql.IndexLookup) sql.Inde intersections = append(intersections, idx.(MergeableLookup).GetIntersections()...) unions = append(unions, idx.(MergeableLookup).GetUnions()...) } + + // TODO: fix logic return &MergeableIndexLookup{ - i.Id, - append(i.Unions, unions...), - append(i.Intersections, intersections...), + Key: i.Key, + Index: i.Index, + Unions: append(i.Unions, unions...), + Intersections: append(i.Intersections, intersections...), } } func (i *MergeableIndexLookup) Union(indexes ...sql.IndexLookup) sql.IndexLookup { var intersections, unions []string for _, idx := range indexes { - unions = append(unions, idx.(*MergeableIndexLookup).Id) + unions = append(unions, idx.(*MergeableIndexLookup).ID()) unions = append(unions, idx.(*MergeableIndexLookup).Unions...) intersections = append(intersections, idx.(*MergeableIndexLookup).Intersections...) } + + // TODO: fix logic return &MergeableIndexLookup{ - i.Id, - append(i.Unions, unions...), - append(i.Intersections, intersections...), + Key: i.Key, + Index: i.Index, + Unions: append(i.Unions, unions...), + Intersections: append(i.Intersections, intersections...), } } @@ -113,6 +129,7 @@ type MergeableDummyIndex struct { func (i MergeableDummyIndex) Database() string { return i.DB } func (i MergeableDummyIndex) Driver() string { return i.DriverName } +func (i MergeableDummyIndex) ColumnExpressions() []sql.Expression { return i.Exprs } func (i MergeableDummyIndex) Expressions() []string { var exprs []string @@ -153,20 +170,11 @@ func (i MergeableDummyIndex) Not(keys ...interface{}) (sql.IndexLookup, error) { } mergeable, _ := lookup.(*MergeableIndexLookup) - return &NegateIndexLookup{Value: mergeable.Id}, nil + return &NegateIndexLookup{Lookup: mergeable}, nil } func (i MergeableDummyIndex) Get(key ...interface{}) (sql.IndexLookup, error) { - if len(key) != 1 { - var parts = make([]string, len(key)) - for i, p := range key { - parts[i] = fmt.Sprint(p) - } - - return &MergeableIndexLookup{Id: strings.Join(parts, ", ")}, nil - } - - return &MergeableIndexLookup{Id: fmt.Sprint(key[0])}, nil + return &MergeableIndexLookup{Key: key, Index: i}, nil } func (i MergeableDummyIndex) Has(sql.Partition, ...interface{}) (bool, error) { diff --git a/memory/negative_index.go b/memory/negative_index.go index c39292d99..bb492ae39 100755 --- a/memory/negative_index.go +++ b/memory/negative_index.go @@ -5,12 +5,12 @@ import ( ) type NegateIndexLookup struct { - Value string + Lookup *MergeableIndexLookup intersections []string unions []string } -func (l *NegateIndexLookup) ID() string { return "not " + l.Value } +func (l *NegateIndexLookup) ID() string { return "not " + l.Lookup.ID() } func (l *NegateIndexLookup) GetUnions() []string { return l.unions } func (l *NegateIndexLookup) GetIntersections() []string { return l.intersections } @@ -42,8 +42,8 @@ func (l *NegateIndexLookup) Intersection(indexes ...sql.IndexLookup) sql.IndexLo unions = append(unions, idx.(MergeableLookup).GetUnions()...) } return &MergeableIndexLookup{ - l.ID(), - append(l.unions, unions...), - append(l.intersections, intersections...), + Index: l.Lookup.Index, + Unions: append(l.unions, unions...), + Intersections: append(l.intersections, intersections...), } } \ No newline at end of file diff --git a/sql/analyzer/assign_indexes_test.go b/sql/analyzer/assign_indexes_test.go index 89fe71027..e43a20aef 100644 --- a/sql/analyzer/assign_indexes_test.go +++ b/sql/analyzer/assign_indexes_test.go @@ -52,7 +52,7 @@ func TestNegateIndex(t *testing.T) { negate, ok := lookupIdxs.lookup.(*memory.NegateIndexLookup) require.True(ok) - require.True(negate.Value == "1") + require.Equal("not 1", negate.ID()) } func TestAssignIndexes(t *testing.T) { @@ -137,14 +137,14 @@ func TestAssignIndexes(t *testing.T) { mergeable, ok := lookupIdxs.lookup.(*memory.MergeableIndexLookup) require.True(ok) - require.True(mergeable.Id == "2") + require.Equal("2", mergeable.ID()) lookupIdxs, ok = result["t2"] require.True(ok) mergeable, ok = lookupIdxs.lookup.(*memory.MergeableIndexLookup) require.True(ok) - require.True(mergeable.Id == "1") + require.Equal("1", mergeable.ID()) node = plan.NewProject( []sql.Expression{}, @@ -236,7 +236,13 @@ func TestGetIndexes(t *testing.T) { ), map[string]*indexLookup{ "t1": &indexLookup{ - &memory.MergeableIndexLookup{Id: "1"}, + &memory.MergeableIndexLookup{ + Key: []interface{}{int64(1)}, + Index: memory.MergeableDummyIndex{ + TableName: "t1", + Exprs: []sql.Expression{col(0, "t1", "bar")}, + }, + }, []sql.Index{indexes[0]}, }, }, @@ -255,7 +261,7 @@ func TestGetIndexes(t *testing.T) { ), map[string]*indexLookup{ "t1": &indexLookup{ - &memory.MergeableIndexLookup{Id: "1", Unions: []string{"2"}}, + &memory.MergeableIndexLookup{Key: []interface{}{1}, Unions: []string{"2"}}, []sql.Index{ indexes[0], indexes[0], @@ -293,7 +299,7 @@ func TestGetIndexes(t *testing.T) { ), map[string]*indexLookup{ "t1": &indexLookup{ - &memory.MergeableIndexLookup{Id: "1", Unions: []string{"2"}}, + &memory.MergeableIndexLookup{Key: []interface{}{1}, Unions: []string{"2"}}, []sql.Index{ indexes[0], }, @@ -314,7 +320,7 @@ func TestGetIndexes(t *testing.T) { ), map[string]*indexLookup{ "t1": &indexLookup{ - &memory.MergeableIndexLookup{Id: "1", Intersections: []string{"2"}}, + &memory.MergeableIndexLookup{Key: []interface{}{1}, Intersections: []string{"2"}}, []sql.Index{ indexes[0], indexes[0], @@ -348,7 +354,7 @@ func TestGetIndexes(t *testing.T) { ), map[string]*indexLookup{ "t1": &indexLookup{ - &memory.MergeableIndexLookup{Id: "1", Unions: []string{"2", "4"}, Intersections: []string{"3"}}, + &memory.MergeableIndexLookup{Key: []interface{}{1}, Unions: []string{"2", "4"}, Intersections: []string{"3"}}, []sql.Index{ indexes[0], indexes[0], @@ -384,7 +390,7 @@ func TestGetIndexes(t *testing.T) { ), map[string]*indexLookup{ "t1": &indexLookup{ - &memory.MergeableIndexLookup{Id: "1", Unions: []string{"2", "3", "4"}}, + &memory.MergeableIndexLookup{Key: []interface{}{1}, Unions: []string{"2", "3", "4"}}, []sql.Index{ indexes[0], indexes[0], @@ -402,7 +408,7 @@ func TestGetIndexes(t *testing.T) { ), map[string]*indexLookup{ "t1": &indexLookup{ - &memory.MergeableIndexLookup{Id: "1", Unions: []string{"2", "3", "4"}}, + &memory.MergeableIndexLookup{Key: []interface{}{1}, Unions: []string{"2", "3", "4"}}, []sql.Index{indexes[0]}, }, }, @@ -433,11 +439,11 @@ func TestGetIndexes(t *testing.T) { ), map[string]*indexLookup{ "t1": &indexLookup{ - &memory.MergeableIndexLookup{Id: "3"}, + &memory.MergeableIndexLookup{Key: []interface{}{3}}, []sql.Index{indexes[0]}, }, "t2": &indexLookup{ - &memory.MergeableIndexLookup{Id: "1, 2"}, + &memory.MergeableIndexLookup{Key: []interface{}{1, 2}}, []sql.Index{indexes[1]}, }, }, @@ -474,11 +480,11 @@ func TestGetIndexes(t *testing.T) { ), map[string]*indexLookup{ "t1": &indexLookup{ - &memory.MergeableIndexLookup{Id: "3"}, + &memory.MergeableIndexLookup{Key: []interface{}{3}}, []sql.Index{indexes[0]}, }, "t2": &indexLookup{ - &memory.MergeableIndexLookup{Id: "5", Unions: []string{"1, 2"}}, + &memory.MergeableIndexLookup{Key: []interface{}{5}, Unions: []string{"1, 2"}}, []sql.Index{ indexes[2], indexes[1], @@ -574,7 +580,7 @@ func TestGetIndexes(t *testing.T) { map[string]*indexLookup{ "t1": &indexLookup{ &memory.NegateIndexLookup{ - Value: "1", + // TODO: fix }, []sql.Index{indexes[0]}, }, @@ -663,10 +669,10 @@ func TestGetIndexes(t *testing.T) { &memory.MergedIndexLookup{ Children: []sql.IndexLookup{ &memory.NegateIndexLookup{ - Value: "10", + // TODO: fix }, &memory.NegateIndexLookup{ - Value: "11", + // TODO: fix }, }, }, @@ -694,7 +700,7 @@ func TestGetIndexes(t *testing.T) { map[string]*indexLookup{ "t1": &indexLookup{ &memory.MergeableIndexLookup{ - Id: "not 10", + Key: []interface{}{10}, Intersections: []string{"not 11"}, }, []sql.Index{ @@ -724,7 +730,7 @@ func TestGetIndexes(t *testing.T) { map[string]*indexLookup{ "t2": &indexLookup{ &memory.NegateIndexLookup{ - Value: "110", + // TODO: fix }, []sql.Index{ indexes[2], @@ -746,7 +752,7 @@ func TestGetIndexes(t *testing.T) { map[string]*indexLookup{ "t1": &indexLookup{ &memory.MergeableIndexLookup{ - Id: "not 1", + Key: []interface{}{1}, Intersections: []string{"not 2", "not 3", "not 4"}, }, []sql.Index{indexes[0]}, @@ -873,11 +879,11 @@ func TestGetMultiColumnIndexes(t *testing.T) { expected := map[string]*indexLookup{ "t1": &indexLookup{ - &memory.MergeableIndexLookup{Id: "5, 6"}, + &memory.MergeableIndexLookup{Key: []interface{}{5, 6}}, []sql.Index{indexes[0]}, }, "t2": &indexLookup{ - &memory.MergeableIndexLookup{Id: "1, 2, 3"}, + &memory.MergeableIndexLookup{Key: []interface{}{1, 2, 3}}, []sql.Index{indexes[1]}, }, "t4": &indexLookup{ @@ -1003,29 +1009,29 @@ func TestIndexesIntersection(t *testing.T) { idx1, idx2 := &memory.MergeableDummyIndex{TableName: "bar"}, &memory.MergeableDummyIndex{TableName: "foo"} left := map[string]*indexLookup{ - "a": &indexLookup{&memory.MergeableIndexLookup{Id: "a"}, nil}, - "b": &indexLookup{&memory.MergeableIndexLookup{Id: "b"}, []sql.Index{idx1}}, + "a": &indexLookup{&memory.MergeableIndexLookup{Key: []interface{}{"a"}}, nil}, + "b": &indexLookup{&memory.MergeableIndexLookup{Key: []interface{}{"b"}}, []sql.Index{idx1}}, "c": &indexLookup{new(DummyIndexLookup), nil}, } right := map[string]*indexLookup{ - "b": &indexLookup{&memory.MergeableIndexLookup{Id: "b2"}, []sql.Index{idx2}}, - "c": &indexLookup{&memory.MergeableIndexLookup{Id: "c"}, nil}, - "d": &indexLookup{&memory.MergeableIndexLookup{Id: "d"}, nil}, + "b": &indexLookup{&memory.MergeableIndexLookup{Key: []interface{}{"b2"}}, []sql.Index{idx2}}, + "c": &indexLookup{&memory.MergeableIndexLookup{Key: []interface{}{"c"}}, nil}, + "d": &indexLookup{&memory.MergeableIndexLookup{Key: []interface{}{"d"}}, nil}, } require.Equal( map[string]*indexLookup{ - "a": &indexLookup{&memory.MergeableIndexLookup{Id: "a"}, nil}, + "a": &indexLookup{&memory.MergeableIndexLookup{Key: []interface{}{"a"}}, nil}, "b": &indexLookup{ &memory.MergeableIndexLookup{ - Id: "b", + Key: []interface{}{"b"}, Intersections: []string{"b2"}, }, []sql.Index{idx1, idx2}, }, "c": &indexLookup{new(DummyIndexLookup), nil}, - "d": &indexLookup{&memory.MergeableIndexLookup{Id: "d"}, nil}, + "d": &indexLookup{&memory.MergeableIndexLookup{Key: []interface{}{"d"}}, nil}, }, indexesIntersection(NewDefault(sql.NewCatalog()), left, right), ) diff --git a/sql/analyzer/pushdown_test.go b/sql/analyzer/pushdown_test.go index 9f8758a7b..84de0d487 100644 --- a/sql/analyzer/pushdown_test.go +++ b/sql/analyzer/pushdown_test.go @@ -189,7 +189,7 @@ func TestPushdownIndexable(t *testing.T) { ), }).(*memory.Table). WithProjection([]string{"i", "f"}).(*memory.Table). - WithIndexLookup(&memory.MergeableIndexLookup{Id: "3.14"}), + WithIndexLookup(&memory.MergeableIndexLookup{Key: []interface{}{3.14}}), ), plan.NewResolvedTable( table2.WithFilters([]sql.Expression{ @@ -201,7 +201,8 @@ func TestPushdownIndexable(t *testing.T) { ), }).(*memory.Table). WithProjection([]string{"i2"}).(*memory.Table). - WithIndexLookup(&memory.NegateIndexLookup{Value: "2"}), + // TODO: fix + WithIndexLookup(&memory.NegateIndexLookup{}), ), ), ), From dc6dc0ef7c837f9e6acb72b76becdd030e6001c3 Mon Sep 17 00:00:00 2001 From: Zach Musgrave Date: Fri, 1 Nov 2019 15:53:40 -0700 Subject: [PATCH 17/42] Patching up all the tests with the new memory index implementations Signed-off-by: Zach Musgrave --- memory/ascend_index.go | 22 ++- memory/descend_index.go | 23 ++- memory/mergeable_index.go | 221 ++++++++++++++-------------- memory/negative_index.go | 32 ++-- sql/analyzer/assign_indexes_test.go | 194 ++++++++++++++++++------ 5 files changed, 319 insertions(+), 173 deletions(-) diff --git a/memory/ascend_index.go b/memory/ascend_index.go index 36be10240..625207b1e 100755 --- a/memory/ascend_index.go +++ b/memory/ascend_index.go @@ -10,6 +10,18 @@ type AscendIndexLookup struct { Lt []interface{} } +func (l *AscendIndexLookup) ID() string { + return l.id +} + +func (l *AscendIndexLookup) GetUnions() []MergeableLookup { + return nil +} + +func (l *AscendIndexLookup) GetIntersections() []MergeableLookup { + return nil +} + func (AscendIndexLookup) Values(sql.Partition) (sql.IndexValueIter, error) { return nil, nil } @@ -23,7 +35,15 @@ func (l *AscendIndexLookup) IsMergeable(sql.IndexLookup) bool { } func (l *AscendIndexLookup) Union(lookups ...sql.IndexLookup) sql.IndexLookup { - return &MergedIndexLookup{append([]sql.IndexLookup{l}, lookups...)} + var unions []MergeableLookup + unions = append(unions, l) + for _, idx := range lookups { + unions = append(unions, idx.(MergeableLookup)) + } + + return &MergedIndexLookup{ + Union: unions, + } } func (AscendIndexLookup) Difference(...sql.IndexLookup) sql.IndexLookup { diff --git a/memory/descend_index.go b/memory/descend_index.go index 16e3024ac..c9b6f6318 100755 --- a/memory/descend_index.go +++ b/memory/descend_index.go @@ -10,6 +10,18 @@ type DescendIndexLookup struct { Lte []interface{} } +func (l DescendIndexLookup) ID() string { + return l.id +} + +func (l DescendIndexLookup) GetUnions() []MergeableLookup { + return nil +} + +func (l DescendIndexLookup) GetIntersections() []MergeableLookup { + return nil +} + func (DescendIndexLookup) Values(sql.Partition) (sql.IndexValueIter, error) { return nil, nil } @@ -23,7 +35,16 @@ func (l *DescendIndexLookup) IsMergeable(sql.IndexLookup) bool { } func (l *DescendIndexLookup) Union(lookups ...sql.IndexLookup) sql.IndexLookup { - return &MergedIndexLookup{append([]sql.IndexLookup{l}, lookups...)} + var unions []MergeableLookup + unions = append(unions, l) + for _, idx := range lookups { + unions = append(unions, idx.(MergeableLookup)) + } + + return &MergedIndexLookup{ + Union: unions, + } + } func (DescendIndexLookup) Difference(...sql.IndexLookup) sql.IndexLookup { diff --git a/memory/mergeable_index.go b/memory/mergeable_index.go index a749f00d6..59c97c737 100755 --- a/memory/mergeable_index.go +++ b/memory/mergeable_index.go @@ -3,122 +3,10 @@ package memory import ( "fmt" "github.com/src-d/go-mysql-server/sql" + "github.com/src-d/go-mysql-server/sql/expression" "strings" ) -type MergeableLookup interface { - ID() string - GetUnions() []string - GetIntersections() []string -} - -// ExpressionsIndex is an index made out of one or more expressions (usually field expressions) -type ExpressionsIndex interface { - ColumnExpressions() []sql.Expression -} - -type MergeableIndexLookup struct { - Key []interface{} - Index ExpressionsIndex - Unions []string - Intersections []string -} - -var _ sql.Mergeable = (*MergeableIndexLookup)(nil) -var _ sql.SetOperations = (*MergeableIndexLookup)(nil) - -func (i *MergeableIndexLookup) ID() string { return strings.Join(i.Indexes(), ",") } -func (i *MergeableIndexLookup) GetUnions() []string { return i.Unions } -func (i *MergeableIndexLookup) GetIntersections() []string { return i.Intersections } - -func (i *MergeableIndexLookup) IsMergeable(lookup sql.IndexLookup) bool { - _, ok := lookup.(MergeableLookup) - return ok -} - -func (i *MergeableIndexLookup) Values(sql.Partition) (sql.IndexValueIter, error) { - return nil, nil -} - -func (i *MergeableIndexLookup) Indexes() []string { - var idxes = make([]string, len(i.Key)) - for i, e := range i.Key { - idxes[i] = fmt.Sprint(e) - } - return idxes -} - -func (i *MergeableIndexLookup) Difference(indexes ...sql.IndexLookup) sql.IndexLookup { - panic("not implemented") -} - -func (i *MergeableIndexLookup) Intersection(indexes ...sql.IndexLookup) sql.IndexLookup { - var intersections, unions []string - for _, idx := range indexes { - intersections = append(intersections, idx.(MergeableLookup).ID()) - intersections = append(intersections, idx.(MergeableLookup).GetIntersections()...) - unions = append(unions, idx.(MergeableLookup).GetUnions()...) - } - - // TODO: fix logic - return &MergeableIndexLookup{ - Key: i.Key, - Index: i.Index, - Unions: append(i.Unions, unions...), - Intersections: append(i.Intersections, intersections...), - } -} - -func (i *MergeableIndexLookup) Union(indexes ...sql.IndexLookup) sql.IndexLookup { - var intersections, unions []string - for _, idx := range indexes { - unions = append(unions, idx.(*MergeableIndexLookup).ID()) - unions = append(unions, idx.(*MergeableIndexLookup).Unions...) - intersections = append(intersections, idx.(*MergeableIndexLookup).Intersections...) - } - - // TODO: fix logic - return &MergeableIndexLookup{ - Key: i.Key, - Index: i.Index, - Unions: append(i.Unions, unions...), - Intersections: append(i.Intersections, intersections...), - } -} - -type MergedIndexLookup struct { - Children []sql.IndexLookup -} - -func (i *MergedIndexLookup) Values(sql.Partition) (sql.IndexValueIter, error) { - return nil, nil -// return &dummyIndexValueIter{}, nil -} - -func (i *MergedIndexLookup) Indexes() []string { - var indexes []string - for _, c := range i.Children { - indexes = append(indexes, c.Indexes()...) - } - return indexes -} - -func (i *MergedIndexLookup) IsMergeable(sql.IndexLookup) bool { - return true -} - -func (i *MergedIndexLookup) Union(lookups ...sql.IndexLookup) sql.IndexLookup { - return &MergedIndexLookup{append(i.Children, lookups...)} -} - -func (MergedIndexLookup) Difference(...sql.IndexLookup) sql.IndexLookup { - panic("mergedIndexLookup.Difference is not implemented") -} - -func (MergedIndexLookup) Intersection(...sql.IndexLookup) sql.IndexLookup { - panic("mergedIndexLookup.Intersection is not implemented") -} - type MergeableDummyIndex struct { DB string // required for engine tests with driver DriverName string // required for engine tests with driver @@ -129,6 +17,7 @@ type MergeableDummyIndex struct { func (i MergeableDummyIndex) Database() string { return i.DB } func (i MergeableDummyIndex) Driver() string { return i.DriverName } +func (i MergeableDummyIndex) MemTable() *Table { return i.Tbl } func (i MergeableDummyIndex) ColumnExpressions() []sql.Expression { return i.Exprs } func (i MergeableDummyIndex) Expressions() []string { @@ -193,4 +82,108 @@ func (i MergeableDummyIndex) ID() string { return "(" + strings.Join(parts, ", ") + ")" } -func (i MergeableDummyIndex) Table() string { return i.TableName } \ No newline at end of file +func (i MergeableDummyIndex) Table() string { return i.TableName } + +type MergeableLookup interface { + ID() string + GetUnions() []MergeableLookup + GetIntersections() []MergeableLookup +} + +// ExpressionsIndex is an index made out of one or more expressions (usually field expressions) +type ExpressionsIndex interface { + MemTable() *Table + ColumnExpressions() []sql.Expression +} + +type MergeableIndexLookup struct { + Key []interface{} + Index ExpressionsIndex +} + +var _ sql.Mergeable = (*MergeableIndexLookup)(nil) +var _ sql.SetOperations = (*MergeableIndexLookup)(nil) + +func (i *MergeableIndexLookup) ID() string { return strings.Join(i.Indexes(), ",") } +func (i *MergeableIndexLookup) GetUnions() []MergeableLookup { return nil } +func (i *MergeableIndexLookup) GetIntersections() []MergeableLookup { return nil } + +func (i *MergeableIndexLookup) IsMergeable(lookup sql.IndexLookup) bool { + _, ok := lookup.(MergeableLookup) + return ok +} + +func (i *MergeableIndexLookup) Values(p sql.Partition) (sql.IndexValueIter, error) { + return &dummyIndexValueIter{ + tbl: i.Index.MemTable(), + partition: p, + matchExpressions: func() []sql.Expression { + var exprs []sql.Expression + for exprI, expr := range i.Index.ColumnExpressions() { + lit, typ := getType(i.Key[exprI]) + exprs = append(exprs, expression.NewEquals(expr, expression.NewLiteral(lit, typ))) + } + return exprs + }}, nil +} + +func (i *MergeableIndexLookup) Indexes() []string { + var idxes = make([]string, len(i.Key)) + for i, e := range i.Key { + idxes[i] = fmt.Sprint(e) + } + return idxes +} + +func (i *MergeableIndexLookup) Difference(...sql.IndexLookup) sql.IndexLookup { + panic("not implemented") +} + +func (i *MergeableIndexLookup) Intersection(indexes ...sql.IndexLookup) sql.IndexLookup { + var intersections []MergeableLookup + intersections = append(intersections, i) + for _, idx := range indexes { + intersections = append(intersections, idx.(MergeableLookup)) + } + + return &MergedIndexLookup{ + Intersection: intersections, + } +} + +func (i *MergeableIndexLookup) Union(indexes ...sql.IndexLookup) sql.IndexLookup { + var unions []MergeableLookup + unions = append(unions, i) + for _, idx := range indexes { + unions = append(unions, idx.(MergeableLookup)) + } + + return &MergedIndexLookup{ + Union: unions, + } +} + +type MergedIndexLookup struct { + Union []MergeableLookup + Intersection []MergeableLookup +} + +func (m *MergedIndexLookup) GetUnions() []MergeableLookup { + panic("implement me") +} + +func (m *MergedIndexLookup) GetIntersections() []MergeableLookup { + panic("implement me") +} + +func (m *MergedIndexLookup) Values(sql.Partition) (sql.IndexValueIter, error) { + panic("implement me") +} + +func (m *MergedIndexLookup) Indexes() []string { + panic("implement me") +} + +func (m *MergedIndexLookup) ID() string { + panic("implement me") +} diff --git a/memory/negative_index.go b/memory/negative_index.go index bb492ae39..a2b34ba92 100755 --- a/memory/negative_index.go +++ b/memory/negative_index.go @@ -5,14 +5,12 @@ import ( ) type NegateIndexLookup struct { - Lookup *MergeableIndexLookup - intersections []string - unions []string + Lookup MergeableLookup } func (l *NegateIndexLookup) ID() string { return "not " + l.Lookup.ID() } -func (l *NegateIndexLookup) GetUnions() []string { return l.unions } -func (l *NegateIndexLookup) GetIntersections() []string { return l.intersections } +func (l *NegateIndexLookup) GetUnions() []MergeableLookup { return l.Lookup.GetUnions() } +func (l *NegateIndexLookup) GetIntersections() []MergeableLookup { return l.Lookup.GetIntersections() } func (*NegateIndexLookup) Values(sql.Partition) (sql.IndexValueIter, error) { return nil, nil @@ -27,7 +25,15 @@ func (*NegateIndexLookup) IsMergeable(sql.IndexLookup) bool { } func (l *NegateIndexLookup) Union(lookups ...sql.IndexLookup) sql.IndexLookup { - return &MergedIndexLookup{append([]sql.IndexLookup{l}, lookups...)} + var unions []MergeableLookup + unions = append(unions, l) + for _, idx := range lookups { + unions = append(unions, idx.(MergeableLookup)) + } + + return &MergedIndexLookup{ + Union: unions, + } } func (*NegateIndexLookup) Difference(...sql.IndexLookup) sql.IndexLookup { @@ -35,15 +41,13 @@ func (*NegateIndexLookup) Difference(...sql.IndexLookup) sql.IndexLookup { } func (l *NegateIndexLookup) Intersection(indexes ...sql.IndexLookup) sql.IndexLookup { - var intersections, unions []string + var intersections []MergeableLookup + intersections = append(intersections, l) for _, idx := range indexes { - intersections = append(intersections, idx.(MergeableLookup).ID()) - intersections = append(intersections, idx.(MergeableLookup).GetIntersections()...) - unions = append(unions, idx.(MergeableLookup).GetUnions()...) + intersections = append(intersections, idx.(MergeableLookup)) } - return &MergeableIndexLookup{ - Index: l.Lookup.Index, - Unions: append(l.unions, unions...), - Intersections: append(l.intersections, intersections...), + + return &MergedIndexLookup{ + Intersection: intersections, } } \ No newline at end of file diff --git a/sql/analyzer/assign_indexes_test.go b/sql/analyzer/assign_indexes_test.go index e43a20aef..3c3e9be0e 100644 --- a/sql/analyzer/assign_indexes_test.go +++ b/sql/analyzer/assign_indexes_test.go @@ -187,6 +187,16 @@ func TestAssignIndexes(t *testing.T) { require.False(ok) } +func mergeableIndexLookup(table string, column string, colIdx int, key ...interface{}) *memory.MergeableIndexLookup { + return &memory.MergeableIndexLookup{ + Key: key, + Index: memory.MergeableDummyIndex{ + TableName: table, + Exprs: []sql.Expression{col(colIdx, table, column)}, + }, + } +} + func TestGetIndexes(t *testing.T) { indexes := []sql.Index { &memory.MergeableDummyIndex{ @@ -236,13 +246,7 @@ func TestGetIndexes(t *testing.T) { ), map[string]*indexLookup{ "t1": &indexLookup{ - &memory.MergeableIndexLookup{ - Key: []interface{}{int64(1)}, - Index: memory.MergeableDummyIndex{ - TableName: "t1", - Exprs: []sql.Expression{col(0, "t1", "bar")}, - }, - }, + mergeableIndexLookup("t1", "bar", 0, int64(1)), []sql.Index{indexes[0]}, }, }, @@ -261,7 +265,12 @@ func TestGetIndexes(t *testing.T) { ), map[string]*indexLookup{ "t1": &indexLookup{ - &memory.MergeableIndexLookup{Key: []interface{}{1}, Unions: []string{"2"}}, + &memory.MergedIndexLookup{ + Union: []memory.MergeableLookup{ + mergeableIndexLookup("t1", "bar", 0, int64(1)), + mergeableIndexLookup("t1", "bar", 0, int64(2)), + }, + }, []sql.Index{ indexes[0], indexes[0], @@ -299,7 +308,12 @@ func TestGetIndexes(t *testing.T) { ), map[string]*indexLookup{ "t1": &indexLookup{ - &memory.MergeableIndexLookup{Key: []interface{}{1}, Unions: []string{"2"}}, + &memory.MergedIndexLookup{ + Union: []memory.MergeableLookup{ + mergeableIndexLookup("t1", "bar", 0, int64(2)), + mergeableIndexLookup("t1", "bar", 0, int64(1)), + }, + }, []sql.Index{ indexes[0], }, @@ -320,7 +334,12 @@ func TestGetIndexes(t *testing.T) { ), map[string]*indexLookup{ "t1": &indexLookup{ - &memory.MergeableIndexLookup{Key: []interface{}{1}, Intersections: []string{"2"}}, + &memory.MergedIndexLookup{ + Intersection: []memory.MergeableLookup{ + &memory.MergeableIndexLookup{Key: []interface{}{1}}, + &memory.MergeableIndexLookup{Key: []interface{}{2}}, + }, + }, []sql.Index{ indexes[0], indexes[0], @@ -354,7 +373,22 @@ func TestGetIndexes(t *testing.T) { ), map[string]*indexLookup{ "t1": &indexLookup{ - &memory.MergeableIndexLookup{Key: []interface{}{1}, Unions: []string{"2", "4"}, Intersections: []string{"3"}}, + &memory.MergedIndexLookup{ + Intersection: []memory.MergeableLookup { + &memory.MergedIndexLookup{ + Union: []memory.MergeableLookup{ + &memory.MergeableIndexLookup{Key: []interface{}{1}}, + &memory.MergeableIndexLookup{Key: []interface{}{2}}, + }, + }, + &memory.MergedIndexLookup{ + Union: []memory.MergeableLookup{ + &memory.MergeableIndexLookup{Key: []interface{}{3}}, + &memory.MergeableIndexLookup{Key: []interface{}{4}}, + }, + }, + }, + }, []sql.Index{ indexes[0], indexes[0], @@ -390,7 +424,22 @@ func TestGetIndexes(t *testing.T) { ), map[string]*indexLookup{ "t1": &indexLookup{ - &memory.MergeableIndexLookup{Key: []interface{}{1}, Unions: []string{"2", "3", "4"}}, + &memory.MergedIndexLookup{ + Union: []memory.MergeableLookup { + &memory.MergedIndexLookup{ + Union: []memory.MergeableLookup{ + &memory.MergeableIndexLookup{Key: []interface{}{1}}, + &memory.MergeableIndexLookup{Key: []interface{}{2}}, + }, + }, + &memory.MergedIndexLookup{ + Union: []memory.MergeableLookup{ + &memory.MergeableIndexLookup{Key: []interface{}{3}}, + &memory.MergeableIndexLookup{Key: []interface{}{4}}, + }, + }, + }, + }, []sql.Index{ indexes[0], indexes[0], @@ -408,7 +457,14 @@ func TestGetIndexes(t *testing.T) { ), map[string]*indexLookup{ "t1": &indexLookup{ - &memory.MergeableIndexLookup{Key: []interface{}{1}, Unions: []string{"2", "3", "4"}}, + &memory.MergedIndexLookup{ + Union: []memory.MergeableLookup{ + &memory.MergeableIndexLookup{Key: []interface{}{1}}, + &memory.MergeableIndexLookup{Key: []interface{}{2}}, + &memory.MergeableIndexLookup{Key: []interface{}{3}}, + &memory.MergeableIndexLookup{Key: []interface{}{4}}, + }, + }, []sql.Index{indexes[0]}, }, }, @@ -484,7 +540,26 @@ func TestGetIndexes(t *testing.T) { []sql.Index{indexes[0]}, }, "t2": &indexLookup{ - &memory.MergeableIndexLookup{Key: []interface{}{5}, Unions: []string{"1, 2"}}, + &memory.MergedIndexLookup{ + Union: []memory.MergeableLookup{ + &memory.MergeableIndexLookup{Key: []interface{}{5}}, + &memory.MergedIndexLookup{ + Intersection: []memory.MergeableLookup{ + &memory.MergeableIndexLookup{Key: []interface{}{1}}, + &memory.MergedIndexLookup{ + Intersection: []memory.MergeableLookup{ + &memory.MergeableIndexLookup{Key: []interface{}{4}}, + &memory.MergedIndexLookup{ + Intersection: []memory.MergeableLookup{ + &memory.MergeableIndexLookup{Key: []interface{}{2}}, + }, + }, + }, + }, + }, + }, + }, + }, []sql.Index{ indexes[2], indexes[1], @@ -553,8 +628,8 @@ func TestGetIndexes(t *testing.T) { ), map[string]*indexLookup{ "t1": &indexLookup{ - &memory.MergedIndexLookup{ - []sql.IndexLookup{ + &memory.MergedIndexLookup { + Intersection: []memory.MergeableLookup{ &memory.AscendIndexLookup{ Gte: []interface{}{int64(1)}, Lt: []interface{}{int64(5)}, @@ -666,13 +741,15 @@ func TestGetIndexes(t *testing.T) { ), map[string]*indexLookup{ "t1": &indexLookup{ - &memory.MergedIndexLookup{ - Children: []sql.IndexLookup{ - &memory.NegateIndexLookup{ - // TODO: fix - }, - &memory.NegateIndexLookup{ - // TODO: fix + &memory.NegateIndexLookup { + Lookup: &memory.MergedIndexLookup{ + Intersection: []memory.MergeableLookup { + &memory.MergeableIndexLookup{ + Key: []interface{}{int64(10)}, + }, + &memory.MergeableIndexLookup{ + Key: []interface{}{int64(11)}, + }, }, }, }, @@ -699,9 +776,17 @@ func TestGetIndexes(t *testing.T) { ), map[string]*indexLookup{ "t1": &indexLookup{ - &memory.MergeableIndexLookup{ - Key: []interface{}{10}, - Intersections: []string{"not 11"}, + &memory.NegateIndexLookup { + Lookup: &memory.MergedIndexLookup{ + Union: []memory.MergeableLookup { + &memory.MergeableIndexLookup{ + Key: []interface{}{int64(10)}, + }, + &memory.MergeableIndexLookup{ + Key: []interface{}{int64(11)}, + }, + }, + }, }, []sql.Index{ indexes[0], @@ -730,7 +815,9 @@ func TestGetIndexes(t *testing.T) { map[string]*indexLookup{ "t2": &indexLookup{ &memory.NegateIndexLookup{ - // TODO: fix + Lookup: &memory.MergeableIndexLookup{ + Key: []interface{}{110}, + }, }, []sql.Index{ indexes[2], @@ -751,9 +838,23 @@ func TestGetIndexes(t *testing.T) { ), map[string]*indexLookup{ "t1": &indexLookup{ - &memory.MergeableIndexLookup{ - Key: []interface{}{1}, - Intersections: []string{"not 2", "not 3", "not 4"}, + &memory.NegateIndexLookup{ + Lookup: &memory.MergedIndexLookup{ + Union: []memory.MergeableLookup { + &memory.MergeableIndexLookup{ + Key: []interface{}{1}, + }, + &memory.MergeableIndexLookup{ + Key: []interface{}{2}, + }, + &memory.MergeableIndexLookup{ + Key: []interface{}{3}, + }, + &memory.MergeableIndexLookup{ + Key: []interface{}{4}, + }, + }, + }, }, []sql.Index{indexes[0]}, }, @@ -887,16 +988,17 @@ func TestGetMultiColumnIndexes(t *testing.T) { []sql.Index{indexes[1]}, }, "t4": &indexLookup{ - &memory.MergedIndexLookup{[]sql.IndexLookup{ - &memory.AscendIndexLookup{ - Gte: []interface{}{int64(1), int64(2)}, - Lt: []interface{}{int64(6), int64(5)}, - }, - &memory.DescendIndexLookup{ - Gt: []interface{}{int64(1), int64(2)}, - Lte: []interface{}{int64(6), int64(5)}, - }, - }}, + &memory.MergedIndexLookup{ + Intersection: []memory.MergeableLookup{ + &memory.AscendIndexLookup{ + Gte: []interface{}{int64(1), int64(2)}, + Lt: []interface{}{int64(6), int64(5)}, + }, + &memory.DescendIndexLookup{ + Gt: []interface{}{int64(1), int64(2)}, + Lte: []interface{}{int64(6), int64(5)}, + }, + }}, []sql.Index{indexes[4]}, }, } @@ -1024,9 +1126,15 @@ func TestIndexesIntersection(t *testing.T) { map[string]*indexLookup{ "a": &indexLookup{&memory.MergeableIndexLookup{Key: []interface{}{"a"}}, nil}, "b": &indexLookup{ - &memory.MergeableIndexLookup{ - Key: []interface{}{"b"}, - Intersections: []string{"b2"}, + &memory.MergedIndexLookup { + Intersection: []memory.MergeableLookup { + &memory.MergeableIndexLookup{ + Key: []interface{}{"b"}, + }, + &memory.MergeableIndexLookup{ + Key: []interface{}{"b2"}, + }, + }, }, []sql.Index{idx1, idx2}, }, From 6c622a15b8442b93331f90ac2ff1a29ce9046463 Mon Sep 17 00:00:00 2001 From: Zach Musgrave Date: Fri, 1 Nov 2019 16:50:28 -0700 Subject: [PATCH 18/42] More progress on getting the tests to pass. Now to clean up the union and intersect functions Signed-off-by: Zach Musgrave --- memory/ascend_index.go | 2 +- memory/descend_index.go | 2 +- memory/mergeable_index.go | 58 +++- memory/negative_index.go | 4 +- sql/analyzer/assign_indexes_test.go | 472 ++++++++++++++-------------- 5 files changed, 283 insertions(+), 255 deletions(-) diff --git a/memory/ascend_index.go b/memory/ascend_index.go index 625207b1e..cb91eb83b 100755 --- a/memory/ascend_index.go +++ b/memory/ascend_index.go @@ -42,7 +42,7 @@ func (l *AscendIndexLookup) Union(lookups ...sql.IndexLookup) sql.IndexLookup { } return &MergedIndexLookup{ - Union: unions, + Unions: unions, } } diff --git a/memory/descend_index.go b/memory/descend_index.go index c9b6f6318..f1f921bdf 100755 --- a/memory/descend_index.go +++ b/memory/descend_index.go @@ -42,7 +42,7 @@ func (l *DescendIndexLookup) Union(lookups ...sql.IndexLookup) sql.IndexLookup { } return &MergedIndexLookup{ - Union: unions, + Unions: unions, } } diff --git a/memory/mergeable_index.go b/memory/mergeable_index.go index 59c97c737..357173b08 100755 --- a/memory/mergeable_index.go +++ b/memory/mergeable_index.go @@ -139,41 +139,69 @@ func (i *MergeableIndexLookup) Difference(...sql.IndexLookup) sql.IndexLookup { panic("not implemented") } -func (i *MergeableIndexLookup) Intersection(indexes ...sql.IndexLookup) sql.IndexLookup { +func (i *MergeableIndexLookup) Intersection(lookups ...sql.IndexLookup) sql.IndexLookup { + return intersection(i, lookups...) +} + +func intersection(left MergeableLookup, lookups ...sql.IndexLookup) sql.IndexLookup { var intersections []MergeableLookup - intersections = append(intersections, i) - for _, idx := range indexes { - intersections = append(intersections, idx.(MergeableLookup)) + intersections = append(intersections, left) + for _, lookup := range lookups { + intersections = append(intersections, lookup.(MergeableLookup)) } return &MergedIndexLookup{ - Intersection: intersections, + Intersections: intersections, } } -func (i *MergeableIndexLookup) Union(indexes ...sql.IndexLookup) sql.IndexLookup { +func (i *MergeableIndexLookup) Union(lookups ...sql.IndexLookup) sql.IndexLookup { + return union(i, lookups...) +} + +func union(left MergeableLookup, lookups ...sql.IndexLookup) sql.IndexLookup { var unions []MergeableLookup - unions = append(unions, i) - for _, idx := range indexes { - unions = append(unions, idx.(MergeableLookup)) + unions = append(unions, left) + for _, lookup := range lookups { + unions = append(unions, lookup.(MergeableLookup)) } return &MergedIndexLookup{ - Union: unions, + Unions: unions, } } type MergedIndexLookup struct { - Union []MergeableLookup - Intersection []MergeableLookup + Unions []MergeableLookup + Intersections []MergeableLookup } +var _ sql.Mergeable = (*MergedIndexLookup)(nil) +var _ sql.SetOperations = (*MergedIndexLookup)(nil) + func (m *MergedIndexLookup) GetUnions() []MergeableLookup { - panic("implement me") + return m.Unions } func (m *MergedIndexLookup) GetIntersections() []MergeableLookup { - panic("implement me") + return m.Intersections +} + +func (m *MergedIndexLookup) Intersection(lookups ...sql.IndexLookup) sql.IndexLookup { + return intersection(m, lookups...) +} + +func (m *MergedIndexLookup) Union(lookups ...sql.IndexLookup) sql.IndexLookup { + return union(m, lookups...) +} + +func (m *MergedIndexLookup) Difference(...sql.IndexLookup) sql.IndexLookup { + panic("not implemented") +} + +func (m *MergedIndexLookup) IsMergeable(lookup sql.IndexLookup) bool { + _, ok := lookup.(MergeableLookup) + return ok } func (m *MergedIndexLookup) Values(sql.Partition) (sql.IndexValueIter, error) { @@ -186,4 +214,4 @@ func (m *MergedIndexLookup) Indexes() []string { func (m *MergedIndexLookup) ID() string { panic("implement me") -} +} \ No newline at end of file diff --git a/memory/negative_index.go b/memory/negative_index.go index a2b34ba92..640dfcacd 100755 --- a/memory/negative_index.go +++ b/memory/negative_index.go @@ -32,7 +32,7 @@ func (l *NegateIndexLookup) Union(lookups ...sql.IndexLookup) sql.IndexLookup { } return &MergedIndexLookup{ - Union: unions, + Unions: unions, } } @@ -48,6 +48,6 @@ func (l *NegateIndexLookup) Intersection(indexes ...sql.IndexLookup) sql.IndexLo } return &MergedIndexLookup{ - Intersection: intersections, + Intersections: intersections, } } \ No newline at end of file diff --git a/sql/analyzer/assign_indexes_test.go b/sql/analyzer/assign_indexes_test.go index 3c3e9be0e..aaa3c98ad 100644 --- a/sql/analyzer/assign_indexes_test.go +++ b/sql/analyzer/assign_indexes_test.go @@ -231,225 +231,225 @@ func TestGetIndexes(t *testing.T) { expected map[string]*indexLookup ok bool }{ - { - eq( - col(0, "t1", "bar"), - col(1, "t1", "baz"), - ), - map[string]*indexLookup{}, - true, - }, - { - eq( - col(0, "t1", "bar"), - lit(1), - ), - map[string]*indexLookup{ - "t1": &indexLookup{ - mergeableIndexLookup("t1", "bar", 0, int64(1)), - []sql.Index{indexes[0]}, - }, - }, - true, - }, - { - or( - eq( - col(0, "t1", "bar"), - lit(1), - ), - eq( - col(0, "t1", "bar"), - lit(2), - ), - ), - map[string]*indexLookup{ - "t1": &indexLookup{ - &memory.MergedIndexLookup{ - Union: []memory.MergeableLookup{ - mergeableIndexLookup("t1", "bar", 0, int64(1)), - mergeableIndexLookup("t1", "bar", 0, int64(2)), - }, - }, - []sql.Index{ - indexes[0], - indexes[0], - }, - }, - }, - true, - }, - { - or( - eq( - col(0, "t3", "foo"), - lit(1), - ), - eq( - col(0, "t3", "foo"), - lit(2), - ), - ), - nil, - true, - }, - { - in( - col(0, "t3", "foo"), - tuple(lit(1), lit(2)), - ), - nil, - true, - }, - { - in( - col(0, "t1", "bar"), - tuple(lit(1), lit(2)), - ), - map[string]*indexLookup{ - "t1": &indexLookup{ - &memory.MergedIndexLookup{ - Union: []memory.MergeableLookup{ - mergeableIndexLookup("t1", "bar", 0, int64(2)), - mergeableIndexLookup("t1", "bar", 0, int64(1)), - }, - }, - []sql.Index{ - indexes[0], - }, - }, - }, - true, - }, - { - and( - eq( - col(0, "t1", "bar"), - lit(1), - ), - eq( - col(0, "t1", "bar"), - lit(2), - ), - ), - map[string]*indexLookup{ - "t1": &indexLookup{ - &memory.MergedIndexLookup{ - Intersection: []memory.MergeableLookup{ - &memory.MergeableIndexLookup{Key: []interface{}{1}}, - &memory.MergeableIndexLookup{Key: []interface{}{2}}, - }, - }, - []sql.Index{ - indexes[0], - indexes[0], - }, - }, - }, - true, - }, - { - and( - or( - eq( - col(0, "t1", "bar"), - lit(1), - ), - eq( - col(0, "t1", "bar"), - lit(2), - ), - ), - or( - eq( - col(0, "t1", "bar"), - lit(3), - ), - eq( - col(0, "t1", "bar"), - lit(4), - ), - ), - ), - map[string]*indexLookup{ - "t1": &indexLookup{ - &memory.MergedIndexLookup{ - Intersection: []memory.MergeableLookup { - &memory.MergedIndexLookup{ - Union: []memory.MergeableLookup{ - &memory.MergeableIndexLookup{Key: []interface{}{1}}, - &memory.MergeableIndexLookup{Key: []interface{}{2}}, - }, - }, - &memory.MergedIndexLookup{ - Union: []memory.MergeableLookup{ - &memory.MergeableIndexLookup{Key: []interface{}{3}}, - &memory.MergeableIndexLookup{Key: []interface{}{4}}, - }, - }, - }, - }, - []sql.Index{ - indexes[0], - indexes[0], - indexes[0], - indexes[0], - }, - }, - }, - true, - }, - { - or( - or( - eq( - col(0, "t1", "bar"), - lit(1), - ), - eq( - col(0, "t1", "bar"), - lit(2), - ), - ), - or( - eq( - col(0, "t1", "bar"), - lit(3), - ), - eq( - col(0, "t1", "bar"), - lit(4), - ), - ), - ), - map[string]*indexLookup{ - "t1": &indexLookup{ - &memory.MergedIndexLookup{ - Union: []memory.MergeableLookup { - &memory.MergedIndexLookup{ - Union: []memory.MergeableLookup{ - &memory.MergeableIndexLookup{Key: []interface{}{1}}, - &memory.MergeableIndexLookup{Key: []interface{}{2}}, - }, - }, - &memory.MergedIndexLookup{ - Union: []memory.MergeableLookup{ - &memory.MergeableIndexLookup{Key: []interface{}{3}}, - &memory.MergeableIndexLookup{Key: []interface{}{4}}, - }, - }, - }, - }, - []sql.Index{ - indexes[0], - indexes[0], - indexes[0], - indexes[0], - }, - }, - }, - true, - }, + // { + // eq( + // col(0, "t1", "bar"), + // col(1, "t1", "baz"), + // ), + // map[string]*indexLookup{}, + // true, + // }, + // { + // eq( + // col(0, "t1", "bar"), + // lit(1), + // ), + // map[string]*indexLookup{ + // "t1": &indexLookup{ + // mergeableIndexLookup("t1", "bar", 0, int64(1)), + // []sql.Index{indexes[0]}, + // }, + // }, + // true, + // }, + // { + // or( + // eq( + // col(0, "t1", "bar"), + // lit(1), + // ), + // eq( + // col(0, "t1", "bar"), + // lit(2), + // ), + // ), + // map[string]*indexLookup{ + // "t1": &indexLookup{ + // &memory.MergedIndexLookup{ + // Union: []memory.MergeableLookup{ + // mergeableIndexLookup("t1", "bar", 0, int64(1)), + // mergeableIndexLookup("t1", "bar", 0, int64(2)), + // }, + // }, + // []sql.Index{ + // indexes[0], + // indexes[0], + // }, + // }, + // }, + // true, + // }, + // { + // or( + // eq( + // col(0, "t3", "foo"), + // lit(1), + // ), + // eq( + // col(0, "t3", "foo"), + // lit(2), + // ), + // ), + // nil, + // true, + // }, + // { + // in( + // col(0, "t3", "foo"), + // tuple(lit(1), lit(2)), + // ), + // nil, + // true, + // }, + // { + // in( + // col(0, "t1", "bar"), + // tuple(lit(1), lit(2)), + // ), + // map[string]*indexLookup{ + // "t1": &indexLookup{ + // &memory.MergedIndexLookup{ + // Union: []memory.MergeableLookup{ + // mergeableIndexLookup("t1", "bar", 0, int64(1)), + // mergeableIndexLookup("t1", "bar", 0, int64(2)), + // }, + // }, + // []sql.Index{ + // indexes[0], + // }, + // }, + // }, + // true, + // }, + // { + // and( + // eq( + // col(0, "t1", "bar"), + // lit(1), + // ), + // eq( + // col(0, "t1", "bar"), + // lit(2), + // ), + // ), + // map[string]*indexLookup{ + // "t1": &indexLookup{ + // &memory.MergedIndexLookup{ + // Intersection: []memory.MergeableLookup{ + // mergeableIndexLookup("t1", "bar", 0, int64(1)), + // mergeableIndexLookup("t1", "bar", 0, int64(2)), + // }, + // }, + // []sql.Index{ + // indexes[0], + // indexes[0], + // }, + // }, + // }, + // true, + // }, + // { + // and( + // or( + // eq( + // col(0, "t1", "bar"), + // lit(1), + // ), + // eq( + // col(0, "t1", "bar"), + // lit(2), + // ), + // ), + // or( + // eq( + // col(0, "t1", "bar"), + // lit(3), + // ), + // eq( + // col(0, "t1", "bar"), + // lit(4), + // ), + // ), + // ), + // map[string]*indexLookup{ + // "t1": &indexLookup{ + // &memory.MergedIndexLookup{ + // Intersection: []memory.MergeableLookup { + // &memory.MergedIndexLookup{ + // Union: []memory.MergeableLookup{ + // mergeableIndexLookup("t1", "bar", 0, int64(1)), + // mergeableIndexLookup("t1", "bar", 0, int64(2)), + // }, + // }, + // &memory.MergedIndexLookup{ + // Union: []memory.MergeableLookup{ + // mergeableIndexLookup("t1", "bar", 0, int64(3)), + // mergeableIndexLookup("t1", "bar", 0, int64(4)), + // }, + // }, + // }, + // }, + // []sql.Index{ + // indexes[0], + // indexes[0], + // indexes[0], + // indexes[0], + // }, + // }, + // }, + // true, + // }, + // { + // or( + // or( + // eq( + // col(0, "t1", "bar"), + // lit(1), + // ), + // eq( + // col(0, "t1", "bar"), + // lit(2), + // ), + // ), + // or( + // eq( + // col(0, "t1", "bar"), + // lit(3), + // ), + // eq( + // col(0, "t1", "bar"), + // lit(4), + // ), + // ), + // ), + // map[string]*indexLookup{ + // "t1": &indexLookup{ + // &memory.MergedIndexLookup{ + // Union: []memory.MergeableLookup { + // &memory.MergedIndexLookup{ + // Union: []memory.MergeableLookup{ + // mergeableIndexLookup("t1", "bar", 0, int64(1)), + // mergeableIndexLookup("t1", "bar", 0, int64(2)), + // }, + // }, + // &memory.MergedIndexLookup{ + // Union: []memory.MergeableLookup{ + // mergeableIndexLookup("t1", "bar", 0, int64(3)), + // mergeableIndexLookup("t1", "bar", 0, int64(4)), + // }, + // }, + // }, + // }, + // []sql.Index{ + // indexes[0], + // indexes[0], + // indexes[0], + // indexes[0], + // }, + // }, + // }, + // true, + // }, { in( col(0, "t1", "bar"), @@ -458,11 +458,11 @@ func TestGetIndexes(t *testing.T) { map[string]*indexLookup{ "t1": &indexLookup{ &memory.MergedIndexLookup{ - Union: []memory.MergeableLookup{ - &memory.MergeableIndexLookup{Key: []interface{}{1}}, - &memory.MergeableIndexLookup{Key: []interface{}{2}}, - &memory.MergeableIndexLookup{Key: []interface{}{3}}, - &memory.MergeableIndexLookup{Key: []interface{}{4}}, + Unions: []memory.MergeableLookup{ + mergeableIndexLookup("t1", "bar", 0, int64(1)), + mergeableIndexLookup("t1", "bar", 0, int64(2)), + mergeableIndexLookup("t1", "bar", 0, int64(3)), + mergeableIndexLookup("t1", "bar", 0, int64(4)), }, }, []sql.Index{indexes[0]}, @@ -495,11 +495,11 @@ func TestGetIndexes(t *testing.T) { ), map[string]*indexLookup{ "t1": &indexLookup{ - &memory.MergeableIndexLookup{Key: []interface{}{3}}, + mergeableIndexLookup("t1", "bar", 0, int64(3)), []sql.Index{indexes[0]}, }, "t2": &indexLookup{ - &memory.MergeableIndexLookup{Key: []interface{}{1, 2}}, + mergeableIndexLookup("t2", "foo", 0, int64(1), int64(2)), []sql.Index{indexes[1]}, }, }, @@ -541,16 +541,16 @@ func TestGetIndexes(t *testing.T) { }, "t2": &indexLookup{ &memory.MergedIndexLookup{ - Union: []memory.MergeableLookup{ + Unions: []memory.MergeableLookup{ &memory.MergeableIndexLookup{Key: []interface{}{5}}, &memory.MergedIndexLookup{ - Intersection: []memory.MergeableLookup{ + Intersections: []memory.MergeableLookup{ &memory.MergeableIndexLookup{Key: []interface{}{1}}, &memory.MergedIndexLookup{ - Intersection: []memory.MergeableLookup{ + Intersections: []memory.MergeableLookup{ &memory.MergeableIndexLookup{Key: []interface{}{4}}, &memory.MergedIndexLookup{ - Intersection: []memory.MergeableLookup{ + Intersections: []memory.MergeableLookup{ &memory.MergeableIndexLookup{Key: []interface{}{2}}, }, }, @@ -629,7 +629,7 @@ func TestGetIndexes(t *testing.T) { map[string]*indexLookup{ "t1": &indexLookup{ &memory.MergedIndexLookup { - Intersection: []memory.MergeableLookup{ + Intersections: []memory.MergeableLookup{ &memory.AscendIndexLookup{ Gte: []interface{}{int64(1)}, Lt: []interface{}{int64(5)}, @@ -743,7 +743,7 @@ func TestGetIndexes(t *testing.T) { "t1": &indexLookup{ &memory.NegateIndexLookup { Lookup: &memory.MergedIndexLookup{ - Intersection: []memory.MergeableLookup { + Intersections: []memory.MergeableLookup { &memory.MergeableIndexLookup{ Key: []interface{}{int64(10)}, }, @@ -778,7 +778,7 @@ func TestGetIndexes(t *testing.T) { "t1": &indexLookup{ &memory.NegateIndexLookup { Lookup: &memory.MergedIndexLookup{ - Union: []memory.MergeableLookup { + Unions: []memory.MergeableLookup { &memory.MergeableIndexLookup{ Key: []interface{}{int64(10)}, }, @@ -840,7 +840,7 @@ func TestGetIndexes(t *testing.T) { "t1": &indexLookup{ &memory.NegateIndexLookup{ Lookup: &memory.MergedIndexLookup{ - Union: []memory.MergeableLookup { + Unions: []memory.MergeableLookup { &memory.MergeableIndexLookup{ Key: []interface{}{1}, }, @@ -989,7 +989,7 @@ func TestGetMultiColumnIndexes(t *testing.T) { }, "t4": &indexLookup{ &memory.MergedIndexLookup{ - Intersection: []memory.MergeableLookup{ + Intersections: []memory.MergeableLookup{ &memory.AscendIndexLookup{ Gte: []interface{}{int64(1), int64(2)}, Lt: []interface{}{int64(6), int64(5)}, @@ -1127,7 +1127,7 @@ func TestIndexesIntersection(t *testing.T) { "a": &indexLookup{&memory.MergeableIndexLookup{Key: []interface{}{"a"}}, nil}, "b": &indexLookup{ &memory.MergedIndexLookup { - Intersection: []memory.MergeableLookup { + Intersections: []memory.MergeableLookup { &memory.MergeableIndexLookup{ Key: []interface{}{"b"}, }, From fa72192f51a90cb8c700e27fe42c845a3923e7a9 Mon Sep 17 00:00:00 2001 From: Daylon Wilkins Date: Fri, 1 Nov 2019 16:45:24 -0700 Subject: [PATCH 19/42] Fixed PK NN behavior to match MySQL Signed-off-by: Daylon Wilkins --- engine_test.go | 21 +++++++++++++++++++++ sql/parse/parse.go | 2 +- sql/parse/parse_test.go | 6 +++--- 3 files changed, 25 insertions(+), 4 deletions(-) diff --git a/engine_test.go b/engine_test.go index b022a3e6b..0b48aabea 100644 --- a/engine_test.go +++ b/engine_test.go @@ -2515,6 +2515,27 @@ func TestCreateTable(t *testing.T) { } require.Equal(s, testTable.Schema()) + + testQuery(t, e, + "CREATE TABLE t4(a INTEGER,"+ + "b TEXT NOT NULL,"+ + "c bool, primary key (a))", + []sql.Row(nil), + ) + + db, err = e.Catalog.Database("mydb") + require.NoError(err) + + testTable, ok = db.Tables()["t4"] + require.True(ok) + + s = sql.Schema{ + {Name: "a", Type: sql.Int32, Nullable: false, PrimaryKey: true, Source: "t4"}, + {Name: "b", Type: sql.Text, Nullable: false, PrimaryKey: false, Source: "t4"}, + {Name: "c", Type: sql.Uint8, Nullable: true, Source: "t4"}, + } + + require.Equal(s, testTable.Schema()) } func TestDropTable(t *testing.T) { diff --git a/sql/parse/parse.go b/sql/parse/parse.go index f4dfe6f0a..517337ef9 100644 --- a/sql/parse/parse.go +++ b/sql/parse/parse.go @@ -545,7 +545,7 @@ func getColumn(cd *sqlparser.ColumnDefinition, indexes []*sqlparser.IndexDefinit } return &sql.Column{ - Nullable: !bool(typ.NotNull), + Nullable: !isPkey && !bool(typ.NotNull), Type: internalTyp, Name: cd.Name.String(), PrimaryKey: isPkey, diff --git a/sql/parse/parse_test.go b/sql/parse/parse_test.go index b66dca943..ab58fe7ff 100644 --- a/sql/parse/parse_test.go +++ b/sql/parse/parse_test.go @@ -72,7 +72,7 @@ var fixtures = map[string]sql.Node{ sql.Schema{{ Name: "a", Type: sql.Int32, - Nullable: true, + Nullable: false, PrimaryKey: true, }, { Name: "b", @@ -87,12 +87,12 @@ var fixtures = map[string]sql.Node{ sql.Schema{{ Name: "a", Type: sql.Int32, - Nullable: true, + Nullable: false, PrimaryKey: true, }, { Name: "b", Type: sql.Text, - Nullable: true, + Nullable: false, PrimaryKey: true, }}, ), From e82528c8d65332342e9cbbb36b8125bb6c5bdec2 Mon Sep 17 00:00:00 2001 From: Zach Musgrave Date: Mon, 4 Nov 2019 11:37:24 -0800 Subject: [PATCH 20/42] Mostly formatting changes, about to attmpe to get rid of MergableIndex Signed-off-by: Zach Musgrave --- memory/ascend_index.go | 14 +++----------- memory/descend_index.go | 15 +++------------ memory/mergeable_index.go | 12 ++++++++++++ 3 files changed, 18 insertions(+), 23 deletions(-) diff --git a/memory/ascend_index.go b/memory/ascend_index.go index cb91eb83b..b796a7528 100755 --- a/memory/ascend_index.go +++ b/memory/ascend_index.go @@ -10,17 +10,9 @@ type AscendIndexLookup struct { Lt []interface{} } -func (l *AscendIndexLookup) ID() string { - return l.id -} - -func (l *AscendIndexLookup) GetUnions() []MergeableLookup { - return nil -} - -func (l *AscendIndexLookup) GetIntersections() []MergeableLookup { - return nil -} +func (l *AscendIndexLookup) ID() string { return l.id } +func (l *AscendIndexLookup) GetUnions() []MergeableLookup { return nil } +func (l *AscendIndexLookup) GetIntersections() []MergeableLookup { return nil } func (AscendIndexLookup) Values(sql.Partition) (sql.IndexValueIter, error) { return nil, nil diff --git a/memory/descend_index.go b/memory/descend_index.go index f1f921bdf..3a070bcb9 100755 --- a/memory/descend_index.go +++ b/memory/descend_index.go @@ -10,17 +10,9 @@ type DescendIndexLookup struct { Lte []interface{} } -func (l DescendIndexLookup) ID() string { - return l.id -} - -func (l DescendIndexLookup) GetUnions() []MergeableLookup { - return nil -} - -func (l DescendIndexLookup) GetIntersections() []MergeableLookup { - return nil -} +func (l DescendIndexLookup) ID() string { return l.id } +func (l DescendIndexLookup) GetUnions() []MergeableLookup { return nil } +func (l DescendIndexLookup) GetIntersections() []MergeableLookup { return nil } func (DescendIndexLookup) Values(sql.Partition) (sql.IndexValueIter, error) { return nil, nil @@ -44,7 +36,6 @@ func (l *DescendIndexLookup) Union(lookups ...sql.IndexLookup) sql.IndexLookup { return &MergedIndexLookup{ Unions: unions, } - } func (DescendIndexLookup) Difference(...sql.IndexLookup) sql.IndexLookup { diff --git a/memory/mergeable_index.go b/memory/mergeable_index.go index 357173b08..8481e9e93 100755 --- a/memory/mergeable_index.go +++ b/memory/mergeable_index.go @@ -107,6 +107,8 @@ var _ sql.SetOperations = (*MergeableIndexLookup)(nil) func (i *MergeableIndexLookup) ID() string { return strings.Join(i.Indexes(), ",") } func (i *MergeableIndexLookup) GetUnions() []MergeableLookup { return nil } func (i *MergeableIndexLookup) GetIntersections() []MergeableLookup { return nil } +func (i *MergeableIndexLookup) ClearUnions() { } +func (i *MergeableIndexLookup) ClearIntersections() { } func (i *MergeableIndexLookup) IsMergeable(lookup sql.IndexLookup) bool { _, ok := lookup.(MergeableLookup) @@ -148,6 +150,7 @@ func intersection(left MergeableLookup, lookups ...sql.IndexLookup) sql.IndexLoo intersections = append(intersections, left) for _, lookup := range lookups { intersections = append(intersections, lookup.(MergeableLookup)) + intersections = append(intersections, lookup.(MergeableLookup).GetIntersections()...) } return &MergedIndexLookup{ @@ -164,6 +167,7 @@ func union(left MergeableLookup, lookups ...sql.IndexLookup) sql.IndexLookup { unions = append(unions, left) for _, lookup := range lookups { unions = append(unions, lookup.(MergeableLookup)) + unions = append(unions, lookup.(MergeableLookup).GetUnions()...) } return &MergedIndexLookup{ @@ -187,6 +191,14 @@ func (m *MergedIndexLookup) GetIntersections() []MergeableLookup { return m.Intersections } +func (m *MergedIndexLookup) ClearUnions() { + m.Unions = nil +} + +func (m *MergedIndexLookup) ClearIntersections() { + m.Intersections = nil +} + func (m *MergedIndexLookup) Intersection(lookups ...sql.IndexLookup) sql.IndexLookup { return intersection(m, lookups...) } From 8c0102be1d82cc555784b194b688fd673b6382d1 Mon Sep 17 00:00:00 2001 From: Zach Musgrave Date: Mon, 4 Nov 2019 12:15:09 -0800 Subject: [PATCH 21/42] Better version of union and intersection. Still have bugs Signed-off-by: Zach Musgrave --- memory/ascend_index.go | 10 +- memory/descend_index.go | 10 +- memory/mergeable_index.go | 57 ++-- memory/negative_index.go | 22 +- sql/analyzer/assign_indexes_test.go | 462 ++++++++++++++-------------- 5 files changed, 262 insertions(+), 299 deletions(-) diff --git a/memory/ascend_index.go b/memory/ascend_index.go index b796a7528..ad70485db 100755 --- a/memory/ascend_index.go +++ b/memory/ascend_index.go @@ -27,15 +27,7 @@ func (l *AscendIndexLookup) IsMergeable(sql.IndexLookup) bool { } func (l *AscendIndexLookup) Union(lookups ...sql.IndexLookup) sql.IndexLookup { - var unions []MergeableLookup - unions = append(unions, l) - for _, idx := range lookups { - unions = append(unions, idx.(MergeableLookup)) - } - - return &MergedIndexLookup{ - Unions: unions, - } + return union(l, lookups...) } func (AscendIndexLookup) Difference(...sql.IndexLookup) sql.IndexLookup { diff --git a/memory/descend_index.go b/memory/descend_index.go index 3a070bcb9..c3f51b3b9 100755 --- a/memory/descend_index.go +++ b/memory/descend_index.go @@ -27,15 +27,7 @@ func (l *DescendIndexLookup) IsMergeable(sql.IndexLookup) bool { } func (l *DescendIndexLookup) Union(lookups ...sql.IndexLookup) sql.IndexLookup { - var unions []MergeableLookup - unions = append(unions, l) - for _, idx := range lookups { - unions = append(unions, idx.(MergeableLookup)) - } - - return &MergedIndexLookup{ - Unions: unions, - } + return union(l, lookups...) } func (DescendIndexLookup) Difference(...sql.IndexLookup) sql.IndexLookup { diff --git a/memory/mergeable_index.go b/memory/mergeable_index.go index 8481e9e93..c82fa5418 100755 --- a/memory/mergeable_index.go +++ b/memory/mergeable_index.go @@ -86,8 +86,8 @@ func (i MergeableDummyIndex) Table() string { return i.TableName } type MergeableLookup interface { ID() string - GetUnions() []MergeableLookup - GetIntersections() []MergeableLookup + // GetUnions() []MergeableLookup + // GetIntersections() []MergeableLookup } // ExpressionsIndex is an index made out of one or more expressions (usually field expressions) @@ -105,8 +105,8 @@ var _ sql.Mergeable = (*MergeableIndexLookup)(nil) var _ sql.SetOperations = (*MergeableIndexLookup)(nil) func (i *MergeableIndexLookup) ID() string { return strings.Join(i.Indexes(), ",") } -func (i *MergeableIndexLookup) GetUnions() []MergeableLookup { return nil } -func (i *MergeableIndexLookup) GetIntersections() []MergeableLookup { return nil } +// func (i *MergeableIndexLookup) GetUnions() []MergeableLookup { return nil } +// func (i *MergeableIndexLookup) GetIntersections() []MergeableLookup { return nil } func (i *MergeableIndexLookup) ClearUnions() { } func (i *MergeableIndexLookup) ClearIntersections() { } @@ -145,16 +145,9 @@ func (i *MergeableIndexLookup) Intersection(lookups ...sql.IndexLookup) sql.Inde return intersection(i, lookups...) } -func intersection(left MergeableLookup, lookups ...sql.IndexLookup) sql.IndexLookup { - var intersections []MergeableLookup - intersections = append(intersections, left) - for _, lookup := range lookups { - intersections = append(intersections, lookup.(MergeableLookup)) - intersections = append(intersections, lookup.(MergeableLookup).GetIntersections()...) - } - +func intersection(left sql.IndexLookup, lookups ...sql.IndexLookup) sql.IndexLookup { return &MergedIndexLookup{ - Intersections: intersections, + Intersections: merge(left, lookups), } } @@ -162,35 +155,37 @@ func (i *MergeableIndexLookup) Union(lookups ...sql.IndexLookup) sql.IndexLookup return union(i, lookups...) } -func union(left MergeableLookup, lookups ...sql.IndexLookup) sql.IndexLookup { - var unions []MergeableLookup - unions = append(unions, left) - for _, lookup := range lookups { - unions = append(unions, lookup.(MergeableLookup)) - unions = append(unions, lookup.(MergeableLookup).GetUnions()...) +func union(left sql.IndexLookup, lookups ...sql.IndexLookup) sql.IndexLookup { + return &MergedIndexLookup{ + Unions: merge(left, lookups), } +} - return &MergedIndexLookup{ - Unions: unions, +func merge(left sql.IndexLookup, lookups []sql.IndexLookup) []sql.IndexLookup { + var merged []sql.IndexLookup + var allLookups []sql.IndexLookup + allLookups = append(allLookups, left) + allLookups = append(allLookups, lookups...) + for _, lookup := range allLookups { + if mil, ok := lookup.(*MergedIndexLookup); ok { + merged = append(merged, mil.Intersections...) + merged = append(merged, mil.Unions...) + } else { + merged = append(merged, lookup) + } } + + return merged } type MergedIndexLookup struct { - Unions []MergeableLookup - Intersections []MergeableLookup + Unions []sql.IndexLookup + Intersections []sql.IndexLookup } var _ sql.Mergeable = (*MergedIndexLookup)(nil) var _ sql.SetOperations = (*MergedIndexLookup)(nil) -func (m *MergedIndexLookup) GetUnions() []MergeableLookup { - return m.Unions -} - -func (m *MergedIndexLookup) GetIntersections() []MergeableLookup { - return m.Intersections -} - func (m *MergedIndexLookup) ClearUnions() { m.Unions = nil } diff --git a/memory/negative_index.go b/memory/negative_index.go index 640dfcacd..7030594b3 100755 --- a/memory/negative_index.go +++ b/memory/negative_index.go @@ -9,8 +9,6 @@ type NegateIndexLookup struct { } func (l *NegateIndexLookup) ID() string { return "not " + l.Lookup.ID() } -func (l *NegateIndexLookup) GetUnions() []MergeableLookup { return l.Lookup.GetUnions() } -func (l *NegateIndexLookup) GetIntersections() []MergeableLookup { return l.Lookup.GetIntersections() } func (*NegateIndexLookup) Values(sql.Partition) (sql.IndexValueIter, error) { return nil, nil @@ -25,15 +23,7 @@ func (*NegateIndexLookup) IsMergeable(sql.IndexLookup) bool { } func (l *NegateIndexLookup) Union(lookups ...sql.IndexLookup) sql.IndexLookup { - var unions []MergeableLookup - unions = append(unions, l) - for _, idx := range lookups { - unions = append(unions, idx.(MergeableLookup)) - } - - return &MergedIndexLookup{ - Unions: unions, - } + return union(l, lookups...) } func (*NegateIndexLookup) Difference(...sql.IndexLookup) sql.IndexLookup { @@ -41,13 +31,5 @@ func (*NegateIndexLookup) Difference(...sql.IndexLookup) sql.IndexLookup { } func (l *NegateIndexLookup) Intersection(indexes ...sql.IndexLookup) sql.IndexLookup { - var intersections []MergeableLookup - intersections = append(intersections, l) - for _, idx := range indexes { - intersections = append(intersections, idx.(MergeableLookup)) - } - - return &MergedIndexLookup{ - Intersections: intersections, - } + return intersection(l, indexes...) } \ No newline at end of file diff --git a/sql/analyzer/assign_indexes_test.go b/sql/analyzer/assign_indexes_test.go index aaa3c98ad..9d09e66dc 100644 --- a/sql/analyzer/assign_indexes_test.go +++ b/sql/analyzer/assign_indexes_test.go @@ -231,225 +231,225 @@ func TestGetIndexes(t *testing.T) { expected map[string]*indexLookup ok bool }{ - // { - // eq( - // col(0, "t1", "bar"), - // col(1, "t1", "baz"), - // ), - // map[string]*indexLookup{}, - // true, - // }, - // { - // eq( - // col(0, "t1", "bar"), - // lit(1), - // ), - // map[string]*indexLookup{ - // "t1": &indexLookup{ - // mergeableIndexLookup("t1", "bar", 0, int64(1)), - // []sql.Index{indexes[0]}, - // }, - // }, - // true, - // }, - // { - // or( - // eq( - // col(0, "t1", "bar"), - // lit(1), - // ), - // eq( - // col(0, "t1", "bar"), - // lit(2), - // ), - // ), - // map[string]*indexLookup{ - // "t1": &indexLookup{ - // &memory.MergedIndexLookup{ - // Union: []memory.MergeableLookup{ - // mergeableIndexLookup("t1", "bar", 0, int64(1)), - // mergeableIndexLookup("t1", "bar", 0, int64(2)), - // }, - // }, - // []sql.Index{ - // indexes[0], - // indexes[0], - // }, - // }, - // }, - // true, - // }, - // { - // or( - // eq( - // col(0, "t3", "foo"), - // lit(1), - // ), - // eq( - // col(0, "t3", "foo"), - // lit(2), - // ), - // ), - // nil, - // true, - // }, - // { - // in( - // col(0, "t3", "foo"), - // tuple(lit(1), lit(2)), - // ), - // nil, - // true, - // }, - // { - // in( - // col(0, "t1", "bar"), - // tuple(lit(1), lit(2)), - // ), - // map[string]*indexLookup{ - // "t1": &indexLookup{ - // &memory.MergedIndexLookup{ - // Union: []memory.MergeableLookup{ - // mergeableIndexLookup("t1", "bar", 0, int64(1)), - // mergeableIndexLookup("t1", "bar", 0, int64(2)), - // }, - // }, - // []sql.Index{ - // indexes[0], - // }, - // }, - // }, - // true, - // }, - // { - // and( - // eq( - // col(0, "t1", "bar"), - // lit(1), - // ), - // eq( - // col(0, "t1", "bar"), - // lit(2), - // ), - // ), - // map[string]*indexLookup{ - // "t1": &indexLookup{ - // &memory.MergedIndexLookup{ - // Intersection: []memory.MergeableLookup{ - // mergeableIndexLookup("t1", "bar", 0, int64(1)), - // mergeableIndexLookup("t1", "bar", 0, int64(2)), - // }, - // }, - // []sql.Index{ - // indexes[0], - // indexes[0], - // }, - // }, - // }, - // true, - // }, - // { - // and( - // or( - // eq( - // col(0, "t1", "bar"), - // lit(1), - // ), - // eq( - // col(0, "t1", "bar"), - // lit(2), - // ), - // ), - // or( - // eq( - // col(0, "t1", "bar"), - // lit(3), - // ), - // eq( - // col(0, "t1", "bar"), - // lit(4), - // ), - // ), - // ), - // map[string]*indexLookup{ - // "t1": &indexLookup{ - // &memory.MergedIndexLookup{ - // Intersection: []memory.MergeableLookup { - // &memory.MergedIndexLookup{ - // Union: []memory.MergeableLookup{ - // mergeableIndexLookup("t1", "bar", 0, int64(1)), - // mergeableIndexLookup("t1", "bar", 0, int64(2)), - // }, - // }, - // &memory.MergedIndexLookup{ - // Union: []memory.MergeableLookup{ - // mergeableIndexLookup("t1", "bar", 0, int64(3)), - // mergeableIndexLookup("t1", "bar", 0, int64(4)), - // }, - // }, - // }, - // }, - // []sql.Index{ - // indexes[0], - // indexes[0], - // indexes[0], - // indexes[0], - // }, - // }, - // }, - // true, - // }, - // { - // or( - // or( - // eq( - // col(0, "t1", "bar"), - // lit(1), - // ), - // eq( - // col(0, "t1", "bar"), - // lit(2), - // ), - // ), - // or( - // eq( - // col(0, "t1", "bar"), - // lit(3), - // ), - // eq( - // col(0, "t1", "bar"), - // lit(4), - // ), - // ), - // ), - // map[string]*indexLookup{ - // "t1": &indexLookup{ - // &memory.MergedIndexLookup{ - // Union: []memory.MergeableLookup { - // &memory.MergedIndexLookup{ - // Union: []memory.MergeableLookup{ - // mergeableIndexLookup("t1", "bar", 0, int64(1)), - // mergeableIndexLookup("t1", "bar", 0, int64(2)), - // }, - // }, - // &memory.MergedIndexLookup{ - // Union: []memory.MergeableLookup{ - // mergeableIndexLookup("t1", "bar", 0, int64(3)), - // mergeableIndexLookup("t1", "bar", 0, int64(4)), - // }, - // }, - // }, - // }, - // []sql.Index{ - // indexes[0], - // indexes[0], - // indexes[0], - // indexes[0], - // }, - // }, - // }, - // true, - // }, + { + eq( + col(0, "t1", "bar"), + col(1, "t1", "baz"), + ), + map[string]*indexLookup{}, + true, + }, + { + eq( + col(0, "t1", "bar"), + lit(1), + ), + map[string]*indexLookup{ + "t1": &indexLookup{ + mergeableIndexLookup("t1", "bar", 0, int64(1)), + []sql.Index{indexes[0]}, + }, + }, + true, + }, + { + or( + eq( + col(0, "t1", "bar"), + lit(1), + ), + eq( + col(0, "t1", "bar"), + lit(2), + ), + ), + map[string]*indexLookup{ + "t1": &indexLookup{ + &memory.MergedIndexLookup{ + Unions: []sql.IndexLookup{ + mergeableIndexLookup("t1", "bar", 0, int64(1)), + mergeableIndexLookup("t1", "bar", 0, int64(2)), + }, + }, + []sql.Index{ + indexes[0], + indexes[0], + }, + }, + }, + true, + }, + { + or( + eq( + col(0, "t3", "foo"), + lit(1), + ), + eq( + col(0, "t3", "foo"), + lit(2), + ), + ), + nil, + true, + }, + { + in( + col(0, "t3", "foo"), + tuple(lit(1), lit(2)), + ), + nil, + true, + }, + { + in( + col(0, "t1", "bar"), + tuple(lit(1), lit(2)), + ), + map[string]*indexLookup{ + "t1": &indexLookup{ + &memory.MergedIndexLookup{ + Unions: []sql.IndexLookup{ + mergeableIndexLookup("t1", "bar", 0, int64(1)), + mergeableIndexLookup("t1", "bar", 0, int64(2)), + }, + }, + []sql.Index{ + indexes[0], + }, + }, + }, + true, + }, + { + and( + eq( + col(0, "t1", "bar"), + lit(1), + ), + eq( + col(0, "t1", "bar"), + lit(2), + ), + ), + map[string]*indexLookup{ + "t1": &indexLookup{ + &memory.MergedIndexLookup{ + Intersections: []sql.IndexLookup{ + mergeableIndexLookup("t1", "bar", 0, int64(1)), + mergeableIndexLookup("t1", "bar", 0, int64(2)), + }, + }, + []sql.Index{ + indexes[0], + indexes[0], + }, + }, + }, + true, + }, + { + and( + or( + eq( + col(0, "t1", "bar"), + lit(1), + ), + eq( + col(0, "t1", "bar"), + lit(2), + ), + ), + or( + eq( + col(0, "t1", "bar"), + lit(3), + ), + eq( + col(0, "t1", "bar"), + lit(4), + ), + ), + ), + map[string]*indexLookup{ + "t1": &indexLookup{ + &memory.MergedIndexLookup{ + Intersections: []sql.IndexLookup { + &memory.MergedIndexLookup{ + Unions: []sql.IndexLookup{ + mergeableIndexLookup("t1", "bar", 0, int64(1)), + mergeableIndexLookup("t1", "bar", 0, int64(2)), + }, + }, + &memory.MergedIndexLookup{ + Unions: []sql.IndexLookup{ + mergeableIndexLookup("t1", "bar", 0, int64(3)), + mergeableIndexLookup("t1", "bar", 0, int64(4)), + }, + }, + }, + }, + []sql.Index{ + indexes[0], + indexes[0], + indexes[0], + indexes[0], + }, + }, + }, + true, + }, + { + or( + or( + eq( + col(0, "t1", "bar"), + lit(1), + ), + eq( + col(0, "t1", "bar"), + lit(2), + ), + ), + or( + eq( + col(0, "t1", "bar"), + lit(3), + ), + eq( + col(0, "t1", "bar"), + lit(4), + ), + ), + ), + map[string]*indexLookup{ + "t1": &indexLookup{ + &memory.MergedIndexLookup{ + Unions: []sql.IndexLookup { + &memory.MergedIndexLookup{ + Unions: []sql.IndexLookup{ + mergeableIndexLookup("t1", "bar", 0, int64(1)), + mergeableIndexLookup("t1", "bar", 0, int64(2)), + }, + }, + &memory.MergedIndexLookup{ + Unions: []sql.IndexLookup{ + mergeableIndexLookup("t1", "bar", 0, int64(3)), + mergeableIndexLookup("t1", "bar", 0, int64(4)), + }, + }, + }, + }, + []sql.Index{ + indexes[0], + indexes[0], + indexes[0], + indexes[0], + }, + }, + }, + true, + }, { in( col(0, "t1", "bar"), @@ -458,7 +458,7 @@ func TestGetIndexes(t *testing.T) { map[string]*indexLookup{ "t1": &indexLookup{ &memory.MergedIndexLookup{ - Unions: []memory.MergeableLookup{ + Unions: []sql.IndexLookup{ mergeableIndexLookup("t1", "bar", 0, int64(1)), mergeableIndexLookup("t1", "bar", 0, int64(2)), mergeableIndexLookup("t1", "bar", 0, int64(3)), @@ -541,16 +541,16 @@ func TestGetIndexes(t *testing.T) { }, "t2": &indexLookup{ &memory.MergedIndexLookup{ - Unions: []memory.MergeableLookup{ + Unions: []sql.IndexLookup{ &memory.MergeableIndexLookup{Key: []interface{}{5}}, &memory.MergedIndexLookup{ - Intersections: []memory.MergeableLookup{ + Intersections: []sql.IndexLookup{ &memory.MergeableIndexLookup{Key: []interface{}{1}}, &memory.MergedIndexLookup{ - Intersections: []memory.MergeableLookup{ + Intersections: []sql.IndexLookup{ &memory.MergeableIndexLookup{Key: []interface{}{4}}, &memory.MergedIndexLookup{ - Intersections: []memory.MergeableLookup{ + Intersections: []sql.IndexLookup{ &memory.MergeableIndexLookup{Key: []interface{}{2}}, }, }, @@ -629,7 +629,7 @@ func TestGetIndexes(t *testing.T) { map[string]*indexLookup{ "t1": &indexLookup{ &memory.MergedIndexLookup { - Intersections: []memory.MergeableLookup{ + Intersections: []sql.IndexLookup{ &memory.AscendIndexLookup{ Gte: []interface{}{int64(1)}, Lt: []interface{}{int64(5)}, @@ -743,7 +743,7 @@ func TestGetIndexes(t *testing.T) { "t1": &indexLookup{ &memory.NegateIndexLookup { Lookup: &memory.MergedIndexLookup{ - Intersections: []memory.MergeableLookup { + Intersections: []sql.IndexLookup { &memory.MergeableIndexLookup{ Key: []interface{}{int64(10)}, }, @@ -778,7 +778,7 @@ func TestGetIndexes(t *testing.T) { "t1": &indexLookup{ &memory.NegateIndexLookup { Lookup: &memory.MergedIndexLookup{ - Unions: []memory.MergeableLookup { + Unions: []sql.IndexLookup { &memory.MergeableIndexLookup{ Key: []interface{}{int64(10)}, }, @@ -840,7 +840,7 @@ func TestGetIndexes(t *testing.T) { "t1": &indexLookup{ &memory.NegateIndexLookup{ Lookup: &memory.MergedIndexLookup{ - Unions: []memory.MergeableLookup { + Unions: []sql.IndexLookup { &memory.MergeableIndexLookup{ Key: []interface{}{1}, }, @@ -873,6 +873,7 @@ func TestGetIndexes(t *testing.T) { a := NewDefault(catalog) + var i int for _, tt := range testCases { t.Run(tt.expr.String(), func(t *testing.T) { require := require.New(t) @@ -884,6 +885,7 @@ func TestGetIndexes(t *testing.T) { } else { require.Error(err) } + i++ }) } } @@ -989,7 +991,7 @@ func TestGetMultiColumnIndexes(t *testing.T) { }, "t4": &indexLookup{ &memory.MergedIndexLookup{ - Intersections: []memory.MergeableLookup{ + Intersections: []sql.IndexLookup{ &memory.AscendIndexLookup{ Gte: []interface{}{int64(1), int64(2)}, Lt: []interface{}{int64(6), int64(5)}, @@ -1127,7 +1129,7 @@ func TestIndexesIntersection(t *testing.T) { "a": &indexLookup{&memory.MergeableIndexLookup{Key: []interface{}{"a"}}, nil}, "b": &indexLookup{ &memory.MergedIndexLookup { - Intersections: []memory.MergeableLookup { + Intersections: []sql.IndexLookup { &memory.MergeableIndexLookup{ Key: []interface{}{"b"}, }, From 78e8b081d4b3e392083fb3ca0d6c3cdf38b51b31 Mon Sep 17 00:00:00 2001 From: Zach Musgrave Date: Mon, 4 Nov 2019 12:34:15 -0800 Subject: [PATCH 22/42] Separated out intersection and union logic again Signed-off-by: Zach Musgrave --- memory/mergeable_index.go | 44 +++++++++++++++++++++++---------------- 1 file changed, 26 insertions(+), 18 deletions(-) diff --git a/memory/mergeable_index.go b/memory/mergeable_index.go index c82fa5418..e10a0e55d 100755 --- a/memory/mergeable_index.go +++ b/memory/mergeable_index.go @@ -145,9 +145,25 @@ func (i *MergeableIndexLookup) Intersection(lookups ...sql.IndexLookup) sql.Inde return intersection(i, lookups...) } +// Intersects the lookups given together, collapsing redundant layers of intersections for lookups that have previously +// been merged. E.g. merging a MergeableIndexLookup with a MergedIndexLookup that has 2 intersections will return a +// MergedIndexLookup with 3 lookups intersected: the left param and the two intersected lookups from the +// MergedIndexLookup. func intersection(left sql.IndexLookup, lookups ...sql.IndexLookup) sql.IndexLookup { + var merged []sql.IndexLookup + var allLookups []sql.IndexLookup + allLookups = append(allLookups, left) + allLookups = append(allLookups, lookups...) + for _, lookup := range allLookups { + if mil, ok := lookup.(*MergedIndexLookup); ok && len(mil.Intersections) > 0 { + merged = append(merged, mil.Intersections...) + } else { + merged = append(merged, lookup) + } + } + return &MergedIndexLookup{ - Intersections: merge(left, lookups), + Intersections: merged, } } @@ -155,29 +171,29 @@ func (i *MergeableIndexLookup) Union(lookups ...sql.IndexLookup) sql.IndexLookup return union(i, lookups...) } +// Unions the lookups given together, collapsing redundant layers of unions for lookups that have previously been +// merged. E.g. merging a MergeableIndexLookup with a MergedIndexLookup that has 2 unions will return a +// MergedIndexLookup with 3 lookups unioned: the left param and the two unioned lookups from the MergedIndexLookup. func union(left sql.IndexLookup, lookups ...sql.IndexLookup) sql.IndexLookup { - return &MergedIndexLookup{ - Unions: merge(left, lookups), - } -} - -func merge(left sql.IndexLookup, lookups []sql.IndexLookup) []sql.IndexLookup { var merged []sql.IndexLookup var allLookups []sql.IndexLookup allLookups = append(allLookups, left) allLookups = append(allLookups, lookups...) for _, lookup := range allLookups { - if mil, ok := lookup.(*MergedIndexLookup); ok { - merged = append(merged, mil.Intersections...) + if mil, ok := lookup.(*MergedIndexLookup); ok && len(mil.Unions) > 0 { merged = append(merged, mil.Unions...) } else { merged = append(merged, lookup) } } - return merged + return &MergedIndexLookup{ + Unions: merged, + } } +// An index lookup that has been merged with another. +// Exactly one of the Unions or Intersections fields should be set. type MergedIndexLookup struct { Unions []sql.IndexLookup Intersections []sql.IndexLookup @@ -186,14 +202,6 @@ type MergedIndexLookup struct { var _ sql.Mergeable = (*MergedIndexLookup)(nil) var _ sql.SetOperations = (*MergedIndexLookup)(nil) -func (m *MergedIndexLookup) ClearUnions() { - m.Unions = nil -} - -func (m *MergedIndexLookup) ClearIntersections() { - m.Intersections = nil -} - func (m *MergedIndexLookup) Intersection(lookups ...sql.IndexLookup) sql.IndexLookup { return intersection(m, lookups...) } From b3534e37c259cc8f70d19460dd9ff39b7ae468c9 Mon Sep 17 00:00:00 2001 From: Zach Musgrave Date: Mon, 4 Nov 2019 12:36:32 -0800 Subject: [PATCH 23/42] Updated expected test result for squashed union logic Signed-off-by: Zach Musgrave --- sql/analyzer/assign_indexes_test.go | 18 +++++------------- 1 file changed, 5 insertions(+), 13 deletions(-) diff --git a/sql/analyzer/assign_indexes_test.go b/sql/analyzer/assign_indexes_test.go index 9d09e66dc..9da60cff5 100644 --- a/sql/analyzer/assign_indexes_test.go +++ b/sql/analyzer/assign_indexes_test.go @@ -425,19 +425,11 @@ func TestGetIndexes(t *testing.T) { map[string]*indexLookup{ "t1": &indexLookup{ &memory.MergedIndexLookup{ - Unions: []sql.IndexLookup { - &memory.MergedIndexLookup{ - Unions: []sql.IndexLookup{ - mergeableIndexLookup("t1", "bar", 0, int64(1)), - mergeableIndexLookup("t1", "bar", 0, int64(2)), - }, - }, - &memory.MergedIndexLookup{ - Unions: []sql.IndexLookup{ - mergeableIndexLookup("t1", "bar", 0, int64(3)), - mergeableIndexLookup("t1", "bar", 0, int64(4)), - }, - }, + Unions: []sql.IndexLookup{ + mergeableIndexLookup("t1", "bar", 0, int64(1)), + mergeableIndexLookup("t1", "bar", 0, int64(2)), + mergeableIndexLookup("t1", "bar", 0, int64(3)), + mergeableIndexLookup("t1", "bar", 0, int64(4)), }, }, []sql.Index{ From 5ecc6bb5183a51d242b80358982d8ca287f1a40d Mon Sep 17 00:00:00 2001 From: Zach Musgrave Date: Mon, 4 Nov 2019 13:17:00 -0800 Subject: [PATCH 24/42] Fixed another test (error in expected value declaration) Signed-off-by: Zach Musgrave --- sql/analyzer/assign_indexes_test.go | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/sql/analyzer/assign_indexes_test.go b/sql/analyzer/assign_indexes_test.go index 9da60cff5..4dbcfd65c 100644 --- a/sql/analyzer/assign_indexes_test.go +++ b/sql/analyzer/assign_indexes_test.go @@ -491,7 +491,16 @@ func TestGetIndexes(t *testing.T) { []sql.Index{indexes[0]}, }, "t2": &indexLookup{ - mergeableIndexLookup("t2", "foo", 0, int64(1), int64(2)), + &memory.MergeableIndexLookup{ + Key: []interface{}{int64(1), int64(2)}, + Index: memory.MergeableDummyIndex{ + TableName: "t2", + Exprs: []sql.Expression{ + col(0, "t2", "foo"), + col(0, "t2", "bar"), + }, + }, + }, []sql.Index{indexes[1]}, }, }, From 1bd5bc9f634ceba20a9df9e29dba0ed7aaca80b8 Mon Sep 17 00:00:00 2001 From: Zach Musgrave Date: Mon, 4 Nov 2019 13:35:17 -0800 Subject: [PATCH 25/42] One more fixed test Signed-off-by: Zach Musgrave --- sql/analyzer/assign_indexes_test.go | 23 +++++++++-------------- 1 file changed, 9 insertions(+), 14 deletions(-) diff --git a/sql/analyzer/assign_indexes_test.go b/sql/analyzer/assign_indexes_test.go index 4dbcfd65c..b518e148c 100644 --- a/sql/analyzer/assign_indexes_test.go +++ b/sql/analyzer/assign_indexes_test.go @@ -537,25 +537,20 @@ func TestGetIndexes(t *testing.T) { ), map[string]*indexLookup{ "t1": &indexLookup{ - &memory.MergeableIndexLookup{Key: []interface{}{3}}, + mergeableIndexLookup("t1", "bar", 0, int64(3)), []sql.Index{indexes[0]}, }, "t2": &indexLookup{ &memory.MergedIndexLookup{ Unions: []sql.IndexLookup{ - &memory.MergeableIndexLookup{Key: []interface{}{5}}, - &memory.MergedIndexLookup{ - Intersections: []sql.IndexLookup{ - &memory.MergeableIndexLookup{Key: []interface{}{1}}, - &memory.MergedIndexLookup{ - Intersections: []sql.IndexLookup{ - &memory.MergeableIndexLookup{Key: []interface{}{4}}, - &memory.MergedIndexLookup{ - Intersections: []sql.IndexLookup{ - &memory.MergeableIndexLookup{Key: []interface{}{2}}, - }, - }, - }, + mergeableIndexLookup("t2", "bar", 0, int64(5)), + &memory.MergeableIndexLookup{ + Key: []interface{}{int64(1), int64(2)}, + Index: memory.MergeableDummyIndex{ + TableName: "t2", + Exprs: []sql.Expression{ + col(0, "t2", "foo"), + col(0, "t2", "bar"), }, }, }, From 966fd5a78844198cda6e45c2001b9c405b3bf454 Mon Sep 17 00:00:00 2001 From: Zach Musgrave Date: Mon, 4 Nov 2019 13:39:56 -0800 Subject: [PATCH 26/42] One more fixed test (misunderstood the semantics of union / intersection for between operation) Signed-off-by: Zach Musgrave --- sql/analyzer/assign_indexes_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/analyzer/assign_indexes_test.go b/sql/analyzer/assign_indexes_test.go index b518e148c..68ff67cc4 100644 --- a/sql/analyzer/assign_indexes_test.go +++ b/sql/analyzer/assign_indexes_test.go @@ -625,7 +625,7 @@ func TestGetIndexes(t *testing.T) { map[string]*indexLookup{ "t1": &indexLookup{ &memory.MergedIndexLookup { - Intersections: []sql.IndexLookup{ + Unions: []sql.IndexLookup{ &memory.AscendIndexLookup{ Gte: []interface{}{int64(1)}, Lt: []interface{}{int64(5)}, From aee8179e7373b82f72de37600ad86dc639c69aef Mon Sep 17 00:00:00 2001 From: Zach Musgrave Date: Mon, 4 Nov 2019 13:58:08 -0800 Subject: [PATCH 27/42] Fixed remaining tests for assign_indexes Signed-off-by: Zach Musgrave --- sql/analyzer/assign_indexes.go | 4 +- sql/analyzer/assign_indexes_test.go | 66 +++++++++++++---------------- 2 files changed, 32 insertions(+), 38 deletions(-) diff --git a/sql/analyzer/assign_indexes.go b/sql/analyzer/assign_indexes.go index 6eb5f686f..932c4be6d 100644 --- a/sql/analyzer/assign_indexes.go +++ b/sql/analyzer/assign_indexes.go @@ -337,6 +337,9 @@ func betweenIndexLookup(index sql.Index, upper, lower []interface{}) (sql.IndexL m, ok := ascendLookup.(sql.Mergeable) if ok && m.IsMergeable(descendLookup) { + // TODO: two bugs here + // 1) Mergeable and SetOperations are separate interfaces, so a bad integrator could generate a type assertion error here + // 2) This should be Intersection, not Union. Not fixing yet because the test index doesn't implement Intersection. return ascendLookup.(sql.SetOperations).Union(descendLookup), nil } } @@ -503,7 +506,6 @@ func getNegatedIndexes(a *Analyzer, not *expression.Not, aliases map[string]sql. return getIndexes(or, aliases, a) default: return nil, nil - } } diff --git a/sql/analyzer/assign_indexes_test.go b/sql/analyzer/assign_indexes_test.go index 68ff67cc4..d43ed4324 100644 --- a/sql/analyzer/assign_indexes_test.go +++ b/sql/analyzer/assign_indexes_test.go @@ -651,7 +651,7 @@ func TestGetIndexes(t *testing.T) { map[string]*indexLookup{ "t1": &indexLookup{ &memory.NegateIndexLookup{ - // TODO: fix + Lookup: mergeableIndexLookup("t1", "bar", 0, int64(1)), }, []sql.Index{indexes[0]}, }, @@ -737,15 +737,13 @@ func TestGetIndexes(t *testing.T) { ), map[string]*indexLookup{ "t1": &indexLookup{ - &memory.NegateIndexLookup { - Lookup: &memory.MergedIndexLookup{ - Intersections: []sql.IndexLookup { - &memory.MergeableIndexLookup{ - Key: []interface{}{int64(10)}, - }, - &memory.MergeableIndexLookup{ - Key: []interface{}{int64(11)}, - }, + &memory.MergedIndexLookup{ + Unions: []sql.IndexLookup{ + &memory.NegateIndexLookup{ + Lookup: mergeableIndexLookup("t1", "bar", 0, int64(10)), + }, + &memory.NegateIndexLookup{ + Lookup: mergeableIndexLookup("t1", "bar", 0, int64(11)), }, }, }, @@ -772,15 +770,13 @@ func TestGetIndexes(t *testing.T) { ), map[string]*indexLookup{ "t1": &indexLookup{ - &memory.NegateIndexLookup { - Lookup: &memory.MergedIndexLookup{ - Unions: []sql.IndexLookup { - &memory.MergeableIndexLookup{ - Key: []interface{}{int64(10)}, - }, - &memory.MergeableIndexLookup{ - Key: []interface{}{int64(11)}, - }, + &memory.MergedIndexLookup{ + Intersections: []sql.IndexLookup{ + &memory.NegateIndexLookup{ + Lookup: mergeableIndexLookup("t1", "bar", 0, int64(10)), + }, + &memory.NegateIndexLookup{ + Lookup: mergeableIndexLookup("t1", "bar", 0, int64(11)), }, }, }, @@ -811,9 +807,7 @@ func TestGetIndexes(t *testing.T) { map[string]*indexLookup{ "t2": &indexLookup{ &memory.NegateIndexLookup{ - Lookup: &memory.MergeableIndexLookup{ - Key: []interface{}{110}, - }, + Lookup: mergeableIndexLookup("t2", "bar", 0, int64(110)), }, []sql.Index{ indexes[2], @@ -834,21 +828,19 @@ func TestGetIndexes(t *testing.T) { ), map[string]*indexLookup{ "t1": &indexLookup{ - &memory.NegateIndexLookup{ - Lookup: &memory.MergedIndexLookup{ - Unions: []sql.IndexLookup { - &memory.MergeableIndexLookup{ - Key: []interface{}{1}, - }, - &memory.MergeableIndexLookup{ - Key: []interface{}{2}, - }, - &memory.MergeableIndexLookup{ - Key: []interface{}{3}, - }, - &memory.MergeableIndexLookup{ - Key: []interface{}{4}, - }, + &memory.MergedIndexLookup{ + Intersections: []sql.IndexLookup{ + &memory.NegateIndexLookup{ + Lookup: mergeableIndexLookup("t1", "bar", 0, int64(1)), + }, + &memory.NegateIndexLookup{ + Lookup: mergeableIndexLookup("t1", "bar", 0, int64(2)), + }, + &memory.NegateIndexLookup{ + Lookup: mergeableIndexLookup("t1", "bar", 0, int64(3)), + }, + &memory.NegateIndexLookup{ + Lookup: mergeableIndexLookup("t1", "bar", 0, int64(4)), }, }, }, From 8ccf6394754215625757fb1f1baaa9cfb1b42f4e Mon Sep 17 00:00:00 2001 From: Zach Musgrave Date: Mon, 4 Nov 2019 16:00:19 -0800 Subject: [PATCH 28/42] Added a second kind of index driver to engine tests. Still getting all the pieces together Signed-off-by: Zach Musgrave --- engine_test.go | 141 ++++++++++++++++++++++++++---------- memory/ascend_index.go | 26 +++++-- memory/descend_index.go | 11 ++- memory/mergeable_index.go | 74 +++++++++++++++---- memory/negative_index.go | 7 +- memory/unmergeable_index.go | 13 +++- 6 files changed, 201 insertions(+), 71 deletions(-) diff --git a/engine_test.go b/engine_test.go index 94d4ba5f1..58ffeebc2 100644 --- a/engine_test.go +++ b/engine_test.go @@ -1597,60 +1597,42 @@ var queries = []struct { } func TestQueries(t *testing.T) { - // Test all queries with these combinations, for a total of 8 runs: + type indexDriverInitalizer func(map[string]*memory.Table) sql.IndexDriver + type indexDriverTestCase struct { + name string + initializer indexDriverInitalizer + } + + // Test all queries with these combinations, for a total of 16 runs: // 1) Partitioned tables / non partitioned tables - // 2) Indexes enabled / disabled + // 2) Mergeable / unmergeable / no indexes // 3) Parallelism on / off // numPartitionsVals := []int{1} // useIndexesVals := []bool{true} // parallelVals := []int{1} numPartitionsVals := []int{1, testNumPartitions} - useIndexesVals := []bool{false, true} + indexDrivers := []*indexDriverTestCase{ + nil, + {"unmergableIndexes", unmergableIndexDriver}, + {"mergableIndexes", mergableIndexDriver}, + } parallelVals := []int{1, 2} for _, numPartitions := range numPartitionsVals { - for _, useIndexes := range useIndexesVals { + for _, indexDriverInit := range indexDrivers { for _, parallelism := range parallelVals { tables := allTestTables(t, numPartitions) - testIndexDriver := memory.NewIndexDriver("mydb", map[string][]sql.Index{ - "mytable": { - newUnmergableIndex(tables, "mytable", - expression.NewGetFieldWithTable(0, sql.Int64, "mytable", "i", false)), - newUnmergableIndex(tables, "mytable", - expression.NewGetFieldWithTable(1, sql.Text, "mytable", "s", false)), - newUnmergableIndex(tables, "mytable", - expression.NewGetFieldWithTable(0, sql.Int64, "mytable", "i", false), - expression.NewGetFieldWithTable(1, sql.Text, "mytable", "s", false)), - }, - "othertable": { - newUnmergableIndex(tables, "othertable", - expression.NewGetFieldWithTable(0, sql.Text, "othertable", "s2", false)), - newUnmergableIndex(tables, "othertable", - expression.NewGetFieldWithTable(1, sql.Text, "othertable", "i2", false)), - newUnmergableIndex(tables, "othertable", - expression.NewGetFieldWithTable(0, sql.Text, "othertable", "s2", false), - expression.NewGetFieldWithTable(1, sql.Text, "othertable", "i2", false)), - }, - "bigtable": { - newUnmergableIndex(tables, "bigtable", - expression.NewGetFieldWithTable(0, sql.Text, "bigtable", "t", false)), - }, - "floattable": { - newUnmergableIndex(tables, "floattable", - expression.NewGetFieldWithTable(2, sql.Text, "floattable", "f64", false)), - }, - "niltable": { - newUnmergableIndex(tables, "niltable", - expression.NewGetFieldWithTable(0, sql.Int64, "niltable", "i", false)), - }, - }) var indexDriver sql.IndexDriver - if useIndexes { - indexDriver = testIndexDriver + if indexDriverInit != nil { + indexDriver = indexDriverInit.initializer(tables) } engine := newEngineWithParallelism(t, parallelism, tables, indexDriver) - testName := fmt.Sprintf("partitions=%d,indexes=%v,parallelism=%v", numPartitions, useIndexes, parallelism) + indexDriverName := "none" + if indexDriverInit != nil { + indexDriverName = indexDriverInit.name + } + testName := fmt.Sprintf("partitions=%d,indexes=%v,parallelism=%v", numPartitions, indexDriverName, parallelism) t.Run(testName, func(t *testing.T) { for _, tt := range queries { testQuery(t, engine, tt.query, tt.expected) @@ -1661,6 +1643,77 @@ func TestQueries(t *testing.T) { } } +func unmergableIndexDriver(tables map[string]*memory.Table) sql.IndexDriver { + return memory.NewIndexDriver("mydb", map[string][]sql.Index{ + "mytable": { + newUnmergableIndex(tables, "mytable", + expression.NewGetFieldWithTable(0, sql.Int64, "mytable", "i", false)), + newUnmergableIndex(tables, "mytable", + expression.NewGetFieldWithTable(1, sql.Text, "mytable", "s", false)), + newUnmergableIndex(tables, "mytable", + expression.NewGetFieldWithTable(0, sql.Int64, "mytable", "i", false), + expression.NewGetFieldWithTable(1, sql.Text, "mytable", "s", false)), + }, + "othertable": { + newUnmergableIndex(tables, "othertable", + expression.NewGetFieldWithTable(0, sql.Text, "othertable", "s2", false)), + newUnmergableIndex(tables, "othertable", + expression.NewGetFieldWithTable(1, sql.Text, "othertable", "i2", false)), + newUnmergableIndex(tables, "othertable", + expression.NewGetFieldWithTable(0, sql.Text, "othertable", "s2", false), + expression.NewGetFieldWithTable(1, sql.Text, "othertable", "i2", false)), + }, + "bigtable": { + newUnmergableIndex(tables, "bigtable", + expression.NewGetFieldWithTable(0, sql.Text, "bigtable", "t", false)), + }, + "floattable": { + newUnmergableIndex(tables, "floattable", + expression.NewGetFieldWithTable(2, sql.Text, "floattable", "f64", false)), + }, + "niltable": { + newUnmergableIndex(tables, "niltable", + expression.NewGetFieldWithTable(0, sql.Int64, "niltable", "i", false)), + }, + }) +} + +func mergableIndexDriver(tables map[string]*memory.Table) sql.IndexDriver { + return memory.NewIndexDriver("mydb", map[string][]sql.Index{ + "mytable": { + newMergableIndex(tables, "mytable", + expression.NewGetFieldWithTable(0, sql.Int64, "mytable", "i", false)), + newMergableIndex(tables, "mytable", + expression.NewGetFieldWithTable(1, sql.Text, "mytable", "s", false)), + newMergableIndex(tables, "mytable", + expression.NewGetFieldWithTable(0, sql.Int64, "mytable", "i", false), + expression.NewGetFieldWithTable(1, sql.Text, "mytable", "s", false)), + }, + "othertable": { + newMergableIndex(tables, "othertable", + expression.NewGetFieldWithTable(0, sql.Text, "othertable", "s2", false)), + newMergableIndex(tables, "othertable", + expression.NewGetFieldWithTable(1, sql.Text, "othertable", "i2", false)), + newMergableIndex(tables, "othertable", + expression.NewGetFieldWithTable(0, sql.Text, "othertable", "s2", false), + expression.NewGetFieldWithTable(1, sql.Text, "othertable", "i2", false)), + }, + "bigtable": { + newMergableIndex(tables, "bigtable", + expression.NewGetFieldWithTable(0, sql.Text, "bigtable", "t", false)), + }, + "floattable": { + newMergableIndex(tables, "floattable", + expression.NewGetFieldWithTable(2, sql.Text, "floattable", "f64", false)), + }, + "niltable": { + newMergableIndex(tables, "niltable", + expression.NewGetFieldWithTable(0, sql.Int64, "niltable", "i", false)), + }, + }) +} + + func newUnmergableIndex(tables map[string]*memory.Table, tableName string, exprs ...sql.Expression) *memory.UnmergeableDummyIndex { return &memory.UnmergeableDummyIndex{ DB: "mydb", @@ -1671,6 +1724,16 @@ func newUnmergableIndex(tables map[string]*memory.Table, tableName string, exprs } } +func newMergableIndex(tables map[string]*memory.Table, tableName string, exprs ...sql.Expression) *memory.MergeableDummyIndex { + return &memory.MergeableDummyIndex { + DB: "mydb", + DriverName: memory.IndexDriverId, + TableName: tableName, + Tbl: tables[tableName], + Exprs: exprs, + } +} + func TestSessionSelectLimit(t *testing.T) { ctx := newCtx() ctx.Session.Set("sql_select_limit", sql.Int64, int64(1)) diff --git a/memory/ascend_index.go b/memory/ascend_index.go index ad70485db..a30ac523a 100755 --- a/memory/ascend_index.go +++ b/memory/ascend_index.go @@ -2,17 +2,19 @@ package memory import ( "github.com/src-d/go-mysql-server/sql" + "github.com/src-d/go-mysql-server/sql/expression" ) type AscendIndexLookup struct { - id string - Gte []interface{} - Lt []interface{} + id string + Gte []interface{} + Lt []interface{} + Index ExpressionsIndex } +var _ memoryIndexLookup = (*AscendIndexLookup)(nil) + func (l *AscendIndexLookup) ID() string { return l.id } -func (l *AscendIndexLookup) GetUnions() []MergeableLookup { return nil } -func (l *AscendIndexLookup) GetIntersections() []MergeableLookup { return nil } func (AscendIndexLookup) Values(sql.Partition) (sql.IndexValueIter, error) { return nil, nil @@ -27,7 +29,19 @@ func (l *AscendIndexLookup) IsMergeable(sql.IndexLookup) bool { } func (l *AscendIndexLookup) Union(lookups ...sql.IndexLookup) sql.IndexLookup { - return union(l, lookups...) + return union(l.Index, l, lookups...) +} + +func (l *AscendIndexLookup) EvalExpression() sql.Expression { + if len(l.Index.ColumnExpressions()) > 1 { + panic("Ascend index unsupported for multi-column indexes") + } + gt, typ := getType(l.Gte[0]) + lt, typ := getType(l.Lt[0]) + return and( + expression.NewGreaterThanOrEqual(l.Index.ColumnExpressions()[0], expression.NewLiteral(gt, typ)), + expression.NewLessThan(l.Index.ColumnExpressions()[0], expression.NewLiteral(lt, typ)), + ) } func (AscendIndexLookup) Difference(...sql.IndexLookup) sql.IndexLookup { diff --git a/memory/descend_index.go b/memory/descend_index.go index c3f51b3b9..970ca814e 100755 --- a/memory/descend_index.go +++ b/memory/descend_index.go @@ -5,14 +5,13 @@ import ( ) type DescendIndexLookup struct { - id string - Gt []interface{} - Lte []interface{} + id string + Gt []interface{} + Lte []interface{} + Index ExpressionsIndex } func (l DescendIndexLookup) ID() string { return l.id } -func (l DescendIndexLookup) GetUnions() []MergeableLookup { return nil } -func (l DescendIndexLookup) GetIntersections() []MergeableLookup { return nil } func (DescendIndexLookup) Values(sql.Partition) (sql.IndexValueIter, error) { return nil, nil @@ -27,7 +26,7 @@ func (l *DescendIndexLookup) IsMergeable(sql.IndexLookup) bool { } func (l *DescendIndexLookup) Union(lookups ...sql.IndexLookup) sql.IndexLookup { - return union(l, lookups...) + return union(l.Index, l, lookups...) } func (DescendIndexLookup) Difference(...sql.IndexLookup) sql.IndexLookup { diff --git a/memory/mergeable_index.go b/memory/mergeable_index.go index e10a0e55d..f111800b9 100755 --- a/memory/mergeable_index.go +++ b/memory/mergeable_index.go @@ -59,7 +59,7 @@ func (i MergeableDummyIndex) Not(keys ...interface{}) (sql.IndexLookup, error) { } mergeable, _ := lookup.(*MergeableIndexLookup) - return &NegateIndexLookup{Lookup: mergeable}, nil + return &NegateIndexLookup{Lookup: mergeable, Index: mergeable.Index}, nil } func (i MergeableDummyIndex) Get(key ...interface{}) (sql.IndexLookup, error) { @@ -86,8 +86,6 @@ func (i MergeableDummyIndex) Table() string { return i.TableName } type MergeableLookup interface { ID() string - // GetUnions() []MergeableLookup - // GetIntersections() []MergeableLookup } // ExpressionsIndex is an index made out of one or more expressions (usually field expressions) @@ -103,12 +101,9 @@ type MergeableIndexLookup struct { var _ sql.Mergeable = (*MergeableIndexLookup)(nil) var _ sql.SetOperations = (*MergeableIndexLookup)(nil) +var _ memoryIndexLookup = (*MergeableIndexLookup)(nil) func (i *MergeableIndexLookup) ID() string { return strings.Join(i.Indexes(), ",") } -// func (i *MergeableIndexLookup) GetUnions() []MergeableLookup { return nil } -// func (i *MergeableIndexLookup) GetIntersections() []MergeableLookup { return nil } -func (i *MergeableIndexLookup) ClearUnions() { } -func (i *MergeableIndexLookup) ClearIntersections() { } func (i *MergeableIndexLookup) IsMergeable(lookup sql.IndexLookup) bool { _, ok := lookup.(MergeableLookup) @@ -129,6 +124,15 @@ func (i *MergeableIndexLookup) Values(p sql.Partition) (sql.IndexValueIter, erro }}, nil } +func (i *MergeableIndexLookup) EvalExpression() sql.Expression { + var exprs []sql.Expression + for exprI, expr := range i.Index.ColumnExpressions() { + lit, typ := getType(i.Key[exprI]) + exprs = append(exprs, expression.NewEquals(expr, expression.NewLiteral(lit, typ))) + } + return and(exprs...) +} + func (i *MergeableIndexLookup) Indexes() []string { var idxes = make([]string, len(i.Key)) for i, e := range i.Key { @@ -142,14 +146,14 @@ func (i *MergeableIndexLookup) Difference(...sql.IndexLookup) sql.IndexLookup { } func (i *MergeableIndexLookup) Intersection(lookups ...sql.IndexLookup) sql.IndexLookup { - return intersection(i, lookups...) + return intersection(i.Index, i, lookups...) } // Intersects the lookups given together, collapsing redundant layers of intersections for lookups that have previously // been merged. E.g. merging a MergeableIndexLookup with a MergedIndexLookup that has 2 intersections will return a // MergedIndexLookup with 3 lookups intersected: the left param and the two intersected lookups from the // MergedIndexLookup. -func intersection(left sql.IndexLookup, lookups ...sql.IndexLookup) sql.IndexLookup { +func intersection(idx ExpressionsIndex, left sql.IndexLookup, lookups ...sql.IndexLookup) sql.IndexLookup { var merged []sql.IndexLookup var allLookups []sql.IndexLookup allLookups = append(allLookups, left) @@ -164,17 +168,18 @@ func intersection(left sql.IndexLookup, lookups ...sql.IndexLookup) sql.IndexLoo return &MergedIndexLookup{ Intersections: merged, + idx: idx, } } func (i *MergeableIndexLookup) Union(lookups ...sql.IndexLookup) sql.IndexLookup { - return union(i, lookups...) + return union(i.Index, i, lookups...) } // Unions the lookups given together, collapsing redundant layers of unions for lookups that have previously been // merged. E.g. merging a MergeableIndexLookup with a MergedIndexLookup that has 2 unions will return a // MergedIndexLookup with 3 lookups unioned: the left param and the two unioned lookups from the MergedIndexLookup. -func union(left sql.IndexLookup, lookups ...sql.IndexLookup) sql.IndexLookup { +func union(idx ExpressionsIndex, left sql.IndexLookup, lookups ...sql.IndexLookup) sql.IndexLookup { var merged []sql.IndexLookup var allLookups []sql.IndexLookup allLookups = append(allLookups, left) @@ -189,6 +194,7 @@ func union(left sql.IndexLookup, lookups ...sql.IndexLookup) sql.IndexLookup { return &MergedIndexLookup{ Unions: merged, + idx: idx, } } @@ -197,17 +203,36 @@ func union(left sql.IndexLookup, lookups ...sql.IndexLookup) sql.IndexLookup { type MergedIndexLookup struct { Unions []sql.IndexLookup Intersections []sql.IndexLookup + idx ExpressionsIndex } var _ sql.Mergeable = (*MergedIndexLookup)(nil) var _ sql.SetOperations = (*MergedIndexLookup)(nil) +var _ memoryIndexLookup = (*MergedIndexLookup)(nil) + +func (m *MergedIndexLookup) EvalExpression() sql.Expression { + var exprs []sql.Expression + if m.Intersections != nil { + for _, lookup := range m.Intersections { + exprs = append(exprs, lookup.(memoryIndexLookup).EvalExpression()) + } + return and(exprs...) + } + if m.Unions != nil { + for _, lookup := range m.Unions { + exprs = append(exprs, lookup.(memoryIndexLookup).EvalExpression()) + } + return or(exprs...) + } + panic("either Unions or Intersections must be non-nil") +} func (m *MergedIndexLookup) Intersection(lookups ...sql.IndexLookup) sql.IndexLookup { - return intersection(m, lookups...) + return intersection(m.idx, m, lookups...) } func (m *MergedIndexLookup) Union(lookups ...sql.IndexLookup) sql.IndexLookup { - return union(m, lookups...) + return union(m.idx, m, lookups...) } func (m *MergedIndexLookup) Difference(...sql.IndexLookup) sql.IndexLookup { @@ -219,8 +244,27 @@ func (m *MergedIndexLookup) IsMergeable(lookup sql.IndexLookup) bool { return ok } -func (m *MergedIndexLookup) Values(sql.Partition) (sql.IndexValueIter, error) { - panic("implement me") +func (m *MergedIndexLookup) Values(p sql.Partition) (sql.IndexValueIter, error) { + return &dummyIndexValueIter{ + tbl: m.idx.MemTable(), + partition: p, + matchExpressions: func() []sql.Expression { + return []sql.Expression { m.EvalExpression() } + }}, nil +} + +func or(expressions ...sql.Expression) sql.Expression { + if len(expressions) == 1 { + return expressions[0] + } + return expression.NewOr(expressions[0], or(expressions[1:]...)) +} + +func and(expressions ...sql.Expression) sql.Expression { + if len(expressions) == 1 { + return expressions[0] + } + return expression.NewAnd(expressions[0], and(expressions[1:]...)) } func (m *MergedIndexLookup) Indexes() []string { diff --git a/memory/negative_index.go b/memory/negative_index.go index 7030594b3..5c6f53048 100755 --- a/memory/negative_index.go +++ b/memory/negative_index.go @@ -5,7 +5,8 @@ import ( ) type NegateIndexLookup struct { - Lookup MergeableLookup + Lookup MergeableLookup + Index ExpressionsIndex } func (l *NegateIndexLookup) ID() string { return "not " + l.Lookup.ID() } @@ -23,7 +24,7 @@ func (*NegateIndexLookup) IsMergeable(sql.IndexLookup) bool { } func (l *NegateIndexLookup) Union(lookups ...sql.IndexLookup) sql.IndexLookup { - return union(l, lookups...) + return union(l.Index, l, lookups...) } func (*NegateIndexLookup) Difference(...sql.IndexLookup) sql.IndexLookup { @@ -31,5 +32,5 @@ func (*NegateIndexLookup) Difference(...sql.IndexLookup) sql.IndexLookup { } func (l *NegateIndexLookup) Intersection(indexes ...sql.IndexLookup) sql.IndexLookup { - return intersection(l, indexes...) + return intersection(l.Index, l, indexes...) } \ No newline at end of file diff --git a/memory/unmergeable_index.go b/memory/unmergeable_index.go index c405d0afb..0b062cca1 100755 --- a/memory/unmergeable_index.go +++ b/memory/unmergeable_index.go @@ -42,13 +42,23 @@ type UnmergeableIndexLookup struct { idx *UnmergeableDummyIndex } +func (u *UnmergeableIndexLookup) EvalExpression() sql.Expression { + return nil +} + +type memoryIndexLookup interface { + EvalExpression() sql.Expression +} + +var _ memoryIndexLookup = (*UnmergeableIndexLookup)(nil) + // dummyIndexValueIter does a very simple and verifiable iteration over the table values for a given index. It does this // by iterating over all the table rows for a partition and evaluating each of them for inclusion in the index. This is // not an efficient way to store an index, and is only suitable for testing the correctness of index code in the engine. type dummyIndexValueIter struct { tbl *Table partition sql.Partition - // Returns a set of expresssions that must match a given row to be included in the value iterator for a lookup + // Returns a set of expressions that must match a given row to be included in the value iterator for a lookup matchExpressions func() []sql.Expression values [][]byte i int @@ -182,4 +192,3 @@ func (u *UnmergeableDummyIndex) ID() string { func (u *UnmergeableDummyIndex) Table() string { return u.TableName } - From 7b8c17bcdda2819253c11ea3f422dd85ec5f6534 Mon Sep 17 00:00:00 2001 From: Zach Musgrave Date: Mon, 4 Nov 2019 16:14:20 -0800 Subject: [PATCH 29/42] Fixed several NPEs in tests Signed-off-by: Zach Musgrave --- memory/ascend_index.go | 15 ++++++++++----- memory/descend_index.go | 30 +++++++++++++++++++++++++----- memory/mergeable_index.go | 12 ++++++------ memory/negative_index.go | 2 +- 4 files changed, 42 insertions(+), 17 deletions(-) diff --git a/memory/ascend_index.go b/memory/ascend_index.go index a30ac523a..a1a24e292 100755 --- a/memory/ascend_index.go +++ b/memory/ascend_index.go @@ -17,7 +17,7 @@ var _ memoryIndexLookup = (*AscendIndexLookup)(nil) func (l *AscendIndexLookup) ID() string { return l.id } func (AscendIndexLookup) Values(sql.Partition) (sql.IndexValueIter, error) { - return nil, nil + panic("unimplemented") } func (l *AscendIndexLookup) Indexes() []string { @@ -36,12 +36,17 @@ func (l *AscendIndexLookup) EvalExpression() sql.Expression { if len(l.Index.ColumnExpressions()) > 1 { panic("Ascend index unsupported for multi-column indexes") } + gt, typ := getType(l.Gte[0]) - lt, typ := getType(l.Lt[0]) - return and( - expression.NewGreaterThanOrEqual(l.Index.ColumnExpressions()[0], expression.NewLiteral(gt, typ)), - expression.NewLessThan(l.Index.ColumnExpressions()[0], expression.NewLiteral(lt, typ)), + gte := expression.NewGreaterThanOrEqual(l.Index.ColumnExpressions()[0], expression.NewLiteral(gt, typ)) + if l.Lt != nil { + lt, _ := getType(l.Lt[0]) + return and( + gte, + expression.NewLessThan(l.Index.ColumnExpressions()[0], expression.NewLiteral(lt, typ)), ) + } + return gte } func (AscendIndexLookup) Difference(...sql.IndexLookup) sql.IndexLookup { diff --git a/memory/descend_index.go b/memory/descend_index.go index 970ca814e..82ecbfd49 100755 --- a/memory/descend_index.go +++ b/memory/descend_index.go @@ -2,6 +2,7 @@ package memory import ( "github.com/src-d/go-mysql-server/sql" + "github.com/src-d/go-mysql-server/sql/expression" ) type DescendIndexLookup struct { @@ -11,10 +12,29 @@ type DescendIndexLookup struct { Index ExpressionsIndex } -func (l DescendIndexLookup) ID() string { return l.id } +var _ memoryIndexLookup = (*DescendIndexLookup)(nil) -func (DescendIndexLookup) Values(sql.Partition) (sql.IndexValueIter, error) { - return nil, nil +func (l *DescendIndexLookup) ID() string { return l.id } + +func (*DescendIndexLookup) Values(sql.Partition) (sql.IndexValueIter, error) { + panic("unimplemented") +} + +func (l *DescendIndexLookup) EvalExpression() sql.Expression { + if len(l.Index.ColumnExpressions()) > 1 { + panic("Descend index unsupported for multi-column indexes") + } + + gt, typ := getType(l.Lte[0]) + lte := expression.NewLessThanOrEqual(l.Index.ColumnExpressions()[0], expression.NewLiteral(gt, typ)) + if l.Gt != nil { + lt, _ := getType(l.Gt[0]) + return and( + lte, + expression.NewGreaterThan(l.Index.ColumnExpressions()[0], expression.NewLiteral(lt, typ)), + ) + } + return lte } func (l *DescendIndexLookup) Indexes() []string { @@ -29,11 +49,11 @@ func (l *DescendIndexLookup) Union(lookups ...sql.IndexLookup) sql.IndexLookup { return union(l.Index, l, lookups...) } -func (DescendIndexLookup) Difference(...sql.IndexLookup) sql.IndexLookup { +func (*DescendIndexLookup) Difference(...sql.IndexLookup) sql.IndexLookup { panic("descendIndexLookup.Difference is not implemented") } -func (DescendIndexLookup) Intersection(...sql.IndexLookup) sql.IndexLookup { +func (*DescendIndexLookup) Intersection(...sql.IndexLookup) sql.IndexLookup { panic("descendIndexLookup.Intersection is not implemented") } diff --git a/memory/mergeable_index.go b/memory/mergeable_index.go index f111800b9..f676290ea 100755 --- a/memory/mergeable_index.go +++ b/memory/mergeable_index.go @@ -29,27 +29,27 @@ func (i MergeableDummyIndex) Expressions() []string { } func (i MergeableDummyIndex) AscendGreaterOrEqual(keys ...interface{}) (sql.IndexLookup, error) { - return &AscendIndexLookup{Gte: keys}, nil + return &AscendIndexLookup{Gte: keys, Index: i}, nil } func (i MergeableDummyIndex) AscendLessThan(keys ...interface{}) (sql.IndexLookup, error) { - return &AscendIndexLookup{Lt: keys}, nil + return &AscendIndexLookup{Lt: keys, Index: i}, nil } func (i MergeableDummyIndex) AscendRange(greaterOrEqual, lessThan []interface{}) (sql.IndexLookup, error) { - return &AscendIndexLookup{Gte: greaterOrEqual, Lt: lessThan}, nil + return &AscendIndexLookup{Gte: greaterOrEqual, Lt: lessThan, Index: i}, nil } func (i MergeableDummyIndex) DescendGreater(keys ...interface{}) (sql.IndexLookup, error) { - return &DescendIndexLookup{Gt: keys}, nil + return &DescendIndexLookup{Gt: keys, Index: i}, nil } func (i MergeableDummyIndex) DescendLessOrEqual(keys ...interface{}) (sql.IndexLookup, error) { - return &DescendIndexLookup{Lte: keys}, nil + return &DescendIndexLookup{Lte: keys, Index: i}, nil } func (i MergeableDummyIndex) DescendRange(lessOrEqual, greaterThan []interface{}) (sql.IndexLookup, error) { - return &DescendIndexLookup{Gt: greaterThan, Lte: lessOrEqual}, nil + return &DescendIndexLookup{Gt: greaterThan, Lte: lessOrEqual, Index: i}, nil } func (i MergeableDummyIndex) Not(keys ...interface{}) (sql.IndexLookup, error) { diff --git a/memory/negative_index.go b/memory/negative_index.go index 5c6f53048..0f7d0647b 100755 --- a/memory/negative_index.go +++ b/memory/negative_index.go @@ -12,7 +12,7 @@ type NegateIndexLookup struct { func (l *NegateIndexLookup) ID() string { return "not " + l.Lookup.ID() } func (*NegateIndexLookup) Values(sql.Partition) (sql.IndexValueIter, error) { - return nil, nil + panic("unimplemented") } func (l *NegateIndexLookup) Indexes() []string { From 5894576faecc2f35592065bca0edeef300aa910c Mon Sep 17 00:00:00 2001 From: Zach Musgrave Date: Mon, 4 Nov 2019 16:28:28 -0800 Subject: [PATCH 30/42] All tests running with mergable indexes without panics. Now to fix failures (probably a mistake in the index Values() implementation) Signed-off-by: Zach Musgrave --- memory/ascend_index.go | 23 ++++++++++++++--------- memory/descend_index.go | 21 +++++++++++++-------- memory/negative_index.go | 17 ++++++++++++++--- 3 files changed, 41 insertions(+), 20 deletions(-) diff --git a/memory/ascend_index.go b/memory/ascend_index.go index a1a24e292..91f287aaa 100755 --- a/memory/ascend_index.go +++ b/memory/ascend_index.go @@ -16,8 +16,13 @@ var _ memoryIndexLookup = (*AscendIndexLookup)(nil) func (l *AscendIndexLookup) ID() string { return l.id } -func (AscendIndexLookup) Values(sql.Partition) (sql.IndexValueIter, error) { - panic("unimplemented") +func (l *AscendIndexLookup) Values(p sql.Partition) (sql.IndexValueIter, error) { + return &dummyIndexValueIter{ + tbl: l.Index.MemTable(), + partition: p, + matchExpressions: func() []sql.Expression { + return []sql.Expression { l.EvalExpression() } + }}, nil } func (l *AscendIndexLookup) Indexes() []string { @@ -37,16 +42,16 @@ func (l *AscendIndexLookup) EvalExpression() sql.Expression { panic("Ascend index unsupported for multi-column indexes") } - gt, typ := getType(l.Gte[0]) - gte := expression.NewGreaterThanOrEqual(l.Index.ColumnExpressions()[0], expression.NewLiteral(gt, typ)) - if l.Lt != nil { - lt, _ := getType(l.Lt[0]) + lt, typ := getType(l.Lt[0]) + ltexpr := expression.NewLessThan(l.Index.ColumnExpressions()[0], expression.NewLiteral(lt, typ)) + if len(l.Gte) > 0 { + gte, _ := getType(l.Gte[0]) return and( - gte, - expression.NewLessThan(l.Index.ColumnExpressions()[0], expression.NewLiteral(lt, typ)), + ltexpr, + expression.NewGreaterThanOrEqual(l.Index.ColumnExpressions()[0], expression.NewLiteral(gte, typ)), ) } - return gte + return ltexpr } func (AscendIndexLookup) Difference(...sql.IndexLookup) sql.IndexLookup { diff --git a/memory/descend_index.go b/memory/descend_index.go index 82ecbfd49..4bb86fa71 100755 --- a/memory/descend_index.go +++ b/memory/descend_index.go @@ -16,8 +16,13 @@ var _ memoryIndexLookup = (*DescendIndexLookup)(nil) func (l *DescendIndexLookup) ID() string { return l.id } -func (*DescendIndexLookup) Values(sql.Partition) (sql.IndexValueIter, error) { - panic("unimplemented") +func (l *DescendIndexLookup) Values(p sql.Partition) (sql.IndexValueIter, error) { + return &dummyIndexValueIter{ + tbl: l.Index.MemTable(), + partition: p, + matchExpressions: func() []sql.Expression { + return []sql.Expression { l.EvalExpression() } + }}, nil } func (l *DescendIndexLookup) EvalExpression() sql.Expression { @@ -25,16 +30,16 @@ func (l *DescendIndexLookup) EvalExpression() sql.Expression { panic("Descend index unsupported for multi-column indexes") } - gt, typ := getType(l.Lte[0]) - lte := expression.NewLessThanOrEqual(l.Index.ColumnExpressions()[0], expression.NewLiteral(gt, typ)) - if l.Gt != nil { + gt, typ := getType(l.Gt[0]) + gtexpr := expression.NewGreaterThan(l.Index.ColumnExpressions()[0], expression.NewLiteral(gt, typ)) + if len(l.Gt) > 0 { lt, _ := getType(l.Gt[0]) return and( - lte, - expression.NewGreaterThan(l.Index.ColumnExpressions()[0], expression.NewLiteral(lt, typ)), + gtexpr, + expression.NewLessThanOrEqual(l.Index.ColumnExpressions()[0], expression.NewLiteral(lt, typ)), ) } - return lte + return gtexpr } func (l *DescendIndexLookup) Indexes() []string { diff --git a/memory/negative_index.go b/memory/negative_index.go index 0f7d0647b..9b0fa2272 100755 --- a/memory/negative_index.go +++ b/memory/negative_index.go @@ -9,10 +9,21 @@ type NegateIndexLookup struct { Index ExpressionsIndex } -func (l *NegateIndexLookup) ID() string { return "not " + l.Lookup.ID() } +var _ memoryIndexLookup = (*NegateIndexLookup)(nil) -func (*NegateIndexLookup) Values(sql.Partition) (sql.IndexValueIter, error) { - panic("unimplemented") +func (l *NegateIndexLookup) ID() string { return "not " + l.Lookup.ID() } + +func (l *NegateIndexLookup) Values(p sql.Partition) (sql.IndexValueIter, error) { + return &dummyIndexValueIter{ + tbl: l.Index.MemTable(), + partition: p, + matchExpressions: func() []sql.Expression { + return []sql.Expression { l.Lookup.(memoryIndexLookup).EvalExpression() } + }}, nil +} + +func (l *NegateIndexLookup) EvalExpression() sql.Expression { + return l.Lookup.(memoryIndexLookup).EvalExpression() } func (l *NegateIndexLookup) Indexes() []string { From feb7eaf89ae6cfc390c1397a9f69b72e697b1f31 Mon Sep 17 00:00:00 2001 From: Zach Musgrave Date: Mon, 4 Nov 2019 16:33:35 -0800 Subject: [PATCH 31/42] Fixed a couple typos in ascend / descend, which fixed the last of the test failures Signed-off-by: Zach Musgrave --- memory/descend_index.go | 6 +++--- memory/negative_index.go | 5 +++-- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/memory/descend_index.go b/memory/descend_index.go index 4bb86fa71..33e140d1d 100755 --- a/memory/descend_index.go +++ b/memory/descend_index.go @@ -32,11 +32,11 @@ func (l *DescendIndexLookup) EvalExpression() sql.Expression { gt, typ := getType(l.Gt[0]) gtexpr := expression.NewGreaterThan(l.Index.ColumnExpressions()[0], expression.NewLiteral(gt, typ)) - if len(l.Gt) > 0 { - lt, _ := getType(l.Gt[0]) + if len(l.Lte) > 0 { + lte, _ := getType(l.Lte[0]) return and( gtexpr, - expression.NewLessThanOrEqual(l.Index.ColumnExpressions()[0], expression.NewLiteral(lt, typ)), + expression.NewLessThanOrEqual(l.Index.ColumnExpressions()[0], expression.NewLiteral(lte, typ)), ) } return gtexpr diff --git a/memory/negative_index.go b/memory/negative_index.go index 9b0fa2272..1c2e3ab86 100755 --- a/memory/negative_index.go +++ b/memory/negative_index.go @@ -2,6 +2,7 @@ package memory import ( "github.com/src-d/go-mysql-server/sql" + "github.com/src-d/go-mysql-server/sql/expression" ) type NegateIndexLookup struct { @@ -18,12 +19,12 @@ func (l *NegateIndexLookup) Values(p sql.Partition) (sql.IndexValueIter, error) tbl: l.Index.MemTable(), partition: p, matchExpressions: func() []sql.Expression { - return []sql.Expression { l.Lookup.(memoryIndexLookup).EvalExpression() } + return []sql.Expression { l.EvalExpression() } }}, nil } func (l *NegateIndexLookup) EvalExpression() sql.Expression { - return l.Lookup.(memoryIndexLookup).EvalExpression() + return expression.NewNot(l.Lookup.(memoryIndexLookup).EvalExpression()) } func (l *NegateIndexLookup) Indexes() []string { From e97734ae38c680422b411938cd49b32a588ec355 Mon Sep 17 00:00:00 2001 From: Zach Musgrave Date: Mon, 4 Nov 2019 16:52:15 -0800 Subject: [PATCH 32/42] Fixed timezone bug in time_test.go Signed-off-by: Zach Musgrave --- sql/expression/function/time_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/expression/function/time_test.go b/sql/expression/function/time_test.go index b9762ec1f..204a3348a 100644 --- a/sql/expression/function/time_test.go +++ b/sql/expression/function/time_test.go @@ -368,7 +368,7 @@ func TestDate(t *testing.T) { {"null date", sql.NewRow(nil), nil, false}, {"invalid type", sql.NewRow([]byte{0, 1, 2}), nil, false}, {"date as string", sql.NewRow(stringDate), "2007-01-02", false}, - {"date as time", sql.NewRow(time.Now()), time.Now().Format("2006-01-02"), false}, + {"date as time", sql.NewRow(time.Now().UTC()), time.Now().UTC().Format("2006-01-02"), false}, {"date as unix timestamp", sql.NewRow(int64(tsDate)), "2009-11-22", false}, } From 0d85dd3f9d79ea4e2d9331e670d65b348f1dd306 Mon Sep 17 00:00:00 2001 From: Zach Musgrave Date: Mon, 4 Nov 2019 17:14:38 -0800 Subject: [PATCH 33/42] Fixing up index assignment tests broken by the addition of new fields to index structs, introduced some new helper functions to cut down on verbosity of test literals Signed-off-by: Zach Musgrave --- memory/mergeable_index.go | 12 ++-- sql/analyzer/assign_indexes_test.go | 99 ++++++++++++++++------------- 2 files changed, 61 insertions(+), 50 deletions(-) diff --git a/memory/mergeable_index.go b/memory/mergeable_index.go index f676290ea..9b36affbc 100755 --- a/memory/mergeable_index.go +++ b/memory/mergeable_index.go @@ -168,7 +168,7 @@ func intersection(idx ExpressionsIndex, left sql.IndexLookup, lookups ...sql.Ind return &MergedIndexLookup{ Intersections: merged, - idx: idx, + Index: idx, } } @@ -194,7 +194,7 @@ func union(idx ExpressionsIndex, left sql.IndexLookup, lookups ...sql.IndexLooku return &MergedIndexLookup{ Unions: merged, - idx: idx, + Index: idx, } } @@ -203,7 +203,7 @@ func union(idx ExpressionsIndex, left sql.IndexLookup, lookups ...sql.IndexLooku type MergedIndexLookup struct { Unions []sql.IndexLookup Intersections []sql.IndexLookup - idx ExpressionsIndex + Index ExpressionsIndex } var _ sql.Mergeable = (*MergedIndexLookup)(nil) @@ -228,11 +228,11 @@ func (m *MergedIndexLookup) EvalExpression() sql.Expression { } func (m *MergedIndexLookup) Intersection(lookups ...sql.IndexLookup) sql.IndexLookup { - return intersection(m.idx, m, lookups...) + return intersection(m.Index, m, lookups...) } func (m *MergedIndexLookup) Union(lookups ...sql.IndexLookup) sql.IndexLookup { - return union(m.idx, m, lookups...) + return union(m.Index, m, lookups...) } func (m *MergedIndexLookup) Difference(...sql.IndexLookup) sql.IndexLookup { @@ -246,7 +246,7 @@ func (m *MergedIndexLookup) IsMergeable(lookup sql.IndexLookup) bool { func (m *MergedIndexLookup) Values(p sql.Partition) (sql.IndexValueIter, error) { return &dummyIndexValueIter{ - tbl: m.idx.MemTable(), + tbl: m.Index.MemTable(), partition: p, matchExpressions: func() []sql.Expression { return []sql.Expression { m.EvalExpression() } diff --git a/sql/analyzer/assign_indexes_test.go b/sql/analyzer/assign_indexes_test.go index d43ed4324..26304abac 100644 --- a/sql/analyzer/assign_indexes_test.go +++ b/sql/analyzer/assign_indexes_test.go @@ -187,13 +187,53 @@ func TestAssignIndexes(t *testing.T) { require.False(ok) } +func intersectionLookupWithKeys(table string, column string, colIdx int, keys ...interface{}) *memory.MergedIndexLookup { + var lookups []sql.IndexLookup + for _, key := range keys { + lookups = append(lookups, mergeableIndexLookup(table, column, colIdx, key)) + } + return &memory.MergedIndexLookup{ + Intersections: lookups, + Index: mergeableIndex(table, column, colIdx), + } +} + +func unionLookupWithKeys(table string, column string, colIdx int, keys ...interface{}) *memory.MergedIndexLookup { + var lookups []sql.IndexLookup + for _, key := range keys { + lookups = append(lookups, mergeableIndexLookup(table, column, colIdx, key)) + } + return &memory.MergedIndexLookup{ + Unions: lookups, + Index: mergeableIndex(table, column, colIdx), + } +} + +func unionLookup(table string, column string, colIdx int, lookups ...sql.IndexLookup) *memory.MergedIndexLookup { + return &memory.MergedIndexLookup{ + Unions: lookups, + Index: mergeableIndex(table, column, colIdx), + } +} + +func intersectionLookup(table string, column string, colIdx int, lookups ...sql.IndexLookup) *memory.MergedIndexLookup { + return &memory.MergedIndexLookup{ + Intersections: lookups, + Index: mergeableIndex(table, column, colIdx), + } +} + func mergeableIndexLookup(table string, column string, colIdx int, key ...interface{}) *memory.MergeableIndexLookup { return &memory.MergeableIndexLookup{ Key: key, - Index: memory.MergeableDummyIndex{ - TableName: table, - Exprs: []sql.Expression{col(colIdx, table, column)}, - }, + Index: mergeableIndex(table, column, colIdx), + } +} + +func mergeableIndex(table string, column string, colIdx int) memory.MergeableDummyIndex { + return memory.MergeableDummyIndex{ + TableName: table, + Exprs: []sql.Expression{col(colIdx, table, column)}, } } @@ -270,6 +310,7 @@ func TestGetIndexes(t *testing.T) { mergeableIndexLookup("t1", "bar", 0, int64(1)), mergeableIndexLookup("t1", "bar", 0, int64(2)), }, + Index: mergeableIndex("t1", "bar", 0), }, []sql.Index{ indexes[0], @@ -308,12 +349,7 @@ func TestGetIndexes(t *testing.T) { ), map[string]*indexLookup{ "t1": &indexLookup{ - &memory.MergedIndexLookup{ - Unions: []sql.IndexLookup{ - mergeableIndexLookup("t1", "bar", 0, int64(1)), - mergeableIndexLookup("t1", "bar", 0, int64(2)), - }, - }, + unionLookupWithKeys("t1", "bar", 0, int64(1), int64(2)), []sql.Index{ indexes[0], }, @@ -334,12 +370,7 @@ func TestGetIndexes(t *testing.T) { ), map[string]*indexLookup{ "t1": &indexLookup{ - &memory.MergedIndexLookup{ - Intersections: []sql.IndexLookup{ - mergeableIndexLookup("t1", "bar", 0, int64(1)), - mergeableIndexLookup("t1", "bar", 0, int64(2)), - }, - }, + intersectionLookupWithKeys("t1", "bar", 0, int64(1), int64(2)), []sql.Index{ indexes[0], indexes[0], @@ -373,22 +404,16 @@ func TestGetIndexes(t *testing.T) { ), map[string]*indexLookup{ "t1": &indexLookup{ - &memory.MergedIndexLookup{ - Intersections: []sql.IndexLookup { - &memory.MergedIndexLookup{ - Unions: []sql.IndexLookup{ + intersectionLookup("t1", "bar", 0, + unionLookup("t1", "bar", 0, mergeableIndexLookup("t1", "bar", 0, int64(1)), mergeableIndexLookup("t1", "bar", 0, int64(2)), - }, - }, - &memory.MergedIndexLookup{ - Unions: []sql.IndexLookup{ + ), + unionLookup("t1", "bar", 0, mergeableIndexLookup("t1", "bar", 0, int64(3)), mergeableIndexLookup("t1", "bar", 0, int64(4)), - }, - }, - }, - }, + ), + ), []sql.Index{ indexes[0], indexes[0], @@ -424,14 +449,7 @@ func TestGetIndexes(t *testing.T) { ), map[string]*indexLookup{ "t1": &indexLookup{ - &memory.MergedIndexLookup{ - Unions: []sql.IndexLookup{ - mergeableIndexLookup("t1", "bar", 0, int64(1)), - mergeableIndexLookup("t1", "bar", 0, int64(2)), - mergeableIndexLookup("t1", "bar", 0, int64(3)), - mergeableIndexLookup("t1", "bar", 0, int64(4)), - }, - }, + unionLookupWithKeys("t1", "bar", 0, int64(1), int64(2), int64(3), int64(4)), []sql.Index{ indexes[0], indexes[0], @@ -449,14 +467,7 @@ func TestGetIndexes(t *testing.T) { ), map[string]*indexLookup{ "t1": &indexLookup{ - &memory.MergedIndexLookup{ - Unions: []sql.IndexLookup{ - mergeableIndexLookup("t1", "bar", 0, int64(1)), - mergeableIndexLookup("t1", "bar", 0, int64(2)), - mergeableIndexLookup("t1", "bar", 0, int64(3)), - mergeableIndexLookup("t1", "bar", 0, int64(4)), - }, - }, + unionLookupWithKeys("t1", "bar", 0, int64(1), int64(2), int64(3), int64(4)), []sql.Index{indexes[0]}, }, }, From 7d34f8d27c1859d97627963dceddec4dd3856a8c Mon Sep 17 00:00:00 2001 From: Zach Musgrave Date: Tue, 5 Nov 2019 10:41:36 -0800 Subject: [PATCH 34/42] Fixed the remaining GetIndexes tests. Signed-off-by: Zach Musgrave --- sql/analyzer/assign_indexes_test.go | 155 ++++++++++++++++------------ 1 file changed, 90 insertions(+), 65 deletions(-) diff --git a/sql/analyzer/assign_indexes_test.go b/sql/analyzer/assign_indexes_test.go index 26304abac..e11c679fc 100644 --- a/sql/analyzer/assign_indexes_test.go +++ b/sql/analyzer/assign_indexes_test.go @@ -552,21 +552,18 @@ func TestGetIndexes(t *testing.T) { []sql.Index{indexes[0]}, }, "t2": &indexLookup{ - &memory.MergedIndexLookup{ - Unions: []sql.IndexLookup{ - mergeableIndexLookup("t2", "bar", 0, int64(5)), - &memory.MergeableIndexLookup{ - Key: []interface{}{int64(1), int64(2)}, - Index: memory.MergeableDummyIndex{ - TableName: "t2", - Exprs: []sql.Expression{ - col(0, "t2", "foo"), - col(0, "t2", "bar"), - }, + unionLookup("t2", "bar", 0, + mergeableIndexLookup("t2", "bar", 0, int64(5)), + &memory.MergeableIndexLookup{ + Key: []interface{}{int64(1), int64(2)}, + Index: memory.MergeableDummyIndex{ + TableName: "t2", + Exprs: []sql.Expression{ + col(0, "t2", "foo"), + col(0, "t2", "bar"), }, }, - }, - }, + }), []sql.Index{ indexes[2], indexes[1], @@ -582,7 +579,10 @@ func TestGetIndexes(t *testing.T) { ), map[string]*indexLookup{ "t1": &indexLookup{ - &memory.DescendIndexLookup{Gt: []interface{}{int64(1)}}, + &memory.DescendIndexLookup{ + Gt: []interface{}{int64(1)}, + Index: mergeableIndex("t1", "bar", 0), + }, []sql.Index{indexes[0]}, }, }, @@ -595,7 +595,10 @@ func TestGetIndexes(t *testing.T) { ), map[string]*indexLookup{ "t1": &indexLookup{ - &memory.AscendIndexLookup{Lt: []interface{}{int64(1)}}, + &memory.AscendIndexLookup{ + Lt: []interface{}{int64(1)}, + Index: mergeableIndex("t1", "bar", 0), + }, []sql.Index{indexes[0]}, }, }, @@ -608,7 +611,10 @@ func TestGetIndexes(t *testing.T) { ), map[string]*indexLookup{ "t1": &indexLookup{ - &memory.AscendIndexLookup{Gte: []interface{}{int64(1)}}, + &memory.AscendIndexLookup{ + Gte: []interface{}{int64(1)}, + Index: mergeableIndex("t1", "bar", 0), + }, []sql.Index{indexes[0]}, }, }, @@ -621,7 +627,10 @@ func TestGetIndexes(t *testing.T) { ), map[string]*indexLookup{ "t1": &indexLookup{ - &memory.DescendIndexLookup{Lte: []interface{}{int64(1)}}, + &memory.DescendIndexLookup{ + Lte: []interface{}{int64(1)}, + Index: mergeableIndex("t1", "bar", 0), + }, []sql.Index{indexes[0]}, }, }, @@ -635,18 +644,18 @@ func TestGetIndexes(t *testing.T) { ), map[string]*indexLookup{ "t1": &indexLookup{ - &memory.MergedIndexLookup { - Unions: []sql.IndexLookup{ - &memory.AscendIndexLookup{ - Gte: []interface{}{int64(1)}, - Lt: []interface{}{int64(5)}, - }, - &memory.DescendIndexLookup{ - Gt: []interface{}{int64(1)}, - Lte: []interface{}{int64(5)}, - }, + unionLookup("t1", "bar", 0, + &memory.AscendIndexLookup{ + Gte: []interface{}{int64(1)}, + Lt: []interface{}{int64(5)}, + Index: mergeableIndex("t1", "bar", 0), }, - }, + &memory.DescendIndexLookup{ + Gt: []interface{}{int64(1)}, + Lte: []interface{}{int64(5)}, + Index: mergeableIndex("t1", "bar", 0), + }, + ), []sql.Index{indexes[0]}, }, }, @@ -663,6 +672,7 @@ func TestGetIndexes(t *testing.T) { "t1": &indexLookup{ &memory.NegateIndexLookup{ Lookup: mergeableIndexLookup("t1", "bar", 0, int64(1)), + Index: mergeableIndex("t1", "bar", 0), }, []sql.Index{indexes[0]}, }, @@ -679,7 +689,10 @@ func TestGetIndexes(t *testing.T) { ), map[string]*indexLookup{ "t1": &indexLookup{ - &memory.DescendIndexLookup{Lte: []interface{}{int64(10)}}, + &memory.DescendIndexLookup{ + Lte: []interface{}{int64(10)}, + Index: mergeableIndex("t1", "bar", 0), + }, []sql.Index{indexes[0]}, }, }, @@ -695,7 +708,10 @@ func TestGetIndexes(t *testing.T) { ), map[string]*indexLookup{ "t1": &indexLookup{ - &memory.AscendIndexLookup{Lt: []interface{}{int64(10)}}, + &memory.AscendIndexLookup{ + Lt: []interface{}{int64(10)}, + Index: mergeableIndex("t1", "bar", 0), + }, []sql.Index{indexes[0]}, }, }, @@ -711,7 +727,10 @@ func TestGetIndexes(t *testing.T) { ), map[string]*indexLookup{ "t1": &indexLookup{ - &memory.DescendIndexLookup{Gt: []interface{}{int64(10)}}, + &memory.DescendIndexLookup{ + Gt: []interface{}{int64(10)}, + Index: mergeableIndex("t1", "bar", 0), + }, []sql.Index{indexes[0]}, }, }, @@ -727,7 +746,10 @@ func TestGetIndexes(t *testing.T) { ), map[string]*indexLookup{ "t1": &indexLookup{ - &memory.AscendIndexLookup{Gte: []interface{}{int64(10)}}, + &memory.AscendIndexLookup{ + Gte: []interface{}{int64(10)}, + Index: mergeableIndex("t1", "bar", 0), + }, []sql.Index{indexes[0]}, }, }, @@ -748,16 +770,16 @@ func TestGetIndexes(t *testing.T) { ), map[string]*indexLookup{ "t1": &indexLookup{ - &memory.MergedIndexLookup{ - Unions: []sql.IndexLookup{ - &memory.NegateIndexLookup{ - Lookup: mergeableIndexLookup("t1", "bar", 0, int64(10)), - }, - &memory.NegateIndexLookup{ - Lookup: mergeableIndexLookup("t1", "bar", 0, int64(11)), - }, + unionLookup("t1", "bar", 0, + &memory.NegateIndexLookup{ + Lookup: mergeableIndexLookup("t1", "bar", 0, int64(10)), + Index: mergeableIndex("t1", "bar", 0), }, - }, + &memory.NegateIndexLookup{ + Lookup: mergeableIndexLookup("t1", "bar", 0, int64(11)), + Index: mergeableIndex("t1", "bar", 0), + }, + ), []sql.Index{ indexes[0], indexes[0], @@ -781,16 +803,16 @@ func TestGetIndexes(t *testing.T) { ), map[string]*indexLookup{ "t1": &indexLookup{ - &memory.MergedIndexLookup{ - Intersections: []sql.IndexLookup{ - &memory.NegateIndexLookup{ - Lookup: mergeableIndexLookup("t1", "bar", 0, int64(10)), - }, - &memory.NegateIndexLookup{ - Lookup: mergeableIndexLookup("t1", "bar", 0, int64(11)), - }, + intersectionLookup("t1", "bar", 0, + &memory.NegateIndexLookup{ + Lookup: mergeableIndexLookup("t1", "bar", 0, int64(10)), + Index: mergeableIndex("t1", "bar", 0), }, - }, + &memory.NegateIndexLookup{ + Lookup: mergeableIndexLookup("t1", "bar", 0, int64(11)), + Index: mergeableIndex("t1", "bar", 0), + }, + ), []sql.Index{ indexes[0], indexes[0], @@ -819,6 +841,7 @@ func TestGetIndexes(t *testing.T) { "t2": &indexLookup{ &memory.NegateIndexLookup{ Lookup: mergeableIndexLookup("t2", "bar", 0, int64(110)), + Index: mergeableIndex("t2", "bar", 0), }, []sql.Index{ indexes[2], @@ -839,22 +862,24 @@ func TestGetIndexes(t *testing.T) { ), map[string]*indexLookup{ "t1": &indexLookup{ - &memory.MergedIndexLookup{ - Intersections: []sql.IndexLookup{ - &memory.NegateIndexLookup{ - Lookup: mergeableIndexLookup("t1", "bar", 0, int64(1)), - }, - &memory.NegateIndexLookup{ - Lookup: mergeableIndexLookup("t1", "bar", 0, int64(2)), - }, - &memory.NegateIndexLookup{ - Lookup: mergeableIndexLookup("t1", "bar", 0, int64(3)), - }, - &memory.NegateIndexLookup{ - Lookup: mergeableIndexLookup("t1", "bar", 0, int64(4)), - }, + intersectionLookup("t1", "bar", 0, + &memory.NegateIndexLookup{ + Lookup: mergeableIndexLookup("t1", "bar", 0, int64(1)), + Index: mergeableIndex("t1", "bar", 0), }, - }, + &memory.NegateIndexLookup{ + Lookup: mergeableIndexLookup("t1", "bar", 0, int64(2)), + Index: mergeableIndex("t1", "bar", 0), + }, + &memory.NegateIndexLookup{ + Lookup: mergeableIndexLookup("t1", "bar", 0, int64(3)), + Index: mergeableIndex("t1", "bar", 0), + }, + &memory.NegateIndexLookup{ + Lookup: mergeableIndexLookup("t1", "bar", 0, int64(4)), + Index: mergeableIndex("t1", "bar", 0), + }, + ), []sql.Index{indexes[0]}, }, }, From a0434f89bfb7da0da880c4f4401f0b1003177056 Mon Sep 17 00:00:00 2001 From: Zach Musgrave Date: Tue, 5 Nov 2019 11:18:19 -0800 Subject: [PATCH 35/42] Fixed last of failing index tests Signed-off-by: Zach Musgrave --- memory/mergeable_index.go | 32 ++++++++++++++--------------- sql/analyzer/assign_indexes_test.go | 27 ++++++++++++++++-------- 2 files changed, 34 insertions(+), 25 deletions(-) diff --git a/memory/mergeable_index.go b/memory/mergeable_index.go index 9b36affbc..c69e9ef0c 100755 --- a/memory/mergeable_index.go +++ b/memory/mergeable_index.go @@ -15,12 +15,12 @@ type MergeableDummyIndex struct { Exprs []sql.Expression } -func (i MergeableDummyIndex) Database() string { return i.DB } -func (i MergeableDummyIndex) Driver() string { return i.DriverName } -func (i MergeableDummyIndex) MemTable() *Table { return i.Tbl } -func (i MergeableDummyIndex) ColumnExpressions() []sql.Expression { return i.Exprs } +func (i *MergeableDummyIndex) Database() string { return i.DB } +func (i *MergeableDummyIndex) Driver() string { return i.DriverName } +func (i *MergeableDummyIndex) MemTable() *Table { return i.Tbl } +func (i *MergeableDummyIndex) ColumnExpressions() []sql.Expression { return i.Exprs } -func (i MergeableDummyIndex) Expressions() []string { +func (i *MergeableDummyIndex) Expressions() []string { var exprs []string for _, e := range i.Exprs { exprs = append(exprs, e.String()) @@ -28,31 +28,31 @@ func (i MergeableDummyIndex) Expressions() []string { return exprs } -func (i MergeableDummyIndex) AscendGreaterOrEqual(keys ...interface{}) (sql.IndexLookup, error) { +func (i *MergeableDummyIndex) AscendGreaterOrEqual(keys ...interface{}) (sql.IndexLookup, error) { return &AscendIndexLookup{Gte: keys, Index: i}, nil } -func (i MergeableDummyIndex) AscendLessThan(keys ...interface{}) (sql.IndexLookup, error) { +func (i *MergeableDummyIndex) AscendLessThan(keys ...interface{}) (sql.IndexLookup, error) { return &AscendIndexLookup{Lt: keys, Index: i}, nil } -func (i MergeableDummyIndex) AscendRange(greaterOrEqual, lessThan []interface{}) (sql.IndexLookup, error) { +func (i *MergeableDummyIndex) AscendRange(greaterOrEqual, lessThan []interface{}) (sql.IndexLookup, error) { return &AscendIndexLookup{Gte: greaterOrEqual, Lt: lessThan, Index: i}, nil } -func (i MergeableDummyIndex) DescendGreater(keys ...interface{}) (sql.IndexLookup, error) { +func (i *MergeableDummyIndex) DescendGreater(keys ...interface{}) (sql.IndexLookup, error) { return &DescendIndexLookup{Gt: keys, Index: i}, nil } -func (i MergeableDummyIndex) DescendLessOrEqual(keys ...interface{}) (sql.IndexLookup, error) { +func (i *MergeableDummyIndex) DescendLessOrEqual(keys ...interface{}) (sql.IndexLookup, error) { return &DescendIndexLookup{Lte: keys, Index: i}, nil } -func (i MergeableDummyIndex) DescendRange(lessOrEqual, greaterThan []interface{}) (sql.IndexLookup, error) { +func (i *MergeableDummyIndex) DescendRange(lessOrEqual, greaterThan []interface{}) (sql.IndexLookup, error) { return &DescendIndexLookup{Gt: greaterThan, Lte: lessOrEqual, Index: i}, nil } -func (i MergeableDummyIndex) Not(keys ...interface{}) (sql.IndexLookup, error) { +func (i *MergeableDummyIndex) Not(keys ...interface{}) (sql.IndexLookup, error) { lookup, err := i.Get(keys...) if err != nil { return nil, err @@ -62,15 +62,15 @@ func (i MergeableDummyIndex) Not(keys ...interface{}) (sql.IndexLookup, error) { return &NegateIndexLookup{Lookup: mergeable, Index: mergeable.Index}, nil } -func (i MergeableDummyIndex) Get(key ...interface{}) (sql.IndexLookup, error) { +func (i *MergeableDummyIndex) Get(key ...interface{}) (sql.IndexLookup, error) { return &MergeableIndexLookup{Key: key, Index: i}, nil } -func (i MergeableDummyIndex) Has(sql.Partition, ...interface{}) (bool, error) { +func (i *MergeableDummyIndex) Has(sql.Partition, ...interface{}) (bool, error) { panic("not implemented") } -func (i MergeableDummyIndex) ID() string { +func (i *MergeableDummyIndex) ID() string { if len(i.Exprs) == 1 { return i.Exprs[0].String() } @@ -82,7 +82,7 @@ func (i MergeableDummyIndex) ID() string { return "(" + strings.Join(parts, ", ") + ")" } -func (i MergeableDummyIndex) Table() string { return i.TableName } +func (i *MergeableDummyIndex) Table() string { return i.TableName } type MergeableLookup interface { ID() string diff --git a/sql/analyzer/assign_indexes_test.go b/sql/analyzer/assign_indexes_test.go index e11c679fc..77cc43cfa 100644 --- a/sql/analyzer/assign_indexes_test.go +++ b/sql/analyzer/assign_indexes_test.go @@ -230,8 +230,8 @@ func mergeableIndexLookup(table string, column string, colIdx int, key ...interf } } -func mergeableIndex(table string, column string, colIdx int) memory.MergeableDummyIndex { - return memory.MergeableDummyIndex{ +func mergeableIndex(table string, column string, colIdx int) *memory.MergeableDummyIndex { + return &memory.MergeableDummyIndex{ TableName: table, Exprs: []sql.Expression{col(colIdx, table, column)}, } @@ -504,7 +504,7 @@ func TestGetIndexes(t *testing.T) { "t2": &indexLookup{ &memory.MergeableIndexLookup{ Key: []interface{}{int64(1), int64(2)}, - Index: memory.MergeableDummyIndex{ + Index: &memory.MergeableDummyIndex{ TableName: "t2", Exprs: []sql.Expression{ col(0, "t2", "foo"), @@ -556,7 +556,7 @@ func TestGetIndexes(t *testing.T) { mergeableIndexLookup("t2", "bar", 0, int64(5)), &memory.MergeableIndexLookup{ Key: []interface{}{int64(1), int64(2)}, - Index: memory.MergeableDummyIndex{ + Index: &memory.MergeableDummyIndex{ TableName: "t2", Exprs: []sql.Expression{ col(0, "t2", "foo"), @@ -942,7 +942,6 @@ func TestGetMultiColumnIndexes(t *testing.T) { }, }, { - TableName: "t3", Exprs: []sql.Expression{col(0, "t3", "foo")}, }, @@ -1006,25 +1005,35 @@ func TestGetMultiColumnIndexes(t *testing.T) { expected := map[string]*indexLookup{ "t1": &indexLookup{ - &memory.MergeableIndexLookup{Key: []interface{}{5, 6}}, + &memory.MergeableIndexLookup{ + Key: []interface{}{int64(5), int64(6)}, + Index: indexes[0], + }, []sql.Index{indexes[0]}, }, "t2": &indexLookup{ - &memory.MergeableIndexLookup{Key: []interface{}{1, 2, 3}}, + &memory.MergeableIndexLookup{ + Key: []interface{}{int64(1), int64(2), int64(3)}, + Index: indexes[1], + }, []sql.Index{indexes[1]}, }, "t4": &indexLookup{ &memory.MergedIndexLookup{ - Intersections: []sql.IndexLookup{ + Unions: []sql.IndexLookup{ &memory.AscendIndexLookup{ Gte: []interface{}{int64(1), int64(2)}, Lt: []interface{}{int64(6), int64(5)}, + Index: indexes[4], }, &memory.DescendIndexLookup{ Gt: []interface{}{int64(1), int64(2)}, Lte: []interface{}{int64(6), int64(5)}, + Index: indexes[4], }, - }}, + }, + Index: indexes[4], + }, []sql.Index{indexes[4]}, }, } From 74ad43867ab69a2428a15f7895061336ffd0b570 Mon Sep 17 00:00:00 2001 From: Zach Musgrave Date: Tue, 5 Nov 2019 13:59:15 -0800 Subject: [PATCH 36/42] Fixed pushdown test Signed-off-by: Zach Musgrave --- engine_test.go | 15 +++++++++------ memory/ascend_index.go | 6 +++--- memory/descend_index.go | 4 ++-- sql/analyzer/assign_indexes.go | 8 +++++--- sql/analyzer/pushdown_test.go | 27 ++++++++++++++++++++++++--- 5 files changed, 43 insertions(+), 17 deletions(-) diff --git a/engine_test.go b/engine_test.go index 58ffeebc2..7362b42a9 100644 --- a/engine_test.go +++ b/engine_test.go @@ -1603,20 +1603,23 @@ func TestQueries(t *testing.T) { initializer indexDriverInitalizer } - // Test all queries with these combinations, for a total of 16 runs: + // Test all queries with these combinations, for a total of 12 runs: // 1) Partitioned tables / non partitioned tables // 2) Mergeable / unmergeable / no indexes // 3) Parallelism on / off - // numPartitionsVals := []int{1} - // useIndexesVals := []bool{true} - // parallelVals := []int{1} - numPartitionsVals := []int{1, testNumPartitions} + numPartitionsVals := []int{ + 1, + testNumPartitions, + } indexDrivers := []*indexDriverTestCase{ nil, {"unmergableIndexes", unmergableIndexDriver}, {"mergableIndexes", mergableIndexDriver}, } - parallelVals := []int{1, 2} + parallelVals := []int{ + 1, + 2, + } for _, numPartitions := range numPartitionsVals { for _, indexDriverInit := range indexDrivers { for _, parallelism := range parallelVals { diff --git a/memory/ascend_index.go b/memory/ascend_index.go index 91f287aaa..ff9ae3b24 100755 --- a/memory/ascend_index.go +++ b/memory/ascend_index.go @@ -54,10 +54,10 @@ func (l *AscendIndexLookup) EvalExpression() sql.Expression { return ltexpr } -func (AscendIndexLookup) Difference(...sql.IndexLookup) sql.IndexLookup { +func (*AscendIndexLookup) Difference(...sql.IndexLookup) sql.IndexLookup { panic("ascendIndexLookup.Difference is not implemented") } -func (AscendIndexLookup) Intersection(...sql.IndexLookup) sql.IndexLookup { - panic("ascendIndexLookup.Intersection is not implemented") +func (l *AscendIndexLookup) Intersection(lookups ...sql.IndexLookup) sql.IndexLookup { + return intersection(l.Index, l, lookups...) } diff --git a/memory/descend_index.go b/memory/descend_index.go index 33e140d1d..061707ef0 100755 --- a/memory/descend_index.go +++ b/memory/descend_index.go @@ -58,7 +58,7 @@ func (*DescendIndexLookup) Difference(...sql.IndexLookup) sql.IndexLookup { panic("descendIndexLookup.Difference is not implemented") } -func (*DescendIndexLookup) Intersection(...sql.IndexLookup) sql.IndexLookup { - panic("descendIndexLookup.Intersection is not implemented") +func (l *DescendIndexLookup) Intersection(lookups ...sql.IndexLookup) sql.IndexLookup { + return intersection(l.Index, l, lookups...) } diff --git a/sql/analyzer/assign_indexes.go b/sql/analyzer/assign_indexes.go index 932c4be6d..18220a0cc 100644 --- a/sql/analyzer/assign_indexes.go +++ b/sql/analyzer/assign_indexes.go @@ -322,6 +322,11 @@ func unifyExpressions(aliases map[string]sql.Expression, expr ...sql.Expression) } func betweenIndexLookup(index sql.Index, upper, lower []interface{}) (sql.IndexLookup, error) { + // TODO: two bugs here + // 1) Mergeable and SetOperations are separate interfaces, so a naive integrator could generate a type assertion + // error in this method + // 2) Since AscendRange and DescendRange both accept an upper and lower bound, there is no good reason to require + // both implementations from an index. One will do fine, no need to require both and merge them. ai, isAscend := index.(sql.AscendIndex) di, isDescend := index.(sql.DescendIndex) if isAscend && isDescend { @@ -337,9 +342,6 @@ func betweenIndexLookup(index sql.Index, upper, lower []interface{}) (sql.IndexL m, ok := ascendLookup.(sql.Mergeable) if ok && m.IsMergeable(descendLookup) { - // TODO: two bugs here - // 1) Mergeable and SetOperations are separate interfaces, so a bad integrator could generate a type assertion error here - // 2) This should be Intersection, not Union. Not fixing yet because the test index doesn't implement Intersection. return ascendLookup.(sql.SetOperations).Union(descendLookup), nil } } diff --git a/sql/analyzer/pushdown_test.go b/sql/analyzer/pushdown_test.go index 84de0d487..086328d53 100644 --- a/sql/analyzer/pushdown_test.go +++ b/sql/analyzer/pushdown_test.go @@ -189,7 +189,23 @@ func TestPushdownIndexable(t *testing.T) { ), }).(*memory.Table). WithProjection([]string{"i", "f"}).(*memory.Table). - WithIndexLookup(&memory.MergeableIndexLookup{Key: []interface{}{3.14}}), + WithIndexLookup( + // TODO: this is arguably a bug. These two indexes should not be mergeable, and fetching the values of + // them will not yield correct results with the current implementation of these indexes. + &memory.MergedIndexLookup{ + Intersections: []sql.IndexLookup{ + &memory.MergeableIndexLookup{ + Key: []interface{}{float64(3.14)}, + Index: idx2, + }, + &memory.DescendIndexLookup{ + Gt: []interface{}{1}, + Index: idx1, + }, + }, + Index: idx2, + }, + ), ), plan.NewResolvedTable( table2.WithFilters([]sql.Expression{ @@ -201,8 +217,13 @@ func TestPushdownIndexable(t *testing.T) { ), }).(*memory.Table). WithProjection([]string{"i2"}).(*memory.Table). - // TODO: fix - WithIndexLookup(&memory.NegateIndexLookup{}), + WithIndexLookup(&memory.NegateIndexLookup{ + Lookup: &memory.MergeableIndexLookup{ + Key: []interface{}{2}, + Index: idx3, + }, + Index: idx3, + }), ), ), ), From 99566c98073c6bf68b24aacd179ddcc0c78fd304 Mon Sep 17 00:00:00 2001 From: Zach Musgrave Date: Tue, 5 Nov 2019 13:59:40 -0800 Subject: [PATCH 37/42] Amended comment in pushdown_test Signed-off-by: Zach Musgrave --- sql/analyzer/pushdown_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/analyzer/pushdown_test.go b/sql/analyzer/pushdown_test.go index 086328d53..d1e7ec522 100644 --- a/sql/analyzer/pushdown_test.go +++ b/sql/analyzer/pushdown_test.go @@ -190,7 +190,7 @@ func TestPushdownIndexable(t *testing.T) { }).(*memory.Table). WithProjection([]string{"i", "f"}).(*memory.Table). WithIndexLookup( - // TODO: this is arguably a bug. These two indexes should not be mergeable, and fetching the values of + // TODO: These two indexes should not be mergeable, and fetching the values of // them will not yield correct results with the current implementation of these indexes. &memory.MergedIndexLookup{ Intersections: []sql.IndexLookup{ From c18edf9ea205ada3e9dd6c47cac9d2cf970bc012 Mon Sep 17 00:00:00 2001 From: Zach Musgrave Date: Tue, 5 Nov 2019 14:10:02 -0800 Subject: [PATCH 38/42] Removed unncessary slice of expressions for row matching Signed-off-by: Zach Musgrave --- memory/ascend_index.go | 9 ++++---- memory/descend_index.go | 9 ++++---- memory/mergeable_index.go | 29 ++++++++++++------------ memory/negative_index.go | 9 ++++---- memory/unmergeable_index.go | 45 +++++++++++++++---------------------- 5 files changed, 44 insertions(+), 57 deletions(-) diff --git a/memory/ascend_index.go b/memory/ascend_index.go index ff9ae3b24..c9a9e617b 100755 --- a/memory/ascend_index.go +++ b/memory/ascend_index.go @@ -18,11 +18,10 @@ func (l *AscendIndexLookup) ID() string { return l.id } func (l *AscendIndexLookup) Values(p sql.Partition) (sql.IndexValueIter, error) { return &dummyIndexValueIter{ - tbl: l.Index.MemTable(), - partition: p, - matchExpressions: func() []sql.Expression { - return []sql.Expression { l.EvalExpression() } - }}, nil + tbl: l.Index.MemTable(), + partition: p, + matchExpression: l.EvalExpression(), + }, nil } func (l *AscendIndexLookup) Indexes() []string { diff --git a/memory/descend_index.go b/memory/descend_index.go index 061707ef0..e57bbbd72 100755 --- a/memory/descend_index.go +++ b/memory/descend_index.go @@ -18,11 +18,10 @@ func (l *DescendIndexLookup) ID() string { return l.id } func (l *DescendIndexLookup) Values(p sql.Partition) (sql.IndexValueIter, error) { return &dummyIndexValueIter{ - tbl: l.Index.MemTable(), - partition: p, - matchExpressions: func() []sql.Expression { - return []sql.Expression { l.EvalExpression() } - }}, nil + tbl: l.Index.MemTable(), + partition: p, + matchExpression: l.EvalExpression(), + }, nil } func (l *DescendIndexLookup) EvalExpression() sql.Expression { diff --git a/memory/mergeable_index.go b/memory/mergeable_index.go index c69e9ef0c..9681ae8c4 100755 --- a/memory/mergeable_index.go +++ b/memory/mergeable_index.go @@ -111,17 +111,17 @@ func (i *MergeableIndexLookup) IsMergeable(lookup sql.IndexLookup) bool { } func (i *MergeableIndexLookup) Values(p sql.Partition) (sql.IndexValueIter, error) { + var exprs []sql.Expression + for exprI, expr := range i.Index.ColumnExpressions() { + lit, typ := getType(i.Key[exprI]) + exprs = append(exprs, expression.NewEquals(expr, expression.NewLiteral(lit, typ))) + } + return &dummyIndexValueIter{ - tbl: i.Index.MemTable(), - partition: p, - matchExpressions: func() []sql.Expression { - var exprs []sql.Expression - for exprI, expr := range i.Index.ColumnExpressions() { - lit, typ := getType(i.Key[exprI]) - exprs = append(exprs, expression.NewEquals(expr, expression.NewLiteral(lit, typ))) - } - return exprs - }}, nil + tbl: i.Index.MemTable(), + partition: p, + matchExpression: and(exprs...), + }, nil } func (i *MergeableIndexLookup) EvalExpression() sql.Expression { @@ -246,11 +246,10 @@ func (m *MergedIndexLookup) IsMergeable(lookup sql.IndexLookup) bool { func (m *MergedIndexLookup) Values(p sql.Partition) (sql.IndexValueIter, error) { return &dummyIndexValueIter{ - tbl: m.Index.MemTable(), - partition: p, - matchExpressions: func() []sql.Expression { - return []sql.Expression { m.EvalExpression() } - }}, nil + tbl: m.Index.MemTable(), + partition: p, + matchExpression: m.EvalExpression(), + }, nil } func or(expressions ...sql.Expression) sql.Expression { diff --git a/memory/negative_index.go b/memory/negative_index.go index 1c2e3ab86..fa872a32d 100755 --- a/memory/negative_index.go +++ b/memory/negative_index.go @@ -16,11 +16,10 @@ func (l *NegateIndexLookup) ID() string { return "not " + l.Lookup.ID() } func (l *NegateIndexLookup) Values(p sql.Partition) (sql.IndexValueIter, error) { return &dummyIndexValueIter{ - tbl: l.Index.MemTable(), - partition: p, - matchExpressions: func() []sql.Expression { - return []sql.Expression { l.EvalExpression() } - }}, nil + tbl: l.Index.MemTable(), + partition: p, + matchExpression: l.EvalExpression(), + }, nil } func (l *NegateIndexLookup) EvalExpression() sql.Expression { diff --git a/memory/unmergeable_index.go b/memory/unmergeable_index.go index 0b062cca1..eddcca168 100755 --- a/memory/unmergeable_index.go +++ b/memory/unmergeable_index.go @@ -58,8 +58,7 @@ var _ memoryIndexLookup = (*UnmergeableIndexLookup)(nil) type dummyIndexValueIter struct { tbl *Table partition sql.Partition - // Returns a set of expressions that must match a given row to be included in the value iterator for a lookup - matchExpressions func() []sql.Expression + matchExpression sql.Expression values [][]byte i int } @@ -89,24 +88,16 @@ func (u *dummyIndexValueIter) initValues() error { } for i, row := range rows { - match := true - for _, matchExpr := range u.matchExpressions() { - ok, err := sql.EvaluateCondition(sql.NewEmptyContext(), matchExpr, row) - if err != nil { - return err - } - - if !ok { - match = false - break - } + ok, err := sql.EvaluateCondition(sql.NewEmptyContext(), u.matchExpression, row) + if err != nil { + return err } - if match { - idxVal := &indexValue{ + if ok { + encoded, err := encodeIndexValue(&indexValue{ Pos: i, - } - encoded, err := encodeIndexValue(idxVal) + }) + if err != nil { return err } @@ -152,17 +143,17 @@ func (u *dummyIndexValueIter) Close() error { } func (u *UnmergeableIndexLookup) Values(p sql.Partition) (sql.IndexValueIter, error) { + var exprs []sql.Expression + for exprI, expr := range u.idx.Exprs { + lit, typ := getType(u.key[exprI]) + exprs = append(exprs, expression.NewEquals(expr, expression.NewLiteral(lit, typ))) + } + return &dummyIndexValueIter{ - tbl: u.idx.Tbl, - partition: p, - matchExpressions: func() []sql.Expression { - var exprs []sql.Expression - for exprI, expr := range u.idx.Exprs { - lit, typ := getType(u.key[exprI]) - exprs = append(exprs, expression.NewEquals(expr, expression.NewLiteral(lit, typ))) - } - return exprs - }}, nil + tbl: u.idx.Tbl, + partition: p, + matchExpression: and(exprs...), + }, nil } func (u *UnmergeableIndexLookup) Indexes() []string { From 2f78b7fb22f2e84893a5d6a384f5830296620b34 Mon Sep 17 00:00:00 2001 From: Zach Musgrave Date: Tue, 5 Nov 2019 15:00:40 -0800 Subject: [PATCH 39/42] Cleaned up and added better comments on a couple types Signed-off-by: Zach Musgrave --- memory/ascend_index.go | 5 +++-- memory/descend_index.go | 5 +++-- memory/index_driver.go | 4 ++++ memory/mergeable_index.go | 37 +++++++++++++++++++++++-------------- memory/negative_index.go | 5 +++-- memory/unmergeable_index.go | 12 ++---------- 6 files changed, 38 insertions(+), 30 deletions(-) diff --git a/memory/ascend_index.go b/memory/ascend_index.go index c9a9e617b..0395f0484 100755 --- a/memory/ascend_index.go +++ b/memory/ascend_index.go @@ -28,8 +28,9 @@ func (l *AscendIndexLookup) Indexes() []string { return []string{l.id} } -func (l *AscendIndexLookup) IsMergeable(sql.IndexLookup) bool { - return true +func (l *AscendIndexLookup) IsMergeable(lookup sql.IndexLookup) bool { + _, ok := lookup.(MergeableLookup) + return ok } func (l *AscendIndexLookup) Union(lookups ...sql.IndexLookup) sql.IndexLookup { diff --git a/memory/descend_index.go b/memory/descend_index.go index e57bbbd72..2d38520cd 100755 --- a/memory/descend_index.go +++ b/memory/descend_index.go @@ -45,8 +45,9 @@ func (l *DescendIndexLookup) Indexes() []string { return []string{l.id} } -func (l *DescendIndexLookup) IsMergeable(sql.IndexLookup) bool { - return true +func (l *DescendIndexLookup) IsMergeable(lookup sql.IndexLookup) bool { + _, ok := lookup.(MergeableLookup) + return ok } func (l *DescendIndexLookup) Union(lookups ...sql.IndexLookup) sql.IndexLookup { diff --git a/memory/index_driver.go b/memory/index_driver.go index e05c2f48a..9fea48bfa 100755 --- a/memory/index_driver.go +++ b/memory/index_driver.go @@ -6,11 +6,15 @@ import ( const IndexDriverId = "MemoryIndexDriver" +// TestIndexDriver is a non-performant index driver meant to aid in verification of engine correctness. It can not +// create or delete indexes, but will use the index types defined in this package to alter how queries are executed, +// retrieving values from the indexes rather than from the tables directly. type TestIndexDriver struct { db string indexes map[string][]sql.Index } +// NewIndexDriver returns a new index driver for database and the indexes given, keyed by the table name. func NewIndexDriver(db string, indexes map[string][]sql.Index) *TestIndexDriver { return &TestIndexDriver{db: db, indexes: indexes} } diff --git a/memory/mergeable_index.go b/memory/mergeable_index.go index 9681ae8c4..d7bc6bb3c 100755 --- a/memory/mergeable_index.go +++ b/memory/mergeable_index.go @@ -84,26 +84,34 @@ func (i *MergeableDummyIndex) ID() string { func (i *MergeableDummyIndex) Table() string { return i.TableName } +// All lookups in this package, except for UnmergeableLookup, are MergeableLookups. The IDs are mostly for testing / +// verification purposes. type MergeableLookup interface { ID() string } -// ExpressionsIndex is an index made out of one or more expressions (usually field expressions) +// ExpressionsIndex is an index made out of one or more expressions (usually field expressions), linked to a Table. type ExpressionsIndex interface { MemTable() *Table ColumnExpressions() []sql.Expression } +// MergeableIndexLookup is a lookup linked to an ExpressionsIndex. It can be merged with any other MergeableIndexLookup. All lookups in this package are Merge type MergeableIndexLookup struct { - Key []interface{} - Index ExpressionsIndex + Key []interface{} + Index ExpressionsIndex +} + +// memoryIndexLookup is a lookup that defines an expression to evaluate which rows are part of the index values +type memoryIndexLookup interface { + EvalExpression() sql.Expression } var _ sql.Mergeable = (*MergeableIndexLookup)(nil) var _ sql.SetOperations = (*MergeableIndexLookup)(nil) var _ memoryIndexLookup = (*MergeableIndexLookup)(nil) -func (i *MergeableIndexLookup) ID() string { return strings.Join(i.Indexes(), ",") } +func (i *MergeableIndexLookup) ID() string { return strings.Join(i.Indexes(), ",") } func (i *MergeableIndexLookup) IsMergeable(lookup sql.IndexLookup) bool { _, ok := lookup.(MergeableLookup) @@ -198,8 +206,9 @@ func union(idx ExpressionsIndex, left sql.IndexLookup, lookups ...sql.IndexLooku } } -// An index lookup that has been merged with another. -// Exactly one of the Unions or Intersections fields should be set. +// MergedIndexLookup is an index lookup that has been merged with another. +// Exactly one of the Unions or Intersections fields should be set, and correspond to a logical AND or OR operation, +// respectively. type MergedIndexLookup struct { Unions []sql.IndexLookup Intersections []sql.IndexLookup @@ -252,6 +261,14 @@ func (m *MergedIndexLookup) Values(p sql.Partition) (sql.IndexValueIter, error) }, nil } +func (m *MergedIndexLookup) Indexes() []string { + panic("implement me") +} + +func (m *MergedIndexLookup) ID() string { + panic("implement me") +} + func or(expressions ...sql.Expression) sql.Expression { if len(expressions) == 1 { return expressions[0] @@ -264,12 +281,4 @@ func and(expressions ...sql.Expression) sql.Expression { return expressions[0] } return expression.NewAnd(expressions[0], and(expressions[1:]...)) -} - -func (m *MergedIndexLookup) Indexes() []string { - panic("implement me") -} - -func (m *MergedIndexLookup) ID() string { - panic("implement me") } \ No newline at end of file diff --git a/memory/negative_index.go b/memory/negative_index.go index fa872a32d..8983b4876 100755 --- a/memory/negative_index.go +++ b/memory/negative_index.go @@ -30,8 +30,9 @@ func (l *NegateIndexLookup) Indexes() []string { return []string{l.ID()} } -func (*NegateIndexLookup) IsMergeable(sql.IndexLookup) bool { - return true +func (*NegateIndexLookup) IsMergeable(lookup sql.IndexLookup) bool { + _, ok := lookup.(MergeableLookup) + return ok } func (l *NegateIndexLookup) Union(lookups ...sql.IndexLookup) sql.IndexLookup { diff --git a/memory/unmergeable_index.go b/memory/unmergeable_index.go index eddcca168..33c97f6bd 100755 --- a/memory/unmergeable_index.go +++ b/memory/unmergeable_index.go @@ -37,21 +37,13 @@ func (u *UnmergeableDummyIndex) Get(key ...interface{}) (sql.IndexLookup, error) }, nil } +// UnmergeableIndexLookup is the only IndexLookup in this package that doesn't implement Mergeable, and therefore +// can't be merged with other lookups. type UnmergeableIndexLookup struct { key []interface{} idx *UnmergeableDummyIndex } -func (u *UnmergeableIndexLookup) EvalExpression() sql.Expression { - return nil -} - -type memoryIndexLookup interface { - EvalExpression() sql.Expression -} - -var _ memoryIndexLookup = (*UnmergeableIndexLookup)(nil) - // dummyIndexValueIter does a very simple and verifiable iteration over the table values for a given index. It does this // by iterating over all the table rows for a partition and evaluating each of them for inclusion in the index. This is // not an efficient way to store an index, and is only suitable for testing the correctness of index code in the engine. From cc348d52b3bbe25d419654c7e6afb45cbe021e8a Mon Sep 17 00:00:00 2001 From: Zach Musgrave Date: Tue, 5 Nov 2019 15:50:21 -0800 Subject: [PATCH 40/42] Renamed UnmergeableDummyIndex and MergeableDummyIndex to remove the Dummy from the name. Moved a few interface assertions left over from assignindex_test into the type files. Signed-off-by: Zach Musgrave --- engine_test.go | 8 +++--- memory/ascend_index.go | 2 +- memory/descend_index.go | 2 +- memory/mergeable_index.go | 43 ++++++++++++++++------------- memory/negative_index.go | 2 +- memory/unmergeable_index.go | 30 ++++++++++---------- sql/analyzer/assign_indexes_test.go | 34 ++++++++++------------- sql/analyzer/pushdown_test.go | 6 ++-- 8 files changed, 64 insertions(+), 63 deletions(-) diff --git a/engine_test.go b/engine_test.go index 7362b42a9..f37e76cdf 100644 --- a/engine_test.go +++ b/engine_test.go @@ -1717,8 +1717,8 @@ func mergableIndexDriver(tables map[string]*memory.Table) sql.IndexDriver { } -func newUnmergableIndex(tables map[string]*memory.Table, tableName string, exprs ...sql.Expression) *memory.UnmergeableDummyIndex { - return &memory.UnmergeableDummyIndex{ +func newUnmergableIndex(tables map[string]*memory.Table, tableName string, exprs ...sql.Expression) *memory.UnmergeableIndex { + return &memory.UnmergeableIndex{ DB: "mydb", DriverName: memory.IndexDriverId, TableName: tableName, @@ -1727,8 +1727,8 @@ func newUnmergableIndex(tables map[string]*memory.Table, tableName string, exprs } } -func newMergableIndex(tables map[string]*memory.Table, tableName string, exprs ...sql.Expression) *memory.MergeableDummyIndex { - return &memory.MergeableDummyIndex { +func newMergableIndex(tables map[string]*memory.Table, tableName string, exprs ...sql.Expression) *memory.MergeableIndex { + return &memory.MergeableIndex{ DB: "mydb", DriverName: memory.IndexDriverId, TableName: tableName, diff --git a/memory/ascend_index.go b/memory/ascend_index.go index 0395f0484..71b74e884 100755 --- a/memory/ascend_index.go +++ b/memory/ascend_index.go @@ -17,7 +17,7 @@ var _ memoryIndexLookup = (*AscendIndexLookup)(nil) func (l *AscendIndexLookup) ID() string { return l.id } func (l *AscendIndexLookup) Values(p sql.Partition) (sql.IndexValueIter, error) { - return &dummyIndexValueIter{ + return &indexValIter{ tbl: l.Index.MemTable(), partition: p, matchExpression: l.EvalExpression(), diff --git a/memory/descend_index.go b/memory/descend_index.go index 2d38520cd..3a8bead19 100755 --- a/memory/descend_index.go +++ b/memory/descend_index.go @@ -17,7 +17,7 @@ var _ memoryIndexLookup = (*DescendIndexLookup)(nil) func (l *DescendIndexLookup) ID() string { return l.id } func (l *DescendIndexLookup) Values(p sql.Partition) (sql.IndexValueIter, error) { - return &dummyIndexValueIter{ + return &indexValIter{ tbl: l.Index.MemTable(), partition: p, matchExpression: l.EvalExpression(), diff --git a/memory/mergeable_index.go b/memory/mergeable_index.go index d7bc6bb3c..2b58dbf1c 100755 --- a/memory/mergeable_index.go +++ b/memory/mergeable_index.go @@ -7,7 +7,7 @@ import ( "strings" ) -type MergeableDummyIndex struct { +type MergeableIndex struct { DB string // required for engine tests with driver DriverName string // required for engine tests with driver Tbl *Table // required for engine tests with driver @@ -15,12 +15,17 @@ type MergeableDummyIndex struct { Exprs []sql.Expression } -func (i *MergeableDummyIndex) Database() string { return i.DB } -func (i *MergeableDummyIndex) Driver() string { return i.DriverName } -func (i *MergeableDummyIndex) MemTable() *Table { return i.Tbl } -func (i *MergeableDummyIndex) ColumnExpressions() []sql.Expression { return i.Exprs } +var _ sql.Index = (*MergeableIndex)(nil) +var _ sql.AscendIndex = (*MergeableIndex)(nil) +var _ sql.DescendIndex = (*MergeableIndex)(nil) +var _ sql.NegateIndex = (*MergeableIndex)(nil) -func (i *MergeableDummyIndex) Expressions() []string { +func (i *MergeableIndex) Database() string { return i.DB } +func (i *MergeableIndex) Driver() string { return i.DriverName } +func (i *MergeableIndex) MemTable() *Table { return i.Tbl } +func (i *MergeableIndex) ColumnExpressions() []sql.Expression { return i.Exprs } + +func (i *MergeableIndex) Expressions() []string { var exprs []string for _, e := range i.Exprs { exprs = append(exprs, e.String()) @@ -28,31 +33,31 @@ func (i *MergeableDummyIndex) Expressions() []string { return exprs } -func (i *MergeableDummyIndex) AscendGreaterOrEqual(keys ...interface{}) (sql.IndexLookup, error) { +func (i *MergeableIndex) AscendGreaterOrEqual(keys ...interface{}) (sql.IndexLookup, error) { return &AscendIndexLookup{Gte: keys, Index: i}, nil } -func (i *MergeableDummyIndex) AscendLessThan(keys ...interface{}) (sql.IndexLookup, error) { +func (i *MergeableIndex) AscendLessThan(keys ...interface{}) (sql.IndexLookup, error) { return &AscendIndexLookup{Lt: keys, Index: i}, nil } -func (i *MergeableDummyIndex) AscendRange(greaterOrEqual, lessThan []interface{}) (sql.IndexLookup, error) { +func (i *MergeableIndex) AscendRange(greaterOrEqual, lessThan []interface{}) (sql.IndexLookup, error) { return &AscendIndexLookup{Gte: greaterOrEqual, Lt: lessThan, Index: i}, nil } -func (i *MergeableDummyIndex) DescendGreater(keys ...interface{}) (sql.IndexLookup, error) { +func (i *MergeableIndex) DescendGreater(keys ...interface{}) (sql.IndexLookup, error) { return &DescendIndexLookup{Gt: keys, Index: i}, nil } -func (i *MergeableDummyIndex) DescendLessOrEqual(keys ...interface{}) (sql.IndexLookup, error) { +func (i *MergeableIndex) DescendLessOrEqual(keys ...interface{}) (sql.IndexLookup, error) { return &DescendIndexLookup{Lte: keys, Index: i}, nil } -func (i *MergeableDummyIndex) DescendRange(lessOrEqual, greaterThan []interface{}) (sql.IndexLookup, error) { +func (i *MergeableIndex) DescendRange(lessOrEqual, greaterThan []interface{}) (sql.IndexLookup, error) { return &DescendIndexLookup{Gt: greaterThan, Lte: lessOrEqual, Index: i}, nil } -func (i *MergeableDummyIndex) Not(keys ...interface{}) (sql.IndexLookup, error) { +func (i *MergeableIndex) Not(keys ...interface{}) (sql.IndexLookup, error) { lookup, err := i.Get(keys...) if err != nil { return nil, err @@ -62,15 +67,15 @@ func (i *MergeableDummyIndex) Not(keys ...interface{}) (sql.IndexLookup, error) return &NegateIndexLookup{Lookup: mergeable, Index: mergeable.Index}, nil } -func (i *MergeableDummyIndex) Get(key ...interface{}) (sql.IndexLookup, error) { +func (i *MergeableIndex) Get(key ...interface{}) (sql.IndexLookup, error) { return &MergeableIndexLookup{Key: key, Index: i}, nil } -func (i *MergeableDummyIndex) Has(sql.Partition, ...interface{}) (bool, error) { +func (i *MergeableIndex) Has(sql.Partition, ...interface{}) (bool, error) { panic("not implemented") } -func (i *MergeableDummyIndex) ID() string { +func (i *MergeableIndex) ID() string { if len(i.Exprs) == 1 { return i.Exprs[0].String() } @@ -82,7 +87,7 @@ func (i *MergeableDummyIndex) ID() string { return "(" + strings.Join(parts, ", ") + ")" } -func (i *MergeableDummyIndex) Table() string { return i.TableName } +func (i *MergeableIndex) Table() string { return i.TableName } // All lookups in this package, except for UnmergeableLookup, are MergeableLookups. The IDs are mostly for testing / // verification purposes. @@ -125,7 +130,7 @@ func (i *MergeableIndexLookup) Values(p sql.Partition) (sql.IndexValueIter, erro exprs = append(exprs, expression.NewEquals(expr, expression.NewLiteral(lit, typ))) } - return &dummyIndexValueIter{ + return &indexValIter{ tbl: i.Index.MemTable(), partition: p, matchExpression: and(exprs...), @@ -254,7 +259,7 @@ func (m *MergedIndexLookup) IsMergeable(lookup sql.IndexLookup) bool { } func (m *MergedIndexLookup) Values(p sql.Partition) (sql.IndexValueIter, error) { - return &dummyIndexValueIter{ + return &indexValIter{ tbl: m.Index.MemTable(), partition: p, matchExpression: m.EvalExpression(), diff --git a/memory/negative_index.go b/memory/negative_index.go index 8983b4876..830b75454 100755 --- a/memory/negative_index.go +++ b/memory/negative_index.go @@ -15,7 +15,7 @@ var _ memoryIndexLookup = (*NegateIndexLookup)(nil) func (l *NegateIndexLookup) ID() string { return "not " + l.Lookup.ID() } func (l *NegateIndexLookup) Values(p sql.Partition) (sql.IndexValueIter, error) { - return &dummyIndexValueIter{ + return &indexValIter{ tbl: l.Index.MemTable(), partition: p, matchExpression: l.EvalExpression(), diff --git a/memory/unmergeable_index.go b/memory/unmergeable_index.go index 33c97f6bd..8eeecf950 100755 --- a/memory/unmergeable_index.go +++ b/memory/unmergeable_index.go @@ -11,7 +11,7 @@ import ( // A very dumb index that iterates over the rows of a table, evaluates its matching expressions against each row, and // stores those values to be later retrieved. Only here to test the functionality of indexed queries. This kind of index // cannot be merged with any other index. -type UnmergeableDummyIndex struct { +type UnmergeableIndex struct { DB string // required for engine tests with driver DriverName string // required for engine tests with driver Tbl *Table // required for engine tests with driver @@ -19,10 +19,12 @@ type UnmergeableDummyIndex struct { Exprs []sql.Expression } -func (u *UnmergeableDummyIndex) Database() string { return u.DB } -func (u *UnmergeableDummyIndex) Driver() string { return u.DriverName } +var _ sql.Index = (*UnmergeableIndex)(nil) -func (u *UnmergeableDummyIndex) Expressions() []string { +func (u *UnmergeableIndex) Database() string { return u.DB } +func (u *UnmergeableIndex) Driver() string { return u.DriverName } + +func (u *UnmergeableIndex) Expressions() []string { var exprs []string for _, e := range u.Exprs { exprs = append(exprs, e.String()) @@ -30,7 +32,7 @@ func (u *UnmergeableDummyIndex) Expressions() []string { return exprs } -func (u *UnmergeableDummyIndex) Get(key ...interface{}) (sql.IndexLookup, error) { +func (u *UnmergeableIndex) Get(key ...interface{}) (sql.IndexLookup, error) { return &UnmergeableIndexLookup{ key: key, idx: u, @@ -41,13 +43,13 @@ func (u *UnmergeableDummyIndex) Get(key ...interface{}) (sql.IndexLookup, error) // can't be merged with other lookups. type UnmergeableIndexLookup struct { key []interface{} - idx *UnmergeableDummyIndex + idx *UnmergeableIndex } // dummyIndexValueIter does a very simple and verifiable iteration over the table values for a given index. It does this // by iterating over all the table rows for a partition and evaluating each of them for inclusion in the index. This is // not an efficient way to store an index, and is only suitable for testing the correctness of index code in the engine. -type dummyIndexValueIter struct { +type indexValIter struct { tbl *Table partition sql.Partition matchExpression sql.Expression @@ -55,7 +57,7 @@ type dummyIndexValueIter struct { i int } -func (u *dummyIndexValueIter) Next() ([]byte, error) { +func (u *indexValIter) Next() ([]byte, error) { err := u.initValues() if err != nil { return nil, err @@ -70,7 +72,7 @@ func (u *dummyIndexValueIter) Next() ([]byte, error) { return nil, io.EOF } -func (u *dummyIndexValueIter) initValues() error { +func (u *indexValIter) initValues() error { if u.values == nil { rows, ok := u.tbl.partitions[string(u.partition.Key())] if !ok { @@ -130,7 +132,7 @@ func getType(val interface{}) (interface{}, sql.Type) { } } -func (u *dummyIndexValueIter) Close() error { +func (u *indexValIter) Close() error { return nil } @@ -141,7 +143,7 @@ func (u *UnmergeableIndexLookup) Values(p sql.Partition) (sql.IndexValueIter, er exprs = append(exprs, expression.NewEquals(expr, expression.NewLiteral(lit, typ))) } - return &dummyIndexValueIter{ + return &indexValIter{ tbl: u.idx.Tbl, partition: p, matchExpression: and(exprs...), @@ -156,11 +158,11 @@ func (u *UnmergeableIndexLookup) Indexes() []string { return idxes } -func (u *UnmergeableDummyIndex) Has(partition sql.Partition, key ...interface{}) (bool, error) { +func (u *UnmergeableIndex) Has(partition sql.Partition, key ...interface{}) (bool, error) { panic("unimplemented") } -func (u *UnmergeableDummyIndex) ID() string { +func (u *UnmergeableIndex) ID() string { if len(u.Exprs) == 1 { return u.Exprs[0].String() } @@ -172,6 +174,6 @@ func (u *UnmergeableDummyIndex) ID() string { return "(" + strings.Join(parts, ", ") + ")" } -func (u *UnmergeableDummyIndex) Table() string { +func (u *UnmergeableIndex) Table() string { return u.TableName } diff --git a/sql/analyzer/assign_indexes_test.go b/sql/analyzer/assign_indexes_test.go index 77cc43cfa..c33a41de6 100644 --- a/sql/analyzer/assign_indexes_test.go +++ b/sql/analyzer/assign_indexes_test.go @@ -14,7 +14,7 @@ func TestNegateIndex(t *testing.T) { require := require.New(t) catalog := sql.NewCatalog() - idx1 := &memory.MergeableDummyIndex{ + idx1 := &memory.MergeableIndex{ TableName: "t1", Exprs: []sql.Expression{ expression.NewGetFieldWithTable(0, sql.Int64, "t1", "foo", false), @@ -59,7 +59,7 @@ func TestAssignIndexes(t *testing.T) { require := require.New(t) catalog := sql.NewCatalog() - idx1 := &memory.MergeableDummyIndex{ + idx1 := &memory.MergeableIndex{ TableName: "t2", Exprs: []sql.Expression{ expression.NewGetFieldWithTable(0, sql.Int64, "t2", "bar", false), @@ -70,7 +70,7 @@ func TestAssignIndexes(t *testing.T) { close(done) <-ready - idx2 := &memory.MergeableDummyIndex{ + idx2 := &memory.MergeableIndex{ TableName: "t1", Exprs: []sql.Expression{ expression.NewGetFieldWithTable(0, sql.Int64, "t1", "foo", false), @@ -82,7 +82,7 @@ func TestAssignIndexes(t *testing.T) { close(done) <-ready - idx3 := &memory.UnmergeableDummyIndex{ + idx3 := &memory.UnmergeableIndex{ TableName: "t1", Exprs: []sql.Expression{ expression.NewGetFieldWithTable(0, sql.Int64, "t1", "bar", false), @@ -230,8 +230,8 @@ func mergeableIndexLookup(table string, column string, colIdx int, key ...interf } } -func mergeableIndex(table string, column string, colIdx int) *memory.MergeableDummyIndex { - return &memory.MergeableDummyIndex{ +func mergeableIndex(table string, column string, colIdx int) *memory.MergeableIndex { + return &memory.MergeableIndex{ TableName: table, Exprs: []sql.Expression{col(colIdx, table, column)}, } @@ -239,26 +239,26 @@ func mergeableIndex(table string, column string, colIdx int) *memory.MergeableDu func TestGetIndexes(t *testing.T) { indexes := []sql.Index { - &memory.MergeableDummyIndex{ + &memory.MergeableIndex{ TableName: "t1", Exprs: []sql.Expression{ col(0, "t1", "bar"), }, }, - &memory.MergeableDummyIndex{ + &memory.MergeableIndex{ TableName: "t2", Exprs: []sql.Expression{ col(0, "t2", "foo"), col(0, "t2", "bar"), }, }, - &memory.MergeableDummyIndex{ + &memory.MergeableIndex{ TableName: "t2", Exprs: []sql.Expression{ col(0, "t2", "bar"), }, }, - &memory.UnmergeableDummyIndex{ + &memory.UnmergeableIndex{ TableName: "t3", Exprs: []sql.Expression{ col(0, "t3", "foo"), @@ -504,7 +504,7 @@ func TestGetIndexes(t *testing.T) { "t2": &indexLookup{ &memory.MergeableIndexLookup{ Key: []interface{}{int64(1), int64(2)}, - Index: &memory.MergeableDummyIndex{ + Index: &memory.MergeableIndex{ TableName: "t2", Exprs: []sql.Expression{ col(0, "t2", "foo"), @@ -556,7 +556,7 @@ func TestGetIndexes(t *testing.T) { mergeableIndexLookup("t2", "bar", 0, int64(5)), &memory.MergeableIndexLookup{ Key: []interface{}{int64(1), int64(2)}, - Index: &memory.MergeableDummyIndex{ + Index: &memory.MergeableIndex{ TableName: "t2", Exprs: []sql.Expression{ col(0, "t2", "foo"), @@ -918,7 +918,7 @@ func TestGetMultiColumnIndexes(t *testing.T) { require := require.New(t) catalog := sql.NewCatalog() - indexes := []*memory.MergeableDummyIndex{ + indexes := []*memory.MergeableIndex{ { TableName: "t1", Exprs: []sql.Expression{ @@ -1134,16 +1134,10 @@ func (DummyIndexLookup) Values(sql.Partition) (sql.IndexValueIter, error) { return nil, nil } -var _ sql.Index = (*memory.MergeableDummyIndex)(nil) -var _ sql.Index = (*memory.UnmergeableDummyIndex)(nil) -var _ sql.AscendIndex = (*memory.MergeableDummyIndex)(nil) -var _ sql.DescendIndex = (*memory.MergeableDummyIndex)(nil) -var _ sql.NegateIndex = (*memory.MergeableDummyIndex)(nil) - func TestIndexesIntersection(t *testing.T) { require := require.New(t) - idx1, idx2 := &memory.MergeableDummyIndex{TableName: "bar"}, &memory.MergeableDummyIndex{TableName: "foo"} + idx1, idx2 := &memory.MergeableIndex{TableName: "bar"}, &memory.MergeableIndex{TableName: "foo"} left := map[string]*indexLookup{ "a": &indexLookup{&memory.MergeableIndexLookup{Key: []interface{}{"a"}}, nil}, diff --git a/sql/analyzer/pushdown_test.go b/sql/analyzer/pushdown_test.go index d1e7ec522..c49839159 100644 --- a/sql/analyzer/pushdown_test.go +++ b/sql/analyzer/pushdown_test.go @@ -105,7 +105,7 @@ func TestPushdownIndexable(t *testing.T) { catalog := sql.NewCatalog() catalog.AddDatabase(db) - idx1 := &memory.MergeableDummyIndex{ + idx1 := &memory.MergeableIndex{ TableName: "mytable", Exprs: []sql.Expression{ expression.NewGetFieldWithTable(0, sql.Int32, "mytable", "i", false), @@ -116,7 +116,7 @@ func TestPushdownIndexable(t *testing.T) { close(done) <-ready - idx2 := &memory.MergeableDummyIndex{ + idx2 := &memory.MergeableIndex{ TableName: "mytable", Exprs: []sql.Expression{ expression.NewGetFieldWithTable(1, sql.Float64, "mytable", "f", false), @@ -127,7 +127,7 @@ func TestPushdownIndexable(t *testing.T) { close(done) <-ready - idx3 := &memory.MergeableDummyIndex{ + idx3 := &memory.MergeableIndex{ TableName: "mytable2", Exprs: []sql.Expression{ expression.NewGetFieldWithTable(0, sql.Int32, "mytable2", "i2", false), From d166b52108f500f405b3e193c9f61f19f021d7a5 Mon Sep 17 00:00:00 2001 From: Zach Musgrave Date: Wed, 6 Nov 2019 11:09:54 -0800 Subject: [PATCH 41/42] Standardized panic messages Signed-off-by: Zach Musgrave --- memory/mergeable_index.go | 4 ++-- memory/unmergeable_index.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/memory/mergeable_index.go b/memory/mergeable_index.go index 2b58dbf1c..e50268ba1 100755 --- a/memory/mergeable_index.go +++ b/memory/mergeable_index.go @@ -267,11 +267,11 @@ func (m *MergedIndexLookup) Values(p sql.Partition) (sql.IndexValueIter, error) } func (m *MergedIndexLookup) Indexes() []string { - panic("implement me") + panic("not implemented") } func (m *MergedIndexLookup) ID() string { - panic("implement me") + panic("not implemented") } func or(expressions ...sql.Expression) sql.Expression { diff --git a/memory/unmergeable_index.go b/memory/unmergeable_index.go index 8eeecf950..b9c9bcb81 100755 --- a/memory/unmergeable_index.go +++ b/memory/unmergeable_index.go @@ -159,7 +159,7 @@ func (u *UnmergeableIndexLookup) Indexes() []string { } func (u *UnmergeableIndex) Has(partition sql.Partition, key ...interface{}) (bool, error) { - panic("unimplemented") + panic("not implemented") } func (u *UnmergeableIndex) ID() string { From 59141259ac045fbf9c2cb3988be881de8f6c6821 Mon Sep 17 00:00:00 2001 From: Zach Musgrave Date: Wed, 6 Nov 2019 11:33:56 -0800 Subject: [PATCH 42/42] Removed unused variable assignments Signed-off-by: Zach Musgrave --- sql/analyzer/assign_indexes_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sql/analyzer/assign_indexes_test.go b/sql/analyzer/assign_indexes_test.go index c33a41de6..58e45487a 100644 --- a/sql/analyzer/assign_indexes_test.go +++ b/sql/analyzer/assign_indexes_test.go @@ -166,7 +166,7 @@ func TestAssignIndexes(t *testing.T) { result, err = assignIndexes(a, node) require.NoError(err) - lookupIdxs, ok = result["t1"] + _, ok = result["t1"] require.False(ok) node = plan.NewProject( @@ -183,7 +183,7 @@ func TestAssignIndexes(t *testing.T) { result, err = assignIndexes(a, node) require.NoError(err) - lookupIdxs, ok = result["t1"] + _, ok = result["t1"] require.False(ok) }