tomas/pocketbase
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

added search skipTotal support

Browse Source
This commit is contained in:
Gani Georgiev
2023-07-21 23:24:36 +03:00
parent 1e4c665b53
commit 437843084b
6 changed files with 225 additions and 82 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/search/provider.go
+90 -46
View File
@@ -7,20 +7,22 @@ import (
"strconv"
"github.com/pocketbase/dbx"
"golang.org/x/sync/errgroup"
)
// DefaultPerPage specifies the default returned search result items.
const DefaultPerPage int = 30
// MaxPerPage specifies the maximum allowed search result items returned in a single page.
const MaxPerPage int = 500
const MaxPerPage int = 1000
// url search query params
const (
PageQueryParam string = "page"
PerPageQueryParam string = "perPage"
SortQueryParam string = "sort"
FilterQueryParam string = "filter"
PageQueryParam string = "page"
PerPageQueryParam string = "perPage"
SortQueryParam string = "sort"
FilterQueryParam string = "filter"
SkipTotalQueryParam string = "skipTotal"
)
// Result defines the returned search result structure.
@@ -36,6 +38,7 @@ type Result struct {
type Provider struct {
fieldResolver FieldResolver
query *dbx.SelectQuery
skipTotal bool
countCol string
page int
perPage int
@@ -57,7 +60,7 @@ type Provider struct {
func NewProvider(fieldResolver FieldResolver) *Provider {
return &Provider{
fieldResolver: fieldResolver,
countCol: "_rowid_",
countCol: "id",
page: 1,
perPage: DefaultPerPage,
sort: []SortField{},
@@ -71,8 +74,16 @@ func (s *Provider) Query(query *dbx.SelectQuery) *Provider {
return s
}
// CountCol allows changing the default column (_rowid_) that is used
// SkipTotal changes the `skipTotal` field of the current search provider.
func (s *Provider) SkipTotal(skipTotal bool) *Provider {
s.skipTotal = skipTotal
return s
}
// CountCol allows changing the default column (id) that is used
// to generated the COUNT SQL query statement.
//
// This field is ignored if skipTotal is true.
func (s *Provider) CountCol(name string) *Provider {
s.countCol = name
return s
@@ -132,30 +143,38 @@ func (s *Provider) Parse(urlQuery string) error {
return err
}
if rawPage := params.Get(PageQueryParam); rawPage != "" {
page, err := strconv.Atoi(rawPage)
if raw := params.Get(SkipTotalQueryParam); raw != "" {
v, err := strconv.ParseBool(raw)
if err != nil {
return err
}
s.Page(page)
s.SkipTotal(v)
}
if rawPerPage := params.Get(PerPageQueryParam); rawPerPage != "" {
perPage, err := strconv.Atoi(rawPerPage)
if raw := params.Get(PageQueryParam); raw != "" {
v, err := strconv.Atoi(raw)
if err != nil {
return err
}
s.PerPage(perPage)
s.Page(v)
}
if rawSort := params.Get(SortQueryParam); rawSort != "" {
for _, sortField := range ParseSortFromString(rawSort) {
if raw := params.Get(PerPageQueryParam); raw != "" {
v, err := strconv.Atoi(raw)
if err != nil {
return err
}
s.PerPage(v)
}
if raw := params.Get(SortQueryParam); raw != "" {
for _, sortField := range ParseSortFromString(raw) {
s.AddSort(sortField)
}
}
if rawFilter := params.Get(FilterQueryParam); rawFilter != "" {
s.AddFilter(FilterData(rawFilter))
if raw := params.Get(FilterQueryParam); raw != "" {
s.AddFilter(FilterData(raw))
}
return nil
@@ -165,10 +184,10 @@ func (s *Provider) Parse(urlQuery string) error {
// the provided `items` slice with the found models.
func (s *Provider) Exec(items any) (*Result, error) {
if s.query == nil {
return nil, errors.New("Query is not set.")
return nil, errors.New("query is not set")
}
// clone provider's query
// shallow clone the provider's query
modelsQuery := *s.query
// build filters
@@ -198,18 +217,9 @@ func (s *Provider) Exec(items any) (*Result, error) {
return nil, err
}
queryInfo := modelsQuery.Info()
// count
var totalCount int64
var baseTable string
if len(queryInfo.From) > 0 {
baseTable = queryInfo.From[0]
}
clone := modelsQuery
countQuery := clone.Distinct(false).Select("COUNT(DISTINCT [[" + baseTable + "." + s.countCol + "]])").OrderBy()
if err := countQuery.Row(&totalCount); err != nil {
return nil, err
// normalize page
if s.page <= 0 {
s.page = 1
}
// normalize perPage
@@ -219,31 +229,65 @@ func (s *Provider) Exec(items any) (*Result, error) {
s.perPage = MaxPerPage
}
totalPages := int(math.Ceil(float64(totalCount) / float64(s.perPage)))
// negative value to differentiate from the zero default
totalCount := -1
totalPages := -1
// normalize page according to the total count
if s.page <= 0 || totalCount == 0 {
s.page = 1
} else if s.page > totalPages {
s.page = totalPages
// prepare a count query from the base one
countQuery := modelsQuery // shallow clone
countExec := func() error {
queryInfo := countQuery.Info()
countCol := s.countCol
if len(queryInfo.From) > 0 {
countCol = queryInfo.From[0] + "." + countCol
}
// note: countQuery is shallow cloned and slice/map in-place modifications should be avoided
err := countQuery.Distinct(false).
Select("COUNT(DISTINCT [[" + countCol + "]])").
OrderBy( /* reset */ ).
Row(&totalCount)
if err != nil {
return err
}
totalPages = int(math.Ceil(float64(totalCount) / float64(s.perPage)))
return nil
}
// apply pagination
modelsQuery.Limit(int64(s.perPage))
modelsQuery.Offset(int64(s.perPage * (s.page - 1)))
// apply pagination to the original query and fetch the models
modelsExec := func() error {
modelsQuery.Limit(int64(s.perPage))
modelsQuery.Offset(int64(s.perPage * (s.page - 1)))
// fetch models
if err := modelsQuery.All(items); err != nil {
return nil, err
return modelsQuery.All(items)
}
return &Result{
if !s.skipTotal {
// execute the 2 queries concurrently
errg := new(errgroup.Group)
errg.SetLimit(2)
errg.Go(countExec)
errg.Go(modelsExec)
if err := errg.Wait(); err != nil {
return nil, err
}
} else {
if err := modelsExec(); err != nil {
return nil, err
}
}
result := &Result{
Page: s.page,
PerPage: s.perPage,
TotalItems: int(totalCount),
TotalItems: totalCount,
TotalPages: totalPages,
Items: items,
}, nil
}
return result, nil
}
// ParseAndExec is a short convenient method to trigger both
tools/search/provider_test.go
+118 -30
View File
@@ -42,11 +42,25 @@ func TestProviderQuery(t *testing.T) {
}
}
func TestProviderSkipTotal(t *testing.T) {
p := NewProvider(&testFieldResolver{})
if p.skipTotal {
t.Fatalf("Expected the default skipTotal to be %v, got %v", false, p.skipTotal)
}
p.SkipTotal(true)
if !p.skipTotal {
t.Fatalf("Expected skipTotal to change to %v, got %v", true, p.skipTotal)
}
}
func TestProviderCountCol(t *testing.T) {
p := NewProvider(&testFieldResolver{})
if p.countCol != "_rowid_" {
t.Fatalf("Expected the default countCol to be %s, got %s", "_rowid_", p.countCol)
if p.countCol != "id" {
t.Fatalf("Expected the default countCol to be %s, got %s", "id", p.countCol)
}
p.CountCol("test")
@@ -229,6 +243,7 @@ func TestProviderExecNonEmptyQuery(t *testing.T) {
perPage int
sort []SortField
filter []FilterData
skipTotal bool
expectError bool
expectResult string
expectQueries []string
@@ -240,23 +255,25 @@ func TestProviderExecNonEmptyQuery(t *testing.T) {
[]SortField{},
[]FilterData{},
false,
false,
`{"page":1,"perPage":10,"totalItems":2,"totalPages":1,"items":[{"test1":1,"test2":"test2.1","test3":""},{"test1":2,"test2":"test2.2","test3":""}]}`,
[]string{
"SELECT COUNT(DISTINCT [[test._rowid_]]) FROM `test` WHERE NOT (`test1` IS NULL)",
"SELECT COUNT(DISTINCT [[test.id]]) FROM `test` WHERE NOT (`test1` IS NULL)",
"SELECT * FROM `test` WHERE NOT (`test1` IS NULL) ORDER BY `test1` ASC LIMIT 10",
},
},
{
"perPage normalization",
10, // will be capped by total count
0, // fallback to default
10,
0, // fallback to default
[]SortField{},
[]FilterData{},
false,
`{"page":1,"perPage":30,"totalItems":2,"totalPages":1,"items":[{"test1":1,"test2":"test2.1","test3":""},{"test1":2,"test2":"test2.2","test3":""}]}`,
false,
`{"page":10,"perPage":30,"totalItems":2,"totalPages":1,"items":[]}`,
[]string{
"SELECT COUNT(DISTINCT [[test._rowid_]]) FROM `test` WHERE NOT (`test1` IS NULL)",
"SELECT * FROM `test` WHERE NOT (`test1` IS NULL) ORDER BY `test1` ASC LIMIT 30",
"SELECT COUNT(DISTINCT [[test.id]]) FROM `test` WHERE NOT (`test1` IS NULL)",
"SELECT * FROM `test` WHERE NOT (`test1` IS NULL) ORDER BY `test1` ASC LIMIT 30 OFFSET 270",
},
},
{
@@ -265,6 +282,7 @@ func TestProviderExecNonEmptyQuery(t *testing.T) {
10,
[]SortField{{"unknown", SortAsc}},
[]FilterData{},
false,
true,
"",
nil,
@@ -275,6 +293,7 @@ func TestProviderExecNonEmptyQuery(t *testing.T) {
10,
[]SortField{},
[]FilterData{"test2 = 'test2.1'", "invalid"},
false,
true,
"",
nil,
@@ -286,10 +305,24 @@ func TestProviderExecNonEmptyQuery(t *testing.T) {
[]SortField{{"test2", SortDesc}},
[]FilterData{"test2 != null", "test1 >= 2"},
false,
false,
`{"page":1,"perPage":` + fmt.Sprint(MaxPerPage) + `,"totalItems":1,"totalPages":1,"items":[{"test1":2,"test2":"test2.2","test3":""}]}`,
[]string{
"SELECT COUNT(DISTINCT [[test._rowid_]]) FROM `test` WHERE ((NOT (`test1` IS NULL)) AND (((test2 != '' AND test2 IS NOT NULL)))) AND (test1 >= 2)",
"SELECT * FROM `test` WHERE ((NOT (`test1` IS NULL)) AND (((test2 != '' AND test2 IS NOT NULL)))) AND (test1 >= 2) ORDER BY `test1` ASC, `test2` DESC LIMIT 500",
"SELECT COUNT(DISTINCT [[test.id]]) FROM `test` WHERE ((NOT (`test1` IS NULL)) AND (((test2 != '' AND test2 IS NOT NULL)))) AND (test1 >= 2)",
"SELECT * FROM `test` WHERE ((NOT (`test1` IS NULL)) AND (((test2 != '' AND test2 IS NOT NULL)))) AND (test1 >= 2) ORDER BY `test1` ASC, `test2` DESC LIMIT " + fmt.Sprint(MaxPerPage),
},
},
{
"valid sort and filter fields (skipTotal=1)",
1,
5555, // will be limited by MaxPerPage
[]SortField{{"test2", SortDesc}},
[]FilterData{"test2 != null", "test1 >= 2"},
true,
false,
`{"page":1,"perPage":` + fmt.Sprint(MaxPerPage) + `,"totalItems":-1,"totalPages":-1,"items":[{"test1":2,"test2":"test2.2","test3":""}]}`,
[]string{
"SELECT * FROM `test` WHERE ((NOT (`test1` IS NULL)) AND (((test2 != '' AND test2 IS NOT NULL)))) AND (test1 >= 2) ORDER BY `test1` ASC, `test2` DESC LIMIT " + fmt.Sprint(MaxPerPage),
},
},
{
@@ -299,22 +332,50 @@ func TestProviderExecNonEmptyQuery(t *testing.T) {
[]SortField{{"test3", SortAsc}},
[]FilterData{"test3 != ''"},
false,
false,
`{"page":1,"perPage":10,"totalItems":0,"totalPages":0,"items":[]}`,
[]string{
"SELECT COUNT(DISTINCT [[test._rowid_]]) FROM `test` WHERE (NOT (`test1` IS NULL)) AND (((test3 != '' AND test3 IS NOT NULL)))",
"SELECT COUNT(DISTINCT [[test.id]]) FROM `test` WHERE (NOT (`test1` IS NULL)) AND (((test3 != '' AND test3 IS NOT NULL)))",
"SELECT * FROM `test` WHERE (NOT (`test1` IS NULL)) AND (((test3 != '' AND test3 IS NOT NULL))) ORDER BY `test1` ASC, `test3` ASC LIMIT 10",
},
},
{
"valid sort and filter fields (zero results; skipTotal=1)",
1,
10,
[]SortField{{"test3", SortAsc}},
[]FilterData{"test3 != ''"},
true,
false,
`{"page":1,"perPage":10,"totalItems":-1,"totalPages":-1,"items":[]}`,
[]string{
"SELECT * FROM `test` WHERE (NOT (`test1` IS NULL)) AND (((test3 != '' AND test3 IS NOT NULL))) ORDER BY `test1` ASC, `test3` ASC LIMIT 10",
},
},
{
"pagination test",
3,
2,
1,
[]SortField{},
[]FilterData{},
false,
false,
`{"page":2,"perPage":1,"totalItems":2,"totalPages":2,"items":[{"test1":2,"test2":"test2.2","test3":""}]}`,
[]string{
"SELECT COUNT(DISTINCT [[test._rowid_]]) FROM `test` WHERE NOT (`test1` IS NULL)",
"SELECT COUNT(DISTINCT [[test.id]]) FROM `test` WHERE NOT (`test1` IS NULL)",
"SELECT * FROM `test` WHERE NOT (`test1` IS NULL) ORDER BY `test1` ASC LIMIT 1 OFFSET 1",
},
},
{
"pagination test (skipTotal=1)",
2,
1,
[]SortField{},
[]FilterData{},
true,
false,
`{"page":2,"perPage":1,"totalItems":-1,"totalPages":-1,"items":[{"test1":2,"test2":"test2.2","test3":""}]}`,
[]string{
"SELECT * FROM `test` WHERE NOT (`test1` IS NULL) ORDER BY `test1` ASC LIMIT 1 OFFSET 1",
},
},
@@ -329,6 +390,7 @@ func TestProviderExecNonEmptyQuery(t *testing.T) {
Page(s.page).
PerPage(s.perPage).
Sort(s.sort).
SkipTotal(s.skipTotal).
Filter(s.filter)
result, err := p.Exec(&[]testTableStruct{})
@@ -378,55 +440,74 @@ func TestProviderParseAndExec(t *testing.T) {
OrderBy("test1 ASC")
scenarios := []struct {
name string
queryString string
expectError bool
expectResult string
}{
// empty
{
"no extra query params (aka. use the provider presets)",
"",
false,
`{"page":1,"perPage":123,"totalItems":2,"totalPages":1,"items":[{"test1":1,"test2":"test2.1","test3":""},{"test1":2,"test2":"test2.2","test3":""}]}`,
`{"page":2,"perPage":123,"totalItems":2,"totalPages":1,"items":[]}`,
},
// invalid query
{
"invalid query",
"invalid;",
true,
"",
},
// invalid page
{
"invalid page",
"page=a",
true,
"",
},
// invalid perPage
{
"invalid perPage",
"perPage=a",
true,
"",
},
// invalid sorting field
{
"invalid skipTotal",
"skipTotal=a",
true,
"",
},
{
"invalid sorting field",
"sort=-unknown",
true,
"",
},
// invalid filter field
{
"invalid filter field",
"filter=unknown>1",
true,
"",
},
// valid query params
{
"page=3&perPage=9999&filter=test1>1&sort=-test2,test3",
"page > existing",
"page=3&perPage=9999",
false,
`{"page":1,"perPage":500,"totalItems":1,"totalPages":1,"items":[{"test1":2,"test2":"test2.2","test3":""}]}`,
`{"page":3,"perPage":1000,"totalItems":2,"totalPages":1,"items":[]}`,
},
{
"valid query params",
"page=1&perPage=9999&filter=test1>1&sort=-test2,test3",
false,
`{"page":1,"perPage":1000,"totalItems":1,"totalPages":1,"items":[{"test1":2,"test2":"test2.2","test3":""}]}`,
},
{
"valid query params with skipTotal=1",
"page=1&perPage=9999&filter=test1>1&sort=-test2,test3&skipTotal=1",
false,
`{"page":1,"perPage":1000,"totalItems":-1,"totalPages":-1,"items":[{"test1":2,"test2":"test2.2","test3":""}]}`,
},
}
for i, s := range scenarios {
for _, s := range scenarios {
testDB.CalledQueries = []string{} // reset
testResolver := &testFieldResolver{}
@@ -441,7 +522,7 @@ func TestProviderParseAndExec(t *testing.T) {
hasErr := err != nil
if hasErr != s.expectError {
t.Errorf("(%d) Expected hasErr %v, got %v (%v)", i, s.expectError, hasErr, err)
t.Errorf("[%s] Expected hasErr %v, got %v (%v)", s.name, s.expectError, hasErr, err)
continue
}
@@ -450,16 +531,21 @@ func TestProviderParseAndExec(t *testing.T) {
}
if testResolver.UpdateQueryCalls != 1 {
t.Errorf("(%d) Expected resolver.Update to be called %d, got %d", i, 1, testResolver.UpdateQueryCalls)
t.Errorf("[%s] Expected resolver.Update to be called %d, got %d", s.name, 1, testResolver.UpdateQueryCalls)
}
if len(testDB.CalledQueries) != 2 {
t.Errorf("(%d) Expected %d db queries, got %d: \n%v", i, 2, len(testDB.CalledQueries), testDB.CalledQueries)
expectedQueries := 2
if provider.skipTotal {
expectedQueries = 1
}
if len(testDB.CalledQueries) != expectedQueries {
t.Errorf("[%s] Expected %d db queries, got %d: \n%v", s.name, expectedQueries, len(testDB.CalledQueries), testDB.CalledQueries)
}
encoded, _ := json.Marshal(result)
if string(encoded) != s.expectResult {
t.Errorf("(%d) Expected result %v, got \n%v", i, s.expectResult, string(encoded))
t.Errorf("[%s] Expected result %v, got \n%v", s.name, s.expectResult, string(encoded))
}
}
}
@@ -481,7 +567,9 @@ type testDB struct {
// NB! Don't forget to call `db.Close()` at the end of the test.
func createTestDB() (*testDB, error) {
sqlDB, err := sql.Open("sqlite", ":memory:")
// using a shared cache to allow multiple connections access to
// the same in memory database https://www.sqlite.org/inmemorydb.html
sqlDB, err := sql.Open("sqlite", "file::memory:?cache=shared")
if err != nil {
return nil, err
}