tomas/pocketbase
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

merge v0.23.0-rc changes

Browse Source
This commit is contained in:
Gani Georgiev
2024-09-29 19:23:19 +03:00
parent ad92992324
commit 844f18cac3
753 changed files with 85141 additions and 63396 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/search/filter.go
+7 -2
View File
@@ -64,9 +64,10 @@ func (f FilterData) BuildExpr(
}
}
if parsedFilterData.Has(raw) {
return buildParsedFilterExpr(parsedFilterData.Get(raw), fieldResolver)
if data, ok := parsedFilterData.GetOk(raw); ok {
return buildParsedFilterExpr(data, fieldResolver)
}
data, err := fexpr.Parse(raw)
if err != nil {
// depending on the users demand we may allow empty expressions
@@ -78,9 +79,11 @@ func (f FilterData) BuildExpr(
return nil, err
}
// store in cache
// (the limit size is arbitrary and it is there to prevent the cache growing too big)
parsedFilterData.SetIfLessThanLimit(raw, data, 500)
return buildParsedFilterExpr(data, fieldResolver)
}
@@ -431,6 +434,8 @@ func mergeParams(params ...dbx.Params) dbx.Params {
return result
}
// @todo consider adding support for custom single character wildcard
//
// wrapLikeParams wraps each provided param value string with `%`
// if the param doesn't contain an explicit wildcard (`%`) character already.
func wrapLikeParams(params dbx.Params) dbx.Params {
tools/search/provider.go
+17 -5
View File
@@ -5,16 +5,20 @@ import (
"math"
"net/url"
"strconv"
"strings"
"github.com/pocketbase/dbx"
"github.com/pocketbase/pocketbase/tools/inflector"
"golang.org/x/sync/errgroup"
)
// DefaultPerPage specifies the default returned search result items.
const DefaultPerPage int = 30
// @todo consider making it configurable
//
// MaxPerPage specifies the maximum allowed search result items returned in a single page.
const MaxPerPage int = 500
const MaxPerPage int = 1000
// url search query params
const (
@@ -27,23 +31,23 @@ const (
// Result defines the returned search result structure.
type Result struct {
Items any `json:"items"`
Page int `json:"page"`
PerPage int `json:"perPage"`
TotalItems int `json:"totalItems"`
TotalPages int `json:"totalPages"`
Items any `json:"items"`
}
// Provider represents a single configured search provider instance.
type Provider struct {
fieldResolver FieldResolver
query *dbx.SelectQuery
skipTotal bool
countCol string
page int
perPage int
sort []SortField
filter []FilterData
page int
perPage int
skipTotal bool
}
// NewProvider creates and returns a new search provider.
@@ -208,6 +212,14 @@ func (s *Provider) Exec(items any) (*Result, error) {
return nil, err
}
if expr != "" {
// ensure that _rowid_ expressions are always prefixed with the first FROM table
if sortField.Name == rowidSortKey && !strings.Contains(expr, ".") {
queryInfo := modelsQuery.Info()
if len(queryInfo.From) > 0 {
expr = "[[" + inflector.Columnify(queryInfo.From[0]) + "]]." + expr
}
}
modelsQuery.AndOrderBy(expr)
}
}
tools/search/provider_test.go
+71 -69
View File
@@ -180,38 +180,39 @@ func TestProviderParse(t *testing.T) {
}
for i, s := range scenarios {
r := &testFieldResolver{}
p := NewProvider(r).
Page(initialPage).
PerPage(initialPerPage).
Sort(initialSort).
Filter(initialFilter)
t.Run(fmt.Sprintf("%d_%s", i, s.query), func(t *testing.T) {
r := &testFieldResolver{}
p := NewProvider(r).
Page(initialPage).
PerPage(initialPerPage).
Sort(initialSort).
Filter(initialFilter)
err := p.Parse(s.query)
err := p.Parse(s.query)
hasErr := err != nil
if hasErr != s.expectError {
t.Errorf("(%d) Expected hasErr %v, got %v (%v)", i, s.expectError, hasErr, err)
continue
}
hasErr := err != nil
if hasErr != s.expectError {
t.Fatalf("Expected hasErr %v, got %v (%v)", s.expectError, hasErr, err)
}
if p.page != s.expectPage {
t.Errorf("(%d) Expected page %v, got %v", i, s.expectPage, p.page)
}
if p.page != s.expectPage {
t.Fatalf("Expected page %v, got %v", s.expectPage, p.page)
}
if p.perPage != s.expectPerPage {
t.Errorf("(%d) Expected perPage %v, got %v", i, s.expectPerPage, p.perPage)
}
if p.perPage != s.expectPerPage {
t.Fatalf("Expected perPage %v, got %v", s.expectPerPage, p.perPage)
}
encodedSort, _ := json.Marshal(p.sort)
if string(encodedSort) != s.expectSort {
t.Errorf("(%d) Expected sort %v, got \n%v", i, s.expectSort, string(encodedSort))
}
encodedSort, _ := json.Marshal(p.sort)
if string(encodedSort) != s.expectSort {
t.Fatalf("Expected sort %v, got \n%v", s.expectSort, string(encodedSort))
}
encodedFilter, _ := json.Marshal(p.filter)
if string(encodedFilter) != s.expectFilter {
t.Errorf("(%d) Expected filter %v, got \n%v", i, s.expectFilter, string(encodedFilter))
}
encodedFilter, _ := json.Marshal(p.filter)
if string(encodedFilter) != s.expectFilter {
t.Fatalf("Expected filter %v, got \n%v", s.expectFilter, string(encodedFilter))
}
})
}
}
@@ -256,7 +257,7 @@ func TestProviderExecNonEmptyQuery(t *testing.T) {
[]FilterData{},
false,
false,
`{"page":1,"perPage":10,"totalItems":2,"totalPages":1,"items":[{"test1":1,"test2":"test2.1","test3":""},{"test1":2,"test2":"test2.2","test3":""}]}`,
`{"items":[{"test1":1,"test2":"test2.1","test3":""},{"test1":2,"test2":"test2.2","test3":""}],"page":1,"perPage":10,"totalItems":2,"totalPages":1}`,
[]string{
"SELECT COUNT(DISTINCT [[test.id]]) FROM `test` WHERE NOT (`test1` IS NULL)",
"SELECT * FROM `test` WHERE NOT (`test1` IS NULL) ORDER BY `test1` ASC LIMIT 10",
@@ -270,7 +271,7 @@ func TestProviderExecNonEmptyQuery(t *testing.T) {
[]FilterData{},
false,
false,
`{"page":10,"perPage":30,"totalItems":2,"totalPages":1,"items":[]}`,
`{"items":[],"page":10,"perPage":30,"totalItems":2,"totalPages":1}`,
[]string{
"SELECT COUNT(DISTINCT [[test.id]]) FROM `test` WHERE NOT (`test1` IS NULL)",
"SELECT * FROM `test` WHERE NOT (`test1` IS NULL) ORDER BY `test1` ASC LIMIT 30 OFFSET 270",
@@ -306,7 +307,7 @@ func TestProviderExecNonEmptyQuery(t *testing.T) {
[]FilterData{"test2 != null", "test1 >= 2"},
false,
false,
`{"page":1,"perPage":` + fmt.Sprint(MaxPerPage) + `,"totalItems":1,"totalPages":1,"items":[{"test1":2,"test2":"test2.2","test3":""}]}`,
`{"items":[{"test1":2,"test2":"test2.2","test3":""}],"page":1,"perPage":` + fmt.Sprint(MaxPerPage) + `,"totalItems":1,"totalPages":1}`,
[]string{
"SELECT COUNT(DISTINCT [[test.id]]) FROM `test` WHERE ((NOT (`test1` IS NULL)) AND (((test2 IS NOT '' AND test2 IS NOT NULL)))) AND (test1 >= 2)",
"SELECT * FROM `test` WHERE ((NOT (`test1` IS NULL)) AND (((test2 IS NOT '' AND test2 IS NOT NULL)))) AND (test1 >= 2) ORDER BY `test1` ASC, `test2` DESC LIMIT " + fmt.Sprint(MaxPerPage),
@@ -320,7 +321,7 @@ func TestProviderExecNonEmptyQuery(t *testing.T) {
[]FilterData{"test2 != null", "test1 >= 2"},
true,
false,
`{"page":1,"perPage":` + fmt.Sprint(MaxPerPage) + `,"totalItems":-1,"totalPages":-1,"items":[{"test1":2,"test2":"test2.2","test3":""}]}`,
`{"items":[{"test1":2,"test2":"test2.2","test3":""}],"page":1,"perPage":` + fmt.Sprint(MaxPerPage) + `,"totalItems":-1,"totalPages":-1}`,
[]string{
"SELECT * FROM `test` WHERE ((NOT (`test1` IS NULL)) AND (((test2 IS NOT '' AND test2 IS NOT NULL)))) AND (test1 >= 2) ORDER BY `test1` ASC, `test2` DESC LIMIT " + fmt.Sprint(MaxPerPage),
},
@@ -333,7 +334,7 @@ func TestProviderExecNonEmptyQuery(t *testing.T) {
[]FilterData{"test3 != ''"},
false,
false,
`{"page":1,"perPage":10,"totalItems":0,"totalPages":0,"items":[]}`,
`{"items":[],"page":1,"perPage":10,"totalItems":0,"totalPages":0}`,
[]string{
"SELECT COUNT(DISTINCT [[test.id]]) FROM `test` WHERE (NOT (`test1` IS NULL)) AND (((test3 IS NOT '' AND test3 IS NOT NULL)))",
"SELECT * FROM `test` WHERE (NOT (`test1` IS NULL)) AND (((test3 IS NOT '' AND test3 IS NOT NULL))) ORDER BY `test1` ASC, `test3` ASC LIMIT 10",
@@ -347,7 +348,7 @@ func TestProviderExecNonEmptyQuery(t *testing.T) {
[]FilterData{"test3 != ''"},
true,
false,
`{"page":1,"perPage":10,"totalItems":-1,"totalPages":-1,"items":[]}`,
`{"items":[],"page":1,"perPage":10,"totalItems":-1,"totalPages":-1}`,
[]string{
"SELECT * FROM `test` WHERE (NOT (`test1` IS NULL)) AND (((test3 IS NOT '' AND test3 IS NOT NULL))) ORDER BY `test1` ASC, `test3` ASC LIMIT 10",
},
@@ -360,7 +361,7 @@ func TestProviderExecNonEmptyQuery(t *testing.T) {
[]FilterData{},
false,
false,
`{"page":2,"perPage":1,"totalItems":2,"totalPages":2,"items":[{"test1":2,"test2":"test2.2","test3":""}]}`,
`{"items":[{"test1":2,"test2":"test2.2","test3":""}],"page":2,"perPage":1,"totalItems":2,"totalPages":2}`,
[]string{
"SELECT COUNT(DISTINCT [[test.id]]) FROM `test` WHERE NOT (`test1` IS NULL)",
"SELECT * FROM `test` WHERE NOT (`test1` IS NULL) ORDER BY `test1` ASC LIMIT 1 OFFSET 1",
@@ -374,7 +375,7 @@ func TestProviderExecNonEmptyQuery(t *testing.T) {
[]FilterData{},
true,
false,
`{"page":2,"perPage":1,"totalItems":-1,"totalPages":-1,"items":[{"test1":2,"test2":"test2.2","test3":""}]}`,
`{"items":[{"test1":2,"test2":"test2.2","test3":""}],"page":2,"perPage":1,"totalItems":-1,"totalPages":-1}`,
[]string{
"SELECT * FROM `test` WHERE NOT (`test1` IS NULL) ORDER BY `test1` ASC LIMIT 1 OFFSET 1",
},
@@ -449,7 +450,7 @@ func TestProviderParseAndExec(t *testing.T) {
"no extra query params (aka. use the provider presets)",
"",
false,
`{"page":2,"perPage":123,"totalItems":2,"totalPages":1,"items":[]}`,
`{"items":[],"page":2,"perPage":123,"totalItems":2,"totalPages":1}`,
},
{
"invalid query",
@@ -491,62 +492,63 @@ func TestProviderParseAndExec(t *testing.T) {
"page > existing",
"page=3&perPage=9999",
false,
`{"page":3,"perPage":500,"totalItems":2,"totalPages":1,"items":[]}`,
`{"items":[],"page":3,"perPage":1000,"totalItems":2,"totalPages":1}`,
},
{
"valid query params",
"page=1&perPage=9999&filter=test1>1&sort=-test2,test3",
false,
`{"page":1,"perPage":500,"totalItems":1,"totalPages":1,"items":[{"test1":2,"test2":"test2.2","test3":""}]}`,
`{"items":[{"test1":2,"test2":"test2.2","test3":""}],"page":1,"perPage":1000,"totalItems":1,"totalPages":1}`,
},
{
"valid query params with skipTotal=1",
"page=1&perPage=9999&filter=test1>1&sort=-test2,test3&skipTotal=1",
false,
`{"page":1,"perPage":500,"totalItems":-1,"totalPages":-1,"items":[{"test1":2,"test2":"test2.2","test3":""}]}`,
`{"items":[{"test1":2,"test2":"test2.2","test3":""}],"page":1,"perPage":1000,"totalItems":-1,"totalPages":-1}`,
},
}
for _, s := range scenarios {
testDB.CalledQueries = []string{} // reset
t.Run(s.name, func(t *testing.T) {
testDB.CalledQueries = []string{} // reset
testResolver := &testFieldResolver{}
provider := NewProvider(testResolver).
Query(query).
Page(2).
PerPage(123).
Sort([]SortField{{"test2", SortAsc}}).
Filter([]FilterData{"test1 > 0"})
testResolver := &testFieldResolver{}
provider := NewProvider(testResolver).
Query(query).
Page(2).
PerPage(123).
Sort([]SortField{{"test2", SortAsc}}).
Filter([]FilterData{"test1 > 0"})
result, err := provider.ParseAndExec(s.queryString, &[]testTableStruct{})
result, err := provider.ParseAndExec(s.queryString, &[]testTableStruct{})
hasErr := err != nil
if hasErr != s.expectError {
t.Errorf("[%s] Expected hasErr %v, got %v (%v)", s.name, s.expectError, hasErr, err)
continue
}
hasErr := err != nil
if hasErr != s.expectError {
t.Fatalf("Expected hasErr %v, got %v (%v)", s.expectError, hasErr, err)
}
if hasErr {
continue
}
if hasErr {
return
}
if testResolver.UpdateQueryCalls != 1 {
t.Errorf("[%s] Expected resolver.Update to be called %d, got %d", s.name, 1, testResolver.UpdateQueryCalls)
}
if testResolver.UpdateQueryCalls != 1 {
t.Fatalf("Expected resolver.Update to be called %d, got %d", 1, testResolver.UpdateQueryCalls)
}
expectedQueries := 2
if provider.skipTotal {
expectedQueries = 1
}
expectedQueries := 2
if provider.skipTotal {
expectedQueries = 1
}
if len(testDB.CalledQueries) != expectedQueries {
t.Errorf("[%s] Expected %d db queries, got %d: \n%v", s.name, expectedQueries, len(testDB.CalledQueries), testDB.CalledQueries)
}
if len(testDB.CalledQueries) != expectedQueries {
t.Fatalf("Expected %d db queries, got %d: \n%v", expectedQueries, len(testDB.CalledQueries), testDB.CalledQueries)
}
encoded, _ := json.Marshal(result)
if string(encoded) != s.expectResult {
t.Errorf("[%s] Expected result %v, got \n%v", s.name, s.expectResult, string(encoded))
}
encoded, _ := json.Marshal(result)
if string(encoded) != s.expectResult {
t.Fatalf("Expected result \n%v\ngot\n%v", s.expectResult, string(encoded))
}
})
}
}
tools/search/simple_field_resolver.go
+1 -1
View File
@@ -76,7 +76,7 @@ func (r *SimpleFieldResolver) UpdateQuery(query *dbx.SelectQuery) error {
// Returns error if `field` is not in `r.allowedFields`.
func (r *SimpleFieldResolver) Resolve(field string) (*ResolverResult, error) {
if !list.ExistInSliceWithRegex(field, r.allowedFields) {
return nil, fmt.Errorf("failed to resolve field %q", field)
return nil, fmt.Errorf("Failed to resolve field %q.", field)
}
parts := strings.Split(field, ".")
tools/search/simple_field_resolver_test.go
+29 -28
View File
@@ -1,6 +1,7 @@
package search_test
import (
"fmt"
"testing"
"github.com/pocketbase/dbx"
@@ -23,22 +24,22 @@ func TestSimpleFieldResolverUpdateQuery(t *testing.T) {
}
for i, s := range scenarios {
db := dbx.NewFromDB(nil, "")
query := db.Select("id").From("test")
t.Run(fmt.Sprintf("%d_%s", i, s.fieldName), func(t *testing.T) {
db := dbx.NewFromDB(nil, "")
query := db.Select("id").From("test")
r.Resolve(s.fieldName)
r.Resolve(s.fieldName)
if err := r.UpdateQuery(nil); err != nil {
t.Errorf("(%d) UpdateQuery failed with error %v", i, err)
continue
}
if err := r.UpdateQuery(nil); err != nil {
t.Fatalf("UpdateQuery failed with error %v", err)
}
rawQuery := query.Build().SQL()
// rawQuery := s.expectQuery
rawQuery := query.Build().SQL()
if rawQuery != s.expectQuery {
t.Errorf("(%d) Expected query %v, got \n%v", i, s.expectQuery, rawQuery)
}
if rawQuery != s.expectQuery {
t.Fatalf("Expected query %v, got \n%v", s.expectQuery, rawQuery)
}
})
}
}
@@ -62,25 +63,25 @@ func TestSimpleFieldResolverResolve(t *testing.T) {
}
for i, s := range scenarios {
r, err := r.Resolve(s.fieldName)
t.Run(fmt.Sprintf("%d_%s", i, s.fieldName), func(t *testing.T) {
r, err := r.Resolve(s.fieldName)
hasErr := err != nil
if hasErr != s.expectError {
t.Errorf("(%d) Expected hasErr %v, got %v (%v)", i, s.expectError, hasErr, err)
continue
}
hasErr := err != nil
if hasErr != s.expectError {
t.Fatalf("Expected hasErr %v, got %v (%v)", s.expectError, hasErr, err)
}
if hasErr {
continue
}
if hasErr {
return
}
if r.Identifier != s.expectName {
t.Errorf("(%d) Expected r.Identifier %q, got %q", i, s.expectName, r.Identifier)
}
if r.Identifier != s.expectName {
t.Fatalf("Expected r.Identifier %q, got %q", s.expectName, r.Identifier)
}
// params should be empty
if len(r.Params) != 0 {
t.Errorf("(%d) Expected 0 r.Params, got %v", i, r.Params)
}
if len(r.Params) != 0 {
t.Fatalf("r.Params should be empty, got %v", r.Params)
}
})
}
}
tools/search/sort.go
+9 -1
View File
@@ -5,7 +5,10 @@ import (
"strings"
)
const randomSortKey string = "@random"
const (
randomSortKey string = "@random"
rowidSortKey string = "@rowid"
)
// sort field directions
const (
@@ -26,6 +29,11 @@ func (s *SortField) BuildExpr(fieldResolver FieldResolver) (string, error) {
return "RANDOM()", nil
}
// special case for the builtin SQLite rowid column
if s.Name == rowidSortKey {
return fmt.Sprintf("[[_rowid_]] %s", s.Direction), nil
}
result, err := fieldResolver.Resolve(s.Name)
// invalidate empty fields and non-column identifiers
tools/search/sort_test.go
+26 -18
View File
@@ -2,6 +2,7 @@ package search_test
import (
"encoding/json"
"fmt"
"testing"
"github.com/pocketbase/pocketbase/tools/search"
@@ -29,27 +30,30 @@ func TestSortFieldBuildExpr(t *testing.T) {
{search.SortField{"test1", search.SortDesc}, false, "[[test1]] DESC"},
// special @random field (ignore direction)
{search.SortField{"@random", search.SortDesc}, false, "RANDOM()"},
// special _rowid_ field
{search.SortField{"@rowid", search.SortDesc}, false, "[[_rowid_]] DESC"},
}
for i, s := range scenarios {
result, err := s.sortField.BuildExpr(resolver)
for _, s := range scenarios {
t.Run(fmt.Sprintf("%s_%s", s.sortField.Name, s.sortField.Name), func(t *testing.T) {
result, err := s.sortField.BuildExpr(resolver)
hasErr := err != nil
if hasErr != s.expectError {
t.Errorf("(%d) Expected hasErr %v, got %v (%v)", i, s.expectError, hasErr, err)
continue
}
hasErr := err != nil
if hasErr != s.expectError {
t.Fatalf("Expected hasErr %v, got %v (%v)", s.expectError, hasErr, err)
}
if result != s.expectExpression {
t.Errorf("(%d) Expected expression %v, got %v", i, s.expectExpression, result)
}
if result != s.expectExpression {
t.Fatalf("Expected expression %v, got %v", s.expectExpression, result)
}
})
}
}
func TestParseSortFromString(t *testing.T) {
scenarios := []struct {
value string
expectedJson string
value string
expected string
}{
{"", `[{"name":"","direction":"ASC"}]`},
{"test", `[{"name":"test","direction":"ASC"}]`},
@@ -57,14 +61,18 @@ func TestParseSortFromString(t *testing.T) {
{"-test", `[{"name":"test","direction":"DESC"}]`},
{"test1,-test2,+test3", `[{"name":"test1","direction":"ASC"},{"name":"test2","direction":"DESC"},{"name":"test3","direction":"ASC"}]`},
{"@random,-test", `[{"name":"@random","direction":"ASC"},{"name":"test","direction":"DESC"}]`},
{"-@rowid,-test", `[{"name":"@rowid","direction":"DESC"},{"name":"test","direction":"DESC"}]`},
}
for i, s := range scenarios {
result := search.ParseSortFromString(s.value)
encoded, _ := json.Marshal(result)
for _, s := range scenarios {
t.Run(s.value, func(t *testing.T) {
result := search.ParseSortFromString(s.value)
encoded, _ := json.Marshal(result)
encodedStr := string(encoded)
if string(encoded) != s.expectedJson {
t.Errorf("(%d) Expected expression %v, got %v", i, s.expectedJson, string(encoded))
}
if encodedStr != s.expected {
t.Fatalf("Expected expression %s, got %s", s.expected, encodedStr)
}
})
}
}