tomas/pocketbase
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

=added experimental expand, filter, fields, custom query and headers parameters support for the realtime subscriptions

Browse Source
This commit is contained in:
Gani Georgiev
2023-10-23 22:46:47 +03:00
parent e6f1b3dfe4
commit 79617e6d99
41 changed files with 553 additions and 257 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/rest/json_serializer.go
+57 -35
View File
@@ -5,7 +5,10 @@ import (
"fmt"
"strings"
// experimental! (need more tests before replacing encoding/json entirely)
// Experimental!
//
// Need more tests before replacing encoding/json entirely.
// Test also encoding/json/v2 once released (see https://github.com/golang/go/discussions/63397)
goccy "github.com/goccy/go-json"
"github.com/labstack/echo/v5"
@@ -45,42 +48,11 @@ func (s *Serializer) Serialize(c echo.Context, i any, indent string) error {
return s.DefaultJSONSerializer.Serialize(c, i, indent)
}
parsedFields, err := parseFields(rawFields)
decoded, err := PickFields(i, rawFields)
if err != nil {
return err
}
// marshalize the provided data to ensure that the related json.Marshaler
// implementations are invoked, and then convert it back to a plain
// json value that we can further operate on.
//
// @todo research other approaches to avoid the double serialization
// ---
encoded, err := json.Marshal(i) // use the std json since goccy has several bugs reported with struct marshaling and it is not safe
if err != nil {
return err
}
var decoded any
if err := goccy.Unmarshal(encoded, &decoded); err != nil {
return err
}
// ---
var isSearchResult bool
switch i.(type) {
case search.Result, *search.Result:
isSearchResult = true
}
if isSearchResult {
if decodedMap, ok := decoded.(map[string]any); ok {
pickFields(decodedMap["items"], parsedFields)
}
} else {
pickFields(decoded, parsedFields)
}
enc := goccy.NewEncoder(c.Response())
if indent != "" {
enc.SetIndent("", indent)
@@ -89,6 +61,56 @@ func (s *Serializer) Serialize(c echo.Context, i any, indent string) error {
return enc.Encode(decoded)
}
// PickFields parses the provided fields string expression and
// returns a new subset of data with only the requested fields.
//
// Fields transformations with modifiers are also supported (see initModifer()).
//
// Example:
//
// data := map[string]any{"a": 1, "b": 2, "c": map[string]any{"c1": 11, "c2": 22}}
// PickFields(data, "a,c.c1") // map[string]any{"a": 1, "c": map[string]any{"c1": 11}}
func PickFields(data any, rawFields string) (any, error) {
parsedFields, err := parseFields(rawFields)
if err != nil {
return nil, err
}
// marshalize the provided data to ensure that the related json.Marshaler
// implementations are invoked, and then convert it back to a plain
// json value that we can further operate on.
//
// @todo research other approaches to avoid the double serialization
// ---
encoded, err := json.Marshal(data) // use the std json since goccy has several bugs reported with struct marshaling and it is not safe
if err != nil {
return nil, err
}
var decoded any
if err := goccy.Unmarshal(encoded, &decoded); err != nil {
return nil, err
}
// ---
// special cases to preserve the same fields format when used with single item or array data.
var isSearchResult bool
switch data.(type) {
case search.Result, *search.Result:
isSearchResult = true
}
if isSearchResult {
if decodedMap, ok := decoded.(map[string]any); ok {
pickParsedFields(decodedMap["items"], parsedFields)
}
} else {
pickParsedFields(decoded, parsedFields)
}
return decoded, nil
}
func parseFields(rawFields string) (map[string]FieldModifier, error) {
t := tokenizer.NewFromString(rawFields)
@@ -145,7 +167,7 @@ func initModifer(rawModifier string) (FieldModifier, error) {
return nil, fmt.Errorf("missing or invalid modifier %q", name)
}
func pickFields(data any, fields map[string]FieldModifier) error {
func pickParsedFields(data any, fields map[string]FieldModifier) error {
switch v := data.(type) {
case map[string]any:
pickMapFields(v, fields)
@@ -233,7 +255,7 @@ DataLoop:
matchingFields[remains] = m
}
if err := pickFields(data[k], matchingFields); err != nil {
if err := pickParsedFields(data[k], matchingFields); err != nil {
return err
}
}
tools/rest/json_serializer_test.go
+127 -59
View File
@@ -1,6 +1,7 @@
package rest_test
import (
"encoding/json"
"io"
"net/http"
"net/http/httptest"
@@ -45,6 +46,22 @@ func TestSerialize(t *testing.T) {
"fields=missing",
`{}`,
},
{
">299 response",
rest.Serializer{},
300,
map[string]any{"a": 1, "b": 2, "c": "test"},
"fields=missing",
`{"a":1,"b":2,"c":"test"}`,
},
{
"<200 response",
rest.Serializer{},
199,
map[string]any{"a": 1, "b": 2, "c": "test"},
"fields=missing",
`{"a":1,"b":2,"c":"test"}`,
},
{
"non map response",
rest.Serializer{},
@@ -69,22 +86,6 @@ func TestSerialize(t *testing.T) {
"fields=missing", // test individual fields trim
`{}`,
},
{
">299 response",
rest.Serializer{},
300,
map[string]any{"a": 1, "b": 2, "c": "test"},
"fields=missing",
`{"a":1,"b":2,"c":"test"}`,
},
{
"<200 response",
rest.Serializer{},
199,
map[string]any{"a": 1, "b": 2, "c": "test"},
"fields=missing",
`{"a":1,"b":2,"c":"test"}`,
},
{
"map with existing and missing fields",
rest.Serializer{},
@@ -101,21 +102,96 @@ func TestSerialize(t *testing.T) {
"custom=a, c ,missing", // test individual fields trim
`{"a":1,"c":"test"}`,
},
}
for _, s := range scenarios {
t.Run(s.name, func(t *testing.T) {
req := httptest.NewRequest(http.MethodPost, "/", nil)
req.URL.RawQuery = s.query
rec := httptest.NewRecorder()
e := echo.New()
c := e.NewContext(req, rec)
c.Response().Status = s.statusCode
if err := s.serializer.Serialize(c, s.data, ""); err != nil {
t.Fatalf("Serialize failure: %v", err)
}
rawBody, err := io.ReadAll(rec.Result().Body)
if err != nil {
t.Fatalf("Failed to read request body: %v", err)
}
if v := strings.TrimSpace(string(rawBody)); v != s.expected {
t.Fatalf("Expected body\n%v \ngot \n%v", s.expected, v)
}
})
}
}
func TestPickFields(t *testing.T) {
scenarios := []struct {
name string
data any
fields string
expectError bool
result string
}{
{
"empty fields",
map[string]any{"a": 1, "b": 2, "c": "test"},
"",
false,
`{"a":1,"b":2,"c":"test"}`,
},
{
"missing fields",
map[string]any{"a": 1, "b": 2, "c": "test"},
"missing",
false,
`{}`,
},
{
"non map data",
"test",
"a,b,test",
false,
`"test"`,
},
{
"non slice of map data",
[]any{"a", "b", "test"},
"a,test",
false,
`["a","b","test"]`,
},
{
"map with no matching field",
map[string]any{"a": 1, "b": 2, "c": "test"},
"missing", // test individual fields trim
false,
`{}`,
},
{
"map with existing and missing fields",
map[string]any{"a": 1, "b": 2, "c": "test"},
"a, c ,missing", // test individual fields trim
false,
`{"a":1,"c":"test"}`,
},
{
"slice of maps with existing and missing fields",
rest.Serializer{},
200,
[]any{
map[string]any{"a": 11, "b": 11, "c": "test1"},
map[string]any{"a": 22, "b": 22, "c": "test2"},
},
"fields=a, c ,missing", // test individual fields trim
"a, c ,missing", // test individual fields trim
false,
`[{"a":11,"c":"test1"},{"a":22,"c":"test2"}]`,
},
{
"nested fields with mixed map and any slices",
rest.Serializer{},
200,
map[string]any{
"a": 1,
"b": 2,
@@ -167,13 +243,12 @@ func TestSerialize(t *testing.T) {
},
},
},
"fields=a, c, anySlice.A, mapSlice.C, mapSlice.D.DA, anySlice.D,fullMap",
"a, c, anySlice.A, mapSlice.C, mapSlice.D.DA, anySlice.D,fullMap",
false,
`{"a":1,"anySlice":[{"A":[1,2,3],"D":{"DA":1,"DB":2}},{"A":"test"}],"c":"test","fullMap":[{"A":[1,2,3],"B":["1","2",3],"C":"test"},{"B":["1","2",3],"D":[{"DA":2},{"DA":3}]}],"mapSlice":[{"C":"test","D":[{"DA":1}]},{"D":[{"DA":2},{"DA":3},{}]}]}`,
},
{
"SearchResult",
rest.Serializer{},
200,
search.Result{
Page: 1,
PerPage: 10,
@@ -184,13 +259,12 @@ func TestSerialize(t *testing.T) {
map[string]any{"a": 22, "b": 22, "c": "test2"},
},
},
"fields=a,c,missing",
"a,c,missing",
false,
`{"items":[{"a":11,"c":"test1"},{"a":22,"c":"test2"}],"page":1,"perPage":10,"totalItems":20,"totalPages":30}`,
},
{
"*SearchResult",
rest.Serializer{},
200,
&search.Result{
Page: 1,
PerPage: 10,
@@ -201,13 +275,12 @@ func TestSerialize(t *testing.T) {
map[string]any{"a": 22, "b": 22, "c": "test2"},
},
},
"fields=a,c",
"a,c",
false,
`{"items":[{"a":11,"c":"test1"},{"a":22,"c":"test2"}],"page":1,"perPage":10,"totalItems":20,"totalPages":30}`,
},
{
"root wildcard",
rest.Serializer{},
200,
&search.Result{
Page: 1,
PerPage: 10,
@@ -218,13 +291,12 @@ func TestSerialize(t *testing.T) {
map[string]any{"a": 22, "b": 22, "c": "test2"},
},
},
"fields=*",
"*",
false,
`{"items":[{"a":11,"b":11,"c":"test1"},{"a":22,"b":22,"c":"test2"}],"page":1,"perPage":10,"totalItems":20,"totalPages":30}`,
},
{
"root wildcard with nested exception",
rest.Serializer{},
200,
map[string]any{
"id": "123",
"title": "lorem",
@@ -233,13 +305,12 @@ func TestSerialize(t *testing.T) {
"title": "rel_title",
},
},
"fields=*,rel.id",
"*,rel.id",
false,
`{"id":"123","rel":{"id":"456"},"title":"lorem"}`,
},
{
"sub wildcard",
rest.Serializer{},
200,
map[string]any{
"id": "123",
"title": "lorem",
@@ -252,13 +323,12 @@ func TestSerialize(t *testing.T) {
},
},
},
"fields=id,rel.*",
"id,rel.*",
false,
`{"id":"123","rel":{"id":"456","sub":{"id":"789","title":"sub_title"},"title":"rel_title"}}`,
},
{
"sub wildcard with nested exception",
rest.Serializer{},
200,
map[string]any{
"id": "123",
"title": "lorem",
@@ -271,21 +341,19 @@ func TestSerialize(t *testing.T) {
},
},
},
"fields=id,rel.*,rel.sub.id",
"id,rel.*,rel.sub.id",
false,
`{"id":"123","rel":{"id":"456","sub":{"id":"789"},"title":"rel_title"}}`,
},
{
"invalid excerpt modifier",
rest.Serializer{},
400,
map[string]any{"a": 1, "b": 2, "c": "test"},
"fields=*:excerpt",
"*:excerpt",
true,
`{"a":1,"b":2,"c":"test"}`,
},
{
"valid excerpt modifier",
rest.Serializer{},
200,
map[string]any{
"id": "123",
"title": "lorem",
@@ -298,32 +366,32 @@ func TestSerialize(t *testing.T) {
},
},
},
"fields=*:excerpt(2),rel.title:excerpt(3, true)",
"*:excerpt(2),rel.title:excerpt(3, true)",
false,
`{"id":"12","rel":{"title":"rel..."},"title":"lo"}`,
},
}
for _, s := range scenarios {
t.Run(s.name, func(t *testing.T) {
req := httptest.NewRequest(http.MethodPost, "/", nil)
req.URL.RawQuery = s.query
rec := httptest.NewRecorder()
result, err := rest.PickFields(s.data, s.fields)
e := echo.New()
c := e.NewContext(req, rec)
c.Response().Status = s.statusCode
if err := s.serializer.Serialize(c, s.data, ""); err != nil {
t.Fatalf("Serialize failure: %v", err)
hasErr := err != nil
if hasErr != s.expectError {
t.Fatalf("Expected hasErr %v, got %v (%v)", s.expectError, hasErr, err)
}
rawBody, err := io.ReadAll(rec.Result().Body)
if hasErr {
return
}
serialized, err := json.Marshal(result)
if err != nil {
t.Fatalf("Failed to read request body: %v", err)
t.Fatal(err)
}
if v := strings.TrimSpace(string(rawBody)); v != s.expected {
t.Fatalf("Expected body\n%v \ngot: \n%v", s.expected, v)
if v := string(serialized); v != s.result {
t.Fatalf("Expected body\n%s \ngot \n%s", s.result, v)
}
})
}