From 9e70c777362508fd4e2ed269c0cbf89e073fadd9 Mon Sep 17 00:00:00 2001 From: Gani Georgiev Date: Mon, 4 Nov 2024 19:03:33 +0200 Subject: [PATCH] added migration to normalize the system collection and field ids --- CHANGELOG.md | 12 +++ core/base_test.go | 8 ++ core/collection_model.go | 3 +- core/log_printer_test.go | 9 ++ migrations/1640988000_init.go | 10 +- migrations/1717233558_v0.23_migrate3.go | 138 ++++++++++++++++++++++++ plugins/migratecmd/templates.go | 4 +- tests/data/data.db | Bin 339968 -> 344064 bytes ui/.env | 2 +- 9 files changed, 177 insertions(+), 9 deletions(-) create mode 100644 migrations/1717233558_v0.23_migrate3.go diff --git a/CHANGELOG.md b/CHANGELOG.md index ea201653..a122cf78 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,15 @@ +## v0.23.0-rc10 + +> [!CAUTION] +> **This is a prerelease intended for test and experimental purposes only!** + +- Restore the CRC32 checksum autogeneration for the collection/field ids in order to maintain deterministic default identifier value and minimize conflicts between custom migrations and full collections snapshots. + _There is a system migration that will attempt to normalize existing system collections ids, but if you already migrated to v0.23.0-rc and have generated a full collections snapshot migration, you have to delete it and regenerate a new one._ + +- Change the behavior of the default generated collections snapshot migration to act as "extend" instead of "replace" to prevent accidental data deletion. + _I think this would be rare but if you want the old behaviour you can edit the generated snapshot file and replace the second argument (`deleteMissing`) of `App.ImportCollection/App.ImportCollectionsByMarshaledJSON` from `false` to `true`._ + + ## v0.23.0-rc9 > [!CAUTION] diff --git a/core/base_test.go b/core/base_test.go index a64a94b4..96d4b448 100644 --- a/core/base_test.go +++ b/core/base_test.go @@ -2,6 +2,7 @@ package core_test import ( "context" + "database/sql" "log/slog" "os" "testing" @@ -9,6 +10,7 @@ import ( _ "unsafe" + "github.com/pocketbase/dbx" "github.com/pocketbase/pocketbase/core" "github.com/pocketbase/pocketbase/tests" "github.com/pocketbase/pocketbase/tools/logger" @@ -351,6 +353,12 @@ func TestBaseAppRefreshSettingsLoggerMinLevelEnabled(t *testing.T) { t.Fatal(err) } + // silence query logs + app.DB().(*dbx.DB).ExecLogFunc = func(ctx context.Context, t time.Duration, sql string, result sql.Result, err error) {} + app.DB().(*dbx.DB).QueryLogFunc = func(ctx context.Context, t time.Duration, sql string, rows *sql.Rows, err error) {} + app.NonconcurrentDB().(*dbx.DB).ExecLogFunc = func(ctx context.Context, t time.Duration, sql string, result sql.Result, err error) {} + app.NonconcurrentDB().(*dbx.DB).QueryLogFunc = func(ctx context.Context, t time.Duration, sql string, rows *sql.Rows, err error) {} + handler, ok := app.Logger().Handler().(*logger.BatchHandler) if !ok { t.Fatalf("Expected BatchHandler, got %v", app.Logger().Handler()) diff --git a/core/collection_model.go b/core/collection_model.go index fe04e77c..f9a004ec 100644 --- a/core/collection_model.go +++ b/core/collection_model.go @@ -488,12 +488,13 @@ func (m *Collection) UnmarshalJSON(b []byte) error { minimal := &struct { Type string `json:"type"` Name string `json:"name"` + Id string `json:"id"` }{} if err := json.Unmarshal(b, minimal); err != nil { return err } - blank := NewCollection(minimal.Type, minimal.Name) + blank := NewCollection(minimal.Type, minimal.Name, minimal.Id) *m = *blank } diff --git a/core/log_printer_test.go b/core/log_printer_test.go index 26d26c2c..35a660d4 100644 --- a/core/log_printer_test.go +++ b/core/log_printer_test.go @@ -2,10 +2,13 @@ package core import ( "context" + "database/sql" "log/slog" "os" "testing" + "time" + "github.com/pocketbase/dbx" "github.com/pocketbase/pocketbase/tools/list" "github.com/pocketbase/pocketbase/tools/logger" ) @@ -51,6 +54,12 @@ func TestBaseAppLoggerLevelDevPrint(t *testing.T) { t.Fatal(err) } + // silence query logs + app.DB().(*dbx.DB).ExecLogFunc = func(ctx context.Context, t time.Duration, sql string, result sql.Result, err error) {} + app.DB().(*dbx.DB).QueryLogFunc = func(ctx context.Context, t time.Duration, sql string, rows *sql.Rows, err error) {} + app.NonconcurrentDB().(*dbx.DB).ExecLogFunc = func(ctx context.Context, t time.Duration, sql string, result sql.Result, err error) {} + app.NonconcurrentDB().(*dbx.DB).QueryLogFunc = func(ctx context.Context, t time.Duration, sql string, rows *sql.Rows, err error) {} + app.Settings().Logs.MinLevel = testLogLevel if err := app.Save(app.Settings()); err != nil { t.Fatal(err) diff --git a/migrations/1640988000_init.go b/migrations/1640988000_init.go index ce8a0cae..c0d01361 100644 --- a/migrations/1640988000_init.go +++ b/migrations/1640988000_init.go @@ -118,7 +118,7 @@ func createParamsTable(txApp core.App) error { } func createMFAsCollection(txApp core.App) error { - col := core.NewBaseCollection(core.CollectionNameMFAs, "_pbc"+core.CollectionNameMFAs) + col := core.NewBaseCollection(core.CollectionNameMFAs) col.System = true ownerRule := "@request.auth.id != '' && recordRef = @request.auth.id && collectionRef = @request.auth.collectionId" @@ -157,7 +157,7 @@ func createMFAsCollection(txApp core.App) error { } func createOTPsCollection(txApp core.App) error { - col := core.NewBaseCollection(core.CollectionNameOTPs, "_pbc"+core.CollectionNameOTPs) + col := core.NewBaseCollection(core.CollectionNameOTPs) col.System = true ownerRule := "@request.auth.id != '' && recordRef = @request.auth.id && collectionRef = @request.auth.collectionId" @@ -198,7 +198,7 @@ func createOTPsCollection(txApp core.App) error { } func createAuthOriginsCollection(txApp core.App) error { - col := core.NewBaseCollection(core.CollectionNameAuthOrigins, "_pbc"+core.CollectionNameAuthOrigins) + col := core.NewBaseCollection(core.CollectionNameAuthOrigins) col.System = true ownerRule := "@request.auth.id != '' && recordRef = @request.auth.id && collectionRef = @request.auth.collectionId" @@ -238,7 +238,7 @@ func createAuthOriginsCollection(txApp core.App) error { } func createExternalAuthsCollection(txApp core.App) error { - col := core.NewBaseCollection(core.CollectionNameExternalAuths, "_pbc"+core.CollectionNameExternalAuths) + col := core.NewBaseCollection(core.CollectionNameExternalAuths) col.System = true ownerRule := "@request.auth.id != '' && recordRef = @request.auth.id && collectionRef = @request.auth.collectionId" @@ -284,7 +284,7 @@ func createExternalAuthsCollection(txApp core.App) error { } func createSuperusersCollection(txApp core.App) error { - superusers := core.NewAuthCollection(core.CollectionNameSuperusers, "_pbc"+core.CollectionNameSuperusers) + superusers := core.NewAuthCollection(core.CollectionNameSuperusers) superusers.System = true superusers.Fields.Add(&core.EmailField{ Name: "email", diff --git a/migrations/1717233558_v0.23_migrate3.go b/migrations/1717233558_v0.23_migrate3.go new file mode 100644 index 00000000..b8ed1795 --- /dev/null +++ b/migrations/1717233558_v0.23_migrate3.go @@ -0,0 +1,138 @@ +package migrations + +import ( + "hash/crc32" + "strconv" + + "github.com/pocketbase/dbx" + "github.com/pocketbase/pocketbase/core" +) + +// note: this migration will be deleted in future version + +func collectionIdChecksum(c *core.Collection) string { + return "pbc_" + strconv.Itoa(int(crc32.ChecksumIEEE([]byte(c.Type+c.Name)))) +} + +func fieldIdChecksum(f core.Field) string { + return f.Type() + strconv.Itoa(int(crc32.ChecksumIEEE([]byte(f.GetName())))) +} + +// normalize system collection and field ids +func init() { + core.SystemMigrations.Register(func(txApp core.App) error { + collections := []*core.Collection{} + err := txApp.CollectionQuery(). + AndWhere(dbx.In( + "name", + core.CollectionNameMFAs, + core.CollectionNameOTPs, + core.CollectionNameExternalAuths, + core.CollectionNameAuthOrigins, + core.CollectionNameSuperusers, + )). + All(&collections) + if err != nil { + return err + } + + for _, c := range collections { + var needUpdate bool + + references, err := txApp.FindCollectionReferences(c, c.Id) + if err != nil { + return err + } + + authOrigins, err := txApp.FindAllAuthOriginsByCollection(c) + if err != nil { + return err + } + + mfas, err := txApp.FindAllMFAsByCollection(c) + if err != nil { + return err + } + + otps, err := txApp.FindAllOTPsByCollection(c) + if err != nil { + return err + } + + originalId := c.Id + + // normalize collection id + if checksum := collectionIdChecksum(c); c.Id != checksum { + c.Id = checksum + needUpdate = true + } + + // normalize system fields + for _, f := range c.Fields { + if !f.GetSystem() { + continue + } + + if checksum := fieldIdChecksum(f); f.GetId() != checksum { + f.SetId(checksum) + needUpdate = true + } + } + + if !needUpdate { + continue + } + + rawExport, err := c.DBExport(txApp) + if err != nil { + return err + } + + _, err = txApp.DB().Update("_collections", rawExport, dbx.HashExp{"id": originalId}).Execute() + if err != nil { + return err + } + + // update collection references + for refCollection, fields := range references { + for _, f := range fields { + relationField, ok := f.(*core.RelationField) + if !ok || relationField.CollectionId == originalId { + continue + } + + relationField.CollectionId = c.Id + } + if err = txApp.Save(refCollection); err != nil { + return err + } + } + + // update mfas references + for _, item := range mfas { + item.SetCollectionRef(c.Id) + if err = txApp.Save(item); err != nil { + return err + } + } + + // update otps references + for _, item := range otps { + item.SetCollectionRef(c.Id) + if err = txApp.Save(item); err != nil { + return err + } + } + + // update authOrigins references + for _, item := range authOrigins { + item.SetCollectionRef(c.Id) + if err = txApp.Save(item); err != nil { + return err + } + } + } + + return nil + }, nil) +} diff --git a/plugins/migratecmd/templates.go b/plugins/migratecmd/templates.go index 0c01e6eb..8297cc45 100644 --- a/plugins/migratecmd/templates.go +++ b/plugins/migratecmd/templates.go @@ -62,7 +62,7 @@ func (p *plugin) jsSnapshotTemplate(collections []*core.Collection) (string, err const template = jsTypesDirective + `migrate((app) => { const snapshot = %s; - return app.importCollections(snapshot, true); + return app.importCollections(snapshot, false); }, (app) => { return null; }) @@ -348,7 +348,7 @@ func init() { m.Register(func(app core.App) error { jsonData := ` + "`%s`" + ` - return app.ImportCollectionsByMarshaledJSON([]byte(jsonData), true) + return app.ImportCollectionsByMarshaledJSON([]byte(jsonData), false) }, func(app core.App) error { return nil }) diff --git a/tests/data/data.db b/tests/data/data.db index e7ad361c9a9fc0d338c647d408dcd1ec0c64023b..50892db4fd8d53d48a3bb3aead48e013f4ee94aa 100644 GIT binary patch delta 1294 zcmaKre`s4(6vyv-_r5eqo80%3bQ#!UwluA8>FWEHUxnam-G-~1fv(K8%q%Hc7IkY$ zmewB>8|Tn5vCUrDMMX=?*g#<EQaN%5>o_8WVcRg0Hq-=9zUNl=a;1IAsQM`2QpW>3WesuN9S9DH2`8?8Yd~uA@xr3+!`VZn7Tekh1t#3g;eh6J>nol^4 zb|~92ur2HM1ghEs?>xzwV+G3YhsG+sXue$6TVFelx+W;3TkoP(hZizUeEe`Q7&o)( z+aeEE(MY!_?iCcD817ZXUP0~&hlJtb5=4!OH-BtIC?-KSCYpb0L_|#jHYP3`#Ge0j z6&@b8<0@eAMPMBGI=+bS;V*F&7d39*04AvolyDnbxpSkT&{#ysDyO33qc16>x`90U zy$!^(4m<%EXBy(0&UGv;IhpsIo_>Z89(CY|u+u!WYZRHo)M-7@j_y$8SB}|9Ht)k;GUrDwQrX4q zBJX|;E#&o2VIT2!pmw$LaTBSykX?V`GOAGO%qLHifll&;HUX z_kI9Cl#J?A-VmntPxju!uoFGk?Of3;^Igb%?r&cKJ8e&NJHN%$_a4%`g)#yz(ry@(bq$z?L)>V%U)46;+Q8?QdcVQWD*bA{2=SYovaHHuh1u$0_Oy{at9Zmro0D5F>BaOf6 z>9HFglGArQoym%{0|uOgGJlOZc-Lj%xXc^@m+-t^=1!RxA45ko<@S@aYaOjG#)>fj ziZM?|$JInF|5Apl<8#-?=O{90YkUqSTl8%QeMadUTy5=p2xlm3;X4fMvt6TH3c5xv zXaXA3-RK3kne!3zKq*=?2gb{8^yt+@mMa*hI)<2Cx=rV(Sjdyg6n`Xh?16e61ijTgmsL1-7%HbIbjiU`$!k^ca# zO_mh`&^3Vj^pAt2wyBCp09XSI0<`|Ug(S9-R_Sr!r+~pDmCl^UBls#IA;iPgx8%&*6DlP3hp1B}}gn`gN{!~usPg-vmP%GzV=K;ee0iMC< zusFpsx7qsbOfa~&*4)d@Xw0yT&_sN8{5O@qFqG^|r_S^#pkMCX44k&PN^ diff --git a/ui/.env b/ui/.env index 57f34659..c494e0f2 100644 --- a/ui/.env +++ b/ui/.env @@ -10,4 +10,4 @@ PB_DOCS_URL = "https://pocketbase.io/docs/" PB_JS_SDK_URL = "https://github.com/pocketbase/js-sdk" PB_DART_SDK_URL = "https://github.com/pocketbase/dart-sdk" PB_RELEASES = "https://github.com/pocketbase/pocketbase/releases" -PB_VERSION = "v0.23.0-rc9" +PB_VERSION = "v0.23.0-rc10"