tomas/pocketbase
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

added split (sync and async) db connections pool

Browse Source
This commit is contained in:
Gani Georgiev
2022-12-15 16:42:35 +02:00
parent e964b019c2
commit b9e257d2b1
13 changed files with 304 additions and 127 deletions
Download Patch File Download Diff File Expand all files Collapse all files
core/app.go
+10
View File
@@ -16,6 +16,11 @@ import (
// App defines the main PocketBase app interface.
type App interface {
// Deprecated:
// This method may get removed in the near future.
// It is recommended to access the logs db instance from app.Dao().DB() or
// if you want more flexibility - app.Dao().AsyncDB() and app.Dao().SyncDB().
//
// DB returns the default app database instance.
DB() *dbx.DB
@@ -26,6 +31,11 @@ type App interface {
// trying to access the request logs table will result in error.
Dao() *daos.Dao
// Deprecated:
// This method may get removed in the near future.
// It is recommended to access the logs db instance from app.LogsDao().DB() or
// if you want more flexibility - app.LogsDao().AsyncDB() and app.LogsDao().SyncDB().
//
// LogsDB returns the app logs database instance.
LogsDB() *dbx.DB
core/base.go
+120 -29
View File
@@ -26,16 +26,18 @@ var _ App = (*BaseApp)(nil)
// BaseApp implements core.App and defines the base PocketBase app structure.
type BaseApp struct {
// configurable parameters
isDebug bool
dataDir string
encryptionEnv string
isDebug bool
dataDir string
encryptionEnv string
dataMaxOpenConns int
dataMaxIdleConns int
logsMaxOpenConns int
logsMaxIdleConns int
// internals
cache *store.Store[any]
settings *settings.Settings
db *dbx.DB
dao *daos.Dao
logsDB *dbx.DB
logsDao *daos.Dao
subscriptionsBroker *subscriptions.Broker
@@ -132,15 +134,30 @@ type BaseApp struct {
onCollectionsAfterImportRequest *hook.Hook[*CollectionsImportEvent]
}
// BaseAppConfig defines a BaseApp configuration option
type BaseAppConfig struct {
DataDir string
EncryptionEnv string
IsDebug bool
DataMaxOpenConns int // default to 600
DataMaxIdleConns int // default 20
LogsMaxOpenConns int // default to 500
LogsMaxIdleConns int // default to 10
}
// NewBaseApp creates and returns a new BaseApp instance
// configured with the provided arguments.
//
// To initialize the app, you need to call `app.Bootstrap()`.
func NewBaseApp(dataDir string, encryptionEnv string, isDebug bool) *BaseApp {
func NewBaseApp(config *BaseAppConfig) *BaseApp {
app := &BaseApp{
dataDir: dataDir,
isDebug: isDebug,
encryptionEnv: encryptionEnv,
dataDir: config.DataDir,
isDebug: config.IsDebug,
encryptionEnv: config.EncryptionEnv,
dataMaxOpenConns: config.DataMaxOpenConns,
dataMaxIdleConns: config.DataMaxIdleConns,
logsMaxOpenConns: config.LogsMaxOpenConns,
logsMaxIdleConns: config.LogsMaxIdleConns,
cache: store.New[any](nil),
settings: settings.New(),
subscriptionsBroker: subscriptions.NewBroker(),
@@ -283,14 +300,20 @@ func (app *BaseApp) Bootstrap() error {
// ResetBootstrapState takes care for releasing initialized app resources
// (eg. closing db connections).
func (app *BaseApp) ResetBootstrapState() error {
if app.db != nil {
if err := app.db.Close(); err != nil {
if app.Dao() != nil {
if err := app.Dao().AsyncDB().(*dbx.DB).Close(); err != nil {
return err
}
if err := app.Dao().SyncDB().(*dbx.DB).Close(); err != nil {
return err
}
}
if app.logsDB != nil {
if err := app.logsDB.Close(); err != nil {
if app.LogsDao() != nil {
if err := app.LogsDao().AsyncDB().(*dbx.DB).Close(); err != nil {
return err
}
if err := app.LogsDao().SyncDB().(*dbx.DB).Close(); err != nil {
return err
}
}
@@ -302,9 +325,23 @@ func (app *BaseApp) ResetBootstrapState() error {
return nil
}
// Deprecated:
// This method may get removed in the near future.
// It is recommended to access the db instance from app.Dao().DB() or
// if you want more flexibility - app.Dao().AsyncDB() and app.Dao().SyncDB().
//
// DB returns the default app database instance.
func (app *BaseApp) DB() *dbx.DB {
return app.db
if app.Dao() == nil {
return nil
}
db, ok := app.Dao().DB().(*dbx.DB)
if !ok {
return nil
}
return db
}
// Dao returns the default app Dao instance.
@@ -312,9 +349,23 @@ func (app *BaseApp) Dao() *daos.Dao {
return app.dao
}
// Deprecated:
// This method may get removed in the near future.
// It is recommended to access the logs db instance from app.LogsDao().DB() or
// if you want more flexibility - app.LogsDao().AsyncDB() and app.LogsDao().SyncDB().
//
// LogsDB returns the app logs database instance.
func (app *BaseApp) LogsDB() *dbx.DB {
return app.logsDB
if app.LogsDao() == nil {
return nil
}
db, ok := app.LogsDao().DB().(*dbx.DB)
if !ok {
return nil
}
return db
}
// LogsDao returns the app logs Dao instance.
@@ -751,41 +802,81 @@ func (app *BaseApp) OnCollectionsAfterImportRequest() *hook.Hook[*CollectionsImp
// -------------------------------------------------------------------
func (app *BaseApp) initLogsDB() error {
var connectErr error
app.logsDB, connectErr = connectDB(filepath.Join(app.DataDir(), "logs.db"))
if connectErr != nil {
return connectErr
maxOpenConns := 500
maxIdleConns := 10
if app.logsMaxOpenConns > 0 {
maxOpenConns = app.logsMaxOpenConns
}
if app.logsMaxIdleConns > 0 {
maxIdleConns = app.logsMaxIdleConns
}
app.logsDao = daos.New(app.logsDB)
asyncDB, err := connectDB(filepath.Join(app.DataDir(), "logs.db"))
if err != nil {
return err
}
asyncDB.DB().SetMaxOpenConns(maxOpenConns)
asyncDB.DB().SetMaxIdleConns(maxIdleConns)
asyncDB.DB().SetConnMaxIdleTime(5 * time.Minute)
syncDB, err := connectDB(filepath.Join(app.DataDir(), "logs.db"))
if err != nil {
return err
}
syncDB.DB().SetMaxOpenConns(1)
syncDB.DB().SetMaxIdleConns(1)
syncDB.DB().SetConnMaxIdleTime(5 * time.Minute)
app.logsDao = daos.NewMultiDB(asyncDB, syncDB)
return nil
}
func (app *BaseApp) initDataDB() error {
var connectErr error
app.db, connectErr = connectDB(filepath.Join(app.DataDir(), "data.db"))
if connectErr != nil {
return connectErr
maxOpenConns := 600
maxIdleConns := 20
if app.dataMaxOpenConns > 0 {
maxOpenConns = app.dataMaxOpenConns
}
if app.dataMaxIdleConns > 0 {
maxIdleConns = app.dataMaxIdleConns
}
asyncDB, err := connectDB(filepath.Join(app.DataDir(), "data.db"))
if err != nil {
return err
}
asyncDB.DB().SetMaxOpenConns(maxOpenConns)
asyncDB.DB().SetMaxIdleConns(maxIdleConns)
asyncDB.DB().SetConnMaxIdleTime(5 * time.Minute)
syncDB, err := connectDB(filepath.Join(app.DataDir(), "data.db"))
if err != nil {
return err
}
syncDB.DB().SetMaxOpenConns(1)
syncDB.DB().SetMaxIdleConns(1)
syncDB.DB().SetConnMaxIdleTime(5 * time.Minute)
if app.IsDebug() {
app.db.QueryLogFunc = func(ctx context.Context, t time.Duration, sql string, rows *sql.Rows, err error) {
syncDB.QueryLogFunc = func(ctx context.Context, t time.Duration, sql string, rows *sql.Rows, err error) {
color.HiBlack("[%.2fms] %v\n", float64(t.Milliseconds()), sql)
}
asyncDB.QueryLogFunc = syncDB.QueryLogFunc
app.db.ExecLogFunc = func(ctx context.Context, t time.Duration, sql string, result sql.Result, err error) {
syncDB.ExecLogFunc = func(ctx context.Context, t time.Duration, sql string, result sql.Result, err error) {
color.HiBlack("[%.2fms] %v\n", float64(t.Milliseconds()), sql)
}
asyncDB.ExecLogFunc = syncDB.ExecLogFunc
}
app.dao = app.createDaoWithHooks(app.db)
app.dao = app.createDaoWithHooks(asyncDB, syncDB)
return nil
}
func (app *BaseApp) createDaoWithHooks(db dbx.Builder) *daos.Dao {
dao := daos.New(db)
func (app *BaseApp) createDaoWithHooks(asyncDB, syncDB dbx.Builder) *daos.Dao {
dao := daos.NewMultiDB(asyncDB, syncDB)
dao.BeforeCreateFunc = func(eventDao *daos.Dao, m models.Model) error {
return app.OnModelBeforeCreate().Trigger(&ModelEvent{eventDao, m})
core/base_test.go
+31 -11
View File
@@ -11,7 +11,11 @@ func TestNewBaseApp(t *testing.T) {
const testDataDir = "./pb_base_app_test_data_dir/"
defer os.RemoveAll(testDataDir)
app := NewBaseApp(testDataDir, "test_env", true)
app := NewBaseApp(&BaseAppConfig{
DataDir: testDataDir,
EncryptionEnv: "test_env",
IsDebug: true,
})
if app.dataDir != testDataDir {
t.Fatalf("expected dataDir %q, got %q", testDataDir, app.dataDir)
@@ -42,7 +46,11 @@ func TestBaseAppBootstrap(t *testing.T) {
const testDataDir = "./pb_base_app_test_data_dir/"
defer os.RemoveAll(testDataDir)
app := NewBaseApp(testDataDir, "pb_test_env", false)
app := NewBaseApp(&BaseAppConfig{
DataDir: testDataDir,
EncryptionEnv: "pb_test_env",
IsDebug: false,
})
defer app.ResetBootstrapState()
// bootstrap
@@ -112,29 +120,33 @@ func TestBaseAppGetters(t *testing.T) {
const testDataDir = "./pb_base_app_test_data_dir/"
defer os.RemoveAll(testDataDir)
app := NewBaseApp(testDataDir, "pb_test_env", false)
app := NewBaseApp(&BaseAppConfig{
DataDir: testDataDir,
EncryptionEnv: "pb_test_env",
IsDebug: false,
})
defer app.ResetBootstrapState()
if err := app.Bootstrap(); err != nil {
t.Fatal(err)
}
if app.db != app.DB() {
t.Fatalf("Expected app.DB %v, got %v", app.DB(), app.db)
}
if app.dao != app.Dao() {
t.Fatalf("Expected app.Dao %v, got %v", app.Dao(), app.dao)
}
if app.logsDB != app.LogsDB() {
t.Fatalf("Expected app.LogsDB %v, got %v", app.LogsDB(), app.logsDB)
if app.dao.AsyncDB() != app.DB() {
t.Fatalf("Expected app.DB %v, got %v", app.DB(), app.dao.AsyncDB())
}
if app.logsDao != app.LogsDao() {
t.Fatalf("Expected app.LogsDao %v, got %v", app.LogsDao(), app.logsDao)
}
if app.logsDao.AsyncDB() != app.LogsDB() {
t.Fatalf("Expected app.LogsDB %v, got %v", app.LogsDB(), app.logsDao.AsyncDB())
}
if app.dataDir != app.DataDir() {
t.Fatalf("Expected app.DataDir %v, got %v", app.DataDir(), app.dataDir)
}
@@ -400,7 +412,11 @@ func TestBaseAppNewMailClient(t *testing.T) {
const testDataDir = "./pb_base_app_test_data_dir/"
defer os.RemoveAll(testDataDir)
app := NewBaseApp(testDataDir, "pb_test_env", false)
app := NewBaseApp(&BaseAppConfig{
DataDir: testDataDir,
EncryptionEnv: "pb_test_env",
IsDebug: false,
})
client1 := app.NewMailClient()
if val, ok := client1.(*mailer.Sendmail); !ok {
@@ -419,7 +435,11 @@ func TestBaseAppNewFilesystem(t *testing.T) {
const testDataDir = "./pb_base_app_test_data_dir/"
defer os.RemoveAll(testDataDir)
app := NewBaseApp(testDataDir, "pb_test_env", false)
app := NewBaseApp(&BaseAppConfig{
DataDir: testDataDir,
EncryptionEnv: "pb_test_env",
IsDebug: false,
})
// local
local, localErr := app.NewFilesystem()
core/db.go
+20
View File
@@ -0,0 +1,20 @@
package core
import (
"github.com/pocketbase/dbx"
)
func initPragmas(db *dbx.DB) error {
// note: the busy_timeout pragma must be first because
// the connection needs to be set to block on busy before WAL mode
// is set in case it hasn't been already set by another connection
_, err := db.NewQuery(`
PRAGMA busy_timeout = 10000;
PRAGMA journal_mode = WAL;
PRAGMA journal_size_limit = 100000000;
PRAGMA synchronous = NORMAL;
PRAGMA foreign_keys = TRUE;
`).Execute()
return err
}
core/db_cgo.go
+9 -24
View File
@@ -3,35 +3,20 @@
package core
import (
"fmt"
"time"
_ "github.com/mattn/go-sqlite3"
"github.com/pocketbase/dbx"
_ "github.com/mattn/go-sqlite3"
)
func connectDB(dbPath string) (*dbx.DB, error) {
// note: the busy_timeout pragma must be first because
// the connection needs to be set to block on busy before WAL mode
// is set in case it hasn't been already set by another connection
pragmas := "_busy_timeout=10000&_journal_mode=WAL&_foreign_keys=1&_synchronous=NORMAL"
db, openErr := dbx.MustOpen("sqlite3", fmt.Sprintf("%s?%s", dbPath, pragmas))
if openErr != nil {
return nil, openErr
db, err := dbx.Open("sqlite3", dbPath)
if err != nil {
return nil, err
}
// use a fixed connection pool to limit the SQLITE_BUSY errors
// and reduce the open file descriptors
// (the limits are arbitrary and may change in the future)
db.DB().SetMaxOpenConns(30)
db.DB().SetMaxIdleConns(30)
db.DB().SetConnMaxIdleTime(5 * time.Minute)
if err := initPragmas(db); err != nil {
db.Close()
return nil, err
}
// additional pragmas not supported through the dsn string
_, err := db.NewQuery(`
pragma journal_size_limit = 100000000;
`).Execute()
return db, err
return db, nil
}
core/db_nocgo.go
+5 -15
View File
@@ -3,30 +3,20 @@
package core
import (
"fmt"
"time"
"github.com/pocketbase/dbx"
_ "modernc.org/sqlite"
)
func connectDB(dbPath string) (*dbx.DB, error) {
// note: the busy_timeout pragma must be first because
// the connection needs to be set to block on busy before WAL mode
// is set in case it hasn't been already set by another connection
pragmas := "_pragma=busy_timeout(10000)&_pragma=journal_mode(WAL)&_pragma=foreign_keys(1)&_pragma=synchronous(NORMAL)&_pragma=journal_size_limit(100000000)"
db, err := dbx.MustOpen("sqlite", fmt.Sprintf("%s?%s", dbPath, pragmas))
db, err := dbx.Open("sqlite", dbPath)
if err != nil {
return nil, err
}
// use a fixed connection pool to limit the SQLITE_BUSY errors
// and reduce the open file descriptors
// (the limits are arbitrary and may change in the future)
db.DB().SetMaxOpenConns(30)
db.DB().SetMaxIdleConns(30)
db.DB().SetConnMaxIdleTime(5 * time.Minute)
if err := initPragmas(db); err != nil {
db.Close()
return nil, err
}
return db, nil
}