From 627b8eeae6b7e3ad11d2b5b0b3f9f46c4aca055f Mon Sep 17 00:00:00 2001 From: tobi <31960611+tsmethurst@users.noreply.github.com> Date: Tue, 17 Jan 2023 13:29:44 +0100 Subject: [PATCH] [feature] Tune sqlite pragmas (#1349) * sqlite pragma tuning * use formatuint * add sqlite busy timeout * fix incorrect cache size format * update envparsing test * add sqlite tuning flags to cli * set sqlite timeout to 30s default --- docs/configuration/database.md | 32 +++++++++ example/config.yaml | 32 +++++++++ internal/config/config.go | 20 +++--- internal/config/defaults.go | 20 +++--- internal/config/flags.go | 4 ++ internal/config/helpers.gen.go | 100 ++++++++++++++++++++++++++ internal/db/bundb/bundb.go | 126 ++++++++++++++++++++------------- test/envparsing.sh | 6 +- testrig/config.go | 19 +++-- testrig/db.go | 12 +--- 10 files changed, 291 insertions(+), 80 deletions(-) diff --git a/docs/configuration/database.md b/docs/configuration/database.md index 336279967..90487c17b 100644 --- a/docs/configuration/database.md +++ b/docs/configuration/database.md @@ -107,4 +107,36 @@ db-tls-mode: "disable" # Examples: ["/path/to/some/cert.crt"] # Default: "" db-tls-ca-cert: "" + +# String. SQLite journaling mode. +# SQLite only -- unused otherwise. +# If set to empty string, the sqlite default will be used. +# See: https://www.sqlite.org/pragma.html#pragma_journal_mode +# Examples: ["DELETE", "TRUNCATE", "PERSIST", "MEMORY", "WAL", "OFF"] +# Default: "WAL" +db-sqlite-journal-mode: "WAL" + +# String. SQLite synchronous mode. +# SQLite only -- unused otherwise. +# If set to empty string, the sqlite default will be used. +# See: https://www.sqlite.org/pragma.html#pragma_synchronous +# Examples: ["OFF", "NORMAL", "FULL", "EXTRA"] +# Default: "NORMAL" +db-sqlite-synchronous: "NORMAL" + +# Byte size. SQlite cache size. +# SQLite only -- unused otherwise. +# If set to empty string or zero, the sqlite default will be used. +# See: https://www.sqlite.org/pragma.html#pragma_cache_size +# Examples: ["32MiB", "0", "64MiB"] +# Default: "64MiB" +db-sqlite-cache-size: "64MiB" + +# Duration. SQlite busy timeout. +# SQLite only -- unused otherwise. +# If set to empty string or zero, the sqlite default will be used. +# See: https://www.sqlite.org/pragma.html#pragma_busy_timeout +# Examples: ["0s", "1s", "30s", "1m", "5m"] +# Default: "5s" +db-sqlite-busy-timeout: "30s" ``` diff --git a/example/config.yaml b/example/config.yaml index 71d26e50c..5ce622a88 100644 --- a/example/config.yaml +++ b/example/config.yaml @@ -164,6 +164,38 @@ db-tls-mode: "disable" # Default: "" db-tls-ca-cert: "" +# String. SQLite journaling mode. +# SQLite only -- unused otherwise. +# If set to empty string, the sqlite default will be used. +# See: https://www.sqlite.org/pragma.html#pragma_journal_mode +# Examples: ["DELETE", "TRUNCATE", "PERSIST", "MEMORY", "WAL", "OFF"] +# Default: "WAL" +db-sqlite-journal-mode: "WAL" + +# String. SQLite synchronous mode. +# SQLite only -- unused otherwise. +# If set to empty string, the sqlite default will be used. +# See: https://www.sqlite.org/pragma.html#pragma_synchronous +# Examples: ["OFF", "NORMAL", "FULL", "EXTRA"] +# Default: "NORMAL" +db-sqlite-synchronous: "NORMAL" + +# Byte size. SQlite cache size. +# SQLite only -- unused otherwise. +# If set to empty string or zero, the sqlite default will be used. +# See: https://www.sqlite.org/pragma.html#pragma_cache_size +# Examples: ["32MiB", "0", "64MiB"] +# Default: "64MiB" +db-sqlite-cache-size: "64MiB" + +# Duration. SQlite busy timeout. +# SQLite only -- unused otherwise. +# If set to empty string or zero, the sqlite default will be used. +# See: https://www.sqlite.org/pragma.html#pragma_busy_timeout +# Examples: ["0s", "1s", "30s", "1m", "5m"] +# Default: "5s" +db-sqlite-busy-timeout: "30s" + cache: gts: ########################### diff --git a/internal/config/config.go b/internal/config/config.go index ec8675f2d..c28cfe419 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -58,14 +58,18 @@ type Configuration struct { TrustedProxies []string `name:"trusted-proxies" usage:"Proxies to trust when parsing x-forwarded headers into real IPs."` SoftwareVersion string `name:"software-version" usage:""` - DbType string `name:"db-type" usage:"Database type: eg., postgres"` - DbAddress string `name:"db-address" usage:"Database ipv4 address, hostname, or filename"` - DbPort int `name:"db-port" usage:"Database port"` - DbUser string `name:"db-user" usage:"Database username"` - DbPassword string `name:"db-password" usage:"Database password"` - DbDatabase string `name:"db-database" usage:"Database name"` - DbTLSMode string `name:"db-tls-mode" usage:"Database tls mode"` - DbTLSCACert string `name:"db-tls-ca-cert" usage:"Path to CA cert for db tls connection"` + DbType string `name:"db-type" usage:"Database type: eg., postgres"` + DbAddress string `name:"db-address" usage:"Database ipv4 address, hostname, or filename"` + DbPort int `name:"db-port" usage:"Database port"` + DbUser string `name:"db-user" usage:"Database username"` + DbPassword string `name:"db-password" usage:"Database password"` + DbDatabase string `name:"db-database" usage:"Database name"` + DbTLSMode string `name:"db-tls-mode" usage:"Database tls mode"` + DbTLSCACert string `name:"db-tls-ca-cert" usage:"Path to CA cert for db tls connection"` + DbSqliteJournalMode string `name:"db-sqlite-journal-mode" usage:"Sqlite only: see https://www.sqlite.org/pragma.html#pragma_journal_mode"` + DbSqliteSynchronous string `name:"db-sqlite-synchronous" usage:"Sqlite only: see https://www.sqlite.org/pragma.html#pragma_synchronous"` + DbSqliteCacheSize bytesize.Size `name:"db-sqlite-cache-size" usage:"Sqlite only: see https://www.sqlite.org/pragma.html#pragma_cache_size"` + DbSqliteBusyTimeout time.Duration `name:"db-sqlite-busy-timeout" usage:"Sqlite only: see https://www.sqlite.org/pragma.html#pragma_busy_timeout"` WebTemplateBaseDir string `name:"web-template-base-dir" usage:"Basedir for html templating files for rendering pages and composing emails."` WebAssetBaseDir string `name:"web-asset-base-dir" usage:"Directory to serve static assets from, accessible at example.org/assets/"` diff --git a/internal/config/defaults.go b/internal/config/defaults.go index 4d61bec05..31f282113 100644 --- a/internal/config/defaults.go +++ b/internal/config/defaults.go @@ -40,14 +40,18 @@ var Defaults = Configuration{ Port: 8080, TrustedProxies: []string{"127.0.0.1/32", "::1"}, // localhost - DbType: "postgres", - DbAddress: "", - DbPort: 5432, - DbUser: "", - DbPassword: "", - DbDatabase: "gotosocial", - DbTLSMode: "disable", - DbTLSCACert: "", + DbType: "postgres", + DbAddress: "", + DbPort: 5432, + DbUser: "", + DbPassword: "", + DbDatabase: "gotosocial", + DbTLSMode: "disable", + DbTLSCACert: "", + DbSqliteJournalMode: "WAL", + DbSqliteSynchronous: "NORMAL", + DbSqliteCacheSize: 64 * bytesize.MiB, + DbSqliteBusyTimeout: time.Second * 30, WebTemplateBaseDir: "./web/template/", WebAssetBaseDir: "./web/assets/", diff --git a/internal/config/flags.go b/internal/config/flags.go index e3d1b20da..a0fde3eed 100644 --- a/internal/config/flags.go +++ b/internal/config/flags.go @@ -51,6 +51,10 @@ func (s *ConfigState) AddGlobalFlags(cmd *cobra.Command) { cmd.PersistentFlags().String(DbDatabaseFlag(), cfg.DbDatabase, fieldtag("DbDatabase", "usage")) cmd.PersistentFlags().String(DbTLSModeFlag(), cfg.DbTLSMode, fieldtag("DbTLSMode", "usage")) cmd.PersistentFlags().String(DbTLSCACertFlag(), cfg.DbTLSCACert, fieldtag("DbTLSCACert", "usage")) + cmd.PersistentFlags().String(DbSqliteJournalModeFlag(), cfg.DbSqliteJournalMode, fieldtag("DbSqliteJournalMode", "usage")) + cmd.PersistentFlags().String(DbSqliteSynchronousFlag(), cfg.DbSqliteSynchronous, fieldtag("DbSqliteSynchronous", "usage")) + cmd.PersistentFlags().Uint64(DbSqliteCacheSizeFlag(), uint64(cfg.DbSqliteCacheSize), fieldtag("DbSqliteCacheSize", "usage")) + cmd.PersistentFlags().Duration(DbSqliteBusyTimeoutFlag(), cfg.DbSqliteBusyTimeout, fieldtag("DbSqliteBusyTimeout", "usage")) }) } diff --git a/internal/config/helpers.gen.go b/internal/config/helpers.gen.go index f340360b2..1da2ff42c 100644 --- a/internal/config/helpers.gen.go +++ b/internal/config/helpers.gen.go @@ -524,6 +524,106 @@ func GetDbTLSCACert() string { return global.GetDbTLSCACert() } // SetDbTLSCACert safely sets the value for global configuration 'DbTLSCACert' field func SetDbTLSCACert(v string) { global.SetDbTLSCACert(v) } +// GetDbSqliteJournalMode safely fetches the Configuration value for state's 'DbSqliteJournalMode' field +func (st *ConfigState) GetDbSqliteJournalMode() (v string) { + st.mutex.Lock() + v = st.config.DbSqliteJournalMode + st.mutex.Unlock() + return +} + +// SetDbSqliteJournalMode safely sets the Configuration value for state's 'DbSqliteJournalMode' field +func (st *ConfigState) SetDbSqliteJournalMode(v string) { + st.mutex.Lock() + defer st.mutex.Unlock() + st.config.DbSqliteJournalMode = v + st.reloadToViper() +} + +// DbSqliteJournalModeFlag returns the flag name for the 'DbSqliteJournalMode' field +func DbSqliteJournalModeFlag() string { return "db-sqlite-journal-mode" } + +// GetDbSqliteJournalMode safely fetches the value for global configuration 'DbSqliteJournalMode' field +func GetDbSqliteJournalMode() string { return global.GetDbSqliteJournalMode() } + +// SetDbSqliteJournalMode safely sets the value for global configuration 'DbSqliteJournalMode' field +func SetDbSqliteJournalMode(v string) { global.SetDbSqliteJournalMode(v) } + +// GetDbSqliteSynchronous safely fetches the Configuration value for state's 'DbSqliteSynchronous' field +func (st *ConfigState) GetDbSqliteSynchronous() (v string) { + st.mutex.Lock() + v = st.config.DbSqliteSynchronous + st.mutex.Unlock() + return +} + +// SetDbSqliteSynchronous safely sets the Configuration value for state's 'DbSqliteSynchronous' field +func (st *ConfigState) SetDbSqliteSynchronous(v string) { + st.mutex.Lock() + defer st.mutex.Unlock() + st.config.DbSqliteSynchronous = v + st.reloadToViper() +} + +// DbSqliteSynchronousFlag returns the flag name for the 'DbSqliteSynchronous' field +func DbSqliteSynchronousFlag() string { return "db-sqlite-synchronous" } + +// GetDbSqliteSynchronous safely fetches the value for global configuration 'DbSqliteSynchronous' field +func GetDbSqliteSynchronous() string { return global.GetDbSqliteSynchronous() } + +// SetDbSqliteSynchronous safely sets the value for global configuration 'DbSqliteSynchronous' field +func SetDbSqliteSynchronous(v string) { global.SetDbSqliteSynchronous(v) } + +// GetDbSqliteCacheSize safely fetches the Configuration value for state's 'DbSqliteCacheSize' field +func (st *ConfigState) GetDbSqliteCacheSize() (v bytesize.Size) { + st.mutex.Lock() + v = st.config.DbSqliteCacheSize + st.mutex.Unlock() + return +} + +// SetDbSqliteCacheSize safely sets the Configuration value for state's 'DbSqliteCacheSize' field +func (st *ConfigState) SetDbSqliteCacheSize(v bytesize.Size) { + st.mutex.Lock() + defer st.mutex.Unlock() + st.config.DbSqliteCacheSize = v + st.reloadToViper() +} + +// DbSqliteCacheSizeFlag returns the flag name for the 'DbSqliteCacheSize' field +func DbSqliteCacheSizeFlag() string { return "db-sqlite-cache-size" } + +// GetDbSqliteCacheSize safely fetches the value for global configuration 'DbSqliteCacheSize' field +func GetDbSqliteCacheSize() bytesize.Size { return global.GetDbSqliteCacheSize() } + +// SetDbSqliteCacheSize safely sets the value for global configuration 'DbSqliteCacheSize' field +func SetDbSqliteCacheSize(v bytesize.Size) { global.SetDbSqliteCacheSize(v) } + +// GetDbSqliteBusyTimeout safely fetches the Configuration value for state's 'DbSqliteBusyTimeout' field +func (st *ConfigState) GetDbSqliteBusyTimeout() (v time.Duration) { + st.mutex.Lock() + v = st.config.DbSqliteBusyTimeout + st.mutex.Unlock() + return +} + +// SetDbSqliteBusyTimeout safely sets the Configuration value for state's 'DbSqliteBusyTimeout' field +func (st *ConfigState) SetDbSqliteBusyTimeout(v time.Duration) { + st.mutex.Lock() + defer st.mutex.Unlock() + st.config.DbSqliteBusyTimeout = v + st.reloadToViper() +} + +// DbSqliteBusyTimeoutFlag returns the flag name for the 'DbSqliteBusyTimeout' field +func DbSqliteBusyTimeoutFlag() string { return "db-sqlite-busy-timeout" } + +// GetDbSqliteBusyTimeout safely fetches the value for global configuration 'DbSqliteBusyTimeout' field +func GetDbSqliteBusyTimeout() time.Duration { return global.GetDbSqliteBusyTimeout() } + +// SetDbSqliteBusyTimeout safely sets the value for global configuration 'DbSqliteBusyTimeout' field +func SetDbSqliteBusyTimeout(v time.Duration) { global.SetDbSqliteBusyTimeout(v) } + // GetWebTemplateBaseDir safely fetches the Configuration value for state's 'WebTemplateBaseDir' field func (st *ConfigState) GetWebTemplateBaseDir() (v string) { st.mutex.Lock() diff --git a/internal/db/bundb/bundb.go b/internal/db/bundb/bundb.go index 1225b2bb0..b6a07bdc6 100644 --- a/internal/db/bundb/bundb.go +++ b/internal/db/bundb/bundb.go @@ -28,9 +28,11 @@ import ( "fmt" "os" "runtime" + "strconv" "strings" "time" + "codeberg.org/gruf/go-bytesize" "github.com/google/uuid" "github.com/jackc/pgx/v4" "github.com/jackc/pgx/v4/stdlib" @@ -49,22 +51,6 @@ import ( "modernc.org/sqlite" ) -const ( - dbTypePostgres = "postgres" - dbTypeSqlite = "sqlite" - - // dbTLSModeDisable does not attempt to make a TLS connection to the database. - dbTLSModeDisable = "disable" - // dbTLSModeEnable attempts to make a TLS connection to the database, but doesn't fail if - // the certificate passed by the database isn't verified. - dbTLSModeEnable = "enable" - // dbTLSModeRequire attempts to make a TLS connection to the database, and requires - // that the certificate presented by the database is valid. - dbTLSModeRequire = "require" - // dbTLSModeUnset means that the TLS mode has not been set. - dbTLSModeUnset = "" -) - var registerTables = []interface{}{ >smodel.AccountToEmoji{}, >smodel.StatusToEmoji{}, @@ -127,26 +113,34 @@ func doMigration(ctx context.Context, db *bun.DB) error { func NewBunDBService(ctx context.Context, state *state.State) (db.DB, error) { var conn *DBConn var err error - dbType := strings.ToLower(config.GetDbType()) + t := strings.ToLower(config.GetDbType()) - switch dbType { - case dbTypePostgres: + switch t { + case "postgres": conn, err = pgConn(ctx) if err != nil { return nil, err } - case dbTypeSqlite: + case "sqlite": conn, err = sqliteConn(ctx) if err != nil { return nil, err } default: - return nil, fmt.Errorf("database type %s not supported for bundb", dbType) + return nil, fmt.Errorf("database type %s not supported for bundb", t) } // Add database query hook conn.DB.AddQueryHook(queryHook{}) + // execute sqlite pragmas *after* adding database hook; + // this allows the pragma queries to be logged + if t == "sqlite" { + if err := sqlitePragmas(ctx, conn); err != nil { + return nil, err + } + } + // table registration is needed for many-to-many, see: // https://bun.uptrace.dev/orm/many-to-many-relation/ for _, t := range registerTables { @@ -230,29 +224,29 @@ func NewBunDBService(ctx context.Context, state *state.State) (db.DB, error) { func sqliteConn(ctx context.Context) (*DBConn, error) { // validate db address has actually been set - dbAddress := config.GetDbAddress() - if dbAddress == "" { + address := config.GetDbAddress() + if address == "" { return nil, fmt.Errorf("'%s' was not set when attempting to start sqlite", config.DbAddressFlag()) } // Drop anything fancy from DB address - dbAddress = strings.Split(dbAddress, "?")[0] - dbAddress = strings.TrimPrefix(dbAddress, "file:") + address = strings.Split(address, "?")[0] + address = strings.TrimPrefix(address, "file:") // Append our own SQLite preferences - dbAddress = "file:" + dbAddress + "?cache=shared" + address = "file:" + address var inMem bool - if dbAddress == "file::memory:?cache=shared" { - dbAddress = fmt.Sprintf("file:%s?mode=memory&cache=shared", uuid.NewString()) - log.Infof("using in-memory database address " + dbAddress) + if address == "file::memory:" { + address = fmt.Sprintf("file:%s?mode=memory&cache=shared", uuid.NewString()) + log.Infof("using in-memory database address " + address) log.Warn("sqlite in-memory database should only be used for debugging") inMem = true } // Open new DB instance - sqldb, err := sql.Open("sqlite", dbAddress) + sqldb, err := sql.Open("sqlite", address) if err != nil { if errWithCode, ok := err.(*sqlite.Error); ok { err = errors.New(sqlite.ErrorCodeString[errWithCode.Code()]) @@ -260,8 +254,6 @@ func sqliteConn(ctx context.Context) (*DBConn, error) { return nil, fmt.Errorf("could not open sqlite db: %s", err) } - tweakConnectionValues(sqldb) - if inMem { // don't close connections on disconnect -- otherwise // the SQLite database will be deleted when there @@ -269,6 +261,7 @@ func sqliteConn(ctx context.Context) (*DBConn, error) { sqldb.SetConnMaxLifetime(0) } + // Wrap Bun database conn in our own wrapper conn := WrapDBConn(bun.NewDB(sqldb, sqlitedialect.New())) // ping to check the db is there and listening @@ -278,11 +271,56 @@ func sqliteConn(ctx context.Context) (*DBConn, error) { } return nil, fmt.Errorf("sqlite ping: %s", err) } - log.Info("connected to SQLITE database") + return conn, nil } +func sqlitePragmas(ctx context.Context, conn *DBConn) error { + var pragmas [][]string + if mode := config.GetDbSqliteJournalMode(); mode != "" { + // Set the user provided SQLite journal mode + pragmas = append(pragmas, []string{"journal_mode", mode}) + } + + if mode := config.GetDbSqliteSynchronous(); mode != "" { + // Set the user provided SQLite synchronous mode + pragmas = append(pragmas, []string{"synchronous", mode}) + } + + if size := config.GetDbSqliteCacheSize(); size > 0 { + // Set the user provided SQLite cache size (in kibibytes) + // Prepend a '-' character to this to indicate to sqlite + // that we're giving kibibytes rather than num pages. + // https://www.sqlite.org/pragma.html#pragma_cache_size + s := "-" + strconv.FormatUint(uint64(size/bytesize.KiB), 10) + pragmas = append(pragmas, []string{"cache_size", s}) + } + + if timeout := config.GetDbSqliteBusyTimeout(); timeout > 0 { + t := strconv.FormatInt(timeout.Milliseconds(), 10) + pragmas = append(pragmas, []string{"busy_timeout", t}) + } + + for _, p := range pragmas { + pk := p[0] + pv := p[1] + + if _, err := conn.DB.ExecContext(ctx, "PRAGMA ?=?", bun.Ident(pk), bun.Safe(pv)); err != nil { + return fmt.Errorf("error executing sqlite pragma %s: %w", pk, err) + } + + var res string + if err := conn.DB.NewRaw("PRAGMA ?", bun.Ident(pk)).Scan(ctx, &res); err != nil { + return fmt.Errorf("error scanning sqlite pragma %s: %w", pv, err) + } + + log.Infof("sqlite pragma %s set to %s", pk, res) + } + + return nil +} + func pgConn(ctx context.Context) (*DBConn, error) { opts, err := deriveBunDBPGOptions() //nolint:contextcheck if err != nil { @@ -291,7 +329,10 @@ func pgConn(ctx context.Context) (*DBConn, error) { sqldb := stdlib.OpenDB(*opts) - tweakConnectionValues(sqldb) + // https://bun.uptrace.dev/postgres/running-bun-in-production.html#database-sql + maxOpenConns := 4 * runtime.GOMAXPROCS(0) + sqldb.SetMaxOpenConns(maxOpenConns) + sqldb.SetMaxIdleConns(maxOpenConns) conn := WrapDBConn(bun.NewDB(sqldb, pgdialect.New())) @@ -311,10 +352,6 @@ func pgConn(ctx context.Context) (*DBConn, error) { // deriveBunDBPGOptions takes an application config and returns either a ready-to-use set of options // with sensible defaults, or an error if it's not satisfied by the provided config. func deriveBunDBPGOptions() (*pgx.ConnConfig, error) { - if strings.ToUpper(config.GetDbType()) != db.DBTypePostgres { - return nil, fmt.Errorf("expected db type of %s but got %s", db.DBTypePostgres, config.DbTypeFlag()) - } - // these are all optional, the db adapter figures out defaults address := config.GetDbAddress() @@ -326,14 +363,14 @@ func deriveBunDBPGOptions() (*pgx.ConnConfig, error) { var tlsConfig *tls.Config switch config.GetDbTLSMode() { - case dbTLSModeDisable, dbTLSModeUnset: + case "", "disable": break // nothing to do - case dbTLSModeEnable: + case "enable": /* #nosec G402 */ tlsConfig = &tls.Config{ InsecureSkipVerify: true, } - case dbTLSModeRequire: + case "require": tlsConfig = &tls.Config{ InsecureSkipVerify: false, ServerName: address, @@ -397,13 +434,6 @@ func deriveBunDBPGOptions() (*pgx.ConnConfig, error) { return cfg, nil } -// https://bun.uptrace.dev/postgres/running-bun-in-production.html#database-sql -func tweakConnectionValues(sqldb *sql.DB) { - maxOpenConns := 4 * runtime.GOMAXPROCS(0) - sqldb.SetMaxOpenConns(maxOpenConns) - sqldb.SetMaxIdleConns(maxOpenConns) -} - /* CONVERSION FUNCTIONS */ diff --git a/test/envparsing.sh b/test/envparsing.sh index b7d3083be..067da5792 100755 --- a/test/envparsing.sh +++ b/test/envparsing.sh @@ -2,7 +2,7 @@ set -eu -EXPECT='{"account-domain":"peepee","accounts-allow-custom-css":true,"accounts-approval-required":false,"accounts-reason-required":false,"accounts-registration-open":true,"advanced-cookies-samesite":"strict","advanced-rate-limit-requests":6969,"advanced-throttling-multiplier":-1,"application-name":"gts","bind-address":"127.0.0.1","cache":{"gts":{"account-max-size":99,"account-sweep-freq":1000000000,"account-ttl":10800000000000,"block-max-size":100,"block-sweep-freq":10000000000,"block-ttl":300000000000,"domain-block-max-size":1000,"domain-block-sweep-freq":60000000000,"domain-block-ttl":86400000000000,"emoji-category-max-size":100,"emoji-category-sweep-freq":10000000000,"emoji-category-ttl":300000000000,"emoji-max-size":500,"emoji-sweep-freq":10000000000,"emoji-ttl":300000000000,"mention-max-size":500,"mention-sweep-freq":10000000000,"mention-ttl":300000000000,"notification-max-size":500,"notification-sweep-freq":10000000000,"notification-ttl":300000000000,"report-max-size":100,"report-sweep-freq":10000000000,"report-ttl":300000000000,"status-max-size":500,"status-sweep-freq":10000000000,"status-ttl":300000000000,"tombstone-max-size":100,"tombstone-sweep-freq":10000000000,"tombstone-ttl":300000000000,"user-max-size":100,"user-sweep-freq":10000000000,"user-ttl":300000000000}},"config-path":"internal/config/testdata/test.yaml","db-address":":memory:","db-database":"gotosocial_prod","db-password":"hunter2","db-port":6969,"db-tls-ca-cert":"","db-tls-mode":"disable","db-type":"sqlite","db-user":"sex-haver","dry-run":false,"email":"","host":"example.com","instance-deliver-to-shared-inboxes":false,"instance-expose-peers":true,"instance-expose-public-timeline":true,"instance-expose-suspended":true,"landing-page-user":"admin","letsencrypt-cert-dir":"/gotosocial/storage/certs","letsencrypt-email-address":"","letsencrypt-enabled":true,"letsencrypt-port":80,"log-db-queries":true,"log-level":"info","media-description-max-chars":5000,"media-description-min-chars":69,"media-emoji-local-max-size":420,"media-emoji-remote-max-size":420,"media-image-max-size":420,"media-remote-cache-days":30,"media-video-max-size":420,"oidc-client-id":"1234","oidc-client-secret":"shhhh its a secret","oidc-enabled":true,"oidc-idp-name":"sex-haver","oidc-issuer":"whoknows","oidc-link-existing":true,"oidc-scopes":["read","write"],"oidc-skip-verification":true,"password":"","path":"","port":6969,"protocol":"http","smtp-from":"queen.rip.in.piss@terfisland.org","smtp-host":"example.com","smtp-password":"hunter2","smtp-port":4269,"smtp-username":"sex-haver","software-version":"","statuses-cw-max-chars":420,"statuses-max-chars":69,"statuses-media-max-files":1,"statuses-poll-max-options":1,"statuses-poll-option-max-chars":50,"storage-backend":"local","storage-local-base-path":"/root/store","storage-s3-access-key":"minio","storage-s3-bucket":"gts","storage-s3-endpoint":"localhost:9000","storage-s3-proxy":true,"storage-s3-secret-key":"miniostorage","storage-s3-use-ssl":false,"syslog-address":"127.0.0.1:6969","syslog-enabled":true,"syslog-protocol":"udp","trusted-proxies":["127.0.0.1/32","docker.host.local"],"username":"","web-asset-base-dir":"/root","web-template-base-dir":"/root"}' +EXPECT='{"account-domain":"peepee","accounts-allow-custom-css":true,"accounts-approval-required":false,"accounts-reason-required":false,"accounts-registration-open":true,"advanced-cookies-samesite":"strict","advanced-rate-limit-requests":6969,"advanced-throttling-multiplier":-1,"application-name":"gts","bind-address":"127.0.0.1","cache":{"gts":{"account-max-size":99,"account-sweep-freq":1000000000,"account-ttl":10800000000000,"block-max-size":100,"block-sweep-freq":10000000000,"block-ttl":300000000000,"domain-block-max-size":1000,"domain-block-sweep-freq":60000000000,"domain-block-ttl":86400000000000,"emoji-category-max-size":100,"emoji-category-sweep-freq":10000000000,"emoji-category-ttl":300000000000,"emoji-max-size":500,"emoji-sweep-freq":10000000000,"emoji-ttl":300000000000,"mention-max-size":500,"mention-sweep-freq":10000000000,"mention-ttl":300000000000,"notification-max-size":500,"notification-sweep-freq":10000000000,"notification-ttl":300000000000,"report-max-size":100,"report-sweep-freq":10000000000,"report-ttl":300000000000,"status-max-size":500,"status-sweep-freq":10000000000,"status-ttl":300000000000,"tombstone-max-size":100,"tombstone-sweep-freq":10000000000,"tombstone-ttl":300000000000,"user-max-size":100,"user-sweep-freq":10000000000,"user-ttl":300000000000}},"config-path":"internal/config/testdata/test.yaml","db-address":":memory:","db-database":"gotosocial_prod","db-password":"hunter2","db-port":6969,"db-sqlite-busy-timeout":1000000000,"db-sqlite-cache-size":0,"db-sqlite-journal-mode":"DELETE","db-sqlite-synchronous":"FULL","db-tls-ca-cert":"","db-tls-mode":"disable","db-type":"sqlite","db-user":"sex-haver","dry-run":false,"email":"","host":"example.com","instance-deliver-to-shared-inboxes":false,"instance-expose-peers":true,"instance-expose-public-timeline":true,"instance-expose-suspended":true,"landing-page-user":"admin","letsencrypt-cert-dir":"/gotosocial/storage/certs","letsencrypt-email-address":"","letsencrypt-enabled":true,"letsencrypt-port":80,"log-db-queries":true,"log-level":"info","media-description-max-chars":5000,"media-description-min-chars":69,"media-emoji-local-max-size":420,"media-emoji-remote-max-size":420,"media-image-max-size":420,"media-remote-cache-days":30,"media-video-max-size":420,"oidc-client-id":"1234","oidc-client-secret":"shhhh its a secret","oidc-enabled":true,"oidc-idp-name":"sex-haver","oidc-issuer":"whoknows","oidc-link-existing":true,"oidc-scopes":["read","write"],"oidc-skip-verification":true,"password":"","path":"","port":6969,"protocol":"http","smtp-from":"queen.rip.in.piss@terfisland.org","smtp-host":"example.com","smtp-password":"hunter2","smtp-port":4269,"smtp-username":"sex-haver","software-version":"","statuses-cw-max-chars":420,"statuses-max-chars":69,"statuses-media-max-files":1,"statuses-poll-max-options":1,"statuses-poll-option-max-chars":50,"storage-backend":"local","storage-local-base-path":"/root/store","storage-s3-access-key":"minio","storage-s3-bucket":"gts","storage-s3-endpoint":"localhost:9000","storage-s3-proxy":true,"storage-s3-secret-key":"miniostorage","storage-s3-use-ssl":false,"syslog-address":"127.0.0.1:6969","syslog-enabled":true,"syslog-protocol":"udp","trusted-proxies":["127.0.0.1/32","docker.host.local"],"username":"","web-asset-base-dir":"/root","web-template-base-dir":"/root"}' # Set all the environment variables to # ensure that these are parsed without panic @@ -22,6 +22,10 @@ GTS_DB_PORT=6969 \ GTS_DB_USER='sex-haver' \ GTS_DB_PASSWORD='hunter2' \ GTS_DB_DATABASE='gotosocial_prod' \ +GTS_DB_SQLITE_JOURNAL_MODE='DELETE' \ +GTS_DB_SQLITE_SYNCHRONOUS='FULL' \ +GTS_DB_SQLITE_CACHE_SIZE=0 \ +GTS_DB_SQLITE_BUSY_TIMEOUT='1s' \ GTS_TLS_MODE='' \ GTS_DB_TLS_CA_CERT='' \ GTS_WEB_TEMPLATE_BASE_DIR='/root' \ diff --git a/testrig/config.go b/testrig/config.go index fff9f8fd3..315e52bb9 100644 --- a/testrig/config.go +++ b/testrig/config.go @@ -19,6 +19,9 @@ package testrig import ( + "time" + + "codeberg.org/gruf/go-bytesize" "github.com/coreos/go-oidc/v3/oidc" "github.com/superseriousbusiness/gotosocial/internal/config" ) @@ -43,12 +46,16 @@ var testDefaults = config.Configuration{ Port: 8080, TrustedProxies: []string{"127.0.0.1/32", "::1"}, - DbType: "sqlite", - DbAddress: ":memory:", - DbPort: 5432, - DbUser: "postgres", - DbPassword: "postgres", - DbDatabase: "postgres", + DbType: "sqlite", + DbAddress: ":memory:", + DbPort: 5432, + DbUser: "postgres", + DbPassword: "postgres", + DbDatabase: "postgres", + DbSqliteJournalMode: "WAL", + DbSqliteSynchronous: "NORMAL", + DbSqliteCacheSize: 64 * bytesize.MiB, + DbSqliteBusyTimeout: time.Second * 30, WebTemplateBaseDir: "./web/template/", WebAssetBaseDir: "./web/assets/", diff --git a/testrig/db.go b/testrig/db.go index 4304050cf..65ec391a7 100644 --- a/testrig/db.go +++ b/testrig/db.go @@ -73,15 +73,11 @@ var testModels = []interface{}{ // value as the port instead. func NewTestDB() db.DB { if alternateAddress := os.Getenv("GTS_DB_ADDRESS"); alternateAddress != "" { - config.Config(func(cfg *config.Configuration) { - cfg.DbAddress = alternateAddress - }) + config.SetDbAddress(alternateAddress) } if alternateDBType := os.Getenv("GTS_DB_TYPE"); alternateDBType != "" { - config.Config(func(cfg *config.Configuration) { - cfg.DbType = alternateDBType - }) + config.SetDbType(alternateDBType) } if alternateDBPort := os.Getenv("GTS_DB_PORT"); alternateDBPort != "" { @@ -89,9 +85,7 @@ func NewTestDB() db.DB { if err != nil { panic(err) } - config.Config(func(cfg *config.Configuration) { - cfg.DbPort = int(port) - }) + config.SetDbPort(int(port)) } var state state.State