mirror of
https://github.com/superseriousbusiness/gotosocial
synced 2024-11-10 06:54:16 +00:00
[performance] cache media attachments (#1525)
* replace concurrency worker pools with base models in State.Workers, update code and tests accordingly * add media attachment caching, slightly tweak default cache config * further tweak default cache config values * replace other media attachment db calls to go through cache * update envparsing test * fix delete media attachment sql * fix media sql query * invalidate cached media entries during status create / update * fix envparsing test * fix typo in panic log message... * add 'updated_at' column during UpdateAttachment * remove unused func --------- Signed-off-by: kim <grufwub@gmail.com>
This commit is contained in:
kim
GitHub
parent
5be59f4a25
commit
a8e6bdfa33
15 changed files with 235 additions and 61 deletions
|
|
@ -239,13 +239,13 @@ cache:
|
|||
# ttl = cached object lifetime
|
||||
# sweep-freq = frequency to look for stale cache objects
|
||||
|
||||
account-max-size: 100
|
||||
account-max-size: 500
|
||||
account-ttl: "5m"
|
||||
account-sweep-freq: "10s"
|
||||
account-sweep-freq: "30s"
|
||||
|
||||
block-max-size: 100
|
||||
block-ttl: "5m"
|
||||
block-sweep-freq: "10s"
|
||||
block-sweep-freq: "30s"
|
||||
|
||||
domain-block-max-size: 1000
|
||||
domain-block-ttl: "24h"
|
||||
|
|
@ -253,35 +253,39 @@ cache:
|
|||
|
||||
emoji-max-size: 500
|
||||
emoji-ttl: "5m"
|
||||
emoji-sweep-freq: "10s"
|
||||
emoji-sweep-freq: "30s"
|
||||
|
||||
emoji-category-max-size: 100
|
||||
emoji-category-ttl: "5m"
|
||||
emoji-category-sweep-freq: "10s"
|
||||
emoji-category-sweep-freq: "30s"
|
||||
|
||||
media-max-size: 500
|
||||
media-ttl: "5m"
|
||||
media-sweep-freq: "30s"
|
||||
|
||||
mention-max-size: 500
|
||||
mention-ttl: "5m"
|
||||
mention-sweep-freq: "10s"
|
||||
mention-sweep-freq: "30s"
|
||||
|
||||
notification-max-size: 500
|
||||
notification-ttl: "5m"
|
||||
notification-sweep-freq: "10s"
|
||||
notification-sweep-freq: "30s"
|
||||
|
||||
report-max-size: 100
|
||||
report-ttl: "5m"
|
||||
report-sweep-freq: "10s"
|
||||
report-sweep-freq: "30s"
|
||||
|
||||
status-max-size: 500
|
||||
status-ttl: "5m"
|
||||
status-sweep-freq: "10s"
|
||||
status-sweep-freq: "30s"
|
||||
|
||||
tombstone-max-size: 100
|
||||
tombstone-ttl: "5m"
|
||||
tombstone-sweep-freq: "10s"
|
||||
tombstone-sweep-freq: "30s"
|
||||
|
||||
user-max-size: 100
|
||||
user-ttl: "5m"
|
||||
user-sweep-freq: "10s"
|
||||
user-sweep-freq: "30s"
|
||||
|
||||
######################
|
||||
##### WEB CONFIG #####
|
||||
|
|
|
|||
24
internal/cache/gts.go
vendored
24
internal/cache/gts.go
vendored
|
|
@ -54,6 +54,9 @@ type GTSCaches interface {
|
|||
// Mention provides access to the gtsmodel Mention database cache.
|
||||
Mention() *result.Cache[*gtsmodel.Mention]
|
||||
|
||||
// Media provides access to the gtsmodel Media database cache.
|
||||
Media() *result.Cache[*gtsmodel.MediaAttachment]
|
||||
|
||||
// Notification provides access to the gtsmodel Notification database cache.
|
||||
Notification() *result.Cache[*gtsmodel.Notification]
|
||||
|
||||
|
|
@ -81,6 +84,7 @@ type gtsCaches struct {
|
|||
domainBlock *domain.BlockCache
|
||||
emoji *result.Cache[*gtsmodel.Emoji]
|
||||
emojiCategory *result.Cache[*gtsmodel.EmojiCategory]
|
||||
media *result.Cache[*gtsmodel.MediaAttachment]
|
||||
mention *result.Cache[*gtsmodel.Mention]
|
||||
notification *result.Cache[*gtsmodel.Notification]
|
||||
report *result.Cache[*gtsmodel.Report]
|
||||
|
|
@ -95,6 +99,7 @@ func (c *gtsCaches) Init() {
|
|||
c.initDomainBlock()
|
||||
c.initEmoji()
|
||||
c.initEmojiCategory()
|
||||
c.initMedia()
|
||||
c.initMention()
|
||||
c.initNotification()
|
||||
c.initReport()
|
||||
|
|
@ -119,6 +124,9 @@ func (c *gtsCaches) Start() {
|
|||
tryUntil("starting gtsmodel.EmojiCategory cache", 5, func() bool {
|
||||
return c.emojiCategory.Start(config.GetCacheGTSEmojiCategorySweepFreq())
|
||||
})
|
||||
tryUntil("starting gtsmodel.MediaAttachment cache", 5, func() bool {
|
||||
return c.media.Start(config.GetCacheGTSMediaSweepFreq())
|
||||
})
|
||||
tryUntil("starting gtsmodel.Mention cache", 5, func() bool {
|
||||
return c.mention.Start(config.GetCacheGTSMentionSweepFreq())
|
||||
})
|
||||
|
|
@ -145,6 +153,7 @@ func (c *gtsCaches) Stop() {
|
|||
tryUntil("stopping gtsmodel.DomainBlock cache", 5, c.domainBlock.Stop)
|
||||
tryUntil("stopping gtsmodel.Emoji cache", 5, c.emoji.Stop)
|
||||
tryUntil("stopping gtsmodel.EmojiCategory cache", 5, c.emojiCategory.Stop)
|
||||
tryUntil("stopping gtsmodel.MediaAttachment cache", 5, c.media.Stop)
|
||||
tryUntil("stopping gtsmodel.Mention cache", 5, c.mention.Stop)
|
||||
tryUntil("stopping gtsmodel.Notification cache", 5, c.notification.Stop)
|
||||
tryUntil("stopping gtsmodel.Report cache", 5, c.report.Stop)
|
||||
|
|
@ -173,6 +182,10 @@ func (c *gtsCaches) EmojiCategory() *result.Cache[*gtsmodel.EmojiCategory] {
|
|||
return c.emojiCategory
|
||||
}
|
||||
|
||||
func (c *gtsCaches) Media() *result.Cache[*gtsmodel.MediaAttachment] {
|
||||
return c.media
|
||||
}
|
||||
|
||||
func (c *gtsCaches) Mention() *result.Cache[*gtsmodel.Mention] {
|
||||
return c.mention
|
||||
}
|
||||
|
|
@ -258,6 +271,17 @@ func (c *gtsCaches) initEmojiCategory() {
|
|||
c.emojiCategory.SetTTL(config.GetCacheGTSEmojiCategoryTTL(), true)
|
||||
}
|
||||
|
||||
func (c *gtsCaches) initMedia() {
|
||||
c.media = result.New([]result.Lookup{
|
||||
{Name: "ID"},
|
||||
}, func(m1 *gtsmodel.MediaAttachment) *gtsmodel.MediaAttachment {
|
||||
m2 := new(gtsmodel.MediaAttachment)
|
||||
*m2 = *m1
|
||||
return m2
|
||||
}, config.GetCacheGTSMediaMaxSize())
|
||||
c.media.SetTTL(config.GetCacheGTSMediaTTL(), true)
|
||||
}
|
||||
|
||||
func (c *gtsCaches) initMention() {
|
||||
c.mention = result.New([]result.Lookup{
|
||||
{Name: "ID"},
|
||||
|
|
|
|||
|
|
@ -177,6 +177,10 @@ type GTSCacheConfiguration struct {
|
|||
EmojiCategoryTTL time.Duration `name:"emoji-category-ttl"`
|
||||
EmojiCategorySweepFreq time.Duration `name:"emoji-category-sweep-freq"`
|
||||
|
||||
MediaMaxSize int `name:"media-max-size"`
|
||||
MediaTTL time.Duration `name:"media-ttl"`
|
||||
MediaSweepFreq time.Duration `name:"media-sweep-freq"`
|
||||
|
||||
MentionMaxSize int `name:"mention-max-size"`
|
||||
MentionTTL time.Duration `name:"mention-ttl"`
|
||||
MentionSweepFreq time.Duration `name:"mention-sweep-freq"`
|
||||
|
|
|
|||
|
|
@ -116,13 +116,13 @@ var Defaults = Configuration{
|
|||
|
||||
Cache: CacheConfiguration{
|
||||
GTS: GTSCacheConfiguration{
|
||||
AccountMaxSize: 100,
|
||||
AccountMaxSize: 500,
|
||||
AccountTTL: time.Minute * 5,
|
||||
AccountSweepFreq: time.Second * 10,
|
||||
AccountSweepFreq: time.Second * 30,
|
||||
|
||||
BlockMaxSize: 100,
|
||||
BlockTTL: time.Minute * 5,
|
||||
BlockSweepFreq: time.Second * 10,
|
||||
BlockSweepFreq: time.Second * 30,
|
||||
|
||||
DomainBlockMaxSize: 1000,
|
||||
DomainBlockTTL: time.Hour * 24,
|
||||
|
|
@ -130,35 +130,39 @@ var Defaults = Configuration{
|
|||
|
||||
EmojiMaxSize: 500,
|
||||
EmojiTTL: time.Minute * 5,
|
||||
EmojiSweepFreq: time.Second * 10,
|
||||
EmojiSweepFreq: time.Second * 30,
|
||||
|
||||
EmojiCategoryMaxSize: 100,
|
||||
EmojiCategoryTTL: time.Minute * 5,
|
||||
EmojiCategorySweepFreq: time.Second * 10,
|
||||
EmojiCategorySweepFreq: time.Second * 30,
|
||||
|
||||
MediaMaxSize: 500,
|
||||
MediaTTL: time.Minute * 5,
|
||||
MediaSweepFreq: time.Second * 30,
|
||||
|
||||
MentionMaxSize: 500,
|
||||
MentionTTL: time.Minute * 5,
|
||||
MentionSweepFreq: time.Second * 10,
|
||||
MentionSweepFreq: time.Second * 30,
|
||||
|
||||
NotificationMaxSize: 500,
|
||||
NotificationTTL: time.Minute * 5,
|
||||
NotificationSweepFreq: time.Second * 10,
|
||||
NotificationSweepFreq: time.Second * 30,
|
||||
|
||||
ReportMaxSize: 100,
|
||||
ReportTTL: time.Minute * 5,
|
||||
ReportSweepFreq: time.Second * 10,
|
||||
ReportSweepFreq: time.Second * 30,
|
||||
|
||||
StatusMaxSize: 500,
|
||||
StatusTTL: time.Minute * 5,
|
||||
StatusSweepFreq: time.Second * 10,
|
||||
StatusSweepFreq: time.Second * 30,
|
||||
|
||||
TombstoneMaxSize: 100,
|
||||
TombstoneTTL: time.Minute * 5,
|
||||
TombstoneSweepFreq: time.Second * 10,
|
||||
TombstoneSweepFreq: time.Second * 30,
|
||||
|
||||
UserMaxSize: 100,
|
||||
UserTTL: time.Minute * 5,
|
||||
UserSweepFreq: time.Second * 10,
|
||||
UserSweepFreq: time.Second * 30,
|
||||
},
|
||||
},
|
||||
|
||||
|
|
|
|||
|
|
@ -2426,6 +2426,81 @@ func GetCacheGTSEmojiCategorySweepFreq() time.Duration {
|
|||
// SetCacheGTSEmojiCategorySweepFreq safely sets the value for global configuration 'Cache.GTS.EmojiCategorySweepFreq' field
|
||||
func SetCacheGTSEmojiCategorySweepFreq(v time.Duration) { global.SetCacheGTSEmojiCategorySweepFreq(v) }
|
||||
|
||||
// GetCacheGTSMediaMaxSize safely fetches the Configuration value for state's 'Cache.GTS.MediaMaxSize' field
|
||||
func (st *ConfigState) GetCacheGTSMediaMaxSize() (v int) {
|
||||
st.mutex.Lock()
|
||||
v = st.config.Cache.GTS.MediaMaxSize
|
||||
st.mutex.Unlock()
|
||||
return
|
||||
}
|
||||
|
||||
// SetCacheGTSMediaMaxSize safely sets the Configuration value for state's 'Cache.GTS.MediaMaxSize' field
|
||||
func (st *ConfigState) SetCacheGTSMediaMaxSize(v int) {
|
||||
st.mutex.Lock()
|
||||
defer st.mutex.Unlock()
|
||||
st.config.Cache.GTS.MediaMaxSize = v
|
||||
st.reloadToViper()
|
||||
}
|
||||
|
||||
// CacheGTSMediaMaxSizeFlag returns the flag name for the 'Cache.GTS.MediaMaxSize' field
|
||||
func CacheGTSMediaMaxSizeFlag() string { return "cache-gts-media-max-size" }
|
||||
|
||||
// GetCacheGTSMediaMaxSize safely fetches the value for global configuration 'Cache.GTS.MediaMaxSize' field
|
||||
func GetCacheGTSMediaMaxSize() int { return global.GetCacheGTSMediaMaxSize() }
|
||||
|
||||
// SetCacheGTSMediaMaxSize safely sets the value for global configuration 'Cache.GTS.MediaMaxSize' field
|
||||
func SetCacheGTSMediaMaxSize(v int) { global.SetCacheGTSMediaMaxSize(v) }
|
||||
|
||||
// GetCacheGTSMediaTTL safely fetches the Configuration value for state's 'Cache.GTS.MediaTTL' field
|
||||
func (st *ConfigState) GetCacheGTSMediaTTL() (v time.Duration) {
|
||||
st.mutex.Lock()
|
||||
v = st.config.Cache.GTS.MediaTTL
|
||||
st.mutex.Unlock()
|
||||
return
|
||||
}
|
||||
|
||||
// SetCacheGTSMediaTTL safely sets the Configuration value for state's 'Cache.GTS.MediaTTL' field
|
||||
func (st *ConfigState) SetCacheGTSMediaTTL(v time.Duration) {
|
||||
st.mutex.Lock()
|
||||
defer st.mutex.Unlock()
|
||||
st.config.Cache.GTS.MediaTTL = v
|
||||
st.reloadToViper()
|
||||
}
|
||||
|
||||
// CacheGTSMediaTTLFlag returns the flag name for the 'Cache.GTS.MediaTTL' field
|
||||
func CacheGTSMediaTTLFlag() string { return "cache-gts-media-ttl" }
|
||||
|
||||
// GetCacheGTSMediaTTL safely fetches the value for global configuration 'Cache.GTS.MediaTTL' field
|
||||
func GetCacheGTSMediaTTL() time.Duration { return global.GetCacheGTSMediaTTL() }
|
||||
|
||||
// SetCacheGTSMediaTTL safely sets the value for global configuration 'Cache.GTS.MediaTTL' field
|
||||
func SetCacheGTSMediaTTL(v time.Duration) { global.SetCacheGTSMediaTTL(v) }
|
||||
|
||||
// GetCacheGTSMediaSweepFreq safely fetches the Configuration value for state's 'Cache.GTS.MediaSweepFreq' field
|
||||
func (st *ConfigState) GetCacheGTSMediaSweepFreq() (v time.Duration) {
|
||||
st.mutex.Lock()
|
||||
v = st.config.Cache.GTS.MediaSweepFreq
|
||||
st.mutex.Unlock()
|
||||
return
|
||||
}
|
||||
|
||||
// SetCacheGTSMediaSweepFreq safely sets the Configuration value for state's 'Cache.GTS.MediaSweepFreq' field
|
||||
func (st *ConfigState) SetCacheGTSMediaSweepFreq(v time.Duration) {
|
||||
st.mutex.Lock()
|
||||
defer st.mutex.Unlock()
|
||||
st.config.Cache.GTS.MediaSweepFreq = v
|
||||
st.reloadToViper()
|
||||
}
|
||||
|
||||
// CacheGTSMediaSweepFreqFlag returns the flag name for the 'Cache.GTS.MediaSweepFreq' field
|
||||
func CacheGTSMediaSweepFreqFlag() string { return "cache-gts-media-sweep-freq" }
|
||||
|
||||
// GetCacheGTSMediaSweepFreq safely fetches the value for global configuration 'Cache.GTS.MediaSweepFreq' field
|
||||
func GetCacheGTSMediaSweepFreq() time.Duration { return global.GetCacheGTSMediaSweepFreq() }
|
||||
|
||||
// SetCacheGTSMediaSweepFreq safely sets the value for global configuration 'Cache.GTS.MediaSweepFreq' field
|
||||
func SetCacheGTSMediaSweepFreq(v time.Duration) { global.SetCacheGTSMediaSweepFreq(v) }
|
||||
|
||||
// GetCacheGTSMentionMaxSize safely fetches the Configuration value for state's 'Cache.GTS.MentionMaxSize' field
|
||||
func (st *ConfigState) GetCacheGTSMentionMaxSize() (v int) {
|
||||
st.mutex.Lock()
|
||||
|
|
|
|||
|
|
@ -34,39 +34,69 @@ type mediaDB struct {
|
|||
state *state.State
|
||||
}
|
||||
|
||||
func (m *mediaDB) newMediaQ(i *gtsmodel.MediaAttachment) *bun.SelectQuery {
|
||||
return m.conn.
|
||||
NewSelect().
|
||||
Model(i)
|
||||
}
|
||||
|
||||
func (m *mediaDB) GetAttachmentByID(ctx context.Context, id string) (*gtsmodel.MediaAttachment, db.Error) {
|
||||
return m.getAttachment(
|
||||
ctx,
|
||||
"ID",
|
||||
func(attachment *gtsmodel.MediaAttachment) error {
|
||||
return m.newMediaQ(attachment).Where("? = ?", bun.Ident("media_attachment.id"), id).Scan(ctx)
|
||||
return m.conn.NewSelect().
|
||||
Model(attachment).
|
||||
Where("? = ?", bun.Ident("media_attachment.id"), id).
|
||||
Scan(ctx)
|
||||
},
|
||||
id,
|
||||
)
|
||||
}
|
||||
|
||||
func (m *mediaDB) getAttachments(ctx context.Context, ids []string) ([]*gtsmodel.MediaAttachment, db.Error) {
|
||||
attachments := make([]*gtsmodel.MediaAttachment, 0, len(ids))
|
||||
func (m *mediaDB) getAttachment(ctx context.Context, lookup string, dbQuery func(*gtsmodel.MediaAttachment) error, keyParts ...any) (*gtsmodel.MediaAttachment, db.Error) {
|
||||
return m.state.Caches.GTS.Media().Load(lookup, func() (*gtsmodel.MediaAttachment, error) {
|
||||
var attachment gtsmodel.MediaAttachment
|
||||
|
||||
for _, id := range ids {
|
||||
// Attempt fetch from DB
|
||||
attachment, err := m.GetAttachmentByID(ctx, id)
|
||||
if err != nil {
|
||||
log.Errorf(ctx, "error getting attachment %q: %v", id, err)
|
||||
continue
|
||||
// Not cached! Perform database query
|
||||
if err := dbQuery(&attachment); err != nil {
|
||||
return nil, m.conn.ProcessError(err)
|
||||
}
|
||||
|
||||
// Append attachment
|
||||
attachments = append(attachments, attachment)
|
||||
return &attachment, nil
|
||||
}, keyParts...)
|
||||
}
|
||||
|
||||
func (m *mediaDB) PutAttachment(ctx context.Context, media *gtsmodel.MediaAttachment) error {
|
||||
return m.state.Caches.GTS.Media().Store(media, func() error {
|
||||
_, err := m.conn.NewInsert().Model(media).Exec(ctx)
|
||||
return m.conn.ProcessError(err)
|
||||
})
|
||||
}
|
||||
|
||||
func (m *mediaDB) UpdateAttachment(ctx context.Context, media *gtsmodel.MediaAttachment, columns ...string) error {
|
||||
media.UpdatedAt = time.Now()
|
||||
if len(columns) > 0 {
|
||||
// If we're updating by column, ensure "updated_at" is included.
|
||||
columns = append(columns, "updated_at")
|
||||
}
|
||||
|
||||
return attachments, nil
|
||||
return m.state.Caches.GTS.Media().Store(media, func() error {
|
||||
_, err := m.conn.NewUpdate().
|
||||
Model(media).
|
||||
Where("? = ?", bun.Ident("media_attachment.id"), media.ID).
|
||||
Column(columns...).
|
||||
Exec(ctx)
|
||||
return m.conn.ProcessError(err)
|
||||
})
|
||||
}
|
||||
|
||||
func (m *mediaDB) DeleteAttachment(ctx context.Context, id string) error {
|
||||
// Attempt to delete from database.
|
||||
if _, err := m.conn.NewDelete().
|
||||
TableExpr("? AS ?", bun.Ident("media_attachments"), bun.Ident("media_attachment")).
|
||||
Where("? = ?", bun.Ident("media_attachment.id"), id).
|
||||
Exec(ctx); err != nil {
|
||||
return m.conn.ProcessError(err)
|
||||
}
|
||||
|
||||
// Invalidate this media item from the cache.
|
||||
m.state.Caches.GTS.Media().Invalidate("ID", id)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *mediaDB) GetRemoteOlderThan(ctx context.Context, olderThan time.Time, limit int) ([]*gtsmodel.MediaAttachment, db.Error) {
|
||||
|
|
@ -183,14 +213,20 @@ func (m *mediaDB) CountLocalUnattachedOlderThan(ctx context.Context, olderThan t
|
|||
return count, nil
|
||||
}
|
||||
|
||||
func (m *mediaDB) getAttachment(ctx context.Context, lookup string, dbQuery func(*gtsmodel.MediaAttachment) error, keyParts ...any) (*gtsmodel.MediaAttachment, db.Error) {
|
||||
// Fetch attachment from database
|
||||
// todo: cache this lookup
|
||||
attachment := new(gtsmodel.MediaAttachment)
|
||||
func (m *mediaDB) getAttachments(ctx context.Context, ids []string) ([]*gtsmodel.MediaAttachment, db.Error) {
|
||||
attachments := make([]*gtsmodel.MediaAttachment, 0, len(ids))
|
||||
|
||||
if err := dbQuery(attachment); err != nil {
|
||||
return nil, m.conn.ProcessError(err)
|
||||
for _, id := range ids {
|
||||
// Attempt fetch from DB
|
||||
attachment, err := m.GetAttachmentByID(ctx, id)
|
||||
if err != nil {
|
||||
log.Errorf(ctx, "error getting attachment %q: %v", id, err)
|
||||
continue
|
||||
}
|
||||
|
||||
// Append attachment
|
||||
attachments = append(attachments, attachment)
|
||||
}
|
||||
|
||||
return attachment, nil
|
||||
return attachments, nil
|
||||
}
|
||||
|
|
|
|||
|
|
@ -188,7 +188,7 @@ func (s *statusDB) getStatus(ctx context.Context, lookup string, dbQuery func(*g
|
|||
}
|
||||
|
||||
func (s *statusDB) PutStatus(ctx context.Context, status *gtsmodel.Status) db.Error {
|
||||
return s.state.Caches.GTS.Status().Store(status, func() error {
|
||||
err := s.state.Caches.GTS.Status().Store(status, func() error {
|
||||
// It is safe to run this database transaction within cache.Store
|
||||
// as the cache does not attempt a mutex lock until AFTER hook.
|
||||
//
|
||||
|
|
@ -248,6 +248,17 @@ func (s *statusDB) PutStatus(ctx context.Context, status *gtsmodel.Status) db.Er
|
|||
return err
|
||||
})
|
||||
})
|
||||
if err != nil {
|
||||
// already processed
|
||||
return err
|
||||
}
|
||||
|
||||
for _, id := range status.AttachmentIDs {
|
||||
// Clear updated media attachment IDs from cache
|
||||
s.state.Caches.GTS.Media().Invalidate("ID", id)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *statusDB) UpdateStatus(ctx context.Context, status *gtsmodel.Status, columns ...string) db.Error {
|
||||
|
|
@ -317,11 +328,18 @@ func (s *statusDB) UpdateStatus(ctx context.Context, status *gtsmodel.Status, co
|
|||
Exec(ctx)
|
||||
return err
|
||||
}); err != nil {
|
||||
// already processed
|
||||
return err
|
||||
}
|
||||
|
||||
// Drop any old value from cache by this ID
|
||||
for _, id := range status.AttachmentIDs {
|
||||
// Clear updated media attachment IDs from cache
|
||||
s.state.Caches.GTS.Media().Invalidate("ID", id)
|
||||
}
|
||||
|
||||
// Drop any old status value from cache by this ID
|
||||
s.state.Caches.GTS.Status().Invalidate("ID", status.ID)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -27,9 +27,18 @@ import (
|
|||
|
||||
// Media contains functions related to creating/getting/removing media attachments.
|
||||
type Media interface {
|
||||
// GetAttachmentByID gets a single attachment by its ID
|
||||
// GetAttachmentByID gets a single attachment by its ID.
|
||||
GetAttachmentByID(ctx context.Context, id string) (*gtsmodel.MediaAttachment, Error)
|
||||
|
||||
// PutAttachment inserts the given attachment into the database.
|
||||
PutAttachment(ctx context.Context, media *gtsmodel.MediaAttachment) error
|
||||
|
||||
// UpdateAttachment will update the given attachment in the database.
|
||||
UpdateAttachment(ctx context.Context, media *gtsmodel.MediaAttachment, columns ...string) error
|
||||
|
||||
// DeleteAttachment deletes the attachment with given ID from the database.
|
||||
DeleteAttachment(ctx context.Context, id string) error
|
||||
|
||||
// GetRemoteOlderThan gets limit n remote media attachments (including avatars and headers) older than the given
|
||||
// olderThan time. These will be returned in order of attachment.created_at descending (newest to oldest in other words).
|
||||
//
|
||||
|
|
|
|||
|
|
@ -123,13 +123,13 @@ func (p *ProcessingMedia) load(ctx context.Context) (*gtsmodel.MediaAttachment,
|
|||
}
|
||||
|
||||
if p.recache {
|
||||
// Existing attachment we're recaching, so only need to update.
|
||||
err = p.mgr.state.DB.UpdateByID(ctx, p.media, p.media.ID)
|
||||
// Existing attachment we're recaching, so only update.
|
||||
err = p.mgr.state.DB.UpdateAttachment(ctx, p.media)
|
||||
return err
|
||||
}
|
||||
|
||||
// New attachment, first time caching.
|
||||
err = p.mgr.state.DB.Put(ctx, p.media)
|
||||
// First time caching this attachment, insert it.
|
||||
err = p.mgr.state.DB.PutAttachment(ctx, p.media)
|
||||
return err
|
||||
})
|
||||
|
||||
|
|
|
|||
|
|
@ -320,7 +320,7 @@ func (m *manager) deleteAttachment(ctx context.Context, attachment *gtsmodel.Med
|
|||
}
|
||||
|
||||
// Delete attachment completely.
|
||||
return m.state.DB.DeleteByID(ctx, attachment.ID, attachment)
|
||||
return m.state.DB.DeleteAttachment(ctx, attachment.ID)
|
||||
}
|
||||
|
||||
func (m *manager) uncacheAttachment(ctx context.Context, attachment *gtsmodel.MediaAttachment) error {
|
||||
|
|
@ -332,7 +332,7 @@ func (m *manager) uncacheAttachment(ctx context.Context, attachment *gtsmodel.Me
|
|||
attachment.UpdatedAt = time.Now()
|
||||
cached := false
|
||||
attachment.Cached = &cached
|
||||
return m.state.DB.UpdateByID(ctx, attachment, attachment.ID, "updated_at", "cached")
|
||||
return m.state.DB.UpdateAttachment(ctx, attachment, "updated_at", "cached")
|
||||
}
|
||||
|
||||
func (m *manager) removeFiles(ctx context.Context, keys ...string) (int, error) {
|
||||
|
|
|
|||
|
|
@ -233,7 +233,7 @@ func (p *Processor) InstancePatch(ctx context.Context, form *apimodel.InstanceSe
|
|||
} else if form.AvatarDescription != nil && ia.AvatarMediaAttachment != nil {
|
||||
// process just the description for the existing avatar
|
||||
ia.AvatarMediaAttachment.Description = *form.AvatarDescription
|
||||
if err := p.state.DB.UpdateByID(ctx, ia.AvatarMediaAttachment, ia.AvatarMediaAttachmentID, "description"); err != nil {
|
||||
if err := p.state.DB.UpdateAttachment(ctx, ia.AvatarMediaAttachment, "description"); err != nil {
|
||||
return nil, gtserror.NewErrorInternalError(fmt.Errorf("db error updating instance avatar description: %s", err))
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -40,7 +40,7 @@ func (p *Processor) Delete(ctx context.Context, mediaAttachmentID string) gtserr
|
|||
}
|
||||
|
||||
// delete the attachment
|
||||
if err := p.state.DB.DeleteByID(ctx, mediaAttachmentID, attachment); err != nil && !errors.Is(err, db.ErrNoEntries) {
|
||||
if err := p.state.DB.DeleteAttachment(ctx, mediaAttachmentID); err != nil && !errors.Is(err, db.ErrNoEntries) {
|
||||
errs = append(errs, fmt.Sprintf("remove attachment: %s", err))
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -49,7 +49,7 @@ func (p *Processor) Unattach(ctx context.Context, account *gtsmodel.Account, med
|
|||
attachment.UpdatedAt = time.Now()
|
||||
attachment.StatusID = ""
|
||||
|
||||
if err := p.state.DB.UpdateByID(ctx, attachment, attachment.ID, updatingColumns...); err != nil {
|
||||
if err := p.state.DB.UpdateAttachment(ctx, attachment, updatingColumns...); err != nil {
|
||||
return nil, gtserror.NewErrorNotFound(fmt.Errorf("db error updating attachment: %s", err))
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@
|
|||
|
||||
set -eu
|
||||
|
||||
EXPECT='{"account-domain":"peepee","accounts-allow-custom-css":true,"accounts-approval-required":false,"accounts-reason-required":false,"accounts-registration-open":true,"advanced-cookies-samesite":"strict","advanced-rate-limit-requests":6969,"advanced-throttling-multiplier":-1,"advanced-throttling-retry-after":10000000000,"application-name":"gts","bind-address":"127.0.0.1","cache":{"gts":{"account-max-size":99,"account-sweep-freq":1000000000,"account-ttl":10800000000000,"block-max-size":100,"block-sweep-freq":10000000000,"block-ttl":300000000000,"domain-block-max-size":1000,"domain-block-sweep-freq":60000000000,"domain-block-ttl":86400000000000,"emoji-category-max-size":100,"emoji-category-sweep-freq":10000000000,"emoji-category-ttl":300000000000,"emoji-max-size":500,"emoji-sweep-freq":10000000000,"emoji-ttl":300000000000,"mention-max-size":500,"mention-sweep-freq":10000000000,"mention-ttl":300000000000,"notification-max-size":500,"notification-sweep-freq":10000000000,"notification-ttl":300000000000,"report-max-size":100,"report-sweep-freq":10000000000,"report-ttl":300000000000,"status-max-size":500,"status-sweep-freq":10000000000,"status-ttl":300000000000,"tombstone-max-size":100,"tombstone-sweep-freq":10000000000,"tombstone-ttl":300000000000,"user-max-size":100,"user-sweep-freq":10000000000,"user-ttl":300000000000}},"config-path":"internal/config/testdata/test.yaml","db-address":":memory:","db-database":"gotosocial_prod","db-max-open-conns-multiplier":3,"db-password":"hunter2","db-port":6969,"db-sqlite-busy-timeout":1000000000,"db-sqlite-cache-size":0,"db-sqlite-journal-mode":"DELETE","db-sqlite-synchronous":"FULL","db-tls-ca-cert":"","db-tls-mode":"disable","db-type":"sqlite","db-user":"sex-haver","dry-run":true,"email":"","host":"example.com","instance-deliver-to-shared-inboxes":false,"instance-expose-peers":true,"instance-expose-public-timeline":true,"instance-expose-suspended":true,"instance-expose-suspended-web":true,"landing-page-user":"admin","letsencrypt-cert-dir":"/gotosocial/storage/certs","letsencrypt-email-address":"","letsencrypt-enabled":true,"letsencrypt-port":80,"log-db-queries":true,"log-level":"info","media-description-max-chars":5000,"media-description-min-chars":69,"media-emoji-local-max-size":420,"media-emoji-remote-max-size":420,"media-image-max-size":420,"media-remote-cache-days":30,"media-video-max-size":420,"oidc-admin-groups":["steamy"],"oidc-client-id":"1234","oidc-client-secret":"shhhh its a secret","oidc-enabled":true,"oidc-idp-name":"sex-haver","oidc-issuer":"whoknows","oidc-link-existing":true,"oidc-scopes":["read","write"],"oidc-skip-verification":true,"password":"","path":"","port":6969,"protocol":"http","request-id-header":"X-Trace-Id","smtp-from":"queen.rip.in.piss@terfisland.org","smtp-host":"example.com","smtp-password":"hunter2","smtp-port":4269,"smtp-username":"sex-haver","software-version":"","statuses-cw-max-chars":420,"statuses-max-chars":69,"statuses-media-max-files":1,"statuses-poll-max-options":1,"statuses-poll-option-max-chars":50,"storage-backend":"local","storage-local-base-path":"/root/store","storage-s3-access-key":"minio","storage-s3-bucket":"gts","storage-s3-endpoint":"localhost:9000","storage-s3-proxy":true,"storage-s3-secret-key":"miniostorage","storage-s3-use-ssl":false,"syslog-address":"127.0.0.1:6969","syslog-enabled":true,"syslog-protocol":"udp","trusted-proxies":["127.0.0.1/32","docker.host.local"],"username":"","web-asset-base-dir":"/root","web-template-base-dir":"/root"}'
|
||||
EXPECT='{"account-domain":"peepee","accounts-allow-custom-css":true,"accounts-approval-required":false,"accounts-reason-required":false,"accounts-registration-open":true,"advanced-cookies-samesite":"strict","advanced-rate-limit-requests":6969,"advanced-throttling-multiplier":-1,"advanced-throttling-retry-after":10000000000,"application-name":"gts","bind-address":"127.0.0.1","cache":{"gts":{"account-max-size":99,"account-sweep-freq":1000000000,"account-ttl":10800000000000,"block-max-size":100,"block-sweep-freq":30000000000,"block-ttl":300000000000,"domain-block-max-size":1000,"domain-block-sweep-freq":60000000000,"domain-block-ttl":86400000000000,"emoji-category-max-size":100,"emoji-category-sweep-freq":30000000000,"emoji-category-ttl":300000000000,"emoji-max-size":500,"emoji-sweep-freq":30000000000,"emoji-ttl":300000000000,"media-max-size":500,"media-sweep-freq":30000000000,"media-ttl":300000000000,"mention-max-size":500,"mention-sweep-freq":30000000000,"mention-ttl":300000000000,"notification-max-size":500,"notification-sweep-freq":30000000000,"notification-ttl":300000000000,"report-max-size":100,"report-sweep-freq":30000000000,"report-ttl":300000000000,"status-max-size":500,"status-sweep-freq":30000000000,"status-ttl":300000000000,"tombstone-max-size":100,"tombstone-sweep-freq":30000000000,"tombstone-ttl":300000000000,"user-max-size":100,"user-sweep-freq":30000000000,"user-ttl":300000000000}},"config-path":"internal/config/testdata/test.yaml","db-address":":memory:","db-database":"gotosocial_prod","db-max-open-conns-multiplier":3,"db-password":"hunter2","db-port":6969,"db-sqlite-busy-timeout":1000000000,"db-sqlite-cache-size":0,"db-sqlite-journal-mode":"DELETE","db-sqlite-synchronous":"FULL","db-tls-ca-cert":"","db-tls-mode":"disable","db-type":"sqlite","db-user":"sex-haver","dry-run":true,"email":"","host":"example.com","instance-deliver-to-shared-inboxes":false,"instance-expose-peers":true,"instance-expose-public-timeline":true,"instance-expose-suspended":true,"instance-expose-suspended-web":true,"landing-page-user":"admin","letsencrypt-cert-dir":"/gotosocial/storage/certs","letsencrypt-email-address":"","letsencrypt-enabled":true,"letsencrypt-port":80,"log-db-queries":true,"log-level":"info","media-description-max-chars":5000,"media-description-min-chars":69,"media-emoji-local-max-size":420,"media-emoji-remote-max-size":420,"media-image-max-size":420,"media-remote-cache-days":30,"media-video-max-size":420,"oidc-admin-groups":["steamy"],"oidc-client-id":"1234","oidc-client-secret":"shhhh its a secret","oidc-enabled":true,"oidc-idp-name":"sex-haver","oidc-issuer":"whoknows","oidc-link-existing":true,"oidc-scopes":["read","write"],"oidc-skip-verification":true,"password":"","path":"","port":6969,"protocol":"http","request-id-header":"X-Trace-Id","smtp-from":"queen.rip.in.piss@terfisland.org","smtp-host":"example.com","smtp-password":"hunter2","smtp-port":4269,"smtp-username":"sex-haver","software-version":"","statuses-cw-max-chars":420,"statuses-max-chars":69,"statuses-media-max-files":1,"statuses-poll-max-options":1,"statuses-poll-option-max-chars":50,"storage-backend":"local","storage-local-base-path":"/root/store","storage-s3-access-key":"minio","storage-s3-bucket":"gts","storage-s3-endpoint":"localhost:9000","storage-s3-proxy":true,"storage-s3-secret-key":"miniostorage","storage-s3-use-ssl":false,"syslog-address":"127.0.0.1:6969","syslog-enabled":true,"syslog-protocol":"udp","trusted-proxies":["127.0.0.1/32","docker.host.local"],"username":"","web-asset-base-dir":"/root","web-template-base-dir":"/root"}'
|
||||
|
||||
# Set all the environment variables to
|
||||
# ensure that these are parsed without panic
|
||||
|
|
|
|||
|
|
@ -34,7 +34,7 @@ func InitTestConfig() {
|
|||
}
|
||||
|
||||
var testDefaults = config.Configuration{
|
||||
LogLevel: "trace",
|
||||
LogLevel: "info",
|
||||
LogDbQueries: true,
|
||||
ApplicationName: "gotosocial",
|
||||
LandingPageUser: "",
|
||||
|
|
|
|||
Loading…
Reference in a new issue