From 58c87bdd7f37e80a57c4802e95eb6c572ccffed4 Mon Sep 17 00:00:00 2001 From: kim <89579420+NyaaaWhatsUpDoc@users.noreply.github.com> Date: Mon, 12 Dec 2022 11:22:19 +0000 Subject: [PATCH] [feature] allow uncaching of other media types (#1234) * simplify pruneRemote, remove unncecessary media trace logging, update RemoteOlderThan() to include headers/avis Signed-off-by: kim * cleanup pruneallmeta, add remote header to pruneremote tests Signed-off-by: kim * fix olderthan duration additions Signed-off-by: kim * fix broken test now that test model header changed Signed-off-by: kim * instead use new remote test account for new header model Signed-off-by: kim * use newer generated ULID for remote_account_3 to ensure it is sorted last Signed-off-by: kim * reorganize serialized keys to match expected test account model order Signed-off-by: kim Signed-off-by: kim --- internal/db/bundb/media.go | 3 +- internal/db/bundb/media_test.go | 4 +- internal/db/media.go | 14 ++--- internal/media/processingmedia.go | 17 +++--- internal/media/prunemeta.go | 30 ++++++----- internal/media/pruneremote.go | 44 ++++++++------- internal/media/pruneremote_test.go | 77 +++++++++++++++----------- internal/media/pruneunusedlocal.go | 7 +-- internal/media/util.go | 17 ------ testrig/testmodels.go | 87 ++++++++++++++++++++++++++++++ 10 files changed, 192 insertions(+), 108 deletions(-) diff --git a/internal/db/bundb/media.go b/internal/db/bundb/media.go index 39e0ad0e3..3be7be39a 100644 --- a/internal/db/bundb/media.go +++ b/internal/db/bundb/media.go @@ -57,8 +57,6 @@ func (m *mediaDB) GetRemoteOlderThan(ctx context.Context, olderThan time.Time, l NewSelect(). Model(&attachments). Where("? = ?", bun.Ident("media_attachment.cached"), true). - Where("? = ?", bun.Ident("media_attachment.avatar"), false). - Where("? = ?", bun.Ident("media_attachment.header"), false). Where("? < ?", bun.Ident("media_attachment.created_at"), olderThan). WhereGroup(" AND ", whereNotEmptyAndNotNull("media_attachment.remote_url")). Order("media_attachment.created_at DESC") @@ -70,6 +68,7 @@ func (m *mediaDB) GetRemoteOlderThan(ctx context.Context, olderThan time.Time, l if err := q.Scan(ctx); err != nil { return nil, m.conn.ProcessError(err) } + return attachments, nil } diff --git a/internal/db/bundb/media_test.go b/internal/db/bundb/media_test.go index d6a4981f8..1676954ce 100644 --- a/internal/db/bundb/media_test.go +++ b/internal/db/bundb/media_test.go @@ -41,7 +41,7 @@ func (suite *MediaTestSuite) TestGetAttachmentByID() { func (suite *MediaTestSuite) TestGetOlder() { attachments, err := suite.db.GetRemoteOlderThan(context.Background(), time.Now(), 20) suite.NoError(err) - suite.Len(attachments, 2) + suite.Len(attachments, 3) } func (suite *MediaTestSuite) TestGetAvisAndHeaders() { @@ -49,7 +49,7 @@ func (suite *MediaTestSuite) TestGetAvisAndHeaders() { attachments, err := suite.db.GetAvatarsAndHeaders(ctx, "", 20) suite.NoError(err) - suite.Len(attachments, 2) + suite.Len(attachments, 3) } func (suite *MediaTestSuite) TestGetLocalUnattachedOlderThan() { diff --git a/internal/db/media.go b/internal/db/media.go index 2f9ed79dc..2d4fbb441 100644 --- a/internal/db/media.go +++ b/internal/db/media.go @@ -29,18 +29,20 @@ import ( type Media interface { // GetAttachmentByID gets a single attachment by its ID GetAttachmentByID(ctx context.Context, id string) (*gtsmodel.MediaAttachment, Error) - // GetRemoteOlderThan gets limit n remote media attachments older than the given olderThan time. - // These will be returned in order of attachment.created_at descending (newest to oldest in other words). + + // GetRemoteOlderThan gets limit n remote media attachments (including avatars and headers) older than the given + // olderThan time. These will be returned in order of attachment.created_at descending (newest to oldest in other words). // // The selected media attachments will be those with both a URL and a RemoteURL filled in. // In other words, media attachments that originated remotely, and that we currently have cached locally. GetRemoteOlderThan(ctx context.Context, olderThan time.Time, limit int) ([]*gtsmodel.MediaAttachment, Error) + // GetAvatarsAndHeaders fetches limit n avatars and headers with an id < maxID. These headers // and avis may be in use or not; the caller should check this if it's important. GetAvatarsAndHeaders(ctx context.Context, maxID string, limit int) ([]*gtsmodel.MediaAttachment, Error) - // GetLocalUnattachedOlderThan fetches limit n local media attachments, older than the given time, which - // aren't header or avatars, and aren't attached to a status. In other words, attachments which were uploaded - // but never used for whatever reason, or attachments that were attached to a status which was subsequently - // deleted. + + // GetLocalUnattachedOlderThan fetches limit n local media attachments (including avatars and headers), older than + // the given time, which aren't header or avatars, and aren't attached to a status. In other words, attachments which were + // uploaded but never used for whatever reason, or attachments that were attached to a status which was subsequently deleted. GetLocalUnattachedOlderThan(ctx context.Context, olderThan time.Time, maxID string, limit int) ([]*gtsmodel.MediaAttachment, Error) } diff --git a/internal/media/processingmedia.go b/internal/media/processingmedia.go index 1fd9423f5..94c8f9a7a 100644 --- a/internal/media/processingmedia.go +++ b/internal/media/processingmedia.go @@ -81,10 +81,8 @@ func (p *ProcessingMedia) AttachmentID() string { // LoadAttachment blocks until the thumbnail and fullsize content // has been processed, and then returns the completed attachment. func (p *ProcessingMedia) LoadAttachment(ctx context.Context) (*gtsmodel.MediaAttachment, error) { - log.Tracef("LoadAttachment: getting lock for attachment %s", p.attachment.URL) p.mu.Lock() defer p.mu.Unlock() - log.Tracef("LoadAttachment: got lock for attachment %s", p.attachment.URL) if err := p.store(ctx); err != nil { return nil, err @@ -98,23 +96,24 @@ func (p *ProcessingMedia) LoadAttachment(ctx context.Context) (*gtsmodel.MediaAt return nil, err } - // store the result in the database before returning it if !p.insertedInDB { if p.recache { - // if it's a recache we should only need to update + // This is an existing media attachment we're recaching, so only need to update it if err := p.database.UpdateByID(ctx, p.attachment, p.attachment.ID); err != nil { return nil, err } } else { - // otherwise we need to really PUT it + // This is a new media attachment we're caching for first time if err := p.database.Put(ctx, p.attachment); err != nil { return nil, err } } + + // Mark this as stored in DB p.insertedInDB = true } - log.Tracef("LoadAttachment: finished, returning attachment %s", p.attachment.URL) + log.Tracef("finished loading attachment %s", p.attachment.URL) return p.attachment, nil } @@ -180,7 +179,7 @@ func (p *ProcessingMedia) loadThumb(ctx context.Context) error { // we're done processing the thumbnail! atomic.StoreInt32(&p.thumbState, int32(complete)) - log.Tracef("loadThumb: finished processing thumbnail for attachment %s", p.attachment.URL) + log.Tracef("finished processing thumbnail for attachment %s", p.attachment.URL) fallthrough case complete: return nil @@ -241,7 +240,7 @@ func (p *ProcessingMedia) loadFullSize(ctx context.Context) error { // we're done processing the full-size image atomic.StoreInt32(&p.fullSizeState, int32(complete)) - log.Tracef("loadFullSize: finished processing full size image for attachment %s", p.attachment.URL) + log.Tracef("finished processing full size image for attachment %s", p.attachment.URL) fallthrough case complete: return nil @@ -362,7 +361,7 @@ func (p *ProcessingMedia) store(ctx context.Context) error { p.attachment.File.FileSize = int(fileSize) p.read = true - log.Tracef("store: finished storing initial data for attachment %s", p.attachment.URL) + log.Tracef("finished storing initial data for attachment %s", p.attachment.URL) return nil } diff --git a/internal/media/prunemeta.go b/internal/media/prunemeta.go index 69d79b8d9..19fbb7e7e 100644 --- a/internal/media/prunemeta.go +++ b/internal/media/prunemeta.go @@ -20,6 +20,7 @@ package media import ( "context" + "errors" "codeberg.org/gruf/go-store/v2/storage" "github.com/superseriousbusiness/gotosocial/internal/db" @@ -28,17 +29,23 @@ import ( ) func (m *manager) PruneAllMeta(ctx context.Context) (int, error) { - var totalPruned int - var maxID string - var attachments []*gtsmodel.MediaAttachment - var err error + var ( + totalPruned int + maxID string + ) + + for { + // select "selectPruneLimit" headers / avatars at a time for pruning + attachments, err := m.db.GetAvatarsAndHeaders(ctx, maxID, selectPruneLimit) + if err != nil && !errors.Is(err, db.ErrNoEntries) { + return totalPruned, err + } else if len(attachments) == 0 { + break + } - // select 20 attachments at a time and prune them - for attachments, err = m.db.GetAvatarsAndHeaders(ctx, maxID, selectPruneLimit); err == nil && len(attachments) != 0; attachments, err = m.db.GetAvatarsAndHeaders(ctx, maxID, selectPruneLimit) { // use the id of the last attachment in the slice as the next 'maxID' value - l := len(attachments) - log.Tracef("PruneAllMeta: got %d attachments with maxID < %s", l, maxID) - maxID = attachments[l-1].ID + log.Tracef("PruneAllMeta: got %d attachments with maxID < %s", len(attachments), maxID) + maxID = attachments[len(attachments)-1].ID // prune each attachment that meets one of the following criteria: // - has no owning account in the database @@ -56,11 +63,6 @@ func (m *manager) PruneAllMeta(ctx context.Context) (int, error) { } } - // make sure we don't have a real error when we leave the loop - if err != nil && err != db.ErrNoEntries { - return totalPruned, err - } - log.Infof("PruneAllMeta: finished pruning avatars + headers: pruned %d entries", totalPruned) return totalPruned, nil } diff --git a/internal/media/pruneremote.go b/internal/media/pruneremote.go index 011ed1dd7..f6108588f 100644 --- a/internal/media/pruneremote.go +++ b/internal/media/pruneremote.go @@ -20,7 +20,8 @@ package media import ( "context" - "fmt" + "errors" + "time" "codeberg.org/gruf/go-store/v2/storage" "github.com/superseriousbusiness/gotosocial/internal/db" @@ -31,21 +32,23 @@ import ( func (m *manager) PruneAllRemote(ctx context.Context, olderThanDays int) (int, error) { var totalPruned int - olderThan, err := parseOlderThan(olderThanDays) - if err != nil { - return totalPruned, fmt.Errorf("PruneAllRemote: error parsing olderThanDays %d: %s", olderThanDays, err) - } + olderThan := time.Now().Add(-time.Hour * 24 * time.Duration(olderThanDays)) log.Infof("PruneAllRemote: pruning media older than %s", olderThan) - // select 20 attachments at a time and prune them - for attachments, err := m.db.GetRemoteOlderThan(ctx, olderThan, selectPruneLimit); err == nil && len(attachments) != 0; attachments, err = m.db.GetRemoteOlderThan(ctx, olderThan, selectPruneLimit) { + for { + // Select "selectPruneLimit" status attacchments at a time for pruning + attachments, err := m.db.GetRemoteOlderThan(ctx, olderThan, selectPruneLimit) + if err != nil && !errors.Is(err, db.ErrNoEntries) { + return totalPruned, err + } else if len(attachments) == 0 { + break + } - // use the age of the oldest attachment (the last one in the slice) as the next 'older than' value - l := len(attachments) - log.Tracef("PruneAllRemote: got %d attachments older than %s", l, olderThan) - olderThan = attachments[l-1].CreatedAt + // use the age of the oldest attachment (last in slice) as the next 'olderThan' value + log.Tracef("PruneAllRemote: got %d status attachments older than %s", len(attachments), olderThan) + olderThan = attachments[len(attachments)-1].CreatedAt - // prune each attachment + // prune each status attachment for _, attachment := range attachments { if err := m.pruneOneRemote(ctx, attachment); err != nil { return totalPruned, err @@ -54,11 +57,6 @@ func (m *manager) PruneAllRemote(ctx context.Context, olderThanDays int) (int, e } } - // make sure we don't have a real error when we leave the loop - if err != nil && err != db.ErrNoEntries { - return totalPruned, err - } - log.Infof("PruneAllRemote: finished pruning remote media: pruned %d entries", totalPruned) return totalPruned, nil } @@ -69,7 +67,7 @@ func (m *manager) pruneOneRemote(ctx context.Context, attachment *gtsmodel.Media if attachment.File.Path != "" { // delete the full size attachment from storage log.Tracef("pruneOneRemote: deleting %s", attachment.File.Path) - if err := m.storage.Delete(ctx, attachment.File.Path); err != nil && err != storage.ErrNotFound { + if err := m.storage.Delete(ctx, attachment.File.Path); err != nil && !errors.Is(err, storage.ErrNotFound) { return err } cached := false @@ -80,7 +78,7 @@ func (m *manager) pruneOneRemote(ctx context.Context, attachment *gtsmodel.Media if attachment.Thumbnail.Path != "" { // delete the thumbnail from storage log.Tracef("pruneOneRemote: deleting %s", attachment.Thumbnail.Path) - if err := m.storage.Delete(ctx, attachment.Thumbnail.Path); err != nil && err != storage.ErrNotFound { + if err := m.storage.Delete(ctx, attachment.Thumbnail.Path); err != nil && !errors.Is(err, storage.ErrNotFound) { return err } cached := false @@ -88,10 +86,10 @@ func (m *manager) pruneOneRemote(ctx context.Context, attachment *gtsmodel.Media changed = true } - // update the attachment to reflect that we no longer have it cached - if changed { - return m.db.UpdateByID(ctx, attachment, attachment.ID, "updated_at", "cached") + if !changed { + return nil } - return nil + // update the attachment to reflect that we no longer have it cached + return m.db.UpdateByID(ctx, attachment, attachment.ID, "updated_at", "cached") } diff --git a/internal/media/pruneremote_test.go b/internal/media/pruneremote_test.go index 4af01c1c5..d4365f72c 100644 --- a/internal/media/pruneremote_test.go +++ b/internal/media/pruneremote_test.go @@ -27,6 +27,7 @@ import ( "codeberg.org/gruf/go-store/v2/storage" "github.com/stretchr/testify/suite" + "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" ) type PruneRemoteTestSuite struct { @@ -34,24 +35,29 @@ type PruneRemoteTestSuite struct { } func (suite *PruneRemoteTestSuite) TestPruneRemote() { - testAttachment := suite.testAttachments["remote_account_1_status_1_attachment_1"] - suite.True(*testAttachment.Cached) + testStatusAttachment := suite.testAttachments["remote_account_1_status_1_attachment_1"] + suite.True(*testStatusAttachment.Cached) + + testHeader := suite.testAttachments["remote_account_3_header"] + suite.True(*testHeader.Cached) totalPruned, err := suite.manager.PruneAllRemote(context.Background(), 1) suite.NoError(err) - suite.Equal(2, totalPruned) + suite.Equal(3, totalPruned) - prunedAttachment, err := suite.db.GetAttachmentByID(context.Background(), testAttachment.ID) + prunedAttachment, err := suite.db.GetAttachmentByID(context.Background(), testStatusAttachment.ID) suite.NoError(err) + suite.False(*prunedAttachment.Cached) - // the media should no longer be cached + prunedAttachment, err = suite.db.GetAttachmentByID(context.Background(), testHeader.ID) + suite.NoError(err) suite.False(*prunedAttachment.Cached) } func (suite *PruneRemoteTestSuite) TestPruneRemoteTwice() { totalPruned, err := suite.manager.PruneAllRemote(context.Background(), 1) suite.NoError(err) - suite.Equal(2, totalPruned) + suite.Equal(3, totalPruned) // final prune should prune nothing, since the first prune already happened totalPrunedAgain, err := suite.manager.PruneAllRemote(context.Background(), 1) @@ -61,16 +67,21 @@ func (suite *PruneRemoteTestSuite) TestPruneRemoteTwice() { func (suite *PruneRemoteTestSuite) TestPruneAndRecache() { ctx := context.Background() - testAttachment := suite.testAttachments["remote_account_1_status_1_attachment_1"] + testStatusAttachment := suite.testAttachments["remote_account_1_status_1_attachment_1"] + testHeader := suite.testAttachments["remote_account_3_header"] totalPruned, err := suite.manager.PruneAllRemote(ctx, 1) suite.NoError(err) - suite.Equal(2, totalPruned) + suite.Equal(3, totalPruned) // media should no longer be stored - _, err = suite.storage.Get(ctx, testAttachment.File.Path) + _, err = suite.storage.Get(ctx, testStatusAttachment.File.Path) suite.ErrorIs(err, storage.ErrNotFound) - _, err = suite.storage.Get(ctx, testAttachment.Thumbnail.Path) + _, err = suite.storage.Get(ctx, testStatusAttachment.Thumbnail.Path) + suite.ErrorIs(err, storage.ErrNotFound) + _, err = suite.storage.Get(ctx, testHeader.File.Path) + suite.ErrorIs(err, storage.ErrNotFound) + _, err = suite.storage.Get(ctx, testHeader.Thumbnail.Path) suite.ErrorIs(err, storage.ErrNotFound) // now recache the image.... @@ -82,34 +93,40 @@ func (suite *PruneRemoteTestSuite) TestPruneAndRecache() { } return io.NopCloser(bytes.NewBuffer(b)), int64(len(b)), nil } - processingRecache, err := suite.manager.RecacheMedia(ctx, data, nil, testAttachment.ID) - suite.NoError(err) - // synchronously load the recached attachment - recachedAttachment, err := processingRecache.LoadAttachment(ctx) - suite.NoError(err) - suite.NotNil(recachedAttachment) + for _, original := range []*gtsmodel.MediaAttachment{ + testStatusAttachment, + testHeader, + } { + processingRecache, err := suite.manager.RecacheMedia(ctx, data, nil, original.ID) + suite.NoError(err) - // recachedAttachment should be basically the same as the old attachment - suite.True(*recachedAttachment.Cached) - suite.Equal(testAttachment.ID, recachedAttachment.ID) - suite.Equal(testAttachment.File.Path, recachedAttachment.File.Path) // file should be stored in the same place - suite.Equal(testAttachment.Thumbnail.Path, recachedAttachment.Thumbnail.Path) // as should the thumbnail - suite.EqualValues(testAttachment.FileMeta, recachedAttachment.FileMeta) // and the filemeta should be the same + // synchronously load the recached attachment + recachedAttachment, err := processingRecache.LoadAttachment(ctx) + suite.NoError(err) + suite.NotNil(recachedAttachment) - // recached files should be back in storage - _, err = suite.storage.Get(ctx, recachedAttachment.File.Path) - suite.NoError(err) - _, err = suite.storage.Get(ctx, recachedAttachment.Thumbnail.Path) - suite.NoError(err) + // recachedAttachment should be basically the same as the old attachment + suite.True(*recachedAttachment.Cached) + suite.Equal(original.ID, recachedAttachment.ID) + suite.Equal(original.File.Path, recachedAttachment.File.Path) // file should be stored in the same place + suite.Equal(original.Thumbnail.Path, recachedAttachment.Thumbnail.Path) // as should the thumbnail + suite.EqualValues(original.FileMeta, recachedAttachment.FileMeta) // and the filemeta should be the same + + // recached files should be back in storage + _, err = suite.storage.Get(ctx, recachedAttachment.File.Path) + suite.NoError(err) + _, err = suite.storage.Get(ctx, recachedAttachment.Thumbnail.Path) + suite.NoError(err) + } } func (suite *PruneRemoteTestSuite) TestPruneOneNonExistent() { ctx := context.Background() - testAttachment := suite.testAttachments["remote_account_1_status_1_attachment_1"] + testStatusAttachment := suite.testAttachments["remote_account_1_status_1_attachment_1"] // Delete this attachment cached on disk - media, err := suite.db.GetAttachmentByID(ctx, testAttachment.ID) + media, err := suite.db.GetAttachmentByID(ctx, testStatusAttachment.ID) suite.NoError(err) suite.True(*media.Cached) err = suite.storage.Delete(ctx, media.File.Path) @@ -118,7 +135,7 @@ func (suite *PruneRemoteTestSuite) TestPruneOneNonExistent() { // Now attempt to prune remote for item with db entry no file totalPruned, err := suite.manager.PruneAllRemote(ctx, 1) suite.NoError(err) - suite.Equal(2, totalPruned) + suite.Equal(3, totalPruned) } func TestPruneRemoteTestSuite(t *testing.T) { diff --git a/internal/media/pruneunusedlocal.go b/internal/media/pruneunusedlocal.go index ba74b7c90..7098648e0 100644 --- a/internal/media/pruneunusedlocal.go +++ b/internal/media/pruneunusedlocal.go @@ -20,7 +20,7 @@ package media import ( "context" - "fmt" + "time" "codeberg.org/gruf/go-store/v2/storage" "github.com/superseriousbusiness/gotosocial/internal/db" @@ -34,10 +34,7 @@ func (m *manager) PruneUnusedLocalAttachments(ctx context.Context) (int, error) var attachments []*gtsmodel.MediaAttachment var err error - olderThan, err := parseOlderThan(UnusedLocalAttachmentCacheDays) - if err != nil { - return totalPruned, fmt.Errorf("PruneUnusedLocalAttachments: error parsing olderThanDays %d: %s", UnusedLocalAttachmentCacheDays, err) - } + olderThan := time.Now().Add(-time.Hour * 24 * time.Duration(UnusedLocalAttachmentCacheDays)) log.Infof("PruneUnusedLocalAttachments: pruning unused local attachments older than %s", olderThan) // select 20 attachments at a time and prune them diff --git a/internal/media/util.go b/internal/media/util.go index 7efd788a1..60661cbc0 100644 --- a/internal/media/util.go +++ b/internal/media/util.go @@ -23,7 +23,6 @@ import ( "errors" "fmt" "io" - "time" "github.com/h2non/filetype" "github.com/superseriousbusiness/gotosocial/internal/log" @@ -134,22 +133,6 @@ func (l *logrusWrapper) Error(err error, msg string, keysAndValues ...interface{ log.Error("media manager cron logger: ", err, msg, keysAndValues) } -func parseOlderThan(olderThanDays int) (time.Time, error) { - // convert days into a duration string - olderThanHoursString := fmt.Sprintf("%dh", olderThanDays*24) - - // parse the duration string into a duration - olderThanHours, err := time.ParseDuration(olderThanHoursString) - if err != nil { - return time.Time{}, err - } - - // 'subtract' that from the time now to give our threshold - olderThan := time.Now().Add(-olderThanHours) - - return olderThan, nil -} - // lengthReader wraps a reader and reads the length of total bytes written as it goes. type lengthReader struct { source io.Reader diff --git a/testrig/testmodels.go b/testrig/testmodels.go index cbaa855e8..b3850dcad 100644 --- a/testrig/testmodels.go +++ b/testrig/testmodels.go @@ -573,6 +573,43 @@ func NewTestAccounts() map[string]*gtsmodel.Account { HideCollections: FalseBool(), SuspensionOrigin: "", }, + "remote_account_3": { + ID: "062G5WYKY35KKD12EMSM3F8PJ8", + Username: "her_fuckin_maj", + Domain: "thequeenisstillalive.technology", + DisplayName: "lizzzieeeeeeeeeeee", + Fields: []gtsmodel.Field{}, + Note: "if i die blame charles don't let that fuck become king", + Memorial: FalseBool(), + MovedToAccountID: "", + CreatedAt: TimeMustParse("2020-08-10T14:13:28+02:00"), + UpdatedAt: TimeMustParse("2022-06-04T13:12:00Z"), + Bot: FalseBool(), + Locked: TrueBool(), + Discoverable: TrueBool(), + Sensitive: FalseBool(), + Language: "en", + URI: "http://thequeenisstillalive.technology/users/her_fuckin_maj", + URL: "http://thequeenisstillalive.technology/@her_fuckin_maj", + LastWebfingeredAt: time.Time{}, + InboxURI: "http://thequeenisstillalive.technology/users/her_fuckin_maj/inbox", + SharedInboxURI: StringPtr(""), + OutboxURI: "http://thequeenisstillalive.technology/users/her_fuckin_maj/outbox", + FollowersURI: "http://thequeenisstillalive.technology/users/her_fuckin_maj/followers", + FollowingURI: "http://thequeenisstillalive.technology/users/her_fuckin_maj/following", + FeaturedCollectionURI: "http://thequeenisstillalive.technology/users/her_fuckin_maj/collections/featured", + ActorType: ap.ActorPerson, + AlsoKnownAs: "", + PrivateKey: &rsa.PrivateKey{}, + PublicKey: &rsa.PublicKey{}, + PublicKeyURI: "http://thequeenisstillalive.technology/users/her_fuckin_maj#main-key", + SensitizedAt: time.Time{}, + SilencedAt: time.Time{}, + SuspendedAt: time.Time{}, + HideCollections: FalseBool(), + SuspensionOrigin: "", + HeaderMediaAttachmentID: "01PFPMWK2FF0D9WMHEJHR07C3R", + }, } var accountsSorted []*gtsmodel.Account @@ -585,6 +622,7 @@ func NewTestAccounts() map[string]*gtsmodel.Account { }) preserializedKeys := []string{ + "MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDA3bAoQMUofndXMXikEU2MOJbfI1uaZbIDrxW0bEO6IOhwe/J0jWJHL2fWc2mbp2NxAH4Db1kZIcl9D0owoRf2cT5k0Y2Dah86dGz4fIedkqGryoWAnEJ2hHKGXGQf2K9OS2L8eaDLGU4CBds0m80vrn153Uiyj7zxWDYqcySM0qQjSg+mvgqpBcxKpd+xACaWNDL8qWvDsBF1D0RuO8hUiXMIKOUoFAGbqe6qWGK0COrEYQTAMydoFuSaAccP70zKQslnSOCKvsOi/iPRKGDNqWINIC/lwqXEIpMj3K+b/A+x41zR7frTgHNLbe4yHWAVNPEwTFningbB/lIyyVmDAgMBAAECggEBALxwnipmRnyvPClMY+RiJ5PGwtqYcGsly82/pwRW98GHX7Rv1lA8x/ZnghxNPbVg0k9ZvMXcaICeu4BejQ2AiKo4sU7OVGc/K+3wTXxoKBU0bJQuV0x24JVuCXvwD7/x9i8Yh0nKCOoH+mkNkcUQKWXaJi0IoXwd5u0kVCAbym1vux/9DcwtydqT4P1EoxEHCXDuRorBP8vYWCZBwRY2etmdAEbHsVpVlNlXWfbGCNMf5e8AecOZre4No8UfTOZkM7YKgjryde3YCmY2zDQI9jExGD2L5nptLizODD5imdpp/IQ7qg6rR3XbIK6CDiKiePEFQibD8XWiz7XVD6JBRokCgYEA0jEAxZseHUyobh1ERHezs2vC2zbiTOfnOpFxhwtNt67dUQZDssTxXF+BymUL8yKi1bnheOTuyASxrgZ7BPdiFvJfhlelSxtxtt1RamY58E179uiel2NPRsR3SL2AsGg+jP+QjJpsJHvYIliXP38G7NVaqaSMFgXfXir7Ty7W0r0CgYEA6uYQWfjmaB66xPrL/oCBaJ+UWM/Zdfw4IETVnRVOxVqGE7AKqC+31fZQ5kIXnNcJNLJ0OJlhGH5vZYp/r4z6qly9BUVolCJcW2YLEOOnChOvKGwlDSXrdGty2f34RXdABwsf/pBHsdpJq70+SE01tTB/8P2NTnRafy9GL/FnwT8CgYEAjJ4D6i8wImHafHBP7441Rl9daNJ66wBqDSCoVrQVNkFiBoauW7at0iKC7ihTqkENtvY4BW0C4gVh6Q6k1lm54agch/+ysWCW3sOJaCkjscPknvZYwubJboqZUqyUn2/eCO4ggi/9ERtZKQEjjnMo6uCBWuSeY01iddlDb2HijfECgYBYQCM4ikiWKaVlyAvIDCOSWRH04/IBX8b+aJ4QrCayAraIwwTd9z+MBUSTnZUdebSdtcXwVb+i4i2b6pLaM48hXkItrswBi39DX20c5UqmgIq4Fxk8fVienpfByqbyAkFt5AIbM72b1jUDbs/tfgSFlDkdI0VpilFNo0ctT/b5JQKBgAxPGtVGzhSQUZWPXjhiBT7MM/1EiLBYhGVrymzd9dmBxj+UyifnRXfIQbOQm3EfI5Z8ZpyS6eqWdi9NTeZi8rg0WleMb/VbOMT3xvTO34vDXvwrQKhFMimX1tY7aKy1udnE2ON2/alq2zWo3zPZfYH1KFdDtGD08GW2M4OO1caa", "MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDGj2wLnDIHnP6wjJ+WmIhp7NGAaKWwfxBWfdMFR+Y0ilkK5ld5igT45UHAmzN3v4HcwHGGpPITD9caDYj5YaGOX+dSdGLgXWwItR0j+ivrHEJmvz8hG6z9wKEZKUUrRw7Ob72S0LOsreq98bjdiWJKHNka27slqQjGyhLQtcg6pe1CLJtnuJH4GEMLj7jJB3/Mqv3vl5CQZ+Js0bXfgw5TF/x/Bzq/8qsxQ1vnmYHJsR0eLPEuDJOvoFPiJZytI09S7qBEJL5PDeVSfjQi3o71sqOzZlEL0b0Ny48rfo/mwJAdkmfcnydRDxeGUEqpAWICCOdUL0+W3/fCffaRZsk1AgMBAAECggEAUuyO6QJgeoF8dGsmMxSc0/ANRp1tpRpLznNZ77ipUYP9z+mG2sFjdjb4kOHASuB18aWFRAAbAQ76fGzuqYe2muk+iFcG/EDH35MUCnRuZxA0QwjX6pHOW2NZZFKyCnLwohJUj74Na65ufMk4tXysydrmaKsfq4i+m5bE6NkiOCtbXsjUGVdJKzkT6X1gEyEPEHgrgVZz9OpRY5nwjZBMcFI6EibFnWdehcuCQLESIX9ll/QzGvTJ1p8xeVJs2ktLWKQ38RewwucNYVLVJmxS1LCPP8x+yHVkOxD66eIncY26sjX+VbyICkaG/ZjKBuoOekOq/T+b6q5ESxWUNfcu+QKBgQDmt3WVBrW6EXKtN1MrVyBoSfn9WHyf8Rfb84t5iNtaWGSyPZK/arUw1DRbI0TdPjct//wMWoUU2/uqcPSzudTaPena3oxjKReXso1hcynHqboCaXJMxWSqDQLumbrVY05C1WFSyhRY0iQS5fIrNzD4+6rmeC2Aj5DKNW5Atda8dwKBgQDcUdhQfjL9SmzzIeAqJUBIfSSI2pSTsZrnrvMtSMkYJbzwYrUdhIVxaS4hXuQYmGgwonLctyvJxVxEMnf+U0nqPgJHE9nGQb5BbK6/LqxBWRJQlc+W6EYodIwvtE5B4JNkPE5757u+xlDdHe2zGUGXSIf4IjBNbSpCu6RcFsGOswKBgEnr4gqbmcJCMOH65fTu930yppxbq6J7Vs+sWrXX+aAazjilrc0S3XcFprjEth3E/10HtbQnlJg4W4wioOSs19wNFk6AG67xzZNXLCFbCrnkUarQKkUawcQSYywbqVcReFPFlmc2RAqpWdGMR2k9R72etQUe4EVeul9veyHUoTbFAoGBAKj3J9NLhaVVb8ri3vzThsJRHzTJlYrTeb5XIO5I1NhtEMK2oLobiQ+aH6O+F2Z5c+Zgn4CABdf/QSyYHAhzLcu0dKC4K5rtjpC0XiwHClovimk9C3BrgGrEP0LSn/XL2p3T1kkWRpkflKKPsl1ZcEEqggSdi7fFkdSN/ZYWaakbAoGBALWVGpA/vXmaZEV/hTDdtDnIHj6RXfKHCsfnyI7AdjUX4gokzdcEvFsEIoI+nnXR/PIAvwqvQw4wiUqQnp2VB8r73YZvW/0npnsidQw3ZjqnyvZ9X8y80nYs7DjSlaG0A8huy2TUdFnJyCMWby30g82kf0b/lhotJg4d3fIDou51", "MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQC6q61hiC7OhlMz7JNnLiL/RwOaFC8955GDvwSMH9Zw3oguWH9nLqkmlJ98cnqRG9ZC0qVo6Gagl7gv6yOHDwD4xZI8JoV2ZfNdDzq4QzoBIzMtRsbSS4IvrF3JP+kDH1tim+CbRMBxiFJgLgS6yeeQlLNvBW+CIYzmeCimZ6CWCr91rZPIprUIdjvhxrM9EQU072Pmzn2gpGM6K5gAReN+LtP+VSBC61x7GQJxBaJNtk11PXkgG99EdFi9vvgEBbM9bdcawvf8jxvjgsgdaDx/1cypDdnaL8eistmyv1YI67bKvrSPCEh55b90hl3o3vW4W5G4gcABoyORON96Y+i9AgMBAAECggEBAKp+tyNH0QiMo13fjFpHR2vFnsKSAPwXj063nx2kzqXUeqlp5yOE+LXmNSzjGpOCy1XJM474BRRUvsP1jkODLq4JNiF+RZP4Vij/CfDWZho33jxSUrIsiUGluxtfJiHV+A++s4zdZK/NhP+XyHYah0gEqUaTvl8q6Zhu0yH5sDCZHDLxDBpgiT5qD3lli8/o2xzzBdaibZdjQyHi9v5Yi3+ysly1tmfmqnkXSsevAubwJu504WxvDUSo7hPpG4a8Xb8ODqL738GIF2UY/olCcGkWqTQEr2pOqG9XbMmlUWnxG62GCfK6KtGfIzCyBBkGO2PZa9aPhVnv2bkYxI4PkLkCgYEAzAp7xH88UbSX31suDRa4jZwgtzhJLeyc3YxO5C4XyWZ89oWrA30V1KvfVwFRavYRJW07a+r0moba+0E1Nj5yZVXPOVu0bWd9ZyMbdH2L6MRZoJWU5bUOwyruulRCkqASZbWo4G05NOVesOyY1bhZGE7RyUW0vOo8tSyyRQ8nUGMCgYEA6jTQbDry4QkUP9tDhvc8+LsobIF1mPLEJui+mT98+9IGar6oeVDKekmNDO0Dx2+miLfjMNhCb5qUc8g036ZsekHt2WuQKunADua0coB00CebMdr6AQFf7QOQ/RuA+/gPJ5G0GzWB3YOQ5gE88tTCO/jBfmikVOZvLtgXUGjo3F8CgYEAl2poMoehQZjc41mMsRXdWukztgPE+pmORzKqENbLvB+cOG01XV9j5fCtyqklvFRioP2QjSNM5aeRtcbMMDbjOaQWJaCSImYcP39kDmxkeRXM1UhruJNGIzsm8Ys55Al53ZSTgAhN3Z0hSfYp7N/i7hD/yXc7Cr5g0qoamPkH2bUCgYApf0oeoyM9tDoeRl9knpHzEFZNQ3LusrUGn96FkLY4eDIi371CIYp+uGGBlM1CnQnI16wtj2PWGnGLQkH8DqTR1LSr/V8B+4DIIyB92TzZVOsunjoFy5SPjj42WpU0D/O/cxWSbJyh/xnBZx7Bd+kibyT5nNjhIiM5DZiz6qK3yQKBgAOO/MFKHKpKOXrtafbqCyculG/ope2u4eBveHKO6ByWcUSbuD9ebtr7Lu5AC5tKUJLkSyRx4EHk71bqP1yOITj8z9wQWdVyLxtVtyj9SUkUNvGwIj+F7NJ5VgHzWVZtvYWDCzrfxkEhKk3DRIIVjqmEohJcaOZoZ2Q/f8sjlId6", "MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQC1NzommDoutE+FVAbgovPb5ioRRS1k93hhH5Mpe4KfAQb1k0aGA/TjrFr2HcRbOtldo6Fe+RRCAm5Go+sBx829zyMEdXbGtR4Pym78xYoRpsCwD+2AK/edQLjsdDf9zXZod9ig/Pe59awYaeuSFyK/w9q94ncuiE7m+MKfXJnTS/qiwwkxWIRm9lBprPIT0DwXCtoh7FdpsOmjLu2QdGADV+9KSDgV5IbVcxwjPY03vHJS4UAIP5eS46TtSrNF3hyM9Q8vGIPAixOVyAY53cRQUxZWU/FIaNjaEgpreUQfK1pgr0gxh1K7IKwmyF3f/JgL0urFljYp2UonzRU5XKHJAgMBAAECggEBAKVT2HLDqTlY+b/LNGcXY+H4b+LHuS2HdUUuuGU9MKN+HWpIziuQSoi4g1hNOgp9ezgqBByQpAHBE/jQraQ3NKZ55xm3TQDm1qFTb8SfOGL4Po2iSm0IL+VA2jWnpjmgjOmshXACusPmtfakE+55uxM3TUa16UQDyfCBfZZEtnaFLTYzJ7KmD2GPot8SCxJBqNmW7AL8pMSIxMC3cRxUbK4R3+KIisXUuB50jZH3zGHxi34e2jA6gDeFmzgHCDJRidHMsCTHTaATzlvVz9YwwNqPQaYY7OFouZXwFxVAxIg/1zVvLc3zx1gWt+UDFeI7h6Eq0h5DZPdUiR4mrhAKd70CgYEAw6WKbPgjzhJI9XVmnu0aMHHH4MK8pbIq4kddChw24yZv1e9qnNTHw3YK17X9Fqog9CU1OX3M/vddfQbc34SorBmtmGYgOfDSuXTct52Ppyl4CRwndYQc0A88Hw+klluTEPY3+NRV6YSzv8vkNMasVuOh0YI1xzbpc+Bb5LL3kwMCgYEA7R4PLYYmtzKAY2YTQOXGBh3xd6UEHgks30W+QzDxvOv75svZt6yDgiwJzXtyrQzbNaH6yca5nfjkqyhnHwpguJ6DK7+S/RnZfVib5MqRwiU7g8l3neKhIXs6xZxfORunDU9T5ntbyNaGv/TJ2cXNw+9VskhBaHfEN/kmaBNNuEMCgYARLuzlfTXH15tI07Lbqn9uWc/wUao381oI3bOyO6Amey2/YHPAqn+RD0EMiRNddjvGta3jCsWCbz9qx7uGdiRKWUcB55ZVAG3BlB3+knwXdnDwe+SLUbsmGvBw2fLesdRM3RM1a5DQHbOb2NCGQhzI1N1VhVYr1QrT/pSTlZRg+QKBgCE05nc/pEhfoC9LakLaauMManaQ+4ShUFFsWPrb7d7BRaPKxJC+biRauny2XxbxB/n410BOvkvrQUre+6ITN/xi5ofH6nPbnOO69woRfFwuDqmkG0ZXKK2hrldiUMuUnc51X5CVkgMMWA6l32bKFsjryZqQF+jjbO1RzRkiKu41AoGAHQer1NyajHEpEfempx8YTsAnOn+Hi33cXAaQoTkS41lX2YK0cBkD18yhubczZcKnMW+GRKZRYXMm0NfwiuIo5oIYWeO6K+rXF+SKptC5mnw/3FhDVnghDAmEqOcRSWnFXARk1WEbFtwG5phDeFrWXsqPzGAjoZ8bhLvKRsrG4OM=", @@ -979,6 +1017,55 @@ func NewTestAttachments() map[string]*gtsmodel.MediaAttachment { Header: FalseBool(), Cached: TrueBool(), }, + "remote_account_3_header": { + ID: "01PFPMWK2FF0D9WMHEJHR07C3R", + StatusID: "", + URL: "http://localhost:8080/fileserver/062G5WYKY35KKD12EMSM3F8PJ8/header/original/01PFPMWK2FF0D9WMHEJHR07C3R.jpeg", + RemoteURL: "http://fossbros-anonymous.io/attachments/small/a499f55b-2d1e-4acd-98d2-1ac2ba6d79b9.jpeg", + CreatedAt: TimeMustParse("2022-06-09T13:12:00Z"), + UpdatedAt: TimeMustParse("2022-06-09T13:12:00Z"), + Type: gtsmodel.FileTypeImage, + FileMeta: gtsmodel.FileMeta{ + Original: gtsmodel.Original{ + Width: 472, + Height: 291, + Size: 137352, + Aspect: 1.6219931271477663, + }, + Small: gtsmodel.Small{ + Width: 472, + Height: 291, + Size: 137352, + Aspect: 1.6219931271477663, + }, + Focus: gtsmodel.Focus{ + X: 0, + Y: 0, + }, + }, + AccountID: "062G5WYKY35KKD12EMSM3F8PJ8", + Description: "tweet from thoughts of dog: i drank. all the water. in my bowl. earlier. but just now. i returned. to the same bowl. and it was. full again.. the bowl. is haunted", + ScheduledStatusID: "", + Blurhash: "LARysgM_IU_3~pD%M_Rj_39FIAt6", + Processing: 2, + File: gtsmodel.File{ + Path: "062G5WYKY35KKD12EMSM3F8PJ8/attachment/original/01PFPMWK2FF0D9WMHEJHR07C3R.jpeg", + ContentType: "image/jpeg", + FileSize: 19310, + UpdatedAt: TimeMustParse("2022-06-09T13:12:00Z"), + }, + Thumbnail: gtsmodel.Thumbnail{ + Path: "062G5WYKY35KKD12EMSM3F8PJ8/attachment/small/01PFPMWK2FF0D9WMHEJHR07C3R.jpeg", + ContentType: "image/jpeg", + FileSize: 20395, + UpdatedAt: TimeMustParse("2022-06-09T13:12:00Z"), + URL: "http://localhost:8080/fileserver/062G5WYKY35KKD12EMSM3F8PJ8/header/small/01PFPMWK2FF0D9WMHEJHR07C3R.jpeg", + RemoteURL: "http://fossbros-anonymous.io/attachments/small/a499f55b-2d1e-4acd-98d2-1ac2ba6d79b9.jpeg", + }, + Avatar: FalseBool(), + Header: TrueBool(), + Cached: TrueBool(), + }, } }