Mirrors/gotosocial
2
0
Fork 0
mirror of https://github.com/superseriousbusiness/gotosocial synced 2024-11-14 00:37:18 +00:00
Code Issues Projects Releases Packages Wiki Activity

Merge pull request #361 from superseriousbusiness/media_refactor

Browse source 
Refactor media handler to allow async media resolution
This commit is contained in:
kim 2022-02-12 18:27:58 +00:00 committed by GitHub
commit 31935ee206
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
244 changed files with 9104 additions and 12940 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
README.md
View file

@ -182,6 +182,7 @@ The following libraries and frameworks are used by GoToSocial, with gratitude
- [google/uuid](https://github.com/google/uuid); UUID generation. [BSD-3-Clause License](https://spdx.org/licenses/BSD-3-Clause.html)
- [go-playground/validator](https://github.com/go-playground/validator); struct validation. [MIT License](https://spdx.org/licenses/MIT.html)
- [gorilla/websocket](https://github.com/gorilla/websocket); Websocket connectivity. [BSD-2-Clause License](https://spdx.org/licenses/BSD-2-Clause.html).
- [gruf/go-runners](https://codeberg.org/gruf/go-runners); worker pool library. [MIT License](https://spdx.org/licenses/MIT.html).
- [gruf/go-store](https://codeberg.org/gruf/go-store); cacheing library. [MIT License](https://spdx.org/licenses/MIT.html).
- [h2non/filetype](https://github.com/h2non/filetype); filetype checking. [MIT License](https://spdx.org/licenses/MIT.html).
- [jackc/pgx](https://github.com/jackc/pgx); Postgres driver. [MIT License](https://spdx.org/licenses/MIT.html).
@ -201,7 +202,7 @@ The following libraries and frameworks are used by GoToSocial, with gratitude
- [spf13/pflag](https://github.com/spf13/pflag); command-line flag utilities. [Apache-2.0 License](https://spdx.org/licenses/Apache-2.0.html).
- [spf13/viper](https://github.com/spf13/viper); configuration management. [Apache-2.0 License](https://spdx.org/licenses/Apache-2.0.html).
- [stretchr/testify](https://github.com/stretchr/testify); test framework. [MIT License](https://spdx.org/licenses/MIT.html).
- [superseriousbusiness/exifremove](https://github.com/superseriousbusiness/exifremove) forked from [scottleedavis/go-exif-remove](https://github.com/scottleedavis/go-exif-remove); EXIF data removal. [MIT License](https://spdx.org/licenses/MIT.html).
- [superseriousbusiness/exif-terminator](https://github.com/superseriousbusiness/exif-terminator); EXIF data removal. [GNU AGPL v3 LICENSE](https://spdx.org/licenses/AGPL-3.0-or-later.html).
- [superseriousbusiness/activity](https://github.com/superseriousbusiness/activity) forked from [go-fed/activity](https://github.com/go-fed/activity); Golang ActivityPub/ActivityStreams library. [BSD-3-Clause License](https://spdx.org/licenses/BSD-3-Clause.html).
- [superseriousbusiness/oauth2](https://github.com/superseriousbusiness/oauth2) forked from [go-oauth2/oauth2](https://github.com/go-oauth2/oauth2); oauth server framework and token handling. [MIT License](https://spdx.org/licenses/MIT.html).
- [go-swagger/go-swagger](https://github.com/go-swagger/go-swagger); Swagger OpenAPI spec generation. [Apache-2.0 License](https://spdx.org/licenses/Apache-2.0.html).

22
cmd/gotosocial/action/server/server.go
View file

@ -24,9 +24,11 @@ import (
"net/http"
"os"
"os/signal"
"path"
"syscall"
"codeberg.org/gruf/go-store/kv"
"codeberg.org/gruf/go-store/storage"
"github.com/sirupsen/logrus"
"github.com/spf13/viper"
"github.com/superseriousbusiness/gotosocial/cmd/gotosocial/action"
@ -97,16 +99,26 @@ var Start action.GTSAction = func(ctx context.Context) error {
// Open the storage backend
storageBasePath := viper.GetString(config.Keys.StorageLocalBasePath)
storage, err := kv.OpenFile(storageBasePath, nil)
storage, err := kv.OpenFile(storageBasePath, &storage.DiskConfig{
// Put the store lockfile in the storage dir itself.
// Normally this would not be safe, since we could end up
// overwriting the lockfile if we store a file called 'store.lock'.
// However, in this case it's OK because the keys are set by
// GtS and not the user, so we know we're never going to overwrite it.
LockFile: path.Join(storageBasePath, "store.lock"),
})
if err != nil {
return fmt.Errorf("error creating storage backend: %s", err)
}
// build backend handlers
mediaHandler := media.New(dbService, storage)
mediaManager, err := media.NewManager(dbService, storage)
if err != nil {
return fmt.Errorf("error creating media manager: %s", err)
}
oauthServer := oauth.New(ctx, dbService)
transportController := transport.NewController(dbService, &federation.Clock{}, http.DefaultClient)
federator := federation.NewFederator(dbService, federatingDB, transportController, typeConverter, mediaHandler)
federator := federation.NewFederator(dbService, federatingDB, transportController, typeConverter, mediaManager)
// decide whether to create a noop email sender (won't send emails) or a real one
var emailSender email.Sender
@ -126,7 +138,7 @@ var Start action.GTSAction = func(ctx context.Context) error {
}
// create and start the message processor using the other services we've created so far
processor := processing.NewProcessor(typeConverter, federator, oauthServer, mediaHandler, storage, dbService, emailSender)
processor := processing.NewProcessor(typeConverter, federator, oauthServer, mediaManager, storage, dbService, emailSender)
if err := processor.Start(ctx); err != nil {
return fmt.Errorf("error starting processor: %s", err)
}
@ -198,7 +210,7 @@ var Start action.GTSAction = func(ctx context.Context) error {
}
}
gts, err := gotosocial.NewServer(dbService, router, federator)
gts, err := gotosocial.NewServer(dbService, router, federator, mediaManager)
if err != nil {
return fmt.Errorf("error creating gotosocial service: %s", err)
}

7
cmd/gotosocial/action/testrig/testrig.go
View file

@ -80,11 +80,12 @@ var Start action.GTSAction = func(ctx context.Context) error {
Body: r,
}, nil
}), dbService)
federator := testrig.NewTestFederator(dbService, transportController, storageBackend)
mediaManager := testrig.NewTestMediaManager(dbService, storageBackend)
federator := testrig.NewTestFederator(dbService, transportController, storageBackend, mediaManager)
emailSender := testrig.NewEmailSender("./web/template/", nil)
processor := testrig.NewTestProcessor(dbService, storageBackend, federator, emailSender)
processor := testrig.NewTestProcessor(dbService, storageBackend, federator, emailSender, mediaManager)
if err := processor.Start(ctx); err != nil {
return fmt.Errorf("error starting processor: %s", err)
}
@ -156,7 +157,7 @@ var Start action.GTSAction = func(ctx context.Context) error {
}
}
gts, err := gotosocial.NewServer(dbService, router, federator)
gts, err := gotosocial.NewServer(dbService, router, federator, mediaManager)
if err != nil {
return fmt.Errorf("error creating gotosocial service: %s", err)
}

21
go.mod
View file

@ -3,8 +3,9 @@ module github.com/superseriousbusiness/gotosocial
go 1.17
require (
codeberg.org/gruf/go-errors v1.0.4
codeberg.org/gruf/go-store v1.1.5
codeberg.org/gruf/go-errors v1.0.5
codeberg.org/gruf/go-runners v1.2.0
codeberg.org/gruf/go-store v1.3.3
github.com/ReneKroon/ttlcache v1.7.0
github.com/buckket/go-blurhash v1.1.0
github.com/coreos/go-oidc/v3 v3.1.0
@ -29,7 +30,7 @@ require (
github.com/spf13/viper v1.10.0
github.com/stretchr/testify v1.7.0
github.com/superseriousbusiness/activity v1.0.1-0.20211113133524-56560b73ace8
github.com/superseriousbusiness/exifremove v0.0.0-20210330092427-6acd27eac203
github.com/superseriousbusiness/exif-terminator v0.1.0
github.com/superseriousbusiness/oauth2/v4 v4.3.2-SSB
github.com/tdewolff/minify/v2 v2.9.22
github.com/uptrace/bun v1.0.19
@ -47,21 +48,19 @@ require (
require (
codeberg.org/gruf/go-bytes v1.0.2 // indirect
codeberg.org/gruf/go-fastpath v1.0.2 // indirect
codeberg.org/gruf/go-format v1.0.3 // indirect
codeberg.org/gruf/go-hashenc v1.0.1 // indirect
codeberg.org/gruf/go-logger v1.3.2 // indirect
codeberg.org/gruf/go-mutexes v1.0.1 // indirect
codeberg.org/gruf/go-nowish v1.1.0 // indirect
codeberg.org/gruf/go-mutexes v1.1.0 // indirect
codeberg.org/gruf/go-pools v1.0.2 // indirect
github.com/aymerick/douceur v0.2.0 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/dsoprea/go-exif v0.0.0-20210625224831-a6301f85c82b // indirect
github.com/dsoprea/go-exif/v2 v2.0.0-20210625224831-a6301f85c82b // indirect
github.com/dsoprea/go-exif/v3 v3.0.0-20210625224831-a6301f85c82b // indirect
github.com/dsoprea/go-iptc v0.0.0-20200610044640-bc9ca208b413 // indirect
github.com/dsoprea/go-jpeg-image-structure v0.0.0-20210512043942-b434301c6836 // indirect
github.com/dsoprea/go-jpeg-image-structure/v2 v2.0.0-20210512043942-b434301c6836 // indirect
github.com/dsoprea/go-logging v0.0.0-20200710184922-b02d349568dd // indirect
github.com/dsoprea/go-photoshop-info-format v0.0.0-20200610045659-121dd752914d // indirect
github.com/dsoprea/go-png-image-structure v0.0.0-20210512210324-29b889a6093d // indirect
github.com/dsoprea/go-utility v0.0.0-20200717064901-2fccff4aa15e // indirect
github.com/dsoprea/go-png-image-structure/v2 v2.0.0-20210512210324-29b889a6093d // indirect
github.com/dsoprea/go-utility/v2 v2.0.0-20200717064901-2fccff4aa15e // indirect
github.com/fsnotify/fsnotify v1.5.1 // indirect
github.com/gin-contrib/sse v0.1.0 // indirect
github.com/go-errors/errors v1.4.1 // indirect

51
go.sum
View file

@ -51,27 +51,26 @@ codeberg.org/gruf/go-bytes v1.0.1/go.mod h1:1v/ibfaosfXSZtRdW2rWaVrDXMc9E3bsi/M9
codeberg.org/gruf/go-bytes v1.0.2 h1:malqE42Ni+h1nnYWBUAJaDDtEzF4aeN4uPN8DfMNNvo=
codeberg.org/gruf/go-bytes v1.0.2/go.mod h1:1v/ibfaosfXSZtRdW2rWaVrDXMc9E3bsi/M9Ekx39cg=
codeberg.org/gruf/go-cache v1.1.2/go.mod h1:/Dbc+xU72Op3hMn6x2PXF3NE9uIDFeS+sXPF00hN/7o=
codeberg.org/gruf/go-errors v1.0.4 h1:jOJCn/GMb6ELLRVlnmpimGRC2CbTreH5/CBZNWh9GZA=
codeberg.org/gruf/go-errors v1.0.4/go.mod h1:rJ08LdIE79Jg8vZ2TGylz/I+tZ1UuMJkGK5mNambIfQ=
codeberg.org/gruf/go-errors v1.0.5 h1:rxV70oQkfasUdggLHxOX2QAoJOMFM7XWxHQR45Zx/Fg=
codeberg.org/gruf/go-errors v1.0.5/go.mod h1:n03EpmvcmfzU3/xJKC0XXtleXXJUNFpT2fgISODvZ1Y=
codeberg.org/gruf/go-fastpath v1.0.1/go.mod h1:edveE/Kp3Eqi0JJm0lXYdkVrB28cNUkcb/bRGFTPqeI=
codeberg.org/gruf/go-fastpath v1.0.2 h1:O3nuYPMXnN89dsgAwVFU5iCGINtPJdITWmbRe2an/iQ=
codeberg.org/gruf/go-fastpath v1.0.2/go.mod h1:edveE/Kp3Eqi0JJm0lXYdkVrB28cNUkcb/bRGFTPqeI=
codeberg.org/gruf/go-format v1.0.3 h1:WoUGzTwZe6SIhILNvtr0qNIA7BOOCgdBlk5bUrfeiio=
codeberg.org/gruf/go-format v1.0.3/go.mod h1:k3TLXp1dqAXdDqxlon0yEM+3FFHdNn0D6BVJTwTy5As=
codeberg.org/gruf/go-hashenc v1.0.1 h1:EBvNe2wW8IPMUqT1XihB6/IM6KMJDLMFBxIUvmsy1f8=
codeberg.org/gruf/go-hashenc v1.0.1/go.mod h1:IfHhPCVScOiYmJLqdCQT9bYVS1nxNTV4ewMUvFWDPtc=
codeberg.org/gruf/go-logger v1.3.1/go.mod h1:tBduUc+Yb9vqGRxY9/FB0ZlYznSteLy/KmIANo7zFjA=
codeberg.org/gruf/go-logger v1.3.2 h1:/2Cg8Tmu6H10lljq/BvHE+76O2d4tDNUDwitN6YUxxk=
codeberg.org/gruf/go-logger v1.3.2/go.mod h1:q4xmTSdaxPzfndSXVF1X2xcyCVk7Nd/PIWCDs/4biMg=
codeberg.org/gruf/go-mutexes v1.0.1 h1:X9bZW74YSEplWWdCrVXAvue5ztw3w5hh+INdXTENu88=
codeberg.org/gruf/go-mutexes v1.0.1/go.mod h1:y2hbGLkWVHhNyxBOIVsA3/y2QMm6RSrYsC3sLVZ4EXM=
codeberg.org/gruf/go-mutexes v1.1.0 h1:kMVWHLxdfGEZTetNVRncdBMeqS4M8dSJxSGbRYXyvKk=
codeberg.org/gruf/go-mutexes v1.1.0/go.mod h1:1j/6/MBeBQUedAtAtysLLnBKogfOZAxdym0E3wlaBD8=
codeberg.org/gruf/go-nowish v1.0.0/go.mod h1:70nvICNcqQ9OHpF07N614Dyk7cpL5ToWU1K1ZVCec2s=
codeberg.org/gruf/go-nowish v1.0.2/go.mod h1:70nvICNcqQ9OHpF07N614Dyk7cpL5ToWU1K1ZVCec2s=
codeberg.org/gruf/go-nowish v1.1.0 h1:rj1z0AXDhLvnxs/DazWFxYAugs6rv5vhgWJkRCgrESg=
codeberg.org/gruf/go-nowish v1.1.0/go.mod h1:70nvICNcqQ9OHpF07N614Dyk7cpL5ToWU1K1ZVCec2s=
codeberg.org/gruf/go-pools v1.0.2 h1:B0X6yoCL9FVmnvyoizb1SYRwMYPWwEJBjPnBMM5ILos=
codeberg.org/gruf/go-pools v1.0.2/go.mod h1:MjUV3H6IASyBeBPCyCr7wjPpSNu8E2N87LG4r4TAyok=
codeberg.org/gruf/go-runners v1.1.1/go.mod h1:9gTrmMnO3d+50C+hVzcmGBf+zTuswReS278E2EMvnmw=
codeberg.org/gruf/go-store v1.1.5 h1:fp28vzGD15OsAF51CCwi7woH+Y3vb0aMl4OFh9JSjA0=
codeberg.org/gruf/go-store v1.1.5/go.mod h1:Q6ev500ddKghDQ8KS4IstL/W9fptDKa2T9oeHP+tXsI=
codeberg.org/gruf/go-runners v1.2.0 h1:tkoPrwYMkVg1o/C4PGTR1YbC11XX4r06uLPOYajBsH4=
codeberg.org/gruf/go-runners v1.2.0/go.mod h1:9gTrmMnO3d+50C+hVzcmGBf+zTuswReS278E2EMvnmw=
codeberg.org/gruf/go-store v1.3.3 h1:fAP9FXy6HiLPxdD7cmpSzyfKXmVvZLjqn0m7HhxVT5M=
codeberg.org/gruf/go-store v1.3.3/go.mod h1:g4+9h3wbwZ6IW0uhpw57xywcqiy4CIj0zQLqqtjEU1M=
dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
@ -146,21 +145,16 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/djherbis/atime v1.1.0/go.mod h1:28OF6Y8s3NQWwacXc5eZTsEsiMzp7LF8MbXE+XJPdBE=
github.com/dsoprea/go-exif v0.0.0-20210131231135-d154f10435cc/go.mod h1:lOaOt7+UEppOgyvRy749v3do836U/hw0YVJNjoyPaEs=
github.com/dsoprea/go-exif v0.0.0-20210625224831-a6301f85c82b h1:hoVHc4m/v8Al8mbWyvKJWr4Z37yM4QUSVh/NY6A5Sbc=
github.com/dsoprea/go-exif v0.0.0-20210625224831-a6301f85c82b/go.mod h1:lOaOt7+UEppOgyvRy749v3do836U/hw0YVJNjoyPaEs=
github.com/dsoprea/go-exif/v2 v2.0.0-20200321225314-640175a69fe4/go.mod h1:Lm2lMM2zx8p4a34ZemkaUV95AnMl4ZvLbCUbwOvLC2E=
github.com/dsoprea/go-exif/v2 v2.0.0-20200604193436-ca8584a0e1c4/go.mod h1:9EXlPeHfblFFnwu5UOqmP2eoZfJyAZ2Ri/Vki33ajO0=
github.com/dsoprea/go-exif/v2 v2.0.0-20210625224831-a6301f85c82b h1:8lVRnnni9zebcpjkrEXrEyxFpRWG/oTpWc2Y3giKomE=
github.com/dsoprea/go-exif/v2 v2.0.0-20210625224831-a6301f85c82b/go.mod h1:oKrjk2kb3rAR5NbtSTLUMvMSbc+k8ZosI3MaVH47noc=
github.com/dsoprea/go-exif/v3 v3.0.0-20200717053412-08f1b6708903/go.mod h1:0nsO1ce0mh5czxGeLo4+OCZ/C6Eo6ZlMWsz7rH/Gxv8=
github.com/dsoprea/go-exif/v3 v3.0.0-20210512043655-120bcdb2a55e/go.mod h1:cg5SNYKHMmzxsr9X6ZeLh/nfBRHHp5PngtEPcujONtk=
github.com/dsoprea/go-exif/v3 v3.0.0-20210428042052-dca55bf8ca15/go.mod h1:cg5SNYKHMmzxsr9X6ZeLh/nfBRHHp5PngtEPcujONtk=
github.com/dsoprea/go-exif/v3 v3.0.0-20210625224831-a6301f85c82b h1:NgNuLvW/gAFKU30ULWW0gtkCt56JfB7FrZ2zyo0wT8I=
github.com/dsoprea/go-exif/v3 v3.0.0-20210625224831-a6301f85c82b/go.mod h1:cg5SNYKHMmzxsr9X6ZeLh/nfBRHHp5PngtEPcujONtk=
github.com/dsoprea/go-iptc v0.0.0-20200609062250-162ae6b44feb/go.mod h1:kYIdx9N9NaOyD7U6D+YtExN7QhRm+5kq7//yOsRXQtM=
github.com/dsoprea/go-iptc v0.0.0-20200610044640-bc9ca208b413 h1:YDRiMEm32T60Kpm35YzOK9ZHgjsS1Qrid+XskNcsdp8=
github.com/dsoprea/go-iptc v0.0.0-20200610044640-bc9ca208b413/go.mod h1:kYIdx9N9NaOyD7U6D+YtExN7QhRm+5kq7//yOsRXQtM=
github.com/dsoprea/go-jpeg-image-structure v0.0.0-20210128210355-86b1014917f2/go.mod h1:ZoOP3yUG0HD1T4IUjIFsz/2OAB2yB4YX6NSm4K+uJRg=
github.com/dsoprea/go-jpeg-image-structure v0.0.0-20210512043942-b434301c6836 h1:OHRfKIVRz2XrhZ6A7fJKHLoKky1giN+VUgU2npF0BvE=
github.com/dsoprea/go-jpeg-image-structure v0.0.0-20210512043942-b434301c6836/go.mod h1:6+tQXZ+I62x13UZ+hemLVoZIuq/usVzvau7bqwUo9P0=
github.com/dsoprea/go-jpeg-image-structure/v2 v2.0.0-20210512043942-b434301c6836 h1:KGCiMMWxODEMmI3+9Ms04l73efoqFVNKKKPbVyOvKrU=
github.com/dsoprea/go-jpeg-image-structure/v2 v2.0.0-20210512043942-b434301c6836/go.mod h1:WaARaUjQuSuDCDFAiU/GwzfxMTJBulfEhqEA2Tx6B4Y=
github.com/dsoprea/go-logging v0.0.0-20190624164917-c4f10aab7696/go.mod h1:Nm/x2ZUNRW6Fe5C3LxdY1PyZY5wmDv/s5dkPJ/VB3iA=
github.com/dsoprea/go-logging v0.0.0-20200517223158-a10564966e9d/go.mod h1:7I+3Pe2o/YSU88W0hWlm9S22W7XI1JFNJ86U0zPKMf8=
github.com/dsoprea/go-logging v0.0.0-20200710184922-b02d349568dd h1:l+vLbuxptsC6VQyQsfD7NnEC8BZuFpz45PgY+pH8YTg=
@ -168,13 +162,11 @@ github.com/dsoprea/go-logging v0.0.0-20200710184922-b02d349568dd/go.mod h1:7I+3P
github.com/dsoprea/go-photoshop-info-format v0.0.0-20200609050348-3db9b63b202c/go.mod h1:pqKB+ijp27cEcrHxhXVgUUMlSDRuGJJp1E+20Lj5H0E=
github.com/dsoprea/go-photoshop-info-format v0.0.0-20200610045659-121dd752914d h1:dg6UMHa50VI01WuPWXPbNJpO8QSyvIF5T5n2IZiqX3A=
github.com/dsoprea/go-photoshop-info-format v0.0.0-20200610045659-121dd752914d/go.mod h1:pqKB+ijp27cEcrHxhXVgUUMlSDRuGJJp1E+20Lj5H0E=
github.com/dsoprea/go-png-image-structure v0.0.0-20200807080309-a98d4e94ac82/go.mod h1:aDYQkL/5gfRNZkoxiLTSWU4Y8/gV/4MVsy/MU9uwTak=
github.com/dsoprea/go-png-image-structure v0.0.0-20210512210324-29b889a6093d h1:8+qI8ant/vZkNSsbwSjIR6XJfWcDVTg/qx/3pRUUZNA=
github.com/dsoprea/go-png-image-structure v0.0.0-20210512210324-29b889a6093d/go.mod h1:yTR3tKgyk20phAFg6IE9ulMA5NjEDD2wyx+okRFLVtw=
github.com/dsoprea/go-utility v0.0.0-20200512094054-1abbbc781176/go.mod h1:95+K3z2L0mqsVYd6yveIv1lmtT3tcQQ3dVakPySffW8=
github.com/dsoprea/go-png-image-structure/v2 v2.0.0-20210512210324-29b889a6093d h1:2zNIgrJTspLxUKoJGl0Ln24+hufPKSjP3cu4++5MeSE=
github.com/dsoprea/go-png-image-structure/v2 v2.0.0-20210512210324-29b889a6093d/go.mod h1:scnx0wQSM7UiCMK66dSdiPZvL2hl6iF5DvpZ7uT59MY=
github.com/dsoprea/go-utility v0.0.0-20200711062821-fab8125e9bdf h1:/w4QxepU4AHh3AuO6/g8y/YIIHH5+aKP3Bj8sg5cqhU=
github.com/dsoprea/go-utility v0.0.0-20200711062821-fab8125e9bdf/go.mod h1:95+K3z2L0mqsVYd6yveIv1lmtT3tcQQ3dVakPySffW8=
github.com/dsoprea/go-utility v0.0.0-20200717064901-2fccff4aa15e h1:ojqYA1mU6LuRm8XzrVOvyfb000y59cbUcu6Wt8sFSAs=
github.com/dsoprea/go-utility v0.0.0-20200717064901-2fccff4aa15e/go.mod h1:KVK+/Hul09ujXAGq+42UBgCTnXkiJZRnLYdURGjQUwo=
github.com/dsoprea/go-utility/v2 v2.0.0-20200717064901-2fccff4aa15e h1:IxIbA7VbCNrwumIYjDoMOdf4KOSkMC6NJE4s8oRbE7E=
github.com/dsoprea/go-utility/v2 v2.0.0-20200717064901-2fccff4aa15e/go.mod h1:uAzdkPTub5Y9yQwXe8W4m2XuP0tK4a9Q/dantD0+uaU=
github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo=
github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
@ -359,7 +351,6 @@ github.com/gorilla/sessions v1.2.1/go.mod h1:dk2InVEVJ0sfLlnXv9EAgkf6ecYs/i80K/z
github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc=
github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw=
github.com/h2non/filetype v1.1.1/go.mod h1:319b3zT68BvV+WRj7cwy856M2ehB3HqNOt6sy1HndBY=
github.com/h2non/filetype v1.1.3 h1:FKkx9QbD7HR/zjK1Ia5XiBsq9zdLi5Kf3zGyFTAFkGg=
github.com/h2non/filetype v1.1.3/go.mod h1:319b3zT68BvV+WRj7cwy856M2ehB3HqNOt6sy1HndBY=
github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q=
@ -658,8 +649,8 @@ github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s
github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=
github.com/superseriousbusiness/activity v1.0.1-0.20211113133524-56560b73ace8 h1:8Bwy6CSsT33/sF5FhjND4vr7jiJCaq4elNTAW4rUzVc=
github.com/superseriousbusiness/activity v1.0.1-0.20211113133524-56560b73ace8/go.mod h1:ZY9xwFDucvp6zTvM6FQZGl8PSOofPBFIAy6gSc85XkY=
github.com/superseriousbusiness/exifremove v0.0.0-20210330092427-6acd27eac203 h1:1SWXcTphBQjYGWRRxLFIAR1LVtQEj4eR7xPtyeOVM/c=
github.com/superseriousbusiness/exifremove v0.0.0-20210330092427-6acd27eac203/go.mod h1:0Xw5cYMOYpgaWs+OOSx41ugycl2qvKTi9tlMMcZhFyY=
github.com/superseriousbusiness/exif-terminator v0.1.0 h1:ePzfV0vcw+tm/haSOGzKbBTKkHAvyQLbCzfsdVkb3hM=
github.com/superseriousbusiness/exif-terminator v0.1.0/go.mod h1:pmlOKzkFZWmqaucLAtrRbZG0R5F3dbrcLWOcd7gAOLI=
github.com/superseriousbusiness/oauth2/v4 v4.3.2-SSB h1:PtW2w6budTvRV2J5QAoSvThTHBuvh8t/+BXIZFAaBSc=
github.com/superseriousbusiness/oauth2/v4 v4.3.2-SSB/go.mod h1:uYC/W92oVRJ49Vh1GcvTqpeFqHi+Ovrl2sMllQWRAEo=
github.com/tdewolff/minify/v2 v2.9.22 h1:PlmaAakaJHdMMdTTwjjsuSwIxKqWPTlvjTj6a/g/ILU=

18
internal/ap/extract.go
View file

@ -395,20 +395,20 @@ func ExtractAttachment(i Attachmentable) (*gtsmodel.MediaAttachment, error) {
attachment.Description = name
}
attachment.Blurhash = ExtractBlurhash(i)
attachment.Processing = gtsmodel.ProcessingStatusReceived
return attachment, nil
}
// func extractBlurhash(i withBlurhash) (string, error) {
// if i.GetTootBlurhashProperty() == nil {
// return "", errors.New("blurhash property was nil")
// }
// if i.GetTootBlurhashProperty().Get() == "" {
// return "", errors.New("empty blurhash string")
// }
// return i.GetTootBlurhashProperty().Get(), nil
// }
// ExtractBlurhash extracts the blurhash value (if present) from a WithBlurhash interface.
func ExtractBlurhash(i WithBlurhash) string {
if i.GetTootBlurhash() == nil {
return ""
}
return i.GetTootBlurhash().Get()
}
// ExtractHashtags returns a slice of tags on the interface.
func ExtractHashtags(i WithTag) ([]*gtsmodel.Tag, error) {

2
internal/ap/extractattachments_test.go
View file

@ -42,7 +42,7 @@ func (suite *ExtractAttachmentsTestSuite) TestExtractAttachments() {
suite.Equal("image/jpeg", attachment1.File.ContentType)
suite.Equal("https://s3-us-west-2.amazonaws.com/plushcity/media_attachments/files/106/867/380/219/163/828/original/88e8758c5f011439.jpg", attachment1.RemoteURL)
suite.Equal("It's a cute plushie.", attachment1.Description)
suite.Empty(attachment1.Blurhash) // atm we discard blurhashes and generate them ourselves during processing
suite.Equal("UxQ0EkRP_4tRxtRjWBt7%hozM_ayV@oLf6WB", attachment1.Blurhash)
}
func (suite *ExtractAttachmentsTestSuite) TestExtractNoAttachments() {

8
internal/ap/interfaces.go
View file

@ -70,6 +70,7 @@ type Attachmentable interface {
WithMediaType
WithURL
WithName
WithBlurhash
}
// Hashtaggable represents the minimum activitypub interface for representing a 'hashtag' tag.
@ -284,9 +285,10 @@ type WithMediaType interface {
GetActivityStreamsMediaType() vocab.ActivityStreamsMediaTypeProperty
}
// type withBlurhash interface {
// GetTootBlurhashProperty() vocab.TootBlurhashProperty
// }
// WithBlurhash represents an activity with TootBlurhashProperty
type WithBlurhash interface {
GetTootBlurhash() vocab.TootBlurhashProperty
}
// type withFocalPoint interface {
// // TODO

21
internal/api/client/account/account_test.go
View file

@ -16,6 +16,7 @@ import (
"github.com/superseriousbusiness/gotosocial/internal/email"
"github.com/superseriousbusiness/gotosocial/internal/federation"
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
"github.com/superseriousbusiness/gotosocial/internal/media"
"github.com/superseriousbusiness/gotosocial/internal/oauth"
"github.com/superseriousbusiness/gotosocial/internal/processing"
"github.com/superseriousbusiness/gotosocial/internal/typeutils"
@ -25,13 +26,14 @@ import (
type AccountStandardTestSuite struct {
// standard suite interfaces
suite.Suite
db db.DB
tc typeutils.TypeConverter
storage *kv.KVStore
federator federation.Federator
processor processing.Processor
emailSender email.Sender
sentEmails map[string]string
db db.DB
tc typeutils.TypeConverter
storage *kv.KVStore
mediaManager media.Manager
federator federation.Federator
processor processing.Processor
emailSender email.Sender
sentEmails map[string]string
// standard suite models
testTokens map[string]*gtsmodel.Token
@ -61,10 +63,11 @@ func (suite *AccountStandardTestSuite) SetupTest() {
suite.db = testrig.NewTestDB()
suite.storage = testrig.NewTestStorage()
testrig.InitTestLog()
suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db), suite.storage)
suite.mediaManager = testrig.NewTestMediaManager(suite.db, suite.storage)
suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db), suite.storage, suite.mediaManager)
suite.sentEmails = make(map[string]string)
suite.emailSender = testrig.NewEmailSender("../../../../web/template/", suite.sentEmails)
suite.processor = testrig.NewTestProcessor(suite.db, suite.storage, suite.federator, suite.emailSender)
suite.processor = testrig.NewTestProcessor(suite.db, suite.storage, suite.federator, suite.emailSender, suite.mediaManager)
suite.accountModule = account.New(suite.processor).(*account.Module)
testrig.StandardDBSetup(suite.db, nil)
testrig.StandardStorageSetup(suite.storage, "../../../../testrig/media")

2
internal/api/client/account/accountverify_test.go
View file

@ -42,7 +42,7 @@ func (suite *AccountVerifyTestSuite) TestAccountVerifyGet() {
// set up the request
recorder := httptest.NewRecorder()
ctx := suite.newContext(recorder, http.MethodPatch, nil, account.UpdateCredentialsPath, "")
ctx := suite.newContext(recorder, http.MethodGet, nil, account.VerifyPath, "")
// call the handler
suite.accountModule.AccountVerifyGETHandler(ctx)

2
internal/api/client/admin/admin.go
View file

@ -58,7 +58,7 @@ func New(processor processing.Processor) api.ClientModule {
// Route attaches all routes from this module to the given router
func (m *Module) Route(r router.Router) error {
r.AttachHandler(http.MethodPost, EmojiPath, m.emojiCreatePOSTHandler)
r.AttachHandler(http.MethodPost, EmojiPath, m.EmojiCreatePOSTHandler)
r.AttachHandler(http.MethodPost, DomainBlocksPath, m.DomainBlocksPOSTHandler)
r.AttachHandler(http.MethodGet, DomainBlocksPath, m.DomainBlocksGETHandler)
r.AttachHandler(http.MethodGet, DomainBlocksPathWithID, m.DomainBlockGETHandler)

123
internal/api/client/admin/admin_test.go Normal file
View file

@ -0,0 +1,123 @@
/*
GoToSocial
Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package admin_test
import (
"bytes"
"fmt"
"net/http"
"net/http/httptest"
"codeberg.org/gruf/go-store/kv"
"github.com/gin-gonic/gin"
"github.com/spf13/viper"
"github.com/stretchr/testify/suite"
"github.com/superseriousbusiness/gotosocial/internal/api/client/admin"
"github.com/superseriousbusiness/gotosocial/internal/config"
"github.com/superseriousbusiness/gotosocial/internal/db"
"github.com/superseriousbusiness/gotosocial/internal/email"
"github.com/superseriousbusiness/gotosocial/internal/federation"
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
"github.com/superseriousbusiness/gotosocial/internal/media"
"github.com/superseriousbusiness/gotosocial/internal/oauth"
"github.com/superseriousbusiness/gotosocial/internal/processing"
"github.com/superseriousbusiness/gotosocial/internal/typeutils"
"github.com/superseriousbusiness/gotosocial/testrig"
)
type AdminStandardTestSuite struct {
// standard suite interfaces
suite.Suite
db db.DB
tc typeutils.TypeConverter
storage *kv.KVStore
mediaManager media.Manager
federator federation.Federator
processor processing.Processor
emailSender email.Sender
sentEmails map[string]string
// standard suite models
testTokens map[string]*gtsmodel.Token
testClients map[string]*gtsmodel.Client
testApplications map[string]*gtsmodel.Application
testUsers map[string]*gtsmodel.User
testAccounts map[string]*gtsmodel.Account
testAttachments map[string]*gtsmodel.MediaAttachment
testStatuses map[string]*gtsmodel.Status
// module being tested
adminModule *admin.Module
}
func (suite *AdminStandardTestSuite) SetupSuite() {
suite.testTokens = testrig.NewTestTokens()
suite.testClients = testrig.NewTestClients()
suite.testApplications = testrig.NewTestApplications()
suite.testUsers = testrig.NewTestUsers()
suite.testAccounts = testrig.NewTestAccounts()
suite.testAttachments = testrig.NewTestAttachments()
suite.testStatuses = testrig.NewTestStatuses()
}
func (suite *AdminStandardTestSuite) SetupTest() {
testrig.InitTestConfig()
testrig.InitTestLog()
suite.db = testrig.NewTestDB()
suite.storage = testrig.NewTestStorage()
suite.mediaManager = testrig.NewTestMediaManager(suite.db, suite.storage)
suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db), suite.storage, suite.mediaManager)
suite.sentEmails = make(map[string]string)
suite.emailSender = testrig.NewEmailSender("../../../../web/template/", suite.sentEmails)
suite.processor = testrig.NewTestProcessor(suite.db, suite.storage, suite.federator, suite.emailSender, suite.mediaManager)
suite.adminModule = admin.New(suite.processor).(*admin.Module)
testrig.StandardDBSetup(suite.db, nil)
testrig.StandardStorageSetup(suite.storage, "../../../../testrig/media")
}
func (suite *AdminStandardTestSuite) TearDownTest() {
testrig.StandardDBTeardown(suite.db)
testrig.StandardStorageTeardown(suite.storage)
}
func (suite *AdminStandardTestSuite) newContext(recorder *httptest.ResponseRecorder, requestMethod string, requestBody []byte, requestPath string, bodyContentType string) *gin.Context {
ctx, _ := gin.CreateTestContext(recorder)
ctx.Set(oauth.SessionAuthorizedAccount, suite.testAccounts["admin_account"])
ctx.Set(oauth.SessionAuthorizedToken, oauth.DBTokenToToken(suite.testTokens["admin_account"]))
ctx.Set(oauth.SessionAuthorizedApplication, suite.testApplications["admin_account"])
ctx.Set(oauth.SessionAuthorizedUser, suite.testUsers["admin_account"])
protocol := viper.GetString(config.Keys.Protocol)
host := viper.GetString(config.Keys.Host)
baseURI := fmt.Sprintf("%s://%s", protocol, host)
requestURI := fmt.Sprintf("%s/%s", baseURI, requestPath)
ctx.Request = httptest.NewRequest(http.MethodPatch, requestURI, bytes.NewReader(requestBody)) // the endpoint we're hitting
if bodyContentType != "" {
ctx.Request.Header.Set("Content-Type", bodyContentType)
}
ctx.Request.Header.Set("accept", "application/json")
return ctx
}

20
internal/api/client/admin/emojicreate.go
View file

@ -27,12 +27,11 @@ import (
"github.com/sirupsen/logrus"
"github.com/superseriousbusiness/gotosocial/internal/api"
"github.com/superseriousbusiness/gotosocial/internal/api/model"
"github.com/superseriousbusiness/gotosocial/internal/media"
"github.com/superseriousbusiness/gotosocial/internal/oauth"
"github.com/superseriousbusiness/gotosocial/internal/validate"
)
// emojiCreateRequest swagger:operation POST /api/v1/admin/custom_emojis emojiCreate
// EmojiCreatePOSTHandler swagger:operation POST /api/v1/admin/custom_emojis emojiCreate
//
// Upload and create a new instance emoji.
//
@ -74,7 +73,9 @@ import (
// description: forbidden
// '400':
// description: bad request
func (m *Module) emojiCreatePOSTHandler(c *gin.Context) {
// '409':
// description: conflict -- domain/shortcode combo for emoji already exists
func (m *Module) EmojiCreatePOSTHandler(c *gin.Context) {
l := logrus.WithFields(logrus.Fields{
"func": "emojiCreatePOSTHandler",
"request_uri": c.Request.RequestURI,
@ -117,10 +118,10 @@ func (m *Module) emojiCreatePOSTHandler(c *gin.Context) {
return
}
apiEmoji, err := m.processor.AdminEmojiCreate(c.Request.Context(), authed, form)
if err != nil {
l.Debugf("error creating emoji: %s", err)
c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
apiEmoji, errWithCode := m.processor.AdminEmojiCreate(c.Request.Context(), authed, form)
if errWithCode != nil {
l.Debugf("error creating emoji: %s", errWithCode.Error())
c.JSON(errWithCode.Code(), gin.H{"error": errWithCode.Safe()})
return
}
@ -133,10 +134,5 @@ func validateCreateEmoji(form *model.EmojiCreateRequest) error {
return errors.New("no emoji given")
}
// a very superficial check to see if the media size limit is exceeded
if form.Image.Size > media.EmojiMaxBytes {
return fmt.Errorf("file size limit exceeded: limit is %d bytes but emoji was %d bytes", media.EmojiMaxBytes, form.Image.Size)
}
return validate.EmojiShortcode(form.Shortcode)
}

128
internal/api/client/admin/emojicreate_test.go Normal file
View file

@ -0,0 +1,128 @@
package admin_test
import (
"context"
"encoding/json"
"io/ioutil"
"net/http"
"net/http/httptest"
"testing"
"github.com/stretchr/testify/suite"
"github.com/superseriousbusiness/gotosocial/internal/api/client/admin"
apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model"
"github.com/superseriousbusiness/gotosocial/internal/db"
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
"github.com/superseriousbusiness/gotosocial/testrig"
)
type EmojiCreateTestSuite struct {
AdminStandardTestSuite
}
func (suite *EmojiCreateTestSuite) TestEmojiCreate() {
// set up the request
requestBody, w, err := testrig.CreateMultipartFormData(
"image", "../../../../testrig/media/rainbow-original.png",
map[string]string{
"shortcode": "new_emoji",
})
if err != nil {
panic(err)
}
bodyBytes := requestBody.Bytes()
recorder := httptest.NewRecorder()
ctx := suite.newContext(recorder, http.MethodPost, bodyBytes, admin.EmojiPath, w.FormDataContentType())
// call the handler
suite.adminModule.EmojiCreatePOSTHandler(ctx)
// 1. we should have OK because our request was valid
suite.Equal(http.StatusOK, recorder.Code)
// 2. we should have no error message in the result body
result := recorder.Result()
defer result.Body.Close()
// check the response
b, err := ioutil.ReadAll(result.Body)
suite.NoError(err)
suite.NotEmpty(b)
// response should be an api model emoji
apiEmoji := &apimodel.Emoji{}
err = json.Unmarshal(b, apiEmoji)
suite.NoError(err)
// appropriate fields should be set
suite.Equal("new_emoji", apiEmoji.Shortcode)
suite.NotEmpty(apiEmoji.URL)
suite.NotEmpty(apiEmoji.StaticURL)
suite.True(apiEmoji.VisibleInPicker)
// emoji should be in the db
dbEmoji := &gtsmodel.Emoji{}
err = suite.db.GetWhere(context.Background(), []db.Where{{Key: "shortcode", Value: "new_emoji"}}, dbEmoji)
suite.NoError(err)
// check fields on the emoji
suite.NotEmpty(dbEmoji.ID)
suite.Equal("new_emoji", dbEmoji.Shortcode)
suite.Empty(dbEmoji.Domain)
suite.Empty(dbEmoji.ImageRemoteURL)
suite.Empty(dbEmoji.ImageStaticRemoteURL)
suite.Equal(apiEmoji.URL, dbEmoji.ImageURL)
suite.Equal(apiEmoji.StaticURL, dbEmoji.ImageStaticURL)
suite.NotEmpty(dbEmoji.ImagePath)
suite.NotEmpty(dbEmoji.ImageStaticPath)
suite.Equal("image/png", dbEmoji.ImageContentType)
suite.Equal("image/png", dbEmoji.ImageStaticContentType)
suite.Equal(36702, dbEmoji.ImageFileSize)
suite.Equal(10413, dbEmoji.ImageStaticFileSize)
suite.False(dbEmoji.Disabled)
suite.NotEmpty(dbEmoji.URI)
suite.True(dbEmoji.VisibleInPicker)
suite.Empty(dbEmoji.CategoryID)
// emoji should be in storage
emojiBytes, err := suite.storage.Get(dbEmoji.ImagePath)
suite.NoError(err)
suite.Len(emojiBytes, dbEmoji.ImageFileSize)
emojiStaticBytes, err := suite.storage.Get(dbEmoji.ImageStaticPath)
suite.NoError(err)
suite.Len(emojiStaticBytes, dbEmoji.ImageStaticFileSize)
}
func (suite *EmojiCreateTestSuite) TestEmojiCreateAlreadyExists() {
// set up the request -- use a shortcode that already exists for an emoji in the database
requestBody, w, err := testrig.CreateMultipartFormData(
"image", "../../../../testrig/media/rainbow-original.png",
map[string]string{
"shortcode": "rainbow",
})
if err != nil {
panic(err)
}
bodyBytes := requestBody.Bytes()
recorder := httptest.NewRecorder()
ctx := suite.newContext(recorder, http.MethodPost, bodyBytes, admin.EmojiPath, w.FormDataContentType())
// call the handler
suite.adminModule.EmojiCreatePOSTHandler(ctx)
suite.Equal(http.StatusConflict, recorder.Code)
result := recorder.Result()
defer result.Body.Close()
// check the response
b, err := ioutil.ReadAll(result.Body)
suite.NoError(err)
suite.NotEmpty(b)
suite.Equal(`{"error":"conflict: emoji with shortcode rainbow already exists"}`, string(b))
}
func TestEmojiCreateTestSuite(t *testing.T) {
suite.Run(t, &EmojiCreateTestSuite{})
}

8
internal/api/client/fileserver/servefile_test.go
View file

@ -51,7 +51,7 @@ type ServeFileTestSuite struct {
federator federation.Federator
tc typeutils.TypeConverter
processor processing.Processor
mediaHandler media.Handler
mediaManager media.Manager
oauthServer oauth.Server
emailSender email.Sender
@ -77,12 +77,12 @@ func (suite *ServeFileTestSuite) SetupSuite() {
testrig.InitTestLog()
suite.db = testrig.NewTestDB()
suite.storage = testrig.NewTestStorage()
suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db), suite.storage)
suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db), suite.storage, testrig.NewTestMediaManager(suite.db, suite.storage))
suite.emailSender = testrig.NewEmailSender("../../../../web/template/", nil)
suite.processor = testrig.NewTestProcessor(suite.db, suite.storage, suite.federator, suite.emailSender)
suite.processor = testrig.NewTestProcessor(suite.db, suite.storage, suite.federator, suite.emailSender, testrig.NewTestMediaManager(suite.db, suite.storage))
suite.tc = testrig.NewTestTypeConverter(suite.db)
suite.mediaHandler = testrig.NewTestMediaHandler(suite.db, suite.storage)
suite.mediaManager = testrig.NewTestMediaManager(suite.db, suite.storage)
suite.oauthServer = testrig.NewTestOauthServer(suite.db)
// setup module being tested

17
internal/api/client/followrequest/followrequest_test.go
View file

@ -33,6 +33,7 @@ import (
"github.com/superseriousbusiness/gotosocial/internal/email"
"github.com/superseriousbusiness/gotosocial/internal/federation"
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
"github.com/superseriousbusiness/gotosocial/internal/media"
"github.com/superseriousbusiness/gotosocial/internal/oauth"
"github.com/superseriousbusiness/gotosocial/internal/processing"
"github.com/superseriousbusiness/gotosocial/testrig"
@ -40,11 +41,12 @@ import (
type FollowRequestStandardTestSuite struct {
suite.Suite
db db.DB
storage *kv.KVStore
federator federation.Federator
processor processing.Processor
emailSender email.Sender
db db.DB
storage *kv.KVStore
mediaManager media.Manager
federator federation.Federator
processor processing.Processor
emailSender email.Sender
// standard suite models
testTokens map[string]*gtsmodel.Token
@ -74,9 +76,10 @@ func (suite *FollowRequestStandardTestSuite) SetupTest() {
testrig.InitTestLog()
suite.db = testrig.NewTestDB()
suite.storage = testrig.NewTestStorage()
suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db), suite.storage)
suite.mediaManager = testrig.NewTestMediaManager(suite.db, suite.storage)
suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db), suite.storage, suite.mediaManager)
suite.emailSender = testrig.NewEmailSender("../../../../web/template/", nil)
suite.processor = testrig.NewTestProcessor(suite.db, suite.storage, suite.federator, suite.emailSender)
suite.processor = testrig.NewTestProcessor(suite.db, suite.storage, suite.federator, suite.emailSender, suite.mediaManager)
suite.followRequestModule = followrequest.New(suite.processor).(*followrequest.Module)
testrig.StandardDBSetup(suite.db, nil)
testrig.StandardStorageSetup(suite.storage, "../../../../testrig/media")

8
internal/api/client/media/mediacreate_test.go
View file

@ -54,9 +54,9 @@ type MediaCreateTestSuite struct {
suite.Suite
db db.DB
storage *kv.KVStore
mediaManager media.Manager
federator federation.Federator
tc typeutils.TypeConverter
mediaHandler media.Handler
oauthServer oauth.Server
emailSender email.Sender
processor processing.Processor
@ -84,11 +84,11 @@ func (suite *MediaCreateTestSuite) SetupSuite() {
suite.db = testrig.NewTestDB()
suite.storage = testrig.NewTestStorage()
suite.tc = testrig.NewTestTypeConverter(suite.db)
suite.mediaHandler = testrig.NewTestMediaHandler(suite.db, suite.storage)
suite.mediaManager = testrig.NewTestMediaManager(suite.db, suite.storage)
suite.oauthServer = testrig.NewTestOauthServer(suite.db)
suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db), suite.storage)
suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db), suite.storage, suite.mediaManager)
suite.emailSender = testrig.NewEmailSender("../../../../web/template/", nil)
suite.processor = testrig.NewTestProcessor(suite.db, suite.storage, suite.federator, suite.emailSender)
suite.processor = testrig.NewTestProcessor(suite.db, suite.storage, suite.federator, suite.emailSender, suite.mediaManager)
// setup module being tested
suite.mediaModule = mediamodule.New(suite.processor).(*mediamodule.Module)

8
internal/api/client/media/mediaupdate_test.go
View file

@ -54,7 +54,7 @@ type MediaUpdateTestSuite struct {
storage *kv.KVStore
federator federation.Federator
tc typeutils.TypeConverter
mediaHandler media.Handler
mediaManager media.Manager
oauthServer oauth.Server
emailSender email.Sender
processor processing.Processor
@ -82,11 +82,11 @@ func (suite *MediaUpdateTestSuite) SetupSuite() {
suite.db = testrig.NewTestDB()
suite.storage = testrig.NewTestStorage()
suite.tc = testrig.NewTestTypeConverter(suite.db)
suite.mediaHandler = testrig.NewTestMediaHandler(suite.db, suite.storage)
suite.mediaManager = testrig.NewTestMediaManager(suite.db, suite.storage)
suite.oauthServer = testrig.NewTestOauthServer(suite.db)
suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db), suite.storage)
suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db), suite.storage, suite.mediaManager)
suite.emailSender = testrig.NewEmailSender("../../../../web/template/", nil)
suite.processor = testrig.NewTestProcessor(suite.db, suite.storage, suite.federator, suite.emailSender)
suite.processor = testrig.NewTestProcessor(suite.db, suite.storage, suite.federator, suite.emailSender, suite.mediaManager)
// setup module being tested
suite.mediaModule = mediamodule.New(suite.processor).(*mediamodule.Module)

19
internal/api/client/status/status_test.go
View file

@ -26,6 +26,7 @@ import (
"github.com/superseriousbusiness/gotosocial/internal/email"
"github.com/superseriousbusiness/gotosocial/internal/federation"
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
"github.com/superseriousbusiness/gotosocial/internal/media"
"github.com/superseriousbusiness/gotosocial/internal/processing"
"github.com/superseriousbusiness/gotosocial/internal/typeutils"
"github.com/superseriousbusiness/gotosocial/testrig"
@ -34,12 +35,13 @@ import (
type StatusStandardTestSuite struct {
// standard suite interfaces
suite.Suite
db db.DB
tc typeutils.TypeConverter
federator federation.Federator
emailSender email.Sender
processor processing.Processor
storage *kv.KVStore
db db.DB
tc typeutils.TypeConverter
mediaManager media.Manager
federator federation.Federator
emailSender email.Sender
processor processing.Processor
storage *kv.KVStore
// standard suite models
testTokens map[string]*gtsmodel.Token
@ -70,9 +72,10 @@ func (suite *StatusStandardTestSuite) SetupTest() {
suite.db = testrig.NewTestDB()
suite.tc = testrig.NewTestTypeConverter(suite.db)
suite.storage = testrig.NewTestStorage()
suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db), suite.storage)
suite.mediaManager = testrig.NewTestMediaManager(suite.db, suite.storage)
suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db), suite.storage, suite.mediaManager)
suite.emailSender = testrig.NewEmailSender("../../../../web/template/", nil)
suite.processor = testrig.NewTestProcessor(suite.db, suite.storage, suite.federator, suite.emailSender)
suite.processor = testrig.NewTestProcessor(suite.db, suite.storage, suite.federator, suite.emailSender, suite.mediaManager)
suite.statusModule = status.New(suite.processor).(*status.Module)
testrig.StandardDBSetup(suite.db, nil)
testrig.StandardStorageSetup(suite.storage, "../../../../testrig/media")

19
internal/api/client/user/user_test.go
View file

@ -26,6 +26,7 @@ import (
"github.com/superseriousbusiness/gotosocial/internal/email"
"github.com/superseriousbusiness/gotosocial/internal/federation"
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
"github.com/superseriousbusiness/gotosocial/internal/media"
"github.com/superseriousbusiness/gotosocial/internal/processing"
"github.com/superseriousbusiness/gotosocial/internal/typeutils"
"github.com/superseriousbusiness/gotosocial/testrig"
@ -33,12 +34,13 @@ import (
type UserStandardTestSuite struct {
suite.Suite
db db.DB
tc typeutils.TypeConverter
federator federation.Federator
emailSender email.Sender
processor processing.Processor
storage *kv.KVStore
db db.DB
tc typeutils.TypeConverter
mediaManager media.Manager
federator federation.Federator
emailSender email.Sender
processor processing.Processor
storage *kv.KVStore
testTokens map[string]*gtsmodel.Token
testClients map[string]*gtsmodel.Client
@ -62,10 +64,11 @@ func (suite *UserStandardTestSuite) SetupTest() {
suite.db = testrig.NewTestDB()
suite.storage = testrig.NewTestStorage()
suite.tc = testrig.NewTestTypeConverter(suite.db)
suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db), suite.storage)
suite.mediaManager = testrig.NewTestMediaManager(suite.db, suite.storage)
suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db), suite.storage, suite.mediaManager)
suite.sentEmails = make(map[string]string)
suite.emailSender = testrig.NewEmailSender("../../../../web/template/", suite.sentEmails)
suite.processor = testrig.NewTestProcessor(suite.db, suite.storage, suite.federator, suite.emailSender)
suite.processor = testrig.NewTestProcessor(suite.db, suite.storage, suite.federator, suite.emailSender, suite.mediaManager)
suite.userModule = user.New(suite.processor).(*user.Module)
testrig.StandardDBSetup(suite.db, suite.testAccounts)
testrig.StandardStorageSetup(suite.storage, "../../../../testrig/media")

16
internal/api/s2s/user/inboxpost_test.go
View file

@ -84,9 +84,9 @@ func (suite *InboxPostTestSuite) TestPostBlock() {
body := bytes.NewReader(bodyJson)
tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db)
federator := testrig.NewTestFederator(suite.db, tc, suite.storage)
federator := testrig.NewTestFederator(suite.db, tc, suite.storage, suite.mediaManager)
emailSender := testrig.NewEmailSender("../../../../web/template/", nil)
processor := testrig.NewTestProcessor(suite.db, suite.storage, federator, emailSender)
processor := testrig.NewTestProcessor(suite.db, suite.storage, federator, emailSender, suite.mediaManager)
userModule := user.New(processor).(*user.Module)
// setup request
@ -184,9 +184,9 @@ func (suite *InboxPostTestSuite) TestPostUnblock() {
body := bytes.NewReader(bodyJson)
tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db)
federator := testrig.NewTestFederator(suite.db, tc, suite.storage)
federator := testrig.NewTestFederator(suite.db, tc, suite.storage, suite.mediaManager)
emailSender := testrig.NewEmailSender("../../../../web/template/", nil)
processor := testrig.NewTestProcessor(suite.db, suite.storage, federator, emailSender)
processor := testrig.NewTestProcessor(suite.db, suite.storage, federator, emailSender, suite.mediaManager)
userModule := user.New(processor).(*user.Module)
// setup request
@ -274,9 +274,9 @@ func (suite *InboxPostTestSuite) TestPostUpdate() {
body := bytes.NewReader(bodyJson)
tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db)
federator := testrig.NewTestFederator(suite.db, tc, suite.storage)
federator := testrig.NewTestFederator(suite.db, tc, suite.storage, suite.mediaManager)
emailSender := testrig.NewEmailSender("../../../../web/template/", nil)
processor := testrig.NewTestProcessor(suite.db, suite.storage, federator, emailSender)
processor := testrig.NewTestProcessor(suite.db, suite.storage, federator, emailSender, suite.mediaManager)
userModule := user.New(processor).(*user.Module)
// setup request
@ -393,9 +393,9 @@ func (suite *InboxPostTestSuite) TestPostDelete() {
body := bytes.NewReader(bodyJson)
tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db)
federator := testrig.NewTestFederator(suite.db, tc, suite.storage)
federator := testrig.NewTestFederator(suite.db, tc, suite.storage, suite.mediaManager)
emailSender := testrig.NewEmailSender("../../../../web/template/", nil)
processor := testrig.NewTestProcessor(suite.db, suite.storage, federator, emailSender)
processor := testrig.NewTestProcessor(suite.db, suite.storage, federator, emailSender, suite.mediaManager)
err = processor.Start(context.Background())
suite.NoError(err)
userModule := user.New(processor).(*user.Module)

12
internal/api/s2s/user/outboxget_test.go
View file

@ -45,9 +45,9 @@ func (suite *OutboxGetTestSuite) TestGetOutbox() {
targetAccount := suite.testAccounts["local_account_1"]
tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db)
federator := testrig.NewTestFederator(suite.db, tc, suite.storage)
federator := testrig.NewTestFederator(suite.db, tc, suite.storage, suite.mediaManager)
emailSender := testrig.NewEmailSender("../../../../web/template/", nil)
processor := testrig.NewTestProcessor(suite.db, suite.storage, federator, emailSender)
processor := testrig.NewTestProcessor(suite.db, suite.storage, federator, emailSender, suite.mediaManager)
userModule := user.New(processor).(*user.Module)
// setup request
@ -100,9 +100,9 @@ func (suite *OutboxGetTestSuite) TestGetOutboxFirstPage() {
targetAccount := suite.testAccounts["local_account_1"]
tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db)
federator := testrig.NewTestFederator(suite.db, tc, suite.storage)
federator := testrig.NewTestFederator(suite.db, tc, suite.storage, suite.mediaManager)
emailSender := testrig.NewEmailSender("../../../../web/template/", nil)
processor := testrig.NewTestProcessor(suite.db, suite.storage, federator, emailSender)
processor := testrig.NewTestProcessor(suite.db, suite.storage, federator, emailSender, suite.mediaManager)
userModule := user.New(processor).(*user.Module)
// setup request
@ -155,9 +155,9 @@ func (suite *OutboxGetTestSuite) TestGetOutboxNextPage() {
targetAccount := suite.testAccounts["local_account_1"]
tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db)
federator := testrig.NewTestFederator(suite.db, tc, suite.storage)
federator := testrig.NewTestFederator(suite.db, tc, suite.storage, suite.mediaManager)
emailSender := testrig.NewEmailSender("../../../../web/template/", nil)
processor := testrig.NewTestProcessor(suite.db, suite.storage, federator, emailSender)
processor := testrig.NewTestProcessor(suite.db, suite.storage, federator, emailSender, suite.mediaManager)
userModule := user.New(processor).(*user.Module)
// setup request

12
internal/api/s2s/user/repliesget_test.go
View file

@ -48,9 +48,9 @@ func (suite *RepliesGetTestSuite) TestGetReplies() {
targetStatus := suite.testStatuses["local_account_1_status_1"]
tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db)
federator := testrig.NewTestFederator(suite.db, tc, suite.storage)
federator := testrig.NewTestFederator(suite.db, tc, suite.storage, suite.mediaManager)
emailSender := testrig.NewEmailSender("../../../../web/template/", nil)
processor := testrig.NewTestProcessor(suite.db, suite.storage, federator, emailSender)
processor := testrig.NewTestProcessor(suite.db, suite.storage, federator, emailSender, suite.mediaManager)
userModule := user.New(processor).(*user.Module)
// setup request
@ -109,9 +109,9 @@ func (suite *RepliesGetTestSuite) TestGetRepliesNext() {
targetStatus := suite.testStatuses["local_account_1_status_1"]
tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db)
federator := testrig.NewTestFederator(suite.db, tc, suite.storage)
federator := testrig.NewTestFederator(suite.db, tc, suite.storage, suite.mediaManager)
emailSender := testrig.NewEmailSender("../../../../web/template/", nil)
processor := testrig.NewTestProcessor(suite.db, suite.storage, federator, emailSender)
processor := testrig.NewTestProcessor(suite.db, suite.storage, federator, emailSender, suite.mediaManager)
userModule := user.New(processor).(*user.Module)
// setup request
@ -173,9 +173,9 @@ func (suite *RepliesGetTestSuite) TestGetRepliesLast() {
targetStatus := suite.testStatuses["local_account_1_status_1"]
tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db)
federator := testrig.NewTestFederator(suite.db, tc, suite.storage)
federator := testrig.NewTestFederator(suite.db, tc, suite.storage, suite.mediaManager)
emailSender := testrig.NewEmailSender("../../../../web/template/", nil)
processor := testrig.NewTestProcessor(suite.db, suite.storage, federator, emailSender)
processor := testrig.NewTestProcessor(suite.db, suite.storage, federator, emailSender, suite.mediaManager)
userModule := user.New(processor).(*user.Module)
// setup request

7
internal/api/s2s/user/user_test.go
View file

@ -27,6 +27,7 @@ import (
"github.com/superseriousbusiness/gotosocial/internal/email"
"github.com/superseriousbusiness/gotosocial/internal/federation"
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
"github.com/superseriousbusiness/gotosocial/internal/media"
"github.com/superseriousbusiness/gotosocial/internal/oauth"
"github.com/superseriousbusiness/gotosocial/internal/processing"
"github.com/superseriousbusiness/gotosocial/internal/typeutils"
@ -38,6 +39,7 @@ type UserStandardTestSuite struct {
suite.Suite
db db.DB
tc typeutils.TypeConverter
mediaManager media.Manager
federator federation.Federator
emailSender email.Sender
processor processing.Processor
@ -77,9 +79,10 @@ func (suite *UserStandardTestSuite) SetupTest() {
suite.db = testrig.NewTestDB()
suite.tc = testrig.NewTestTypeConverter(suite.db)
suite.storage = testrig.NewTestStorage()
suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db), suite.storage)
suite.mediaManager = testrig.NewTestMediaManager(suite.db, suite.storage)
suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db), suite.storage, suite.mediaManager)
suite.emailSender = testrig.NewEmailSender("../../../../web/template/", nil)
suite.processor = testrig.NewTestProcessor(suite.db, suite.storage, suite.federator, suite.emailSender)
suite.processor = testrig.NewTestProcessor(suite.db, suite.storage, suite.federator, suite.emailSender, suite.mediaManager)
suite.userModule = user.New(suite.processor).(*user.Module)
suite.oauthServer = testrig.NewTestOauthServer(suite.db)
suite.securityModule = security.New(suite.db, suite.oauthServer).(*security.Module)

4
internal/api/s2s/user/userget_test.go
View file

@ -46,9 +46,9 @@ func (suite *UserGetTestSuite) TestGetUser() {
targetAccount := suite.testAccounts["local_account_1"]
tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db)
federator := testrig.NewTestFederator(suite.db, tc, suite.storage)
federator := testrig.NewTestFederator(suite.db, tc, suite.storage, suite.mediaManager)
emailSender := testrig.NewEmailSender("../../../../web/template/", nil)
processor := testrig.NewTestProcessor(suite.db, suite.storage, federator, emailSender)
processor := testrig.NewTestProcessor(suite.db, suite.storage, federator, emailSender, suite.mediaManager)
userModule := user.New(processor).(*user.Module)
// setup request

7
internal/api/s2s/webfinger/webfinger_test.go
View file

@ -32,6 +32,7 @@ import (
"github.com/superseriousbusiness/gotosocial/internal/email"
"github.com/superseriousbusiness/gotosocial/internal/federation"
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
"github.com/superseriousbusiness/gotosocial/internal/media"
"github.com/superseriousbusiness/gotosocial/internal/oauth"
"github.com/superseriousbusiness/gotosocial/internal/processing"
"github.com/superseriousbusiness/gotosocial/internal/typeutils"
@ -43,6 +44,7 @@ type WebfingerStandardTestSuite struct {
suite.Suite
db db.DB
tc typeutils.TypeConverter
mediaManager media.Manager
federator federation.Federator
emailSender email.Sender
processor processing.Processor
@ -80,9 +82,10 @@ func (suite *WebfingerStandardTestSuite) SetupTest() {
suite.db = testrig.NewTestDB()
suite.tc = testrig.NewTestTypeConverter(suite.db)
suite.storage = testrig.NewTestStorage()
suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db), suite.storage)
suite.mediaManager = testrig.NewTestMediaManager(suite.db, suite.storage)
suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db), suite.storage, suite.mediaManager)
suite.emailSender = testrig.NewEmailSender("../../../../web/template/", nil)
suite.processor = testrig.NewTestProcessor(suite.db, suite.storage, suite.federator, suite.emailSender)
suite.processor = testrig.NewTestProcessor(suite.db, suite.storage, suite.federator, suite.emailSender, suite.mediaManager)
suite.webfingerModule = webfinger.New(suite.processor).(*webfinger.Module)
suite.oauthServer = testrig.NewTestOauthServer(suite.db)
suite.securityModule = security.New(suite.db, suite.oauthServer).(*security.Module)

4
internal/api/s2s/webfinger/webfingerget_test.go
View file

@ -69,7 +69,7 @@ func (suite *WebfingerGetTestSuite) TestFingerUser() {
func (suite *WebfingerGetTestSuite) TestFingerUserWithDifferentAccountDomainByHost() {
viper.Set(config.Keys.Host, "gts.example.org")
viper.Set(config.Keys.AccountDomain, "example.org")
suite.processor = processing.NewProcessor(suite.tc, suite.federator, testrig.NewTestOauthServer(suite.db), testrig.NewTestMediaHandler(suite.db, suite.storage), suite.storage, suite.db, suite.emailSender)
suite.processor = processing.NewProcessor(suite.tc, suite.federator, testrig.NewTestOauthServer(suite.db), testrig.NewTestMediaManager(suite.db, suite.storage), suite.storage, suite.db, suite.emailSender)
suite.webfingerModule = webfinger.New(suite.processor).(*webfinger.Module)
targetAccount := accountDomainAccount()
@ -103,7 +103,7 @@ func (suite *WebfingerGetTestSuite) TestFingerUserWithDifferentAccountDomainByHo
func (suite *WebfingerGetTestSuite) TestFingerUserWithDifferentAccountDomainByAccountDomain() {
viper.Set(config.Keys.Host, "gts.example.org")
viper.Set(config.Keys.AccountDomain, "example.org")
suite.processor = processing.NewProcessor(suite.tc, suite.federator, testrig.NewTestOauthServer(suite.db), testrig.NewTestMediaHandler(suite.db, suite.storage), suite.storage, suite.db, suite.emailSender)
suite.processor = processing.NewProcessor(suite.tc, suite.federator, testrig.NewTestOauthServer(suite.db), testrig.NewTestMediaManager(suite.db, suite.storage), suite.storage, suite.db, suite.emailSender)
suite.webfingerModule = webfinger.New(suite.processor).(*webfinger.Module)
targetAccount := accountDomainAccount()

2
internal/db/bundb/errors.go
View file

@ -35,7 +35,7 @@ func processSQLiteError(err error) db.Error {
// Handle supplied error code:
switch sqliteErr.Code() {
case sqlite3.SQLITE_CONSTRAINT_UNIQUE:
case sqlite3.SQLITE_CONSTRAINT_UNIQUE, sqlite3.SQLITE_CONSTRAINT_PRIMARYKEY:
return db.NewErrAlreadyExists(err.Error())
default:
return err

9
internal/db/bundb/trace.go
View file

@ -20,7 +20,6 @@ package bundb
import (
"context"
"database/sql"
"time"
"github.com/sirupsen/logrus"
@ -48,13 +47,5 @@ func (q *debugQueryHook) AfterQuery(_ context.Context, event *bun.QueryEvent) {
"operation": event.Operation(),
})
if event.Err != nil && event.Err != sql.ErrNoRows {
// if there's an error the it'll be handled in the application logic,
// but we can still debug log it here alongside the query
l = l.WithField("query", event.Query)
l.Debug(event.Err)
return
}
l.Tracef("[%s] %s", dur, event.Operation())
}

8
internal/federation/dereference.go
View file

@ -26,12 +26,8 @@ import (
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
)
func (f *federator) GetRemoteAccount(ctx context.Context, username string, remoteAccountID *url.URL, refresh bool) (*gtsmodel.Account, bool, error) {
return f.dereferencer.GetRemoteAccount(ctx, username, remoteAccountID, refresh)
}
func (f *federator) EnrichRemoteAccount(ctx context.Context, username string, account *gtsmodel.Account) (*gtsmodel.Account, error) {
return f.dereferencer.EnrichRemoteAccount(ctx, username, account)
func (f *federator) GetRemoteAccount(ctx context.Context, username string, remoteAccountID *url.URL, blocking bool, refresh bool) (*gtsmodel.Account, error) {
return f.dereferencer.GetRemoteAccount(ctx, username, remoteAccountID, blocking, refresh)
}
func (f *federator) GetRemoteStatus(ctx context.Context, username string, remoteStatusID *url.URL, refresh, includeParent bool) (*gtsmodel.Status, ap.Statusable, bool, error) {

321
internal/federation/dereferencing/account.go
View file

@ -23,8 +23,11 @@ import (
"encoding/json"
"errors"
"fmt"
"io"
"net/url"
"strings"
"sync"
"time"
"github.com/sirupsen/logrus"
"github.com/superseriousbusiness/activity/streams"
@ -32,6 +35,7 @@ import (
"github.com/superseriousbusiness/gotosocial/internal/ap"
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
"github.com/superseriousbusiness/gotosocial/internal/id"
"github.com/superseriousbusiness/gotosocial/internal/media"
"github.com/superseriousbusiness/gotosocial/internal/transport"
)
@ -42,94 +46,97 @@ func instanceAccount(account *gtsmodel.Account) bool {
(account.Username == "internal.fetch" && strings.Contains(account.Note, "internal service actor"))
}
// EnrichRemoteAccount takes an account that's already been inserted into the database in a minimal form,
// and populates it with additional fields, media, etc.
//
// EnrichRemoteAccount is mostly useful for calling after an account has been initially created by
// the federatingDB's Create function, or during the federated authorization flow.
func (d *deref) EnrichRemoteAccount(ctx context.Context, username string, account *gtsmodel.Account) (*gtsmodel.Account, error) {
// if we're dealing with an instance account, we don't need to update anything
if instanceAccount(account) {
return account, nil
}
if err := d.PopulateAccountFields(ctx, account, username, false); err != nil {
return nil, err
}
updated, err := d.db.UpdateAccount(ctx, account)
if err != nil {
logrus.Errorf("EnrichRemoteAccount: error updating account: %s", err)
return account, nil
}
return updated, nil
}
// GetRemoteAccount completely dereferences a remote account, converts it to a GtS model account,
// puts it in the database, and returns it to a caller. The boolean indicates whether the account is new
// to us or not. If we haven't seen the account before, bool will be true. If we have seen the account before,
// it will be false.
// puts it in the database, and returns it to a caller.
//
// Refresh indicates whether--if the account exists in our db already--it should be refreshed by calling
// the remote instance again.
// the remote instance again. Blocking indicates whether the function should block until processing of
// the fetched account is complete.
//
// SIDE EFFECTS: remote account will be stored in the database, or updated if it already exists (and refresh is true).
func (d *deref) GetRemoteAccount(ctx context.Context, username string, remoteAccountID *url.URL, refresh bool) (*gtsmodel.Account, bool, error) {
func (d *deref) GetRemoteAccount(ctx context.Context, username string, remoteAccountID *url.URL, blocking bool, refresh bool) (*gtsmodel.Account, error) {
new := true
// check if we already have the account in our db
maybeAccount, err := d.db.GetAccountByURI(ctx, remoteAccountID.String())
// check if we already have the account in our db, and just return it unless we'd doing a refresh
remoteAccount, err := d.db.GetAccountByURI(ctx, remoteAccountID.String())
if err == nil {
// we've seen this account before so it's not new
new = false
if !refresh {
// we're not being asked to refresh, but just in case we don't have the avatar/header cached yet....
maybeAccount, err = d.EnrichRemoteAccount(ctx, username, maybeAccount)
return maybeAccount, new, err
// make sure the account fields are populated before returning:
// even if we're not doing a refresh, the caller might want to block
// until everything is loaded
changed, err := d.populateAccountFields(ctx, remoteAccount, username, refresh, blocking)
if err != nil {
return nil, fmt.Errorf("GetRemoteAccount: error populating remoteAccount fields: %s", err)
}
if changed {
updatedAccount, err := d.db.UpdateAccount(ctx, remoteAccount)
if err != nil {
return nil, fmt.Errorf("GetRemoteAccount: error updating remoteAccount: %s", err)
}
return updatedAccount, err
}
return remoteAccount, nil
}
}
accountable, err := d.dereferenceAccountable(ctx, username, remoteAccountID)
if err != nil {
return nil, new, fmt.Errorf("FullyDereferenceAccount: error dereferencing accountable: %s", err)
}
gtsAccount, err := d.typeConverter.ASRepresentationToAccount(ctx, accountable, refresh)
if err != nil {
return nil, new, fmt.Errorf("FullyDereferenceAccount: error converting accountable to account: %s", err)
}
if new {
// generate a new id since we haven't seen this account before, and do a put
// we haven't seen this account before: dereference it from remote
accountable, err := d.dereferenceAccountable(ctx, username, remoteAccountID)
if err != nil {
return nil, fmt.Errorf("GetRemoteAccount: error dereferencing accountable: %s", err)
}
newAccount, err := d.typeConverter.ASRepresentationToAccount(ctx, accountable, refresh)
if err != nil {
return nil, fmt.Errorf("GetRemoteAccount: error converting accountable to account: %s", err)
}
ulid, err := id.NewRandomULID()
if err != nil {
return nil, new, fmt.Errorf("FullyDereferenceAccount: error generating new id for account: %s", err)
return nil, fmt.Errorf("GetRemoteAccount: error generating new id for account: %s", err)
}
gtsAccount.ID = ulid
newAccount.ID = ulid
if err := d.PopulateAccountFields(ctx, gtsAccount, username, refresh); err != nil {
return nil, new, fmt.Errorf("FullyDereferenceAccount: error populating further account fields: %s", err)
if _, err := d.populateAccountFields(ctx, newAccount, username, refresh, blocking); err != nil {
return nil, fmt.Errorf("GetRemoteAccount: error populating further account fields: %s", err)
}
if err := d.db.Put(ctx, gtsAccount); err != nil {
return nil, new, fmt.Errorf("FullyDereferenceAccount: error putting new account: %s", err)
}
} else {
// take the id we already have and do an update
gtsAccount.ID = maybeAccount.ID
if err := d.PopulateAccountFields(ctx, gtsAccount, username, refresh); err != nil {
return nil, new, fmt.Errorf("FullyDereferenceAccount: error populating further account fields: %s", err)
if err := d.db.Put(ctx, newAccount); err != nil {
return nil, fmt.Errorf("GetRemoteAccount: error putting new account: %s", err)
}
gtsAccount, err = d.db.UpdateAccount(ctx, gtsAccount)
if err != nil {
return nil, false, fmt.Errorf("EnrichRemoteAccount: error updating account: %s", err)
}
return newAccount, nil
}
return gtsAccount, new, nil
// we have seen this account before, but we have to refresh it
refreshedAccountable, err := d.dereferenceAccountable(ctx, username, remoteAccountID)
if err != nil {
return nil, fmt.Errorf("GetRemoteAccount: error dereferencing refreshedAccountable: %s", err)
}
refreshedAccount, err := d.typeConverter.ASRepresentationToAccount(ctx, refreshedAccountable, refresh)
if err != nil {
return nil, fmt.Errorf("GetRemoteAccount: error converting refreshedAccountable to refreshedAccount: %s", err)
}
refreshedAccount.ID = remoteAccount.ID
changed, err := d.populateAccountFields(ctx, refreshedAccount, username, refresh, blocking)
if err != nil {
return nil, fmt.Errorf("GetRemoteAccount: error populating further refreshedAccount fields: %s", err)
}
if changed {
updatedAccount, err := d.db.UpdateAccount(ctx, refreshedAccount)
if err != nil {
return nil, fmt.Errorf("GetRemoteAccount: error updating refreshedAccount: %s", err)
}
return updatedAccount, nil
}
return refreshedAccount, nil
}
// dereferenceAccountable calls remoteAccountID with a GET request, and tries to parse whatever
@ -200,71 +207,189 @@ func (d *deref) dereferenceAccountable(ctx context.Context, username string, rem
return nil, fmt.Errorf("DereferenceAccountable: type name %s not supported", t.GetTypeName())
}
// PopulateAccountFields populates any fields on the given account that weren't populated by the initial
// populateAccountFields populates any fields on the given account that weren't populated by the initial
// dereferencing. This includes things like header and avatar etc.
func (d *deref) PopulateAccountFields(ctx context.Context, account *gtsmodel.Account, requestingUsername string, refresh bool) error {
l := logrus.WithFields(logrus.Fields{
"func": "PopulateAccountFields",
"requestingUsername": requestingUsername,
})
func (d *deref) populateAccountFields(ctx context.Context, account *gtsmodel.Account, requestingUsername string, blocking bool, refresh bool) (bool, error) {
// if we're dealing with an instance account, just bail, we don't need to do anything
if instanceAccount(account) {
return false, nil
}
accountURI, err := url.Parse(account.URI)
if err != nil {
return fmt.Errorf("PopulateAccountFields: couldn't parse account URI %s: %s", account.URI, err)
return false, fmt.Errorf("populateAccountFields: couldn't parse account URI %s: %s", account.URI, err)
}
if blocked, err := d.db.IsDomainBlocked(ctx, accountURI.Host); blocked || err != nil {
return fmt.Errorf("PopulateAccountFields: domain %s is blocked", accountURI.Host)
return false, fmt.Errorf("populateAccountFields: domain %s is blocked", accountURI.Host)
}
t, err := d.transportController.NewTransportForUsername(ctx, requestingUsername)
if err != nil {
return fmt.Errorf("PopulateAccountFields: error getting transport for user: %s", err)
return false, fmt.Errorf("populateAccountFields: error getting transport for user: %s", err)
}
// fetch the header and avatar
if err := d.fetchHeaderAndAviForAccount(ctx, account, t, refresh); err != nil {
// if this doesn't work, just skip it -- we can do it later
l.Debugf("error fetching header/avi for account: %s", err)
changed, err := d.fetchRemoteAccountMedia(ctx, account, t, refresh, blocking)
if err != nil {
return false, fmt.Errorf("populateAccountFields: error fetching header/avi for account: %s", err)
}
return nil
return changed, nil
}
// fetchHeaderAndAviForAccount fetches the header and avatar for a remote account, using a transport
// on behalf of requestingUsername.
// fetchRemoteAccountMedia fetches and stores the header and avatar for a remote account,
// using a transport on behalf of requestingUsername.
//
// The returned boolean indicates whether anything changed -- in other words, whether the
// account should be updated in the database.
//
// targetAccount's AvatarMediaAttachmentID and HeaderMediaAttachmentID will be updated as necessary.
//
// SIDE EFFECTS: remote header and avatar will be stored in local storage.
func (d *deref) fetchHeaderAndAviForAccount(ctx context.Context, targetAccount *gtsmodel.Account, t transport.Transport, refresh bool) error {
// If refresh is true, then the media will be fetched again even if it's already been fetched before.
//
// If blocking is true, then the calls to the media manager made by this function will be blocking:
// in other words, the function won't return until the header and the avatar have been fully processed.
func (d *deref) fetchRemoteAccountMedia(ctx context.Context, targetAccount *gtsmodel.Account, t transport.Transport, blocking bool, refresh bool) (bool, error) {
changed := false
accountURI, err := url.Parse(targetAccount.URI)
if err != nil {
return fmt.Errorf("fetchHeaderAndAviForAccount: couldn't parse account URI %s: %s", targetAccount.URI, err)
return changed, fmt.Errorf("fetchRemoteAccountMedia: couldn't parse account URI %s: %s", targetAccount.URI, err)
}
if blocked, err := d.db.IsDomainBlocked(ctx, accountURI.Host); blocked || err != nil {
return fmt.Errorf("fetchHeaderAndAviForAccount: domain %s is blocked", accountURI.Host)
return changed, fmt.Errorf("fetchRemoteAccountMedia: domain %s is blocked", accountURI.Host)
}
if targetAccount.AvatarRemoteURL != "" && (targetAccount.AvatarMediaAttachmentID == "" || refresh) {
a, err := d.mediaHandler.ProcessRemoteHeaderOrAvatar(ctx, t, &gtsmodel.MediaAttachment{
RemoteURL: targetAccount.AvatarRemoteURL,
Avatar: true,
}, targetAccount.ID)
if err != nil {
return fmt.Errorf("error processing avatar for user: %s", err)
var processingMedia *media.ProcessingMedia
d.dereferencingAvatarsLock.Lock() // LOCK HERE
// first check if we're already processing this media
if alreadyProcessing, ok := d.dereferencingAvatars[targetAccount.ID]; ok {
// we're already on it, no worries
processingMedia = alreadyProcessing
}
targetAccount.AvatarMediaAttachmentID = a.ID
if processingMedia == nil {
// we're not already processing it so start now
avatarIRI, err := url.Parse(targetAccount.AvatarRemoteURL)
if err != nil {
d.dereferencingAvatarsLock.Unlock()
return changed, err
}
data := func(innerCtx context.Context) (io.Reader, int, error) {
return t.DereferenceMedia(innerCtx, avatarIRI)
}
avatar := true
newProcessing, err := d.mediaManager.ProcessMedia(ctx, data, targetAccount.ID, &media.AdditionalMediaInfo{
RemoteURL: &targetAccount.AvatarRemoteURL,
Avatar: &avatar,
})
if err != nil {
d.dereferencingAvatarsLock.Unlock()
return changed, err
}
// store it in our map to indicate it's in process
d.dereferencingAvatars[targetAccount.ID] = newProcessing
processingMedia = newProcessing
}
d.dereferencingAvatarsLock.Unlock() // UNLOCK HERE
// block until loaded if required...
if blocking {
if err := lockAndLoad(ctx, d.dereferencingAvatarsLock, processingMedia, d.dereferencingAvatars, targetAccount.ID); err != nil {
return changed, err
}
} else {
// ...otherwise do it async
go func() {
dlCtx, done := context.WithDeadline(context.Background(), time.Now().Add(1*time.Minute))
if err := lockAndLoad(dlCtx, d.dereferencingAvatarsLock, processingMedia, d.dereferencingAvatars, targetAccount.ID); err != nil {
logrus.Errorf("fetchRemoteAccountMedia: error during async lock and load of avatar: %s", err)
}
done()
}()
}
targetAccount.AvatarMediaAttachmentID = processingMedia.AttachmentID()
changed = true
}
if targetAccount.HeaderRemoteURL != "" && (targetAccount.HeaderMediaAttachmentID == "" || refresh) {
a, err := d.mediaHandler.ProcessRemoteHeaderOrAvatar(ctx, t, &gtsmodel.MediaAttachment{
RemoteURL: targetAccount.HeaderRemoteURL,
Header: true,
}, targetAccount.ID)
if err != nil {
return fmt.Errorf("error processing header for user: %s", err)
var processingMedia *media.ProcessingMedia
d.dereferencingHeadersLock.Lock() // LOCK HERE
// first check if we're already processing this media
if alreadyProcessing, ok := d.dereferencingHeaders[targetAccount.ID]; ok {
// we're already on it, no worries
processingMedia = alreadyProcessing
}
targetAccount.HeaderMediaAttachmentID = a.ID
if processingMedia == nil {
// we're not already processing it so start now
headerIRI, err := url.Parse(targetAccount.HeaderRemoteURL)
if err != nil {
d.dereferencingAvatarsLock.Unlock()
return changed, err
}
data := func(innerCtx context.Context) (io.Reader, int, error) {
return t.DereferenceMedia(innerCtx, headerIRI)
}
header := true
newProcessing, err := d.mediaManager.ProcessMedia(ctx, data, targetAccount.ID, &media.AdditionalMediaInfo{
RemoteURL: &targetAccount.HeaderRemoteURL,
Header: &header,
})
if err != nil {
d.dereferencingAvatarsLock.Unlock()
return changed, err
}
// store it in our map to indicate it's in process
d.dereferencingHeaders[targetAccount.ID] = newProcessing
processingMedia = newProcessing
}
d.dereferencingHeadersLock.Unlock() // UNLOCK HERE
// block until loaded if required...
if blocking {
if err := lockAndLoad(ctx, d.dereferencingHeadersLock, processingMedia, d.dereferencingHeaders, targetAccount.ID); err != nil {
return changed, err
}
} else {
// ...otherwise do it async
go func() {
dlCtx, done := context.WithDeadline(context.Background(), time.Now().Add(1*time.Minute))
if err := lockAndLoad(dlCtx, d.dereferencingHeadersLock, processingMedia, d.dereferencingHeaders, targetAccount.ID); err != nil {
logrus.Errorf("fetchRemoteAccountMedia: error during async lock and load of header: %s", err)
}
done()
}()
}
targetAccount.HeaderMediaAttachmentID = processingMedia.AttachmentID()
changed = true
}
return nil
return changed, nil
}
func lockAndLoad(ctx context.Context, lock *sync.Mutex, processing *media.ProcessingMedia, processingMap map[string]*media.ProcessingMedia, accountID string) error {
// whatever happens, remove the in-process media from the map
defer func() {
lock.Lock()
delete(processingMap, accountID)
lock.Unlock()
}()
// try and load it
_, err := processing.LoadAttachment(ctx)
return err
}

3
internal/federation/dereferencing/account_test.go
View file

@ -35,11 +35,10 @@ func (suite *AccountTestSuite) TestDereferenceGroup() {
fetchingAccount := suite.testAccounts["local_account_1"]
groupURL := testrig.URLMustParse("https://unknown-instance.com/groups/some_group")
group, new, err := suite.dereferencer.GetRemoteAccount(context.Background(), fetchingAccount.Username, groupURL, false)
group, err := suite.dereferencer.GetRemoteAccount(context.Background(), fetchingAccount.Username, groupURL, false, false)
suite.NoError(err)
suite.NotNil(group)
suite.NotNil(group)
suite.True(new)
// group values should be set
suite.Equal("https://unknown-instance.com/groups/some_group", group.URI)

104
internal/federation/dereferencing/attachment.go
View file

@ -1,104 +0,0 @@
/*
GoToSocial
Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package dereferencing
import (
"context"
"errors"
"fmt"
"net/url"
"github.com/sirupsen/logrus"
"github.com/superseriousbusiness/gotosocial/internal/db"
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
)
func (d *deref) GetRemoteAttachment(ctx context.Context, requestingUsername string, minAttachment *gtsmodel.MediaAttachment) (*gtsmodel.MediaAttachment, error) {
if minAttachment.RemoteURL == "" {
return nil, fmt.Errorf("GetRemoteAttachment: minAttachment remote URL was empty")
}
remoteAttachmentURL := minAttachment.RemoteURL
l := logrus.WithFields(logrus.Fields{
"username": requestingUsername,
"remoteAttachmentURL": remoteAttachmentURL,
})
// return early if we already have the attachment somewhere
maybeAttachment := &gtsmodel.MediaAttachment{}
where := []db.Where{
{
Key: "remote_url",
Value: remoteAttachmentURL,
},
}
if err := d.db.GetWhere(ctx, where, maybeAttachment); err == nil {
// we already the attachment in the database
l.Debugf("GetRemoteAttachment: attachment already exists with id %s", maybeAttachment.ID)
return maybeAttachment, nil
}
a, err := d.RefreshAttachment(ctx, requestingUsername, minAttachment)
if err != nil {
return nil, fmt.Errorf("GetRemoteAttachment: error refreshing attachment: %s", err)
}
if err := d.db.Put(ctx, a); err != nil {
var alreadyExistsError *db.ErrAlreadyExists
if !errors.As(err, &alreadyExistsError) {
return nil, fmt.Errorf("GetRemoteAttachment: error inserting attachment: %s", err)
}
}
return a, nil
}
func (d *deref) RefreshAttachment(ctx context.Context, requestingUsername string, minAttachment *gtsmodel.MediaAttachment) (*gtsmodel.MediaAttachment, error) {
// it just doesn't exist or we have to refresh
if minAttachment.AccountID == "" {
return nil, fmt.Errorf("RefreshAttachment: minAttachment account ID was empty")
}
if minAttachment.File.ContentType == "" {
return nil, fmt.Errorf("RefreshAttachment: minAttachment.file.contentType was empty")
}
t, err := d.transportController.NewTransportForUsername(ctx, requestingUsername)
if err != nil {
return nil, fmt.Errorf("RefreshAttachment: error creating transport: %s", err)
}
derefURI, err := url.Parse(minAttachment.RemoteURL)
if err != nil {
return nil, err
}
attachmentBytes, err := t.DereferenceMedia(ctx, derefURI, minAttachment.File.ContentType)
if err != nil {
return nil, fmt.Errorf("RefreshAttachment: error dereferencing media: %s", err)
}
a, err := d.mediaHandler.ProcessAttachment(ctx, attachmentBytes, minAttachment)
if err != nil {
return nil, fmt.Errorf("RefreshAttachment: error processing attachment: %s", err)
}
return a, nil
}

64
internal/federation/dereferencing/dereferencer.go
View file

@ -33,42 +33,14 @@ import (
// Dereferencer wraps logic and functionality for doing dereferencing of remote accounts, statuses, etc, from federated instances.
type Dereferencer interface {
GetRemoteAccount(ctx context.Context, username string, remoteAccountID *url.URL, refresh bool) (*gtsmodel.Account, bool, error)
EnrichRemoteAccount(ctx context.Context, username string, account *gtsmodel.Account) (*gtsmodel.Account, error)
GetRemoteAccount(ctx context.Context, username string, remoteAccountID *url.URL, blocking bool, refresh bool) (*gtsmodel.Account, error)
GetRemoteStatus(ctx context.Context, username string, remoteStatusID *url.URL, refresh, includeParent bool) (*gtsmodel.Status, ap.Statusable, bool, error)
EnrichRemoteStatus(ctx context.Context, username string, status *gtsmodel.Status, includeParent bool) (*gtsmodel.Status, error)
GetRemoteInstance(ctx context.Context, username string, remoteInstanceURI *url.URL) (*gtsmodel.Instance, error)
// GetRemoteAttachment takes a minimal attachment struct and converts it into a fully fleshed out attachment, stored in the database and instance storage.
//
// The parameter minAttachment must have at least the following fields defined:
// * minAttachment.RemoteURL
// * minAttachment.AccountID
// * minAttachment.File.ContentType
//
// The returned attachment will have an ID generated for it, so no need to generate one beforehand.
// A blurhash will also be generated for the attachment.
//
// Most other fields will be preserved on the passed attachment, including:
// * minAttachment.StatusID
// * minAttachment.CreatedAt
// * minAttachment.UpdatedAt
// * minAttachment.FileMeta
// * minAttachment.AccountID
// * minAttachment.Description
// * minAttachment.ScheduledStatusID
// * minAttachment.Thumbnail.RemoteURL
// * minAttachment.Avatar
// * minAttachment.Header
//
// GetRemoteAttachment will return early if an attachment with the same value as minAttachment.RemoteURL
// is found in the database -- then that attachment will be returned and nothing else will be changed or stored.
GetRemoteAttachment(ctx context.Context, requestingUsername string, minAttachment *gtsmodel.MediaAttachment) (*gtsmodel.MediaAttachment, error)
// RefreshAttachment is like GetRemoteAttachment, but the attachment will always be dereferenced again,
// whether or not it was already stored in the database.
RefreshAttachment(ctx context.Context, requestingUsername string, minAttachment *gtsmodel.MediaAttachment) (*gtsmodel.MediaAttachment, error)
GetRemoteMedia(ctx context.Context, requestingUsername string, accountID string, remoteURL string, ai *media.AdditionalMediaInfo) (*media.ProcessingMedia, error)
DereferenceAnnounce(ctx context.Context, announce *gtsmodel.Status, requestingUsername string) error
DereferenceThread(ctx context.Context, username string, statusIRI *url.URL) error
@ -77,21 +49,29 @@ type Dereferencer interface {
}
type deref struct {
db db.DB
typeConverter typeutils.TypeConverter
transportController transport.Controller
mediaHandler media.Handler
handshakes map[string][]*url.URL
handshakeSync *sync.Mutex // mutex to lock/unlock when checking or updating the handshakes map
db db.DB
typeConverter typeutils.TypeConverter
transportController transport.Controller
mediaManager media.Manager
dereferencingAvatars map[string]*media.ProcessingMedia
dereferencingAvatarsLock *sync.Mutex
dereferencingHeaders map[string]*media.ProcessingMedia
dereferencingHeadersLock *sync.Mutex
handshakes map[string][]*url.URL
handshakeSync *sync.Mutex // mutex to lock/unlock when checking or updating the handshakes map
}
// NewDereferencer returns a Dereferencer initialized with the given parameters.
func NewDereferencer(db db.DB, typeConverter typeutils.TypeConverter, transportController transport.Controller, mediaHandler media.Handler) Dereferencer {
func NewDereferencer(db db.DB, typeConverter typeutils.TypeConverter, transportController transport.Controller, mediaManager media.Manager) Dereferencer {
return &deref{
db: db,
typeConverter: typeConverter,
transportController: transportController,
mediaHandler: mediaHandler,
handshakeSync: &sync.Mutex{},
db: db,
typeConverter: typeConverter,
transportController: transportController,
mediaManager: mediaManager,
dereferencingAvatars: make(map[string]*media.ProcessingMedia),
dereferencingAvatarsLock: &sync.Mutex{},
dereferencingHeaders: make(map[string]*media.ProcessingMedia),
dereferencingHeadersLock: &sync.Mutex{},
handshakeSync: &sync.Mutex{},
}
}

2
internal/federation/dereferencing/dereferencer_test.go
View file

@ -64,7 +64,7 @@ func (suite *DereferencerStandardTestSuite) SetupTest() {
suite.db = testrig.NewTestDB()
suite.storage = testrig.NewTestStorage()
suite.dereferencer = dereferencing.NewDereferencer(suite.db, testrig.NewTestTypeConverter(suite.db), suite.mockTransportController(), testrig.NewTestMediaHandler(suite.db, suite.storage))
suite.dereferencer = dereferencing.NewDereferencer(suite.db, testrig.NewTestTypeConverter(suite.db), suite.mockTransportController(), testrig.NewTestMediaManager(suite.db, suite.storage))
testrig.StandardDBSetup(suite.db, nil)
}

55
internal/federation/dereferencing/media.go Normal file
View file

@ -0,0 +1,55 @@
/*
GoToSocial
Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package dereferencing
import (
"context"
"fmt"
"io"
"net/url"
"github.com/superseriousbusiness/gotosocial/internal/media"
)
func (d *deref) GetRemoteMedia(ctx context.Context, requestingUsername string, accountID string, remoteURL string, ai *media.AdditionalMediaInfo) (*media.ProcessingMedia, error) {
if accountID == "" {
return nil, fmt.Errorf("GetRemoteMedia: account ID was empty")
}
t, err := d.transportController.NewTransportForUsername(ctx, requestingUsername)
if err != nil {
return nil, fmt.Errorf("GetRemoteMedia: error creating transport: %s", err)
}
derefURI, err := url.Parse(remoteURL)
if err != nil {
return nil, fmt.Errorf("GetRemoteMedia: error parsing url: %s", err)
}
dataFunc := func(innerCtx context.Context) (io.Reader, int, error) {
return t.DereferenceMedia(innerCtx, derefURI)
}
processingMedia, err := d.mediaManager.ProcessMedia(ctx, dataFunc, accountID, ai)
if err != nil {
return nil, fmt.Errorf("GetRemoteMedia: error processing attachment: %s", err)
}
return processingMedia, nil
}

90
internal/federation/dereferencing/attachment_test.go → internal/federation/dereferencing/media_test.go
View file

@ -20,17 +20,22 @@ package dereferencing_test
import (
"context"
"fmt"
"testing"
"time"
"github.com/stretchr/testify/suite"
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
"github.com/superseriousbusiness/gotosocial/internal/media"
)
type AttachmentTestSuite struct {
DereferencerStandardTestSuite
}
func (suite *AttachmentTestSuite) TestDereferenceAttachmentOK() {
func (suite *AttachmentTestSuite) TestDereferenceAttachmentBlocking() {
ctx := context.Background()
fetchingAccount := suite.testAccounts["local_account_1"]
attachmentOwner := "01FENS9F666SEQ6TYQWEEY78GM"
@ -38,19 +43,20 @@ func (suite *AttachmentTestSuite) TestDereferenceAttachmentOK() {
attachmentContentType := "image/jpeg"
attachmentURL := "https://s3-us-west-2.amazonaws.com/plushcity/media_attachments/files/106/867/380/219/163/828/original/88e8758c5f011439.jpg"
attachmentDescription := "It's a cute plushie."
attachmentBlurhash := "LwP?p=aK_4%N%MRjWXt7%hozM_a}"
minAttachment := &gtsmodel.MediaAttachment{
RemoteURL: attachmentURL,
AccountID: attachmentOwner,
StatusID: attachmentStatus,
File: gtsmodel.File{
ContentType: attachmentContentType,
},
Description: attachmentDescription,
}
attachment, err := suite.dereferencer.GetRemoteAttachment(context.Background(), fetchingAccount.Username, minAttachment)
media, err := suite.dereferencer.GetRemoteMedia(ctx, fetchingAccount.Username, attachmentOwner, attachmentURL, &media.AdditionalMediaInfo{
StatusID: &attachmentStatus,
RemoteURL: &attachmentURL,
Description: &attachmentDescription,
Blurhash: &attachmentBlurhash,
})
suite.NoError(err)
// make a blocking call to load the attachment from the in-process media
attachment, err := media.LoadAttachment(ctx)
suite.NoError(err)
suite.NotNil(attachment)
suite.Equal(attachmentOwner, attachment.AccountID)
@ -65,7 +71,7 @@ func (suite *AttachmentTestSuite) TestDereferenceAttachmentOK() {
suite.Equal(2071680, attachment.FileMeta.Original.Size)
suite.Equal(1245, attachment.FileMeta.Original.Height)
suite.Equal(1664, attachment.FileMeta.Original.Width)
suite.Equal("LwP?p=aK_4%N%MRjWXt7%hozM_a}", attachment.Blurhash)
suite.Equal(attachmentBlurhash, attachment.Blurhash)
suite.Equal(gtsmodel.ProcessingStatusProcessed, attachment.Processing)
suite.NotEmpty(attachment.File.Path)
suite.Equal(attachmentContentType, attachment.File.ContentType)
@ -91,7 +97,7 @@ func (suite *AttachmentTestSuite) TestDereferenceAttachmentOK() {
suite.Equal(2071680, dbAttachment.FileMeta.Original.Size)
suite.Equal(1245, dbAttachment.FileMeta.Original.Height)
suite.Equal(1664, dbAttachment.FileMeta.Original.Width)
suite.Equal("LwP?p=aK_4%N%MRjWXt7%hozM_a}", dbAttachment.Blurhash)
suite.Equal(attachmentBlurhash, dbAttachment.Blurhash)
suite.Equal(gtsmodel.ProcessingStatusProcessed, dbAttachment.Processing)
suite.NotEmpty(dbAttachment.File.Path)
suite.Equal(attachmentContentType, dbAttachment.File.ContentType)
@ -101,6 +107,62 @@ func (suite *AttachmentTestSuite) TestDereferenceAttachmentOK() {
suite.NotEmpty(dbAttachment.Type)
}
func (suite *AttachmentTestSuite) TestDereferenceAttachmentAsync() {
ctx := context.Background()
fetchingAccount := suite.testAccounts["local_account_1"]
attachmentOwner := "01FENS9F666SEQ6TYQWEEY78GM"
attachmentStatus := "01FENS9NTTVNEX1YZV7GB63MT8"
attachmentContentType := "image/jpeg"
attachmentURL := "https://s3-us-west-2.amazonaws.com/plushcity/media_attachments/files/106/867/380/219/163/828/original/88e8758c5f011439.jpg"
attachmentDescription := "It's a cute plushie."
attachmentBlurhash := "LwP?p=aK_4%N%MRjWXt7%hozM_a}"
processingMedia, err := suite.dereferencer.GetRemoteMedia(ctx, fetchingAccount.Username, attachmentOwner, attachmentURL, &media.AdditionalMediaInfo{
StatusID: &attachmentStatus,
RemoteURL: &attachmentURL,
Description: &attachmentDescription,
Blurhash: &attachmentBlurhash,
})
suite.NoError(err)
attachmentID := processingMedia.AttachmentID()
// wait for the media to finish processing
for finished := processingMedia.Finished(); !finished; finished = processingMedia.Finished() {
time.Sleep(10 * time.Millisecond)
fmt.Printf("\n\nnot finished yet...\n\n")
}
fmt.Printf("\n\nfinished!\n\n")
// now get the attachment from the database
attachment, err := suite.db.GetAttachmentByID(ctx, attachmentID)
suite.NoError(err)
suite.NotNil(attachment)
suite.Equal(attachmentOwner, attachment.AccountID)
suite.Equal(attachmentStatus, attachment.StatusID)
suite.Equal(attachmentURL, attachment.RemoteURL)
suite.NotEmpty(attachment.URL)
suite.NotEmpty(attachment.Blurhash)
suite.NotEmpty(attachment.ID)
suite.NotEmpty(attachment.CreatedAt)
suite.NotEmpty(attachment.UpdatedAt)
suite.Equal(1.336546184738956, attachment.FileMeta.Original.Aspect)
suite.Equal(2071680, attachment.FileMeta.Original.Size)
suite.Equal(1245, attachment.FileMeta.Original.Height)
suite.Equal(1664, attachment.FileMeta.Original.Width)
suite.Equal(attachmentBlurhash, attachment.Blurhash)
suite.Equal(gtsmodel.ProcessingStatusProcessed, attachment.Processing)
suite.NotEmpty(attachment.File.Path)
suite.Equal(attachmentContentType, attachment.File.ContentType)
suite.Equal(attachmentDescription, attachment.Description)
suite.NotEmpty(attachment.Thumbnail.Path)
suite.NotEmpty(attachment.Type)
}
func TestAttachmentTestSuite(t *testing.T) {
suite.Run(t, new(AttachmentTestSuite))
}

21
internal/federation/dereferencing/status.go
View file

@ -32,6 +32,7 @@ import (
"github.com/superseriousbusiness/gotosocial/internal/ap"
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
"github.com/superseriousbusiness/gotosocial/internal/id"
"github.com/superseriousbusiness/gotosocial/internal/media"
)
// EnrichRemoteStatus takes a status that's already been inserted into the database in a minimal form,
@ -88,7 +89,7 @@ func (d *deref) GetRemoteStatus(ctx context.Context, username string, remoteStat
}
// do this so we know we have the remote account of the status in the db
_, _, err = d.GetRemoteAccount(ctx, username, accountURI, false)
_, err = d.GetRemoteAccount(ctx, username, accountURI, true, false)
if err != nil {
return nil, statusable, new, fmt.Errorf("GetRemoteStatus: couldn't derive status author: %s", err)
}
@ -331,7 +332,7 @@ func (d *deref) populateStatusMentions(ctx context.Context, status *gtsmodel.Sta
if targetAccount == nil {
// we didn't find the account in our database already
// check if we can get the account remotely (dereference it)
if a, _, err := d.GetRemoteAccount(ctx, requestingUsername, targetAccountURI, false); err != nil {
if a, err := d.GetRemoteAccount(ctx, requestingUsername, targetAccountURI, false, false); err != nil {
errs = append(errs, err.Error())
} else {
logrus.Debugf("populateStatusMentions: got target account %s with id %s through GetRemoteAccount", targetAccountURI, a.ID)
@ -393,9 +394,21 @@ func (d *deref) populateStatusAttachments(ctx context.Context, status *gtsmodel.
a.AccountID = status.AccountID
a.StatusID = status.ID
attachment, err := d.GetRemoteAttachment(ctx, requestingUsername, a)
processingMedia, err := d.GetRemoteMedia(ctx, requestingUsername, a.AccountID, a.RemoteURL, &media.AdditionalMediaInfo{
CreatedAt: &a.CreatedAt,
StatusID: &a.StatusID,
RemoteURL: &a.RemoteURL,
Description: &a.Description,
Blurhash: &a.Blurhash,
})
if err != nil {
logrus.Errorf("populateStatusAttachments: couldn't get remote attachment %s: %s", a.RemoteURL, err)
logrus.Errorf("populateStatusAttachments: couldn't get remote media %s: %s", a.RemoteURL, err)
continue
}
attachment, err := processingMedia.LoadAttachment(ctx)
if err != nil {
logrus.Errorf("populateStatusAttachments: couldn't load remote attachment %s: %s", a.RemoteURL, err)
continue
}

2
internal/federation/federatingprotocol.go
View file

@ -153,7 +153,7 @@ func (f *federator) AuthenticatePostInbox(ctx context.Context, w http.ResponseWr
}
}
requestingAccount, _, err := f.GetRemoteAccount(ctx, username, publicKeyOwnerURI, false)
requestingAccount, err := f.GetRemoteAccount(ctx, username, publicKeyOwnerURI, false, false)
if err != nil {
return nil, false, fmt.Errorf("couldn't get requesting account %s: %s", publicKeyOwnerURI, err)
}

11
internal/federation/federator.go
View file

@ -57,8 +57,7 @@ type Federator interface {
DereferenceRemoteThread(ctx context.Context, username string, statusURI *url.URL) error
DereferenceAnnounce(ctx context.Context, announce *gtsmodel.Status, requestingUsername string) error
GetRemoteAccount(ctx context.Context, username string, remoteAccountID *url.URL, refresh bool) (*gtsmodel.Account, bool, error)
EnrichRemoteAccount(ctx context.Context, username string, account *gtsmodel.Account) (*gtsmodel.Account, error)
GetRemoteAccount(ctx context.Context, username string, remoteAccountID *url.URL, blocking bool, refresh bool) (*gtsmodel.Account, error)
GetRemoteStatus(ctx context.Context, username string, remoteStatusID *url.URL, refresh, includeParent bool) (*gtsmodel.Status, ap.Statusable, bool, error)
EnrichRemoteStatus(ctx context.Context, username string, status *gtsmodel.Status, includeParent bool) (*gtsmodel.Status, error)
@ -78,13 +77,13 @@ type federator struct {
typeConverter typeutils.TypeConverter
transportController transport.Controller
dereferencer dereferencing.Dereferencer
mediaHandler media.Handler
mediaManager media.Manager
actor pub.FederatingActor
}
// NewFederator returns a new federator
func NewFederator(db db.DB, federatingDB federatingdb.DB, transportController transport.Controller, typeConverter typeutils.TypeConverter, mediaHandler media.Handler) Federator {
dereferencer := dereferencing.NewDereferencer(db, typeConverter, transportController, mediaHandler)
func NewFederator(db db.DB, federatingDB federatingdb.DB, transportController transport.Controller, typeConverter typeutils.TypeConverter, mediaManager media.Manager) Federator {
dereferencer := dereferencing.NewDereferencer(db, typeConverter, transportController, mediaManager)
clock := &Clock{}
f := &federator{
@ -94,7 +93,7 @@ func NewFederator(db db.DB, federatingDB federatingdb.DB, transportController tr
typeConverter: typeConverter,
transportController: transportController,
dereferencer: dereferencer,
mediaHandler: mediaHandler,
mediaManager: mediaManager,
}
actor := newFederatingActor(f, f, federatingDB, clock)
f.actor = actor

4
internal/federation/federator_test.go
View file

@ -78,7 +78,7 @@ func (suite *ProtocolTestSuite) TestPostInboxRequestBodyHook() {
return nil, nil
}), suite.db)
// setup module being tested
federator := federation.NewFederator(suite.db, testrig.NewTestFederatingDB(suite.db), tc, suite.typeConverter, testrig.NewTestMediaHandler(suite.db, suite.storage))
federator := federation.NewFederator(suite.db, testrig.NewTestFederatingDB(suite.db), tc, suite.typeConverter, testrig.NewTestMediaManager(suite.db, suite.storage))
// setup request
ctx := context.Background()
@ -107,7 +107,7 @@ func (suite *ProtocolTestSuite) TestAuthenticatePostInbox() {
tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db)
// now setup module being tested, with the mock transport controller
federator := federation.NewFederator(suite.db, testrig.NewTestFederatingDB(suite.db), tc, suite.typeConverter, testrig.NewTestMediaHandler(suite.db, suite.storage))
federator := federation.NewFederator(suite.db, testrig.NewTestFederatingDB(suite.db), tc, suite.typeConverter, testrig.NewTestMediaManager(suite.db, suite.storage))
request := httptest.NewRequest(http.MethodPost, "http://localhost:8080/users/the_mighty_zork/inbox", nil)
// we need these headers for the request to be validated

26
internal/gotosocial/gotosocial.go
View file

@ -23,6 +23,7 @@ import (
"github.com/superseriousbusiness/gotosocial/internal/db"
"github.com/superseriousbusiness/gotosocial/internal/federation"
"github.com/superseriousbusiness/gotosocial/internal/media"
"github.com/superseriousbusiness/gotosocial/internal/router"
)
@ -41,19 +42,21 @@ type Server interface {
// NewServer returns a new gotosocial server, initialized with the given configuration.
// An error will be returned the caller if something goes wrong during initialization
// eg., no db or storage connection, port for router already in use, etc.
func NewServer(db db.DB, apiRouter router.Router, federator federation.Federator) (Server, error) {
func NewServer(db db.DB, apiRouter router.Router, federator federation.Federator, mediaManager media.Manager) (Server, error) {
return &gotosocial{
db: db,
apiRouter: apiRouter,
federator: federator,
db: db,
apiRouter: apiRouter,
federator: federator,
mediaManager: mediaManager,
}, nil
}
// gotosocial fulfils the gotosocial interface.
type gotosocial struct {
db db.DB
apiRouter router.Router
federator federation.Federator
db db.DB
apiRouter router.Router
federator federation.Federator
mediaManager media.Manager
}
// Start starts up the gotosocial server. If something goes wrong
@ -63,13 +66,16 @@ func (gts *gotosocial) Start(ctx context.Context) error {
return nil
}
// Stop closes down the gotosocial server, first closing the router
// then the database. If something goes wrong while stopping, an
// error will be returned.
// Stop closes down the gotosocial server, first closing the router,
// then the media manager, then the database.
// If something goes wrong while stopping, an error will be returned.
func (gts *gotosocial) Stop(ctx context.Context) error {
if err := gts.apiRouter.Stop(ctx); err != nil {
return err
}
if err := gts.mediaManager.Stop(); err != nil {
return err
}
if err := gts.db.Stop(ctx); err != nil {
return err
}

13
internal/gtserror/withcode.go
View file

@ -122,3 +122,16 @@ func NewErrorInternalError(original error, helpText ...string) WithCode {
code: http.StatusInternalServerError,
}
}
// NewErrorConflict returns an ErrorWithCode 409 with the given original error and optional help text.
func NewErrorConflict(original error, helpText ...string) WithCode {
safe := "conflict"
if helpText != nil {
safe = safe + ": " + strings.Join(helpText, ": ")
}
return withCode{
original: original,
safe: errors.New(safe),
code: http.StatusConflict,
}
}

318
internal/media/handler.go
View file

@ -1,318 +0,0 @@
/*
GoToSocial
Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package media
import (
"context"
"errors"
"fmt"
"net/url"
"strings"
"time"
"codeberg.org/gruf/go-store/kv"
"github.com/sirupsen/logrus"
"github.com/superseriousbusiness/gotosocial/internal/db"
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
"github.com/superseriousbusiness/gotosocial/internal/id"
"github.com/superseriousbusiness/gotosocial/internal/transport"
"github.com/superseriousbusiness/gotosocial/internal/uris"
)
// EmojiMaxBytes is the maximum permitted bytes of an emoji upload (50kb)
const EmojiMaxBytes = 51200
type Size string
const (
SizeSmall Size = "small" // SizeSmall is the key for small/thumbnail versions of media
SizeOriginal Size = "original" // SizeOriginal is the key for original/fullsize versions of media and emoji
SizeStatic Size = "static" // SizeStatic is the key for static (non-animated) versions of emoji
)
type Type string
const (
TypeAttachment Type = "attachment" // TypeAttachment is the key for media attachments
TypeHeader Type = "header" // TypeHeader is the key for profile header requests
TypeAvatar Type = "avatar" // TypeAvatar is the key for profile avatar requests
TypeEmoji Type = "emoji" // TypeEmoji is the key for emoji type requests
)
// Handler provides an interface for parsing, storing, and retrieving media objects like photos, videos, and gifs.
type Handler interface {
// ProcessHeaderOrAvatar takes a new header image for an account, checks it out, removes exif data from it,
// puts it in whatever storage backend we're using, sets the relevant fields in the database for the new image,
// and then returns information to the caller about the new header.
ProcessHeaderOrAvatar(ctx context.Context, attachment []byte, accountID string, mediaType Type, remoteURL string) (*gtsmodel.MediaAttachment, error)
// ProcessLocalAttachment takes a new attachment and the requesting account, checks it out, removes exif data from it,
// puts it in whatever storage backend we're using, sets the relevant fields in the database for the new media,
// and then returns information to the caller about the attachment. It's the caller's responsibility to put the returned struct
// in the database.
ProcessAttachment(ctx context.Context, attachmentBytes []byte, minAttachment *gtsmodel.MediaAttachment) (*gtsmodel.MediaAttachment, error)
// ProcessLocalEmoji takes a new emoji and a shortcode, cleans it up, puts it in storage, and creates a new
// *gts.Emoji for it, then returns it to the caller. It's the caller's responsibility to put the returned struct
// in the database.
ProcessLocalEmoji(ctx context.Context, emojiBytes []byte, shortcode string) (*gtsmodel.Emoji, error)
ProcessRemoteHeaderOrAvatar(ctx context.Context, t transport.Transport, currentAttachment *gtsmodel.MediaAttachment, accountID string) (*gtsmodel.MediaAttachment, error)
}
type mediaHandler struct {
db db.DB
storage *kv.KVStore
}
// New returns a new handler with the given db and storage
func New(database db.DB, storage *kv.KVStore) Handler {
return &mediaHandler{
db: database,
storage: storage,
}
}
/*
INTERFACE FUNCTIONS
*/
// ProcessHeaderOrAvatar takes a new header image for an account, checks it out, removes exif data from it,
// puts it in whatever storage backend we're using, sets the relevant fields in the database for the new image,
// and then returns information to the caller about the new header.
func (mh *mediaHandler) ProcessHeaderOrAvatar(ctx context.Context, attachment []byte, accountID string, mediaType Type, remoteURL string) (*gtsmodel.MediaAttachment, error) {
l := logrus.WithField("func", "SetHeaderForAccountID")
if mediaType != TypeHeader && mediaType != TypeAvatar {
return nil, errors.New("header or avatar not selected")
}
// make sure we have a type we can handle
contentType, err := parseContentType(attachment)
if err != nil {
return nil, err
}
if !SupportedImageType(contentType) {
return nil, fmt.Errorf("%s is not an accepted image type", contentType)
}
if len(attachment) == 0 {
return nil, fmt.Errorf("passed reader was of size 0")
}
l.Tracef("read %d bytes of file", len(attachment))
// process it
ma, err := mh.processHeaderOrAvi(attachment, contentType, mediaType, accountID, remoteURL)
if err != nil {
return nil, fmt.Errorf("error processing %s: %s", mediaType, err)
}
// set it in the database
if err := mh.db.SetAccountHeaderOrAvatar(ctx, ma, accountID); err != nil {
return nil, fmt.Errorf("error putting %s in database: %s", mediaType, err)
}
return ma, nil
}
// ProcessAttachment takes a new attachment and the owning account, checks it out, removes exif data from it,
// puts it in whatever storage backend we're using, sets the relevant fields in the database for the new media,
// and then returns information to the caller about the attachment.
func (mh *mediaHandler) ProcessAttachment(ctx context.Context, attachmentBytes []byte, minAttachment *gtsmodel.MediaAttachment) (*gtsmodel.MediaAttachment, error) {
contentType, err := parseContentType(attachmentBytes)
if err != nil {
return nil, err
}
minAttachment.File.ContentType = contentType
mainType := strings.Split(contentType, "/")[0]
switch mainType {
// case MIMEVideo:
// if !SupportedVideoType(contentType) {
// return nil, fmt.Errorf("video type %s not supported", contentType)
// }
// if len(attachment) == 0 {
// return nil, errors.New("video was of size 0")
// }
// return mh.processVideoAttachment(attachment, accountID, contentType, remoteURL)
case MIMEImage:
if !SupportedImageType(contentType) {
return nil, fmt.Errorf("image type %s not supported", contentType)
}
if len(attachmentBytes) == 0 {
return nil, errors.New("image was of size 0")
}
return mh.processImageAttachment(attachmentBytes, minAttachment)
default:
break
}
return nil, fmt.Errorf("content type %s not (yet) supported", contentType)
}
// ProcessLocalEmoji takes a new emoji and a shortcode, cleans it up, puts it in storage, and creates a new
// *gts.Emoji for it, then returns it to the caller. It's the caller's responsibility to put the returned struct
// in the database.
func (mh *mediaHandler) ProcessLocalEmoji(ctx context.Context, emojiBytes []byte, shortcode string) (*gtsmodel.Emoji, error) {
var clean []byte
var err error
var original *imageAndMeta
var static *imageAndMeta
// check content type of the submitted emoji and make sure it's supported by us
contentType, err := parseContentType(emojiBytes)
if err != nil {
return nil, err
}
if !supportedEmojiType(contentType) {
return nil, fmt.Errorf("content type %s not supported for emojis", contentType)
}
if len(emojiBytes) == 0 {
return nil, errors.New("emoji was of size 0")
}
if len(emojiBytes) > EmojiMaxBytes {
return nil, fmt.Errorf("emoji size %d bytes exceeded max emoji size of %d bytes", len(emojiBytes), EmojiMaxBytes)
}
// clean any exif data from png but leave gifs alone
switch contentType {
case MIMEPng:
if clean, err = purgeExif(emojiBytes); err != nil {
return nil, fmt.Errorf("error cleaning exif data: %s", err)
}
case MIMEGif:
clean = emojiBytes
default:
return nil, errors.New("media type unrecognized")
}
// unlike with other attachments we don't need to derive anything here because we don't care about the width/height etc
original = &imageAndMeta{
image: clean,
}
static, err = deriveStaticEmoji(clean, contentType)
if err != nil {
return nil, fmt.Errorf("error deriving static emoji: %s", err)
}
// since emoji aren't 'owned' by an account, but we still want to use the same pattern for serving them through the filserver,
// (ie., fileserver/ACCOUNT_ID/etc etc) we need to fetch the INSTANCE ACCOUNT from the database. That is, the account that's created
// with the same username as the instance hostname, which doesn't belong to any particular user.
instanceAccount, err := mh.db.GetInstanceAccount(ctx, "")
if err != nil {
return nil, fmt.Errorf("error fetching instance account: %s", err)
}
// the file extension (either png or gif)
extension := strings.Split(contentType, "/")[1]
// generate a ulid for the new emoji
newEmojiID, err := id.NewRandomULID()
if err != nil {
return nil, err
}
// activitypub uri for the emoji -- unrelated to actually serving the image
// will be something like https://example.org/emoji/01FPSVBK3H8N7V8XK6KGSQ86EC
emojiURI := uris.GenerateURIForEmoji(newEmojiID)
// serve url and storage path for the original emoji -- can be png or gif
emojiURL := uris.GenerateURIForAttachment(instanceAccount.ID, string(TypeEmoji), string(SizeOriginal), newEmojiID, extension)
emojiPath := fmt.Sprintf("%s/%s/%s/%s.%s", instanceAccount.ID, TypeEmoji, SizeOriginal, newEmojiID, extension)
// serve url and storage path for the static version -- will always be png
emojiStaticURL := uris.GenerateURIForAttachment(instanceAccount.ID, string(TypeEmoji), string(SizeStatic), newEmojiID, "png")
emojiStaticPath := fmt.Sprintf("%s/%s/%s/%s.png", instanceAccount.ID, TypeEmoji, SizeStatic, newEmojiID)
// Store the original emoji
if err := mh.storage.Put(emojiPath, original.image); err != nil {
return nil, fmt.Errorf("storage error: %s", err)
}
// Store the static emoji
if err := mh.storage.Put(emojiStaticPath, static.image); err != nil {
return nil, fmt.Errorf("storage error: %s", err)
}
// and finally return the new emoji data to the caller -- it's up to them what to do with it
e := &gtsmodel.Emoji{
ID: newEmojiID,
Shortcode: shortcode,
Domain: "", // empty because this is a local emoji
CreatedAt: time.Now(),
UpdatedAt: time.Now(),
ImageRemoteURL: "", // empty because this is a local emoji
ImageStaticRemoteURL: "", // empty because this is a local emoji
ImageURL: emojiURL,
ImageStaticURL: emojiStaticURL,
ImagePath: emojiPath,
ImageStaticPath: emojiStaticPath,
ImageContentType: contentType,
ImageStaticContentType: MIMEPng, // static version will always be a png
ImageFileSize: len(original.image),
ImageStaticFileSize: len(static.image),
ImageUpdatedAt: time.Now(),
Disabled: false,
URI: emojiURI,
VisibleInPicker: true,
CategoryID: "", // empty because this is a new emoji -- no category yet
}
return e, nil
}
func (mh *mediaHandler) ProcessRemoteHeaderOrAvatar(ctx context.Context, t transport.Transport, currentAttachment *gtsmodel.MediaAttachment, accountID string) (*gtsmodel.MediaAttachment, error) {
if !currentAttachment.Header && !currentAttachment.Avatar {
return nil, errors.New("provided attachment was set to neither header nor avatar")
}
if currentAttachment.Header && currentAttachment.Avatar {
return nil, errors.New("provided attachment was set to both header and avatar")
}
var headerOrAvi Type
if currentAttachment.Header {
headerOrAvi = TypeHeader
} else if currentAttachment.Avatar {
headerOrAvi = TypeAvatar
}
if currentAttachment.RemoteURL == "" {
return nil, errors.New("no remote URL on media attachment to dereference")
}
remoteIRI, err := url.Parse(currentAttachment.RemoteURL)
if err != nil {
return nil, fmt.Errorf("error parsing attachment url %s: %s", currentAttachment.RemoteURL, err)
}
// for content type, we assume we don't know what to expect...
expectedContentType := "*/*"
if currentAttachment.File.ContentType != "" {
// ... and then narrow it down if we do
expectedContentType = currentAttachment.File.ContentType
}
attachmentBytes, err := t.DereferenceMedia(ctx, remoteIRI, expectedContentType)
if err != nil {
return nil, fmt.Errorf("dereferencing remote media with url %s: %s", remoteIRI.String(), err)
}
return mh.ProcessHeaderOrAvatar(ctx, attachmentBytes, accountID, headerOrAvi, currentAttachment.RemoteURL)
}

198
internal/media/image.go Normal file
View file

@ -0,0 +1,198 @@
/*
GoToSocial
Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package media
import (
"bytes"
"errors"
"fmt"
"image"
"image/gif"
"image/jpeg"
"image/png"
"io"
"github.com/buckket/go-blurhash"
"github.com/nfnt/resize"
)
const (
thumbnailMaxWidth = 512
thumbnailMaxHeight = 512
)
type imageMeta struct {
width int
height int
size int
aspect float64
blurhash string // defined only for calls to deriveThumbnail if createBlurhash is true
small []byte // defined only for calls to deriveStaticEmoji or deriveThumbnail
}
func decodeGif(r io.Reader) (*imageMeta, error) {
gif, err := gif.DecodeAll(r)
if err != nil {
return nil, err
}
// use the first frame to get the static characteristics
width := gif.Config.Width
height := gif.Config.Height
size := width * height
aspect := float64(width) / float64(height)
return &imageMeta{
width: width,
height: height,
size: size,
aspect: aspect,
}, nil
}
func decodeImage(r io.Reader, contentType string) (*imageMeta, error) {
var i image.Image
var err error
switch contentType {
case mimeImageJpeg:
i, err = jpeg.Decode(r)
case mimeImagePng:
i, err = png.Decode(r)
default:
err = fmt.Errorf("content type %s not recognised", contentType)
}
if err != nil {
return nil, err
}
if i == nil {
return nil, errors.New("processed image was nil")
}
width := i.Bounds().Size().X
height := i.Bounds().Size().Y
size := width * height
aspect := float64(width) / float64(height)
return &imageMeta{
width: width,
height: height,
size: size,
aspect: aspect,
}, nil
}
// deriveThumbnail returns a byte slice and metadata for a thumbnail
// of a given jpeg, png, or gif, or an error if something goes wrong.
//
// If createBlurhash is true, then a blurhash will also be generated from a tiny
// version of the image. This costs precious CPU cycles, so only use it if you
// really need a blurhash and don't have one already.
//
// If createBlurhash is false, then the blurhash field on the returned ImageAndMeta
// will be an empty string.
func deriveThumbnail(r io.Reader, contentType string, createBlurhash bool) (*imageMeta, error) {
var i image.Image
var err error
switch contentType {
case mimeImageJpeg:
i, err = jpeg.Decode(r)
case mimeImagePng:
i, err = png.Decode(r)
case mimeImageGif:
i, err = gif.Decode(r)
default:
err = fmt.Errorf("content type %s can't be thumbnailed", contentType)
}
if err != nil {
return nil, err
}
if i == nil {
return nil, errors.New("processed image was nil")
}
thumb := resize.Thumbnail(thumbnailMaxWidth, thumbnailMaxHeight, i, resize.NearestNeighbor)
width := thumb.Bounds().Size().X
height := thumb.Bounds().Size().Y
size := width * height
aspect := float64(width) / float64(height)
im := &imageMeta{
width: width,
height: height,
size: size,
aspect: aspect,
}
if createBlurhash {
// for generating blurhashes, it's more cost effective to lose detail rather than
// pass a big image into the blurhash algorithm, so make a teeny tiny version
tiny := resize.Thumbnail(32, 32, thumb, resize.NearestNeighbor)
bh, err := blurhash.Encode(4, 3, tiny)
if err != nil {
return nil, err
}
im.blurhash = bh
}
out := &bytes.Buffer{}
if err := jpeg.Encode(out, thumb, &jpeg.Options{
// Quality isn't extremely important for thumbnails, so 75 is "good enough"
Quality: 75,
}); err != nil {
return nil, err
}
im.small = out.Bytes()
return im, nil
}
// deriveStaticEmojji takes a given gif or png of an emoji, decodes it, and re-encodes it as a static png.
func deriveStaticEmoji(r io.Reader, contentType string) (*imageMeta, error) {
var i image.Image
var err error
switch contentType {
case mimeImagePng:
i, err = png.Decode(r)
if err != nil {
return nil, err
}
case mimeImageGif:
i, err = gif.Decode(r)
if err != nil {
return nil, err
}
default:
return nil, fmt.Errorf("content type %s not allowed for emoji", contentType)
}
out := &bytes.Buffer{}
if err := png.Encode(out, i); err != nil {
return nil, err
}
return &imageMeta{
small: out.Bytes(),
}, nil
}

176
internal/media/manager.go Normal file
View file

@ -0,0 +1,176 @@
/*
GoToSocial
Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package media
import (
"context"
"errors"
"runtime"
"codeberg.org/gruf/go-runners"
"codeberg.org/gruf/go-store/kv"
"github.com/sirupsen/logrus"
"github.com/superseriousbusiness/gotosocial/internal/db"
)
// Manager provides an interface for managing media: parsing, storing, and retrieving media objects like photos, videos, and gifs.
type Manager interface {
// ProcessMedia begins the process of decoding and storing the given data as an attachment.
// It will return a pointer to a Media struct upon which further actions can be performed, such as getting
// the finished media, thumbnail, attachment, etc.
//
// data should be a function that the media manager can call to return raw bytes of a piece of media.
//
// accountID should be the account that the media belongs to.
//
// ai is optional and can be nil. Any additional information about the attachment provided will be put in the database.
ProcessMedia(ctx context.Context, data DataFunc, accountID string, ai *AdditionalMediaInfo) (*ProcessingMedia, error)
ProcessEmoji(ctx context.Context, data DataFunc, shortcode string, id string, uri string, ai *AdditionalEmojiInfo) (*ProcessingEmoji, error)
// NumWorkers returns the total number of workers available to this manager.
NumWorkers() int
// QueueSize returns the total capacity of the queue.
QueueSize() int
// JobsQueued returns the number of jobs currently in the task queue.
JobsQueued() int
// ActiveWorkers returns the number of workers currently performing jobs.
ActiveWorkers() int
// Stop stops the underlying worker pool of the manager. It should be called
// when closing GoToSocial in order to cleanly finish any in-progress jobs.
// It will block until workers are finished processing.
Stop() error
}
type manager struct {
db db.DB
storage *kv.KVStore
pool runners.WorkerPool
numWorkers int
queueSize int
}
// NewManager returns a media manager with the given db and underlying storage.
//
// A worker pool will also be initialized for the manager, to ensure that only
// a limited number of media will be processed in parallel.
//
// The number of workers will be the number of CPUs available to the Go runtime,
// divided by 2 (rounding down, but always at least 1).
//
// The length of the queue will be the number of workers multiplied by 10.
//
// So for an 8 core machine, the media manager will get 4 workers, and a queue of length 40.
// For a 4 core machine, this will be 2 workers, and a queue length of 20.
// For a single or 2-core machine, the media manager will get 1 worker, and a queue of length 10.
func NewManager(database db.DB, storage *kv.KVStore) (Manager, error) {
numWorkers := runtime.NumCPU() / 2
// make sure we always have at least 1 worker even on single-core machines
if numWorkers == 0 {
numWorkers = 1
}
queueSize := numWorkers * 10
m := &manager{
db: database,
storage: storage,
pool: runners.NewWorkerPool(numWorkers, queueSize),
numWorkers: numWorkers,
queueSize: queueSize,
}
if start := m.pool.Start(); !start {
return nil, errors.New("could not start worker pool")
}
logrus.Debugf("started media manager worker pool with %d workers and queue capacity of %d", numWorkers, queueSize)
return m, nil
}
func (m *manager) ProcessMedia(ctx context.Context, data DataFunc, accountID string, ai *AdditionalMediaInfo) (*ProcessingMedia, error) {
processingMedia, err := m.preProcessMedia(ctx, data, accountID, ai)
if err != nil {
return nil, err
}
logrus.Tracef("ProcessMedia: about to enqueue media with attachmentID %s, queue length is %d", processingMedia.AttachmentID(), m.pool.Queue())
m.pool.Enqueue(func(innerCtx context.Context) {
select {
case <-innerCtx.Done():
// if the inner context is done that means the worker pool is closing, so we should just return
return
default:
// start loading the media already for the caller's convenience
if _, err := processingMedia.LoadAttachment(innerCtx); err != nil {
logrus.Errorf("ProcessMedia: error processing media with attachmentID %s: %s", processingMedia.AttachmentID(), err)
}
}
})
logrus.Tracef("ProcessMedia: succesfully queued media with attachmentID %s, queue length is %d", processingMedia.AttachmentID(), m.pool.Queue())
return processingMedia, nil
}
func (m *manager) ProcessEmoji(ctx context.Context, data DataFunc, shortcode string, id string, uri string, ai *AdditionalEmojiInfo) (*ProcessingEmoji, error) {
processingEmoji, err := m.preProcessEmoji(ctx, data, shortcode, id, uri, ai)
if err != nil {
return nil, err
}
logrus.Tracef("ProcessEmoji: about to enqueue emoji with id %s, queue length is %d", processingEmoji.EmojiID(), m.pool.Queue())
m.pool.Enqueue(func(innerCtx context.Context) {
select {
case <-innerCtx.Done():
// if the inner context is done that means the worker pool is closing, so we should just return
return
default:
// start loading the emoji already for the caller's convenience
if _, err := processingEmoji.LoadEmoji(innerCtx); err != nil {
logrus.Errorf("ProcessEmoji: error processing emoji with id %s: %s", processingEmoji.EmojiID(), err)
}
}
})
logrus.Tracef("ProcessEmoji: succesfully queued emoji with id %s, queue length is %d", processingEmoji.EmojiID(), m.pool.Queue())
return processingEmoji, nil
}
func (m *manager) NumWorkers() int {
return m.numWorkers
}
func (m *manager) QueueSize() int {
return m.queueSize
}
func (m *manager) JobsQueued() int {
return m.pool.Queue()
}
func (m *manager) ActiveWorkers() int {
return m.pool.Workers()
}
func (m *manager) Stop() error {
logrus.Info("stopping media manager worker pool")
stopped := m.pool.Stop()
if !stopped {
return errors.New("could not stop media manager worker pool")
}
return nil
}

363
internal/media/manager_test.go Normal file
View file

@ -0,0 +1,363 @@
/*
GoToSocial
Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package media_test
import (
"bytes"
"context"
"fmt"
"io"
"os"
"path"
"testing"
"time"
"codeberg.org/gruf/go-store/kv"
"codeberg.org/gruf/go-store/storage"
"github.com/stretchr/testify/suite"
gtsmodel "github.com/superseriousbusiness/gotosocial/internal/db/bundb/migrations/20211113114307_init"
"github.com/superseriousbusiness/gotosocial/internal/media"
)
type ManagerTestSuite struct {
MediaStandardTestSuite
}
func (suite *ManagerTestSuite) TestSimpleJpegProcessBlocking() {
ctx := context.Background()
data := func(_ context.Context) (io.Reader, int, error) {
// load bytes from a test image
b, err := os.ReadFile("./test/test-jpeg.jpg")
if err != nil {
panic(err)
}
return bytes.NewBuffer(b), len(b), nil
}
accountID := "01FS1X72SK9ZPW0J1QQ68BD264"
// process the media with no additional info provided
processingMedia, err := suite.manager.ProcessMedia(ctx, data, accountID, nil)
suite.NoError(err)
// fetch the attachment id from the processing media
attachmentID := processingMedia.AttachmentID()
// do a blocking call to fetch the attachment
attachment, err := processingMedia.LoadAttachment(ctx)
suite.NoError(err)
suite.NotNil(attachment)
// make sure it's got the stuff set on it that we expect
// the attachment ID and accountID we expect
suite.Equal(attachmentID, attachment.ID)
suite.Equal(accountID, attachment.AccountID)
// file meta should be correctly derived from the image
suite.EqualValues(gtsmodel.Original{
Width: 1920, Height: 1080, Size: 2073600, Aspect: 1.7777777777777777,
}, attachment.FileMeta.Original)
suite.EqualValues(gtsmodel.Small{
Width: 512, Height: 288, Size: 147456, Aspect: 1.7777777777777777,
}, attachment.FileMeta.Small)
suite.Equal("image/jpeg", attachment.File.ContentType)
suite.Equal(269739, attachment.File.FileSize)
suite.Equal("LjBzUo#6RQR._NvzRjWF?urqV@a$", attachment.Blurhash)
// now make sure the attachment is in the database
dbAttachment, err := suite.db.GetAttachmentByID(ctx, attachmentID)
suite.NoError(err)
suite.NotNil(dbAttachment)
// make sure the processed file is in storage
processedFullBytes, err := suite.storage.Get(attachment.File.Path)
suite.NoError(err)
suite.NotEmpty(processedFullBytes)
// load the processed bytes from our test folder, to compare
processedFullBytesExpected, err := os.ReadFile("./test/test-jpeg-processed.jpg")
suite.NoError(err)
suite.NotEmpty(processedFullBytesExpected)
// the bytes in storage should be what we expected
suite.Equal(processedFullBytesExpected, processedFullBytes)
// now do the same for the thumbnail and make sure it's what we expected
processedThumbnailBytes, err := suite.storage.Get(attachment.Thumbnail.Path)
suite.NoError(err)
suite.NotEmpty(processedThumbnailBytes)
processedThumbnailBytesExpected, err := os.ReadFile("./test/test-jpeg-thumbnail.jpg")
suite.NoError(err)
suite.NotEmpty(processedThumbnailBytesExpected)
suite.Equal(processedThumbnailBytesExpected, processedThumbnailBytes)
}
func (suite *ManagerTestSuite) TestSimpleJpegProcessAsync() {
ctx := context.Background()
data := func(_ context.Context) (io.Reader, int, error) {
// load bytes from a test image
b, err := os.ReadFile("./test/test-jpeg.jpg")
if err != nil {
panic(err)
}
return bytes.NewBuffer(b), len(b), nil
}
accountID := "01FS1X72SK9ZPW0J1QQ68BD264"
// process the media with no additional info provided
processingMedia, err := suite.manager.ProcessMedia(ctx, data, accountID, nil)
suite.NoError(err)
// fetch the attachment id from the processing media
attachmentID := processingMedia.AttachmentID()
// wait for the media to finish processing
for finished := processingMedia.Finished(); !finished; finished = processingMedia.Finished() {
time.Sleep(10 * time.Millisecond)
fmt.Printf("\n\nnot finished yet...\n\n")
}
fmt.Printf("\n\nfinished!\n\n")
// fetch the attachment from the database
attachment, err := suite.db.GetAttachmentByID(ctx, attachmentID)
suite.NoError(err)
suite.NotNil(attachment)
// make sure it's got the stuff set on it that we expect
// the attachment ID and accountID we expect
suite.Equal(attachmentID, attachment.ID)
suite.Equal(accountID, attachment.AccountID)
// file meta should be correctly derived from the image
suite.EqualValues(gtsmodel.Original{
Width: 1920, Height: 1080, Size: 2073600, Aspect: 1.7777777777777777,
}, attachment.FileMeta.Original)
suite.EqualValues(gtsmodel.Small{
Width: 512, Height: 288, Size: 147456, Aspect: 1.7777777777777777,
}, attachment.FileMeta.Small)
suite.Equal("image/jpeg", attachment.File.ContentType)
suite.Equal(269739, attachment.File.FileSize)
suite.Equal("LjBzUo#6RQR._NvzRjWF?urqV@a$", attachment.Blurhash)
// now make sure the attachment is in the database
dbAttachment, err := suite.db.GetAttachmentByID(ctx, attachmentID)
suite.NoError(err)
suite.NotNil(dbAttachment)
// make sure the processed file is in storage
processedFullBytes, err := suite.storage.Get(attachment.File.Path)
suite.NoError(err)
suite.NotEmpty(processedFullBytes)
// load the processed bytes from our test folder, to compare
processedFullBytesExpected, err := os.ReadFile("./test/test-jpeg-processed.jpg")
suite.NoError(err)
suite.NotEmpty(processedFullBytesExpected)
// the bytes in storage should be what we expected
suite.Equal(processedFullBytesExpected, processedFullBytes)
// now do the same for the thumbnail and make sure it's what we expected
processedThumbnailBytes, err := suite.storage.Get(attachment.Thumbnail.Path)
suite.NoError(err)
suite.NotEmpty(processedThumbnailBytes)
processedThumbnailBytesExpected, err := os.ReadFile("./test/test-jpeg-thumbnail.jpg")
suite.NoError(err)
suite.NotEmpty(processedThumbnailBytesExpected)
suite.Equal(processedThumbnailBytesExpected, processedThumbnailBytes)
}
func (suite *ManagerTestSuite) TestSimpleJpegQueueSpamming() {
// in this test, we spam the manager queue with 50 new media requests, just to see how it holds up
ctx := context.Background()
b, err := os.ReadFile("./test/test-jpeg.jpg")
if err != nil {
panic(err)
}
data := func(_ context.Context) (io.Reader, int, error) {
// load bytes from a test image
return bytes.NewReader(b), len(b), nil
}
accountID := "01FS1X72SK9ZPW0J1QQ68BD264"
spam := 50
inProcess := []*media.ProcessingMedia{}
for i := 0; i < spam; i++ {
// process the media with no additional info provided
processingMedia, err := suite.manager.ProcessMedia(ctx, data, accountID, nil)
suite.NoError(err)
inProcess = append(inProcess, processingMedia)
}
for _, processingMedia := range inProcess {
fmt.Printf("\n\n\nactive workers: %d, queue length: %d\n\n\n", suite.manager.ActiveWorkers(), suite.manager.JobsQueued())
// fetch the attachment id from the processing media
attachmentID := processingMedia.AttachmentID()
// do a blocking call to fetch the attachment
attachment, err := processingMedia.LoadAttachment(ctx)
suite.NoError(err)
suite.NotNil(attachment)
// make sure it's got the stuff set on it that we expect
// the attachment ID and accountID we expect
suite.Equal(attachmentID, attachment.ID)
suite.Equal(accountID, attachment.AccountID)
// file meta should be correctly derived from the image
suite.EqualValues(gtsmodel.Original{
Width: 1920, Height: 1080, Size: 2073600, Aspect: 1.7777777777777777,
}, attachment.FileMeta.Original)
suite.EqualValues(gtsmodel.Small{
Width: 512, Height: 288, Size: 147456, Aspect: 1.7777777777777777,
}, attachment.FileMeta.Small)
suite.Equal("image/jpeg", attachment.File.ContentType)
suite.Equal(269739, attachment.File.FileSize)
suite.Equal("LjBzUo#6RQR._NvzRjWF?urqV@a$", attachment.Blurhash)
// now make sure the attachment is in the database
dbAttachment, err := suite.db.GetAttachmentByID(ctx, attachmentID)
suite.NoError(err)
suite.NotNil(dbAttachment)
// make sure the processed file is in storage
processedFullBytes, err := suite.storage.Get(attachment.File.Path)
suite.NoError(err)
suite.NotEmpty(processedFullBytes)
// load the processed bytes from our test folder, to compare
processedFullBytesExpected, err := os.ReadFile("./test/test-jpeg-processed.jpg")
suite.NoError(err)
suite.NotEmpty(processedFullBytesExpected)
// the bytes in storage should be what we expected
suite.Equal(processedFullBytesExpected, processedFullBytes)
// now do the same for the thumbnail and make sure it's what we expected
processedThumbnailBytes, err := suite.storage.Get(attachment.Thumbnail.Path)
suite.NoError(err)
suite.NotEmpty(processedThumbnailBytes)
processedThumbnailBytesExpected, err := os.ReadFile("./test/test-jpeg-thumbnail.jpg")
suite.NoError(err)
suite.NotEmpty(processedThumbnailBytesExpected)
suite.Equal(processedThumbnailBytesExpected, processedThumbnailBytes)
}
}
func (suite *ManagerTestSuite) TestSimpleJpegProcessBlockingWithDiskStorage() {
ctx := context.Background()
data := func(_ context.Context) (io.Reader, int, error) {
// load bytes from a test image
b, err := os.ReadFile("./test/test-jpeg.jpg")
if err != nil {
panic(err)
}
return bytes.NewBuffer(b), len(b), nil
}
accountID := "01FS1X72SK9ZPW0J1QQ68BD264"
temp := fmt.Sprintf("%s/gotosocial-test", os.TempDir())
defer os.RemoveAll(temp)
diskStorage, err := kv.OpenFile(temp, &storage.DiskConfig{
LockFile: path.Join(temp, "store.lock"),
})
if err != nil {
panic(err)
}
diskManager, err := media.NewManager(suite.db, diskStorage)
if err != nil {
panic(err)
}
suite.manager = diskManager
// process the media with no additional info provided
processingMedia, err := diskManager.ProcessMedia(ctx, data, accountID, nil)
suite.NoError(err)
// fetch the attachment id from the processing media
attachmentID := processingMedia.AttachmentID()
// do a blocking call to fetch the attachment
attachment, err := processingMedia.LoadAttachment(ctx)
suite.NoError(err)
suite.NotNil(attachment)
// make sure it's got the stuff set on it that we expect
// the attachment ID and accountID we expect
suite.Equal(attachmentID, attachment.ID)
suite.Equal(accountID, attachment.AccountID)
// file meta should be correctly derived from the image
suite.EqualValues(gtsmodel.Original{
Width: 1920, Height: 1080, Size: 2073600, Aspect: 1.7777777777777777,
}, attachment.FileMeta.Original)
suite.EqualValues(gtsmodel.Small{
Width: 512, Height: 288, Size: 147456, Aspect: 1.7777777777777777,
}, attachment.FileMeta.Small)
suite.Equal("image/jpeg", attachment.File.ContentType)
suite.Equal(269739, attachment.File.FileSize)
suite.Equal("LjBzUo#6RQR._NvzRjWF?urqV@a$", attachment.Blurhash)
// now make sure the attachment is in the database
dbAttachment, err := suite.db.GetAttachmentByID(ctx, attachmentID)
suite.NoError(err)
suite.NotNil(dbAttachment)
// make sure the processed file is in storage
processedFullBytes, err := diskStorage.Get(attachment.File.Path)
suite.NoError(err)
suite.NotEmpty(processedFullBytes)
// load the processed bytes from our test folder, to compare
processedFullBytesExpected, err := os.ReadFile("./test/test-jpeg-processed.jpg")
suite.NoError(err)
suite.NotEmpty(processedFullBytesExpected)
// the bytes in storage should be what we expected
suite.Equal(processedFullBytesExpected, processedFullBytes)
// now do the same for the thumbnail and make sure it's what we expected
processedThumbnailBytes, err := diskStorage.Get(attachment.Thumbnail.Path)
suite.NoError(err)
suite.NotEmpty(processedThumbnailBytes)
processedThumbnailBytesExpected, err := os.ReadFile("./test/test-jpeg-thumbnail.jpg")
suite.NoError(err)
suite.NotEmpty(processedThumbnailBytesExpected)
suite.Equal(processedThumbnailBytesExpected, processedThumbnailBytes)
}
func TestManagerTestSuite(t *testing.T) {
suite.Run(t, &ManagerTestSuite{})
}

54
internal/media/media_test.go Normal file
View file

@ -0,0 +1,54 @@
/*
GoToSocial
Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package media_test
import (
"codeberg.org/gruf/go-store/kv"
"github.com/stretchr/testify/suite"
"github.com/superseriousbusiness/gotosocial/internal/db"
"github.com/superseriousbusiness/gotosocial/internal/media"
"github.com/superseriousbusiness/gotosocial/testrig"
)
type MediaStandardTestSuite struct {
suite.Suite
db db.DB
storage *kv.KVStore
manager media.Manager
}
func (suite *MediaStandardTestSuite) SetupSuite() {
testrig.InitTestConfig()
testrig.InitTestLog()
suite.db = testrig.NewTestDB()
suite.storage = testrig.NewTestStorage()
}
func (suite *MediaStandardTestSuite) SetupTest() {
testrig.StandardStorageSetup(suite.storage, "../../testrig/media")
testrig.StandardDBSetup(suite.db, nil)
suite.manager = testrig.NewTestMediaManager(suite.db, suite.storage)
}
func (suite *MediaStandardTestSuite) TearDownTest() {
testrig.StandardDBTeardown(suite.db)
testrig.StandardStorageTeardown(suite.storage)
}

143
internal/media/processicon.go
View file

@ -1,143 +0,0 @@
/*
GoToSocial
Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package media
import (
"errors"
"fmt"
"strings"
"time"
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
"github.com/superseriousbusiness/gotosocial/internal/id"
"github.com/superseriousbusiness/gotosocial/internal/uris"
)
func (mh *mediaHandler) processHeaderOrAvi(imageBytes []byte, contentType string, mediaType Type, accountID string, remoteURL string) (*gtsmodel.MediaAttachment, error) {
var isHeader bool
var isAvatar bool
switch mediaType {
case TypeHeader:
isHeader = true
case TypeAvatar:
isAvatar = true
default:
return nil, errors.New("header or avatar not selected")
}
var clean []byte
var err error
var original *imageAndMeta
switch contentType {
case MIMEJpeg:
if clean, err = purgeExif(imageBytes); err != nil {
return nil, fmt.Errorf("error cleaning exif data: %s", err)
}
original, err = deriveImage(clean, contentType)
case MIMEPng:
if clean, err = purgeExif(imageBytes); err != nil {
return nil, fmt.Errorf("error cleaning exif data: %s", err)
}
original, err = deriveImage(clean, contentType)
case MIMEGif:
clean = imageBytes
original, err = deriveGif(clean, contentType)
default:
return nil, errors.New("media type unrecognized")
}
if err != nil {
return nil, fmt.Errorf("error parsing image: %s", err)
}
small, err := deriveThumbnail(clean, contentType, 256, 256)
if err != nil {
return nil, fmt.Errorf("error deriving thumbnail: %s", err)
}
// now put it in storage, take a new id for the name of the file so we don't store any unnecessary info about it
extension := strings.Split(contentType, "/")[1]
newMediaID, err := id.NewRandomULID()
if err != nil {
return nil, err
}
originalURL := uris.GenerateURIForAttachment(accountID, string(mediaType), string(SizeOriginal), newMediaID, extension)
smallURL := uris.GenerateURIForAttachment(accountID, string(mediaType), string(SizeSmall), newMediaID, extension)
// we store the original...
originalPath := fmt.Sprintf("%s/%s/%s/%s.%s", accountID, mediaType, SizeOriginal, newMediaID, extension)
if err := mh.storage.Put(originalPath, original.image); err != nil {
return nil, fmt.Errorf("storage error: %s", err)
}
// and a thumbnail...
smallPath := fmt.Sprintf("%s/%s/%s/%s.%s", accountID, mediaType, SizeSmall, newMediaID, extension)
if err := mh.storage.Put(smallPath, small.image); err != nil {
return nil, fmt.Errorf("storage error: %s", err)
}
ma := &gtsmodel.MediaAttachment{
ID: newMediaID,
StatusID: "",
URL: originalURL,
RemoteURL: remoteURL,
CreatedAt: time.Now(),
UpdatedAt: time.Now(),
Type: gtsmodel.FileTypeImage,
FileMeta: gtsmodel.FileMeta{
Original: gtsmodel.Original{
Width: original.width,
Height: original.height,
Size: original.size,
Aspect: original.aspect,
},
Small: gtsmodel.Small{
Width: small.width,
Height: small.height,
Size: small.size,
Aspect: small.aspect,
},
},
AccountID: accountID,
Description: "",
ScheduledStatusID: "",
Blurhash: small.blurhash,
Processing: 2,
File: gtsmodel.File{
Path: originalPath,
ContentType: contentType,
FileSize: len(original.image),
UpdatedAt: time.Now(),
},
Thumbnail: gtsmodel.Thumbnail{
Path: smallPath,
ContentType: contentType,
FileSize: len(small.image),
UpdatedAt: time.Now(),
URL: smallURL,
RemoteURL: "",
},
Avatar: isAvatar,
Header: isHeader,
}
return ma, nil
}

133
internal/media/processimage.go
View file

@ -1,133 +0,0 @@
/*
GoToSocial
Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package media
import (
"errors"
"fmt"
"strings"
"time"
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
"github.com/superseriousbusiness/gotosocial/internal/id"
"github.com/superseriousbusiness/gotosocial/internal/uris"
)
func (mh *mediaHandler) processImageAttachment(data []byte, minAttachment *gtsmodel.MediaAttachment) (*gtsmodel.MediaAttachment, error) {
var clean []byte
var err error
var original *imageAndMeta
var small *imageAndMeta
contentType := minAttachment.File.ContentType
switch contentType {
case MIMEJpeg, MIMEPng:
if clean, err = purgeExif(data); err != nil {
return nil, fmt.Errorf("error cleaning exif data: %s", err)
}
original, err = deriveImage(clean, contentType)
if err != nil {
return nil, fmt.Errorf("error parsing image: %s", err)
}
case MIMEGif:
clean = data
original, err = deriveGif(clean, contentType)
if err != nil {
return nil, fmt.Errorf("error parsing gif: %s", err)
}
default:
return nil, errors.New("media type unrecognized")
}
small, err = deriveThumbnail(clean, contentType, 512, 512)
if err != nil {
return nil, fmt.Errorf("error deriving thumbnail: %s", err)
}
// now put it in storage, take a new id for the name of the file so we don't store any unnecessary info about it
extension := strings.Split(contentType, "/")[1]
newMediaID, err := id.NewRandomULID()
if err != nil {
return nil, err
}
originalURL := uris.GenerateURIForAttachment(minAttachment.AccountID, string(TypeAttachment), string(SizeOriginal), newMediaID, extension)
smallURL := uris.GenerateURIForAttachment(minAttachment.AccountID, string(TypeAttachment), string(SizeSmall), newMediaID, "jpeg") // all thumbnails/smalls are encoded as jpeg
// we store the original...
originalPath := fmt.Sprintf("%s/%s/%s/%s.%s", minAttachment.AccountID, TypeAttachment, SizeOriginal, newMediaID, extension)
if err := mh.storage.Put(originalPath, original.image); err != nil {
return nil, fmt.Errorf("storage error: %s", err)
}
// and a thumbnail...
smallPath := fmt.Sprintf("%s/%s/%s/%s.jpeg", minAttachment.AccountID, TypeAttachment, SizeSmall, newMediaID) // all thumbnails/smalls are encoded as jpeg
if err := mh.storage.Put(smallPath, small.image); err != nil {
return nil, fmt.Errorf("storage error: %s", err)
}
minAttachment.FileMeta.Original = gtsmodel.Original{
Width: original.width,
Height: original.height,
Size: original.size,
Aspect: original.aspect,
}
minAttachment.FileMeta.Small = gtsmodel.Small{
Width: small.width,
Height: small.height,
Size: small.size,
Aspect: small.aspect,
}
attachment := &gtsmodel.MediaAttachment{
ID: newMediaID,
StatusID: minAttachment.StatusID,
URL: originalURL,
RemoteURL: minAttachment.RemoteURL,
CreatedAt: minAttachment.CreatedAt,
UpdatedAt: minAttachment.UpdatedAt,
Type: gtsmodel.FileTypeImage,
FileMeta: minAttachment.FileMeta,
AccountID: minAttachment.AccountID,
Description: minAttachment.Description,
ScheduledStatusID: minAttachment.ScheduledStatusID,
Blurhash: small.blurhash,
Processing: 2,
File: gtsmodel.File{
Path: originalPath,
ContentType: contentType,
FileSize: len(original.image),
UpdatedAt: time.Now(),
},
Thumbnail: gtsmodel.Thumbnail{
Path: smallPath,
ContentType: MIMEJpeg, // all thumbnails/smalls are encoded as jpeg
FileSize: len(small.image),
UpdatedAt: time.Now(),
URL: smallURL,
RemoteURL: minAttachment.Thumbnail.RemoteURL,
},
Avatar: minAttachment.Avatar,
Header: minAttachment.Header,
}
return attachment, nil
}

290
internal/media/processingemoji.go Normal file
View file

@ -0,0 +1,290 @@
/*
GoToSocial
Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package media
import (
"bytes"
"context"
"fmt"
"io"
"strings"
"sync"
"sync/atomic"
"time"
"codeberg.org/gruf/go-store/kv"
"github.com/superseriousbusiness/gotosocial/internal/db"
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
"github.com/superseriousbusiness/gotosocial/internal/uris"
)
// ProcessingEmoji represents an emoji currently processing. It exposes
// various functions for retrieving data from the process.
type ProcessingEmoji struct {
mu sync.Mutex
// id of this instance's account -- pinned for convenience here so we only need to fetch it once
instanceAccountID string
/*
below fields should be set on newly created media;
emoji will be updated incrementally as media goes through processing
*/
emoji *gtsmodel.Emoji
data DataFunc
read bool // bool indicating that data function has been triggered already
/*
below fields represent the processing state of the static of the emoji
*/
staticState int32
/*
below pointers to database and storage are maintained so that
the media can store and update itself during processing steps
*/
database db.DB
storage *kv.KVStore
err error // error created during processing, if any
// track whether this emoji has already been put in the databse
insertedInDB bool
}
// EmojiID returns the ID of the underlying emoji without blocking processing.
func (p *ProcessingEmoji) EmojiID() string {
return p.emoji.ID
}
// LoadEmoji blocks until the static and fullsize image
// has been processed, and then returns the completed emoji.
func (p *ProcessingEmoji) LoadEmoji(ctx context.Context) (*gtsmodel.Emoji, error) {
p.mu.Lock()
defer p.mu.Unlock()
if err := p.store(ctx); err != nil {
return nil, err
}
if err := p.loadStatic(ctx); err != nil {
return nil, err
}
// store the result in the database before returning it
if !p.insertedInDB {
if err := p.database.Put(ctx, p.emoji); err != nil {
return nil, err
}
p.insertedInDB = true
}
return p.emoji, nil
}
// Finished returns true if processing has finished for both the thumbnail
// and full fized version of this piece of media.
func (p *ProcessingEmoji) Finished() bool {
return atomic.LoadInt32(&p.staticState) == int32(complete)
}
func (p *ProcessingEmoji) loadStatic(ctx context.Context) error {
staticState := atomic.LoadInt32(&p.staticState)
switch processState(staticState) {
case received:
// stream the original file out of storage...
stored, err := p.storage.GetStream(p.emoji.ImagePath)
if err != nil {
p.err = fmt.Errorf("loadStatic: error fetching file from storage: %s", err)
atomic.StoreInt32(&p.staticState, int32(errored))
return p.err
}
// we haven't processed a static version of this emoji yet so do it now
static, err := deriveStaticEmoji(stored, p.emoji.ImageContentType)
if err != nil {
p.err = fmt.Errorf("loadStatic: error deriving static: %s", err)
atomic.StoreInt32(&p.staticState, int32(errored))
return p.err
}
if err := stored.Close(); err != nil {
p.err = fmt.Errorf("loadStatic: error closing stored full size: %s", err)
atomic.StoreInt32(&p.staticState, int32(errored))
return p.err
}
// put the static in storage
if err := p.storage.Put(p.emoji.ImageStaticPath, static.small); err != nil {
p.err = fmt.Errorf("loadStatic: error storing static: %s", err)
atomic.StoreInt32(&p.staticState, int32(errored))
return p.err
}
p.emoji.ImageStaticFileSize = len(static.small)
// we're done processing the static version of the emoji!
atomic.StoreInt32(&p.staticState, int32(complete))
fallthrough
case complete:
return nil
case errored:
return p.err
}
return fmt.Errorf("static processing status %d unknown", p.staticState)
}
// store calls the data function attached to p if it hasn't been called yet,
// and updates the underlying attachment fields as necessary. It will then stream
// bytes from p's reader directly into storage so that it can be retrieved later.
func (p *ProcessingEmoji) store(ctx context.Context) error {
// check if we've already done this and bail early if we have
if p.read {
return nil
}
// execute the data function to get the reader out of it
reader, fileSize, err := p.data(ctx)
if err != nil {
return fmt.Errorf("store: error executing data function: %s", err)
}
// extract no more than 261 bytes from the beginning of the file -- this is the header
firstBytes := make([]byte, maxFileHeaderBytes)
if _, err := reader.Read(firstBytes); err != nil {
return fmt.Errorf("store: error reading initial %d bytes: %s", maxFileHeaderBytes, err)
}
// now we have the file header we can work out the content type from it
contentType, err := parseContentType(firstBytes)
if err != nil {
return fmt.Errorf("store: error parsing content type: %s", err)
}
// bail if this is a type we can't process
if !supportedEmoji(contentType) {
return fmt.Errorf("store: content type %s was not valid for an emoji", contentType)
}
// extract the file extension
split := strings.Split(contentType, "/")
extension := split[1] // something like 'gif'
// set some additional fields on the emoji now that
// we know more about what the underlying image actually is
p.emoji.ImageURL = uris.GenerateURIForAttachment(p.instanceAccountID, string(TypeEmoji), string(SizeOriginal), p.emoji.ID, extension)
p.emoji.ImagePath = fmt.Sprintf("%s/%s/%s/%s.%s", p.instanceAccountID, TypeEmoji, SizeOriginal, p.emoji.ID, extension)
p.emoji.ImageContentType = contentType
p.emoji.ImageFileSize = fileSize
// concatenate the first bytes with the existing bytes still in the reader (thanks Mara)
multiReader := io.MultiReader(bytes.NewBuffer(firstBytes), reader)
// store this for now -- other processes can pull it out of storage as they please
if err := p.storage.PutStream(p.emoji.ImagePath, multiReader); err != nil {
return fmt.Errorf("store: error storing stream: %s", err)
}
// if the original reader is a readcloser, close it since we're done with it now
if rc, ok := reader.(io.ReadCloser); ok {
if err := rc.Close(); err != nil {
return fmt.Errorf("store: error closing readcloser: %s", err)
}
}
p.read = true
return nil
}
func (m *manager) preProcessEmoji(ctx context.Context, data DataFunc, shortcode string, id string, uri string, ai *AdditionalEmojiInfo) (*ProcessingEmoji, error) {
instanceAccount, err := m.db.GetInstanceAccount(ctx, "")
if err != nil {
return nil, fmt.Errorf("preProcessEmoji: error fetching this instance account from the db: %s", err)
}
// populate initial fields on the emoji -- some of these will be overwritten as we proceed
emoji := &gtsmodel.Emoji{
ID: id,
CreatedAt: time.Now(),
UpdatedAt: time.Now(),
Shortcode: shortcode,
Domain: "", // assume our own domain unless told otherwise
ImageRemoteURL: "",
ImageStaticRemoteURL: "",
ImageURL: "", // we don't know yet
ImageStaticURL: uris.GenerateURIForAttachment(instanceAccount.ID, string(TypeEmoji), string(SizeStatic), id, mimePng), // all static emojis are encoded as png
ImagePath: "", // we don't know yet
ImageStaticPath: fmt.Sprintf("%s/%s/%s/%s.%s", instanceAccount.ID, TypeEmoji, SizeStatic, id, mimePng), // all static emojis are encoded as png
ImageContentType: "", // we don't know yet
ImageStaticContentType: mimeImagePng, // all static emojis are encoded as png
ImageFileSize: 0,
ImageStaticFileSize: 0,
ImageUpdatedAt: time.Now(),
Disabled: false,
URI: uri,
VisibleInPicker: true,
CategoryID: "",
}
// check if we have additional info to add to the emoji,
// and overwrite some of the emoji fields if so
if ai != nil {
if ai.CreatedAt != nil {
emoji.CreatedAt = *ai.CreatedAt
}
if ai.Domain != nil {
emoji.Domain = *ai.Domain
}
if ai.ImageRemoteURL != nil {
emoji.ImageRemoteURL = *ai.ImageRemoteURL
}
if ai.ImageStaticRemoteURL != nil {
emoji.ImageStaticRemoteURL = *ai.ImageStaticRemoteURL
}
if ai.Disabled != nil {
emoji.Disabled = *ai.Disabled
}
if ai.VisibleInPicker != nil {
emoji.VisibleInPicker = *ai.VisibleInPicker
}
if ai.CategoryID != nil {
emoji.CategoryID = *ai.CategoryID
}
}
processingEmoji := &ProcessingEmoji{
instanceAccountID: instanceAccount.ID,
emoji: emoji,
data: data,
staticState: int32(received),
database: m.db,
storage: m.storage,
}
return processingEmoji, nil
}

413
internal/media/processingmedia.go Normal file
View file

@ -0,0 +1,413 @@
/*
GoToSocial
Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package media
import (
"bytes"
"context"
"fmt"
"io"
"strings"
"sync"
"sync/atomic"
"time"
"codeberg.org/gruf/go-store/kv"
terminator "github.com/superseriousbusiness/exif-terminator"
"github.com/superseriousbusiness/gotosocial/internal/db"
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
"github.com/superseriousbusiness/gotosocial/internal/id"
"github.com/superseriousbusiness/gotosocial/internal/uris"
)
// ProcessingMedia represents a piece of media that is currently being processed. It exposes
// various functions for retrieving data from the process.
type ProcessingMedia struct {
mu sync.Mutex
/*
below fields should be set on newly created media;
attachment will be updated incrementally as media goes through processing
*/
attachment *gtsmodel.MediaAttachment
data DataFunc
read bool // bool indicating that data function has been triggered already
thumbState int32 // the processing state of the media thumbnail
fullSizeState int32 // the processing state of the full-sized media
/*
below pointers to database and storage are maintained so that
the media can store and update itself during processing steps
*/
database db.DB
storage *kv.KVStore
err error // error created during processing, if any
// track whether this media has already been put in the databse
insertedInDB bool
}
// AttachmentID returns the ID of the underlying media attachment without blocking processing.
func (p *ProcessingMedia) AttachmentID() string {
return p.attachment.ID
}
// LoadAttachment blocks until the thumbnail and fullsize content
// has been processed, and then returns the completed attachment.
func (p *ProcessingMedia) LoadAttachment(ctx context.Context) (*gtsmodel.MediaAttachment, error) {
p.mu.Lock()
defer p.mu.Unlock()
if err := p.store(ctx); err != nil {
return nil, err
}
if err := p.loadThumb(ctx); err != nil {
return nil, err
}
if err := p.loadFullSize(ctx); err != nil {
return nil, err
}
// store the result in the database before returning it
if !p.insertedInDB {
if err := p.database.Put(ctx, p.attachment); err != nil {
return nil, err
}
p.insertedInDB = true
}
return p.attachment, nil
}
// Finished returns true if processing has finished for both the thumbnail
// and full fized version of this piece of media.
func (p *ProcessingMedia) Finished() bool {
return atomic.LoadInt32(&p.thumbState) == int32(complete) && atomic.LoadInt32(&p.fullSizeState) == int32(complete)
}
func (p *ProcessingMedia) loadThumb(ctx context.Context) error {
thumbState := atomic.LoadInt32(&p.thumbState)
switch processState(thumbState) {
case received:
// we haven't processed a thumbnail for this media yet so do it now
// check if we need to create a blurhash or if there's already one set
var createBlurhash bool
if p.attachment.Blurhash == "" {
// no blurhash created yet
createBlurhash = true
}
// stream the original file out of storage...
stored, err := p.storage.GetStream(p.attachment.File.Path)
if err != nil {
p.err = fmt.Errorf("loadThumb: error fetching file from storage: %s", err)
atomic.StoreInt32(&p.thumbState, int32(errored))
return p.err
}
// ... and into the derive thumbnail function
thumb, err := deriveThumbnail(stored, p.attachment.File.ContentType, createBlurhash)
if err != nil {
p.err = fmt.Errorf("loadThumb: error deriving thumbnail: %s", err)
atomic.StoreInt32(&p.thumbState, int32(errored))
return p.err
}
if err := stored.Close(); err != nil {
p.err = fmt.Errorf("loadThumb: error closing stored full size: %s", err)
atomic.StoreInt32(&p.thumbState, int32(errored))
return p.err
}
// put the thumbnail in storage
if err := p.storage.Put(p.attachment.Thumbnail.Path, thumb.small); err != nil {
p.err = fmt.Errorf("loadThumb: error storing thumbnail: %s", err)
atomic.StoreInt32(&p.thumbState, int32(errored))
return p.err
}
// set appropriate fields on the attachment based on the thumbnail we derived
if createBlurhash {
p.attachment.Blurhash = thumb.blurhash
}
p.attachment.FileMeta.Small = gtsmodel.Small{
Width: thumb.width,
Height: thumb.height,
Size: thumb.size,
Aspect: thumb.aspect,
}
p.attachment.Thumbnail.FileSize = len(thumb.small)
// we're done processing the thumbnail!
atomic.StoreInt32(&p.thumbState, int32(complete))
fallthrough
case complete:
return nil
case errored:
return p.err
}
return fmt.Errorf("loadThumb: thumbnail processing status %d unknown", p.thumbState)
}
func (p *ProcessingMedia) loadFullSize(ctx context.Context) error {
fullSizeState := atomic.LoadInt32(&p.fullSizeState)
switch processState(fullSizeState) {
case received:
var err error
var decoded *imageMeta
// stream the original file out of storage...
stored, err := p.storage.GetStream(p.attachment.File.Path)
if err != nil {
p.err = fmt.Errorf("loadFullSize: error fetching file from storage: %s", err)
atomic.StoreInt32(&p.fullSizeState, int32(errored))
return p.err
}
// decode the image
ct := p.attachment.File.ContentType
switch ct {
case mimeImageJpeg, mimeImagePng:
decoded, err = decodeImage(stored, ct)
case mimeImageGif:
decoded, err = decodeGif(stored)
default:
err = fmt.Errorf("loadFullSize: content type %s not a processible image type", ct)
}
if err != nil {
p.err = err
atomic.StoreInt32(&p.fullSizeState, int32(errored))
return p.err
}
if err := stored.Close(); err != nil {
p.err = fmt.Errorf("loadFullSize: error closing stored full size: %s", err)
atomic.StoreInt32(&p.fullSizeState, int32(errored))
return p.err
}
// set appropriate fields on the attachment based on the image we derived
p.attachment.FileMeta.Original = gtsmodel.Original{
Width: decoded.width,
Height: decoded.height,
Size: decoded.size,
Aspect: decoded.aspect,
}
p.attachment.File.UpdatedAt = time.Now()
p.attachment.Processing = gtsmodel.ProcessingStatusProcessed
// we're done processing the full-size image
atomic.StoreInt32(&p.fullSizeState, int32(complete))
fallthrough
case complete:
return nil
case errored:
return p.err
}
return fmt.Errorf("loadFullSize: full size processing status %d unknown", p.fullSizeState)
}
// store calls the data function attached to p if it hasn't been called yet,
// and updates the underlying attachment fields as necessary. It will then stream
// bytes from p's reader directly into storage so that it can be retrieved later.
func (p *ProcessingMedia) store(ctx context.Context) error {
// check if we've already done this and bail early if we have
if p.read {
return nil
}
// execute the data function to get the reader out of it
reader, fileSize, err := p.data(ctx)
if err != nil {
return fmt.Errorf("store: error executing data function: %s", err)
}
// extract no more than 261 bytes from the beginning of the file -- this is the header
firstBytes := make([]byte, maxFileHeaderBytes)
if _, err := reader.Read(firstBytes); err != nil {
return fmt.Errorf("store: error reading initial %d bytes: %s", maxFileHeaderBytes, err)
}
// now we have the file header we can work out the content type from it
contentType, err := parseContentType(firstBytes)
if err != nil {
return fmt.Errorf("store: error parsing content type: %s", err)
}
// bail if this is a type we can't process
if !supportedImage(contentType) {
return fmt.Errorf("store: media type %s not (yet) supported", contentType)
}
// extract the file extension
split := strings.Split(contentType, "/")
if len(split) != 2 {
return fmt.Errorf("store: content type %s was not valid", contentType)
}
extension := split[1] // something like 'jpeg'
// concatenate the cleaned up first bytes with the existing bytes still in the reader (thanks Mara)
multiReader := io.MultiReader(bytes.NewBuffer(firstBytes), reader)
// we'll need to clean exif data from the first bytes; while we're
// here, we can also use the extension to derive the attachment type
var clean io.Reader
switch extension {
case mimeGif:
p.attachment.Type = gtsmodel.FileTypeGif
clean = multiReader // nothing to clean from a gif
case mimeJpeg, mimePng:
p.attachment.Type = gtsmodel.FileTypeImage
purged, err := terminator.Terminate(multiReader, fileSize, extension)
if err != nil {
return fmt.Errorf("store: exif error: %s", err)
}
clean = purged
default:
return fmt.Errorf("store: couldn't process %s", extension)
}
// now set some additional fields on the attachment since
// we know more about what the underlying media actually is
p.attachment.URL = uris.GenerateURIForAttachment(p.attachment.AccountID, string(TypeAttachment), string(SizeOriginal), p.attachment.ID, extension)
p.attachment.File.Path = fmt.Sprintf("%s/%s/%s/%s.%s", p.attachment.AccountID, TypeAttachment, SizeOriginal, p.attachment.ID, extension)
p.attachment.File.ContentType = contentType
p.attachment.File.FileSize = fileSize
// store this for now -- other processes can pull it out of storage as they please
if err := p.storage.PutStream(p.attachment.File.Path, clean); err != nil {
return fmt.Errorf("store: error storing stream: %s", err)
}
// if the original reader is a readcloser, close it since we're done with it now
if rc, ok := reader.(io.ReadCloser); ok {
if err := rc.Close(); err != nil {
return fmt.Errorf("store: error closing readcloser: %s", err)
}
}
p.read = true
return nil
}
func (m *manager) preProcessMedia(ctx context.Context, data DataFunc, accountID string, ai *AdditionalMediaInfo) (*ProcessingMedia, error) {
id, err := id.NewRandomULID()
if err != nil {
return nil, err
}
file := gtsmodel.File{
Path: "", // we don't know yet because it depends on the uncalled DataFunc
ContentType: "", // we don't know yet because it depends on the uncalled DataFunc
UpdatedAt: time.Now(),
}
thumbnail := gtsmodel.Thumbnail{
URL: uris.GenerateURIForAttachment(accountID, string(TypeAttachment), string(SizeSmall), id, mimeJpeg), // all thumbnails are encoded as jpeg,
Path: fmt.Sprintf("%s/%s/%s/%s.%s", accountID, TypeAttachment, SizeSmall, id, mimeJpeg), // all thumbnails are encoded as jpeg,
ContentType: mimeJpeg,
UpdatedAt: time.Now(),
}
// populate initial fields on the media attachment -- some of these will be overwritten as we proceed
attachment := &gtsmodel.MediaAttachment{
ID: id,
CreatedAt: time.Now(),
UpdatedAt: time.Now(),
StatusID: "",
URL: "", // we don't know yet because it depends on the uncalled DataFunc
RemoteURL: "",
Type: gtsmodel.FileTypeUnknown, // we don't know yet because it depends on the uncalled DataFunc
FileMeta: gtsmodel.FileMeta{},
AccountID: accountID,
Description: "",
ScheduledStatusID: "",
Blurhash: "",
Processing: gtsmodel.ProcessingStatusReceived,
File: file,
Thumbnail: thumbnail,
Avatar: false,
Header: false,
}
// check if we have additional info to add to the attachment,
// and overwrite some of the attachment fields if so
if ai != nil {
if ai.CreatedAt != nil {
attachment.CreatedAt = *ai.CreatedAt
}
if ai.StatusID != nil {
attachment.StatusID = *ai.StatusID
}
if ai.RemoteURL != nil {
attachment.RemoteURL = *ai.RemoteURL
}
if ai.Description != nil {
attachment.Description = *ai.Description
}
if ai.ScheduledStatusID != nil {
attachment.ScheduledStatusID = *ai.ScheduledStatusID
}
if ai.Blurhash != nil {
attachment.Blurhash = *ai.Blurhash
}
if ai.Avatar != nil {
attachment.Avatar = *ai.Avatar
}
if ai.Header != nil {
attachment.Header = *ai.Header
}
if ai.FocusX != nil {
attachment.FileMeta.Focus.X = *ai.FocusX
}
if ai.FocusY != nil {
attachment.FileMeta.Focus.Y = *ai.FocusY
}
}
processingMedia := &ProcessingMedia{
attachment: attachment,
data: data,
thumbState: int32(received),
fullSizeState: int32(received),
database: m.db,
storage: m.storage,
}
return processingMedia, nil
}

23
internal/media/processvideo.go
View file

@ -1,23 +0,0 @@
/*
GoToSocial
Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package media
// func (mh *mediaHandler) processVideoAttachment(data []byte, accountID string, contentType string, remoteURL string) (*gtsmodel.MediaAttachment, error) {
// return nil, nil
// }

1
internal/media/test/test-corrupted.jpg
View file

File diff suppressed because one or more lines are too long

BIN
internal/media/test/test-jpeg-blurhash.jpg
View file

Binary file not shown.
Side by side

Before

Width:  |  Height:  |  Size: 8.6 KiB

BIN
internal/media/test/test-with-exif.jpg
View file

Binary file not shown.
Side by side

Before

Width:  |  Height:  |  Size: 1.2 MiB

BIN
internal/media/test/test-without-exif.jpg
View file

Binary file not shown.
Side by side

Before

Width:  |  Height:  |  Size: 1.2 MiB

121
internal/media/types.go Normal file
View file

@ -0,0 +1,121 @@
/*
GoToSocial
Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package media
import (
"context"
"io"
"time"
)
// maxFileHeaderBytes represents the maximum amount of bytes we want
// to examine from the beginning of a file to determine its type.
//
// See: https://en.wikipedia.org/wiki/File_format#File_header
// and https://github.com/h2non/filetype
const maxFileHeaderBytes = 261
// mime consts
const (
mimeImage = "image"
mimeJpeg = "jpeg"
mimeImageJpeg = mimeImage + "/" + mimeJpeg
mimeGif = "gif"
mimeImageGif = mimeImage + "/" + mimeGif
mimePng = "png"
mimeImagePng = mimeImage + "/" + mimePng
)
type processState int32
const (
received processState = iota // processing order has been received but not done yet
complete // processing order has been completed successfully
errored // processing order has been completed with an error
)
// EmojiMaxBytes is the maximum permitted bytes of an emoji upload (50kb)
// const EmojiMaxBytes = 51200
type Size string
const (
SizeSmall Size = "small" // SizeSmall is the key for small/thumbnail versions of media
SizeOriginal Size = "original" // SizeOriginal is the key for original/fullsize versions of media and emoji
SizeStatic Size = "static" // SizeStatic is the key for static (non-animated) versions of emoji
)
type Type string
const (
TypeAttachment Type = "attachment" // TypeAttachment is the key for media attachments
TypeHeader Type = "header" // TypeHeader is the key for profile header requests
TypeAvatar Type = "avatar" // TypeAvatar is the key for profile avatar requests
TypeEmoji Type = "emoji" // TypeEmoji is the key for emoji type requests
)
// AdditionalMediaInfo represents additional information that should be added to an attachment
// when processing a piece of media.
type AdditionalMediaInfo struct {
// Time that this media was created; defaults to time.Now().
CreatedAt *time.Time
// ID of the status to which this media is attached; defaults to "".
StatusID *string
// URL of the media on a remote instance; defaults to "".
RemoteURL *string
// Image description of this media; defaults to "".
Description *string
// Blurhash of this media; defaults to "".
Blurhash *string
// ID of the scheduled status to which this media is attached; defaults to "".
ScheduledStatusID *string
// Mark this media as in-use as an avatar; defaults to false.
Avatar *bool
// Mark this media as in-use as a header; defaults to false.
Header *bool
// X focus coordinate for this media; defaults to 0.
FocusX *float32
// Y focus coordinate for this media; defaults to 0.
FocusY *float32
}
// AdditionalMediaInfo represents additional information
// that should be added to an emoji when processing it.
type AdditionalEmojiInfo struct {
// Time that this emoji was created; defaults to time.Now().
CreatedAt *time.Time
// Domain the emoji originated from. Blank for this instance's domain. Defaults to "".
Domain *string
// URL of this emoji on a remote instance; defaults to "".
ImageRemoteURL *string
// URL of the static version of this emoji on a remote instance; defaults to "".
ImageStaticRemoteURL *string
// Whether this emoji should be disabled (not shown) on this instance; defaults to false.
Disabled *bool
// Whether this emoji should be visible in the instance's emoji picker; defaults to true.
VisibleInPicker *bool
// ID of the category this emoji should be placed in; defaults to "".
CategoryID *string
}
// DataFunc represents a function used to retrieve the raw bytes of a piece of media.
type DataFunc func(ctx context.Context) (reader io.Reader, fileSize int, err error)

247
internal/media/util.go
View file

@ -19,50 +19,22 @@
package media
import (
"bytes"
"errors"
"fmt"
"image"
"image/gif"
"image/jpeg"
"image/png"
"github.com/buckket/go-blurhash"
"github.com/h2non/filetype"
"github.com/nfnt/resize"
"github.com/superseriousbusiness/exifremove/pkg/exifremove"
)
const (
// MIMEImage is the mime type for image
MIMEImage = "image"
// MIMEJpeg is the jpeg image mime type
MIMEJpeg = "image/jpeg"
// MIMEGif is the gif image mime type
MIMEGif = "image/gif"
// MIMEPng is the png image mime type
MIMEPng = "image/png"
// MIMEVideo is the mime type for video
MIMEVideo = "video"
// MIMEMp4 is the mp4 video mime type
MIMEMp4 = "video/mp4"
// MIMEMpeg is the mpeg video mime type
MIMEMpeg = "video/mpeg"
// MIMEWebm is the webm video mime type
MIMEWebm = "video/webm"
)
// parseContentType parses the MIME content type from a file, returning it as a string in the form (eg., "image/jpeg").
// Returns an error if the content type is not something we can process.
func parseContentType(content []byte) (string, error) {
head := make([]byte, 261)
_, err := bytes.NewReader(content).Read(head)
if err != nil {
return "", fmt.Errorf("could not read first magic bytes of file: %s", err)
//
// Fileheader should be no longer than 262 bytes; anything more than this is inefficient.
func parseContentType(fileHeader []byte) (string, error) {
if fhLength := len(fileHeader); fhLength > maxFileHeaderBytes {
return "", fmt.Errorf("parseContentType requires %d bytes max, we got %d", maxFileHeaderBytes, fhLength)
}
kind, err := filetype.Match(head)
kind, err := filetype.Match(fileHeader)
if err != nil {
return "", err
}
@ -74,13 +46,13 @@ func parseContentType(content []byte) (string, error) {
return kind.MIME.Value, nil
}
// SupportedImageType checks mime type of an image against a slice of accepted types,
// supportedImage checks mime type of an image against a slice of accepted types,
// and returns True if the mime type is accepted.
func SupportedImageType(mimeType string) bool {
func supportedImage(mimeType string) bool {
acceptedImageTypes := []string{
MIMEJpeg,
MIMEGif,
MIMEPng,
mimeImageJpeg,
mimeImageGif,
mimeImagePng,
}
for _, accepted := range acceptedImageTypes {
if mimeType == accepted {
@ -90,27 +62,11 @@ func SupportedImageType(mimeType string) bool {
return false
}
// SupportedVideoType checks mime type of a video against a slice of accepted types,
// and returns True if the mime type is accepted.
func SupportedVideoType(mimeType string) bool {
acceptedVideoTypes := []string{
MIMEMp4,
MIMEMpeg,
MIMEWebm,
}
for _, accepted := range acceptedVideoTypes {
if mimeType == accepted {
return true
}
}
return false
}
// supportedEmojiType checks that the content type is image/png -- the only type supported for emoji.
func supportedEmojiType(mimeType string) bool {
// supportedEmoji checks that the content type is image/png or image/gif -- the only types supported for emoji.
func supportedEmoji(mimeType string) bool {
acceptedEmojiTypes := []string{
MIMEGif,
MIMEPng,
mimeImageGif,
mimeImagePng,
}
for _, accepted := range acceptedEmojiTypes {
if mimeType == accepted {
@ -120,179 +76,6 @@ func supportedEmojiType(mimeType string) bool {
return false
}
// purgeExif is a little wrapper for the action of removing exif data from an image.
// Only pass pngs or jpegs to this function.
func purgeExif(b []byte) ([]byte, error) {
if len(b) == 0 {
return nil, errors.New("passed image was not valid")
}
clean, err := exifremove.Remove(b)
if err != nil {
return nil, fmt.Errorf("could not purge exif from image: %s", err)
}
if len(clean) == 0 {
return nil, errors.New("purged image was not valid")
}
return clean, nil
}
func deriveGif(b []byte, extension string) (*imageAndMeta, error) {
var g *gif.GIF
var err error
switch extension {
case MIMEGif:
g, err = gif.DecodeAll(bytes.NewReader(b))
if err != nil {
return nil, err
}
default:
return nil, fmt.Errorf("extension %s not recognised", extension)
}
// use the first frame to get the static characteristics
width := g.Config.Width
height := g.Config.Height
size := width * height
aspect := float64(width) / float64(height)
return &imageAndMeta{
image: b,
width: width,
height: height,
size: size,
aspect: aspect,
}, nil
}
func deriveImage(b []byte, contentType string) (*imageAndMeta, error) {
var i image.Image
var err error
switch contentType {
case MIMEJpeg:
i, err = jpeg.Decode(bytes.NewReader(b))
if err != nil {
return nil, err
}
case MIMEPng:
i, err = png.Decode(bytes.NewReader(b))
if err != nil {
return nil, err
}
default:
return nil, fmt.Errorf("content type %s not recognised", contentType)
}
width := i.Bounds().Size().X
height := i.Bounds().Size().Y
size := width * height
aspect := float64(width) / float64(height)
return &imageAndMeta{
image: b,
width: width,
height: height,
size: size,
aspect: aspect,
}, nil
}
// deriveThumbnail returns a byte slice and metadata for a thumbnail of width x and height y,
// of a given jpeg, png, or gif, or an error if something goes wrong.
//
// Note that the aspect ratio of the image will be retained,
// so it will not necessarily be a square, even if x and y are set as the same value.
func deriveThumbnail(b []byte, contentType string, x uint, y uint) (*imageAndMeta, error) {
var i image.Image
var err error
switch contentType {
case MIMEJpeg:
i, err = jpeg.Decode(bytes.NewReader(b))
if err != nil {
return nil, err
}
case MIMEPng:
i, err = png.Decode(bytes.NewReader(b))
if err != nil {
return nil, err
}
case MIMEGif:
i, err = gif.Decode(bytes.NewReader(b))
if err != nil {
return nil, err
}
default:
return nil, fmt.Errorf("content type %s not recognised", contentType)
}
thumb := resize.Thumbnail(x, y, i, resize.NearestNeighbor)
width := thumb.Bounds().Size().X
height := thumb.Bounds().Size().Y
size := width * height
aspect := float64(width) / float64(height)
tiny := resize.Thumbnail(32, 32, thumb, resize.NearestNeighbor)
bh, err := blurhash.Encode(4, 3, tiny)
if err != nil {
return nil, err
}
out := &bytes.Buffer{}
if err := jpeg.Encode(out, thumb, &jpeg.Options{
Quality: 75,
}); err != nil {
return nil, err
}
return &imageAndMeta{
image: out.Bytes(),
width: width,
height: height,
size: size,
aspect: aspect,
blurhash: bh,
}, nil
}
// deriveStaticEmojji takes a given gif or png of an emoji, decodes it, and re-encodes it as a static png.
func deriveStaticEmoji(b []byte, contentType string) (*imageAndMeta, error) {
var i image.Image
var err error
switch contentType {
case MIMEPng:
i, err = png.Decode(bytes.NewReader(b))
if err != nil {
return nil, err
}
case MIMEGif:
i, err = gif.Decode(bytes.NewReader(b))
if err != nil {
return nil, err
}
default:
return nil, fmt.Errorf("content type %s not allowed for emoji", contentType)
}
out := &bytes.Buffer{}
if err := png.Encode(out, i); err != nil {
return nil, err
}
return &imageAndMeta{
image: out.Bytes(),
}, nil
}
type imageAndMeta struct {
image []byte
width int
height int
size int
aspect float64
blurhash string
}
// ParseMediaType converts s to a recognized MediaType, or returns an error if unrecognized
func ParseMediaType(s string) (Type, error) {
switch s {

150
internal/media/util_test.go
View file

@ -1,150 +0,0 @@
/*
GoToSocial
Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package media
import (
"io/ioutil"
"testing"
"github.com/spf13/viper"
"github.com/superseriousbusiness/gotosocial/internal/config"
"github.com/superseriousbusiness/gotosocial/internal/log"
"github.com/stretchr/testify/suite"
)
type MediaUtilTestSuite struct {
suite.Suite
}
/*
TEST INFRASTRUCTURE
*/
// SetupSuite sets some variables on the suite that we can use as consts (more or less) throughout
func (suite *MediaUtilTestSuite) SetupSuite() {
// doesn't use testrig.InitTestLog() helper to prevent import cycle
viper.Set(config.Keys.LogLevel, "trace")
err := log.Initialize()
if err != nil {
panic(err)
}
}
func (suite *MediaUtilTestSuite) TearDownSuite() {
}
// SetupTest creates a db connection and creates necessary tables before each test
func (suite *MediaUtilTestSuite) SetupTest() {
}
// TearDownTest drops tables to make sure there's no data in the db
func (suite *MediaUtilTestSuite) TearDownTest() {
}
/*
ACTUAL TESTS
*/
func (suite *MediaUtilTestSuite) TestParseContentTypeOK() {
f, err := ioutil.ReadFile("./test/test-jpeg.jpg")
suite.NoError(err)
ct, err := parseContentType(f)
suite.NoError(err)
suite.Equal("image/jpeg", ct)
}
func (suite *MediaUtilTestSuite) TestParseContentTypeNotOK() {
f, err := ioutil.ReadFile("./test/test-corrupted.jpg")
suite.NoError(err)
ct, err := parseContentType(f)
suite.NotNil(err)
suite.Equal("", ct)
suite.Equal("filetype unknown", err.Error())
}
func (suite *MediaUtilTestSuite) TestRemoveEXIF() {
// load and validate image
b, err := ioutil.ReadFile("./test/test-with-exif.jpg")
suite.NoError(err)
// clean it up and validate the clean version
clean, err := purgeExif(b)
suite.NoError(err)
// compare it to our stored sample
sampleBytes, err := ioutil.ReadFile("./test/test-without-exif.jpg")
suite.NoError(err)
suite.EqualValues(sampleBytes, clean)
}
func (suite *MediaUtilTestSuite) TestDeriveImageFromJPEG() {
// load image
b, err := ioutil.ReadFile("./test/test-jpeg.jpg")
suite.NoError(err)
// clean it up and validate the clean version
imageAndMeta, err := deriveImage(b, "image/jpeg")
suite.NoError(err)
suite.Equal(1920, imageAndMeta.width)
suite.Equal(1080, imageAndMeta.height)
suite.Equal(1.7777777777777777, imageAndMeta.aspect)
suite.Equal(2073600, imageAndMeta.size)
// assert that the final image is what we would expect
sampleBytes, err := ioutil.ReadFile("./test/test-jpeg-processed.jpg")
suite.NoError(err)
suite.EqualValues(sampleBytes, imageAndMeta.image)
}
func (suite *MediaUtilTestSuite) TestDeriveThumbnailFromJPEG() {
// load image
b, err := ioutil.ReadFile("./test/test-jpeg.jpg")
suite.NoError(err)
// clean it up and validate the clean version
imageAndMeta, err := deriveThumbnail(b, "image/jpeg", 512, 512)
suite.NoError(err)
suite.Equal(512, imageAndMeta.width)
suite.Equal(288, imageAndMeta.height)
suite.Equal(1.7777777777777777, imageAndMeta.aspect)
suite.Equal(147456, imageAndMeta.size)
suite.Equal("LjBzUo#6RQR._NvzRjWF?urqV@a$", imageAndMeta.blurhash)
sampleBytes, err := ioutil.ReadFile("./test/test-jpeg-thumbnail.jpg")
suite.NoError(err)
suite.EqualValues(sampleBytes, imageAndMeta.image)
}
func (suite *MediaUtilTestSuite) TestSupportedImageTypes() {
ok := SupportedImageType("image/jpeg")
suite.True(ok)
ok = SupportedImageType("image/bmp")
suite.False(ok)
}
func TestMediaUtilTestSuite(t *testing.T) {
suite.Run(t, new(MediaUtilTestSuite))
}

6
internal/processing/account/account.go
View file

@ -77,7 +77,7 @@ type Processor interface {
type processor struct {
tc typeutils.TypeConverter
mediaHandler media.Handler
mediaManager media.Manager
fromClientAPI chan messages.FromClientAPI
oauthServer oauth.Server
filter visibility.Filter
@ -87,10 +87,10 @@ type processor struct {
}
// New returns a new account processor.
func New(db db.DB, tc typeutils.TypeConverter, mediaHandler media.Handler, oauthServer oauth.Server, fromClientAPI chan messages.FromClientAPI, federator federation.Federator) Processor {
func New(db db.DB, tc typeutils.TypeConverter, mediaManager media.Manager, oauthServer oauth.Server, fromClientAPI chan messages.FromClientAPI, federator federation.Federator) Processor {
return &processor{
tc: tc,
mediaHandler: mediaHandler,
mediaManager: mediaManager,
fromClientAPI: fromClientAPI,
oauthServer: oauthServer,
filter: visibility.NewFilter(db),

8
internal/processing/account/account_test.go
View file

@ -41,7 +41,7 @@ type AccountStandardTestSuite struct {
db db.DB
tc typeutils.TypeConverter
storage *kv.KVStore
mediaHandler media.Handler
mediaManager media.Manager
oauthServer oauth.Server
fromClientAPIChan chan messages.FromClientAPI
httpClient pub.HttpClient
@ -80,15 +80,15 @@ func (suite *AccountStandardTestSuite) SetupTest() {
suite.db = testrig.NewTestDB()
suite.tc = testrig.NewTestTypeConverter(suite.db)
suite.storage = testrig.NewTestStorage()
suite.mediaHandler = testrig.NewTestMediaHandler(suite.db, suite.storage)
suite.mediaManager = testrig.NewTestMediaManager(suite.db, suite.storage)
suite.oauthServer = testrig.NewTestOauthServer(suite.db)
suite.fromClientAPIChan = make(chan messages.FromClientAPI, 100)
suite.httpClient = testrig.NewMockHTTPClient(nil)
suite.transportController = testrig.NewTestTransportController(suite.httpClient, suite.db)
suite.federator = testrig.NewTestFederator(suite.db, suite.transportController, suite.storage)
suite.federator = testrig.NewTestFederator(suite.db, suite.transportController, suite.storage, suite.mediaManager)
suite.sentEmails = make(map[string]string)
suite.emailSender = testrig.NewEmailSender("../../../web/template/", suite.sentEmails)
suite.accountProcessor = account.New(suite.db, suite.tc, suite.mediaHandler, suite.oauthServer, suite.fromClientAPIChan, suite.federator)
suite.accountProcessor = account.New(suite.db, suite.tc, suite.mediaManager, suite.oauthServer, suite.fromClientAPIChan, suite.federator)
testrig.StandardDBSetup(suite.db, nil)
testrig.StandardStorageSetup(suite.storage, "../../../testrig/media")
}

8
internal/processing/account/get.go
View file

@ -22,6 +22,7 @@ import (
"context"
"errors"
"fmt"
"net/url"
apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model"
"github.com/superseriousbusiness/gotosocial/internal/db"
@ -56,7 +57,12 @@ func (p *processor) Get(ctx context.Context, requestingAccount *gtsmodel.Account
// last-minute check to make sure we have remote account header/avi cached
if targetAccount.Domain != "" {
a, err := p.federator.EnrichRemoteAccount(ctx, requestingAccount.Username, targetAccount)
targetAccountURI, err := url.Parse(targetAccount.URI)
if err != nil {
return nil, fmt.Errorf("error parsing url %s: %s", targetAccount.URI, err)
}
a, err := p.federator.GetRemoteAccount(ctx, requestingAccount.Username, targetAccountURI, true, false)
if err == nil {
targetAccount = a
}

71
internal/processing/account/update.go
View file

@ -19,9 +19,7 @@
package account
import (
"bytes"
"context"
"errors"
"fmt"
"io"
"mime/multipart"
@ -137,68 +135,57 @@ func (p *processor) Update(ctx context.Context, account *gtsmodel.Account, form
// parsing and checking the image, and doing the necessary updates in the database for this to become
// the account's new avatar image.
func (p *processor) UpdateAvatar(ctx context.Context, avatar *multipart.FileHeader, accountID string) (*gtsmodel.MediaAttachment, error) {
var err error
maxImageSize := viper.GetInt(config.Keys.MediaImageMaxSize)
if int(avatar.Size) > maxImageSize {
err = fmt.Errorf("avatar with size %d exceeded max image size of %d bytes", avatar.Size, maxImageSize)
return nil, err
}
f, err := avatar.Open()
if err != nil {
return nil, fmt.Errorf("could not read provided avatar: %s", err)
return nil, fmt.Errorf("UpdateAvatar: avatar with size %d exceeded max image size of %d bytes", avatar.Size, maxImageSize)
}
// extract the bytes
buf := new(bytes.Buffer)
size, err := io.Copy(buf, f)
if err != nil {
return nil, fmt.Errorf("could not read provided avatar: %s", err)
}
if size == 0 {
return nil, errors.New("could not read provided avatar: size 0 bytes")
dataFunc := func(innerCtx context.Context) (io.Reader, int, error) {
f, err := avatar.Open()
return f, int(avatar.Size), err
}
// do the setting
avatarInfo, err := p.mediaHandler.ProcessHeaderOrAvatar(ctx, buf.Bytes(), accountID, media.TypeAvatar, "")
if err != nil {
return nil, fmt.Errorf("error processing avatar: %s", err)
isAvatar := true
ai := &media.AdditionalMediaInfo{
Avatar: &isAvatar,
}
return avatarInfo, f.Close()
processingMedia, err := p.mediaManager.ProcessMedia(ctx, dataFunc, accountID, ai)
if err != nil {
return nil, fmt.Errorf("UpdateAvatar: error processing avatar: %s", err)
}
return processingMedia.LoadAttachment(ctx)
}
// UpdateHeader does the dirty work of checking the header part of an account update form,
// parsing and checking the image, and doing the necessary updates in the database for this to become
// the account's new header image.
func (p *processor) UpdateHeader(ctx context.Context, header *multipart.FileHeader, accountID string) (*gtsmodel.MediaAttachment, error) {
var err error
maxImageSize := viper.GetInt(config.Keys.MediaImageMaxSize)
if int(header.Size) > maxImageSize {
err = fmt.Errorf("header with size %d exceeded max image size of %d bytes", header.Size, maxImageSize)
return nil, err
}
f, err := header.Open()
if err != nil {
return nil, fmt.Errorf("could not read provided header: %s", err)
return nil, fmt.Errorf("UpdateHeader: header with size %d exceeded max image size of %d bytes", header.Size, maxImageSize)
}
// extract the bytes
buf := new(bytes.Buffer)
size, err := io.Copy(buf, f)
if err != nil {
return nil, fmt.Errorf("could not read provided header: %s", err)
}
if size == 0 {
return nil, errors.New("could not read provided header: size 0 bytes")
dataFunc := func(innerCtx context.Context) (io.Reader, int, error) {
f, err := header.Open()
return f, int(header.Size), err
}
// do the setting
headerInfo, err := p.mediaHandler.ProcessHeaderOrAvatar(ctx, buf.Bytes(), accountID, media.TypeHeader, "")
if err != nil {
return nil, fmt.Errorf("error processing header: %s", err)
isHeader := true
ai := &media.AdditionalMediaInfo{
Header: &isHeader,
}
return headerInfo, f.Close()
processingMedia, err := p.mediaManager.ProcessMedia(ctx, dataFunc, accountID, ai)
if err != nil {
return nil, fmt.Errorf("UpdateHeader: error processing header: %s", err)
}
if err != nil {
return nil, fmt.Errorf("UpdateHeader: error processing header: %s", err)
}
return processingMedia.LoadAttachment(ctx)
}
func (p *processor) processNote(ctx context.Context, note string, accountID string) (string, error) {

2
internal/processing/admin.go
View file

@ -26,7 +26,7 @@ import (
"github.com/superseriousbusiness/gotosocial/internal/oauth"
)
func (p *processor) AdminEmojiCreate(ctx context.Context, authed *oauth.Auth, form *apimodel.EmojiCreateRequest) (*apimodel.Emoji, error) {
func (p *processor) AdminEmojiCreate(ctx context.Context, authed *oauth.Auth, form *apimodel.EmojiCreateRequest) (*apimodel.Emoji, gtserror.WithCode) {
return p.adminProcessor.EmojiCreate(ctx, authed.Account, authed.User, form)
}

8
internal/processing/admin/admin.go
View file

@ -38,21 +38,21 @@ type Processor interface {
DomainBlocksGet(ctx context.Context, account *gtsmodel.Account, export bool) ([]*apimodel.DomainBlock, gtserror.WithCode)
DomainBlockGet(ctx context.Context, account *gtsmodel.Account, id string, export bool) (*apimodel.DomainBlock, gtserror.WithCode)
DomainBlockDelete(ctx context.Context, account *gtsmodel.Account, id string) (*apimodel.DomainBlock, gtserror.WithCode)
EmojiCreate(ctx context.Context, account *gtsmodel.Account, user *gtsmodel.User, form *apimodel.EmojiCreateRequest) (*apimodel.Emoji, error)
EmojiCreate(ctx context.Context, account *gtsmodel.Account, user *gtsmodel.User, form *apimodel.EmojiCreateRequest) (*apimodel.Emoji, gtserror.WithCode)
}
type processor struct {
tc typeutils.TypeConverter
mediaHandler media.Handler
mediaManager media.Manager
fromClientAPI chan messages.FromClientAPI
db db.DB
}
// New returns a new admin processor.
func New(db db.DB, tc typeutils.TypeConverter, mediaHandler media.Handler, fromClientAPI chan messages.FromClientAPI) Processor {
func New(db db.DB, tc typeutils.TypeConverter, mediaManager media.Manager, fromClientAPI chan messages.FromClientAPI) Processor {
return &processor{
tc: tc,
mediaHandler: mediaHandler,
mediaManager: mediaManager,
fromClientAPI: fromClientAPI,
db: db,
}

52
internal/processing/admin/emoji.go
View file

@ -19,55 +19,53 @@
package admin
import (
"bytes"
"context"
"errors"
"fmt"
"io"
apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model"
"github.com/superseriousbusiness/gotosocial/internal/db"
"github.com/superseriousbusiness/gotosocial/internal/gtserror"
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
"github.com/superseriousbusiness/gotosocial/internal/id"
"github.com/superseriousbusiness/gotosocial/internal/uris"
)
func (p *processor) EmojiCreate(ctx context.Context, account *gtsmodel.Account, user *gtsmodel.User, form *apimodel.EmojiCreateRequest) (*apimodel.Emoji, error) {
if user.Admin {
return nil, fmt.Errorf("user %s not an admin", user.ID)
func (p *processor) EmojiCreate(ctx context.Context, account *gtsmodel.Account, user *gtsmodel.User, form *apimodel.EmojiCreateRequest) (*apimodel.Emoji, gtserror.WithCode) {
if !user.Admin {
return nil, gtserror.NewErrorNotAuthorized(fmt.Errorf("user %s not an admin", user.ID), "user is not an admin")
}
// open the emoji and extract the bytes from it
f, err := form.Image.Open()
if err != nil {
return nil, fmt.Errorf("error opening emoji: %s", err)
}
buf := new(bytes.Buffer)
size, err := io.Copy(buf, f)
if err != nil {
return nil, fmt.Errorf("error reading emoji: %s", err)
}
if size == 0 {
return nil, errors.New("could not read provided emoji: size 0 bytes")
data := func(innerCtx context.Context) (io.Reader, int, error) {
f, err := form.Image.Open()
return f, int(form.Image.Size), err
}
// allow the mediaHandler to work its magic of processing the emoji bytes, and putting them in whatever storage backend we're using
emoji, err := p.mediaHandler.ProcessLocalEmoji(ctx, buf.Bytes(), form.Shortcode)
emojiID, err := id.NewRandomULID()
if err != nil {
return nil, fmt.Errorf("error reading emoji: %s", err)
return nil, gtserror.NewErrorInternalError(fmt.Errorf("error creating id for new emoji: %s", err), "error creating emoji ID")
}
emojiID, err := id.NewULID()
emojiURI := uris.GenerateURIForEmoji(emojiID)
processingEmoji, err := p.mediaManager.ProcessEmoji(ctx, data, form.Shortcode, emojiID, emojiURI, nil)
if err != nil {
return nil, err
return nil, gtserror.NewErrorInternalError(fmt.Errorf("error processing emoji: %s", err), "error processing emoji")
}
emoji, err := processingEmoji.LoadEmoji(ctx)
if err != nil {
var alreadyExistsError *db.ErrAlreadyExists
if errors.As(err, &alreadyExistsError) {
return nil, gtserror.NewErrorConflict(fmt.Errorf("emoji with shortcode %s already exists", form.Shortcode), fmt.Sprintf("emoji with shortcode %s already exists", form.Shortcode))
}
return nil, gtserror.NewErrorInternalError(fmt.Errorf("error loading emoji: %s", err), "error loading emoji")
}
emoji.ID = emojiID
apiEmoji, err := p.tc.EmojiToAPIEmoji(ctx, emoji)
if err != nil {
return nil, fmt.Errorf("error converting emoji to apitype: %s", err)
}
if err := p.db.Put(ctx, emoji); err != nil {
return nil, fmt.Errorf("database error while processing emoji: %s", err)
return nil, gtserror.NewErrorInternalError(fmt.Errorf("error converting emoji: %s", err), "error converting emoji to api representation")
}
return &apiEmoji, nil

2
internal/processing/federation/getfollowers.go
View file

@ -41,7 +41,7 @@ func (p *processor) GetFollowers(ctx context.Context, requestedUsername string,
return nil, gtserror.NewErrorNotAuthorized(errors.New("not authorized"), "not authorized")
}
requestingAccount, _, err := p.federator.GetRemoteAccount(ctx, requestedUsername, requestingAccountURI, false)
requestingAccount, err := p.federator.GetRemoteAccount(ctx, requestedUsername, requestingAccountURI, false, false)
if err != nil {
return nil, gtserror.NewErrorNotAuthorized(err)
}

2
internal/processing/federation/getfollowing.go
View file

@ -41,7 +41,7 @@ func (p *processor) GetFollowing(ctx context.Context, requestedUsername string,
return nil, gtserror.NewErrorNotAuthorized(errors.New("not authorized"), "not authorized")
}
requestingAccount, _, err := p.federator.GetRemoteAccount(ctx, requestedUsername, requestingAccountURI, false)
requestingAccount, err := p.federator.GetRemoteAccount(ctx, requestedUsername, requestingAccountURI, false, false)
if err != nil {
return nil, gtserror.NewErrorNotAuthorized(err)
}

2
internal/processing/federation/getoutbox.go
View file

@ -42,7 +42,7 @@ func (p *processor) GetOutbox(ctx context.Context, requestedUsername string, pag
return nil, gtserror.NewErrorNotAuthorized(errors.New("not authorized"), "not authorized")
}
requestingAccount, _, err := p.federator.GetRemoteAccount(ctx, requestedUsername, requestingAccountURI, false)
requestingAccount, err := p.federator.GetRemoteAccount(ctx, requestedUsername, requestingAccountURI, false, false)
if err != nil {
return nil, gtserror.NewErrorNotAuthorized(err)
}

2
internal/processing/federation/getstatus.go
View file

@ -43,7 +43,7 @@ func (p *processor) GetStatus(ctx context.Context, requestedUsername string, req
return nil, gtserror.NewErrorNotAuthorized(errors.New("not authorized"), "not authorized")
}
requestingAccount, _, err := p.federator.GetRemoteAccount(ctx, requestedUsername, requestingAccountURI, false)
requestingAccount, err := p.federator.GetRemoteAccount(ctx, requestedUsername, requestingAccountURI, false, false)
if err != nil {
return nil, gtserror.NewErrorNotAuthorized(err)
}

2
internal/processing/federation/getstatusreplies.go
View file

@ -43,7 +43,7 @@ func (p *processor) GetStatusReplies(ctx context.Context, requestedUsername stri
return nil, gtserror.NewErrorNotAuthorized(errors.New("not authorized"), "not authorized")
}
requestingAccount, _, err := p.federator.GetRemoteAccount(ctx, requestedUsername, requestingAccountURI, false)
requestingAccount, err := p.federator.GetRemoteAccount(ctx, requestedUsername, requestingAccountURI, false, false)
if err != nil {
return nil, gtserror.NewErrorNotAuthorized(err)
}

2
internal/processing/federation/getuser.go
View file

@ -54,7 +54,7 @@ func (p *processor) GetUser(ctx context.Context, requestedUsername string, reque
// if we're not already handshaking/dereferencing a remote account, dereference it now
if !p.federator.Handshaking(ctx, requestedUsername, requestingAccountURI) {
requestingAccount, _, err := p.federator.GetRemoteAccount(ctx, requestedUsername, requestingAccountURI, false)
requestingAccount, err := p.federator.GetRemoteAccount(ctx, requestedUsername, requestingAccountURI, false, false)
if err != nil {
return nil, gtserror.NewErrorNotAuthorized(err)
}

109
internal/processing/fromfederator.go
View file

@ -22,6 +22,7 @@ import (
"context"
"errors"
"fmt"
"net/url"
"github.com/sirupsen/logrus"
"github.com/superseriousbusiness/gotosocial/internal/ap"
@ -114,6 +115,30 @@ func (p *processor) processCreateStatusFromFederator(ctx context.Context, federa
}
}
// make sure the account is pinned
if status.Account == nil {
a, err := p.db.GetAccountByID(ctx, status.AccountID)
if err != nil {
return err
}
status.Account = a
}
// do a BLOCKING get of the remote account to make sure the avi and header are cached
if status.Account.Domain != "" {
remoteAccountID, err := url.Parse(status.Account.URI)
if err != nil {
return err
}
a, err := p.federator.GetRemoteAccount(ctx, federatorMsg.ReceivingAccount.Username, remoteAccountID, true, false)
if err != nil {
return err
}
status.Account = a
}
if err := p.timelineStatus(ctx, status); err != nil {
return err
}
@ -132,6 +157,30 @@ func (p *processor) processCreateFaveFromFederator(ctx context.Context, federato
return errors.New("like was not parseable as *gtsmodel.StatusFave")
}
// make sure the account is pinned
if incomingFave.Account == nil {
a, err := p.db.GetAccountByID(ctx, incomingFave.AccountID)
if err != nil {
return err
}
incomingFave.Account = a
}
// do a BLOCKING get of the remote account to make sure the avi and header are cached
if incomingFave.Account.Domain != "" {
remoteAccountID, err := url.Parse(incomingFave.Account.URI)
if err != nil {
return err
}
a, err := p.federator.GetRemoteAccount(ctx, federatorMsg.ReceivingAccount.Username, remoteAccountID, true, false)
if err != nil {
return err
}
incomingFave.Account = a
}
if err := p.notifyFave(ctx, incomingFave); err != nil {
return err
}
@ -146,6 +195,30 @@ func (p *processor) processCreateFollowRequestFromFederator(ctx context.Context,
return errors.New("incomingFollowRequest was not parseable as *gtsmodel.FollowRequest")
}
// make sure the account is pinned
if followRequest.Account == nil {
a, err := p.db.GetAccountByID(ctx, followRequest.AccountID)
if err != nil {
return err
}
followRequest.Account = a
}
// do a BLOCKING get of the remote account to make sure the avi and header are cached
if followRequest.Account.Domain != "" {
remoteAccountID, err := url.Parse(followRequest.Account.URI)
if err != nil {
return err
}
a, err := p.federator.GetRemoteAccount(ctx, federatorMsg.ReceivingAccount.Username, remoteAccountID, true, false)
if err != nil {
return err
}
followRequest.Account = a
}
if followRequest.TargetAccount == nil {
a, err := p.db.GetAccountByID(ctx, followRequest.TargetAccountID)
if err != nil {
@ -153,9 +226,8 @@ func (p *processor) processCreateFollowRequestFromFederator(ctx context.Context,
}
followRequest.TargetAccount = a
}
targetAccount := followRequest.TargetAccount
if targetAccount.Locked {
if followRequest.TargetAccount.Locked {
// if the account is locked just notify the follow request and nothing else
return p.notifyFollowRequest(ctx, followRequest)
}
@ -170,7 +242,7 @@ func (p *processor) processCreateFollowRequestFromFederator(ctx context.Context,
return err
}
return p.notifyFollow(ctx, follow, targetAccount)
return p.notifyFollow(ctx, follow, followRequest.TargetAccount)
}
// processCreateAnnounceFromFederator handles Activity Create and Object Announce
@ -180,6 +252,30 @@ func (p *processor) processCreateAnnounceFromFederator(ctx context.Context, fede
return errors.New("announce was not parseable as *gtsmodel.Status")
}
// make sure the account is pinned
if incomingAnnounce.Account == nil {
a, err := p.db.GetAccountByID(ctx, incomingAnnounce.AccountID)
if err != nil {
return err
}
incomingAnnounce.Account = a
}
// do a BLOCKING get of the remote account to make sure the avi and header are cached
if incomingAnnounce.Account.Domain != "" {
remoteAccountID, err := url.Parse(incomingAnnounce.Account.URI)
if err != nil {
return err
}
a, err := p.federator.GetRemoteAccount(ctx, federatorMsg.ReceivingAccount.Username, remoteAccountID, true, false)
if err != nil {
return err
}
incomingAnnounce.Account = a
}
if err := p.federator.DereferenceAnnounce(ctx, incomingAnnounce, federatorMsg.ReceivingAccount.Username); err != nil {
return fmt.Errorf("error dereferencing announce from federator: %s", err)
}
@ -232,7 +328,12 @@ func (p *processor) processUpdateAccountFromFederator(ctx context.Context, feder
return errors.New("profile was not parseable as *gtsmodel.Account")
}
if _, err := p.federator.EnrichRemoteAccount(ctx, federatorMsg.ReceivingAccount.Username, incomingAccount); err != nil {
incomingAccountURL, err := url.Parse(incomingAccount.URI)
if err != nil {
return err
}
if _, err := p.federator.GetRemoteAccount(ctx, federatorMsg.ReceivingAccount.Username, incomingAccountURL, false, true); err != nil {
return fmt.Errorf("error enriching updated account from federator: %s", err)
}

54
internal/processing/media/create.go
View file

@ -19,56 +19,39 @@
package media
import (
"bytes"
"context"
"errors"
"fmt"
"io"
"time"
apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model"
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
"github.com/superseriousbusiness/gotosocial/internal/text"
"github.com/superseriousbusiness/gotosocial/internal/media"
)
func (p *processor) Create(ctx context.Context, account *gtsmodel.Account, form *apimodel.AttachmentRequest) (*apimodel.Attachment, error) {
// open the attachment and extract the bytes from it
f, err := form.File.Open()
if err != nil {
return nil, fmt.Errorf("error opening attachment: %s", err)
}
buf := new(bytes.Buffer)
size, err := io.Copy(buf, f)
if err != nil {
return nil, fmt.Errorf("error reading attachment: %s", err)
}
if size == 0 {
return nil, errors.New("could not read provided attachment: size 0 bytes")
data := func(innerCtx context.Context) (io.Reader, int, error) {
f, err := form.File.Open()
return f, int(form.File.Size), err
}
// now parse the focus parameter
focusx, focusy, err := parseFocus(form.Focus)
focusX, focusY, err := parseFocus(form.Focus)
if err != nil {
return nil, fmt.Errorf("couldn't parse attachment focus: %s", err)
return nil, fmt.Errorf("could not parse focus value %s: %s", form.Focus, err)
}
minAttachment := &gtsmodel.MediaAttachment{
CreatedAt: time.Now(),
UpdatedAt: time.Now(),
AccountID: account.ID,
Description: text.SanitizeCaption(form.Description),
FileMeta: gtsmodel.FileMeta{
Focus: gtsmodel.Focus{
X: focusx,
Y: focusy,
},
},
// process the media attachment and load it immediately
media, err := p.mediaManager.ProcessMedia(ctx, data, account.ID, &media.AdditionalMediaInfo{
Description: &form.Description,
FocusX: &focusX,
FocusY: &focusY,
})
if err != nil {
return nil, err
}
// allow the mediaHandler to work its magic of processing the attachment bytes, and putting them in whatever storage backend we're using
attachment, err := p.mediaHandler.ProcessAttachment(ctx, buf.Bytes(), minAttachment)
attachment, err := media.LoadAttachment(ctx)
if err != nil {
return nil, fmt.Errorf("error reading attachment: %s", err)
return nil, err
}
// prepare the frontend representation now -- if there are any errors here at least we can bail without
@ -78,10 +61,5 @@ func (p *processor) Create(ctx context.Context, account *gtsmodel.Account, form
return nil, fmt.Errorf("error parsing media attachment to frontend type: %s", err)
}
// now we can confidently put the attachment in the database
if err := p.db.Put(ctx, attachment); err != nil {
return nil, fmt.Errorf("error storing media attachment in db: %s", err)
}
return &apiAttachment, nil
}

6
internal/processing/media/media.go
View file

@ -43,16 +43,16 @@ type Processor interface {
type processor struct {
tc typeutils.TypeConverter
mediaHandler media.Handler
mediaManager media.Manager
storage *kv.KVStore
db db.DB
}
// New returns a new media processor.
func New(db db.DB, tc typeutils.TypeConverter, mediaHandler media.Handler, storage *kv.KVStore) Processor {
func New(db db.DB, tc typeutils.TypeConverter, mediaManager media.Manager, storage *kv.KVStore) Processor {
return &processor{
tc: tc,
mediaHandler: mediaHandler,
mediaManager: mediaManager,
storage: storage,
db: db,
}

14
internal/processing/processor.go
View file

@ -96,7 +96,7 @@ type Processor interface {
AccountBlockRemove(ctx context.Context, authed *oauth.Auth, targetAccountID string) (*apimodel.Relationship, gtserror.WithCode)
// AdminEmojiCreate handles the creation of a new instance emoji by an admin, using the given form.
AdminEmojiCreate(ctx context.Context, authed *oauth.Auth, form *apimodel.EmojiCreateRequest) (*apimodel.Emoji, error)
AdminEmojiCreate(ctx context.Context, authed *oauth.Auth, form *apimodel.EmojiCreateRequest) (*apimodel.Emoji, gtserror.WithCode)
// AdminDomainBlockCreate handles the creation of a new domain block by an admin, using the given form.
AdminDomainBlockCreate(ctx context.Context, authed *oauth.Auth, form *apimodel.DomainBlockCreateRequest) (*apimodel.DomainBlock, gtserror.WithCode)
// AdminDomainBlocksImport handles the import of multiple domain blocks by an admin, using the given form.
@ -235,7 +235,7 @@ type processor struct {
stop chan interface{}
tc typeutils.TypeConverter
oauthServer oauth.Server
mediaHandler media.Handler
mediaManager media.Manager
storage *kv.KVStore
statusTimelines timeline.Manager
db db.DB
@ -259,7 +259,7 @@ func NewProcessor(
tc typeutils.TypeConverter,
federator federation.Federator,
oauthServer oauth.Server,
mediaHandler media.Handler,
mediaManager media.Manager,
storage *kv.KVStore,
db db.DB,
emailSender email.Sender) Processor {
@ -268,9 +268,9 @@ func NewProcessor(
statusProcessor := status.New(db, tc, fromClientAPI)
streamingProcessor := streaming.New(db, oauthServer)
accountProcessor := account.New(db, tc, mediaHandler, oauthServer, fromClientAPI, federator)
adminProcessor := admin.New(db, tc, mediaHandler, fromClientAPI)
mediaProcessor := mediaProcessor.New(db, tc, mediaHandler, storage)
accountProcessor := account.New(db, tc, mediaManager, oauthServer, fromClientAPI, federator)
adminProcessor := admin.New(db, tc, mediaManager, fromClientAPI)
mediaProcessor := mediaProcessor.New(db, tc, mediaManager, storage)
userProcessor := user.New(db, emailSender)
federationProcessor := federationProcessor.New(db, tc, federator, fromFederator)
filter := visibility.NewFilter(db)
@ -282,7 +282,7 @@ func NewProcessor(
stop: make(chan interface{}),
tc: tc,
oauthServer: oauthServer,
mediaHandler: mediaHandler,
mediaManager: mediaManager,
storage: storage,
statusTimelines: timeline.NewManager(StatusGrabFunction(db), StatusFilterFunction(db, filter), StatusPrepareFunction(db, tc), StatusSkipInsertFunction()),
db: db,

8
internal/processing/processor_test.go
View file

@ -47,11 +47,11 @@ type ProcessingStandardTestSuite struct {
suite.Suite
db db.DB
storage *kv.KVStore
mediaManager media.Manager
typeconverter typeutils.TypeConverter
transportController transport.Controller
federator federation.Federator
oauthServer oauth.Server
mediaHandler media.Handler
timelineManager timeline.Manager
emailSender email.Sender
@ -216,12 +216,12 @@ func (suite *ProcessingStandardTestSuite) SetupTest() {
})
suite.transportController = testrig.NewTestTransportController(httpClient, suite.db)
suite.federator = testrig.NewTestFederator(suite.db, suite.transportController, suite.storage)
suite.mediaManager = testrig.NewTestMediaManager(suite.db, suite.storage)
suite.federator = testrig.NewTestFederator(suite.db, suite.transportController, suite.storage, suite.mediaManager)
suite.oauthServer = testrig.NewTestOauthServer(suite.db)
suite.mediaHandler = testrig.NewTestMediaHandler(suite.db, suite.storage)
suite.emailSender = testrig.NewEmailSender("../../web/template/", nil)
suite.processor = processing.NewProcessor(suite.typeconverter, suite.federator, suite.oauthServer, suite.mediaHandler, suite.storage, suite.db, suite.emailSender)
suite.processor = processing.NewProcessor(suite.typeconverter, suite.federator, suite.oauthServer, suite.mediaManager, suite.storage, suite.db, suite.emailSender)
testrig.StandardDBSetup(suite.db, suite.testAccounts)
testrig.StandardStorageSetup(suite.storage, "../../testrig/media")

4
internal/processing/search.go
View file

@ -148,7 +148,7 @@ func (p *processor) searchAccountByURI(ctx context.Context, authed *oauth.Auth,
if resolve {
// we don't have it locally so try and dereference it
account, _, err := p.federator.GetRemoteAccount(ctx, authed.Account.Username, uri, true)
account, err := p.federator.GetRemoteAccount(ctx, authed.Account.Username, uri, true, true)
if err != nil {
return nil, fmt.Errorf("searchAccountByURI: error dereferencing account with uri %s: %s", uri.String(), err)
}
@ -203,7 +203,7 @@ func (p *processor) searchAccountByMention(ctx context.Context, authed *oauth.Au
}
// we don't have it locally so try and dereference it
account, _, err := p.federator.GetRemoteAccount(ctx, authed.Account.Username, acctURI, true)
account, err := p.federator.GetRemoteAccount(ctx, authed.Account.Username, acctURI, true, true)
if err != nil {
return nil, fmt.Errorf("searchAccountByMention: error dereferencing account with uri %s: %s", acctURI.String(), err)
}

22
internal/transport/derefmedia.go
View file

@ -21,25 +21,22 @@ package transport
import (
"context"
"fmt"
"io/ioutil"
"io"
"net/http"
"net/url"
"github.com/sirupsen/logrus"
)
func (t *transport) DereferenceMedia(ctx context.Context, iri *url.URL, expectedContentType string) ([]byte, error) {
func (t *transport) DereferenceMedia(ctx context.Context, iri *url.URL) (io.ReadCloser, int, error) {
l := logrus.WithField("func", "DereferenceMedia")
l.Debugf("performing GET to %s", iri.String())
req, err := http.NewRequestWithContext(ctx, "GET", iri.String(), nil)
if err != nil {
return nil, err
}
if expectedContentType == "" {
req.Header.Add("Accept", "*/*")
} else {
req.Header.Add("Accept", expectedContentType)
return nil, 0, err
}
req.Header.Add("Accept", "*/*") // we don't know what kind of media we're going to get here
req.Header.Add("Date", t.clock.Now().UTC().Format("Mon, 02 Jan 2006 15:04:05")+" GMT")
req.Header.Add("User-Agent", fmt.Sprintf("%s %s", t.appAgent, t.gofedAgent))
req.Header.Set("Host", iri.Host)
@ -47,15 +44,14 @@ func (t *transport) DereferenceMedia(ctx context.Context, iri *url.URL, expected
err = t.getSigner.SignRequest(t.privkey, t.pubKeyID, req, nil)
t.getSignerMu.Unlock()
if err != nil {
return nil, err
return nil, 0, err
}
resp, err := t.client.Do(req)
if err != nil {
return nil, err
return nil, 0, err
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return nil, fmt.Errorf("GET request to %s failed (%d): %s", iri.String(), resp.StatusCode, resp.Status)
return nil, 0, fmt.Errorf("GET request to %s failed (%d): %s", iri.String(), resp.StatusCode, resp.Status)
}
return ioutil.ReadAll(resp.Body)
return resp.Body, int(resp.ContentLength), nil
}

5
internal/transport/transport.go
View file

@ -21,6 +21,7 @@ package transport
import (
"context"
"crypto"
"io"
"net/url"
"sync"
@ -33,8 +34,8 @@ import (
// functionality for fetching remote media.
type Transport interface {
pub.Transport
// DereferenceMedia fetches the bytes of the given media attachment IRI, with the expectedContentType.
DereferenceMedia(ctx context.Context, iri *url.URL, expectedContentType string) ([]byte, error)
// DereferenceMedia fetches the given media attachment IRI, returning the reader and filesize.
DereferenceMedia(ctx context.Context, iri *url.URL) (io.ReadCloser, int, error)
// DereferenceInstance dereferences remote instance information, first by checking /api/v1/instance, and then by checking /.well-known/nodeinfo.
DereferenceInstance(ctx context.Context, iri *url.URL) (*gtsmodel.Instance, error)
// Finger performs a webfinger request with the given username and domain, and returns the bytes from the response body.

95
internal/typeutils/internaltoas.go
View file

@ -25,6 +25,7 @@ import (
"fmt"
"net/url"
"github.com/sirupsen/logrus"
"github.com/spf13/viper"
"github.com/superseriousbusiness/activity/pub"
"github.com/superseriousbusiness/activity/streams"
@ -215,62 +216,68 @@ func (c *converter) AccountToAS(ctx context.Context, a *gtsmodel.Account) (vocab
// Used as profile avatar.
if a.AvatarMediaAttachmentID != "" {
if a.AvatarMediaAttachment == nil {
avatar := &gtsmodel.MediaAttachment{}
if err := c.db.GetByID(ctx, a.AvatarMediaAttachmentID, avatar); err != nil {
avatar, err := c.db.GetAttachmentByID(ctx, a.AvatarMediaAttachmentID)
if err == nil {
a.AvatarMediaAttachment = avatar
} else {
logrus.Errorf("AccountToAS: error getting Avatar with id %s: %s", a.AvatarMediaAttachmentID, err)
}
}
if a.AvatarMediaAttachment != nil {
iconProperty := streams.NewActivityStreamsIconProperty()
iconImage := streams.NewActivityStreamsImage()
mediaType := streams.NewActivityStreamsMediaTypeProperty()
mediaType.Set(a.AvatarMediaAttachment.File.ContentType)
iconImage.SetActivityStreamsMediaType(mediaType)
avatarURLProperty := streams.NewActivityStreamsUrlProperty()
avatarURL, err := url.Parse(a.AvatarMediaAttachment.URL)
if err != nil {
return nil, err
}
a.AvatarMediaAttachment = avatar
avatarURLProperty.AppendIRI(avatarURL)
iconImage.SetActivityStreamsUrl(avatarURLProperty)
iconProperty.AppendActivityStreamsImage(iconImage)
person.SetActivityStreamsIcon(iconProperty)
}
iconProperty := streams.NewActivityStreamsIconProperty()
iconImage := streams.NewActivityStreamsImage()
mediaType := streams.NewActivityStreamsMediaTypeProperty()
mediaType.Set(a.AvatarMediaAttachment.File.ContentType)
iconImage.SetActivityStreamsMediaType(mediaType)
avatarURLProperty := streams.NewActivityStreamsUrlProperty()
avatarURL, err := url.Parse(a.AvatarMediaAttachment.URL)
if err != nil {
return nil, err
}
avatarURLProperty.AppendIRI(avatarURL)
iconImage.SetActivityStreamsUrl(avatarURLProperty)
iconProperty.AppendActivityStreamsImage(iconImage)
person.SetActivityStreamsIcon(iconProperty)
}
// image
// Used as profile header.
if a.HeaderMediaAttachmentID != "" {
if a.HeaderMediaAttachment == nil {
header := &gtsmodel.MediaAttachment{}
if err := c.db.GetByID(ctx, a.HeaderMediaAttachmentID, header); err != nil {
header, err := c.db.GetAttachmentByID(ctx, a.HeaderMediaAttachmentID)
if err == nil {
a.HeaderMediaAttachment = header
} else {
logrus.Errorf("AccountToAS: error getting Header with id %s: %s", a.HeaderMediaAttachmentID, err)
}
}
if a.HeaderMediaAttachment != nil {
headerProperty := streams.NewActivityStreamsImageProperty()
headerImage := streams.NewActivityStreamsImage()
mediaType := streams.NewActivityStreamsMediaTypeProperty()
mediaType.Set(a.HeaderMediaAttachment.File.ContentType)
headerImage.SetActivityStreamsMediaType(mediaType)
headerURLProperty := streams.NewActivityStreamsUrlProperty()
headerURL, err := url.Parse(a.HeaderMediaAttachment.URL)
if err != nil {
return nil, err
}
a.HeaderMediaAttachment = header
headerURLProperty.AppendIRI(headerURL)
headerImage.SetActivityStreamsUrl(headerURLProperty)
headerProperty.AppendActivityStreamsImage(headerImage)
person.SetActivityStreamsImage(headerProperty)
}
headerProperty := streams.NewActivityStreamsImageProperty()
headerImage := streams.NewActivityStreamsImage()
mediaType := streams.NewActivityStreamsMediaTypeProperty()
mediaType.Set(a.HeaderMediaAttachment.File.ContentType)
headerImage.SetActivityStreamsMediaType(mediaType)
headerURLProperty := streams.NewActivityStreamsUrlProperty()
headerURL, err := url.Parse(a.HeaderMediaAttachment.URL)
if err != nil {
return nil, err
}
headerURLProperty.AppendIRI(headerURL)
headerImage.SetActivityStreamsUrl(headerURLProperty)
headerProperty.AppendActivityStreamsImage(headerImage)
person.SetActivityStreamsImage(headerProperty)
}
return person, nil

31
internal/typeutils/internaltofrontend.go
View file

@ -96,35 +96,40 @@ func (c *converter) AccountToAPIAccountPublic(ctx context.Context, a *gtsmodel.A
lastStatusAt = lastPosted.Format(time.RFC3339)
}
// build the avatar and header URLs
// set account avatar fields if available
var aviURL string
var aviURLStatic string
if a.AvatarMediaAttachmentID != "" {
// make sure avi is pinned to this account
if a.AvatarMediaAttachment == nil {
avi, err := c.db.GetAttachmentByID(ctx, a.AvatarMediaAttachmentID)
if err != nil {
return nil, fmt.Errorf("error retrieving avatar: %s", err)
if err == nil {
a.AvatarMediaAttachment = avi
} else {
logrus.Errorf("AccountToAPIAccountPublic: error getting Avatar with id %s: %s", a.AvatarMediaAttachmentID, err)
}
a.AvatarMediaAttachment = avi
}
aviURL = a.AvatarMediaAttachment.URL
aviURLStatic = a.AvatarMediaAttachment.Thumbnail.URL
if a.AvatarMediaAttachment != nil {
aviURL = a.AvatarMediaAttachment.URL
aviURLStatic = a.AvatarMediaAttachment.Thumbnail.URL
}
}
// set account header fields if available
var headerURL string
var headerURLStatic string
if a.HeaderMediaAttachmentID != "" {
// make sure header is pinned to this account
if a.HeaderMediaAttachment == nil {
avi, err := c.db.GetAttachmentByID(ctx, a.HeaderMediaAttachmentID)
if err != nil {
return nil, fmt.Errorf("error retrieving avatar: %s", err)
if err == nil {
a.HeaderMediaAttachment = avi
} else {
logrus.Errorf("AccountToAPIAccountPublic: error getting Header with id %s: %s", a.HeaderMediaAttachmentID, err)
}
a.HeaderMediaAttachment = avi
}
headerURL = a.HeaderMediaAttachment.URL
headerURLStatic = a.HeaderMediaAttachment.Thumbnail.URL
if a.HeaderMediaAttachment != nil {
headerURL = a.HeaderMediaAttachment.URL
headerURLStatic = a.HeaderMediaAttachment.Thumbnail.URL
}
}
// get the fields set on this account

5
testrig/federator.go
View file

@ -22,10 +22,11 @@ import (
"codeberg.org/gruf/go-store/kv"
"github.com/superseriousbusiness/gotosocial/internal/db"
"github.com/superseriousbusiness/gotosocial/internal/federation"
"github.com/superseriousbusiness/gotosocial/internal/media"
"github.com/superseriousbusiness/gotosocial/internal/transport"
)
// NewTestFederator returns a federator with the given database and (mock!!) transport controller.
func NewTestFederator(db db.DB, tc transport.Controller, storage *kv.KVStore) federation.Federator {
return federation.NewFederator(db, NewTestFederatingDB(db), tc, NewTestTypeConverter(db), NewTestMediaHandler(db, storage))
func NewTestFederator(db db.DB, tc transport.Controller, storage *kv.KVStore, mediaManager media.Manager) federation.Federator {
return federation.NewFederator(db, NewTestFederatingDB(db), tc, NewTestTypeConverter(db), mediaManager)
}

10
testrig/mediahandler.go
View file

@ -24,7 +24,11 @@ import (
"github.com/superseriousbusiness/gotosocial/internal/media"
)
// NewTestMediaHandler returns a media handler with the default test config, and the given db and storage.
func NewTestMediaHandler(db db.DB, storage *kv.KVStore) media.Handler {
return media.New(db, storage)
// NewTestMediaManager returns a media handler with the default test config, and the given db and storage.
func NewTestMediaManager(db db.DB, storage *kv.KVStore) media.Manager {
m, err := media.NewManager(db, storage)
if err != nil {
panic(err)
}
return m
}

5
testrig/processor.go
View file

@ -23,10 +23,11 @@ import (
"github.com/superseriousbusiness/gotosocial/internal/db"
"github.com/superseriousbusiness/gotosocial/internal/email"
"github.com/superseriousbusiness/gotosocial/internal/federation"
"github.com/superseriousbusiness/gotosocial/internal/media"
"github.com/superseriousbusiness/gotosocial/internal/processing"
)
// NewTestProcessor returns a Processor suitable for testing purposes
func NewTestProcessor(db db.DB, storage *kv.KVStore, federator federation.Federator, emailSender email.Sender) processing.Processor {
return processing.NewProcessor(NewTestTypeConverter(db), federator, NewTestOauthServer(db), NewTestMediaHandler(db, storage), storage, db, emailSender)
func NewTestProcessor(db db.DB, storage *kv.KVStore, federator federation.Federator, emailSender email.Sender, mediaManager media.Manager) processing.Processor {
return processing.NewProcessor(NewTestTypeConverter(db), federator, NewTestOauthServer(db), mediaManager, storage, db, emailSender)
}

82
testrig/storage.go
View file

@ -19,20 +19,16 @@
package testrig
import (
"bytes"
"errors"
"fmt"
"io"
"os"
"codeberg.org/gruf/go-store/kv"
"codeberg.org/gruf/go-store/storage"
"codeberg.org/gruf/go-store/util"
)
// NewTestStorage returns a new in memory storage with the default test config
func NewTestStorage() *kv.KVStore {
storage, err := kv.OpenStorage(&inMemStorage{storage: map[string][]byte{}, overwrite: false})
storage, err := kv.OpenStorage(storage.OpenMemory(200, false))
if err != nil {
panic(err)
}
@ -113,79 +109,3 @@ func StandardStorageTeardown(s *kv.KVStore) {
}
}
}
type inMemStorage struct {
storage map[string][]byte
overwrite bool
}
func (s *inMemStorage) Clean() error {
return nil
}
func (s *inMemStorage) ReadBytes(key string) ([]byte, error) {
b, ok := s.storage[key]
if !ok {
return nil, errors.New("key not found")
}
return b, nil
}
func (s *inMemStorage) ReadStream(key string) (io.ReadCloser, error) {
b, err := s.ReadBytes(key)
if err != nil {
return nil, err
}
return util.NopReadCloser(bytes.NewReader(b)), nil
}
func (s *inMemStorage) WriteBytes(key string, value []byte) error {
if _, ok := s.storage[key]; ok && !s.overwrite {
return errors.New("key already in storage")
}
s.storage[key] = copyBytes(value)
return nil
}
func (s *inMemStorage) WriteStream(key string, r io.Reader) error {
b, err := io.ReadAll(r)
if err != nil {
return err
}
return s.WriteBytes(key, b)
}
func (s *inMemStorage) Stat(key string) (bool, error) {
_, ok := s.storage[key]
return ok, nil
}
func (s *inMemStorage) Remove(key string) error {
if _, ok := s.storage[key]; !ok {
return errors.New("key not found")
}
delete(s.storage, key)
return nil
}
func (s *inMemStorage) WalkKeys(opts storage.WalkKeysOptions) error {
if opts.WalkFn == nil {
return errors.New("invalid walkfn")
}
for key := range s.storage {
opts.WalkFn(entry(key))
}
return nil
}
type entry string
func (e entry) Key() string {
return string(e)
}
func copyBytes(b []byte) []byte {
p := make([]byte, len(b))
copy(p, b)
return p
}

10
testrig/testmodels.go
View file

@ -66,6 +66,16 @@ func NewTestTokens() map[string]*gtsmodel.Token {
AccessCreateAt: time.Now(),
AccessExpiresAt: time.Now().Add(72 * time.Hour),
},
"admin_account": {
ID: "01FS4TP8ANA5VE92EAPA9E0M7Q",
ClientID: "01F8MGWSJCND9BWBD4WGJXBM93",
UserID: "01F8MGWYWKVKS3VS8DV1AMYPGE",
RedirectURI: "http://localhost:8080",
Scope: "read write follow push admin",
Access: "AININALKNENFNF98717NAMG4LWE4NJITMWUXM2M4MTRHZDEX",
AccessCreateAt: time.Now(),
AccessExpiresAt: time.Now().Add(72 * time.Hour),
},
}
return tokens
}

27
vendor/codeberg.org/gruf/go-errors/data.go generated vendored
View file

@ -4,17 +4,9 @@ import (
"fmt"
"sync"
"codeberg.org/gruf/go-bytes"
"codeberg.org/gruf/go-logger"
"codeberg.org/gruf/go-format"
)
// global logfmt data formatter.
var logfmt = logger.TextFormat{
Strict: false,
Verbose: true,
MaxDepth: 5,
}
// KV is a structure for setting key-value pairs in ErrorData.
type KV struct {
Key string
@ -31,7 +23,7 @@ type ErrorData interface {
Append(...KV)
// Implement byte slice representation formatter.
logger.Formattable
format.Formattable
// Implement string representation formatter.
fmt.Stringer
@ -89,13 +81,22 @@ func (d *errorData) Append(kvs ...KV) {
}
func (d *errorData) AppendFormat(b []byte) []byte {
buf := bytes.Buffer{B: b}
buf := format.Buffer{B: b}
d.mu.Lock()
buf.B = append(buf.B, '{')
// Append data as kv pairs
for i := range d.data {
logfmt.AppendKey(&buf, d.data[i].Key)
logfmt.AppendValue(&buf, d.data[i].Value)
key := d.data[i].Key
val := d.data[i].Value
format.Appendf(&buf, "{:k}={:v} ", key, val)
}
// Drop trailing space
if len(d.data) > 0 {
buf.Truncate(1)
}
buf.B = append(buf.B, '}')
d.mu.Unlock()
return buf.B

0
vendor/codeberg.org/gruf/go-logger/LICENSE → vendor/codeberg.org/gruf/go-format/LICENSE generated vendored
View file

16
vendor/codeberg.org/gruf/go-format/README.md generated vendored Normal file
View file

@ -0,0 +1,16 @@
# go-format
String formatting package using Rust-style formatting directives.
Output is generally more visually-friendly than `"fmt"`, while performance is neck-and-neck.
README is WIP.
## todos
- improved verbose printing of number types
- more test cases
- improved verbose printing of string ptr types

81
vendor/codeberg.org/gruf/go-format/buffer.go generated vendored Normal file
View file

@ -0,0 +1,81 @@
package format
import (
"io"
"unicode/utf8"
"unsafe"
)
// ensure we conform to io.Writer.
var _ io.Writer = (*Buffer)(nil)
// Buffer is a simple wrapper around a byte slice.
type Buffer struct {
B []byte
}
// Write will append given byte slice to buffer, fulfilling io.Writer.
func (buf *Buffer) Write(b []byte) (int, error) {
buf.B = append(buf.B, b...)
return len(b), nil
}
// AppendByte appends given byte to the buffer.
func (buf *Buffer) AppendByte(b byte) {
buf.B = append(buf.B, b)
}
// AppendRune appends given rune to the buffer.
func (buf *Buffer) AppendRune(r rune) {
if r < utf8.RuneSelf {
buf.B = append(buf.B, byte(r))
return
}
l := buf.Len()
for i := 0; i < utf8.UTFMax; i++ {
buf.B = append(buf.B, 0)
}
n := utf8.EncodeRune(buf.B[l:buf.Len()], r)
buf.B = buf.B[:l+n]
}
// Append will append given byte slice to the buffer.
func (buf *Buffer) Append(b []byte) {
buf.B = append(buf.B, b...)
}
// AppendString appends given string to the buffer.
func (buf *Buffer) AppendString(s string) {
buf.B = append(buf.B, s...)
}
// Len returns the length of the buffer's underlying byte slice.
func (buf *Buffer) Len() int {
return len(buf.B)
}
// Cap returns the capacity of the buffer's underlying byte slice.
func (buf *Buffer) Cap() int {
return cap(buf.B)
}
// Truncate will reduce the length of the buffer by 'n'.
func (buf *Buffer) Truncate(n int) {
if n > len(buf.B) {
n = len(buf.B)
}
buf.B = buf.B[:buf.Len()-n]
}
// Reset will reset the buffer length to 0 (retains capacity).
func (buf *Buffer) Reset() {
buf.B = buf.B[:0]
}
// String returns the underlying byte slice as a string. Please note
// this value is tied directly to the underlying byte slice, if you
// write to the buffer then returned string values will also change.
func (buf *Buffer) String() string {
return *(*string)(unsafe.Pointer(&buf.B))
}

565
vendor/codeberg.org/gruf/go-format/format.go generated vendored Normal file
View file

@ -0,0 +1,565 @@
package format
import (
"reflect"
"strconv"
"unsafe"
)
// Formattable defines a type capable of being formatted and appended to a byte buffer.
type Formattable interface {
AppendFormat([]byte) []byte
}
// format is the object passed among the append___ formatting functions.
type format struct {
flags uint8 // 'isKey' and 'verbose' flags
drefs uint8 // current value deref count
curd uint8 // current depth
maxd uint8 // maximum depth
buf *Buffer // out buffer
}
const (
// flag bit constants.
isKeyBit = uint8(1) << 0
isValBit = uint8(1) << 1
vboseBit = uint8(1) << 2
panicBit = uint8(1) << 3
)
// AtMaxDepth returns whether format is currently at max depth.
func (f format) AtMaxDepth() bool {
return f.curd > f.maxd
}
// Derefs returns no. times current value has been dereferenced.
func (f format) Derefs() uint8 {
return f.drefs
}
// IsKey returns whether the isKey flag is set.
func (f format) IsKey() bool {
return (f.flags & isKeyBit) != 0
}
// IsValue returns whether the isVal flag is set.
func (f format) IsValue() bool {
return (f.flags & isValBit) != 0
}
// Verbose returns whether the verbose flag is set.
func (f format) Verbose() bool {
return (f.flags & vboseBit) != 0
}
// Panic returns whether the panic flag is set.
func (f format) Panic() bool {
return (f.flags & panicBit) != 0
}
// SetIsKey returns format instance with the isKey bit set to value.
func (f format) SetIsKey() format {
return format{
flags: f.flags & ^isValBit | isKeyBit,
curd: f.curd,
maxd: f.maxd,
buf: f.buf,
}
}
// SetIsValue returns format instance with the isVal bit set to value.
func (f format) SetIsValue() format {
return format{
flags: f.flags & ^isKeyBit | isValBit,
curd: f.curd,
maxd: f.maxd,
buf: f.buf,
}
}
// SetPanic returns format instance with the panic bit set to value.
func (f format) SetPanic() format {
return format{
flags: f.flags | panicBit /* handle panic as value */ | isValBit & ^isKeyBit,
curd: f.curd,
maxd: f.maxd,
buf: f.buf,
}
}
// IncrDepth returns format instance with depth incremented.
func (f format) IncrDepth() format {
return format{
flags: f.flags,
curd: f.curd + 1,
maxd: f.maxd,
buf: f.buf,
}
}
// IncrDerefs returns format instance with dereference count incremented.
func (f format) IncrDerefs() format {
return format{
flags: f.flags,
drefs: f.drefs + 1,
curd: f.curd,
maxd: f.maxd,
buf: f.buf,
}
}
// appendType appends a type using supplied type str.
func appendType(fmt format, t string) {
for i := uint8(0); i < fmt.Derefs(); i++ {
fmt.buf.AppendByte('*')
}
fmt.buf.AppendString(t)
}
// appendNilType Appends nil to buf, type included if verbose.
func appendNilType(fmt format, t string) {
if fmt.Verbose() {
fmt.buf.AppendByte('(')
appendType(fmt, t)
fmt.buf.AppendString(`)(nil)`)
} else {
fmt.buf.AppendString(`nil`)
}
}
// appendByte Appends a single byte to buf.
func appendByte(fmt format, b byte) {
if fmt.IsValue() || fmt.Verbose() {
fmt.buf.AppendString(`'` + string(b) + `'`)
} else {
fmt.buf.AppendByte(b)
}
}
// appendBytes Appends a quoted byte slice to buf.
func appendBytes(fmt format, b []byte) {
if b == nil {
// Bytes CAN be nil formatted
appendNilType(fmt, `[]byte`)
} else {
// Append bytes as slice
fmt.buf.AppendByte('[')
for _, b := range b {
fmt.buf.AppendByte(b)
fmt.buf.AppendByte(',')
}
if len(b) > 0 {
fmt.buf.Truncate(1)
}
fmt.buf.AppendByte(']')
}
}
// appendString Appends an escaped, double-quoted string to buf.
func appendString(fmt format, s string) {
switch {
// Key in a key-value pair
case fmt.IsKey():
if !strconv.CanBackquote(s) {
// Requires quoting AND escaping
fmt.buf.B = strconv.AppendQuote(fmt.buf.B, s)
} else if containsSpaceOrTab(s) {
// Contains space, needs quotes
fmt.buf.AppendString(`"` + s + `"`)
} else {
// All else write as-is
fmt.buf.AppendString(s)
}
// Value in a key-value pair (always escape+quote)
case fmt.IsValue():
fmt.buf.B = strconv.AppendQuote(fmt.buf.B, s)
// Verbose but neither key nor value (always quote)
case fmt.Verbose():
fmt.buf.AppendString(`"` + s + `"`)
// All else
default:
fmt.buf.AppendString(s)
}
}
// appendBool Appends a formatted bool to buf.
func appendBool(fmt format, b bool) {
fmt.buf.B = strconv.AppendBool(fmt.buf.B, b)
}
// appendInt Appends a formatted int to buf.
func appendInt(fmt format, i int64) {
fmt.buf.B = strconv.AppendInt(fmt.buf.B, i, 10)
}
// appendUint Appends a formatted uint to buf.
func appendUint(fmt format, u uint64) {
fmt.buf.B = strconv.AppendUint(fmt.buf.B, u, 10)
}
// appendFloat Appends a formatted float to buf.
func appendFloat(fmt format, f float64) {
fmt.buf.B = strconv.AppendFloat(fmt.buf.B, f, 'G', -1, 64)
}
// appendComplex Appends a formatted complex128 to buf.
func appendComplex(fmt format, c complex128) {
appendFloat(fmt, real(c))
fmt.buf.AppendByte('+')
appendFloat(fmt, imag(c))
fmt.buf.AppendByte('i')
}
// isNil will safely check if 'v' is nil without dealing with weird Go interface nil bullshit.
func isNil(i interface{}) bool {
e := *(*struct {
_ unsafe.Pointer // type
v unsafe.Pointer // value
})(unsafe.Pointer(&i))
return (e.v == nil)
}
// appendIfaceOrReflectValue will attempt to append as interface, falling back to reflection.
func appendIfaceOrRValue(fmt format, i interface{}) {
if !appendIface(fmt, i) {
appendRValue(fmt, reflect.ValueOf(i))
}
}
// appendValueNext checks for interface methods before performing appendRValue, checking + incr depth.
func appendRValueOrIfaceNext(fmt format, v reflect.Value) {
// Check we haven't hit max
if fmt.AtMaxDepth() {
fmt.buf.AppendString("...")
return
}
// Incr the depth
fmt = fmt.IncrDepth()
// Make actual call
if !v.CanInterface() || !appendIface(fmt, v.Interface()) {
appendRValue(fmt, v)
}
}
// appendIface parses and Appends a formatted interface value to buf.
func appendIface(fmt format, i interface{}) (ok bool) {
ok = true // default
catchPanic := func() {
if r := recover(); r != nil {
// DON'T recurse catchPanic()
if fmt.Panic() {
panic(r)
}
// Attempt to decode panic into buf
fmt.buf.AppendString(`!{PANIC=`)
appendIfaceOrRValue(fmt.SetPanic(), r)
fmt.buf.AppendByte('}')
// Ensure return
ok = true
}
}
switch i := i.(type) {
// Nil type
case nil:
fmt.buf.AppendString(`nil`)
// Reflect types
case reflect.Type:
if isNil(i) /* safer nil check */ {
appendNilType(fmt, `reflect.Type`)
} else {
appendType(fmt, `reflect.Type`)
fmt.buf.AppendString(`(` + i.String() + `)`)
}
case reflect.Value:
appendType(fmt, `reflect.Value`)
fmt.buf.AppendByte('(')
fmt.flags |= vboseBit
appendRValue(fmt, i)
fmt.buf.AppendByte(')')
// Bytes and string types
case byte:
appendByte(fmt, i)
case []byte:
appendBytes(fmt, i)
case string:
appendString(fmt, i)
// Int types
case int:
appendInt(fmt, int64(i))
case int8:
appendInt(fmt, int64(i))
case int16:
appendInt(fmt, int64(i))
case int32:
appendInt(fmt, int64(i))
case int64:
appendInt(fmt, i)
// Uint types
case uint:
appendUint(fmt, uint64(i))
// case uint8 :: this is 'byte'
case uint16:
appendUint(fmt, uint64(i))
case uint32:
appendUint(fmt, uint64(i))
case uint64:
appendUint(fmt, i)
// Float types
case float32:
appendFloat(fmt, float64(i))
case float64:
appendFloat(fmt, i)
// Bool type
case bool:
appendBool(fmt, i)
// Complex types
case complex64:
appendComplex(fmt, complex128(i))
case complex128:
appendComplex(fmt, i)
// Method types
case error:
switch {
case fmt.Verbose():
ok = false
case isNil(i) /* use safer nil check */ :
appendNilType(fmt, reflect.TypeOf(i).String())
default:
defer catchPanic()
appendString(fmt, i.Error())
}
case Formattable:
switch {
case fmt.Verbose():
ok = false
case isNil(i) /* use safer nil check */ :
appendNilType(fmt, reflect.TypeOf(i).String())
default:
defer catchPanic()
fmt.buf.B = i.AppendFormat(fmt.buf.B)
}
case interface{ String() string }:
switch {
case fmt.Verbose():
ok = false
case isNil(i) /* use safer nil check */ :
appendNilType(fmt, reflect.TypeOf(i).String())
default:
defer catchPanic()
appendString(fmt, i.String())
}
// No quick handler
default:
ok = false
}
return ok
}
// appendReflectValue will safely append a reflected value.
func appendRValue(fmt format, v reflect.Value) {
switch v.Kind() {
// String and byte types
case reflect.Uint8:
appendByte(fmt, byte(v.Uint()))
case reflect.String:
appendString(fmt, v.String())
// Float tpyes
case reflect.Float32, reflect.Float64:
appendFloat(fmt, v.Float())
// Int types
case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
appendInt(fmt, v.Int())
// Uint types
case reflect.Uint, reflect.Uint16, reflect.Uint32, reflect.Uint64:
appendUint(fmt, v.Uint())
// Complex types
case reflect.Complex64, reflect.Complex128:
appendComplex(fmt, v.Complex())
// Bool type
case reflect.Bool:
appendBool(fmt, v.Bool())
// Slice and array types
case reflect.Array:
appendArrayType(fmt, v)
case reflect.Slice:
if v.IsNil() {
appendNilType(fmt, v.Type().String())
} else {
appendArrayType(fmt, v)
}
// Map types
case reflect.Map:
if v.IsNil() {
appendNilType(fmt, v.Type().String())
} else {
appendMapType(fmt, v)
}
// Struct types
case reflect.Struct:
appendStructType(fmt, v)
// Deref'able ptr types
case reflect.Ptr, reflect.Interface:
if v.IsNil() {
appendNilType(fmt, v.Type().String())
} else {
appendRValue(fmt.IncrDerefs(), v.Elem())
}
// 'raw' pointer types
case reflect.UnsafePointer:
appendType(fmt, `unsafe.Pointer`)
fmt.buf.AppendByte('(')
if u := v.Pointer(); u != 0 {
fmt.buf.AppendString("0x")
fmt.buf.B = strconv.AppendUint(fmt.buf.B, uint64(u), 16)
} else {
fmt.buf.AppendString(`nil`)
}
fmt.buf.AppendByte(')')
case reflect.Uintptr:
appendType(fmt, `uintptr`)
fmt.buf.AppendByte('(')
if u := v.Uint(); u != 0 {
fmt.buf.AppendString("0x")
fmt.buf.B = strconv.AppendUint(fmt.buf.B, u, 16)
} else {
fmt.buf.AppendString(`nil`)
}
fmt.buf.AppendByte(')')
// Generic types we don't *exactly* handle
case reflect.Func, reflect.Chan:
if v.IsNil() {
appendNilType(fmt, v.Type().String())
} else {
fmt.buf.AppendString(v.String())
}
// Unhandled kind
default:
fmt.buf.AppendString(v.String())
}
}
// appendArrayType Appends an array of unknown type (parsed by reflection) to buf, unlike appendSliceType does NOT catch nil slice.
func appendArrayType(fmt format, v reflect.Value) {
// get no. elements
n := v.Len()
fmt.buf.AppendByte('[')
// Append values
for i := 0; i < n; i++ {
appendRValueOrIfaceNext(fmt.SetIsValue(), v.Index(i))
fmt.buf.AppendByte(',')
}
// Drop last comma
if n > 0 {
fmt.buf.Truncate(1)
}
fmt.buf.AppendByte(']')
}
// appendMapType Appends a map of unknown types (parsed by reflection) to buf.
func appendMapType(fmt format, v reflect.Value) {
// Prepend type if verbose
if fmt.Verbose() {
appendType(fmt, v.Type().String())
}
// Get a map iterator
r := v.MapRange()
n := v.Len()
fmt.buf.AppendByte('{')
// Iterate pairs
for r.Next() {
appendRValueOrIfaceNext(fmt.SetIsKey(), r.Key())
fmt.buf.AppendByte('=')
appendRValueOrIfaceNext(fmt.SetIsValue(), r.Value())
fmt.buf.AppendByte(' ')
}
// Drop last space
if n > 0 {
fmt.buf.Truncate(1)
}
fmt.buf.AppendByte('}')
}
// appendStructType Appends a struct (as a set of key-value fields) to buf.
func appendStructType(fmt format, v reflect.Value) {
// Get value type & no. fields
t := v.Type()
n := v.NumField()
// Prepend type if verbose
if fmt.Verbose() {
appendType(fmt, v.Type().String())
}
fmt.buf.AppendByte('{')
// Iterate fields
for i := 0; i < n; i++ {
vfield := v.Field(i)
tfield := t.Field(i)
// Append field name
fmt.buf.AppendString(tfield.Name)
fmt.buf.AppendByte('=')
appendRValueOrIfaceNext(fmt.SetIsValue(), vfield)
// Iter written count
fmt.buf.AppendByte(' ')
}
// Drop last space
if n > 0 {
fmt.buf.Truncate(1)
}
fmt.buf.AppendByte('}')
}
// containsSpaceOrTab checks if "s" contains space or tabs.
func containsSpaceOrTab(s string) bool {
for _, r := range s {
if r == ' ' || r == '\t' {
return true
}
}
return false
}

352
vendor/codeberg.org/gruf/go-format/formatter.go generated vendored Normal file
View file

@ -0,0 +1,352 @@
package format
import (
"strings"
)
// Formatter allows configuring value and string formatting.
type Formatter struct {
// MaxDepth specifies the max depth of fields the formatter will iterate.
// Once max depth is reached, value will simply be formatted as "...".
// e.g.
//
// MaxDepth=1
// type A struct{
// Nested B
// }
// type B struct{
// Nested C
// }
// type C struct{
// Field string
// }
//
// Append(&buf, A{}) => {Nested={Nested={Field=...}}}
MaxDepth uint8
}
// Append will append formatted form of supplied values into 'buf'.
func (f Formatter) Append(buf *Buffer, v ...interface{}) {
for _, v := range v {
appendIfaceOrRValue(format{maxd: f.MaxDepth, buf: buf}, v)
buf.AppendByte(' ')
}
if len(v) > 0 {
buf.Truncate(1)
}
}
// Appendf will append the formatted string with supplied values into 'buf'.
// Supported format directives:
// - '{}' => format supplied arg, in place
// - '{0}' => format arg at index 0 of supplied, in place
// - '{:?}' => format supplied arg verbosely, in place
// - '{:k}' => format supplied arg as key, in place
// - '{:v}' => format supplied arg as value, in place
//
// To escape either of '{}' simply append an additional brace e.g.
// - '{{' => '{'
// - '}}' => '}'
// - '{{}}' => '{}'
// - '{{:?}}' => '{:?}'
//
// More formatting directives might be included in the future.
func (f Formatter) Appendf(buf *Buffer, s string, a ...interface{}) {
const (
// ground state
modeNone = uint8(0)
// prev reached '{'
modeOpen = uint8(1)
// prev reached '}'
modeClose = uint8(2)
// parsing directive index
modeIdx = uint8(3)
// parsing directive operands
modeOp = uint8(4)
)
var (
// mode is current parsing mode
mode uint8
// arg is the current arg index
arg int
// carg is current directive-set arg index
carg int
// last is the trailing cursor to see slice windows
last int
// idx is the current index in 's'
idx int
// fmt is the base argument formatter
fmt = format{
maxd: f.MaxDepth,
buf: buf,
}
// NOTE: these functions are defined here as function
// locals as it turned out to be better for performance
// doing it this way, than encapsulating their logic in
// some kind of parsing structure. Maybe if the parser
// was pooled along with the buffers it might work out
// better, but then it makes more internal functions i.e.
// .Append() .Appendf() less accessible outside package.
//
// Currently, passing '-gcflags "-l=4"' causes a not
// insignificant decrease in ns/op, which is likely due
// to more aggressive function inlining, which this
// function can obviously stand to benefit from :)
// Str returns current string window slice, and updates
// the trailing cursor 'last' to current 'idx'
Str = func() string {
str := s[last:idx]
last = idx
return str
}
// MoveUp moves the trailing cursor 'last' just past 'idx'
MoveUp = func() {
last = idx + 1
}
// MoveUpTo moves the trailing cursor 'last' either up to
// closest '}', or current 'idx', whichever is furthest
MoveUpTo = func() {
if i := strings.IndexByte(s[idx:], '}'); i >= 0 {
idx += i
}
MoveUp()
}
// ParseIndex parses an integer from the current string
// window, updating 'last' to 'idx'. The string window
// is ASSUMED to contain only valid ASCII numbers. This
// only returns false if number exceeds platform int size
ParseIndex = func() bool {
// Get current window
str := Str()
if len(str) < 1 {
return true
}
// Index HAS to fit within platform int
if !can32bitInt(str) && !can64bitInt(str) {
return false
}
// Build integer from string
carg = 0
for _, c := range []byte(str) {
carg = carg*10 + int(c-'0')
}
return true
}
// ParseOp parses operands from the current string
// window, updating 'last' to 'idx'. The string window
// is ASSUMED to contain only valid operand ASCII. This
// returns success on parsing of operand logic
ParseOp = func() bool {
// Get current window
str := Str()
if len(str) < 1 {
return true
}
// (for now) only
// accept length = 1
if len(str) > 1 {
return false
}
switch str[0] {
case 'k':
fmt.flags |= isKeyBit
case 'v':
fmt.flags |= isValBit
case '?':
fmt.flags |= vboseBit
}
return true
}
// AppendArg will take either the directive-set, or
// iterated arg index, check within bounds of 'a' and
// append the that argument formatted to the buffer.
// On failure, it will append an error string
AppendArg = func() {
// Look for idx
if carg < 0 {
carg = arg
}
// Incr idx
arg++
if carg < len(a) {
// Append formatted argument value
appendIfaceOrRValue(fmt, a[carg])
} else {
// No argument found for index
buf.AppendString(`!{MISSING_ARG}`)
}
}
// Reset will reset the mode to ground, the flags
// to empty and parsed 'carg' to empty
Reset = func() {
mode = modeNone
fmt.flags = 0
carg = -1
}
)
for idx = 0; idx < len(s); idx++ {
// Get next char
c := s[idx]
switch mode {
// Ground mode
case modeNone:
switch c {
case '{':
// Enter open mode
buf.AppendString(Str())
mode = modeOpen
MoveUp()
case '}':
// Enter close mode
buf.AppendString(Str())
mode = modeClose
MoveUp()
}
// Encountered open '{'
case modeOpen:
switch c {
case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9':
// Starting index
mode = modeIdx
MoveUp()
case '{':
// Escaped bracket
buf.AppendByte('{')
mode = modeNone
MoveUp()
case '}':
// Format arg
AppendArg()
Reset()
MoveUp()
case ':':
// Starting operands
mode = modeOp
MoveUp()
default:
// Bad char, missing a close
buf.AppendString(`!{MISSING_CLOSE}`)
mode = modeNone
MoveUpTo()
}
// Encountered close '}'
case modeClose:
switch c {
case '}':
// Escaped close bracket
buf.AppendByte('}')
mode = modeNone
MoveUp()
default:
// Missing an open bracket
buf.AppendString(`!{MISSING_OPEN}`)
mode = modeNone
MoveUp()
}
// Preparing index
case modeIdx:
switch c {
case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9':
case ':':
if !ParseIndex() {
// Unable to parse an integer
buf.AppendString(`!{BAD_INDEX}`)
mode = modeNone
MoveUpTo()
} else {
// Starting operands
mode = modeOp
MoveUp()
}
case '}':
if !ParseIndex() {
// Unable to parse an integer
buf.AppendString(`!{BAD_INDEX}`)
} else {
// Format arg
AppendArg()
}
Reset()
MoveUp()
default:
// Not a valid index character
buf.AppendString(`!{BAD_INDEX}`)
mode = modeNone
MoveUpTo()
}
// Preparing operands
case modeOp:
switch c {
case 'k', 'v', '?':
// TODO: set flags as received
case '}':
if !ParseOp() {
// Unable to parse operands
buf.AppendString(`!{BAD_OPERAND}`)
} else {
// Format arg
AppendArg()
}
Reset()
MoveUp()
default:
// Not a valid operand char
buf.AppendString(`!{BAD_OPERAND}`)
Reset()
MoveUpTo()
}
}
}
// Append any remaining
buf.AppendString(s[last:])
}
// formatter is the default formatter instance.
var formatter = Formatter{
MaxDepth: 10,
}
// Append will append formatted form of supplied values into 'buf' using default formatter.
// See Formatter.Append() for more documentation.
func Append(buf *Buffer, v ...interface{}) {
formatter.Append(buf, v...)
}
// Appendf will append the formatted string with supplied values into 'buf' using default formatter.
// See Formatter.Appendf() for more documentation.
func Appendf(buf *Buffer, s string, a ...interface{}) {
formatter.Appendf(buf, s, a...)
}

88
vendor/codeberg.org/gruf/go-format/print.go generated vendored Normal file
View file

@ -0,0 +1,88 @@
package format
import (
"io"
"os"
"sync"
)
// pool is the global printer buffer pool.
var pool = sync.Pool{
New: func() interface{} {
return &Buffer{}
},
}
// getBuf fetches a buffer from pool.
func getBuf() *Buffer {
return pool.Get().(*Buffer)
}
// putBuf places a Buffer back in pool.
func putBuf(buf *Buffer) {
if buf.Cap() > 64<<10 {
return // drop large
}
buf.Reset()
pool.Put(buf)
}
// Sprint will format supplied values, returning this string.
func Sprint(v ...interface{}) string {
buf := Buffer{}
Append(&buf, v...)
return buf.String()
}
// Sprintf will format supplied format string and args, returning this string.
// See Formatter.Appendf() for more documentation.
func Sprintf(s string, a ...interface{}) string {
buf := Buffer{}
Appendf(&buf, s, a...)
return buf.String()
}
// Print will format supplied values, print this to os.Stdout.
func Print(v ...interface{}) {
Fprint(os.Stdout, v...) //nolint
}
// Printf will format supplied format string and args, printing this to os.Stdout.
// See Formatter.Appendf() for more documentation.
func Printf(s string, a ...interface{}) {
Fprintf(os.Stdout, s, a...) //nolint
}
// Println will format supplied values, append a trailing newline and print this to os.Stdout.
func Println(v ...interface{}) {
Fprintln(os.Stdout, v...) //nolint
}
// Fprint will format supplied values, writing this to an io.Writer.
func Fprint(w io.Writer, v ...interface{}) (int, error) {
buf := getBuf()
Append(buf, v...)
n, err := w.Write(buf.B)
putBuf(buf)
return n, err
}
// Fprintf will format supplied format string and args, writing this to an io.Writer.
// See Formatter.Appendf() for more documentation.
func Fprintf(w io.Writer, s string, a ...interface{}) (int, error) {
buf := getBuf()
Appendf(buf, s, a...)
n, err := w.Write(buf.B)
putBuf(buf)
return n, err
}
// Println will format supplied values, append a trailing newline and writer this to an io.Writer.
func Fprintln(w io.Writer, v ...interface{}) (int, error) {
buf := getBuf()
Append(buf, v...)
buf.AppendByte('\n')
n, err := w.Write(buf.B)
putBuf(buf)
return n, err
}

13
vendor/codeberg.org/gruf/go-format/util.go generated vendored Normal file
View file

@ -0,0 +1,13 @@
package format
import "strconv"
// can32bitInt returns whether it's possible for 's' to contain an int on 32bit platforms.
func can32bitInt(s string) bool {
return strconv.IntSize == 32 && (0 < len(s) && len(s) < 10)
}
// can64bitInt returns whether it's possible for 's' to contain an int on 64bit platforms.
func can64bitInt(s string) bool {
return strconv.IntSize == 64 && (0 < len(s) && len(s) < 19)
}

13
vendor/codeberg.org/gruf/go-logger/README.md generated vendored
View file

@ -1,13 +0,0 @@
Fast levelled logging package with customizable formatting.
Supports logging in 2 modes:
- no locks, fastest possible logging, no guarantees for io.Writer thread safety
- mutex locks during writes, still far faster than standard library logger
Running without locks isn't likely to cause you any issues*, but if it does, you can wrap your `io.Writer` using `AddSafety()` when instantiating your new Logger. Even when running the benchmarks, this library has no printing issues without locks, so in most cases you'll be fine, but the safety is there if you need it.
*most logging libraries advertising high speeds are likely not performing mutex locks, which is why with this library you have the option to opt-in/out of them.
Note there are 2 uses of the unsafe package:
- safer interface nil value checks, uses similar logic to reflect package to check if the value in the internal fat pointer is nil
- casting a byte slice to string to allow sharing of similar byte and string methods, performs same logic as `strings.Builder{}.String()`

21
vendor/codeberg.org/gruf/go-logger/clock.go generated vendored
View file

@ -1,21 +0,0 @@
package logger
import (
"sync"
"time"
"codeberg.org/gruf/go-nowish"
)
var (
clock = nowish.Clock{}
clockOnce = sync.Once{}
)
// startClock starts the global nowish clock.
func startClock() {
clockOnce.Do(func() {
clock.Start(time.Millisecond * 100)
clock.SetFormat("2006-01-02 15:04:05")
})
}

107
vendor/codeberg.org/gruf/go-logger/default.go generated vendored
View file

@ -1,107 +0,0 @@
package logger
import (
"os"
"sync"
)
var (
instance *Logger
instanceOnce = sync.Once{}
)
// Default returns the default Logger instance.
func Default() *Logger {
instanceOnce.Do(func() { instance = New(os.Stdout) })
return instance
}
// Debug prints the provided arguments with the debug prefix to the global Logger instance.
func Debug(a ...interface{}) {
Default().Debug(a...)
}
// Debugf prints the provided format string and arguments with the debug prefix to the global Logger instance.
func Debugf(s string, a ...interface{}) {
Default().Debugf(s, a...)
}
// Info prints the provided arguments with the info prefix to the global Logger instance.
func Info(a ...interface{}) {
Default().Info(a...)
}
// Infof prints the provided format string and arguments with the info prefix to the global Logger instance.
func Infof(s string, a ...interface{}) {
Default().Infof(s, a...)
}
// Warn prints the provided arguments with the warn prefix to the global Logger instance.
func Warn(a ...interface{}) {
Default().Warn(a...)
}
// Warnf prints the provided format string and arguments with the warn prefix to the global Logger instance.
func Warnf(s string, a ...interface{}) {
Default().Warnf(s, a...)
}
// Error prints the provided arguments with the error prefix to the global Logger instance.
func Error(a ...interface{}) {
Default().Error(a...)
}
// Errorf prints the provided format string and arguments with the error prefix to the global Logger instance.
func Errorf(s string, a ...interface{}) {
Default().Errorf(s, a...)
}
// Fatal prints the provided arguments with the fatal prefix to the global Logger instance before exiting the program with os.Exit(1).
func Fatal(a ...interface{}) {
Default().Fatal(a...)
}
// Fatalf prints the provided format string and arguments with the fatal prefix to the global Logger instance before exiting the program with os.Exit(1).
func Fatalf(s string, a ...interface{}) {
Default().Fatalf(s, a...)
}
// Log prints the provided arguments with the supplied log level to the global Logger instance.
func Log(lvl LEVEL, a ...interface{}) {
Default().Log(lvl, a...)
}
// Logf prints the provided format string and arguments with the supplied log level to the global Logger instance.
func Logf(lvl LEVEL, s string, a ...interface{}) {
Default().Logf(lvl, s, a...)
}
// LogFields prints the provided fields formatted as key-value pairs at the supplied log level to the global Logger instance.
func LogFields(lvl LEVEL, fields map[string]interface{}) {
Default().LogFields(lvl, fields)
}
// LogValues prints the provided values formatted as-so at the supplied log level to the global Logger instance.
func LogValues(lvl LEVEL, a ...interface{}) {
Default().LogValues(lvl, a...)
}
// Print simply prints provided arguments to the global Logger instance.
func Print(a ...interface{}) {
Default().Print(a...)
}
// Printf simply prints provided the provided format string and arguments to the global Logger instance.
func Printf(s string, a ...interface{}) {
Default().Printf(s, a...)
}
// PrintFields prints the provided fields formatted as key-value pairs to the global Logger instance.
func PrintFields(fields map[string]interface{}) {
Default().PrintFields(fields)
}
// PrintValues prints the provided values formatted as-so to the global Logger instance.
func PrintValues(a ...interface{}) {
Default().PrintValues(a...)
}

385
vendor/codeberg.org/gruf/go-logger/entry.go generated vendored
View file

@ -1,385 +0,0 @@
package logger
import (
"context"
"fmt"
"time"
"codeberg.org/gruf/go-bytes"
)
// Entry defines an entry in the log, it is NOT safe for concurrent use
type Entry struct {
ctx context.Context
lvl LEVEL
buf *bytes.Buffer
log *Logger
}
// Context returns the current set Entry context.Context
func (e *Entry) Context() context.Context {
return e.ctx
}
// WithContext updates Entry context value to the supplied
func (e *Entry) WithContext(ctx context.Context) *Entry {
e.ctx = ctx
return e
}
// Level appends the supplied level to the log entry, and sets the entry level.
// Please note this CAN be called and append log levels multiple times
func (e *Entry) Level(lvl LEVEL) *Entry {
e.log.Format.AppendLevel(e.buf, lvl)
e.buf.WriteByte(' ')
e.lvl = lvl
return e
}
// Timestamp appends the current timestamp to the log entry. Please note this
// CAN be called and append the timestamp multiple times
func (e *Entry) Timestamp() *Entry {
e.log.Format.AppendTimestamp(e.buf, clock.NowFormat())
e.buf.WriteByte(' ')
return e
}
// TimestampIf performs Entry.Timestamp() only IF timestamping is enabled for the Logger.
// Please note this CAN be called multiple times
func (e *Entry) TimestampIf() *Entry {
if e.log.Timestamp {
e.Timestamp()
}
return e
}
// Hooks applies currently set Hooks to the Entry. Please note this CAN be
// called and perform the Hooks multiple times
func (e *Entry) Hooks() *Entry {
for _, hook := range e.log.Hooks {
hook.Do(e)
}
return e
}
// Byte appends a byte value to the log entry
func (e *Entry) Byte(value byte) *Entry {
e.log.Format.AppendByte(e.buf, value)
e.buf.WriteByte(' ')
return e
}
// ByteField appends a byte value as key-value pair to the log entry
func (e *Entry) ByteField(key string, value byte) *Entry {
e.log.Format.AppendKey(e.buf, key)
e.log.Format.AppendByte(e.buf, value)
e.buf.WriteByte(' ')
return e
}
// Bytes appends a byte slice value as to the log entry
func (e *Entry) Bytes(value []byte) *Entry {
e.log.Format.AppendBytes(e.buf, value)
e.buf.WriteByte(' ')
return e
}
// BytesField appends a byte slice value as key-value pair to the log entry
func (e *Entry) BytesField(key string, value []byte) *Entry {
e.log.Format.AppendKey(e.buf, key)
e.log.Format.AppendBytes(e.buf, value)
e.buf.WriteByte(' ')
return e
}
// Str appends a string value to the log entry
func (e *Entry) Str(value string) *Entry {
e.log.Format.AppendString(e.buf, value)
e.buf.WriteByte(' ')
return e
}
// StrField appends a string value as key-value pair to the log entry
func (e *Entry) StrField(key string, value string) *Entry {
e.log.Format.AppendKey(e.buf, key)
e.log.Format.AppendString(e.buf, value)
e.buf.WriteByte(' ')
return e
}
// Strs appends a string slice value to the log entry
func (e *Entry) Strs(value []string) *Entry {
e.log.Format.AppendStrings(e.buf, value)
e.buf.WriteByte(' ')
return e
}
// StrsField appends a string slice value as key-value pair to the log entry
func (e *Entry) StrsField(key string, value []string) *Entry {
e.log.Format.AppendKey(e.buf, key)
e.log.Format.AppendStrings(e.buf, value)
e.buf.WriteByte(' ')
return e
}
// Int appends an int value to the log entry
func (e *Entry) Int(value int) *Entry {
e.log.Format.AppendInt(e.buf, value)
e.buf.WriteByte(' ')
return e
}
// IntField appends an int value as key-value pair to the log entry
func (e *Entry) IntField(key string, value int) *Entry {
e.log.Format.AppendKey(e.buf, key)
e.log.Format.AppendInt(e.buf, value)
e.buf.WriteByte(' ')
return e
}
// Ints appends an int slice value to the log entry
func (e *Entry) Ints(value []int) *Entry {
e.log.Format.AppendInts(e.buf, value)
e.buf.WriteByte(' ')
return e
}
// IntsField appends an int slice value as key-value pair to the log entry
func (e *Entry) IntsField(key string, value []int) *Entry {
e.log.Format.AppendKey(e.buf, key)
e.log.Format.AppendInts(e.buf, value)
e.buf.WriteByte(' ')
return e
}
// Uint appends a uint value to the log entry
func (e *Entry) Uint(value uint) *Entry {
e.log.Format.AppendUint(e.buf, value)
e.buf.WriteByte(' ')
return e
}
// UintField appends a uint value as key-value pair to the log entry
func (e *Entry) UintField(key string, value uint) *Entry {
e.log.Format.AppendKey(e.buf, key)
e.log.Format.AppendUint(e.buf, value)
e.buf.WriteByte(' ')
return e
}
// Uints appends a uint slice value to the log entry
func (e *Entry) Uints(value []uint) *Entry {
e.log.Format.AppendUints(e.buf, value)
e.buf.WriteByte(' ')
return e
}
// UintsField appends a uint slice value as key-value pair to the log entry
func (e *Entry) UintsField(key string, value []uint) *Entry {
e.log.Format.AppendKey(e.buf, key)
e.log.Format.AppendUints(e.buf, value)
e.buf.WriteByte(' ')
return e
}
// Float appends a float value to the log entry
func (e *Entry) Float(value float64) *Entry {
e.log.Format.AppendFloat(e.buf, value)
e.buf.WriteByte(' ')
return e
}
// FloatField appends a float value as key-value pair to the log entry
func (e *Entry) FloatField(key string, value float64) *Entry {
e.log.Format.AppendKey(e.buf, key)
e.log.Format.AppendFloat(e.buf, value)
e.buf.WriteByte(' ')
return e
}
// Floats appends a float slice value to the log entry
func (e *Entry) Floats(value []float64) *Entry {
e.log.Format.AppendFloats(e.buf, value)
e.buf.WriteByte(' ')
return e
}
// FloatsField appends a float slice value as key-value pair to the log entry
func (e *Entry) FloatsField(key string, value []float64) *Entry {
e.log.Format.AppendKey(e.buf, key)
e.log.Format.AppendFloats(e.buf, value)
e.buf.WriteByte(' ')
return e
}
// Bool appends a bool value to the log entry
func (e *Entry) Bool(value bool) *Entry {
e.log.Format.AppendBool(e.buf, value)
e.buf.WriteByte(' ')
return e
}
// BoolField appends a bool value as key-value pair to the log entry
func (e *Entry) BoolField(key string, value bool) *Entry {
e.log.Format.AppendKey(e.buf, key)
e.log.Format.AppendBool(e.buf, value)
e.buf.WriteByte(' ')
return e
}
// Bools appends a bool slice value to the log entry
func (e *Entry) Bools(value []bool) *Entry {
e.log.Format.AppendBools(e.buf, value)
e.buf.WriteByte(' ')
return e
}
// BoolsField appends a bool slice value as key-value pair to the log entry
func (e *Entry) BoolsField(key string, value []bool) *Entry {
e.log.Format.AppendKey(e.buf, key)
e.log.Format.AppendBools(e.buf, value)
e.buf.WriteByte(' ')
return e
}
// Time appends a time.Time value to the log entry
func (e *Entry) Time(value time.Time) *Entry {
e.log.Format.AppendTime(e.buf, value)
e.buf.WriteByte(' ')
return e
}
// TimeField appends a time.Time value as key-value pair to the log entry
func (e *Entry) TimeField(key string, value time.Time) *Entry {
e.log.Format.AppendKey(e.buf, key)
e.log.Format.AppendTime(e.buf, value)
e.buf.WriteByte(' ')
return e
}
// Times appends a time.Time slice value to the log entry
func (e *Entry) Times(value []time.Time) *Entry {
e.log.Format.AppendTimes(e.buf, value)
e.buf.WriteByte(' ')
return e
}
// TimesField appends a time.Time slice value as key-value pair to the log entry
func (e *Entry) TimesField(key string, value []time.Time) *Entry {
e.log.Format.AppendKey(e.buf, key)
e.log.Format.AppendTimes(e.buf, value)
e.buf.WriteByte(' ')
return e
}
// DurationField appends a time.Duration value to the log entry
func (e *Entry) Duration(value time.Duration) *Entry {
e.log.Format.AppendDuration(e.buf, value)
e.buf.WriteByte(' ')
return e
}
// DurationField appends a time.Duration value as key-value pair to the log entry
func (e *Entry) DurationField(key string, value time.Duration) *Entry {
e.log.Format.AppendKey(e.buf, key)
e.log.Format.AppendDuration(e.buf, value)
e.buf.WriteByte(' ')
return e
}
// Durations appends a time.Duration slice value to the log entry
func (e *Entry) Durations(value []time.Duration) *Entry {
e.log.Format.AppendDurations(e.buf, value)
e.buf.WriteByte(' ')
return e
}
// DurationsField appends a time.Duration slice value as key-value pair to the log entry
func (e *Entry) DurationsField(key string, value []time.Duration) *Entry {
e.log.Format.AppendKey(e.buf, key)
e.log.Format.AppendDurations(e.buf, value)
e.buf.WriteByte(' ')
return e
}
// Field appends an interface value as key-value pair to the log entry
func (e *Entry) Field(key string, value interface{}) *Entry {
e.log.Format.AppendKey(e.buf, key)
e.log.Format.AppendValue(e.buf, value)
e.buf.WriteByte(' ')
return e
}
// Fields appends a map of key-value pairs to the log entry
func (e *Entry) Fields(fields map[string]interface{}) *Entry {
for key, value := range fields {
e.Field(key, value)
}
return e
}
// Values appends the given values to the log entry formatted as values, without a key.
func (e *Entry) Values(values ...interface{}) *Entry {
for _, value := range values {
e.log.Format.AppendValue(e.buf, value)
e.buf.WriteByte(' ')
}
return e
}
// Append will append the given args formatted using fmt.Sprint(a...) to the Entry.
func (e *Entry) Append(a ...interface{}) *Entry {
fmt.Fprint(e.buf, a...)
e.buf.WriteByte(' ')
return e
}
// Appendf will append the given format string and args using fmt.Sprintf(s, a...) to the Entry.
func (e *Entry) Appendf(s string, a ...interface{}) *Entry {
fmt.Fprintf(e.buf, s, a...)
e.buf.WriteByte(' ')
return e
}
// Msg appends the fmt.Sprint() formatted final message to the log and calls .Send()
func (e *Entry) Msg(a ...interface{}) {
e.log.Format.AppendMsg(e.buf, a...)
e.Send()
}
// Msgf appends the fmt.Sprintf() formatted final message to the log and calls .Send()
func (e *Entry) Msgf(s string, a ...interface{}) {
e.log.Format.AppendMsgf(e.buf, s, a...)
e.Send()
}
// Send triggers write of the log entry, skipping if the entry's log LEVEL
// is below the currently set Logger level, and releases the Entry back to
// the Logger's Entry pool. So it is NOT safe to continue using this Entry
// object after calling .Send(), .Msg() or .Msgf()
func (e *Entry) Send() {
// If nothing to do, return
if e.lvl < e.log.Level || e.buf.Len() < 1 {
e.reset()
return
}
// Ensure a final new line
if e.buf.B[e.buf.Len()-1] != '\n' {
e.buf.WriteByte('\n')
}
// Write, reset and release
e.log.Output.Write(e.buf.B)
e.reset()
}
func (e *Entry) reset() {
// Reset all
e.ctx = nil
e.buf.Reset()
e.lvl = unset
// Release to pool
e.log.pool.Put(e)
}

Some files were not shown because too many files have changed in this diff Show more