Mirrors/gotosocial
2
0
Fork 0
mirror of https://github.com/superseriousbusiness/gotosocial synced 2024-11-10 06:54:16 +00:00
Code Issues Projects Releases Packages Wiki Activity

[feature] support processing of (many) more media types (#3090)

Browse source 
* initial work replacing our media decoding / encoding pipeline with ffprobe + ffmpeg

* specify the video codec to use when generating static image from emoji

* update go-storage library (fixes incompatibility after updating go-iotools)

* maintain image aspect ratio when generating a thumbnail for it

* update readme to show go-ffmpreg

* fix a bunch of media tests, move filesize checking to callers of media manager for more flexibility

* remove extra debug from error message

* fix up incorrect function signatures

* update PutFile to just use regular file copy, as changes are file is on separate partition

* fix remaining tests, remove some unneeded tests now we're working with ffmpeg/ffprobe

* update more tests, add more code comments

* add utilities to generate processed emoji / media outputs

* fix remaining tests

* add test for opus media file, add license header to utility cmds

* limit the number of concurrently available ffmpeg / ffprobe instances

* reduce number of instances

* further reduce number of instances

* fix envparsing test with configuration variables

* update docs and configuration with new media-{local,remote}-max-size variables
This commit is contained in:
kim 2024-07-12 09:39:47 +00:00 committed by GitHub
parent 5bc567196b
commit cde2fb6244
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
376 changed files with 8026 additions and 54091 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
README.md
View file

@ -261,6 +261,7 @@ The following open source libraries, frameworks, and tools are used by GoToSocia
- [gruf/go-debug](https://codeberg.org/gruf/go-debug); debug build tag. [MIT License](https://spdx.org/licenses/MIT.html).
- [gruf/go-errors](https://codeberg.org/gruf/go-errors); context-like error w/ value wrapping [MIT License](https://spdx.org/licenses/MIT.html).
- [gruf/go-fastcopy](https://codeberg.org/gruf/go-fastcopy); performant (buffer pooled) I/O copying [MIT License](https://spdx.org/licenses/MIT.html).
- [gruf/go-ffmpreg](https://codeberg.org/gruf/go-ffmpreg); embedded ffmpeg / ffprobe WASM binaries [GPL-3.0 License](https://spdx.org/licenses/GPL-3.0-only.html).
- [gruf/go-kv](https://codeberg.org/gruf/go-kv); log field formatting. [MIT License](https://spdx.org/licenses/MIT.html).
- [gruf/go-list](https://codeberg.org/gruf/go-list); generic doubly linked list. [MIT License](https://spdx.org/licenses/MIT.html).
- [gruf/go-mutexes](https://codeberg.org/gruf/go-mutexes); safemutex & mutex map. [MIT License](https://spdx.org/licenses/MIT.html).

45
cmd/gotosocial/action/server/server.go
View file

@ -24,12 +24,14 @@ import (
"net/http"
"os"
"os/signal"
"runtime"
"strings"
"syscall"
"time"
"github.com/KimMachineGun/automemlimit/memlimit"
"github.com/gin-gonic/gin"
"github.com/ncruces/go-sqlite3"
"github.com/superseriousbusiness/gotosocial/cmd/gotosocial/action"
"github.com/superseriousbusiness/gotosocial/internal/api"
apiutil "github.com/superseriousbusiness/gotosocial/internal/api/util"
@ -37,6 +39,7 @@ import (
"github.com/superseriousbusiness/gotosocial/internal/filter/spam"
"github.com/superseriousbusiness/gotosocial/internal/filter/visibility"
"github.com/superseriousbusiness/gotosocial/internal/gtserror"
"github.com/superseriousbusiness/gotosocial/internal/media/ffmpeg"
"github.com/superseriousbusiness/gotosocial/internal/messages"
"github.com/superseriousbusiness/gotosocial/internal/metrics"
"github.com/superseriousbusiness/gotosocial/internal/middleware"
@ -66,14 +69,15 @@ import (
// Start creates and starts a gotosocial server
var Start action.GTSAction = func(ctx context.Context) error {
if _, err := maxprocs.Set(maxprocs.Logger(nil)); err != nil {
log.Warnf(ctx, "could not set CPU limits from cgroup: %s", err)
}
// Set GOMAXPROCS / GOMEMLIMIT
// to match container limits.
setLimits(ctx)
if _, err := memlimit.SetGoMemLimitWithOpts(); err != nil {
if !strings.Contains(err.Error(), "cgroup mountpoint does not exist") {
log.Warnf(ctx, "could not set Memory limits from cgroup: %s", err)
}
// Compile WASM modules ahead of first use
// to prevent unexpected initial slowdowns.
log.Info(ctx, "precompiling WebAssembly")
if err := precompileWASM(ctx); err != nil {
return err
}
var (
@ -429,3 +433,30 @@ var Start action.GTSAction = func(ctx context.Context) error {
return nil
}
func setLimits(ctx context.Context) {
if _, err := maxprocs.Set(maxprocs.Logger(nil)); err != nil {
log.Warnf(ctx, "could not set CPU limits from cgroup: %s", err)
}
if _, err := memlimit.SetGoMemLimitWithOpts(); err != nil {
if !strings.Contains(err.Error(), "cgroup mountpoint does not exist") {
log.Warnf(ctx, "could not set Memory limits from cgroup: %s", err)
}
}
}
func precompileWASM(ctx context.Context) error {
// TODO: make max number instances configurable
maxprocs := runtime.GOMAXPROCS(0)
if err := sqlite3.Initialize(); err != nil {
return gtserror.Newf("error compiling sqlite3: %w", err)
}
if err := ffmpeg.InitFfmpeg(ctx, maxprocs); err != nil {
return gtserror.Newf("error compiling ffmpeg: %w", err)
}
if err := ffmpeg.InitFfprobe(ctx, maxprocs); err != nil {
return gtserror.Newf("error compiling ffprobe: %w", err)
}
return nil
}

122
cmd/process-emoji/main.go Normal file
View file

@ -0,0 +1,122 @@
// GoToSocial
// Copyright (C) GoToSocial Authors admin@gotosocial.org
// SPDX-License-Identifier: AGPL-3.0-or-later
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package main
import (
"context"
"io"
"os"
"os/signal"
"syscall"
"codeberg.org/gruf/go-storage/memory"
"github.com/superseriousbusiness/gotosocial/internal/config"
"github.com/superseriousbusiness/gotosocial/internal/db/bundb"
"github.com/superseriousbusiness/gotosocial/internal/log"
"github.com/superseriousbusiness/gotosocial/internal/media"
"github.com/superseriousbusiness/gotosocial/internal/state"
"github.com/superseriousbusiness/gotosocial/internal/storage"
"github.com/superseriousbusiness/gotosocial/internal/util"
)
func main() {
ctx := context.Background()
ctx, cncl := signal.NotifyContext(ctx, syscall.SIGTERM, syscall.SIGINT)
defer cncl()
if len(os.Args) != 3 {
log.Panic(ctx, "Usage: go run ./cmd/process-emoji <input-file> <output-static>")
}
var st storage.Driver
st.Storage = memory.Open(10, true)
var state state.State
state.Storage = &st
state.Caches.Init()
var err error
config.SetHost("example.com")
config.SetStorageBackend("disk")
config.SetStorageLocalBasePath("/tmp/gotosocial")
config.SetDbType("sqlite")
config.SetDbAddress(":memory:")
state.DB, err = bundb.NewBunDBService(ctx, &state)
if err != nil {
log.Panic(ctx, err)
}
if err := state.DB.CreateInstanceAccount(ctx); err != nil {
log.Panicf(ctx, "error creating instance account: %s", err)
}
if err := state.DB.CreateInstanceInstance(ctx); err != nil {
log.Panicf(ctx, "error creating instance instance: %s", err)
}
if err := state.DB.CreateInstanceApplication(ctx); err != nil {
log.Panicf(ctx, "error creating instance application: %s", err)
}
mgr := media.NewManager(&state)
processing, err := mgr.CreateEmoji(ctx,
"emoji",
"example.com",
func(ctx context.Context) (reader io.ReadCloser, err error) {
return os.Open(os.Args[1])
},
media.AdditionalEmojiInfo{
URI: util.Ptr("example.com/emoji"),
},
)
if err != nil {
log.Panic(ctx, err)
}
emoji, err := processing.Load(ctx)
if err != nil {
log.Panic(ctx, err)
}
copyFile(ctx, &st, emoji.ImageStaticPath, os.Args[2])
}
func copyFile(ctx context.Context, st *storage.Driver, key string, path string) {
rc, err := st.GetStream(ctx, key)
if err != nil {
log.Panic(ctx, err)
}
defer rc.Close()
_ = os.Remove(path)
output, err := os.Create(path)
if err != nil {
log.Panic(ctx, err)
}
defer output.Close()
_, err = io.Copy(output, rc)
if err != nil {
log.Panic(ctx, err)
}
}

124
cmd/process-media/main.go Normal file
View file

@ -0,0 +1,124 @@
// GoToSocial
// Copyright (C) GoToSocial Authors admin@gotosocial.org
// SPDX-License-Identifier: AGPL-3.0-or-later
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package main
import (
"context"
"io"
"os"
"os/signal"
"syscall"
"codeberg.org/gruf/go-storage/memory"
"github.com/superseriousbusiness/gotosocial/internal/config"
"github.com/superseriousbusiness/gotosocial/internal/db/bundb"
"github.com/superseriousbusiness/gotosocial/internal/log"
"github.com/superseriousbusiness/gotosocial/internal/media"
"github.com/superseriousbusiness/gotosocial/internal/state"
"github.com/superseriousbusiness/gotosocial/internal/storage"
)
func main() {
ctx := context.Background()
ctx, cncl := signal.NotifyContext(ctx, syscall.SIGTERM, syscall.SIGINT)
defer cncl()
if len(os.Args) != 4 {
log.Panic(ctx, "Usage: go run ./cmd/process-media <input-file> <output-processed> <output-thumbnail>")
}
var st storage.Driver
st.Storage = memory.Open(10, true)
var state state.State
state.Storage = &st
state.Caches.Init()
var err error
config.SetHost("example.com")
config.SetStorageBackend("disk")
config.SetStorageLocalBasePath("/tmp/gotosocial")
config.SetDbType("sqlite")
config.SetDbAddress(":memory:")
state.DB, err = bundb.NewBunDBService(ctx, &state)
if err != nil {
log.Panic(ctx, err)
}
if err := state.DB.CreateInstanceAccount(ctx); err != nil {
log.Panicf(ctx, "error creating instance account: %s", err)
}
if err := state.DB.CreateInstanceInstance(ctx); err != nil {
log.Panicf(ctx, "error creating instance instance: %s", err)
}
if err := state.DB.CreateInstanceApplication(ctx); err != nil {
log.Panicf(ctx, "error creating instance application: %s", err)
}
account, err := state.DB.GetInstanceAccount(ctx, "")
if err != nil {
log.Panic(ctx, err)
}
mgr := media.NewManager(&state)
processing, err := mgr.CreateMedia(ctx,
account.ID,
func(ctx context.Context) (reader io.ReadCloser, err error) {
return os.Open(os.Args[1])
},
media.AdditionalMediaInfo{},
)
if err != nil {
log.Panic(ctx, err)
}
media, err := processing.Load(ctx)
if err != nil {
log.Panic(ctx, err)
}
copyFile(ctx, &st, media.File.Path, os.Args[2])
copyFile(ctx, &st, media.Thumbnail.Path, os.Args[3])
}
func copyFile(ctx context.Context, st *storage.Driver, key string, path string) {
rc, err := st.GetStream(ctx, key)
if err != nil {
log.Panic(ctx, err)
}
defer rc.Close()
_ = os.Remove(path)
output, err := os.Create(path)
if err != nil {
log.Panic(ctx, err)
}
defer output.Close()
_, err = io.Copy(output, rc)
if err != nil {
log.Panic(ctx, err)
}
}

23
docs/configuration/media.md
View file

@ -7,25 +7,24 @@
##### MEDIA CONFIG #####
########################
# Config pertaining to media uploads (videos, image, image descriptions, emoji).
# Config pertaining to media uploads (media, image descriptions, emoji).
# Size. Maximum allowed image upload size in bytes.
#
# Raising this limit may cause other servers to not fetch media
# attached to a post.
#
# Examples: [2097152, 10485760, 10MB, 10MiB]
# Default: 10MiB (10485760 bytes)
media-image-max-size: 10MiB
# Size. Maximum allowed video upload size in bytes.
# Size. Max size in bytes of media uploads via API.
#
# Raising this limit may cause other servers to not fetch media
# attached to a post.
#
# Examples: [2097152, 10485760, 40MB, 40MiB]
# Default: 40MiB (41943040 bytes)
media-video-max-size: 40MiB
media-local-max-size: 40MiB
# Size. Max size in bytes of media to download from other instances.
#
# Lowering this limit may cause your instance not to fetch post media.
#
# Examples: [2097152, 10485760, 40MB, 40MiB]
# Default: 40MiB (41943040 bytes)
media-remote-max-size: 40MiB
# Int. Minimum amount of characters required as an image or video description.
# Examples: [500, 1000, 1500]

23
example/config.yaml
View file

@ -444,25 +444,24 @@ accounts-custom-css-length: 10000
##### MEDIA CONFIG #####
########################
# Config pertaining to media uploads (videos, image, image descriptions, emoji).
# Config pertaining to media uploads (media, image descriptions, emoji).
# Size. Maximum allowed image upload size in bytes.
#
# Raising this limit may cause other servers to not fetch media
# attached to a post.
#
# Examples: [2097152, 10485760, 10MB, 10MiB]
# Default: 10MiB (10485760 bytes)
media-image-max-size: 10MiB
# Size. Maximum allowed video upload size in bytes.
# Size. Max size in bytes of media uploads via API.
#
# Raising this limit may cause other servers to not fetch media
# attached to a post.
#
# Examples: [2097152, 10485760, 40MB, 40MiB]
# Default: 40MiB (41943040 bytes)
media-video-max-size: 40MiB
media-local-max-size: 40MiB
# Size. Max size in bytes of media to download from other instances.
#
# Lowering this limit may cause your instance not to fetch post media.
#
# Examples: [2097152, 10485760, 40MB, 40MiB]
# Default: 40MiB (41943040 bytes)
media-remote-max-size: 40MiB
# Int. Minimum amount of characters required as an image or video description.
# Examples: [500, 1000, 1500]

23
go.mod
View file

@ -12,20 +12,20 @@ require (
codeberg.org/gruf/go-debug v1.3.0
codeberg.org/gruf/go-errors/v2 v2.3.2
codeberg.org/gruf/go-fastcopy v1.1.2
codeberg.org/gruf/go-iotools v0.0.0-20230811115124-5d4223615a7f
codeberg.org/gruf/go-ffmpreg v0.2.2
codeberg.org/gruf/go-iotools v0.0.0-20240710125620-934ae9c654cf
codeberg.org/gruf/go-kv v1.6.4
codeberg.org/gruf/go-list v0.0.0-20240425093752-494db03d641f
codeberg.org/gruf/go-logger/v2 v2.2.1
codeberg.org/gruf/go-mempool v0.0.0-20240507125005-cef10d64a760
codeberg.org/gruf/go-mimetypes v1.2.0
codeberg.org/gruf/go-mutexes v1.5.1
codeberg.org/gruf/go-runners v1.6.2
codeberg.org/gruf/go-sched v1.2.3
codeberg.org/gruf/go-storage v0.1.1
codeberg.org/gruf/go-storage v0.1.2
codeberg.org/gruf/go-structr v0.8.7
codeberg.org/superseriousbusiness/exif-terminator v0.7.0
github.com/DmitriyVTitov/size v1.5.0
github.com/KimMachineGun/automemlimit v0.6.1
github.com/abema/go-mp4 v1.2.0
github.com/buckket/go-blurhash v1.1.0
github.com/coreos/go-oidc/v3 v3.10.0
github.com/disintegration/imaging v1.6.2
@ -39,7 +39,6 @@ require (
github.com/google/uuid v1.6.0
github.com/gorilla/feeds v1.2.0
github.com/gorilla/websocket v1.5.2
github.com/h2non/filetype v1.1.3
github.com/jackc/pgx/v5 v5.6.0
github.com/microcosm-cc/bluemonday v1.0.27
github.com/miekg/dns v1.1.61
@ -56,6 +55,7 @@ require (
github.com/superseriousbusiness/oauth2/v4 v4.3.2-SSB.0.20230227143000-f4900831d6c8
github.com/tdewolff/minify/v2 v2.20.34
github.com/technologize/otel-go-contrib v1.1.1
github.com/tetratelabs/wazero v1.7.3
github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80
github.com/ulule/limiter/v3 v3.11.2
github.com/uptrace/bun v1.2.1
@ -74,7 +74,6 @@ require (
go.opentelemetry.io/otel/trace v1.26.0
go.uber.org/automaxprocs v1.5.3
golang.org/x/crypto v0.25.0
golang.org/x/image v0.18.0
golang.org/x/net v0.27.0
golang.org/x/oauth2 v0.21.0
golang.org/x/text v0.16.0
@ -107,17 +106,11 @@ require (
github.com/coreos/go-systemd/v22 v22.3.2 // indirect
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
github.com/docker/go-units v0.5.0 // indirect
github.com/dsoprea/go-exif/v3 v3.0.0-20210625224831-a6301f85c82b // indirect
github.com/dsoprea/go-iptc v0.0.0-20200610044640-bc9ca208b413 // indirect
github.com/dsoprea/go-logging v0.0.0-20200710184922-b02d349568dd // indirect
github.com/dsoprea/go-photoshop-info-format v0.0.0-20200610045659-121dd752914d // indirect
github.com/dsoprea/go-utility/v2 v2.0.0-20200717064901-2fccff4aa15e // indirect
github.com/dustin/go-humanize v1.0.1 // indirect
github.com/felixge/httpsnoop v1.0.4 // indirect
github.com/fsnotify/fsnotify v1.7.0 // indirect
github.com/gabriel-vasile/mimetype v1.4.3 // indirect
github.com/gin-contrib/sse v0.1.0 // indirect
github.com/go-errors/errors v1.4.1 // indirect
github.com/go-fed/httpsig v1.1.0 // indirect
github.com/go-ini/ini v1.67.0 // indirect
github.com/go-jose/go-jose/v4 v4.0.1 // indirect
@ -137,11 +130,9 @@ require (
github.com/go-playground/locales v0.14.1 // indirect
github.com/go-playground/universal-translator v0.18.1 // indirect
github.com/go-playground/validator/v10 v10.20.0 // indirect
github.com/go-xmlfmt/xmlfmt v0.0.0-20211206191508-7fd73a941850 // indirect
github.com/goccy/go-json v0.10.3 // indirect
github.com/godbus/dbus/v5 v5.0.4 // indirect
github.com/golang-jwt/jwt v3.2.2+incompatible // indirect
github.com/golang/geo v0.0.0-20210211234256-740aa86cb551 // indirect
github.com/gorilla/context v1.1.2 // indirect
github.com/gorilla/css v1.0.1 // indirect
github.com/gorilla/handlers v1.5.2 // indirect
@ -196,10 +187,7 @@ require (
github.com/spf13/cast v1.6.0 // indirect
github.com/spf13/pflag v1.0.5 // indirect
github.com/subosito/gotenv v1.6.0 // indirect
github.com/superseriousbusiness/go-jpeg-image-structure/v2 v2.0.0-20220321154430-d89a106fdabe // indirect
github.com/superseriousbusiness/go-png-image-structure/v2 v2.0.1-SSB // indirect
github.com/tdewolff/parse/v2 v2.7.15 // indirect
github.com/tetratelabs/wazero v1.7.3 // indirect
github.com/tmthrgd/go-hex v0.0.0-20190904060850-447a3041c3bc // indirect
github.com/toqueteos/webbrowser v1.2.0 // indirect
github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
@ -213,6 +201,7 @@ require (
go.uber.org/multierr v1.11.0 // indirect
golang.org/x/arch v0.8.0 // indirect
golang.org/x/exp v0.0.0-20240222234643-814bf88cf225 // indirect
golang.org/x/image v0.18.0 // indirect
golang.org/x/mod v0.18.0 // indirect
golang.org/x/sync v0.7.0 // indirect
golang.org/x/sys v0.22.0 // indirect

61
go.sum
View file

@ -52,8 +52,10 @@ codeberg.org/gruf/go-fastcopy v1.1.2 h1:YwmYXPsyOcRBxKEE2+w1bGAZfclHVaPijFsOVOcn
codeberg.org/gruf/go-fastcopy v1.1.2/go.mod h1:GDDYR0Cnb3U/AIfGM3983V/L+GN+vuwVMvrmVABo21s=
codeberg.org/gruf/go-fastpath/v2 v2.0.0 h1:iAS9GZahFhyWEH0KLhFEJR+txx1ZhMXxYzu2q5Qo9c0=
codeberg.org/gruf/go-fastpath/v2 v2.0.0/go.mod h1:3pPqu5nZjpbRrOqvLyAK7puS1OfEtQvjd6342Cwz56Q=
codeberg.org/gruf/go-iotools v0.0.0-20230811115124-5d4223615a7f h1:Kazm/PInN2m1SannRMRe3DQGQc9V2EuetsQ9KAi+pBQ=
codeberg.org/gruf/go-iotools v0.0.0-20230811115124-5d4223615a7f/go.mod h1:B8uq4yHtIcKXhBZT9C/SYisz25lldLHMVpwZPz4ADLQ=
codeberg.org/gruf/go-ffmpreg v0.2.2 h1:K4I/7+BuzPLOVjL3hzTFdL8Z9wC0oRCK3xMKNVE86TE=
codeberg.org/gruf/go-ffmpreg v0.2.2/go.mod h1:oPMfBkOK7xmR/teT/dKW6SeMFpRos9ceR/OuUrxBfcQ=
codeberg.org/gruf/go-iotools v0.0.0-20240710125620-934ae9c654cf h1:84s/ii8N6lYlskZjHH+DG6jyia8w2mXMZlRwFn8Gs3A=
codeberg.org/gruf/go-iotools v0.0.0-20240710125620-934ae9c654cf/go.mod h1:zZAICsp5rY7+hxnws2V0ePrWxE0Z2Z/KXcN3p/RQCfk=
codeberg.org/gruf/go-kv v1.6.4 h1:3NZiW8HVdBM3kpOiLb7XfRiihnzZWMAixdCznguhILk=
codeberg.org/gruf/go-kv v1.6.4/go.mod h1:O/YkSvKiS9XsRolM3rqCd9YJmND7dAXu9z+PrlYO4bc=
codeberg.org/gruf/go-list v0.0.0-20240425093752-494db03d641f h1:Ss6Z+vygy+jOGhj96d/GwsYYDd22QmIcH74zM7/nQkw=
@ -68,18 +70,18 @@ codeberg.org/gruf/go-maps v1.0.3 h1:VDwhnnaVNUIy5O93CvkcE2IZXnMB1+IJjzfop9V12es=
codeberg.org/gruf/go-maps v1.0.3/go.mod h1:D5LNDxlC9rsDuVQVM6JObaVGAdHB6g2dTdOdkh1aXWA=
codeberg.org/gruf/go-mempool v0.0.0-20240507125005-cef10d64a760 h1:m2/UCRXhjDwAg4vyji6iKCpomKw6P4PmBOUi5DvAMH4=
codeberg.org/gruf/go-mempool v0.0.0-20240507125005-cef10d64a760/go.mod h1:E3RcaCFNq4zXpvaJb8lfpPqdUAmSkP5F1VmMiEUYTEk=
codeberg.org/gruf/go-mimetypes v1.2.0 h1:3rZGXY/SkNYbamiddWXs2gETXIBkGIeWYnbWpp2OEbc=
codeberg.org/gruf/go-mimetypes v1.2.0/go.mod h1:YiUWRj/nAdJQc+UFRvcsL6xXZsbc6b6Ic739ycEO8Yg=
codeberg.org/gruf/go-mutexes v1.5.1 h1:xICU0WXhWr6wf+Iror4eE3xT+xnXNPrO6o77D/G6QuY=
codeberg.org/gruf/go-mutexes v1.5.1/go.mod h1:rPEqQ/y6CmGITaZ3GPTMQVsoZAOzbsAHyIaLsJcOqVE=
codeberg.org/gruf/go-runners v1.6.2 h1:oQef9niahfHu/wch14xNxlRMP8i+ABXH1Cb9PzZ4oYo=
codeberg.org/gruf/go-runners v1.6.2/go.mod h1:Tq5PrZ/m/rBXbLZz0u5if+yP3nG5Sf6S8O/GnyEePeQ=
codeberg.org/gruf/go-sched v1.2.3 h1:H5ViDxxzOBR3uIyGBCf0eH8b1L8wMybOXcdtUUTXZHk=
codeberg.org/gruf/go-sched v1.2.3/go.mod h1:vT9uB6KWFIIwnG9vcPY2a0alYNoqdL1mSzRM8I+PK7A=
codeberg.org/gruf/go-storage v0.1.1 h1:CSX1PMMg/7vqqK8aCFtq94xCrOB3xhj7eWIvzILdLpY=
codeberg.org/gruf/go-storage v0.1.1/go.mod h1:145IWMUOc6YpIiZIiCIEwkkNZZPiSbwMnZxRjSc5q6c=
codeberg.org/gruf/go-storage v0.1.2 h1:dIOVOKq1CJpRmuhbB8Zok3mmo8V6VV/nX5GLIm6hywA=
codeberg.org/gruf/go-storage v0.1.2/go.mod h1:LRDpFHqRJi0f+35c3ltBH2e/pGfwY5dGlNlgCJ/R1DA=
codeberg.org/gruf/go-structr v0.8.7 h1:agYCI6tSXU4JHVYPwZk3Og5rrBePNVv5iPWsDu7ZJIw=
codeberg.org/gruf/go-structr v0.8.7/go.mod h1:O0FTNgzUnUKwWey4dEW99QD8rPezKPi5sxCVxYOJ1Fg=
codeberg.org/superseriousbusiness/exif-terminator v0.7.0 h1:Y6VApSXhKqExG0H2hZ2JelRK4xmWdjDQjn13CpEfzko=
codeberg.org/superseriousbusiness/exif-terminator v0.7.0/go.mod h1:gCWKduudUWFzsnixoMzu0FYVdxHWG+AbXnZ50DqxsUE=
dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
@ -94,8 +96,6 @@ github.com/Masterminds/semver/v3 v3.2.1 h1:RN9w6+7QoMeJVGyfmbcgs28Br8cvmnucEXnY0
github.com/Masterminds/semver/v3 v3.2.1/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYrf8m9wsX0PNOMQ=
github.com/Masterminds/sprig/v3 v3.2.3 h1:eL2fZNezLomi0uOLqjQoN6BfsDD+fyLtgbJMAj9n6YA=
github.com/Masterminds/sprig/v3 v3.2.3/go.mod h1:rXcFaZ2zZbLRJv/xSysmlgIM1u11eBaRMhvYXJNkGuM=
github.com/abema/go-mp4 v1.2.0 h1:gi4X8xg/m179N/J15Fn5ugywN9vtI6PLk6iLldHGLAk=
github.com/abema/go-mp4 v1.2.0/go.mod h1:vPl9t5ZK7K0x68jh12/+ECWBCXoWuIDtNgPtU2f04ws=
github.com/ajg/form v1.5.1 h1:t9c7v8JUKu/XxOGBU0yjNpaMloxGEJhUkqFRq0ibGeU=
github.com/ajg/form v1.5.1/go.mod h1:uL1WgH+h2mgNtvBq0339dVnzXdBETtL2LeUXaIv25UY=
github.com/andybalholm/brotli v1.0.0/go.mod h1:loMXtMfwqflxFJPmdbJO0a3KNoPuLBgiu3qAvBg8x/Y=
@ -138,7 +138,6 @@ github.com/coreos/go-oidc/v3 v3.10.0/go.mod h1:5j11xcw0D3+SGxn6Z/WFADsgcWVMyNAlS
github.com/coreos/go-systemd/v22 v22.3.2 h1:D9/bQk5vlXQFZ6Kwuu6zaiXJ9oTPe68++AzAJc1DzSI=
github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY=
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@ -148,22 +147,6 @@ github.com/disintegration/imaging v1.6.2 h1:w1LecBlG2Lnp8B3jk5zSuNqd7b4DXhcjwek1
github.com/disintegration/imaging v1.6.2/go.mod h1:44/5580QXChDfwIclfc/PCwrr44amcmDAg8hxG0Ewe4=
github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4=
github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
github.com/dsoprea/go-exif/v2 v2.0.0-20200321225314-640175a69fe4/go.mod h1:Lm2lMM2zx8p4a34ZemkaUV95AnMl4ZvLbCUbwOvLC2E=
github.com/dsoprea/go-exif/v3 v3.0.0-20200717053412-08f1b6708903/go.mod h1:0nsO1ce0mh5czxGeLo4+OCZ/C6Eo6ZlMWsz7rH/Gxv8=
github.com/dsoprea/go-exif/v3 v3.0.0-20210428042052-dca55bf8ca15/go.mod h1:cg5SNYKHMmzxsr9X6ZeLh/nfBRHHp5PngtEPcujONtk=
github.com/dsoprea/go-exif/v3 v3.0.0-20210625224831-a6301f85c82b h1:NgNuLvW/gAFKU30ULWW0gtkCt56JfB7FrZ2zyo0wT8I=
github.com/dsoprea/go-exif/v3 v3.0.0-20210625224831-a6301f85c82b/go.mod h1:cg5SNYKHMmzxsr9X6ZeLh/nfBRHHp5PngtEPcujONtk=
github.com/dsoprea/go-iptc v0.0.0-20200610044640-bc9ca208b413 h1:YDRiMEm32T60Kpm35YzOK9ZHgjsS1Qrid+XskNcsdp8=
github.com/dsoprea/go-iptc v0.0.0-20200610044640-bc9ca208b413/go.mod h1:kYIdx9N9NaOyD7U6D+YtExN7QhRm+5kq7//yOsRXQtM=
github.com/dsoprea/go-logging v0.0.0-20190624164917-c4f10aab7696/go.mod h1:Nm/x2ZUNRW6Fe5C3LxdY1PyZY5wmDv/s5dkPJ/VB3iA=
github.com/dsoprea/go-logging v0.0.0-20200517223158-a10564966e9d/go.mod h1:7I+3Pe2o/YSU88W0hWlm9S22W7XI1JFNJ86U0zPKMf8=
github.com/dsoprea/go-logging v0.0.0-20200710184922-b02d349568dd h1:l+vLbuxptsC6VQyQsfD7NnEC8BZuFpz45PgY+pH8YTg=
github.com/dsoprea/go-logging v0.0.0-20200710184922-b02d349568dd/go.mod h1:7I+3Pe2o/YSU88W0hWlm9S22W7XI1JFNJ86U0zPKMf8=
github.com/dsoprea/go-photoshop-info-format v0.0.0-20200610045659-121dd752914d h1:dg6UMHa50VI01WuPWXPbNJpO8QSyvIF5T5n2IZiqX3A=
github.com/dsoprea/go-photoshop-info-format v0.0.0-20200610045659-121dd752914d/go.mod h1:pqKB+ijp27cEcrHxhXVgUUMlSDRuGJJp1E+20Lj5H0E=
github.com/dsoprea/go-utility v0.0.0-20200711062821-fab8125e9bdf/go.mod h1:95+K3z2L0mqsVYd6yveIv1lmtT3tcQQ3dVakPySffW8=
github.com/dsoprea/go-utility/v2 v2.0.0-20200717064901-2fccff4aa15e h1:IxIbA7VbCNrwumIYjDoMOdf4KOSkMC6NJE4s8oRbE7E=
github.com/dsoprea/go-utility/v2 v2.0.0-20200717064901-2fccff4aa15e/go.mod h1:uAzdkPTub5Y9yQwXe8W4m2XuP0tK4a9Q/dantD0+uaU=
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
@ -197,11 +180,6 @@ github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE
github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI=
github.com/gin-gonic/gin v1.10.0 h1:nTuyha1TYqgedzytsKYqna+DfLos46nTv2ygFy86HFU=
github.com/gin-gonic/gin v1.10.0/go.mod h1:4PMNQiOhvDRa013RKVbsiNwoyezlm2rm0uX/T7kzp5Y=
github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q=
github.com/go-errors/errors v1.0.2/go.mod h1:psDX2osz5VnTOnFWbDeWwS7yejl+uV3FEWEp4lssFEs=
github.com/go-errors/errors v1.1.1/go.mod h1:psDX2osz5VnTOnFWbDeWwS7yejl+uV3FEWEp4lssFEs=
github.com/go-errors/errors v1.4.1 h1:IvVlgbzSsaUNudsw5dcXSzF3EWyXTi5XrAdngnuhRyg=
github.com/go-errors/errors v1.4.1/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og=
github.com/go-fed/httpsig v1.1.0 h1:9M+hb0jkEICD8/cAiNqEB66R87tTINszBRTjwjQzWcI=
github.com/go-fed/httpsig v1.1.0/go.mod h1:RCMrTZvN1bJYtofsG4rd5NaO5obxQ5xBkdiS7xsT7bM=
github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
@ -254,8 +232,6 @@ github.com/go-swagger/go-swagger v0.31.0 h1:H8eOYQnY2u7vNKWDNykv2xJP3pBhRG/R+SOC
github.com/go-swagger/go-swagger v0.31.0/go.mod h1:WSigRRWEig8zV6t6Sm8Y+EmUjlzA/HoaZJ5edupq7po=
github.com/go-test/deep v1.0.8 h1:TDsG77qcSprGbC6vTN8OuXp5g+J+b5Pcguhf7Zt61VM=
github.com/go-test/deep v1.0.8/go.mod h1:5C2ZWiW0ErCdrYzpqxLbTX7MG14M9iiw8DgHncVwcsE=
github.com/go-xmlfmt/xmlfmt v0.0.0-20211206191508-7fd73a941850 h1:PSPmmucxGiFBtbQcttHTUc4LQ3P09AW+ldO2qspyKdY=
github.com/go-xmlfmt/xmlfmt v0.0.0-20211206191508-7fd73a941850/go.mod h1:aUCEOzzezBEjDBbFBoSiya/gduyIiWYRP6CnSFIV8AM=
github.com/goccy/go-json v0.10.3 h1:KZ5WoDbxAIgm2HNbYckL0se1fHD6rz5j4ywS6ebzDqA=
github.com/goccy/go-json v0.10.3/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M=
github.com/godbus/dbus/v5 v5.0.4 h1:9349emZab16e7zQvpmsbtjc18ykshndd8y2PG3sgJbA=
@ -263,10 +239,6 @@ github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5x
github.com/golang-jwt/jwt v3.2.1+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I=
github.com/golang-jwt/jwt v3.2.2+incompatible h1:IfV12K8xAKAnZqdXVzCZ+TOjboZ2keLg81eXfW3O+oY=
github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I=
github.com/golang/geo v0.0.0-20190916061304-5b978397cfec/go.mod h1:QZ0nwyI2jOfgRAoBvP+ab5aRr7c9x7lhGEJrKvBwjWI=
github.com/golang/geo v0.0.0-20200319012246-673a6f80352d/go.mod h1:QZ0nwyI2jOfgRAoBvP+ab5aRr7c9x7lhGEJrKvBwjWI=
github.com/golang/geo v0.0.0-20210211234256-740aa86cb551 h1:gtexQ/VGyN+VVFRXSFiguSNcXmS6rkKT+X7FdIrTtfo=
github.com/golang/geo v0.0.0-20210211234256-740aa86cb551/go.mod h1:QZ0nwyI2jOfgRAoBvP+ab5aRr7c9x7lhGEJrKvBwjWI=
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
@ -324,7 +296,6 @@ github.com/google/pprof v0.0.0-20240409012703-83162a5b38cd h1:gbpYu9NMq8jhDVbvlG
github.com/google/pprof v0.0.0-20240409012703-83162a5b38cd/go.mod h1:kf6iHlnVGwgKolg33glAes7Yg/8iWP8ukqeldJSO7jw=
github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
@ -348,8 +319,6 @@ github.com/gorilla/websocket v1.5.2 h1:qoW6V1GT3aZxybsbC6oLnailWnB+qTMVwMreOso9X
github.com/gorilla/websocket v1.5.2/go.mod h1:0n9H61RBAcf5/38py2MCYbxzPIY9rOkpvvMT24Rqs30=
github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.1 h1:/c3QmbOGMGTOumP2iT/rCwB7b0QDGLKzqOmktBjT+Is=
github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.1/go.mod h1:5SN9VR2LTsRFsrEC6FHgRbTWrTHu6tqPeKxEQv15giM=
github.com/h2non/filetype v1.1.3 h1:FKkx9QbD7HR/zjK1Ia5XiBsq9zdLi5Kf3zGyFTAFkGg=
github.com/h2non/filetype v1.1.3/go.mod h1:319b3zT68BvV+WRj7cwy856M2ehB3HqNOt6sy1HndBY=
github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k=
@ -376,7 +345,6 @@ github.com/jackc/pgx/v5 v5.6.0 h1:SWJzexBzPL5jb0GEsrPMLIsi/3jOo7RHlzTjcAeDrPY=
github.com/jackc/pgx/v5 v5.6.0/go.mod h1:DNZ/vlrUnhWCoFGxHAG8U2ljioxukquj7utPDgtQdTw=
github.com/jackc/puddle/v2 v2.2.1 h1:RhxXJtFG022u4ibrCSMSiu5aOq1i77R3OHKNJj77OAk=
github.com/jackc/puddle/v2 v2.2.1/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4=
github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
github.com/jessevdk/go-flags v1.5.0 h1:1jKYvbxEjfUl0fmqTCOfonvskHHXMjBySTLW4y9LFvc=
github.com/jessevdk/go-flags v1.5.0/go.mod h1:Fw0T6WPc1dYxT4mKEZRfG5kJhaTDP9pj1c2EWnYs/m4=
github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E=
@ -404,7 +372,6 @@ github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORN
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
@ -459,8 +426,6 @@ github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7J
github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=
github.com/opencontainers/runtime-spec v1.0.2 h1:UfAcuLBJB9Coz72x1hgl8O5RVzTdNiaglX6v2DM6FI0=
github.com/opencontainers/runtime-spec v1.0.2/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0=
github.com/orcaman/writerseeker v0.0.0-20200621085525-1d3f536ff85e h1:s2RNOM/IGdY0Y6qfTeUKhDawdHDpK9RGBdx80qN4Ttw=
github.com/orcaman/writerseeker v0.0.0-20200621085525-1d3f536ff85e/go.mod h1:nBdnFKj15wFbf94Rwfq4m30eAcyY9V/IyKAGQFtqkW0=
github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 h1:onHthvaw9LFnH4t2DcNVpwGmV9E1BkGknEliJkfwQj0=
github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58/go.mod h1:DXv8WO4yhMYhSNPKjeNKa5WY9YCIEBRbNzFFPJbWO6Y=
github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM=
@ -540,13 +505,8 @@ github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsT
github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8=
github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU=
github.com/sunfish-shogi/bufseekio v0.0.0-20210207115823-a4185644b365/go.mod h1:dEzdXgvImkQ3WLI+0KQpmEx8T/C/ma9KeS3AfmU899I=
github.com/superseriousbusiness/activity v1.7.0-gts h1:DsCvzksTWptn7JUDTFIIiJ7xkh0A22VZs5KI3q67p+4=
github.com/superseriousbusiness/activity v1.7.0-gts/go.mod h1:AZw0Xb4Oju8rmaJCZ21gc5CPg47MmNgyac+Hx5jo8VM=
github.com/superseriousbusiness/go-jpeg-image-structure/v2 v2.0.0-20220321154430-d89a106fdabe h1:ksl2oCx/Qo8sNDc3Grb8WGKBM9nkvhCm25uvlT86azE=
github.com/superseriousbusiness/go-jpeg-image-structure/v2 v2.0.0-20220321154430-d89a106fdabe/go.mod h1:gH4P6gN1V+wmIw5o97KGaa1RgXB/tVpC2UNzijhg3E4=
github.com/superseriousbusiness/go-png-image-structure/v2 v2.0.1-SSB h1:8psprYSK1KdOSH7yQ4PbJq0YYaGQY+gzdW/B0ExDb/8=
github.com/superseriousbusiness/go-png-image-structure/v2 v2.0.1-SSB/go.mod h1:ymKGfy9kg4dIdraeZRAdobMS/flzLk3VcRPLpEWOAXg=
github.com/superseriousbusiness/httpsig v1.2.0-SSB h1:BinBGKbf2LSuVT5+MuH0XynHN9f0XVshx2CTDtkaWj0=
github.com/superseriousbusiness/httpsig v1.2.0-SSB/go.mod h1:+rxfATjFaDoDIVaJOTSP0gj6UrbicaYPEptvCLC9F28=
github.com/superseriousbusiness/oauth2/v4 v4.3.2-SSB.0.20230227143000-f4900831d6c8 h1:nTIhuP157oOFcscuoK1kCme1xTeGIzztSw70lX9NrDQ=
@ -739,7 +699,6 @@ golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLL
golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20200320220750-118fecf932d8/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
@ -972,13 +931,9 @@ gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA=
gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
gopkg.in/mcuadros/go-syslog.v2 v2.3.0 h1:kcsiS+WsTKyIEPABJBJtoG0KkOS6yzvJ+/eZlhD79kk=
gopkg.in/mcuadros/go-syslog.v2 v2.3.0/go.mod h1:l5LPIyOOyIdQquNg+oU6Z3524YwrcqEm0aKH+5zpt2U=
gopkg.in/src-d/go-billy.v4 v4.3.2 h1:0SQA1pRztfTFx2miS8sA97XvooFeNOmvUenF4o0EcVg=
gopkg.in/src-d/go-billy.v4 v4.3.2/go.mod h1:nDjArDMp+XMs1aFAESLRjfGSgfvoYN0hDfzEk0GjC98=
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.7/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=

12
internal/api/client/admin/emojicreate_test.go
View file

@ -90,10 +90,10 @@ func (suite *EmojiCreateTestSuite) TestEmojiCreateNewCategory() {
suite.Equal(apiEmoji.StaticURL, dbEmoji.ImageStaticURL)
suite.NotEmpty(dbEmoji.ImagePath)
suite.NotEmpty(dbEmoji.ImageStaticPath)
suite.Equal("image/png", dbEmoji.ImageContentType)
suite.Equal("image/apng", dbEmoji.ImageContentType)
suite.Equal("image/png", dbEmoji.ImageStaticContentType)
suite.Equal(36702, dbEmoji.ImageFileSize)
suite.Equal(10413, dbEmoji.ImageStaticFileSize)
suite.Equal(6092, dbEmoji.ImageStaticFileSize)
suite.False(*dbEmoji.Disabled)
suite.NotEmpty(dbEmoji.URI)
suite.True(*dbEmoji.VisibleInPicker)
@ -163,10 +163,10 @@ func (suite *EmojiCreateTestSuite) TestEmojiCreateExistingCategory() {
suite.Equal(apiEmoji.StaticURL, dbEmoji.ImageStaticURL)
suite.NotEmpty(dbEmoji.ImagePath)
suite.NotEmpty(dbEmoji.ImageStaticPath)
suite.Equal("image/png", dbEmoji.ImageContentType)
suite.Equal("image/apng", dbEmoji.ImageContentType)
suite.Equal("image/png", dbEmoji.ImageStaticContentType)
suite.Equal(36702, dbEmoji.ImageFileSize)
suite.Equal(10413, dbEmoji.ImageStaticFileSize)
suite.Equal(6092, dbEmoji.ImageStaticFileSize)
suite.False(*dbEmoji.Disabled)
suite.NotEmpty(dbEmoji.URI)
suite.True(*dbEmoji.VisibleInPicker)
@ -236,10 +236,10 @@ func (suite *EmojiCreateTestSuite) TestEmojiCreateNoCategory() {
suite.Equal(apiEmoji.StaticURL, dbEmoji.ImageStaticURL)
suite.NotEmpty(dbEmoji.ImagePath)
suite.NotEmpty(dbEmoji.ImageStaticPath)
suite.Equal("image/png", dbEmoji.ImageContentType)
suite.Equal("image/apng", dbEmoji.ImageContentType)
suite.Equal("image/png", dbEmoji.ImageStaticContentType)
suite.Equal(36702, dbEmoji.ImageFileSize)
suite.Equal(10413, dbEmoji.ImageStaticFileSize)
suite.Equal(6092, dbEmoji.ImageStaticFileSize)
suite.False(*dbEmoji.Disabled)
suite.NotEmpty(dbEmoji.URI)
suite.True(*dbEmoji.VisibleInPicker)

2
internal/api/client/admin/emojidelete_test.go
View file

@ -62,7 +62,7 @@ func (suite *EmojiDeleteTestSuite) TestEmojiDelete1() {
"id": "01F8MH9H8E4VG3KDYJR9EGPXCQ",
"disabled": false,
"updated_at": "2021-09-20T10:40:37.000Z",
"total_file_size": 47115,
"total_file_size": 42794,
"content_type": "image/png",
"uri": "http://localhost:8080/emoji/01F8MH9H8E4VG3KDYJR9EGPXCQ"
}`, dst.String())

4
internal/api/client/admin/emojiget_test.go
View file

@ -60,7 +60,7 @@ func (suite *EmojiGetTestSuite) TestEmojiGet1() {
"id": "01F8MH9H8E4VG3KDYJR9EGPXCQ",
"disabled": false,
"updated_at": "2021-09-20T10:40:37.000Z",
"total_file_size": 47115,
"total_file_size": 42794,
"content_type": "image/png",
"uri": "http://localhost:8080/emoji/01F8MH9H8E4VG3KDYJR9EGPXCQ"
}`, dst.String())
@ -92,7 +92,7 @@ func (suite *EmojiGetTestSuite) TestEmojiGet2() {
"disabled": false,
"domain": "fossbros-anonymous.io",
"updated_at": "2020-03-18T12:12:00.000Z",
"total_file_size": 21697,
"total_file_size": 19854,
"content_type": "image/png",
"uri": "http://fossbros-anonymous.io/emoji/01GD5KP5CQEE1R3X43Y1EHS2CW"
}`, dst.String())

30
internal/api/client/admin/emojiupdate_test.go
View file

@ -100,19 +100,19 @@ func (suite *EmojiUpdateTestSuite) TestEmojiUpdateNewCategory() {
suite.Equal("image/png", dbEmoji.ImageContentType)
suite.Equal("image/png", dbEmoji.ImageStaticContentType)
suite.Equal(36702, dbEmoji.ImageFileSize)
suite.Equal(10413, dbEmoji.ImageStaticFileSize)
suite.Equal(6092, dbEmoji.ImageStaticFileSize)
suite.False(*dbEmoji.Disabled)
suite.NotEmpty(dbEmoji.URI)
suite.True(*dbEmoji.VisibleInPicker)
suite.NotEmpty(dbEmoji.CategoryID)
// emoji should be in storage
emojiBytes, err := suite.storage.Get(ctx, dbEmoji.ImagePath)
entry, err := suite.storage.Storage.Stat(ctx, dbEmoji.ImagePath)
suite.NoError(err)
suite.Len(emojiBytes, dbEmoji.ImageFileSize)
emojiStaticBytes, err := suite.storage.Get(ctx, dbEmoji.ImageStaticPath)
suite.Equal(int64(dbEmoji.ImageFileSize), entry.Size)
entry, err = suite.storage.Storage.Stat(ctx, dbEmoji.ImageStaticPath)
suite.NoError(err)
suite.Len(emojiStaticBytes, dbEmoji.ImageStaticFileSize)
suite.Equal(int64(dbEmoji.ImageStaticFileSize), entry.Size)
}
func (suite *EmojiUpdateTestSuite) TestEmojiUpdateSwitchCategory() {
@ -177,19 +177,19 @@ func (suite *EmojiUpdateTestSuite) TestEmojiUpdateSwitchCategory() {
suite.Equal("image/png", dbEmoji.ImageContentType)
suite.Equal("image/png", dbEmoji.ImageStaticContentType)
suite.Equal(36702, dbEmoji.ImageFileSize)
suite.Equal(10413, dbEmoji.ImageStaticFileSize)
suite.Equal(6092, dbEmoji.ImageStaticFileSize)
suite.False(*dbEmoji.Disabled)
suite.NotEmpty(dbEmoji.URI)
suite.True(*dbEmoji.VisibleInPicker)
suite.NotEmpty(dbEmoji.CategoryID)
// emoji should be in storage
emojiBytes, err := suite.storage.Get(ctx, dbEmoji.ImagePath)
entry, err := suite.storage.Storage.Stat(ctx, dbEmoji.ImagePath)
suite.NoError(err)
suite.Len(emojiBytes, dbEmoji.ImageFileSize)
emojiStaticBytes, err := suite.storage.Get(ctx, dbEmoji.ImageStaticPath)
suite.Equal(int64(dbEmoji.ImageFileSize), entry.Size)
entry, err = suite.storage.Storage.Stat(ctx, dbEmoji.ImageStaticPath)
suite.NoError(err)
suite.Len(emojiStaticBytes, dbEmoji.ImageStaticFileSize)
suite.Equal(int64(dbEmoji.ImageStaticFileSize), entry.Size)
}
func (suite *EmojiUpdateTestSuite) TestEmojiUpdateCopyRemoteToLocal() {
@ -255,19 +255,19 @@ func (suite *EmojiUpdateTestSuite) TestEmojiUpdateCopyRemoteToLocal() {
suite.Equal("image/png", dbEmoji.ImageContentType)
suite.Equal("image/png", dbEmoji.ImageStaticContentType)
suite.Equal(10889, dbEmoji.ImageFileSize)
suite.Equal(10672, dbEmoji.ImageStaticFileSize)
suite.Equal(8965, dbEmoji.ImageStaticFileSize)
suite.False(*dbEmoji.Disabled)
suite.NotEmpty(dbEmoji.URI)
suite.True(*dbEmoji.VisibleInPicker)
suite.NotEmpty(dbEmoji.CategoryID)
// emoji should be in storage
emojiBytes, err := suite.storage.Get(ctx, dbEmoji.ImagePath)
entry, err := suite.storage.Storage.Stat(ctx, dbEmoji.ImagePath)
suite.NoError(err)
suite.Len(emojiBytes, dbEmoji.ImageFileSize)
emojiStaticBytes, err := suite.storage.Get(ctx, dbEmoji.ImageStaticPath)
suite.Equal(int64(dbEmoji.ImageFileSize), entry.Size)
entry, err = suite.storage.Storage.Stat(ctx, dbEmoji.ImageStaticPath)
suite.NoError(err)
suite.Len(emojiStaticBytes, dbEmoji.ImageStaticFileSize)
suite.Equal(int64(dbEmoji.ImageStaticFileSize), entry.Size)
}
func (suite *EmojiUpdateTestSuite) TestEmojiUpdateDisableEmoji() {

7
internal/api/client/instance/instancepatch.go
View file

@ -182,13 +182,6 @@ func validateInstanceUpdate(form *apimodel.InstanceSettingsUpdateRequest) error
return errors.New("empty form submitted")
}
if form.Avatar != nil {
maxImageSize := config.GetMediaImageMaxSize()
if size := form.Avatar.Size; size > int64(maxImageSize) {
return fmt.Errorf("file size limit exceeded: limit is %d bytes but desired instance avatar was %d bytes", maxImageSize, size)
}
}
if form.AvatarDescription != nil {
maxDescriptionChars := config.GetMediaDescriptionMaxChars()
if length := len([]rune(*form.AvatarDescription)); length > maxDescriptionChars {

14
internal/api/client/instance/instancepatch_test.go
View file

@ -109,7 +109,7 @@ func (suite *InstancePatchTestSuite) TestInstancePatch1() {
"image/webp",
"video/mp4"
],
"image_size_limit": 10485760,
"image_size_limit": 41943040,
"image_matrix_limit": 16777216,
"video_size_limit": 41943040,
"video_frame_rate_limit": 60,
@ -230,7 +230,7 @@ func (suite *InstancePatchTestSuite) TestInstancePatch2() {
"image/webp",
"video/mp4"
],
"image_size_limit": 10485760,
"image_size_limit": 41943040,
"image_matrix_limit": 16777216,
"video_size_limit": 41943040,
"video_frame_rate_limit": 60,
@ -351,7 +351,7 @@ func (suite *InstancePatchTestSuite) TestInstancePatch3() {
"image/webp",
"video/mp4"
],
"image_size_limit": 10485760,
"image_size_limit": 41943040,
"image_matrix_limit": 16777216,
"video_size_limit": 41943040,
"video_frame_rate_limit": 60,
@ -523,7 +523,7 @@ func (suite *InstancePatchTestSuite) TestInstancePatch6() {
"image/webp",
"video/mp4"
],
"image_size_limit": 10485760,
"image_size_limit": 41943040,
"image_matrix_limit": 16777216,
"video_size_limit": 41943040,
"video_frame_rate_limit": 60,
@ -666,7 +666,7 @@ func (suite *InstancePatchTestSuite) TestInstancePatch8() {
"image/webp",
"video/mp4"
],
"image_size_limit": 10485760,
"image_size_limit": 41943040,
"image_matrix_limit": 16777216,
"video_size_limit": 41943040,
"video_frame_rate_limit": 60,
@ -754,7 +754,7 @@ func (suite *InstancePatchTestSuite) TestInstancePatch8() {
"url": "http://localhost:8080/fileserver/01AY6P665V14JJR0AFVRT7311Y/attachment/original/`+instanceAccount.AvatarMediaAttachment.ID+`.gif",`+`
"thumbnail_type": "image/gif",
"thumbnail_description": "A bouncing little green peglin.",
"blurhash": "LG9t;qRS4YtO.4WDRlt5IXoxtPj["
"blurhash": "LtJ[eKxu_4xt9Yj]M{WBt8WBM{WB"
}`, string(instanceV2ThumbnailJson))
// double extra special bonus: now update the image description without changing the image
@ -824,7 +824,7 @@ func (suite *InstancePatchTestSuite) TestInstancePatch9() {
"image/webp",
"video/mp4"
],
"image_size_limit": 10485760,
"image_size_limit": 41943040,
"image_matrix_limit": 16777216,
"video_size_limit": 41943040,
"video_frame_rate_limit": 60,

13
internal/api/client/media/mediacreate.go
View file

@ -153,22 +153,9 @@ func validateCreateMedia(form *apimodel.AttachmentRequest) error {
return errors.New("no attachment given")
}
maxVideoSize := config.GetMediaVideoMaxSize()
maxImageSize := config.GetMediaImageMaxSize()
minDescriptionChars := config.GetMediaDescriptionMinChars()
maxDescriptionChars := config.GetMediaDescriptionMaxChars()
// a very superficial check to see if no size limits are exceeded
// we still don't actually know which media types we're dealing with but the other handlers will go into more detail there
maxSize := maxVideoSize
if maxImageSize > maxSize {
maxSize = maxImageSize
}
if form.File.Size > int64(maxSize) {
return fmt.Errorf("file size limit exceeded: limit is %d bytes but attachment was %d bytes", maxSize, form.File.Size)
}
if length := len([]rune(form.Description)); length > maxDescriptionChars {
return fmt.Errorf("image description length must be between %d and %d characters (inclusive), but provided image description was %d chars", minDescriptionChars, maxDescriptionChars, length)
}

4
internal/api/client/media/mediacreate_test.go
View file

@ -206,7 +206,7 @@ func (suite *MediaCreateTestSuite) TestMediaCreateSuccessful() {
Y: 0.5,
},
}, *attachmentReply.Meta)
suite.Equal("LiBzRk#6V[WF_NvzV@WY_3rqV@a$", *attachmentReply.Blurhash)
suite.Equal("LjCGfG#6RkRn_NvzRjWF?urqV@a$", *attachmentReply.Blurhash)
suite.NotEmpty(attachmentReply.ID)
suite.NotEmpty(attachmentReply.URL)
suite.NotEmpty(attachmentReply.PreviewURL)
@ -291,7 +291,7 @@ func (suite *MediaCreateTestSuite) TestMediaCreateSuccessfulV2() {
Y: 0.5,
},
}, *attachmentReply.Meta)
suite.Equal("LiBzRk#6V[WF_NvzV@WY_3rqV@a$", *attachmentReply.Blurhash)
suite.Equal("LjCGfG#6RkRn_NvzRjWF?urqV@a$", *attachmentReply.Blurhash)
suite.NotEmpty(attachmentReply.ID)
suite.Nil(attachmentReply.URL)
suite.NotEmpty(attachmentReply.PreviewURL)

4
internal/cleaner/media_test.go
View file

@ -373,13 +373,13 @@ func (suite *MediaTestSuite) TestUncacheAndRecache() {
suite.True(storage.IsNotFound(err))
// now recache the image....
data := func(_ context.Context) (io.ReadCloser, int64, error) {
data := func(_ context.Context) (io.ReadCloser, error) {
// load bytes from a test image
b, err := os.ReadFile("../../testrig/media/thoughtsofdog-original.jpg")
if err != nil {
panic(err)
}
return io.NopCloser(bytes.NewBuffer(b)), int64(len(b)), nil
return io.NopCloser(bytes.NewBuffer(b)), nil
}
for _, original := range []*gtsmodel.MediaAttachment{

4
internal/config/config.go
View file

@ -92,13 +92,13 @@ type Configuration struct {
AccountsAllowCustomCSS bool `name:"accounts-allow-custom-css" usage:"Allow accounts to enable custom CSS for their profile pages and statuses."`
AccountsCustomCSSLength int `name:"accounts-custom-css-length" usage:"Maximum permitted length (characters) of custom CSS for accounts."`
MediaImageMaxSize bytesize.Size `name:"media-image-max-size" usage:"Max size of accepted images in bytes"`
MediaVideoMaxSize bytesize.Size `name:"media-video-max-size" usage:"Max size of accepted videos in bytes"`
MediaDescriptionMinChars int `name:"media-description-min-chars" usage:"Min required chars for an image description"`
MediaDescriptionMaxChars int `name:"media-description-max-chars" usage:"Max permitted chars for an image description"`
MediaRemoteCacheDays int `name:"media-remote-cache-days" usage:"Number of days to locally cache media from remote instances. If set to 0, remote media will be kept indefinitely."`
MediaEmojiLocalMaxSize bytesize.Size `name:"media-emoji-local-max-size" usage:"Max size in bytes of emojis uploaded to this instance via the admin API."`
MediaEmojiRemoteMaxSize bytesize.Size `name:"media-emoji-remote-max-size" usage:"Max size in bytes of emojis to download from other instances."`
MediaLocalMaxSize bytesize.Size `name:"media-local-max-size" usage:"Max size in bytes of media uploaded to this instance via API"`
MediaRemoteMaxSize bytesize.Size `name:"media-remote-max-size" usage:"Max size in bytes of media to download from other instances"`
MediaCleanupFrom string `name:"media-cleanup-from" usage:"Time of day from which to start running media cleanup/prune jobs. Should be in the format 'hh:mm:ss', eg., '15:04:05'."`
MediaCleanupEvery time.Duration `name:"media-cleanup-every" usage:"Period to elapse between cleanups, starting from media-cleanup-at."`

4
internal/config/defaults.go
View file

@ -71,11 +71,11 @@ var Defaults = Configuration{
AccountsAllowCustomCSS: false,
AccountsCustomCSSLength: 10000,
MediaImageMaxSize: 10 * bytesize.MiB,
MediaVideoMaxSize: 40 * bytesize.MiB,
MediaDescriptionMinChars: 0,
MediaDescriptionMaxChars: 1500,
MediaRemoteCacheDays: 7,
MediaLocalMaxSize: 40 * bytesize.MiB,
MediaRemoteMaxSize: 40 * bytesize.MiB,
MediaEmojiLocalMaxSize: 50 * bytesize.KiB,
MediaEmojiRemoteMaxSize: 100 * bytesize.KiB,
MediaCleanupFrom: "00:00", // Midnight.

4
internal/config/flags.go
View file

@ -97,11 +97,11 @@ func (s *ConfigState) AddServerFlags(cmd *cobra.Command) {
cmd.Flags().Bool(AccountsAllowCustomCSSFlag(), cfg.AccountsAllowCustomCSS, fieldtag("AccountsAllowCustomCSS", "usage"))
// Media
cmd.Flags().Uint64(MediaImageMaxSizeFlag(), uint64(cfg.MediaImageMaxSize), fieldtag("MediaImageMaxSize", "usage"))
cmd.Flags().Uint64(MediaVideoMaxSizeFlag(), uint64(cfg.MediaVideoMaxSize), fieldtag("MediaVideoMaxSize", "usage"))
cmd.Flags().Int(MediaDescriptionMinCharsFlag(), cfg.MediaDescriptionMinChars, fieldtag("MediaDescriptionMinChars", "usage"))
cmd.Flags().Int(MediaDescriptionMaxCharsFlag(), cfg.MediaDescriptionMaxChars, fieldtag("MediaDescriptionMaxChars", "usage"))
cmd.Flags().Int(MediaRemoteCacheDaysFlag(), cfg.MediaRemoteCacheDays, fieldtag("MediaRemoteCacheDays", "usage"))
cmd.Flags().Uint64(MediaLocalMaxSizeFlag(), uint64(cfg.MediaLocalMaxSize), fieldtag("MediaLocalMaxSize", "usage"))
cmd.Flags().Uint64(MediaRemoteMaxSizeFlag(), uint64(cfg.MediaRemoteMaxSize), fieldtag("MediaRemoteMaxSize", "usage"))
cmd.Flags().Uint64(MediaEmojiLocalMaxSizeFlag(), uint64(cfg.MediaEmojiLocalMaxSize), fieldtag("MediaEmojiLocalMaxSize", "usage"))
cmd.Flags().Uint64(MediaEmojiRemoteMaxSizeFlag(), uint64(cfg.MediaEmojiRemoteMaxSize), fieldtag("MediaEmojiRemoteMaxSize", "usage"))
cmd.Flags().String(MediaCleanupFromFlag(), cfg.MediaCleanupFrom, fieldtag("MediaCleanupFrom", "usage"))

100
internal/config/helpers.gen.go
View file

@ -1075,56 +1075,6 @@ func GetAccountsCustomCSSLength() int { return global.GetAccountsCustomCSSLength
// SetAccountsCustomCSSLength safely sets the value for global configuration 'AccountsCustomCSSLength' field
func SetAccountsCustomCSSLength(v int) { global.SetAccountsCustomCSSLength(v) }
// GetMediaImageMaxSize safely fetches the Configuration value for state's 'MediaImageMaxSize' field
func (st *ConfigState) GetMediaImageMaxSize() (v bytesize.Size) {
st.mutex.RLock()
v = st.config.MediaImageMaxSize
st.mutex.RUnlock()
return
}
// SetMediaImageMaxSize safely sets the Configuration value for state's 'MediaImageMaxSize' field
func (st *ConfigState) SetMediaImageMaxSize(v bytesize.Size) {
st.mutex.Lock()
defer st.mutex.Unlock()
st.config.MediaImageMaxSize = v
st.reloadToViper()
}
// MediaImageMaxSizeFlag returns the flag name for the 'MediaImageMaxSize' field
func MediaImageMaxSizeFlag() string { return "media-image-max-size" }
// GetMediaImageMaxSize safely fetches the value for global configuration 'MediaImageMaxSize' field
func GetMediaImageMaxSize() bytesize.Size { return global.GetMediaImageMaxSize() }
// SetMediaImageMaxSize safely sets the value for global configuration 'MediaImageMaxSize' field
func SetMediaImageMaxSize(v bytesize.Size) { global.SetMediaImageMaxSize(v) }
// GetMediaVideoMaxSize safely fetches the Configuration value for state's 'MediaVideoMaxSize' field
func (st *ConfigState) GetMediaVideoMaxSize() (v bytesize.Size) {
st.mutex.RLock()
v = st.config.MediaVideoMaxSize
st.mutex.RUnlock()
return
}
// SetMediaVideoMaxSize safely sets the Configuration value for state's 'MediaVideoMaxSize' field
func (st *ConfigState) SetMediaVideoMaxSize(v bytesize.Size) {
st.mutex.Lock()
defer st.mutex.Unlock()
st.config.MediaVideoMaxSize = v
st.reloadToViper()
}
// MediaVideoMaxSizeFlag returns the flag name for the 'MediaVideoMaxSize' field
func MediaVideoMaxSizeFlag() string { return "media-video-max-size" }
// GetMediaVideoMaxSize safely fetches the value for global configuration 'MediaVideoMaxSize' field
func GetMediaVideoMaxSize() bytesize.Size { return global.GetMediaVideoMaxSize() }
// SetMediaVideoMaxSize safely sets the value for global configuration 'MediaVideoMaxSize' field
func SetMediaVideoMaxSize(v bytesize.Size) { global.SetMediaVideoMaxSize(v) }
// GetMediaDescriptionMinChars safely fetches the Configuration value for state's 'MediaDescriptionMinChars' field
func (st *ConfigState) GetMediaDescriptionMinChars() (v int) {
st.mutex.RLock()
@ -1250,6 +1200,56 @@ func GetMediaEmojiRemoteMaxSize() bytesize.Size { return global.GetMediaEmojiRem
// SetMediaEmojiRemoteMaxSize safely sets the value for global configuration 'MediaEmojiRemoteMaxSize' field
func SetMediaEmojiRemoteMaxSize(v bytesize.Size) { global.SetMediaEmojiRemoteMaxSize(v) }
// GetMediaLocalMaxSize safely fetches the Configuration value for state's 'MediaLocalMaxSize' field
func (st *ConfigState) GetMediaLocalMaxSize() (v bytesize.Size) {
st.mutex.RLock()
v = st.config.MediaLocalMaxSize
st.mutex.RUnlock()
return
}
// SetMediaLocalMaxSize safely sets the Configuration value for state's 'MediaLocalMaxSize' field
func (st *ConfigState) SetMediaLocalMaxSize(v bytesize.Size) {
st.mutex.Lock()
defer st.mutex.Unlock()
st.config.MediaLocalMaxSize = v
st.reloadToViper()
}
// MediaLocalMaxSizeFlag returns the flag name for the 'MediaLocalMaxSize' field
func MediaLocalMaxSizeFlag() string { return "media-local-max-size" }
// GetMediaLocalMaxSize safely fetches the value for global configuration 'MediaLocalMaxSize' field
func GetMediaLocalMaxSize() bytesize.Size { return global.GetMediaLocalMaxSize() }
// SetMediaLocalMaxSize safely sets the value for global configuration 'MediaLocalMaxSize' field
func SetMediaLocalMaxSize(v bytesize.Size) { global.SetMediaLocalMaxSize(v) }
// GetMediaRemoteMaxSize safely fetches the Configuration value for state's 'MediaRemoteMaxSize' field
func (st *ConfigState) GetMediaRemoteMaxSize() (v bytesize.Size) {
st.mutex.RLock()
v = st.config.MediaRemoteMaxSize
st.mutex.RUnlock()
return
}
// SetMediaRemoteMaxSize safely sets the Configuration value for state's 'MediaRemoteMaxSize' field
func (st *ConfigState) SetMediaRemoteMaxSize(v bytesize.Size) {
st.mutex.Lock()
defer st.mutex.Unlock()
st.config.MediaRemoteMaxSize = v
st.reloadToViper()
}
// MediaRemoteMaxSizeFlag returns the flag name for the 'MediaRemoteMaxSize' field
func MediaRemoteMaxSizeFlag() string { return "media-remote-max-size" }
// GetMediaRemoteMaxSize safely fetches the value for global configuration 'MediaRemoteMaxSize' field
func GetMediaRemoteMaxSize() bytesize.Size { return global.GetMediaRemoteMaxSize() }
// SetMediaRemoteMaxSize safely sets the value for global configuration 'MediaRemoteMaxSize' field
func SetMediaRemoteMaxSize(v bytesize.Size) { global.SetMediaRemoteMaxSize(v) }
// GetMediaCleanupFrom safely fetches the Configuration value for state's 'MediaCleanupFrom' field
func (st *ConfigState) GetMediaCleanupFrom() (v string) {
st.mutex.RLock()

15
internal/federation/dereferencing/emoji.go
View file

@ -23,6 +23,7 @@ import (
"io"
"net/url"
"github.com/superseriousbusiness/gotosocial/internal/config"
"github.com/superseriousbusiness/gotosocial/internal/db"
"github.com/superseriousbusiness/gotosocial/internal/gtserror"
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
@ -90,9 +91,12 @@ func (d *Dereferencer) GetEmoji(
return nil, err
}
// Get maximum supported remote emoji size.
maxsz := config.GetMediaEmojiRemoteMaxSize()
// Prepare data function to dereference remote emoji media.
data := func(context.Context) (io.ReadCloser, int64, error) {
return tsport.DereferenceMedia(ctx, url)
data := func(context.Context) (io.ReadCloser, error) {
return tsport.DereferenceMedia(ctx, url, int64(maxsz))
}
// Pass along for safe processing.
@ -171,9 +175,12 @@ func (d *Dereferencer) RefreshEmoji(
return nil, err
}
// Get maximum supported remote emoji size.
maxsz := config.GetMediaEmojiRemoteMaxSize()
// Prepare data function to dereference remote emoji media.
data := func(context.Context) (io.ReadCloser, int64, error) {
return tsport.DereferenceMedia(ctx, url)
data := func(context.Context) (io.ReadCloser, error) {
return tsport.DereferenceMedia(ctx, url, int64(maxsz))
}
// Pass along for safe processing.

2
internal/federation/dereferencing/emoji_test.go
View file

@ -75,7 +75,7 @@ func (suite *EmojiTestSuite) TestDereferenceEmojiBlocking() {
suite.Equal("image/gif", emoji.ImageContentType)
suite.Equal("image/png", emoji.ImageStaticContentType)
suite.Equal(37796, emoji.ImageFileSize)
suite.Equal(7951, emoji.ImageStaticFileSize)
suite.Equal(9824, emoji.ImageStaticFileSize)
suite.WithinDuration(time.Now(), emoji.UpdatedAt, 10*time.Second)
suite.False(*emoji.Disabled)
suite.Equal(emojiURI, emoji.URI)

15
internal/federation/dereferencing/media.go
View file

@ -22,6 +22,7 @@ import (
"io"
"net/url"
"github.com/superseriousbusiness/gotosocial/internal/config"
"github.com/superseriousbusiness/gotosocial/internal/gtserror"
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
"github.com/superseriousbusiness/gotosocial/internal/media"
@ -69,12 +70,15 @@ func (d *Dereferencer) GetMedia(
return nil, gtserror.Newf("failed getting transport for %s: %w", requestUser, err)
}
// Get maximum supported remote media size.
maxsz := config.GetMediaRemoteMaxSize()
// Start processing remote attachment at URL.
processing, err := d.mediaManager.CreateMedia(
ctx,
accountID,
func(ctx context.Context) (io.ReadCloser, int64, error) {
return tsport.DereferenceMedia(ctx, url)
func(ctx context.Context) (io.ReadCloser, error) {
return tsport.DereferenceMedia(ctx, url, int64(maxsz))
},
info,
)
@ -163,11 +167,14 @@ func (d *Dereferencer) RefreshMedia(
return nil, gtserror.Newf("failed getting transport for %s: %w", requestUser, err)
}
// Get maximum supported remote media size.
maxsz := config.GetMediaRemoteMaxSize()
// Start processing remote attachment recache.
processing := d.mediaManager.RecacheMedia(
media,
func(ctx context.Context) (io.ReadCloser, int64, error) {
return tsport.DereferenceMedia(ctx, url)
func(ctx context.Context) (io.ReadCloser, error) {
return tsport.DereferenceMedia(ctx, url, int64(maxsz))
},
)

37
internal/httpclient/client.go
View file

@ -31,7 +31,6 @@ import (
"strings"
"time"
"codeberg.org/gruf/go-bytesize"
"codeberg.org/gruf/go-cache/v3"
errorsv2 "codeberg.org/gruf/go-errors/v2"
"codeberg.org/gruf/go-iotools"
@ -89,9 +88,6 @@ type Config struct {
// WriteBufferSize: see http.Transport{}.WriteBufferSize.
WriteBufferSize int
// MaxBodySize determines the maximum fetchable body size.
MaxBodySize int64
// Timeout: see http.Client{}.Timeout.
Timeout time.Duration
@ -111,7 +107,6 @@ type Config struct {
type Client struct {
client http.Client
badHosts cache.TTLCache[string, struct{}]
bodyMax int64
retries uint
}
@ -137,11 +132,6 @@ func New(cfg Config) *Client {
cfg.MaxIdleConns = cfg.MaxOpenConnsPerHost * 10
}
if cfg.MaxBodySize <= 0 {
// By default set this to a reasonable 40MB.
cfg.MaxBodySize = int64(40 * bytesize.MiB)
}
// Protect the dialer
// with IP range sanitizer.
d.Control = (&Sanitizer{
@ -151,7 +141,6 @@ func New(cfg Config) *Client {
// Prepare client fields.
c.client.Timeout = cfg.Timeout
c.bodyMax = cfg.MaxBodySize
// Prepare transport TLS config.
tlsClientConfig := &tls.Config{
@ -377,31 +366,15 @@ func (c *Client) do(r *Request) (rsp *http.Response, retry bool, err error) {
rbody := (io.Reader)(rsp.Body)
cbody := (io.Closer)(rsp.Body)
var limit int64
if limit = rsp.ContentLength; limit < 0 {
// If unknown, use max as reader limit.
limit = c.bodyMax
}
// Don't trust them, limit body reads.
rbody = io.LimitReader(rbody, limit)
// Wrap closer to ensure entire body drained BEFORE close.
// Wrap closer to ensure body drained BEFORE close.
cbody = iotools.CloserAfterCallback(cbody, func() {
_, _ = discard.ReadFrom(rbody)
})
// Wrap body with limit.
rsp.Body = &struct {
io.Reader
io.Closer
}{rbody, cbody}
// Check response body not too large.
if rsp.ContentLength > c.bodyMax {
_ = rsp.Body.Close()
return nil, false, ErrBodyTooLarge
// Set the wrapped response body.
rsp.Body = &iotools.ReadCloserType{
Reader: rbody,
Closer: cbody,
}
return rsp, true, nil

39
internal/httpclient/client_test.go
View file

@ -48,44 +48,19 @@ var bodies = []string{
"body with\r\nnewlines",
}
func TestHTTPClientSmallBody(t *testing.T) {
func TestHTTPClientBody(t *testing.T) {
for _, body := range bodies {
_TestHTTPClientWithBody(t, []byte(body), int(^uint16(0)))
testHTTPClientWithBody(t, []byte(body))
}
}
func TestHTTPClientExactBody(t *testing.T) {
for _, body := range bodies {
_TestHTTPClientWithBody(t, []byte(body), len(body))
}
}
func TestHTTPClientLargeBody(t *testing.T) {
for _, body := range bodies {
_TestHTTPClientWithBody(t, []byte(body), len(body)-1)
}
}
func _TestHTTPClientWithBody(t *testing.T, body []byte, max int) {
func testHTTPClientWithBody(t *testing.T, body []byte) {
var (
handler http.HandlerFunc
expect []byte
expectErr error
)
// If this is a larger body, reslice and
// set error so we know what to expect
expect = body
if max < len(body) {
expect = expect[:max]
expectErr = httpclient.ErrBodyTooLarge
}
// Create new HTTP client with maximum body size
client := httpclient.New(httpclient.Config{
MaxBodySize: int64(max),
DisableCompression: true,
AllowRanges: []netip.Prefix{
// Loopback (used by server)
@ -110,10 +85,8 @@ func _TestHTTPClientWithBody(t *testing.T, body []byte, max int) {
// Perform the test request
rsp, err := client.Do(req)
if !errors.Is(err, expectErr) {
if err != nil {
t.Fatalf("error performing client request: %v", err)
} else if err != nil {
return // expected error
}
defer rsp.Body.Close()
@ -124,8 +97,8 @@ func _TestHTTPClientWithBody(t *testing.T, body []byte, max int) {
}
// Check actual response body matches expected
if !bytes.Equal(expect, check) {
t.Errorf("response body did not match expected: expect=%q actual=%q", string(expect), string(check))
if !bytes.Equal(body, check) {
t.Errorf("response body did not match expected: expect=%q actual=%q", string(body), string(check))
}
}

313
internal/media/ffmpeg.go Normal file
View file

@ -0,0 +1,313 @@
// GoToSocial
// Copyright (C) GoToSocial Authors admin@gotosocial.org
// SPDX-License-Identifier: AGPL-3.0-or-later
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package media
import (
"context"
"encoding/json"
"errors"
"os"
"path"
"strconv"
"strings"
"codeberg.org/gruf/go-byteutil"
"codeberg.org/gruf/go-ffmpreg/wasm"
_ffmpeg "github.com/superseriousbusiness/gotosocial/internal/media/ffmpeg"
"github.com/superseriousbusiness/gotosocial/internal/gtserror"
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
"github.com/tetratelabs/wazero"
)
// ffmpegClearMetadata generates a copy (in-place) of input media with all metadata cleared.
func ffmpegClearMetadata(ctx context.Context, filepath string, ext string) error {
// Get directory from filepath.
dirpath := path.Dir(filepath)
// Generate output file path with ext.
outpath := filepath + "_cleaned." + ext
// Clear metadata with ffmpeg.
if err := ffmpeg(ctx, dirpath,
"-loglevel", "error",
"-i", filepath,
"-map_metadata", "-1",
"-codec", "copy",
"-y",
outpath,
); err != nil {
return err
}
// Move the new output file path to original location.
if err := os.Rename(outpath, filepath); err != nil {
return gtserror.Newf("error renaming %s: %w", outpath, err)
}
return nil
}
// ffmpegGenerateThumb generates a thumbnail jpeg from input media of any type, useful for any media.
func ffmpegGenerateThumb(ctx context.Context, filepath string, width, height int) (string, error) {
// Get directory from filepath.
dirpath := path.Dir(filepath)
// Generate output frame file path.
outpath := filepath + "_thumb.jpg"
// Generate thumb with ffmpeg.
if err := ffmpeg(ctx, dirpath,
"-loglevel", "error",
"-i", filepath,
"-filter:v", "thumbnail=n=10",
"-filter:v", "scale="+strconv.Itoa(width)+":"+strconv.Itoa(height),
"-qscale:v", "12", // ~ 70% quality
"-frames:v", "1",
"-y",
outpath,
); err != nil {
return "", err
}
return outpath, nil
}
// ffmpegGenerateStatic generates a static png from input image of any type, useful for emoji.
func ffmpegGenerateStatic(ctx context.Context, filepath string) (string, error) {
// Get directory from filepath.
dirpath := path.Dir(filepath)
// Generate output static file path.
outpath := filepath + "_static.png"
// Generate static with ffmpeg.
if err := ffmpeg(ctx, dirpath,
"-loglevel", "error",
"-i", filepath,
"-codec:v", "png", // specifically NOT 'apng'
"-frames:v", "1", // in case animated, only take 1 frame
"-y",
outpath,
); err != nil {
return "", err
}
return outpath, nil
}
// ffmpeg calls `ffmpeg [args...]` (WASM) with directory path mounted in runtime.
func ffmpeg(ctx context.Context, dirpath string, args ...string) error {
var stderr byteutil.Buffer
rc, err := _ffmpeg.Ffmpeg(ctx, wasm.Args{
Stderr: &stderr,
Args: args,
Config: func(modcfg wazero.ModuleConfig) wazero.ModuleConfig {
fscfg := wazero.NewFSConfig()
fscfg = fscfg.WithDirMount(dirpath, dirpath)
modcfg = modcfg.WithFSConfig(fscfg)
return modcfg
},
})
if err != nil {
return gtserror.Newf("error running: %w", err)
} else if rc != 0 {
return gtserror.Newf("non-zero return code %d (%s)", rc, stderr.B)
}
return nil
}
// ffprobe calls `ffprobe` (WASM) on filepath, returning parsed JSON output.
func ffprobe(ctx context.Context, filepath string) (*ffprobeResult, error) {
var stdout byteutil.Buffer
// Get directory from filepath.
dirpath := path.Dir(filepath)
// Run ffprobe on our given file at path.
_, err := _ffmpeg.Ffprobe(ctx, wasm.Args{
Stdout: &stdout,
Args: []string{
"-i", filepath,
"-loglevel", "quiet",
"-print_format", "json",
"-show_streams",
"-show_format",
"-show_error",
},
Config: func(modcfg wazero.ModuleConfig) wazero.ModuleConfig {
fscfg := wazero.NewFSConfig()
fscfg = fscfg.WithReadOnlyDirMount(dirpath, dirpath)
modcfg = modcfg.WithFSConfig(fscfg)
return modcfg
},
})
if err != nil {
return nil, gtserror.Newf("error running: %w", err)
}
var result ffprobeResult
// Unmarshal the ffprobe output as our result type.
if err := json.Unmarshal(stdout.B, &result); err != nil {
return nil, gtserror.Newf("error unmarshaling json: %w", err)
}
return &result, nil
}
// ffprobeResult contains parsed JSON data from
// result of calling `ffprobe` on a media file.
type ffprobeResult struct {
Streams []ffprobeStream `json:"streams"`
Format *ffprobeFormat `json:"format"`
Error *ffprobeError `json:"error"`
}
// ImageMeta extracts image metadata contained within ffprobe'd media result streams.
func (res *ffprobeResult) ImageMeta() (width int, height int, err error) {
for _, stream := range res.Streams {
if stream.Width > width {
width = stream.Width
}
if stream.Height > height {
height = stream.Height
}
}
if width == 0 || height == 0 {
err = errors.New("invalid image stream(s)")
}
return
}
// VideoMeta extracts video metadata contained within ffprobe'd media result streams.
func (res *ffprobeResult) VideoMeta() (width, height int, framerate float32, err error) {
for _, stream := range res.Streams {
if stream.Width > width {
width = stream.Width
}
if stream.Height > height {
height = stream.Height
}
if fr := stream.GetFrameRate(); fr > 0 {
if framerate == 0 || fr < framerate {
framerate = fr
}
}
}
if width == 0 || height == 0 || framerate == 0 {
err = errors.New("invalid video stream(s)")
}
return
}
type ffprobeStream struct {
CodecName string `json:"codec_name"`
AvgFrameRate string `json:"avg_frame_rate"`
Width int `json:"width"`
Height int `json:"height"`
// + unused fields.
}
// GetFrameRate calculates float32 framerate value from stream json string.
func (str *ffprobeStream) GetFrameRate() float32 {
if str.AvgFrameRate != "" {
var (
// numerator
num float32
// denominator
den float32
)
// Check for a provided inequality, i.e. numerator / denominator.
if p := strings.SplitN(str.AvgFrameRate, "/", 2); len(p) == 2 {
n, _ := strconv.ParseFloat(p[0], 32)
d, _ := strconv.ParseFloat(p[1], 32)
num, den = float32(n), float32(d)
} else {
n, _ := strconv.ParseFloat(p[0], 32)
num = float32(n)
}
return num / den
}
return 0
}
type ffprobeFormat struct {
Filename string `json:"filename"`
FormatName string `json:"format_name"`
Duration string `json:"duration"`
BitRate string `json:"bit_rate"`
// + unused fields
}
// GetFileType determines file type and extension to use for media data.
func (fmt *ffprobeFormat) GetFileType() (gtsmodel.FileType, string) {
switch fmt.FormatName {
case "mov,mp4,m4a,3gp,3g2,mj2":
return gtsmodel.FileTypeVideo, "mp4"
case "apng":
return gtsmodel.FileTypeImage, "apng"
case "png_pipe":
return gtsmodel.FileTypeImage, "png"
case "image2", "jpeg_pipe":
return gtsmodel.FileTypeImage, "jpeg"
case "webp_pipe":
return gtsmodel.FileTypeImage, "webp"
case "gif":
return gtsmodel.FileTypeImage, "gif"
case "mp3":
return gtsmodel.FileTypeAudio, "mp3"
case "ogg":
return gtsmodel.FileTypeAudio, "ogg"
default:
return gtsmodel.FileTypeUnknown, fmt.FormatName
}
}
// GetDuration calculates float32 framerate value from format json string.
func (fmt *ffprobeFormat) GetDuration() float32 {
if fmt.Duration != "" {
dur, _ := strconv.ParseFloat(fmt.Duration, 32)
return float32(dur)
}
return 0
}
// GetBitRate calculates uint64 bitrate value from format json string.
func (fmt *ffprobeFormat) GetBitRate() uint64 {
if fmt.BitRate != "" {
r, _ := strconv.ParseUint(fmt.BitRate, 10, 64)
return r
}
return 0
}
type ffprobeError struct {
Code int `json:"code"`
String string `json:"string"`
}
func (err *ffprobeError) Error() string {
return err.String + " (" + strconv.Itoa(err.Code) + ")"
}

46
internal/media/ffmpeg/cache.go Normal file
View file

@ -0,0 +1,46 @@
// GoToSocial
// Copyright (C) GoToSocial Authors admin@gotosocial.org
// SPDX-License-Identifier: AGPL-3.0-or-later
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package ffmpeg
import (
"os"
"github.com/tetratelabs/wazero"
)
// shared WASM compilation cache.
var cache wazero.CompilationCache
func initCache() {
if cache != nil {
return
}
if dir := os.Getenv("WAZERO_COMPILATION_CACHE"); dir != "" {
var err error
// Use on-filesystem compilation cache given by env.
cache, err = wazero.NewCompilationCacheWithDir(dir)
if err != nil {
panic(err)
}
} else {
// Use in-memory compilation cache.
cache = wazero.NewCompilationCache()
}
}

92
internal/media/ffmpeg/ffmpeg.go Normal file
View file

@ -0,0 +1,92 @@
// GoToSocial
// Copyright (C) GoToSocial Authors admin@gotosocial.org
// SPDX-License-Identifier: AGPL-3.0-or-later
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package ffmpeg
import (
"context"
ffmpeglib "codeberg.org/gruf/go-ffmpreg/embed/ffmpeg"
"codeberg.org/gruf/go-ffmpreg/util"
"codeberg.org/gruf/go-ffmpreg/wasm"
"github.com/tetratelabs/wazero"
"github.com/tetratelabs/wazero/api"
"github.com/tetratelabs/wazero/imports/wasi_snapshot_preview1"
)
// InitFfmpeg initializes the ffmpeg WebAssembly instance pool,
// with given maximum limiting the number of concurrent instances.
func InitFfmpeg(ctx context.Context, max int) error {
initCache() // ensure compilation cache initialized
return ffmpegPool.Init(ctx, max)
}
// Ffmpeg runs the given arguments with an instance of ffmpeg.
func Ffmpeg(ctx context.Context, args wasm.Args) (uint32, error) {
return ffmpegPool.Run(ctx, args)
}
var ffmpegPool = wasmInstancePool{
inst: wasm.Instantiator{
// WASM module name.
Module: "ffmpeg",
// Per-instance WebAssembly runtime (with shared cache).
Runtime: func(ctx context.Context) wazero.Runtime {
// Prepare config with cache.
cfg := wazero.NewRuntimeConfig()
cfg = cfg.WithCoreFeatures(ffmpeglib.CoreFeatures)
cfg = cfg.WithCompilationCache(cache)
// Instantiate runtime with our config.
rt := wazero.NewRuntimeWithConfig(ctx, cfg)
// Prepare default "env" host module.
env := rt.NewHostModuleBuilder("env")
env = env.NewFunctionBuilder().
WithGoModuleFunction(
api.GoModuleFunc(util.Wasm_Tempnam),
[]api.ValueType{api.ValueTypeI32, api.ValueTypeI32},
[]api.ValueType{api.ValueTypeI32},
).
Export("tempnam")
// Instantiate "env" module in our runtime.
_, err := env.Instantiate(context.Background())
if err != nil {
panic(err)
}
// Instantiate the wasi snapshot preview 1 in runtime.
_, err = wasi_snapshot_preview1.Instantiate(ctx, rt)
if err != nil {
panic(err)
}
return rt
},
// Per-run module configuration.
Config: wazero.NewModuleConfig,
// Embedded WASM.
Source: ffmpeglib.B,
},
}

92
internal/media/ffmpeg/ffprobe.go Normal file
View file

@ -0,0 +1,92 @@
// GoToSocial
// Copyright (C) GoToSocial Authors admin@gotosocial.org
// SPDX-License-Identifier: AGPL-3.0-or-later
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package ffmpeg
import (
"context"
ffprobelib "codeberg.org/gruf/go-ffmpreg/embed/ffprobe"
"codeberg.org/gruf/go-ffmpreg/util"
"codeberg.org/gruf/go-ffmpreg/wasm"
"github.com/tetratelabs/wazero"
"github.com/tetratelabs/wazero/api"
"github.com/tetratelabs/wazero/imports/wasi_snapshot_preview1"
)
// InitFfprobe initializes the ffprobe WebAssembly instance pool,
// with given maximum limiting the number of concurrent instances.
func InitFfprobe(ctx context.Context, max int) error {
initCache() // ensure compilation cache initialized
return ffprobePool.Init(ctx, max)
}
// Ffprobe runs the given arguments with an instance of ffprobe.
func Ffprobe(ctx context.Context, args wasm.Args) (uint32, error) {
return ffprobePool.Run(ctx, args)
}
var ffprobePool = wasmInstancePool{
inst: wasm.Instantiator{
// WASM module name.
Module: "ffprobe",
// Per-instance WebAssembly runtime (with shared cache).
Runtime: func(ctx context.Context) wazero.Runtime {
// Prepare config with cache.
cfg := wazero.NewRuntimeConfig()
cfg = cfg.WithCoreFeatures(ffprobelib.CoreFeatures)
cfg = cfg.WithCompilationCache(cache)
// Instantiate runtime with our config.
rt := wazero.NewRuntimeWithConfig(ctx, cfg)
// Prepare default "env" host module.
env := rt.NewHostModuleBuilder("env")
env = env.NewFunctionBuilder().
WithGoModuleFunction(
api.GoModuleFunc(util.Wasm_Tempnam),
[]api.ValueType{api.ValueTypeI32, api.ValueTypeI32},
[]api.ValueType{api.ValueTypeI32},
).
Export("tempnam")
// Instantiate "env" module in our runtime.
_, err := env.Instantiate(context.Background())
if err != nil {
panic(err)
}
// Instantiate the wasi snapshot preview 1 in runtime.
_, err = wasi_snapshot_preview1.Instantiate(ctx, rt)
if err != nil {
panic(err)
}
return rt
},
// Per-run module configuration.
Config: wazero.NewModuleConfig,
// Embedded WASM.
Source: ffprobelib.B,
},
}

75
internal/media/ffmpeg/pool.go Normal file
View file

@ -0,0 +1,75 @@
// GoToSocial
// Copyright (C) GoToSocial Authors admin@gotosocial.org
// SPDX-License-Identifier: AGPL-3.0-or-later
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package ffmpeg
import (
"context"
"codeberg.org/gruf/go-ffmpreg/wasm"
)
// wasmInstancePool wraps a wasm.Instantiator{} and a
// channel of wasm.Instance{}s to provide a concurrency
// safe pool of WebAssembly module instances capable of
// compiling new instances on-the-fly, with a predetermined
// maximum number of concurrent instances at any one time.
type wasmInstancePool struct {
inst wasm.Instantiator
pool chan *wasm.Instance
}
func (p *wasmInstancePool) Init(ctx context.Context, sz int) error {
p.pool = make(chan *wasm.Instance, sz)
for i := 0; i < sz; i++ {
inst, err := p.inst.New(ctx)
if err != nil {
return err
}
p.pool <- inst
}
return nil
}
func (p *wasmInstancePool) Run(ctx context.Context, args wasm.Args) (uint32, error) {
var inst *wasm.Instance
select {
// Context canceled.
case <-ctx.Done():
return 0, ctx.Err()
// Acquire instance.
case inst = <-p.pool:
// Ensure instance is
// ready for running.
if inst.IsClosed() {
var err error
inst, err = p.inst.New(ctx)
if err != nil {
return 0, err
}
}
}
// Release instance to pool on end.
defer func() { p.pool <- inst }()
// Pass args to instance.
return inst.Run(ctx, args)
}

189
internal/media/image.go
View file

@ -1,189 +0,0 @@
// GoToSocial
// Copyright (C) GoToSocial Authors admin@gotosocial.org
// SPDX-License-Identifier: AGPL-3.0-or-later
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package media
import (
"bufio"
"image"
"image/color"
"image/draw"
"image/jpeg"
"image/png"
"io"
"sync"
"github.com/buckket/go-blurhash"
"github.com/disintegration/imaging"
"github.com/superseriousbusiness/gotosocial/internal/iotools"
// import to init webp encode/decoding.
_ "golang.org/x/image/webp"
)
var (
// pngEncoder provides our global PNG encoding with
// specified compression level, and memory pooled buffers.
pngEncoder = png.Encoder{
CompressionLevel: png.DefaultCompression,
BufferPool: &pngEncoderBufferPool{},
}
// jpegBufferPool is a memory pool
// of byte buffers for JPEG encoding.
jpegBufferPool sync.Pool
)
// gtsImage is a thin wrapper around the standard library image
// interface to provide our own useful helper functions for image
// size and aspect ratio calculations, streamed encoding to various
// types, and creating reduced size thumbnail images.
type gtsImage struct{ image image.Image }
// blankImage generates a blank image of given dimensions.
func blankImage(width int, height int) *gtsImage {
// create a rectangle with the same dimensions as the video
img := image.NewRGBA(image.Rect(0, 0, width, height))
// fill the rectangle with our desired fill color.
draw.Draw(img, img.Bounds(), &image.Uniform{
color.RGBA{42, 43, 47, 0},
}, image.Point{}, draw.Src)
return &gtsImage{image: img}
}
// decodeImage will decode image from reader stream and return image wrapped in our own gtsImage{} type.
func decodeImage(r io.Reader, opts ...imaging.DecodeOption) (*gtsImage, error) {
img, err := imaging.Decode(r, opts...)
if err != nil {
return nil, err
}
return &gtsImage{image: img}, nil
}
// Width returns the image width in pixels.
func (m *gtsImage) Width() int {
return m.image.Bounds().Size().X
}
// Height returns the image height in pixels.
func (m *gtsImage) Height() int {
return m.image.Bounds().Size().Y
}
// Size returns the total number of image pixels.
func (m *gtsImage) Size() int {
return m.image.Bounds().Size().X *
m.image.Bounds().Size().Y
}
// AspectRatio returns the image ratio of width:height.
func (m *gtsImage) AspectRatio() float32 {
// note: we cast bounds to float64 to prevent truncation
// and only at the end aspect ratio do we cast to float32
// (as the sizes are likely to be much larger than ratio).
return float32(float64(m.image.Bounds().Size().X) /
float64(m.image.Bounds().Size().Y))
}
// Thumbnail returns a small sized copy of gtsImage{}, limited to 512x512 if not small enough.
func (m *gtsImage) Thumbnail() *gtsImage {
const (
// max thumb
// dimensions.
maxWidth = 512
maxHeight = 512
)
// Check the receiving image is within max thumnail bounds.
if m.Width() <= maxWidth && m.Height() <= maxHeight {
return &gtsImage{image: imaging.Clone(m.image)}
}
// Image is too large, needs to be resized to thumbnail max.
img := imaging.Fit(m.image, maxWidth, maxHeight, imaging.Linear)
return &gtsImage{image: img}
}
// Blurhash calculates the blurhash for the receiving image data.
func (m *gtsImage) Blurhash() (string, error) {
// for generating blurhashes, it's more cost effective to
// lose detail since it's blurry, so make a tiny version.
tiny := imaging.Resize(m.image, 32, 0, imaging.NearestNeighbor)
// Encode blurhash from resized version
return blurhash.Encode(4, 3, tiny)
}
// ToJPEG creates a new streaming JPEG encoder from receiving image, and a size ptr
// which stores the number of bytes written during the image encoding process.
func (m *gtsImage) ToJPEG(opts *jpeg.Options) io.Reader {
return iotools.StreamWriteFunc(func(w io.Writer) error {
// Get encoding buffer
bw := getJPEGBuffer(w)
// Encode JPEG to buffered writer.
err := jpeg.Encode(bw, m.image, opts)
// Replace buffer.
//
// NOTE: jpeg.Encode() already
// performs a bufio.Writer.Flush().
putJPEGBuffer(bw)
return err
})
}
// ToPNG creates a new streaming PNG encoder from receiving image, and a size ptr
// which stores the number of bytes written during the image encoding process.
func (m *gtsImage) ToPNG() io.Reader {
return iotools.StreamWriteFunc(func(w io.Writer) error {
return pngEncoder.Encode(w, m.image)
})
}
// getJPEGBuffer fetches a reset JPEG encoding buffer from global JPEG buffer pool.
func getJPEGBuffer(w io.Writer) *bufio.Writer {
v := jpegBufferPool.Get()
if v == nil {
v = bufio.NewWriter(nil)
}
buf := v.(*bufio.Writer)
buf.Reset(w)
return buf
}
// putJPEGBuffer resets the given bufio writer and places in global JPEG buffer pool.
func putJPEGBuffer(buf *bufio.Writer) {
buf.Reset(nil)
jpegBufferPool.Put(buf)
}
// pngEncoderBufferPool implements png.EncoderBufferPool.
type pngEncoderBufferPool sync.Pool
func (p *pngEncoderBufferPool) Get() *png.EncoderBuffer {
buf, _ := (*sync.Pool)(p).Get().(*png.EncoderBuffer)
return buf
}
func (p *pngEncoderBufferPool) Put(buf *png.EncoderBuffer) {
(*sync.Pool)(p).Put(buf)
}

30
internal/media/manager.go
View file

@ -314,21 +314,26 @@ func (m *Manager) RefreshEmoji(
// Since this is a refresh we will end up storing new images at new
// paths, so we should wrap closer to delete old paths at completion.
wrapped := func(ctx context.Context) (io.ReadCloser, int64, error) {
wrapped := func(ctx context.Context) (io.ReadCloser, error) {
// Call original data func.
rc, sz, err := data(ctx)
// Call original func.
rc, err := data(ctx)
if err != nil {
return nil, 0, err
return nil, err
}
// Wrap closer to cleanup old data.
c := iotools.CloserFunc(func() error {
// Cast as separated reader / closer types.
rct, ok := rc.(*iotools.ReadCloserType)
// First try close original.
if rc.Close(); err != nil {
return err
}
if !ok {
// Allocate new read closer type.
rct = new(iotools.ReadCloserType)
rct.Reader = rc
rct.Closer = rc
}
// Wrap underlying io.Closer type to cleanup old data.
rct.Closer = iotools.CloserCallback(rct.Closer, func() {
// Remove any *old* emoji image file path now stream is closed.
if err := m.state.Storage.Delete(ctx, oldPath); err != nil &&
@ -341,12 +346,9 @@ func (m *Manager) RefreshEmoji(
!storage.IsNotFound(err) {
log.Errorf(ctx, "error deleting old static emoji %s from storage: %v", shortcodeDomain, err)
}
return nil
})
// Return newly wrapped readcloser and size.
return iotools.ReadCloser(rc, c), sz, nil
return rct, nil
}
// Use a new ID to create a new path

821
internal/media/manager_test.go
View file

File diff suppressed because it is too large Load diff

211
internal/media/png-stripper.go
View file

@ -1,211 +0,0 @@
// GoToSocial
// Copyright (C) GoToSocial Authors admin@gotosocial.org
// SPDX-License-Identifier: AGPL-3.0-or-later
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package media
/*
The code in this file is taken from the following source:
https://github.com/google/wuffs/blob/414a011491ff513b86d8694c5d71800f3cb5a715/script/strip-png-ancillary-chunks.go
It presents a workaround for this issue: https://github.com/golang/go/issues/43382
The license for the copied code is reproduced below:
Copyright 2021 The Wuffs Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
https://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
// strip-png-ancillary-chunks.go copies PNG data from stdin to stdout, removing
// any ancillary chunks.
//
// Specification-compliant PNG decoders are required to honor critical chunks
// but may ignore ancillary (non-critical) chunks. Stripping out ancillary
// chunks before decoding should mean that different PNG decoders will agree on
// the decoded output regardless of which ancillary chunk types they choose to
// honor. Specifically, some PNG decoders may implement color and gamma
// correction but not all do.
//
// This program will strip out all ancillary chunks, but it should be
// straightforward to copy-paste-and-modify it to strip out only certain chunk
// types (e.g. only "tRNS" transparency chunks).
//
// --------
//
// A PNG file consists of an 8-byte magic identifier and then a series of
// chunks. Each chunk is:
//
// - a 4-byte uint32 payload length N.
// - a 4-byte chunk type (e.g. "gAMA" for gamma correction metadata).
// - an N-byte payload.
// - a 4-byte CRC-32 checksum of the previous (N + 4) bytes, including the
// chunk type but excluding the payload length.
//
// Chunk types consist of 4 ASCII letters. The upper-case / lower-case bit of
// the first letter denote critical or ancillary chunks: "IDAT" and "PLTE" are
// critical, "gAMA" and "tEXt" are ancillary. See
// https://www.w3.org/TR/2003/REC-PNG-20031110/#5Chunk-naming-conventions
//
// --------
import (
"encoding/binary"
"io"
)
const (
chunkTypeIHDR = 0x49484452
chunkTypePLTE = 0x504C5445
chunkTypeIDAT = 0x49444154
chunkTypeIEND = 0x49454E44
chunkTypeTRNS = 0x74524e53
)
func isNecessaryChunkType(chunkType uint32) bool {
switch chunkType {
case chunkTypeIHDR:
return true
case chunkTypePLTE:
return true
case chunkTypeIDAT:
return true
case chunkTypeIEND:
return true
case chunkTypeTRNS:
return true
}
return false
}
// pngAncillaryChunkStripper wraps another io.Reader to strip ancillary chunks,
// if the data is in the PNG file format. If the data isn't PNG, it is passed
// through unmodified.
type pngAncillaryChunkStripper struct {
// Reader is the wrapped io.Reader.
Reader io.Reader
// stickyErr is the first error returned from the wrapped io.Reader.
stickyErr error
// buffer[rIndex:wIndex] holds data read from the wrapped io.Reader that
// wasn't passed through yet.
buffer [8]byte
rIndex int
wIndex int
// pending and discard is the number of remaining bytes for (and whether to
// discard or pass through) the current chunk-in-progress.
pending int64
discard bool
// notPNG is set true if the data stream doesn't start with the 8-byte PNG
// magic identifier. If true, the wrapped io.Reader's data (including the
// first up-to-8 bytes) is passed through without modification.
notPNG bool
// seenMagic is whether we've seen the 8-byte PNG magic identifier.
seenMagic bool
}
// Read implements io.Reader.
func (r *pngAncillaryChunkStripper) Read(p []byte) (int, error) {
for {
// If the wrapped io.Reader returned a non-nil error, drain r.buffer
// (what data we have) and return that error (if fully drained).
if r.stickyErr != nil {
n := copy(p, r.buffer[r.rIndex:r.wIndex])
r.rIndex += n
if r.rIndex < r.wIndex {
return n, nil
}
return n, r.stickyErr
}
// Handle trivial requests, including draining our buffer.
if len(p) == 0 {
return 0, nil
} else if r.rIndex < r.wIndex {
n := copy(p, r.buffer[r.rIndex:r.wIndex])
r.rIndex += n
return n, nil
}
// From here onwards, our buffer is drained: r.rIndex == r.wIndex.
// Handle non-PNG input.
if r.notPNG {
return r.Reader.Read(p)
}
// Continue processing any PNG chunk that's in progress, whether
// discarding it or passing it through.
for r.pending > 0 {
if int64(len(p)) > r.pending {
p = p[:r.pending]
}
n, err := r.Reader.Read(p)
r.pending -= int64(n)
r.stickyErr = err
if r.discard {
continue
}
return n, err
}
// We're either expecting the 8-byte PNG magic identifier or the 4-byte
// PNG chunk length + 4-byte PNG chunk type. Either way, read 8 bytes.
r.rIndex = 0
r.wIndex, r.stickyErr = io.ReadFull(r.Reader, r.buffer[:8])
if r.stickyErr != nil {
// Undo io.ReadFull converting io.EOF to io.ErrUnexpectedEOF.
if r.stickyErr == io.ErrUnexpectedEOF {
r.stickyErr = io.EOF
}
continue
}
// Process those 8 bytes, either:
// - a PNG chunk (if we've already seen the PNG magic identifier),
// - the PNG magic identifier itself (if the input is a PNG) or
// - something else (if it's not a PNG).
//nolint:gocritic
if r.seenMagic {
// The number of pending bytes is equal to (N + 4) because of the 4
// byte trailer, a checksum.
r.pending = int64(binary.BigEndian.Uint32(r.buffer[:4])) + 4
chunkType := binary.BigEndian.Uint32(r.buffer[4:])
r.discard = !isNecessaryChunkType(chunkType)
if r.discard {
r.rIndex = r.wIndex
}
} else if string(r.buffer[:8]) == "\x89PNG\x0D\x0A\x1A\x0A" {
r.seenMagic = true
} else {
r.notPNG = true
}
}
}

216
internal/media/processingemoji.go
View file

@ -18,16 +18,10 @@
package media
import (
"bytes"
"context"
"io"
"slices"
"codeberg.org/gruf/go-bytesize"
errorsv2 "codeberg.org/gruf/go-errors/v2"
"codeberg.org/gruf/go-runners"
"github.com/h2non/filetype"
"github.com/superseriousbusiness/gotosocial/internal/config"
"github.com/superseriousbusiness/gotosocial/internal/gtscontext"
"github.com/superseriousbusiness/gotosocial/internal/gtserror"
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
@ -125,19 +119,8 @@ func (p *ProcessingEmoji) load(ctx context.Context) (
// full-size media attachment details.
//
// This will update p.emoji as it goes.
if err = p.store(ctx); err != nil {
return err
}
// Finish processing by reloading media into
// memory to get dimension and generate a thumb.
//
// This will update p.emoji as it goes.
if err = p.finish(ctx); err != nil {
return err //nolint:revive
}
return nil
err = p.store(ctx)
return err
})
emoji = p.emoji
return
@ -147,80 +130,66 @@ func (p *ProcessingEmoji) load(ctx context.Context) (
// and updates the underlying attachment fields as necessary. It will then stream
// bytes from p's reader directly into storage so that it can be retrieved later.
func (p *ProcessingEmoji) store(ctx context.Context) error {
// Load media from provided data fun
rc, sz, err := p.dataFn(ctx)
// Load media from data func.
rc, err := p.dataFn(ctx)
if err != nil {
return gtserror.Newf("error executing data function: %w", err)
}
var (
// predfine temporary media
// file path variables so we
// can remove them on error.
temppath string
staticpath string
)
defer func() {
// Ensure data reader gets closed on return.
if err := rc.Close(); err != nil {
log.Errorf(ctx, "error closing data reader: %v", err)
if err := remove(temppath, staticpath); err != nil {
log.Errorf(ctx, "error(s) cleaning up files: %v", err)
}
}()
var maxSize bytesize.Size
if p.emoji.IsLocal() {
// this is a local emoji upload
maxSize = config.GetMediaEmojiLocalMaxSize()
} else {
// this is a remote incoming emoji
maxSize = config.GetMediaEmojiRemoteMaxSize()
}
// Check that provided size isn't beyond max. We check beforehand
// so that we don't attempt to stream the emoji into storage if not needed.
if sz > 0 && sz > int64(maxSize) {
sz := bytesize.Size(sz) // improves log readability
return gtserror.Newf("given emoji size %s greater than max allowed %s", sz, maxSize)
}
// Prepare to read bytes from
// file header or magic number.
fileSize := int(sz)
hdrBuf := newHdrBuf(fileSize)
// Read into buffer as much as possible.
//
// UnexpectedEOF means we couldn't read up to the
// given size, but we may still have read something.
//
// EOF means we couldn't read anything at all.
//
// Any other error likely means the connection messed up.
//
// In other words, rather counterintuitively, we
// can only proceed on no error or unexpected error!
n, err := io.ReadFull(rc, hdrBuf)
// Drain reader to tmp file
// (this reader handles close).
temppath, err = drainToTmp(rc)
if err != nil {
if err != io.ErrUnexpectedEOF {
return gtserror.Newf("error reading first bytes of incoming media: %w", err)
}
// Initial file size was misreported, so we didn't read
// fully into hdrBuf. Reslice it to the size we did read.
hdrBuf = hdrBuf[:n]
fileSize = n
p.emoji.ImageFileSize = fileSize
return gtserror.Newf("error draining data to tmp: %w", err)
}
// Parse file type info from header buffer.
// This should only ever error if the buffer
// is empty (ie., the attachment is 0 bytes).
info, err := filetype.Match(hdrBuf)
// Pass input file through ffprobe to
// parse further metadata information.
result, err := ffprobe(ctx, temppath)
if err != nil {
return gtserror.Newf("error parsing file type: %w", err)
return gtserror.Newf("error ffprobing data: %w", err)
}
// Ensure supported emoji img type.
if !slices.Contains(SupportedEmojiMIMETypes, info.MIME.Value) {
return gtserror.Newf("unsupported emoji filetype: %s", info.Extension)
switch {
// No errors parsing data.
case result.Error == nil:
// Data type unhandleable by ffprobe.
case result.Error.Code == -1094995529:
log.Warn(ctx, "unsupported data type")
return nil
default:
return gtserror.Newf("ffprobe error: %w", err)
}
// Recombine header bytes with remaining stream
r := io.MultiReader(bytes.NewReader(hdrBuf), rc)
var ext string
// Set media type from ffprobe format data.
fileType, ext := result.Format.GetFileType()
if fileType != gtsmodel.FileTypeImage {
return gtserror.Newf("unsupported emoji filetype: %s (%s)", fileType, ext)
}
// Generate a static image from input emoji path.
staticpath, err = ffmpegGenerateStatic(ctx, temppath)
if err != nil {
return gtserror.Newf("error generating emoji static: %w", err)
}
var pathID string
if p.newPathID != "" {
@ -244,95 +213,50 @@ func (p *ProcessingEmoji) store(ctx context.Context) error {
string(TypeEmoji),
string(SizeOriginal),
pathID,
info.Extension,
ext,
)
// File shouldn't already exist in storage at this point,
// but we do a check as it's worth logging / cleaning up.
if have, _ := p.mgr.state.Storage.Has(ctx, p.emoji.ImagePath); have {
log.Warnf(ctx, "emoji already exists at: %s", p.emoji.ImagePath)
// Attempt to remove existing emoji at storage path (might be broken / out-of-date)
if err := p.mgr.state.Storage.Delete(ctx, p.emoji.ImagePath); err != nil {
return gtserror.Newf("error removing emoji %s from storage: %v", p.emoji.ImagePath, err)
}
}
// Write the final image reader stream to our storage.
sz, err = p.mgr.state.Storage.PutStream(ctx, p.emoji.ImagePath, r)
// Copy temporary file into storage at path.
filesz, err := p.mgr.state.Storage.PutFile(ctx,
p.emoji.ImagePath,
temppath,
)
if err != nil {
return gtserror.Newf("error writing emoji to storage: %w", err)
}
// Perform final size check in case none was
// given previously, or size was mis-reported.
// (error here will later perform p.cleanup()).
if sz > int64(maxSize) {
sz := bytesize.Size(sz) // improves log readability
return gtserror.Newf("written emoji size %s greater than max allowed %s", sz, maxSize)
// Copy static emoji file into storage at path.
staticsz, err := p.mgr.state.Storage.PutFile(ctx,
p.emoji.ImageStaticPath,
staticpath,
)
if err != nil {
return gtserror.Newf("error writing static to storage: %w", err)
}
// Set final determined file sizes.
p.emoji.ImageFileSize = int(filesz)
p.emoji.ImageStaticFileSize = int(staticsz)
// Fill in remaining emoji data now it's stored.
p.emoji.ImageURL = uris.URIForAttachment(
instanceAccID,
string(TypeEmoji),
string(SizeOriginal),
pathID,
info.Extension,
ext,
)
p.emoji.ImageContentType = info.MIME.Value
p.emoji.ImageFileSize = int(sz)
// Get mimetype for the file container
// type, falling back to generic data.
p.emoji.ImageContentType = getMimeType(ext)
// We can now consider this cached.
p.emoji.Cached = util.Ptr(true)
return nil
}
func (p *ProcessingEmoji) finish(ctx context.Context) error {
// Get a stream to the original file for further processing.
rc, err := p.mgr.state.Storage.GetStream(ctx, p.emoji.ImagePath)
if err != nil {
return gtserror.Newf("error loading file from storage: %w", err)
}
defer rc.Close()
// Decode the image from storage.
staticImg, err := decodeImage(rc)
if err != nil {
return gtserror.Newf("error decoding image: %w", err)
}
// staticImg should be in-memory by
// now so we're done with storage.
if err := rc.Close(); err != nil {
return gtserror.Newf("error closing file: %w", err)
}
// Static img shouldn't exist in storage at this point,
// but we do a check as it's worth logging / cleaning up.
if have, _ := p.mgr.state.Storage.Has(ctx, p.emoji.ImageStaticPath); have {
log.Warnf(ctx, "static emoji already exists at: %s", p.emoji.ImageStaticPath)
// Attempt to remove existing thumbnail (might be broken / out-of-date).
if err := p.mgr.state.Storage.Delete(ctx, p.emoji.ImageStaticPath); err != nil {
return gtserror.Newf("error removing static emoji %s from storage: %v", p.emoji.ImageStaticPath, err)
}
}
// Create emoji PNG encoder stream.
enc := staticImg.ToPNG()
// Stream-encode the PNG static emoji image into our storage driver.
sz, err := p.mgr.state.Storage.PutStream(ctx, p.emoji.ImageStaticPath, enc)
if err != nil {
return gtserror.Newf("error stream-encoding static emoji to storage: %w", err)
}
// Set final written thumb size.
p.emoji.ImageStaticFileSize = int(sz)
return nil
}
// cleanup will remove any traces of processing emoji from storage,
// and perform any other necessary cleanup steps after failure.
func (p *ProcessingEmoji) cleanup(ctx context.Context) {

467
internal/media/processingmedia.go
View file

@ -18,18 +18,12 @@
package media
import (
"bytes"
"cmp"
"context"
"image/jpeg"
"io"
"time"
errorsv2 "codeberg.org/gruf/go-errors/v2"
"codeberg.org/gruf/go-runners"
terminator "codeberg.org/superseriousbusiness/exif-terminator"
"github.com/disintegration/imaging"
"github.com/h2non/filetype"
"github.com/superseriousbusiness/gotosocial/internal/gtscontext"
"github.com/superseriousbusiness/gotosocial/internal/gtserror"
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
@ -145,19 +139,8 @@ func (p *ProcessingMedia) load(ctx context.Context) (
// full-size media attachment details.
//
// This will update p.media as it goes.
if err = p.store(ctx); err != nil {
return err
}
// Finish processing by reloading media into
// memory to get dimension and generate a thumb.
//
// This will update p.media as it goes.
if err = p.finish(ctx); err != nil {
return err //nolint:revive
}
return nil
err = p.store(ctx)
return err
})
media = p.media
return
@ -167,89 +150,224 @@ func (p *ProcessingMedia) load(ctx context.Context) (
// and updates the underlying attachment fields as necessary. It will then stream
// bytes from p's reader directly into storage so that it can be retrieved later.
func (p *ProcessingMedia) store(ctx context.Context) error {
// Load media from provided data fun
rc, sz, err := p.dataFn(ctx)
// Load media from data func.
rc, err := p.dataFn(ctx)
if err != nil {
return gtserror.Newf("error executing data function: %w", err)
}
var (
// predfine temporary media
// file path variables so we
// can remove them on error.
temppath string
thumbpath string
)
defer func() {
// Ensure data reader gets closed on return.
if err := rc.Close(); err != nil {
log.Errorf(ctx, "error closing data reader: %v", err)
if err := remove(temppath, thumbpath); err != nil {
log.Errorf(ctx, "error(s) cleaning up files: %v", err)
}
}()
// Assume we're given correct file
// size, we can overwrite this later
// once we know THE TRUTH.
fileSize := int(sz)
p.media.File.FileSize = fileSize
// Prepare to read bytes from
// file header or magic number.
hdrBuf := newHdrBuf(fileSize)
// Read into buffer as much as possible.
//
// UnexpectedEOF means we couldn't read up to the
// given size, but we may still have read something.
//
// EOF means we couldn't read anything at all.
//
// Any other error likely means the connection messed up.
//
// In other words, rather counterintuitively, we
// can only proceed on no error or unexpected error!
n, err := io.ReadFull(rc, hdrBuf)
// Drain reader to tmp file
// (this reader handles close).
temppath, err = drainToTmp(rc)
if err != nil {
if err != io.ErrUnexpectedEOF {
return gtserror.Newf("error reading first bytes of incoming media: %w", err)
return gtserror.Newf("error draining data to tmp: %w", err)
}
// Pass input file through ffprobe to
// parse further metadata information.
result, err := ffprobe(ctx, temppath)
if err != nil {
return gtserror.Newf("error ffprobing data: %w", err)
}
switch {
// No errors parsing data.
case result.Error == nil:
// Data type unhandleable by ffprobe.
case result.Error.Code == -1094995529:
log.Warn(ctx, "unsupported data type")
return nil
default:
return gtserror.Newf("ffprobe error: %w", err)
}
var ext string
// Set the media type from ffprobe format data.
p.media.Type, ext = result.Format.GetFileType()
if p.media.Type == gtsmodel.FileTypeUnknown {
// Return early (deleting file)
// for unhandled file types.
return nil
}
switch p.media.Type {
case gtsmodel.FileTypeImage:
// Pass file through ffmpeg clearing
// any excess metadata (e.g. EXIF).
if err := ffmpegClearMetadata(ctx,
temppath, ext,
); err != nil {
return gtserror.Newf("error cleaning metadata: %w", err)
}
// Initial file size was misreported, so we didn't read
// fully into hdrBuf. Reslice it to the size we did read.
hdrBuf = hdrBuf[:n]
fileSize = n
p.media.File.FileSize = fileSize
}
// Extract image metadata from streams.
width, height, err := result.ImageMeta()
if err != nil {
return err
}
p.media.FileMeta.Original.Width = width
p.media.FileMeta.Original.Height = height
p.media.FileMeta.Original.Size = (width * height)
p.media.FileMeta.Original.Aspect = float32(width) / float32(height)
// Parse file type info from header buffer.
// This should only ever error if the buffer
// is empty (ie., the attachment is 0 bytes).
info, err := filetype.Match(hdrBuf)
if err != nil {
return gtserror.Newf("error parsing file type: %w", err)
}
// Determine thumbnail dimensions to use.
thumbWidth, thumbHeight := thumbSize(width, height)
p.media.FileMeta.Small.Width = thumbWidth
p.media.FileMeta.Small.Height = thumbHeight
p.media.FileMeta.Small.Size = (thumbWidth * thumbHeight)
p.media.FileMeta.Small.Aspect = float32(thumbWidth) / float32(thumbHeight)
// Recombine header bytes with remaining stream
r := io.MultiReader(bytes.NewReader(hdrBuf), rc)
// Generate a thumbnail image from input image path.
thumbpath, err = ffmpegGenerateThumb(ctx, temppath,
thumbWidth,
thumbHeight,
)
if err != nil {
return gtserror.Newf("error generating image thumb: %w", err)
}
// Assume we'll put
// this file in storage.
store := true
case gtsmodel.FileTypeVideo:
// Pass file through ffmpeg clearing
// any excess metadata (e.g. EXIF).
if err := ffmpegClearMetadata(ctx,
temppath, ext,
); err != nil {
return gtserror.Newf("error cleaning metadata: %w", err)
}
switch info.Extension {
case "mp4":
// No problem.
// Extract video metadata we can from streams.
width, height, framerate, err := result.VideoMeta()
if err != nil {
return err
}
p.media.FileMeta.Original.Width = width
p.media.FileMeta.Original.Height = height
p.media.FileMeta.Original.Size = (width * height)
p.media.FileMeta.Original.Aspect = float32(width) / float32(height)
p.media.FileMeta.Original.Framerate = &framerate
case "gif":
// No problem
// Extract total duration from format.
duration := result.Format.GetDuration()
p.media.FileMeta.Original.Duration = &duration
case "jpg", "jpeg", "png", "webp":
if fileSize > 0 {
// A file size was provided so we can clean
// exif data from image as we're streaming it.
r, err = terminator.Terminate(r, fileSize, info.Extension)
// Extract total bitrate from format.
bitrate := result.Format.GetBitRate()
p.media.FileMeta.Original.Bitrate = &bitrate
// Determine thumbnail dimensions to use.
thumbWidth, thumbHeight := thumbSize(width, height)
p.media.FileMeta.Small.Width = thumbWidth
p.media.FileMeta.Small.Height = thumbHeight
p.media.FileMeta.Small.Size = (thumbWidth * thumbHeight)
p.media.FileMeta.Small.Aspect = float32(thumbWidth) / float32(thumbHeight)
// Extract a thumbnail frame from input video path.
thumbpath, err = ffmpegGenerateThumb(ctx, temppath,
thumbWidth,
thumbHeight,
)
if err != nil {
return gtserror.Newf("error extracting video frame: %w", err)
}
case gtsmodel.FileTypeAudio:
// Extract total duration from format.
duration := result.Format.GetDuration()
p.media.FileMeta.Original.Duration = &duration
// Extract total bitrate from format.
bitrate := result.Format.GetBitRate()
p.media.FileMeta.Original.Bitrate = &bitrate
// Extract image metadata from streams (if any),
// this will only exist for embedded album art.
width, height, _ := result.ImageMeta()
if width > 0 && height > 0 {
// Determine thumbnail dimensions to use.
thumbWidth, thumbHeight := thumbSize(width, height)
p.media.FileMeta.Small.Width = thumbWidth
p.media.FileMeta.Small.Height = thumbHeight
p.media.FileMeta.Small.Size = (thumbWidth * thumbHeight)
p.media.FileMeta.Small.Aspect = float32(thumbWidth) / float32(thumbHeight)
// Generate a thumbnail image from input image path.
thumbpath, err = ffmpegGenerateThumb(ctx, temppath,
thumbWidth,
thumbHeight,
)
if err != nil {
return gtserror.Newf("error cleaning exif data: %w", err)
return gtserror.Newf("error generating image thumb: %w", err)
}
}
default:
// The file is not a supported format that we can process, so we can't do much with it.
log.Warnf(ctx, "unsupported media extension '%s'; not caching locally", info.Extension)
store = false
log.Warnf(ctx, "unsupported type: %s (%s)", p.media.Type, result.Format.FormatName)
return nil
}
// Calculate final media attachment file path.
p.media.File.Path = uris.StoragePathForAttachment(
p.media.AccountID,
string(TypeAttachment),
string(SizeOriginal),
p.media.ID,
ext,
)
// Copy temporary file into storage at path.
filesz, err := p.mgr.state.Storage.PutFile(ctx,
p.media.File.Path,
temppath,
)
if err != nil {
return gtserror.Newf("error writing media to storage: %w", err)
}
// Set final determined file size.
p.media.File.FileSize = int(filesz)
if thumbpath != "" {
// Note that neither thumbnail storage
// nor a blurhash are needed for audio.
if p.media.Blurhash == "" {
// Generate blurhash (if not already) from thumbnail.
p.media.Blurhash, err = generateBlurhash(thumbpath)
if err != nil {
return gtserror.Newf("error generating thumb blurhash: %w", err)
}
}
// Copy thumbnail file into storage at path.
thumbsz, err := p.mgr.state.Storage.PutFile(ctx,
p.media.Thumbnail.Path,
thumbpath,
)
if err != nil {
return gtserror.Newf("error writing thumb to storage: %w", err)
}
// Set final determined thumbnail size.
p.media.Thumbnail.FileSize = int(thumbsz)
}
// Fill in correct attachment
@ -259,194 +377,17 @@ func (p *ProcessingMedia) store(ctx context.Context) error {
string(TypeAttachment),
string(SizeOriginal),
p.media.ID,
info.Extension,
ext,
)
// Prefer discovered MIME, fallback to generic data stream.
mime := cmp.Or(info.MIME.Value, "application/octet-stream")
p.media.File.ContentType = mime
// Calculate final media attachment file path.
p.media.File.Path = uris.StoragePathForAttachment(
p.media.AccountID,
string(TypeAttachment),
string(SizeOriginal),
p.media.ID,
info.Extension,
)
// We should only try to store the file if it's
// a format we can keep processing, otherwise be
// a bit cheeky: don't store it and let users
// click through to the remote server instead.
if !store {
return nil
}
// File shouldn't already exist in storage at this point,
// but we do a check as it's worth logging / cleaning up.
if have, _ := p.mgr.state.Storage.Has(ctx, p.media.File.Path); have {
log.Warnf(ctx, "media already exists at: %s", p.media.File.Path)
// Attempt to remove existing media at storage path (might be broken / out-of-date)
if err := p.mgr.state.Storage.Delete(ctx, p.media.File.Path); err != nil {
return gtserror.Newf("error removing media %s from storage: %v", p.media.File.Path, err)
}
}
// Write the final reader stream to our storage driver.
sz, err = p.mgr.state.Storage.PutStream(ctx, p.media.File.Path, r)
if err != nil {
return gtserror.Newf("error writing media to storage: %w", err)
}
// Set actual written size
// as authoritative file size.
p.media.File.FileSize = int(sz)
// Get mimetype for the file container
// type, falling back to generic data.
p.media.File.ContentType = getMimeType(ext)
// We can now consider this cached.
p.media.Cached = util.Ptr(true)
return nil
}
func (p *ProcessingMedia) finish(ctx context.Context) error {
// Nothing else to do if
// media was not cached.
if !*p.media.Cached {
return nil
}
// Get a stream to the original file for further processing.
rc, err := p.mgr.state.Storage.GetStream(ctx, p.media.File.Path)
if err != nil {
return gtserror.Newf("error loading file from storage: %w", err)
}
defer rc.Close()
// fullImg is the processed version of
// the original (stripped + reoriented).
var fullImg *gtsImage
// Depending on the content type, we
// can do various types of decoding.
switch p.media.File.ContentType {
// .jpeg, .gif, .webp image type
case mimeImageJpeg, mimeImageGif, mimeImageWebp:
fullImg, err = decodeImage(rc,
imaging.AutoOrientation(true),
)
if err != nil {
return gtserror.Newf("error decoding image: %w", err)
}
// Mark as no longer unknown type now
// we know for sure we can decode it.
p.media.Type = gtsmodel.FileTypeImage
// .png image (requires ancillary chunk stripping)
case mimeImagePng:
fullImg, err = decodeImage(
&pngAncillaryChunkStripper{Reader: rc},
imaging.AutoOrientation(true),
)
if err != nil {
return gtserror.Newf("error decoding image: %w", err)
}
// Mark as no longer unknown type now
// we know for sure we can decode it.
p.media.Type = gtsmodel.FileTypeImage
// .mp4 video type
case mimeVideoMp4:
video, err := decodeVideoFrame(rc)
if err != nil {
return gtserror.Newf("error decoding video: %w", err)
}
// Set video frame as image.
fullImg = video.frame
// Set video metadata in attachment info.
p.media.FileMeta.Original.Duration = &video.duration
p.media.FileMeta.Original.Framerate = &video.framerate
p.media.FileMeta.Original.Bitrate = &video.bitrate
// Mark as no longer unknown type now
// we know for sure we can decode it.
p.media.Type = gtsmodel.FileTypeVideo
}
// fullImg should be in-memory by
// now so we're done with storage.
if err := rc.Close(); err != nil {
return gtserror.Newf("error closing file: %w", err)
}
// Set full-size dimensions in attachment info.
p.media.FileMeta.Original.Width = fullImg.Width()
p.media.FileMeta.Original.Height = fullImg.Height()
p.media.FileMeta.Original.Size = fullImg.Size()
p.media.FileMeta.Original.Aspect = fullImg.AspectRatio()
// Get smaller thumbnail image
thumbImg := fullImg.Thumbnail()
// Garbage collector, you may
// now take our large son.
fullImg = nil
// Only generate blurhash
// from thumb if necessary.
if p.media.Blurhash == "" {
hash, err := thumbImg.Blurhash()
if err != nil {
return gtserror.Newf("error generating blurhash: %w", err)
}
// Set the attachment blurhash.
p.media.Blurhash = hash
}
// Thumbnail shouldn't exist in storage at this point,
// but we do a check as it's worth logging / cleaning up.
if have, _ := p.mgr.state.Storage.Has(ctx, p.media.Thumbnail.Path); have {
log.Warnf(ctx, "thumbnail already exists at: %s", p.media.Thumbnail.Path)
// Attempt to remove existing thumbnail (might be broken / out-of-date).
if err := p.mgr.state.Storage.Delete(ctx, p.media.Thumbnail.Path); err != nil {
return gtserror.Newf("error removing thumbnail %s from storage: %v", p.media.Thumbnail.Path, err)
}
}
// Create a thumbnail JPEG encoder stream.
enc := thumbImg.ToJPEG(&jpeg.Options{
// Good enough for
// a thumbnail.
Quality: 70,
})
// Stream-encode the JPEG thumbnail image into our storage driver.
sz, err := p.mgr.state.Storage.PutStream(ctx, p.media.Thumbnail.Path, enc)
if err != nil {
return gtserror.Newf("error stream-encoding thumbnail to storage: %w", err)
}
// Set final written thumb size.
p.media.Thumbnail.FileSize = int(sz)
// Set thumbnail dimensions in attachment info.
p.media.FileMeta.Small = gtsmodel.Small{
Width: thumbImg.Width(),
Height: thumbImg.Height(),
Size: thumbImg.Size(),
Aspect: thumbImg.AspectRatio(),
}
// Finally set the attachment as processed.
// Finally set the attachment as finished processing.
p.media.Processing = gtsmodel.ProcessingStatusProcessed
return nil

10
internal/media/refetch.go
View file

@ -24,12 +24,13 @@ import (
"io"
"net/url"
"github.com/superseriousbusiness/gotosocial/internal/config"
"github.com/superseriousbusiness/gotosocial/internal/db"
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
"github.com/superseriousbusiness/gotosocial/internal/log"
)
type DereferenceMedia func(ctx context.Context, iri *url.URL) (io.ReadCloser, int64, error)
type DereferenceMedia func(ctx context.Context, iri *url.URL, maxsz int64) (io.ReadCloser, error)
// RefetchEmojis iterates through remote emojis (for the given domain, or all if domain is empty string).
//
@ -48,6 +49,9 @@ func (m *Manager) RefetchEmojis(ctx context.Context, domain string, dereferenceM
refetchIDs []string
)
// Get max supported remote emoji media size.
maxsz := config.GetMediaEmojiRemoteMaxSize()
// page through emojis 20 at a time, looking for those with missing images
for {
// Fetch next block of emojis from database
@ -107,8 +111,8 @@ func (m *Manager) RefetchEmojis(ctx context.Context, domain string, dereferenceM
continue
}
dataFunc := func(ctx context.Context) (reader io.ReadCloser, fileSize int64, err error) {
return dereferenceMedia(ctx, emojiImageIRI)
dataFunc := func(ctx context.Context) (reader io.ReadCloser, err error) {
return dereferenceMedia(ctx, emojiImageIRI, int64(maxsz))
}
processingEmoji, err := m.RefreshEmoji(ctx, emoji, dataFunc, AdditionalEmojiInfo{

BIN
internal/media/test/birdnest-processed.mp4
View file

Binary file not shown.

BIN
internal/media/test/birdnest-thumbnail.jpg
View file

Binary file not shown.
Side by side Swipe Overlay

Before

Width:  |  Height:  |  Size: 2.8 KiB

After

Width:  |  Height:  |  Size: 9.9 KiB

BIN
internal/media/test/gts_pixellated-static.png
View file

Binary file not shown.
Side by side Swipe Overlay

Before

Width:  |  Height:  |  Size: 1,010 B

After

Width:  |  Height:  |  Size: 1.5 KiB

BIN
internal/media/test/longer-mp4-processed.mp4
View file

Binary file not shown.

BIN
internal/media/test/longer-mp4-thumbnail.jpg
View file

Binary file not shown.
Side by side Swipe Overlay

Before

Width:  |  Height:  |  Size: 2.8 KiB

After

Width:  |  Height:  |  Size: 3.7 KiB

BIN
internal/media/test/nb-flag-static.png
View file

Binary file not shown.
Side by side Swipe Overlay

Before

Width:  |  Height:  |  Size: 878 B

After

Width:  |  Height:  |  Size: 709 B

BIN
internal/media/test/rainbow-static.png
View file

Binary file not shown.
Side by side Swipe Overlay

Before

Width:  |  Height:  |  Size: 10 KiB

After

Width:  |  Height:  |  Size: 5.9 KiB

BIN
internal/media/test/test-jpeg-thumbnail.jpg
View file

Binary file not shown.
Side by side Swipe Overlay

Before

Width:  |  Height:  |  Size: 20 KiB

After

Width:  |  Height:  |  Size: 11 KiB

BIN
internal/media/test/test-mp4-processed.mp4
View file

Binary file not shown.

BIN
internal/media/test/test-mp4-thumbnail.jpg
View file

Binary file not shown.
Side by side Swipe Overlay

Before

Width:  |  Height:  |  Size: 1.9 KiB

After

Width:  |  Height:  |  Size: 4.4 KiB

BIN
internal/media/test/test-opus-original.opus Normal file
View file

Binary file not shown.

BIN
internal/media/test/test-opus-processed.opus Normal file
View file

Binary file not shown.

BIN
internal/media/test/test-png-alphachannel-processed.png
View file

Binary file not shown.
Side by side Swipe Overlay

Before

Width:  |  Height:  |  Size: 18 KiB

After

Width:  |  Height:  |  Size: 18 KiB

BIN
internal/media/test/test-png-alphachannel-thumbnail.jpg
View file

Binary file not shown.
Side by side Swipe Overlay

Before

Width:  |  Height:  |  Size: 5.8 KiB

After

Width:  |  Height:  |  Size: 7.8 KiB

BIN
internal/media/test/test-png-noalphachannel-thumbnail.jpg
View file

Binary file not shown.
Side by side Swipe Overlay

Before

Width:  |  Height:  |  Size: 5.8 KiB

After

Width:  |  Height:  |  Size: 7.9 KiB

2
internal/media/types.go
View file

@ -144,4 +144,4 @@ type AdditionalEmojiInfo struct {
}
// DataFunc represents a function used to retrieve the raw bytes of a piece of media.
type DataFunc func(ctx context.Context) (reader io.ReadCloser, fileSize int64, err error)
type DataFunc func(ctx context.Context) (reader io.ReadCloser, err error)

176
internal/media/util.go
View file

@ -17,25 +17,161 @@
package media
// newHdrBuf returns a buffer of suitable size to
// read bytes from a file header or magic number.
//
// File header is *USUALLY* 261 bytes at the start
// of a file; magic number can be much less than
// that (just a few bytes).
//
// To cover both cases, this function returns a buffer
// suitable for whichever is smallest: the first 261
// bytes of the file, or the whole file.
//
// See:
//
// - https://en.wikipedia.org/wiki/File_format#File_header
// - https://github.com/h2non/filetype.
func newHdrBuf(fileSize int) []byte {
bufSize := 261
if fileSize > 0 && fileSize < bufSize {
bufSize = fileSize
import (
"cmp"
"errors"
"fmt"
"image"
"image/jpeg"
"io"
"os"
"codeberg.org/gruf/go-bytesize"
"codeberg.org/gruf/go-iotools"
"codeberg.org/gruf/go-mimetypes"
"github.com/buckket/go-blurhash"
"github.com/disintegration/imaging"
)
// thumbSize returns the dimensions to use for an input
// image of given width / height, for its outgoing thumbnail.
// This maintains the original image aspect ratio.
func thumbSize(width, height int) (int, int) {
const (
maxThumbWidth = 512
maxThumbHeight = 512
)
switch {
// Simplest case, within bounds!
case width < maxThumbWidth &&
height < maxThumbHeight:
return width, height
// Width is larger side.
case width > height:
p := float32(width) / float32(maxThumbWidth)
return maxThumbWidth, int(float32(height) / p)
// Height is larger side.
case height > width:
p := float32(height) / float32(maxThumbHeight)
return int(float32(width) / p), maxThumbHeight
// Square.
default:
return maxThumbWidth, maxThumbHeight
}
return make([]byte, bufSize)
}
// jpegDecode decodes the JPEG at filepath into parsed image.Image.
func jpegDecode(filepath string) (image.Image, error) {
// Open the file at given path.
file, err := os.Open(filepath)
if err != nil {
return nil, err
}
// Decode image from file.
img, err := jpeg.Decode(file)
// Done with file.
_ = file.Close()
return img, err
}
// generateBlurhash generates a blurhash for JPEG at filepath.
func generateBlurhash(filepath string) (string, error) {
// Decode JPEG file at given path.
img, err := jpegDecode(filepath)
if err != nil {
return "", err
}
// for generating blurhashes, it's more cost effective to
// lose detail since it's blurry, so make a tiny version.
tiny := imaging.Resize(img, 64, 64, imaging.NearestNeighbor)
// Drop the larger image
// ref as soon as possible
// to allow GC to claim.
img = nil //nolint
// Generate blurhash for thumbnail.
return blurhash.Encode(4, 3, tiny)
}
// getMimeType returns a suitable mimetype for file extension.
func getMimeType(ext string) string {
const defaultType = "application/octet-stream"
return cmp.Or(mimetypes.MimeTypes[ext], defaultType)
}
// drainToTmp drains data from given reader into a new temp file
// and closes it, returning the path of the resulting temp file.
//
// Note that this function specifically makes attempts to unwrap the
// io.ReadCloser as much as it can to underlying type, to maximise
// chance that Linux's sendfile syscall can be utilised for optimal
// draining of data source to temporary file storage.
func drainToTmp(rc io.ReadCloser) (string, error) {
tmp, err := os.CreateTemp(os.TempDir(), "gotosocial-*")
if err != nil {
return "", err
}
// Close readers
// on func return.
defer tmp.Close()
defer rc.Close()
// Extract file path.
path := tmp.Name()
// Limited reader (if any).
var lr *io.LimitedReader
var limit int64
// Reader type to use
// for draining to tmp.
rd := (io.Reader)(rc)
// Check if reader is actually wrapped,
// (as our http client wraps close func).
rct, ok := rc.(*iotools.ReadCloserType)
if ok {
// Get unwrapped.
rd = rct.Reader
// Extract limited reader if wrapped.
lr, limit = iotools.GetReaderLimit(rd)
}
// Drain reader into tmp.
_, err = tmp.ReadFrom(rd)
if err != nil {
return path, err
}
// Check to see if limit was reached,
// (produces more useful error messages).
if lr != nil && !iotools.AtEOF(lr.R) {
return path, fmt.Errorf("reached read limit %s", bytesize.Size(limit))
}
return path, nil
}
// remove only removes paths if not-empty.
func remove(paths ...string) error {
var errs []error
for _, path := range paths {
if path != "" {
if err := os.Remove(path); err != nil {
errs = append(errs, fmt.Errorf("error removing %s: %w", path, err))
}
}
}
return errors.Join(errs...)
}

141
internal/media/video.go
View file

@ -1,141 +0,0 @@
// GoToSocial
// Copyright (C) GoToSocial Authors admin@gotosocial.org
// SPDX-License-Identifier: AGPL-3.0-or-later
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package media
import (
"fmt"
"io"
"github.com/abema/go-mp4"
"github.com/superseriousbusiness/gotosocial/internal/iotools"
"github.com/superseriousbusiness/gotosocial/internal/log"
)
type gtsVideo struct {
frame *gtsImage
duration float32 // in seconds
bitrate uint64
framerate float32
}
// decodeVideoFrame decodes and returns an image from a single frame in the given video stream.
// (note: currently this only returns a blank image resized to fit video dimensions).
func decodeVideoFrame(r io.Reader) (*gtsVideo, error) {
// Check if video stream supports
// seeking, usually when *os.File.
rsc, ok := r.(io.ReadSeekCloser)
if !ok {
var err error
// Store stream to temporary location
// in order that we can get seek-reads.
rsc, err = iotools.TempFileSeeker(r)
if err != nil {
return nil, fmt.Errorf("error creating temp file seeker: %w", err)
}
defer func() {
// Ensure temp. read seeker closed.
if err := rsc.Close(); err != nil {
log.Errorf(nil, "error closing temp file seeker: %s", err)
}
}()
}
// probe the video file to extract useful metadata from it; for methodology, see:
// https://github.com/abema/go-mp4/blob/7d8e5a7c5e644e0394261b0cf72fef79ce246d31/mp4tool/probe/probe.go#L85-L154
info, err := mp4.Probe(rsc)
if err != nil {
return nil, fmt.Errorf("error during mp4 probe: %w", err)
}
var (
width int
height int
videoBitrate uint64
audioBitrate uint64
video gtsVideo
)
for _, tr := range info.Tracks {
if tr.AVC == nil {
// audio track
if br := tr.Samples.GetBitrate(tr.Timescale); br > audioBitrate {
audioBitrate = br
} else if br := info.Segments.GetBitrate(tr.TrackID, tr.Timescale); br > audioBitrate {
audioBitrate = br
}
if d := float64(tr.Duration) / float64(tr.Timescale); d > float64(video.duration) {
video.duration = float32(d)
}
continue
}
// video track
if w := int(tr.AVC.Width); w > width {
width = w
}
if h := int(tr.AVC.Height); h > height {
height = h
}
if br := tr.Samples.GetBitrate(tr.Timescale); br > videoBitrate {
videoBitrate = br
} else if br := info.Segments.GetBitrate(tr.TrackID, tr.Timescale); br > videoBitrate {
videoBitrate = br
}
if d := float64(tr.Duration) / float64(tr.Timescale); d > float64(video.duration) {
video.framerate = float32(len(tr.Samples)) / float32(d)
video.duration = float32(d)
}
}
// overall bitrate should be audio + video combined
// (since they're both playing at the same time)
video.bitrate = audioBitrate + videoBitrate
// Check for empty video metadata.
var empty []string
if width == 0 {
empty = append(empty, "width")
}
if height == 0 {
empty = append(empty, "height")
}
if video.duration == 0 {
empty = append(empty, "duration")
}
if video.framerate == 0 {
empty = append(empty, "framerate")
}
if video.bitrate == 0 {
empty = append(empty, "bitrate")
}
if len(empty) > 0 {
return nil, fmt.Errorf("error determining video metadata: %v", empty)
}
// Create new empty "frame" image.
// TODO: decode frame from video file.
video.frame = blankImage(width, height)
return &video, nil
}

50
internal/processing/account/update.go
View file

@ -24,7 +24,7 @@ import (
"io"
"mime/multipart"
"codeberg.org/gruf/go-bytesize"
"codeberg.org/gruf/go-iotools"
"github.com/superseriousbusiness/gotosocial/internal/ap"
apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model"
"github.com/superseriousbusiness/gotosocial/internal/config"
@ -365,21 +365,31 @@ func (p *Processor) UpdateAvatar(
*gtsmodel.MediaAttachment,
gtserror.WithCode,
) {
max := config.GetMediaImageMaxSize()
if sz := bytesize.Size(avatar.Size); sz > max {
text := fmt.Sprintf("size %s exceeds max media size %s", sz, max)
// Get maximum supported local media size.
maxsz := config.GetMediaLocalMaxSize()
// Ensure media within size bounds.
if avatar.Size > int64(maxsz) {
text := fmt.Sprintf("media exceeds configured max size: %s", maxsz)
return nil, gtserror.NewErrorBadRequest(errors.New(text), text)
}
data := func(_ context.Context) (io.ReadCloser, int64, error) {
f, err := avatar.Open()
return f, avatar.Size, err
// Open multipart file reader.
mpfile, err := avatar.Open()
if err != nil {
err := gtserror.Newf("error opening multipart file: %w", err)
return nil, gtserror.NewErrorInternalError(err)
}
// Wrap the multipart file reader to ensure is limited to max.
rc, _, _ := iotools.UpdateReadCloserLimit(mpfile, int64(maxsz))
// Write to instance storage.
return p.c.StoreLocalMedia(ctx,
account.ID,
data,
func(ctx context.Context) (reader io.ReadCloser, err error) {
return rc, nil
},
media.AdditionalMediaInfo{
Avatar: util.Ptr(true),
Description: description,
@ -400,21 +410,31 @@ func (p *Processor) UpdateHeader(
*gtsmodel.MediaAttachment,
gtserror.WithCode,
) {
max := config.GetMediaImageMaxSize()
if sz := bytesize.Size(header.Size); sz > max {
text := fmt.Sprintf("size %s exceeds max media size %s", sz, max)
// Get maximum supported local media size.
maxsz := config.GetMediaLocalMaxSize()
// Ensure media within size bounds.
if header.Size > int64(maxsz) {
text := fmt.Sprintf("media exceeds configured max size: %s", maxsz)
return nil, gtserror.NewErrorBadRequest(errors.New(text), text)
}
data := func(_ context.Context) (io.ReadCloser, int64, error) {
f, err := header.Open()
return f, header.Size, err
// Open multipart file reader.
mpfile, err := header.Open()
if err != nil {
err := gtserror.Newf("error opening multipart file: %w", err)
return nil, gtserror.NewErrorInternalError(err)
}
// Wrap the multipart file reader to ensure is limited to max.
rc, _, _ := iotools.UpdateReadCloserLimit(mpfile, int64(maxsz))
// Write to instance storage.
return p.c.StoreLocalMedia(ctx,
account.ID,
data,
func(ctx context.Context) (reader io.ReadCloser, err error) {
return rc, nil
},
media.AdditionalMediaInfo{
Header: util.Ptr(true),
Description: description,

64
internal/processing/admin/emoji.go
View file

@ -25,7 +25,10 @@ import (
"mime/multipart"
"strings"
"codeberg.org/gruf/go-bytesize"
"codeberg.org/gruf/go-iotools"
apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model"
"github.com/superseriousbusiness/gotosocial/internal/config"
"github.com/superseriousbusiness/gotosocial/internal/db"
"github.com/superseriousbusiness/gotosocial/internal/gtserror"
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
@ -41,10 +44,26 @@ func (p *Processor) EmojiCreate(
form *apimodel.EmojiCreateRequest,
) (*apimodel.Emoji, gtserror.WithCode) {
// Simply read provided form data for emoji data source.
data := func(_ context.Context) (io.ReadCloser, int64, error) {
f, err := form.Image.Open()
return f, form.Image.Size, err
// Get maximum supported local emoji size.
maxsz := config.GetMediaEmojiLocalMaxSize()
// Ensure media within size bounds.
if form.Image.Size > int64(maxsz) {
text := fmt.Sprintf("emoji exceeds configured max size: %s", maxsz)
return nil, gtserror.NewErrorBadRequest(errors.New(text), text)
}
// Open multipart file reader.
mpfile, err := form.Image.Open()
if err != nil {
err := gtserror.Newf("error opening multipart file: %w", err)
return nil, gtserror.NewErrorInternalError(err)
}
// Wrap the multipart file reader to ensure is limited to max.
rc, _, _ := iotools.UpdateReadCloserLimit(mpfile, int64(maxsz))
data := func(context.Context) (io.ReadCloser, error) {
return rc, nil
}
// Attempt to create the new local emoji.
@ -285,14 +304,23 @@ func (p *Processor) emojiUpdateCopy(
return nil, gtserror.NewErrorNotFound(err)
}
// Get maximum supported local emoji size.
maxsz := config.GetMediaEmojiLocalMaxSize()
// Ensure target emoji image within size bounds.
if bytesize.Size(target.ImageFileSize) > maxsz {
text := fmt.Sprintf("emoji exceeds configured max size: %s", maxsz)
return nil, gtserror.NewErrorBadRequest(errors.New(text), text)
}
// Data function for copying just streams media
// out of storage into an additional location.
//
// This means that data for the copy persists even
// if the remote copied emoji gets deleted at some point.
data := func(ctx context.Context) (io.ReadCloser, int64, error) {
data := func(ctx context.Context) (io.ReadCloser, error) {
rc, err := p.state.Storage.GetStream(ctx, target.ImagePath)
return rc, int64(target.ImageFileSize), err
return rc, err
}
// Attempt to create the new local emoji.
@ -413,10 +441,26 @@ func (p *Processor) emojiUpdateModify(
// Updating image and maybe categoryID.
// We can do both at the same time :)
// Simply read provided form data for emoji data source.
data := func(_ context.Context) (io.ReadCloser, int64, error) {
f, err := image.Open()
return f, image.Size, err
// Get maximum supported local emoji size.
maxsz := config.GetMediaEmojiLocalMaxSize()
// Ensure media within size bounds.
if image.Size > int64(maxsz) {
text := fmt.Sprintf("emoji exceeds configured max size: %s", maxsz)
return nil, gtserror.NewErrorBadRequest(errors.New(text), text)
}
// Open multipart file reader.
mpfile, err := image.Open()
if err != nil {
err := gtserror.Newf("error opening multipart file: %w", err)
return nil, gtserror.NewErrorInternalError(err)
}
// Wrap the multipart file reader to ensure is limited to max.
rc, _, _ := iotools.UpdateReadCloserLimit(mpfile, int64(maxsz))
data := func(context.Context) (io.ReadCloser, error) {
return rc, nil
}
// Prepare emoji model for recache from new data.

4
internal/processing/admin/media.go
View file

@ -21,6 +21,7 @@ import (
"context"
"fmt"
"github.com/superseriousbusiness/gotosocial/internal/gtscontext"
"github.com/superseriousbusiness/gotosocial/internal/gtserror"
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
"github.com/superseriousbusiness/gotosocial/internal/log"
@ -35,8 +36,9 @@ func (p *Processor) MediaRefetch(ctx context.Context, requestingAccount *gtsmode
}
go func() {
ctx := gtscontext.WithValues(context.Background(), ctx)
log.Info(ctx, "starting emoji refetch")
refetched, err := p.media.RefetchEmojis(context.Background(), domain, transport.DereferenceMedia)
refetched, err := p.media.RefetchEmojis(ctx, domain, transport.DereferenceMedia)
if err != nil {
log.Errorf(ctx, "error refetching emojis: %s", err)
} else {

33
internal/processing/media/create.go
View file

@ -19,10 +19,13 @@ package media
import (
"context"
"errors"
"fmt"
"io"
"codeberg.org/gruf/go-iotools"
apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model"
"github.com/superseriousbusiness/gotosocial/internal/config"
"github.com/superseriousbusiness/gotosocial/internal/gtserror"
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
"github.com/superseriousbusiness/gotosocial/internal/media"
@ -30,21 +33,39 @@ import (
// Create creates a new media attachment belonging to the given account, using the request form.
func (p *Processor) Create(ctx context.Context, account *gtsmodel.Account, form *apimodel.AttachmentRequest) (*apimodel.Attachment, gtserror.WithCode) {
data := func(_ context.Context) (io.ReadCloser, int64, error) {
f, err := form.File.Open()
return f, form.File.Size, err
// Get maximum supported local media size.
maxsz := config.GetMediaLocalMaxSize()
// Ensure media within size bounds.
if form.File.Size > int64(maxsz) {
text := fmt.Sprintf("media exceeds configured max size: %s", maxsz)
return nil, gtserror.NewErrorBadRequest(errors.New(text), text)
}
// Parse focus details from API form input.
focusX, focusY, err := parseFocus(form.Focus)
if err != nil {
err := fmt.Errorf("could not parse focus value %s: %s", form.Focus, err)
return nil, gtserror.NewErrorBadRequest(err, err.Error())
text := fmt.Sprintf("could not parse focus value %s: %s", form.Focus, err)
return nil, gtserror.NewErrorBadRequest(errors.New(text), text)
}
// Open multipart file reader.
mpfile, err := form.File.Open()
if err != nil {
err := gtserror.Newf("error opening multipart file: %w", err)
return nil, gtserror.NewErrorInternalError(err)
}
// Wrap the multipart file reader to ensure is limited to max.
rc, _, _ := iotools.UpdateReadCloserLimit(mpfile, int64(maxsz))
// Create local media and write to instance storage.
attachment, errWithCode := p.c.StoreLocalMedia(ctx,
account.ID,
data,
func(ctx context.Context) (reader io.ReadCloser, err error) {
return rc, nil
},
media.AdditionalMediaInfo{
Description: &form.Description,
FocusX: &focusX,

27
internal/processing/media/getfile_test.go
View file

@ -18,7 +18,6 @@
package media_test
import (
"bytes"
"context"
"io"
"path"
@ -87,9 +86,9 @@ func (suite *GetFileTestSuite) TestGetRemoteFileUncached() {
MediaSize: string(media.SizeOriginal),
FileName: fileName,
})
suite.NoError(errWithCode)
suite.NotNil(content)
b, err := io.ReadAll(content.Content)
suite.NoError(err)
suite.NoError(content.Content.Close())
@ -111,7 +110,7 @@ func (suite *GetFileTestSuite) TestGetRemoteFileUncached() {
suite.True(*dbAttachment.Cached)
// the file should be back in storage at the same path as before
refreshedBytes, err := suite.storage.Get(ctx, testAttachment.File.Path)
refreshedBytes, err := suite.storage.Get(ctx, dbAttachment.File.Path)
suite.NoError(err)
suite.Equal(suite.testRemoteAttachments[testAttachment.RemoteURL].Data, refreshedBytes)
}
@ -139,32 +138,26 @@ func (suite *GetFileTestSuite) TestGetRemoteFileUncachedInterrupted() {
MediaSize: string(media.SizeOriginal),
FileName: fileName,
})
suite.NoError(errWithCode)
suite.NotNil(content)
// only read the first kilobyte and then stop
b := make([]byte, 0, 1024)
if !testrig.WaitFor(func() bool {
read, err := io.CopyN(bytes.NewBuffer(b), content.Content, 1024)
return err == nil && read == 1024
}) {
suite.FailNow("timed out trying to read first 1024 bytes")
}
_, err = io.CopyN(io.Discard, content.Content, 1024)
suite.NoError(err)
// close the reader
suite.NoError(content.Content.Close())
err = content.Content.Close()
suite.NoError(err)
// the attachment should still be updated in the database even though the caller hung up
var dbAttachment *gtsmodel.MediaAttachment
if !testrig.WaitFor(func() bool {
dbAttachment, _ := suite.db.GetAttachmentByID(ctx, testAttachment.ID)
dbAttachment, _ = suite.db.GetAttachmentByID(ctx, testAttachment.ID)
return *dbAttachment.Cached
}) {
suite.FailNow("timed out waiting for attachment to be updated")
}
// the file should be back in storage at the same path as before
refreshedBytes, err := suite.storage.Get(ctx, testAttachment.File.Path)
refreshedBytes, err := suite.storage.Get(ctx, dbAttachment.File.Path)
suite.NoError(err)
suite.Equal(suite.testRemoteAttachments[testAttachment.RemoteURL].Data, refreshedBytes)
}
@ -196,9 +189,9 @@ func (suite *GetFileTestSuite) TestGetRemoteFileThumbnailUncached() {
MediaSize: string(media.SizeSmall),
FileName: fileName,
})
suite.NoError(errWithCode)
suite.NotNil(content)
b, err := io.ReadAll(content.Content)
suite.NoError(err)
suite.NoError(content.Content.Close())

25
internal/storage/storage.go
View file

@ -24,6 +24,7 @@ import (
"io"
"mime"
"net/url"
"os"
"path"
"syscall"
"time"
@ -95,6 +96,30 @@ func (d *Driver) PutStream(ctx context.Context, key string, r io.Reader) (int64,
return d.Storage.WriteStream(ctx, key, r)
}
// PutFile moves the contents of file at path, to storage.Driver{} under given key.
func (d *Driver) PutFile(ctx context.Context, key string, filepath string) (int64, error) {
// Open file at path for reading.
file, err := os.Open(filepath)
if err != nil {
return 0, gtserror.Newf("error opening file %s: %w", filepath, err)
}
// Write the file data to storage under key. Note
// that for disk.DiskStorage{} this should end up
// being a highly optimized Linux sendfile syscall.
sz, err := d.Storage.WriteStream(ctx, key, file)
if err != nil {
err = gtserror.Newf("error writing file %s: %w", key, err)
}
// Close the file: done with it.
if e := file.Close(); e != nil {
log.Errorf(ctx, "error closing file %s: %v", filepath, e)
}
return sz, err
}
// Delete attempts to remove the supplied key (and corresponding value) from storage.
func (d *Driver) Delete(ctx context.Context, key string) error {
return d.Storage.Remove(ctx, key)

22
internal/transport/derefmedia.go
View file

@ -23,30 +23,42 @@ import (
"net/http"
"net/url"
"codeberg.org/gruf/go-bytesize"
"codeberg.org/gruf/go-iotools"
"github.com/superseriousbusiness/gotosocial/internal/gtserror"
)
func (t *transport) DereferenceMedia(ctx context.Context, iri *url.URL) (io.ReadCloser, int64, error) {
func (t *transport) DereferenceMedia(ctx context.Context, iri *url.URL, maxsz int64) (io.ReadCloser, error) {
// Build IRI just once
iriStr := iri.String()
// Prepare HTTP request to this media's IRI
req, err := http.NewRequestWithContext(ctx, "GET", iriStr, nil)
if err != nil {
return nil, 0, err
return nil, err
}
req.Header.Add("Accept", "*/*") // we don't know what kind of media we're going to get here
// Perform the HTTP request
rsp, err := t.GET(req)
if err != nil {
return nil, 0, err
return nil, err
}
// Check for an expected status code
if rsp.StatusCode != http.StatusOK {
return nil, 0, gtserror.NewFromResponse(rsp)
return nil, gtserror.NewFromResponse(rsp)
}
return rsp.Body, rsp.ContentLength, nil
// Check media within size limit.
if rsp.ContentLength > maxsz {
_ = rsp.Body.Close() // close early.
sz := bytesize.Size(maxsz) // nicer log format
return nil, gtserror.Newf("media body exceeds max size %s", sz)
}
// Update response body with maximum supported media size.
rsp.Body, _, _ = iotools.UpdateReadCloserLimit(rsp.Body, maxsz)
return rsp.Body, nil
}

4
internal/transport/transport.go
View file

@ -67,8 +67,8 @@ type Transport interface {
// Dereference fetches the ActivityStreams object located at this IRI with a GET request.
Dereference(ctx context.Context, iri *url.URL) (*http.Response, error)
// DereferenceMedia fetches the given media attachment IRI, returning the reader and filesize.
DereferenceMedia(ctx context.Context, iri *url.URL) (io.ReadCloser, int64, error)
// DereferenceMedia fetches the given media attachment IRI, returning the reader limited to given max.
DereferenceMedia(ctx context.Context, iri *url.URL, maxsz int64) (io.ReadCloser, error)
// DereferenceInstance dereferences remote instance information, first by checking /api/v1/instance, and then by checking /.well-known/nodeinfo.
DereferenceInstance(ctx context.Context, iri *url.URL) (*gtsmodel.Instance, error)

8
internal/typeutils/internaltofrontend.go
View file

@ -1385,9 +1385,9 @@ func (c *Converter) InstanceToAPIV1Instance(ctx context.Context, i *gtsmodel.Ins
instance.Configuration.Statuses.CharactersReservedPerURL = instanceStatusesCharactersReservedPerURL
instance.Configuration.Statuses.SupportedMimeTypes = instanceStatusesSupportedMimeTypes
instance.Configuration.MediaAttachments.SupportedMimeTypes = media.SupportedMIMETypes
instance.Configuration.MediaAttachments.ImageSizeLimit = int(config.GetMediaImageMaxSize())
instance.Configuration.MediaAttachments.ImageSizeLimit = int(config.GetMediaRemoteMaxSize())
instance.Configuration.MediaAttachments.ImageMatrixLimit = instanceMediaAttachmentsImageMatrixLimit
instance.Configuration.MediaAttachments.VideoSizeLimit = int(config.GetMediaVideoMaxSize())
instance.Configuration.MediaAttachments.VideoSizeLimit = int(config.GetMediaRemoteMaxSize())
instance.Configuration.MediaAttachments.VideoFrameRateLimit = instanceMediaAttachmentsVideoFrameRateLimit
instance.Configuration.MediaAttachments.VideoMatrixLimit = instanceMediaAttachmentsVideoMatrixLimit
instance.Configuration.Polls.MaxOptions = config.GetStatusesPollMaxOptions()
@ -1525,9 +1525,9 @@ func (c *Converter) InstanceToAPIV2Instance(ctx context.Context, i *gtsmodel.Ins
instance.Configuration.Statuses.CharactersReservedPerURL = instanceStatusesCharactersReservedPerURL
instance.Configuration.Statuses.SupportedMimeTypes = instanceStatusesSupportedMimeTypes
instance.Configuration.MediaAttachments.SupportedMimeTypes = media.SupportedMIMETypes
instance.Configuration.MediaAttachments.ImageSizeLimit = int(config.GetMediaImageMaxSize())
instance.Configuration.MediaAttachments.ImageSizeLimit = int(config.GetMediaRemoteMaxSize())
instance.Configuration.MediaAttachments.ImageMatrixLimit = instanceMediaAttachmentsImageMatrixLimit
instance.Configuration.MediaAttachments.VideoSizeLimit = int(config.GetMediaVideoMaxSize())
instance.Configuration.MediaAttachments.VideoSizeLimit = int(config.GetMediaRemoteMaxSize())
instance.Configuration.MediaAttachments.VideoFrameRateLimit = instanceMediaAttachmentsVideoFrameRateLimit
instance.Configuration.MediaAttachments.VideoMatrixLimit = instanceMediaAttachmentsVideoMatrixLimit
instance.Configuration.Polls.MaxOptions = config.GetStatusesPollMaxOptions()

8
internal/typeutils/internaltofrontend_test.go
View file

@ -1217,7 +1217,7 @@ func (suite *InternalToFrontendTestSuite) TestInstanceV1ToFrontend() {
"image/webp",
"video/mp4"
],
"image_size_limit": 10485760,
"image_size_limit": 41943040,
"image_matrix_limit": 16777216,
"video_size_limit": 41943040,
"video_frame_rate_limit": 60,
@ -1342,7 +1342,7 @@ func (suite *InternalToFrontendTestSuite) TestInstanceV2ToFrontend() {
"image/webp",
"video/mp4"
],
"image_size_limit": 10485760,
"image_size_limit": 41943040,
"image_matrix_limit": 16777216,
"video_size_limit": 41943040,
"video_frame_rate_limit": 60,
@ -1433,7 +1433,7 @@ func (suite *InternalToFrontendTestSuite) TestEmojiToFrontendAdmin1() {
"id": "01F8MH9H8E4VG3KDYJR9EGPXCQ",
"disabled": false,
"updated_at": "2021-09-20T10:40:37.000Z",
"total_file_size": 47115,
"total_file_size": 42794,
"content_type": "image/png",
"uri": "http://localhost:8080/emoji/01F8MH9H8E4VG3KDYJR9EGPXCQ"
}`, string(b))
@ -1455,7 +1455,7 @@ func (suite *InternalToFrontendTestSuite) TestEmojiToFrontendAdmin2() {
"disabled": false,
"domain": "fossbros-anonymous.io",
"updated_at": "2020-03-18T12:12:00.000Z",
"total_file_size": 21697,
"total_file_size": 19854,
"content_type": "image/png",
"uri": "http://fossbros-anonymous.io/emoji/01GD5KP5CQEE1R3X43Y1EHS2CW"
}`, string(b))

8
test/envparsing.sh
View file

@ -122,9 +122,9 @@ EXPECT=$(cat << "EOF"
"media-description-min-chars": 69,
"media-emoji-local-max-size": 420,
"media-emoji-remote-max-size": 420,
"media-image-max-size": 420,
"media-local-max-size": 420,
"media-remote-cache-days": 30,
"media-video-max-size": 420,
"media-remote-max-size": 420,
"metrics-auth-enabled": false,
"metrics-auth-password": "",
"metrics-auth-username": "",
@ -233,10 +233,10 @@ GTS_ACCOUNTS_ALLOW_CUSTOM_CSS=true \
GTS_ACCOUNTS_CUSTOM_CSS_LENGTH=5000 \
GTS_ACCOUNTS_REGISTRATION_OPEN=true \
GTS_ACCOUNTS_REASON_REQUIRED=false \
GTS_MEDIA_IMAGE_MAX_SIZE=420 \
GTS_MEDIA_VIDEO_MAX_SIZE=420 \
GTS_MEDIA_DESCRIPTION_MIN_CHARS=69 \
GTS_MEDIA_DESCRIPTION_MAX_CHARS=5000 \
GTS_MEDIA_LOCAL_MAX_SIZE=420 \
GTS_MEDIA_REMOTE_MAX_SIZE=420 \
GTS_MEDIA_REMOTE_CACHE_DAYS=30 \
GTS_MEDIA_EMOJI_LOCAL_MAX_SIZE=420 \
GTS_MEDIA_EMOJI_REMOTE_MAX_SIZE=420 \

20
testrig/config.go
View file

@ -18,6 +18,7 @@
package testrig
import (
"context"
"os"
"strconv"
"time"
@ -26,8 +27,23 @@ import (
"github.com/coreos/go-oidc/v3/oidc"
"github.com/superseriousbusiness/gotosocial/internal/config"
"github.com/superseriousbusiness/gotosocial/internal/language"
"github.com/superseriousbusiness/gotosocial/internal/media/ffmpeg"
)
func init() {
ctx := context.Background()
// Ensure global ffmpeg WASM pool initialized.
if err := ffmpeg.InitFfmpeg(ctx, 1); err != nil {
panic(err)
}
// Ensure global ffmpeg WASM pool initialized.
if err := ffmpeg.InitFfprobe(ctx, 1); err != nil {
panic(err)
}
}
// InitTestConfig initializes viper
// configuration with test defaults.
func InitTestConfig() {
@ -86,11 +102,11 @@ func testDefaults() config.Configuration {
AccountsAllowCustomCSS: true,
AccountsCustomCSSLength: 10000,
MediaImageMaxSize: 10485760, // 10MiB
MediaVideoMaxSize: 41943040, // 40MiB
MediaDescriptionMinChars: 0,
MediaDescriptionMaxChars: 500,
MediaRemoteCacheDays: 7,
MediaLocalMaxSize: 40 * bytesize.MiB,
MediaRemoteMaxSize: 40 * bytesize.MiB,
MediaEmojiLocalMaxSize: 51200, // 50KiB
MediaEmojiRemoteMaxSize: 102400, // 100KiB
MediaCleanupFrom: "00:00", // midnight.

BIN
testrig/media/cowlick-small.jpeg
View file

Binary file not shown.
Side by side Swipe Overlay

Before

Width:  |  Height:  |  Size: 5.1 KiB

After

Width:  |  Height:  |  Size: 6.8 KiB

BIN
testrig/media/kip-static.png
View file

Binary file not shown.
Side by side Swipe Overlay

Before

Width:  |  Height:  |  Size: 802 B

After

Width:  |  Height:  |  Size: 1 KiB

BIN
testrig/media/ohyou-small.jpg Executable file → Normal file
View file

Binary file not shown.
Side by side Swipe Overlay

Before

Width:  |  Height:  |  Size: 6 KiB

After

Width:  |  Height:  |  Size: 7.5 KiB

BIN
testrig/media/rainbow-static.png Executable file → Normal file
View file

Binary file not shown.
Side by side Swipe Overlay

Before

Width:  |  Height:  |  Size: 10 KiB

After

Width:  |  Height:  |  Size: 5.9 KiB

BIN
testrig/media/sloth-small.jpg
View file

Binary file not shown.
Side by side Swipe Overlay

Before

Width:  |  Height:  |  Size: 50 KiB

After

Width:  |  Height:  |  Size: 35 KiB

BIN
testrig/media/team-fortress-small.jpg
View file

Binary file not shown.
Side by side Swipe Overlay

Before

Width:  |  Height:  |  Size: 41 KiB

After

Width:  |  Height:  |  Size: 9.7 KiB

BIN
testrig/media/thoughtsofdog-small.jpg
View file

Binary file not shown.
Side by side Swipe Overlay

Before

Width:  |  Height:  |  Size: 19 KiB

After

Width:  |  Height:  |  Size: 12 KiB

BIN
testrig/media/trent-small.jpg Executable file → Normal file
View file

Binary file not shown.
Side by side Swipe Overlay

Before

Width:  |  Height:  |  Size: 8.6 KiB

After

Width:  |  Height:  |  Size: 9.5 KiB

BIN
testrig/media/welcome-small.jpg Executable file → Normal file
View file

Binary file not shown.
Side by side Swipe Overlay

Before

Width:  |  Height:  |  Size: 6.7 KiB

After

Width:  |  Height:  |  Size: 7.4 KiB

BIN
testrig/media/yell-static.png
View file

Binary file not shown.
Side by side Swipe Overlay

Before

Width:  |  Height:  |  Size: 11 KiB

After

Width:  |  Height:  |  Size: 8.8 KiB

BIN
testrig/media/zork-small.jpg
View file

Binary file not shown.
Side by side Swipe Overlay

Before

Width:  |  Height:  |  Size: 15 KiB

After

Width:  |  Height:  |  Size: 29 KiB

6
testrig/testmodels.go
View file

@ -1028,7 +1028,7 @@ func NewTestAttachments() map[string]*gtsmodel.MediaAttachment {
Thumbnail: gtsmodel.Thumbnail{
Path: "01F8MH5ZK5VRH73AKHQM6Y9VNX/attachment/small/01FVW7RXPQ8YJHTEXYPE7Q8ZY0.jpg",
ContentType: "image/jpeg",
FileSize: 19312,
FileSize: 11751,
URL: "http://localhost:8080/fileserver/01F8MH5ZK5VRH73AKHQM6Y9VNX/attachment/small/01FVW7RXPQ8YJHTEXYPE7Q8ZY0.jpg",
RemoteURL: "http://fossbros-anonymous.io/attachments/small/a499f55b-2d1e-4acd-98d2-1ac2ba6d79b9.jpg",
},
@ -1205,7 +1205,7 @@ func NewTestEmojis() map[string]*gtsmodel.Emoji {
ImageContentType: "image/png",
ImageStaticContentType: "image/png",
ImageFileSize: 36702,
ImageStaticFileSize: 10413,
ImageStaticFileSize: 6092,
Disabled: util.Ptr(false),
URI: "http://localhost:8080/emoji/01F8MH9H8E4VG3KDYJR9EGPXCQ",
VisibleInPicker: util.Ptr(true),
@ -1227,7 +1227,7 @@ func NewTestEmojis() map[string]*gtsmodel.Emoji {
ImageContentType: "image/png",
ImageStaticContentType: "image/png",
ImageFileSize: 10889,
ImageStaticFileSize: 10808,
ImageStaticFileSize: 8965,
Disabled: util.Ptr(false),
URI: "http://fossbros-anonymous.io/emoji/01GD5KP5CQEE1R3X43Y1EHS2CW",
VisibleInPicker: util.Ptr(false),

147
vendor/codeberg.org/superseriousbusiness/exif-terminator/LICENSE → vendor/codeberg.org/gruf/go-ffmpreg/LICENSE generated vendored
View file

@ -1,21 +1,23 @@
GNU AFFERO GENERAL PUBLIC LICENSE
Version 3, 19 November 2007
GNU GENERAL PUBLIC LICENSE
Version 3, 29 June 2007
Copyright (C) 2007 Free Software Foundation, Inc. <http://fsf.org/>
Copyright (C) 2007 Free Software Foundation, Inc. <https://fsf.org/>
Everyone is permitted to copy and distribute verbatim copies
of this license document, but changing it is not allowed.
Preamble
The GNU Affero General Public License is a free, copyleft license for
software and other kinds of works, specifically designed to ensure
cooperation with the community in the case of network server software.
The GNU General Public License is a free, copyleft license for
software and other kinds of works.
The licenses for most software and other practical works are designed
to take away your freedom to share and change the works. By contrast,
our General Public Licenses are intended to guarantee your freedom to
the GNU General Public License is intended to guarantee your freedom to
share and change all versions of a program--to make sure it remains free
software for all its users.
software for all its users. We, the Free Software Foundation, use the
GNU General Public License for most of our software; it applies also to
any other work released this way by its authors. You can apply it to
your programs, too.
When we speak of free software, we are referring to freedom, not
price. Our General Public Licenses are designed to make sure that you
@ -24,34 +26,44 @@ them if you wish), that you receive source code or can get it if you
want it, that you can change the software or use pieces of it in new
free programs, and that you know you can do these things.
Developers that use our General Public Licenses protect your rights
with two steps: (1) assert copyright on the software, and (2) offer
you this License which gives you legal permission to copy, distribute
and/or modify the software.
To protect your rights, we need to prevent others from denying you
these rights or asking you to surrender the rights. Therefore, you have
certain responsibilities if you distribute copies of the software, or if
you modify it: responsibilities to respect the freedom of others.
A secondary benefit of defending all users' freedom is that
improvements made in alternate versions of the program, if they
receive widespread use, become available for other developers to
incorporate. Many developers of free software are heartened and
encouraged by the resulting cooperation. However, in the case of
software used on network servers, this result may fail to come about.
The GNU General Public License permits making a modified version and
letting the public access it on a server without ever releasing its
source code to the public.
For example, if you distribute copies of such a program, whether
gratis or for a fee, you must pass on to the recipients the same
freedoms that you received. You must make sure that they, too, receive
or can get the source code. And you must show them these terms so they
know their rights.
The GNU Affero General Public License is designed specifically to
ensure that, in such cases, the modified source code becomes available
to the community. It requires the operator of a network server to
provide the source code of the modified version running there to the
users of that server. Therefore, public use of a modified version, on
a publicly accessible server, gives the public access to the source
code of the modified version.
Developers that use the GNU GPL protect your rights with two steps:
(1) assert copyright on the software, and (2) offer you this License
giving you legal permission to copy, distribute and/or modify it.
An older license, called the Affero General Public License and
published by Affero, was designed to accomplish similar goals. This is
a different license, not a version of the Affero GPL, but Affero has
released a new version of the Affero GPL which permits relicensing under
this license.
For the developers' and authors' protection, the GPL clearly explains
that there is no warranty for this free software. For both users' and
authors' sake, the GPL requires that modified versions be marked as
changed, so that their problems will not be attributed erroneously to
authors of previous versions.
Some devices are designed to deny users access to install or run
modified versions of the software inside them, although the manufacturer
can do so. This is fundamentally incompatible with the aim of
protecting users' freedom to change the software. The systematic
pattern of such abuse occurs in the area of products for individuals to
use, which is precisely where it is most unacceptable. Therefore, we
have designed this version of the GPL to prohibit the practice for those
products. If such problems arise substantially in other domains, we
stand ready to extend this provision to those domains in future versions
of the GPL, as needed to protect the freedom of users.
Finally, every program is threatened constantly by software patents.
States should not allow patents to restrict development and use of
software on general-purpose computers, but in those that do, we wish to
avoid the special danger that patents applied to a free program could
make it effectively proprietary. To prevent this, the GPL assures that
patents cannot be used to render the program non-free.
The precise terms and conditions for copying, distribution and
modification follow.
@ -60,7 +72,7 @@ modification follow.
0. Definitions.
"This License" refers to version 3 of the GNU Affero General Public License.
"This License" refers to version 3 of the GNU General Public License.
"Copyright" also means copyright-like laws that apply to other kinds of
works, such as semiconductor masks.
@ -537,45 +549,35 @@ to collect a royalty for further conveying from those to whom you convey
the Program, the only way you could satisfy both those terms and this
License would be to refrain entirely from conveying the Program.
13. Remote Network Interaction; Use with the GNU General Public License.
Notwithstanding any other provision of this License, if you modify the
Program, your modified version must prominently offer all users
interacting with it remotely through a computer network (if your version
supports such interaction) an opportunity to receive the Corresponding
Source of your version by providing access to the Corresponding Source
from a network server at no charge, through some standard or customary
means of facilitating copying of software. This Corresponding Source
shall include the Corresponding Source for any work covered by version 3
of the GNU General Public License that is incorporated pursuant to the
following paragraph.
13. Use with the GNU Affero General Public License.
Notwithstanding any other provision of this License, you have
permission to link or combine any covered work with a work licensed
under version 3 of the GNU General Public License into a single
under version 3 of the GNU Affero General Public License into a single
combined work, and to convey the resulting work. The terms of this
License will continue to apply to the part which is the covered work,
but the work with which it is combined will remain governed by version
3 of the GNU General Public License.
but the special requirements of the GNU Affero General Public License,
section 13, concerning interaction through a network will apply to the
combination as such.
14. Revised Versions of this License.
The Free Software Foundation may publish revised and/or new versions of
the GNU Affero General Public License from time to time. Such new versions
will be similar in spirit to the present version, but may differ in detail to
the GNU General Public License from time to time. Such new versions will
be similar in spirit to the present version, but may differ in detail to
address new problems or concerns.
Each version is given a distinguishing version number. If the
Program specifies that a certain numbered version of the GNU Affero General
Program specifies that a certain numbered version of the GNU General
Public License "or any later version" applies to it, you have the
option of following the terms and conditions either of that numbered
version or of any later version published by the Free Software
Foundation. If the Program does not specify a version number of the
GNU Affero General Public License, you may choose any version ever published
GNU General Public License, you may choose any version ever published
by the Free Software Foundation.
If the Program specifies that a proxy can decide which future
versions of the GNU Affero General Public License can be used, that proxy's
versions of the GNU General Public License can be used, that proxy's
public statement of acceptance of a version permanently authorizes you
to choose that version for the Program.
@ -633,29 +635,40 @@ the "copyright" line and a pointer to where the full notice is found.
Copyright (C) <year> <name of author>
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published by
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
GNU General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
Also add information on how to contact you by electronic and paper mail.
If your software can interact with users remotely through a computer
network, you should also make sure that it provides a way for users to
get its source. For example, if your program is a web application, its
interface could display a "Source" link that leads users to an archive
of the code. There are many ways you could offer source, and different
solutions will be better for different programs; see section 13 for the
specific requirements.
If the program does terminal interaction, make it output a short
notice like this when it starts in an interactive mode:
<program> Copyright (C) <year> <name of author>
This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'.
This is free software, and you are welcome to redistribute it
under certain conditions; type `show c' for details.
The hypothetical commands `show w' and `show c' should show the appropriate
parts of the General Public License. Of course, your program's commands
might be different; for a GUI interface, you would use an "about box".
You should also get your employer (if you work as a programmer) or school,
if any, to sign a "copyright disclaimer" for the program, if necessary.
For more information on this, and how to apply and follow the GNU AGPL, see
<http://www.gnu.org/licenses/>.
For more information on this, and how to apply and follow the GNU GPL, see
<https://www.gnu.org/licenses/>.
The GNU General Public License does not permit incorporating your program
into proprietary programs. If your program is a subroutine library, you
may consider it more useful to permit linking proprietary applications with
the library. If this is what you want to do, use the GNU Lesser General
Public License instead of this License. But first, please read
<https://www.gnu.org/licenses/why-not-lgpl.html>.

BIN
vendor/codeberg.org/gruf/go-ffmpreg/embed/ffmpeg/ffmpeg.wasm generated vendored Normal file
View file

Binary file not shown.

38
vendor/codeberg.org/gruf/go-ffmpreg/embed/ffmpeg/lib.go generated vendored Normal file
View file

@ -0,0 +1,38 @@
package ffmpeg
import (
_ "embed"
"os"
"github.com/tetratelabs/wazero/api"
"github.com/tetratelabs/wazero/experimental"
)
func init() {
// Check for WASM source file path.
path := os.Getenv("FFMPEG_WASM")
if path == "" {
return
}
var err error
// Read file into memory.
B, err = os.ReadFile(path)
if err != nil {
panic(err)
}
}
// CoreFeatures is the WebAssembly Core specification
// features this embedded binary was compiled with.
const CoreFeatures = api.CoreFeatureSIMD |
api.CoreFeatureBulkMemoryOperations |
api.CoreFeatureNonTrappingFloatToIntConversion |
api.CoreFeatureMutableGlobal |
api.CoreFeatureReferenceTypes |
api.CoreFeatureSignExtensionOps |
experimental.CoreFeaturesThreads
//go:embed ffmpeg.wasm
var B []byte

BIN
vendor/codeberg.org/gruf/go-ffmpreg/embed/ffprobe/ffprobe.wasm generated vendored Normal file
View file

Binary file not shown.

38
vendor/codeberg.org/gruf/go-ffmpreg/embed/ffprobe/lib.go generated vendored Normal file
View file

@ -0,0 +1,38 @@
package ffprobe
import (
_ "embed"
"os"
"github.com/tetratelabs/wazero/api"
"github.com/tetratelabs/wazero/experimental"
)
func init() {
// Check for WASM source file path.
path := os.Getenv("FFPROBE_WASM")
if path == "" {
return
}
var err error
// Read file into memory.
B, err = os.ReadFile(path)
if err != nil {
panic(err)
}
}
// CoreFeatures is the WebAssembly Core specification
// features this embedded binary was compiled with.
const CoreFeatures = api.CoreFeatureSIMD |
api.CoreFeatureBulkMemoryOperations |
api.CoreFeatureNonTrappingFloatToIntConversion |
api.CoreFeatureMutableGlobal |
api.CoreFeatureReferenceTypes |
api.CoreFeatureSignExtensionOps |
experimental.CoreFeaturesThreads
//go:embed ffprobe.wasm
var B []byte

109
vendor/codeberg.org/gruf/go-ffmpreg/ffmpeg/ffmpeg.go generated vendored Normal file
View file

@ -0,0 +1,109 @@
package ffmpeg
import (
"context"
"codeberg.org/gruf/go-ffmpreg/embed/ffmpeg"
"codeberg.org/gruf/go-ffmpreg/internal"
"codeberg.org/gruf/go-ffmpreg/util"
"codeberg.org/gruf/go-ffmpreg/wasm"
"github.com/tetratelabs/wazero"
"github.com/tetratelabs/wazero/api"
"github.com/tetratelabs/wazero/imports/wasi_snapshot_preview1"
)
// pool of WASM module instances.
var pool = wasm.InstancePool{
Instantiator: wasm.Instantiator{
// WASM module name.
Module: "ffmpeg",
// Per-instance WebAssembly runtime (with shared cache).
Runtime: func(ctx context.Context) wazero.Runtime {
// Prepare config with cache.
cfg := wazero.NewRuntimeConfig()
cfg = cfg.WithCoreFeatures(ffmpeg.CoreFeatures)
cfg = cfg.WithCompilationCache(internal.Cache)
// Instantiate runtime with our config.
rt := wazero.NewRuntimeWithConfig(ctx, cfg)
// Prepare default "env" host module.
env := rt.NewHostModuleBuilder("env")
env = env.NewFunctionBuilder().
WithGoModuleFunction(
api.GoModuleFunc(util.Wasm_Tempnam),
[]api.ValueType{api.ValueTypeI32, api.ValueTypeI32},
[]api.ValueType{api.ValueTypeI32},
).
Export("tempnam")
// Instantiate "env" module in our runtime.
_, err := env.Instantiate(context.Background())
if err != nil {
panic(err)
}
// Instantiate the wasi snapshot preview 1 in runtime.
_, err = wasi_snapshot_preview1.Instantiate(ctx, rt)
if err != nil {
panic(err)
}
return rt
},
// Per-run module configuration.
Config: wazero.NewModuleConfig,
// Embedded WASM.
Source: ffmpeg.B,
},
}
// Precompile ensures at least compiled ffmpeg
// instance is available in the global pool.
func Precompile(ctx context.Context) error {
inst, err := pool.Get(ctx)
if err != nil {
return err
}
pool.Put(inst)
return nil
}
// Get fetches new ffmpeg instance from pool, prefering cached if available.
func Get(ctx context.Context) (*wasm.Instance, error) { return pool.Get(ctx) }
// Put places the given ffmpeg instance in pool.
func Put(inst *wasm.Instance) { pool.Put(inst) }
// Run will run the given args against an ffmpeg instance from pool.
func Run(ctx context.Context, args wasm.Args) (uint32, error) {
inst, err := pool.Get(ctx)
if err != nil {
return 0, err
}
rc, err := inst.Run(ctx, args)
pool.Put(inst)
return rc, err
}
// Cached returns a cached instance (if any) from pool.
func Cached() *wasm.Instance { return pool.Cached() }
// Free drops all instances
// cached in instance pool.
func Free() {
ctx := context.Background()
for {
inst := pool.Cached()
if inst == nil {
return
}
_ = inst.Close(ctx)
}
}

108
vendor/codeberg.org/gruf/go-ffmpreg/ffprobe/ffprobe.go generated vendored Normal file
View file

@ -0,0 +1,108 @@
package ffprobe
import (
"context"
"codeberg.org/gruf/go-ffmpreg/embed/ffprobe"
"codeberg.org/gruf/go-ffmpreg/internal"
"codeberg.org/gruf/go-ffmpreg/util"
"codeberg.org/gruf/go-ffmpreg/wasm"
"github.com/tetratelabs/wazero"
"github.com/tetratelabs/wazero/api"
"github.com/tetratelabs/wazero/imports/wasi_snapshot_preview1"
)
// pool of WASM module instances.
var pool = wasm.InstancePool{
Instantiator: wasm.Instantiator{
// WASM module name.
Module: "ffprobe",
// Per-instance WebAssembly runtime (with shared cache).
Runtime: func(ctx context.Context) wazero.Runtime {
// Prepare config with cache.
cfg := wazero.NewRuntimeConfig()
cfg = cfg.WithCoreFeatures(ffprobe.CoreFeatures)
cfg = cfg.WithCompilationCache(internal.Cache)
// Instantiate runtime with our config.
rt := wazero.NewRuntimeWithConfig(ctx, cfg)
// Prepare default "env" host module.
env := rt.NewHostModuleBuilder("env")
env = env.NewFunctionBuilder().
WithGoModuleFunction(
api.GoModuleFunc(util.Wasm_Tempnam),
[]api.ValueType{api.ValueTypeI32, api.ValueTypeI32},
[]api.ValueType{api.ValueTypeI32},
).
Export("tempnam")
// Instantiate "env" module in our runtime.
_, err := env.Instantiate(context.Background())
if err != nil {
panic(err)
}
// Instantiate the wasi snapshot preview 1 in runtime.
_, err = wasi_snapshot_preview1.Instantiate(ctx, rt)
if err != nil {
panic(err)
}
return rt
},
// Per-run module configuration.
Config: wazero.NewModuleConfig,
// Embedded WASM.
Source: ffprobe.B,
},
}
// Precompile ensures at least compiled ffprobe
// instance is available in the global pool.
func Precompile(ctx context.Context) error {
inst, err := pool.Get(ctx)
if err != nil {
return err
}
pool.Put(inst)
return nil
}
// Get fetches new ffprobe instance from pool, prefering cached if available.
func Get(ctx context.Context) (*wasm.Instance, error) { return pool.Get(ctx) }
// Put places the given ffprobe instance in pool.
func Put(inst *wasm.Instance) { pool.Put(inst) }
// Run will run the given args against an ffprobe instance from pool.
func Run(ctx context.Context, args wasm.Args) (uint32, error) {
inst, err := pool.Get(ctx)
if err != nil {
return 0, err
}
rc, err := inst.Run(ctx, args)
pool.Put(inst)
return rc, err
}
// Cached returns a cached instance (if any) from pool.
func Cached() *wasm.Instance { return pool.Cached() }
// Free drops all instances
// cached in instance pool.
func Free() {
ctx := context.Background()
for {
inst := pool.Cached()
if inst == nil {
return
}
_ = inst.Close(ctx)
}
}

25
vendor/codeberg.org/gruf/go-ffmpreg/internal/wasm.go generated vendored Normal file
View file

@ -0,0 +1,25 @@
package internal
import (
"os"
"github.com/tetratelabs/wazero"
)
func init() {
var err error
if dir := os.Getenv("WAZERO_COMPILATION_CACHE"); dir != "" {
// Use on-filesystem compilation cache given by env.
Cache, err = wazero.NewCompilationCacheWithDir(dir)
if err != nil {
panic(err)
}
} else {
// Use in-memory compilation cache.
Cache = wazero.NewCompilationCache()
}
}
// Shared WASM compilation cache.
var Cache wazero.CompilationCache

65
vendor/codeberg.org/gruf/go-ffmpreg/util/funcs.go generated vendored Normal file
View file

@ -0,0 +1,65 @@
package util
import (
"context"
"os"
"path"
"strconv"
"time"
"github.com/tetratelabs/wazero/api"
)
// Wasm_Tempnam wraps Go_Tempnam to fulfill wazero's api.GoModuleFunc,
// the argument definition is (i32, i32) and return definition is (i32).
// NOTE: the calling module MUST have access to exported malloc / free.
func Wasm_Tempnam(ctx context.Context, mod api.Module, stack []uint64) {
dirptr := api.DecodeU32(stack[0])
pfxptr := api.DecodeU32(stack[1])
dir := readString(ctx, mod, dirptr, 0)
pfx := readString(ctx, mod, pfxptr, 0)
tmpstr := Go_Tempnam(dir, pfx)
tmpptr := writeString(ctx, mod, tmpstr)
stack[0] = api.EncodeU32(tmpptr)
}
// Go_Tempname is functionally similar to C's tempnam.
func Go_Tempnam(dir, prefix string) string {
now := time.Now().Unix()
prefix = path.Join(dir, prefix)
for i := 0; i < 1000; i++ {
n := murmur2(uint32(now + int64(i)))
name := prefix + strconv.FormatUint(uint64(n), 10)
_, err := os.Stat(name)
if err == nil {
continue
} else if os.IsNotExist(err) {
return name
} else {
panic(err)
}
}
panic("too many attempts")
}
// murmur2 is a simple uint32 murmur2 hash
// impl with fixed seed and input size.
func murmur2(k uint32) (h uint32) {
const (
// seed ^ bitlen
s = uint32(2147483647) ^ 8
M = 0x5bd1e995
R = 24
)
h = s
k *= M
k ^= k >> R
k *= M
h *= M
h ^= k
h ^= h >> 13
h *= M
h ^= h >> 15
return
}

81
vendor/codeberg.org/gruf/go-ffmpreg/util/wasm.go generated vendored Normal file
View file

@ -0,0 +1,81 @@
package util
import (
"bytes"
"context"
"github.com/tetratelabs/wazero/api"
)
// NOTE:
// the below functions are not very well optimized
// for repeated calls. this is relying on the fact
// that the only place they get used (tempnam), is
// not called very often, should only be once per run
// so calls to ExportedFunction() and Call() instead
// of caching api.Function and using CallWithStack()
// will work out the same (if only called once).
// maxaddr is the maximum
// wasm32 memory address.
const maxaddr = ^uint32(0)
func malloc(ctx context.Context, mod api.Module, sz uint32) uint32 {
stack, err := mod.ExportedFunction("malloc").Call(ctx, uint64(sz))
if err != nil {
panic(err)
}
ptr := api.DecodeU32(stack[0])
if ptr == 0 {
panic("out of memory")
}
return ptr
}
func free(ctx context.Context, mod api.Module, ptr uint32) {
if ptr != 0 {
mod.ExportedFunction("free").Call(ctx, uint64(ptr))
}
}
func view(ctx context.Context, mod api.Module, ptr uint32, n uint32) []byte {
if n == 0 {
n = maxaddr - ptr
}
mem := mod.Memory()
b, ok := mem.Read(ptr, n)
if !ok {
panic("out of range")
}
return b
}
func read(ctx context.Context, mod api.Module, ptr, n uint32) []byte {
return bytes.Clone(view(ctx, mod, ptr, n))
}
func readString(ctx context.Context, mod api.Module, ptr, n uint32) string {
return string(view(ctx, mod, ptr, n))
}
func write(ctx context.Context, mod api.Module, b []byte) uint32 {
mem := mod.Memory()
len := uint32(len(b))
ptr := malloc(ctx, mod, len)
ok := mem.Write(ptr, b)
if !ok {
panic("out of range")
}
return ptr
}
func writeString(ctx context.Context, mod api.Module, str string) uint32 {
mem := mod.Memory()
len := uint32(len(str) + 1)
ptr := malloc(ctx, mod, len)
ok := mem.WriteString(ptr, str)
if !ok {
panic("out of range")
}
return ptr
}

181
vendor/codeberg.org/gruf/go-ffmpreg/wasm/instance.go generated vendored Normal file
View file

@ -0,0 +1,181 @@
package wasm
import (
"context"
"errors"
"io"
"sync"
"github.com/tetratelabs/wazero"
"github.com/tetratelabs/wazero/sys"
)
type Args struct {
// Standard FDs.
Stdin io.Reader
Stdout io.Writer
Stderr io.Writer
// CLI args.
Args []string
// Optional further module configuration function.
// (e.g. to mount filesystem dir, set env vars, etc).
Config func(wazero.ModuleConfig) wazero.ModuleConfig
}
type Instantiator struct {
// Module ...
Module string
// Runtime ...
Runtime func(context.Context) wazero.Runtime
// Config ...
Config func() wazero.ModuleConfig
// Source ...
Source []byte
}
func (inst *Instantiator) New(ctx context.Context) (*Instance, error) {
switch {
case inst.Module == "":
panic("missing module name")
case inst.Runtime == nil:
panic("missing runtime instantiator")
case inst.Config == nil:
panic("missing module configuration")
case len(inst.Source) == 0:
panic("missing module source")
}
// Create new host runtime.
rt := inst.Runtime(ctx)
// Compile guest module from WebAssembly source.
mod, err := rt.CompileModule(ctx, inst.Source)
if err != nil {
return nil, err
}
return &Instance{
inst: inst,
wzrt: rt,
cmod: mod,
}, nil
}
type InstancePool struct {
Instantiator
pool []*Instance
lock sync.Mutex
}
func (p *InstancePool) Get(ctx context.Context) (*Instance, error) {
for {
// Check for cached.
inst := p.Cached()
if inst == nil {
break
}
// Check if closed.
if inst.IsClosed() {
continue
}
return inst, nil
}
// Must create new instance.
return p.Instantiator.New(ctx)
}
func (p *InstancePool) Put(inst *Instance) {
if inst.inst != &p.Instantiator {
panic("instance and pool instantiators do not match")
}
p.lock.Lock()
p.pool = append(p.pool, inst)
p.lock.Unlock()
}
func (p *InstancePool) Cached() *Instance {
var inst *Instance
p.lock.Lock()
if len(p.pool) > 0 {
inst = p.pool[len(p.pool)-1]
p.pool = p.pool[:len(p.pool)-1]
}
p.lock.Unlock()
return inst
}
// Instance ...
//
// NOTE: Instance is NOT concurrency
// safe. One at a time please!!
type Instance struct {
inst *Instantiator
wzrt wazero.Runtime
cmod wazero.CompiledModule
}
func (inst *Instance) Run(ctx context.Context, args Args) (uint32, error) {
if inst.inst == nil {
panic("not initialized")
}
// Check instance open.
if inst.IsClosed() {
return 0, errors.New("instance closed")
}
// Prefix binary name as argv0 to args.
cargs := make([]string, len(args.Args)+1)
copy(cargs[1:], args.Args)
cargs[0] = inst.inst.Module
// Create base module config.
modcfg := inst.inst.Config()
modcfg = modcfg.WithName(inst.inst.Module)
modcfg = modcfg.WithArgs(cargs...)
modcfg = modcfg.WithStdin(args.Stdin)
modcfg = modcfg.WithStdout(args.Stdout)
modcfg = modcfg.WithStderr(args.Stderr)
if args.Config != nil {
// Pass through config fn.
modcfg = args.Config(modcfg)
}
// Instantiate the module from precompiled wasm module data.
mod, err := inst.wzrt.InstantiateModule(ctx, inst.cmod, modcfg)
if mod != nil {
// Close module.
mod.Close(ctx)
}
// Check for a returned exit code error.
if err, ok := err.(*sys.ExitError); ok {
return err.ExitCode(), nil
}
return 0, err
}
func (inst *Instance) IsClosed() bool {
return (inst.wzrt == nil || inst.cmod == nil)
}
func (inst *Instance) Close(ctx context.Context) error {
if inst.IsClosed() {
return nil
}
err1 := inst.cmod.Close(ctx)
err2 := inst.wzrt.Close(ctx)
return errors.Join(err1, err2)
}

9
vendor/codeberg.org/gruf/go-iotools/close.go generated vendored
View file

@ -2,6 +2,13 @@ package iotools
import "io"
// NopCloser is an empty
// implementation of io.Closer,
// that simply does nothing!
type NopCloser struct{}
func (NopCloser) Close() error { return nil }
// CloserFunc is a function signature which allows
// a function to implement the io.Closer type.
type CloserFunc func() error
@ -10,6 +17,7 @@ func (c CloserFunc) Close() error {
return c()
}
// CloserCallback wraps io.Closer to add a callback deferred to call just after Close().
func CloserCallback(c io.Closer, cb func()) io.Closer {
return CloserFunc(func() error {
defer cb()
@ -17,6 +25,7 @@ func CloserCallback(c io.Closer, cb func()) io.Closer {
})
}
// CloserAfterCallback wraps io.Closer to add a callback called just before Close().
func CloserAfterCallback(c io.Closer, cb func()) io.Closer {
return CloserFunc(func() (err error) {
defer func() { err = c.Close() }()

85
vendor/codeberg.org/gruf/go-iotools/helpers.go generated vendored Normal file
View file

@ -0,0 +1,85 @@
package iotools
import "io"
// AtEOF returns true when reader at EOF,
// this is checked with a 0 length read.
func AtEOF(r io.Reader) bool {
_, err := r.Read(nil)
return (err == io.EOF)
}
// GetReadCloserLimit attempts to cast io.Reader to access its io.LimitedReader with limit.
func GetReaderLimit(r io.Reader) (*io.LimitedReader, int64) {
lr, ok := r.(*io.LimitedReader)
if !ok {
return nil, -1
}
return lr, lr.N
}
// UpdateReaderLimit attempts to update the limit of a reader for existing, newly wrapping if necessary.
func UpdateReaderLimit(r io.Reader, limit int64) (*io.LimitedReader, int64) {
lr, ok := r.(*io.LimitedReader)
if !ok {
lr = &io.LimitedReader{r, limit}
return lr, limit
}
if limit < lr.N {
// Update existing.
lr.N = limit
}
return lr, lr.N
}
// GetReadCloserLimit attempts to unwrap io.ReadCloser to access its io.LimitedReader with limit.
func GetReadCloserLimit(rc io.ReadCloser) (*io.LimitedReader, int64) {
rct, ok := rc.(*ReadCloserType)
if !ok {
return nil, -1
}
lr, ok := rct.Reader.(*io.LimitedReader)
if !ok {
return nil, -1
}
return lr, lr.N
}
// UpdateReadCloserLimit attempts to update the limit of a readcloser for existing, newly wrapping if necessary.
func UpdateReadCloserLimit(rc io.ReadCloser, limit int64) (io.ReadCloser, *io.LimitedReader, int64) {
// Check for our wrapped ReadCloserType.
if rct, ok := rc.(*ReadCloserType); ok {
// Attempt to update existing wrapped limit reader.
if lr, ok := rct.Reader.(*io.LimitedReader); ok {
if limit < lr.N {
// Update existing.
lr.N = limit
}
return rct, lr, lr.N
}
// Wrap the reader type with new limit.
lr := &io.LimitedReader{rct.Reader, limit}
rct.Reader = lr
return rct, lr, lr.N
}
// Wrap separated types.
rct := &ReadCloserType{
Reader: rc,
Closer: rc,
}
// Wrap separated reader part with limit.
lr := &io.LimitedReader{rct.Reader, limit}
rct.Reader = lr
return rct, lr, lr.N
}

21
vendor/codeberg.org/gruf/go-iotools/read.go generated vendored
View file

@ -4,6 +4,16 @@ import (
"io"
)
// ReadCloserType implements io.ReadCloser
// by combining the two underlying interfaces,
// while providing an exported type to still
// access the underlying original io.Reader or
// io.Closer separately (e.g. without wrapping).
type ReadCloserType struct {
io.Reader
io.Closer
}
// ReaderFunc is a function signature which allows
// a function to implement the io.Reader type.
type ReaderFunc func([]byte) (int, error)
@ -22,15 +32,10 @@ func (rf ReaderFromFunc) ReadFrom(r io.Reader) (int64, error) {
// ReadCloser wraps an io.Reader and io.Closer in order to implement io.ReadCloser.
func ReadCloser(r io.Reader, c io.Closer) io.ReadCloser {
return &struct {
io.Reader
io.Closer
}{r, c}
return &ReadCloserType{r, c}
}
// NopReadCloser wraps an io.Reader to implement io.ReadCloser with empty io.Closer implementation.
// NopReadCloser wraps io.Reader with NopCloser{} in ReadCloserType.
func NopReadCloser(r io.Reader) io.ReadCloser {
return ReadCloser(r, CloserFunc(func() error {
return nil
}))
return &ReadCloserType{r, NopCloser{}}
}

25
vendor/codeberg.org/gruf/go-iotools/size.go generated vendored Normal file
View file

@ -0,0 +1,25 @@
package iotools
type Sizer interface {
Size() int64
}
// SizerFunc is a function signature which allows
// a function to implement the Sizer type.
type SizerFunc func() int64
func (s SizerFunc) Size() int64 {
return s()
}
type Lengther interface {
Len() int
}
// LengthFunc is a function signature which allows
// a function to implement the Lengther type.
type LengthFunc func() int
func (l LengthFunc) Len() int {
return l()
}

9
vendor/codeberg.org/gruf/go-iotools/write.go generated vendored
View file

@ -28,7 +28,10 @@ func WriteCloser(w io.Writer, c io.Closer) io.WriteCloser {
// NopWriteCloser wraps an io.Writer to implement io.WriteCloser with empty io.Closer implementation.
func NopWriteCloser(w io.Writer) io.WriteCloser {
return WriteCloser(w, CloserFunc(func() error {
return nil
}))
return &nopWriteCloser{w}
}
// nopWriteCloser implements io.WriteCloser with a no-op Close().
type nopWriteCloser struct{ io.Writer }
func (wc *nopWriteCloser) Close() error { return nil }

5
vendor/codeberg.org/gruf/go-mimetypes/README.md generated vendored Normal file
View file

@ -0,0 +1,5 @@
# go-mimetypes
A generated lookup map of file extensions to mimetypes, from data provided at: https://raw.githubusercontent.com/micnic/mime.json/master/index.json
This allows determining mimetype without relying on OS mimetype lookups.

42
vendor/codeberg.org/gruf/go-mimetypes/get-mime-types.sh generated vendored Normal file
View file

@ -0,0 +1,42 @@
#!/bin/sh
# Mime types JSON source
URL='https://raw.githubusercontent.com/micnic/mime.json/master/index.json'
# Define intro to file
FILE='
// This is an automatically generated file, do not edit
package mimetypes
// MimeTypes is a map of file extensions to mime types.
var MimeTypes = map[string]string{
'
# Set break on new-line
IFS='
'
for line in $(curl -fL "$URL" | grep -E '".+"\s*:\s*".+"'); do
# Trim final whitespace
line=$(echo "$line" | sed -e 's|\s*$||')
# Ensure it ends in a comma
[ "${line%,}" = "$line" ] && line="${line},"
# Add to file
FILE="${FILE}${line}
"
done
# Add final statement to file
FILE="${FILE}
}
"
# Write to file
echo "$FILE" > 'mime.gen.go'
# Check for valid go
gofumpt -w 'mime.gen.go'

1207
vendor/codeberg.org/gruf/go-mimetypes/mime.gen.go generated vendored Normal file
View file

File diff suppressed because it is too large Load diff

47
vendor/codeberg.org/gruf/go-mimetypes/mime.go generated vendored Normal file
View file

@ -0,0 +1,47 @@
package mimetypes
import "path"
// PreferredExts defines preferred file
// extensions for input mime types (as there
// can be multiple extensions per mime type).
var PreferredExts = map[string]string{
MimeTypes["mp3"]: "mp3", // audio/mpeg
MimeTypes["mpeg"]: "mpeg", // video/mpeg
}
// GetForFilename returns mimetype for given filename.
func GetForFilename(filename string) (string, bool) {
ext := path.Ext(filename)
if len(ext) < 1 {
return "", false
}
mime, ok := MimeTypes[ext[1:]]
return mime, ok
}
// GetFileExt returns the file extension to use for mimetype. Relying first upon
// the 'PreferredExts' map. It simply returns the first match there may multiple.
func GetFileExt(mimeType string) (string, bool) {
ext, ok := PreferredExts[mimeType]
if ok {
return ext, true
}
for ext, mime := range MimeTypes {
if mime == mimeType {
return ext, true
}
}
return "", false
}
// GetFileExts returns known file extensions used for mimetype.
func GetFileExts(mimeType string) []string {
var exts []string
for ext, mime := range MimeTypes {
if mime == mimeType {
exts = append(exts, ext)
}
}
return exts
}

Some files were not shown because too many files have changed in this diff Show more