mirror of
https://github.com/superseriousbusiness/gotosocial
synced 2024-11-27 06:40:32 +00:00
07727753b9
* Add whereNotEmptyAndNotNull * Add GetRemoteOlderThanDays * Add GetRemoteOlderThanDays * Add PruneRemote to Manager interface * Start implementing PruneRemote * add new attachment + status to tests * fix up and test GetRemoteOlderThan * fix bad import * PruneRemote: return number pruned * add Cached column to mediaattachment * update + test pruneRemote * update mediaTest * use Cached column * upstep bun to latest version * embed structs in mediaAttachment * migrate mediaAttachment to new format * don't default cached to true * select only remote media * update db dependencies * step bun back to last working version * update pruneRemote to use Cached field * fix storage path of test attachments * add recache logic to manager * fix trimmed aspect ratio * test prune and recache * return errwithcode * tidy up different paths for emoji vs attachment * fix incorrect thumbnail type being stored * expose TransportController to media processor * implement tee-ing recached content * add thoughts of dog to test fedi attachments * test get remote files * add comment on PruneRemote * add postData cleanup to recache * test thumbnail fetching * add incredible diagram * go mod tidy * buffer pipes for recache streaming * test for client stops reading after 1kb * add media-remote-cache-days to config * add cron package * wrap logrus so it's available to cron * start and stop cron jobs gracefully
434 lines
11 KiB
Go
434 lines
11 KiB
Go
package cron
|
|
|
|
import (
|
|
"fmt"
|
|
"math"
|
|
"strconv"
|
|
"strings"
|
|
"time"
|
|
)
|
|
|
|
// Configuration options for creating a parser. Most options specify which
|
|
// fields should be included, while others enable features. If a field is not
|
|
// included the parser will assume a default value. These options do not change
|
|
// the order fields are parse in.
|
|
type ParseOption int
|
|
|
|
const (
|
|
Second ParseOption = 1 << iota // Seconds field, default 0
|
|
SecondOptional // Optional seconds field, default 0
|
|
Minute // Minutes field, default 0
|
|
Hour // Hours field, default 0
|
|
Dom // Day of month field, default *
|
|
Month // Month field, default *
|
|
Dow // Day of week field, default *
|
|
DowOptional // Optional day of week field, default *
|
|
Descriptor // Allow descriptors such as @monthly, @weekly, etc.
|
|
)
|
|
|
|
var places = []ParseOption{
|
|
Second,
|
|
Minute,
|
|
Hour,
|
|
Dom,
|
|
Month,
|
|
Dow,
|
|
}
|
|
|
|
var defaults = []string{
|
|
"0",
|
|
"0",
|
|
"0",
|
|
"*",
|
|
"*",
|
|
"*",
|
|
}
|
|
|
|
// A custom Parser that can be configured.
|
|
type Parser struct {
|
|
options ParseOption
|
|
}
|
|
|
|
// NewParser creates a Parser with custom options.
|
|
//
|
|
// It panics if more than one Optional is given, since it would be impossible to
|
|
// correctly infer which optional is provided or missing in general.
|
|
//
|
|
// Examples
|
|
//
|
|
// // Standard parser without descriptors
|
|
// specParser := NewParser(Minute | Hour | Dom | Month | Dow)
|
|
// sched, err := specParser.Parse("0 0 15 */3 *")
|
|
//
|
|
// // Same as above, just excludes time fields
|
|
// subsParser := NewParser(Dom | Month | Dow)
|
|
// sched, err := specParser.Parse("15 */3 *")
|
|
//
|
|
// // Same as above, just makes Dow optional
|
|
// subsParser := NewParser(Dom | Month | DowOptional)
|
|
// sched, err := specParser.Parse("15 */3")
|
|
//
|
|
func NewParser(options ParseOption) Parser {
|
|
optionals := 0
|
|
if options&DowOptional > 0 {
|
|
optionals++
|
|
}
|
|
if options&SecondOptional > 0 {
|
|
optionals++
|
|
}
|
|
if optionals > 1 {
|
|
panic("multiple optionals may not be configured")
|
|
}
|
|
return Parser{options}
|
|
}
|
|
|
|
// Parse returns a new crontab schedule representing the given spec.
|
|
// It returns a descriptive error if the spec is not valid.
|
|
// It accepts crontab specs and features configured by NewParser.
|
|
func (p Parser) Parse(spec string) (Schedule, error) {
|
|
if len(spec) == 0 {
|
|
return nil, fmt.Errorf("empty spec string")
|
|
}
|
|
|
|
// Extract timezone if present
|
|
var loc = time.Local
|
|
if strings.HasPrefix(spec, "TZ=") || strings.HasPrefix(spec, "CRON_TZ=") {
|
|
var err error
|
|
i := strings.Index(spec, " ")
|
|
eq := strings.Index(spec, "=")
|
|
if loc, err = time.LoadLocation(spec[eq+1 : i]); err != nil {
|
|
return nil, fmt.Errorf("provided bad location %s: %v", spec[eq+1:i], err)
|
|
}
|
|
spec = strings.TrimSpace(spec[i:])
|
|
}
|
|
|
|
// Handle named schedules (descriptors), if configured
|
|
if strings.HasPrefix(spec, "@") {
|
|
if p.options&Descriptor == 0 {
|
|
return nil, fmt.Errorf("parser does not accept descriptors: %v", spec)
|
|
}
|
|
return parseDescriptor(spec, loc)
|
|
}
|
|
|
|
// Split on whitespace.
|
|
fields := strings.Fields(spec)
|
|
|
|
// Validate & fill in any omitted or optional fields
|
|
var err error
|
|
fields, err = normalizeFields(fields, p.options)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
field := func(field string, r bounds) uint64 {
|
|
if err != nil {
|
|
return 0
|
|
}
|
|
var bits uint64
|
|
bits, err = getField(field, r)
|
|
return bits
|
|
}
|
|
|
|
var (
|
|
second = field(fields[0], seconds)
|
|
minute = field(fields[1], minutes)
|
|
hour = field(fields[2], hours)
|
|
dayofmonth = field(fields[3], dom)
|
|
month = field(fields[4], months)
|
|
dayofweek = field(fields[5], dow)
|
|
)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
return &SpecSchedule{
|
|
Second: second,
|
|
Minute: minute,
|
|
Hour: hour,
|
|
Dom: dayofmonth,
|
|
Month: month,
|
|
Dow: dayofweek,
|
|
Location: loc,
|
|
}, nil
|
|
}
|
|
|
|
// normalizeFields takes a subset set of the time fields and returns the full set
|
|
// with defaults (zeroes) populated for unset fields.
|
|
//
|
|
// As part of performing this function, it also validates that the provided
|
|
// fields are compatible with the configured options.
|
|
func normalizeFields(fields []string, options ParseOption) ([]string, error) {
|
|
// Validate optionals & add their field to options
|
|
optionals := 0
|
|
if options&SecondOptional > 0 {
|
|
options |= Second
|
|
optionals++
|
|
}
|
|
if options&DowOptional > 0 {
|
|
options |= Dow
|
|
optionals++
|
|
}
|
|
if optionals > 1 {
|
|
return nil, fmt.Errorf("multiple optionals may not be configured")
|
|
}
|
|
|
|
// Figure out how many fields we need
|
|
max := 0
|
|
for _, place := range places {
|
|
if options&place > 0 {
|
|
max++
|
|
}
|
|
}
|
|
min := max - optionals
|
|
|
|
// Validate number of fields
|
|
if count := len(fields); count < min || count > max {
|
|
if min == max {
|
|
return nil, fmt.Errorf("expected exactly %d fields, found %d: %s", min, count, fields)
|
|
}
|
|
return nil, fmt.Errorf("expected %d to %d fields, found %d: %s", min, max, count, fields)
|
|
}
|
|
|
|
// Populate the optional field if not provided
|
|
if min < max && len(fields) == min {
|
|
switch {
|
|
case options&DowOptional > 0:
|
|
fields = append(fields, defaults[5]) // TODO: improve access to default
|
|
case options&SecondOptional > 0:
|
|
fields = append([]string{defaults[0]}, fields...)
|
|
default:
|
|
return nil, fmt.Errorf("unknown optional field")
|
|
}
|
|
}
|
|
|
|
// Populate all fields not part of options with their defaults
|
|
n := 0
|
|
expandedFields := make([]string, len(places))
|
|
copy(expandedFields, defaults)
|
|
for i, place := range places {
|
|
if options&place > 0 {
|
|
expandedFields[i] = fields[n]
|
|
n++
|
|
}
|
|
}
|
|
return expandedFields, nil
|
|
}
|
|
|
|
var standardParser = NewParser(
|
|
Minute | Hour | Dom | Month | Dow | Descriptor,
|
|
)
|
|
|
|
// ParseStandard returns a new crontab schedule representing the given
|
|
// standardSpec (https://en.wikipedia.org/wiki/Cron). It requires 5 entries
|
|
// representing: minute, hour, day of month, month and day of week, in that
|
|
// order. It returns a descriptive error if the spec is not valid.
|
|
//
|
|
// It accepts
|
|
// - Standard crontab specs, e.g. "* * * * ?"
|
|
// - Descriptors, e.g. "@midnight", "@every 1h30m"
|
|
func ParseStandard(standardSpec string) (Schedule, error) {
|
|
return standardParser.Parse(standardSpec)
|
|
}
|
|
|
|
// getField returns an Int with the bits set representing all of the times that
|
|
// the field represents or error parsing field value. A "field" is a comma-separated
|
|
// list of "ranges".
|
|
func getField(field string, r bounds) (uint64, error) {
|
|
var bits uint64
|
|
ranges := strings.FieldsFunc(field, func(r rune) bool { return r == ',' })
|
|
for _, expr := range ranges {
|
|
bit, err := getRange(expr, r)
|
|
if err != nil {
|
|
return bits, err
|
|
}
|
|
bits |= bit
|
|
}
|
|
return bits, nil
|
|
}
|
|
|
|
// getRange returns the bits indicated by the given expression:
|
|
// number | number "-" number [ "/" number ]
|
|
// or error parsing range.
|
|
func getRange(expr string, r bounds) (uint64, error) {
|
|
var (
|
|
start, end, step uint
|
|
rangeAndStep = strings.Split(expr, "/")
|
|
lowAndHigh = strings.Split(rangeAndStep[0], "-")
|
|
singleDigit = len(lowAndHigh) == 1
|
|
err error
|
|
)
|
|
|
|
var extra uint64
|
|
if lowAndHigh[0] == "*" || lowAndHigh[0] == "?" {
|
|
start = r.min
|
|
end = r.max
|
|
extra = starBit
|
|
} else {
|
|
start, err = parseIntOrName(lowAndHigh[0], r.names)
|
|
if err != nil {
|
|
return 0, err
|
|
}
|
|
switch len(lowAndHigh) {
|
|
case 1:
|
|
end = start
|
|
case 2:
|
|
end, err = parseIntOrName(lowAndHigh[1], r.names)
|
|
if err != nil {
|
|
return 0, err
|
|
}
|
|
default:
|
|
return 0, fmt.Errorf("too many hyphens: %s", expr)
|
|
}
|
|
}
|
|
|
|
switch len(rangeAndStep) {
|
|
case 1:
|
|
step = 1
|
|
case 2:
|
|
step, err = mustParseInt(rangeAndStep[1])
|
|
if err != nil {
|
|
return 0, err
|
|
}
|
|
|
|
// Special handling: "N/step" means "N-max/step".
|
|
if singleDigit {
|
|
end = r.max
|
|
}
|
|
if step > 1 {
|
|
extra = 0
|
|
}
|
|
default:
|
|
return 0, fmt.Errorf("too many slashes: %s", expr)
|
|
}
|
|
|
|
if start < r.min {
|
|
return 0, fmt.Errorf("beginning of range (%d) below minimum (%d): %s", start, r.min, expr)
|
|
}
|
|
if end > r.max {
|
|
return 0, fmt.Errorf("end of range (%d) above maximum (%d): %s", end, r.max, expr)
|
|
}
|
|
if start > end {
|
|
return 0, fmt.Errorf("beginning of range (%d) beyond end of range (%d): %s", start, end, expr)
|
|
}
|
|
if step == 0 {
|
|
return 0, fmt.Errorf("step of range should be a positive number: %s", expr)
|
|
}
|
|
|
|
return getBits(start, end, step) | extra, nil
|
|
}
|
|
|
|
// parseIntOrName returns the (possibly-named) integer contained in expr.
|
|
func parseIntOrName(expr string, names map[string]uint) (uint, error) {
|
|
if names != nil {
|
|
if namedInt, ok := names[strings.ToLower(expr)]; ok {
|
|
return namedInt, nil
|
|
}
|
|
}
|
|
return mustParseInt(expr)
|
|
}
|
|
|
|
// mustParseInt parses the given expression as an int or returns an error.
|
|
func mustParseInt(expr string) (uint, error) {
|
|
num, err := strconv.Atoi(expr)
|
|
if err != nil {
|
|
return 0, fmt.Errorf("failed to parse int from %s: %s", expr, err)
|
|
}
|
|
if num < 0 {
|
|
return 0, fmt.Errorf("negative number (%d) not allowed: %s", num, expr)
|
|
}
|
|
|
|
return uint(num), nil
|
|
}
|
|
|
|
// getBits sets all bits in the range [min, max], modulo the given step size.
|
|
func getBits(min, max, step uint) uint64 {
|
|
var bits uint64
|
|
|
|
// If step is 1, use shifts.
|
|
if step == 1 {
|
|
return ^(math.MaxUint64 << (max + 1)) & (math.MaxUint64 << min)
|
|
}
|
|
|
|
// Else, use a simple loop.
|
|
for i := min; i <= max; i += step {
|
|
bits |= 1 << i
|
|
}
|
|
return bits
|
|
}
|
|
|
|
// all returns all bits within the given bounds. (plus the star bit)
|
|
func all(r bounds) uint64 {
|
|
return getBits(r.min, r.max, 1) | starBit
|
|
}
|
|
|
|
// parseDescriptor returns a predefined schedule for the expression, or error if none matches.
|
|
func parseDescriptor(descriptor string, loc *time.Location) (Schedule, error) {
|
|
switch descriptor {
|
|
case "@yearly", "@annually":
|
|
return &SpecSchedule{
|
|
Second: 1 << seconds.min,
|
|
Minute: 1 << minutes.min,
|
|
Hour: 1 << hours.min,
|
|
Dom: 1 << dom.min,
|
|
Month: 1 << months.min,
|
|
Dow: all(dow),
|
|
Location: loc,
|
|
}, nil
|
|
|
|
case "@monthly":
|
|
return &SpecSchedule{
|
|
Second: 1 << seconds.min,
|
|
Minute: 1 << minutes.min,
|
|
Hour: 1 << hours.min,
|
|
Dom: 1 << dom.min,
|
|
Month: all(months),
|
|
Dow: all(dow),
|
|
Location: loc,
|
|
}, nil
|
|
|
|
case "@weekly":
|
|
return &SpecSchedule{
|
|
Second: 1 << seconds.min,
|
|
Minute: 1 << minutes.min,
|
|
Hour: 1 << hours.min,
|
|
Dom: all(dom),
|
|
Month: all(months),
|
|
Dow: 1 << dow.min,
|
|
Location: loc,
|
|
}, nil
|
|
|
|
case "@daily", "@midnight":
|
|
return &SpecSchedule{
|
|
Second: 1 << seconds.min,
|
|
Minute: 1 << minutes.min,
|
|
Hour: 1 << hours.min,
|
|
Dom: all(dom),
|
|
Month: all(months),
|
|
Dow: all(dow),
|
|
Location: loc,
|
|
}, nil
|
|
|
|
case "@hourly":
|
|
return &SpecSchedule{
|
|
Second: 1 << seconds.min,
|
|
Minute: 1 << minutes.min,
|
|
Hour: all(hours),
|
|
Dom: all(dom),
|
|
Month: all(months),
|
|
Dow: all(dow),
|
|
Location: loc,
|
|
}, nil
|
|
|
|
}
|
|
|
|
const every = "@every "
|
|
if strings.HasPrefix(descriptor, every) {
|
|
duration, err := time.ParseDuration(descriptor[len(every):])
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to parse duration %s: %s", descriptor, err)
|
|
}
|
|
return Every(duration), nil
|
|
}
|
|
|
|
return nil, fmt.Errorf("unrecognized descriptor: %s", descriptor)
|
|
}
|