Ac rewrite

* Full revamp of filtering, and autocalibration settings.

* Fix concurrency issue in calibration

* Fix linting

CHANGELOG.md

@@ -16,6 +16,7 @@
     - Added full line colors
     - Added `-json` to emit newline delimited JSON output
     - Added 500 Internal Server Error to list of status codes matched by default
+    - New autocalibration options: `-ach`, `-ack` and `-acs`. Revamped the whole autocalibration process
   - Changed
     - Fixed an issue where output file was created regardless of `-or`
     - Fixed an issue where output (often a lot of it) would be printed after entering interactive mode

ffufrc.example

@@ -27,6 +27,9 @@
         "randomtest",
         "admin"
     ]
+    autocalibration_strategy = "basic"
+    autocalibration_keyword = "FUZZ"
+    autocalibration_perhost = false
     colors = false
     delay = ""
     maxtime = 0

help.go

@@ -61,7 +61,7 @@ func Usage() {
 		Description:   "",
 		Flags:         make([]UsageFlag, 0),
 		Hidden:        false,
-		ExpectedFlags: []string{"ac", "acc", "c", "config", "json", "maxtime", "maxtime-job", "noninteractive", "p", "rate", "s", "sa", "se", "sf", "t", "v", "V"},
+		ExpectedFlags: []string{"ac", "acc", "ack", "ach", "acs", "c", "config", "json", "maxtime", "maxtime-job", "noninteractive", "p", "rate", "s", "sa", "se", "sf", "t", "v", "V"},
 	}
 	u_compat := UsageSection{
 		Name:          "COMPATIBILITY OPTIONS",

main.go

@@ -4,13 +4,13 @@ import (
 	"context"
 	"flag"
 	"fmt"
+	"github.com/ffuf/ffuf/pkg/filter"
 	"io/ioutil"
 	"log"
 	"os"
 	"strings"
 
 	"github.com/ffuf/ffuf/pkg/ffuf"
-	"github.com/ffuf/ffuf/pkg/filter"
 	"github.com/ffuf/ffuf/pkg/input"
 	"github.com/ffuf/ffuf/pkg/interactive"
 	"github.com/ffuf/ffuf/pkg/output"
@@ -62,6 +62,7 @@ func ParseFlags(opts *ffuf.ConfigOptions) *ffuf.ConfigOptions {
 	flag.BoolVar(&ignored, "k", false, "Dummy flag for backwards compatibility")
 	flag.BoolVar(&opts.Output.OutputSkipEmptyFile, "or", opts.Output.OutputSkipEmptyFile, "Don't create the output file if we don't have results")
 	flag.BoolVar(&opts.General.AutoCalibration, "ac", opts.General.AutoCalibration, "Automatically calibrate filtering options")
+	flag.BoolVar(&opts.General.AutoCalibrationPerHost, "ach", opts.General.AutoCalibration, "Per host autocalibration")
 	flag.BoolVar(&opts.General.Colors, "c", opts.General.Colors, "Colorize output.")
 	flag.BoolVar(&opts.General.Json, "json", opts.General.Json, "JSON output, printing newline-delimited JSON records")
 	flag.BoolVar(&opts.General.Noninteractive, "noninteractive", opts.General.Noninteractive, "Disable the interactive console functionality")
@@ -84,6 +85,8 @@ func ParseFlags(opts *ffuf.ConfigOptions) *ffuf.ConfigOptions {
 	flag.IntVar(&opts.HTTP.RecursionDepth, "recursion-depth", opts.HTTP.RecursionDepth, "Maximum recursion depth.")
 	flag.IntVar(&opts.HTTP.Timeout, "timeout", opts.HTTP.Timeout, "HTTP request timeout in seconds.")
 	flag.IntVar(&opts.Input.InputNum, "input-num", opts.Input.InputNum, "Number of inputs to test. Used in conjunction with --input-cmd.")
+	flag.StringVar(&opts.General.AutoCalibrationKeyword, "ack", opts.General.AutoCalibrationKeyword, "Autocalibration keyword")
+	flag.StringVar(&opts.General.AutoCalibrationStrategy, "acs", opts.General.AutoCalibrationStrategy, "Autocalibration strategy: \"basic\" or \"advanced\"")
 	flag.StringVar(&opts.General.ConfigFile, "config", "", "Load configuration from a file")
 	flag.StringVar(&opts.Filter.Lines, "fl", opts.Filter.Lines, "Filter by amount of lines in response. Comma separated list of line counts and ranges")
 	flag.StringVar(&opts.Filter.Regexp, "fr", opts.Filter.Regexp, "Filter regexp")
@@ -195,17 +198,13 @@ func main() {
 		fmt.Fprintf(os.Stderr, "Encountered error(s): %s\n", err)
 		os.Exit(1)
 	}
-	if err := filter.SetupFilters(opts, conf); err != nil {
+	if err := SetupFilters(opts, conf); err != nil {
 		fmt.Fprintf(os.Stderr, "Encountered error(s): %s\n", err)
 		Usage()
 		fmt.Fprintf(os.Stderr, "Encountered error(s): %s\n", err)
 		os.Exit(1)
 	}
 
-	if err := filter.CalibrateIfNeeded(job); err != nil {
-		fmt.Fprintf(os.Stderr, "Error in autocalibration, exiting: %s\n", err)
-		os.Exit(1)
-	}
 	if !conf.Noninteractive {
 		go func() {
 			err := interactive.Handle(job)
@@ -233,3 +232,104 @@ func prepareJob(conf *ffuf.Config) (*ffuf.Job, error) {
 	job.Output = output.NewOutputProviderByName("stdout", conf)
 	return job, errs.ErrorOrNil()
 }
+
+func SetupFilters(parseOpts *ffuf.ConfigOptions, conf *ffuf.Config) error {
+	errs := ffuf.NewMultierror()
+	conf.MatcherManager = filter.NewMatcherManager()
+	// If any other matcher is set, ignore -mc default value
+	matcherSet := false
+	statusSet := false
+	warningIgnoreBody := false
+	flag.Visit(func(f *flag.Flag) {
+		if f.Name == "mc" {
+			statusSet = true
+		}
+		if f.Name == "ms" {
+			matcherSet = true
+			warningIgnoreBody = true
+		}
+		if f.Name == "ml" {
+			matcherSet = true
+			warningIgnoreBody = true
+		}
+		if f.Name == "mr" {
+			matcherSet = true
+		}
+		if f.Name == "mt" {
+			matcherSet = true
+		}
+		if f.Name == "mw" {
+			matcherSet = true
+			warningIgnoreBody = true
+		}
+	})
+	// Only set default matchers if no
+	if statusSet || !matcherSet {
+		if err := conf.MatcherManager.AddMatcher("status", parseOpts.Matcher.Status); err != nil {
+			errs.Add(err)
+		}
+	}
+
+	if parseOpts.Filter.Status != "" {
+		if err := conf.MatcherManager.AddFilter("status", parseOpts.Filter.Status, false); err != nil {
+			errs.Add(err)
+		}
+	}
+	if parseOpts.Filter.Size != "" {
+		warningIgnoreBody = true
+		if err := conf.MatcherManager.AddFilter("size", parseOpts.Filter.Size, false); err != nil {
+			errs.Add(err)
+		}
+	}
+	if parseOpts.Filter.Regexp != "" {
+		if err := conf.MatcherManager.AddFilter("regexp", parseOpts.Filter.Regexp, false); err != nil {
+			errs.Add(err)
+		}
+	}
+	if parseOpts.Filter.Words != "" {
+		warningIgnoreBody = true
+		if err := conf.MatcherManager.AddFilter("word", parseOpts.Filter.Words, false); err != nil {
+			errs.Add(err)
+		}
+	}
+	if parseOpts.Filter.Lines != "" {
+		warningIgnoreBody = true
+		if err := conf.MatcherManager.AddFilter("line", parseOpts.Filter.Lines, false); err != nil {
+			errs.Add(err)
+		}
+	}
+	if parseOpts.Filter.Time != "" {
+		if err := conf.MatcherManager.AddFilter("time", parseOpts.Filter.Time, false); err != nil {
+			errs.Add(err)
+		}
+	}
+	if parseOpts.Matcher.Size != "" {
+		if err := conf.MatcherManager.AddMatcher("size", parseOpts.Matcher.Size); err != nil {
+			errs.Add(err)
+		}
+	}
+	if parseOpts.Matcher.Regexp != "" {
+		if err := conf.MatcherManager.AddMatcher("regexp", parseOpts.Matcher.Regexp); err != nil {
+			errs.Add(err)
+		}
+	}
+	if parseOpts.Matcher.Words != "" {
+		if err := conf.MatcherManager.AddMatcher("word", parseOpts.Matcher.Words); err != nil {
+			errs.Add(err)
+		}
+	}
+	if parseOpts.Matcher.Lines != "" {
+		if err := conf.MatcherManager.AddMatcher("line", parseOpts.Matcher.Lines); err != nil {
+			errs.Add(err)
+		}
+	}
+	if parseOpts.Matcher.Time != "" {
+		if err := conf.MatcherManager.AddFilter("time", parseOpts.Matcher.Time, false); err != nil {
+			errs.Add(err)
+		}
+	}
+	if conf.IgnoreBody && warningIgnoreBody {
+		fmt.Printf("*** Warning: possible undesired combination of -ignore-body and the response options: fl,fs,fw,ml,ms and mw.\n")
+	}
+	return errs.ErrorOrNil()
+}

pkg/ffuf/autocalibration.go

@@ -0,0 +1,231 @@
+package ffuf
+
+import (
+	"fmt"
+	"log"
+	"math/rand"
+	"strconv"
+	"time"
+)
+
+func (j *Job) autoCalibrationStrings() map[string][]string {
+	rand.Seed(time.Now().UnixNano())
+	cInputs := make(map[string][]string)
+	if len(j.Config.AutoCalibrationStrings) < 1 {
+		cInputs["basic_admin"] = append(cInputs["basic_admin"], "admin"+RandomString(16))
+		cInputs["basic_admin"] = append(cInputs["basic_admin"], "admin"+RandomString(8))
+		cInputs["htaccess"] = append(cInputs["htaccess"], ".htaccess"+RandomString(16))
+		cInputs["htaccess"] = append(cInputs["htaccess"], ".htaccess"+RandomString(8))
+		cInputs["basic_random"] = append(cInputs["basic_random"], RandomString(16))
+		cInputs["basic_random"] = append(cInputs["basic_random"], RandomString(8))
+		if j.Config.AutoCalibrationStrategy == "advanced" {
+			// Add directory tests and .htaccess too
+			cInputs["admin_dir"] = append(cInputs["admin_dir"], "admin"+RandomString(16)+"/")
+			cInputs["admin_dir"] = append(cInputs["admin_dir"], "admin"+RandomString(8)+"/")
+			cInputs["random_dir"] = append(cInputs["random_dir"], RandomString(16)+"/")
+			cInputs["random_dir"] = append(cInputs["random_dir"], RandomString(8)+"/")
+		}
+	} else {
+		cInputs["custom"] = append(cInputs["custom"], j.Config.AutoCalibrationStrings...)
+	}
+	return cInputs
+}
+
+func (j *Job) calibrationRequest(inputs map[string][]byte) (Response, error) {
+	basereq := BaseRequest(j.Config)
+	req, err := j.Runner.Prepare(inputs, &basereq)
+	if err != nil {
+		j.Output.Error(fmt.Sprintf("Encountered an error while preparing autocalibration request: %s\n", err))
+		j.incError()
+		log.Printf("%s", err)
+		return Response{}, err
+	}
+	resp, err := j.Runner.Execute(&req)
+	if err != nil {
+		j.Output.Error(fmt.Sprintf("Encountered an error while executing autocalibration request: %s\n", err))
+		j.incError()
+		log.Printf("%s", err)
+		return Response{}, err
+	}
+	// Only calibrate on responses that would be matched otherwise
+	if j.isMatch(resp) {
+		return resp, nil
+	}
+	return resp, fmt.Errorf("Response wouldn't be matched")
+}
+
+//CalibrateForHost runs autocalibration for a specific host
+func (j *Job) CalibrateForHost(host string, input map[string][]byte) error {
+	if j.Config.MatcherManager.CalibratedForDomain(host) {
+		return nil
+	}
+	if input[j.Config.AutoCalibrationKeyword] == nil {
+		return fmt.Errorf("Autocalibration keyword \"%s\" not found in the request.", j.Config.AutoCalibrationKeyword)
+	}
+	cStrings := j.autoCalibrationStrings()
+	for _, v := range cStrings {
+		responses := make([]Response, 0)
+		for _, cs := range v {
+			input[j.Config.AutoCalibrationKeyword] = []byte(cs)
+			resp, err := j.calibrationRequest(input)
+			if err != nil {
+				continue
+			}
+			responses = append(responses, resp)
+			err = j.calibrateFilters(responses, true)
+			if err != nil {
+				j.Output.Error(fmt.Sprintf("%s", err))
+			}
+		}
+	}
+	j.Config.MatcherManager.SetCalibratedForHost(host, true)
+	return nil
+}
+
+//CalibrateResponses returns slice of Responses for randomly generated filter autocalibration requests
+func (j *Job) Calibrate(input map[string][]byte) error {
+	if j.Config.MatcherManager.Calibrated() {
+		return nil
+	}
+	cInputs := j.autoCalibrationStrings()
+
+	for _, v := range cInputs {
+		responses := make([]Response, 0)
+		for _, cs := range v {
+			input[j.Config.AutoCalibrationKeyword] = []byte(cs)
+			resp, err := j.calibrationRequest(input)
+			if err != nil {
+				continue
+			}
+			responses = append(responses, resp)
+			err = j.calibrateFilters(responses, false)
+			if err != nil {
+				j.Output.Error(fmt.Sprintf("%s", err))
+			}
+		}
+	}
+	j.Config.MatcherManager.SetCalibrated(true)
+	return nil
+}
+
+//CalibrateIfNeeded runs a self-calibration task for filtering options (if needed) by requesting random resources and
+//  configuring the filters accordingly
+func (j *Job) CalibrateIfNeeded(host string, input map[string][]byte) error {
+	j.calibMutex.Lock()
+	defer j.calibMutex.Unlock()
+	if !j.Config.AutoCalibration {
+		return nil
+	}
+	if j.Config.AutoCalibrationPerHost {
+		return j.CalibrateForHost(host, input)
+	}
+	return j.Calibrate(input)
+}
+
+func (j *Job) calibrateFilters(responses []Response, perHost bool) error {
+	// Work down from the most specific common denominator
+	if len(responses) > 0 {
+		// Content length
+		baselineSize := responses[0].ContentLength
+		sizeMatch := true
+		for _, r := range responses {
+			if baselineSize != r.ContentLength {
+				sizeMatch = false
+			}
+		}
+		if sizeMatch {
+			if perHost {
+				// Check if already filtered
+				for _, f := range j.Config.MatcherManager.FiltersForDomain(responses[0].Request.Host) {
+					match, _ := f.Filter(&responses[0])
+					if match {
+						// Already filtered
+						return nil
+					}
+				}
+				_ = j.Config.MatcherManager.AddPerDomainFilter(responses[0].Request.Host, "size", strconv.FormatInt(baselineSize, 10))
+				return nil
+			} else {
+				// Check if already filtered
+				for _, f := range j.Config.MatcherManager.GetFilters() {
+					match, _ := f.Filter(&responses[0])
+					if match {
+						// Already filtered
+						return nil
+					}
+				}
+				_ = j.Config.MatcherManager.AddFilter("size", strconv.FormatInt(baselineSize, 10), false)
+				return nil
+			}
+		}
+
+		// Content words
+		baselineWords := responses[0].ContentWords
+		wordsMatch := true
+		for _, r := range responses {
+			if baselineWords != r.ContentWords {
+				wordsMatch = false
+			}
+		}
+		if wordsMatch {
+			if perHost {
+				// Check if already filtered
+				for _, f := range j.Config.MatcherManager.FiltersForDomain(responses[0].Request.Host) {
+					match, _ := f.Filter(&responses[0])
+					if match {
+						// Already filtered
+						return nil
+					}
+				}
+				_ = j.Config.MatcherManager.AddPerDomainFilter(responses[0].Request.Host, "word", strconv.FormatInt(baselineWords, 10))
+				return nil
+			} else {
+				// Check if already filtered
+				for _, f := range j.Config.MatcherManager.GetFilters() {
+					match, _ := f.Filter(&responses[0])
+					if match {
+						// Already filtered
+						return nil
+					}
+				}
+				_ = j.Config.MatcherManager.AddFilter("word", strconv.FormatInt(baselineSize, 10), false)
+				return nil
+			}
+		}
+
+		// Content lines
+		baselineLines := responses[0].ContentLines
+		linesMatch := true
+		for _, r := range responses {
+			if baselineLines != r.ContentLines {
+				linesMatch = false
+			}
+		}
+		if linesMatch {
+			if perHost {
+				// Check if already filtered
+				for _, f := range j.Config.MatcherManager.FiltersForDomain(responses[0].Request.Host) {
+					match, _ := f.Filter(&responses[0])
+					if match {
+						// Already filtered
+						return nil
+					}
+				}
+				_ = j.Config.MatcherManager.AddPerDomainFilter(responses[0].Request.Host, "line", strconv.FormatInt(baselineLines, 10))
+				return nil
+			} else {
+				// Check if already filtered
+				for _, f := range j.Config.MatcherManager.GetFilters() {
+					match, _ := f.Filter(&responses[0])
+					if match {
+						// Already filtered
+						return nil
+					}
+				}
+				_ = j.Config.MatcherManager.AddFilter("line", strconv.FormatInt(baselineSize, 10), false)
+				return nil
+			}
+		}
+	}
+	return fmt.Errorf("No common filtering values found")
+}

pkg/ffuf/config.go

@@ -5,54 +5,56 @@ import (
 )
 
 type Config struct {
-	AutoCalibration        bool                      `json:"autocalibration"`
-	AutoCalibrationStrings []string                  `json:"autocalibration_strings"`
-	Cancel                 context.CancelFunc        `json:"-"`
-	Colors                 bool                      `json:"colors"`
-	CommandKeywords        []string                  `json:"-"`
-	CommandLine            string                    `json:"cmdline"`
-	ConfigFile             string                    `json:"configfile"`
-	Context                context.Context           `json:"-"`
-	Data                   string                    `json:"postdata"`
-	Delay                  optRange                  `json:"delay"`
-	DirSearchCompat        bool                      `json:"dirsearch_compatibility"`
-	Extensions             []string                  `json:"extensions"`
-	Filters                map[string]FilterProvider `json:"filters"`
-	FollowRedirects        bool                      `json:"follow_redirects"`
-	Headers                map[string]string         `json:"headers"`
-	IgnoreBody             bool                      `json:"ignorebody"`
-	IgnoreWordlistComments bool                      `json:"ignore_wordlist_comments"`
-	InputMode              string                    `json:"inputmode"`
-	InputNum               int                       `json:"cmd_inputnum"`
-	InputProviders         []InputProviderConfig     `json:"inputproviders"`
-	InputShell             string                    `json:"inputshell"`
-	Json                   bool                      `json:"json"`
-	Matchers               map[string]FilterProvider `json:"matchers"`
-	MaxTime                int                       `json:"maxtime"`
-	MaxTimeJob             int                       `json:"maxtime_job"`
-	Method                 string                    `json:"method"`
-	Noninteractive         bool                      `json:"noninteractive"`
-	OutputDirectory        string                    `json:"outputdirectory"`
-	OutputFile             string                    `json:"outputfile"`
-	OutputFormat           string                    `json:"outputformat"`
-	OutputSkipEmptyFile    bool                      `json:"OutputSkipEmptyFile"`
-	ProgressFrequency      int                       `json:"-"`
-	ProxyURL               string                    `json:"proxyurl"`
-	Quiet                  bool                      `json:"quiet"`
-	Rate                   int64                     `json:"rate"`
-	Recursion              bool                      `json:"recursion"`
-	RecursionDepth         int                       `json:"recursion_depth"`
-	RecursionStrategy      string                    `json:"recursion_strategy"`
-	ReplayProxyURL         string                    `json:"replayproxyurl"`
-	SNI                    string                    `json:"sni"`
-	StopOn403              bool                      `json:"stop_403"`
-	StopOnAll              bool                      `json:"stop_all"`
-	StopOnErrors           bool                      `json:"stop_errors"`
-	Threads                int                       `json:"threads"`
-	Timeout                int                       `json:"timeout"`
-	Url                    string                    `json:"url"`
-	Verbose                bool                      `json:"verbose"`
-	Http2                  bool                      `json:"http2"`
+	AutoCalibration         bool                  `json:"autocalibration"`
+	AutoCalibrationKeyword  string                `json:"autocalibration_keyword"`
+	AutoCalibrationPerHost  bool                  `json:"autocalibration_perhost"`
+	AutoCalibrationStrategy string                `json:"autocalibration_strategy"`
+	AutoCalibrationStrings  []string              `json:"autocalibration_strings"`
+	Cancel                  context.CancelFunc    `json:"-"`
+	Colors                  bool                  `json:"colors"`
+	CommandKeywords         []string              `json:"-"`
+	CommandLine             string                `json:"cmdline"`
+	ConfigFile              string                `json:"configfile"`
+	Context                 context.Context       `json:"-"`
+	Data                    string                `json:"postdata"`
+	Delay                   optRange              `json:"delay"`
+	DirSearchCompat         bool                  `json:"dirsearch_compatibility"`
+	Extensions              []string              `json:"extensions"`
+	FollowRedirects         bool                  `json:"follow_redirects"`
+	Headers                 map[string]string     `json:"headers"`
+	IgnoreBody              bool                  `json:"ignorebody"`
+	IgnoreWordlistComments  bool                  `json:"ignore_wordlist_comments"`
+	InputMode               string                `json:"inputmode"`
+	InputNum                int                   `json:"cmd_inputnum"`
+	InputProviders          []InputProviderConfig `json:"inputproviders"`
+	InputShell              string                `json:"inputshell"`
+	Json                    bool                  `json:"json"`
+	MatcherManager          MatcherManager        `json:"matchers"`
+	MaxTime                 int                   `json:"maxtime"`
+	MaxTimeJob              int                   `json:"maxtime_job"`
+	Method                  string                `json:"method"`
+	Noninteractive          bool                  `json:"noninteractive"`
+	OutputDirectory         string                `json:"outputdirectory"`
+	OutputFile              string                `json:"outputfile"`
+	OutputFormat            string                `json:"outputformat"`
+	OutputSkipEmptyFile     bool                  `json:"OutputSkipEmptyFile"`
+	ProgressFrequency       int                   `json:"-"`
+	ProxyURL                string                `json:"proxyurl"`
+	Quiet                   bool                  `json:"quiet"`
+	Rate                    int64                 `json:"rate"`
+	Recursion               bool                  `json:"recursion"`
+	RecursionDepth          int                   `json:"recursion_depth"`
+	RecursionStrategy       string                `json:"recursion_strategy"`
+	ReplayProxyURL          string                `json:"replayproxyurl"`
+	SNI                     string                `json:"sni"`
+	StopOn403               bool                  `json:"stop_403"`
+	StopOnAll               bool                  `json:"stop_all"`
+	StopOnErrors            bool                  `json:"stop_errors"`
+	Threads                 int                   `json:"threads"`
+	Timeout                 int                   `json:"timeout"`
+	Url                     string                `json:"url"`
+	Verbose                 bool                  `json:"verbose"`
+	Http2                   bool                  `json:"http2"`
 }
 
 type InputProviderConfig struct {
@@ -64,6 +66,8 @@ type InputProviderConfig struct {
 
 func NewConfig(ctx context.Context, cancel context.CancelFunc) Config {
 	var conf Config
+	conf.AutoCalibrationKeyword = "FUZZ"
+	conf.AutoCalibrationStrategy = "basic"
 	conf.AutoCalibrationStrings = make([]string, 0)
 	conf.CommandKeywords = make([]string, 0)
 	conf.Context = ctx
@@ -72,7 +76,6 @@ func NewConfig(ctx context.Context, cancel context.CancelFunc) Config {
 	conf.Delay = optRange{0, 0, false, false}
 	conf.DirSearchCompat = false
 	conf.Extensions = make([]string, 0)
-	conf.Filters = make(map[string]FilterProvider)
 	conf.FollowRedirects = false
 	conf.Headers = make(map[string]string)
 	conf.IgnoreWordlistComments = false
@@ -81,7 +84,6 @@ func NewConfig(ctx context.Context, cancel context.CancelFunc) Config {
 	conf.InputShell = ""
 	conf.InputProviders = make([]InputProviderConfig, 0)
 	conf.Json = false
-	conf.Matchers = make(map[string]FilterProvider)
 	conf.MaxTime = 0
 	conf.MaxTimeJob = 0
 	conf.Method = "GET"

pkg/ffuf/interfaces.go

@@ -2,6 +2,21 @@ package ffuf
 
 import "time"
 
+//MatcherManager provides functions for managing matchers and filters
+type MatcherManager interface {
+	SetCalibrated(calibrated bool)
+	SetCalibratedForHost(host string, calibrated bool)
+	AddFilter(name string, option string, replace bool) error
+	AddPerDomainFilter(domain string, name string, option string) error
+	RemoveFilter(name string)
+	AddMatcher(name string, option string) error
+	GetFilters() map[string]FilterProvider
+	GetMatchers() map[string]FilterProvider
+	FiltersForDomain(domain string) map[string]FilterProvider
+	CalibratedForDomain(domain string) bool
+	Calibrated() bool
+}
+
 //FilterProvider is a generic interface for both Matchers and Filters
 type FilterProvider interface {
 	Filter(response *Response) (bool, error)

pkg/ffuf/job.go

@@ -36,6 +36,7 @@ type Job struct {
 	queuepos             int
 	skipQueue            bool
 	currentDepth         int
+	calibMutex           sync.Mutex
 	pauseWg              sync.WaitGroup
 }
 
@@ -325,7 +326,15 @@ func (j *Job) updateProgress() {
 
 func (j *Job) isMatch(resp Response) bool {
 	matched := false
-	for _, m := range j.Config.Matchers {
+	var matchers map[string]FilterProvider
+	var filters map[string]FilterProvider
+	if j.Config.AutoCalibrationPerHost {
+		filters = j.Config.MatcherManager.FiltersForDomain(resp.Request.Host)
+	} else {
+		filters = j.Config.MatcherManager.GetFilters()
+	}
+	matchers = j.Config.MatcherManager.GetMatchers()
+	for _, m := range matchers {
 		match, err := m.Filter(&resp)
 		if err != nil {
 			continue
@@ -338,7 +347,7 @@ func (j *Job) isMatch(resp Response) bool {
 	if !matched {
 		return false
 	}
-	for _, f := range j.Config.Filters {
+	for _, f := range filters {
 		fv, err := f.Filter(&resp)
 		if err != nil {
 			continue
@@ -360,6 +369,7 @@ func (j *Job) runTask(input map[string][]byte, position int, retried bool) {
 		log.Printf("%s", err)
 		return
 	}
+
 	resp, err := j.Runner.Execute(&req)
 	if err != nil {
 		if retried {
@@ -386,6 +396,10 @@ func (j *Job) runTask(input map[string][]byte, position int, retried bool) {
 		}
 	}
 	j.pauseWg.Wait()
+
+	// Handle autocalibration, must be done after the actual request to ensure sane value in req.Host
+	_ = j.CalibrateIfNeeded(req.Host, input)
+
 	if j.isMatch(resp) {
 		// Re-send request through replay-proxy if needed
 		if j.ReplayRunner != nil {
@@ -444,47 +458,6 @@ func (j *Job) handleDefaultRecursionJob(resp Response) {
 	}
 }
 
-//CalibrateResponses returns slice of Responses for randomly generated filter autocalibration requests
-func (j *Job) CalibrateResponses() ([]Response, error) {
-	basereq := BaseRequest(j.Config)
-	cInputs := make([]string, 0)
-	rand.Seed(time.Now().UnixNano())
-	if len(j.Config.AutoCalibrationStrings) < 1 {
-		cInputs = append(cInputs, "admin"+RandomString(16)+"/")
-		cInputs = append(cInputs, ".htaccess"+RandomString(16))
-		cInputs = append(cInputs, RandomString(16)+"/")
-		cInputs = append(cInputs, RandomString(16))
-	} else {
-		cInputs = append(cInputs, j.Config.AutoCalibrationStrings...)
-	}
-
-	results := make([]Response, 0)
-	for _, input := range cInputs {
-		inputs := make(map[string][]byte, len(j.Config.InputProviders))
-		for _, v := range j.Config.InputProviders {
-			inputs[v.Keyword] = []byte(input)
-		}
-
-		req, err := j.Runner.Prepare(inputs, &basereq)
-		if err != nil {
-			j.Output.Error(fmt.Sprintf("Encountered an error while preparing request: %s\n", err))
-			j.incError()
-			log.Printf("%s", err)
-			return results, err
-		}
-		resp, err := j.Runner.Execute(&req)
-		if err != nil {
-			return results, err
-		}
-
-		// Only calibrate on responses that would be matched otherwise
-		if j.isMatch(resp) {
-			results = append(results, resp)
-		}
-	}
-	return results, nil
-}
-
 // CheckStop stops the job if stopping conditions are met
 func (j *Job) CheckStop() {
 	if j.Counter > 50 {

pkg/ffuf/optionsparser.go

@@ -44,23 +44,26 @@ type HTTPOptions struct {
 }
 
 type GeneralOptions struct {
-	AutoCalibration        bool
-	AutoCalibrationStrings []string
-	Colors                 bool
-	ConfigFile             string `toml:"-"`
-	Delay                  string
-	Json                   bool
-	MaxTime                int
-	MaxTimeJob             int
-	Noninteractive         bool
-	Quiet                  bool
-	Rate                   int
-	ShowVersion            bool `toml:"-"`
-	StopOn403              bool
-	StopOnAll              bool
-	StopOnErrors           bool
-	Threads                int
-	Verbose                bool
+	AutoCalibration         bool
+	AutoCalibrationKeyword  string
+	AutoCalibrationPerHost  bool
+	AutoCalibrationStrategy string
+	AutoCalibrationStrings  []string
+	Colors                  bool
+	ConfigFile              string `toml:"-"`
+	Delay                   string
+	Json                    bool
+	MaxTime                 int
+	MaxTimeJob              int
+	Noninteractive          bool
+	Quiet                   bool
+	Rate                    int
+	ShowVersion             bool `toml:"-"`
+	StopOn403               bool
+	StopOnAll               bool
+	StopOnErrors            bool
+	Threads                 int
+	Verbose                 bool
 }
 
 type InputOptions struct {
@@ -112,6 +115,8 @@ func NewConfigOptions() *ConfigOptions {
 	c.Filter.Time = ""
 	c.Filter.Words = ""
 	c.General.AutoCalibration = false
+	c.General.AutoCalibrationKeyword = "FUZZ"
+	c.General.AutoCalibrationStrategy = "basic"
 	c.General.Colors = false
 	c.General.Delay = ""
 	c.General.Json = false
@@ -445,6 +450,8 @@ func ConfigFromOptions(parseOpts *ConfigOptions, ctx context.Context, cancel con
 	conf.RecursionDepth = parseOpts.HTTP.RecursionDepth
 	conf.RecursionStrategy = parseOpts.HTTP.RecursionStrategy
 	conf.AutoCalibration = parseOpts.General.AutoCalibration
+	conf.AutoCalibrationPerHost = parseOpts.General.AutoCalibrationPerHost
+	conf.AutoCalibrationStrategy = parseOpts.General.AutoCalibrationStrategy
 	conf.Threads = parseOpts.General.Threads
 	conf.Timeout = parseOpts.HTTP.Timeout
 	conf.MaxTime = parseOpts.General.MaxTime
@@ -454,6 +461,11 @@ func ConfigFromOptions(parseOpts *ConfigOptions, ctx context.Context, cancel con
 	conf.Json = parseOpts.General.Json
 	conf.Http2 = parseOpts.HTTP.Http2
 
+	if conf.AutoCalibrationPerHost {
+		// AutoCalibrationPerHost implies AutoCalibration
+		conf.AutoCalibration = true
+	}
+
 	// Handle copy as curl situation where POST method is implied by --data flag. If method is set to anything but GET, NOOP
 	if len(conf.Data) > 0 &&
 		conf.Method == "GET" &&
@@ -557,6 +569,7 @@ func parseRawRequest(parseOpts *ConfigOptions, conf *Config) error {
 	conf.Data = string(b)
 
 	// Remove newline (typically added by the editor) at the end of the file
+	//nolint:gosimple // we specifically want to remove just a single newline, not all of them
 	if strings.HasSuffix(conf.Data, "\r\n") {
 		conf.Data = conf.Data[:len(conf.Data)-2]
 	} else if strings.HasSuffix(conf.Data, "\n") {

pkg/filter/filter.go

@@ -1,14 +1,56 @@
 package filter
 
 import (
-	"flag"
 	"fmt"
-	"strconv"
-	"strings"
-
 	"github.com/ffuf/ffuf/pkg/ffuf"
+	"sync"
 )
 
+// MatcherManager handles both filters and matchers.
+type MatcherManager struct {
+	IsCalibrated     bool
+	Mutex            sync.Mutex
+	Matchers         map[string]ffuf.FilterProvider
+	Filters          map[string]ffuf.FilterProvider
+	PerDomainFilters map[string]*PerDomainFilter
+}
+
+type PerDomainFilter struct {
+	IsCalibrated bool
+	Filters      map[string]ffuf.FilterProvider
+}
+
+func NewPerDomainFilter(globfilters map[string]ffuf.FilterProvider) *PerDomainFilter {
+	return &PerDomainFilter{IsCalibrated: false, Filters: globfilters}
+}
+
+func (p *PerDomainFilter) SetCalibrated(value bool) {
+	p.IsCalibrated = value
+}
+
+func NewMatcherManager() ffuf.MatcherManager {
+	return &MatcherManager{
+		IsCalibrated:     false,
+		Matchers:         make(map[string]ffuf.FilterProvider),
+		Filters:          make(map[string]ffuf.FilterProvider),
+		PerDomainFilters: make(map[string]*PerDomainFilter),
+	}
+}
+
+func (f *MatcherManager) SetCalibrated(value bool) {
+	f.IsCalibrated = value
+}
+
+func (f *MatcherManager) SetCalibratedForHost(host string, value bool) {
+	if f.PerDomainFilters[host] != nil {
+		f.PerDomainFilters[host].IsCalibrated = value
+	} else {
+		newFilter := NewPerDomainFilter(f.Filters)
+		newFilter.IsCalibrated = true
+		f.PerDomainFilters[host] = newFilter
+	}
+}
+
 func NewFilterByName(name string, value string) (ffuf.FilterProvider, error) {
 	if name == "status" {
 		return NewStatusFilter(value)
@@ -31,195 +73,102 @@ func NewFilterByName(name string, value string) (ffuf.FilterProvider, error) {
 	return nil, fmt.Errorf("Could not create filter with name %s", name)
 }
 
-//AddFilter adds a new filter to Config
-func AddFilter(conf *ffuf.Config, name string, option string) error {
+//AddFilter adds a new filter to MatcherManager
+func (f *MatcherManager) AddFilter(name string, option string, replace bool) error {
+	f.Mutex.Lock()
+	defer f.Mutex.Unlock()
 	newf, err := NewFilterByName(name, option)
 	if err == nil {
 		// valid filter create or append
-		if conf.Filters[name] == nil {
-			conf.Filters[name] = newf
+		if f.Filters[name] == nil || replace {
+			f.Filters[name] = newf
 		} else {
-			newoption := conf.Filters[name].Repr() + "," + option
+			newoption := f.Filters[name].Repr() + "," + option
 			newerf, err := NewFilterByName(name, newoption)
 			if err == nil {
-				conf.Filters[name] = newerf
+				f.Filters[name] = newerf
 			}
 		}
 	}
 	return err
 }
 
+//AddPerDomainFilter adds a new filter to PerDomainFilter configuration
+func (f *MatcherManager) AddPerDomainFilter(domain string, name string, option string) error {
+	f.Mutex.Lock()
+	defer f.Mutex.Unlock()
+	var pdFilters *PerDomainFilter
+	if filter, ok := f.PerDomainFilters[domain]; ok {
+		pdFilters = filter
+	} else {
+		pdFilters = NewPerDomainFilter(f.Filters)
+	}
+	newf, err := NewFilterByName(name, option)
+	if err == nil {
+		// valid filter create or append
+		if pdFilters.Filters[name] == nil {
+			pdFilters.Filters[name] = newf
+		} else {
+			newoption := pdFilters.Filters[name].Repr() + "," + option
+			newerf, err := NewFilterByName(name, newoption)
+			if err == nil {
+				pdFilters.Filters[name] = newerf
+			}
+		}
+	}
+	f.PerDomainFilters[domain] = pdFilters
+	return err
+}
+
 //RemoveFilter removes a filter of a given type
-func RemoveFilter(conf *ffuf.Config, name string) {
-	delete(conf.Filters, name)
+func (f *MatcherManager) RemoveFilter(name string) {
+	f.Mutex.Lock()
+	defer f.Mutex.Unlock()
+	delete(f.Filters, name)
 }
 
 //AddMatcher adds a new matcher to Config
-func AddMatcher(conf *ffuf.Config, name string, option string) error {
+func (f *MatcherManager) AddMatcher(name string, option string) error {
+	f.Mutex.Lock()
+	defer f.Mutex.Unlock()
 	newf, err := NewFilterByName(name, option)
 	if err == nil {
-		conf.Matchers[name] = newf
+		// valid filter create or append
+		if f.Matchers[name] == nil {
+			f.Matchers[name] = newf
+		} else {
+			newoption := f.Matchers[name].Repr() + "," + option
+			newerf, err := NewFilterByName(name, newoption)
+			if err == nil {
+				f.Matchers[name] = newerf
+			}
+		}
 	}
 	return err
 }
 
-//CalibrateIfNeeded runs a self-calibration task for filtering options (if needed) by requesting random resources and acting accordingly
-func CalibrateIfNeeded(j *ffuf.Job) error {
-	var err error
-	if !j.Config.AutoCalibration {
-		return nil
-	}
-	// Handle the calibration
-	responses, err := j.CalibrateResponses()
-	if err != nil {
-		return err
-	}
-	if len(responses) > 0 {
-		err = calibrateFilters(j, responses)
-	}
-	return err
+func (f *MatcherManager) GetFilters() map[string]ffuf.FilterProvider {
+	return f.Filters
 }
 
-func calibrateFilters(j *ffuf.Job, responses []ffuf.Response) error {
-	sizeCalib := make([]string, 0)
-	wordCalib := make([]string, 0)
-	lineCalib := make([]string, 0)
-	for _, r := range responses {
-		if r.ContentLength > 0 {
-			// Only add if we have an actual size of responses
-			sizeCalib = append(sizeCalib, strconv.FormatInt(r.ContentLength, 10))
-		}
-		if r.ContentWords > 0 {
-			// Only add if we have an actual word length of response
-			wordCalib = append(wordCalib, strconv.FormatInt(r.ContentWords, 10))
-		}
-		if r.ContentLines > 1 {
-			// Only add if we have an actual word length of response
-			lineCalib = append(lineCalib, strconv.FormatInt(r.ContentLines, 10))
-		}
-	}
-
-	//Remove duplicates
-	sizeCalib = ffuf.UniqStringSlice(sizeCalib)
-	wordCalib = ffuf.UniqStringSlice(wordCalib)
-	lineCalib = ffuf.UniqStringSlice(lineCalib)
-
-	if len(sizeCalib) > 0 {
-		err := AddFilter(j.Config, "size", strings.Join(sizeCalib, ","))
-		if err != nil {
-			return err
-		}
-	}
-	if len(wordCalib) > 0 {
-		err := AddFilter(j.Config, "word", strings.Join(wordCalib, ","))
-		if err != nil {
-			return err
-		}
-	}
-	if len(lineCalib) > 0 {
-		err := AddFilter(j.Config, "line", strings.Join(lineCalib, ","))
-		if err != nil {
-			return err
-		}
-	}
-	return nil
+func (f *MatcherManager) GetMatchers() map[string]ffuf.FilterProvider {
+	return f.Matchers
 }
 
-func SetupFilters(parseOpts *ffuf.ConfigOptions, conf *ffuf.Config) error {
-	errs := ffuf.NewMultierror()
-	// If any other matcher is set, ignore -mc default value
-	matcherSet := false
-	statusSet := false
-	warningIgnoreBody := false
-	flag.Visit(func(f *flag.Flag) {
-		if f.Name == "mc" {
-			statusSet = true
-		}
-		if f.Name == "ms" {
-			matcherSet = true
-			warningIgnoreBody = true
-		}
-		if f.Name == "ml" {
-			matcherSet = true
-			warningIgnoreBody = true
-		}
-		if f.Name == "mr" {
-			matcherSet = true
-		}
-		if f.Name == "mt" {
-			matcherSet = true
-		}
-		if f.Name == "mw" {
-			matcherSet = true
-			warningIgnoreBody = true
-		}
-	})
-	if statusSet || !matcherSet {
-		if err := AddMatcher(conf, "status", parseOpts.Matcher.Status); err != nil {
-			errs.Add(err)
-		}
+func (f *MatcherManager) FiltersForDomain(domain string) map[string]ffuf.FilterProvider {
+	if f.PerDomainFilters[domain] == nil {
+		return f.Filters
 	}
-
-	if parseOpts.Filter.Status != "" {
-		if err := AddFilter(conf, "status", parseOpts.Filter.Status); err != nil {
-			errs.Add(err)
-		}
-	}
-	if parseOpts.Filter.Size != "" {
-		warningIgnoreBody = true
-		if err := AddFilter(conf, "size", parseOpts.Filter.Size); err != nil {
-			errs.Add(err)
-		}
-	}
-	if parseOpts.Filter.Regexp != "" {
-		if err := AddFilter(conf, "regexp", parseOpts.Filter.Regexp); err != nil {
-			errs.Add(err)
-		}
-	}
-	if parseOpts.Filter.Words != "" {
-		warningIgnoreBody = true
-		if err := AddFilter(conf, "word", parseOpts.Filter.Words); err != nil {
-			errs.Add(err)
-		}
-	}
-	if parseOpts.Filter.Lines != "" {
-		warningIgnoreBody = true
-		if err := AddFilter(conf, "line", parseOpts.Filter.Lines); err != nil {
-			errs.Add(err)
-		}
-	}
-	if parseOpts.Filter.Time != "" {
-		if err := AddFilter(conf, "time", parseOpts.Filter.Time); err != nil {
-			errs.Add(err)
-		}
-	}
-	if parseOpts.Matcher.Size != "" {
-		if err := AddMatcher(conf, "size", parseOpts.Matcher.Size); err != nil {
-			errs.Add(err)
-		}
-	}
-	if parseOpts.Matcher.Regexp != "" {
-		if err := AddMatcher(conf, "regexp", parseOpts.Matcher.Regexp); err != nil {
-			errs.Add(err)
-		}
-	}
-	if parseOpts.Matcher.Words != "" {
-		if err := AddMatcher(conf, "word", parseOpts.Matcher.Words); err != nil {
-			errs.Add(err)
-		}
-	}
-	if parseOpts.Matcher.Lines != "" {
-		if err := AddMatcher(conf, "line", parseOpts.Matcher.Lines); err != nil {
-			errs.Add(err)
-		}
-	}
-	if parseOpts.Matcher.Time != "" {
-		if err := AddFilter(conf, "time", parseOpts.Matcher.Time); err != nil {
-			errs.Add(err)
-		}
-	}
-	if conf.IgnoreBody && warningIgnoreBody {
-		fmt.Printf("*** Warning: possible undesired combination of -ignore-body and the response options: fl,fs,fw,ml,ms and mw.\n")
-	}
-	return errs.ErrorOrNil()
+	return f.PerDomainFilters[domain].Filters
+}
+
+func (f *MatcherManager) CalibratedForDomain(domain string) bool {
+	if f.PerDomainFilters[domain] != nil {
+		return f.PerDomainFilters[domain].IsCalibrated
+	}
+	return false
+}
+
+func (f *MatcherManager) Calibrated() bool {
+	return f.IsCalibrated
 }

pkg/interactive/termhandler.go

@@ -8,7 +8,6 @@ import (
 	"time"
 
 	"github.com/ffuf/ffuf/pkg/ffuf"
-	"github.com/ffuf/ffuf/pkg/filter"
 )
 
 type interactive struct {
@@ -81,7 +80,7 @@ func (i *interactive) handleInput(in []byte) {
 			} else if len(args) > 2 {
 				i.Job.Output.Error("Too many arguments for \"fc\"")
 			} else {
-				i.updateFilter("status", args[1])
+				i.updateFilter("status", args[1], true)
 				i.Job.Output.Info("New status code filter value set")
 			}
 		case "afc":
@@ -99,7 +98,7 @@ func (i *interactive) handleInput(in []byte) {
 			} else if len(args) > 2 {
 				i.Job.Output.Error("Too many arguments for \"fl\"")
 			} else {
-				i.updateFilter("line", args[1])
+				i.updateFilter("line", args[1], true)
 				i.Job.Output.Info("New line count filter value set")
 			}
 		case "afl":
@@ -117,7 +116,7 @@ func (i *interactive) handleInput(in []byte) {
 			} else if len(args) > 2 {
 				i.Job.Output.Error("Too many arguments for \"fw\"")
 			} else {
-				i.updateFilter("word", args[1])
+				i.updateFilter("word", args[1], true)
 				i.Job.Output.Info("New word count filter value set")
 			}
 		case "afw":
@@ -135,7 +134,7 @@ func (i *interactive) handleInput(in []byte) {
 			} else if len(args) > 2 {
 				i.Job.Output.Error("Too many arguments for \"fs\"")
 			} else {
-				i.updateFilter("size", args[1])
+				i.updateFilter("size", args[1], true)
 				i.Job.Output.Info("New response size filter value set")
 			}
 		case "afs":
@@ -153,7 +152,7 @@ func (i *interactive) handleInput(in []byte) {
 			} else if len(args) > 2 {
 				i.Job.Output.Error("Too many arguments for \"ft\"")
 			} else {
-				i.updateFilter("time", args[1])
+				i.updateFilter("time", args[1], true)
 				i.Job.Output.Info("New response time filter value set")
 			}
 		case "aft":
@@ -192,19 +191,10 @@ func (i *interactive) handleInput(in []byte) {
 	}
 }
 
-func (i *interactive) updateFilter(name, value string) {
-	if value == "none" {
-		filter.RemoveFilter(i.Job.Config, name)
-	} else {
-		newFc, err := filter.NewFilterByName(name, value)
-		if err != nil {
-			i.Job.Output.Error(fmt.Sprintf("Error while setting new filter value: %s", err))
-			return
-		} else {
-			i.Job.Config.Filters[name] = newFc
-		}
-
-		results := make([]ffuf.Result, 0)
+func (i *interactive) refreshResults() {
+	results := make([]ffuf.Result, 0)
+	filters := i.Job.Config.MatcherManager.GetFilters()
+	for _, filter := range filters {
 		for _, res := range i.Job.Output.GetCurrentResults() {
 			fakeResp := &ffuf.Response{
 				StatusCode:    res.StatusCode,
@@ -212,22 +202,26 @@ func (i *interactive) updateFilter(name, value string) {
 				ContentWords:  res.ContentWords,
 				ContentLength: res.ContentLength,
 			}
-			filterOut, _ := newFc.Filter(fakeResp)
+			filterOut, _ := filter.Filter(fakeResp)
 			if !filterOut {
 				results = append(results, res)
 			}
 		}
-		i.Job.Output.SetCurrentResults(results)
 	}
+	i.Job.Output.SetCurrentResults(results)
+}
+
+func (i *interactive) updateFilter(name, value string, replace bool) {
+	if value == "none" {
+		i.Job.Config.MatcherManager.RemoveFilter(name)
+	} else {
+		_ = i.Job.Config.MatcherManager.AddFilter(name, value, replace)
+	}
+	i.refreshResults()
 }
 
 func (i *interactive) appendFilter(name, value string) {
-	if oldFc, found := i.Job.Config.Filters[name]; found {
-		oldVal := oldFc.Repr()
-		i.updateFilter(name, strings.Join([]string{oldVal, value}, ","))
-	} else {
-		i.updateFilter(name, value)
-	}
+	i.updateFilter(name, value, false)
 }
 
 func (i *interactive) printQueue() {
@@ -270,7 +264,7 @@ func (i *interactive) printPrompt() {
 
 func (i *interactive) printHelp() {
 	var fc, fl, fs, ft, fw string
-	for name, filter := range i.Job.Config.Filters {
+	for name, filter := range i.Job.Config.MatcherManager.GetFilters() {
 		switch name {
 		case "status":
 			fc = "(active: " + filter.Repr() + ")"

pkg/output/stdout.go

@@ -124,11 +124,11 @@ func (s *Stdoutput) Banner() {
 	}
 
 	// Print matchers
-	for _, f := range s.config.Matchers {
+	for _, f := range s.config.MatcherManager.GetMatchers() {
 		printOption([]byte("Matcher"), []byte(f.ReprVerbose()))
 	}
 	// Print filters
-	for _, f := range s.config.Filters {
+	for _, f := range s.config.MatcherManager.GetFilters() {
 		printOption([]byte("Filter"), []byte(f.ReprVerbose()))
 	}
 	fmt.Fprintf(os.Stderr, "%s\n\n", BANNER_SEP)