Feature to autocalibrate the size and word count filters (#30)

README.md

@@ -136,6 +136,7 @@ The only dependency of ffuf is Go 1.11. No dependencies outside of Go standard l
 
 - master
    - New
+      - New CLI flag: -ac to autocalibrate response size and word filters based on few preset URLs.
 
    - Changed
 

main.go

@@ -76,6 +76,7 @@ func main() {
 	flag.BoolVar(&conf.StopOnErrors, "se", false, "Stop on spurious errors")
 	flag.BoolVar(&conf.StopOnAll, "sa", false, "Stop on all error cases. Implies -sf and -se")
 	flag.BoolVar(&conf.FollowRedirects, "r", false, "Follow redirects")
+	flag.BoolVar(&conf.AutoCalibration, "ac", false, "Automatically calibrate filtering options")
 	flag.IntVar(&conf.Threads, "t", 40, "Number of concurrent threads.")
 	flag.BoolVar(&opts.showVersion, "V", false, "Show version information.")
 	flag.Parse()
@@ -100,10 +101,44 @@ func main() {
 		flag.Usage()
 		os.Exit(1)
 	}
+
+	if conf.AutoCalibration {
+		// Handle the calibration
+		responses, err := job.CalibrateResponses()
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "Error in autocalibration, exiting: %s\n", err)
+			os.Exit(1)
+		}
+		if len(responses) > 0 {
+			calibrateFilters(responses, &conf)
+		}
+	}
+
 	// Job handles waiting for goroutines to complete itself
 	job.Start()
 }
 
+func calibrateFilters(responses []ffuf.Response, conf *ffuf.Config) {
+	sizeCalib := make([]string, 0)
+	wordCalib := make([]string, 0)
+	for _, r := range responses {
+		if r.ContentLength > 1 {
+			// Only add if we have an actual size of responses
+			sizeCalib = append(sizeCalib, strconv.FormatInt(r.ContentLength, 10))
+		}
+		if r.ContentWords > 1 {
+			// Only add if we have an actual word length of response
+			wordCalib = append(wordCalib, strconv.FormatInt(r.ContentWords, 10))
+		}
+	}
+	if len(sizeCalib) > 0 {
+		addFilter(conf, "size", strings.Join(sizeCalib, ","))
+	}
+	if len(wordCalib) > 0 {
+		addFilter(conf, "word", strings.Join(wordCalib, ","))
+	}
+}
+
 func prepareJob(conf *ffuf.Config) (*ffuf.Job, error) {
 	errs := ffuf.NewMultierror()
 	// TODO: implement error handling for runnerprovider and outputprovider

pkg/ffuf/config.go

@@ -33,6 +33,7 @@ type Config struct {
 	StopOnErrors    bool
 	StopOnAll       bool
 	FollowRedirects bool
+	AutoCalibration bool
 	Delay           optRange
 	Filters         []FilterProvider
 	Matchers        []FilterProvider

pkg/ffuf/interfaces.go

@@ -26,5 +26,5 @@ type OutputProvider interface {
 	Progress(status string)
 	Error(errstring string)
 	Warning(warnstring string)
-	Result(resp Response) bool
+	Result(resp Response)
 }

pkg/ffuf/job.go

@@ -137,6 +137,62 @@ func (j *Job) updateProgress() {
 	j.Output.Progress(progString)
 }
 
+//Calibrate runs a self-calibration task for filtering options, requesting random resources and acting accordingly
+func (j *Job) CalibrateResponses() ([]Response, error) {
+	cInputs := make([]string, 0)
+	cInputs = append(cInputs, "admin"+randomString(16)+"/")
+	cInputs = append(cInputs, ".htaccess"+randomString(16))
+	cInputs = append(cInputs, randomString(16)+"/")
+	cInputs = append(cInputs, randomString(16))
+
+	results := make([]Response, 0)
+	for _, input := range cInputs {
+		req, err := j.Runner.Prepare([]byte(input))
+		if err != nil {
+			j.Output.Error(fmt.Sprintf("Encountered an error while preparing request: %s\n", err))
+			j.incError()
+			return results, err
+		}
+		resp, err := j.Runner.Execute(&req)
+		if err != nil {
+			return results, err
+		}
+
+		// Only calibrate on responses that would be matched otherwise
+		if j.isMatch(resp) {
+			results = append(results, resp)
+		}
+	}
+	return results, nil
+}
+
+func (j *Job) isMatch(resp Response) bool {
+	matched := false
+	for _, m := range j.Config.Matchers {
+		match, err := m.Filter(&resp)
+		if err != nil {
+			continue
+		}
+		if match {
+			matched = true
+		}
+	}
+	// The response was not matched, return before running filters
+	if !matched {
+		return false
+	}
+	for _, f := range j.Config.Filters {
+		fv, err := f.Filter(&resp)
+		if err != nil {
+			continue
+		}
+		if fv {
+			return false
+		}
+	}
+	return true
+}
+
 func (j *Job) runTask(input []byte, retried bool) {
 	req, err := j.Runner.Prepare(input)
 	if err != nil {
@@ -162,7 +218,8 @@ func (j *Job) runTask(input []byte, retried bool) {
 			j.inc403()
 		}
 	}
-	if j.Output.Result(resp) {
+	if j.isMatch(resp) {
+		j.Output.Result(resp)
 		// Refresh the progress indicator as we printed something out
 		j.updateProgress()
 	}

pkg/ffuf/util.go

@@ -0,0 +1,16 @@
+package ffuf
+
+import (
+	"math/rand"
+)
+
+//used for random string generation in calibration function
+var chars = []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ")
+
+func randomString(n int) string {
+	s := make([]rune, n)
+	for i := range s {
+		s[i] = chars[rand.Intn(len(chars))]
+	}
+	return string(s)
+}

pkg/output/stdout.go

@@ -103,32 +103,10 @@ func (s *Stdoutput) Finalize() error {
 	return nil
 }
 
-func (s *Stdoutput) Result(resp ffuf.Response) bool {
+func (s *Stdoutput) Result(resp ffuf.Response) {
-	matched := false
+	// Output the result
-	for _, m := range s.config.Matchers {
-		match, err := m.Filter(&resp)
-		if err != nil {
-			continue
-		}
-		if match {
-			matched = true
-		}
-	}
-	// The response was not matched, return before running filters
-	if !matched {
-		return false
-	}
-	for _, f := range s.config.Filters {
-		fv, err := f.Filter(&resp)
-		if err != nil {
-			continue
-		}
-		if fv {
-			return false
-		}
-	}
-	// Response survived the filtering, output the result
 	s.printResult(resp)
+	// Check if we need the data later
 	if s.config.OutputFile != "" {
 		// No need to store results if we're not going to use them later
 		sResult := Result{
@@ -139,7 +117,6 @@ func (s *Stdoutput) Result(resp ffuf.Response) bool {
 		}
 		s.Results = append(s.Results, sResult)
 	}
-	return true
 }
 
 func (s *Stdoutput) printResult(resp ffuf.Response) {