New input provider --input-cmd (#40)

* New input provider: command

* Set env var and move to Windows and POSIX constants for shell instead of CLI flag.

* Display position instead of input payload when --input-cmd is used

* Update README

* Fix README and flags help

* Add an example to README

README.md

@@ -66,9 +66,32 @@ This is a very straightforward operation, again by using the `FUZZ` keyword. Thi
 ffuf -w /path/to/postdata.txt -X POST -d "username=admin\&password=FUZZ" https://target/login.php -fc 401
 ```
 
+### Using external mutator to produce test cases
+
+For this example, we'll fuzz JSON data that's sent over POST. [Radamsa](https://gitlab.com/akihe/radamsa) is used as the mutator.
+
+When `--input-cmd` is used, ffuf will display matches as their position. This same position value will be available for the callee as an environment variable `$FFUF_NUM`. We'll use this position value as the seed for the mutator. Files example1.txt and example2.txt contain valid JSON payloads. We are matching all the responses, but filtering out response code `400 - Bad request`:
+
+```
+ffuf --input-cmd 'radamsa --seed $FFUF_NUM example1.txt example2.txt' -H "Content-Type: application/json" -X POST -u https://ffuf.io.fi/ -mc all -fc 400
+```
+
+It of course isn't very efficient to call the mutator for each payload, so we can also pre-generate the payloads, still using [Radamsa](https://gitlab.com/akihe/radamsa) as an example:
+
+```
+# Generate 1000 example payloads
+radamsa -n 1000 -o %n.txt example1.txt example2.txt
+
+# This results into files 1.txt ... 1000.txt
+# Now we can just read the payload data in a loop from file for ffuf
+
+ffuf --input-cmd 'cat $FFUF_NUM.txt' -H "Content-Type: application/json" -X POST -u https://ffuf.io.fi/ -mc all -fc 400
+```
+
 ## Usage
 
 To define the test case for ffuf, use the keyword `FUZZ` anywhere in the URL (`-u`), headers (`-H`), or POST data (`-d`).
+
 ```
   -D	DirSearch style wordlist compatibility mode. Used in conjunction with -e flag. Replaces %EXT% in wordlist entry with each of the extensions provided by -e.
   -H "Name: Value"
@@ -79,8 +102,12 @@ To define the test case for ffuf, use the keyword `FUZZ` anywhere in the URL (`-
   -ac
     	Automatically calibrate filtering options
   -c	Colorize output.
+  -compressed
+    	Dummy flag for copy as curl functionality (ignored) (default true)
   -d string
-    	POST data.
+    	POST data
+  -data string
+    	POST data (alias of -d)
   -e string
     	Comma separated list of extensions to apply. Each extension provided will extend the wordlist entry once.
   -fc string
@@ -91,6 +118,10 @@ To define the test case for ffuf, use the keyword `FUZZ` anywhere in the URL (`-
     	Filter HTTP response size
   -fw string
     	Filter by amount of words in response
+  -input-cmd string
+    	Command producing the input. --input-num is required when using this input method. Overrides -w.
+  -input-num int
+    	Number of inputs to test. Used in conjunction with --input-cmd. (default 100)
   -k	TLS identity verification
   -mc string
     	Match HTTP status codes from respose, use "all" to match every response code. (default "200,204,301,302,307,401,403")
@@ -144,6 +175,8 @@ The only dependency of ffuf is Go 1.11. No dependencies outside of Go standard l
       - New CLI flag: -timeout to specify custom timeouts for all HTTP requests.
       - New CLI flag: --data for compatibility with copy as curl functionality of browsers.
       - New CLI flag: --compress, dummy flag that does nothing. for compatibility with copy as curl.
+      - New CLI flags: --input-cmd, and --input-num to handle input generation using external commands. Mutators for example. Environment variable FFUF_NUM will be updated on every call of the command.
+      - When --input-cmd is used, display position instead of the payload in results. The output file (of all formats) will include the payload in addition to the position however.
 
    - Changed
       - Wordlist can also be read from standard input

main.go

@@ -55,7 +55,7 @@ func main() {
 	flag.BoolVar(&conf.DirSearchCompat, "D", false, "DirSearch style wordlist compatibility mode. Used in conjunction with -e flag. Replaces %EXT% in wordlist entry with each of the extensions provided by -e.")
 	flag.Var(&opts.headers, "H", "Header `\"Name: Value\"`, separated by colon. Multiple -H flags are accepted.")
 	flag.StringVar(&conf.Url, "u", "", "Target URL")
-	flag.StringVar(&conf.Wordlist, "w", "", "Wordlist path")
+	flag.StringVar(&conf.Wordlist, "w", "", "Wordlist file path or - to read from standard input")
 	flag.BoolVar(&conf.TLSVerify, "k", false, "TLS identity verification")
 	flag.StringVar(&opts.delay, "p", "", "Seconds of `delay` between requests, or a range of random delay. For example \"0.1\" or \"0.1-2.0\"")
 	flag.StringVar(&opts.filterStatus, "fc", "", "Filter HTTP status codes from response")
@@ -66,6 +66,8 @@ func main() {
 	flag.StringVar(&conf.Data, "data", "", "POST data (alias of -d)")
 	flag.BoolVar(&conf.Colors, "c", false, "Colorize output.")
 	flag.BoolVar(&ignored, "compressed", true, "Dummy flag for copy as curl functionality (ignored)")
+	flag.StringVar(&conf.InputCommand, "input-cmd", "", "Command producing the input. --input-num is required when using this input method. Overrides -w.")
+	flag.IntVar(&conf.InputNum, "input-num", 100, "Number of inputs to test. Used in conjunction with --input-cmd.")
 	flag.StringVar(&opts.matcherStatus, "mc", "200,204,301,302,307,401,403", "Match HTTP status codes from respose, use \"all\" to match every response code.")
 	flag.StringVar(&opts.matcherSize, "ms", "", "Match HTTP response size")
 	flag.StringVar(&opts.matcherRegexp, "mr", "", "Match regexp")
@@ -116,11 +118,17 @@ func main() {
 
 func prepareJob(conf *ffuf.Config) (*ffuf.Job, error) {
 	errs := ffuf.NewMultierror()
+	var err error
+	var inputprovider ffuf.InputProvider
 	// TODO: implement error handling for runnerprovider and outputprovider
 	// We only have http runner right now
 	runprovider := runner.NewRunnerByName("http", conf)
-	// We only have wordlist inputprovider right now
-	inputprovider, err := input.NewInputProviderByName("wordlist", conf)
+	// Initialize the correct inputprovider
+	if len(conf.InputCommand) > 0 {
+		inputprovider, err = input.NewInputProviderByName("command", conf)
+	} else {
+		inputprovider, err = input.NewInputProviderByName("wordlist", conf)
+	}
 	if err != nil {
 		errs.Add(fmt.Errorf("%s", err))
 	}
@@ -189,8 +197,8 @@ func prepareConfig(parseOpts *cliOptions, conf *ffuf.Config) error {
 	if len(conf.Url) == 0 {
 		errs.Add(fmt.Errorf("-u flag is required"))
 	}
-	if len(conf.Wordlist) == 0 {
-		errs.Add(fmt.Errorf("-w flag is required"))
+	if len(conf.Wordlist) == 0 && len(conf.InputCommand) == 0 {
+		errs.Add(fmt.Errorf("Either -w or --input-cmd flag is required"))
 	}
 	// prepare extensions
 	if parseOpts.extensions != "" {

pkg/ffuf/config.go

@@ -27,6 +27,8 @@ type Config struct {
 	Quiet             bool
 	Colors            bool
 	Wordlist          string
+	InputCommand      string
+	InputNum          int
 	OutputFile        string
 	OutputFormat      string
 	StopOn403         bool
@@ -59,6 +61,8 @@ func NewConfig(ctx context.Context) Config {
 	conf.StopOnErrors = false
 	conf.StopOnAll = false
 	conf.FollowRedirects = false
+	conf.InputCommand = ""
+	conf.InputNum = 0
 	conf.ProxyURL = http.ProxyFromEnvironment
 	conf.Filters = make([]FilterProvider, 0)
 	conf.Delay = optRange{0, 0, false, false}

pkg/ffuf/interfaces.go

@@ -15,6 +15,7 @@ type RunnerProvider interface {
 //InputProvider interface handles the input data for RunnerProvider
 type InputProvider interface {
 	Next() bool
+	Position() int
 	Value() []byte
 	Total() int
 }

pkg/ffuf/job.go

@@ -79,12 +79,13 @@ func (j *Job) Start() {
 		}
 		limiter <- true
 		nextInput := j.Input.Value()
+		nextPosition := j.Input.Position()
 		wg.Add(1)
 		j.Counter++
 		go func() {
 			defer func() { <-limiter }()
 			defer wg.Done()
-			j.runTask([]byte(nextInput), false)
+			j.runTask([]byte(nextInput), nextPosition, false)
 			if j.Config.Delay.HasDelay {
 				var sleepDurationMS time.Duration
 				if j.Config.Delay.IsRange {
@@ -156,8 +157,9 @@ func (j *Job) isMatch(resp Response) bool {
 	return true
 }
 
-func (j *Job) runTask(input []byte, retried bool) {
+func (j *Job) runTask(input []byte, position int, retried bool) {
 	req, err := j.Runner.Prepare(input)
+	req.Position = position
 	if err != nil {
 		j.Output.Error(fmt.Sprintf("Encountered an error while preparing request: %s\n", err))
 		j.incError()
@@ -168,7 +170,7 @@ func (j *Job) runTask(input []byte, retried bool) {
 		if retried {
 			j.incError()
 		} else {
-			j.runTask(input, true)
+			j.runTask(input, position, true)
 		}
 		return
 	}

pkg/ffuf/request.go

@@ -2,11 +2,12 @@ package ffuf
 
 // Request holds the meaningful data that is passed for runner for making the query
 type Request struct {
-	Method  string
-	Url     string
-	Headers map[string]string
-	Data    []byte
-	Input   []byte
+	Method   string
+	Url      string
+	Headers  map[string]string
+	Data     []byte
+	Input    []byte
+	Position int
 }
 
 func NewRequest(conf *Config) Request {

pkg/input/command.go

@@ -0,0 +1,54 @@
+package input
+
+import (
+	"bytes"
+	"os"
+	"os/exec"
+	"strconv"
+
+	"github.com/ffuf/ffuf/pkg/ffuf"
+)
+
+type CommandInput struct {
+	config *ffuf.Config
+	count  int
+}
+
+func NewCommandInput(conf *ffuf.Config) (*CommandInput, error) {
+	var cmd CommandInput
+	cmd.config = conf
+	cmd.count = -1
+	return &cmd, nil
+}
+
+//Position will return the current position in the input list
+func (c *CommandInput) Position() int {
+	return c.count
+}
+
+//Next will increment the cursor position, and return a boolean telling if there's iterations left
+func (c *CommandInput) Next() bool {
+	c.count++
+	if c.count >= c.config.InputNum {
+		return false
+	}
+	return true
+}
+
+//Value returns the input from command stdoutput
+func (c *CommandInput) Value() []byte {
+	var stdout bytes.Buffer
+	os.Setenv("FFUF_NUM", strconv.Itoa(c.count))
+	cmd := exec.Command(SHELL_CMD, SHELL_ARG, c.config.InputCommand)
+	cmd.Stdout = &stdout
+	err := cmd.Run()
+	if err != nil {
+		return []byte("")
+	}
+	return stdout.Bytes()
+}
+
+//Total returns the size of wordlist
+func (c *CommandInput) Total() int {
+	return c.config.InputNum
+}

pkg/input/const.go

@@ -0,0 +1,8 @@
+// +build !windows
+
+package input
+
+const (
+	SHELL_CMD = "/bin/sh"
+	SHELL_ARG = "-c"
+)

pkg/input/const_windows.go

@@ -0,0 +1,8 @@
+// +build windows
+
+package input
+
+const (
+	SHELL_CMD = "cmd.exe"
+	SHELL_ARG = "/C"
+)

pkg/input/input.go

@@ -5,6 +5,10 @@ import (
 )
 
 func NewInputProviderByName(name string, conf *ffuf.Config) (ffuf.InputProvider, error) {
-	// We have only one inputprovider at the moment
-	return NewWordlistInput(conf)
+	if name == "command" {
+		return NewCommandInput(conf)
+	} else {
+		// Default to wordlist
+		return NewWordlistInput(conf)
+	}
 }

pkg/input/wordlist.go

@@ -37,6 +37,11 @@ func NewWordlistInput(conf *ffuf.Config) (*WordlistInput, error) {
 	return &wl, err
 }
 
+//Position will return the current position in the input list
+func (w *WordlistInput) Position() int {
+	return w.position
+}
+
 //Next will increment the cursor position, and return a boolean telling if there's words left in the list
 func (w *WordlistInput) Next() bool {
 	w.position++

pkg/output/file_csv.go

@@ -9,7 +9,7 @@ import (
 	"github.com/ffuf/ffuf/pkg/ffuf"
 )
 
-var header = []string{"input", "status_code", "content_length", "content_words"}
+var header = []string{"input", "position", "status_code", "content_length", "content_words"}
 
 func writeCSV(config *ffuf.Config, res []Result, encode bool) error {
 	f, err := os.Create(config.OutputFile)
@@ -44,6 +44,7 @@ func base64encode(in string) string {
 func toCSV(r Result) []string {
 	return []string{
 		r.Input,
+		strconv.Itoa(r.Position),
 		strconv.FormatInt(r.StatusCode, 10),
 		strconv.FormatInt(r.ContentLength, 10),
 		strconv.FormatInt(r.ContentWords, 10),

pkg/output/stdout.go

@@ -3,6 +3,7 @@ package output
 import (
 	"fmt"
 	"os"
+	"strconv"
 	"time"
 
 	"github.com/ffuf/ffuf/pkg/ffuf"
@@ -27,6 +28,7 @@ type Stdoutput struct {
 
 type Result struct {
 	Input         string `json:"input"`
+	Position      int    `json:"position"`
 	StatusCode    int64  `json:"status"`
 	ContentLength int64  `json:"length"`
 	ContentWords  int64  `json:"words"`
@@ -127,6 +129,7 @@ func (s *Stdoutput) Result(resp ffuf.Response) {
 		// No need to store results if we're not going to use them later
 		sResult := Result{
 			Input:         string(resp.Request.Input),
+			Position:      resp.Request.Position,
 			StatusCode:    resp.StatusCode,
 			ContentLength: resp.ContentLength,
 			ContentWords:  resp.ContentWords,
@@ -144,11 +147,22 @@ func (s *Stdoutput) printResult(resp ffuf.Response) {
 }
 
 func (s *Stdoutput) resultQuiet(resp ffuf.Response) {
-	fmt.Println(string(resp.Request.Input))
+	if len(s.config.InputCommand) > 0 {
+		// If we're using external command for input, display the position instead of input
+		fmt.Println(strconv.Itoa(resp.Request.Position))
+	} else {
+		fmt.Println(string(resp.Request.Input))
+	}
 }
 
 func (s *Stdoutput) resultNormal(resp ffuf.Response) {
-	res_str := fmt.Sprintf("%s%-23s [Status: %s, Size: %d, Words: %d]", TERMINAL_CLEAR_LINE, resp.Request.Input, s.colorizeStatus(resp.StatusCode), resp.ContentLength, resp.ContentWords)
+	var res_str string
+	if len(s.config.InputCommand) > 0 {
+		// If we're using external command for input, display the position instead of input
+		res_str = fmt.Sprintf("%s%-23s [Status: %s, Size: %d, Words: %d]", TERMINAL_CLEAR_LINE, strconv.Itoa(resp.Request.Position), s.colorizeStatus(resp.StatusCode), resp.ContentLength, resp.ContentWords)
+	} else {
+		res_str = fmt.Sprintf("%s%-23s [Status: %s, Size: %d, Words: %d]", TERMINAL_CLEAR_LINE, resp.Request.Input, s.colorizeStatus(resp.StatusCode), resp.ContentLength, resp.ContentWords)
+	}
 	fmt.Println(res_str)
 }