Merge branch 'master' into raw

CHANGELOG.md

@@ -2,10 +2,14 @@
 - master
   - New
     - New cli flag `-raw` to omit urlencoding for URIs
+    - Integration with `github.com/ffuf/pencode` library, added `-enc` cli flag to do various in-fly encodings for input data
   - Changed
     - Explicitly allow TLS1.0 
     - Fix markdown output file format
-  
+    - Fix csv output file format
+    - Fixed divide by 0 error when setting rate limit to 0 manually.
+    - Automatic brotli and deflate decompression
+
 - v2.0.0
   - New
     - Added a new, dynamic keyword `FFUFHASH` that generates hash from job configuration and wordlist position to map blind payloads back to the initial request.

CONTRIBUTORS.md

@@ -14,10 +14,13 @@
 * [Daviey](https://github.com/Daviey)
 * [delic](https://github.com/delic)
 * [denandz](https://github.com/denandz)
+* [Ephex2](https://github.com/Ephex2)
 * [erbbysam](https://github.com/erbbysam)
 * [eur0pa](https://github.com/eur0pa)
+* [gserrg](https://github.com/gserrg)
 * [fabiobauer](https://github.com/fabiobauer)
 * [fang0654](https://github.com/fang0654)
+* [haseobang](https://github.com/haseobang)
 * [Hazegard](https://github.com/Hazegard)
 * [helpermika](https://github.com/helpermika)
 * [h1x](https://github.com/h1x-lnx)
@@ -41,9 +44,9 @@
 * [putsi](https://github.com/putsi)
 * [SakiiR](https://github.com/SakiiR)
 * [seblw](https://github.com/seblw)
+* [Serizao](https://github.com/Serizao)
 * [Shaked](https://github.com/Shaked)
 * [Skyehopper](https://github.com/Skyehopper)
 * [SolomonSklash](https://github.com/SolomonSklash)
 * [TomNomNom](https://github.com/tomnomnom)
 * [xfgusta](https://github.com/xfgusta)
-

go.mod

@@ -5,6 +5,8 @@ go 1.17
 require (
 	github.com/PuerkitoBio/goquery v1.8.0
 	github.com/adrg/xdg v0.4.0
+	github.com/andybalholm/brotli v1.0.5
+	github.com/ffuf/pencode v0.0.0-20230421231718-2cea7e60a693
 	github.com/pelletier/go-toml v1.9.5
 )
 

go.sum

@@ -2,11 +2,15 @@ github.com/PuerkitoBio/goquery v1.8.0 h1:PJTF7AmFCFKk1N6V6jmKfrNH9tV5pNE6lZMkG0g
 github.com/PuerkitoBio/goquery v1.8.0/go.mod h1:ypIiRMtY7COPGk+I/YbZLbxsxn9g5ejnI2HSMtkjZvI=
 github.com/adrg/xdg v0.4.0 h1:RzRqFcjH4nE5C6oTAxhBtoE2IRyjBSa62SCbyPidvls=
 github.com/adrg/xdg v0.4.0/go.mod h1:N6ag73EX4wyxeaoeHctc1mas01KZgsj5tYiAIwqJE/E=
+github.com/andybalholm/brotli v1.0.5 h1:8uQZIdzKmjc/iuPu7O2ioW48L81FgatrcpfFmiq/cCs=
+github.com/andybalholm/brotli v1.0.5/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig=
 github.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x004T2c=
 github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/ffuf/pencode v0.0.0-20230421231718-2cea7e60a693 h1:fdlgw33oLPzRpoHa4ppDFX5EcmzHHychPrO5xXmzxqc=
+github.com/ffuf/pencode v0.0.0-20230421231718-2cea7e60a693/go.mod h1:Qmgn2URTRtZ5wMntUke1+/G7z8rofTFHG1EvN3addNY=
 github.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3ve8=
 github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=

help.go

@@ -54,7 +54,7 @@ func Usage() {
 		Description:   "Options controlling the HTTP request and its parts.",
 		Flags:         make([]UsageFlag, 0),
 		Hidden:        false,
-		ExpectedFlags: []string{"H", "X", "b", "d", "r", "u", "raw", "recursion", "recursion-depth", "recursion-strategy", "replay-proxy", "timeout", "ignore-body", "x", "sni", "http2"},
+		ExpectedFlags: []string{"cc", "ck", "H", "X", "b", "d", "r", "u", "raw", "recursion", "recursion-depth", "recursion-strategy", "replay-proxy", "timeout", "ignore-body", "x", "sni", "http2"},
 	}
 	u_general := UsageSection{
 		Name:          "GENERAL OPTIONS",
@@ -89,7 +89,7 @@ func Usage() {
 		Description:   "Options for input data for fuzzing. Wordlists and input generators.",
 		Flags:         make([]UsageFlag, 0),
 		Hidden:        false,
-		ExpectedFlags: []string{"D", "ic", "input-cmd", "input-num", "input-shell", "mode", "request", "request-proto", "e", "w"},
+		ExpectedFlags: []string{"D", "enc", "ic", "input-cmd", "input-num", "input-shell", "mode", "request", "request-proto", "e", "w"},
 	}
 	u_output := UsageSection{
 		Name:          "OUTPUT OPTIONS",

main.go

@@ -51,13 +51,14 @@ func (m *wordlistFlag) Set(value string) error {
 func ParseFlags(opts *ffuf.ConfigOptions) *ffuf.ConfigOptions {
 	var ignored bool
 	var cookies, autocalibrationstrings, headers, inputcommands multiStringFlag
-	var wordlists wordlistFlag
+	var wordlists, encoders wordlistFlag
 
 	cookies = opts.HTTP.Cookies
 	autocalibrationstrings = opts.General.AutoCalibrationStrings
 	headers = opts.HTTP.Headers
 	inputcommands = opts.Input.Inputcommands
 	wordlists = opts.Input.Wordlists
+	encoders = opts.Input.Encoders
 
 	flag.BoolVar(&ignored, "compressed", true, "Dummy flag for copy as curl functionality (ignored)")
 	flag.BoolVar(&ignored, "i", true, "Dummy flag for copy as curl functionality (ignored)")
@@ -89,6 +90,8 @@ func ParseFlags(opts *ffuf.ConfigOptions) *ffuf.ConfigOptions {
 	flag.IntVar(&opts.HTTP.Timeout, "timeout", opts.HTTP.Timeout, "HTTP request timeout in seconds.")
 	flag.IntVar(&opts.Input.InputNum, "input-num", opts.Input.InputNum, "Number of inputs to test. Used in conjunction with --input-cmd.")
 	flag.StringVar(&opts.General.AutoCalibrationKeyword, "ack", opts.General.AutoCalibrationKeyword, "Autocalibration keyword")
+	flag.StringVar(&opts.HTTP.ClientCert, "cc", "", "Client cert for authentication. Client key needs to be defined as well for this to work")
+	flag.StringVar(&opts.HTTP.ClientKey, "ck", "", "Client key for authentication. Client certificate needs to be defined as well for this to work")
 	flag.StringVar(&opts.General.AutoCalibrationStrategy, "acs", opts.General.AutoCalibrationStrategy, "Autocalibration strategy: \"basic\" or \"advanced\"")
 	flag.StringVar(&opts.General.ConfigFile, "config", "", "Load configuration from a file")
 	flag.StringVar(&opts.General.ScraperFile, "scraperfile", "", "Custom scraper file path")
@@ -134,6 +137,7 @@ func ParseFlags(opts *ffuf.ConfigOptions) *ffuf.ConfigOptions {
 	flag.Var(&headers, "H", "Header `\"Name: Value\"`, separated by colon. Multiple -H flags are accepted.")
 	flag.Var(&inputcommands, "input-cmd", "Command producing the input. --input-num is required when using this input method. Overrides -w.")
 	flag.Var(&wordlists, "w", "Wordlist file path and (optional) keyword separated by colon. eg. '/path/to/wordlist:KEYWORD'")
+	flag.Var(&encoders, "enc", "Encoders for keywords, eg. 'FUZZ:urlencode b64encode'")
 	flag.Usage = Usage
 	flag.Parse()
 
@@ -142,6 +146,7 @@ func ParseFlags(opts *ffuf.ConfigOptions) *ffuf.ConfigOptions {
 	opts.HTTP.Headers = headers
 	opts.Input.Inputcommands = inputcommands
 	opts.Input.Wordlists = wordlists
+	opts.Input.Encoders = encoders
 	return opts
 }
 

pkg/ffuf/config.go

@@ -20,6 +20,7 @@ type Config struct {
 	Debuglog                string                `json:"debuglog"`
 	Delay                   optRange              `json:"delay"`
 	DirSearchCompat         bool                  `json:"dirsearch_compatibility"`
+	Encoders                []string              `json:"encoders"`
 	Extensions              []string              `json:"extensions"`
 	FilterMode              string                `json:"fmode"`
 	FollowRedirects         bool                  `json:"follow_redirects"`
@@ -64,12 +65,15 @@ type Config struct {
 	Verbose                 bool                  `json:"verbose"`
 	Wordlists               []string              `json:"wordlists"`
 	Http2                   bool                  `json:"http2"`
+	ClientCert              string                `json:"client-cert"`
+	ClientKey               string                `json:"client-key"`
 }
 
 type InputProviderConfig struct {
 	Name     string `json:"name"`
 	Keyword  string `json:"keyword"`
 	Value    string `json:"value"`
+	Encoders string `json:"encoders"`
 	Template string `json:"template"` // the templating string used for sniper mode (usually "§")
 }
 
@@ -85,6 +89,7 @@ func NewConfig(ctx context.Context, cancel context.CancelFunc) Config {
 	conf.Debuglog = ""
 	conf.Delay = optRange{0, 0, false, false}
 	conf.DirSearchCompat = false
+	conf.Encoders = make([]string, 0)
 	conf.Extensions = make([]string, 0)
 	conf.FilterMode = "or"
 	conf.FollowRedirects = false

pkg/ffuf/optionsparser.go

@@ -42,6 +42,8 @@ type HTTPOptions struct {
 	Timeout           int      `json:"timeout"`
 	URL               string   `json:"url"`
 	Http2             bool     `json:"http2"`
+	ClientCert        string   `json:"client-cert"`
+	ClientKey         string   `json:"client-key"`
 }
 
 type GeneralOptions struct {
@@ -72,6 +74,7 @@ type GeneralOptions struct {
 
 type InputOptions struct {
 	DirSearchCompat        bool     `json:"dirsearch_compat"`
+	Encoders               []string `json:"encoders"`
 	Extensions             string   `json:"extensions"`
 	IgnoreWordlistComments bool     `json:"ignore_wordlist_comments"`
 	InputMode              string   `json:"input_mode"`
@@ -156,6 +159,7 @@ func NewConfigOptions() *ConfigOptions {
 	c.HTTP.URL = ""
 	c.HTTP.Http2 = false
 	c.Input.DirSearchCompat = false
+	c.Input.Encoders = []string{}
 	c.Input.Extensions = ""
 	c.Input.IgnoreWordlistComments = false
 	c.Input.InputMode = "clusterbomb"
@@ -227,7 +231,14 @@ func ConfigFromOptions(parseOpts *ConfigOptions, ctx context.Context, cancel con
 			errs.Add(fmt.Errorf("sniper mode only supports one input command"))
 		}
 	}
-
+	tmpEncoders := make(map[string]string)
+	for _, e := range parseOpts.Input.Encoders {
+		if strings.Contains(e, ":") {
+			key := strings.Split(e, ":")[0]
+			val := strings.Split(e, ":")[1]
+			tmpEncoders[key] = val
+		}
+	}
 	tmpWordlists := make([]string, 0)
 	for _, v := range parseOpts.Input.Wordlists {
 		var wl []string
@@ -267,19 +278,31 @@ func ConfigFromOptions(parseOpts *ConfigOptions, ctx context.Context, cancel con
 			if conf.InputMode == "sniper" {
 				errs.Add(fmt.Errorf("sniper mode does not support wordlist keywords"))
 			} else {
-				conf.InputProviders = append(conf.InputProviders, InputProviderConfig{
+				newp := InputProviderConfig{
 					Name:    "wordlist",
 					Value:   wl[0],
 					Keyword: wl[1],
-				})
+				}
+				// Add encoders if set
+				enc, ok := tmpEncoders[wl[1]]
+				if ok {
+					newp.Encoders = enc
+				}
+				conf.InputProviders = append(conf.InputProviders, newp)
 			}
 		} else {
-			conf.InputProviders = append(conf.InputProviders, InputProviderConfig{
+			newp := InputProviderConfig{
 				Name:     "wordlist",
 				Value:    wl[0],
 				Keyword:  "FUZZ",
 				Template: template,
-			})
+			}
+			// Add encoders if set
+			enc, ok := tmpEncoders["FUZZ"]
+			if ok {
+				newp.Encoders = enc
+			}
+			conf.InputProviders = append(conf.InputProviders, newp)
 		}
 		tmpWordlists = append(tmpWordlists, strings.Join(wl, ":"))
 	}
@@ -291,20 +314,30 @@ func ConfigFromOptions(parseOpts *ConfigOptions, ctx context.Context, cancel con
 			if conf.InputMode == "sniper" {
 				errs.Add(fmt.Errorf("sniper mode does not support command keywords"))
 			} else {
-				conf.InputProviders = append(conf.InputProviders, InputProviderConfig{
+				newp := InputProviderConfig{
 					Name:    "command",
 					Value:   ic[0],
 					Keyword: ic[1],
-				})
+				}
+				enc, ok := tmpEncoders[ic[1]]
+				if ok {
+					newp.Encoders = enc
+				}
+				conf.InputProviders = append(conf.InputProviders, newp)
 				conf.CommandKeywords = append(conf.CommandKeywords, ic[0])
 			}
 		} else {
-			conf.InputProviders = append(conf.InputProviders, InputProviderConfig{
+			newp := InputProviderConfig{
 				Name:     "command",
 				Value:    ic[0],
 				Keyword:  "FUZZ",
 				Template: template,
-			})
+			}
+			enc, ok := tmpEncoders["FUZZ"]
+			if ok {
+				newp.Encoders = enc
+			}
+			conf.InputProviders = append(conf.InputProviders, newp)
 			conf.CommandKeywords = append(conf.CommandKeywords, "FUZZ")
 		}
 	}
@@ -332,6 +365,15 @@ func ConfigFromOptions(parseOpts *ConfigOptions, ctx context.Context, cancel con
 		conf.SNI = parseOpts.HTTP.SNI
 	}
 
+	// prepare cert
+	if parseOpts.HTTP.ClientCert != "" {
+		conf.ClientCert = parseOpts.HTTP.ClientCert
+	}
+	if parseOpts.HTTP.ClientKey != "" {
+		conf.ClientKey = parseOpts.HTTP.ClientKey
+	}
+
+
 	//Prepare headers and make canonical
 	for _, v := range parseOpts.HTTP.Headers {
 		hs := strings.SplitN(v, ":", 2)

pkg/ffuf/rate.go

@@ -65,7 +65,12 @@ func (r *RateThrottle) CurrentRate() int64 {
 }
 
 func (r *RateThrottle) ChangeRate(rate int) {
-	ratemicros := 1000000 / rate
+	ratemicros := 0 // set default to 0, avoids integer divide by 0 error
+
+	if rate != 0 {
+		ratemicros = 1000000 / rate
+	}
+
 	r.RateLimiter.Stop()
 	r.RateLimiter = time.NewTicker(time.Microsecond * time.Duration(ratemicros))
 	r.Config.Rate = int64(rate)

pkg/input/input.go

@@ -2,12 +2,15 @@ package input
 
 import (
 	"fmt"
-
 	"github.com/ffuf/ffuf/v2/pkg/ffuf"
+	"strings"
+
+	"github.com/ffuf/pencode/pkg/pencode"
 )
 
 type MainInputProvider struct {
 	Providers   []ffuf.InternalInputProvider
+	Encoders    map[string]*pencode.Chain
 	Config      *ffuf.Config
 	position    int
 	msbIterator int
@@ -25,7 +28,7 @@ func NewInputProvider(conf *ffuf.Config) (ffuf.InputProvider, ffuf.Multierror) {
 		errs.Add(fmt.Errorf("Input mode (-mode) %s not recognized", conf.InputMode))
 		return &MainInputProvider{}, errs
 	}
-	mainip := MainInputProvider{Config: conf, msbIterator: 0}
+	mainip := MainInputProvider{Config: conf, msbIterator: 0, Encoders: make(map[string]*pencode.Chain)}
 	// Initialize the correct inputprovider
 	for _, v := range conf.InputProviders {
 		err := mainip.AddProvider(v)
@@ -48,6 +51,14 @@ func (i *MainInputProvider) AddProvider(provider ffuf.InputProviderConfig) error
 		}
 		i.Providers = append(i.Providers, newwl)
 	}
+	if len(provider.Encoders) > 0 {
+		chain := pencode.NewChain()
+		err := chain.Initialize(strings.Split(strings.TrimSpace(provider.Encoders), " "))
+		if err != nil {
+			return err
+		}
+		i.Encoders[provider.Keyword] = chain
+	}
 	return nil
 }
 
@@ -103,6 +114,18 @@ func (i *MainInputProvider) Value() map[string][]byte {
 	if i.Config.InputMode == "pitchfork" {
 		retval = i.pitchforkValue()
 	}
+	if len(i.Encoders) > 0 {
+		for key, val := range retval {
+			chain, ok := i.Encoders[key]
+			if ok {
+				tmpVal, err := chain.Encode([]byte(val))
+				if err != nil {
+					fmt.Printf("ERROR: %s\n", err)
+				}
+				retval[key] = tmpVal
+			}
+		}
+	}
 	return retval
 }
 

pkg/output/file_csv.go

@@ -9,7 +9,7 @@ import (
 	"github.com/ffuf/ffuf/v2/pkg/ffuf"
 )
 
-var staticheaders = []string{"url", "redirectlocation", "position", "status_code", "content_length", "content_words", "content_lines", "content_type", "duration", "resultfile"}
+var staticheaders = []string{"url", "redirectlocation", "position", "status_code", "content_length", "content_words", "content_lines", "content_type", "duration", "resultfile", "Ffufhash"}
 
 func writeCSV(filename string, config *ffuf.Config, res []ffuf.Result, encode bool) error {
 	header := make([]string, 0)
@@ -53,8 +53,13 @@ func base64encode(in []byte) string {
 
 func toCSV(r ffuf.Result) []string {
 	res := make([]string, 0)
-	for _, v := range r.Input {
-		res = append(res, string(v))
+	ffufhash := ""
+	for k, v := range r.Input {
+		if k == "FFUFHASH" {
+			ffufhash = string(v)
+		} else {
+			res = append(res, string(v))
+		}
 	}
 	res = append(res, r.Url)
 	res = append(res, r.RedirectLocation)
@@ -66,5 +71,6 @@ func toCSV(r ffuf.Result) []string {
 	res = append(res, r.ContentType)
 	res = append(res, r.Duration.String())
 	res = append(res, r.ResultFile)
+	res = append(res, ffufhash)
 	return res
 }

pkg/output/file_html.go

@@ -108,7 +108,7 @@ const (
                 <div style="display:none">
 |result_raw|{{ $result.StatusCode }}{{ range $keyword, $value := $result.Input }}|{{ $value | printf "%s" }}{{ end }}|{{ $result.Url }}|{{ $result.RedirectLocation }}|{{ $result.Position }}|{{ $result.ContentLength }}|{{ $result.ContentWords }}|{{ $result.ContentLines }}|{{ $result.ContentType }}|{{ $result.Duration }}|{{ $result.ResultFile }}|{{ $result.ScraperData }}|{{ $result.FfufHash }}|
                 </div>
-                <tr class="result-{{ $result.StatusCode }}" style="background-color: {{$result.HTMLColor}};">
+                <tr class="result-{{ $result.StatusCode }}" style="background-color: {{ $result.HTMLColor }};">
                     <td><font color="black" class="status-code">{{ $result.StatusCode }}</font></td>
                     {{ range $keyword, $value := $result.Input }}
                         <td>{{ $value | printf "%s" }}</td>

pkg/runner/simple.go

@@ -2,6 +2,7 @@ package runner
 
 import (
 	"bytes"
+	"compress/flate"
 	"compress/gzip"
 	"crypto/tls"
 	"fmt"
@@ -17,6 +18,8 @@ import (
 	"time"
 
 	"github.com/ffuf/ffuf/v2/pkg/ffuf"
+
+	"github.com/andybalholm/brotli"
 )
 
 // Download results < 5MB
@@ -43,6 +46,13 @@ func NewSimpleRunner(conf *ffuf.Config, replay bool) ffuf.RunnerProvider {
 			proxyURL = http.ProxyURL(pu)
 		}
 	}
+	cert := []tls.Certificate{}
+
+	if conf.ClientCert != "" && conf.ClientKey != "" {
+		tmp, _ := tls.LoadX509KeyPair(conf.ClientCert, conf.ClientKey)
+		cert = []tls.Certificate{tmp}
+	}
+
 	simplerunner.config = conf
 	simplerunner.client = &http.Client{
 		CheckRedirect: func(req *http.Request, via []*http.Request) error { return http.ErrUseLastResponse },
@@ -62,6 +72,7 @@ func NewSimpleRunner(conf *ffuf.Config, replay bool) ffuf.RunnerProvider {
 				MinVersion:         tls.VersionTLS10,
 				Renegotiation:      tls.RenegotiateOnceAsClient,
 				ServerName:         conf.SNI,
+				Certificates:       cert,
 			},
 		}}
 
@@ -169,6 +180,18 @@ func (r *SimpleRunner) Execute(req *ffuf.Request) (ffuf.Response, error) {
 			// fallback to raw data
 			bodyReader = httpresp.Body
 		}
+	} else if httpresp.Header.Get("Content-Encoding") == "br" {
+		bodyReader = io.NopCloser(brotli.NewReader(httpresp.Body))
+		if err != nil {
+			// fallback to raw data
+			bodyReader = httpresp.Body
+		}
+	} else if httpresp.Header.Get("Content-Encoding") == "deflate" {
+		bodyReader = flate.NewReader(httpresp.Body)
+		if err != nil {
+			// fallback to raw data
+			bodyReader = httpresp.Body
+		}
 	} else {
 		bodyReader = httpresp.Body
 	}