Mirrors/ffuf
2
0
Fork 0
mirror of https://github.com/ffuf/ffuf synced 2024-11-10 06:04:17 +00:00
Code Issues Projects Releases Packages Wiki Activity

[FEATURE] Extensible Auto-calibration strategies (#694)

Browse source 
* blacklist detection

* added option to help.go

* refactored -blacklist-detection to autocalibrationstrategy extra

* "No common filtering values found" fixed

* added wildcard not found detection

* custom auto-calibration strategies

* Make linter happy

---------

Co-authored-by: Joona Hoikkala <5235109+joohoi@users.noreply.github.com>
This commit is contained in:
Aristos Miliaresis 2023-09-15 18:26:45 +03:00 committed by GitHub
parent a7dea16d62
commit e80fdc47c0
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
9 changed files with 192 additions and 116 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
CHANGELOG.md
View file

@ -1,6 +1,7 @@
## Changelog
- master
- New
- autocalibration-strategy refactored to support extensible strategy configuration
- New cli flag `-raw` to omit urlencoding for URIs
- Integration with `github.com/ffuf/pencode` library, added `-enc` cli flag to do various in-fly encodings for input data
- Changed

11
main.go
View file

@ -50,7 +50,8 @@ func (m *wordlistFlag) Set(value string) error {
// ParseFlags parses the command line flags and (re)populates the ConfigOptions struct
func ParseFlags(opts *ffuf.ConfigOptions) *ffuf.ConfigOptions {
var ignored bool
var cookies, autocalibrationstrings, headers, inputcommands multiStringFlag
var cookies, autocalibrationstrings, autocalibrationstrategies, headers, inputcommands multiStringFlag
var wordlists, encoders wordlistFlag
cookies = opts.HTTP.Cookies
@ -92,7 +93,6 @@ func ParseFlags(opts *ffuf.ConfigOptions) *ffuf.ConfigOptions {
flag.StringVar(&opts.General.AutoCalibrationKeyword, "ack", opts.General.AutoCalibrationKeyword, "Autocalibration keyword")
flag.StringVar(&opts.HTTP.ClientCert, "cc", "", "Client cert for authentication. Client key needs to be defined as well for this to work")
flag.StringVar(&opts.HTTP.ClientKey, "ck", "", "Client key for authentication. Client certificate needs to be defined as well for this to work")
flag.StringVar(&opts.General.AutoCalibrationStrategy, "acs", opts.General.AutoCalibrationStrategy, "Autocalibration strategy: \"basic\" or \"advanced\"")
flag.StringVar(&opts.General.ConfigFile, "config", "", "Load configuration from a file")
flag.StringVar(&opts.General.ScraperFile, "scraperfile", "", "Custom scraper file path")
flag.StringVar(&opts.General.Scrapers, "scrapers", opts.General.Scrapers, "Active scraper groups")
@ -132,6 +132,7 @@ func ParseFlags(opts *ffuf.ConfigOptions) *ffuf.ConfigOptions {
flag.StringVar(&opts.Output.OutputFile, "o", opts.Output.OutputFile, "Write output to file")
flag.StringVar(&opts.Output.OutputFormat, "of", opts.Output.OutputFormat, "Output file format. Available formats: json, ejson, html, md, csv, ecsv (or, 'all' for all formats)")
flag.Var(&autocalibrationstrings, "acc", "Custom auto-calibration string. Can be used multiple times. Implies -ac")
flag.Var(&autocalibrationstrategies, "acs", "Custom auto-calibration strategies. Can be used multiple times. Implies -ac")
flag.Var(&cookies, "b", "Cookie data `\"NAME1=VALUE1; NAME2=VALUE2\"` for copy as curl functionality.")
flag.Var(&cookies, "cookie", "Cookie data (alias of -b)")
flag.Var(&headers, "H", "Header `\"Name: Value\"`, separated by colon. Multiple -H flags are accepted.")
@ -142,6 +143,12 @@ func ParseFlags(opts *ffuf.ConfigOptions) *ffuf.ConfigOptions {
flag.Parse()
opts.General.AutoCalibrationStrings = autocalibrationstrings
if len(autocalibrationstrategies) > 0 {
opts.General.AutoCalibrationStrategies = []string {}
for _, strategy := range autocalibrationstrategies {
opts.General.AutoCalibrationStrategies = append(opts.General.AutoCalibrationStrategies, strings.Split(strategy, ",")...)
}
}
opts.HTTP.Cookies = cookies
opts.HTTP.Headers = headers
opts.Input.Inputcommands = inputcommands

79
pkg/ffuf/autocalibration.go
View file

@ -6,31 +6,80 @@ import (
"math/rand"
"strconv"
"time"
"encoding/json"
"path/filepath"
"os"
)
type AutocalibrationStrategy map[string][]string
func (j *Job) autoCalibrationStrings() map[string][]string {
rand.Seed(time.Now().UnixNano())
cInputs := make(map[string][]string)
if len(j.Config.AutoCalibrationStrings) < 1 {
cInputs["basic_admin"] = append(cInputs["basic_admin"], "admin"+RandomString(16))
cInputs["basic_admin"] = append(cInputs["basic_admin"], "admin"+RandomString(8))
cInputs["htaccess"] = append(cInputs["htaccess"], ".htaccess"+RandomString(16))
cInputs["htaccess"] = append(cInputs["htaccess"], ".htaccess"+RandomString(8))
cInputs["basic_random"] = append(cInputs["basic_random"], RandomString(16))
cInputs["basic_random"] = append(cInputs["basic_random"], RandomString(8))
if j.Config.AutoCalibrationStrategy == "advanced" {
// Add directory tests and .htaccess too
cInputs["admin_dir"] = append(cInputs["admin_dir"], "admin"+RandomString(16)+"/")
cInputs["admin_dir"] = append(cInputs["admin_dir"], "admin"+RandomString(8)+"/")
cInputs["random_dir"] = append(cInputs["random_dir"], RandomString(16)+"/")
cInputs["random_dir"] = append(cInputs["random_dir"], RandomString(8)+"/")
}
} else {
if len(j.Config.AutoCalibrationStrings) > 0 {
cInputs["custom"] = append(cInputs["custom"], j.Config.AutoCalibrationStrings...)
return cInputs
}
for _, strategy := range j.Config.AutoCalibrationStrategies {
jsonStrategy, err := os.ReadFile(filepath.Join(AUTOCALIBDIR, strategy+".json"))
if err != nil {
j.Output.Warning(fmt.Sprintf("Skipping strategy \"%s\" because of error: %s\n", strategy, err))
continue
}
tmpStrategy := AutocalibrationStrategy{}
err = json.Unmarshal(jsonStrategy, &tmpStrategy)
if err != nil {
j.Output.Warning(fmt.Sprintf("Skipping strategy \"%s\" because of error: %s\n", strategy, err))
continue
}
cInputs = mergeMaps(cInputs, tmpStrategy)
}
return cInputs
}
func setupDefaultAutocalibrationStrategies() error {
basic_strategy := AutocalibrationStrategy {
"basic_admin": []string{"admin"+RandomString(16), "admin"+RandomString(8)},
"htaccess": []string{".htaccess"+RandomString(16), ".htaccess"+RandomString(8)},
"basic_random": []string{RandomString(16), RandomString(8)},
}
basic_strategy_json, err := json.Marshal(basic_strategy)
if err != nil {
return err
}
advanced_strategy := AutocalibrationStrategy {
"basic_admin": []string{"admin"+RandomString(16), "admin"+RandomString(8)},
"htaccess": []string{".htaccess"+RandomString(16), ".htaccess"+RandomString(8)},
"basic_random": []string{RandomString(16), RandomString(8)},
"admin_dir": []string{"admin"+RandomString(16)+"/", "admin"+RandomString(8)+"/"},
"random_dir": []string{RandomString(16)+"/", RandomString(8)+"/"},
}
advanced_strategy_json, err := json.Marshal(advanced_strategy)
if err != nil {
return err
}
basic_strategy_file := filepath.Join(AUTOCALIBDIR, "basic.json")
if !FileExists(basic_strategy_file) {
err = os.WriteFile(filepath.Join(AUTOCALIBDIR, "basic.json"), basic_strategy_json, 0640)
return err
}
advanced_strategy_file := filepath.Join(AUTOCALIBDIR, "advanced.json")
if !FileExists(advanced_strategy_file) {
err = os.WriteFile(filepath.Join(AUTOCALIBDIR, "advanced.json"), advanced_strategy_json, 0640)
return err
}
return nil
}
func (j *Job) calibrationRequest(inputs map[string][]byte) (Response, error) {
basereq := BaseRequest(j.Config)
req, err := j.Runner.Prepare(inputs, &basereq)

126
pkg/ffuf/config.go
View file

@ -5,68 +5,68 @@ import (
)
type Config struct {
AutoCalibration bool `json:"autocalibration"`
AutoCalibrationKeyword string `json:"autocalibration_keyword"`
AutoCalibrationPerHost bool `json:"autocalibration_perhost"`
AutoCalibrationStrategy string `json:"autocalibration_strategy"`
AutoCalibrationStrings []string `json:"autocalibration_strings"`
Cancel context.CancelFunc `json:"-"`
Colors bool `json:"colors"`
CommandKeywords []string `json:"-"`
CommandLine string `json:"cmdline"`
ConfigFile string `json:"configfile"`
Context context.Context `json:"-"`
Data string `json:"postdata"`
Debuglog string `json:"debuglog"`
Delay optRange `json:"delay"`
DirSearchCompat bool `json:"dirsearch_compatibility"`
Encoders []string `json:"encoders"`
Extensions []string `json:"extensions"`
FilterMode string `json:"fmode"`
FollowRedirects bool `json:"follow_redirects"`
Headers map[string]string `json:"headers"`
IgnoreBody bool `json:"ignorebody"`
IgnoreWordlistComments bool `json:"ignore_wordlist_comments"`
InputMode string `json:"inputmode"`
InputNum int `json:"cmd_inputnum"`
InputProviders []InputProviderConfig `json:"inputproviders"`
InputShell string `json:"inputshell"`
Json bool `json:"json"`
MatcherManager MatcherManager `json:"matchers"`
MatcherMode string `json:"mmode"`
MaxTime int `json:"maxtime"`
MaxTimeJob int `json:"maxtime_job"`
Method string `json:"method"`
Noninteractive bool `json:"noninteractive"`
OutputDirectory string `json:"outputdirectory"`
OutputFile string `json:"outputfile"`
OutputFormat string `json:"outputformat"`
OutputSkipEmptyFile bool `json:"OutputSkipEmptyFile"`
ProgressFrequency int `json:"-"`
ProxyURL string `json:"proxyurl"`
Quiet bool `json:"quiet"`
Rate int64 `json:"rate"`
Raw bool `json:"raw"`
Recursion bool `json:"recursion"`
RecursionDepth int `json:"recursion_depth"`
RecursionStrategy string `json:"recursion_strategy"`
ReplayProxyURL string `json:"replayproxyurl"`
RequestFile string `json:"requestfile"`
RequestProto string `json:"requestproto"`
ScraperFile string `json:"scraperfile"`
Scrapers string `json:"scrapers"`
SNI string `json:"sni"`
StopOn403 bool `json:"stop_403"`
StopOnAll bool `json:"stop_all"`
StopOnErrors bool `json:"stop_errors"`
Threads int `json:"threads"`
Timeout int `json:"timeout"`
Url string `json:"url"`
Verbose bool `json:"verbose"`
Wordlists []string `json:"wordlists"`
Http2 bool `json:"http2"`
ClientCert string `json:"client-cert"`
ClientKey string `json:"client-key"`
AutoCalibration bool `json:"autocalibration"`
AutoCalibrationKeyword string `json:"autocalibration_keyword"`
AutoCalibrationPerHost bool `json:"autocalibration_perhost"`
AutoCalibrationStrategies []string `json:"autocalibration_strategies"`
AutoCalibrationStrings []string `json:"autocalibration_strings"`
Cancel context.CancelFunc `json:"-"`
Colors bool `json:"colors"`
CommandKeywords []string `json:"-"`
CommandLine string `json:"cmdline"`
ConfigFile string `json:"configfile"`
Context context.Context `json:"-"`
Data string `json:"postdata"`
Debuglog string `json:"debuglog"`
Delay optRange `json:"delay"`
DirSearchCompat bool `json:"dirsearch_compatibility"`
Encoders []string `json:"encoders"`
Extensions []string `json:"extensions"`
FilterMode string `json:"fmode"`
FollowRedirects bool `json:"follow_redirects"`
Headers map[string]string `json:"headers"`
IgnoreBody bool `json:"ignorebody"`
IgnoreWordlistComments bool `json:"ignore_wordlist_comments"`
InputMode string `json:"inputmode"`
InputNum int `json:"cmd_inputnum"`
InputProviders []InputProviderConfig `json:"inputproviders"`
InputShell string `json:"inputshell"`
Json bool `json:"json"`
MatcherManager MatcherManager `json:"matchers"`
MatcherMode string `json:"mmode"`
MaxTime int `json:"maxtime"`
MaxTimeJob int `json:"maxtime_job"`
Method string `json:"method"`
Noninteractive bool `json:"noninteractive"`
OutputDirectory string `json:"outputdirectory"`
OutputFile string `json:"outputfile"`
OutputFormat string `json:"outputformat"`
OutputSkipEmptyFile bool `json:"OutputSkipEmptyFile"`
ProgressFrequency int `json:"-"`
ProxyURL string `json:"proxyurl"`
Quiet bool `json:"quiet"`
Rate int64 `json:"rate"`
Raw bool `json:"raw"`
Recursion bool `json:"recursion"`
RecursionDepth int `json:"recursion_depth"`
RecursionStrategy string `json:"recursion_strategy"`
ReplayProxyURL string `json:"replayproxyurl"`
RequestFile string `json:"requestfile"`
RequestProto string `json:"requestproto"`
ScraperFile string `json:"scraperfile"`
Scrapers string `json:"scrapers"`
SNI string `json:"sni"`
StopOn403 bool `json:"stop_403"`
StopOnAll bool `json:"stop_all"`
StopOnErrors bool `json:"stop_errors"`
Threads int `json:"threads"`
Timeout int `json:"timeout"`
Url string `json:"url"`
Verbose bool `json:"verbose"`
Wordlists []string `json:"wordlists"`
Http2 bool `json:"http2"`
ClientCert string `json:"client-cert"`
ClientKey string `json:"client-key"`
}
type InputProviderConfig struct {
@ -80,7 +80,7 @@ type InputProviderConfig struct {
func NewConfig(ctx context.Context, cancel context.CancelFunc) Config {
var conf Config
conf.AutoCalibrationKeyword = "FUZZ"
conf.AutoCalibrationStrategy = "basic"
conf.AutoCalibrationStrategies = []string{"basic"}
conf.AutoCalibrationStrings = make([]string, 0)
conf.CommandKeywords = make([]string, 0)
conf.Context = ctx

2
pkg/ffuf/configmarshaller.go
View file

@ -31,7 +31,7 @@ func (c *Config) ToOptions() ConfigOptions {
o.General.AutoCalibration = c.AutoCalibration
o.General.AutoCalibrationKeyword = c.AutoCalibrationKeyword
o.General.AutoCalibrationPerHost = c.AutoCalibrationPerHost
o.General.AutoCalibrationStrategy = c.AutoCalibrationStrategy
o.General.AutoCalibrationStrategies = c.AutoCalibrationStrategies
o.General.AutoCalibrationStrings = c.AutoCalibrationStrings
o.General.Colors = c.Colors
o.General.ConfigFile = ""

1
pkg/ffuf/constants.go
View file

@ -13,4 +13,5 @@ var (
CONFIGDIR = filepath.Join(xdg.ConfigHome, "ffuf")
HISTORYDIR = filepath.Join(CONFIGDIR, "history")
SCRAPERDIR = filepath.Join(CONFIGDIR, "scraper")
AUTOCALIBDIR = filepath.Join(CONFIGDIR, "autocalibration")
)

58
pkg/ffuf/optionsparser.go
View file

@ -47,29 +47,29 @@ type HTTPOptions struct {
}
type GeneralOptions struct {
AutoCalibration bool `json:"autocalibration"`
AutoCalibrationKeyword string `json:"autocalibration_keyword"`
AutoCalibrationPerHost bool `json:"autocalibration_per_host"`
AutoCalibrationStrategy string `json:"autocalibration_strategy"`
AutoCalibrationStrings []string `json:"autocalibration_strings"`
Colors bool `json:"colors"`
ConfigFile string `toml:"-" json:"config_file"`
Delay string `json:"delay"`
Json bool `json:"json"`
MaxTime int `json:"maxtime"`
MaxTimeJob int `json:"maxtime_job"`
Noninteractive bool `json:"noninteractive"`
Quiet bool `json:"quiet"`
Rate int `json:"rate"`
ScraperFile string `json:"scraperfile"`
Scrapers string `json:"scrapers"`
Searchhash string `json:"-"`
ShowVersion bool `toml:"-" json:"-"`
StopOn403 bool `json:"stop_on_403"`
StopOnAll bool `json:"stop_on_all"`
StopOnErrors bool `json:"stop_on_errors"`
Threads int `json:"threads"`
Verbose bool `json:"verbose"`
AutoCalibration bool `json:"autocalibration"`
AutoCalibrationKeyword string `json:"autocalibration_keyword"`
AutoCalibrationPerHost bool `json:"autocalibration_per_host"`
AutoCalibrationStrategies []string `json:"autocalibration_strategies"`
AutoCalibrationStrings []string `json:"autocalibration_strings"`
Colors bool `json:"colors"`
ConfigFile string `toml:"-" json:"config_file"`
Delay string `json:"delay"`
Json bool `json:"json"`
MaxTime int `json:"maxtime"`
MaxTimeJob int `json:"maxtime_job"`
Noninteractive bool `json:"noninteractive"`
Quiet bool `json:"quiet"`
Rate int `json:"rate"`
ScraperFile string `json:"scraperfile"`
Scrapers string `json:"scrapers"`
Searchhash string `json:"-"`
ShowVersion bool `toml:"-" json:"-"`
StopOn403 bool `json:"stop_on_403"`
StopOnAll bool `json:"stop_on_all"`
StopOnErrors bool `json:"stop_on_errors"`
Threads int `json:"threads"`
Verbose bool `json:"verbose"`
}
type InputOptions struct {
@ -126,7 +126,7 @@ func NewConfigOptions() *ConfigOptions {
c.Filter.Words = ""
c.General.AutoCalibration = false
c.General.AutoCalibrationKeyword = "FUZZ"
c.General.AutoCalibrationStrategy = "basic"
c.General.AutoCalibrationStrategies = []string{"basic"}
c.General.Colors = false
c.General.Delay = ""
c.General.Json = false
@ -466,10 +466,18 @@ func ConfigFromOptions(parseOpts *ConfigOptions, ctx context.Context, cancel con
if len(parseOpts.General.AutoCalibrationStrings) > 0 {
conf.AutoCalibrationStrings = parseOpts.General.AutoCalibrationStrings
}
// Auto-calibration strategies
if len(parseOpts.General.AutoCalibrationStrategies) > 0 {
conf.AutoCalibrationStrategies = parseOpts.General.AutoCalibrationStrategies
}
// Using -acc implies -ac
if len(parseOpts.General.AutoCalibrationStrings) > 0 {
conf.AutoCalibration = true
}
// Using -acs implies -ac
if len(parseOpts.General.AutoCalibrationStrategies) > 0 {
conf.AutoCalibration = true
}
if parseOpts.General.Rate < 0 {
conf.Rate = 0
@ -522,7 +530,7 @@ func ConfigFromOptions(parseOpts *ConfigOptions, ctx context.Context, cancel con
conf.RecursionStrategy = parseOpts.HTTP.RecursionStrategy
conf.AutoCalibration = parseOpts.General.AutoCalibration
conf.AutoCalibrationPerHost = parseOpts.General.AutoCalibrationPerHost
conf.AutoCalibrationStrategy = parseOpts.General.AutoCalibrationStrategy
conf.AutoCalibrationStrategies = parseOpts.General.AutoCalibrationStrategies
conf.Threads = parseOpts.General.Threads
conf.Timeout = parseOpts.HTTP.Timeout
conf.MaxTime = parseOpts.General.MaxTime

19
pkg/ffuf/util.go
View file

@ -93,6 +93,14 @@ func CheckOrCreateConfigDir() error {
return err
}
err = createConfigDir(SCRAPERDIR)
if err != nil {
return err
}
err = createConfigDir(AUTOCALIBDIR)
if err != nil {
return err
}
err = setupDefaultAutocalibrationStrategies()
return err
}
@ -116,3 +124,14 @@ func StrInSlice(key string, slice []string) bool {
}
return false
}
func mergeMaps(m1 map[string][]string, m2 map[string][]string) map[string][]string {
merged := make(map[string][]string)
for k, v := range m1 {
merged[k] = v
}
for key, value := range m2 {
merged[key] = value
}
return merged
}

11
pkg/input/input.go
View file

@ -65,7 +65,7 @@ func (i *MainInputProvider) AddProvider(provider ffuf.InputProviderConfig) error
// ActivateKeywords enables / disables wordlists based on list of active keywords
func (i *MainInputProvider) ActivateKeywords(kws []string) {
for _, p := range i.Providers {
if sliceContains(kws, p.Keyword()) {
if ffuf.StrInSlice(p.Keyword(), kws) {
p.Active()
} else {
p.Disable()
@ -254,12 +254,3 @@ func (i *MainInputProvider) Total() int {
return count
}
// sliceContains is a helper function that returns true if a string is included in a string slice
func sliceContains(sslice []string, str string) bool {
for _, v := range sslice {
if v == str {
return true
}
}
return false
}