Automatic brotli and deflate decompression (#720)

CHANGELOG.md

@@ -7,7 +7,7 @@
     - Fix markdown output file format
     - Fix csv output file format
     - Fixed divide by 0 error when setting rate limit to 0 manually.
-
+    - Automatic brotli and deflate decompression
 
 - v2.0.0
   - New

go.mod

@@ -5,6 +5,7 @@ go 1.17
 require (
 	github.com/PuerkitoBio/goquery v1.8.0
 	github.com/adrg/xdg v0.4.0
+	github.com/andybalholm/brotli v1.0.5
 	github.com/ffuf/pencode v0.0.0-20230421231718-2cea7e60a693
 	github.com/pelletier/go-toml v1.9.5
 )

go.sum

@@ -2,6 +2,8 @@ github.com/PuerkitoBio/goquery v1.8.0 h1:PJTF7AmFCFKk1N6V6jmKfrNH9tV5pNE6lZMkG0g
 github.com/PuerkitoBio/goquery v1.8.0/go.mod h1:ypIiRMtY7COPGk+I/YbZLbxsxn9g5ejnI2HSMtkjZvI=
 github.com/adrg/xdg v0.4.0 h1:RzRqFcjH4nE5C6oTAxhBtoE2IRyjBSa62SCbyPidvls=
 github.com/adrg/xdg v0.4.0/go.mod h1:N6ag73EX4wyxeaoeHctc1mas01KZgsj5tYiAIwqJE/E=
+github.com/andybalholm/brotli v1.0.5 h1:8uQZIdzKmjc/iuPu7O2ioW48L81FgatrcpfFmiq/cCs=
+github.com/andybalholm/brotli v1.0.5/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig=
 github.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x004T2c=
 github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=

pkg/runner/simple.go

@@ -2,6 +2,7 @@ package runner
 
 import (
 	"bytes"
+	"compress/flate"
 	"compress/gzip"
 	"crypto/tls"
 	"fmt"
@@ -17,6 +18,8 @@ import (
 	"time"
 
 	"github.com/ffuf/ffuf/v2/pkg/ffuf"
+
+	"github.com/andybalholm/brotli"
 )
 
 // Download results < 5MB
@@ -47,7 +50,7 @@ func NewSimpleRunner(conf *ffuf.Config, replay bool) ffuf.RunnerProvider {
 
 	if conf.ClientCert != "" && conf.ClientKey != "" {
 		tmp, _ := tls.LoadX509KeyPair(conf.ClientCert, conf.ClientKey)
-		cert =  []tls.Certificate{tmp}
+		cert = []tls.Certificate{tmp}
 	}
 
 	simplerunner.config = conf
@@ -69,7 +72,7 @@ func NewSimpleRunner(conf *ffuf.Config, replay bool) ffuf.RunnerProvider {
 				MinVersion:         tls.VersionTLS10,
 				Renegotiation:      tls.RenegotiateOnceAsClient,
 				ServerName:         conf.SNI,
-				Certificates: cert,
+				Certificates:       cert,
 			},
 		}}
 
@@ -171,6 +174,18 @@ func (r *SimpleRunner) Execute(req *ffuf.Request) (ffuf.Response, error) {
 			// fallback to raw data
 			bodyReader = httpresp.Body
 		}
+	} else if httpresp.Header.Get("Content-Encoding") == "br" {
+		bodyReader = io.NopCloser(brotli.NewReader(httpresp.Body))
+		if err != nil {
+			// fallback to raw data
+			bodyReader = httpresp.Body
+		}
+	} else if httpresp.Header.Get("Content-Encoding") == "deflate" {
+		bodyReader = flate.NewReader(httpresp.Body)
+		if err != nil {
+			// fallback to raw data
+			bodyReader = httpresp.Body
+		}
 	} else {
 		bodyReader = httpresp.Body
 	}