Mirrors/ffuf
2
0
Fork 0
mirror of https://github.com/ffuf/ffuf synced 2024-11-10 06:04:17 +00:00
Code Issues Projects Releases Packages Wiki Activity

Automatic brotli and deflate decompression (#720)

Browse source
This commit is contained in:
Joona Hoikkala 2023-09-15 17:08:44 +03:00 committed by GitHub
parent 6731988cb5
commit 3fdb4e2b6a
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
4 changed files with 23 additions and 5 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
CHANGELOG.md
View file

@ -7,7 +7,7 @@
- Fix markdown output file format - Fix markdown output file format
- Fix csv output file format - Fix csv output file format
- Fixed divide by 0 error when setting rate limit to 0 manually. - Fixed divide by 0 error when setting rate limit to 0 manually.
- Automatic brotli and deflate decompression
- v2.0.0 - v2.0.0
- New - New

1
go.mod
View file

@ -5,6 +5,7 @@ go 1.17
require ( require (
github.com/PuerkitoBio/goquery v1.8.0 github.com/PuerkitoBio/goquery v1.8.0
github.com/adrg/xdg v0.4.0 github.com/adrg/xdg v0.4.0
github.com/andybalholm/brotli v1.0.5
github.com/ffuf/pencode v0.0.0-20230421231718-2cea7e60a693 github.com/ffuf/pencode v0.0.0-20230421231718-2cea7e60a693
github.com/pelletier/go-toml v1.9.5 github.com/pelletier/go-toml v1.9.5
) )

2
go.sum
View file

@ -2,6 +2,8 @@ github.com/PuerkitoBio/goquery v1.8.0 h1:PJTF7AmFCFKk1N6V6jmKfrNH9tV5pNE6lZMkG0g
github.com/PuerkitoBio/goquery v1.8.0/go.mod h1:ypIiRMtY7COPGk+I/YbZLbxsxn9g5ejnI2HSMtkjZvI= github.com/PuerkitoBio/goquery v1.8.0/go.mod h1:ypIiRMtY7COPGk+I/YbZLbxsxn9g5ejnI2HSMtkjZvI=
github.com/adrg/xdg v0.4.0 h1:RzRqFcjH4nE5C6oTAxhBtoE2IRyjBSa62SCbyPidvls= github.com/adrg/xdg v0.4.0 h1:RzRqFcjH4nE5C6oTAxhBtoE2IRyjBSa62SCbyPidvls=
github.com/adrg/xdg v0.4.0/go.mod h1:N6ag73EX4wyxeaoeHctc1mas01KZgsj5tYiAIwqJE/E= github.com/adrg/xdg v0.4.0/go.mod h1:N6ag73EX4wyxeaoeHctc1mas01KZgsj5tYiAIwqJE/E=
github.com/andybalholm/brotli v1.0.5 h1:8uQZIdzKmjc/iuPu7O2ioW48L81FgatrcpfFmiq/cCs=
github.com/andybalholm/brotli v1.0.5/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig=
github.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x004T2c= github.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x004T2c=
github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA= github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=

15
pkg/runner/simple.go
View file

@ -2,6 +2,7 @@ package runner
import ( import (
"bytes" "bytes"
"compress/flate"
"compress/gzip" "compress/gzip"
"crypto/tls" "crypto/tls"
"fmt" "fmt"
@ -17,6 +18,8 @@ import (
"time" "time"
"github.com/ffuf/ffuf/v2/pkg/ffuf" "github.com/ffuf/ffuf/v2/pkg/ffuf"
"github.com/andybalholm/brotli"
) )
// Download results < 5MB // Download results < 5MB
@ -171,6 +174,18 @@ func (r *SimpleRunner) Execute(req *ffuf.Request) (ffuf.Response, error) {
// fallback to raw data // fallback to raw data
bodyReader = httpresp.Body bodyReader = httpresp.Body
} }
} else if httpresp.Header.Get("Content-Encoding") == "br" {
bodyReader = io.NopCloser(brotli.NewReader(httpresp.Body))
if err != nil {
// fallback to raw data
bodyReader = httpresp.Body
}
} else if httpresp.Header.Get("Content-Encoding") == "deflate" {
bodyReader = flate.NewReader(httpresp.Body)
if err != nil {
// fallback to raw data
bodyReader = httpresp.Body
}
} else { } else {
bodyReader = httpresp.Body bodyReader = httpresp.Body
} }