mirror of
https://github.com/anchore/syft
synced 2024-11-10 14:24:12 +00:00
Add support for detecting nested java archives (#77)
* with sb build app * test nested jar support * pin jdk version during parse test (but dont compare version)
This commit is contained in:
Alex Goodman
GitHub
parent
e8d11eec69
commit
66a16a67fd
22 changed files with 1285 additions and 620 deletions
227
imgbom/cataloger/java/archive_parser.go
Normal file
227
imgbom/cataloger/java/archive_parser.go
Normal file
|
|
@ -0,0 +1,227 @@
|
|||
package java
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
"strings"
|
||||
|
||||
"github.com/anchore/imgbom/imgbom/pkg"
|
||||
"github.com/anchore/imgbom/internal"
|
||||
"github.com/anchore/imgbom/internal/file"
|
||||
)
|
||||
|
||||
var archiveFormatGlobs = []string{
|
||||
"*.jar",
|
||||
"*.war",
|
||||
"*.ear",
|
||||
"*.jpi",
|
||||
"*.hpi",
|
||||
}
|
||||
|
||||
type archiveParser struct {
|
||||
discoveredPkgs internal.StringSet
|
||||
fileManifest file.ZipFileManifest
|
||||
virtualPath string
|
||||
archivePath string
|
||||
contentPath string
|
||||
fileInfo archiveFilename
|
||||
detectNested bool
|
||||
}
|
||||
|
||||
func parseJavaArchive(virtualPath string, reader io.Reader) ([]pkg.Package, error) {
|
||||
parser, cleanupFn, err := newJavaArchiveParser(virtualPath, reader, true)
|
||||
// note: even on error, we should always run cleanup functions
|
||||
defer cleanupFn()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return parser.parse()
|
||||
}
|
||||
|
||||
func uniquePkgKey(p *pkg.Package) string {
|
||||
if p == nil {
|
||||
return ""
|
||||
}
|
||||
return fmt.Sprintf("%s|%s", p.Name, p.Version)
|
||||
}
|
||||
|
||||
func newJavaArchiveParser(virtualPath string, reader io.Reader, detectNested bool) (*archiveParser, func(), error) {
|
||||
contentPath, archivePath, cleanupFn, err := saveArchiveToTmp(reader)
|
||||
if err != nil {
|
||||
return nil, cleanupFn, fmt.Errorf("unable to process java archive: %w", err)
|
||||
}
|
||||
|
||||
fileManifest, err := file.NewZipFileManifest(archivePath)
|
||||
if err != nil {
|
||||
return nil, cleanupFn, fmt.Errorf("unable to read files from java archive: %w", err)
|
||||
}
|
||||
|
||||
return &archiveParser{
|
||||
discoveredPkgs: internal.NewStringSet(),
|
||||
fileManifest: fileManifest,
|
||||
virtualPath: virtualPath,
|
||||
archivePath: archivePath,
|
||||
contentPath: contentPath,
|
||||
fileInfo: newJavaArchiveFilename(virtualPath),
|
||||
detectNested: detectNested,
|
||||
}, cleanupFn, nil
|
||||
}
|
||||
|
||||
func (j *archiveParser) parse() ([]pkg.Package, error) {
|
||||
var pkgs = make([]pkg.Package, 0)
|
||||
|
||||
// find the parent package from the java manifest
|
||||
parentPkg, err := j.discoverMainPackage()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("could not generate package from %s: %w", j.virtualPath, err)
|
||||
}
|
||||
|
||||
// don't add the parent package yet, we still may discover aux info to add to the metadata (but still track it as added to prevent duplicates)
|
||||
parentKey := uniquePkgKey(parentPkg)
|
||||
if parentKey != "" {
|
||||
j.discoveredPkgs.Add(parentKey)
|
||||
}
|
||||
|
||||
// find aux packages from pom.properties
|
||||
auxPkgs, err := j.discoverPkgsFromPomProperties(parentPkg)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
pkgs = append(pkgs, auxPkgs...)
|
||||
|
||||
// find nested java archive packages
|
||||
nestedPkgs, err := j.discoverPkgsFromNestedArchives(parentPkg)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
pkgs = append(pkgs, nestedPkgs...)
|
||||
|
||||
// lastly, add the parent package to the list (assuming the parent exists)
|
||||
if parentPkg != nil {
|
||||
// only the parent package gets the type, nested packages may be of a different package type (or not of a package type at all, since they may not be bundled)
|
||||
parentPkg.Type = j.fileInfo.pkgType()
|
||||
pkgs = append([]pkg.Package{*parentPkg}, pkgs...)
|
||||
}
|
||||
|
||||
return pkgs, nil
|
||||
}
|
||||
|
||||
func (j *archiveParser) discoverMainPackage() (*pkg.Package, error) {
|
||||
// search and parse java manifest files
|
||||
manifestMatches := j.fileManifest.GlobMatch(manifestPath)
|
||||
if len(manifestMatches) > 1 {
|
||||
return nil, fmt.Errorf("found multiple manifests in the jar: %+v", manifestMatches)
|
||||
} else if len(manifestMatches) == 0 {
|
||||
// we did not find any manifests, but that may not be a problem (there may be other information to generate packages for)
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
// fetch the manifest file
|
||||
contents, err := file.ContentsFromZip(j.archivePath, manifestMatches...)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("unable to extract java manifests (%s): %w", j.virtualPath, err)
|
||||
}
|
||||
|
||||
// parse the manifest file into a rich object
|
||||
manifestContents := contents[manifestMatches[0]]
|
||||
manifest, err := parseJavaManifest(strings.NewReader(manifestContents))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to parse java manifest (%s): %w", j.virtualPath, err)
|
||||
}
|
||||
|
||||
return &pkg.Package{
|
||||
Name: selectName(manifest, j.fileInfo),
|
||||
Version: selectVersion(manifest, j.fileInfo),
|
||||
Language: pkg.Java,
|
||||
Metadata: pkg.JavaMetadata{
|
||||
Manifest: manifest,
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (j *archiveParser) discoverPkgsFromPomProperties(parentPkg *pkg.Package) ([]pkg.Package, error) {
|
||||
var pkgs = make([]pkg.Package, 0)
|
||||
parentKey := uniquePkgKey(parentPkg)
|
||||
|
||||
// search and parse pom.properties files & fetch the contents
|
||||
contents, err := file.ContentsFromZip(j.archivePath, j.fileManifest.GlobMatch(pomPropertiesGlob)...)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("unable to extract pom.properties: %w", err)
|
||||
}
|
||||
|
||||
// parse the manifest file into a rich object
|
||||
for propsPath, propsContents := range contents {
|
||||
propsObj, err := parsePomProperties(propsPath, strings.NewReader(propsContents))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to parse pom.properties (%s): %w", j.virtualPath, err)
|
||||
}
|
||||
|
||||
if propsObj != nil {
|
||||
if propsObj.Version != "" && propsObj.ArtifactID != "" {
|
||||
// TODO: if there is no parentPkg (no java manifest) one of these poms could be the parent. We should discover the right parent and attach the correct info accordingly to each discovered package
|
||||
|
||||
// discovered props = new package
|
||||
p := pkg.Package{
|
||||
Name: propsObj.ArtifactID,
|
||||
Version: propsObj.Version,
|
||||
Language: pkg.Java,
|
||||
Metadata: pkg.JavaMetadata{
|
||||
PomProperties: propsObj,
|
||||
Parent: parentPkg,
|
||||
},
|
||||
}
|
||||
|
||||
pkgKey := uniquePkgKey(&p)
|
||||
|
||||
if !j.discoveredPkgs.Contains(pkgKey) {
|
||||
// only keep packages we haven't seen yet
|
||||
pkgs = append(pkgs, p)
|
||||
} else if pkgKey == parentKey {
|
||||
// we've run across more information about our parent package, add this info to the parent package metadata
|
||||
parentMetadata, ok := parentPkg.Metadata.(pkg.JavaMetadata)
|
||||
if ok {
|
||||
parentMetadata.PomProperties = propsObj
|
||||
parentPkg.Metadata = parentMetadata
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return pkgs, nil
|
||||
}
|
||||
|
||||
func (j *archiveParser) discoverPkgsFromNestedArchives(parentPkg *pkg.Package) ([]pkg.Package, error) {
|
||||
var pkgs = make([]pkg.Package, 0)
|
||||
|
||||
if !j.detectNested {
|
||||
return pkgs, nil
|
||||
}
|
||||
|
||||
// search and parse pom.properties files & fetch the contents
|
||||
readers, err := file.ExtractFromZipToUniqueTempFile(j.archivePath, j.contentPath, j.fileManifest.GlobMatch(archiveFormatGlobs...)...)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("unable to extract files from zip: %w", err)
|
||||
}
|
||||
|
||||
// discover nested artifacts
|
||||
for archivePath, archiveReader := range readers {
|
||||
nestedPath := fmt.Sprintf("%s:%s", j.virtualPath, archivePath)
|
||||
nestedPkgs, err := parseJavaArchive(nestedPath, archiveReader)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("unable to process nested java archive (%s): %w", archivePath, err)
|
||||
}
|
||||
|
||||
// attach the parent package to all discovered packages that are not already associated with a java archive
|
||||
for _, p := range nestedPkgs {
|
||||
if metadata, ok := p.Metadata.(pkg.JavaMetadata); ok {
|
||||
if metadata.Parent == nil {
|
||||
metadata.Parent = parentPkg
|
||||
}
|
||||
p.Metadata = metadata
|
||||
}
|
||||
pkgs = append(pkgs, p)
|
||||
}
|
||||
}
|
||||
|
||||
return pkgs, nil
|
||||
}
|
||||
535
imgbom/cataloger/java/archive_parser_test.go
Normal file
535
imgbom/cataloger/java/archive_parser_test.go
Normal file
|
|
@ -0,0 +1,535 @@
|
|||
package java
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"syscall"
|
||||
"testing"
|
||||
|
||||
"github.com/anchore/imgbom/internal"
|
||||
|
||||
"github.com/anchore/imgbom/imgbom/pkg"
|
||||
"github.com/go-test/deep"
|
||||
"github.com/gookit/color"
|
||||
)
|
||||
|
||||
func generateJavaBuildFixture(t *testing.T, fixturePath string) {
|
||||
if _, err := os.Stat(fixturePath); !os.IsNotExist(err) {
|
||||
// fixture already exists...
|
||||
return
|
||||
}
|
||||
|
||||
makeTask := strings.TrimPrefix(fixturePath, "test-fixtures/java-builds/")
|
||||
t.Logf(color.Bold.Sprintf("Generating Fixture from 'make %s'", makeTask))
|
||||
|
||||
cwd, err := os.Getwd()
|
||||
if err != nil {
|
||||
t.Errorf("unable to get cwd: %+v", err)
|
||||
}
|
||||
|
||||
cmd := exec.Command("make", makeTask)
|
||||
cmd.Dir = filepath.Join(cwd, "test-fixtures/java-builds/")
|
||||
|
||||
stderr, err := cmd.StderrPipe()
|
||||
if err != nil {
|
||||
t.Fatalf("could not get stderr: %+v", err)
|
||||
}
|
||||
stdout, err := cmd.StdoutPipe()
|
||||
if err != nil {
|
||||
t.Fatalf("could not get stdout: %+v", err)
|
||||
}
|
||||
|
||||
err = cmd.Start()
|
||||
if err != nil {
|
||||
t.Fatalf("failed to start cmd: %+v", err)
|
||||
}
|
||||
|
||||
show := func(label string, reader io.ReadCloser) {
|
||||
scanner := bufio.NewScanner(reader)
|
||||
scanner.Split(bufio.ScanLines)
|
||||
for scanner.Scan() {
|
||||
t.Logf("%s: %s", label, scanner.Text())
|
||||
}
|
||||
}
|
||||
go show("out", stdout)
|
||||
go show("err", stderr)
|
||||
|
||||
if err := cmd.Wait(); err != nil {
|
||||
if exiterr, ok := err.(*exec.ExitError); ok {
|
||||
// The program has exited with an exit code != 0
|
||||
|
||||
// This works on both Unix and Windows. Although package
|
||||
// syscall is generally platform dependent, WaitStatus is
|
||||
// defined for both Unix and Windows and in both cases has
|
||||
// an ExitStatus() method with the same signature.
|
||||
if status, ok := exiterr.Sys().(syscall.WaitStatus); ok {
|
||||
if status.ExitStatus() != 0 {
|
||||
t.Fatalf("failed to generate fixture: rc=%d", status.ExitStatus())
|
||||
}
|
||||
}
|
||||
} else {
|
||||
t.Fatalf("unable to get generate fixture result: %+v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseJar(t *testing.T) {
|
||||
tests := []struct {
|
||||
fixture string
|
||||
expected map[string]pkg.Package
|
||||
ignoreExtras []string
|
||||
}{
|
||||
{
|
||||
fixture: "test-fixtures/java-builds/packages/example-jenkins-plugin.hpi",
|
||||
ignoreExtras: []string{
|
||||
"Plugin-Version", // has dynamic date
|
||||
"Build-Jdk", // can't guarantee the JDK used at build time
|
||||
},
|
||||
expected: map[string]pkg.Package{
|
||||
"example-jenkins-plugin": {
|
||||
Name: "example-jenkins-plugin",
|
||||
Version: "1.0-SNAPSHOT",
|
||||
Language: pkg.Java,
|
||||
Type: pkg.JenkinsPluginPkg,
|
||||
Metadata: pkg.JavaMetadata{
|
||||
Manifest: &pkg.JavaManifest{
|
||||
ManifestVersion: "1.0",
|
||||
SpecTitle: "The Jenkins Plugins Parent POM Project",
|
||||
ImplTitle: "example-jenkins-plugin",
|
||||
ImplVersion: "1.0-SNAPSHOT",
|
||||
Extra: map[string]string{
|
||||
"Archiver-Version": "Plexus Archiver",
|
||||
"Plugin-License-Url": "https://opensource.org/licenses/MIT",
|
||||
"Plugin-License-Name": "MIT License",
|
||||
"Created-By": "Apache Maven",
|
||||
"Built-By": "?",
|
||||
//"Build-Jdk": "14.0.1",
|
||||
"Jenkins-Version": "2.164.3",
|
||||
"Minimum-Java-Version": "1.8",
|
||||
"Plugin-Developers": "",
|
||||
"Plugin-ScmUrl": "https://github.com/jenkinsci/plugin-pom/example-jenkins-plugin",
|
||||
"Extension-Name": "example-jenkins-plugin",
|
||||
"Short-Name": "example-jenkins-plugin",
|
||||
"Group-Id": "io.jenkins.plugins",
|
||||
"Plugin-Dependencies": "structs:1.20",
|
||||
//"Plugin-Version": "1.0-SNAPSHOT (private-07/09/2020 13:30-?)",
|
||||
"Hudson-Version": "2.164.3",
|
||||
"Long-Name": "TODO Plugin",
|
||||
},
|
||||
},
|
||||
PomProperties: &pkg.PomProperties{
|
||||
Path: "META-INF/maven/io.jenkins.plugins/example-jenkins-plugin/pom.properties",
|
||||
GroupID: "io.jenkins.plugins",
|
||||
ArtifactID: "example-jenkins-plugin",
|
||||
Version: "1.0-SNAPSHOT",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
fixture: "test-fixtures/java-builds/packages/example-java-app-gradle-0.1.0.jar",
|
||||
expected: map[string]pkg.Package{
|
||||
"example-java-app-gradle": {
|
||||
Name: "example-java-app-gradle",
|
||||
Version: "0.1.0",
|
||||
Language: pkg.Java,
|
||||
Type: pkg.JavaPkg,
|
||||
Metadata: pkg.JavaMetadata{
|
||||
Manifest: &pkg.JavaManifest{
|
||||
ManifestVersion: "1.0",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
fixture: "test-fixtures/java-builds/packages/example-java-app-maven-0.1.0.jar",
|
||||
ignoreExtras: []string{
|
||||
"Build-Jdk", // can't guarantee the JDK used at build time
|
||||
},
|
||||
expected: map[string]pkg.Package{
|
||||
"example-java-app-maven": {
|
||||
Name: "example-java-app-maven",
|
||||
Version: "0.1.0",
|
||||
Language: pkg.Java,
|
||||
Type: pkg.JavaPkg,
|
||||
Metadata: pkg.JavaMetadata{
|
||||
Manifest: &pkg.JavaManifest{
|
||||
ManifestVersion: "1.0",
|
||||
Extra: map[string]string{
|
||||
"Archiver-Version": "Plexus Archiver",
|
||||
"Created-By": "Apache Maven 3.6.3",
|
||||
"Built-By": "?",
|
||||
//"Build-Jdk": "14.0.1",
|
||||
"Main-Class": "hello.HelloWorld",
|
||||
},
|
||||
},
|
||||
PomProperties: &pkg.PomProperties{
|
||||
Path: "META-INF/maven/org.anchore/example-java-app-maven/pom.properties",
|
||||
GroupID: "org.anchore",
|
||||
ArtifactID: "example-java-app-maven",
|
||||
Version: "0.1.0",
|
||||
},
|
||||
},
|
||||
},
|
||||
"joda-time": {
|
||||
Name: "joda-time",
|
||||
Version: "2.9.2",
|
||||
Language: pkg.Java,
|
||||
Type: pkg.UnknownPkg,
|
||||
Metadata: pkg.JavaMetadata{
|
||||
PomProperties: &pkg.PomProperties{
|
||||
Path: "META-INF/maven/joda-time/joda-time/pom.properties",
|
||||
GroupID: "joda-time",
|
||||
ArtifactID: "joda-time",
|
||||
Version: "2.9.2",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.fixture, func(t *testing.T) {
|
||||
|
||||
generateJavaBuildFixture(t, test.fixture)
|
||||
|
||||
fixture, err := os.Open(test.fixture)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to open fixture: %+v", err)
|
||||
}
|
||||
|
||||
parser, cleanupFn, err := newJavaArchiveParser(fixture.Name(), fixture, false)
|
||||
defer cleanupFn()
|
||||
if err != nil {
|
||||
t.Fatalf("should not have filed... %+v", err)
|
||||
}
|
||||
|
||||
actual, err := parser.parse()
|
||||
if err != nil {
|
||||
t.Fatalf("failed to parse java archive: %+v", err)
|
||||
}
|
||||
|
||||
if len(actual) != len(test.expected) {
|
||||
for _, a := range actual {
|
||||
t.Log(" ", a)
|
||||
}
|
||||
t.Fatalf("unexpected package count: %d!=%d", len(actual), len(test.expected))
|
||||
}
|
||||
|
||||
var parent *pkg.Package
|
||||
for _, a := range actual {
|
||||
if strings.Contains(a.Name, "example-") {
|
||||
parent = &a
|
||||
}
|
||||
}
|
||||
|
||||
if parent == nil {
|
||||
t.Fatal("could not find the parent pkg")
|
||||
}
|
||||
|
||||
for _, a := range actual {
|
||||
e, ok := test.expected[a.Name]
|
||||
if !ok {
|
||||
t.Errorf("entry not found: %s", a.Name)
|
||||
continue
|
||||
}
|
||||
|
||||
if a.Name != parent.Name && a.Metadata.(pkg.JavaMetadata).Parent != nil && a.Metadata.(pkg.JavaMetadata).Parent.Name != parent.Name {
|
||||
t.Errorf("mismatched parent: %+v", a.Metadata.(pkg.JavaMetadata).Parent)
|
||||
}
|
||||
|
||||
// we need to compare the other fields without parent attached
|
||||
metadata := a.Metadata.(pkg.JavaMetadata)
|
||||
metadata.Parent = nil
|
||||
|
||||
// ignore select fields
|
||||
for _, field := range test.ignoreExtras {
|
||||
if metadata.Manifest != nil && metadata.Manifest.Extra != nil {
|
||||
if _, ok := metadata.Manifest.Extra[field]; ok {
|
||||
delete(metadata.Manifest.Extra, field)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// write censored data back
|
||||
a.Metadata = metadata
|
||||
|
||||
diffs := deep.Equal(a, e)
|
||||
if len(diffs) > 0 {
|
||||
t.Errorf("diffs found for %q", a.Name)
|
||||
for _, d := range diffs {
|
||||
t.Errorf("diff: %+v", d)
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseNestedJar(t *testing.T) {
|
||||
tests := []struct {
|
||||
fixture string
|
||||
expected []pkg.Package
|
||||
ignoreExtras []string
|
||||
}{
|
||||
{
|
||||
fixture: "test-fixtures/java-builds/packages/spring-boot-0.0.1-SNAPSHOT.jar",
|
||||
expected: []pkg.Package{
|
||||
{
|
||||
Name: "spring-boot",
|
||||
Version: "0.0.1-SNAPSHOT",
|
||||
},
|
||||
{
|
||||
Name: "spring-boot-starter",
|
||||
Version: "2.2.2.RELEASE",
|
||||
},
|
||||
{
|
||||
Name: "jul-to-slf4j",
|
||||
Version: "1.7.29",
|
||||
},
|
||||
{
|
||||
Name: "tomcat-embed-websocket",
|
||||
Version: "9.0.29",
|
||||
},
|
||||
{
|
||||
Name: "spring-boot-starter-validation",
|
||||
Version: "2.2.2.RELEASE",
|
||||
},
|
||||
{
|
||||
Name: "hibernate-validator",
|
||||
Version: "6.0.18.Final",
|
||||
},
|
||||
{
|
||||
Name: "jboss-logging",
|
||||
Version: "3.4.1.Final",
|
||||
},
|
||||
{
|
||||
Name: "spring-expression",
|
||||
Version: "5.2.2.RELEASE",
|
||||
},
|
||||
{
|
||||
Name: "jakarta.validation-api",
|
||||
Version: "2.0.1",
|
||||
},
|
||||
{
|
||||
Name: "spring-web",
|
||||
Version: "5.2.2.RELEASE",
|
||||
},
|
||||
{
|
||||
Name: "spring-boot-starter-actuator",
|
||||
Version: "2.2.2.RELEASE",
|
||||
},
|
||||
{
|
||||
Name: "log4j-api",
|
||||
Version: "2.12.1",
|
||||
},
|
||||
{
|
||||
Name: "snakeyaml",
|
||||
Version: "1.25",
|
||||
},
|
||||
{
|
||||
Name: "jackson-core",
|
||||
Version: "2.10.1",
|
||||
},
|
||||
{
|
||||
Name: "jackson-datatype-jsr310",
|
||||
Version: "2.10.1",
|
||||
},
|
||||
{
|
||||
Name: "spring-aop",
|
||||
Version: "5.2.2.RELEASE",
|
||||
},
|
||||
{
|
||||
Name: "spring-boot-actuator-autoconfigure",
|
||||
Version: "2.2.2.RELEASE",
|
||||
},
|
||||
{
|
||||
Name: "spring-jcl",
|
||||
Version: "5.2.2.RELEASE",
|
||||
},
|
||||
{
|
||||
Name: "spring-boot",
|
||||
Version: "2.2.2.RELEASE",
|
||||
},
|
||||
{
|
||||
Name: "spring-boot-starter-logging",
|
||||
Version: "2.2.2.RELEASE",
|
||||
},
|
||||
{
|
||||
Name: "jakarta.annotation-api",
|
||||
Version: "1.3.5",
|
||||
},
|
||||
{
|
||||
Name: "spring-webmvc",
|
||||
Version: "5.2.2.RELEASE",
|
||||
},
|
||||
{
|
||||
Name: "HdrHistogram",
|
||||
Version: "2.1.11",
|
||||
},
|
||||
{
|
||||
Name: "spring-boot-starter-web",
|
||||
Version: "2.2.2.RELEASE",
|
||||
},
|
||||
{
|
||||
Name: "logback-classic",
|
||||
Version: "1.2.3",
|
||||
},
|
||||
{
|
||||
Name: "log4j-to-slf4j",
|
||||
Version: "2.12.1",
|
||||
},
|
||||
{
|
||||
Name: "spring-boot-starter-json",
|
||||
Version: "2.2.2.RELEASE",
|
||||
},
|
||||
{
|
||||
Name: "jackson-databind",
|
||||
Version: "2.10.1",
|
||||
},
|
||||
{
|
||||
Name: "jackson-module-parameter-names",
|
||||
Version: "2.10.1",
|
||||
},
|
||||
{
|
||||
Name: "LatencyUtils",
|
||||
Version: "2.0.3",
|
||||
},
|
||||
{
|
||||
Name: "spring-boot-autoconfigure",
|
||||
Version: "2.2.2.RELEASE",
|
||||
},
|
||||
{
|
||||
Name: "jackson-datatype-jdk8",
|
||||
Version: "2.10.1",
|
||||
},
|
||||
{
|
||||
Name: "tomcat-embed-core",
|
||||
Version: "9.0.29",
|
||||
},
|
||||
{
|
||||
Name: "tomcat-embed-el",
|
||||
Version: "9.0.29",
|
||||
},
|
||||
{
|
||||
Name: "spring-beans",
|
||||
Version: "5.2.2.RELEASE",
|
||||
},
|
||||
{
|
||||
Name: "spring-boot-actuator",
|
||||
Version: "2.2.2.RELEASE",
|
||||
},
|
||||
{
|
||||
Name: "slf4j-api",
|
||||
Version: "1.7.29",
|
||||
},
|
||||
{
|
||||
Name: "spring-core",
|
||||
Version: "5.2.2.RELEASE",
|
||||
},
|
||||
{
|
||||
Name: "logback-core",
|
||||
Version: "1.2.3",
|
||||
},
|
||||
{
|
||||
Name: "micrometer-core",
|
||||
Version: "1.3.1",
|
||||
},
|
||||
{
|
||||
Name: "pcollections",
|
||||
Version: "3.1.0",
|
||||
},
|
||||
{
|
||||
Name: "jackson-annotations",
|
||||
Version: "2.10.1",
|
||||
},
|
||||
{
|
||||
Name: "spring-boot-starter-tomcat",
|
||||
Version: "2.2.2.RELEASE",
|
||||
},
|
||||
{
|
||||
Name: "classmate",
|
||||
Version: "1.5.1",
|
||||
},
|
||||
{
|
||||
Name: "spring-context",
|
||||
Version: "5.2.2.RELEASE",
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.fixture, func(t *testing.T) {
|
||||
|
||||
generateJavaBuildFixture(t, test.fixture)
|
||||
|
||||
fixture, err := os.Open(test.fixture)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to open fixture: %+v", err)
|
||||
}
|
||||
|
||||
actual, err := parseJavaArchive(fixture.Name(), fixture)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to parse java archive: %+v", err)
|
||||
}
|
||||
|
||||
nameVersionPairSet := internal.NewStringSet()
|
||||
|
||||
makeKey := func(p *pkg.Package) string {
|
||||
if p == nil {
|
||||
t.Fatal("cannot make key for nil pkg")
|
||||
}
|
||||
return fmt.Sprintf("%s|%s", p.Name, p.Version)
|
||||
}
|
||||
|
||||
for _, e := range test.expected {
|
||||
nameVersionPairSet.Add(makeKey(&e))
|
||||
}
|
||||
|
||||
if len(actual) != len(nameVersionPairSet) {
|
||||
for _, a := range actual {
|
||||
t.Log(" ", a)
|
||||
}
|
||||
t.Fatalf("unexpected package count: %d!=%d", len(actual), len(nameVersionPairSet))
|
||||
}
|
||||
|
||||
for _, a := range actual {
|
||||
actualKey := makeKey(&a)
|
||||
|
||||
if !nameVersionPairSet.Contains(actualKey) {
|
||||
t.Errorf("unexpected pkg: %q", actualKey)
|
||||
}
|
||||
|
||||
metadata := a.Metadata.(pkg.JavaMetadata)
|
||||
if actualKey == "spring-boot|0.0.1-SNAPSHOT" {
|
||||
if metadata.Parent != nil {
|
||||
t.Errorf("expected no parent for root pkg, got %q", makeKey(metadata.Parent))
|
||||
}
|
||||
} else {
|
||||
if metadata.Parent == nil {
|
||||
t.Errorf("unassigned error for pkg=%q", actualKey)
|
||||
} else if makeKey(metadata.Parent) != "spring-boot|0.0.1-SNAPSHOT" {
|
||||
// NB: this is a hard-coded condition to simplify the test harness
|
||||
if a.Name == "pcollections" {
|
||||
if metadata.Parent.Name != "micrometer-core" {
|
||||
t.Errorf("nested 'pcollections' pkg has wrong parent: %q", metadata.Parent.Name)
|
||||
}
|
||||
} else {
|
||||
t.Errorf("bad parent for pkg=%q parent=%q", actualKey, makeKey(metadata.Parent))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
|
@ -12,12 +12,9 @@ type Cataloger struct {
|
|||
}
|
||||
|
||||
func NewCataloger() *Cataloger {
|
||||
globParsers := map[string]common.ParserFn{
|
||||
"*.jar": parseJavaArchive,
|
||||
"*.war": parseJavaArchive,
|
||||
"*.ear": parseJavaArchive,
|
||||
"*.jpi": parseJavaArchive,
|
||||
"*.hpi": parseJavaArchive,
|
||||
globParsers := make(map[string]common.ParserFn)
|
||||
for _, pattern := range archiveFormatGlobs {
|
||||
globParsers[pattern] = parseJavaArchive
|
||||
}
|
||||
|
||||
return &Cataloger{
|
||||
|
|
|
|||
|
|
@ -6,8 +6,6 @@ import (
|
|||
"io"
|
||||
"strings"
|
||||
|
||||
"github.com/anchore/imgbom/internal/file"
|
||||
|
||||
"github.com/anchore/imgbom/imgbom/pkg"
|
||||
"github.com/mitchellh/mapstructure"
|
||||
)
|
||||
|
|
@ -101,38 +99,3 @@ func selectVersion(manifest *pkg.JavaManifest, filenameObj archiveFilename) stri
|
|||
}
|
||||
return version
|
||||
}
|
||||
|
||||
func newPackageFromJavaManifest(virtualPath, archivePath string, fileManifest file.ZipManifest) (*pkg.Package, error) {
|
||||
// search and parse java manifest files
|
||||
manifestMatches := fileManifest.GlobMatch(manifestPath)
|
||||
if len(manifestMatches) > 1 {
|
||||
return nil, fmt.Errorf("found multiple manifests in the jar: %+v", manifestMatches)
|
||||
} else if len(manifestMatches) == 0 {
|
||||
// we did not find any manifests, but that may not be a problem (there may be other information to generate packages for)
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
// fetch the manifest file
|
||||
contents, err := file.ExtractFilesFromZip(archivePath, manifestMatches...)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("unable to extract java manifests (%s): %w", virtualPath, err)
|
||||
}
|
||||
|
||||
// parse the manifest file into a rich object
|
||||
manifestContents := contents[manifestMatches[0]]
|
||||
manifest, err := parseJavaManifest(strings.NewReader(manifestContents))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to parse java manifest (%s): %w", virtualPath, err)
|
||||
}
|
||||
|
||||
filenameObj := newJavaArchiveFilename(virtualPath)
|
||||
|
||||
return &pkg.Package{
|
||||
Name: selectName(manifest, filenameObj),
|
||||
Version: selectVersion(manifest, filenameObj),
|
||||
Language: pkg.Java,
|
||||
Metadata: pkg.JavaMetadata{
|
||||
Manifest: manifest,
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,116 +0,0 @@
|
|||
package java
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
"strings"
|
||||
|
||||
"github.com/anchore/imgbom/imgbom/pkg"
|
||||
"github.com/anchore/imgbom/internal"
|
||||
"github.com/anchore/imgbom/internal/file"
|
||||
)
|
||||
|
||||
func uniquePkgKey(p *pkg.Package) string {
|
||||
if p == nil {
|
||||
return ""
|
||||
}
|
||||
return fmt.Sprintf("%s|%s", p.Name, p.Version)
|
||||
}
|
||||
|
||||
func parseJavaArchive(virtualPath string, reader io.Reader) ([]pkg.Package, error) {
|
||||
var pkgs = make([]pkg.Package, 0)
|
||||
discoveredPkgs := internal.NewStringSet()
|
||||
|
||||
_, archivePath, cleanupFn, err := saveArchiveToTmp(reader)
|
||||
// note: even on error, we should always run cleanup functions
|
||||
defer cleanupFn()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("unable to process jar: %w", err)
|
||||
}
|
||||
|
||||
fileManifest, err := file.ZipFileManifest(archivePath)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("unable to read files from jar: %w", err)
|
||||
}
|
||||
|
||||
// find the parent package from the java manifest
|
||||
parentPkg, err := newPackageFromJavaManifest(virtualPath, archivePath, fileManifest)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("could not generate package from %s: %w", virtualPath, err)
|
||||
}
|
||||
|
||||
// don't add the parent package yet, we still may discover aux info to add to the metadata (but still track it as added to prevent duplicates)
|
||||
parentKey := uniquePkgKey(parentPkg)
|
||||
if parentKey != "" {
|
||||
discoveredPkgs.Add(parentKey)
|
||||
}
|
||||
|
||||
// find aux packages from pom.properties
|
||||
auxPkgs, err := newPackagesFromPomProperties(parentPkg, discoveredPkgs, virtualPath, archivePath, fileManifest)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
pkgs = append(pkgs, auxPkgs...)
|
||||
|
||||
// TODO: search for nested jars... but only in ears? or all the time? and remember we need to capture pkg metadata and type appropriately for each
|
||||
|
||||
// lastly, add the parent package to the list (assuming the parent exists)
|
||||
if parentPkg != nil {
|
||||
// only the parent package gets the type, nested packages may be of a different package type (or not of a package type at all, since they may not be bundled)
|
||||
parentPkg.Type = newJavaArchiveFilename(virtualPath).pkgType()
|
||||
pkgs = append([]pkg.Package{*parentPkg}, pkgs...)
|
||||
}
|
||||
|
||||
return pkgs, nil
|
||||
}
|
||||
|
||||
func newPackagesFromPomProperties(parentPkg *pkg.Package, discoveredPkgs internal.StringSet, virtualPath, archivePath string, fileManifest file.ZipManifest) ([]pkg.Package, error) {
|
||||
var pkgs = make([]pkg.Package, 0)
|
||||
parentKey := uniquePkgKey(parentPkg)
|
||||
|
||||
// search and parse pom.properties files & fetch the contents
|
||||
contents, err := file.ExtractFilesFromZip(archivePath, fileManifest.GlobMatch(pomPropertiesGlob)...)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("unable to extract pom.properties: %w", err)
|
||||
}
|
||||
|
||||
// parse the manifest file into a rich object
|
||||
for propsPath, propsContents := range contents {
|
||||
propsObj, err := parsePomProperties(propsPath, strings.NewReader(propsContents))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to parse pom.properties (%s): %w", virtualPath, err)
|
||||
}
|
||||
|
||||
if propsObj != nil {
|
||||
if propsObj.Version != "" && propsObj.ArtifactID != "" {
|
||||
// TODO: if there is no parentPkg (no java manifest) one of these poms could be the parent. We should discover the right parent and attach the correct info accordingly to each discovered package
|
||||
|
||||
// discovered props = new package
|
||||
p := pkg.Package{
|
||||
Name: propsObj.ArtifactID,
|
||||
Version: propsObj.Version,
|
||||
Language: pkg.Java,
|
||||
Metadata: pkg.JavaMetadata{
|
||||
PomProperties: propsObj,
|
||||
Parent: parentPkg,
|
||||
},
|
||||
}
|
||||
|
||||
pkgKey := uniquePkgKey(&p)
|
||||
|
||||
if !discoveredPkgs.Contains(pkgKey) {
|
||||
// only keep packages we haven't seen yet
|
||||
pkgs = append(pkgs, p)
|
||||
} else if pkgKey == parentKey {
|
||||
// we've run across more information about our parent package, add this info to the parent package metadata
|
||||
parentMetadata, ok := parentPkg.Metadata.(pkg.JavaMetadata)
|
||||
if ok {
|
||||
parentMetadata.PomProperties = propsObj
|
||||
parentPkg.Metadata = parentMetadata
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return pkgs, nil
|
||||
}
|
||||
|
|
@ -1,255 +0,0 @@
|
|||
package java
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"github.com/anchore/imgbom/imgbom/pkg"
|
||||
"github.com/go-test/deep"
|
||||
"github.com/gookit/color"
|
||||
"io"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"syscall"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func generateJavaBuildFixture(t *testing.T, fixturePath string) {
|
||||
if _, err := os.Stat(fixturePath); !os.IsNotExist(err) {
|
||||
// fixture already exists...
|
||||
return
|
||||
}
|
||||
|
||||
makeTask := strings.TrimPrefix(fixturePath, "test-fixtures/java-builds/")
|
||||
t.Logf(color.Bold.Sprintf("Generating Fixture from 'make %s'", makeTask))
|
||||
|
||||
cwd, err := os.Getwd()
|
||||
if err != nil {
|
||||
t.Errorf("unable to get cwd: %+v", err)
|
||||
}
|
||||
|
||||
cmd := exec.Command("make", makeTask)
|
||||
cmd.Dir = filepath.Join(cwd, "test-fixtures/java-builds/")
|
||||
|
||||
stderr, err := cmd.StderrPipe()
|
||||
if err != nil {
|
||||
t.Fatalf("could not get stderr: %+v", err)
|
||||
}
|
||||
stdout, err := cmd.StdoutPipe()
|
||||
if err != nil {
|
||||
t.Fatalf("could not get stdout: %+v", err)
|
||||
}
|
||||
|
||||
err = cmd.Start()
|
||||
if err != nil {
|
||||
t.Fatalf("failed to start cmd: %+v", err)
|
||||
}
|
||||
|
||||
show := func(label string, reader io.ReadCloser) {
|
||||
scanner := bufio.NewScanner(reader)
|
||||
scanner.Split(bufio.ScanLines)
|
||||
for scanner.Scan() {
|
||||
t.Logf("%s: %s", label, scanner.Text())
|
||||
}
|
||||
}
|
||||
go show("out", stdout)
|
||||
go show("err", stderr)
|
||||
|
||||
if err := cmd.Wait(); err != nil {
|
||||
if exiterr, ok := err.(*exec.ExitError); ok {
|
||||
// The program has exited with an exit code != 0
|
||||
|
||||
// This works on both Unix and Windows. Although package
|
||||
// syscall is generally platform dependent, WaitStatus is
|
||||
// defined for both Unix and Windows and in both cases has
|
||||
// an ExitStatus() method with the same signature.
|
||||
if status, ok := exiterr.Sys().(syscall.WaitStatus); ok {
|
||||
if status.ExitStatus() != 0 {
|
||||
t.Fatalf("failed to generate fixture: rc=%d", status.ExitStatus())
|
||||
}
|
||||
}
|
||||
} else {
|
||||
t.Fatalf("unable to get generate fixture result: %+v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseJar(t *testing.T) {
|
||||
tests := []struct {
|
||||
fixture string
|
||||
expected map[string]pkg.Package
|
||||
ignoreExtras []string
|
||||
}{
|
||||
{
|
||||
fixture: "test-fixtures/java-builds/packages/example-jenkins-plugin.hpi",
|
||||
ignoreExtras: []string{"Plugin-Version"}, // has dynamic date
|
||||
expected: map[string]pkg.Package{
|
||||
"example-jenkins-plugin": {
|
||||
Name: "example-jenkins-plugin",
|
||||
Version: "1.0-SNAPSHOT",
|
||||
Language: pkg.Java,
|
||||
Type: pkg.JenkinsPluginPkg,
|
||||
Metadata: pkg.JavaMetadata{
|
||||
Manifest: &pkg.JavaManifest{
|
||||
ManifestVersion: "1.0",
|
||||
SpecTitle: "The Jenkins Plugins Parent POM Project",
|
||||
ImplTitle: "example-jenkins-plugin",
|
||||
ImplVersion: "1.0-SNAPSHOT",
|
||||
Extra: map[string]string{
|
||||
"Archiver-Version": "Plexus Archiver",
|
||||
"Plugin-License-Url": "https://opensource.org/licenses/MIT",
|
||||
"Plugin-License-Name": "MIT License",
|
||||
"Created-By": "Apache Maven",
|
||||
"Built-By": "?",
|
||||
"Build-Jdk": "14.0.1",
|
||||
"Jenkins-Version": "2.164.3",
|
||||
"Minimum-Java-Version": "1.8",
|
||||
"Plugin-Developers": "",
|
||||
"Plugin-ScmUrl": "https://github.com/jenkinsci/plugin-pom/example-jenkins-plugin",
|
||||
"Extension-Name": "example-jenkins-plugin",
|
||||
"Short-Name": "example-jenkins-plugin",
|
||||
"Group-Id": "io.jenkins.plugins",
|
||||
"Plugin-Dependencies": "structs:1.20",
|
||||
//"Plugin-Version": "1.0-SNAPSHOT (private-07/09/2020 13:30-?)",
|
||||
"Hudson-Version": "2.164.3",
|
||||
"Long-Name": "TODO Plugin",
|
||||
},
|
||||
},
|
||||
PomProperties: &pkg.PomProperties{
|
||||
Path: "META-INF/maven/io.jenkins.plugins/example-jenkins-plugin/pom.properties",
|
||||
GroupID: "io.jenkins.plugins",
|
||||
ArtifactID: "example-jenkins-plugin",
|
||||
Version: "1.0-SNAPSHOT",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
fixture: "test-fixtures/java-builds/packages/example-java-app-gradle-0.1.0.jar",
|
||||
expected: map[string]pkg.Package{
|
||||
"example-java-app-gradle": {
|
||||
Name: "example-java-app-gradle",
|
||||
Version: "0.1.0",
|
||||
Language: pkg.Java,
|
||||
Type: pkg.JavaPkg,
|
||||
Metadata: pkg.JavaMetadata{
|
||||
Manifest: &pkg.JavaManifest{
|
||||
ManifestVersion: "1.0",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
fixture: "test-fixtures/java-builds/packages/example-java-app-maven-0.1.0.jar",
|
||||
expected: map[string]pkg.Package{
|
||||
"example-java-app-maven": {
|
||||
Name: "example-java-app-maven",
|
||||
Version: "0.1.0",
|
||||
Language: pkg.Java,
|
||||
Type: pkg.JavaPkg,
|
||||
Metadata: pkg.JavaMetadata{
|
||||
Manifest: &pkg.JavaManifest{
|
||||
ManifestVersion: "1.0",
|
||||
Extra: map[string]string{
|
||||
"Archiver-Version": "Plexus Archiver",
|
||||
"Created-By": "Apache Maven 3.6.3",
|
||||
"Built-By": "?",
|
||||
"Build-Jdk": "14.0.1",
|
||||
"Main-Class": "hello.HelloWorld",
|
||||
},
|
||||
},
|
||||
PomProperties: &pkg.PomProperties{
|
||||
Path: "META-INF/maven/org.anchore/example-java-app-maven/pom.properties",
|
||||
GroupID: "org.anchore",
|
||||
ArtifactID: "example-java-app-maven",
|
||||
Version: "0.1.0",
|
||||
},
|
||||
},
|
||||
},
|
||||
"joda-time": {
|
||||
Name: "joda-time",
|
||||
Version: "2.9.2",
|
||||
Language: pkg.Java,
|
||||
Type: pkg.UnknownPkg,
|
||||
Metadata: pkg.JavaMetadata{
|
||||
PomProperties: &pkg.PomProperties{
|
||||
Path: "META-INF/maven/joda-time/joda-time/pom.properties",
|
||||
GroupID: "joda-time",
|
||||
ArtifactID: "joda-time",
|
||||
Version: "2.9.2",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.fixture, func(t *testing.T) {
|
||||
|
||||
generateJavaBuildFixture(t, test.fixture)
|
||||
|
||||
fixture, err := os.Open(test.fixture)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to open fixture: %+v", err)
|
||||
}
|
||||
|
||||
actual, err := parseJavaArchive(fixture.Name(), fixture)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to parse java archive: %+v", err)
|
||||
}
|
||||
|
||||
if len(actual) != len(test.expected) {
|
||||
for _, a := range actual {
|
||||
t.Log(" ", a)
|
||||
}
|
||||
t.Fatalf("unexpected package count: %d!=%d", len(actual), 1)
|
||||
}
|
||||
|
||||
var parent *pkg.Package
|
||||
for _, a := range actual {
|
||||
if strings.Contains(a.Name, "example-") {
|
||||
parent = &a
|
||||
}
|
||||
}
|
||||
|
||||
if parent == nil {
|
||||
t.Fatal("could not find the parent pkg")
|
||||
}
|
||||
|
||||
for _, a := range actual {
|
||||
e, ok := test.expected[a.Name]
|
||||
if !ok {
|
||||
t.Errorf("entry not found: %s", a.Name)
|
||||
continue
|
||||
}
|
||||
|
||||
if a.Name != parent.Name && a.Metadata.(pkg.JavaMetadata).Parent != nil && a.Metadata.(pkg.JavaMetadata).Parent.Name != parent.Name {
|
||||
t.Errorf("mismatched parent: %+v", a.Metadata.(pkg.JavaMetadata).Parent)
|
||||
}
|
||||
|
||||
// we need to compare the other fields without parent attached
|
||||
metadata := a.Metadata.(pkg.JavaMetadata)
|
||||
metadata.Parent = nil
|
||||
|
||||
// ignore select fields
|
||||
for _, field := range test.ignoreExtras {
|
||||
delete(metadata.Manifest.Extra, field)
|
||||
}
|
||||
|
||||
// write censored data back
|
||||
a.Metadata = metadata
|
||||
|
||||
diffs := deep.Equal(a, e)
|
||||
if len(diffs) > 0 {
|
||||
t.Errorf("diffs found for %q", a.Name)
|
||||
for _, d := range diffs {
|
||||
t.Errorf("diff: %+v", d)
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
|
@ -4,14 +4,22 @@ ifndef PKGSDIR
|
|||
$(error PKGSDIR is not set)
|
||||
endif
|
||||
|
||||
all: $(PKGSDIR)/example-java-app-maven-0.1.0.jar $(PKGSDIR)/example-java-app-gradle-0.1.0.jar $(PKGSDIR)/example-jenkins-plugin.hpi
|
||||
all: $(PKGSDIR)/example-java-app-maven-0.1.0.jar $(PKGSDIR)/example-java-app-gradle-0.1.0.jar $(PKGSDIR)/example-jenkins-plugin.hpi $(PKGSDIR)/spring-boot-0.0.1-SNAPSHOT.jar
|
||||
|
||||
clean: clean-examples
|
||||
rm -f $(PKGSDIR)/*
|
||||
|
||||
clean-examples: clean-gradle clean-maven clean-jenkins
|
||||
clean-examples: clean-gradle clean-maven clean-jenkins clean-nestedjar
|
||||
|
||||
.PHONY: maven gradle clean clean-gradle clean-maven clean-jenkins clean-examples
|
||||
.PHONY: maven gradle clean clean-gradle clean-maven clean-jenkins clean-examples clean-nestedjar
|
||||
|
||||
# Nested jar...
|
||||
|
||||
$(PKGSDIR)/spring-boot-0.0.1-SNAPSHOT.jar:
|
||||
./build-example-sb-app-nestedjar.sh $(PKGSDIR)
|
||||
|
||||
clean-nestedjar:
|
||||
rm -rf example-sb-app/target
|
||||
|
||||
# Maven...
|
||||
$(PKGSDIR)/example-java-app-maven-0.1.0.jar:
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@ set -uxe
|
|||
# note: this can be easily done in a 1-liner, however circle CI does NOT allow volume mounts from the host in docker executors (since they are on remote hosts, where the host files are inaccessible)
|
||||
|
||||
PKGSDIR=$1
|
||||
CTRID=$(docker create -u "$(id -u):$(id -g)" -e MAVEN_CONFIG=/tmp/.m2 -v /example-java-app -w /example-java-app maven:openjdk mvn -Duser.home=/tmp -DskipTests package)
|
||||
CTRID=$(docker create -u "$(id -u):$(id -g)" -e MAVEN_CONFIG=/tmp/.m2 -v /example-java-app -w /example-java-app maven:3.6.3-openjdk-14 mvn -Duser.home=/tmp -DskipTests package)
|
||||
|
||||
function cleanup() {
|
||||
docker rm "${CTRID}"
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@ set -uxe
|
|||
# note: this can be easily done in a 1-liner, however circle CI does NOT allow volume mounts from the host in docker executors (since they are on remote hosts, where the host files are inaccessible)
|
||||
|
||||
PKGSDIR=$1
|
||||
CTRID=$(docker create -u "$(id -u):$(id -g)" -e MAVEN_CONFIG=/tmp/.m2 -v /example-jenkins-plugin -w /example-jenkins-plugin maven:openjdk mvn -Duser.home=/tmp -DskipTests package)
|
||||
CTRID=$(docker create -u "$(id -u):$(id -g)" -e MAVEN_CONFIG=/tmp/.m2 -v /example-jenkins-plugin -w /example-jenkins-plugin maven:3.6.3-openjdk-14 mvn -Duser.home=/tmp -DskipTests package)
|
||||
|
||||
function cleanup() {
|
||||
docker rm "${CTRID}"
|
||||
|
|
|
|||
|
|
@ -0,0 +1,19 @@
|
|||
#!/usr/bin/env bash
|
||||
set -uxe
|
||||
|
||||
# note: this can be easily done in a 1-liner, however circle CI does NOT allow volume mounts from the host in docker executors (since they are on remote hosts, where the host files are inaccessible)
|
||||
|
||||
PKGSDIR=$1
|
||||
CTRID=$(docker create -u "$(id -u):$(id -g)" -e MAVEN_CONFIG=/tmp/.m2 -v /example-sb-app -w /example-sb-app maven:3.6.3-openjdk-14 mvn -Duser.home=/tmp -DskipTests package spring-boot:repackage)
|
||||
|
||||
function cleanup() {
|
||||
docker rm "${CTRID}"
|
||||
}
|
||||
|
||||
trap cleanup EXIT
|
||||
set +e
|
||||
|
||||
docker cp "$(pwd)/example-sb-app" "${CTRID}:/"
|
||||
docker start -a "${CTRID}"
|
||||
mkdir -p "$PKGSDIR"
|
||||
docker cp "${CTRID}:/example-sb-app/target/spring-boot-0.0.1-SNAPSHOT.jar" "$PKGSDIR"
|
||||
6
imgbom/cataloger/java/test-fixtures/java-builds/example-sb-app/.gitignore
vendored
Normal file
6
imgbom/cataloger/java/test-fixtures/java-builds/example-sb-app/.gitignore
vendored
Normal file
|
|
@ -0,0 +1,6 @@
|
|||
# maven build creates this when in a container volume
|
||||
/?/
|
||||
/.gradle/
|
||||
/build/
|
||||
target/
|
||||
dependency-reduced-pom.xml
|
||||
|
|
@ -0,0 +1,71 @@
|
|||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
<parent>
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
<artifactId>spring-boot-starter-parent</artifactId>
|
||||
<version>2.2.2.RELEASE</version>
|
||||
<relativePath/> <!-- lookup parent from repository -->
|
||||
</parent>
|
||||
<groupId>com.example</groupId>
|
||||
<artifactId>spring-boot</artifactId>
|
||||
<version>0.0.1-SNAPSHOT</version>
|
||||
<name>spring-boot</name>
|
||||
<description>Demo project for Spring Boot</description>
|
||||
|
||||
<properties>
|
||||
<java.version>1.8</java.version>
|
||||
</properties>
|
||||
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
<artifactId>spring-boot-starter-web</artifactId>
|
||||
</dependency>
|
||||
|
||||
<!-- tag::actuator[] -->
|
||||
<dependency>
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
<artifactId>spring-boot-starter-actuator</artifactId>
|
||||
</dependency>
|
||||
<!-- end::actuator[] -->
|
||||
|
||||
<!-- tag::tests[] -->
|
||||
<dependency>
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
<artifactId>spring-boot-starter-test</artifactId>
|
||||
<scope>test</scope>
|
||||
<exclusions>
|
||||
<exclusion>
|
||||
<groupId>org.junit.vintage</groupId>
|
||||
<artifactId>junit-vintage-engine</artifactId>
|
||||
</exclusion>
|
||||
</exclusions>
|
||||
</dependency>
|
||||
<!-- end::tests[] -->
|
||||
</dependencies>
|
||||
|
||||
<build>
|
||||
<plugins>
|
||||
|
||||
<plugin>
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
<artifactId>spring-boot-maven-plugin</artifactId>
|
||||
<version>1.0.2.RELEASE</version>
|
||||
<configuration>
|
||||
<mainClass>${start-class}</mainClass>
|
||||
<layout>ZIP</layout>
|
||||
</configuration>
|
||||
<executions>
|
||||
<execution>
|
||||
<goals>
|
||||
<goal>repackage</goal>
|
||||
</goals>
|
||||
</execution>
|
||||
</executions>
|
||||
</plugin>
|
||||
</plugins>
|
||||
</build>
|
||||
|
||||
</project>
|
||||
|
|
@ -0,0 +1,33 @@
|
|||
package com.example.springboot;
|
||||
|
||||
import java.util.Arrays;
|
||||
|
||||
import org.springframework.boot.CommandLineRunner;
|
||||
import org.springframework.boot.SpringApplication;
|
||||
import org.springframework.boot.autoconfigure.SpringBootApplication;
|
||||
import org.springframework.context.ApplicationContext;
|
||||
import org.springframework.context.annotation.Bean;
|
||||
|
||||
@SpringBootApplication
|
||||
public class Application {
|
||||
|
||||
public static void main(String[] args) {
|
||||
SpringApplication.run(Application.class, args);
|
||||
}
|
||||
|
||||
@Bean
|
||||
public CommandLineRunner commandLineRunner(ApplicationContext ctx) {
|
||||
return args -> {
|
||||
|
||||
System.out.println("Let's inspect the beans provided by Spring Boot:");
|
||||
|
||||
String[] beanNames = ctx.getBeanDefinitionNames();
|
||||
Arrays.sort(beanNames);
|
||||
for (String beanName : beanNames) {
|
||||
System.out.println(beanName);
|
||||
}
|
||||
|
||||
};
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,14 @@
|
|||
package com.example.springboot;
|
||||
|
||||
import org.springframework.web.bind.annotation.RestController;
|
||||
import org.springframework.web.bind.annotation.RequestMapping;
|
||||
|
||||
@RestController
|
||||
public class HelloController {
|
||||
|
||||
@RequestMapping("/")
|
||||
public String index() {
|
||||
return "Greetings from Spring Boot!";
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,38 @@
|
|||
package com.example.springboot;
|
||||
|
||||
import static org.assertj.core.api.Assertions.*;
|
||||
|
||||
import java.net.URL;
|
||||
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.boot.test.context.SpringBootTest;
|
||||
import org.springframework.boot.test.web.client.TestRestTemplate;
|
||||
import org.springframework.boot.web.server.LocalServerPort;
|
||||
import org.springframework.http.ResponseEntity;
|
||||
|
||||
@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT)
|
||||
public class HelloControllerIT {
|
||||
|
||||
@LocalServerPort
|
||||
private int port;
|
||||
|
||||
private URL base;
|
||||
|
||||
@Autowired
|
||||
private TestRestTemplate template;
|
||||
|
||||
@BeforeEach
|
||||
public void setUp() throws Exception {
|
||||
this.base = new URL("http://localhost:" + port + "/");
|
||||
}
|
||||
|
||||
@Test
|
||||
public void getHello() throws Exception {
|
||||
ResponseEntity<String> response = template.getForEntity(base.toString(),
|
||||
String.class);
|
||||
assertThat(response.getBody()).isEqualTo("Greetings from Spring Boot!");
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,29 @@
|
|||
package com.example.springboot;
|
||||
|
||||
import static org.hamcrest.Matchers.equalTo;
|
||||
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content;
|
||||
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status;
|
||||
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.boot.test.autoconfigure.web.servlet.AutoConfigureMockMvc;
|
||||
import org.springframework.boot.test.context.SpringBootTest;
|
||||
import org.springframework.http.MediaType;
|
||||
import org.springframework.test.web.servlet.MockMvc;
|
||||
import org.springframework.test.web.servlet.request.MockMvcRequestBuilders;
|
||||
|
||||
@SpringBootTest
|
||||
@AutoConfigureMockMvc
|
||||
public class HelloControllerTest {
|
||||
|
||||
@Autowired
|
||||
private MockMvc mvc;
|
||||
|
||||
@Test
|
||||
public void getHello() throws Exception {
|
||||
mvc.perform(MockMvcRequestBuilders.get("/").accept(MediaType.APPLICATION_JSON))
|
||||
.andExpect(status().isOk())
|
||||
.andExpect(content().string(equalTo("Greetings from Spring Boot!")));
|
||||
}
|
||||
}
|
||||
|
|
@ -3,9 +3,10 @@
|
|||
package integration
|
||||
|
||||
import (
|
||||
"github.com/anchore/imgbom/internal"
|
||||
"testing"
|
||||
|
||||
"github.com/anchore/imgbom/internal"
|
||||
|
||||
"github.com/anchore/go-testutils"
|
||||
"github.com/anchore/imgbom/imgbom/cataloger"
|
||||
"github.com/anchore/imgbom/imgbom/pkg"
|
||||
|
|
@ -48,6 +49,7 @@ func TestLanguageImage(t *testing.T) {
|
|||
pkgLanguage: pkg.Java,
|
||||
pkgInfo: map[string]string{
|
||||
"example-java-app-maven": "0.1.0",
|
||||
"example-jenkins-plugin": "1.0-SNAPSHOT", // the jeninks HPI file has a nested JAR of the same name
|
||||
},
|
||||
},
|
||||
{
|
||||
|
|
|
|||
58
internal/file/zip_file_manifest.go
Normal file
58
internal/file/zip_file_manifest.go
Normal file
|
|
@ -0,0 +1,58 @@
|
|||
package file
|
||||
|
||||
import (
|
||||
"archive/zip"
|
||||
"fmt"
|
||||
"os"
|
||||
"sort"
|
||||
|
||||
"github.com/anchore/imgbom/internal"
|
||||
|
||||
"github.com/anchore/imgbom/internal/log"
|
||||
)
|
||||
|
||||
type ZipFileManifest map[string]os.FileInfo
|
||||
|
||||
func newZipManifest() ZipFileManifest {
|
||||
return make(ZipFileManifest)
|
||||
}
|
||||
|
||||
func (z ZipFileManifest) Add(entry string, info os.FileInfo) {
|
||||
z[entry] = info
|
||||
}
|
||||
|
||||
func (z ZipFileManifest) GlobMatch(patterns ...string) []string {
|
||||
uniqueMatches := internal.NewStringSet()
|
||||
|
||||
for _, pattern := range patterns {
|
||||
for entry := range z {
|
||||
if GlobMatch(pattern, entry) {
|
||||
uniqueMatches.Add(entry)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
results := uniqueMatches.ToSlice()
|
||||
sort.Strings(results)
|
||||
|
||||
return results
|
||||
}
|
||||
|
||||
func NewZipFileManifest(archivePath string) (ZipFileManifest, error) {
|
||||
zipReader, err := zip.OpenReader(archivePath)
|
||||
manifest := newZipManifest()
|
||||
if err != nil {
|
||||
return manifest, fmt.Errorf("unable to open zip archive (%s): %w", archivePath, err)
|
||||
}
|
||||
defer func() {
|
||||
err = zipReader.Close()
|
||||
if err != nil {
|
||||
log.Errorf("unable to close zip archive (%s): %w", archivePath, err)
|
||||
}
|
||||
}()
|
||||
|
||||
for _, file := range zipReader.Reader.File {
|
||||
manifest.Add(file.Name, file.FileInfo())
|
||||
}
|
||||
return manifest, nil
|
||||
}
|
||||
222
internal/file/zip_file_traversal.go
Normal file
222
internal/file/zip_file_traversal.go
Normal file
|
|
@ -0,0 +1,222 @@
|
|||
package file
|
||||
|
||||
import (
|
||||
"archive/zip"
|
||||
"bytes"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"github.com/anchore/imgbom/internal/log"
|
||||
)
|
||||
|
||||
const (
|
||||
_ = iota
|
||||
KB = 1 << (10 * iota)
|
||||
MB
|
||||
GB
|
||||
)
|
||||
|
||||
const perFileReadLimit = 2 * GB
|
||||
|
||||
type zipTraversalRequest map[string]struct{}
|
||||
|
||||
func newZipTraverseRequest(paths ...string) zipTraversalRequest {
|
||||
results := make(zipTraversalRequest)
|
||||
for _, p := range paths {
|
||||
results[p] = struct{}{}
|
||||
}
|
||||
return results
|
||||
}
|
||||
|
||||
func TraverseFilesInZip(archivePath string, visitor func(*zip.File) error, paths ...string) error {
|
||||
request := newZipTraverseRequest(paths...)
|
||||
|
||||
zipReader, err := zip.OpenReader(archivePath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("unable to open zip archive (%s): %w", archivePath, err)
|
||||
}
|
||||
defer func() {
|
||||
err = zipReader.Close()
|
||||
if err != nil {
|
||||
log.Errorf("unable to close zip archive (%s): %w", archivePath, err)
|
||||
}
|
||||
}()
|
||||
|
||||
for _, file := range zipReader.Reader.File {
|
||||
// if no paths are given then assume that all files should be traversed
|
||||
if len(paths) > 0 {
|
||||
if _, ok := request[file.Name]; !ok {
|
||||
// this file path is not of interest
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
if err = visitor(file); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func ExtractFromZipToUniqueTempFile(archivePath, dir string, paths ...string) (map[string]io.Reader, error) {
|
||||
results := make(map[string]io.Reader)
|
||||
|
||||
// don't allow for full traversal, only select traversal from given paths
|
||||
if len(paths) == 0 {
|
||||
return results, nil
|
||||
}
|
||||
|
||||
visitor := func(file *zip.File) error {
|
||||
tempfilePrefix := filepath.Base(filepath.Clean(file.Name)) + "-"
|
||||
|
||||
tempFile, err := ioutil.TempFile(dir, tempfilePrefix)
|
||||
if err != nil {
|
||||
return fmt.Errorf("unable to create temp file: %w", err)
|
||||
}
|
||||
|
||||
zippedFile, err := file.Open()
|
||||
if err != nil {
|
||||
return fmt.Errorf("unable to read file=%q from zip=%q: %w", file.Name, archivePath, err)
|
||||
}
|
||||
|
||||
if file.FileInfo().IsDir() {
|
||||
return fmt.Errorf("unable to extract directories, only files: %s", file.Name)
|
||||
}
|
||||
|
||||
// limit the zip reader on each file read to prevent decompression bomb attacks
|
||||
numBytes, err := io.Copy(tempFile, io.LimitReader(zippedFile, perFileReadLimit))
|
||||
if numBytes >= perFileReadLimit || errors.Is(err, io.EOF) {
|
||||
return fmt.Errorf("zip read limit hit (potential decompression bomb attack)")
|
||||
}
|
||||
if err != nil {
|
||||
return fmt.Errorf("unable to copy source=%q for zip=%q: %w", file.Name, archivePath, err)
|
||||
}
|
||||
|
||||
// the file pointer is at the end due to the copy operation, reset back to the beginning
|
||||
_, err = tempFile.Seek(0, io.SeekStart)
|
||||
if err != nil {
|
||||
return fmt.Errorf("unable to reset file pointer (%s): %w", tempFile.Name(), err)
|
||||
}
|
||||
|
||||
results[file.Name] = tempFile
|
||||
|
||||
err = zippedFile.Close()
|
||||
if err != nil {
|
||||
return fmt.Errorf("unable to close source file=%q from zip=%q: %w", file.Name, archivePath, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
return results, TraverseFilesInZip(archivePath, visitor, paths...)
|
||||
}
|
||||
|
||||
func ContentsFromZip(archivePath string, paths ...string) (map[string]string, error) {
|
||||
results := make(map[string]string)
|
||||
|
||||
// don't allow for full traversal, only select traversal from given paths
|
||||
if len(paths) == 0 {
|
||||
return results, nil
|
||||
}
|
||||
|
||||
visitor := func(file *zip.File) error {
|
||||
zippedFile, err := file.Open()
|
||||
if err != nil {
|
||||
return fmt.Errorf("unable to read file=%q from zip=%q: %w", file.Name, archivePath, err)
|
||||
}
|
||||
|
||||
if file.FileInfo().IsDir() {
|
||||
return fmt.Errorf("unable to extract directories, only files: %s", file.Name)
|
||||
}
|
||||
|
||||
var buffer bytes.Buffer
|
||||
|
||||
// limit the zip reader on each file read to prevent decompression bomb attacks
|
||||
numBytes, err := io.Copy(&buffer, io.LimitReader(zippedFile, perFileReadLimit))
|
||||
if numBytes >= perFileReadLimit || errors.Is(err, io.EOF) {
|
||||
return fmt.Errorf("zip read limit hit (potential decompression bomb attack)")
|
||||
}
|
||||
if err != nil {
|
||||
return fmt.Errorf("unable to copy source=%q for zip=%q: %w", file.Name, archivePath, err)
|
||||
}
|
||||
|
||||
results[file.Name] = buffer.String()
|
||||
|
||||
err = zippedFile.Close()
|
||||
if err != nil {
|
||||
return fmt.Errorf("unable to close source file=%q from zip=%q: %w", file.Name, archivePath, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
return results, TraverseFilesInZip(archivePath, visitor, paths...)
|
||||
}
|
||||
|
||||
func UnzipToDir(archivePath, targetDir string) error {
|
||||
visitor := func(file *zip.File) error {
|
||||
// the zip-slip attack protection is still being erroneously detected
|
||||
// nolint:gosec
|
||||
expandedFilePath := filepath.Clean(filepath.Join(targetDir, file.Name))
|
||||
|
||||
// protect against zip slip attacks (traversing unintended parent paths from maliciously crafted relative-path entries)
|
||||
if !strings.HasPrefix(expandedFilePath, filepath.Clean(targetDir)+string(os.PathSeparator)) {
|
||||
return fmt.Errorf("potential zip slip attack: %q", expandedFilePath)
|
||||
}
|
||||
|
||||
err := extractSingleFile(file, expandedFilePath, archivePath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
return TraverseFilesInZip(archivePath, visitor)
|
||||
}
|
||||
|
||||
func extractSingleFile(file *zip.File, expandedFilePath, archivePath string) error {
|
||||
zippedFile, err := file.Open()
|
||||
if err != nil {
|
||||
return fmt.Errorf("unable to read file=%q from zip=%q: %w", file.Name, archivePath, err)
|
||||
}
|
||||
|
||||
if file.FileInfo().IsDir() {
|
||||
err = os.MkdirAll(expandedFilePath, file.Mode())
|
||||
if err != nil {
|
||||
return fmt.Errorf("unable to create dir=%q from zip=%q: %w", expandedFilePath, archivePath, err)
|
||||
}
|
||||
} else {
|
||||
// Open an output file for writing
|
||||
outputFile, err := os.OpenFile(
|
||||
expandedFilePath,
|
||||
os.O_WRONLY|os.O_CREATE|os.O_TRUNC,
|
||||
file.Mode(),
|
||||
)
|
||||
if err != nil {
|
||||
return fmt.Errorf("unable to create dest file=%q from zip=%q: %w", expandedFilePath, archivePath, err)
|
||||
}
|
||||
|
||||
// limit the zip reader on each file read to prevent decompression bomb attacks
|
||||
numBytes, err := io.Copy(outputFile, io.LimitReader(zippedFile, perFileReadLimit))
|
||||
if numBytes >= perFileReadLimit || errors.Is(err, io.EOF) {
|
||||
return fmt.Errorf("zip read limit hit (potential decompression bomb attack)")
|
||||
}
|
||||
if err != nil {
|
||||
return fmt.Errorf("unable to copy source=%q to dest=%q for zip=%q: %w", file.Name, outputFile.Name(), archivePath, err)
|
||||
}
|
||||
|
||||
err = outputFile.Close()
|
||||
if err != nil {
|
||||
return fmt.Errorf("unable to close dest file=%q from zip=%q: %w", outputFile.Name(), archivePath, err)
|
||||
}
|
||||
}
|
||||
|
||||
err = zippedFile.Close()
|
||||
if err != nil {
|
||||
return fmt.Errorf("unable to close source file=%q from zip=%q: %w", file.Name, archivePath, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
|
@ -3,7 +3,6 @@ package file
|
|||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/json"
|
||||
"github.com/go-test/deep"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
|
|
@ -12,6 +11,8 @@ import (
|
|||
"strings"
|
||||
"syscall"
|
||||
"testing"
|
||||
|
||||
"github.com/go-test/deep"
|
||||
)
|
||||
|
||||
func generateFixture(t *testing.T, archivePath string) {
|
||||
|
|
@ -188,7 +189,7 @@ func TestExtractFilesFromZipFile(t *testing.T) {
|
|||
bFilePath: "B file...",
|
||||
}
|
||||
|
||||
actual, err := ExtractFilesFromZip(archivePath, aFilePath, bFilePath)
|
||||
actual, err := ContentsFromZip(archivePath, aFilePath, bFilePath)
|
||||
if err != nil {
|
||||
t.Fatalf("unable to extract from unzip archive: %+v", err)
|
||||
}
|
||||
|
|
@ -236,7 +237,7 @@ func TestZipFileManifest(t *testing.T) {
|
|||
filepath.Join("zip-source", "b-file.txt"),
|
||||
}
|
||||
|
||||
actual, err := ZipFileManifest(archivePath)
|
||||
actual, err := NewZipFileManifest(archivePath)
|
||||
if err != nil {
|
||||
t.Fatalf("unable to extract from unzip archive: %+v", err)
|
||||
}
|
||||
|
|
@ -1,197 +0,0 @@
|
|||
package file
|
||||
|
||||
import (
|
||||
"archive/zip"
|
||||
"bytes"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"github.com/anchore/imgbom/internal/log"
|
||||
)
|
||||
|
||||
const (
|
||||
_ = iota
|
||||
KB = 1 << (10 * iota)
|
||||
MB
|
||||
GB
|
||||
)
|
||||
|
||||
const readLimit = 2 * GB
|
||||
|
||||
type extractRequest map[string]struct{}
|
||||
|
||||
func newExtractRequest(paths ...string) extractRequest {
|
||||
results := make(extractRequest)
|
||||
for _, p := range paths {
|
||||
results[p] = struct{}{}
|
||||
}
|
||||
return results
|
||||
}
|
||||
|
||||
func ExtractFilesFromZip(archivePath string, paths ...string) (map[string]string, error) {
|
||||
request := newExtractRequest(paths...)
|
||||
|
||||
results := make(map[string]string)
|
||||
zipReader, err := zip.OpenReader(archivePath)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("unable to open zip archive (%s): %w", archivePath, err)
|
||||
}
|
||||
defer func() {
|
||||
err = zipReader.Close()
|
||||
if err != nil {
|
||||
log.Errorf("unable to close zip archive (%s): %w", archivePath, err)
|
||||
}
|
||||
}()
|
||||
|
||||
for _, file := range zipReader.Reader.File {
|
||||
if _, ok := request[file.Name]; !ok {
|
||||
// this file path is not of interest
|
||||
continue
|
||||
}
|
||||
|
||||
zippedFile, err := file.Open()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("unable to read file=%q from zip=%q: %w", file.Name, archivePath, err)
|
||||
}
|
||||
|
||||
if file.FileInfo().IsDir() {
|
||||
return nil, fmt.Errorf("unable to extract directories, only files: %s", file.Name)
|
||||
}
|
||||
|
||||
var buffer bytes.Buffer
|
||||
|
||||
// limit the zip reader on each file read to prevent decompression bomb attacks
|
||||
numBytes, err := io.Copy(&buffer, io.LimitReader(zippedFile, readLimit))
|
||||
if numBytes >= readLimit || errors.Is(err, io.EOF) {
|
||||
return nil, fmt.Errorf("zip read limit hit (potential decompression bomb attack)")
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("unable to copy source=%q for zip=%q: %w", file.Name, archivePath, err)
|
||||
}
|
||||
|
||||
results[file.Name] = buffer.String()
|
||||
|
||||
err = zippedFile.Close()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("unable to close source file=%q from zip=%q: %w", file.Name, archivePath, err)
|
||||
}
|
||||
}
|
||||
return results, nil
|
||||
}
|
||||
|
||||
func UnzipToDir(archivePath, targetDir string) error {
|
||||
zipReader, err := zip.OpenReader(archivePath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("unable to open zip archive (%s): %w", archivePath, err)
|
||||
}
|
||||
defer func() {
|
||||
err = zipReader.Close()
|
||||
if err != nil {
|
||||
log.Errorf("unable to close zip archive (%s): %w", archivePath, err)
|
||||
}
|
||||
}()
|
||||
|
||||
for _, file := range zipReader.Reader.File {
|
||||
// the zip-slip attack protection is still being erroneously detected
|
||||
// nolint:gosec
|
||||
expandedFilePath := filepath.Clean(filepath.Join(targetDir, file.Name))
|
||||
|
||||
// protect against zip slip attacks (traversing unintended parent paths from maliciously crafted relative-path entries)
|
||||
if !strings.HasPrefix(expandedFilePath, filepath.Clean(targetDir)+string(os.PathSeparator)) {
|
||||
return fmt.Errorf("potential zip slip attack: %q", expandedFilePath)
|
||||
}
|
||||
|
||||
err = extractSingleFile(file, expandedFilePath, archivePath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func extractSingleFile(file *zip.File, expandedFilePath, archivePath string) error {
|
||||
zippedFile, err := file.Open()
|
||||
if err != nil {
|
||||
return fmt.Errorf("unable to read file=%q from zip=%q: %w", file.Name, archivePath, err)
|
||||
}
|
||||
|
||||
if file.FileInfo().IsDir() {
|
||||
err = os.MkdirAll(expandedFilePath, file.Mode())
|
||||
if err != nil {
|
||||
return fmt.Errorf("unable to create dir=%q from zip=%q: %w", expandedFilePath, archivePath, err)
|
||||
}
|
||||
} else {
|
||||
// Open an output file for writing
|
||||
outputFile, err := os.OpenFile(
|
||||
expandedFilePath,
|
||||
os.O_WRONLY|os.O_CREATE|os.O_TRUNC,
|
||||
file.Mode(),
|
||||
)
|
||||
if err != nil {
|
||||
return fmt.Errorf("unable to create dest file=%q from zip=%q: %w", expandedFilePath, archivePath, err)
|
||||
}
|
||||
|
||||
// limit the zip reader on each file read to prevent decompression bomb attacks
|
||||
numBytes, err := io.Copy(outputFile, io.LimitReader(zippedFile, readLimit))
|
||||
if numBytes >= readLimit || errors.Is(err, io.EOF) {
|
||||
return fmt.Errorf("zip read limit hit (potential decompression bomb attack)")
|
||||
}
|
||||
if err != nil {
|
||||
return fmt.Errorf("unable to copy source=%q to dest=%q for zip=%q: %w", file.Name, outputFile.Name(), archivePath, err)
|
||||
}
|
||||
|
||||
err = outputFile.Close()
|
||||
if err != nil {
|
||||
return fmt.Errorf("unable to close dest file=%q from zip=%q: %w", outputFile.Name(), archivePath, err)
|
||||
}
|
||||
}
|
||||
|
||||
err = zippedFile.Close()
|
||||
if err != nil {
|
||||
return fmt.Errorf("unable to close source file=%q from zip=%q: %w", file.Name, archivePath, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type ZipManifest map[string]os.FileInfo
|
||||
|
||||
func newZipManifest() ZipManifest {
|
||||
return make(ZipManifest)
|
||||
}
|
||||
|
||||
func (z ZipManifest) Add(entry string, info os.FileInfo) {
|
||||
z[entry] = info
|
||||
}
|
||||
|
||||
func (z ZipManifest) GlobMatch(pattern string) []string {
|
||||
results := make([]string, 0)
|
||||
for entry := range z {
|
||||
if GlobMatch(pattern, entry) {
|
||||
results = append(results, entry)
|
||||
}
|
||||
}
|
||||
return results
|
||||
}
|
||||
|
||||
func ZipFileManifest(archivePath string) (ZipManifest, error) {
|
||||
zipReader, err := zip.OpenReader(archivePath)
|
||||
manifest := newZipManifest()
|
||||
if err != nil {
|
||||
return manifest, fmt.Errorf("unable to open zip archive (%s): %w", archivePath, err)
|
||||
}
|
||||
defer func() {
|
||||
err = zipReader.Close()
|
||||
if err != nil {
|
||||
log.Errorf("unable to close zip archive (%s): %w", archivePath, err)
|
||||
}
|
||||
}()
|
||||
|
||||
for _, file := range zipReader.Reader.File {
|
||||
manifest.Add(file.Name, file.FileInfo())
|
||||
}
|
||||
return manifest, nil
|
||||
}
|
||||
|
|
@ -26,3 +26,13 @@ func (s StringSet) Contains(i string) bool {
|
|||
_, ok := s[i]
|
||||
return ok
|
||||
}
|
||||
|
||||
func (s StringSet) ToSlice() []string {
|
||||
ret := make([]string, len(s))
|
||||
idx := 0
|
||||
for v := range s {
|
||||
ret[idx] = v
|
||||
idx++
|
||||
}
|
||||
return ret
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue