Mirrors/syft
2
0
Fork 0
mirror of https://github.com/anchore/syft synced 2024-11-10 06:14:16 +00:00
Code Issues Projects Releases Packages Wiki Activity

[wip] source api refactor

Browse source 
Signed-off-by: Alex Goodman <alex.goodman@anchore.com>
This commit is contained in:
Alex Goodman 2023-04-30 13:03:40 -04:00
parent a07bfe7dfa
commit d49ba172f1
No known key found for this signature in database
GPG key ID: 5CB45AE22BAB7EA7
35 changed files with 1302 additions and 838 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
cmd/syft/cli/attest/attest.go
View file

@ -3,6 +3,7 @@ package attest
import (
"context"
"fmt"
"github.com/anchore/syft/syft/source/scheme"
"os"
"os/exec"
"strings"
@ -52,7 +53,7 @@ func Run(_ context.Context, app *config.Application, args []string) error {
return fmt.Errorf("could not generate source input for packages command: %w", err)
}
if si.Scheme != source.ImageScheme {
if si.Scheme != scheme.ContainerImageScheme {
return fmt.Errorf("attestations are only supported for oci images at this time")
}

14
cmd/syft/cli/eventloop/tasks.go
View file

@ -12,7 +12,7 @@ import (
"github.com/anchore/syft/syft/source"
)
type Task func(*sbom.Artifacts, *source.Source) ([]artifact.Relationship, error)
type Task func(*sbom.Artifacts, source.Source) ([]artifact.Relationship, error)
func Tasks(app *config.Application) ([]Task, error) {
var tasks []Task
@ -44,7 +44,7 @@ func generateCatalogPackagesTask(app *config.Application) (Task, error) {
return nil, nil
}
task := func(results *sbom.Artifacts, src *source.Source) ([]artifact.Relationship, error) {
task := func(results *sbom.Artifacts, src source.Source) ([]artifact.Relationship, error) {
packageCatalog, relationships, theDistro, err := syft.CatalogPackages(src, app.ToCatalogerConfig())
results.PackageCatalog = packageCatalog
@ -63,7 +63,7 @@ func generateCatalogFileMetadataTask(app *config.Application) (Task, error) {
metadataCataloger := file.NewMetadataCataloger()
task := func(results *sbom.Artifacts, src *source.Source) ([]artifact.Relationship, error) {
task := func(results *sbom.Artifacts, src source.Source) ([]artifact.Relationship, error) {
resolver, err := src.FileResolver(app.FileMetadata.Cataloger.ScopeOpt)
if err != nil {
return nil, err
@ -109,7 +109,7 @@ func generateCatalogFileDigestsTask(app *config.Application) (Task, error) {
return nil, err
}
task := func(results *sbom.Artifacts, src *source.Source) ([]artifact.Relationship, error) {
task := func(results *sbom.Artifacts, src source.Source) ([]artifact.Relationship, error) {
resolver, err := src.FileResolver(app.FileMetadata.Cataloger.ScopeOpt)
if err != nil {
return nil, err
@ -141,7 +141,7 @@ func generateCatalogSecretsTask(app *config.Application) (Task, error) {
return nil, err
}
task := func(results *sbom.Artifacts, src *source.Source) ([]artifact.Relationship, error) {
task := func(results *sbom.Artifacts, src source.Source) ([]artifact.Relationship, error) {
resolver, err := src.FileResolver(app.Secrets.Cataloger.ScopeOpt)
if err != nil {
return nil, err
@ -168,7 +168,7 @@ func generateCatalogContentsTask(app *config.Application) (Task, error) {
return nil, err
}
task := func(results *sbom.Artifacts, src *source.Source) ([]artifact.Relationship, error) {
task := func(results *sbom.Artifacts, src source.Source) ([]artifact.Relationship, error) {
resolver, err := src.FileResolver(app.FileContents.Cataloger.ScopeOpt)
if err != nil {
return nil, err
@ -185,7 +185,7 @@ func generateCatalogContentsTask(app *config.Application) (Task, error) {
return task, nil
}
func RunTask(t Task, a *sbom.Artifacts, src *source.Source, c chan<- artifact.Relationship, errs chan<- error) {
func RunTask(t Task, a *sbom.Artifacts, src source.Source, c chan<- artifact.Relationship, errs chan<- error) {
defer close(c)
relationships, err := t(a, src)

17
cmd/syft/cli/packages/packages.go
View file

@ -3,6 +3,7 @@ package packages
import (
"context"
"fmt"
"github.com/anchore/syft/syft/source/scheme"
"github.com/wagoodman/go-partybus"
@ -42,7 +43,7 @@ func Run(_ context.Context, app *config.Application, args []string) error {
// could be an image or a directory, with or without a scheme
userInput := args[0]
si, err := source.ParseInputWithName(userInput, app.Platform, app.Name, app.DefaultImagePullSource)
si, err := scheme.Parse(userInput, app.Platform, app.Name, app.DefaultImagePullSource)
if err != nil {
return fmt.Errorf("could not generate source input for packages command: %w", err)
}
@ -61,14 +62,14 @@ func Run(_ context.Context, app *config.Application, args []string) error {
)
}
func execWorker(app *config.Application, si source.Input, writer sbom.Writer) <-chan error {
func execWorker(app *config.Application, si scheme.Input, writer sbom.Writer) <-chan error {
errs := make(chan error)
go func() {
defer close(errs)
src, cleanup, err := source.New(si, app.Registry.ToOptions(), app.Exclusions)
if cleanup != nil {
defer cleanup()
src, err := scheme.NewSource(si, app.Registry.ToOptions(), app.Exclusions)
if src != nil {
defer src.Close()
}
if err != nil {
errs <- fmt.Errorf("failed to construct source from user input %q: %w", si.UserInput, err)
@ -93,14 +94,14 @@ func execWorker(app *config.Application, si source.Input, writer sbom.Writer) <-
return errs
}
func GenerateSBOM(src *source.Source, errs chan error, app *config.Application) (*sbom.SBOM, error) {
func GenerateSBOM(src source.Source, errs chan error, app *config.Application) (*sbom.SBOM, error) {
tasks, err := eventloop.Tasks(app)
if err != nil {
return nil, err
}
s := sbom.SBOM{
Source: src.Metadata,
Source: src.Describe(),
Descriptor: sbom.Descriptor{
Name: internal.ApplicationName,
Version: version.FromBuild().Version,
@ -113,7 +114,7 @@ func GenerateSBOM(src *source.Source, errs chan error, app *config.Application)
return &s, nil
}
func buildRelationships(s *sbom.SBOM, src *source.Source, tasks []eventloop.Task, errs chan error) {
func buildRelationships(s *sbom.SBOM, src source.Source, tasks []eventloop.Task, errs chan error) {
var relationships []<-chan artifact.Relationship
for _, task := range tasks {
c := make(chan artifact.Relationship)

5
syft/formats/common/cyclonedxhelpers/decoder.go
View file

@ -2,6 +2,7 @@ package cyclonedxhelpers
import (
"fmt"
"github.com/anchore/syft/syft/source/scheme"
"io"
"github.com/CycloneDX/cyclonedx-go"
@ -244,12 +245,12 @@ func extractComponents(meta *cyclonedx.Metadata) source.Metadata {
switch c.Type {
case cyclonedx.ComponentTypeContainer:
return source.Metadata{
Scheme: source.ImageScheme,
Scheme: scheme.ContainerImageScheme,
ImageMetadata: image,
}
case cyclonedx.ComponentTypeFile:
return source.Metadata{
Scheme: source.FileScheme, // or source.DirectoryScheme
Scheme: scheme.FileScheme, // or source.DirectoryScheme
Path: c.Name,
ImageMetadata: image,
}

5
syft/formats/common/cyclonedxhelpers/format.go
View file

@ -1,6 +1,7 @@
package cyclonedxhelpers
import (
"github.com/anchore/syft/syft/source/scheme"
"time"
"github.com/CycloneDX/cyclonedx-go"
@ -173,7 +174,7 @@ func toDependencies(relationships []artifact.Relationship) []cyclonedx.Dependenc
func toBomDescriptorComponent(srcMetadata source.Metadata) *cyclonedx.Component {
name := srcMetadata.Name
switch srcMetadata.Scheme {
case source.ImageScheme:
case scheme.ContainerImageScheme:
if name == "" {
name = srcMetadata.ImageMetadata.UserInput
}
@ -187,7 +188,7 @@ func toBomDescriptorComponent(srcMetadata source.Metadata) *cyclonedx.Component
Name: name,
Version: srcMetadata.ImageMetadata.ManifestDigest,
}
case source.DirectoryScheme, source.FileScheme:
case scheme.DirectoryScheme, scheme.FileScheme:
if name == "" {
name = srcMetadata.Path
}

5
syft/formats/common/spdxhelpers/document_name.go
View file

@ -2,6 +2,7 @@ package spdxhelpers
import (
"github.com/anchore/syft/syft/source"
"github.com/anchore/syft/syft/source/scheme"
)
func DocumentName(srcMetadata source.Metadata) string {
@ -10,9 +11,9 @@ func DocumentName(srcMetadata source.Metadata) string {
}
switch srcMetadata.Scheme {
case source.ImageScheme:
case scheme.ContainerImageScheme:
return srcMetadata.ImageMetadata.UserInput
case source.DirectoryScheme, source.FileScheme:
case scheme.DirectoryScheme, scheme.FileScheme:
return srcMetadata.Path
default:
return "unknown"

9
syft/formats/common/spdxhelpers/document_name_test.go
View file

@ -2,6 +2,7 @@ package spdxhelpers
import (
"fmt"
"github.com/anchore/syft/syft/source/scheme"
"strings"
"testing"
@ -13,7 +14,7 @@ import (
func Test_DocumentName(t *testing.T) {
allSchemes := strset.New()
for _, s := range source.AllSchemes {
for _, s := range scheme.AllSchemes {
allSchemes.Add(string(s))
}
testedSchemes := strset.New()
@ -28,7 +29,7 @@ func Test_DocumentName(t *testing.T) {
name: "image",
inputName: "my-name",
srcMetadata: source.Metadata{
Scheme: source.ImageScheme,
Scheme: scheme.ContainerImageScheme,
ImageMetadata: source.ImageMetadata{
UserInput: "image-repo/name:tag",
ID: "id",
@ -41,7 +42,7 @@ func Test_DocumentName(t *testing.T) {
name: "directory",
inputName: "my-name",
srcMetadata: source.Metadata{
Scheme: source.DirectoryScheme,
Scheme: scheme.DirectoryScheme,
Path: "some/path/to/place",
},
expected: "some/path/to/place",
@ -50,7 +51,7 @@ func Test_DocumentName(t *testing.T) {
name: "file",
inputName: "my-name",
srcMetadata: source.Metadata{
Scheme: source.FileScheme,
Scheme: scheme.FileScheme,
Path: "some/path/to/place",
},
expected: "some/path/to/place",

7
syft/formats/common/spdxhelpers/document_namespace.go
View file

@ -2,6 +2,7 @@ package spdxhelpers
import (
"fmt"
"github.com/anchore/syft/syft/source/scheme"
"net/url"
"path"
"strings"
@ -27,11 +28,11 @@ func DocumentNamespace(name string, srcMetadata source.Metadata) string {
name = cleanName(name)
input := "unknown-source-type"
switch srcMetadata.Scheme {
case source.ImageScheme:
case scheme.ContainerImageScheme:
input = inputImage
case source.DirectoryScheme:
case scheme.DirectoryScheme:
input = inputDirectory
case source.FileScheme:
case scheme.FileScheme:
input = inputFile
}

9
syft/formats/common/spdxhelpers/document_namespace_test.go
View file

@ -2,6 +2,7 @@ package spdxhelpers
import (
"fmt"
"github.com/anchore/syft/syft/source/scheme"
"strings"
"testing"
@ -13,7 +14,7 @@ import (
func Test_documentNamespace(t *testing.T) {
allSchemes := strset.New()
for _, s := range source.AllSchemes {
for _, s := range scheme.AllSchemes {
allSchemes.Add(string(s))
}
testedSchemes := strset.New()
@ -28,7 +29,7 @@ func Test_documentNamespace(t *testing.T) {
name: "image",
inputName: "my-name",
srcMetadata: source.Metadata{
Scheme: source.ImageScheme,
Scheme: scheme.ContainerImageScheme,
ImageMetadata: source.ImageMetadata{
UserInput: "image-repo/name:tag",
ID: "id",
@ -41,7 +42,7 @@ func Test_documentNamespace(t *testing.T) {
name: "directory",
inputName: "my-name",
srcMetadata: source.Metadata{
Scheme: source.DirectoryScheme,
Scheme: scheme.DirectoryScheme,
Path: "some/path/to/place",
},
expected: "https://anchore.com/syft/dir/my-name-",
@ -50,7 +51,7 @@ func Test_documentNamespace(t *testing.T) {
name: "file",
inputName: "my-name",
srcMetadata: source.Metadata{
Scheme: source.FileScheme,
Scheme: scheme.FileScheme,
Path: "some/path/to/place",
},
expected: "https://anchore.com/syft/file/my-name-",

15
syft/formats/common/spdxhelpers/to_syft_model.go
View file

@ -2,6 +2,7 @@ package spdxhelpers
import (
"errors"
"github.com/anchore/syft/syft/source/scheme"
"net/url"
"strconv"
"strings"
@ -27,7 +28,7 @@ func ToSyftModel(doc *spdx.Document) (*sbom.SBOM, error) {
spdxIDMap := make(map[string]interface{})
src := source.Metadata{Scheme: source.UnknownScheme}
src := source.Metadata{Scheme: scheme.UnknownScheme}
src.Scheme = extractSchemeFromNamespace(doc.DocumentNamespace)
s := &sbom.SBOM{
@ -53,24 +54,24 @@ func ToSyftModel(doc *spdx.Document) (*sbom.SBOM, error) {
// image, directory, for example. This is our best effort to determine
// the scheme. Syft-generated SBOMs have in the namespace
// field a type encoded, which we try to identify here.
func extractSchemeFromNamespace(ns string) source.Scheme {
func extractSchemeFromNamespace(ns string) scheme.Scheme {
u, err := url.Parse(ns)
if err != nil {
return source.UnknownScheme
return scheme.UnknownScheme
}
parts := strings.Split(u.Path, "/")
for _, p := range parts {
switch p {
case inputFile:
return source.FileScheme
return scheme.FileScheme
case inputImage:
return source.ImageScheme
return scheme.ContainerImageScheme
case inputDirectory:
return source.DirectoryScheme
return scheme.DirectoryScheme
}
}
return source.UnknownScheme
return scheme.UnknownScheme
}
func findLinuxReleaseByPURL(doc *spdx.Document) *linux.Release {

15
syft/formats/common/spdxhelpers/to_syft_model_test.go
View file

@ -1,6 +1,7 @@
package spdxhelpers
import (
"github.com/anchore/syft/syft/source/scheme"
"testing"
"github.com/spdx/tools-golang/spdx"
@ -196,31 +197,31 @@ func Test_extractMetadata(t *testing.T) {
func TestExtractSourceFromNamespaces(t *testing.T) {
tests := []struct {
namespace string
expected source.Scheme
expected scheme.Scheme
}{
{
namespace: "https://anchore.com/syft/file/d42b01d0-7325-409b-b03f-74082935c4d3",
expected: source.FileScheme,
expected: scheme.FileScheme,
},
{
namespace: "https://anchore.com/syft/image/d42b01d0-7325-409b-b03f-74082935c4d3",
expected: source.ImageScheme,
expected: scheme.ContainerImageScheme,
},
{
namespace: "https://anchore.com/syft/dir/d42b01d0-7325-409b-b03f-74082935c4d3",
expected: source.DirectoryScheme,
expected: scheme.DirectoryScheme,
},
{
namespace: "https://another-host/blob/123",
expected: source.UnknownScheme,
expected: scheme.UnknownScheme,
},
{
namespace: "bla bla",
expected: source.UnknownScheme,
expected: scheme.UnknownScheme,
},
{
namespace: "",
expected: source.UnknownScheme,
expected: scheme.UnknownScheme,
},
}

7
syft/formats/github/encoder.go
View file

@ -2,6 +2,7 @@ package github
import (
"fmt"
"github.com/anchore/syft/syft/source/scheme"
"strings"
"time"
@ -85,15 +86,15 @@ func toPath(s source.Metadata, p pkg.Package) string {
}
packagePath = strings.TrimPrefix(packagePath, "/")
switch s.Scheme {
case source.ImageScheme:
case scheme.ContainerImageScheme:
image := strings.ReplaceAll(s.ImageMetadata.UserInput, ":/", "//")
return fmt.Sprintf("%s:/%s", image, packagePath)
case source.FileScheme:
case scheme.FileScheme:
if isArchive(inputPath) {
return fmt.Sprintf("%s:/%s", inputPath, packagePath)
}
return inputPath
case source.DirectoryScheme:
case scheme.DirectoryScheme:
if inputPath != "" {
return fmt.Sprintf("%s/%s", inputPath, packagePath)
}

13
syft/formats/github/encoder_test.go
View file

@ -2,6 +2,7 @@ package github
import (
"encoding/json"
"github.com/anchore/syft/syft/source/scheme"
"testing"
"github.com/stretchr/testify/assert"
@ -16,7 +17,7 @@ import (
func Test_toGithubModel(t *testing.T) {
s := sbom.SBOM{
Source: source.Metadata{
Scheme: source.ImageScheme,
Scheme: scheme.ContainerImageScheme,
ImageMetadata: source.ImageMetadata{
UserInput: "ubuntu:18.04",
Architecture: "amd64",
@ -135,27 +136,27 @@ func Test_toGithubModel(t *testing.T) {
// Just test the other schemes:
s.Source.Path = "."
s.Source.Scheme = source.DirectoryScheme
s.Source.Scheme = scheme.DirectoryScheme
actual = toGithubModel(&s)
assert.Equal(t, "etc", actual.Manifests["etc"].Name)
s.Source.Path = "./artifacts"
s.Source.Scheme = source.DirectoryScheme
s.Source.Scheme = scheme.DirectoryScheme
actual = toGithubModel(&s)
assert.Equal(t, "artifacts/etc", actual.Manifests["artifacts/etc"].Name)
s.Source.Path = "/artifacts"
s.Source.Scheme = source.DirectoryScheme
s.Source.Scheme = scheme.DirectoryScheme
actual = toGithubModel(&s)
assert.Equal(t, "/artifacts/etc", actual.Manifests["/artifacts/etc"].Name)
s.Source.Path = "./executable"
s.Source.Scheme = source.FileScheme
s.Source.Scheme = scheme.FileScheme
actual = toGithubModel(&s)
assert.Equal(t, "executable", actual.Manifests["executable"].Name)
s.Source.Path = "./archive.tar.gz"
s.Source.Scheme = source.FileScheme
s.Source.Scheme = scheme.FileScheme
actual = toGithubModel(&s)
assert.Equal(t, "archive.tar.gz:/etc", actual.Manifests["archive.tar.gz:/etc"].Name)
}

3
syft/formats/spdxtagvalue/encoder_test.go
View file

@ -2,6 +2,7 @@ package spdxtagvalue
import (
"flag"
"github.com/anchore/syft/syft/source/scheme"
"regexp"
"testing"
@ -53,7 +54,7 @@ func TestSPDXJSONSPDXIDs(t *testing.T) {
},
Relationships: nil,
Source: source.Metadata{
Scheme: source.DirectoryScheme,
Scheme: scheme.DirectoryScheme,
Path: "foobar/baz", // in this case, foobar is used as the spdx docment name
},
Descriptor: sbom.Descriptor{

3
syft/formats/syftjson/encoder_test.go
View file

@ -2,6 +2,7 @@ package syftjson
import (
"flag"
"github.com/anchore/syft/syft/source/scheme"
"regexp"
"testing"
@ -166,7 +167,7 @@ func TestEncodeFullJSONDocument(t *testing.T) {
},
Source: source.Metadata{
ID: "c2b46b4eb06296933b7cf0722683964e9ecbd93265b9ef6ae9642e3952afbba0",
Scheme: source.ImageScheme,
Scheme: scheme.ContainerImageScheme,
ImageMetadata: source.ImageMetadata{
UserInput: "user-image-input",
ID: "sha256:c2b46b4eb06296933b7cf0722683964e9ecbd93265b9ef6ae9642e3952afbba0",

7
syft/formats/syftjson/to_format_model.go
View file

@ -2,6 +2,7 @@ package syftjson
import (
"fmt"
"github.com/anchore/syft/syft/source/scheme"
"sort"
"strconv"
@ -241,7 +242,7 @@ func toRelationshipModel(relationships []artifact.Relationship) []model.Relation
// toSourceModel creates a new source object to be represented into JSON.
func toSourceModel(src source.Metadata) (model.Source, error) {
switch src.Scheme {
case source.ImageScheme:
case scheme.ContainerImageScheme:
metadata := src.ImageMetadata
// ensure that empty collections are not shown as null
if metadata.RepoDigests == nil {
@ -255,13 +256,13 @@ func toSourceModel(src source.Metadata) (model.Source, error) {
Type: "image",
Target: metadata,
}, nil
case source.DirectoryScheme:
case scheme.DirectoryScheme:
return model.Source{
ID: src.ID,
Type: "directory",
Target: src.Path,
}, nil
case source.FileScheme:
case scheme.FileScheme:
return model.Source{
ID: src.ID,
Type: "file",

9
syft/formats/syftjson/to_format_model_test.go
View file

@ -1,6 +1,7 @@
package syftjson
import (
"github.com/anchore/syft/syft/source/scheme"
"testing"
"github.com/scylladb/go-set/strset"
@ -14,7 +15,7 @@ import (
func Test_toSourceModel(t *testing.T) {
allSchemes := strset.New()
for _, s := range source.AllSchemes {
for _, s := range scheme.AllSchemes {
allSchemes.Add(string(s))
}
testedSchemes := strset.New()
@ -28,7 +29,7 @@ func Test_toSourceModel(t *testing.T) {
name: "directory",
src: source.Metadata{
ID: "test-id",
Scheme: source.DirectoryScheme,
Scheme: scheme.DirectoryScheme,
Path: "some/path",
},
expected: model.Source{
@ -41,7 +42,7 @@ func Test_toSourceModel(t *testing.T) {
name: "file",
src: source.Metadata{
ID: "test-id",
Scheme: source.FileScheme,
Scheme: scheme.FileScheme,
Path: "some/path",
},
expected: model.Source{
@ -54,7 +55,7 @@ func Test_toSourceModel(t *testing.T) {
name: "image",
src: source.Metadata{
ID: "test-id",
Scheme: source.ImageScheme,
Scheme: scheme.ContainerImageScheme,
ImageMetadata: source.ImageMetadata{
UserInput: "user-input",
ID: "id...",

7
syft/formats/syftjson/to_syft_model.go
View file

@ -1,6 +1,7 @@
package syftjson
import (
"github.com/anchore/syft/syft/source/scheme"
"os"
"strconv"
"strings"
@ -227,7 +228,7 @@ func toSyftSourceData(s model.Source) *source.Metadata {
}
return &source.Metadata{
ID: s.ID,
Scheme: source.DirectoryScheme,
Scheme: scheme.DirectoryScheme,
Path: path,
}
case "file":
@ -238,7 +239,7 @@ func toSyftSourceData(s model.Source) *source.Metadata {
}
return &source.Metadata{
ID: s.ID,
Scheme: source.FileScheme,
Scheme: scheme.FileScheme,
Path: path,
}
case "image":
@ -249,7 +250,7 @@ func toSyftSourceData(s model.Source) *source.Metadata {
}
return &source.Metadata{
ID: s.ID,
Scheme: source.ImageScheme,
Scheme: scheme.ContainerImageScheme,
ImageMetadata: metadata,
}
}

9
syft/formats/syftjson/to_syft_model_test.go
View file

@ -1,6 +1,7 @@
package syftjson
import (
"github.com/anchore/syft/syft/source/scheme"
"testing"
"github.com/scylladb/go-set/strset"
@ -16,7 +17,7 @@ import (
func Test_toSyftSourceData(t *testing.T) {
allSchemes := strset.New()
for _, s := range source.AllSchemes {
for _, s := range scheme.AllSchemes {
allSchemes.Add(string(s))
}
testedSchemes := strset.New()
@ -29,7 +30,7 @@ func Test_toSyftSourceData(t *testing.T) {
{
name: "directory",
expected: source.Metadata{
Scheme: source.DirectoryScheme,
Scheme: scheme.DirectoryScheme,
Path: "some/path",
},
src: model.Source{
@ -40,7 +41,7 @@ func Test_toSyftSourceData(t *testing.T) {
{
name: "file",
expected: source.Metadata{
Scheme: source.FileScheme,
Scheme: scheme.FileScheme,
Path: "some/path",
},
src: model.Source{
@ -51,7 +52,7 @@ func Test_toSyftSourceData(t *testing.T) {
{
name: "image",
expected: source.Metadata{
Scheme: source.ImageScheme,
Scheme: scheme.ContainerImageScheme,
ImageMetadata: source.ImageMetadata{
UserInput: "user-input",
ID: "id...",

6
syft/formats/text/encoder.go
View file

@ -2,11 +2,11 @@ package text
import (
"fmt"
"github.com/anchore/syft/syft/source/scheme"
"io"
"text/tabwriter"
"github.com/anchore/syft/syft/sbom"
"github.com/anchore/syft/syft/source"
)
func encoder(output io.Writer, s sbom.SBOM) error {
@ -15,9 +15,9 @@ func encoder(output io.Writer, s sbom.SBOM) error {
w.Init(output, 0, 8, 0, '\t', tabwriter.AlignRight)
switch s.Source.Scheme {
case source.DirectoryScheme, source.FileScheme:
case scheme.DirectoryScheme, scheme.FileScheme:
fmt.Fprintf(w, "[Path: %s]\n", s.Source.Path)
case source.ImageScheme:
case scheme.ContainerImageScheme:
fmt.Fprintln(w, "[Image]")
for idx, l := range s.Source.ImageMetadata.Layers {

19
syft/lib.go
View file

@ -34,7 +34,7 @@ import (
// CatalogPackages takes an inventory of packages from the given image from a particular perspective
// (e.g. squashed source, all-layers source). Returns the discovered set of packages, the identified Linux
// distribution, and the source object used to wrap the data source.
func CatalogPackages(src *source.Source, cfg cataloger.Config) (*pkg.Collection, []artifact.Relationship, *linux.Release, error) {
func CatalogPackages(src source.Source, cfg cataloger.Config) (*pkg.Collection, []artifact.Relationship, *linux.Release, error) {
resolver, err := src.FileResolver(cfg.Search.Scope)
if err != nil {
return nil, nil, nil, fmt.Errorf("unable to determine resolver while cataloging packages: %w", err)
@ -54,18 +54,23 @@ func CatalogPackages(src *source.Source, cfg cataloger.Config) (*pkg.Collection,
catalogers = cataloger.AllCatalogers(cfg)
} else {
// otherwise conditionally use the correct set of loggers based on the input type (container image or directory)
switch src.Metadata.Scheme {
case source.ImageScheme:
// TODO: this is bad, we should not be using the concrete type to determine the cataloger set
// instead this should be a caller concern (pass the catalogers you want to use). The SBOM build PR will do this.
switch src.(type) {
case *source.StereoscopeImageSource:
log.Info("cataloging image")
catalogers = cataloger.ImageCatalogers(cfg)
case source.FileScheme:
case *source.FileSource:
log.Info("cataloging file")
catalogers = cataloger.AllCatalogers(cfg)
case source.DirectoryScheme:
case *source.DirectorySource:
log.Info("cataloging directory")
catalogers = cataloger.DirectoryCatalogers(cfg)
default:
return nil, nil, nil, fmt.Errorf("unable to determine cataloger set from scheme=%+v", src.Metadata.Scheme)
// TODO: no
panic("REMOVE ME")
}
}
@ -76,7 +81,7 @@ func CatalogPackages(src *source.Source, cfg cataloger.Config) (*pkg.Collection,
return catalog, relationships, release, err
}
func newSourceRelationshipsFromCatalog(src *source.Source, c *pkg.Collection) []artifact.Relationship {
func newSourceRelationshipsFromCatalog(src source.Source, c *pkg.Collection) []artifact.Relationship {
relationships := make([]artifact.Relationship, 0) // Should we pre-allocate this by giving catalog a Len() method?
for p := range c.Enumerate() {
relationships = append(relationships, artifact.Relationship{

2
syft/sbom/sbom.go
View file

@ -15,7 +15,7 @@ import (
type SBOM struct {
Artifacts Artifacts
Relationships []artifact.Relationship
Source source.Metadata
Source source.Description
Descriptor Descriptor
}

18
syft/source/config.go Normal file
View file

@ -0,0 +1,18 @@
package source
type ExcludeConfig struct {
Paths []string
}
type ImageInterpreter interface {
Metadata() ImageMetadata
}
type PathInterpreter interface {
Metadata() PathMetadata
}
type PathMetadata struct {
Path string
Base string
}

8
syft/source/description.go Normal file
View file

@ -0,0 +1,8 @@
package source
// Description represents any static source data that helps describe "what" was cataloged.
type Description struct {
ID string `hash:"ignore"` // the id generated from the parent source struct
Name string
Metadata interface{}
}

152
syft/source/directory_source.go Normal file
View file

@ -0,0 +1,152 @@
package source
import (
"fmt"
"github.com/anchore/syft/syft/artifact"
"github.com/bmatcuk/doublestar/v4"
"github.com/opencontainers/go-digest"
"os"
"path/filepath"
"strings"
"sync"
)
var (
_ Source = (*DirectorySource)(nil)
_ PathInterpreter = (*DirectorySource)(nil)
)
type DirectoryConfig struct {
Path string
Base string
Exclude ExcludeConfig
Name string // ? can this be done differently?
}
// TODO: add fs.FS support
type DirectorySource struct {
id artifact.ID
config DirectoryConfig
resolver *directoryResolver
mutex *sync.Mutex
// implements PathInterpreter
}
func NewFromDirectory(cfg DirectoryConfig) (Source, error) {
fi, err := os.Stat(cfg.Path)
if err != nil {
return nil, fmt.Errorf("unable to stat path=%q: %w", cfg.Path, err)
}
if !fi.IsDir() {
return nil, fmt.Errorf("given path is not a directory (path=%q): %w", cfg.Path, err)
}
return &DirectorySource{
id: artifact.ID(strings.TrimPrefix(digest.FromString(cfg.Path).String(), "sha256:")),
config: cfg,
mutex: &sync.Mutex{},
}, nil
}
func (s DirectorySource) ID() artifact.ID {
return s.id
}
func (s DirectorySource) Metadata() PathMetadata {
return PathMetadata{
Path: s.config.Path,
Base: s.config.Base,
}
}
func (s DirectorySource) Describe() Description {
return Description{
ID: string(s.id),
Name: s.config.Path,
Metadata: s.Metadata(),
}
}
func (s *DirectorySource) FileResolver(scope Scope) (FileResolver, error) {
s.mutex.Lock()
defer s.mutex.Unlock()
if s.resolver == nil {
exclusionFunctions, err := getDirectoryExclusionFunctions(s.config.Path, s.config.Exclude.Paths)
if err != nil {
return nil, err
}
resolver, err := newDirectoryResolver(s.config.Path, s.config.Base, exclusionFunctions...)
if err != nil {
return nil, fmt.Errorf("unable to create directory resolver: %w", err)
}
s.resolver = resolver
}
return s.resolver, nil
}
func (s *DirectorySource) Close() error {
s.mutex.Lock()
defer s.mutex.Unlock()
s.resolver = nil
return nil
}
func getDirectoryExclusionFunctions(root string, exclusions []string) ([]pathIndexVisitor, error) {
if len(exclusions) == 0 {
return nil, nil
}
// this is what directoryResolver.indexTree is doing to get the absolute path:
root, err := filepath.Abs(root)
if err != nil {
return nil, err
}
// this handles Windows file paths by converting them to C:/something/else format
root = filepath.ToSlash(root)
if !strings.HasSuffix(root, "/") {
root += "/"
}
var errors []string
for idx, exclusion := range exclusions {
// check exclusions for supported paths, these are all relative to the "scan root"
if strings.HasPrefix(exclusion, "./") || strings.HasPrefix(exclusion, "*/") || strings.HasPrefix(exclusion, "**/") {
exclusion = strings.TrimPrefix(exclusion, "./")
exclusions[idx] = root + exclusion
} else {
errors = append(errors, exclusion)
}
}
if errors != nil {
return nil, fmt.Errorf("invalid exclusion pattern(s): '%s' (must start with one of: './', '*/', or '**/')", strings.Join(errors, "', '"))
}
return []pathIndexVisitor{
func(path string, info os.FileInfo, _ error) error {
for _, exclusion := range exclusions {
// this is required to handle Windows filepaths
path = filepath.ToSlash(path)
matches, err := doublestar.Match(exclusion, path)
if err != nil {
return nil
}
if matches {
if info != nil && info.IsDir() {
return filepath.SkipDir
}
return errSkipPath
}
}
return nil
},
}, nil
}

161
syft/source/file_source.go Normal file
View file

@ -0,0 +1,161 @@
package source
import (
"fmt"
"github.com/anchore/syft/internal/log"
"github.com/anchore/syft/syft/artifact"
"github.com/mholt/archiver/v3"
"github.com/opencontainers/go-digest"
"os"
"strings"
"sync"
)
var (
_ Source = (*FileSource)(nil)
_ PathInterpreter = (*FileSource)(nil)
)
type FileConfig struct {
Path string
Exclude ExcludeConfig
Name string // ? can this be done differently?
// base??
}
// TODO: add fs.FS support
type FileSource struct {
id artifact.ID
config FileConfig
resolver *directoryResolver
mutex *sync.Mutex
closer func() error
analysisPath string
}
func NewFromFile(cfg FileConfig) (Source, error) {
fileMeta, err := os.Stat(cfg.Path)
if err != nil {
return nil, fmt.Errorf("unable to stat path=%q: %w", cfg.Path, err)
}
if fileMeta.IsDir() {
return nil, fmt.Errorf("given path is a directory (path=%q): %w", cfg.Path, err)
}
analysisPath, cleanupFn := fileAnalysisPath(cfg.Path)
return &FileSource{
id: artifact.ID(strings.TrimPrefix(digestOfFileContents(cfg.Path), "sha256:")),
config: FileConfig{},
mutex: &sync.Mutex{},
closer: cleanupFn,
analysisPath: analysisPath,
}, nil
}
func (s FileSource) ID() artifact.ID {
return s.id
}
func (s FileSource) Metadata() PathMetadata {
return PathMetadata{
Path: s.config.Path,
Base: "",
}
}
func (s FileSource) Describe() Description {
return Description{
ID: string(s.id),
Name: s.config.Path,
Metadata: s.Metadata(),
}
}
func (s FileSource) FileResolver(scope Scope) (FileResolver, error) {
s.mutex.Lock()
defer s.mutex.Unlock()
if s.resolver == nil {
exclusionFunctions, err := getDirectoryExclusionFunctions(s.analysisPath, s.config.Exclude.Paths)
if err != nil {
return nil, err
}
resolver, err := newDirectoryResolver(s.analysisPath, "", exclusionFunctions...)
if err != nil {
return nil, fmt.Errorf("unable to create directory resolver: %w", err)
}
s.resolver = resolver
}
return s.resolver, nil
}
func (s FileSource) Close() error {
if s.closer == nil {
return nil
}
s.resolver = nil
return s.closer()
}
// fileAnalysisPath returns the path given, or in the case the path is an archive, the location where the archive
// contents have been made available. A cleanup function is provided for any temp files created (if any).
func fileAnalysisPath(path string) (string, func() error) {
var analysisPath = path
var cleanupFn = func() error { return nil }
// if the given file is an archive (as indicated by the file extension and not MIME type) then unarchive it and
// use the contents as the source. Note: this does NOT recursively unarchive contents, only the given path is
// unarchived.
envelopedUnarchiver, err := archiver.ByExtension(path)
if unarchiver, ok := envelopedUnarchiver.(archiver.Unarchiver); err == nil && ok {
if tar, ok := unarchiver.(*archiver.Tar); ok {
// when tar files are extracted, if there are multiple entries at the same
// location, the last entry wins
// NOTE: this currently does not display any messages if an overwrite happens
tar.OverwriteExisting = true
}
unarchivedPath, tmpCleanup, err := unarchiveToTmp(path, unarchiver)
if err != nil {
log.Warnf("file could not be unarchived: %+v", err)
} else {
log.Debugf("source path is an archive")
analysisPath = unarchivedPath
}
if tmpCleanup != nil {
cleanupFn = tmpCleanup
}
}
return analysisPath, cleanupFn
}
func digestOfFileContents(path string) string {
file, err := os.Open(path)
if err != nil {
return digest.FromString(path).String()
}
defer file.Close()
di, err := digest.FromReader(file)
if err != nil {
return digest.FromString(path).String()
}
return di.String()
}
func unarchiveToTmp(path string, unarchiver archiver.Unarchiver) (string, func() error, error) {
tempDir, err := os.MkdirTemp("", "syft-archive-contents-")
if err != nil {
return "", func() error { return nil }, fmt.Errorf("unable to create tempdir for archive processing: %w", err)
}
cleanupFn := func() error {
return os.RemoveAll(tempDir)
}
return tempDir, cleanupFn, unarchiver.Unarchive(path, tempDir) // TODO: does not work, use v4 for io.FS support
}

208
syft/source/image_source.go Normal file
View file

@ -0,0 +1,208 @@
package source
import (
"context"
"fmt"
"github.com/anchore/stereoscope"
"github.com/anchore/stereoscope/pkg/image"
"github.com/anchore/syft/syft/artifact"
"github.com/bmatcuk/doublestar/v4"
"github.com/opencontainers/go-digest"
"strings"
)
var (
_ Source = (*StereoscopeImageSource)(nil)
_ ImageInterpreter = (*StereoscopeImageSource)(nil)
)
type StereoscopeImageConfig struct {
Reference string
From image.Source
Platform *image.Platform
RegistryOptions *image.RegistryOptions // TODO: takes platform? as string?
Exclude ExcludeConfig
Name string // ? can this be done differently?
}
type StereoscopeImageSource struct {
id artifact.ID
config StereoscopeImageConfig
image *image.Image
metadata ImageMetadata
}
func NewFromImage(cfg StereoscopeImageConfig) (Source, error) {
ctx := context.TODO()
var opts []stereoscope.Option
if cfg.RegistryOptions != nil {
opts = append(opts, stereoscope.WithRegistryOptions(*cfg.RegistryOptions))
}
if cfg.Platform != nil {
opts = append(opts, stereoscope.WithPlatform(cfg.Platform.String()))
}
img, err := stereoscope.GetImageFromSource(ctx, cfg.Reference, cfg.From, opts...)
if err != nil {
return nil, fmt.Errorf("unable to load image: %w", err)
}
metadata := imageMetadataFromStereoscopeImage(img, cfg.Reference)
return &StereoscopeImageSource{
id: artifactIDFromStereoscopeImage(metadata),
config: cfg,
image: img,
metadata: metadata,
}, nil
}
func (s StereoscopeImageSource) ID() artifact.ID {
return s.id
}
func (s StereoscopeImageSource) Metadata() ImageMetadata {
return s.metadata
}
func (s StereoscopeImageSource) Describe() Description {
return Description{
ID: string(s.id),
Name: s.config.Reference,
Metadata: s.Metadata(),
}
}
func (s StereoscopeImageSource) FileResolver(scope Scope) (FileResolver, error) {
var resolver FileResolver
var err error
switch scope {
case SquashedScope:
resolver, err = newImageSquashResolver(s.image)
case AllLayersScope:
resolver, err = newAllLayersResolver(s.image)
default:
return nil, fmt.Errorf("bad image scope provided: %+v", scope)
}
if err != nil {
return nil, err
}
// image tree contains all paths, so we filter out the excluded entries afterward
if len(s.config.Exclude.Paths) > 0 {
resolver = NewExcludingResolver(resolver, getImageExclusionFunction(s.config.Exclude.Paths))
}
return resolver, nil
}
func (s StereoscopeImageSource) Close() error {
if s.image == nil {
return nil
}
return s.image.Cleanup()
}
func imageMetadataFromStereoscopeImage(img *image.Image, reference string) ImageMetadata {
tags := make([]string, len(img.Metadata.Tags))
for idx, tag := range img.Metadata.Tags {
tags[idx] = tag.String()
}
layers := make([]LayerMetadata, len(img.Layers))
for _, l := range img.Layers {
layers = append(layers,
LayerMetadata{
MediaType: string(l.Metadata.MediaType),
Digest: l.Metadata.Digest,
Size: l.Metadata.Size,
},
)
}
return ImageMetadata{
ID: img.Metadata.ID,
UserInput: reference,
ManifestDigest: img.Metadata.ManifestDigest,
Size: img.Metadata.Size,
MediaType: string(img.Metadata.MediaType),
Tags: tags,
Layers: layers,
RawConfig: img.Metadata.RawConfig,
RawManifest: img.Metadata.RawManifest,
RepoDigests: img.Metadata.RepoDigests,
Architecture: img.Metadata.Architecture,
Variant: img.Metadata.Variant,
OS: img.Metadata.OS,
}
}
func artifactIDFromStereoscopeImage(metadata ImageMetadata) artifact.ID {
var input string
manifestDigest := digest.FromBytes(metadata.RawManifest).String()
if manifestDigest != "" {
input = manifestDigest
} else {
// calculate chain ID for image sources where manifestDigest is not available
// https://github.com/opencontainers/image-spec/blob/main/config.md#layer-chainid
input = calculateChainID(metadata.Layers)
if input == "" {
// TODO what happens here if image has no layers?
// is this case possible?
input = digest.FromString(metadata.UserInput).String()
}
}
return artifact.ID(strings.TrimPrefix(input, "sha256:"))
}
func calculateChainID(lm []LayerMetadata) string {
if len(lm) < 1 {
return ""
}
// DiffID(L0) = digest of layer 0
// https://github.com/anchore/stereoscope/blob/1b1b744a919964f38d14e1416fb3f25221b761ce/pkg/image/layer_metadata.go#L19-L32
chainID := lm[0].Digest
id := chain(chainID, lm[1:])
return id
}
func chain(chainID string, layers []LayerMetadata) string {
if len(layers) < 1 {
return chainID
}
chainID = digest.FromString(layers[0].Digest + " " + chainID).String()
return chain(chainID, layers[1:])
}
func getImageExclusionFunction(exclusions []string) func(string) bool {
if len(exclusions) == 0 {
return nil
}
// add subpath exclusions
for _, exclusion := range exclusions {
exclusions = append(exclusions, exclusion+"/**")
}
return func(path string) bool {
for _, exclusion := range exclusions {
matches, err := doublestar.Match(exclusion, path)
if err != nil {
return false
}
if matches {
return true
}
}
return false
}
}

11
syft/source/metadata.go
View file

@ -1,11 +0,0 @@
package source
// Metadata represents any static source data that helps describe "what" was cataloged.
type Metadata struct {
ID string `hash:"ignore"` // the id generated from the parent source struct
Scheme Scheme // the source data scheme type (directory or image)
ImageMetadata ImageMetadata // all image info (image only)
Path string // the root path to be cataloged (directory only)
Base string // the base path to be cataloged (directory only)
Name string
}

74
syft/source/scheme.go
View file

@ -1,74 +0,0 @@
package source
import (
"fmt"
"strings"
"github.com/mitchellh/go-homedir"
"github.com/spf13/afero"
"github.com/anchore/stereoscope/pkg/image"
)
// Scheme represents the optional prefixed string at the beginning of a user request (e.g. "docker:").
type Scheme string
const (
// UnknownScheme is the default scheme
UnknownScheme Scheme = "UnknownScheme"
// DirectoryScheme indicates the source being cataloged is a directory on the root filesystem
DirectoryScheme Scheme = "DirectoryScheme"
// ImageScheme indicates the source being cataloged is a container image
ImageScheme Scheme = "ImageScheme"
// FileScheme indicates the source being cataloged is a single file
FileScheme Scheme = "FileScheme"
)
var AllSchemes = []Scheme{
DirectoryScheme,
ImageScheme,
FileScheme,
}
func DetectScheme(fs afero.Fs, imageDetector sourceDetector, userInput string) (Scheme, image.Source, string, error) {
switch {
case strings.HasPrefix(userInput, "dir:"):
dirLocation, err := homedir.Expand(strings.TrimPrefix(userInput, "dir:"))
if err != nil {
return UnknownScheme, image.UnknownSource, "", fmt.Errorf("unable to expand directory path: %w", err)
}
return DirectoryScheme, image.UnknownSource, dirLocation, nil
case strings.HasPrefix(userInput, "file:"):
fileLocation, err := homedir.Expand(strings.TrimPrefix(userInput, "file:"))
if err != nil {
return UnknownScheme, image.UnknownSource, "", fmt.Errorf("unable to expand directory path: %w", err)
}
return FileScheme, image.UnknownSource, fileLocation, nil
}
// try the most specific sources first and move out towards more generic sources.
// first: let's try the image detector, which has more scheme parsing internal to stereoscope
source, imageSpec, err := imageDetector(userInput)
if err == nil && source != image.UnknownSource {
return ImageScheme, source, imageSpec, nil
}
// next: let's try more generic sources (dir, file, etc.)
location, err := homedir.Expand(userInput)
if err != nil {
return UnknownScheme, image.UnknownSource, "", fmt.Errorf("unable to expand potential directory path: %w", err)
}
fileMeta, err := fs.Stat(location)
if err != nil {
return UnknownScheme, source, "", nil
}
if fileMeta.IsDir() {
return DirectoryScheme, source, location, nil
}
return FileScheme, source, location, nil
}

140
syft/source/scheme/scheme.go Normal file
View file

@ -0,0 +1,140 @@
package scheme
import (
"fmt"
"strings"
"github.com/mitchellh/go-homedir"
"github.com/spf13/afero"
"github.com/anchore/stereoscope/pkg/image"
)
// Scheme represents the optional prefixed string at the beginning of a user request (e.g. "docker:").
type Scheme string
type sourceDetector func(string) (image.Source, string, error)
const (
// UnknownScheme is the default scheme
UnknownScheme Scheme = "UnknownScheme"
// DirectoryScheme indicates the source being cataloged is a directory on the root filesystem
DirectoryScheme Scheme = "DirectoryScheme"
// ContainerImageScheme indicates the source being cataloged is a container image
ContainerImageScheme Scheme = "ContainerImageScheme"
// FileScheme indicates the source being cataloged is a single file
FileScheme Scheme = "FileScheme"
)
var AllSchemes = []Scheme{
DirectoryScheme,
ContainerImageScheme,
FileScheme,
}
// Input is an object that captures the detected user input regarding source location, scheme, and provider type.
// It acts as a struct input for some source constructors.
type Input struct {
UserInput string
Scheme Scheme
ImageSource image.Source
Location string
Platform string
Name string
}
// Parse generates a source Input that can be used as an argument to generate a new source
// from specific providers including a registry, with an explicit name.
func Parse(userInput string, platform, name, defaultImageSource string) (*Input, error) {
fs := afero.NewOsFs()
scheme, source, location, err := detect(fs, image.DetectSource, userInput)
if err != nil {
return nil, err
}
if source == image.UnknownSource {
// only run for these two scheme
// only check on packages command, attest we automatically try to pull from userInput
switch scheme {
case ContainerImageScheme, UnknownScheme:
scheme = ContainerImageScheme
location = userInput
if defaultImageSource != "" {
source = parseDefaultImageSource(defaultImageSource)
} else {
source = image.DetermineDefaultImagePullSource(userInput)
}
default:
}
}
if scheme != ContainerImageScheme && platform != "" {
return nil, fmt.Errorf("cannot specify a platform for a non-image source")
}
// collect user input for downstream consumption
return &Input{
UserInput: userInput,
Scheme: scheme,
ImageSource: source,
Location: location,
Platform: platform,
Name: name,
}, nil
}
func detect(fs afero.Fs, imageDetector sourceDetector, userInput string) (Scheme, image.Source, string, error) {
switch {
case strings.HasPrefix(userInput, "dir:"):
dirLocation, err := homedir.Expand(strings.TrimPrefix(userInput, "dir:"))
if err != nil {
return UnknownScheme, image.UnknownSource, "", fmt.Errorf("unable to expand directory path: %w", err)
}
return DirectoryScheme, image.UnknownSource, dirLocation, nil
case strings.HasPrefix(userInput, "file:"):
fileLocation, err := homedir.Expand(strings.TrimPrefix(userInput, "file:"))
if err != nil {
return UnknownScheme, image.UnknownSource, "", fmt.Errorf("unable to expand directory path: %w", err)
}
return FileScheme, image.UnknownSource, fileLocation, nil
}
// try the most specific sources first and move out towards more generic sources.
// first: let's try the image detector, which has more scheme parsing internal to stereoscope
source, imageSpec, err := imageDetector(userInput)
if err == nil && source != image.UnknownSource {
return ContainerImageScheme, source, imageSpec, nil
}
// next: let's try more generic sources (dir, file, etc.)
location, err := homedir.Expand(userInput)
if err != nil {
return UnknownScheme, image.UnknownSource, "", fmt.Errorf("unable to expand potential directory path: %w", err)
}
fileMeta, err := fs.Stat(location)
if err != nil {
return UnknownScheme, source, "", nil
}
if fileMeta.IsDir() {
return DirectoryScheme, source, location, nil
}
return FileScheme, source, location, nil
}
func parseDefaultImageSource(defaultImageSource string) image.Source {
switch defaultImageSource {
case "registry":
return image.OciRegistrySource
case "docker":
return image.DockerDaemonSource
case "podman":
return image.PodmanDaemonSource
default:
return image.UnknownSource
}
}

28
syft/source/scheme_test.go → syft/source/scheme/scheme_test.go
View file

@ -1,4 +1,4 @@
package source
package scheme
import (
"os"
@ -34,7 +34,7 @@ func TestDetectScheme(t *testing.T) {
src: image.DockerDaemonSource,
ref: "wagoodman/dive:latest",
},
expectedScheme: ImageScheme,
expectedScheme: ContainerImageScheme,
expectedLocation: "wagoodman/dive:latest",
},
{
@ -44,7 +44,7 @@ func TestDetectScheme(t *testing.T) {
src: image.DockerDaemonSource,
ref: "wagoodman/dive",
},
expectedScheme: ImageScheme,
expectedScheme: ContainerImageScheme,
expectedLocation: "wagoodman/dive",
},
{
@ -54,7 +54,7 @@ func TestDetectScheme(t *testing.T) {
src: image.OciRegistrySource,
ref: "wagoodman/dive:latest",
},
expectedScheme: ImageScheme,
expectedScheme: ContainerImageScheme,
expectedLocation: "wagoodman/dive:latest",
},
{
@ -64,7 +64,7 @@ func TestDetectScheme(t *testing.T) {
src: image.DockerDaemonSource,
ref: "wagoodman/dive:latest",
},
expectedScheme: ImageScheme,
expectedScheme: ContainerImageScheme,
expectedLocation: "wagoodman/dive:latest",
},
{
@ -74,7 +74,7 @@ func TestDetectScheme(t *testing.T) {
src: image.DockerDaemonSource,
ref: "wagoodman/dive",
},
expectedScheme: ImageScheme,
expectedScheme: ContainerImageScheme,
expectedLocation: "wagoodman/dive",
},
{
@ -84,7 +84,7 @@ func TestDetectScheme(t *testing.T) {
src: image.DockerDaemonSource,
ref: "latest",
},
expectedScheme: ImageScheme,
expectedScheme: ContainerImageScheme,
// we expected to be able to handle this case better, however, I don't see a way to do this
// the user will need to provide more explicit input (docker:docker:latest)
expectedLocation: "latest",
@ -96,7 +96,7 @@ func TestDetectScheme(t *testing.T) {
src: image.DockerDaemonSource,
ref: "docker:latest",
},
expectedScheme: ImageScheme,
expectedScheme: ContainerImageScheme,
// we expected to be able to handle this case better, however, I don't see a way to do this
// the user will need to provide more explicit input (docker:docker:latest)
expectedLocation: "docker:latest",
@ -108,7 +108,7 @@ func TestDetectScheme(t *testing.T) {
src: image.OciTarballSource,
ref: "some/path-to-file",
},
expectedScheme: ImageScheme,
expectedScheme: ContainerImageScheme,
expectedLocation: "some/path-to-file",
},
{
@ -119,7 +119,7 @@ func TestDetectScheme(t *testing.T) {
ref: "some/path-to-dir",
},
dirs: []string{"some/path-to-dir"},
expectedScheme: ImageScheme,
expectedScheme: ContainerImageScheme,
expectedLocation: "some/path-to-dir",
},
{
@ -140,7 +140,7 @@ func TestDetectScheme(t *testing.T) {
src: image.DockerDaemonSource,
ref: "some/path-to-dir",
},
expectedScheme: ImageScheme,
expectedScheme: ContainerImageScheme,
expectedLocation: "some/path-to-dir",
},
{
@ -150,7 +150,7 @@ func TestDetectScheme(t *testing.T) {
src: image.PodmanDaemonSource,
ref: "something:latest",
},
expectedScheme: ImageScheme,
expectedScheme: ContainerImageScheme,
expectedLocation: "something:latest",
},
{
@ -214,7 +214,7 @@ func TestDetectScheme(t *testing.T) {
src: image.OciDirectorySource,
ref: "~/some-path",
},
expectedScheme: ImageScheme,
expectedScheme: ContainerImageScheme,
expectedLocation: "~/some-path",
},
{
@ -288,7 +288,7 @@ func TestDetectScheme(t *testing.T) {
}
}
actualScheme, actualSource, actualLocation, err := DetectScheme(fs, imageDetector, test.userInput)
actualScheme, actualSource, actualLocation, err := detect(fs, imageDetector, test.userInput)
if err != nil {
t.Fatalf("unexpected err : %+v", err)
}

59
syft/source/scheme/source.go Normal file
View file

@ -0,0 +1,59 @@
package scheme
import (
"fmt"
"github.com/anchore/stereoscope/pkg/image"
"github.com/anchore/syft/syft/source"
)
// NewSource produces a Source based on userInput like dir: or image:tag
func NewSource(in Input, registryOptions *image.RegistryOptions, exclusions []string) (source.Source, error) {
var err error
var src source.Source
switch in.Scheme {
case FileScheme:
src, err = source.NewFromFile(
source.FileConfig{
Path: in.Location,
Exclude: source.ExcludeConfig{
Paths: exclusions,
},
Name: in.Name,
},
)
case DirectoryScheme:
src, err = source.NewFromDirectory(
source.DirectoryConfig{
Path: in.Location,
Base: in.Location,
Exclude: source.ExcludeConfig{},
Name: in.Name,
},
)
case ContainerImageScheme:
var platform *image.Platform
if in.Platform != "" {
platform, err = image.NewPlatform(in.Platform)
if err != nil {
return nil, fmt.Errorf("unable to parse platform: %w", err)
}
}
src, err = source.NewFromImage(
source.StereoscopeImageConfig{
Reference: in.Location,
From: in.ImageSource,
Platform: platform,
RegistryOptions: registryOptions,
Exclude: source.ExcludeConfig{
Paths: exclusions,
},
Name: in.Name,
},
)
default:
err = fmt.Errorf("unable to process input for scanning: %q", in.UserInput)
}
return src, err
}

2
syft/source/scope.go
View file

@ -10,7 +10,7 @@ const (
UnknownScope Scope = "UnknownScope"
// SquashedScope indicates to only catalog content visible from the squashed filesystem representation (what can be seen only within the container at runtime)
SquashedScope Scope = "Squashed"
// AllLayersScope indicates to catalog content on all layers, irregardless if it is visible from the container at runtime.
// AllLayersScope indicates to catalog content on all layers, regardless if it is visible from the container at runtime.
AllLayersScope Scope = "AllLayers"
)

889
syft/source/source.go
View file

@ -6,159 +6,140 @@ within this package.
package source
import (
"context"
"fmt"
"os"
"path/filepath"
"strings"
"sync"
"github.com/bmatcuk/doublestar/v4"
"github.com/mholt/archiver/v3"
digest "github.com/opencontainers/go-digest"
"github.com/spf13/afero"
"github.com/anchore/stereoscope"
"github.com/anchore/stereoscope/pkg/image"
"github.com/anchore/syft/internal/log"
"github.com/anchore/syft/syft/artifact"
"io"
"strings"
)
// Source is an object that captures the data source to be cataloged, configuration, and a specific resolver used
// in cataloging (based on the data source and configuration)
type Source struct {
id artifact.ID `hash:"ignore"`
Image *image.Image `hash:"ignore"` // the image object to be cataloged (image only)
Metadata Metadata
directoryResolver *directoryResolver `hash:"ignore"`
path string
base string
mutex *sync.Mutex
Exclusions []string `hash:"ignore"`
type Source interface {
artifact.Identifiable
FileResolver(Scope) (FileResolver, error)
Describe() Description
io.Closer
}
// Input is an object that captures the detected user input regarding source location, scheme, and provider type.
// It acts as a struct input for some source constructors.
type Input struct {
UserInput string
Scheme Scheme
ImageSource image.Source
Location string
Platform string
Name string
}
//// Input is an object that captures the detected user input regarding source location, scheme, and provider type.
//// It acts as a struct input for some source constructors.
//type Input struct {
// UserInput string
// Scheme Scheme
// ImageSource image.Source
// Location string
// Platform string
// Name string
//}
//
//// ParseInput generates a source Input that can be used as an argument to generate a new source
//// from specific providers including a registry.
//func ParseInput(userInput string, platform string) (*Input, error) {
// return ParseInputWithName(userInput, platform, "", "")
//}
//
//// ParseInputWithName generates a source Input that can be used as an argument to generate a new source
//// from specific providers including a registry, with an explicit name.
//func ParseInputWithName(userInput string, platform, name, defaultImageSource string) (*Input, error) {
// fs := afero.NewOsFs()
// scheme, source, location, err := DetectScheme(fs, image.DetectSource, userInput)
// if err != nil {
// return nil, err
// }
//
// if source == image.UnknownSource {
// // only run for these two scheme
// // only check on packages command, attest we automatically try to pull from userInput
// switch scheme {
// case ImageScheme, UnknownScheme:
// scheme = ImageScheme
// location = userInput
// if defaultImageSource != "" {
// source = parseDefaultImageSource(defaultImageSource)
// } else {
// imagePullSource := image.DetermineDefaultImagePullSource(userInput)
// source = imagePullSource
// }
// if location == "" {
// location = userInput
// }
// default:
// }
// }
//
// if scheme != ImageScheme && platform != "" {
// return nil, fmt.Errorf("cannot specify a platform for a non-image source")
// }
//
// // collect user input for downstream consumption
// return &Input{
// UserInput: userInput,
// Scheme: scheme,
// ImageSource: source,
// Location: location,
// Platform: platform,
// Name: name,
// }, nil
//}
//
//func parseDefaultImageSource(defaultImageSource string) image.Source {
// switch defaultImageSource {
// case "registry":
// return image.OciRegistrySource
// case "docker":
// return image.DockerDaemonSource
// case "podman":
// return image.PodmanDaemonSource
// default:
// return image.UnknownSource
// }
//}
//
//type sourceDetector func(string) (image.Source, string, error)
// ParseInput generates a source Input that can be used as an argument to generate a new source
// from specific providers including a registry.
func ParseInput(userInput string, platform string) (*Input, error) {
return ParseInputWithName(userInput, platform, "", "")
}
//func NewFromRegistry(in Input, registryOptions *image.RegistryOptions, exclusions []string) (*Source, func(), error) {
// source, cleanupFn, err := generateImageSource(in, registryOptions)
// if source != nil {
// source.Exclusions = exclusions
// }
// return source, cleanupFn, err
//}
//
//// New produces a Source based on userInput like dir: or image:tag
//func New(in Input, registryOptions *image.RegistryOptions, exclusions []string) (*Source, func(), error) {
// var err error
// fs := afero.NewOsFs()
// var source *Source
// cleanupFn := func() {}
//
// switch in.Scheme {
// case FileScheme:
// source, cleanupFn, err = generateFileSource(fs, in)
// case DirectoryScheme:
// source, cleanupFn, err = generateDirectorySource(fs, in)
// case ImageScheme:
// source, cleanupFn, err = generateImageSource(in, registryOptions)
// default:
// err = fmt.Errorf("unable to process input for scanning: %q", in.UserInput)
// }
//
// if err == nil {
// source.Exclusions = exclusions
// }
//
// return source, cleanupFn, err
//}
// ParseInputWithName generates a source Input that can be used as an argument to generate a new source
// from specific providers including a registry, with an explicit name.
func ParseInputWithName(userInput string, platform, name, defaultImageSource string) (*Input, error) {
fs := afero.NewOsFs()
scheme, source, location, err := DetectScheme(fs, image.DetectSource, userInput)
if err != nil {
return nil, err
}
if source == image.UnknownSource {
// only run for these two scheme
// only check on packages command, attest we automatically try to pull from userInput
switch scheme {
case ImageScheme, UnknownScheme:
scheme = ImageScheme
location = userInput
if defaultImageSource != "" {
source = parseDefaultImageSource(defaultImageSource)
} else {
imagePullSource := image.DetermineDefaultImagePullSource(userInput)
source = imagePullSource
}
if location == "" {
location = userInput
}
default:
}
}
if scheme != ImageScheme && platform != "" {
return nil, fmt.Errorf("cannot specify a platform for a non-image source")
}
// collect user input for downstream consumption
return &Input{
UserInput: userInput,
Scheme: scheme,
ImageSource: source,
Location: location,
Platform: platform,
Name: name,
}, nil
}
func parseDefaultImageSource(defaultImageSource string) image.Source {
switch defaultImageSource {
case "registry":
return image.OciRegistrySource
case "docker":
return image.DockerDaemonSource
case "podman":
return image.PodmanDaemonSource
default:
return image.UnknownSource
}
}
type sourceDetector func(string) (image.Source, string, error)
func NewFromRegistry(in Input, registryOptions *image.RegistryOptions, exclusions []string) (*Source, func(), error) {
source, cleanupFn, err := generateImageSource(in, registryOptions)
if source != nil {
source.Exclusions = exclusions
}
return source, cleanupFn, err
}
// New produces a Source based on userInput like dir: or image:tag
func New(in Input, registryOptions *image.RegistryOptions, exclusions []string) (*Source, func(), error) {
var err error
fs := afero.NewOsFs()
var source *Source
cleanupFn := func() {}
switch in.Scheme {
case FileScheme:
source, cleanupFn, err = generateFileSource(fs, in)
case DirectoryScheme:
source, cleanupFn, err = generateDirectorySource(fs, in)
case ImageScheme:
source, cleanupFn, err = generateImageSource(in, registryOptions)
default:
err = fmt.Errorf("unable to process input for scanning: %q", in.UserInput)
}
if err == nil {
source.Exclusions = exclusions
}
return source, cleanupFn, err
}
func generateImageSource(in Input, registryOptions *image.RegistryOptions) (*Source, func(), error) {
img, cleanup, err := getImageWithRetryStrategy(in, registryOptions)
if err != nil || img == nil {
return nil, cleanup, fmt.Errorf("could not fetch image %q: %w", in.Location, err)
}
s, err := NewFromImageWithName(img, in.Location, in.Name)
if err != nil {
return nil, cleanup, fmt.Errorf("could not populate source with image: %w", err)
}
return &s, cleanup, nil
}
//func generateImageSource(in Input, registryOptions *image.RegistryOptions) (*Source, func(), error) {
// img, cleanup, err := getImageWithRetryStrategy(in, registryOptions)
// if err != nil || img == nil {
// return nil, cleanup, fmt.Errorf("could not fetch image %q: %w", in.Location, err)
// }
//
// s, err := NewFromImageWithName(img, in.Location, in.Name)
// if err != nil {
// return nil, cleanup, fmt.Errorf("could not populate source with image: %w", err)
// }
//
// return &s, cleanup, nil
//}
func parseScheme(userInput string) string {
parts := strings.SplitN(userInput, ":", 2)
@ -169,416 +150,216 @@ func parseScheme(userInput string) string {
return parts[0]
}
func getImageWithRetryStrategy(in Input, registryOptions *image.RegistryOptions) (*image.Image, func(), error) {
ctx := context.TODO()
//func generateDirectorySource(fs afero.Fs, in Input) (*Source, func(), error) {
// fileMeta, err := fs.Stat(in.Location)
// if err != nil {
// return nil, func() {}, fmt.Errorf("unable to stat dir=%q: %w", in.Location, err)
// }
//
// if !fileMeta.IsDir() {
// return nil, func() {}, fmt.Errorf("given path is not a directory (path=%q): %w", in.Location, err)
// }
//
// s, err := NewFromDirectoryWithName(in.Location, in.Name)
// if err != nil {
// return nil, func() {}, fmt.Errorf("could not populate source from path=%q: %w", in.Location, err)
// }
//
// return &s, func() {}, nil
//}
var opts []stereoscope.Option
if registryOptions != nil {
opts = append(opts, stereoscope.WithRegistryOptions(*registryOptions))
}
//func generateFileSource(fs afero.Fs, in Input) (*Source, func(), error) {
// fileMeta, err := fs.Stat(in.Location)
// if err != nil {
// return nil, func() {}, fmt.Errorf("unable to stat dir=%q: %w", in.Location, err)
// }
//
// if fileMeta.IsDir() {
// return nil, func() {}, fmt.Errorf("given path is not a directory (path=%q): %w", in.Location, err)
// }
//
// s, cleanupFn := NewFromFileWithName(in.Location, in.Name)
//
// return &s, cleanupFn, nil
//}
if in.Platform != "" {
opts = append(opts, stereoscope.WithPlatform(in.Platform))
}
//// NewFromDirectory creates a new source object tailored to catalog a given filesystem directory recursively.
//func NewFromDirectory(path string) (Source, error) {
// return NewFromDirectoryWithName(path, "")
//}
//
//// NewFromDirectory creates a new source object tailored to catalog a given filesystem directory recursively.
//func NewFromDirectoryRoot(path string) (Source, error) {
// return NewFromDirectoryRootWithName(path, "")
//}
//
//// NewFromDirectoryWithName creates a new source object tailored to catalog a given filesystem directory recursively, with an explicitly provided name.
//func NewFromDirectoryWithName(path string, name string) (Source, error) {
// s := Source{
// mutex: &sync.Mutex{},
// Metadata: Metadata{
// Name: name,
// Scheme: DirectoryScheme,
// Path: path,
// },
// path: path,
// }
// s.SetID()
// return s, nil
//}
//
//// NewFromDirectoryRootWithName creates a new source object tailored to catalog a given filesystem directory recursively, with an explicitly provided name.
//func NewFromDirectoryRootWithName(path string, name string) (Source, error) {
// s := Source{
// mutex: &sync.Mutex{},
// Metadata: Metadata{
// Name: name,
// Scheme: DirectoryScheme,
// Path: path,
// Base: path,
// },
// path: path,
// base: path,
// }
// s.SetID()
// return s, nil
//}
img, err := stereoscope.GetImageFromSource(ctx, in.Location, in.ImageSource, opts...)
cleanup := func() {
if err := img.Cleanup(); err != nil {
log.Warnf("unable to cleanup image=%q: %w", in.UserInput, err)
}
}
if err == nil {
// Success on the first try!
return img, cleanup, nil
}
//// NewFromFile creates a new source object tailored to catalog a file.
//func NewFromFile(path string) (Source, func()) {
// return NewFromFileWithName(path, "")
//}
//
//// NewFromFileWithName creates a new source object tailored to catalog a file, with an explicitly provided name.
//func NewFromFileWithName(path string, name string) (Source, func()) {
// analysisPath, cleanupFn := fileAnalysisPath(path)
//
// s := Source{
// mutex: &sync.Mutex{},
// Metadata: Metadata{
// Name: name,
// Scheme: FileScheme,
// Path: path,
// },
// path: analysisPath,
// }
//
// s.SetID()
// return s, cleanupFn
//}
scheme := parseScheme(in.UserInput)
if !(scheme == "docker" || scheme == "registry") {
// Image retrieval failed, and we shouldn't retry it. It's most likely that the
// user _did_ intend the parsed scheme, but there was a legitimate failure with
// using the scheme to load the image. Alert the user to this failure, so they
// can fix the problem.
return nil, nil, err
}
//
//// NewFromImage creates a new source object tailored to catalog a given container image, relative to the
//// option given (e.g. all-layers, squashed, etc)
//func NewFromImage(img *image.Image, userImageStr string) (Source, error) {
// return NewFromImageWithName(img, userImageStr, "")
//}
//
//// NewFromImageWithName creates a new source object tailored to catalog a given container image, relative to the
//// option given (e.g. all-layers, squashed, etc), with an explicit name.
//func NewFromImageWithName(img *image.Image, userImageStr string, name string) (Source, error) {
// if img == nil {
// return Source{}, fmt.Errorf("no image given")
// }
//
// s := Source{
// Image: img,
// Metadata: Metadata{
// Name: name,
// Scheme: ImageScheme,
// metadata: NewImageMetadata(img, userImageStr),
// },
// }
// s.SetID()
// return s, nil
//}
// Maybe the user wanted "docker" or "registry" to refer to an _image name_
// (e.g. "docker:latest"), not a scheme. We'll retry image retrieval with this
// alternative interpretation, in an attempt to avoid unnecessary user friction.
log.Warnf(
"scheme %q specified, but it coincides with a common image name; re-examining user input %q"+
" without scheme parsing because image retrieval using scheme parsing was unsuccessful: %v",
scheme,
in.UserInput,
err,
)
// We need to determine the image source again, such that this determination
// doesn't take scheme parsing into account.
in.ImageSource = image.DetermineDefaultImagePullSource(in.UserInput)
img, err = stereoscope.GetImageFromSource(ctx, in.UserInput, in.ImageSource, opts...)
cleanup = func() {
if err := img.Cleanup(); err != nil {
log.Warnf("unable to cleanup image=%q: %w", in.UserInput, err)
}
}
return img, cleanup, err
}
func generateDirectorySource(fs afero.Fs, in Input) (*Source, func(), error) {
fileMeta, err := fs.Stat(in.Location)
if err != nil {
return nil, func() {}, fmt.Errorf("unable to stat dir=%q: %w", in.Location, err)
}
if !fileMeta.IsDir() {
return nil, func() {}, fmt.Errorf("given path is not a directory (path=%q): %w", in.Location, err)
}
s, err := NewFromDirectoryWithName(in.Location, in.Name)
if err != nil {
return nil, func() {}, fmt.Errorf("could not populate source from path=%q: %w", in.Location, err)
}
return &s, func() {}, nil
}
func generateFileSource(fs afero.Fs, in Input) (*Source, func(), error) {
fileMeta, err := fs.Stat(in.Location)
if err != nil {
return nil, func() {}, fmt.Errorf("unable to stat dir=%q: %w", in.Location, err)
}
if fileMeta.IsDir() {
return nil, func() {}, fmt.Errorf("given path is not a directory (path=%q): %w", in.Location, err)
}
s, cleanupFn := NewFromFileWithName(in.Location, in.Name)
return &s, cleanupFn, nil
}
// NewFromDirectory creates a new source object tailored to catalog a given filesystem directory recursively.
func NewFromDirectory(path string) (Source, error) {
return NewFromDirectoryWithName(path, "")
}
// NewFromDirectory creates a new source object tailored to catalog a given filesystem directory recursively.
func NewFromDirectoryRoot(path string) (Source, error) {
return NewFromDirectoryRootWithName(path, "")
}
// NewFromDirectoryWithName creates a new source object tailored to catalog a given filesystem directory recursively, with an explicitly provided name.
func NewFromDirectoryWithName(path string, name string) (Source, error) {
s := Source{
mutex: &sync.Mutex{},
Metadata: Metadata{
Name: name,
Scheme: DirectoryScheme,
Path: path,
},
path: path,
}
s.SetID()
return s, nil
}
// NewFromDirectoryRootWithName creates a new source object tailored to catalog a given filesystem directory recursively, with an explicitly provided name.
func NewFromDirectoryRootWithName(path string, name string) (Source, error) {
s := Source{
mutex: &sync.Mutex{},
Metadata: Metadata{
Name: name,
Scheme: DirectoryScheme,
Path: path,
Base: path,
},
path: path,
base: path,
}
s.SetID()
return s, nil
}
// NewFromFile creates a new source object tailored to catalog a file.
func NewFromFile(path string) (Source, func()) {
return NewFromFileWithName(path, "")
}
// NewFromFileWithName creates a new source object tailored to catalog a file, with an explicitly provided name.
func NewFromFileWithName(path string, name string) (Source, func()) {
analysisPath, cleanupFn := fileAnalysisPath(path)
s := Source{
mutex: &sync.Mutex{},
Metadata: Metadata{
Name: name,
Scheme: FileScheme,
Path: path,
},
path: analysisPath,
}
s.SetID()
return s, cleanupFn
}
// fileAnalysisPath returns the path given, or in the case the path is an archive, the location where the archive
// contents have been made available. A cleanup function is provided for any temp files created (if any).
func fileAnalysisPath(path string) (string, func()) {
var analysisPath = path
var cleanupFn = func() {}
// if the given file is an archive (as indicated by the file extension and not MIME type) then unarchive it and
// use the contents as the source. Note: this does NOT recursively unarchive contents, only the given path is
// unarchived.
envelopedUnarchiver, err := archiver.ByExtension(path)
if unarchiver, ok := envelopedUnarchiver.(archiver.Unarchiver); err == nil && ok {
if tar, ok := unarchiver.(*archiver.Tar); ok {
// when tar files are extracted, if there are multiple entries at the same
// location, the last entry wins
// NOTE: this currently does not display any messages if an overwrite happens
tar.OverwriteExisting = true
}
unarchivedPath, tmpCleanup, err := unarchiveToTmp(path, unarchiver)
if err != nil {
log.Warnf("file could not be unarchived: %+v", err)
} else {
log.Debugf("source path is an archive")
analysisPath = unarchivedPath
}
if tmpCleanup != nil {
cleanupFn = tmpCleanup
}
}
return analysisPath, cleanupFn
}
// NewFromImage creates a new source object tailored to catalog a given container image, relative to the
// option given (e.g. all-layers, squashed, etc)
func NewFromImage(img *image.Image, userImageStr string) (Source, error) {
return NewFromImageWithName(img, userImageStr, "")
}
// NewFromImageWithName creates a new source object tailored to catalog a given container image, relative to the
// option given (e.g. all-layers, squashed, etc), with an explicit name.
func NewFromImageWithName(img *image.Image, userImageStr string, name string) (Source, error) {
if img == nil {
return Source{}, fmt.Errorf("no image given")
}
s := Source{
Image: img,
Metadata: Metadata{
Name: name,
Scheme: ImageScheme,
ImageMetadata: NewImageMetadata(img, userImageStr),
},
}
s.SetID()
return s, nil
}
func (s *Source) ID() artifact.ID {
if s.id == "" {
s.SetID()
}
return s.id
}
func (s *Source) SetID() {
var d string
switch s.Metadata.Scheme {
case DirectoryScheme:
d = digest.FromString(s.Metadata.Path).String()
case FileScheme:
// attempt to use the digest of the contents of the file as the ID
file, err := os.Open(s.Metadata.Path)
if err != nil {
d = digest.FromString(s.Metadata.Path).String()
break
}
defer file.Close()
di, err := digest.FromReader(file)
if err != nil {
d = digest.FromString(s.Metadata.Path).String()
break
}
d = di.String()
case ImageScheme:
manifestDigest := digest.FromBytes(s.Metadata.ImageMetadata.RawManifest).String()
if manifestDigest != "" {
d = manifestDigest
break
}
// calcuate chain ID for image sources where manifestDigest is not available
// https://github.com/opencontainers/image-spec/blob/main/config.md#layer-chainid
d = calculateChainID(s.Metadata.ImageMetadata.Layers)
if d == "" {
// TODO what happens here if image has no layers?
// Is this case possible
d = digest.FromString(s.Metadata.ImageMetadata.UserInput).String()
}
default: // for UnknownScheme we hash the struct
id, _ := artifact.IDByHash(s)
d = string(id)
}
s.id = artifact.ID(strings.TrimPrefix(d, "sha256:"))
s.Metadata.ID = strings.TrimPrefix(d, "sha256:")
}
func calculateChainID(lm []LayerMetadata) string {
if len(lm) < 1 {
return ""
}
// DiffID(L0) = digest of layer 0
// https://github.com/anchore/stereoscope/blob/1b1b744a919964f38d14e1416fb3f25221b761ce/pkg/image/layer_metadata.go#L19-L32
chainID := lm[0].Digest
id := chain(chainID, lm[1:])
return id
}
func chain(chainID string, layers []LayerMetadata) string {
if len(layers) < 1 {
return chainID
}
chainID = digest.FromString(layers[0].Digest + " " + chainID).String()
return chain(chainID, layers[1:])
}
func (s *Source) FileResolver(scope Scope) (FileResolver, error) {
switch s.Metadata.Scheme {
case DirectoryScheme, FileScheme:
s.mutex.Lock()
defer s.mutex.Unlock()
if s.directoryResolver == nil {
exclusionFunctions, err := getDirectoryExclusionFunctions(s.path, s.Exclusions)
if err != nil {
return nil, err
}
resolver, err := newDirectoryResolver(s.path, s.base, exclusionFunctions...)
if err != nil {
return nil, fmt.Errorf("unable to create directory resolver: %w", err)
}
s.directoryResolver = resolver
}
return s.directoryResolver, nil
case ImageScheme:
var resolver FileResolver
var err error
switch scope {
case SquashedScope:
resolver, err = newImageSquashResolver(s.Image)
case AllLayersScope:
resolver, err = newAllLayersResolver(s.Image)
default:
return nil, fmt.Errorf("bad image scope provided: %+v", scope)
}
if err != nil {
return nil, err
}
// image tree contains all paths, so we filter out the excluded entries afterwards
if len(s.Exclusions) > 0 {
resolver = NewExcludingResolver(resolver, getImageExclusionFunction(s.Exclusions))
}
return resolver, nil
}
return nil, fmt.Errorf("unable to determine FilePathResolver with current scheme=%q", s.Metadata.Scheme)
}
func unarchiveToTmp(path string, unarchiver archiver.Unarchiver) (string, func(), error) {
tempDir, err := os.MkdirTemp("", "syft-archive-contents-")
if err != nil {
return "", func() {}, fmt.Errorf("unable to create tempdir for archive processing: %w", err)
}
cleanupFn := func() {
if err := os.RemoveAll(tempDir); err != nil {
log.Warnf("unable to cleanup archive tempdir: %+v", err)
}
}
return tempDir, cleanupFn, unarchiver.Unarchive(path, tempDir)
}
func getImageExclusionFunction(exclusions []string) func(string) bool {
if len(exclusions) == 0 {
return nil
}
// add subpath exclusions
for _, exclusion := range exclusions {
exclusions = append(exclusions, exclusion+"/**")
}
return func(path string) bool {
for _, exclusion := range exclusions {
matches, err := doublestar.Match(exclusion, path)
if err != nil {
return false
}
if matches {
return true
}
}
return false
}
}
func getDirectoryExclusionFunctions(root string, exclusions []string) ([]pathIndexVisitor, error) {
if len(exclusions) == 0 {
return nil, nil
}
// this is what directoryResolver.indexTree is doing to get the absolute path:
root, err := filepath.Abs(root)
if err != nil {
return nil, err
}
// this handles Windows file paths by converting them to C:/something/else format
root = filepath.ToSlash(root)
if !strings.HasSuffix(root, "/") {
root += "/"
}
var errors []string
for idx, exclusion := range exclusions {
// check exclusions for supported paths, these are all relative to the "scan root"
if strings.HasPrefix(exclusion, "./") || strings.HasPrefix(exclusion, "*/") || strings.HasPrefix(exclusion, "**/") {
exclusion = strings.TrimPrefix(exclusion, "./")
exclusions[idx] = root + exclusion
} else {
errors = append(errors, exclusion)
}
}
if errors != nil {
return nil, fmt.Errorf("invalid exclusion pattern(s): '%s' (must start with one of: './', '*/', or '**/')", strings.Join(errors, "', '"))
}
return []pathIndexVisitor{
func(path string, info os.FileInfo, _ error) error {
for _, exclusion := range exclusions {
// this is required to handle Windows filepaths
path = filepath.ToSlash(path)
matches, err := doublestar.Match(exclusion, path)
if err != nil {
return nil
}
if matches {
if info != nil && info.IsDir() {
return filepath.SkipDir
}
return errSkipPath
}
}
return nil
},
}, nil
}
//func (s *Source) ID() artifact.ID {
// if s.id == "" {
// s.SetID()
// }
// return s.id
//}
//
//func (s *Source) SetID() {
// var d string
// switch s.Metadata.Scheme {
// case DirectoryScheme:
// d = digest.FromString(s.Metadata.Path).String()
// case FileScheme:
// // attempt to use the digest of the contents of the file as the ID
// file, err := os.Open(s.Metadata.Path)
// if err != nil {
// d = digest.FromString(s.Metadata.Path).String()
// break
// }
// defer file.Close()
// di, err := digest.FromReader(file)
// if err != nil {
// d = digest.FromString(s.Metadata.Path).String()
// break
// }
// d = di.String()
// case ImageScheme:
// manifestDigest := digest.FromBytes(s.Metadata.ImageMetadata.RawManifest).String()
// if manifestDigest != "" {
// d = manifestDigest
// break
// }
//
// // calcuate chain ID for image sources where manifestDigest is not available
// // https://github.com/opencontainers/image-spec/blob/main/config.md#layer-chainid
// d = calculateChainID(s.Metadata.ImageMetadata.Layers)
// if d == "" {
// // TODO what happens here if image has no layers?
// // Is this case possible
// d = digest.FromString(s.Metadata.ImageMetadata.UserInput).String()
// }
// default: // for UnknownScheme we hash the struct
// id, _ := artifact.IDByHash(s)
// d = string(id)
// }
//
// s.id = artifact.ID(strings.TrimPrefix(d, "sha256:"))
// s.Metadata.ID = strings.TrimPrefix(d, "sha256:")
//}
//
//func (s *Source) FileResolver(scope Scope) (FileResolver, error) {
// switch s.Metadata.Scheme {
// case DirectoryScheme, FileScheme:
// s.mutex.Lock()
// defer s.mutex.Unlock()
// if s.directoryResolver == nil {
// exclusionFunctions, err := getDirectoryExclusionFunctions(s.path, s.Exclusions)
// if err != nil {
// return nil, err
// }
// resolver, err := newDirectoryResolver(s.path, s.base, exclusionFunctions...)
// if err != nil {
// return nil, fmt.Errorf("unable to create directory resolver: %w", err)
// }
// s.directoryResolver = resolver
// }
// return s.directoryResolver, nil
// case ImageScheme:
// var resolver FileResolver
// var err error
// switch scope {
// case SquashedScope:
// resolver, err = newImageSquashResolver(s.Image)
// case AllLayersScope:
// resolver, err = newAllLayersResolver(s.Image)
// default:
// return nil, fmt.Errorf("bad image scope provided: %+v", scope)
// }
// if err != nil {
// return nil, err
// }
// // image tree contains all paths, so we filter out the excluded entries afterwards
// if len(s.Exclusions) > 0 {
// resolver = NewExcludingResolver(resolver, getImageExclusionFunction(s.Exclusions))
// }
// return resolver, nil
// }
// return nil, fmt.Errorf("unable to determine FilePathResolver with current scheme=%q", s.Metadata.Scheme)
//}

201
syft/source/source_test.go
View file

@ -23,44 +23,43 @@ import (
"github.com/anchore/stereoscope/pkg/image"
"github.com/anchore/stereoscope/pkg/imagetest"
"github.com/anchore/syft/syft/artifact"
)
func TestParseInput(t *testing.T) {
tests := []struct {
name string
input string
platform string
expected Scheme
errFn require.ErrorAssertionFunc
}{
{
name: "ParseInput parses a file input",
input: "test-fixtures/image-simple/file-1.txt",
expected: FileScheme,
},
{
name: "errors out when using platform for non-image scheme",
input: "test-fixtures/image-simple/file-1.txt",
platform: "arm64",
errFn: require.Error,
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
if test.errFn == nil {
test.errFn = require.NoError
}
sourceInput, err := ParseInput(test.input, test.platform)
test.errFn(t, err)
if test.expected != "" {
require.NotNil(t, sourceInput)
assert.Equal(t, sourceInput.Scheme, test.expected)
}
})
}
}
//func TestParseInput(t *testing.T) {
// tests := []struct {
// name string
// input string
// platform string
// expected scheme.Scheme
// errFn require.ErrorAssertionFunc
// }{
// {
// name: "ParseInput parses a file input",
// input: "test-fixtures/image-simple/file-1.txt",
// expected: scheme.FileScheme,
// },
// {
// name: "errors out when using platform for non-image scheme",
// input: "test-fixtures/image-simple/file-1.txt",
// platform: "arm64",
// errFn: require.Error,
// },
// }
//
// for _, test := range tests {
// t.Run(test.name, func(t *testing.T) {
// if test.errFn == nil {
// test.errFn = require.NoError
// }
// sourceInput, err := ParseInput(test.input, test.platform)
// test.errFn(t, err)
// if test.expected != "" {
// require.NotNil(t, sourceInput)
// assert.Equal(t, sourceInput.Scheme, test.expected)
// }
// })
// }
//}
func TestNewFromImageFails(t *testing.T) {
t.Run("no image given", func(t *testing.T) {
@ -71,71 +70,71 @@ func TestNewFromImageFails(t *testing.T) {
})
}
func TestSetID(t *testing.T) {
layer := image.NewLayer(nil)
layer.Metadata = image.LayerMetadata{
Digest: "sha256:6f4fb385d4e698647bf2a450749dfbb7bc2831ec9a730ef4046c78c08d468e89",
}
img := image.Image{
Layers: []*image.Layer{layer},
}
tests := []struct {
name string
input *Source
expected artifact.ID
}{
{
name: "source.SetID sets the ID for FileScheme",
input: &Source{
Metadata: Metadata{
Scheme: FileScheme,
Path: "test-fixtures/image-simple/file-1.txt",
},
},
expected: artifact.ID("55096713247489add592ce977637be868497132b36d1e294a3831925ec64319a"),
},
{
name: "source.SetID sets the ID for ImageScheme",
input: &Source{
Image: &img,
Metadata: Metadata{
Scheme: ImageScheme,
},
},
expected: artifact.ID("e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"),
},
{
name: "source.SetID sets the ID for DirectoryScheme",
input: &Source{
Image: &img,
Metadata: Metadata{
Scheme: DirectoryScheme,
Path: "test-fixtures/image-simple",
},
},
expected: artifact.ID("91db61e5e0ae097ef764796ce85e442a93f2a03e5313d4c7307e9b413f62e8c4"),
},
{
name: "source.SetID sets the ID for UnknownScheme",
input: &Source{
Image: &img,
Metadata: Metadata{
Scheme: UnknownScheme,
Path: "test-fixtures/image-simple",
},
},
expected: artifact.ID("1b0dc351e6577b01"),
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
test.input.SetID()
assert.Equal(t, test.expected, test.input.ID())
})
}
}
//func TestSetID(t *testing.T) {
// layer := image.NewLayer(nil)
// layer.Metadata = image.LayerMetadata{
// Digest: "sha256:6f4fb385d4e698647bf2a450749dfbb7bc2831ec9a730ef4046c78c08d468e89",
// }
// img := image.Image{
// Layers: []*image.Layer{layer},
// }
//
// tests := []struct {
// name string
// input *Source
// expected artifact.ID
// }{
// {
// name: "source.SetID sets the ID for FileScheme",
// input: &Source{
// Metadata: Metadata{
// Scheme: scheme.FileScheme,
// Path: "test-fixtures/image-simple/file-1.txt",
// },
// },
// expected: artifact.ID("55096713247489add592ce977637be868497132b36d1e294a3831925ec64319a"),
// },
// {
// name: "source.SetID sets the ID for ImageScheme",
// input: &Source{
// Image: &img,
// Metadata: Metadata{
// Scheme: scheme.ImageScheme,
// },
// },
// expected: artifact.ID("e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"),
// },
// {
// name: "source.SetID sets the ID for DirectoryScheme",
// input: &Source{
// Image: &img,
// Metadata: Metadata{
// Scheme: scheme.DirectoryScheme,
// Path: "test-fixtures/image-simple",
// },
// },
// expected: artifact.ID("91db61e5e0ae097ef764796ce85e442a93f2a03e5313d4c7307e9b413f62e8c4"),
// },
// {
// name: "source.SetID sets the ID for UnknownScheme",
// input: &Source{
// Image: &img,
// Metadata: Metadata{
// Scheme: scheme.UnknownScheme,
// Path: "test-fixtures/image-simple",
// },
// },
// expected: artifact.ID("1b0dc351e6577b01"),
// },
// }
//
// for _, test := range tests {
// t.Run(test.name, func(t *testing.T) {
// test.input.SetID()
// assert.Equal(t, test.expected, test.input.ID())
// })
// }
//}
func TestNewFromImage(t *testing.T) {
layer := image.NewLayer(nil)