add cli testing

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

test/cli/json_schema_test.go

@@ -0,0 +1,91 @@
+package cli
+
+import (
+	"fmt"
+	"path"
+	"strings"
+	"testing"
+
+	"github.com/anchore/stereoscope/pkg/imagetest"
+	"github.com/anchore/syft/internal"
+	"github.com/xeipuuv/gojsonschema"
+)
+
+// this is the path to the json schema directory relative to the root of the repo
+const jsonSchemaPath = "schema/json"
+
+func TestJSONSchema(t *testing.T) {
+
+	imageFixture := func(t *testing.T) string {
+		fixtureImageName := "image-pkg-coverage"
+		imagetest.GetFixtureImage(t, "docker-archive", fixtureImageName)
+		tarPath := imagetest.GetFixtureImageTarPath(t, fixtureImageName)
+		return "docker-archive:" + tarPath
+	}
+
+	tests := []struct {
+		name       string
+		subcommand string
+		args       []string
+		fixture    func(*testing.T) string
+	}{
+		{
+			name:       "packages:image:docker-archive:pkg-coverage",
+			subcommand: "packages",
+			args:       []string{"-o", "json"},
+			fixture:    imageFixture,
+		},
+		{
+			name:       "power-user:image:docker-archive:pkg-coverage",
+			subcommand: "power-user",
+			fixture:    imageFixture,
+		},
+		{
+			name:       "packages:dir:pkg-coverage",
+			subcommand: "packages",
+			args:       []string{"-o", "json"},
+			fixture: func(t *testing.T) string {
+				return "dir:test-fixtures/image-pkg-coverage"
+			},
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			fixtureRef := test.fixture(t)
+			args := []string{
+				test.subcommand, fixtureRef, "-q",
+			}
+			for _, a := range test.args {
+				args = append(args, a)
+			}
+
+			_, stdout, _ := runSyftCommand(t, nil, args...)
+
+			if len(strings.Trim(stdout, "\n ")) < 100 {
+				t.Fatalf("bad syft output: %q", stdout)
+			}
+
+			validateAgainstV1Schema(t, stdout)
+		})
+	}
+}
+
+func validateAgainstV1Schema(t testing.TB, json string) {
+	fullSchemaPath := path.Join(repoRoot(t), jsonSchemaPath, fmt.Sprintf("schema-%s.json", internal.JSONSchemaVersion))
+	schemaLoader := gojsonschema.NewReferenceLoader(fmt.Sprintf("file://%s", fullSchemaPath))
+	documentLoader := gojsonschema.NewStringLoader(json)
+
+	result, err := gojsonschema.Validate(schemaLoader, documentLoader)
+	if err != nil {
+		t.Fatal("unable to validate json schema:", err.Error())
+	}
+
+	if !result.Valid() {
+		t.Errorf("failed json schema validation:")
+		t.Errorf("JSON:\n%s\n", json)
+		for _, desc := range result.Errors() {
+			t.Errorf("  - %s\n", desc)
+		}
+	}
+}

test/cli/packages_cmd_test.go

@@ -0,0 +1,139 @@
+package cli
+
+import (
+	"strings"
+	"testing"
+
+	"github.com/anchore/syft/syft/source"
+)
+
+func TestPackagesCmdFlags(t *testing.T) {
+	request := "docker-archive:" + getFixtureImage(t, "image-pkg-coverage")
+
+	tests := []struct {
+		name       string
+		args       []string
+		env        map[string]string
+		assertions []traitAssertion
+	}{
+		{
+			name: "json-output-flag",
+			args: []string{"packages", "-o", "json", request},
+			assertions: []traitAssertion{
+				assertJsonReport,
+				assertSource(source.SquashedScope),
+				assertSuccessfulReturnCode,
+			},
+		},
+		{
+			name: "output-env-binding",
+			env: map[string]string{
+				"SYFT_OUTPUT": "json",
+			},
+			args: []string{"packages", request},
+			assertions: []traitAssertion{
+				assertJsonReport,
+				assertSuccessfulReturnCode,
+			},
+		},
+		{
+			name: "table-output-flag",
+			args: []string{"packages", "-o", "table", request},
+			assertions: []traitAssertion{
+				assertTableReport,
+				assertSuccessfulReturnCode,
+			},
+		},
+		{
+			name: "default-output-flag",
+			args: []string{"packages", request},
+			assertions: []traitAssertion{
+				assertTableReport,
+				assertSuccessfulReturnCode,
+			},
+		},
+		{
+			name: "squashed-scope-flag",
+			args: []string{"packages", "-o", "json", "-s", "squashed", request},
+			assertions: []traitAssertion{
+				assertSource(source.SquashedScope),
+				assertSuccessfulReturnCode,
+			},
+		},
+		{
+			name: "all-layers-scope-flag",
+			args: []string{"packages", "-o", "json", "-s", "all-layers", request},
+			assertions: []traitAssertion{
+				assertSource(source.AllLayersScope),
+				assertSuccessfulReturnCode,
+			},
+		},
+		{
+			name: "packages-scope-env-binding",
+			env: map[string]string{
+				"SYFT_PACKAGES_SCOPE": "all-layers",
+			},
+			args: []string{"packages", "-o", "json", request},
+			assertions: []traitAssertion{
+				assertSource(source.AllLayersScope),
+				assertSuccessfulReturnCode,
+			},
+		},
+		{
+			name: "attempt-upload-on-cli-switches",
+			args: []string{"packages", "-vv", "-H", "localhost:8080", "-u", "the-username", "-d", "test-fixtures/image-pkg-coverage/Dockerfile", "--overwrite-existing-image", request},
+			env: map[string]string{
+				"SYFT_ANCHORE_PATH":     "path/to/api",
+				"SYFT_ANCHORE_PASSWORD": "the-password",
+			},
+			assertions: []traitAssertion{
+				// we cannot easily assert a successful upload behavior, so instead we are doing the next best thing
+				// and asserting that the parsed configuration has the expected values and we see log entries
+				// indicating an upload attempt.
+				assertNotInOutput("the-username"),
+				assertNotInOutput("the-password"),
+				assertInOutput("uploading results to localhost:8080"),
+				assertInOutput(`dockerfile: test-fixtures/image-pkg-coverage/Dockerfile`),
+				assertInOutput(`overwrite-existing-image: true`),
+				assertInOutput(`path: path/to/api`),
+				assertInOutput(`host: localhost:8080`),
+				assertFailingReturnCode, // upload can't go anywhere, so if this passes that would be surprising
+			},
+		},
+		{
+			name: "dockerfile-without-upload-is-invalid",
+			args: []string{"packages", "-vv", "-d", "test-fixtures/image-pkg-coverage/Dockerfile", request},
+			assertions: []traitAssertion{
+
+				assertNotInOutput("uploading results to localhost:8080"),
+				assertInOutput("invalid application config: cannot provide dockerfile option without enabling upload"),
+				assertFailingReturnCode,
+			},
+		},
+		{
+			name: "attempt-upload-with-env-host-set",
+			args: []string{"packages", "-vv", request},
+			env: map[string]string{
+				"SYFT_ANCHORE_HOST": "localhost:8080",
+			},
+			assertions: []traitAssertion{
+				assertInOutput("uploading results to localhost:8080"),
+				assertFailingReturnCode, // upload can't go anywhere, so if this passes that would be surprising
+			},
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			cmd, stdout, stderr := runSyftCommand(t, test.env, test.args...)
+			for _, traitFn := range test.assertions {
+				traitFn(t, stdout, stderr, cmd.ProcessState.ExitCode())
+			}
+			if t.Failed() {
+				t.Log("STDOUT:\n", stdout)
+				t.Log("STDERR:\n", stderr)
+				t.Log("COMMAND:", strings.Join(cmd.Args, " "))
+			}
+		})
+	}
+}

test/cli/power_user_cmd_test.go

@@ -0,0 +1,49 @@
+package cli
+
+import (
+	"strings"
+	"testing"
+)
+
+func TestPowerUserCmdFlags(t *testing.T) {
+	request := "docker-archive:" + getFixtureImage(t, "image-pkg-coverage")
+
+	tests := []struct {
+		name       string
+		args       []string
+		env        map[string]string
+		assertions []traitAssertion
+	}{
+		{
+			name: "json-output-flag-fails",
+			args: []string{"power-user", "-o", "json", request},
+			assertions: []traitAssertion{
+				assertFailingReturnCode,
+			},
+		},
+		{
+			name: "default-results",
+			args: []string{"power-user", request},
+			assertions: []traitAssertion{
+				assertInOutput(`"type": "regularFile"`),         // proof of file-metadata data
+				assertInOutput(`"algorithm": "sha256"`),         // proof of file-metadata default digest algorithm of sha256
+				assertInOutput(`"metadataType": "ApkMetadata"`), // proof of package artifacts data
+				assertSuccessfulReturnCode,
+			},
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			cmd, stdout, stderr := runSyftCommand(t, test.env, test.args...)
+			for _, traitFn := range test.assertions {
+				traitFn(t, stdout, stderr, cmd.ProcessState.ExitCode())
+			}
+			if t.Failed() {
+				t.Log("STDOUT:\n", stdout)
+				t.Log("STDERR:\n", stderr)
+				t.Log("COMMAND:", strings.Join(cmd.Args, " "))
+			}
+		})
+	}
+}

test/cli/root_cmd_test.go

@@ -0,0 +1,91 @@
+package cli
+
+import (
+	"strings"
+	"testing"
+
+	"github.com/sergi/go-diff/diffmatchpatch"
+)
+
+func TestRootCmdAliasesToPackagesSubcommand(t *testing.T) {
+	request := "docker-archive:" + getFixtureImage(t, "image-pkg-coverage")
+	deprecationWarning := "The root command is deprecated"
+
+	_, aliasStdout, aliasStderr := runSyftCommand(t, nil, request)
+
+	if !strings.Contains(aliasStderr, deprecationWarning) {
+		t.Errorf("missing root-packages alias deprecation warning")
+	}
+
+	_, pkgsStdout, pkgsStderr := runSyftCommand(t, nil, "packages", request)
+
+	if strings.Contains(pkgsStderr, deprecationWarning) {
+		t.Errorf("packages command should not have deprecation warning")
+	}
+
+	if aliasStdout != pkgsStdout {
+		t.Errorf("packages and root command should have same report output but do not!")
+		dmp := diffmatchpatch.New()
+		diffs := dmp.DiffMain(aliasStdout, pkgsStdout, true)
+		t.Error(dmp.DiffPrettyText(diffs))
+	}
+}
+
+func TestPersistentFlags(t *testing.T) {
+	request := "docker-archive:" + getFixtureImage(t, "image-pkg-coverage")
+
+	tests := []struct {
+		name       string
+		args       []string
+		env        map[string]string
+		assertions []traitAssertion
+	}{
+		{
+			name: "quiet-flag",
+			// note: the root command will always show the deprecation warning, so the packages command is used instead
+			args: []string{"packages", "-q", request},
+			assertions: []traitAssertion{
+				func(tb testing.TB, stdout, stderr string, rc int) {
+					// ensure there is no status
+					if len(stderr) != 0 {
+						tb.Errorf("should have seen no stderr output, got %d bytes", len(stderr))
+					}
+					// ensure there is still a report
+					if len(stdout) == 0 {
+						tb.Errorf("should have seen a report on stdout, got nothing")
+					}
+				},
+			},
+		},
+		{
+			name: "info-log-flag",
+			args: []string{"-v", request},
+			assertions: []traitAssertion{
+				assertLoggingLevel("info"),
+				assertSuccessfulReturnCode,
+			},
+		},
+		{
+			name: "debug-log-flag",
+			args: []string{"-vv", request},
+			assertions: []traitAssertion{
+				assertLoggingLevel("debug"),
+				assertSuccessfulReturnCode,
+			},
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			cmd, stdout, stderr := runSyftCommand(t, test.env, test.args...)
+			for _, traitFn := range test.assertions {
+				traitFn(t, stdout, stderr, cmd.ProcessState.ExitCode())
+			}
+			if t.Failed() {
+				t.Log("STDOUT:\n", stdout)
+				t.Log("STDERR:\n", stderr)
+				t.Log("COMMAND:", strings.Join(cmd.Args, " "))
+			}
+		})
+	}
+}

test/cli/test-fixtures/image-pkg-coverage

@@ -0,0 +1 @@
+../../integration/test-fixtures/image-pkg-coverage

test/cli/trait_assertions_test.go

@@ -0,0 +1,82 @@
+package cli
+
+import (
+	"encoding/json"
+	"fmt"
+	"regexp"
+	"strings"
+	"testing"
+
+	"github.com/acarl005/stripansi"
+	"github.com/anchore/syft/syft/source"
+)
+
+type traitAssertion func(tb testing.TB, stdout, stderr string, rc int)
+
+func assertJsonReport(tb testing.TB, stdout, _ string, _ int) {
+	var data interface{}
+
+	if err := json.Unmarshal([]byte(stdout), &data); err != nil {
+		tb.Errorf("expected to find a JSON report, but was unmarshalable: %+v", err)
+	}
+}
+
+func assertTableReport(tb testing.TB, stdout, _ string, _ int) {
+	if !strings.Contains(stdout, "NAME") || !strings.Contains(stdout, "VERSION") || !strings.Contains(stdout, "TYPE") {
+		tb.Errorf("expected to find a table report, but did not")
+	}
+}
+
+func assertSource(scope source.Scope) traitAssertion {
+	return func(tb testing.TB, stdout, stderr string, rc int) {
+		// we can only verify source with the json report
+		assertJsonReport(tb, stdout, stderr, rc)
+
+		if !strings.Contains(stdout, fmt.Sprintf(`"scope": "%s"`, scope.String())) {
+			tb.Errorf("JSON report did not indicate the %q scope", scope)
+		}
+	}
+}
+
+func assertLoggingLevel(level string) traitAssertion {
+	// match examples:
+	//  "[0000]  INFO"
+	//  "[0012] DEBUG"
+	logPattern := regexp.MustCompile(`(?m)^\[\d\d\d\d\]\s+` + strings.ToUpper(level))
+	return func(tb testing.TB, _, stderr string, _ int) {
+		if !logPattern.MatchString(stripansi.Strip(stderr)) {
+			tb.Errorf("output did not indicate the %q logging level", level)
+		}
+	}
+}
+
+func assertNotInOutput(data string) traitAssertion {
+	return func(tb testing.TB, stdout, stderr string, _ int) {
+		if strings.Contains(stripansi.Strip(stderr), data) {
+			tb.Errorf("data=%q was found in stderr, but should not have been there", data)
+		}
+		if strings.Contains(stripansi.Strip(stdout), data) {
+			tb.Errorf("data=%q was found in stdout, but should not have been there", data)
+		}
+	}
+}
+
+func assertInOutput(data string) traitAssertion {
+	return func(tb testing.TB, stdout, stderr string, _ int) {
+		if !strings.Contains(stripansi.Strip(stderr), data) && !strings.Contains(stripansi.Strip(stdout), data) {
+			tb.Errorf("data=%q was NOT found in any output, but should have been there", data)
+		}
+	}
+}
+
+func assertFailingReturnCode(tb testing.TB, _, _ string, rc int) {
+	if rc == 0 {
+		tb.Errorf("expected a failure but got rc=%d", rc)
+	}
+}
+
+func assertSuccessfulReturnCode(tb testing.TB, _, _ string, rc int) {
+	if rc != 0 {
+		tb.Errorf("expected no failure but got rc=%d", rc)
+	}
+}

test/cli/utils_test.go

@@ -0,0 +1,73 @@
+package cli
+
+import (
+	"bytes"
+	"fmt"
+	"os"
+	"os/exec"
+	"path"
+	"path/filepath"
+	"runtime"
+	"strings"
+	"testing"
+
+	"github.com/anchore/stereoscope/pkg/imagetest"
+)
+
+func getFixtureImage(t testing.TB, fixtureImageName string) string {
+	imagetest.GetFixtureImage(t, "docker-archive", fixtureImageName)
+	return imagetest.GetFixtureImageTarPath(t, fixtureImageName)
+}
+
+func runSyftCommand(t testing.TB, env map[string]string, args ...string) (*exec.Cmd, string, string) {
+	cmd := getSyftCommand(t, args...)
+	if env != nil {
+		var envList []string
+		for key, val := range env {
+			if key == "" {
+				continue
+			}
+			envList = append(envList, fmt.Sprintf("%s=%s", key, val))
+		}
+		cmd.Env = envList
+	}
+	var stdout, stderr bytes.Buffer
+	cmd.Stdout = &stdout
+	cmd.Stderr = &stderr
+
+	// ignore errors since this may be what the test expects
+	cmd.Run()
+
+	return cmd, stdout.String(), stderr.String()
+}
+
+func getSyftCommand(t testing.TB, args ...string) *exec.Cmd {
+
+	var binaryLocation string
+	if os.Getenv("SYFT_BINARY_LOCATION") != "" {
+		// SYFT_BINARY_LOCATION is relative to the repository root. (e.g., "snapshot/syft-linux_amd64/syft")
+		// This value is transformed due to the CLI tests' need for a path relative to the test directory.
+		binaryLocation = path.Join(repoRoot(t), os.Getenv("SYFT_BINARY_LOCATION"))
+	} else {
+		os := runtime.GOOS
+		if os == "darwin" {
+			os = "macos_darwin"
+		}
+
+		binaryLocation = path.Join(repoRoot(t), fmt.Sprintf("snapshot/syft-%s_%s/syft", os, runtime.GOARCH))
+	}
+	return exec.Command(binaryLocation, args...)
+}
+
+func repoRoot(t testing.TB) string {
+	t.Helper()
+	root, err := exec.Command("git", "rev-parse", "--show-toplevel").Output()
+	if err != nil {
+		t.Fatalf("unable to find repo root dir: %+v", err)
+	}
+	absRepoRoot, err := filepath.Abs(strings.TrimSpace(string(root)))
+	if err != nil {
+		t.Fatal("unable to get abs path to repo root:", err)
+	}
+	return absRepoRoot
+}