Merge pull request #22 from anchore/dpkg-analyzer

add dpkg analyzer

go.mod

@@ -4,8 +4,10 @@ go 1.14
 
 require (
 	github.com/anchore/stereoscope v0.0.0-20200518155435-f6c722e4572b
+	github.com/go-test/deep v1.0.6
 	github.com/golang/protobuf v1.4.2 // indirect
 	github.com/hashicorp/go-multierror v1.1.0
+	github.com/mitchellh/mapstructure v1.1.2
 	github.com/opencontainers/go-digest v1.0.0 // indirect
 	github.com/sirupsen/logrus v1.6.0 // indirect
 	github.com/spf13/cobra v1.0.0

go.sum

@@ -133,6 +133,8 @@ github.com/go-openapi/swag v0.0.0-20160704191624-1d0bd113de87/go.mod h1:DXUve3Dp
 github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk=
 github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk=
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
+github.com/go-test/deep v1.0.6 h1:UHSEyLZUwX9Qoi99vVwvewiMC8mM2bf7XEM2nqvzEn8=
+github.com/go-test/deep v1.0.6/go.mod h1:QV8Hv/iy04NyLBxAdO9njL0iVPN1S4d/A3NVv1V36o8=
 github.com/godbus/dbus v0.0.0-20190422162347-ade71ed3457e/go.mod h1:bBOAhwG1umN6/6ZUMtDFBMQR8jRg9O75tm9K00oMsK4=
 github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
 github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=

imgbom/analyzer/controller.go

@@ -1,7 +1,7 @@
 package analyzer
 
 import (
-	"github.com/anchore/imgbom/imgbom/analyzer/dummy"
+	"github.com/anchore/imgbom/imgbom/analyzer/dpkg"
 	"github.com/anchore/imgbom/imgbom/pkg"
 	"github.com/anchore/imgbom/imgbom/scope"
 	"github.com/anchore/stereoscope/pkg/file"
@@ -14,7 +14,7 @@ func init() {
 	controllerInstance = controller{
 		analyzers: make([]Analyzer, 0),
 	}
-	controllerInstance.add(dummy.NewAnalyzer())
+	controllerInstance.add(dpkg.NewAnalyzer())
 }
 
 func Analyze(s scope.Scope) (pkg.Catalog, error) {

imgbom/analyzer/dpkg/analyzer.go

@@ -0,0 +1,60 @@
+package dpkg
+
+import (
+	"strings"
+
+	"github.com/anchore/imgbom/imgbom/pkg"
+	"github.com/anchore/stereoscope/pkg/file"
+	"github.com/anchore/stereoscope/pkg/tree"
+)
+
+type Analyzer struct {
+	selectedFiles []file.Reference
+}
+
+func NewAnalyzer() *Analyzer {
+	return &Analyzer{}
+}
+
+func (a *Analyzer) SelectFiles(trees []*tree.FileTree) []file.Reference {
+	files := make([]file.Reference, 0)
+	for _, tree := range trees {
+		// TODO: extract into function/slice/etc
+		file := tree.File("/var/lib/dpkg/status")
+		if file != nil {
+			files = append(files, *file)
+		}
+	}
+
+	a.selectedFiles = files
+
+	return files
+}
+
+func (a *Analyzer) Analyze(contents map[file.Reference]string) ([]pkg.Package, error) {
+	packages := make([]pkg.Package, 0)
+	for _, reference := range a.selectedFiles {
+		content, ok := contents[reference]
+		if !ok {
+			// TODO: this needs handling
+			panic(reference)
+		}
+
+		entries, err := ParseEntries(strings.NewReader(content))
+		if err != nil {
+			// TODO: punt for now, we need to handle this
+			panic(err)
+		}
+		for _, entry := range entries {
+			packages = append(packages, pkg.Package{
+				Name:     entry.Package,
+				Version:  entry.Version,
+				Type:     pkg.DebPkg,
+				Source:   []file.Reference{reference},
+				Metadata: entry,
+			})
+		}
+
+	}
+	return packages, nil
+}

imgbom/analyzer/dpkg/parser.go

@@ -0,0 +1,94 @@
+package dpkg
+
+import (
+	"bufio"
+	"fmt"
+	"io"
+	"strings"
+
+	"github.com/anchore/imgbom/imgbom/pkg"
+	"github.com/mitchellh/mapstructure"
+)
+
+var endOfPackages = fmt.Errorf("no more packages to read")
+
+func ParseEntries(reader io.Reader) ([]pkg.DpkgMetadata, error) {
+	buffedReader := bufio.NewReader(reader)
+	var entries = make([]pkg.DpkgMetadata, 0)
+
+	for {
+		entry, err := parseEntry(buffedReader)
+		if err != nil {
+			if err == endOfPackages {
+				break
+			}
+			return nil, err
+		}
+		entries = append(entries, entry)
+	}
+
+	return entries, nil
+}
+
+func parseEntry(reader *bufio.Reader) (entry pkg.DpkgMetadata, err error) {
+	dpkgFields := make(map[string]string)
+	var key string
+
+	for {
+		line, err := reader.ReadString('\n')
+		if err != nil {
+			if err == io.EOF {
+				return pkg.DpkgMetadata{}, endOfPackages
+			}
+			return pkg.DpkgMetadata{}, err
+		}
+
+		line = strings.TrimRight(line, "\n")
+
+		// empty line indicates end of entry
+		if len(line) == 0 {
+			// if the entry has not started, keep parsing lines
+			if len(dpkgFields) == 0 {
+				continue
+			}
+			break
+		}
+
+		switch {
+		case strings.HasPrefix(line, " "):
+			// a field-body continuation
+			if len(key) == 0 {
+				return pkg.DpkgMetadata{}, fmt.Errorf("no match for continuation: line: '%s'", line)
+			}
+
+			val, ok := dpkgFields[key]
+			if !ok {
+				return pkg.DpkgMetadata{}, fmt.Errorf("no previous key exists, expecting: %s", key)
+			}
+			// concatenate onto previous value
+			val = fmt.Sprintf("%s\n %s", val, strings.TrimSpace(line))
+			dpkgFields[key] = val
+		default:
+			// parse a new key
+			if i := strings.Index(line, ":"); i > 0 {
+				key = strings.TrimSpace(line[0:i])
+				val := strings.TrimSpace(line[i+1:])
+
+				if _, ok := dpkgFields[key]; ok {
+					return pkg.DpkgMetadata{}, fmt.Errorf("duplicate key discovered: %s", key)
+				}
+
+				dpkgFields[key] = val
+			} else {
+				return pkg.DpkgMetadata{}, fmt.Errorf("cannot parse field from line: '%s'", line)
+			}
+		}
+	}
+
+	err = mapstructure.Decode(dpkgFields, &entry)
+	if err != nil {
+		return pkg.DpkgMetadata{}, err
+	}
+
+	return entry, nil
+}

imgbom/analyzer/dpkg/parser_test.go

@@ -0,0 +1,145 @@
+package dpkg
+
+import (
+	"bufio"
+	"os"
+	"testing"
+
+	"github.com/anchore/imgbom/imgbom/pkg"
+	"github.com/go-test/deep"
+)
+
+func compareEntries(t *testing.T, left, right pkg.DpkgMetadata) {
+	t.Helper()
+	if diff := deep.Equal(left, right); diff != nil {
+		t.Error(diff)
+	}
+}
+
+func TestSinglePackage(t *testing.T) {
+	tests := []struct {
+		name     string
+		expected pkg.DpkgMetadata
+	}{
+		{
+			name: "Test Single Package",
+			expected: pkg.DpkgMetadata{
+				Package:        "apt",
+				Status:         "install ok installed",
+				Priority:       "required",
+				InstalledSize:  "4064",
+				Maintainer:     "APT Development Team <deity@lists.debian.org>",
+				Architecture:   "amd64",
+				Version:        "1.8.2",
+				ReplacesPkgs:   "apt-transport-https (<< 1.5~alpha4~), apt-utils (<< 1.3~exp2~)",
+				ProvidesPkgs:   "apt-transport-https (= 1.8.2)",
+				DependsPkgs:    "adduser, gpgv | gpgv2 | gpgv1, debian-archive-keyring, libapt-pkg5.0 (>= 1.7.0~alpha3~), libc6 (>= 2.15), libgcc1 (>= 1:3.0), libgnutls30 (>= 3.6.6), libseccomp2 (>= 1.0.1), libstdc++6 (>= 5.2)",
+				RecommendsPkgs: "ca-certificates",
+				SuggestsPkgs:   "apt-doc, aptitude | synaptic | wajig, dpkg-dev (>= 1.17.2), gnupg | gnupg2 | gnupg1, powermgmt-base",
+				ConfigFiles: `
+ /etc/apt/apt.conf.d/01autoremove 76120d358bc9037bb6358e737b3050b5
+ /etc/cron.daily/apt-compat 49e9b2cfa17849700d4db735d04244f3
+ /etc/kernel/postinst.d/apt-auto-removal 4ad976a68f045517cf4696cec7b8aa3a
+ /etc/logrotate.d/apt 179f2ed4f85cbaca12fa3d69c2a4a1c3`,
+			},
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			file, err := os.Open("test-fixtures/single")
+			if err != nil {
+				t.Fatal("Unable to read test_fixtures/single: ", err)
+			}
+			defer func() {
+				err := file.Close()
+				if err != nil {
+					panic(err)
+				}
+			}()
+
+			reader := bufio.NewReader(file)
+
+			entry, err := parseEntry(reader)
+			if err != nil {
+				t.Fatal("Unable to read file contents: ", err)
+			}
+
+			compareEntries(t, entry, test.expected)
+
+		})
+	}
+}
+
+func TestMultiplePackages(t *testing.T) {
+	tests := []struct {
+		name     string
+		expected []pkg.DpkgMetadata
+	}{
+		{
+			name: "Test Multiple Package",
+			expected: []pkg.DpkgMetadata{
+				{
+					Package:       "tzdata",
+					Status:        "install ok installed",
+					Priority:      "required",
+					InstalledSize: "3036",
+					Maintainer:    "GNU Libc Maintainers <debian-glibc@lists.debian.org>",
+					Architecture:  "all",
+					Version:       "2020a-0+deb10u1",
+					ReplacesPkgs:  "libc0.1, libc0.3, libc6, libc6.1",
+					ProvidesPkgs:  "tzdata-buster",
+					DependsPkgs:   "debconf (>= 0.5) | debconf-2.0",
+				},
+				{
+					Package:       "util-linux",
+					Status:        "install ok installed",
+					Priority:      "required",
+					InstalledSize: "4327",
+					Maintainer:    "LaMont Jones <lamont@debian.org>",
+					Architecture:  "amd64",
+					Version:       "2.33.1-0.1",
+					ReplacesPkgs:  "bash-completion (<< 1:2.8), initscripts (<< 2.88dsf-59.2~), login (<< 1:4.5-1.1~), mount (<< 2.29.2-3~), s390-tools (<< 2.2.0-1~), setpriv (<< 2.32.1-0.2~), sysvinit-utils (<< 2.88dsf-59.1~)",
+					DependsPkgs:   "fdisk, login (>= 1:4.5-1.1~)",
+					SuggestsPkgs:  "dosfstools, kbd | console-tools, util-linux-locales",
+					ConfigFiles: `
+ /etc/default/hwclock 3916544450533eca69131f894db0ca12
+ /etc/init.d/hwclock.sh 1ca5c0743fa797ffa364db95bb8d8d8e
+ /etc/pam.d/runuser b8b44b045259525e0fae9e38fdb2aeeb
+ /etc/pam.d/runuser-l 2106ea05877e8913f34b2c77fa02be45
+ /etc/pam.d/su ce6dcfda3b190a27a455bb38a45ff34a
+ /etc/pam.d/su-l 756fef5687fecc0d986e5951427b0c4f`,
+				},
+			},
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			file, err := os.Open("test-fixtures/multiple")
+			if err != nil {
+				t.Fatal("Unable to read: ", err)
+			}
+			defer func() {
+				err := file.Close()
+				if err != nil {
+					panic(err)
+				}
+			}()
+
+			entries, err := ParseEntries(file)
+			if err != nil {
+				t.Fatal("Unable to read file contents: ", err)
+			}
+
+			if len(entries) != 2 {
+				t.Fatalf("unexpected number of entries: %d", len(entries))
+			}
+
+			for idx, entry := range entries {
+				compareEntries(t, entry, test.expected[idx])
+			}
+
+		})
+	}
+}

imgbom/analyzer/dpkg/test-fixtures/multiple

@@ -0,0 +1,51 @@
+Package: tzdata
+Status: install ok installed
+Priority: required
+Section: localization
+Installed-Size: 3036
+Maintainer: GNU Libc Maintainers <debian-glibc@lists.debian.org>
+Architecture: all
+Multi-Arch: foreign
+Version: 2020a-0+deb10u1
+Replaces: libc0.1, libc0.3, libc6, libc6.1
+Provides: tzdata-buster
+Depends: debconf (>= 0.5) | debconf-2.0
+Description: time zone and daylight-saving time data
+ This package contains data required for the implementation of
+ standard local time for many representative locations around the
+ globe. It is updated periodically to reflect changes made by
+ political bodies to time zone boundaries, UTC offsets, and
+ daylight-saving rules.
+Homepage: https://www.iana.org/time-zones
+
+Package: util-linux
+Essential: yes
+Status: install ok installed
+Priority: required
+Section: utils
+Installed-Size: 4327
+Maintainer: LaMont Jones <lamont@debian.org>
+Architecture: amd64
+Multi-Arch: foreign
+Version: 2.33.1-0.1
+Replaces: bash-completion (<< 1:2.8), initscripts (<< 2.88dsf-59.2~), login (<< 1:4.5-1.1~), mount (<< 2.29.2-3~), s390-tools (<< 2.2.0-1~), setpriv (<< 2.32.1-0.2~), sysvinit-utils (<< 2.88dsf-59.1~)
+Depends: fdisk, login (>= 1:4.5-1.1~)
+Pre-Depends: libaudit1 (>= 1:2.2.1), libblkid1 (>= 2.31.1), libc6 (>= 2.25), libcap-ng0 (>= 0.7.9), libmount1 (>= 2.25), libpam0g (>= 0.99.7.1), libselinux1 (>= 2.6-3~), libsmartcols1 (>= 2.33), libsystemd0, libtinfo6 (>= 6), libudev1 (>= 183), libuuid1 (>= 2.16), zlib1g (>= 1:1.1.4)
+Suggests: dosfstools, kbd | console-tools, util-linux-locales
+Breaks: bash-completion (<< 1:2.8), grml-debootstrap (<< 0.68), mount (<< 2.29.2-3~), s390-tools (<< 2.2.0-1~), setpriv (<< 2.32.1-0.2~), sysvinit-utils (<< 2.88dsf-59.4~)
+Conffiles:
+ /etc/default/hwclock 3916544450533eca69131f894db0ca12
+ /etc/init.d/hwclock.sh 1ca5c0743fa797ffa364db95bb8d8d8e
+ /etc/pam.d/runuser b8b44b045259525e0fae9e38fdb2aeeb
+ /etc/pam.d/runuser-l 2106ea05877e8913f34b2c77fa02be45
+ /etc/pam.d/su ce6dcfda3b190a27a455bb38a45ff34a
+ /etc/pam.d/su-l 756fef5687fecc0d986e5951427b0c4f
+Description: miscellaneous system utilities
+ This package contains a number of important utilities, most of which
+ are oriented towards maintenance of your system. Some of the more
+ important utilities included in this package allow you to view kernel
+ messages, create new filesystems, view block device information,
+ interface with real time clock, etc.
+
+
+

imgbom/analyzer/dpkg/test-fixtures/single

@@ -0,0 +1,34 @@
+Package: apt
+Status: install ok installed
+Priority: required
+Section: admin
+Installed-Size: 4064
+Maintainer: APT Development Team <deity@lists.debian.org>
+Architecture: amd64
+Version: 1.8.2
+Replaces: apt-transport-https (<< 1.5~alpha4~), apt-utils (<< 1.3~exp2~)
+Provides: apt-transport-https (= 1.8.2)
+Depends: adduser, gpgv | gpgv2 | gpgv1, debian-archive-keyring, libapt-pkg5.0 (>= 1.7.0~alpha3~), libc6 (>= 2.15), libgcc1 (>= 1:3.0), libgnutls30 (>= 3.6.6), libseccomp2 (>= 1.0.1), libstdc++6 (>= 5.2)
+Recommends: ca-certificates
+Suggests: apt-doc, aptitude | synaptic | wajig, dpkg-dev (>= 1.17.2), gnupg | gnupg2 | gnupg1, powermgmt-base
+Breaks: apt-transport-https (<< 1.5~alpha4~), apt-utils (<< 1.3~exp2~), aptitude (<< 0.8.10)
+Conffiles:
+ /etc/apt/apt.conf.d/01autoremove 76120d358bc9037bb6358e737b3050b5
+ /etc/cron.daily/apt-compat 49e9b2cfa17849700d4db735d04244f3
+ /etc/kernel/postinst.d/apt-auto-removal 4ad976a68f045517cf4696cec7b8aa3a
+ /etc/logrotate.d/apt 179f2ed4f85cbaca12fa3d69c2a4a1c3
+Description: commandline package manager
+ This package provides commandline tools for searching and
+ managing as well as querying information about packages
+ as a low-level access to all features of the libapt-pkg library.
+ .
+ These include:
+  * apt-get for retrieval of packages and information about them
+    from authenticated sources and for installation, upgrade and
+    removal of packages together with their dependencies
+  * apt-cache for querying available information about installed
+    as well as installable packages
+  * apt-cdrom to use removable media as a source for packages
+  * apt-config as an interface to the configuration settings
+  * apt-key as an interface to manage authentication keys
+

imgbom/pkg/metadata.go

@@ -0,0 +1,20 @@
+package pkg
+
+// TODO: consider keeping the remaining values as an embedded map
+// Available fields are described at http://manpages.ubuntu.com/manpages/xenial/man1/dpkg-query.1.html
+// in the --showformat section
+type DpkgMetadata struct {
+	Package        string `mapstructure:"Package"`
+	Architecture   string `mapstructure:"Architecture"`
+	DependsPkgs    string `mapstructure:"Depends"`
+	InstalledSize  string `mapstructure:"Installed-Size"`
+	Maintainer     string `mapstructure:"Maintainer"`
+	Priority       string `mapstructure:"Priority"`
+	ProvidesPkgs   string `mapstructure:"Provides"`
+	RecommendsPkgs string `mapstructure:"Recommends"`
+	ReplacesPkgs   string `mapstructure:"Replaces"`
+	Status         string `mapstructure:"Status"`
+	SuggestsPkgs   string `mapstructure:"Suggests"`
+	Version        string `mapstructure:"Version"`
+	ConfigFiles    string `mapstructure:"Conffiles"`
+}