add rpmdb support; enhance integration tests

Makefile

@@ -32,9 +32,13 @@ test: unit integration ## Run all tests (currently unit & integration)
 help:
 	@grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "$(BOLD)$(CYAN)%-25s$(RESET)%s\n", $$1, $$2}'
 
-ci-bootstrap: bootstrap
+ci-bootstrap: ci-lib-dependencies bootstrap
 	sudo apt install -y bc
 
+ci-lib-dependencies:
+	# libdb5.3-dev and libssl-dev are required for Berkeley DB C bindings for RPM DB support
+	sudo apt install -y libdb5.3-dev libssl-dev
+
 bootstrap: ## Download and install all project dependencies (+ prep tooling in the ./tmp dir)
 	$(call title,Downloading dependencies)
 	# prep temp dirs

go.mod

@@ -12,6 +12,7 @@ require (
 	github.com/gopherjs/gopherjs v0.0.0-20190910122728-9d188e94fb99 // indirect
 	github.com/hashicorp/go-multierror v1.1.0
 	github.com/hashicorp/go-version v1.2.0
+	github.com/knqyf263/go-rpmdb v0.0.0-20190501070121-10a1c42a10dc
 	github.com/mitchellh/go-homedir v1.1.0
 	github.com/mitchellh/mapstructure v1.3.1
 	github.com/sergi/go-diff v1.1.0

go.sum

@@ -530,6 +530,10 @@ github.com/klauspost/compress v1.4.0/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0
 github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
 github.com/klauspost/cpuid v0.0.0-20180405133222-e7e905edc00e/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
 github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
+github.com/knqyf263/berkeleydb v0.0.0-20190501065933-fafe01fb9662 h1:UGS0RbPHwXJkq8tcba8OD0nvVUWLf2h7uUJznuHPPB0=
+github.com/knqyf263/berkeleydb v0.0.0-20190501065933-fafe01fb9662/go.mod h1:bu1CcN4tUtoRcI/B/RFHhxMNKFHVq/c3SV+UTyduoXg=
+github.com/knqyf263/go-rpmdb v0.0.0-20190501070121-10a1c42a10dc h1:pumO9pqmRAjvic6oove22RGh9wDZQnj96XQjJSbSEPs=
+github.com/knqyf263/go-rpmdb v0.0.0-20190501070121-10a1c42a10dc/go.mod h1:MrSSvdMpTSymaQWk1yFr9sxFSyQmKMj6jkbvGrchBV8=
 github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/konsorten/go-windows-terminal-sequences v1.0.3 h1:CE8S1cTafDpPvMhIxNJKvHsGVBgn1xWYf1NbHQhywc8=
@@ -1087,6 +1091,7 @@ golang.org/x/tools v0.0.0-20200502202811-ed308ab3e770/go.mod h1:EkVYQZoAsY45+roY
 golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
 golang.org/x/tools v0.0.0-20200527183253-8e7acdbce89d h1:SR+e35rACZFBohNb4Om1ibX6N3iO0FtdbwqGSuD9dBU=
 golang.org/x/tools v0.0.0-20200527183253-8e7acdbce89d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/xerrors v0.0.0-20190410155217-1f06c39b4373/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=

imgbom/cataloger/controller.go

@@ -4,6 +4,7 @@ import (
 	"github.com/anchore/imgbom/imgbom/cataloger/bundler"
 	"github.com/anchore/imgbom/imgbom/cataloger/dpkg"
 	"github.com/anchore/imgbom/imgbom/cataloger/python"
+	"github.com/anchore/imgbom/imgbom/cataloger/rpmdb"
 	"github.com/anchore/imgbom/imgbom/event"
 	"github.com/anchore/imgbom/imgbom/pkg"
 	"github.com/anchore/imgbom/imgbom/scope"
@@ -44,6 +45,7 @@ func newController() controller {
 	ctrlr.add(dpkg.NewCataloger())
 	ctrlr.add(bundler.NewCataloger())
 	ctrlr.add(python.NewCataloger())
+	ctrlr.add(rpmdb.NewCataloger())
 	return ctrlr
 }
 

imgbom/cataloger/rpmdb/cataloger.go

@@ -0,0 +1,34 @@
+package rpmdb
+
+import (
+	"github.com/anchore/imgbom/imgbom/cataloger/common"
+	"github.com/anchore/imgbom/imgbom/pkg"
+	"github.com/anchore/imgbom/imgbom/scope"
+	"github.com/anchore/stereoscope/pkg/file"
+)
+
+type Cataloger struct {
+	cataloger common.GenericCataloger
+}
+
+func NewCataloger() *Cataloger {
+	pathParsers := map[string]common.ParserFn{
+		"/var/lib/rpm/Packages": parseRpmDB,
+	}
+
+	return &Cataloger{
+		cataloger: common.NewGenericCataloger(pathParsers, nil),
+	}
+}
+
+func (a *Cataloger) Name() string {
+	return "rpmdb-cataloger"
+}
+
+func (a *Cataloger) SelectFiles(resolver scope.FileResolver) []file.Reference {
+	return a.cataloger.SelectFiles(resolver)
+}
+
+func (a *Cataloger) Catalog(contents map[file.Reference]string) ([]pkg.Package, error) {
+	return a.cataloger.Catalog(contents, a.Name())
+}

imgbom/cataloger/rpmdb/parse_rpmdb.go

@@ -0,0 +1,62 @@
+package rpmdb
+
+import (
+	"fmt"
+	"io"
+	"io/ioutil"
+	"os"
+
+	"github.com/anchore/imgbom/imgbom/pkg"
+	"github.com/anchore/imgbom/internal"
+	"github.com/anchore/imgbom/internal/log"
+	rpmdb "github.com/knqyf263/go-rpmdb/pkg"
+)
+
+func parseRpmDB(reader io.Reader) ([]pkg.Package, error) {
+	f, err := ioutil.TempFile("", internal.ApplicationName+"-rpmdb")
+	if err != nil {
+		return nil, fmt.Errorf("failed to create temp rpmdb file: %w", err)
+	}
+
+	defer func() {
+		err = os.Remove(f.Name())
+		if err != nil {
+			log.Errorf("failed to remove temp rpmdb file: %+v", err)
+		}
+	}()
+
+	_, err = io.Copy(f, reader)
+	if err != nil {
+		return nil, fmt.Errorf("failed to copy rpmdb contents to temp file: %w", err)
+	}
+
+	db := rpmdb.DB{}
+	err = db.Open(f.Name())
+	if err != nil {
+		return nil, err
+	}
+
+	pkgList, err := db.ListPackages()
+	if err != nil {
+		return nil, err
+	}
+
+	allPkgs := make([]pkg.Package, 0)
+
+	for _, entry := range pkgList {
+		p := pkg.Package{
+			Name:    entry.Name,
+			Version: entry.Version,
+			Type:    pkg.RpmPkg,
+			Metadata: pkg.RpmMetadata{
+				Epoch:   entry.Epoch,
+				Arch:    entry.Arch,
+				Release: entry.Release,
+			},
+		}
+
+		allPkgs = append(allPkgs, p)
+	}
+
+	return allPkgs, nil
+}

imgbom/cataloger/rpmdb/parse_rpmdb_test.go

@@ -0,0 +1,50 @@
+package rpmdb
+
+import (
+	"github.com/anchore/imgbom/imgbom/pkg"
+	"github.com/go-test/deep"
+	"os"
+	"testing"
+)
+
+func TestParseRpmDB(t *testing.T) {
+	expected := map[string]pkg.Package{
+		"dive": {
+			Name:    "dive",
+			Version: "0.9.2",
+			Type:    pkg.RpmPkg,
+			Metadata: pkg.RpmMetadata{
+				Epoch:   0,
+				Arch:    "x86_64",
+				Release: "1",
+			},
+		},
+	}
+
+	fixture, err := os.Open("test-fixtures/Packages")
+	if err != nil {
+		t.Fatalf("failed to open fixture: %+v", err)
+	}
+
+	actual, err := parseRpmDB(fixture)
+	if err != nil {
+		t.Fatalf("failed to parse rpmdb: %+v", err)
+	}
+
+	if len(actual) != 1 {
+		for _, a := range actual {
+			t.Log("   ", a)
+		}
+		t.Fatalf("unexpected package count: %d!=%d", len(actual), 1)
+	}
+
+	for _, a := range actual {
+		e := expected[a.Name]
+		diffs := deep.Equal(a, e)
+		if len(diffs) > 0 {
+			for _, d := range diffs {
+				t.Errorf("diff: %+v", d)
+			}
+		}
+	}
+}

imgbom/cataloger/rpmdb/test-fixtures/generate-fixture.sh

@@ -0,0 +1,23 @@
+#!/usr/bin/env bash
+set -eux
+
+docker create --name generate-rpmdb-fixture centos:latest sh -c 'tail -f /dev/null'
+
+function cleanup {
+  docker kill generate-rpmdb-fixture
+  docker rm generate-rpmdb-fixture
+}
+trap cleanup EXIT
+
+docker start generate-rpmdb-fixture
+docker exec -i --tty=false generate-rpmdb-fixture bash <<-EOF
+  mkdir -p /scratch
+  cd /scratch
+  rpm --initdb --dbpath /scratch
+  curl -sSLO https://github.com/wagoodman/dive/releases/download/v0.9.2/dive_0.9.2_linux_amd64.rpm
+  rpm --dbpath /scratch -ivh dive_0.9.2_linux_amd64.rpm
+  rm dive_0.9.2_linux_amd64.rpm
+  rpm --dbpath /scratch -qa
+EOF
+
+docker cp generate-rpmdb-fixture:/scratch/Packages .

imgbom/pkg/language.go

@@ -2,8 +2,8 @@ package pkg
 
 const (
 	UnknownLanguage Language = iota
-	Java
-	JavaScript
+	//Java
+	//JavaScript
 	Python
 	Ruby
 )
@@ -12,15 +12,15 @@ type Language uint
 
 var languageStr = []string{
 	"UnknownLanguage",
-	"java",
-	"javascript",
+	//"java",
+	//"javascript",
 	"python",
 	"ruby",
 }
 
 var AllLanguages = []Language{
-	Java,
-	JavaScript,
+	//Java,
+	//JavaScript,
 	Python,
 	Ruby,
 }

imgbom/pkg/metadata.go

@@ -8,3 +8,9 @@ type DpkgMetadata struct {
 	Source  string `mapstructure:"Source"`
 	Version string `mapstructure:"Version"`
 }
+
+type RpmMetadata struct {
+	Epoch   int    `mapstructure:"Epoch"`
+	Arch    string `mapstructure:"Arch"`
+	Release string `mapstructure:"Release"`
+}

imgbom/pkg/type.go

@@ -2,11 +2,11 @@ package pkg
 
 const (
 	UnknownPkg Type = iota
-	ApkPkg
+	//ApkPkg
 	BundlerPkg
 	DebPkg
 	EggPkg
-	PacmanPkg
+	//PacmanPkg
 	RpmPkg
 	WheelPkg
 )
@@ -15,21 +15,21 @@ type Type uint
 
 var typeStr = []string{
 	"UnknownPackage",
-	"apk",
+	//"apk",
 	"bundle",
 	"deb",
 	"egg",
-	"pacman",
+	//"pacman",
 	"rpm",
 	"wheel",
 }
 
 var AllPkgs = []Type{
-	ApkPkg,
+	//ApkPkg,
 	BundlerPkg,
 	DebPkg,
 	EggPkg,
-	PacmanPkg,
+	//PacmanPkg,
 	RpmPkg,
 	WheelPkg,
 }

integration/fixture_image_language_pkgs_test.go

@@ -3,6 +3,7 @@
 package integration
 
 import (
+	"github.com/anchore/imgbom/internal"
 	"testing"
 
 	"github.com/anchore/go-testutils"
@@ -12,7 +13,7 @@ import (
 )
 
 func TestLanguageImage(t *testing.T) {
-	img, cleanup := testutils.GetFixtureImage(t, "docker-archive", "image-language-pkgs")
+	img, cleanup := testutils.GetFixtureImage(t, "docker-archive", "image-pkg-coverage")
 	defer cleanup()
 
 	s, err := scope.NewImageScope(img, scope.AllLayersScope)
@@ -27,6 +28,20 @@ func TestLanguageImage(t *testing.T) {
 		pkgLanguage pkg.Language
 		pkgInfo     map[string]string
 	}{
+		{
+			name:    "find rpmdb packages",
+			pkgType: pkg.RpmPkg,
+			pkgInfo: map[string]string{
+				"dive": "0.9.2",
+			},
+		},
+		{
+			name:    "find dpkg packages",
+			pkgType: pkg.DebPkg,
+			pkgInfo: map[string]string{
+				"apt": "1.8.2",
+			},
+		},
 		{
 			name:        "find python wheel packages",
 			pkgType:     pkg.WheelPkg,
@@ -102,13 +117,28 @@ func TestLanguageImage(t *testing.T) {
 			},
 		},
 	}
+
+	observedLanguages := internal.NewStringSet()
+	definedLanguages := internal.NewStringSet()
+	for _, l := range pkg.AllLanguages {
+		definedLanguages.Add(l.String())
+	}
+
+	observedPkgs := internal.NewStringSet()
+	definedPkgs := internal.NewStringSet()
+	for _, p := range pkg.AllPkgs {
+		definedPkgs.Add(p.String())
+	}
+
 	for _, c := range cases {
 		t.Run(c.name, func(t *testing.T) {
-
 			pkgCount := 0
 
 			for a := range catalog.Enumerate(c.pkgType) {
 
+				observedLanguages.Add(a.Language.String())
+				observedPkgs.Add(a.Type.String())
+
 				expectedVersion, ok := c.pkgInfo[a.Name]
 				if !ok {
 					t.Errorf("unexpected package found: %s", a.Name)
@@ -138,9 +168,17 @@ func TestLanguageImage(t *testing.T) {
 		})
 	}
 
+	observedLanguages.Remove(pkg.UnknownLanguage.String())
+	definedLanguages.Remove(pkg.UnknownLanguage.String())
+	observedPkgs.Remove(pkg.UnknownPkg.String())
+	definedPkgs.Remove(pkg.UnknownPkg.String())
+
 	// ensure that integration test cases stay in sync with the available catalogers
-	if len(cataloger.Catalogers()) < len(cases) {
-		t.Fatalf("probably missed a cataloger during testing, double check that all catalogers are included in testing")
+	if len(observedLanguages) < len(definedLanguages) {
+		t.Errorf("language coverage incomplete (languages=%d, coverage=%d)", len(definedLanguages), len(observedLanguages))
 	}
 
+	if len(observedPkgs) < len(definedPkgs) {
+		t.Errorf("package coverage incomplete (packages=%d, coverage=%d)", len(definedPkgs), len(observedPkgs))
+	}
 }

integration/test-fixtures/image-pkg-coverage/var/lib/dpkg/status

@@ -0,0 +1,35 @@
+Package: apt
+Status: install ok installed
+Priority: required
+Section: admin
+Installed-Size: 4064
+Maintainer: APT Development Team <deity@lists.debian.org>
+Architecture: amd64
+Version: 1.8.2
+Source: apt-dev
+Replaces: apt-transport-https (<< 1.5~alpha4~), apt-utils (<< 1.3~exp2~)
+Provides: apt-transport-https (= 1.8.2)
+Depends: adduser, gpgv | gpgv2 | gpgv1, debian-archive-keyring, libapt-pkg5.0 (>= 1.7.0~alpha3~), libc6 (>= 2.15), libgcc1 (>= 1:3.0), libgnutls30 (>= 3.6.6), libseccomp2 (>= 1.0.1), libstdc++6 (>= 5.2)
+Recommends: ca-certificates
+Suggests: apt-doc, aptitude | synaptic | wajig, dpkg-dev (>= 1.17.2), gnupg | gnupg2 | gnupg1, powermgmt-base
+Breaks: apt-transport-https (<< 1.5~alpha4~), apt-utils (<< 1.3~exp2~), aptitude (<< 0.8.10)
+Conffiles:
+ /etc/apt/apt.conf.d/01autoremove 76120d358bc9037bb6358e737b3050b5
+ /etc/cron.daily/apt-compat 49e9b2cfa17849700d4db735d04244f3
+ /etc/kernel/postinst.d/apt-auto-removal 4ad976a68f045517cf4696cec7b8aa3a
+ /etc/logrotate.d/apt 179f2ed4f85cbaca12fa3d69c2a4a1c3
+Description: commandline package manager
+ This package provides commandline tools for searching and
+ managing as well as querying information about packages
+ as a low-level access to all features of the libapt-pkg library.
+ .
+ These include:
+  * apt-get for retrieval of packages and information about them
+    from authenticated sources and for installation, upgrade and
+    removal of packages together with their dependencies
+  * apt-cache for querying available information about installed
+    as well as installable packages
+  * apt-cdrom to use removable media as a source for packages
+  * apt-config as an interface to the configuration settings
+  * apt-key as an interface to manage authentication keys
+

integration/test-fixtures/image-pkg-coverage/var/lib/rpm/generate-fixture.sh

@@ -0,0 +1,23 @@
+#!/usr/bin/env bash
+set -eux
+
+docker create --name generate-rpmdb-fixture centos:latest sh -c 'tail -f /dev/null'
+
+function cleanup {
+  docker kill generate-rpmdb-fixture
+  docker rm generate-rpmdb-fixture
+}
+trap cleanup EXIT
+
+docker start generate-rpmdb-fixture
+docker exec -i --tty=false generate-rpmdb-fixture bash <<-EOF
+  mkdir -p /scratch
+  cd /scratch
+  rpm --initdb --dbpath /scratch
+  curl -sSLO https://github.com/wagoodman/dive/releases/download/v0.9.2/dive_0.9.2_linux_amd64.rpm
+  rpm --dbpath /scratch -ivh dive_0.9.2_linux_amd64.rpm
+  rm dive_0.9.2_linux_amd64.rpm
+  rpm --dbpath /scratch -qa
+EOF
+
+docker cp generate-rpmdb-fixture:/scratch/Packages .