From e9373eac961377ad49a23d04eca30979c3dc46a6 Mon Sep 17 00:00:00 2001 From: Alfredo Deza Date: Fri, 15 May 2020 16:44:01 -0400 Subject: [PATCH] initial implementation of dpkg parser Signed-off-by: Alfredo Deza --- imgbom/analyzer/dpkg/parser.go | 148 ++++++++++++++++++++ imgbom/analyzer/dpkg/parser_test.go | 130 +++++++++++++++++ imgbom/analyzer/dpkg/test-fixtures/multiple | 51 +++++++ imgbom/analyzer/dpkg/test-fixtures/single | 34 +++++ 4 files changed, 363 insertions(+) create mode 100644 imgbom/analyzer/dpkg/parser.go create mode 100644 imgbom/analyzer/dpkg/parser_test.go create mode 100644 imgbom/analyzer/dpkg/test-fixtures/multiple create mode 100644 imgbom/analyzer/dpkg/test-fixtures/single diff --git a/imgbom/analyzer/dpkg/parser.go b/imgbom/analyzer/dpkg/parser.go new file mode 100644 index 000000000..181a7ff8f --- /dev/null +++ b/imgbom/analyzer/dpkg/parser.go @@ -0,0 +1,148 @@ +package dpkg + +import ( + "bufio" + "fmt" + "io" + "strings" + + "github.com/mitchellh/mapstructure" +) + +// TODO: consider keeping the remaining values as an embedded map +type DpkgEntry struct { + Package string `mapstructure:"Package"` + Architecture string `mapstructure:"Architecture"` + DependsPkgs string `mapstructure:"Depends"` + InstalledSize string `mapstructure:"Installed-Size"` + Maintainer string `mapstructure:"Maintainer"` + Priority string `mapstructure:"Priority"` + ProvidesPkgs string `mapstructure:"Provides"` + RecommendsPkgs string `mapstructure:"Recommends"` + ReplacesPkgs string `mapstructure:"Replaces"` + Status string `mapstructure:"Status"` + SuggestsPkgs string `mapstructure:"Suggests"` + Version string `mapstructure:"Version"` + ConfigFiles string `mapstructure:"Conffiles"` +} + +// dpkg-query recognized fields +// Architecture +// Bugs +// Conffiles (internal) +// Config-Version (internal) +// Conflicts +// Breaks +// Depends +// Description +// Enhances +// Essential +// Filename (internal, front-end related) +// Homepage +// Installed-Size +// MD5sum (internal, front-end related) +// MSDOS-Filename (internal, front-end related) +// Maintainer +// Origin +// Package +// Pre-Depends +// Priority +// Provides +// Recommends +// Replaces +// Revision (obsolete) +// Section +// Size (internal, front-end related) +// Source +// Status (internal) +// Suggests +// Tag (usually not in .deb but in repository Packages files) +// Triggers-Awaited (internal) +// Triggers-Pending (internal) +// Version +// + +var EndOfPackages = fmt.Errorf("no more packages to read") + +func Read(reader io.Reader) (entry DpkgEntry, err error) { + buff := bufio.NewReader(reader) + dpkgFields := make(map[string]string) + var key string + + for { + line, ioerr := buff.ReadString('\n') + fmt.Printf("line:'%+v' err:'%+v'\n", line, ioerr) + if ioerr != nil { + if ioerr == io.EOF { + return DpkgEntry{}, EndOfPackages + } + return DpkgEntry{}, ioerr + } + + line = strings.TrimRight(line, "\n") + + // stop if there is no contents in line + if len(line) == 0 { + break + } + + switch { + case strings.HasPrefix(line, " "): + // a field-body continuation + if len(key) == 0 { + return DpkgEntry{}, fmt.Errorf("no match for continuation: line: '%s'", line) + } + + val, ok := dpkgFields[key] + if !ok { + return DpkgEntry{}, fmt.Errorf("no previous key exists, expecting: %s", key) + } + // concatenate onto previous value + val = fmt.Sprintf("%s\n %s", val, strings.TrimSpace(line)) + dpkgFields[key] = val + default: + // parse a new key + if i := strings.Index(line, ":"); i > 0 { + key = strings.TrimSpace(line[0:i]) + val := strings.TrimSpace(line[i+1:]) + + if _, ok := dpkgFields[key]; ok { + return DpkgEntry{}, fmt.Errorf("duplicate key discovered: %s", key) + } + + dpkgFields[key] = val + } else { + return DpkgEntry{}, fmt.Errorf("cannot parse field from line: '%s'", line) + } + } + } + + fmt.Println("OUTOFLOOP") + + // map -> struct + err = mapstructure.Decode(dpkgFields, &entry) + if err != nil { + return DpkgEntry{}, err + } + + return entry, nil +} + +func ReadAllDpkgEntries(reader io.Reader) ([]DpkgEntry, error) { + var entries = make([]DpkgEntry, 0) + + for { + // Read() until error + entry, err := Read(reader) + fmt.Printf("entry:'%+v'\n\terr:%+v\n", entry, err) + if err != nil { + if err == EndOfPackages { + break + } + return nil, err + } + entries = append(entries, entry) + } + + return entries, nil +} diff --git a/imgbom/analyzer/dpkg/parser_test.go b/imgbom/analyzer/dpkg/parser_test.go new file mode 100644 index 000000000..8da505fbb --- /dev/null +++ b/imgbom/analyzer/dpkg/parser_test.go @@ -0,0 +1,130 @@ +package dpkg + +import ( + "os" + "testing" + + "github.com/go-test/deep" +) + +func compareEntries(t *testing.T, left, right DpkgEntry) { + t.Helper() + if diff := deep.Equal(left, right); diff != nil { + t.Error(diff) + } +} + +func TestSinglePackage(t *testing.T) { + tests := []struct { + name string + expected DpkgEntry + }{ + { + name: "Test Single Package", + expected: DpkgEntry{ + Package: "apt", + Status: "install ok installed", + Priority: "required", + InstalledSize: "4064", + Maintainer: "APT Development Team ", + Architecture: "amd64", + Version: "1.8.2", + ReplacesPkgs: "apt-transport-https (<< 1.5~alpha4~), apt-utils (<< 1.3~exp2~)", + ProvidesPkgs: "apt-transport-https (= 1.8.2)", + DependsPkgs: "adduser, gpgv | gpgv2 | gpgv1, debian-archive-keyring, libapt-pkg5.0 (>= 1.7.0~alpha3~), libc6 (>= 2.15), libgcc1 (>= 1:3.0), libgnutls30 (>= 3.6.6), libseccomp2 (>= 1.0.1), libstdc++6 (>= 5.2)", + RecommendsPkgs: "ca-certificates", + SuggestsPkgs: "apt-doc, aptitude | synaptic | wajig, dpkg-dev (>= 1.17.2), gnupg | gnupg2 | gnupg1, powermgmt-base", + ConfigFiles: ` + /etc/apt/apt.conf.d/01autoremove 76120d358bc9037bb6358e737b3050b5 + /etc/cron.daily/apt-compat 49e9b2cfa17849700d4db735d04244f3 + /etc/kernel/postinst.d/apt-auto-removal 4ad976a68f045517cf4696cec7b8aa3a + /etc/logrotate.d/apt 179f2ed4f85cbaca12fa3d69c2a4a1c3`, + }, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + file, err := os.Open("test-fixtures/single") + if err != nil { + t.Fatal("Unable to read test_fixtures/single: ", err) + } + + entry, err := Read(file) + if err != nil { + t.Fatal("Unable to read file contents: ", err) + } + + compareEntries(t, entry, test.expected) + + }) + } +} + +func TestMultiplePackage(t *testing.T) { + tests := []struct { + name string + expected []DpkgEntry + }{ + { + name: "Test Multiple Package", + expected: []DpkgEntry{ + { + Package: "tzdata", + Status: "install ok installed", + Priority: "required", + InstalledSize: "3036", + Maintainer: "GNU Libc Maintainers ", + Architecture: "all", + Version: "2020a-0+deb10u1", + ReplacesPkgs: "libc0.1, libc0.3, libc6, libc6.1", + ProvidesPkgs: "tzdata-buster", + DependsPkgs: "debconf (>= 0.5) | debconf-2.0", + }, + { + Package: "util-linux", + Status: "install ok installed", + Priority: "required", + InstalledSize: "4327", + Maintainer: "LaMont Jones ", + Architecture: "amd64", + Version: "2.33.1-0.1", + ReplacesPkgs: "bash-completion (<< 1:2.8), initscripts (<< 2.88dsf-59.2~), login (<< 1:4.5-1.1~), mount (<< 2.29.2-3~), s390-tools (<< 2.2.0-1~), setpriv (<< 2.32.1-0.2~), sysvinit-utils (<< 2.88dsf-59.1~)", + DependsPkgs: "fdisk, login (>= 1:4.5-1.1~)", + SuggestsPkgs: "dosfstools, kbd | console-tools, util-linux-locales", + ConfigFiles: ` + /etc/default/hwclock 3916544450533eca69131f894db0ca12 + /etc/init.d/hwclock.sh 1ca5c0743fa797ffa364db95bb8d8d8e + /etc/pam.d/runuser b8b44b045259525e0fae9e38fdb2aeeb + /etc/pam.d/runuser-l 2106ea05877e8913f34b2c77fa02be45 + /etc/pam.d/su ce6dcfda3b190a27a455bb38a45ff34a + /etc/pam.d/su-l 756fef5687fecc0d986e5951427b0c4f`, + }, + }, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + file, err := os.Open("test-fixtures/multiple") + if err != nil { + t.Fatal("Unable to read: ", err) + } + defer file.Close() + + entries, err := ReadAllDpkgEntries(file) + if err != nil { + t.Fatal("Unable to read file contents: ", err) + } + + if len(entries) != 2 { + t.Fatalf("unexpected number of entries: %d", len(entries)) + } + + for idx, entry := range entries { + compareEntries(t, entry, test.expected[idx]) + } + + }) + } +} diff --git a/imgbom/analyzer/dpkg/test-fixtures/multiple b/imgbom/analyzer/dpkg/test-fixtures/multiple new file mode 100644 index 000000000..8c515e695 --- /dev/null +++ b/imgbom/analyzer/dpkg/test-fixtures/multiple @@ -0,0 +1,51 @@ +Package: tzdata +Status: install ok installed +Priority: required +Section: localization +Installed-Size: 3036 +Maintainer: GNU Libc Maintainers +Architecture: all +Multi-Arch: foreign +Version: 2020a-0+deb10u1 +Replaces: libc0.1, libc0.3, libc6, libc6.1 +Provides: tzdata-buster +Depends: debconf (>= 0.5) | debconf-2.0 +Description: time zone and daylight-saving time data + This package contains data required for the implementation of + standard local time for many representative locations around the + globe. It is updated periodically to reflect changes made by + political bodies to time zone boundaries, UTC offsets, and + daylight-saving rules. +Homepage: https://www.iana.org/time-zones + +Package: util-linux +Essential: yes +Status: install ok installed +Priority: required +Section: utils +Installed-Size: 4327 +Maintainer: LaMont Jones +Architecture: amd64 +Multi-Arch: foreign +Version: 2.33.1-0.1 +Replaces: bash-completion (<< 1:2.8), initscripts (<< 2.88dsf-59.2~), login (<< 1:4.5-1.1~), mount (<< 2.29.2-3~), s390-tools (<< 2.2.0-1~), setpriv (<< 2.32.1-0.2~), sysvinit-utils (<< 2.88dsf-59.1~) +Depends: fdisk, login (>= 1:4.5-1.1~) +Pre-Depends: libaudit1 (>= 1:2.2.1), libblkid1 (>= 2.31.1), libc6 (>= 2.25), libcap-ng0 (>= 0.7.9), libmount1 (>= 2.25), libpam0g (>= 0.99.7.1), libselinux1 (>= 2.6-3~), libsmartcols1 (>= 2.33), libsystemd0, libtinfo6 (>= 6), libudev1 (>= 183), libuuid1 (>= 2.16), zlib1g (>= 1:1.1.4) +Suggests: dosfstools, kbd | console-tools, util-linux-locales +Breaks: bash-completion (<< 1:2.8), grml-debootstrap (<< 0.68), mount (<< 2.29.2-3~), s390-tools (<< 2.2.0-1~), setpriv (<< 2.32.1-0.2~), sysvinit-utils (<< 2.88dsf-59.4~) +Conffiles: + /etc/default/hwclock 3916544450533eca69131f894db0ca12 + /etc/init.d/hwclock.sh 1ca5c0743fa797ffa364db95bb8d8d8e + /etc/pam.d/runuser b8b44b045259525e0fae9e38fdb2aeeb + /etc/pam.d/runuser-l 2106ea05877e8913f34b2c77fa02be45 + /etc/pam.d/su ce6dcfda3b190a27a455bb38a45ff34a + /etc/pam.d/su-l 756fef5687fecc0d986e5951427b0c4f +Description: miscellaneous system utilities + This package contains a number of important utilities, most of which + are oriented towards maintenance of your system. Some of the more + important utilities included in this package allow you to view kernel + messages, create new filesystems, view block device information, + interface with real time clock, etc. + + + diff --git a/imgbom/analyzer/dpkg/test-fixtures/single b/imgbom/analyzer/dpkg/test-fixtures/single new file mode 100644 index 000000000..79590d82e --- /dev/null +++ b/imgbom/analyzer/dpkg/test-fixtures/single @@ -0,0 +1,34 @@ +Package: apt +Status: install ok installed +Priority: required +Section: admin +Installed-Size: 4064 +Maintainer: APT Development Team +Architecture: amd64 +Version: 1.8.2 +Replaces: apt-transport-https (<< 1.5~alpha4~), apt-utils (<< 1.3~exp2~) +Provides: apt-transport-https (= 1.8.2) +Depends: adduser, gpgv | gpgv2 | gpgv1, debian-archive-keyring, libapt-pkg5.0 (>= 1.7.0~alpha3~), libc6 (>= 2.15), libgcc1 (>= 1:3.0), libgnutls30 (>= 3.6.6), libseccomp2 (>= 1.0.1), libstdc++6 (>= 5.2) +Recommends: ca-certificates +Suggests: apt-doc, aptitude | synaptic | wajig, dpkg-dev (>= 1.17.2), gnupg | gnupg2 | gnupg1, powermgmt-base +Breaks: apt-transport-https (<< 1.5~alpha4~), apt-utils (<< 1.3~exp2~), aptitude (<< 0.8.10) +Conffiles: + /etc/apt/apt.conf.d/01autoremove 76120d358bc9037bb6358e737b3050b5 + /etc/cron.daily/apt-compat 49e9b2cfa17849700d4db735d04244f3 + /etc/kernel/postinst.d/apt-auto-removal 4ad976a68f045517cf4696cec7b8aa3a + /etc/logrotate.d/apt 179f2ed4f85cbaca12fa3d69c2a4a1c3 +Description: commandline package manager + This package provides commandline tools for searching and + managing as well as querying information about packages + as a low-level access to all features of the libapt-pkg library. + . + These include: + * apt-get for retrieval of packages and information about them + from authenticated sources and for installation, upgrade and + removal of packages together with their dependencies + * apt-cache for querying available information about installed + as well as installable packages + * apt-cdrom to use removable media as a source for packages + * apt-config as an interface to the configuration settings + * apt-key as an interface to manage authentication keys +