From d272024b018e52092c968ade7918ea768950d244 Mon Sep 17 00:00:00 2001 From: Dominik Richter Date: Wed, 10 Feb 2016 20:36:43 +0100 Subject: [PATCH 1/5] rework resolver connection I.e. we want to get access to the actual directory handler, with full exposure of the underlying directory resolver. e.g. Get the InspecProfileDirectory handler (which provides access to tests, metadata, libraries), but be able to get all data with that alone (e.g. an ArchiveHelper for ZIP which reads all files/folders from zip) --- lib/inspec/targets/core.rb | 5 ++-- lib/inspec/targets/dir.rb | 47 +++++++++++++++++++++++++++++++++--- lib/inspec/targets/folder.rb | 41 +++++++++---------------------- 3 files changed, 57 insertions(+), 36 deletions(-) diff --git a/lib/inspec/targets/core.rb b/lib/inspec/targets/core.rb index 101e1bc09..25b4b3dd9 100644 --- a/lib/inspec/targets/core.rb +++ b/lib/inspec/targets/core.rb @@ -14,9 +14,8 @@ module Inspec def self.find_handler(target) resolver = find_resolver(target) - return resolver unless resolver.is_a?DirsResolver - files = resolver.get_files(target) - DirsHelper.get_handler(files) + return resolver unless resolver.ancestors.include? DirsResolver + resolver.from_target(target).handler end def self.resolve(targets, opts = {}) diff --git a/lib/inspec/targets/dir.rb b/lib/inspec/targets/dir.rb index a3b01ea80..6f4ec9b04 100644 --- a/lib/inspec/targets/dir.rb +++ b/lib/inspec/targets/dir.rb @@ -18,7 +18,7 @@ module Inspec::Targets extend Modulator def self.get_handler(paths) - modules.values.find { |x| x.handles?(paths) } + modules.values.reverse.find { |x| x.handles?(paths) } end end @@ -27,8 +27,45 @@ module Inspec::Targets # # These resolvers must implement the required methods of this class. class DirsResolver - def get_files(_target) - fail NotImplementedError, "Directory resolver #{self.class} must implement #get_files" + def files + fail NotImplementedError, "Directory resolver #{self.class} must implement #files" + end + + def resolve(_path) + fail NotImplementedError, "Directory resolver #{self.class} must implement #content" + end + + def handler + DirsHelper.get_handler(files) || + fail("Don't know how to handle #{self}") + end + + def self.from_target(target) + new(target) + end + + def self.resolve(target, opts) + r = new(target, opts) + handler = r.handler + files = r.files + + res = [] + { + test: __collect(handler, :get_filenames, files), + library: __collect(handler, :get_libraries, files), + metadata: __collect(handler, :get_metadata, files), + }.each do |as, list| + Array(list).each { |path| + res.push(r.resolve(path, as: as)) + } + end + + res + end + + def self.__collect(handler, getter, files) + return [] unless handler.respond_to? getter + handler.method(getter).call(files) end end @@ -97,7 +134,9 @@ module Inspec::Targets end def get_filenames(paths) - paths.find_all { |x| x.end_with?('.rb') and !x.include?('/') } + # TODO: eventually remove the metadata.rb exception here + # when we have fully phased out metadata.rb in 1.0 + paths.find_all { |x| x.end_with?('.rb') && !x.include?('/') && x != 'metadata.rb' } end end DirsHelper.add_module('flat-ruby-folder', FlatDir.new) diff --git a/lib/inspec/targets/folder.rb b/lib/inspec/targets/folder.rb index bafec8e4a..39a9cf086 100644 --- a/lib/inspec/targets/folder.rb +++ b/lib/inspec/targets/folder.rb @@ -7,49 +7,32 @@ require 'inspec/targets/file' module Inspec::Targets class FolderHelper < DirsResolver - def handles?(target) + def self.handles?(target) File.directory?(target) end - def get_files(target) + attr_reader :files + + def initialize(target, _opts) + @base_folder = target # find all files in the folder files = Dir[File.join(target, '**', '*')] # remove the prefix prefix = File.join(target, '') - files.map { |x| x.sub(prefix, '') } + @files = files.map { |x| x.sub(prefix, '') } + @file_handler = Inspec::Targets.modules['file'] end - def resolve(target, _opts = {}) - # get the dirs helper - files = get_files(target) - helper = DirsHelper.get_handler(files) || - fail("Don't know how to handle folder #{target}") - - # get all test file contents - file_handler = Inspec::Targets.modules['file'] - res = { - test: collect(helper, files, :get_filenames), - library: collect(helper, files, :get_libraries), - metadata: collect(helper, files, :get_metadata), - }.map { |as, list| - file_handler.resolve_all(list, base_folder: target, as: as) - } - - # flatten the outer list layer - res.inject(&:+) + def resolve(path, opts = {}) + o = (opts || {}) + o[:base_folder] = @base_folder + @file_handler.resolve(path, o) end def to_s 'Folder Loader' end - - private - - def collect(helper, files, getter) - return [] unless helper.respond_to? getter - helper.method(getter).call(files) - end end - Inspec::Targets.add_module('folder', FolderHelper.new) + Inspec::Targets.add_module('folder', FolderHelper) end From 6bd757c58532c3f8de9988530f5b9d67f21024cb Mon Sep 17 00:00:00 2001 From: Dominik Richter Date: Wed, 10 Feb 2016 20:36:54 +0100 Subject: [PATCH 2/5] improve documentation on target resolvers --- lib/inspec/targets/core.rb | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/lib/inspec/targets/core.rb b/lib/inspec/targets/core.rb index 25b4b3dd9..ca12c4a71 100644 --- a/lib/inspec/targets/core.rb +++ b/lib/inspec/targets/core.rb @@ -8,16 +8,42 @@ module Inspec module Targets extend Modulator + # Retrieve the resolver for a given target. Resolvers are any type of + # instance, that can take a target in the #resolve method, and turn + # it into raw InSpec code. It will actually retrieve all file's + # contents, including libraries and metadata. + # + # @param [any] target any target you are pointing to; typically a string + # @return [Inspec::Targets::*] the resolver handling this target def self.find_resolver(target) modules.values.find { |m| m.handles?(target) } end + # Retrieve the target handler, i.e. the component that is going to handle + # all aspects of a given target. Unlike the resolver, this provides + # access to an object, that understands the target and is able to read + # all contents, while providing additional methods which InSpec itself + # does not care about (think: plugins). + # + # There is a special case, where you might be pointing to a URL containing + # a ZIP file which has a full InSpec Profile embedded. This will be + # resolved to a directory helper (Inspec::Targets::DirsResolver). This + # method will retrieve the right handler for this directory and provide it + # with full access all contents (i.e. the DirsResolver will be embedded). + # + # @param [any] target any target you are pointing to; typically a string + # @return [Inspec::Targets::*] the handler for this target def self.find_handler(target) resolver = find_resolver(target) return resolver unless resolver.ancestors.include? DirsResolver resolver.from_target(target).handler end + # Turn targets into actionable InSpec code, library-data and metadata. + # + # @param [any] targets any targets you are pointing to; typically strings + # @param [Hash] additional options for loading + # @return [Array] an array of resolved data, with :content, :ref, :type def self.resolve(targets, opts = {}) Array(targets).map do |target| handler = find_resolver(target) || From 19a0a18db113199548a1632406825faf65edb025 Mon Sep 17 00:00:00 2001 From: Dominik Richter Date: Wed, 10 Feb 2016 22:30:13 +0100 Subject: [PATCH 3/5] sync archive+tar+zip helpers to new dir-resolver --- lib/inspec/targets/archive.rb | 38 +++++++++++------------------------ lib/inspec/targets/tar.rb | 4 ++-- lib/inspec/targets/zip.rb | 4 ++-- 3 files changed, 16 insertions(+), 30 deletions(-) diff --git a/lib/inspec/targets/archive.rb b/lib/inspec/targets/archive.rb index 5e5607223..3f6c1733b 100644 --- a/lib/inspec/targets/archive.rb +++ b/lib/inspec/targets/archive.rb @@ -8,40 +8,26 @@ require 'inspec/targets/dir' module Inspec::Targets class ArchiveHelper < DirsResolver - def get_files(target) - files, rootdir = structure(target) + attr_reader :files + + def initialize(target, _opts) + @target = target + files, @rootdir = structure(target) # remove trailing slashes files = files.collect { |f| f.chomp('/') } # remove leading directory - files.collect { |f| - Pathname(f).relative_path_from(Pathname(rootdir)).to_s + root = Pathname(@rootdir) + @files = files.collect { |f| + Pathname(f).relative_path_from(root).to_s } end - def resolve(target, _opts = {}) - files = get_files(target) - helper = DirsHelper.get_handler(files) || - fail("Don't know how to handle folder #{target}") - - _, rootdir = structure(target) - - res = { - test: collect(helper, files, :get_filenames), - library: collect(helper, files, :get_libraries), - metadata: collect(helper, files, :get_metadata), - }.map { |as, list| - content(target, list, rootdir, base_folder: target, as: as) - } - - res.flatten - end - - # FIXME(sr) dedup inspec/targets/folder - def collect(helper, files, getter) - return [] unless helper.respond_to? getter - helper.method(getter).call(files) + def resolve(path, opts = {}) + o = (opts || {}) + o[:base_folder] = @target + content(@target, [path], @rootdir, o) end end end diff --git a/lib/inspec/targets/tar.rb b/lib/inspec/targets/tar.rb index 554be91bf..90d8818e3 100644 --- a/lib/inspec/targets/tar.rb +++ b/lib/inspec/targets/tar.rb @@ -8,7 +8,7 @@ require 'inspec/targets/archive' module Inspec::Targets class TarHelper < ArchiveHelper - def handles?(target) + def self.handles?(target) File.file?(target) && target.end_with?('.tar.gz', '.tgz') end @@ -59,5 +59,5 @@ module Inspec::Targets end end - Inspec::Targets.add_module('tar', TarHelper.new) + Inspec::Targets.add_module('tar', TarHelper) end diff --git a/lib/inspec/targets/zip.rb b/lib/inspec/targets/zip.rb index 84c5ae72e..404111f96 100644 --- a/lib/inspec/targets/zip.rb +++ b/lib/inspec/targets/zip.rb @@ -8,7 +8,7 @@ require 'inspec/targets/archive' module Inspec::Targets class ZipHelper < ArchiveHelper - def handles?(target) + def self.handles?(target) File.file?(target) and target.end_with?('.zip') end @@ -51,5 +51,5 @@ module Inspec::Targets end end - Inspec::Targets.add_module('zip', ZipHelper.new) + Inspec::Targets.add_module('zip', ZipHelper) end From 3efd0961f0c6d4356f23381e406c4f268e89a248 Mon Sep 17 00:00:00 2001 From: Dominik Richter Date: Wed, 10 Feb 2016 22:49:51 +0100 Subject: [PATCH 4/5] make sure archive resolvers return one file only --- lib/inspec/targets/archive.rb | 4 ++-- lib/inspec/targets/folder.rb | 2 +- lib/inspec/targets/tar.rb | 20 +++++++++----------- lib/inspec/targets/zip.rb | 10 +++++----- 4 files changed, 17 insertions(+), 19 deletions(-) diff --git a/lib/inspec/targets/archive.rb b/lib/inspec/targets/archive.rb index 3f6c1733b..c4c263d69 100644 --- a/lib/inspec/targets/archive.rb +++ b/lib/inspec/targets/archive.rb @@ -10,7 +10,7 @@ module Inspec::Targets class ArchiveHelper < DirsResolver attr_reader :files - def initialize(target, _opts) + def initialize(target, _opts = {}) @target = target files, @rootdir = structure(target) @@ -27,7 +27,7 @@ module Inspec::Targets def resolve(path, opts = {}) o = (opts || {}) o[:base_folder] = @target - content(@target, [path], @rootdir, o) + content(@target, path, @rootdir, o) end end end diff --git a/lib/inspec/targets/folder.rb b/lib/inspec/targets/folder.rb index 39a9cf086..44f41690c 100644 --- a/lib/inspec/targets/folder.rb +++ b/lib/inspec/targets/folder.rb @@ -13,7 +13,7 @@ module Inspec::Targets attr_reader :files - def initialize(target, _opts) + def initialize(target, _opts = {}) @base_folder = target # find all files in the folder files = Dir[File.join(target, '**', '*')] diff --git a/lib/inspec/targets/tar.rb b/lib/inspec/targets/tar.rb index 90d8818e3..e8af85708 100644 --- a/lib/inspec/targets/tar.rb +++ b/lib/inspec/targets/tar.rb @@ -30,22 +30,20 @@ module Inspec::Targets [files, rootdir] end - def content(input, files, rootdir = nil, opts = {}) - content = [] + def content(input, path, rootdir = nil, opts = {}) + content = nil Gem::Package::TarReader.new(Zlib::GzipReader.open(input)) do |tar| tar.each do |entry| if entry.directory? # nothing to do elsif entry.file? - if files.include?(entry.full_name.gsub(rootdir, '')) - h = { - # NB if some file is empty, return empty-string, not nil - content: entry.read || '', - type: opts[:as] || :test, - ref: entry.full_name, - } - content.push(h) - end + next unless path == entry.full_name.gsub(rootdir, '') + content = { + # NB if some file is empty, return empty-string, not nil + content: entry.read || '', + type: opts[:as] || :test, + ref: entry.full_name, + } elsif entry.header.typeflag == '2' # ignore symlinks for now end diff --git a/lib/inspec/targets/zip.rb b/lib/inspec/targets/zip.rb index 404111f96..cd094a1f0 100644 --- a/lib/inspec/targets/zip.rb +++ b/lib/inspec/targets/zip.rb @@ -12,18 +12,18 @@ module Inspec::Targets File.file?(target) and target.end_with?('.zip') end - def content(input, files, rootdir = nil, opts = {}) - content = [] + def content(input, path, rootdir = nil, opts = {}) + content = nil ::Zip::InputStream.open(input) do |io| while (entry = io.get_next_entry) - next if !files.include?(entry.name.gsub(rootdir, '')) - h = { + next unless path == entry.name.gsub(rootdir, '') + content = { # NB if some file is empty, return empty-string, not nil content: io.read || '', type: opts[:as] || :test, ref: entry.name, } - content.push(h) + abort end end content From 137bee74ca97b1d1af4ed126653a1994dc8acb55 Mon Sep 17 00:00:00 2001 From: Dominik Richter Date: Wed, 10 Feb 2016 23:46:55 +0100 Subject: [PATCH 5/5] add content resolver to dir helper --- lib/inspec/targets/dir.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/inspec/targets/dir.rb b/lib/inspec/targets/dir.rb index 6f4ec9b04..c9b225f49 100644 --- a/lib/inspec/targets/dir.rb +++ b/lib/inspec/targets/dir.rb @@ -60,6 +60,7 @@ module Inspec::Targets } end + return handler.resolve_contents(res) if handler.respond_to?(:resolve_contents) res end