From 9c1b82e7d4def9889e73eef9c8ed80024d25c59e Mon Sep 17 00:00:00 2001 From: Steven Danna Date: Fri, 19 Aug 2016 15:21:04 +0100 Subject: [PATCH] Add prototype of inspec.lock This adds a basic prototype of inspec.lock. When the lockfile exists on disk, the dependencies tree is constructed using the information in the lock file rather than using the resolver. Signed-off-by: Steven Danna --- lib/fetchers/local.rb | 10 +- lib/fetchers/tar.rb | 8 ++ lib/fetchers/url.rb | 4 + lib/fetchers/zip.rb | 8 ++ lib/inspec/cli.rb | 7 ++ lib/inspec/dependencies/dependency_set.rb | 67 ++++++++++- lib/inspec/dependencies/lockfile.rb | 86 ++++++++++++++ lib/inspec/dependencies/requirement.rb | 133 ++++++++++++++-------- lib/inspec/dependencies/resolver.rb | 9 +- lib/inspec/dependencies/vendor_index.rb | 15 ++- lib/inspec/profile.rb | 34 +++++- 11 files changed, 309 insertions(+), 72 deletions(-) create mode 100644 lib/inspec/dependencies/lockfile.rb diff --git a/lib/fetchers/local.rb b/lib/fetchers/local.rb index df289d3cc..c18a7be74 100644 --- a/lib/fetchers/local.rb +++ b/lib/fetchers/local.rb @@ -10,7 +10,14 @@ module Fetchers attr_reader :files def self.resolve(target) - unless target.is_a?(String) && File.exist?(target) + return nil unless target.is_a?(String) + + # Support "urls" in the form of file:// + if target.start_with?('file://') + target = target.gsub(%r{^file://}, '') + end + + if !File.exist?(target) nil else new(target) @@ -18,6 +25,7 @@ module Fetchers end def initialize(target) + @target = target if File.file?(target) @files = [target] else diff --git a/lib/fetchers/tar.rb b/lib/fetchers/tar.rb index c6f057a59..73525b254 100644 --- a/lib/fetchers/tar.rb +++ b/lib/fetchers/tar.rb @@ -28,6 +28,14 @@ module Fetchers end end + def url + if parent + parent.url + else + 'file://target' + end + end + def read(file) @contents[file] ||= read_from_tar(file) end diff --git a/lib/fetchers/url.rb b/lib/fetchers/url.rb index 6d5866f59..706c479bc 100644 --- a/lib/fetchers/url.rb +++ b/lib/fetchers/url.rb @@ -102,5 +102,9 @@ module Fetchers @target = url @archive = self.class.download_archive(url, opts) end + + def url + @target + end end end diff --git a/lib/fetchers/zip.rb b/lib/fetchers/zip.rb index 0ce199b71..708a1f84d 100644 --- a/lib/fetchers/zip.rb +++ b/lib/fetchers/zip.rb @@ -29,6 +29,14 @@ module Fetchers end end + def url + if parent + parent.url + else + 'file://target' + end + end + def read(file) @contents[file] ||= read_from_zip(file) end diff --git a/lib/inspec/cli.rb b/lib/inspec/cli.rb index 98b1ca95c..9e44d378f 100644 --- a/lib/inspec/cli.rb +++ b/lib/inspec/cli.rb @@ -93,6 +93,13 @@ class Inspec::InspecCLI < Inspec::BaseCLI # rubocop:disable Metrics/ClassLength exit 1 unless result[:summary][:valid] end + desc 'vendor', 'Download all dependencies and generate a lockfile' + def vendor(path = nil) + profile = Inspec::Profile.for_target('./', opts) + lockfile = profile.generate_lockfile(path) + File.write('inspec.lock', lockfile.to_yaml) + end + desc 'archive PATH', 'archive a profile to tar.gz (default) or zip' profile_options option :output, aliases: :o, type: :string, diff --git a/lib/inspec/dependencies/dependency_set.rb b/lib/inspec/dependencies/dependency_set.rb index 25b5bdb6a..f9abe0aed 100644 --- a/lib/inspec/dependencies/dependency_set.rb +++ b/lib/inspec/dependencies/dependency_set.rb @@ -1,5 +1,6 @@ # encoding: utf-8 require 'inspec/dependencies/vendor_index' +require 'inspec/dependencies/requirement' require 'inspec/dependencies/resolver' module Inspec @@ -10,17 +11,71 @@ module Inspec # VendorIndex and the Resolver. # class DependencySet - attr_reader :list, :vendor_path + # + # Return a dependency set given a lockfile. + # + # @param lockfile [Inspec::Lockfile] A lockfile to generate the dependency set from + # @param cwd [String] Current working directory for relative path includes + # @param vendor_path [String] Path to the vendor directory + # + def self.from_lockfile(lockfile, cwd, vendor_path) + vendor_index = VendorIndex.new(vendor_path) + dep_tree = lockfile.deps.map do |dep| + Inspec::Requirement.from_lock_entry(dep, cwd, vendor_index) + end + # Flatten tree because that is all we know how to deal with for + # right now. Last dep seen for a given name wins right now. + dep_list = flatten_dep_tree(dep_tree) + new(cwd, vendor_path, dep_list) + end + + def self.flatten_dep_tree(dep_tree) + dep_list = {} + dep_tree.each do |d| + dep_list[d.name] = d + dep_list.merge!(flatten_dep_tree(d.dependencies)) + end + dep_list + end + + attr_reader :vendor_path + attr_writer :dep_list # initialize # # @param cwd [String] current working directory for relative path includes # @param vendor_path [String] path which contains vendored dependencies # @return [dependencies] this - def initialize(cwd, vendor_path) + def initialize(cwd, vendor_path, dep_list = nil) @cwd = cwd - @vendor_path = vendor_path || File.join(Dir.home, '.inspec', 'cache') - @list = nil + @vendor_path = vendor_path + @dep_list = dep_list + @dep_graph = nil + end + + # + # Returns a flat list of all dependencies since that is all we + # know how to load at the moment. + # + def list + @dep_list ||= begin + return nil if @dep_graph.nil? + arr = @dep_graph.map(&:payload) + Hash[arr.map { |e| [e.name, e] }] + end + end + + def to_array + return [] if @dep_graph.nil? + @dep_graph.map do |v| + # Resolver's list of dependency includes dependencies that + # we'll find further down the tree We don't want those at the + # top level as they should already be included in the to_hash + # output of the nodes that connect them. + if v.incoming_edges.empty? + v.payload.to_hash + end + end.compact end # @@ -29,10 +84,12 @@ module Inspec # # @param dependencies [Gem::Dependency] list of dependencies # @return [nil] + # def vendor(dependencies) return nil if dependencies.nil? || dependencies.empty? @vendor_index ||= VendorIndex.new(@vendor_path) - @list = Resolver.resolve(dependencies, @vendor_index, @cwd) + @dep_graph = Resolver.resolve(dependencies, @vendor_index, @cwd) + list end end end diff --git a/lib/inspec/dependencies/lockfile.rb b/lib/inspec/dependencies/lockfile.rb new file mode 100644 index 000000000..d9918c165 --- /dev/null +++ b/lib/inspec/dependencies/lockfile.rb @@ -0,0 +1,86 @@ +# encoding: utf-8 +require 'yaml' + +module Inspec + class Lockfile + # When we finalize this feature, we should set these to 1 + MINIMUM_SUPPORTED_VERSION = 0 + CURRENT_LOCKFILE_VERSION = 0 + + def self.from_dependency_set(dep_set) + lockfile_content = { + 'lockfile_version' => CURRENT_LOCKFILE_VERSION, + 'depends' => dep_set.to_array, + } + new(lockfile_content) + end + + def self.from_file(path) + parsed_content = YAML.load(File.read(path)) + version = parsed_content['lockfile_version'] + fail "No lockfile_version set in #{path}!" if version.nil? + validate_lockfile_version!(version.to_i) + new(parsed_content) + end + + def self.validate_lockfile_version!(version) + if version < MINIMUM_SUPPORTED_VERSION + fail < CURRENT_LOCKFILE_VERSION + fail < CURRENT_LOCKFILE_VERSION, + 'depends' => @deps, + }.to_yaml + end + + private + + def parse_content_hash(lockfile_content_hash) + case version + when 0 + parse_content_hash_0(lockfile_content_hash) + else + # If we've gotten here, there is likely a mistake in the + # lockfile version validation in the constructor. + fail "No lockfile parser for version #{version}" + end + end + + def parse_content_hash_0(lockfile_content_hash) + @deps = lockfile_content_hash['depends'] + end + end +end diff --git a/lib/inspec/dependencies/requirement.rb b/lib/inspec/dependencies/requirement.rb index f0c10471e..5c51fd1fe 100644 --- a/lib/inspec/dependencies/requirement.rb +++ b/lib/inspec/dependencies/requirement.rb @@ -1,19 +1,42 @@ # encoding: utf-8 require 'inspec/fetcher' +require 'digest' module Inspec # # Inspec::Requirement represents a given profile dependency, where # appropriate we delegate to Inspec::Profile directly. # - class Requirement + class Requirement # rubocop:disable Metrics/ClassLength attr_reader :name, :dep, :cwd, :opts + attr_writer :dependencies + + def self.from_metadata(dep, vendor_index, opts) + fail 'Cannot load empty dependency.' if dep.nil? || dep.empty? + name = dep[:name] || fail('You must provide a name for all dependencies') + version = dep[:version] + new(name, version, vendor_index, opts[:cwd], opts.merge(dep)) + end + + def self.from_lock_entry(entry, cwd, vendor_index) + req = new(entry['name'], + entry['version_constraints'], + vendor_index, + cwd, { url: entry['resolved_source'] }) + + locked_deps = [] + Array(entry['dependencies']).each do |dep_entry| + locked_deps << Inspec::Requirement.from_lock_entry(dep_entry, cwd, vendor_index) + end + + req.lock_deps(locked_deps) + req + end def initialize(name, version_constraints, vendor_index, cwd, opts) @name = name - @dep = Gem::Dependency.new(name, - Gem::Requirement.new(Array(version_constraints)), - :runtime) + @version_requirement = Gem::Requirement.new(Array(version_constraints)) + @dep = Gem::Dependency.new(name, @version_requirement, :runtime) @vendor_index = vendor_index @opts = opts @cwd = cwd @@ -24,64 +47,81 @@ module Inspec @dep.match?(params[:name], params[:version]) end + def to_hash + h = { + 'name' => name, + 'resolved_source' => source_url, + 'version_constraints' => @version_requirement.to_s, + } + + if !dependencies.empty? + h['dependencies'] = dependencies.map(&:to_hash) + end + + if is_vendored? + h['content_hash'] = content_hash + end + h + end + + def lock_deps(dep_array) + @dependencies = dep_array + end + + def is_vendored? + @vendor_index.exists?(@name, source_url) + end + + def content_hash + @content_hash ||= begin + archive_path = @vendor_index.archive_entry_for(@name, source_url) + fail "No vendored archive path for #{self}, cannot take content hash" if archive_path.nil? + Digest::SHA256.hexdigest File.read(archive_path) + end + end + def source_url - case source_type - when :local_path - "file://#{File.expand_path(opts[:path])}" - when :url - @opts[:url] + if opts[:path] + "file://#{opts[:path]}" + elsif opts[:url] + opts[:url] end end def local_path - @local_path ||= case source_type - when :local_path - File.expand_path(opts[:path], @cwd) + @local_path ||= if fetcher.class == Fetchers::Local + File.expand_path(fetcher.target, @cwd) else @vendor_index.prefered_entry_for(@name, source_url) end end - def source_type - @source_type ||= if @opts[:path] - :local_path - elsif opts[:url] - :url - else - fail "Cannot determine source type from #{opts}" - end - end - - def fetcher_class - @fetcher_class ||= case source_type - when :local_path - Fetchers::Local - when :url - Fetchers::Url - else - fail "No known fetcher for dependency #{name} with source_type #{source_type}" - end - - fail "No fetcher for #{name} (options: #{opts})" if @fetcher_class.nil? - @fetcher_class + def fetcher + @fetcher ||= Inspec::Fetcher.resolve(source_url) + fail "No fetcher for #{name} (options: #{opts})" if @fetcher.nil? + @fetcher end def pull - case source_type - when :local_path + # TODO(ssd): Dispatch on the class here is gross. Seems like + # Fetcher is missing an API we want. + if fetcher.class == Fetchers::Local || @vendor_index.exists?(@name, source_url) local_path else - if @vendor_index.exists?(@name, source_url) - local_path - else - archive = fetcher_class.download_archive(source_url) - @vendor_index.add(@name, source_url, archive.path) - end + @vendor_index.add(@name, source_url, fetcher.archive.path) + end + end + + def dependencies + return @dependencies unless @dependencies.nil? + + @dependencies = profile.metadata.dependencies.map do |r| + Inspec::Requirement.from_metadata(r, @vendor_index, cwd: @cwd) end end def to_s - dep.to_s + "#{dep} (#{source_url})" end def path @@ -92,12 +132,5 @@ module Inspec return nil if path.nil? @profile ||= Inspec::Profile.for_target(path, {}) end - - def self.from_metadata(dep, vendor_index, opts) - fail 'Cannot load empty dependency.' if dep.nil? || dep.empty? - name = dep[:name] || fail('You must provide a name for all dependencies') - version = dep[:version] - new(name, version, vendor_index, opts[:cwd], opts.merge(dep)) - end end end diff --git a/lib/inspec/dependencies/resolver.rb b/lib/inspec/dependencies/resolver.rb index 268865720..02c30b7ef 100644 --- a/lib/inspec/dependencies/resolver.rb +++ b/lib/inspec/dependencies/resolver.rb @@ -18,7 +18,7 @@ module Inspec req || fail("Cannot initialize dependency: #{req}") end - new(vendor_index, opts.merge(cwd: cwd)).resolve(reqs) + new(vendor_index, opts).resolve(reqs) end def initialize(vendor_index, opts = {}) @@ -26,7 +26,6 @@ module Inspec @debug_mode = false @vendor_index = vendor_index - @cwd = opts[:cwd] || './' @resolver = Molinillo::Resolver.new(self, self) @search_cache = {} end @@ -39,8 +38,6 @@ module Inspec requirements.each(&:pull) @base_dep_graph = Molinillo::DependencyGraph.new @dep_graph = @resolver.resolve(requirements, @base_dep_graph) - arr = @dep_graph.map(&:payload) - Hash[arr.map { |e| [e.name, e] }] rescue Molinillo::VersionConflict => e raise VersionConflict.new(e.conflicts.keys.uniq, e.message) rescue Molinillo::CircularDependencyError => e @@ -91,9 +88,7 @@ module Inspec # @return [Array] the dependencies that are required by the given # `specification`. def dependencies_for(specification) - specification.profile.metadata.dependencies.map do |r| - Inspec::Requirement.from_metadata(r, @vendor_index, cwd: @cwd) - end + specification.dependencies end # Determines whether the given `requirement` is satisfied by the given diff --git a/lib/inspec/dependencies/vendor_index.rb b/lib/inspec/dependencies/vendor_index.rb index d65b5d2a4..bcaca2264 100644 --- a/lib/inspec/dependencies/vendor_index.rb +++ b/lib/inspec/dependencies/vendor_index.rb @@ -18,9 +18,9 @@ module Inspec # class VendorIndex attr_reader :path - def initialize(path) - @path = path - FileUtils.mkdir_p(path) unless File.directory?(path) + def initialize(path = nil) + @path = path || File.join(Dir.home, '.inspec', 'cache') + FileUtils.mkdir_p(@path) unless File.directory?(@path) end def add(name, source, path_from) @@ -42,7 +42,14 @@ module Inspec path = base_path_for(name, source_url) if File.directory?(path) path - elsif File.exist?("#{path}.tar.gz") + else + archive_entry_for(name, source_url) + end + end + + def archive_entry_for(name, source_url) + path = base_path_for(name, source_url) + if File.exist?("#{path}.tar.gz") "#{path}.tar.gz" elsif File.exist?("#{path}.zip") "#{path}.zip" diff --git a/lib/inspec/profile.rb b/lib/inspec/profile.rb index cd1872d0b..7e386727c 100644 --- a/lib/inspec/profile.rb +++ b/lib/inspec/profile.rb @@ -8,6 +8,7 @@ require 'inspec/polyfill' require 'inspec/fetcher' require 'inspec/source_reader' require 'inspec/metadata' +require 'inspec/dependencies/lockfile' require 'inspec/dependencies/dependency_set' module Inspec @@ -211,6 +212,32 @@ module Inspec @locked_dependencies ||= load_dependencies end + def lockfile_exists? + File.exist?(lockfile_path) + end + + def lockfile_path + File.join(cwd, 'inspec.lock') + end + + def cwd + @target.is_a?(String) && File.directory?(@target) ? @target : nil + end + + def lockfile + @lockfile ||= if lockfile_exists? + Inspec::Lockfile.from_file(lockfile_path) + else + generate_lockfile + end + end + + def generate_lockfile(vendor_path = nil) + res = Inspec::DependencySet.new(cwd, vendor_path) + res.vendor(metadata.dependencies) + Inspec::Lockfile.from_dependency_set(res) + end + private # Create an archive name for this profile and an additional options @@ -225,7 +252,7 @@ module Inspec name = params[:name] || fail('Cannot create an archive without a profile name! Please '\ - 'specify the name in metadata or use --output to create the archive.') + 'specify the name in metadata or use --output to create the archive.') ext = opts[:zip] ? 'zip' : 'tar.gz' slug = name.downcase.strip.tr(' ', '-').gsub(/[^\w-]/, '_') Pathname.new(Dir.pwd).join("#{slug}.#{ext}") @@ -297,10 +324,7 @@ module Inspec end def load_dependencies - cwd = @target.is_a?(String) && File.directory?(@target) ? @target : nil - res = Inspec::DependencySet.new(cwd, nil) - res.vendor(metadata.dependencies) - res + Inspec::DependencySet.from_lockfile(lockfile, cwd, nil) end end end