Add prototype of inspec.lock

This adds a basic prototype of inspec.lock. When the lockfile exists on
disk, the dependencies tree is constructed using the information in the
lock file rather than using the resolver.

Signed-off-by: Steven Danna <steve@chef.io>

lib/fetchers/local.rb

@@ -10,7 +10,14 @@ module Fetchers
     attr_reader :files
 
     def self.resolve(target)
-      unless target.is_a?(String) && File.exist?(target)
+      return nil unless target.is_a?(String)
+
+      # Support "urls" in the form of file://
+      if target.start_with?('file://')
+        target = target.gsub(%r{^file://}, '')
+      end
+
+      if !File.exist?(target)
         nil
       else
         new(target)
@@ -18,6 +25,7 @@ module Fetchers
     end
 
     def initialize(target)
+      @target = target
       if File.file?(target)
         @files = [target]
       else

lib/fetchers/tar.rb

@@ -28,6 +28,14 @@ module Fetchers
       end
     end
 
+    def url
+      if parent
+        parent.url
+      else
+        'file://target'
+      end
+    end
+
     def read(file)
       @contents[file] ||= read_from_tar(file)
     end

lib/fetchers/url.rb

@@ -102,5 +102,9 @@ module Fetchers
       @target = url
       @archive = self.class.download_archive(url, opts)
     end
+
+    def url
+      @target
+    end
   end
 end

lib/fetchers/zip.rb

@@ -29,6 +29,14 @@ module Fetchers
       end
     end
 
+    def url
+      if parent
+        parent.url
+      else
+        'file://target'
+      end
+    end
+
     def read(file)
       @contents[file] ||= read_from_zip(file)
     end

lib/inspec/cli.rb

@@ -93,6 +93,13 @@ class Inspec::InspecCLI < Inspec::BaseCLI # rubocop:disable Metrics/ClassLength
     exit 1 unless result[:summary][:valid]
   end
 
+  desc 'vendor', 'Download all dependencies and generate a lockfile'
+  def vendor(path = nil)
+    profile = Inspec::Profile.for_target('./', opts)
+    lockfile = profile.generate_lockfile(path)
+    File.write('inspec.lock', lockfile.to_yaml)
+  end
+
   desc 'archive PATH', 'archive a profile to tar.gz (default) or zip'
   profile_options
   option :output, aliases: :o, type: :string,

lib/inspec/dependencies/dependency_set.rb

@@ -1,5 +1,6 @@
 # encoding: utf-8
 require 'inspec/dependencies/vendor_index'
+require 'inspec/dependencies/requirement'
 require 'inspec/dependencies/resolver'
 
 module Inspec
@@ -10,17 +11,71 @@ module Inspec
   # VendorIndex and the Resolver.
   #
   class DependencySet
-    attr_reader :list, :vendor_path
+    #
+    # Return a dependency set given a lockfile.
+    #
+    # @param lockfile [Inspec::Lockfile] A lockfile to generate the dependency set from
+    # @param cwd [String] Current working directory for relative path includes
+    # @param vendor_path [String] Path to the vendor directory
+    #
+    def self.from_lockfile(lockfile, cwd, vendor_path)
+      vendor_index = VendorIndex.new(vendor_path)
+      dep_tree = lockfile.deps.map do |dep|
+        Inspec::Requirement.from_lock_entry(dep, cwd, vendor_index)
+      end
 
+      # Flatten tree because that is all we know how to deal with for
+      # right now.  Last dep seen for a given name wins right now.
+      dep_list = flatten_dep_tree(dep_tree)
+      new(cwd, vendor_path, dep_list)
+    end
+
+    def self.flatten_dep_tree(dep_tree)
+      dep_list = {}
+      dep_tree.each do |d|
+        dep_list[d.name] = d
+        dep_list.merge!(flatten_dep_tree(d.dependencies))
+      end
+      dep_list
+    end
+
+    attr_reader :vendor_path
+    attr_writer :dep_list
     # initialize
     #
     # @param cwd [String] current working directory for relative path includes
     # @param vendor_path [String] path which contains vendored dependencies
     # @return [dependencies] this
-    def initialize(cwd, vendor_path)
+    def initialize(cwd, vendor_path, dep_list = nil)
       @cwd = cwd
-      @vendor_path = vendor_path || File.join(Dir.home, '.inspec', 'cache')
+      @vendor_path = vendor_path
-      @list = nil
+      @dep_list = dep_list
+      @dep_graph = nil
+    end
+
+    #
+    # Returns a flat list of all dependencies since that is all we
+    # know how to load at the moment.
+    #
+    def list
+      @dep_list ||= begin
+                      return nil if @dep_graph.nil?
+                      arr = @dep_graph.map(&:payload)
+                      Hash[arr.map { |e| [e.name, e] }]
+                    end
+    end
+
+    def to_array
+      return [] if @dep_graph.nil?
+      @dep_graph.map do |v|
+        # Resolver's list of dependency includes dependencies that
+        # we'll find further down the tree We don't want those at the
+        # top level as they should already be included in the to_hash
+        # output of the nodes that connect them.
+        if v.incoming_edges.empty?
+          v.payload.to_hash
+        end
+      end.compact
     end
 
     #
@@ -29,10 +84,12 @@ module Inspec
     #
     # @param dependencies [Gem::Dependency] list of dependencies
     # @return [nil]
+    #
     def vendor(dependencies)
       return nil if dependencies.nil? || dependencies.empty?
       @vendor_index ||= VendorIndex.new(@vendor_path)
-      @list = Resolver.resolve(dependencies, @vendor_index, @cwd)
+      @dep_graph = Resolver.resolve(dependencies, @vendor_index, @cwd)
+      list
     end
   end
 end

lib/inspec/dependencies/lockfile.rb

@@ -0,0 +1,86 @@
+# encoding: utf-8
+require 'yaml'
+
+module Inspec
+  class Lockfile
+    # When we finalize this feature, we should set these to 1
+    MINIMUM_SUPPORTED_VERSION = 0
+    CURRENT_LOCKFILE_VERSION = 0
+
+    def self.from_dependency_set(dep_set)
+      lockfile_content = {
+        'lockfile_version' => CURRENT_LOCKFILE_VERSION,
+        'depends' => dep_set.to_array,
+      }
+      new(lockfile_content)
+    end
+
+    def self.from_file(path)
+      parsed_content = YAML.load(File.read(path))
+      version = parsed_content['lockfile_version']
+      fail "No lockfile_version set in #{path}!" if version.nil?
+      validate_lockfile_version!(version.to_i)
+      new(parsed_content)
+    end
+
+    def self.validate_lockfile_version!(version)
+      if version < MINIMUM_SUPPORTED_VERSION
+        fail <<EOF
+This lockfile specifies a lockfile_version of #{version} which is
+lower than the minimum supported version #{MINIMUM_SUPPORTED_VERSION}.
+
+Please create a new lockfile for this project by running:
+
+    inspec vendor
+EOF
+      elsif version == 0
+        # Remove this case once this feature stablizes
+        $stderr.puts <<EOF
+WARNING: This is a version 0 lockfile. Thank you for trying the
+experimental dependency management feature. Please be aware you may
+need to regenerate this lockfile in future versions as the feature is
+currently in development.
+EOF
+      elsif version > CURRENT_LOCKFILE_VERSION
+        fail <<EOF
+This lockfile claims to be version #{version} which is greater than
+the most recent lockfile version(#{CURRENT_LOCKFILE_VERSION}).
+
+This may happen if you are using an older version of inspec than was
+used to create the lockfile.
+EOF
+      end
+    end
+
+    attr_reader :version, :deps
+    def initialize(lockfile_content_hash)
+      version = lockfile_content_hash['lockfile_version']
+      @version = version.to_i
+      parse_content_hash(lockfile_content_hash)
+    end
+
+    def to_yaml
+      {
+        'lockfile_version' => CURRENT_LOCKFILE_VERSION,
+        'depends' => @deps,
+      }.to_yaml
+    end
+
+    private
+
+    def parse_content_hash(lockfile_content_hash)
+      case version
+      when 0
+        parse_content_hash_0(lockfile_content_hash)
+      else
+        # If we've gotten here, there is likely a mistake in the
+        # lockfile version validation in the constructor.
+        fail "No lockfile parser for version #{version}"
+      end
+    end
+
+    def parse_content_hash_0(lockfile_content_hash)
+      @deps = lockfile_content_hash['depends']
+    end
+  end
+end

lib/inspec/dependencies/requirement.rb

@@ -1,19 +1,42 @@
 # encoding: utf-8
 require 'inspec/fetcher'
+require 'digest'
 
 module Inspec
   #
   # Inspec::Requirement represents a given profile dependency, where
   # appropriate we delegate to Inspec::Profile directly.
   #
-  class Requirement
+  class Requirement # rubocop:disable Metrics/ClassLength
     attr_reader :name, :dep, :cwd, :opts
+    attr_writer :dependencies
+
+    def self.from_metadata(dep, vendor_index, opts)
+      fail 'Cannot load empty dependency.' if dep.nil? || dep.empty?
+      name = dep[:name] || fail('You must provide a name for all dependencies')
+      version = dep[:version]
+      new(name, version, vendor_index, opts[:cwd], opts.merge(dep))
+    end
+
+    def self.from_lock_entry(entry, cwd, vendor_index)
+      req = new(entry['name'],
+                entry['version_constraints'],
+                vendor_index,
+                cwd, { url: entry['resolved_source'] })
+
+      locked_deps = []
+      Array(entry['dependencies']).each do |dep_entry|
+        locked_deps << Inspec::Requirement.from_lock_entry(dep_entry, cwd, vendor_index)
+      end
+
+      req.lock_deps(locked_deps)
+      req
+    end
 
     def initialize(name, version_constraints, vendor_index, cwd, opts)
       @name = name
-      @dep = Gem::Dependency.new(name,
+      @version_requirement = Gem::Requirement.new(Array(version_constraints))
-                                 Gem::Requirement.new(Array(version_constraints)),
+      @dep = Gem::Dependency.new(name, @version_requirement, :runtime)
-                                 :runtime)
       @vendor_index = vendor_index
       @opts = opts
       @cwd = cwd
@@ -24,64 +47,81 @@ module Inspec
       @dep.match?(params[:name], params[:version])
     end
 
+    def to_hash
+      h = {
+        'name' => name,
+        'resolved_source' => source_url,
+        'version_constraints' => @version_requirement.to_s,
+      }
+
+      if !dependencies.empty?
+        h['dependencies'] = dependencies.map(&:to_hash)
+      end
+
+      if is_vendored?
+        h['content_hash'] = content_hash
+      end
+      h
+    end
+
+    def lock_deps(dep_array)
+      @dependencies = dep_array
+    end
+
+    def is_vendored?
+      @vendor_index.exists?(@name, source_url)
+    end
+
+    def content_hash
+      @content_hash ||= begin
+                          archive_path = @vendor_index.archive_entry_for(@name, source_url)
+                          fail "No vendored archive path for #{self}, cannot take content hash" if archive_path.nil?
+                          Digest::SHA256.hexdigest File.read(archive_path)
+                        end
+    end
+
     def source_url
-      case source_type
+      if opts[:path]
-      when :local_path
+        "file://#{opts[:path]}"
-        "file://#{File.expand_path(opts[:path])}"
+      elsif opts[:url]
-      when :url
+        opts[:url]
-        @opts[:url]
       end
     end
 
     def local_path
-      @local_path ||= case source_type
+      @local_path ||= if fetcher.class == Fetchers::Local
-                      when :local_path
+                        File.expand_path(fetcher.target, @cwd)
-                        File.expand_path(opts[:path], @cwd)
                       else
                         @vendor_index.prefered_entry_for(@name, source_url)
                       end
     end
 
-    def source_type
+    def fetcher
-      @source_type ||= if @opts[:path]
+      @fetcher ||= Inspec::Fetcher.resolve(source_url)
-                         :local_path
+      fail "No fetcher for #{name} (options: #{opts})" if @fetcher.nil?
-                       elsif opts[:url]
+      @fetcher
-                         :url
-                       else
-                         fail "Cannot determine source type from #{opts}"
-                       end
-    end
-
-    def fetcher_class
-      @fetcher_class ||= case source_type
-                         when :local_path
-                           Fetchers::Local
-                         when :url
-                           Fetchers::Url
-                         else
-                           fail "No known fetcher for dependency #{name} with source_type #{source_type}"
-                         end
-
-      fail "No fetcher for #{name} (options: #{opts})" if @fetcher_class.nil?
-      @fetcher_class
     end
 
     def pull
-      case source_type
+      # TODO(ssd): Dispatch on the class here is gross.  Seems like
-      when :local_path
+      # Fetcher is missing an API we want.
+      if fetcher.class == Fetchers::Local || @vendor_index.exists?(@name, source_url)
         local_path
       else
-        if @vendor_index.exists?(@name, source_url)
+        @vendor_index.add(@name, source_url, fetcher.archive.path)
-          local_path
+      end
-        else
+    end
-          archive = fetcher_class.download_archive(source_url)
+
-          @vendor_index.add(@name, source_url, archive.path)
+    def dependencies
-        end
+      return @dependencies unless @dependencies.nil?
+
+      @dependencies = profile.metadata.dependencies.map do |r|
+        Inspec::Requirement.from_metadata(r, @vendor_index, cwd: @cwd)
       end
     end
 
     def to_s
-      dep.to_s
+      "#{dep} (#{source_url})"
     end
 
     def path
@@ -92,12 +132,5 @@ module Inspec
       return nil if path.nil?
       @profile ||= Inspec::Profile.for_target(path, {})
     end
-
-    def self.from_metadata(dep, vendor_index, opts)
-      fail 'Cannot load empty dependency.' if dep.nil? || dep.empty?
-      name = dep[:name] || fail('You must provide a name for all dependencies')
-      version = dep[:version]
-      new(name, version, vendor_index, opts[:cwd], opts.merge(dep))
-    end
   end
 end

lib/inspec/dependencies/resolver.rb

@@ -18,7 +18,7 @@ module Inspec
         req || fail("Cannot initialize dependency: #{req}")
       end
 
-      new(vendor_index, opts.merge(cwd: cwd)).resolve(reqs)
+      new(vendor_index, opts).resolve(reqs)
     end
 
     def initialize(vendor_index, opts = {})
@@ -26,7 +26,6 @@ module Inspec
       @debug_mode = false
 
       @vendor_index = vendor_index
-      @cwd = opts[:cwd] || './'
       @resolver = Molinillo::Resolver.new(self, self)
       @search_cache = {}
     end
@@ -39,8 +38,6 @@ module Inspec
       requirements.each(&:pull)
       @base_dep_graph = Molinillo::DependencyGraph.new
       @dep_graph = @resolver.resolve(requirements, @base_dep_graph)
-      arr = @dep_graph.map(&:payload)
-      Hash[arr.map { |e| [e.name, e] }]
     rescue Molinillo::VersionConflict => e
       raise VersionConflict.new(e.conflicts.keys.uniq, e.message)
     rescue Molinillo::CircularDependencyError => e
@@ -91,9 +88,7 @@ module Inspec
     # @return [Array<Object>] the dependencies that are required by the given
     #   `specification`.
     def dependencies_for(specification)
-      specification.profile.metadata.dependencies.map do |r|
+      specification.dependencies
-        Inspec::Requirement.from_metadata(r, @vendor_index, cwd: @cwd)
-      end
     end
 
     # Determines whether the given `requirement` is satisfied by the given

lib/inspec/dependencies/vendor_index.rb

@@ -18,9 +18,9 @@ module Inspec
   #
   class VendorIndex
     attr_reader :path
-    def initialize(path)
+    def initialize(path = nil)
-      @path = path
+      @path = path || File.join(Dir.home, '.inspec', 'cache')
-      FileUtils.mkdir_p(path) unless File.directory?(path)
+      FileUtils.mkdir_p(@path) unless File.directory?(@path)
     end
 
     def add(name, source, path_from)
@@ -42,7 +42,14 @@ module Inspec
       path = base_path_for(name, source_url)
       if File.directory?(path)
         path
-      elsif File.exist?("#{path}.tar.gz")
+      else
+        archive_entry_for(name, source_url)
+      end
+    end
+
+    def archive_entry_for(name, source_url)
+      path = base_path_for(name, source_url)
+      if File.exist?("#{path}.tar.gz")
         "#{path}.tar.gz"
       elsif File.exist?("#{path}.zip")
         "#{path}.zip"

lib/inspec/profile.rb

@@ -8,6 +8,7 @@ require 'inspec/polyfill'
 require 'inspec/fetcher'
 require 'inspec/source_reader'
 require 'inspec/metadata'
+require 'inspec/dependencies/lockfile'
 require 'inspec/dependencies/dependency_set'
 
 module Inspec
@@ -211,6 +212,32 @@ module Inspec
       @locked_dependencies ||= load_dependencies
     end
 
+    def lockfile_exists?
+      File.exist?(lockfile_path)
+    end
+
+    def lockfile_path
+      File.join(cwd, 'inspec.lock')
+    end
+
+    def cwd
+      @target.is_a?(String) && File.directory?(@target) ? @target : nil
+    end
+
+    def lockfile
+      @lockfile ||= if lockfile_exists?
+                      Inspec::Lockfile.from_file(lockfile_path)
+                    else
+                      generate_lockfile
+                    end
+    end
+
+    def generate_lockfile(vendor_path = nil)
+      res = Inspec::DependencySet.new(cwd, vendor_path)
+      res.vendor(metadata.dependencies)
+      Inspec::Lockfile.from_dependency_set(res)
+    end
+
     private
 
     # Create an archive name for this profile and an additional options
@@ -225,7 +252,7 @@ module Inspec
 
       name = params[:name] ||
              fail('Cannot create an archive without a profile name! Please '\
-             'specify the name in metadata or use --output to create the archive.')
+                  'specify the name in metadata or use --output to create the archive.')
       ext = opts[:zip] ? 'zip' : 'tar.gz'
       slug = name.downcase.strip.tr(' ', '-').gsub(/[^\w-]/, '_')
       Pathname.new(Dir.pwd).join("#{slug}.#{ext}")
@@ -297,10 +324,7 @@ module Inspec
     end
 
     def load_dependencies
-      cwd = @target.is_a?(String) && File.directory?(@target) ? @target : nil
+      Inspec::DependencySet.from_lockfile(lockfile, cwd, nil)
-      res = Inspec::DependencySet.new(cwd, nil)
-      res.vendor(metadata.dependencies)
-      res
     end
   end
 end