Merge pull request #891 from chef/dr/depends

introduce dependency resolution (experimental)

examples/inheritance/inspec.yml

@@ -8,3 +8,6 @@ summary: Demonstrates the use of InSpec profile inheritance
 version: 1.0.0
 supports:
   - os-family: unix
+depends:
+  - name: profile
+    path: ../profile

inspec.gemspec

@@ -34,6 +34,7 @@ Gem::Specification.new do |spec|
   spec.add_dependency 'rspec-its', '~> 1.2'
   spec.add_dependency 'pry', '~> 0'
   spec.add_dependency 'hashie', '~> 3.4'
+  spec.add_dependency 'molinillo', '~> 0.5'
 
   spec.add_development_dependency 'mocha', '~> 1.1'
 end

lib/inspec/dependencies.rb

@@ -0,0 +1,307 @@
+# encoding: utf-8
+# author: Dominik Richter
+# author: Christoph Hartmann
+
+require 'logger'
+require 'fileutils'
+require 'molinillo'
+require 'inspec/errors'
+
+module Inspec
+  class Resolver
+    def self.resolve(requirements, vendor_index, cwd, opts = {})
+      reqs = requirements.map do |req|
+        Requirement.from_metadata(req, cwd: cwd) ||
+          fail("Cannot initialize dependency: #{req}")
+      end
+
+      new(vendor_index, opts).resolve(reqs)
+    end
+
+    def initialize(vendor_index, opts = {})
+      @logger = opts[:logger] || Logger.new(nil)
+      @debug_mode = false # TODO: hardcoded for now, grab from options
+
+      @vendor_index = vendor_index
+      @resolver = Molinillo::Resolver.new(self, self)
+      @search_cache = {}
+    end
+
+    # Resolve requirements.
+    #
+    # @param requirements [Array(Inspec::requirement)] Array of requirements
+    # @return [Array(String)] list of resolved dependency paths
+    def resolve(requirements)
+      requirements.each(&:pull)
+      @base_dep_graph = Molinillo::DependencyGraph.new
+      @dep_graph = @resolver.resolve(requirements, @base_dep_graph)
+      arr = @dep_graph.map(&:payload)
+      Hash[arr.map { |e| [e.name, e] }]
+    rescue Molinillo::VersionConflict => e
+      raise VersionConflict.new(e.conflicts.keys.uniq, e.message)
+    rescue Molinillo::CircularDependencyError => e
+      names = e.dependencies.sort_by(&:name).map { |d| "profile '#{d.name}'" }
+      raise CyclicDependencyError,
+            'Your profile has requirements that depend on each other, creating '\
+            "an infinite loop. Please remove #{names.count > 1 ? 'either ' : ''} "\
+            "#{names.join(' or ')} and try again."
+    end
+
+    # --------------------------------------------------------------------------
+    # SpecificationProvider
+
+    # Search for the specifications that match the given dependency.
+    # The specifications in the returned array will be considered in reverse
+    # order, so the latest version ought to be last.
+    # @note This method should be 'pure', i.e. the return value should depend
+    #   only on the `dependency` parameter.
+    #
+    # @param [Object] dependency
+    # @return [Array<Object>] the specifications that satisfy the given
+    #   `dependency`.
+    def search_for(dep)
+      unless dep.is_a?(Inspec::Requirement)
+        fail 'Internal error: Dependency resolver requires an Inspec::Requirement object for #search_for(dependency)'
+      end
+      @search_cache[dep] ||= uncached_search_for(dep)
+    end
+
+    def uncached_search_for(dep)
+      # pre-cached and specified dependencies
+      return [dep] unless dep.profile.nil?
+
+      results = @vendor_index.find(dep)
+      return [] unless results.any?
+
+      # TODO: load dep from vendor index
+      # vertex = @dep_graph.vertex_named(dep.name)
+      # locked_requirement = vertex.payload.requirement if vertex
+      fail NotImplementedError, "load dependency #{dep} from vendor index"
+    end
+
+    # Returns the dependencies of `specification`.
+    # @note This method should be 'pure', i.e. the return value should depend
+    #   only on the `specification` parameter.
+    #
+    # @param [Object] specification
+    # @return [Array<Object>] the dependencies that are required by the given
+    #   `specification`.
+    def dependencies_for(specification)
+      specification.profile.metadata.dependencies
+    end
+
+    # Determines whether the given `requirement` is satisfied by the given
+    # `spec`, in the context of the current `activated` dependency graph.
+    #
+    # @param [Object] requirement
+    # @param [DependencyGraph] activated the current dependency graph in the
+    #   resolution process.
+    # @param [Object] spec
+    # @return [Boolean] whether `requirement` is satisfied by `spec` in the
+    #   context of the current `activated` dependency graph.
+    def requirement_satisfied_by?(requirement, _activated, spec)
+      requirement.matches_spec?(spec) || spec.is_a?(Inspec::Profile)
+    end
+
+    # Returns the name for the given `dependency`.
+    # @note This method should be 'pure', i.e. the return value should depend
+    #   only on the `dependency` parameter.
+    #
+    # @param [Object] dependency
+    # @return [String] the name for the given `dependency`.
+    def name_for(dependency)
+      unless dependency.is_a?(Inspec::Requirement)
+        fail 'Internal error: Dependency resolver requires an Inspec::Requirement object for #name_for(dependency)'
+      end
+      dependency.name
+    end
+
+    # @return [String] the name of the source of explicit dependencies, i.e.
+    #   those passed to {Resolver#resolve} directly.
+    def name_for_explicit_dependency_source
+      'inspec.yml'
+    end
+
+    # @return [String] the name of the source of 'locked' dependencies, i.e.
+    #   those passed to {Resolver#resolve} directly as the `base`
+    def name_for_locking_dependency_source
+      'inspec.lock'
+    end
+
+    # Sort dependencies so that the ones that are easiest to resolve are first.
+    # Easiest to resolve is (usually) defined by:
+    #   1) Is this dependency already activated?
+    #   2) How relaxed are the requirements?
+    #   3) Are there any conflicts for this dependency?
+    #   4) How many possibilities are there to satisfy this dependency?
+    #
+    # @param [Array<Object>] dependencies
+    # @param [DependencyGraph] activated the current dependency graph in the
+    #   resolution process.
+    # @param [{String => Array<Conflict>}] conflicts
+    # @return [Array<Object>] a sorted copy of `dependencies`.
+    def sort_dependencies(dependencies, activated, conflicts)
+      dependencies.sort_by do |dependency|
+        name = name_for(dependency)
+        [
+          activated.vertex_named(name).payload ? 0 : 1,
+          # amount_constrained(dependency), # TODO
+          conflicts[name] ? 0 : 1,
+          # activated.vertex_named(name).payload ? 0 : search_for(dependency).count, # TODO
+        ]
+      end
+    end
+
+    # Returns whether this dependency, which has no possible matching
+    # specifications, can safely be ignored.
+    #
+    # @param [Object] dependency
+    # @return [Boolean] whether this dependency can safely be skipped.
+    def allow_missing?(_dependency)
+      # TODO
+      false
+    end
+
+    # --------------------------------------------------------------------------
+    # UI
+
+    include Molinillo::UI
+
+    # The {IO} object that should be used to print output. `STDOUT`, by default.
+    #
+    # @return [IO]
+    def output
+      self
+    end
+
+    def print(what = '')
+      @logger.info(what)
+    end
+    alias puts print
+  end
+
+  class Package
+    def initialize(path, version)
+      @path = path
+      @version = version
+    end
+  end
+
+  class VendorIndex
+    attr_reader :list, :path
+    def initialize(path)
+      @path = path
+      FileUtils.mkdir_p(path) unless File.directory?(path)
+      @list = Dir[File.join(path, '*')].map { |x| load_path(x) }
+    end
+
+    def find(_dependency)
+      # TODO
+      fail NotImplementedError, '#find(dependency) on VendorIndex seeks implementation.'
+    end
+
+    private
+
+    def load_path(_path)
+      # TODO
+      fail NotImplementedError, '#load_path(path) on VendorIndex wants to be implemented.'
+    end
+  end
+
+  class Requirement
+    attr_reader :name, :dep, :cwd, :opts
+    def initialize(name, dep, cwd, opts)
+      @name = name
+      @dep = Gem::Dependency.new(name, Gem::Requirement.new(Array(dep)), :runtime)
+      @opts = opts
+      @cwd = cwd
+    end
+
+    def matches_spec?(spec)
+      params = spec.profile.metadata.params
+      @dep.match?(params[:name], params[:version])
+    end
+
+    def pull
+      case
+      when @opts[:path] then pull_path(@opts[:path])
+      else
+        # TODO: should default to supermarket
+        fail 'You must specify the source of the dependency (for now...)'
+      end
+    end
+
+    def path
+      @path || pull
+    end
+
+    def profile
+      return nil if path.nil?
+      @profile ||= Inspec::Profile.for_target(path, {})
+    end
+
+    def self.from_metadata(dep, opts)
+      fail 'Cannot load empty dependency.' if dep.nil? || dep.empty?
+      name = dep[:name] || fail('You must provide a name for all dependencies')
+      version = dep[:version]
+      new(name, version, opts[:cwd], dep)
+    end
+
+    def to_s
+      @dep.to_s
+    end
+
+    private
+
+    def pull_path(path)
+      abspath = File.absolute_path(path, @cwd)
+      fail "Dependency path doesn't exist: #{path}" unless File.exist?(abspath)
+      fail "Dependency path isn't a folder: #{path}" unless File.directory?(abspath)
+      @path = abspath
+      true
+    end
+  end
+
+  class SupermarketDependency
+    def initialize(url, requirement)
+      @url = url
+      @requirement = requirement
+    end
+
+    def self.load(dep)
+      return nil if dep.nil?
+      sname = dep[:supermarket]
+      return nil if sname.nil?
+      surl = dep[:supermarket_url] || 'default_url...'
+      requirement = dep[:version]
+      url = surl + '/' + sname
+      new(url, requirement)
+    end
+  end
+
+  class Dependencies
+    attr_reader :list, :vendor_path
+
+    # initialize
+    #
+    # @param cwd [String] current working directory for relative path includes
+    # @param vendor_path [String] path which contains vendored dependencies
+    # @return [dependencies] this
+    def initialize(cwd, vendor_path)
+      @cwd = cwd
+      @vendor_path = vendor_path || File.join(Dir.home, '.inspec', 'cache')
+      @list = nil
+    end
+
+    # 1. Get dependencies, pull things to a local cache if necessary
+    # 2. Resolve dependencies
+    #
+    # @param dependencies [Gem::Dependency] list of dependencies
+    # @return [nil]
+    def vendor(dependencies)
+      return if dependencies.nil? || dependencies.empty?
+      @vendor_index ||= VendorIndex.new(@vendor_path)
+      @list = Resolver.resolve(dependencies, @vendor_index, @cwd)
+    end
+  end
+end

lib/inspec/dsl.rb

@@ -6,12 +6,12 @@
 
 module Inspec::DSL
   def require_controls(id, &block)
-    opts = { profile_id: id, include_all: false, backend: @backend, conf: @conf }
+    opts = { profile_id: id, include_all: false, backend: @backend, conf: @conf, dependencies: @dependencies }
     ::Inspec::DSL.load_spec_files_for_profile(self, opts, &block)
   end
 
   def include_controls(id, &block)
-    opts = { profile_id: id, include_all: true, backend: @backend, conf: @conf }
+    opts = { profile_id: id, include_all: true, backend: @backend, conf: @conf, dependencies: @dependencies }
     ::Inspec::DSL.load_spec_files_for_profile(self, opts, &block)
   end
 
@@ -33,8 +33,9 @@ module Inspec::DSL
 
   def self.load_spec_files_for_profile(bind_context, opts, &block)
     # get all spec files
-    target = get_reference_profile(opts[:profile_id], opts[:conf])
-    profile = Inspec::Profile.for_target(target, opts)
+    target = opts[:dependencies].list[opts[:profile_id]] ||
+             fail("Can't find profile #{opts[:profile_id].inspect}, please add it as a dependency.")
+    profile = Inspec::Profile.for_target(target.path, opts)
     context = load_profile_context(opts[:profile_id], profile, opts)
 
     # if we don't want all the rules, then just make 1 pass to get all rule_IDs
@@ -62,22 +63,6 @@ module Inspec::DSL
     end
   end
 
-  def self.get_reference_profile(id, opts)
-    profiles_path = opts['profiles_path'] ||
-                    fail('You must supply a --profiles-path to inherit from other profiles.')
-    abs_path = File.expand_path(profiles_path.to_s)
-    unless File.directory? abs_path
-      fail("Cannot find profiles path #{abs_path}")
-    end
-
-    id_path = File.join(abs_path, id)
-    unless File.directory? id_path
-      fail("Cannot find referenced profile #{id} in #{id_path}")
-    end
-
-    id_path
-  end
-
   def self.load_profile_context(id, profile, opts)
     ctx = Inspec::ProfileContext.new(id, opts[:backend], opts[:conf])
     profile.libraries.each do |path, content|

lib/inspec/errors.rb

@@ -0,0 +1,17 @@
+# encoding: utf-8
+# author: Dominik Richter
+# author: Christoph Hartmann
+
+module Inspec
+  class Error < StandardError; end
+
+  # dependency resolution
+  class CyclicDependencyError < Error; end
+  class VersionConflict < Error
+    attr_reader :conflicts
+    def initialize(conflicts, msg = nil)
+      super(msg)
+      @conflicts = conflicts
+    end
+  end
+end

lib/inspec/metadata.rb

@@ -36,6 +36,10 @@ module Inspec
       end
     end
 
+    def dependencies
+      params[:depends] || []
+    end
+
     def supports(sth, version = nil)
       # Ignore supports with metadata.rb. This file is legacy and the way it
       # it handles `supports` deprecated. A deprecation warning will be printed

lib/inspec/profile.rb

@@ -8,6 +8,7 @@ require 'inspec/polyfill'
 require 'inspec/fetcher'
 require 'inspec/source_reader'
 require 'inspec/metadata'
+require 'inspec/dependencies'
 
 module Inspec
   class Profile # rubocop:disable Metrics/ClassLength
@@ -206,6 +207,10 @@ module Inspec
       true
     end
 
+    def locked_dependencies
+      @locked_dependencies ||= load_dependencies
+    end
+
     private
 
     # Create an archive name for this profile and an additional options
@@ -290,5 +295,12 @@ module Inspec
       }
       groups[file][:controls].push(id)
     end
+
+    def load_dependencies
+      cwd = File.directory?(@target) ? @target : nil
+      res = Inspec::Dependencies.new(cwd, nil)
+      res.vendor(metadata.dependencies)
+      res
+    end
   end
 end

lib/inspec/profile_context.rb

@@ -22,6 +22,8 @@ module Inspec
       @backend = backend
       @conf = conf.dup
       @rules = {}
+      @dependencies = {}
+      @dependencies = conf['profile'].locked_dependencies unless conf['profile'].nil?
       @require_loader = ::Inspec::RequireLoader.new
       @attributes = []
       reload_dsl
@@ -30,7 +32,7 @@ module Inspec
     def reload_dsl
       resources_dsl = Inspec::Resource.create_dsl(@backend)
       ctx = create_context(resources_dsl, rule_context(resources_dsl))
-      @profile_context = ctx.new(@backend, @conf, @require_loader)
+      @profile_context = ctx.new(@backend, @conf, @dependencies, @require_loader)
     end
 
     def load_libraries(libs)
@@ -136,9 +138,10 @@ module Inspec
         include Inspec::DSL
         include resources_dsl
 
-        def initialize(backend, conf, require_loader) # rubocop:disable Lint/NestedMethodDefinition, Lint/DuplicateMethods
+        def initialize(backend, conf, dependencies, require_loader) # rubocop:disable Lint/NestedMethodDefinition, Lint/DuplicateMethods
           @backend = backend
           @conf = conf
+          @dependencies = dependencies
           @require_loader = require_loader
           @skip_profile = false
         end

test/functional/inheritance_test.rb

@@ -8,19 +8,6 @@ describe 'example inheritance profile' do
   include FunctionalHelper
   let(:path) { File.join(examples_path, 'inheritance') }
 
-  [
-    'archive %s --overwrite',
-    'check %s',
-    'json %s',
-  ].each do |cmd|
-    it cmd[/^\w/] + ' fails without --profiles-path' do
-      out = inspec(format(cmd, path))
-      out.stderr.must_include 'You must supply a --profiles-path to inherit'
-      # out.stdout.must_equal '' => we still get partial output
-      out.exit_status.must_equal 1
-    end
-  end
-
   it 'check succeeds with --profiles-path' do
     out = inspec('check ' + path + ' --profiles-path ' + examples_path)
     out.stderr.must_equal ''
@@ -28,6 +15,13 @@ describe 'example inheritance profile' do
     out.exit_status.must_equal 0
   end
 
+  it 'check succeeds without --profiles-path using inspec.yml' do
+    out = inspec('check ' + path)
+    out.stderr.must_equal ''
+    out.stdout.must_match /Valid.*true/
+    out.exit_status.must_equal 0
+  end
+
   it 'archive is successful with --profiles-path' do
     out = inspec('archive ' + path + ' --output ' + dst.path + ' --profiles-path ' + examples_path)
     out.stderr.must_equal ''
@@ -37,6 +31,15 @@ describe 'example inheritance profile' do
     File.exist?(dst.path).must_equal true
   end
 
+  it 'archive is successful without --profiles-path using inspec.yml' do
+    out = inspec('archive ' + path + ' --output ' + dst.path)
+    out.stderr.must_equal ''
+    out.stdout.must_include 'Generate archive '+dst.path
+    out.stdout.must_include 'Finished archive generation.'
+    out.exit_status.must_equal 0
+    File.exist?(dst.path).must_equal true
+  end
+
   it 'read the profile json with --profiles-path' do
     out = inspec('json ' + path + ' --profiles-path '+examples_path)
     out.stderr.must_equal ''
@@ -46,4 +49,14 @@ describe 'example inheritance profile' do
     hm['name'].must_equal 'inheritance'
     hm['controls'].length.must_equal 3
   end
+
+  it 'read the profile json without --profiles-path using inspec.yml' do
+    out = inspec('json ' + path)
+    out.stderr.must_equal ''
+    out.exit_status.must_equal 0
+    s = out.stdout
+    hm = JSON.load(s)
+    hm['name'].must_equal 'inheritance'
+    hm['controls'].length.must_equal 3
+  end
 end