Mirrors/inspec
2
0
Fork 0
mirror of https://github.com/inspec/inspec synced 2024-11-22 20:53:11 +00:00
Code Issues Projects Releases Packages Wiki Activity

Load all dependent libraries, even if include_context isn't called

Browse source 
The goal of these changes is to ensure that the libraries from
dependencies are loaded even if their controls are never included.  To
facilitate this, we break up the loading into seperate steps, and move
the loading code into the Profile which has acceess to the dependency
information.

Signed-off-by: Steven Danna <steve@chef.io>
This commit is contained in:
Steven Danna 2016-08-31 12:17:44 +01:00 committed by Christoph Hartmann
parent 384ccb610c
commit 5fdf659df1
15 changed files with 167 additions and 221 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
lib/inspec/base_cli.rb
View file

@ -70,7 +70,7 @@ module Inspec
o[:logger].level = get_log_level(o.log_level)
runner = Inspec::Runner.new(o)
targets.each { |target| runner.add_target(target, opts) }
targets.each { |target| runner.add_target(target) }
exit runner.run
rescue RuntimeError, Train::UserError => e
$stderr.puts e.message

27
lib/inspec/dependencies/dependency_set.rb
View file

@ -18,14 +18,22 @@ module Inspec
# @param cwd [String] Current working directory for relative path includes
# @param vendor_path [String] Path to the vendor directory
#
def self.from_lockfile(lockfile, cwd, vendor_path)
def self.from_lockfile(lockfile, cwd, vendor_path, backend)
vendor_index = VendorIndex.new(vendor_path)
dep_tree = lockfile.deps.map do |dep|
Inspec::Requirement.from_lock_entry(dep, cwd, vendor_index)
Inspec::Requirement.from_lock_entry(dep, cwd, vendor_index, backend)
end
dep_list = flatten_dep_tree(dep_tree)
new(cwd, vendor_path, dep_list)
new(cwd, vendor_path, dep_list, backend)
end
def self.from_array(dependencies, cwd, vendor_path, backend)
dep_list = {}
dependencies.each do |d|
dep_list[d.name] = d
end
new(cwd, vendor_path, dep_list, backend)
end
# This is experimental code to test the working of the
@ -50,14 +58,21 @@ module Inspec
# @param cwd [String] current working directory for relative path includes
# @param vendor_path [String] path which contains vendored dependencies
# @return [dependencies] this
def initialize(cwd, vendor_path, dep_list = nil)
def initialize(cwd, vendor_path, dep_list, backend)
@cwd = cwd
@vendor_path = vendor_path
@dep_list = dep_list
@backend = backend
end
def each
@dep_list.each do |_k, v|
yield v.profile
end
end
def list
@dep_list
@dep_list || {}
end
def to_array
@ -77,7 +92,7 @@ module Inspec
def vendor(dependencies)
return nil if dependencies.nil? || dependencies.empty?
@vendor_index ||= VendorIndex.new(@vendor_path)
@dep_list = Resolver.resolve(dependencies, @vendor_index, @cwd)
@dep_list = Resolver.resolve(dependencies, @vendor_index, @cwd, @backend)
end
end
end

19
lib/inspec/dependencies/requirement.rb
View file

@ -1,5 +1,6 @@
# encoding: utf-8
require 'inspec/fetcher'
require 'inspec/dependencies/dependency_set'
require 'digest'
module Inspec
@ -18,17 +19,18 @@ module Inspec
new(name, version, vendor_index, opts[:cwd], opts.merge(dep))
end
def self.from_lock_entry(entry, cwd, vendor_index)
def self.from_lock_entry(entry, cwd, vendor_index, backend)
req = new(entry['name'],
entry['version_constraints'],
vendor_index,
cwd, { url: entry['resolved_source'] })
cwd,
{ url: entry['resolved_source'],
backend: backend })
locked_deps = []
Array(entry['dependencies']).each do |dep_entry|
locked_deps << Inspec::Requirement.from_lock_entry(dep_entry, cwd, vendor_index)
locked_deps << Inspec::Requirement.from_lock_entry(dep_entry, cwd, vendor_index, backend)
end
req.lock_deps(locked_deps)
req
end
@ -38,6 +40,7 @@ module Inspec
@version_requirement = Gem::Requirement.new(Array(version_constraints))
@dep = Gem::Dependency.new(name, @version_requirement, :runtime)
@vendor_index = vendor_index
@backend = opts[:backend]
@opts = opts
@cwd = cwd
end
@ -123,7 +126,7 @@ module Inspec
def dependencies
@dependencies ||= profile.metadata.dependencies.map do |r|
Inspec::Requirement.from_metadata(r, @vendor_index, cwd: @cwd)
Inspec::Requirement.from_metadata(r, @vendor_index, cwd: @cwd, backend: @backend)
end
end
@ -137,7 +140,11 @@ module Inspec
def profile
return nil if path.nil?
@profile ||= Inspec::Profile.for_target(path, {})
opts = { backend: @backend }
if !@dependencies.nil?
opts[:dependencies] = Inspec::DependencySet.from_array(@dependencies, @cwd, @vendor_index, @backend)
end
@profile ||= Inspec::Profile.for_target(path, opts)
end
end
end

4
lib/inspec/dependencies/resolver.rb
View file

@ -23,9 +23,9 @@ module Inspec
# implementation of the fetcher being used.
#
class Resolver
def self.resolve(dependencies, vendor_index, working_dir)
def self.resolve(dependencies, vendor_index, working_dir, backend)
reqs = dependencies.map do |dep|
req = Inspec::Requirement.from_metadata(dep, vendor_index, cwd: working_dir)
req = Inspec::Requirement.from_metadata(dep, vendor_index, cwd: working_dir, backend: backend)
req || fail("Cannot initialize dependency: #{req}")
end
new.resolve(reqs)

8
lib/inspec/dsl.rb
View file

@ -34,21 +34,17 @@ Dependencies available from this context are:
EOF
end
context = Inspec::ProfileContext.for_profile(dep_entry.profile, opts[:backend])
context = dep_entry.profile.runner_context
# if we don't want all the rules, then just make 1 pass to get all rule_IDs
# that we want to keep from the original
filter_included_controls(context, dep_entry.profile, &block) if !opts[:include_all]
# interpret the block and skip/modify as required
context.load(block) if block_given?
bind_context.add_subcontext(context)
end
def self.filter_included_controls(context, profile, &block)
mock = Inspec::Backend.create({ backend: 'mock' })
include_ctx = Inspec::ProfileContext.for_profile(profile, mock)
include_ctx = profile.runner_context
include_ctx.load(block) if block_given?
# remove all rules that were not registered
context.rules.keys.each do |id|

87
lib/inspec/profile.rb
View file

@ -8,6 +8,9 @@ require 'inspec/polyfill'
require 'inspec/fetcher'
require 'inspec/source_reader'
require 'inspec/metadata'
require 'inspec/backend'
require 'inspec/rule'
require 'inspec/profile_context'
require 'inspec/dependencies/lockfile'
require 'inspec/dependencies/dependency_set'
@ -31,7 +34,7 @@ module Inspec
reader
end
def self.for_target(target, opts)
def self.for_target(target, opts = {})
new(resolve_target(target), opts.merge(target: target))
end
@ -47,9 +50,14 @@ module Inspec
@target = @options.delete(:target)
@logger = @options[:logger] || Logger.new(nil)
@source_reader = source_reader
if options[:dependencies]
@locked_dependencies = options[:dependencies]
end
@controls = options[:controls] || []
@profile_id = @options[:id]
@runner_context = nil
@backend = @options[:backend] || Inspec::Backend.create(options)
Metadata.finalize(@source_reader.metadata, @profile_id)
@runner_context = @options[:profile_context] || Inspec::ProfileContext.for_profile(self, @backend)
end
def name
@ -60,6 +68,46 @@ module Inspec
@params ||= load_params
end
def collect_tests(include_list = @controls)
if !@tests_collected
locked_dependencies.each(&:collect_tests)
tests.each do |path, content|
next if content.nil? || content.empty?
abs_path = source_reader.target.abs_path(path)
@runner_context.load(content, abs_path, nil)
end
@tests_collected = true
end
filter_controls(@runner_context.all_rules, include_list)
end
def filter_controls(controls_array, include_list)
return controls_array if include_list.nil? || include_list.empty?
controls_array.select do |c|
id = ::Inspec::Rule.rule_id(c)
include_list.include?(id)
end
end
def load_libraries
locked_dependencies.each do |d|
c = d.load_libraries
@runner_context.add_resources(c)
end
libs = libraries.map do |path, content|
[content, path]
end
@runner_context.load_libraries(libs)
@runner_context
end
def to_s
"Inspec::Profile<#{name}>"
end
def info
res = params.dup
# add information about the controls
@ -248,11 +296,15 @@ module Inspec
# @return [Inspec::Lockfile]
#
def generate_lockfile(vendor_path = nil)
res = Inspec::DependencySet.new(cwd, vendor_path)
res = Inspec::DependencySet.new(cwd, vendor_path, nil, @backend)
res.vendor(metadata.dependencies)
Inspec::Lockfile.from_dependency_set(res)
end
def load_dependencies
Inspec::DependencySet.from_lockfile(lockfile, cwd, nil, @backend)
end
private
# Create an archive name for this profile and an additional options
@ -281,34 +333,17 @@ module Inspec
params
end
#
# Returns a new runner for the current profile.
#
# @params [Symbol] The type of backend to use when constructing a
# new runner.
# @returns [Inspec::Runner]
#
def runner_for_profile(backend = :mock)
opts = @options.dup
opts[:ignore_supports] = true
r = Runner.new(id: @profile_id,
backend: backend,
test_collector: opts.delete(:test_collector))
r.add_profile(self, opts)
r
end
def load_checks_params(params)
load_libraries
tests = collect_tests
params[:controls] = controls = {}
params[:groups] = groups = {}
prefix = @source_reader.target.prefix || ''
# Load from the runner_context if it exists
runner = @runner_context || runner_for_profile
runner.all_rules.each do |rule|
tests.each do |rule|
f = load_rule_filepath(prefix, rule)
load_rule(rule, f, controls, groups)
end
params[:attributes] = runner.attributes
params[:attributes] = @runner_context.attributes
params
end
@ -337,9 +372,5 @@ module Inspec
}
groups[file][:controls].push(id)
end
def load_dependencies
Inspec::DependencySet.from_lockfile(lockfile, cwd, nil)
end
end
end

51
lib/inspec/profile_context.rb
View file

@ -12,8 +12,7 @@ require 'inspec/objects/attribute'
module Inspec
class ProfileContext # rubocop:disable Metrics/ClassLength
def self.for_profile(profile, backend)
c = new(profile.name, backend, { 'profile' => profile })
c.load_profile_content(profile)
new(profile.name, backend, { 'profile' => profile })
end
attr_reader :attributes, :rules, :profile_id, :resource_registry
@ -28,8 +27,6 @@ module Inspec
@conf = conf.dup
@rules = {}
@subcontexts = []
@dependencies = {}
@dependencies = conf['profile'].locked_dependencies unless conf['profile'].nil?
@require_loader = ::Inspec::RequireLoader.new
@attributes = []
# A local resource registry that only contains resources defined
@ -37,6 +34,14 @@ module Inspec
@resource_registry = Inspec::Resource.new_registry
end
def dependencies
if @conf['profile'].nil?
{}
else
@conf['profile'].locked_dependencies
end
end
def to_resources_dsl
Inspec::Resource.create_dsl(@backend, @resource_registry)
end
@ -45,7 +50,7 @@ module Inspec
@profile_execution_context ||= begin
resources_dsl = to_resources_dsl
ctx = create_context(resources_dsl)
ctx.new(@backend, @conf, @dependencies, @require_loader)
ctx.new(@backend, @conf, dependencies, @require_loader)
end
end
@ -59,35 +64,16 @@ module Inspec
ret
end
def add_subcontext(context)
Inspec::Log.debug("Adding subcontext #{context} to #{self}")
def add_resources(context)
@resource_registry.merge!(context.resource_registry)
profile_execution_context.add_resources(context)
reload_dsl
end
def add_subcontext(context)
@subcontexts << context
end
def load_profile_content(profile)
# load_attributes_from_profile(profile)
load_libraries_from_profile(profile)
load_tests_from_profile(profile)
self
end
def load_tests_from_profile(profile)
profile.tests.each do |path, content|
next if content.nil? || content.empty?
abs_path = profile.source_reader.target.abs_path(path)
load(content, abs_path, nil)
end
end
def load_libraries_from_profile(profile)
libs = profile.libraries.map do |path, content|
[content, path]
end
load_libraries(libs)
end
def load_libraries(libs)
lib_prefix = 'libraries' + File::SEPARATOR
autoloads = []
@ -249,14 +235,17 @@ module Inspec
res
end
define_method :add_subcontext do |context|
Inspec::Log.debug("Adding resource_dsl from #{context} with registry #{context.resource_registry.keys}")
define_method :add_resources do |context|
self.class.class_eval do
include context.to_resources_dsl
end
rule_class.class_eval do
include context.to_resources_dsl
end
end
define_method :add_subcontext do |context|
profile_context_owner.add_subcontext(context)
end

4
lib/inspec/rule.rb
View file

@ -33,6 +33,10 @@ module Inspec
instance_eval(&block) if block_given?
end
def to_s
Inspec::Rule.rule_id(self)
end
def id(*_)
# never overwrite the ID
@id

87
lib/inspec/runner.rb
View file

@ -30,11 +30,18 @@ module Inspec
#
class Runner # rubocop:disable Metrics/ClassLength
extend Forwardable
def_delegator :@test_collector, :report
def_delegator :@test_collector, :reset
attr_reader :backend, :rules, :attributes
def initialize(conf = {})
@rules = []
@conf = conf.dup
@conf[:logger] ||= Logger.new(nil)
@target_profiles = []
@controls = @conf[:controls] || []
@ignore_supports = @conf[:ignore_supports]
@test_collector = @conf.delete(:test_collector) || begin
require 'inspec/runner_rspec'
@ -57,6 +64,26 @@ module Inspec
@test_collector.backend = @backend
end
def run(with = nil)
Inspec::Log.debug "Starting run with targets: #{@target_profiles.map(&:to_s)}"
Inspec::Log.debug "Backend is #{@backend}"
all_controls = []
@target_profiles.each do |profile|
@test_collector.add_profile(profile)
profile.locked_dependencies
profile.load_libraries
@attributes |= profile.runner_context.attributes
all_controls += profile.collect_tests
end
all_controls.each do |rule|
register_rule(rule)
end
@test_collector.run(with)
end
# determine all attributes before the execution, fetch data from secrets backend
def load_attributes(options)
attributes = {}
@ -74,8 +101,7 @@ module Inspec
#
# add_target allows the user to add a target whose tests will be
# run when the user calls the run method. This is the main entry
# point to profile loading.
# run when the user calls the run method.
#
# A target is a path or URL that points to a profile. Using this
# target we generate a Profile and a ProfileContext. The content
@ -94,49 +120,15 @@ module Inspec
# TODO: Deduplicate/clarify the loading code that exists in here,
# the ProfileContext, the Profile, and Inspec::DSL
#
# TODO: Libraries of dependent profiles should be loaded even when
# include_content/required_content aren't called.
#
# @params target [String] A path or URL to a profile or raw test.
#
# @params options [Hash] An options hash. The relevant options
# considered by this call path are:
#
# - `:ignore_supports`: A boolean that controls whether to ignore
# the profile's metadata regarding supported Inspec versions or
# platforms.
#
# - `:controls`: An array of controls to run from this
# profile. Any returned rules that are not part of these
# controls will be filtered before being passed to @test_collector.
# @params _opts [Hash] Unused, but still here to avoid breaking kitchen-inspec
#
# @eturns [Inspec::ProfileContext]
#
def add_target(target, options = {})
profile = Inspec::Profile.for_target(target, options)
def add_target(target, _opts = [])
profile = Inspec::Profile.for_target(target, backend: @backend, controls: @controls)
fail "Could not resolve #{target} to valid input." if profile.nil?
add_profile(profile, options)
end
# Returns the profile context used to initialize this profile.
def add_profile(profile, options = {})
return if !options[:ignore_supports] && !supports_profile?(profile)
@test_collector.add_profile(profile)
add_content_from_profile(profile, options[:controls])
end
def add_content_from_profile(profile, controls)
return if profile.tests.nil? || profile.tests.empty?
ctx = Inspec::ProfileContext.for_profile(profile, @backend)
profile.runner_context = ctx
@attributes |= ctx.attributes
filter_controls(ctx.all_rules, controls).each do |rule|
register_rule(rule)
end
ctx
@target_profiles << profile if supports_profile?(profile)
end
#
@ -154,7 +146,7 @@ module Inspec
end
def supports_profile?(profile)
return true if profile.metadata.nil?
return true if profile.metadata.nil? || @ignore_supports
if !profile.metadata.supports_runtime?
fail 'This profile requires InSpec version '\
@ -187,20 +179,8 @@ module Inspec
new_tests
end
def_delegator :@test_collector, :run
def_delegator :@test_collector, :report
def_delegator :@test_collector, :reset
private
def filter_controls(controls_array, include_list)
return controls_array if include_list.nil? || include_list.empty?
controls_array.select do |c|
id = ::Inspec::Rule.rule_id(c)
include_list.include?(id)
end
end
def block_source_info(block)
return {} if block.nil? || !block.respond_to?(:source_location)
opts = {}
@ -245,6 +225,7 @@ module Inspec
end
def register_rule(rule)
Inspec::Log.debug "Registering rule #{rule}"
@rules << rule
checks = ::Inspec::Rule.prepare_checks(rule)
examples = checks.map do |m, a, b|

2
test/docker_test.rb
View file

@ -50,7 +50,7 @@ class DockerTester
puts "--> run test on docker #{container.id}"
opts = { 'target' => "docker://#{container.id}" }
runner = Inspec::Runner.new(opts)
@tests.each { |test| runner.add_target(test, opts) }
@tests.each { |test| runner.add_target(test) }
runner.tests.map { |g| g.run(report) }
end
end

1
test/helper.rb
View file

@ -292,6 +292,7 @@ class MockLoader
def self.load_profile(name, opts = {})
opts[:test_collector] = Inspec::RunnerMock.new
opts[:backend] = Inspec::Backend.create(opts)
Inspec::Profile.for_target(profile_path(name), opts)
end

2
test/unit/dsl/control_test.rb
View file

@ -10,7 +10,7 @@ describe 'controls' do
'inspec.yml' => "name: mock",
'controls/mock.rb' => "control '1' do\n#{content}\nend\n",
}
opts = { test_collector: Inspec::RunnerMock.new }
opts = { test_collector: Inspec::RunnerMock.new, backend: Inspec::Backend.create({}) }
Inspec::Profile.for_target(data, opts)
.params[:controls]['1']
end

2
test/unit/mock/profiles/dependencies/profile_b/controls/example.rb
View file

@ -17,6 +17,6 @@ end
control 'profileb-2' do
describe gordon_config do
its('version') { should eq('1.0') }
its('version') { should eq('2.0') }
end
end

45
test/unit/mock/profiles/dependencies/profile_c/libraries/gordon_config.rb
View file

@ -1,54 +1,15 @@
require 'yaml'
puts "Loading gordon_config from c"
# Custom resource based on the InSpec resource DSL
class GordonConfig < Inspec.resource(1)
name 'gordon_config'
desc "
Gordon's resource description ...
"
desc "Gordon's resource description ..."
example "
describe gordon_config do
its('version') { should eq('1.0') }
its('file_size') { should > 1 }
end
"
# Load the configuration file on initialization
def initialize
@params = {}
@path = '/tmp/gordon/config.yaml'
@file = inspec.file(@path)
return skip_resource "Can't find file \"#{@path}\"" if !@file.file?
# Protect from invalid YAML content
begin
@params = YAML.load(@file.content)
# Add two extra matchers
@params['file_size'] = @file.size
@params['file_path'] = @path
@params['ruby'] = 'RUBY IS HERE TO HELP ME!'
rescue Exception
return skip_resource "#{@file}: #{$!}"
end
end
# Example method called by 'it { should exist }'
# Returns true or false from the 'File.exists?' method
def exists?
return File.exists?(@path)
end
# Example matcher for the number of commas in the file
def comma_count
text = @file.content
return text.count(',')
end
# Expose all parameters
def method_missing(name)
return @params[name.to_s]
def version
"1.0"
end
end

47
test/unit/mock/profiles/dependencies/profile_d/libraries/gordon_config.rb
View file

@ -1,54 +1,15 @@
require 'yaml'
puts "Loading gordon_config from d"
# Custom resource based on the InSpec resource DSL
class GordonConfig < Inspec.resource(1)
name 'gordon_config'
desc "
Gordon's resource description ...
"
desc "Gordon's resource description ..."
example "
describe gordon_config do
its('version') { should eq('1.0') }
its('file_size') { should > 1 }
its('version') { should eq('2.0') }
end
"
# Load the configuration file on initialization
def initialize
@params = {}
@path = '/tmp/gordon/config2.yaml'
@file = inspec.file(@path)
return skip_resource "Can't find file \"#{@path}\"" if !@file.file?
# Protect from invalid YAML content
begin
@params = YAML.load(@file.content)
# Add two extra matchers
@params['file_size'] = @file.size
@params['file_path'] = @path
@params['ruby'] = 'RUBY IS HERE TO HELP ME!'
rescue Exception
return skip_resource "#{@file}: #{$!}"
end
end
# Example method called by 'it { should exist }'
# Returns true or false from the 'File.exists?' method
def exists?
return File.exists?(@path)
end
# Example matcher for the number of commas in the file
def comma_count
text = @file.content
return text.count(',')
end
# Expose all parameters
def method_missing(name)
return @params[name.to_s]
def version
"2.0"
end
end