diff --git a/lib/inspec/profile.rb b/lib/inspec/profile.rb index 1cfa362f2..298ede7c1 100644 --- a/lib/inspec/profile.rb +++ b/lib/inspec/profile.rb @@ -55,24 +55,16 @@ module Inspec def info res = params.dup - rules = {} - res[:rules].each do |gid, group| - next if gid.to_s.empty? - rules[gid] = { title: gid, rules: {} } - group.each do |id, rule| - next if id.to_s.empty? - data = rule.dup - data.delete(:checks) - data[:impact] ||= 0.5 - data[:impact] = 1.0 if data[:impact] > 1.0 - data[:impact] = 0.0 if data[:impact] < 0.0 - rules[gid][:rules][id] = data - # TODO: temporarily flatten the group down; replace this with - # proper hierarchy later on - rules[gid][:title] = data[:group_title] - end + controls = res[:controls].map do |id, rule| + next if id.to_s.empty? + data = rule.dup + data.delete(:checks) + data[:impact] ||= 0.5 + data[:impact] = 1.0 if data[:impact] > 1.0 + data[:impact] = 0.0 if data[:impact] < 0.0 + [id, data] end - res[:rules] = rules + res[:controls] = Hash[controls.compact] res end @@ -137,7 +129,7 @@ module Inspec warn.call(@target, 0, 0, nil, 'Profile uses deprecated `test` directory, rename it to `controls`.') end - count = rules_count + count = controls_count result[:summary][:controls] = count if count == 0 warn.call(nil, nil, nil, nil, 'No controls or tests were defined.') @@ -146,18 +138,15 @@ module Inspec end # iterate over hash of groups - params[:rules].each { |group, controls| - @logger.info "Verify all controls in #{group}" - controls.each { |id, control| - sfile, sline = control[:source_location] - error.call(sfile, sline, nil, id, 'Avoid controls with empty IDs') if id.nil? or id.empty? - next if id.start_with? '(generated ' - warn.call(sfile, sline, nil, id, "Control #{id} has no title") if control[:title].to_s.empty? - warn.call(sfile, sline, nil, id, "Control #{id} has no description") if control[:desc].to_s.empty? - warn.call(sfile, sline, nil, id, "Control #{id} has impact > 1.0") if control[:impact].to_f > 1.0 - warn.call(sfile, sline, nil, id, "Control #{id} has impact < 0.0") if control[:impact].to_f < 0.0 - warn.call(sfile, sline, nil, id, "Control #{id} has no tests defined") if control[:checks].nil? or control[:checks].empty? - } + params[:controls].each { |id, control| + sfile, sline = control[:source_location] + error.call(sfile, sline, nil, id, 'Avoid controls with empty IDs') if id.nil? or id.empty? + next if id.start_with? '(generated ' + warn.call(sfile, sline, nil, id, "Control #{id} has no title") if control[:title].to_s.empty? + warn.call(sfile, sline, nil, id, "Control #{id} has no description") if control[:desc].to_s.empty? + warn.call(sfile, sline, nil, id, "Control #{id} has impact > 1.0") if control[:impact].to_f > 1.0 + warn.call(sfile, sline, nil, id, "Control #{id} has impact < 0.0") if control[:impact].to_f < 0.0 + warn.call(sfile, sline, nil, id, "Control #{id} has no tests defined") if control[:checks].nil? or control[:checks].empty? } # profile is valid if we could not find any error @@ -167,8 +156,8 @@ module Inspec result end - def rules_count - params[:rules].values.map { |hm| hm.values.length }.inject(:+) || 0 + def controls_count + params[:controls].values.length end # generates a archive of a folder profile @@ -233,7 +222,8 @@ module Inspec def load_params params = @source_reader.metadata.params params[:name] = @profile_id unless @profile_id.nil? - params[:rules] = rules = {} + params[:controls] = controls = {} + params[:groups] = groups = {} prefix = @source_reader.target.prefix || '' # we're checking a profile, we don't care if it runs on the host machine @@ -246,25 +236,39 @@ module Inspec ) runner.add_profile(self, opts) - runner.rules.each do |id, rule| - file = rule.instance_variable_get(:@__file) - file = file[prefix.length..-1] if file.start_with?(prefix) - rules[file] ||= {} - rules[file][id] = { - title: rule.title, - desc: rule.desc, - impact: rule.impact, - refs: rule.ref, - tags: rule.tag, - checks: Inspec::Rule.checks(rule), - code: rule.instance_variable_get(:@__code), - source_location: rule.instance_variable_get(:@__source_location), - group_title: rule.instance_variable_get(:@__group_title), - } + runner.rules.values.each do |rule| + f = load_rule_filepath(prefix, rule) + load_rule(rule, f, controls, groups) end @profile_id ||= params[:name] params end + + def load_rule_filepath(prefix, rule) + file = rule.instance_variable_get(:@__file) + file = file[prefix.length..-1] if file.start_with?(prefix) + file + end + + def load_rule(rule, file, controls, groups) + id = Inspec::Rule.rule_id(rule) + controls[id] = { + title: rule.title, + desc: rule.desc, + impact: rule.impact, + refs: rule.ref, + tags: rule.tag, + checks: Inspec::Rule.checks(rule), + code: rule.instance_variable_get(:@__code), + source_location: rule.instance_variable_get(:@__source_location), + } + + groups[file] ||= { + title: rule.instance_variable_get(:@__group_title), + controls: [], + } + groups[file][:controls].push(id) + end end end diff --git a/lib/inspec/rspec_json_formatter.rb b/lib/inspec/rspec_json_formatter.rb index 4df0d89ea..80d543ad5 100644 --- a/lib/inspec/rspec_json_formatter.rb +++ b/lib/inspec/rspec_json_formatter.rb @@ -5,43 +5,46 @@ require 'rspec/core' require 'rspec/core/formatters/json_formatter' -# Extend the basic RSpec JSON Formatter -# to give us an ID in its output -# TODO: remove once RSpec has IDs in stable (probably v3.3/v4.0) -module RSpec::Core::Formatters - class JsonFormatter - private +# Vanilla RSpec JSON formatter with a slight extension to show example IDs. +# TODO: Remove these lines when RSpec includes the ID natively +class InspecRspecVanilla < RSpec::Core::Formatters::JsonFormatter + RSpec::Core::Formatters.register self, :message, :dump_summary, :dump_profile, :stop, :close - def format_example(example) - { - description: example.description, - full_description: example.full_description, - status: example.execution_result.status.to_s, - file_path: example.metadata['file_path'], - line_number: example.metadata['line_number'], - run_time: example.execution_result.run_time, - pending_message: example.execution_result.pending_message, - id: example.metadata[:id], - impact: example.metadata[:impact], - } - end + private + + def format_example(example) + res = super(example) + res[:id] = example.metadata[:id] + res end end -class InspecRspecFormatter < RSpec::Core::Formatters::JsonFormatter +# Minimal JSON formatter for inspec. Only contains limited information about +# examples without any extras. +class InspecRspecJson < RSpec::Core::Formatters::JsonFormatter RSpec::Core::Formatters.register self, :message, :dump_summary, :dump_profile, :stop, :close - def add_profile(profile) - @profiles ||= [] - @profiles.push(profile) + def dump_summary(summary) + @output_hash[:version] = Inspec::VERSION + @output_hash[:summary] = { + duration: summary.duration, + example_count: summary.example_count, + failure_count: summary.failure_count, + skip_count: summary.pending_count, + } end - def dump_summary(summary) - super(summary) - @output_hash[:profiles] = Array(@profiles).map do |profile| - r = profile.params.dup - r.delete(:rules) - r + def stop(notification) + @output_hash[:controls] = notification.examples.map do |example| + format_example(example).tap do |hash| + e = example.exception + next unless e + hash[:message] = e.message + + next if e.is_a? RSpec::Expectations::ExpectationNotMetError + hash[:exception] = e.class.name + hash[:backtrace] = e.backtrace + end end end @@ -50,21 +53,70 @@ class InspecRspecFormatter < RSpec::Core::Formatters::JsonFormatter def format_example(example) res = { id: example.metadata[:id], - title: example.metadata[:title], - desc: example.metadata[:desc], - code: example.metadata[:code], - impact: example.metadata[:impact], status: example.execution_result.status.to_s, code_desc: example.full_description, - ref: example.metadata['file_path'], - ref_line: example.metadata['line_number'], - run_time: example.execution_result.run_time, - start_time: example.execution_result.started_at.to_s, } - # pending messages are embedded in the resources description - res[:pending] = example.metadata[:description] if res[:status] == 'pending' + unless (pid = example.metadata[:profile_id]).nil? + res[:profile_id] = pid + end + + if res[:status] == 'pending' + res[:status] = 'skipped' + res[:skip_message] = example.metadata[:description] + res[:resource] = example.metadata[:described_class].to_s + end res end end + +class InspecRspecFullJson < InspecRspecJson + RSpec::Core::Formatters.register self, :message, :dump_summary, :dump_profile, :stop, :close + + def add_profile(profile) + @profiles ||= [] + @profiles.push(profile) + end + + def dump_one_example(example, profiles, missing) + profile = profiles[example[:profile_id]] + return missing.push(example) if profile.nil? || profile[:controls].nil? + + control = profile[:controls][example[:id]] + return missing.push(example) if control.nil? + + control[:results] ||= [] + example.delete(:id) + example.delete(:profile_id) + control[:results].push(example) + end + + def profile_info(profile) + info = profile.info.dup + [info[:name], info] + end + + def dump_summary(summary) + super(summary) + @profiles ||= [] + examples = @output_hash.delete(:controls) + profiles = Hash[@profiles.map { |x| profile_info(x) }] + missing = [] + + examples.each do |example| + dump_one_example(example, profiles, missing) + end + + @output_hash[:profiles] = profiles + end + + private + + def format_example(example) + super(example).tap do |res| + res[:run_time] = example.execution_result.run_time + res[:start_time] = example.execution_result.started_at.to_s + end + end +end diff --git a/lib/inspec/runner.rb b/lib/inspec/runner.rb index a35936b79..576431f28 100644 --- a/lib/inspec/runner.rb +++ b/lib/inspec/runner.rb @@ -73,6 +73,7 @@ module Inspec @test_collector.add_profile(profile) options[:metadata] = profile.metadata + options[:profile] = profile libs = profile.libraries.map do |k, v| { ref: k, content: v } @@ -126,7 +127,10 @@ module Inspec def filter_controls(controls_map, include_list) return controls_map if include_list.nil? || include_list.empty? - controls_map.select { |k, _| include_list.include?(k) } + controls_map.select do |_, c| + id = ::Inspec::Rule.rule_id(c) + include_list.include?(id) + end end def block_source_info(block) @@ -188,7 +192,7 @@ module Inspec # scope. dsl = Inspec::Resource.create_dsl(backend) example.send(:include, dsl) - @test_collector.add_test(example, rule_id, rule) + @test_collector.add_test(example, rule) end end end diff --git a/lib/inspec/runner_mock.rb b/lib/inspec/runner_mock.rb index 482eb883b..7114b4670 100644 --- a/lib/inspec/runner_mock.rb +++ b/lib/inspec/runner_mock.rb @@ -14,7 +14,7 @@ module Inspec @profiles.push(profile) end - def add_test(example, _rule_id, _rule) + def add_test(example, _rule) @tests.push(example) end diff --git a/lib/inspec/runner_rspec.rb b/lib/inspec/runner_rspec.rb index 7a904da9b..14474c9c3 100644 --- a/lib/inspec/runner_rspec.rb +++ b/lib/inspec/runner_rspec.rb @@ -33,7 +33,7 @@ module Inspec # @return [nil] def add_profile(profile) RSpec.configuration.formatters - .find_all { |c| c.is_a? InspecRspecFormatter } + .find_all { |c| c.is_a? InspecRspecFullJson } .each do |fmt| fmt.add_profile(profile) end @@ -44,8 +44,8 @@ module Inspec # @param [RSpecExampleGroup] example test # @param [String] rule_id the ID associated with this check # @return [nil] - def add_test(example, rule_id, rule) - set_rspec_ids(example, rule_id, rule) + def add_test(example, rule) + set_rspec_ids(example, rule) @tests.example_groups.push(example) end @@ -81,6 +81,12 @@ module Inspec RSpec.configuration.reset end + FORMATTERS = { + 'json' => 'InspecRspecJson', + 'fulljson' => 'InspecRspecFullJson', + 'rspecjson' => 'InspecRspecVanilla', + }.freeze + # Configure the output formatter and stream to be used with RSpec. # # @return [nil] @@ -91,8 +97,7 @@ module Inspec RSpec.configuration.output_stream = @conf['output'] end - format = @conf['format'] || 'progress' - format = 'InspecRspecFormatter' if format == 'fulljson' + format = FORMATTERS[@conf['format']] || @conf['format'] || 'progress' RSpec.configuration.add_formatter(format) RSpec.configuration.color = @conf['color'] @@ -109,27 +114,26 @@ module Inspec # by the InSpec adjusted json formatter (rspec_json_formatter). # # @param [RSpecExampleGroup] example object which contains a check - # @param [Type] id describe id # @return [Type] description of returned object - def set_rspec_ids(example, id, rule) - example.metadata[:id] = id - example.metadata[:impact] = rule.impact - example.metadata[:title] = rule.title - example.metadata[:desc] = rule.desc - example.metadata[:code] = rule.instance_variable_get(:@__code) - example.metadata[:source_location] = rule.instance_variable_get(:@__source_location) + def set_rspec_ids(example, rule) + assign_rspec_ids(example.metadata, rule) example.filtered_examples.each do |e| - e.metadata[:id] = id - e.metadata[:impact] = rule.impact - e.metadata[:title] = rule.title - e.metadata[:desc] = rule.desc - e.metadata[:code] = rule.instance_variable_get(:@__code) - e.metadata[:source_location] = rule.instance_variable_get(:@__source_location) + assign_rspec_ids(e.metadata, rule) end example.children.each do |child| - set_rspec_ids(child, id, rule) + set_rspec_ids(child, rule) end end + + def assign_rspec_ids(metadata, rule) + metadata[:id] = ::Inspec::Rule.rule_id(rule) + metadata[:profile_id] = ::Inspec::Rule.profile_id(rule) + metadata[:impact] = rule.impact + metadata[:title] = rule.title + metadata[:desc] = rule.desc + metadata[:code] = rule.instance_variable_get(:@__code) + metadata[:source_location] = rule.instance_variable_get(:@__source_location) + end end class RSpecReporter < RSpec::Core::Formatters::JsonFormatter diff --git a/test/functional/inspec_exec_test.rb b/test/functional/inspec_exec_test.rb index 85164f3fb..cc0ed37cf 100644 --- a/test/functional/inspec_exec_test.rb +++ b/test/functional/inspec_exec_test.rb @@ -54,45 +54,63 @@ describe 'inspec exec' do describe 'execute a profile with json formatting' do let(:json) { JSON.load(inspec('exec ' + example_profile + ' --format json').stdout) } - let(:examples) { json['examples'] } - let(:ex1) { examples.find{|x| x['id'] == 'profile/tmp-1.0'} } - let(:ex2) { examples.find{|x| x['id'] =~ /generated/} } - let(:ex3) { examples.find{|x| x['id'] == 'profile/gordon-1.0'} } + let(:controls) { json['controls'] } + let(:ex1) { controls.find{|x| x['id'] == 'tmp-1.0'} } + let(:ex2) { controls.find{|x| x['id'] =~ /generated/} } + let(:ex3) { controls.find{|x| x['id'] == 'gordon-1.0'} } it 'must have 5 examples' do - json['examples'].length.must_equal 5 + json['controls'].length.must_equal 5 end - it 'id in json' do - examples.find { |ex| !ex.key? 'id' }.must_be :nil? + it 'has an id' do + controls.find { |ex| !ex.key? 'id' }.must_be :nil? end - it 'impact in json' do - ex1['impact'].must_equal 0.7 - ex2['impact'].must_be :nil? + it 'has a profile_id' do + controls.find { |ex| !ex.key? 'profile_id' }.must_be :nil? end - it 'status in json' do + it 'has a code_desc' do + ex1['code_desc'].must_equal 'File /tmp should be directory' + controls.find { |ex| !ex.key? 'code_desc' }.must_be :nil? + end + + it 'has a status' do ex1['status'].must_equal 'passed' - ex3['status'].must_equal 'pending' + ex3['status'].must_equal 'skipped' end - it 'pending message in json' do - ex1['pending_message'].must_be :nil? - ex3['pending_message'].must_equal 'Not yet implemented' + it 'has a skip_message' do + ex1['skip_message'].must_be :nil? + ex3['skip_message'].must_equal "Can't find file \"/tmp/gordon/config.yaml\"" end end + it 'can execute the profile with the fulljson formatter' do + out = inspec('exec ' + example_profile + ' --format fulljson') + out.stderr.must_equal '' + out.exit_status.must_equal 0 + JSON.load(out.stdout).must_be_kind_of Hash + end + describe 'execute a profile with fulljson formatting' do let(:json) { JSON.load(inspec('exec ' + example_profile + ' --format fulljson').stdout) } - let(:examples) { json['examples'] } - let(:metadata) { json['profiles'][0] } - let(:ex1) { examples.find{|x| x['id'] == 'tmp-1.0'} } - let(:ex2) { examples.find{|x| x['id'] =~ /generated/} } - let(:ex3) { examples.find{|x| x['id'] == 'gordon-1.0'} } + let(:profile) { json['profiles']['profile'] } + let(:controls) { profile['controls'] } + let(:ex1) { controls['tmp-1.0'] } + let(:ex2) { + k = controls.keys.find { |x| x =~ /generated/ } + controls[k] + } + let(:ex3) { controls['gordon-1.0'] } + let(:check_result) { ex1['results'][0] } it 'has all the metadata' do - metadata.must_equal({ + controls = profile.delete('controls') + key = controls.keys.find { |x| x =~ /generated from example.rb/ } + + profile.must_equal({ "name" => "profile", "title" => "InSpec Example Profile", "maintainer" => "Chef Software, Inc.", @@ -101,16 +119,21 @@ describe 'inspec exec' do "license" => "Apache 2 license", "summary" => "Demonstrates the use of InSpec Compliance Profile", "version" => "1.0.0", - "supports" => [{"os-family" => "unix"}] + "supports" => [{"os-family" => "unix"}], + "groups" => { + "controls/meta.rb" => {"title"=>"SSH Server Configuration", "controls"=>["ssh-1"]}, + "controls/example.rb" => {"title"=>"/tmp profile", "controls"=>["tmp-1.0", key]}, + "controls/gordon.rb" => {"title"=>"Gordon Config Checks", "controls"=>["gordon-1.0"]}, + }, }) end - it 'must have 5 examples' do - json['examples'].length.must_equal 5 + it 'must have 4 controls' do + controls.length.must_equal 4 end - it 'id in json' do - examples.find { |ex| !ex.key? 'id' }.must_be :nil? + it 'has an id for every control' do + controls.keys.find(&:nil?).must_be :nil? end it 'title in json' do @@ -134,30 +157,35 @@ describe 'inspec exec' do ex2['impact'].must_be :nil? end - it 'status in json' do - ex1['status'].must_equal 'passed' - ex3['status'].must_equal 'pending' + it 'source location in json' do + ex1['source_location'][0].must_match %r{examples/profile/controls/example.rb$} end - it 'ref in json' do - ex1['ref'].must_match %r{examples/profile/controls/example.rb$} + it 'source line in json' do + ex1['source_location'][1].must_equal 8 end - it 'ref_line in json' do - ex1['ref_line'].must_equal 16 + it 'has all needed results' do + ex1['results'].length.must_equal 1 + ex2['results'].length.must_equal 1 + ex3['results'].length.must_equal 2 end - it 'run_time in json' do - ex1['run_time'].wont_be :nil? + it 'has a status in its check result' do + check_result['status'].must_equal 'passed' end - it 'start_time in json' do - ex1['start_time'].wont_be :nil? + it 'has a code description in its check result' do + check_result['code_desc'].must_equal 'File /tmp should be directory' end - it 'pending message in json' do - ex1['pending'].must_be :nil? - ex3['pending'].must_equal "Can't find file \"/tmp/gordon/config.yaml\"" + it 'has a run_time in its check result' do + check_result['run_time'].must_be > 0 + check_result['run_time'].must_be < 1 + end + + it 'has a start_time in its check result' do + check_result['start_time'].wont_be :nil? end end diff --git a/test/unit/control_test.rb b/test/unit/control_test.rb index 7bd724f2c..2709cc158 100644 --- a/test/unit/control_test.rb +++ b/test/unit/control_test.rb @@ -12,7 +12,7 @@ describe 'controls' do } opts = { test_collector: Inspec::RunnerMock.new } Inspec::Profile.for_target(data, opts) - .params[:rules].values[0]['1'] + .params[:rules]['1'] end it 'works with empty refs' do diff --git a/test/unit/profile_test.rb b/test/unit/profile_test.rb index 74cb6c4a4..bdc8eddf2 100644 --- a/test/unit/profile_test.rb +++ b/test/unit/profile_test.rb @@ -183,7 +183,6 @@ describe Inspec::Profile do logger.expect :info, nil, ["Checking profile in #{home}/mock/profiles/#{profile_id}"] logger.expect :info, nil, ['Metadata OK.'] logger.expect :info, nil, ['Found 1 controls.'] - logger.expect :info, nil, ["Verify all controls in controls/filesystem_spec.rb"] logger.expect :info, nil, ['Control definitions OK.'] result = MockLoader.load_profile(profile_id, {logger: logger}).check @@ -209,7 +208,6 @@ describe Inspec::Profile do logger.expect :info, nil, ["Checking profile in #{home}/mock/profiles/#{profile_id}"] logger.expect :info, nil, ['Metadata OK.'] logger.expect :info, nil, ['Found 1 controls.'] - logger.expect :info, nil, ["Verify all controls in controls/filesystem_spec.rb"] logger.expect :info, nil, ['Control definitions OK.'] result = MockLoader.load_profile(profile_id, {logger: logger}).check @@ -235,7 +233,6 @@ describe Inspec::Profile do logger.expect :info, nil, ["Checking profile in #{home}/mock/profiles/#{profile_id}"] logger.expect :info, nil, ['Metadata OK.'] logger.expect :info, nil, ['Found 1 controls.'] - logger.expect :info, nil, ["Verify all controls in controls/filesystem_spec.rb"] logger.expect :info, nil, ['Control definitions OK.'] result = MockLoader.load_profile(profile_id, {logger: logger}).check