inspec --format [json|fulljson|rspecjson] overhaul

Full rewrite of all formatters. Create a minimal JSON, a full JSON, and a fallback RSpec formatter. The latter is only needed for corner cases and should not really be used. The former 2 are for (1) running `inspec json` followed by `inspec exec` (`--format json`) and (2) running just `inspec exec --format fulljson`.
This commit is contained in:
Dominik Richter 2016-04-09 20:10:04 +02:00 committed by Christoph Hartmann
parent a809097d12
commit 20d08a63b5
8 changed files with 243 additions and 154 deletions

View file

@ -55,24 +55,16 @@ module Inspec
def info
res = params.dup
rules = {}
res[:rules].each do |gid, group|
next if gid.to_s.empty?
rules[gid] = { title: gid, rules: {} }
group.each do |id, rule|
next if id.to_s.empty?
data = rule.dup
data.delete(:checks)
data[:impact] ||= 0.5
data[:impact] = 1.0 if data[:impact] > 1.0
data[:impact] = 0.0 if data[:impact] < 0.0
rules[gid][:rules][id] = data
# TODO: temporarily flatten the group down; replace this with
# proper hierarchy later on
rules[gid][:title] = data[:group_title]
end
controls = res[:controls].map do |id, rule|
next if id.to_s.empty?
data = rule.dup
data.delete(:checks)
data[:impact] ||= 0.5
data[:impact] = 1.0 if data[:impact] > 1.0
data[:impact] = 0.0 if data[:impact] < 0.0
[id, data]
end
res[:rules] = rules
res[:controls] = Hash[controls.compact]
res
end
@ -137,7 +129,7 @@ module Inspec
warn.call(@target, 0, 0, nil, 'Profile uses deprecated `test` directory, rename it to `controls`.')
end
count = rules_count
count = controls_count
result[:summary][:controls] = count
if count == 0
warn.call(nil, nil, nil, nil, 'No controls or tests were defined.')
@ -146,18 +138,15 @@ module Inspec
end
# iterate over hash of groups
params[:rules].each { |group, controls|
@logger.info "Verify all controls in #{group}"
controls.each { |id, control|
sfile, sline = control[:source_location]
error.call(sfile, sline, nil, id, 'Avoid controls with empty IDs') if id.nil? or id.empty?
next if id.start_with? '(generated '
warn.call(sfile, sline, nil, id, "Control #{id} has no title") if control[:title].to_s.empty?
warn.call(sfile, sline, nil, id, "Control #{id} has no description") if control[:desc].to_s.empty?
warn.call(sfile, sline, nil, id, "Control #{id} has impact > 1.0") if control[:impact].to_f > 1.0
warn.call(sfile, sline, nil, id, "Control #{id} has impact < 0.0") if control[:impact].to_f < 0.0
warn.call(sfile, sline, nil, id, "Control #{id} has no tests defined") if control[:checks].nil? or control[:checks].empty?
}
params[:controls].each { |id, control|
sfile, sline = control[:source_location]
error.call(sfile, sline, nil, id, 'Avoid controls with empty IDs') if id.nil? or id.empty?
next if id.start_with? '(generated '
warn.call(sfile, sline, nil, id, "Control #{id} has no title") if control[:title].to_s.empty?
warn.call(sfile, sline, nil, id, "Control #{id} has no description") if control[:desc].to_s.empty?
warn.call(sfile, sline, nil, id, "Control #{id} has impact > 1.0") if control[:impact].to_f > 1.0
warn.call(sfile, sline, nil, id, "Control #{id} has impact < 0.0") if control[:impact].to_f < 0.0
warn.call(sfile, sline, nil, id, "Control #{id} has no tests defined") if control[:checks].nil? or control[:checks].empty?
}
# profile is valid if we could not find any error
@ -167,8 +156,8 @@ module Inspec
result
end
def rules_count
params[:rules].values.map { |hm| hm.values.length }.inject(:+) || 0
def controls_count
params[:controls].values.length
end
# generates a archive of a folder profile
@ -233,7 +222,8 @@ module Inspec
def load_params
params = @source_reader.metadata.params
params[:name] = @profile_id unless @profile_id.nil?
params[:rules] = rules = {}
params[:controls] = controls = {}
params[:groups] = groups = {}
prefix = @source_reader.target.prefix || ''
# we're checking a profile, we don't care if it runs on the host machine
@ -246,25 +236,39 @@ module Inspec
)
runner.add_profile(self, opts)
runner.rules.each do |id, rule|
file = rule.instance_variable_get(:@__file)
file = file[prefix.length..-1] if file.start_with?(prefix)
rules[file] ||= {}
rules[file][id] = {
title: rule.title,
desc: rule.desc,
impact: rule.impact,
refs: rule.ref,
tags: rule.tag,
checks: Inspec::Rule.checks(rule),
code: rule.instance_variable_get(:@__code),
source_location: rule.instance_variable_get(:@__source_location),
group_title: rule.instance_variable_get(:@__group_title),
}
runner.rules.values.each do |rule|
f = load_rule_filepath(prefix, rule)
load_rule(rule, f, controls, groups)
end
@profile_id ||= params[:name]
params
end
def load_rule_filepath(prefix, rule)
file = rule.instance_variable_get(:@__file)
file = file[prefix.length..-1] if file.start_with?(prefix)
file
end
def load_rule(rule, file, controls, groups)
id = Inspec::Rule.rule_id(rule)
controls[id] = {
title: rule.title,
desc: rule.desc,
impact: rule.impact,
refs: rule.ref,
tags: rule.tag,
checks: Inspec::Rule.checks(rule),
code: rule.instance_variable_get(:@__code),
source_location: rule.instance_variable_get(:@__source_location),
}
groups[file] ||= {
title: rule.instance_variable_get(:@__group_title),
controls: [],
}
groups[file][:controls].push(id)
end
end
end

View file

@ -5,43 +5,46 @@
require 'rspec/core'
require 'rspec/core/formatters/json_formatter'
# Extend the basic RSpec JSON Formatter
# to give us an ID in its output
# TODO: remove once RSpec has IDs in stable (probably v3.3/v4.0)
module RSpec::Core::Formatters
class JsonFormatter
private
# Vanilla RSpec JSON formatter with a slight extension to show example IDs.
# TODO: Remove these lines when RSpec includes the ID natively
class InspecRspecVanilla < RSpec::Core::Formatters::JsonFormatter
RSpec::Core::Formatters.register self, :message, :dump_summary, :dump_profile, :stop, :close
def format_example(example)
{
description: example.description,
full_description: example.full_description,
status: example.execution_result.status.to_s,
file_path: example.metadata['file_path'],
line_number: example.metadata['line_number'],
run_time: example.execution_result.run_time,
pending_message: example.execution_result.pending_message,
id: example.metadata[:id],
impact: example.metadata[:impact],
}
end
private
def format_example(example)
res = super(example)
res[:id] = example.metadata[:id]
res
end
end
class InspecRspecFormatter < RSpec::Core::Formatters::JsonFormatter
# Minimal JSON formatter for inspec. Only contains limited information about
# examples without any extras.
class InspecRspecJson < RSpec::Core::Formatters::JsonFormatter
RSpec::Core::Formatters.register self, :message, :dump_summary, :dump_profile, :stop, :close
def add_profile(profile)
@profiles ||= []
@profiles.push(profile)
def dump_summary(summary)
@output_hash[:version] = Inspec::VERSION
@output_hash[:summary] = {
duration: summary.duration,
example_count: summary.example_count,
failure_count: summary.failure_count,
skip_count: summary.pending_count,
}
end
def dump_summary(summary)
super(summary)
@output_hash[:profiles] = Array(@profiles).map do |profile|
r = profile.params.dup
r.delete(:rules)
r
def stop(notification)
@output_hash[:controls] = notification.examples.map do |example|
format_example(example).tap do |hash|
e = example.exception
next unless e
hash[:message] = e.message
next if e.is_a? RSpec::Expectations::ExpectationNotMetError
hash[:exception] = e.class.name
hash[:backtrace] = e.backtrace
end
end
end
@ -50,21 +53,70 @@ class InspecRspecFormatter < RSpec::Core::Formatters::JsonFormatter
def format_example(example)
res = {
id: example.metadata[:id],
title: example.metadata[:title],
desc: example.metadata[:desc],
code: example.metadata[:code],
impact: example.metadata[:impact],
status: example.execution_result.status.to_s,
code_desc: example.full_description,
ref: example.metadata['file_path'],
ref_line: example.metadata['line_number'],
run_time: example.execution_result.run_time,
start_time: example.execution_result.started_at.to_s,
}
# pending messages are embedded in the resources description
res[:pending] = example.metadata[:description] if res[:status] == 'pending'
unless (pid = example.metadata[:profile_id]).nil?
res[:profile_id] = pid
end
if res[:status] == 'pending'
res[:status] = 'skipped'
res[:skip_message] = example.metadata[:description]
res[:resource] = example.metadata[:described_class].to_s
end
res
end
end
class InspecRspecFullJson < InspecRspecJson
RSpec::Core::Formatters.register self, :message, :dump_summary, :dump_profile, :stop, :close
def add_profile(profile)
@profiles ||= []
@profiles.push(profile)
end
def dump_one_example(example, profiles, missing)
profile = profiles[example[:profile_id]]
return missing.push(example) if profile.nil? || profile[:controls].nil?
control = profile[:controls][example[:id]]
return missing.push(example) if control.nil?
control[:results] ||= []
example.delete(:id)
example.delete(:profile_id)
control[:results].push(example)
end
def profile_info(profile)
info = profile.info.dup
[info[:name], info]
end
def dump_summary(summary)
super(summary)
@profiles ||= []
examples = @output_hash.delete(:controls)
profiles = Hash[@profiles.map { |x| profile_info(x) }]
missing = []
examples.each do |example|
dump_one_example(example, profiles, missing)
end
@output_hash[:profiles] = profiles
end
private
def format_example(example)
super(example).tap do |res|
res[:run_time] = example.execution_result.run_time
res[:start_time] = example.execution_result.started_at.to_s
end
end
end

View file

@ -73,6 +73,7 @@ module Inspec
@test_collector.add_profile(profile)
options[:metadata] = profile.metadata
options[:profile] = profile
libs = profile.libraries.map do |k, v|
{ ref: k, content: v }
@ -126,7 +127,10 @@ module Inspec
def filter_controls(controls_map, include_list)
return controls_map if include_list.nil? || include_list.empty?
controls_map.select { |k, _| include_list.include?(k) }
controls_map.select do |_, c|
id = ::Inspec::Rule.rule_id(c)
include_list.include?(id)
end
end
def block_source_info(block)
@ -188,7 +192,7 @@ module Inspec
# scope.
dsl = Inspec::Resource.create_dsl(backend)
example.send(:include, dsl)
@test_collector.add_test(example, rule_id, rule)
@test_collector.add_test(example, rule)
end
end
end

View file

@ -14,7 +14,7 @@ module Inspec
@profiles.push(profile)
end
def add_test(example, _rule_id, _rule)
def add_test(example, _rule)
@tests.push(example)
end

View file

@ -33,7 +33,7 @@ module Inspec
# @return [nil]
def add_profile(profile)
RSpec.configuration.formatters
.find_all { |c| c.is_a? InspecRspecFormatter }
.find_all { |c| c.is_a? InspecRspecFullJson }
.each do |fmt|
fmt.add_profile(profile)
end
@ -44,8 +44,8 @@ module Inspec
# @param [RSpecExampleGroup] example test
# @param [String] rule_id the ID associated with this check
# @return [nil]
def add_test(example, rule_id, rule)
set_rspec_ids(example, rule_id, rule)
def add_test(example, rule)
set_rspec_ids(example, rule)
@tests.example_groups.push(example)
end
@ -81,6 +81,12 @@ module Inspec
RSpec.configuration.reset
end
FORMATTERS = {
'json' => 'InspecRspecJson',
'fulljson' => 'InspecRspecFullJson',
'rspecjson' => 'InspecRspecVanilla',
}.freeze
# Configure the output formatter and stream to be used with RSpec.
#
# @return [nil]
@ -91,8 +97,7 @@ module Inspec
RSpec.configuration.output_stream = @conf['output']
end
format = @conf['format'] || 'progress'
format = 'InspecRspecFormatter' if format == 'fulljson'
format = FORMATTERS[@conf['format']] || @conf['format'] || 'progress'
RSpec.configuration.add_formatter(format)
RSpec.configuration.color = @conf['color']
@ -109,27 +114,26 @@ module Inspec
# by the InSpec adjusted json formatter (rspec_json_formatter).
#
# @param [RSpecExampleGroup] example object which contains a check
# @param [Type] id describe id
# @return [Type] description of returned object
def set_rspec_ids(example, id, rule)
example.metadata[:id] = id
example.metadata[:impact] = rule.impact
example.metadata[:title] = rule.title
example.metadata[:desc] = rule.desc
example.metadata[:code] = rule.instance_variable_get(:@__code)
example.metadata[:source_location] = rule.instance_variable_get(:@__source_location)
def set_rspec_ids(example, rule)
assign_rspec_ids(example.metadata, rule)
example.filtered_examples.each do |e|
e.metadata[:id] = id
e.metadata[:impact] = rule.impact
e.metadata[:title] = rule.title
e.metadata[:desc] = rule.desc
e.metadata[:code] = rule.instance_variable_get(:@__code)
e.metadata[:source_location] = rule.instance_variable_get(:@__source_location)
assign_rspec_ids(e.metadata, rule)
end
example.children.each do |child|
set_rspec_ids(child, id, rule)
set_rspec_ids(child, rule)
end
end
def assign_rspec_ids(metadata, rule)
metadata[:id] = ::Inspec::Rule.rule_id(rule)
metadata[:profile_id] = ::Inspec::Rule.profile_id(rule)
metadata[:impact] = rule.impact
metadata[:title] = rule.title
metadata[:desc] = rule.desc
metadata[:code] = rule.instance_variable_get(:@__code)
metadata[:source_location] = rule.instance_variable_get(:@__source_location)
end
end
class RSpecReporter < RSpec::Core::Formatters::JsonFormatter

View file

@ -54,45 +54,63 @@ describe 'inspec exec' do
describe 'execute a profile with json formatting' do
let(:json) { JSON.load(inspec('exec ' + example_profile + ' --format json').stdout) }
let(:examples) { json['examples'] }
let(:ex1) { examples.find{|x| x['id'] == 'profile/tmp-1.0'} }
let(:ex2) { examples.find{|x| x['id'] =~ /generated/} }
let(:ex3) { examples.find{|x| x['id'] == 'profile/gordon-1.0'} }
let(:controls) { json['controls'] }
let(:ex1) { controls.find{|x| x['id'] == 'tmp-1.0'} }
let(:ex2) { controls.find{|x| x['id'] =~ /generated/} }
let(:ex3) { controls.find{|x| x['id'] == 'gordon-1.0'} }
it 'must have 5 examples' do
json['examples'].length.must_equal 5
json['controls'].length.must_equal 5
end
it 'id in json' do
examples.find { |ex| !ex.key? 'id' }.must_be :nil?
it 'has an id' do
controls.find { |ex| !ex.key? 'id' }.must_be :nil?
end
it 'impact in json' do
ex1['impact'].must_equal 0.7
ex2['impact'].must_be :nil?
it 'has a profile_id' do
controls.find { |ex| !ex.key? 'profile_id' }.must_be :nil?
end
it 'status in json' do
it 'has a code_desc' do
ex1['code_desc'].must_equal 'File /tmp should be directory'
controls.find { |ex| !ex.key? 'code_desc' }.must_be :nil?
end
it 'has a status' do
ex1['status'].must_equal 'passed'
ex3['status'].must_equal 'pending'
ex3['status'].must_equal 'skipped'
end
it 'pending message in json' do
ex1['pending_message'].must_be :nil?
ex3['pending_message'].must_equal 'Not yet implemented'
it 'has a skip_message' do
ex1['skip_message'].must_be :nil?
ex3['skip_message'].must_equal "Can't find file \"/tmp/gordon/config.yaml\""
end
end
it 'can execute the profile with the fulljson formatter' do
out = inspec('exec ' + example_profile + ' --format fulljson')
out.stderr.must_equal ''
out.exit_status.must_equal 0
JSON.load(out.stdout).must_be_kind_of Hash
end
describe 'execute a profile with fulljson formatting' do
let(:json) { JSON.load(inspec('exec ' + example_profile + ' --format fulljson').stdout) }
let(:examples) { json['examples'] }
let(:metadata) { json['profiles'][0] }
let(:ex1) { examples.find{|x| x['id'] == 'tmp-1.0'} }
let(:ex2) { examples.find{|x| x['id'] =~ /generated/} }
let(:ex3) { examples.find{|x| x['id'] == 'gordon-1.0'} }
let(:profile) { json['profiles']['profile'] }
let(:controls) { profile['controls'] }
let(:ex1) { controls['tmp-1.0'] }
let(:ex2) {
k = controls.keys.find { |x| x =~ /generated/ }
controls[k]
}
let(:ex3) { controls['gordon-1.0'] }
let(:check_result) { ex1['results'][0] }
it 'has all the metadata' do
metadata.must_equal({
controls = profile.delete('controls')
key = controls.keys.find { |x| x =~ /generated from example.rb/ }
profile.must_equal({
"name" => "profile",
"title" => "InSpec Example Profile",
"maintainer" => "Chef Software, Inc.",
@ -101,16 +119,21 @@ describe 'inspec exec' do
"license" => "Apache 2 license",
"summary" => "Demonstrates the use of InSpec Compliance Profile",
"version" => "1.0.0",
"supports" => [{"os-family" => "unix"}]
"supports" => [{"os-family" => "unix"}],
"groups" => {
"controls/meta.rb" => {"title"=>"SSH Server Configuration", "controls"=>["ssh-1"]},
"controls/example.rb" => {"title"=>"/tmp profile", "controls"=>["tmp-1.0", key]},
"controls/gordon.rb" => {"title"=>"Gordon Config Checks", "controls"=>["gordon-1.0"]},
},
})
end
it 'must have 5 examples' do
json['examples'].length.must_equal 5
it 'must have 4 controls' do
controls.length.must_equal 4
end
it 'id in json' do
examples.find { |ex| !ex.key? 'id' }.must_be :nil?
it 'has an id for every control' do
controls.keys.find(&:nil?).must_be :nil?
end
it 'title in json' do
@ -134,30 +157,35 @@ describe 'inspec exec' do
ex2['impact'].must_be :nil?
end
it 'status in json' do
ex1['status'].must_equal 'passed'
ex3['status'].must_equal 'pending'
it 'source location in json' do
ex1['source_location'][0].must_match %r{examples/profile/controls/example.rb$}
end
it 'ref in json' do
ex1['ref'].must_match %r{examples/profile/controls/example.rb$}
it 'source line in json' do
ex1['source_location'][1].must_equal 8
end
it 'ref_line in json' do
ex1['ref_line'].must_equal 16
it 'has all needed results' do
ex1['results'].length.must_equal 1
ex2['results'].length.must_equal 1
ex3['results'].length.must_equal 2
end
it 'run_time in json' do
ex1['run_time'].wont_be :nil?
it 'has a status in its check result' do
check_result['status'].must_equal 'passed'
end
it 'start_time in json' do
ex1['start_time'].wont_be :nil?
it 'has a code description in its check result' do
check_result['code_desc'].must_equal 'File /tmp should be directory'
end
it 'pending message in json' do
ex1['pending'].must_be :nil?
ex3['pending'].must_equal "Can't find file \"/tmp/gordon/config.yaml\""
it 'has a run_time in its check result' do
check_result['run_time'].must_be > 0
check_result['run_time'].must_be < 1
end
it 'has a start_time in its check result' do
check_result['start_time'].wont_be :nil?
end
end

View file

@ -12,7 +12,7 @@ describe 'controls' do
}
opts = { test_collector: Inspec::RunnerMock.new }
Inspec::Profile.for_target(data, opts)
.params[:rules].values[0]['1']
.params[:rules]['1']
end
it 'works with empty refs' do

View file

@ -183,7 +183,6 @@ describe Inspec::Profile do
logger.expect :info, nil, ["Checking profile in #{home}/mock/profiles/#{profile_id}"]
logger.expect :info, nil, ['Metadata OK.']
logger.expect :info, nil, ['Found 1 controls.']
logger.expect :info, nil, ["Verify all controls in controls/filesystem_spec.rb"]
logger.expect :info, nil, ['Control definitions OK.']
result = MockLoader.load_profile(profile_id, {logger: logger}).check
@ -209,7 +208,6 @@ describe Inspec::Profile do
logger.expect :info, nil, ["Checking profile in #{home}/mock/profiles/#{profile_id}"]
logger.expect :info, nil, ['Metadata OK.']
logger.expect :info, nil, ['Found 1 controls.']
logger.expect :info, nil, ["Verify all controls in controls/filesystem_spec.rb"]
logger.expect :info, nil, ['Control definitions OK.']
result = MockLoader.load_profile(profile_id, {logger: logger}).check
@ -235,7 +233,6 @@ describe Inspec::Profile do
logger.expect :info, nil, ["Checking profile in #{home}/mock/profiles/#{profile_id}"]
logger.expect :info, nil, ['Metadata OK.']
logger.expect :info, nil, ['Found 1 controls.']
logger.expect :info, nil, ["Verify all controls in controls/filesystem_spec.rb"]
logger.expect :info, nil, ['Control definitions OK.']
result = MockLoader.load_profile(profile_id, {logger: logger}).check