Merge pull request #671 from chef/dr/formatter-redesign

JSON formatter redesign

lib/inspec/cli.rb

@@ -15,8 +15,6 @@ class Inspec::InspecCLI < Inspec::BaseCLI # rubocop:disable Metrics/ClassLength
     desc: 'Show diagnostics (versions, configurations)'
 
   desc 'json PATH', 'read all tests in PATH and generate a JSON summary'
-  option :id, type: :string,
-    desc: 'Attach a profile ID to all test results'
   option :output, aliases: :o, type: :string,
     desc: 'Save the created profile to a path'
   option :controls, type: :array,

lib/inspec/dsl.rb

@@ -31,37 +31,6 @@ module Inspec::DSL
     end
   end
 
-  # Register a given rule with RSpec and
-  # let it run. This happens after everything
-  # else is merged in.
-  def self.execute_rule(r, profile_id)
-    checks = ::Inspec::Rule.prepare_checks(r)
-    fid = InspecBaseRule.full_id(r, profile_id)
-    checks.each do |m, a, b|
-      # check if the resource is skippable and skipped
-      cres = rule_from_check(m, a, b)
-      set_rspec_ids(cres, fid) if m == 'describe'
-    end
-  end
-
-  # merge two rules completely; all defined
-  # fields from src will be overwritten in dst
-  def self.merge_rules(dst, src)
-    InspecBaseRule.merge dst, src
-  end
-
-  # Attach an ID attribute to the
-  # metadata of all examples
-  # TODO: remove this once IDs are in rspec-core
-  def self.set_rspec_ids(obj, id)
-    obj.examples.each {|ex|
-      ex.metadata[:id] = id
-    }
-    obj.children.each {|c|
-      set_rspec_ids(c, id)
-    }
-  end
-
   def self.load_spec_files_for_profile(bind_context, opts, &block)
     # get all spec files
     target = get_reference_profile(opts[:profile_id], opts[:conf])
@@ -121,24 +90,3 @@ module Inspec::DSL
     ctx
   end
 end
-
-module Inspec::GlobalDSL
-  def __register_rule(r)
-    # make sure the profile id is attached to the rule
-    ::Inspec::DSL.execute_rule(r, __profile_id)
-  end
-
-  def __unregister_rule(_id)
-  end
-end
-
-module Inspec::DSLHelper
-  def self.bind_dsl(scope)
-    (class << scope; self; end).class_exec do
-      include Inspec::DSL
-      include Inspec::GlobalDSL
-    end
-  end
-end
-
-::Inspec::DSLHelper.bind_dsl(self)

lib/inspec/profile.rb

@@ -11,7 +11,6 @@ require 'inspec/metadata'
 module Inspec
   class Profile # rubocop:disable Metrics/ClassLength
     extend Forwardable
-    attr_reader :path
 
     def self.resolve_target(target, opts)
       # Fetchers retrieve file contents
@@ -35,6 +34,7 @@ module Inspec
     end
 
     attr_reader :source_reader
+    attr_accessor :runner_context
     def_delegator :@source_reader, :tests
     def_delegator :@source_reader, :libraries
     def_delegator :@source_reader, :metadata
@@ -46,6 +46,7 @@ module Inspec
       @logger = @options[:logger] || Logger.new(nil)
       @source_reader = source_reader
       @profile_id = @options[:id]
+      @runner_context = nil
       Metadata.finalize(@source_reader.metadata, @profile_id)
     end
 
@@ -55,24 +56,16 @@ module Inspec
 
     def info
       res = params.dup
-      rules = {}
+      controls = res[:controls].map do |id, rule|
-      res[:rules].each do |gid, group|
-        next if gid.to_s.empty?
-        rules[gid] = { title: gid, rules: {} }
-        group.each do |id, rule|
         next if id.to_s.empty?
         data = rule.dup
         data.delete(:checks)
         data[:impact] ||= 0.5
         data[:impact] = 1.0 if data[:impact] > 1.0
         data[:impact] = 0.0 if data[:impact] < 0.0
-          rules[gid][:rules][id] = data
+        [id, data]
-          # TODO: temporarily flatten the group down; replace this with
-          # proper hierarchy later on
-          rules[gid][:title] = data[:group_title]
       end
-      end
+      res[:controls] = Hash[controls.compact]
-      res[:rules] = rules
       res
     end
 
@@ -137,7 +130,7 @@ module Inspec
         warn.call(@target, 0, 0, nil, 'Profile uses deprecated `test` directory, rename it to `controls`.')
       end
 
-      count = rules_count
+      count = controls_count
       result[:summary][:controls] = count
       if count == 0
         warn.call(nil, nil, nil, nil, 'No controls or tests were defined.')
@@ -146,9 +139,7 @@ module Inspec
       end
 
       # iterate over hash of groups
-      params[:rules].each { |group, controls|
+      params[:controls].each { |id, control|
-        @logger.info "Verify all controls in #{group}"
-        controls.each { |id, control|
         sfile, sline = control[:source_location]
         error.call(sfile, sline, nil, id, 'Avoid controls with empty IDs') if id.nil? or id.empty?
         next if id.start_with? '(generated '
@@ -158,7 +149,6 @@ module Inspec
         warn.call(sfile, sline, nil, id, "Control #{id} has impact < 0.0") if control[:impact].to_f < 0.0
         warn.call(sfile, sline, nil, id, "Control #{id} has no tests defined") if control[:checks].nil? or control[:checks].empty?
       }
-      }
 
       # profile is valid if we could not find any error
       result[:summary][:valid] = result[:errors].empty?
@@ -167,8 +157,8 @@ module Inspec
       result
     end
 
-    def rules_count
+    def controls_count
-      params[:rules].values.map { |hm| hm.values.length }.inject(:+) || 0
+      params[:controls].values.length
     end
 
     # generates a archive of a folder profile
@@ -233,9 +223,17 @@ module Inspec
     def load_params
       params = @source_reader.metadata.params
       params[:name] = @profile_id unless @profile_id.nil?
-      params[:rules] = rules = {}
+      load_checks_params(params)
+      @profile_id ||= params[:name]
+      params
+    end
+
+    def load_checks_params(params)
+      params[:controls] = controls = {}
+      params[:groups] = groups = {}
       prefix = @source_reader.target.prefix || ''
 
+      if @runner_context.nil?
         # we're checking a profile, we don't care if it runs on the host machine
         opts = @options.dup
         opts[:ignore_supports] = true
@@ -245,12 +243,28 @@ module Inspec
           test_collector: opts.delete(:test_collector),
         )
         runner.add_profile(self, opts)
+        runner.rules.values.each do |rule|
+          f = load_rule_filepath(prefix, rule)
+          load_rule(rule, f, controls, groups)
+        end
+      else
+        # load from context
+        @runner_context.rules.values.each do |rule|
+          f = load_rule_filepath(prefix, rule)
+          load_rule(rule, f, controls, groups)
+        end
+      end
+    end
 
-      runner.rules.each do |id, rule|
+    def load_rule_filepath(prefix, rule)
       file = rule.instance_variable_get(:@__file)
       file = file[prefix.length..-1] if file.start_with?(prefix)
-        rules[file] ||= {}
+      file
-        rules[file][id] = {
+    end
+
+    def load_rule(rule, file, controls, groups)
+      id = Inspec::Rule.rule_id(rule)
+      controls[id] = {
         title: rule.title,
         desc: rule.desc,
         impact: rule.impact,
@@ -259,12 +273,13 @@ module Inspec
         checks: Inspec::Rule.checks(rule),
         code: rule.instance_variable_get(:@__code),
         source_location: rule.instance_variable_get(:@__source_location),
-          group_title: rule.instance_variable_get(:@__group_title),
       }
-      end
 
-      @profile_id ||= params[:name]
+      groups[file] ||= {
-      params
+        title: rule.instance_variable_get(:@__group_title),
+        controls: [],
+      }
+      groups[file][:controls].push(id)
     end
   end
 end

lib/inspec/profile_context.rb

@@ -41,24 +41,19 @@ module Inspec
     end
 
     def unregister_rule(id)
-      full_id = Inspec::Rule.full_id(@profile_id, id)
+      @rules.delete(full_id(@profile_id, id))
-      @rules[full_id] = nil
     end
 
     def register_rule(r)
       # get the full ID
       r.instance_variable_set(:@__file, @current_load[:file])
       r.instance_variable_set(:@__group_title, @current_load[:title])
-      full_id = Inspec::Rule.full_id(@profile_id, r)
-      if full_id.nil?
-        # TODO: error
-        return
-      end
 
       # add the rule to the registry
-      existing = @rules[full_id]
+      fid = full_id(Inspec::Rule.profile_id(r), Inspec::Rule.rule_id(r))
+      existing = @rules[fid]
       if existing.nil?
-        @rules[full_id] = r
+        @rules[fid] = r
       else
         Inspec::Rule.merge(existing, r)
       end
@@ -70,6 +65,11 @@ module Inspec
 
     private
 
+    def full_id(pid, rid)
+      return rid.to_s if pid.to_s.empty?
+      pid.to_s + '/' + rid.to_s
+    end
+
     # Create the context for controls. This includes all components of the DSL,
     # including matchers and resources.
     #
@@ -93,6 +93,7 @@ module Inspec
     # @return [ProfileContextClass]
     def create_context(resources_dsl, rule_class) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
       profile_context_owner = self
+      profile_id = @profile_id
 
       # rubocop:disable Lint/NestedMethodDefinition
       Class.new do
@@ -116,7 +117,7 @@ module Inspec
         define_method :control do |*args, &block|
           id = args[0]
           opts = args[1] || {}
-          register_control(rule_class.new(id, opts, &block))
+          register_control(rule_class.new(id, profile_id, opts, &block))
         end
 
         define_method :describe do |*args, &block|
@@ -124,7 +125,7 @@ module Inspec
           id = "(generated from #{loc} #{SecureRandom.hex})"
 
           res = nil
-          rule = rule_class.new(id, {}) do
+          rule = rule_class.new(id, profile_id, {}) do
             res = describe(*args, &block)
           end
           register_control(rule, &block)

lib/inspec/rspec_json_formatter.rb

@@ -5,43 +5,46 @@
 require 'rspec/core'
 require 'rspec/core/formatters/json_formatter'
 
-# Extend the basic RSpec JSON Formatter
+# Vanilla RSpec JSON formatter with a slight extension to show example IDs.
-# to give us an ID in its output
+# TODO: Remove these lines when RSpec includes the ID natively
-# TODO: remove once RSpec has IDs in stable (probably v3.3/v4.0)
+class InspecRspecVanilla < RSpec::Core::Formatters::JsonFormatter
-module RSpec::Core::Formatters
+  RSpec::Core::Formatters.register self, :message, :dump_summary, :dump_profile, :stop, :close
-  class JsonFormatter
+
   private
 
   def format_example(example)
-      {
+    res = super(example)
-        description: example.description,
+    res[:id] = example.metadata[:id]
-        full_description: example.full_description,
+    res
-        status: example.execution_result.status.to_s,
-        file_path: example.metadata['file_path'],
-        line_number: example.metadata['line_number'],
-        run_time: example.execution_result.run_time,
-        pending_message: example.execution_result.pending_message,
-        id: example.metadata[:id],
-        impact: example.metadata[:impact],
-      }
-    end
   end
 end
 
-class InspecRspecFormatter < RSpec::Core::Formatters::JsonFormatter
+# Minimal JSON formatter for inspec. Only contains limited information about
+# examples without any extras.
+class InspecRspecMiniJson < RSpec::Core::Formatters::JsonFormatter
   RSpec::Core::Formatters.register self, :message, :dump_summary, :dump_profile, :stop, :close
 
-  def add_profile(profile)
+  def dump_summary(summary)
-    @profiles ||= []
+    @output_hash[:version] = Inspec::VERSION
-    @profiles.push(profile)
+    @output_hash[:summary] = {
+      duration: summary.duration,
+      example_count: summary.example_count,
+      failure_count: summary.failure_count,
+      skip_count: summary.pending_count,
+    }
   end
 
-  def dump_summary(summary)
+  def stop(notification)
-    super(summary)
+    @output_hash[:controls] = notification.examples.map do |example|
-    @output_hash[:profiles] = Array(@profiles).map do |profile|
+      format_example(example).tap do |hash|
-      r = profile.params.dup
+        e = example.exception
-      r.delete(:rules)
+        next unless e
-      r
+        hash[:message] = e.message
+
+        next if e.is_a? RSpec::Expectations::ExpectationNotMetError
+        hash[:exception] = e.class.name
+        hash[:backtrace] = e.backtrace
+      end
     end
   end
 
@@ -50,21 +53,71 @@ class InspecRspecFormatter < RSpec::Core::Formatters::JsonFormatter
   def format_example(example)
     res = {
       id: example.metadata[:id],
-      title: example.metadata[:title],
-      desc: example.metadata[:desc],
-      code: example.metadata[:code],
-      impact: example.metadata[:impact],
       status: example.execution_result.status.to_s,
       code_desc: example.full_description,
-      ref: example.metadata['file_path'],
-      ref_line: example.metadata['line_number'],
-      run_time: example.execution_result.run_time,
-      start_time: example.execution_result.started_at.to_s,
     }
 
-    # pending messages are embedded in the resources description
+    unless (pid = example.metadata[:profile_id]).nil?
-    res[:pending] = example.metadata[:description] if res[:status] == 'pending'
+      res[:profile_id] = pid
+    end
+
+    if res[:status] == 'pending'
+      res[:status] = 'skipped'
+      res[:skip_message] = example.metadata[:description]
+      res[:resource] = example.metadata[:described_class].to_s
+    end
 
     res
   end
 end
+
+class InspecRspecJson < InspecRspecMiniJson
+  RSpec::Core::Formatters.register self, :message, :dump_summary, :dump_profile, :stop, :close
+
+  def add_profile(profile)
+    @profiles ||= []
+    @profiles.push(profile)
+  end
+
+  def dump_one_example(example, profiles, missing)
+    profile = profiles[example[:profile_id]]
+    return missing.push(example) if profile.nil? || profile[:controls].nil?
+
+    control = profile[:controls][example[:id]]
+    return missing.push(example) if control.nil?
+
+    control[:results] ||= []
+    example.delete(:id)
+    example.delete(:profile_id)
+    control[:results].push(example)
+  end
+
+  def profile_info(profile)
+    info = profile.info.dup
+    [info[:name], info]
+  end
+
+  def dump_summary(summary)
+    super(summary)
+    @profiles ||= []
+    examples = @output_hash.delete(:controls)
+    profiles = Hash[@profiles.map { |x| profile_info(x) }]
+    missing = []
+
+    examples.each do |example|
+      dump_one_example(example, profiles, missing)
+    end
+
+    @output_hash[:profiles] = profiles
+    @output_hash[:other_checks] = missing
+  end
+
+  private
+
+  def format_example(example)
+    super(example).tap do |res|
+      res[:run_time]   = example.execution_result.run_time
+      res[:start_time] = example.execution_result.started_at.to_s
+    end
+  end
+end

lib/inspec/rule.rb

@@ -12,8 +12,7 @@ module Inspec
   class Rule # rubocop:disable Metrics/ClassLength
     include ::RSpec::Matchers
 
-    def initialize(id, _opts, &block)
+    def initialize(id, profile_id, _opts, &block)
-      @id = id
       @impact = nil
       @title = nil
       @desc = nil
@@ -24,7 +23,8 @@ module Inspec
       @__block = block
       @__code = __get_block_source(&block)
       @__source_location = __get_block_source_location(&block)
-      @__rule_id = nil
+      @__rule_id = id
+      @__profile_id = profile_id
       @__checks = []
       @__skip_rule = nil
 
@@ -119,6 +119,10 @@ module Inspec
       rule.instance_variable_set(:@__rule_id, value)
     end
 
+    def self.profile_id(rule)
+      rule.instance_variable_get(:@__profile_id)
+    end
+
     def self.checks(rule)
       rule.instance_variable_get(:@__checks)
     end
@@ -167,32 +171,6 @@ module Inspec
       set_skip_rule(dst, sr) unless sr.nil?
     end
 
-    # Get the full id consisting of profile id + rule id
-    # for the rule. If the rule's profile id is empty,
-    # the given profile_id will be used instead and also
-    # set for the rule.
-    def self.full_id(profile_id, rule)
-      if rule.is_a?(String) or rule.nil?
-        rid = rule
-      else
-        # As the profile context is exclusively pulled with a
-        # profile ID, attach it to the rule if necessary.
-        rid = rule.instance_variable_get(:@id)
-        if rid.nil?
-          # TODO: Message about skipping this rule
-          # due to missing ID
-          return nil
-        end
-      end
-      pid = rule_id(rule)
-      pid = set_rule_id(rule, profile_id) if pid.nil?
-
-      # if we don't have a profile id, just return the rule's ID
-      return rid if pid.nil? or pid.empty?
-      # otherwise combine them
-      "#{pid}/#{rid}"
-    end
-
     private
 
     def __add_check(describe_or_expect, values, block)

lib/inspec/runner.rb

@@ -18,7 +18,6 @@ module Inspec
     attr_reader :backend, :rules
     def initialize(conf = {})
       @rules = {}
-      @profile_id = conf[:id]
       @conf = conf.dup
       @conf[:logger] ||= Logger.new(nil)
 
@@ -74,6 +73,7 @@ module Inspec
 
       @test_collector.add_profile(profile)
       options[:metadata] = profile.metadata
+      options[:profile] = profile
 
       libs = profile.libraries.map do |k, v|
         { ref: k, content: v }
@@ -88,7 +88,10 @@ module Inspec
     end
 
     def create_context(options = {})
-      Inspec::ProfileContext.new(@profile_id, @backend, @conf.merge(options))
+      meta = options['metadata']
+      profile_id = nil
+      profile_id = meta.params[:name] unless meta.nil?
+      Inspec::ProfileContext.new(profile_id, @backend, @conf.merge(options))
     end
 
     def add_content(tests, libs, options = {})
@@ -101,6 +104,11 @@ module Inspec
         ctx.reload_dsl
       end
 
+      # hand the context to the profile for further evaluation
+      unless (profile = options['profile']).nil?
+        profile.runner_context = ctx
+      end
+
       # evaluate the test content
       tests = [tests] unless tests.is_a? Array
       tests.each { |t| add_test_to_context(t, ctx) }
@@ -124,7 +132,10 @@ module Inspec
 
     def filter_controls(controls_map, include_list)
       return controls_map if include_list.nil? || include_list.empty?
-      controls_map.select { |k, _| include_list.include?(k) }
+      controls_map.select do |_, c|
+        id = ::Inspec::Rule.rule_id(c)
+        include_list.include?(id)
+      end
     end
 
     def block_source_info(block)
@@ -186,7 +197,7 @@ module Inspec
         # scope.
         dsl = Inspec::Resource.create_dsl(backend)
         example.send(:include, dsl)
-        @test_collector.add_test(example, rule_id, rule)
+        @test_collector.add_test(example, rule)
       end
     end
   end

lib/inspec/runner_mock.rb

@@ -14,7 +14,7 @@ module Inspec
       @profiles.push(profile)
     end
 
-    def add_test(example, _rule_id, _rule)
+    def add_test(example, _rule)
       @tests.push(example)
     end
 

lib/inspec/runner_rspec.rb

@@ -7,10 +7,8 @@ require 'rspec/its'
 require 'inspec/rspec_json_formatter'
 
 # There be dragons!! Or borgs, or something...
-# This file and all its contents cannot yet be tested. Once it is included
+# This file and all its contents cannot be unit-tested. both test-suits
-# in our unit test suite, it deactivates all other checks completely.
+# collide and disable all unit tests that have been added.
-# To circumvent this, we need functional tests which tackle the RSpec runner
-# or a separate suite of unit tests to which get along with this.
 
 module Inspec
   class RunnerRspec
@@ -35,7 +33,7 @@ module Inspec
     # @return [nil]
     def add_profile(profile)
       RSpec.configuration.formatters
-           .find_all { |c| c.is_a? InspecRspecFormatter }
+           .find_all { |c| c.is_a? InspecRspecJson }
            .each do |fmt|
         fmt.add_profile(profile)
       end
@@ -46,8 +44,8 @@ module Inspec
     # @param [RSpecExampleGroup] example test
     # @param [String] rule_id the ID associated with this check
     # @return [nil]
-    def add_test(example, rule_id, rule)
+    def add_test(example, rule)
-      set_rspec_ids(example, rule_id, rule)
+      set_rspec_ids(example, rule)
       @tests.example_groups.push(example)
     end
 
@@ -83,6 +81,12 @@ module Inspec
       RSpec.configuration.reset
     end
 
+    FORMATTERS = {
+      'json-min' => 'InspecRspecMiniJson',
+      'json' => 'InspecRspecJson',
+      'json-rspec' => 'InspecRspecVanilla',
+    }.freeze
+
     # Configure the output formatter and stream to be used with RSpec.
     #
     # @return [nil]
@@ -93,8 +97,7 @@ module Inspec
         RSpec.configuration.output_stream = @conf['output']
       end
 
-      format = @conf['format'] || 'progress'
+      format = FORMATTERS[@conf['format']] || @conf['format'] || 'progress'
-      format = 'InspecRspecFormatter' if format == 'fulljson'
       RSpec.configuration.add_formatter(format)
       RSpec.configuration.color = @conf['color']
 
@@ -111,27 +114,26 @@ module Inspec
     # by the InSpec adjusted json formatter (rspec_json_formatter).
     #
     # @param [RSpecExampleGroup] example object which contains a check
-    # @param [Type] id describe id
     # @return [Type] description of returned object
-    def set_rspec_ids(example, id, rule)
+    def set_rspec_ids(example, rule)
-      example.metadata[:id] = id
+      assign_rspec_ids(example.metadata, rule)
-      example.metadata[:impact] = rule.impact
-      example.metadata[:title] = rule.title
-      example.metadata[:desc] = rule.desc
-      example.metadata[:code] = rule.instance_variable_get(:@__code)
-      example.metadata[:source_location] = rule.instance_variable_get(:@__source_location)
       example.filtered_examples.each do |e|
-        e.metadata[:id] = id
+        assign_rspec_ids(e.metadata, rule)
-        e.metadata[:impact] = rule.impact
-        e.metadata[:title] = rule.title
-        e.metadata[:desc] = rule.desc
-        e.metadata[:code] = rule.instance_variable_get(:@__code)
-        e.metadata[:source_location] = rule.instance_variable_get(:@__source_location)
       end
       example.children.each do |child|
-        set_rspec_ids(child, id, rule)
+        set_rspec_ids(child, rule)
       end
     end
+
+    def assign_rspec_ids(metadata, rule)
+      metadata[:id] = ::Inspec::Rule.rule_id(rule)
+      metadata[:profile_id] = ::Inspec::Rule.profile_id(rule)
+      metadata[:impact] = rule.impact
+      metadata[:title] = rule.title
+      metadata[:desc] = rule.desc
+      metadata[:code] = rule.instance_variable_get(:@__code)
+      metadata[:source_location] = rule.instance_variable_get(:@__source_location)
+    end
   end
 
   class RSpecReporter < RSpec::Core::Formatters::JsonFormatter

lib/utils/base_cli.rb

@@ -45,8 +45,6 @@ module Inspec
     end
 
     def self.exec_options
-      option :id, type: :string,
-        desc: 'Attach a profile ID to all test results'
       target_options
       profile_options
       option :controls, type: :array,

test/functional/helper.rb

@@ -20,6 +20,7 @@ module FunctionalHelper
   let(:examples_path) { File.join(repo_path, 'examples') }
 
   let(:example_profile) { File.join(examples_path, 'profile') }
+  let(:example_control) { File.join(example_profile, 'controls', 'example.rb') }
   let(:inheritance_profile) { File.join(examples_path, 'profile') }
 
   let(:dst) {

test/functional/inheritance_test.rb

@@ -44,6 +44,6 @@ describe 'example inheritance profile' do
     s = out.stdout
     hm = JSON.load(s)
     hm['name'].must_equal 'inheritance'
-    hm['rules'].length.must_equal 1 # TODO: flatten out or search deeper!
+    hm['controls'].length.must_equal 3
   end
 end

test/functional/inspec_exec_json_test.rb

@@ -0,0 +1,122 @@
+# encoding: utf-8
+# author: Dominik Richter
+# author: Christoph Hartmann
+
+require 'functional/helper'
+
+describe 'inspec exec with json formatter' do
+  include FunctionalHelper
+
+  it 'can execute a simple file with the json formatter' do
+    out = inspec('exec ' + example_control + ' --format json')
+    out.stderr.must_equal ''
+    out.exit_status.must_equal 0
+    JSON.load(out.stdout).must_be_kind_of Hash
+  end
+
+  it 'can execute the profile with the json formatter' do
+    out = inspec('exec ' + example_profile + ' --format json')
+    out.stderr.must_equal ''
+    out.exit_status.must_equal 0
+    JSON.load(out.stdout).must_be_kind_of Hash
+  end
+
+  describe 'execute a profile with json formatting' do
+    let(:json) { JSON.load(inspec('exec ' + example_profile + ' --format json').stdout) }
+    let(:profile) { json['profiles']['profile'] }
+    let(:controls) { profile['controls'] }
+    let(:ex1) { controls['tmp-1.0'] }
+    let(:ex2) {
+      k = controls.keys.find { |x| x =~ /generated/ }
+      controls[k]
+    }
+    let(:ex3) { profile['controls']['gordon-1.0'] }
+    let(:check_result) {
+      ex3['results'].find { |x| x['resource'] == 'gordon_config' }
+    }
+
+    it 'has all the metadata' do
+      actual = profile.dup
+      key = actual.delete('controls').keys
+                  .find { |x| x =~ /generated from example.rb/ }
+
+      actual.must_equal({
+        "name" => "profile",
+        "title" => "InSpec Example Profile",
+        "maintainer" => "Chef Software, Inc.",
+        "copyright" => "Chef Software, Inc.",
+        "copyright_email" => "support@chef.io",
+        "license" => "Apache 2 license",
+        "summary" => "Demonstrates the use of InSpec Compliance Profile",
+        "version" => "1.0.0",
+        "supports" => [{"os-family" => "unix"}],
+        "groups" => {
+          "controls/meta.rb" => {"title"=>"SSH Server Configuration", "controls"=>["ssh-1"]},
+          "controls/example.rb" => {"title"=>"/tmp profile", "controls"=>["tmp-1.0", key]},
+          "controls/gordon.rb" => {"title"=>"Gordon Config Checks", "controls"=>["gordon-1.0"]},
+        },
+      })
+    end
+
+    it 'must have 4 controls' do
+      controls.length.must_equal 4
+    end
+
+    it 'has an id for every control' do
+      controls.keys.find(&:nil?).must_be :nil?
+    end
+
+    it 'has no missing checks' do
+      json['other_checks'].must_equal([])
+    end
+
+    it 'has results for every control' do
+      ex1['results'].length.must_equal 1
+      ex2['results'].length.must_equal 1
+      ex3['results'].length.must_equal 2
+    end
+
+    it 'has the right result for tmp-1.0' do
+      actual = ex1.dup
+
+      src = actual.delete('source_location')
+      src[0].must_match %r{examples/profile/controls/example.rb$}
+      src[1].must_equal 8
+
+      result = actual.delete('results')[0]
+      result.wont_be :nil?
+      result['status'].must_equal 'passed'
+      result['code_desc'].must_equal 'File /tmp should be directory'
+      result['run_time'].wont_be :nil?
+      result['start_time'].wont_be :nil?
+
+      actual.must_equal({
+        "title" => "Create /tmp directory",
+        "desc" => "An optional description...",
+        "impact" => 0.7,
+        "refs" => [
+          {
+            "url" => "http://...",
+            "ref" => "Document A-12"
+          }
+        ],
+        "tags" => {
+          "data" => "temp data",
+          "security" => nil
+        },
+        "code" => "control \"tmp-1.0\" do                        # A unique ID for this control\n  impact 0.7                                # The criticality, if this control fails.\n  title \"Create /tmp directory\"             # A human-readable title\n  desc \"An optional description...\"         # Describe why this is needed\n  tag data: \"temp data\"                     # A tag allows you to associate key information\n  tag \"security\"                            # to the test\n  ref \"Document A-12\", url: 'http://...'    # Additional references\n\n  describe file('/tmp') do                  # The actual test\n    it { should be_directory }\n  end\nend\n",
+      })
+    end
+  end
+
+  describe 'with a profile that is not supported on this OS/platform' do
+    let(:out) { inspec('exec ' + File.join(profile_path, 'skippy-profile-os') + ' --format json') }
+    let(:json) { JSON.load(out.stdout) }
+
+    # TODO: failure handling in json formatters...
+
+    it 'never runs the actual resource' do
+      File.exist?('/tmp/inspec_test_DONT_CREATE').must_equal false
+    end
+  end
+end

test/functional/inspec_exec_test.rb

@@ -22,15 +22,13 @@ describe 'inspec exec' do
     out.stdout.must_include '1 example, 0 failures'
   end
 
-  it 'can execute the profile with the json formatter' do
+  it 'can execute the profile with the mini json formatter' do
-    out = inspec('exec ' + example_profile + ' --format json')
+    out = inspec('exec ' + example_profile + ' --format json-min')
     out.stderr.must_equal ''
     out.exit_status.must_equal 0
     JSON.load(out.stdout).must_be_kind_of Hash
   end
 
-  let(:example_control) { File.join(example_profile, 'controls', 'example.rb') }
-
   it 'can execute a simple file with the default formatter' do
     out = inspec('exec ' + example_control)
     out.stderr.must_equal ''
@@ -38,126 +36,45 @@ describe 'inspec exec' do
     out.stdout.must_include '2 examples, 0 failures'
   end
 
-  it 'can execute a simple file with the json formatter' do
+  it 'can execute a simple file with the mini json formatter' do
-    out = inspec('exec ' + example_control + ' --format json')
+    out = inspec('exec ' + example_control + ' --format json-min')
     out.stderr.must_equal ''
     out.exit_status.must_equal 0
     JSON.load(out.stdout).must_be_kind_of Hash
   end
 
-  it 'can execute a simple file with the fulljson formatter' do
+  describe 'execute a profile with mini json formatting' do
-    out = inspec('exec ' + example_control + ' --format fulljson')
+    let(:json) { JSON.load(inspec('exec ' + example_profile + ' --format json-min').stdout) }
-    out.stderr.must_equal ''
+    let(:controls) { json['controls'] }
-    out.exit_status.must_equal 0
+    let(:ex1) { controls.find{|x| x['id'] == 'tmp-1.0'} }
-    JSON.load(out.stdout).must_be_kind_of Hash
+    let(:ex2) { controls.find{|x| x['id'] =~ /generated/} }
-  end
+    let(:ex3) { controls.find{|x| x['id'] == 'gordon-1.0'} }
-
-  describe 'execute a profile with json formatting' do
-    let(:json) { JSON.load(inspec('exec ' + example_profile + ' --format json').stdout) }
-    let(:examples) { json['examples'] }
-    let(:ex1) { examples.find{|x| x['id'] == 'tmp-1.0'} }
-    let(:ex2) { examples.find{|x| x['id'] =~ /generated/} }
-    let(:ex3) { examples.find{|x| x['id'] == 'gordon-1.0'} }
 
     it 'must have 5 examples' do
-      json['examples'].length.must_equal 5
+      json['controls'].length.must_equal 5
     end
 
-    it 'id in json' do
+    it 'has an id' do
-      examples.find { |ex| !ex.key? 'id' }.must_be :nil?
+      controls.find { |ex| !ex.key? 'id' }.must_be :nil?
     end
 
-    it 'impact in json' do
+    it 'has a profile_id' do
-      ex1['impact'].must_equal 0.7
+      controls.find { |ex| !ex.key? 'profile_id' }.must_be :nil?
-      ex2['impact'].must_be :nil?
     end
 
-    it 'status in json' do
+    it 'has a code_desc' do
+      ex1['code_desc'].must_equal 'File /tmp should be directory'
+      controls.find { |ex| !ex.key? 'code_desc' }.must_be :nil?
+    end
+
+    it 'has a status' do
       ex1['status'].must_equal 'passed'
-      ex3['status'].must_equal 'pending'
+      ex3['status'].must_equal 'skipped'
     end
 
-    it 'pending message in json' do
+    it 'has a skip_message' do
-      ex1['pending_message'].must_be :nil?
+      ex1['skip_message'].must_be :nil?
-      ex3['pending_message'].must_equal 'Not yet implemented'
+      ex3['skip_message'].must_equal "Can't find file \"/tmp/gordon/config.yaml\""
-    end
-  end
-
-  describe 'execute a profile with fulljson formatting' do
-    let(:json) { JSON.load(inspec('exec ' + example_profile + ' --format fulljson').stdout) }
-    let(:examples) { json['examples'] }
-    let(:metadata) { json['profiles'][0] }
-    let(:ex1) { examples.find{|x| x['id'] == 'tmp-1.0'} }
-    let(:ex2) { examples.find{|x| x['id'] =~ /generated/} }
-    let(:ex3) { examples.find{|x| x['id'] == 'gordon-1.0'} }
-
-    it 'has all the metadata' do
-      metadata.must_equal({
-        "name" => "profile",
-        "title" => "InSpec Example Profile",
-        "maintainer" => "Chef Software, Inc.",
-        "copyright" => "Chef Software, Inc.",
-        "copyright_email" => "support@chef.io",
-        "license" => "Apache 2 license",
-        "summary" => "Demonstrates the use of InSpec Compliance Profile",
-        "version" => "1.0.0",
-        "supports" => [{"os-family" => "unix"}]
-      })
-    end
-
-    it 'must have 5 examples' do
-      json['examples'].length.must_equal 5
-    end
-
-    it 'id in json' do
-      examples.find { |ex| !ex.key? 'id' }.must_be :nil?
-    end
-
-    it 'title in json' do
-      ex3['title'].must_equal 'Verify the version number of Gordon'
-    end
-
-    it 'desc in json' do
-      ex3['desc'].must_equal 'An optional description...'
-    end
-
-    it 'code in json' do
-      ex3['code'].wont_be :nil?
-    end
-
-    it 'code_desc in json' do
-      ex3['code_desc'].wont_be :nil?
-    end
-
-    it 'impact in json' do
-      ex1['impact'].must_equal 0.7
-      ex2['impact'].must_be :nil?
-    end
-
-    it 'status in json' do
-      ex1['status'].must_equal 'passed'
-      ex3['status'].must_equal 'pending'
-    end
-
-    it 'ref in json' do
-      ex1['ref'].must_match %r{examples/profile/controls/example.rb$}
-    end
-
-    it 'ref_line in json' do
-      ex1['ref_line'].must_equal 16
-    end
-
-    it 'run_time in json' do
-      ex1['run_time'].wont_be :nil?
-    end
-
-    it 'start_time in json' do
-      ex1['start_time'].wont_be :nil?
-    end
-
-    it 'pending message in json' do
-      ex1['pending'].must_be :nil?
-      ex3['pending'].must_equal "Can't find file \"/tmp/gordon/config.yaml\""
     end
   end
 
@@ -171,17 +88,6 @@ describe 'inspec exec' do
     end
   end
 
-  describe 'with a profile that is not supported on this OS/platform' do
-    let(:out) { inspec('exec ' + File.join(profile_path, 'skippy-profile-os') + ' --format fulljson') }
-    let(:json) { JSON.load(out.stdout) }
-
-    # TODO: failure handling in json formatters...
-
-    it 'never runs the actual resource' do
-      File.exist?('/tmp/inspec_test_DONT_CREATE').must_equal false
-    end
-  end
-
   describe 'with a profile that is supported on this version of inspec' do
     let(:out) { inspec('exec ' + File.join(profile_path, 'supported_inspec')) }
 

test/functional/inspec_json_profile_test.rb

@@ -42,36 +42,36 @@ describe 'inspec json' do
       json['copyright'].must_equal 'Chef Software, Inc.'
     end
 
-    it 'has rules' do
+    it 'has controls' do
-      json['rules'].length.must_equal 3 # TODO: flatten out or search deeper!
+      json['controls'].length.must_equal 4
     end
 
-    describe 'a rule' do
+    describe 'a control' do
-      let(:rule) { json['rules']['controls/example.rb']['rules']['tmp-1.0'] }
+      let(:control) { json['controls']['tmp-1.0'] }
 
       it 'has a title' do
-        rule['title'].must_equal 'Create /tmp directory'
+        control['title'].must_equal 'Create /tmp directory'
       end
 
       it 'has a description' do
-        rule['desc'].must_equal 'An optional description...'
+        control['desc'].must_equal 'An optional description...'
       end
 
       it 'has an impact' do
-        rule['impact'].must_equal 0.7
+        control['impact'].must_equal 0.7
       end
 
       it 'has a ref' do
-        rule['refs'].must_equal([{'ref' => 'Document A-12', 'url' => 'http://...'}])
+        control['refs'].must_equal([{'ref' => 'Document A-12', 'url' => 'http://...'}])
       end
 
       it 'has a source location' do
         loc = File.join(example_profile, '/controls/example.rb')
-        rule['source_location'].must_equal [loc, 8]
+        control['source_location'].must_equal [loc, 8]
       end
 
       it 'has a the source code' do
-        rule['code'].must_match /\Acontrol \"tmp-1.0\" do.*end\n\Z/m
+        control['code'].must_match /\Acontrol \"tmp-1.0\" do.*end\n\Z/m
       end
     end
   end
@@ -86,10 +86,8 @@ describe 'inspec json' do
 
     it 'only has one control included' do
       json = JSON.load(out.stdout)
-      grps = json['rules']
+      json['controls'].keys.must_equal %w{tmp-1.0}
-      grps.keys.must_equal ['controls/example.rb']
+      json['groups'].keys.must_equal %w{controls/example.rb}
-      rules = grps.values[0]['rules']
-      rules.keys.must_equal ['tmp-1.0']
     end
   end
 
@@ -99,6 +97,6 @@ describe 'inspec json' do
     out.exit_status.must_equal 0
     hm = JSON.load(File.read(dst.path))
     hm['name'].must_equal 'profile'
-    hm['rules'].length.must_equal 3 # TODO: flatten out or search deeper!
+    hm['controls'].length.must_equal 4
   end
 end

test/unit/control_test.rb

@@ -12,7 +12,7 @@ describe 'controls' do
     }
     opts = { test_collector: Inspec::RunnerMock.new }
     Inspec::Profile.for_target(data, opts)
-                   .params[:rules].values[0]['1']
+                   .params[:controls]['1']
   end
 
   it 'works with empty refs' do

test/unit/profile_context_test.rb

@@ -177,13 +177,13 @@ describe Inspec::ProfileContext do
 
     it 'provides the control keyword in the global DSL' do
       profile.load('control 1')
-      profile.rules.keys.must_equal [1]
+      profile.rules.keys.must_equal ['1']
       profile.rules.values[0].must_be_kind_of Inspec::Rule
     end
 
     it 'provides the rule keyword in the global DSL (legacy mode)' do
       profile.load('rule 1')
-      profile.rules.keys.must_equal [1]
+      profile.rules.keys.must_equal ['1']
       profile.rules.values[0].must_be_kind_of Inspec::Rule
     end
   end

test/unit/profile_test.rb

@@ -16,8 +16,8 @@ describe Inspec::Profile do
       profile.params[:name].must_be_nil
     end
 
-    it 'has no rules' do
+    it 'has no controls' do
-      profile.params[:rules].must_equal({})
+      profile.params[:controls].must_equal({})
     end
   end
 
@@ -28,8 +28,8 @@ describe Inspec::Profile do
       profile.params[:name].must_be_nil
     end
 
-    it 'has no rules' do
+    it 'has no controls' do
-      profile.params[:rules].must_equal({})
+      profile.params[:controls].must_equal({})
     end
   end
 
@@ -41,8 +41,8 @@ describe Inspec::Profile do
       profile.params[:name].must_equal 'yumyum profile'
     end
 
-    it 'has no rules' do
+    it 'has no controls' do
-      profile.params[:rules].must_equal({})
+      profile.params[:controls].must_equal({})
     end
 
     it 'can overwrite the profile ID' do
@@ -59,8 +59,8 @@ describe Inspec::Profile do
       profile.params[:name].must_equal 'metadata profile'
     end
 
-    it 'has no rules' do
+    it 'has no controls' do
-      profile.params[:rules].must_equal({})
+      profile.params[:controls].must_equal({})
     end
   end
 
@@ -179,11 +179,10 @@ describe Inspec::Profile do
     describe 'a complete metadata profile with controls' do
       let(:profile_id) { 'complete-profile' }
 
-      it 'prints ok messages and counts the rules' do
+      it 'prints ok messages and counts the controls' do
         logger.expect :info, nil, ["Checking profile in #{home}/mock/profiles/#{profile_id}"]
         logger.expect :info, nil, ['Metadata OK.']
         logger.expect :info, nil, ['Found 1 controls.']
-        logger.expect :info, nil, ["Verify all controls in controls/filesystem_spec.rb"]
         logger.expect :info, nil, ['Control definitions OK.']
 
         result = MockLoader.load_profile(profile_id, {logger: logger}).check
@@ -205,11 +204,10 @@ describe Inspec::Profile do
       let(:profile_path) { MockLoader.profile_tgz(profile_id) }
       let(:profile) { MockLoader.load_profile(profile_path, {logger: logger}) }
 
-      it 'prints ok messages and counts the rules' do
+      it 'prints ok messages and counts the controls' do
         logger.expect :info, nil, ["Checking profile in #{home}/mock/profiles/#{profile_id}"]
         logger.expect :info, nil, ['Metadata OK.']
         logger.expect :info, nil, ['Found 1 controls.']
-        logger.expect :info, nil, ["Verify all controls in controls/filesystem_spec.rb"]
         logger.expect :info, nil, ['Control definitions OK.']
 
         result = MockLoader.load_profile(profile_id, {logger: logger}).check
@@ -231,11 +229,10 @@ describe Inspec::Profile do
       let(:profile_path) { MockLoader.profile_zip(profile_id) }
       let(:profile) { MockLoader.load_profile(profile_path, {logger: logger}) }
 
-      it 'prints ok messages and counts the rules' do
+      it 'prints ok messages and counts the controls' do
         logger.expect :info, nil, ["Checking profile in #{home}/mock/profiles/#{profile_id}"]
         logger.expect :info, nil, ['Metadata OK.']
         logger.expect :info, nil, ['Found 1 controls.']
-        logger.expect :info, nil, ["Verify all controls in controls/filesystem_spec.rb"]
         logger.expect :info, nil, ['Control definitions OK.']
 
         result = MockLoader.load_profile(profile_id, {logger: logger}).check