Merge pull request #371 from chef/dr/separate-rspec

separate RSpec handling in runner

lib/inspec/profile.rb

@@ -38,6 +38,7 @@ module Inspec
       @runner = Runner.new(
         id: @profile_id,
         backend: :mock,
+        test_collector: @options.delete(:test_collector),
       )
       @runner.add_tests([@path], @options)
       @runner.rules.each do |id, rule|

lib/inspec/profile_context.rb

@@ -4,11 +4,10 @@
 
 require 'inspec/rule'
 require 'inspec/dsl'
-require 'rspec/core/dsl'
 require 'securerandom'
 
 module Inspec
-  class ProfileContext # rubocop:disable Metrics/ClassLength
+  class ProfileContext
     attr_reader :rules, :only_ifs
     def initialize(profile_id, backend, profile_registry = {}, only_ifs = [])
       if backend.nil?
@@ -25,9 +24,8 @@ module Inspec
     end
 
     def reload_dsl
-      dsl = create_inner_dsl(@backend)
-      outer_dsl = create_outer_dsl(dsl)
-      ctx = create_context(outer_dsl)
+      resources_dsl = Inspec::Resource.create_dsl(@backend)
+      ctx = create_context(resources_dsl, rule_context(resources_dsl))
       @profile_context = ctx.new
     end
 
@@ -66,36 +64,42 @@ module Inspec
 
     private
 
-    # Creates the inner DSL which includes all resources for
-    # creating tests. It is always connected to one target,
-    # which is specified via the backend argument.
+    # Create the context for controls. This includes all components of the DSL,
+    # including matchers and resources.
     #
-    # @param backend [BackendRunner] exposing the target to resources
-    # @return [InnerDSLModule]
-    def create_inner_dsl(backend)
-      Module.new do
-        Inspec::Resource.registry.each do |id, r|
-          define_method id.to_sym do |*args|
-            r.new(backend, id.to_s, *args)
-          end
-        end
+    # @param [ResourcesDSL] resources_dsl which has all resources to attach
+    # @return [RuleContext] the inner context of rules
+    def rule_context(resources_dsl)
+      require 'rspec/core/dsl'
+      Class.new(Inspec::Rule) do
+        include RSpec::Core::DSL
+        include resources_dsl
       end
     end
 
-    # Creates the outer DSL which includes all methods for creating
-    # tests and control structures.
+    # Creates the heart of the profile context:
+    # An instantiated object which has all resources registered to it
+    # and exposes them to the a test file. The profile context serves as a
+    # container for all profiles which are registered. Within the context
+    # profiles get access to all DSL calls for creating tests and controls.
     #
-    # @param dsl [InnerDSLModule] which contains all resources
-    # @return [OuterDSLClass]
-    def create_outer_dsl(dsl)
-      rule_class = Class.new(Inspec::Rule) do
-        include RSpec::Core::DSL
-        include dsl
-      end
+    # @param outer_dsl [OuterDSLClass]
+    # @return [ProfileContextClass]
+    def create_context(resources_dsl, rule_class)
+      profile_context_owner = self
 
       # rubocop:disable Lint/NestedMethodDefinition
       Class.new do
-        include dsl
+        include Inspec::DSL
+        include resources_dsl
+
+        define_method :title do |arg|
+          profile_context_owner.set_header(:title, arg)
+        end
+
+        def to_s
+          'Profile Context Run'
+        end
 
         define_method :control do |*args, &block|
           id = args[0]
@@ -107,7 +111,7 @@ module Inspec
           # controls.
           return if @skip_profile && os[:family] != 'unknown'
 
-          __register_rule rule_class.new(id, opts, &block)
+          profile_context_owner.register_rule(rule_class.new(id, opts, &block))
         end
 
         alias_method :rule, :control
@@ -119,7 +123,7 @@ module Inspec
           rule = rule_class.new(id, {}) do
             describe(*args, &block)
           end
-          __register_rule rule, &block
+          profile_context_owner.register_rule(rule, &block)
         end
 
         # TODO: mock method for attributes; import attribute handling
@@ -128,7 +132,7 @@ module Inspec
         end
 
         def skip_control(id)
-          __unregister_rule id
+          profile_context_owner.unregister_rule(id)
         end
 
         alias_method :skip_rule, :skip_control
@@ -140,38 +144,5 @@ module Inspec
       end
       # rubocop:enable all
     end
-
-    # Creates the heart of the profile context:
-    # An instantiated object which has all resources registered to it
-    # and exposes them to the a test file. The profile context serves as a
-    # container for all profiles which are registered. Within the context
-    # profiles get access to all DSL calls for creating tests and controls.
-    #
-    # @param outer_dsl [OuterDSLClass]
-    # @return [ProfileContextClass]
-    def create_context(outer_dsl)
-      profile_context_owner = self
-
-      # rubocop:disable Lint/NestedMethodDefinition
-      Class.new(outer_dsl) do
-        include Inspec::DSL
-
-        define_method :title do |arg|
-          profile_context_owner.set_header(:title, arg)
-        end
-
-        define_method :__register_rule do |*args|
-          profile_context_owner.register_rule(*args)
-        end
-        define_method :__unregister_rule do |*args|
-          profile_context_owner.unregister_rule(*args)
-        end
-
-        def to_s
-          'Profile Context Run'
-        end
-      end
-      # rubocop:enable all
-    end
   end
 end

lib/inspec/resource.rb

@@ -11,8 +11,31 @@ module Inspec
     def self.registry
       @registry ||= {}
     end
+
+    # Creates the inner DSL which includes all resources for
+    # creating tests. It is always connected to one target,
+    # which is specified via the backend argument.
+    #
+    # @param backend [BackendRunner] exposing the target to resources
+    # @return [ResourcesDSL]
+    def self.create_dsl(backend)
+      # need the local name, to use it in the module creation further down
+      my_registry = registry
+      Module.new do
+        my_registry.each do |id, r|
+          define_method id.to_sym do |*args|
+            r.new(backend, id.to_s, *args)
+          end
+        end
+      end
+    end
   end
 
+  # Retrieve the base class for creating a new resource.
+  # Create classes that inherit from this class.
+  #
+  # @param [int] version the resource version to use
+  # @return [Resource] base class for creating a new resource
   def self.resource(version)
     if version != 1
       fail 'Only resource version 1 is supported!'

lib/inspec/runner.rb

@@ -10,27 +10,28 @@ require 'inspec/profile_context'
 require 'inspec/targets'
 require 'inspec/metadata'
 # spec requirements
-require 'rspec'
-require 'rspec/its'
-require 'inspec/rspec_json_formatter'
 
 module Inspec
   class Runner # rubocop:disable Metrics/ClassLength
-    attr_reader :tests, :backend, :rules
+    attr_reader :backend, :rules
     def initialize(conf = {})
       @rules = {}
       @profile_id = conf[:id]
       @conf = conf.dup
       @conf[:logger] ||= Logger.new(nil)
-      @tests = RSpec::Core::World.new
 
-      # resets "pending examples" in reporter
-      RSpec.configuration.reset
+      @test_collector = @conf.delete(:test_collector) || begin
+        require 'inspec/runner_rspec'
+        RunnerRspec.new(@conf)
+      end
 
-      configure_output
       configure_transport
     end
 
+    def tests
+      @test_collector.tests
+    end
+
     def normalize_map(hm)
       res = {}
       hm.each {|k, v|
@@ -39,10 +40,6 @@ module Inspec
       res
     end
 
-    def configure_output
-      RSpec.configuration.add_formatter(@conf['format'] || 'progress')
-    end
-
     def configure_transport
       @backend = Inspec::Backend.create(@conf)
     end
@@ -105,16 +102,12 @@ module Inspec
 
       # process the resulting rules
       ctx.rules.each do |rule_id, rule|
-        register_rule(ctx, rule_id, rule)
+        register_rule(rule_id, rule)
       end
     end
 
-    def run
-      run_with(RSpec::Core::Runner.new(nil))
-    end
-
-    def run_with(rspec_runner)
-      rspec_runner.run_specs(@tests.ordered_example_groups)
+    def run(with = nil)
+      @test_collector.run(with)
     end
 
     private
@@ -130,14 +123,14 @@ module Inspec
       if !arg.empty? &&
          arg[0].respond_to?(:resource_skipped) &&
          !arg[0].resource_skipped.nil?
-        return RSpec::Core::ExampleGroup.describe(*arg, opts) do
+        return @test_collector.example_group(*arg, opts) do
           it arg[0].resource_skipped
         end
       else
         # add the resource
         case method_name
         when 'describe'
-          return RSpec::Core::ExampleGroup.describe(*arg, opts, &block)
+          return @test_collector.example_group(*arg, opts, &block)
         when 'expect'
           return block.example_group
         else
@@ -148,7 +141,7 @@ module Inspec
       nil
     end
 
-    def register_rule(ctx, rule_id, rule)
+    def register_rule(rule_id, rule)
       @rules[rule_id] = rule
       checks = rule.instance_variable_get(:@checks)
       checks.each do |m, a, b|
@@ -161,21 +154,10 @@ module Inspec
         # the scope of this run, thus not gaining ony of the DSL pieces.
         # To circumvent this, the full DSL is attached to the example's
         # scope.
-        dsl = ctx.method(:create_inner_dsl).call(backend)
+        dsl = Inspec::Resource.create_dsl(backend)
         example.send(:include, dsl)
 
-        set_rspec_ids(example, rule_id)
-        @tests.register(example)
-      end
-    end
-
-    def set_rspec_ids(example, id)
-      example.metadata[:id] = id
-      example.filtered_examples.each do |e|
-        e.metadata[:id] = id
-      end
-      example.children.each do |child|
-        set_rspec_ids(child, id)
+        @test_collector.add_test(example, rule_id)
       end
     end
   end

lib/inspec/runner_mock.rb

@@ -0,0 +1,31 @@
+# encoding: utf-8
+# author: Dominik Richter
+# author: Christoph Hartmann
+
+module Inspec
+  class RunnerMock
+    attr_reader :tests
+    def initialize
+      @tests = []
+    end
+
+    def add_test(example, _rule_id)
+      @tests.push(example)
+    end
+
+    def example_group(*in_args, &in_block)
+      Class.new do
+        define_method :args do
+          in_args
+        end
+        define_method :block do
+          in_block
+        end
+      end
+    end
+
+    def run(_with = nil)
+      puts 'uhm.... nothing or something... dunno, ask your admin'
+    end
+  end
+end

lib/inspec/runner_rspec.rb

@@ -0,0 +1,94 @@
+# encoding: utf-8
+# author: Dominik Richter
+# author: Christoph Hartmann
+
+require 'rspec/core'
+require 'rspec/its'
+require 'inspec/rspec_json_formatter'
+
+# There be dragons!! Or borgs, or something...
+# This file and all its contents cannot yet be tested. Once it is included
+# in our unit test suite, it deactivates all other checks completely.
+# To circumvent this, we need functional tests which tackle the RSpec runner
+# or a separate suite of unit tests to which get along with this.
+
+module Inspec
+  class RunnerRspec
+    def initialize(conf)
+      @conf = conf
+      reset_tests
+      configure_output
+    end
+
+    # Create a new RSpec example group from arguments and block.
+    #
+    # @param [Type] *args list of arguments for this example
+    # @param [Type] &block the block associated with this example group
+    # @return [RSpecExampleGroup]
+    def example_group(*args, &block)
+      RSpec::Core::ExampleGroup.describe(*args, &block)
+    end
+
+    # Add an example group to the list of registered tests.
+    #
+    # @param [RSpecExampleGroup] example test
+    # @param [String] rule_id the ID associated with this check
+    # @return [nil]
+    def add_test(example, rule_id)
+      set_rspec_ids(example, rule_id)
+      @tests.register(example)
+    end
+
+    # Retrieve the list of tests that have been added.
+    #
+    # @return [Array] full list of tests
+    def tests
+      @tests.ordered_example_groups
+    end
+
+    # Run all registered tests with an optional test runner.
+    #
+    # @param [RSpecRunner] with is an optional RSpecRunner
+    # @return [int] 0 if all went well; otherwise nonzero
+    def run(with = nil)
+      with ||= RSpec::Core::Runner.new(nil)
+      with.run_specs(tests)
+    end
+
+    private
+
+    # Empty the list of registered tests.
+    #
+    # @return [nil]
+    def reset_tests
+      @tests = RSpec::Core::World.new
+      # resets "pending examples" in reporter
+      RSpec.configuration.reset
+    end
+
+    # Configure the output formatter and stream to be used with RSpec.
+    #
+    # @return [nil]
+    def configure_output
+      RSpec.configuration.add_formatter(@conf['format'] || 'progress')
+    end
+
+    # Make sure that all RSpec example groups use the provided ID.
+    # At the time of creation, we didn't yet have full ID support in RSpec,
+    # which is why they were added to metadata directly. This is evaluated
+    # by the InSpec adjusted json formatter (rspec_json_formatter).
+    #
+    # @param [RSpecExampleGroup] example object which contains a check
+    # @param [Type] id describe id
+    # @return [Type] description of returned object
+    def set_rspec_ids(example, id)
+      example.metadata[:id] = id
+      example.filtered_examples.each do |e|
+        e.metadata[:id] = id
+      end
+      example.children.each do |child|
+        set_rspec_ids(child, id)
+      end
+    end
+  end
+end

test/docker_run.rb

@@ -52,7 +52,11 @@ class DockerRunner
         res = block.call(name, container)
       # special rescue block to handle not implemented error
       rescue NotImplementedError => err
-        raise err.message
+        stop_container(container)
+        raise err.message + "\n" + err.backtrace.join("\n")
+      rescue StandardError => err
+        stop_container(container)
+        raise err.message + "\n" + err.backtrace.join("\n")
       end
       # always stop the container
       stop_container(container)

test/docker_test.rb

@@ -43,8 +43,7 @@ class DockerTester
     opts = { 'target' => "docker://#{container.id}" }
     runner = Inspec::Runner.new(opts)
     runner.add_tests(@tests)
-    tests = runner.tests.ordered_example_groups
-    tests.map { |g| g.run(report) }
+    runner.tests.map { |g| g.run(report) }
   end
 end
 

test/unit/profile_test.rb

@@ -3,30 +3,21 @@
 # author: Dominik Richter
 
 require 'helper'
+require 'inspec/profile_context'
+require 'inspec/runner'
+require 'inspec/runner_mock'
 
 describe Inspec::Profile do
-  before {
-    # mock up the profile runner
-    # TODO: try to take the real profile runner here;
-    # currently it's stopped at test runner conflicts
-    class Inspec::Profile::Runner
-      def initialize(opts) end
-      def add_tests(tests, options=nil) end
-      def rules
-        {}
-      end
-    end
-  }
-
   let(:logger) { Minitest::Mock.new }
   let(:home) { File.dirname(__FILE__) }
 
   def load_profile(name, opts = {})
+    opts[:test_collector] = Inspec::RunnerMock.new
     Inspec::Profile.from_path("#{home}/mock/profiles/#{name}", opts)
   end
 
-  describe 'with empty profile' do
-    let(:profile) { load_profile('empty') }
+  describe 'with empty profile (legacy mode)' do
+    let(:profile) { load_profile('legacy-empty-metadata') }
 
     it 'has no metadata' do
       profile.params[:name].must_be_nil
@@ -37,8 +28,8 @@ describe Inspec::Profile do
     end
   end
 
-  describe 'with normal metadata in profile' do
-    let(:profile) { load_profile('metadata') }
+  describe 'with normal metadata in profile (legacy mode)' do
+    let(:profile) { load_profile('legacy-metadata') }
 
     it 'has metadata' do
       profile.params[:name].must_equal 'metadata profile'
@@ -50,11 +41,11 @@ describe Inspec::Profile do
   end
 
   describe 'when checking' do
-    describe 'an empty profile' do
-      let(:profile) { load_profile('empty', {logger: logger}) }
+    describe 'an empty profile (legacy mode)' do
+      let(:profile_id) { 'legacy-empty-metadata' }
 
       it 'prints loads of warnings' do
-        logger.expect :info, nil, ["Checking profile in #{home}/mock/profiles/empty"]
+        logger.expect :info, nil, ["Checking profile in #{home}/mock/profiles/#{profile_id}"]
         logger.expect :warn, nil, ['The use of `metadata.rb` is deprecated. Use `inspec.yml`.']
         logger.expect :error, nil, ['Missing profile name in metadata.rb']
         logger.expect :error, nil, ['Missing profile version in metadata.rb']
@@ -64,16 +55,17 @@ describe Inspec::Profile do
         logger.expect :warn, nil, ['Missing profile copyright in metadata.rb']
         logger.expect :warn, nil, ['No controls or tests were defined.']
 
-        profile.check
+        load_profile(profile_id, {logger: logger}).check
         logger.verify
       end
     end
 
     describe 'a complete metadata profile (legacy mode)' do
-      let(:profile) { load_profile('complete-meta', {logger: logger}) }
+      let(:profile_id) { 'legacy-complete-metadata' }
+      let(:profile) { load_profile(profile_id, {logger: logger}) }
 
       it 'prints ok messages' do
-        logger.expect :info, nil, ["Checking profile in #{home}/mock/profiles/complete-meta"]
+        logger.expect :info, nil, ["Checking profile in #{home}/mock/profiles/#{profile_id}"]
         logger.expect :warn, nil, ['The use of `metadata.rb` is deprecated. Use `inspec.yml`.']
         logger.expect :info, nil, ['Metadata OK.']
         logger.expect :warn, nil, ["Profile uses deprecated `test` directory, rename it to `controls`."]
@@ -89,20 +81,16 @@ describe Inspec::Profile do
     end
 
     describe 'a complete metadata profile with controls' do
-      let(:profile) { load_profile('complete-profile', {logger: logger, ignore_supports: true}) }
+      let(:profile_id) { 'complete-profile' }
 
       it 'prints ok messages and counts the rules' do
-        logger.expect :info, nil, ["Checking profile in #{home}/mock/profiles/complete-profile"]
+        logger.expect :info, nil, ["Checking profile in #{home}/mock/profiles/#{profile_id}"]
         logger.expect :info, nil, ['Metadata OK.']
+        logger.expect :info, nil, ['Found 1 rules.']
+        logger.expect :debug, nil, ["Verify all rules in  #{home}/mock/profiles/#{profile_id}/controls/filesystem_spec.rb"]
+        logger.expect :info, nil, ['Rule definitions OK.']
 
-        # TODO: cannot load rspec in unit tests, therefore we get a loading warn
-        # RSpec does not work with minitest tests
-        logger.expect :warn, nil, ['No controls or tests were defined.']
-        # we expect that this should work:
-        # logger.expect :info, nil, ['Found 1 rules.']
-        # logger.expect :info, nil, ['Rule definitions OK.']
-
-        profile.check
+        load_profile(profile_id, {logger: logger, ignore_supports: true}).check
         logger.verify
       end
     end