Merge pull request #497 from chef/dr/or

add `describe.one`: collection of tests with at least one passing

docs/dsl_inspec.rst

@@ -45,9 +45,22 @@ where
 * ``its('Port')`` is the matcher; ``{ should eq('22') }`` is the test. A ``describe`` block must contain at least one matcher, but may contain as many as required
 
 
-Author Tests
+Advanced concepts
------------------------------------------------------
+=====================================================
-It is recommended that test files are located in the ``/tests`` directory. When writing controls, the ``impact``, ``title``, ``desc`` metadata are _optional_, but are highly recommended.
+
+With inspec it is possible to check if at least one of a collection of checks is true. For example: If a setting is configured in two different locations, you may want to test if either configuration A or configuration B have been set. This is accomplished via ``describe.one``. It defines a block of tests with at least one valid check.
+
+.. code-block:: ruby
+
+   describe.one do
+     describe ConfigurationA do
+       its('setting_1') { should eq true }
+     end
+
+     describe ConfigurationB do
+       its('setting_2') { should eq true }
+     end
+   end
 
 Examples
 =====================================================

lib/inspec/describe.rb

@@ -0,0 +1,27 @@
+# encoding: utf-8
+# author: Dominik Richter
+# author: Christoph Hartmann
+
+module Inspec
+  class DescribeBase
+    def initialize(action)
+      @action = action
+      @checks = []
+    end
+
+    # Evaluate the given block and collect all checks. These will be registered
+    # with the callback function under the 'describe.one' name.
+    #
+    # @param [Proc] ruby block containing checks (e.g. via describe)
+    # @return [nil]
+    def one(&block)
+      return unless block_given?
+      instance_eval(&block)
+      @action.call('describe.one', @checks, nil)
+    end
+
+    def describe(*args, &block)
+      @checks.push(['describe', args, block])
+    end
+  end
+end

lib/inspec/expect.rb

@@ -0,0 +1,45 @@
+# encoding: utf-8
+# copyright: 2016, Chef Software Inc.
+# author: Dominik Richter
+# author: Christoph Hartmann
+
+require 'rspec/expectations'
+
+module Inspec
+  class Expect
+    attr_reader :calls, :value, :block
+    def initialize(value, &block)
+      @value = value
+      @block = block
+      @calls = []
+    end
+
+    def to(*args, &block)
+      @calls.push([:to, args, block, caller])
+    end
+
+    def not_to(*args, &block)
+      @calls.push([:not_to, args, block, caller])
+    end
+
+    def example_group
+      that = self
+
+      opts = { 'caller' => calls[0][3] }
+      if !calls[0][3].nil? && !calls[0][3].empty? &&
+         (m = calls[0][3][0].match(/^([^:]*):(\d+):/))
+        opts['file_path'] = m[0]
+        opts['line_number'] = m[1]
+      end
+
+      RSpec::Core::ExampleGroup.describe(that.value, opts) do
+        that.calls.each do |method, args, block, clr|
+          it(nil, caller: clr) do
+            x = expect(that.value, &that.block).method(method)
+            x.call(*args, &block)
+          end
+        end
+      end
+    end
+  end
+end

lib/inspec/profile_context.rb

@@ -85,7 +85,7 @@ module Inspec
     #
     # @param outer_dsl [OuterDSLClass]
     # @return [ProfileContextClass]
-    def create_context(resources_dsl, rule_class)
+    def create_context(resources_dsl, rule_class) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
       profile_context_owner = self
 
       # rubocop:disable Lint/NestedMethodDefinition
@@ -117,13 +117,15 @@ module Inspec
         alias_method :rule, :control
 
         define_method :describe do |*args, &block|
-          path = block.source_location[0]
+          loc = block_location(block, caller[0])
-          line = block.source_location[1]
+          id = "(generated from #{loc} #{SecureRandom.hex})"
-          id = "(generated from #{File.basename(path)}:#{line} #{SecureRandom.hex})"
+
+          res = nil
           rule = rule_class.new(id, {}) do
-            describe(*args, &block)
+            res = describe(*args, &block)
           end
           profile_context_owner.register_rule(rule, &block)
+          res
         end
 
         # TODO: mock method for attributes; import attribute handling
@@ -141,6 +143,17 @@ module Inspec
           return unless block_given?
           @skip_profile = !yield
         end
+
+        private
+
+        def block_location(block, alternate_caller)
+          if block.nil?
+            alternate_caller[/^(.+:\d+):in .+$/, 1] || 'unknown'
+          else
+            path, line = block.source_location
+            "#{File.basename(path)}:#{line}"
+          end
+        end
       end
       # rubocop:enable all
     end

lib/inspec/rule.rb

@@ -4,47 +4,11 @@
 # author: Dominik Richter
 # author: Christoph Hartmann
 
-require 'rspec/expectations'
 require 'method_source'
+require 'inspec/describe'
+require 'inspec/expect'
 
 module Inspec
-  class ExpectationTarget
-    attr_reader :calls, :value, :block
-    def initialize(value, &block)
-      @value = value
-      @block = block
-      @calls = []
-    end
-
-    def to(*args, &block)
-      @calls.push([:to, args, block, caller])
-    end
-
-    def not_to(*args, &block)
-      @calls.push([:not_to, args, block, caller])
-    end
-
-    def example_group
-      that = self
-
-      opts = { 'caller' => calls[0][3] }
-      if !calls[0][3].nil? && !calls[0][3].empty? &&
-         (m = calls[0][3][0].match(/^([^:]*):(\d+):/))
-        opts['file_path'] = m[0]
-        opts['line_number'] = m[1]
-      end
-
-      RSpec::Core::ExampleGroup.describe(that.value, opts) do
-        that.calls.each do |method, args, block, clr|
-          it(nil, caller: clr) do
-            x = expect(that.value, &that.block).method(method)
-            x.call(*args, &block)
-          end
-        end
-      end
-    end
-  end
-
   class Rule
     include ::RSpec::Matchers
 
@@ -83,13 +47,32 @@ module Inspec
       @desc
     end
 
-    def describe(value, &block)
+    # Describe will add one or more tests to this control. There is 2 ways
-      @checks.push(['describe', [value], block])
+    # of calling it:
+    #
+    #   describe resource do ... end
+    #
+    # or
+    #
+    #   describe.one do ... end
+    #
+    # @param [any] Resource to be describe, string, or nil
+    # @param [Proc] An optional block containing tests for the described resource
+    # @return [nil|DescribeBase] if called without arguments, returns DescribeBase
+    def describe(*values, &block)
+      if values.empty? && !block_given?
+        dsl = self.class.ancestors[1]
+        Class.new(DescribeBase) do
+          include dsl
+        end.new(method(:add_check))
+      else
+        add_check('describe', values, block)
+      end
     end
 
     def expect(value, &block)
-      target = ExpectationTarget.new(value, &block)
+      target = Inspec::Expect.new(value, &block)
-      @checks.push(['expect', [value], target])
+      add_check('expect', [value], target)
       target
     end
 
@@ -148,6 +131,10 @@ module Inspec
 
     private
 
+    def add_check(describe_or_expect, values, block)
+      @checks.push([describe_or_expect, values, block])
+    end
+
     # Idio(ma)tic unindent
     # TODO: replace this
     #

lib/inspec/runner.rb

@@ -13,7 +13,7 @@ require 'inspec/metadata'
 # spec requirements
 
 module Inspec
-  class Runner
+  class Runner # rubocop:disable Metrics/ClassLength
     extend Forwardable
     attr_reader :backend, :rules
     def initialize(conf = {})
@@ -96,13 +96,17 @@ module Inspec
 
     private
 
-    def get_check_example(method_name, arg, block)
+    def block_source_info(block)
+      return {} if block.nil? || !block.respond_to?(:source_location)
       opts = {}
-      if !block.nil? && block.respond_to?(:source_location)
+      file_path, line = block.source_location
-        file_path, line = block.source_location
+      opts['file_path'] = file_path
-        opts['file_path'] = file_path
+      opts['line_number'] = line
-        opts['line_number'] = line
+      opts
-      end
+    end
+
+    def get_check_example(method_name, arg, block)
+      opts = block_source_info(block)
 
       if !arg.empty? &&
          arg[0].respond_to?(:resource_skipped) &&
@@ -117,6 +121,16 @@ module Inspec
           return @test_collector.example_group(*arg, opts, &block)
         when 'expect'
           return block.example_group
+        when 'describe.one'
+          tests = arg.map do |x|
+            @test_collector.example_group(x[1][0], block_source_info(x[2]), &x[2])
+          end
+          return nil if tests.empty?
+          ok_tests = tests.find_all(&:run)
+          # return all tests if none succeeds; we will just report full failure
+          return tests if ok_tests.empty?
+          # otherwise return all working tests
+          return ok_tests
         else
           fail "A rule was registered with #{method_name.inspect}, "\
                "which isn't understood and cannot be processed."
@@ -128,10 +142,11 @@ module Inspec
     def register_rule(rule_id, rule)
       @rules[rule_id] = rule
       checks = rule.instance_variable_get(:@checks)
-      checks.each do |m, a, b|
+      examples = checks.map do |m, a, b|
-        # resource skipping
+        get_check_example(m, a, b)
-        example = get_check_example(m, a, b)
+      end.flatten.compact
 
+      examples.each do |example|
         # TODO: Remove this!! It is very dangerous to do this here.
         # The goal of this is to make the audit DSL available to all
         # describe blocks. Right now, these blocks are executed outside
@@ -140,7 +155,6 @@ module Inspec
         # scope.
         dsl = Inspec::Resource.create_dsl(backend)
         example.send(:include, dsl)
-
         @test_collector.add_test(example, rule_id)
       end
     end

test/unit/profile_context_test.rb

@@ -5,10 +5,64 @@
 require 'helper'
 require 'inspec/profile_context'
 
+class Module
+  include Minitest::Spec::DSL
+end
+
+module DescribeOneTest
+  it 'loads an empty describe.one' do
+    profile.load(format(context_format, 'describe.one'))
+    get_checks.must_equal([])
+  end
+
+  it 'loads an empty describe.one block' do
+    profile.load(format(context_format, 'describe.one do; end'))
+    get_checks.must_equal([['describe.one', [], nil]])
+  end
+
+  it 'loads a simple describe.one block' do
+    profile.load(format(context_format, '
+      describe.one do
+        describe true do; it { should eq true }; end
+      end'))
+    c = get_checks[0]
+    c[0].must_equal 'describe.one'
+    childs = c[1]
+    childs.length.must_equal 1
+    childs[0][0].must_equal 'describe'
+    childs[0][1].must_equal [true]
+  end
+
+  it 'loads a complex describe.one block' do
+    profile.load(format(context_format, '
+      describe.one do
+        describe 0 do; it { should eq true }; end
+        describe 1 do; it { should eq true }; end
+        describe 2 do; it { should eq true }; end
+      end'))
+    c = get_checks[0]
+    c[0].must_equal 'describe.one'
+    childs = c[1]
+    childs.length.must_equal 3
+    childs.each_with_index do |ci, idx|
+      ci[0].must_equal 'describe'
+      ci[1].must_equal [idx]
+    end
+  end
+end
+
 describe Inspec::ProfileContext do
   let(:backend) { MockLoader.new.backend }
   let(:profile) { Inspec::ProfileContext.new(nil, backend) }
 
+  def get_rule
+    profile.rules.values[0]
+  end
+
+  def get_checks
+    get_rule.instance_variable_get(:@checks)
+  end
+
   it 'must be able to load empty content' do
     profile.load('', 'dummy', 1).must_be_nil
   end
@@ -18,6 +72,10 @@ describe Inspec::ProfileContext do
       proc { profile.load(call) }
     end
 
+    let(:context_format) { '%s' }
+
+    include DescribeOneTest
+
     it 'must provide os resource' do
       load('print os[:family]').must_output 'ubuntu'
     end
@@ -30,6 +88,13 @@ describe Inspec::ProfileContext do
       load('print command("").stdout').must_output ''
     end
 
+    it 'supports empty describe calls' do
+      load('describe').must_output ''
+      profile.rules.keys.length.must_equal 1
+      profile.rules.keys[0].must_match /^\(generated from unknown:1 [0-9a-f]+\)$/
+      profile.rules.values[0].must_be_kind_of Inspec::Rule
+    end
+
     it 'provides the describe keyword in the global DSL' do
       load('describe true do; it { should_eq true }; end')
         .must_output ''
@@ -61,6 +126,13 @@ describe Inspec::ProfileContext do
 
   describe 'rule DSL' do
     let(:rule_id) { rand.to_s }
+    let(:context_format) { "rule #{rule_id.inspect} do\n%s\nend" }
+
+    def get_rule
+      profile.rules[rule_id]
+    end
+
+    include DescribeOneTest
 
     it 'doesnt add any checks if none are provided' do
       profile.load("rule #{rule_id.inspect}")
@@ -68,17 +140,20 @@ describe Inspec::ProfileContext do
       rule.instance_variable_get(:@checks).must_equal([])
     end
 
+    describe 'supports empty describe blocks' do
+      it 'doesnt crash, but doesnt add anything either' do
+        profile.load(format(context_format, 'describe'))
+        profile.rules.keys.must_include(rule_id)
+        get_checks.must_equal([])
+      end
+    end
+
     describe 'adds a check via describe' do
-      let(:cmd) {<<-EOF
-        rule #{rule_id.inspect} do
-          describe os[:family] { it { must_equal 'ubuntu' } }
-        end
-      EOF
-      }
       let(:check) {
-        profile.load(cmd)
+        profile.load(format(context_format,
-        rule = profile.rules[rule_id]
+          "describe os[:family] { it { must_equal 'ubuntu' } }"
-        rule.instance_variable_get(:@checks)[0]
+          ))
+        get_checks[0]
       }
 
       it 'registers the check with describe' do
@@ -95,16 +170,11 @@ describe Inspec::ProfileContext do
     end
 
     describe 'adds a check via expect' do
-      let(:cmd) {<<-EOF
-        rule #{rule_id.inspect} do
-          expect(os[:family]).to eq('ubuntu')
-        end
-      EOF
-      }
       let(:check) {
-        profile.load(cmd)
+        profile.load(format(context_format,
-        rule = profile.rules[rule_id]
+          "expect(os[:family]).to eq('ubuntu')"
-        rule.instance_variable_get(:@checks)[0]
+          ))
+        get_checks[0]
       }
 
       it 'registers the check with describe' do
@@ -116,23 +186,18 @@ describe Inspec::ProfileContext do
       end
 
       it 'registers the check with the provided proc' do
-        check[2].must_be_kind_of Inspec::ExpectationTarget
+        check[2].must_be_kind_of Inspec::Expect
       end
     end
 
     describe 'adds a check via describe + expect' do
-      let(:cmd) {<<-EOF
-        rule #{rule_id.inspect} do
-          describe 'the actual test' do
-            expect(os[:family]).to eq('ubuntu')
-          end
-        end
-      EOF
-      }
       let(:check) {
-        profile.load(cmd)
+        profile.load(format(context_format,
-        rule = profile.rules[rule_id]
+          "describe 'the actual test' do
-        rule.instance_variable_get(:@checks)[0]
+            expect(os[:family]).to eq('ubuntu')
+          end"
+          ))
+        get_checks[0]
       }
 
       it 'registers the check with describe' do