Streaming reporter with progress bar functionality added

Signed-off-by: Nikita Mathur <nikita.mathur@chef.io>

dev-docs/plugins.md

@@ -105,6 +105,10 @@ The entry point is the file that will be `require`d at load time (*not* activati
 require_relative 'inspec-my-plugin/plugin'
 ```
 
+### Types of plugins
+
+Types of plugins that are handled within the loader logic are `bundle`, `core`, `user_gem` or `system_gem`.
+
 ### Plugin Definition File
 
 The plugin definition file uses the plugin DSL to declare a small amount of metadata, followed by as many activation hooks as your plugin needs.

inspec.gemspec

@@ -31,6 +31,9 @@ Gem::Specification.new do |spec|
   spec.add_dependency "cookstyle"
   spec.add_dependency "rake"
 
+  # progress bar streaming reporter plugin support
+  spec.add_dependency "progress_bar", "~> 1.3.3"
+
   # Used for Azure profile until integrated into train
   spec.add_dependency "faraday_middleware", ">= 0.12.2", "< 1.1"
 

lib/inspec/formatters/base.rb

@@ -14,6 +14,8 @@ module Inspec::Formatters
       @profiles = []
       @profiles_info = nil
       @backend = nil
+      @all_controls_count = 0
+      @control_checks_count_map = {}
     end
 
     # RSpec Override: #dump_summary
@@ -79,6 +81,26 @@ module Inspec::Formatters
       @profiles.push(profile)
     end
 
+    # These control count related methods are called via runner rspec library of inspec
+    # And these are used within streaming plugins to determine end of control
+    ######### Start of control count related methods
+    def set_controls_count(controls_count)
+      @all_controls_count = controls_count
+    end
+
+    def set_control_checks_count_map(mapping)
+      @control_checks_count_map = mapping
+    end
+
+    def get_controls_count
+      @all_controls_count
+    end
+
+    def get_control_checks_count_map
+      @control_checks_count_map
+    end
+    ######### end of control count related methods
+
     # Return all the collected output to the caller
     def results
       run_data

lib/inspec/plugin/v2/plugin_types/streaming_reporter.rb

@@ -1,10 +1,49 @@
 module Inspec::Plugin::V2::PluginType
-  class StreamingReporter < Inspec::Plugin::V2::PluginBase # TBD Superclass may need to change
+  class StreamingReporter < Inspec::Plugin::V2::PluginBase
     register_plugin_type(:streaming_reporter)
 
     #====================================================================#
     #                StreamingReporter plugin type API
     #====================================================================#
     # Implementation classes must implement these methods.
+
+    def initialize_streaming_reporter
+      @running_controls_list = []
+      @control_checks_count_map = {}
+      @controls_count = nil
+    end
+
+    private
+
+    # method to identify when the control started running
+    # this will be useful in executing operations on control's level start
+    def control_started?(control_id)
+      if @running_controls_list.include? control_id
+        false
+      else
+        @running_controls_list.push(control_id)
+        true
+      end
+    end
+
+    # method to identify when the control ended running
+    # this will be useful in executing operations on control's level end
+    def control_ended?(control_id)
+      set_control_checks_count_map_value
+      @control_checks_count_map[control_id] -= 1
+      @control_checks_count_map[control_id] == 0
+    end
+
+    # method to identify total no. of controls
+    def controls_count
+      @controls_count ||= RSpec.configuration.formatters.grep(Inspec::Formatters::Base).first.get_controls_count
+    end
+
+    # this method is used in the logic of determining end of control
+    def set_control_checks_count_map_value
+      if @control_checks_count_map.empty?
+        @control_checks_count_map = RSpec.configuration.formatters.grep(Inspec::Formatters::Base).first.get_control_checks_count_map
+      end
+    end
   end
 end

lib/inspec/runner.rb

@@ -126,9 +126,25 @@ module Inspec
         end
       end
 
+      controls_count = 0
+      control_checks_count_map = {}
+
       all_controls.each do |rule|
-        register_rule(rule) unless rule.nil?
+        unless rule.nil?
+          register_rule(rule)
+          checks = ::Inspec::Rule.prepare_checks(rule)
+          unless checks.empty?
+            # controls with empty tests are avoided
+            # checks represent tests within control
+            controls_count += 1
+            control_checks_count_map[rule.to_s] = checks.count
+          end
+        end
       end
+
+      # this sets data via runner-rspec into base RSpec formatter object, which gets used up within streaming plugins
+      @test_collector.set_controls_count(controls_count)
+      @test_collector.set_control_checks_count_map(control_checks_count_map)
     end
 
     def run(with = nil)

lib/inspec/runner_rspec.rb

@@ -42,6 +42,21 @@ module Inspec
       end
     end
 
+    # These control count related methods are called from load logic of runner library of inspec
+    ######### Start of control count related methods
+    def set_controls_count(controls_count)
+      formatters.each do |fmt|
+        fmt.set_controls_count(controls_count)
+      end
+    end
+
+    def set_control_checks_count_map(mapping)
+      formatters.each do |fmt|
+        fmt.set_control_checks_count_map(mapping)
+      end
+    end
+    ######### end of control count related methods
+
     def backend
       formatters.first.backend
     end

lib/plugins/inspec-streaming-reporter-progress-bar/README.md

@@ -0,0 +1,28 @@
+# StreamingReporterProgressBar Plugin
+
+This plugin was generated by `inspec init plugin`, and apparently the author, 'Nikita Mathur', did not update the README.
+
+## To Install This Plugin
+
+Assuming it has been published to RubyGems, you can install this gem using:
+
+```
+you@machine $ inspec plugin install inspec-streaming-reporter-progress-bar
+```
+
+## What This Plugin Does
+
+No idea.
+
+## Developing This Plugin
+
+The generated plugin contains everything a real-world, industrial grade plugin would have, including:
+
+* an (possibly incomplete) implementation of one or more InSpec Plugin Types
+* documentation (you are reading it now)
+* tests, at the unit and functional level
+* a .gemspec, for packaging and publishing it as a gem
+* a Gemfile, for managing its dependencies
+* a Rakefile, for running development tasks
+* Rubocop linting support for using the base InSpec project rubocop.yml (See Rakefile)
+

lib/plugins/inspec-streaming-reporter-progress-bar/lib/inspec-streaming-reporter-progress-bar.rb

@@ -0,0 +1,16 @@
+# This file is known as the "entry point."
+# This is the file InSpec will try to load if it
+# thinks your plugin is installed.
+
+# The *only* thing this file should do is setup the
+# load path, then load the plugin definition file.
+
+# Next two lines simply add the path of the gem to the load path.
+# This is not needed when being loaded as a gem; but when doing
+# plugin development, you may need it.  Either way, it's harmless.
+libdir = File.dirname(__FILE__)
+# ["/Users/nmathur/chef/inspec/vendor/bundle/ruby/2.7.0/gems/ruby-progressbar-1.11.0/lib"]
+# /Users/nmathur/.rbenv/versions/2.7.4/lib/ruby/gems/2.7.0/gems/progress_bar-1.3.3/lib/progress_bar.rb
+$LOAD_PATH.unshift(libdir) unless $LOAD_PATH.include?(libdir)
+
+require "inspec-streaming-reporter-progress-bar/plugin"

lib/plugins/inspec-streaming-reporter-progress-bar/lib/inspec-streaming-reporter-progress-bar/plugin.rb

@@ -0,0 +1,46 @@
+# Plugin Definition file
+# The purpose of this file is to declare to InSpec what plugin_types (capabilities)
+# are included in this plugin, and provide activator that will load them as needed.
+
+# It is important that this file load successfully and *quickly*.
+# Your plugin's functionality may never be used on this InSpec run; so we keep things
+# fast and light by only loading heavy things when they are needed.
+
+# Presumably this is light
+require "inspec-streaming-reporter-progress-bar/version"
+
+# The InspecPlugins namespace is where all plugins should declare themselves.
+# The "Inspec" capitalization is used throughout the InSpec source code; yes, it's
+# strange.
+module InspecPlugins
+  # Pick a reasonable namespace here for your plugin. A reasonable choice
+  # would be the CamelCase version of your plugin gem name.
+  # inspec-streaming-reporter-progress-bar => StreamingReporterProgressBar
+  module StreamingReporterProgressBar
+    # This simple class handles the plugin definition, so calling it simply Plugin is OK.
+    #   Inspec.plugin returns various Classes, intended to be superclasses for various
+    # plugin components. Here, the one-arg form gives you the Plugin Definition superclass,
+    # which mainly gives you access to the activator / plugin_type DSL.
+    #   The number '2' says you are asking for version 2 of the plugin API. If there are
+    # future versions, InSpec promises plugin API v2 will work for at least two more InSpec
+    # major versions.
+    class Plugin < ::Inspec.plugin(2)
+      # Internal machine name of the plugin. InSpec will use this in errors, etc.
+      plugin_name :"inspec-streaming-reporter-progress-bar"
+
+      # Define a new Streaming Reporter.
+      # The argument here will be used to match against the CLI --reporter option.
+      # `--reporter streaming_reporter` will load your streaming reporter and perform streaming real-time on each passing, failing or pending test.
+      streaming_reporter :progress_bar do
+        # Calling this activator doesn't mean the reporter is being executed - just
+        # that we should be ready to do so. So, load the file that defines the
+        # functionality.
+        require "inspec-streaming-reporter-progress-bar/streaming_reporter"
+
+        # Having loaded our functionality, return a class that will let the
+        # reporting engine tap into it.
+        InspecPlugins::StreamingReporterProgressBar::StreamingReporter
+      end
+    end
+  end
+end

lib/plugins/inspec-streaming-reporter-progress-bar/lib/inspec-streaming-reporter-progress-bar/streaming_reporter.rb

@@ -0,0 +1,110 @@
+require "progress_bar"
+module InspecPlugins::StreamingReporterProgressBar
+  # This class will provide the actual Streaming Reporter implementation.
+  # Its superclass is provided by another call to Inspec.plugin,
+  # this time with two args.  The first arg specifies we are requesting
+  # version 2 of the Plugins API.  The second says we are making a
+  # Streaming Reporter plugin component, so please make available any DSL needed
+  # for that.
+
+  class StreamingReporter < Inspec.plugin(2, :streaming_reporter)
+    # Registering these methods with RSpec::Core::Formatters class is mandatory
+    RSpec::Core::Formatters.register self, :example_passed, :example_failed, :example_pending
+
+    case RUBY_PLATFORM
+    when /windows|mswin|msys|mingw|cygwin/
+      # Most currently available Windows terminals have poor support
+      # for ANSI extended colors
+      COLORS = {
+        "failed" => "\033[0;1;31m",
+        "passed" => "\033[0;1;32m",
+        "skipped" => "\033[0;37m",
+        "reset" => "\033[0m",
+      }.freeze
+
+      # Most currently available Windows terminals have poor support
+      # for UTF-8 characters so use these boring indicators
+      INDICATORS = {
+        "failed" => "[FAIL]",
+        "skipped" => "[SKIP]",
+        "passed" => "[PASS]",
+      }.freeze
+    else
+      # Extended colors for everyone else
+      COLORS = {
+        "failed" => "\033[38;5;9m",
+        "passed" => "\033[38;5;41m",
+        "skipped" => "\033[38;5;247m",
+        "reset" => "\033[0m",
+      }.freeze
+
+      # Groovy UTF-8 characters for everyone else...
+      # ...even though they probably only work on Mac
+      INDICATORS = {
+        "failed" => "×",
+        "skipped" => "↺",
+        "passed" => "✔",
+      }.freeze
+    end
+
+    def initialize(output)
+      @bar = nil
+      @status_mapping = {}
+      initialize_streaming_reporter
+    end
+
+    def example_passed(notification)
+      control_id = notification.example.metadata[:id]
+      set_status_mapping(control_id, "passed")
+      show_progress(control_id) if control_ended?(control_id)
+    end
+
+    def example_failed(notification)
+      control_id = notification.example.metadata[:id]
+      set_status_mapping(control_id, "failed")
+      show_progress(control_id) if control_ended?(control_id)
+    end
+
+    def example_pending(notification)
+      control_id = notification.example.metadata[:id]
+      set_status_mapping(control_id, "skipped")
+      show_progress(control_id) if control_ended?(control_id)
+    end
+
+    private
+
+    def show_progress(control_id)
+      @bar ||= ProgressBar.new(controls_count, :bar, :counter, :percentage)
+      sleep 0.1
+      @bar.increment!
+      @bar.puts format_it(control_id)
+    end
+
+    def format_it(control_id)
+      control_status = if @status_mapping[control_id].include? "failed"
+                         "failed"
+                       elsif @status_mapping[control_id].include? "skipped"
+                         "skipped"
+                       elsif @status_mapping[control_id].include? "passed"
+                         "passed"
+                       end
+
+      indicator = INDICATORS[control_status]
+      message_to_format = ""
+      message_to_format += "#{indicator}  "
+      message_to_format += control_id.to_s.lstrip.force_encoding(Encoding::UTF_8)
+      format_with_color(control_status, message_to_format)
+    end
+
+    def format_with_color(color_name, text)
+      "#{COLORS[color_name]}#{text}#{COLORS["reset"]}"
+    end
+
+    # status mapping with control id to decide the final state of the control
+    def set_status_mapping(control_id, status)
+      @status_mapping[control_id] = [] if @status_mapping[control_id].nil?
+      @status_mapping[control_id].push(status)
+    end
+
+  end
+end

lib/plugins/inspec-streaming-reporter-progress-bar/lib/inspec-streaming-reporter-progress-bar/version.rb

@@ -0,0 +1,8 @@
+# This file simply makes it easier for CI engines to update
+# the version stamp, and provide a clean way for the gemspec
+# to learn the current version.
+module InspecPlugins
+  module StreamingReporterProgressBar
+    VERSION = "0.1.0".freeze
+  end
+end

lib/plugins/inspec-streaming-reporter-progress-bar/test/fixtures/README.md

@@ -0,0 +1,24 @@
+# Test Fixtures Area
+
+In this directory, you would place things that you need during testing.  For example, if you were making a plugin that counts the number of controls in a profile, you might have a directory tree like this:
+
+```
+  fixtures/
+    profiles/
+      zero-controls/
+        inspec.yml
+        controls/
+      twelve-controls/
+        inspec.yml
+        controls/
+          nine.rb
+          three.rb
+```
+
+When writing your functional tests, you can point InSpec at the various test fixture profiles, and know that when it points at the zero-controls profile, it should find no controls; and when pointed at the twelve-controls profile, it should find 12.
+
+## Using test fixtures provided with the `inspec` source code
+
+InSpec itself ships with many test fixtures - not just profiles, but attribute files, configuration directories, and more.  Examine them at [the fixtures directory](https://github.com/inspec/inspec/tree/master/test/fixtures)
+
+To use them, see the helper.rb file included in the example at test/helper.rb .

lib/plugins/inspec-streaming-reporter-progress-bar/test/functional/README.md

@@ -0,0 +1,12 @@
+# Functional Testing Area for Plugins
+
+## What are functional tests?
+
+Functional tests are tests that verify that your plugin works _as would be seen by a user_. Functional tests generally do not have inside knowledge about the inner workings of the plugin.  However a functional test is very interested in changes that your plugin makes to the outside world: exit codes, command output, changes to files on the filesystem, etc.
+
+To be picked up by the Rake tasks as tests, each test file should end in `_test.rb`.
+
+## Unit vs Functional Tests
+
+A practical difference between unit tests and functional tests is that unit tests all run within one process, while functional tests might exercise a CLI plugin by shelling out to an `inspec` invocation in a subprocess, and examining the results.
+

lib/plugins/inspec-streaming-reporter-progress-bar/test/helper.rb

@@ -0,0 +1,24 @@
+# Test helper file for non-core plugins
+
+# This file's job is to collect any libraries needed for testing, as well as provide
+# any utilities to make testing a plugin easier.
+
+# InSpec core provides a number of such libraries and facilities, in the file
+# lib/plugins/shared/core_plugin_test_helper.rb . So, one job in this file is
+# to locate and load that file.
+require "inspec/../plugins/shared/core_plugin_test_helper"
+
+# Also load the InSpec plugin system. We need this so we can unit-test the plugin
+# classes, which will rely on the plugin system.
+require "inspec/plugin/v2"
+
+# Caution: loading all of InSpec (i.e. require 'inspec') may cause interference with
+# minitest/spec; one symptom would be appearing to have no tests.
+# See https://github.com/inspec/inspec/issues/3380
+
+# You can select from a number of test harnesses.  Since InSpec uses Spec-style controls
+# in profile code, you will probably want to use something like minitest/spec, which provides
+# Spec-style tests.
+require "minitest/autorun" # loads all styles and runs tests automatically
+
+# You might want to put some debugging tools here.  We run tests to find bugs, after all.

lib/plugins/inspec-streaming-reporter-progress-bar/test/unit/README.md

@@ -0,0 +1,15 @@
+# Unit Testing Area for Plugins
+
+## What Tests are Provided?
+
+ * plugin_def_test.rb - Would be useful in any plugin.  Verifies that the plugin is properly detected and registered.
+ 
+
+## What are Unit Tests?
+
+Unit tests are tests that verify that the individual components of your plugin work as intended.  To be picked up by the Rake tasks as tests, each test file should end in `_test.rb`.
+
+## Unit vs Functional Tests
+
+A practical difference between unit tests and functional tests is that unit tests all run within one process, while functional tests might exercise a CLI plugin by shelling out to an inspec command in a subprocess, and examining the results.
+

lib/plugins/inspec-streaming-reporter-progress-bar/test/unit/plugin_def_test.rb

@@ -0,0 +1,51 @@
+# This unit test performs some tests to verify that
+# the inspec-resource-lister plugin is configured correctly.
+
+# Include our test harness
+require_relative "../helper"
+
+# Load the class under test, the Plugin definition.
+require "inspec-streaming-reporter-progress-bar/plugin"
+
+# Because InSpec is a Spec-style test suite, we're going to use Minitest::Spec
+# here, for familiar look and feel. However, this isn't InSpec (or RSpec) code.
+
+describe InspecPlugins::StreamingReporterProgressBar::Plugin do
+
+  # When writing tests, you can use `let` to create variables that you
+  # can reference easily.
+
+  # Internally, plugins are always known by a Symbol name. Convert here.
+  let(:plugin_name) { :"inspec-streaming-reporter-progress-bar" }
+
+  # The Registry knows about all plugins that ship with InSpec by
+  # default, as well as any that are installed by the user. When a
+  # plugin definition is loaded, it will also self-register.
+  let(:registry) { Inspec::Plugin::V2::Registry.instance }
+
+  # The plugin status record tells us what the Registry knows.
+  # Note that you can use previously-defined 'let's.
+  let(:status) { registry[plugin_name] }
+
+  # OK, actual tests!
+
+  # Does the Registry know about us at all?
+  it "should be registered" do
+    registry.known_plugin?(plugin_name)
+  end
+
+  # Some tests through here use minitest Expectations, which attach to all
+  # Objects, and begin with 'must' (positive) or 'wont' (negative)
+  # See http://docs.seattlerb.org/minitest/Minitest/Expectations.html
+
+  # The plugin system had an undocumented v1 API; this should be a v2 example.
+  it "should be an api-v2 plugin" do
+    status.api_generation.must_equal(2)
+  end
+
+  # Plugins can support several different activator, each of which has a type.
+  # Since this is (primarily) a CliCommand plugin, we'd expect to see that among our types.
+  it "should include a cli_command activator" do
+    status.plugin_types.must_include(:cli_command)
+  end
+end