Merge pull request #5863 from inspec/nm/progress-bar

CFINSPEC-10 Added Progress Bar streaming reporter plugin

dev-docs/plugins.md

@@ -105,6 +105,10 @@ The entry point is the file that will be `require`d at load time (*not* activati
 require_relative 'inspec-my-plugin/plugin'
 ```
 
+### Types of plugins
+
+Types of plugins that are handled within the loader logic are `bundle`, `core`, `user_gem` or `system_gem`.
+
 ### Plugin Definition File
 
 The plugin definition file uses the plugin DSL to declare a small amount of metadata, followed by as many activation hooks as your plugin needs.

docs-chef-io/content/inspec/cli.md

@@ -342,7 +342,7 @@ This subcommand has the following additional options:
 * ``--proxy-command=PROXY_COMMAND``
     Specifies the command to use to connect to the server.
 * ``--reporter=one two:/output/file/path``
-    Enable one or more output reporters: cli, documentation, html, progress, json, json-min, json-rspec, junit, yaml.
+    Enable one or more output reporters: cli, documentation, html, progress, progress-bar, json, json-min, json-rspec, junit, yaml.
 * ``--reporter-backtrace-inclusion``, ``--no-reporter-backtrace-inclusion``
     Include a code backtrace in report data (default: true).
 * ``--reporter-include-source``

docs-chef-io/content/inspec/reporters.md

@@ -90,6 +90,12 @@ Output cli to screen and write json to a file.
     }
 }
 ```
+
+Output real-time progress to screen with a progress bar.
+```bash
+inspec exec example_profile --reporter progress-bar
+```
+
 ## Reporter Options
 
 The following are CLI options that may be used to modify reporter behavior. Many of these options allow you to limit the size of the report, because some reporters (such as the json-automate reporter) have a limit on the total size of the report that can be processed.
@@ -172,6 +178,10 @@ This legacy reporter outputs nonstandard JUnit XML and is provided only for back
 
 This reporter is very condensed and gives you a `.`(pass), `f`(fail), or `*`(skip) character per test and a small summary at the end.
 
+### progress-bar
+
+This reporter outputs real-time progress of a running InSpec profile using a progress bar and prints running control's ID with an indicator of control's status (Passed, failed or skipped).
+
 ### json-rspec
 
 This reporter includes all information from the rspec runner. Unlike the json reporter this includes rspec specific details.

inspec.gemspec

@@ -31,6 +31,9 @@ Gem::Specification.new do |spec|
   spec.add_dependency "cookstyle"
   spec.add_dependency "rake"
 
+  # progress bar streaming reporter plugin support
+  spec.add_dependency "progress_bar", "~> 1.3.3"
+
   # Used for Azure profile until integrated into train
   spec.add_dependency "faraday_middleware", ">= 0.12.2", "< 1.1"
 

lib/inspec/base_cli.rb

@@ -162,7 +162,7 @@ module Inspec
         desc: "A list of tags names that are part of controls to filter and run controls, or a list of /regexes/ to match against tags names of controls. Ignore all other tests."
       option :reporter, type: :array,
         banner: "one two:/output/file/path",
-        desc: "Enable one or more output reporters: cli, documentation, html, progress, json, json-min, json-rspec, junit, yaml"
+        desc: "Enable one or more output reporters: cli, documentation, html, progress, progress-bar, json, json-min, json-rspec, junit, yaml"
       option :reporter_message_truncation, type: :string,
         desc: "Number of characters to truncate failure messages and code_desc in report data to (default: no truncation)"
       option :reporter_backtrace_inclusion, type: :boolean,

lib/inspec/formatters/base.rb

@@ -15,6 +15,8 @@ module Inspec::Formatters
       @profiles = []
       @profiles_info = nil
       @backend = nil
+      @all_controls_count = nil
+      @control_checks_count_map = {}
     end
 
     # RSpec Override: #dump_summary
@@ -80,6 +82,26 @@ module Inspec::Formatters
       @profiles.push(profile)
     end
 
+    # These control count related methods are called via runner rspec library of inspec
+    # And these are used within streaming plugins to determine end of control
+    ######### Start of control count related methods
+    def set_controls_count(controls_count)
+      @all_controls_count = controls_count
+    end
+
+    def set_control_checks_count_map(mapping)
+      @control_checks_count_map = mapping
+    end
+
+    def get_controls_count
+      @all_controls_count
+    end
+
+    def get_control_checks_count_map
+      @control_checks_count_map
+    end
+    ######### end of control count related methods
+
     # Return all the collected output to the caller
     def results
       run_data

lib/inspec/plugin/v2/plugin_types/streaming_reporter.rb

@@ -1,10 +1,53 @@
 module Inspec::Plugin::V2::PluginType
-  class StreamingReporter < Inspec::Plugin::V2::PluginBase # TBD Superclass may need to change
+  class StreamingReporter < Inspec::Plugin::V2::PluginBase
     register_plugin_type(:streaming_reporter)
 
     #====================================================================#
     #                StreamingReporter plugin type API
     #====================================================================#
     # Implementation classes must implement these methods.
+
+    def initialize_streaming_reporter
+      @running_controls_list = []
+      @control_checks_count_map = {}
+      @controls_count = nil
+    end
+
+    private
+
+    # method to identify when the control started running
+    # this will be useful in executing operations on control's level start
+    def control_started?(control_id)
+      if @running_controls_list.include? control_id
+        false
+      else
+        @running_controls_list.push(control_id)
+        true
+      end
+    end
+
+    # method to identify when the control ended running
+    # this will be useful in executing operations on control's level end
+    def control_ended?(control_id)
+      set_control_checks_count_map_value
+      unless @control_checks_count_map[control_id].nil?
+        @control_checks_count_map[control_id] -= 1
+        @control_checks_count_map[control_id] == 0
+      else
+        false
+      end
+    end
+
+    # method to identify total no. of controls
+    def controls_count
+      @controls_count ||= RSpec.configuration.formatters.grep(Inspec::Formatters::Base).first.get_controls_count
+    end
+
+    # this method is used in the logic of determining end of control
+    def set_control_checks_count_map_value
+      if @control_checks_count_map.empty?
+        @control_checks_count_map = RSpec.configuration.formatters.grep(Inspec::Formatters::Base).first.get_control_checks_count_map
+      end
+    end
   end
 end

lib/inspec/runner.rb

@@ -126,9 +126,25 @@ module Inspec
         end
       end
 
+      controls_count = 0
+      control_checks_count_map = {}
+
       all_controls.each do |rule|
-        register_rule(rule) unless rule.nil?
+        unless rule.nil?
+          register_rule(rule)
+          checks = ::Inspec::Rule.prepare_checks(rule)
+          unless checks.empty?
+            # controls with empty tests are avoided
+            # checks represent tests within control
+            controls_count += 1
+            control_checks_count_map[rule.to_s] = checks.count
+          end
+        end
       end
+
+      # this sets data via runner-rspec into base RSpec formatter object, which gets used up within streaming plugins
+      @test_collector.set_controls_count(controls_count)
+      @test_collector.set_control_checks_count_map(control_checks_count_map)
     end
 
     def run(with = nil)

lib/inspec/runner_rspec.rb

@@ -42,6 +42,21 @@ module Inspec
       end
     end
 
+    # These control count related methods are called from load logic of runner library of inspec
+    ######### Start of control count related methods
+    def set_controls_count(controls_count)
+      formatters.each do |fmt|
+        fmt.set_controls_count(controls_count)
+      end
+    end
+
+    def set_control_checks_count_map(mapping)
+      formatters.each do |fmt|
+        fmt.set_control_checks_count_map(mapping)
+      end
+    end
+    ######### end of control count related methods
+
     def backend
       formatters.first.backend
     end

lib/plugins/inspec-streaming-reporter-progress-bar/README.md

@@ -0,0 +1,5 @@
+# StreamingReporterProgressBar Plugin
+
+## What This Plugin Does
+
+This plugin is a streaming reporter plugin which shows the real-time progress of a running InSpec profile using a progress bar. It also outputs the ID of a running control with an indicator showing if the control has passed, failed or skipped.

lib/plugins/inspec-streaming-reporter-progress-bar/lib/inspec-streaming-reporter-progress-bar.rb

@@ -0,0 +1,15 @@
+# This file is known as the "entry point."
+# This is the file InSpec will try to load if it
+# thinks your plugin is installed.
+
+# The *only* thing this file should do is setup the
+# load path, then load the plugin definition file.
+
+# Next two lines simply add the path of the gem to the load path.
+# This is not needed when being loaded as a gem; but when doing
+# plugin development, you may need it.  Either way, it's harmless.
+libdir = File.dirname(__FILE__)
+
+$LOAD_PATH.unshift(libdir) unless $LOAD_PATH.include?(libdir)
+
+require "inspec-streaming-reporter-progress-bar/plugin"

lib/plugins/inspec-streaming-reporter-progress-bar/lib/inspec-streaming-reporter-progress-bar/plugin.rb

@@ -0,0 +1,13 @@
+require "inspec-streaming-reporter-progress-bar/version"
+
+module InspecPlugins
+  module StreamingReporterProgressBar
+    class Plugin < ::Inspec.plugin(2)
+      plugin_name :"inspec-streaming-reporter-progress-bar"
+      streaming_reporter :"progress-bar" do
+        require "inspec-streaming-reporter-progress-bar/streaming_reporter"
+        InspecPlugins::StreamingReporterProgressBar::StreamingReporter
+      end
+    end
+  end
+end

lib/plugins/inspec-streaming-reporter-progress-bar/lib/inspec-streaming-reporter-progress-bar/streaming_reporter.rb

@@ -0,0 +1,112 @@
+require "progress_bar"
+module InspecPlugins::StreamingReporterProgressBar
+  # This class will provide the actual Streaming Reporter implementation.
+  # Its superclass is provided by another call to Inspec.plugin,
+  # this time with two args.  The first arg specifies we are requesting
+  # version 2 of the Plugins API.  The second says we are making a
+  # Streaming Reporter plugin component, so please make available any DSL needed
+  # for that.
+
+  class StreamingReporter < Inspec.plugin(2, :streaming_reporter)
+    # Registering these methods with RSpec::Core::Formatters class is mandatory
+    RSpec::Core::Formatters.register self, :example_passed, :example_failed, :example_pending
+
+    case RUBY_PLATFORM
+    when /windows|mswin|msys|mingw|cygwin/
+      # Most currently available Windows terminals have poor support
+      # for ANSI extended colors
+      COLORS = {
+        "failed" => "\033[0;1;31m",
+        "passed" => "\033[0;1;32m",
+        "skipped" => "\033[0;37m",
+        "reset" => "\033[0m",
+      }.freeze
+
+      # Most currently available Windows terminals have poor support
+      # for UTF-8 characters so use these boring indicators
+      INDICATORS = {
+        "failed" => "[FAIL]",
+        "skipped" => "[SKIP]",
+        "passed" => "[PASS]",
+      }.freeze
+    else
+      # Extended colors for everyone else
+      COLORS = {
+        "failed" => "\033[38;5;9m",
+        "passed" => "\033[38;5;41m",
+        "skipped" => "\033[38;5;247m",
+        "reset" => "\033[0m",
+      }.freeze
+
+      # Groovy UTF-8 characters for everyone else...
+      # ...even though they probably only work on Mac
+      INDICATORS = {
+        "failed" => "×",
+        "skipped" => "↺",
+        "passed" => "✔",
+      }.freeze
+    end
+
+    def initialize(output)
+      @bar = nil
+      @status_mapping = {}
+      initialize_streaming_reporter
+    end
+
+    def example_passed(notification)
+      control_id = notification.example.metadata[:id]
+      set_status_mapping(control_id, "passed")
+      show_progress(control_id) if control_ended?(control_id)
+    end
+
+    def example_failed(notification)
+      control_id = notification.example.metadata[:id]
+      set_status_mapping(control_id, "failed")
+      show_progress(control_id) if control_ended?(control_id)
+    end
+
+    def example_pending(notification)
+      control_id = notification.example.metadata[:id]
+      set_status_mapping(control_id, "skipped")
+      show_progress(control_id) if control_ended?(control_id)
+    end
+
+    private
+
+    def show_progress(control_id)
+      @bar ||= ProgressBar.new(controls_count, :bar, :counter, :percentage)
+      sleep 0.1
+      @bar.increment!
+      @bar.puts format_it(control_id)
+    rescue Exception => ex
+      raise "Exception in Progress Bar streaming reporter: #{ex}"
+    end
+
+    def format_it(control_id)
+      control_status = if @status_mapping[control_id].include? "failed"
+                         "failed"
+                       elsif @status_mapping[control_id].include? "skipped"
+                         "skipped"
+                       elsif @status_mapping[control_id].include? "passed"
+                         "passed"
+                       end
+
+      indicator = INDICATORS[control_status]
+      message_to_format = ""
+      message_to_format += "#{indicator}  "
+      message_to_format += control_id.to_s.lstrip.force_encoding(Encoding::UTF_8)
+      format_with_color(control_status, message_to_format)
+    end
+
+    def format_with_color(color_name, text)
+      "#{COLORS[color_name]}#{text}#{COLORS["reset"]}"
+    end
+
+    # status mapping with control id to decide the final state of the control
+    def set_status_mapping(control_id, status)
+      @status_mapping[control_id] = [] if @status_mapping[control_id].nil?
+      @status_mapping[control_id].push(status)
+    end
+
+  end
+end

lib/plugins/inspec-streaming-reporter-progress-bar/lib/inspec-streaming-reporter-progress-bar/version.rb

@@ -0,0 +1,8 @@
+# This file simply makes it easier for CI engines to update
+# the version stamp, and provide a clean way for the gemspec
+# to learn the current version.
+module InspecPlugins
+  module StreamingReporterProgressBar
+    VERSION = "0.1.0".freeze
+  end
+end

test/functional/inspec_exec_streaming_progress_bar_test.rb

@@ -0,0 +1,80 @@
+require "functional/helper"
+
+describe "inspec exec with streaming progress bar reporter" do
+  include FunctionalHelper
+
+  parallelize_me!
+
+  it "can execute a simple file and validate the streaming progress bar schema" do
+    skip_windows!
+
+    out = inspec("exec " + example_control + " --reporter progress-bar --no-create-lockfile")
+    _(out.stderr).must_include "[100.00%]"
+    _(out.stderr).must_include "[38;5;41m"
+    _(out.stderr).wont_include "[38;5;9m"
+    assert_exit_code 0, out
+  end
+
+  it "can execute a simple file while using end of options after reporter streaming progress bar option" do
+    skip_windows!
+
+    out = inspec("exec --no-create-lockfile --reporter progress-bar -- " + example_control)
+    _(out.stderr).must_include "[100.00%]"
+    _(out.stderr).must_include "[38;5;41m"
+    _(out.stderr).wont_include "[38;5;9m"
+    assert_exit_code 0, out
+  end
+
+  it "can execute a profile with dependent profiles" do
+    profile = File.join(profile_path, "dependencies", "inheritance")
+    out = inspec("exec " + profile + " --reporter progress-bar --no-create-lockfile")
+    _(out.stderr).must_include "[100.00%]"
+    _(out.stderr).must_include "[6/6]"
+    assert_exit_code 0, out
+  end
+
+  it "can execute a profile with --tags filters" do
+    profile = File.join(profile_path, "control-tags")
+    out = inspec("exec " + profile + " --tags tag1 --reporter progress-bar --no-create-lockfile")
+    _(out.stderr).must_include "[100.00%]"
+    _(out.stderr).must_include "[1/1]"
+    assert_exit_code 0, out
+  end
+
+  it "can execute a profile with --controls filters" do
+    out = inspec("exec " + File.join(profile_path, "controls-option-test") + " --no-create-lockfile --controls foo --reporter progress-bar")
+    _(out.stderr).must_include "[100.00%]"
+    _(out.stderr).must_include "[1/1]"
+    assert_exit_code 0, out
+  end
+
+  it "can execute multiple profiles" do
+    out = inspec("exec " + File.join(profile_path, "dependencies", "inheritance") + " " + File.join(profile_path, "controls-option-test") + " --no-create-lockfile --reporter progress-bar")
+    _(out.stderr).must_include "[100.00%]"
+    _(out.stderr).must_include "[11/11]"
+    assert_exit_code 0, out
+  end
+
+  it "can execute and print proper output when tests are failed" do
+    skip_windows!
+
+    out = inspec("exec " + File.join(profile_path, "control-tags") + " --tags tag18 --no-create-lockfile --reporter progress-bar")
+    _(out.stderr).must_include "[100.00%]"
+    _(out.stderr).must_include "[38;5;9m"
+    _(out.stderr).wont_include "[38;5;247m"
+    _(out.stderr).wont_include "[38;5;41m"
+    assert_exit_code 100, out
+  end
+
+  it "can execute and print proper output when tests are skipped" do
+    skip_windows!
+
+    out = inspec("exec " + File.join(profile_path, "skippy-controls") + " --no-create-lockfile --reporter progress-bar")
+    _(out.stderr).must_include "[100.00%]"
+    _(out.stderr).must_include "[38;5;247m"
+    _(out.stderr).wont_include "[38;5;41m"
+    _(out.stderr).wont_include "[38;5;9m"
+    assert_exit_code 101, out
+  end
+
+end