inspec/test/functional/inspec_exec_test.rb

require "functional/helper"
require "helpers/mock_loader"

describe "inspec exec" do
  parallelize_me!

  include FunctionalHelper
  let(:looks_like_a_stacktrace) { %r{lib/inspec/.+\.rb:\d+:in} }

  attr_accessor :out

  def inspec(commandline, prefix = nil)
    @stdout = @stderr = nil
    self.out = super
  end

  def stdout
    @stdout ||= out.stdout
      .force_encoding(Encoding::UTF_8)
      .gsub(/\e\[(\d+)(;\d+)*m/, "") # strip ANSI color codes
  end

  def stderr
    @stderr ||= out.stderr
      .force_encoding(Encoding::UTF_8)
      .gsub(/\e\[(\d+)(;\d+)*m/, "") # strip ANSI color codes
  end

  before do
    prof = "test/fixtures/profiles"
    FileUtils.rm_f "#{prof}/aws-profile/inspec.lock"
    FileUtils.rm_f "#{prof}/simple-inheritance/inspec.lock"
    FileUtils.rm_f "#{prof}/simple-metadata/inspec.lock"
  end

  it "handles '=' character on input" do
    # This test handles a bug discovered at: https://github.com/inspec/inspec/issues/5131
    inspec(
      "exec " +
      File.join(profile_path, "inputs", "with_various_values") +
      " --no-create-lockfile" +
      " --input my_input='ab=cde'"
    )

    _(stdout).must_include "0 failures"
  end

  it "can execute the profile" do
    inspec("exec " + complete_profile + " --no-create-lockfile")

    _(stdout).must_include "Host example.com"
    _(stdout).must_include "1 successful control, "\
      "0 control failures, 0 controls skipped"
    _(stderr).must_be_empty

    assert_exit_code 0, out
  end

  it "executes a minimum metadata-only profile" do
    inspec("exec " + File.join(profile_path, "simple-metadata") + " --no-create-lockfile")

    _(stdout).must_equal "
Profile: yumyum profile
Version: (not specified)
Target:  local://

     No tests executed.

Test Summary: 0 successful, 0 failures, 0 skipped
"
    _(stderr).must_equal ""

    assert_exit_code 0, out
  end

  it "can execute the profile and write to directory" do
    outpath = Dir.tmpdir
    inspec("exec #{complete_profile} --no-create-lockfile --reporter json:#{outpath}/foo/bar/test.json")

    _(File.exist?("#{outpath}/foo/bar/test.json")).must_equal true
    _(File.stat("#{outpath}/foo/bar/test.json").size).must_be :>, 0

    _(stderr).must_equal ""

    assert_exit_code 0, out
  end

  it "can execute --help after exec command" do
    inspec("exec --help")

    _(stdout).must_include "Usage:\n  inspec exec LOCATIONS"

    _(stderr).must_equal ""

    assert_exit_code 0, out
  end

  it "can execute help after exec command" do
    inspec("exec help")

    _(stdout).must_include "Usage:\n  inspec exec LOCATIONS"

    _(stderr).must_equal ""

    assert_exit_code 0, out
  end

  it "can execute help before exec command" do
    inspec("help exec")

    _(stdout).must_include "Usage:\n  inspec exec LOCATIONS"

    _(stderr).must_equal ""

    assert_exit_code 0, out
  end

  it "can execute the profile with a target_id passthrough" do
    inspec("exec #{complete_profile} --no-create-lockfile --target-id 1d3e399f-4d71-4863-ac54-84d437fbc444")

    _(stdout).must_include "Target ID: 1d3e399f-4d71-4863-ac54-84d437fbc444"

    _(stderr).must_equal ""

    assert_exit_code 0, out
  end

  it "executes a metadata-only profile" do
    inspec("exec " + File.join(profile_path, "complete-metadata") + " --no-create-lockfile")

    _(stdout).must_equal "
Profile: title (name)
Version: 1.2.3
Target:  local://

     No tests executed.

Test Summary: 0 successful, 0 failures, 0 skipped
"

    _(stderr).must_equal ""

    assert_exit_code 0, out
  end

  it "executes a profile and reads inputs" do
    inspec("exec #{File.join(examples_path, "profile-attribute")} --no-create-lockfile --input-file #{File.join(examples_path, "profile-attribute.yml")}")

    _(stdout).must_include "Test Summary: 2 successful, 0 failures, 0 skipped"

    _(stderr).must_equal ""

    assert_exit_code 0, out
  end

  it "executes a specs-only profile" do
    inspec("exec " + File.join(profile_path, "spec_only") + " --no-create-lockfile")
    _(stdout).must_include "Target:  local://"
    _(stdout).must_include "working"
    _(stdout).must_include "✔  is expected to eq \"working\""
    _(stdout).must_include "skippy\n"
    _(stdout).must_include "↺  This will be skipped intentionally"
    _(stdout).must_include "failing"
    _(stdout).must_include "×  is expected to eq \"as intended\""
    _(stdout).must_include "Test Summary: 1 successful, 1 failure, 1 skipped\n"

    _(stderr).must_equal ""

    assert_exit_code 100, out
  end

  it "executes only specified controls when selecting the controls by literal names" do
    inspec("exec " + File.join(profile_path, "controls-option-test") + " --no-create-lockfile --controls foo")
    _(out.stdout).must_include "foo"
    _(out.stdout).wont_include "bar"
    _(out.stdout).wont_include "only-describe"
    _(stderr).must_equal ""

    assert_exit_code 0, out
  end

  it "executes only specified controls when selecting the controls by regex" do
    inspec("exec " + File.join(profile_path, "controls-option-test") + " --no-create-lockfile --controls '/^11_pass/'")
    _(out.stdout).must_include "11_pass"
    _(out.stdout).must_include "11_pass2"
    _(out.stdout).wont_include "bar"
    _(out.stdout).wont_include "only-describe"
    _(stderr).must_equal ""

    assert_exit_code 0, out
  end

  # it filters the control from its dependent profile_c
  it "executes only specified controls from parent and child profile when selecting the controls by regex" do
    inspec("exec " + File.join(profile_path, "dependencies", "profile_a") + " --no-create-lockfile --controls '/^profilec/'")
    _(out.stdout).must_include "profilec-1"
    _(out.stdout).wont_include "profilea-1"
    _(out.stdout).wont_include "only-describe"
    _(stderr).must_equal ""

    assert_exit_code 0, out
  end

  # it filters the control from its dependent profile_c
  it "executes only specified controls from parent and child profile when selecting the controls by id" do
    inspec("exec " + File.join(profile_path, "dependencies", "profile_a") + " --no-create-lockfile --controls 'profilec-1'")
    _(out.stdout).must_include "profilec-1"
    _(out.stdout).wont_include "profilea-1"
    _(out.stdout).wont_include "only-describe"
    _(stderr).must_equal ""

    assert_exit_code 0, out
  end

  # it filters the control from its dependent profile_c
  it "executes only specified controls from parent and child profile when selecting the controls by space seprated id" do
    inspec("exec " + File.join(profile_path, "dependencies", "profile_a") + " --no-create-lockfile --controls 'profilec-1' 'profilea-1'")
    _(out.stdout).must_include "profilec-1"
    _(out.stdout).must_include "profilea-1"
    _(out.stdout).wont_include "profilea-2"
    _(out.stdout).wont_include "only-describe"
    _(stderr).must_equal ""

    assert_exit_code 0, out
  end

  # it filters the control from its dependent profile_c
  it "executes only specified controls of required dependent profile when selecting the controls by space seprated id" do
    inspec("exec " + File.join(profile_path, "dependencies", "require_controls_test") + " --no-create-lockfile --controls 'profileb-2'")
    _(out.stdout).must_include "profileb-2"
    _(out.stdout).wont_include "profilea-1"
    _(out.stdout).wont_include "profilea-2"
    _(out.stdout).wont_include "only-describe"
    _(stderr).must_equal ""

    assert_exit_code 0, out
  end

  it "executes only specified controls when selecting passing controls by literal names" do
    inspec("exec " + File.join(profile_path, "filter_table") + " --no-create-lockfile --controls 2943_pass_undeclared_field_in_hash 2943_pass_irregular_row_key")

    _(stdout).must_include "\nProfile Summary: 2 successful controls, 0 control failures, 0 controls skipped\n"

    _(stderr).must_equal ""

    assert_exit_code 0, out
  end

  it "executes only specified controls when selecting failing controls by literal names" do
    inspec("exec " + File.join(profile_path, "filter_table") + " --no-create-lockfile --controls 2943_fail_derail_check")

    _(stdout).must_include "\nProfile Summary: 0 successful controls, 1 control failure, 0 controls skipped"

    _(stderr).must_equal ""

    assert_exit_code 100, out
  end

  it "executes only specified controls when selecting passing controls by regex" do
    inspec("exec " + File.join(profile_path, "filter_table") + " --no-create-lockfile --controls '/^2943_pass/'")

    _(stdout).must_include "Profile Summary: 6 successful controls, 0 control failures, 0 controls skipped"

    assert_exit_code 0, out
  end

  it "executes only specified controls when selecting failing controls by regex" do
    inspec("exec " + File.join(profile_path, "filter_table") + " --no-create-lockfile --controls '/^2943_fail/'")

    _(stdout).must_include "Profile Summary: 0 successful controls, 1 control failure, 0 controls skipped"

    _(stderr).must_equal ""

    assert_exit_code 100, out
  end

  it "executes only specified controls when selecting the controls by literal single tag name" do
    inspec("exec " + File.join(profile_path, "control-tags") + " --no-create-lockfile --tags tag1")
    _(stdout).must_include "true is expected to eq true\n"
    _(stdout).must_include "Test Summary: 1 successful, 0 failures, 0 skipped\n"
    _(stderr).must_equal ""

    assert_exit_code 0, out
  end

  it "executes only specified controls when selecting the controls by literal multiple tag names" do
    inspec("exec " + File.join(profile_path, "control-tags") + " --no-create-lockfile --tags tag1 tag5 tag6 tag17 'tagname with space'")
    _(stdout).must_include "true is expected to eq true\n"
    _(stdout).must_include "Test Summary: 4 successful, 0 failures, 0 skipped\n"
    _(stderr).must_equal ""

    assert_exit_code 0, out
  end

  it "executes only specified controls when selecting the controls by using regex on tags" do
    inspec("exec " + File.join(profile_path, "control-tags") + " --no-create-lockfile --tags '/\s+/'")
    _(stdout).must_include "true is expected to eq true\n"
    _(stdout).must_include "Test Summary: 2 successful, 0 failures, 0 skipped\n"
    _(stderr).must_equal ""

    assert_exit_code 0, out
  end

  it "executes only specified controls when selecting failing controls by using literal name of tag" do
    inspec("exec " + File.join(profile_path, "control-tags") + " --no-create-lockfile --tags tag18")
    _(stdout).must_include "true is expected to eq false\n"
    _(stdout).must_include "Test Summary: 0 successful, 1 failure, 0 skipped\n"
    _(stderr).must_equal ""

    assert_exit_code 100, out
  end

  it "executes only specified controls when selecting failing controls by using regex on tags" do
    inspec("exec " + File.join(profile_path, "control-tags") + " --no-create-lockfile --tags '/(18)/'")
    _(stdout).must_include "true is expected to eq false\n"
    _(stdout).must_include "Test Summary: 0 successful, 1 failure, 0 skipped\n"
    _(stderr).must_equal ""

    assert_exit_code 100, out
  end

  it "executes profile successfully when tags are used with single element array, punctuations and linefeeds" do
    inspec("exec " + File.join(profile_path, "control-tags") + " --no-create-lockfile --tags tag1 'Line with a comma,error' CCI-000366")
    _(stdout).must_include "true is expected to eq true\n"
    _(stdout).must_include "Test Summary: 1 successful, 0 failures, 0 skipped\n"
    _(stderr).must_equal ""

    assert_exit_code 0, out
  end

  it "executes only specified controls of included dependent profile by using literal names of tags" do
    inspec("exec " + File.join(profile_path, "dependencies", "profile_a") + " --no-create-lockfile --tags tag-profilea1 tag-profilec1")
    _(stdout).must_include "✔  profilea-1: Create / directory\n"
    _(stdout).must_include "✔  profilec-1: Create /tmp directory\n"
    _(stdout).must_include "✔  File / is expected to be directory\n"
    _(stdout).wont_include "✔  profilea-2: example_config\n"
    _(stdout).must_include "Test Summary: 2 successful, 0 failures, 0 skipped\n"
    _(stderr).must_equal ""

    assert_exit_code 0, out
  end

  it "executes only specified controls of included dependent profile by using regex on tags" do
    inspec("exec " + File.join(profile_path, "dependencies", "profile_a") + " --no-create-lockfile --tags '/^tag-profilea/'")
    _(stdout).must_include "✔  profilea-1: Create / directory\n"
    _(stdout).must_include "✔  profilea-2: example_config\n"
    _(stdout).wont_include "✔  profilec-1: Create /tmp directory\n"
    _(stdout).must_include "Test Summary: 2 successful, 0 failures, 0 skipped\n"
    _(stderr).must_equal ""

    assert_exit_code 0, out
  end

  it "executes only specified controls of required dependent profile by using literal names of tags" do
    inspec("exec " + File.join(profile_path, "dependencies", "require_controls_test") + " --no-create-lockfile --tags tag-profileb2")
    _(stdout).must_include "✔  profileb-2: example_config\n"
    _(stdout).must_include "✔  example_config version is expected to eq \"2.0\"\n"
    _(stdout).wont_include "✔  profilea-1: Create / directory\n"
    _(stdout).wont_include "✔  profilea-2: example_config\n"
    _(stdout).must_include "Test Summary: 2 successful, 0 failures, 0 skipped\n"
    _(stderr).must_equal ""

    assert_exit_code 0, out
  end

  it "executes only specified controls of required dependent profile by using regex on tags" do
    inspec("exec " + File.join(profile_path, "dependencies", "require_controls_test") + " --no-create-lockfile --tags '/^tag-profileb/'")
    _(stdout).must_include "✔  profileb-2: example_config\n"
    _(stdout).must_include "✔  example_config version is expected to eq \"2.0\"\n"
    _(stdout).wont_include "✔  profilea-1: Create / directory\n"
    _(stdout).wont_include "✔  profilea-2: example_config\n"
    _(stdout).must_include "Test Summary: 2 successful, 0 failures, 0 skipped\n"
    _(stderr).must_equal ""

    assert_exit_code 0, out
  end

  it "reports whan a profile cannot be loaded" do
    inspec("exec " + File.join(profile_path, "raise_outside_control") + " --no-create-lockfile")
    _(stdout).must_match(/Profile:[\W]+InSpec Profile \(raise_outside_control\)/)

    _(stdout).must_include "ERROR: Failed to load profile raise_outside_control: Failed to load source for controls/raises.rb: Something unforeseen..."

    assert_exit_code 102, out
  end

  it "can execute a simple file with the default formatter" do
    inspec("exec " + example_control + " --no-create-lockfile")

    _(stdout).must_include "\nProfile Summary: 1 successful control, 0 control failures, 0 controls skipped\n"
    _(stdout).must_include "\nTest Summary: 2 successful, 0 failures, 0 skipped\n"

    _(stderr).must_equal ""

    assert_exit_code 0, out
  end

  it "does not vendor profiles when using the a local path dependecy" do
    Dir.mktmpdir do |tmpdir|
      command = "exec " + inheritance_profile + " --no-create-lockfile " \
        "--input-file=#{examples_path}/profile-attribute.yml"
      inspec_with_env(command, INSPEC_CONFIG_DIR: tmpdir)

      if is_windows?
        _(stdout).must_include "No tests executed."
        assert_exit_code 1, out
      else
        _(stdout).must_include "Profile Summary: 2 successful controls, 0 control failures, 0 controls skipped\n"
        _(stdout).must_include "Test Summary: 5 successful, 0 failures, 0 skipped\n"
        assert_exit_code 0, out
      end

      cache_dir = File.join(tmpdir, "cache")
      _(Dir.exist?(cache_dir)).must_equal true
      _(Dir.glob(File.join(cache_dir, "**", "*"))).must_be_empty

      _(stderr).must_equal ""
    end
  end

  describe "with a profile that is not supported on this OS/platform" do
    let(:out) { inspec("exec " + File.join(profile_path, "skippy-profile-os") + " --no-create-lockfile") }
    let(:json) { JSON.load(stdout) }

    it "exits with skip message" do
      _(stdout).must_include("Skipping profile: 'skippy' on unsupported platform:")

      _(stderr).must_equal ""

      assert_exit_code 101, out
    end
  end

  describe "with a profile that contains skipped controls" do
    let(:out) { inspec("exec " + File.join(profile_path, "skippy-controls") + " --no-create-lockfile") }
    let(:json) { JSON.load(stdout) }

    it "exits with an error" do
      _(stdout).must_include "skippy\n     ↺  This will be skipped super intentionally.\n"
      _(stdout).must_include "Profile Summary: 0 successful controls, 0 control failures, 1 control skipped\nTest Summary: 0 successful, 0 failures, 1 skipped\n"

      _(stderr).must_equal ""

      assert_exit_code 101, out
    end
  end

  describe "with a profile that contains skipped controls and the --no-distinct-exit flag" do
    let(:out) { inspec("exec " + File.join(profile_path, "skippy-controls") + " --no-distinct-exit --no-create-lockfile") }

    it "exits with code 0 and skipped tests in output" do
      _(stdout).must_include "Profile Summary: 0 successful controls, 0 control failures, 1 control skipped\nTest Summary: 0 successful, 0 failures, 1 skipped\n"

      _(stderr).must_equal ""

      assert_exit_code 0, out
    end
  end

  describe "with a profile that contains failing controls and the --no-distinct-exit flag" do
    let(:out) { inspec("exec " + File.join(profile_path, "failures") + " --no-distinct-exit --no-create-lockfile") }

    it "exits with code 1" do
      _(stdout).must_include "Profile Summary: 0 successful controls, 4 control failures, 0 controls skipped"

      _(stderr).must_equal ""

      assert_exit_code 1, out
    end
  end

  describe "with a profile that contains skipped resources" do
    let(:out) { inspec("exec " + File.join(profile_path, "aws-profile")) }
    it "exits with an error" do
      skip if ENV["NO_AWS"]
      _(stdout).must_include "Unsupported resource/backend combination: aws_iam_users"
      _(stdout).must_include "Unsupported resource/backend combination: aws_iam_access_keys"
      _(stdout).must_include "Unsupported resource/backend combination: aws_s3_bucket"
      _(stdout).must_include "3 failures"

      assert_exit_code 100, out
    end
  end

  describe "with a profile that is supported on this version of inspec" do
    let(:out) { inspec("exec " + File.join(profile_path, "supported_inspec") + " --no-create-lockfile") }

    it "exits cleanly" do
      _(stderr).must_equal ""

      assert_exit_code 0, out
    end
  end

  describe "with a profile that is not supported on this version of inspec" do
    let(:out) { inspec("exec " + File.join(profile_path, "unsupported_inspec") + " --no-create-lockfile") }

    it "does not support this profile" do
      _(stderr).must_equal "This profile requires Chef InSpec version >= 99.0.0. You are running Chef InSpec v#{Inspec::VERSION}.\n"

      assert_exit_code 1, out
    end
  end

  describe "with a profile that loads a library and reference" do
    let(:out) { inspec("exec " + File.join(profile_path, "library") + " --no-create-lockfile") }

    it "executes the profile without error" do
      _(stderr).must_equal ""

      assert_exit_code 0, out
    end
  end

  describe "with a profile that inherits core resource into custom reosuce" do
    let(:out) { inspec("exec " + File.join(profile_path, "custom-resource-inheritance") + " --no-create-lockfile") }
    it "executes the custom resoruc without error" do
      _(stdout).must_equal "
Profile: InSpec Profile (custom-resource-inheritance)
Version: 0.1.0
Target:  local://

  Node Json
     ✔  name is expected to eq \"hello\"
     ✔  [\"meta\", \"creator\"] is expected to eq \"John Doe\"

Test Summary: 2 successful, 0 failures, 0 skipped
"
      _(stderr).must_equal ""
      assert_exit_code 0, out
    end
  end

  describe "given a profile with controls and anonymous describe blocks" do
    let(:out) { inspec("exec " + example_control + " --no-create-lockfile") }

    it "prints the control results, then the anonymous describe block results" do
      _(stdout).must_match(/Profile: tests from .*test.fixtures.profiles.old-examples.profile.controls.example-tmp.rb/)
      _(stdout).must_include "
Version: (not specified)
Target:  local://

  \xE2\x9C\x94  tmp-1.0: Create / directory
     \xE2\x9C\x94  File / is expected to be directory

  File /
     \xE2\x9C\x94  is expected to be directory

Profile Summary: 1 successful control, 0 control failures, 0 controls skipped
Test Summary: 2 successful, 0 failures, 0 skipped\n"
    end
  end

  describe "given a profile with an anonymous describe block" do
    let(:out) { inspec("exec " + failure_control + " --no-create-lockfile") }

    it "prints the exception message when a test has a syntax error" do
      _(stdout).must_include "undefined method `should_nota' "
    end
  end

  describe "given an inherited profile that has more that one test per control block" do
    let(:out) { inspec("exec " + simple_inheritance + " --no-create-lockfile") }

    it "should print all the results" do
      _(stdout).must_include "×  tmp-1.0: Create / directory (1 failed)"
      _(stdout).must_include "×  is expected not to be directory\n"
      _(stdout).must_include "×  File / \n     undefined method `should_nota'"
      _(stdout).must_include "×  is expected not to be directory\n     expected `File /.directory?` to be falsey, got true"
      _(stdout).must_include "×  7 is expected to cmp >= 9\n"
      _(stdout).must_include "×  7 is expected not to cmp == /^\\d$/\n"
      _(stdout).must_include "✔  7 is expected to cmp == \"7\""
      _(stdout).must_include "expected: %s" % ["01147"]
      _(stdout).must_include "got: %s" % [is_windows? ? "040755" : "0755"]
    end
  end

  describe "when passing in two profiles given an inherited profile that has more that one test per control block" do
    let(:out) { inspec("exec " + File.join(profile_path, "dependencies", "profile_d") + " " + simple_inheritance + " --no-create-lockfile") }

    it "should print all the results" do
      _(stdout).must_include "×  tmp-1.0: Create / directory (1 failed)"
      _(stdout).must_include "×  cmp-1.0: Using the cmp matcher for numbers (2 failed)"
      _(stdout).must_include "×  File / \n     undefined method `should_nota'"
      _(stdout).must_include "×  is expected not to be directory\n     expected `File /.directory?` to be falsey, got true"
      _(stdout).must_include "✔  profiled-1: Create / directory (profile d)"
    end
  end

  describe "given an inherited profile" do
    let(:out) { inspec("exec " + simple_inheritance) }

    it "should print the profile information and then the test results" do
      _(stdout).must_include "  ×  tmp-1.0: Create / directory (1 failed)\n     ✔  File / is expected to be directory\n     ×  File / is expected not to be directory\n"
    end
  end

  describe "using namespaced resources" do
    it "works" do
      inspec("exec " + File.join(profile_path, "dependencies", "resource-namespace") + " --no-create-lockfile")

      _(stdout).must_include "Profile Summary: 1 successful control, 0 control failures, 0 controls skipped\n"

      _(stderr).must_equal ""

      assert_exit_code 0, out
    end
  end

  describe "with require_controls" do
    it "does not run rules you did not include" do
      inspec("exec " + File.join(profile_path, "dependencies", "require_controls_test") + " --no-create-lockfile")

      _(stdout).must_include "Profile Summary: 1 successful control, 0 control failures, 0 controls skipped\n"

      _(stderr).must_equal ""

      assert_exit_code 0, out
    end
  end

  describe "with a 2-level dependency tree" do
    it "correctly runs tests from the whole tree" do
      inspec("exec " + File.join(profile_path, "dependencies", "inheritance") + " --no-create-lockfile")

      _(stdout).must_include "Profile Summary: 6 successful controls, 0 control failures, 0 controls skipped\n"

      _(stderr).must_equal ""

      assert_exit_code 0, out
    end
  end

  describe "when using profiles on the supermarket" do
    it "can run supermarket profiles directly from the command line" do

      skip_windows! # Breakage confirmed, only on CI: https://buildkite.com/chef-oss/inspec-inspec-master-verify/builds/2355#2c9d032e-4a24-4e7c-aef2-1c9e2317d9e2
      inspec("exec supermarket://nathenharvey/tmp-compliance-profile --no-create-lockfile")

      if is_windows?
        _(stdout).must_include "Profile Summary: 1 successful control, 1 control failure, 0 controls skipped\n"
      else
        _(stdout).must_include "Profile Summary: 2 successful controls, 0 control failures, 0 controls skipped\n"
      end

      _(stderr).must_equal ""

      if is_windows?
        assert_exit_code 100, out # references root
      else
        assert_exit_code 0, out
      end
    end

    it "can run supermarket profiles directly from the command line with --supermarket_url option" do

      skip_windows! # Breakage confirmed, only on CI: https://buildkite.com/chef-oss/inspec-inspec-master-verify/builds/2355#2c9d032e-4a24-4e7c-aef2-1c9e2317d9e2
      inspec("exec supermarket://nathenharvey/tmp-compliance-profile --supermarket_url='https://supermarket.chef.io' --no-create-lockfile")

      if is_windows?
        _(stdout).must_include "Profile Summary: 1 successful control, 1 control failure, 0 controls skipped\n"
      else
        _(stdout).must_include "Profile Summary: 2 successful controls, 0 control failures, 0 controls skipped\n"
      end

      _(stderr).must_equal ""

      if is_windows?
        assert_exit_code 100, out # references root
      else
        assert_exit_code 0, out
      end
    end

    it "can run supermarket profiles from inspec.yml" do
      skip_windows! # Breakage confirmed, only on CI: https://buildkite.com/chef-oss/inspec-inspec-master-verify/builds/2355#2c9d032e-4a24-4e7c-aef2-1c9e2317d9e2

      inspec("exec #{File.join(profile_path, "supermarket-dep")} --no-create-lockfile")

      if is_windows?
        _(stdout).must_include "Profile Summary: 1 successful control, 1 control failure, 0 controls skipped\n"
      else
        _(stdout).must_include "Profile Summary: 2 successful controls, 0 control failures, 0 controls skipped\n"
      end

      _(stderr).must_equal ""

      if is_windows?
        assert_exit_code 1, out
      else
        assert_exit_code 0, out
      end
    end
  end

  describe "when a dependency does not support our backend platform" do
    it "skips the controls from that profile" do
      inspec("exec #{File.join(profile_path, "profile-support-skip")} --no-create-lockfile")

      _(stdout).must_include "WARN: Skipping profile"
      _(stdout).must_include "0 successful, 0 failures, 0 skipped\n"

      _(stderr).must_equal ""

      assert_exit_code 0, out
    end
  end

  describe "when trying to use --sudo with a local target" do
    it "must print an error and exit" do
      inspec("exec #{File.join(profile_path, "profile-support-skip")} --sudo")
      str = "Sudo is only valid when running against a remote host. To run this locally with elevated privileges, run the command with `sudo ...`.\n"

      _(stderr).must_include str

      assert_exit_code 1, out
      # TODO: check for stacktrace
    end
  end

  describe "when --no-color is used" do
    it "does not output color control characters" do
      inspec("exec " + File.join(profile_path, "simple-metadata") + " --no-color")

      _(stdout).wont_include "\e[38"

      _(stderr).must_equal ""

      assert_exit_code 0, out
    end
  end

  describe "when --password is used" do
    it "raises an exception if no password is provided" do
      inspec("exec " + complete_profile + " --password")

      _(stderr).must_include "Please provide a value for --password. For example: --password=hello."

      assert_exit_code 1, out
    end
  end

  describe "when --sudo-password is used" do
    it "raises an exception if no sudo password is provided" do
      inspec("exec " + complete_profile + " --sudo-password")

      _(stderr).must_include "Please provide a value for --sudo-password. For example: --sudo-password=hello."

      assert_exit_code 1, out
    end
  end

  describe "when --bastion-host and --proxy_command is used" do
    it "raises an exception when both flags are provided" do
      inspec("exec " + complete_profile + " -t ssh://dummy@dummy --password dummy --proxy_command dummy --bastion_host dummy")

      _(stderr).must_include "Client error, can't connect to 'ssh' backend: Only one of proxy_command or bastion_host needs to be specified"

      assert_exit_code 1, out
    end
  end

  describe "when --winrm-transport is used" do
    it "raises an exception when an invalid transport is given" do
      inspec("exec " + complete_profile + " -t winrm://administrator@dummy --password dummy --winrm-transport nonesuch")

      _(stderr).must_include "Client error, can't connect to 'winrm' backend: Unsupported transport type: :nonesuch\n"

      assert_exit_code 1, out
    end
  end

  describe "with sensitive resources" do
    it "hides sensitive output" do
      inspec("exec " + sensitive_profile + " --no-create-lockfile")

      _(stdout).must_include '×  is expected to eq "billy"'
      _(stdout).must_include 'expected: "billy"'
      _(stdout).must_include 'got: "bob"'
      _(stdout).must_include '×  is expected to eq "secret"'
      _(stdout).must_include "*** sensitive output suppressed ***"
      _(stdout).must_include "\nTest Summary: 2 successful, 2 failures, 0 skipped\n"

      _(stderr).must_equal ""

      assert_exit_code 100, out
    end
  end

  describe "with a profile that loads dependencies" do
    let(:out) { inspec("exec " + File.join(profile_path, "profile-support-skip") + " --no-create-lockfile --reporter json") }
    let(:json) { JSON.load(stdout) }
    let(:controls) { json["profiles"][0]["controls"] }

    it "skips loaded inherited profiles on unsupported platforms" do
      _(json["profiles"][0]["depends"][0]["name"]).must_equal "unsupported_inspec"
      _(controls).must_be_empty

      _(stderr).must_include "WARN: Skipping profile"
    end
  end

  describe "with a profile containing exceptions in the controls" do
    let(:out) { inspec("exec " + File.join(profile_path, "exception-in-control") + " --no-create-lockfile --reporter json") }
    let(:json) { JSON.load(stdout) }
    let(:controls) { json["profiles"][0]["controls"] }

    it "completes the run with failed controls but no exception" do
      _(stderr).must_be_empty

      _(controls.count).must_equal 10
      _(controls.select { |c| c["results"][0]["status"] == "failed" }.count).must_be :>, 1
      _(controls.select { |c| c["results"][0]["status"] == "passed" }.count).must_be :>, 1

      assert_exit_code 100, out
    end
  end

  describe "with a profile containing exceptions outside controls" do
    let(:out) { inspec("exec " + File.join(profile_path, "raise_outside_control") + " --no-create-lockfile") }
    it "gives the failure reason" do
      _(stdout).must_include "Failure Message: Failed to load source for controls/raises.rb: Something unforeseen..."
    end

    it "exits non-zero" do
      assert_exit_code 102, out
    end
  end

  describe "when running both a valid profile and one that fails to load" do
    let(:out) { inspec("exec " + File.join(profile_path, "raise_outside_control") + " " + File.join(profile_path, "basic_profile") + " --no-create-lockfile --no-color") }
    it "gives the failure reason for the failing profile" do
      _(stdout).must_include "Failure Message: Failed to load source for controls/raises.rb: Something unforeseen..."
    end

    it "reports results for the working profile" do
      _(stdout).must_include "Profile Summary: 1 successful control"
    end

    it "exits non-zero" do
      assert_exit_code 102, out
    end
  end

  describe "with a profile containing control overrides" do
    let(:out) { inspec("exec " + File.join(profile_path, "wrapper-override") + " --no-create-lockfile --vendor-cache " + File.join(profile_path, "wrapper-override", "vendor") + " --reporter json") }
    let(:json) { JSON.load(stdout) }
    let(:controls) { json["profiles"][0]["controls"] }
    let(:child_profile) { json["profiles"].select { |p| p["name"] == "myprofile1" }.first }
    let(:child_control) { child_profile["controls"].select { |c| c["title"] == "Profile 1 - Control 2-updated" }.first }
    let(:override) { controls.select { |c| c["title"] == "Profile 1 - Control 2-updated" }.first }

    it "completes the run with parent control overrides" do
      _(stderr).must_be_empty

      if is_windows?
        assert_exit_code 100, out
      else
        assert_exit_code 0, out
      end

      _(controls.count).must_equal 2

      # check for json override
      # TODO: Brittle test expects the leading spaces.
      expected_value = <<-END
  control 'pro1-con2' do
    impact 0.999
    title 'Profile 1 - Control 2-updated'
    desc 'Profile 1 - Control 2 description-updated'
    desc 'overwrite me', 'it is overwritten'
    desc 'new entry', 'this is appended to the description list'
    tag 'password-updated'
    ref 'Section 3.5.2.1', url: 'https://example.com'
    describe file('/etc/passwd') do
      it { should exist }
    end
  end
      END
      _(override["code"]).must_equal expected_value
      _(override["impact"]).must_equal 0.999
      _(override["descriptions"]).must_equal([
        { "label" => "default",
          "data" => "Profile 1 - Control 2 description-updated",
        },
        { "label" => "overwrite me",
          "data" => "it is overwritten",
        },
        { "label" => "new entry",
          "data" => "this is appended to the description list",
        },
      ])
      _(override["title"]).must_equal "Profile 1 - Control 2-updated"
      tags_assert = { "password" => nil, "password-updated" => nil }
      _(override["tags"]).must_equal tags_assert
      _(child_profile["parent_profile"]).must_equal "wrapper-override"

      # check for original code on child profile
      expected_value = <<~END
        control 'pro1-con2' do
          impact 0.9
          title 'Profile 1 - Control 2'
          desc 'Profile 1 - Control 2 description'
          desc 'overwrite me', 'overwrite this'
          tag 'password'
          describe file('/etc/passwdddddddddd') do
            it { should exist }
          end
        end
      END
      _(child_control["code"]).must_equal expected_value

      _(stderr).must_be_empty

      if is_windows?
        assert_exit_code 100, out
      else
        assert_exit_code 0, out
      end
    end
  end

  describe "when using multiple custom resources with each other" do
    let(:out) { inspec("exec " + File.join(examples_path, "custom-resource") + " --no-create-lockfile") }

    it "completes the run with failed controls but no exception" do
      _(stderr).must_be_empty

      assert_exit_code 0, out
    end
  end

  describe "when using a profile with non-UTF characters and wrong encoding" do
    let(:out) { inspec("exec " + File.join(profile_path, "wrong-char-profile") + " --no-create-lockfile") }

    it "completes the run with failed controls but no exception" do
      _(stderr).must_be_empty

      assert_exit_code 0, out
    end
  end

  describe "when targeting private GitHub profiles" do
    let(:private_profile) do
      URI.parse("https://github.com/chef/inspec-test-profile-private.git")
    end

    # This tests requires that a private SSH key be provided for a user that has
    # access to the private profile repo
    if ENV["INSPEC_TEST_SSH_KEY_PATH"]
      it "can use SSH + Git" do
        target = "git@" + private_profile.host + ":" + private_profile.path
        ssh_prefix = 'GIT_SSH_COMMAND="ssh -i ' +
          ENV["INSPEC_TEST_SSH_KEY_PATH"] +
          '"'
        inspec_command = "exec " + target + " --reporter json-min"
        inspec(inspec_command, ssh_prefix)

        _(JSON.parse(stdout)["controls"][0]["status"]).must_equal "passed"

        _(stderr).must_equal ""

        assert_exit_code 0, out
      end
    end

    # This tests requires that a GitHub API token be provided for a user that
    # has access to the private profile repo
    if ENV["INSPEC_TEST_GITHUB_TOKEN"]
      it "can use HTTPS + token + Git" do
        private_profile.userinfo = ENV["INSPEC_TEST_GITHUB_TOKEN"]
        inspec_command = "exec " + private_profile.to_s + " --reporter json-min"
        inspec(inspec_command)

        _(JSON.parse(stdout)["controls"][0]["status"]).must_equal "passed"

        _(stderr).must_equal ""

        assert_exit_code 0, out
      end
    end
  end

  describe "when specifying a config file" do
    let(:run_result) { run_inspec_process("exec " + File.join(profile_path, "simple-metadata") + " " + cli_args, json: true, env: env) }
    let(:seen_target_id) { @json["platform"]["target_id"] }
    let(:platform_uuid) { inspec_os_uuid }
    let(:stderr) { run_result.stderr }
    let(:env) { {} }

    describe "when using the legacy --json-config" do
      let(:cli_args) { "--json-config " + File.join(config_dir_path, "json-config", "good.json") }
      it "should override the custom target ID value with platform uuid" do
        _(stderr).must_be_empty # TODO: one day deprecate the --json-config option
        _(seen_target_id).wont_equal "from-config-file"
        _(seen_target_id).must_equal platform_uuid
      end
    end

    describe "when using the --config option to read from a custom file" do
      let(:cli_args) { "--config " + File.join(config_dir_path, "json-config", "good.json") }
      it "should override the custom target ID value with platform uuid" do
        _(stderr).must_be_empty
        _(seen_target_id).wont_equal "from-config-file"
        _(seen_target_id).must_equal platform_uuid
      end
    end

    unless windows?
      describe "when using the --config option to read from STDIN" do
        let(:json_path) { File.join(config_dir_path, "json-config", "good.json") }
        let(:cli_args) { "--config -" }
        let(:opts) { { prefix: "cat " + json_path + " | ", json: true, env: env } }
        let(:njopts) { opts.merge(json: false) }
        let(:platform_uuid) { inspec_os_uuid }

        # DO NOT use the `let`-defined run_result through here
        # If you do, it will execute twice, and cause STDIN to read empty on the second time
        it "exec should override the the custom target ID value if platform uuid is present." do
          result = run_inspec_process( "exec " + File.join(profile_path, "simple-metadata") + " " + cli_args + " ", opts )

          _(result.stderr).must_be_empty
          _(@json["platform"]["target_id"]).wont_equal "from-config-file"
          _(@json["platform"]["target_id"]).must_equal platform_uuid
        end

        it "detect should exit 0" do
          result = run_inspec_process( "detect " + cli_args + " ", njopts )

          _(result.stderr).must_be_empty

          assert_exit_code 0, result
        end

        it "shell should exit 0" do
          result = run_inspec_process( 'shell -c "platform.family" ' + cli_args + " ", njopts )

          _(result.stderr).must_be_empty

          assert_exit_code 0, result
        end
      end
    end

    describe "when reading from the default location" do
      # Should read from File.join(config_dir_path, 'fakehome-2', '.inspec', 'config.json')
      let(:env) { { "HOME" => File.join(config_dir_path, "fakehome-2") } }
      let(:cli_args) { "" }
      let(:platform_uuid) { inspec_os_uuid }
      it "should override override the homedir target ID value with platform uuid" do
        _(stderr).must_be_empty
        _(seen_target_id).wont_equal "from-fakehome-config-file"
        _(seen_target_id).must_equal platform_uuid
      end
    end

    describe "when --config points to a nonexistent location" do
      let(:cli_args) { "--config " + "no/such/path" }
      it "should issue an error with the file path" do
        _(stderr).wont_match looks_like_a_stacktrace
        _(stderr).must_include "Could not read configuration file" # Should specify error
        _(stderr).must_include "no/such/path" # Should include error value seen

        assert_exit_code 1, run_result
      end
    end

    describe "when --config points to a malformed file" do
      let(:cli_args) { "--config " + File.join(config_dir_path, "json-config", "malformed.json") }
      it "should issue an error with the parse message" do
        _(stderr).wont_match looks_like_a_stacktrace
        _(stderr).must_include "Failed to load JSON"
        _(stderr).must_include "Config was:"

        assert_exit_code 1, run_result
      end
    end

    describe "when --config points to an invalid file" do
      let(:cli_args) { "--config " + File.join(config_dir_path, "json-config", "invalid.json") }
      it "should issue an error with the parse message" do
        _(stderr).wont_match looks_like_a_stacktrace
        _(stderr).must_include "Unrecognized top-level configuration"
        _(stderr).must_include "this_key_is_invalid"

        assert_exit_code 1, run_result
      end
    end
  end

  describe "when specifying the execution target" do
    let(:local_plat) do
      json = run_inspec_process("detect --format json", {}).stdout
      JSON.parse(json).slice("name", "release").merge({ "target_id" => inspec_os_uuid })
    end
    let(:run_result) { run_inspec_process("exec " + File.join(profile_path, "simple-metadata") + " " + cli_args, json: true) }
    let(:seen_platform) { run_result; @json["platform"].select { |k, v| %w{name release target_id}.include? k } }
    let(:stderr) { run_result.stderr }

    describe "when neither target nor backend is specified" do
      let(:cli_args) { "" }
      it "should connect to the local platform" do
        _(seen_platform).must_equal local_plat
      end
    end

    describe "when local:// is specified" do
      let(:cli_args) { " -t local:// " }
      it "should connect to the local platform" do
        _(seen_platform).must_equal local_plat
      end
    end

    describe "when an unrecognized backend is specified" do
      let(:cli_args) { "-b garble " }
      it "should exit with an error" do
        _(stderr).wont_match looks_like_a_stacktrace
        # "Can't find train plugin garble. Please install it first"
        _(stderr).must_include "Can't find train plugin"
        _(stderr).must_include "garble"

        assert_exit_code 1, run_result
      end
    end

    describe "when an unrecognized target schema is specified" do
      let(:cli_args) { "-t garble:// " }
      it "should exit with an error" do
        _(stderr).wont_match looks_like_a_stacktrace
        # "Can't find train plugin garble. Please install it first"
        _(stderr).must_include "Can't find train plugin"
        _(stderr).must_include "garble"

        assert_exit_code 1, run_result
      end
    end

    describe "when a schemaless URI is specified" do
      let(:cli_args) { "-t garble " }
      it "should exit with an error" do
        _(stderr).wont_match looks_like_a_stacktrace
        # "Could not recognize a backend from the target garble - use a URI
        # format with the backend name as the URI schema.  Example: 'ssh://somehost.com'
        # or 'transport://credset' or 'transport://' if credentials are provided
        #  outside of InSpec."
        _(stderr).must_include "Could not recognize a backend"
        _(stderr).must_include "garble"
        _(stderr).must_include "ssh://somehost.com"
        _(stderr).must_include "transport://credset"

        assert_exit_code 1, run_result
      end
    end

    describe "when a target URI with a known credset is used" do
      let(:cli_args) { "--target mock://mycredset" + " --config " + File.join(config_dir_path, "json-config", "mock-credset.json") }
      it "should connect to the mock platform" do
        _(seen_platform).must_equal({ "name" => "mock", "release" => "unknown", "target_id" => "from-mock-credset-config-file" })
      end
    end
  end

  describe "when targeting cloud resource packs" do
    let(:cloud_path) { profile_path + "/cloud/" }
    let(:run_result) { run_inspec_process("exec " + cloud_profile + " " + args, env: env) }
    let(:env) { {} }

    describe "when targeting aws" do
      let(:cloud_profile) { cloud_path + "test-aws" }
      # Use log level FATAL to absorb WARNs from deprecataions and ERRORs from not having credentials set.
      # An actual stacktrace then will appear as sole stderr output
      let(:args) { "-t aws://fakecreds --log-level fatal " }
      it "should fail to connect to aws due to lack of creds and stacktrace" do
        _(run_result.stderr).wont_match looks_like_a_stacktrace
      end
    end

    describe "when targeting azure" do
      let(:cloud_profile) { cloud_path + "test-azure" }
      let(:args) { "-t azure://" }
      it "should fail to connect to azure due to lack of creds but not stacktrace" do
        _(run_result.stderr).must_equal "Tenant id cannot be nil\n"
      end
    end

    describe "when targeting gcp" do
      let(:cloud_profile) { cloud_path + "test-gcp" }
      let(:args) { "-t gcp:// --input gcp_project_id=fakeproject" }
      let(:env) { { GOOGLE_AUTH_SUPPRESS_CREDENTIALS_WARNINGS: 1 } }
      it "should fail to connect to gcp due to lack of creds but not stacktrace" do
        _(run_result.stderr).must_include "Could not load the default credentials."
      end
    end

  end

  describe "when evaluating profiles with only_if" do
    let(:run_result) { run_inspec_process("exec #{profile}", json: true) }
    describe "when running a profile with a variety of skips" do
      let(:profile) { "#{profile_path}/only_if/skip-control" }
      it "should correctly skip in individual controls" do
        run_result
        _(@json.dig("profiles", 0, "controls", 0, "results", 0, "status")).must_equal "passed"
        _(@json.dig("profiles", 0, "controls", 1, "results", 0, "status")).must_equal "skipped"
        _(@json.dig("profiles", 0, "controls", 1, "results", 0, "skip_message")).must_equal "Skipped control due to only_if condition."
        _(@json.dig("profiles", 0, "controls", 2, "results", 0, "status")).must_equal "skipped"
        _(@json.dig("profiles", 0, "controls", 2, "results", 0, "skip_message")).must_equal "Skipped control due to only_if condition: here is a message"
        # 1/0 in test body
        _(@json.dig("profiles", 0, "controls", 3, "results", 0, "status")).must_equal "skipped"
        _(@json.dig("profiles", 0, "controls", 3, "results", 0, "skip_message")).must_equal "Skipped control due to only_if condition."
        # 1/0 in resource declaration but it follows the only_if
        _(@json.dig("profiles", 0, "controls", 4, "results", 0, "status")).must_equal "skipped"
        _(@json.dig("profiles", 0, "controls", 4, "results", 0, "skip_message")).must_equal "Skipped control due to only_if condition."
        # resource declaration but it precedes the only_if
        _(@json.dig("profiles", 0, "controls", 5, "results", 0, "status")).must_equal "skipped"
        _(@json.dig("profiles", 0, "controls", 5, "results", 0, "skip_message")).must_equal "Skipped control due to only_if condition."
        # multiple only_ifs
        _(@json.dig("profiles", 0, "controls", 6, "results", 0, "status")).must_equal "skipped"
        _(@json.dig("profiles", 0, "controls", 6, "results", 0, "skip_message")).must_equal "Skipped control due to only_if condition: here is a different message"
      end
    end
    describe "when running a profile with an only_if at the top-level" do
      let(:profile) { "#{profile_path}/only_if/skip-file" }
      it "should correctly skip entire files" do
        run_result
        # first control is in a separate file
        _(@json.dig("profiles", 0, "controls", 0, "results", 0, "status")).must_equal "passed"
        # Latter three are in the same file
        _(@json.dig("profiles", 0, "controls", 1, "results", 0, "status")).must_equal "skipped"
        _(@json.dig("profiles", 0, "controls", 2, "results", 0, "status")).must_equal "skipped"
        _(@json.dig("profiles", 0, "controls", 3, "results", 0, "status")).must_equal "skipped"
      end
    end
  end

  describe "when running a profile using timeouts on a command resource" do
    let(:profile) { "#{profile_path}/timeouts" }

    describe "when using the DSL command resource option" do
      let(:run_result) { run_inspec_process("exec #{profile}") }

      it "properly timesout an inlined command resource" do
        # Command timeout not available on local windows pipe train transports
        skip if windows?
        _(run_result.stderr).must_be_empty

        # Control with inline timeout should be interrupted correctly
        _(run_result.stdout).must_include "Command `sleep 10; echo oops` timed out after 2 seconds"
        # Subsequent control must still run correctly
        _(run_result.stdout).must_include "Command: `echo hello` exit_status is expected to cmp == 0"
      end
    end

    describe "when using the CLI option to override the command timeout" do
      let(:run_result) { run_inspec_process("exec #{profile} --command-timeout 1") }
      it "properly overrides the DSL setting with the CLI timeout option" do
        # Command timeout not available on local windows pipe train transports
        skip if windows?
        _(run_result.stderr).must_be_empty

        # Command timeout should be interrupted correctly, with CLI timeout applied
        _(run_result.stdout).must_include "Command `sleep 10; echo oops` timed out after 1 seconds"
        # Subsequent control must still run correctly
        _(run_result.stdout).must_include "Command: `echo hello` exit_status is expected to cmp == 0"
      end
    end
  end

  describe "when using the --reporter-include-source option with the CLI reporter" do
    let(:profile) { "#{profile_path}/sorted-results/sort-me-1" } # A profile with controls separated in multiple files
    let(:run_result) { run_inspec_process("exec #{profile} --reporter-include-source") }
    it "includes the control source code" do
      _(run_result.stderr).must_be_empty

      expected = %r{Control Source from .+test/fixtures/profiles/sorted-results/sort-me-1/controls/a-uvw.rb:1..6}
      _(run_result.stdout).must_match expected
      expected = <<EOT
     control "w" do
       describe "anything" do
         it { should eq "anything" }
       end
     end
EOT
      _(run_result.stdout).must_include expected

      expected = %r{Control Source from .+test/fixtures/profiles/sorted-results/sort-me-1/controls/c-rst.rb:1..6}
      _(run_result.stdout).must_match expected
      expected = <<EOT
     control "r" do
       describe "anything" do
         it { should eq "anything" }
       end
     end
EOT
      _(run_result.stdout).must_include expected

    end
  end

  describe "when evalutating profiles that reference resource_id" do
    let(:run_result) { run_inspec_process("exec #{profile}", json: true) }
    let(:profile) { "#{profile_path}/resource_ids" } # A profile with custom resources and test controls that exercise resource ids
    it "should evaluate all test controls correctly" do
      _(run_result.stderr).must_be_empty
      assert_json_controls_passing
    end
  end
end