2016-03-25 00:31:19 +00:00
|
|
|
|
# author: Dominik Richter
|
|
|
|
|
# author: Christoph Hartmann
|
|
|
|
|
|
|
|
|
|
require 'helper'
|
2018-09-12 22:04:16 +00:00
|
|
|
|
require 'rbconfig'
|
2018-11-16 22:03:40 +00:00
|
|
|
|
require 'json'
|
|
|
|
|
require 'fileutils'
|
|
|
|
|
require 'yaml'
|
|
|
|
|
require 'tmpdir'
|
2016-03-25 00:31:19 +00:00
|
|
|
|
|
|
|
|
|
require 'minitest/hell'
|
|
|
|
|
class Minitest::Test
|
|
|
|
|
parallelize_me!
|
|
|
|
|
end
|
|
|
|
|
|
|
|
|
|
class Module
|
|
|
|
|
include Minitest::Spec::DSL
|
|
|
|
|
end
|
|
|
|
|
|
2018-11-16 22:03:40 +00:00
|
|
|
|
module Inspec
|
|
|
|
|
class FuncTestRunResult
|
|
|
|
|
attr_reader :train_result
|
|
|
|
|
attr_reader :payload
|
|
|
|
|
|
|
|
|
|
extend Forwardable
|
|
|
|
|
def_delegator :train_result, :stdout
|
|
|
|
|
def_delegator :train_result, :stderr
|
|
|
|
|
def_delegator :train_result, :exit_status
|
|
|
|
|
|
|
|
|
|
def initialize(train_result)
|
|
|
|
|
@train_result = train_result
|
|
|
|
|
@payload = OpenStruct.new
|
|
|
|
|
end
|
2019-01-10 01:14:42 +00:00
|
|
|
|
|
|
|
|
|
# Intentional failure to cause CI to print output
|
|
|
|
|
def diagnose!
|
|
|
|
|
msg = ''
|
|
|
|
|
msg += "\nInvocation:\n" + payload.invocation
|
|
|
|
|
msg += "\nSTDOUT:\n" + stdout
|
|
|
|
|
msg += "\nSTDERR:\n" + stderr
|
|
|
|
|
msg.must_equal ''
|
|
|
|
|
end
|
2019-04-23 15:19:51 +00:00
|
|
|
|
|
|
|
|
|
def stderr_ignore_deprecations
|
2019-04-25 18:48:35 +00:00
|
|
|
|
suffix = stderr.end_with?("\n") ? "\n" : ''
|
|
|
|
|
stderr.split("\n").reject { |l| l.include? ' DEPRECATION: ' }.join("\n") + suffix
|
2019-04-23 15:19:51 +00:00
|
|
|
|
end
|
|
|
|
|
|
|
|
|
|
def stdout_ignore_deprecations
|
2019-04-25 18:48:35 +00:00
|
|
|
|
suffix = stdout.end_with?("\n") ? "\n" : ''
|
|
|
|
|
stdout.split("\n").reject { |l| l.include? ' DEPRECATION: ' }.join("\n") + suffix
|
2019-04-23 15:19:51 +00:00
|
|
|
|
end
|
2019-03-07 16:55:18 +00:00
|
|
|
|
|
|
|
|
|
# This works if you use json: true on an exec call
|
|
|
|
|
def must_have_all_controls_passing
|
|
|
|
|
# Strategy: assemble an array of tests that failed or skipped, and insist it is empty
|
|
|
|
|
# result.payload.json['profiles'][0]['controls'][0]['results'][0]['status']
|
|
|
|
|
failed_tests = []
|
|
|
|
|
payload.json['profiles'].each do |profile_struct|
|
|
|
|
|
profile_name = profile_struct['name']
|
|
|
|
|
profile_struct['controls'].each do |control_struct|
|
|
|
|
|
control_name = control_struct['id']
|
|
|
|
|
control_struct['results'].compact.each do |test_struct|
|
|
|
|
|
test_desc = test_struct['code_desc']
|
|
|
|
|
if test_struct['status'] != 'passed'
|
|
|
|
|
failed_tests << "#{profile_name}/#{control_name}/#{test_desc}"
|
|
|
|
|
end
|
|
|
|
|
end
|
|
|
|
|
end
|
|
|
|
|
end
|
|
|
|
|
|
|
|
|
|
failed_tests.must_be_empty
|
|
|
|
|
end
|
2018-11-16 22:03:40 +00:00
|
|
|
|
end
|
|
|
|
|
end
|
|
|
|
|
|
2016-03-25 00:31:19 +00:00
|
|
|
|
module FunctionalHelper
|
2018-11-08 17:00:14 +00:00
|
|
|
|
let(:repo_path) do
|
|
|
|
|
path = File.expand_path(File.join( __FILE__, '..', '..', '..'))
|
|
|
|
|
# fix for vagrant repo pathing
|
|
|
|
|
path.gsub!('//vboxsrv', 'C:') if is_windows?
|
|
|
|
|
path
|
|
|
|
|
end
|
2019-04-30 22:33:07 +00:00
|
|
|
|
let(:inspec_path) { File.join(repo_path, 'inspec-bin', 'bin', 'inspec') }
|
|
|
|
|
libdir = File.expand_path "lib"
|
|
|
|
|
let(:exec_inspec) { [Gem.ruby, "-I#{libdir}", inspec_path].join " " }
|
Plugins API v2: Loader, Base API, and Test Harness (#3278)
* Functional tests for userdir option
* Accepts --config-dir CLI option
* Actually loads a config file from the config dir, more cases to test
* Able to load config and verify contents from config-dir
* Functional tests to ensure precedence for config options
* Enable setting config dir via env var
* .inspec, not .inspec.d
* Begin converting PluginCtl to PluginLoader/Registry
* Able to load and partially validate the plugins.json file
* More work on the plugin loader
* Break the world, move next gen stuff to plugin/
* Be sure to require base cli in bundled plugins
* Move test file
* Revert changes to v1 plugin, so we can have a separate one
* Checkpoint commit
* Move v2 plugin work to v2 area
* Move plugins v1 code into an isolated directory
* rubocop fixes
* Rip out the stuff about a user-dir config file, just use a plugin file
* Two psuedocode test file
* Working base API, moock plugin type, and loader.
* Adjust load path to be more welcoming
* Silence circular depencency warning, which was breaking a unit test
* Linting
* Fix plugin type registry, add tests to cover
* Feedback from Jerry
Signed-off-by: Clinton Wolfe <clintoncwolfe@gmail.com>
2018-08-16 22:16:32 +00:00
|
|
|
|
let(:mock_path) { File.join(repo_path, 'test', 'unit', 'mock') }
|
|
|
|
|
let(:profile_path) { File.join(mock_path, 'profiles') }
|
2019-03-18 15:04:02 +00:00
|
|
|
|
let(:examples_path) { File.join(profile_path, 'old-examples') }
|
2017-08-15 18:41:24 +00:00
|
|
|
|
let(:integration_test_path) { File.join(repo_path, 'test', 'integration', 'default') }
|
2016-03-25 00:31:19 +00:00
|
|
|
|
|
2019-03-18 15:04:02 +00:00
|
|
|
|
let(:example_profile) { File.join(examples_path, 'profile') }
|
|
|
|
|
let(:meta_profile) { File.join(examples_path, 'meta-profile') }
|
2016-04-20 04:50:18 +00:00
|
|
|
|
let(:example_control) { File.join(example_profile, 'controls', 'example.rb') }
|
2019-03-18 15:04:02 +00:00
|
|
|
|
let(:inheritance_profile) { File.join(examples_path, 'inheritance') }
|
2016-09-01 12:25:56 +00:00
|
|
|
|
let(:failure_control) { File.join(profile_path, 'failures', 'controls', 'failures.rb') }
|
2016-09-01 22:54:03 +00:00
|
|
|
|
let(:simple_inheritance) { File.join(profile_path, 'simple-inheritance') }
|
2019-03-18 15:04:02 +00:00
|
|
|
|
let(:sensitive_profile) { File.join(examples_path, 'profile-sensitive') }
|
Plugins API v2: Loader, Base API, and Test Harness (#3278)
* Functional tests for userdir option
* Accepts --config-dir CLI option
* Actually loads a config file from the config dir, more cases to test
* Able to load config and verify contents from config-dir
* Functional tests to ensure precedence for config options
* Enable setting config dir via env var
* .inspec, not .inspec.d
* Begin converting PluginCtl to PluginLoader/Registry
* Able to load and partially validate the plugins.json file
* More work on the plugin loader
* Break the world, move next gen stuff to plugin/
* Be sure to require base cli in bundled plugins
* Move test file
* Revert changes to v1 plugin, so we can have a separate one
* Checkpoint commit
* Move v2 plugin work to v2 area
* Move plugins v1 code into an isolated directory
* rubocop fixes
* Rip out the stuff about a user-dir config file, just use a plugin file
* Two psuedocode test file
* Working base API, moock plugin type, and loader.
* Adjust load path to be more welcoming
* Silence circular depencency warning, which was breaking a unit test
* Linting
* Fix plugin type registry, add tests to cover
* Feedback from Jerry
Signed-off-by: Clinton Wolfe <clintoncwolfe@gmail.com>
2018-08-16 22:16:32 +00:00
|
|
|
|
let(:config_dir_path) { File.join(mock_path, 'config_dirs') }
|
2016-03-25 00:31:19 +00:00
|
|
|
|
|
|
|
|
|
let(:dst) {
|
|
|
|
|
# create a temporary path, but we only want an auto-clean helper
|
|
|
|
|
# so remove the file and give back the path
|
|
|
|
|
res = Tempfile.new('inspec-shred')
|
2018-09-14 00:19:02 +00:00
|
|
|
|
res.close
|
2016-03-25 00:31:19 +00:00
|
|
|
|
FileUtils.rm(res.path)
|
|
|
|
|
TMP_CACHE[res.path] = res
|
|
|
|
|
}
|
|
|
|
|
|
2018-09-12 22:04:16 +00:00
|
|
|
|
def convert_windows_output(text)
|
|
|
|
|
text = text.force_encoding("UTF-8")
|
|
|
|
|
text.gsub!("[PASS]", '✔')
|
|
|
|
|
text.gsub!("\033[0;1;32m", "\033[38;5;41m")
|
|
|
|
|
text.gsub!("[SKIP]", '↺')
|
|
|
|
|
text.gsub!("\033[0;37m", "\033[38;5;247m")
|
|
|
|
|
text.gsub!("[FAIL]", '×')
|
|
|
|
|
text.gsub!("\033[0;1;31m", "\033[38;5;9m")
|
|
|
|
|
end
|
|
|
|
|
|
2018-11-08 17:00:14 +00:00
|
|
|
|
def self.is_windows?
|
2018-09-12 22:04:16 +00:00
|
|
|
|
RbConfig::CONFIG['host_os'] =~ /mswin|mingw|cygwin/
|
|
|
|
|
end
|
|
|
|
|
|
2018-11-08 17:00:14 +00:00
|
|
|
|
def is_windows?
|
|
|
|
|
FunctionalHelper.is_windows?
|
|
|
|
|
end
|
|
|
|
|
|
2016-11-09 14:41:48 +00:00
|
|
|
|
def inspec(commandline, prefix = nil)
|
2018-09-12 22:04:16 +00:00
|
|
|
|
if is_windows?
|
2019-01-10 01:14:42 +00:00
|
|
|
|
invocation = "cmd /C \"#{prefix} bundle exec ruby #{exec_inspec} #{commandline}\""
|
|
|
|
|
result = CMD.run_command(invocation)
|
2018-09-12 22:04:16 +00:00
|
|
|
|
result.stdout.encode!(universal_newline: true)
|
|
|
|
|
result.stderr.encode!(universal_newline: true)
|
|
|
|
|
convert_windows_output(result.stdout)
|
|
|
|
|
# remove the CLIXML header trash in windows
|
|
|
|
|
result.stderr.gsub!("#< CLIXML\n", '')
|
2019-01-10 01:14:42 +00:00
|
|
|
|
ftrr = Inspec::FuncTestRunResult.new(result)
|
2018-09-12 22:04:16 +00:00
|
|
|
|
else
|
2019-01-10 01:14:42 +00:00
|
|
|
|
invocation = "#{prefix} #{exec_inspec} #{commandline}"
|
|
|
|
|
ftrr = Inspec::FuncTestRunResult.new(CMD.run_command(invocation))
|
2018-09-12 22:04:16 +00:00
|
|
|
|
end
|
2019-01-10 01:14:42 +00:00
|
|
|
|
ftrr.payload.invocation = invocation
|
|
|
|
|
ftrr
|
2016-03-25 00:31:19 +00:00
|
|
|
|
end
|
2017-06-12 12:01:26 +00:00
|
|
|
|
|
Plugins API v2: Loader, Base API, and Test Harness (#3278)
* Functional tests for userdir option
* Accepts --config-dir CLI option
* Actually loads a config file from the config dir, more cases to test
* Able to load config and verify contents from config-dir
* Functional tests to ensure precedence for config options
* Enable setting config dir via env var
* .inspec, not .inspec.d
* Begin converting PluginCtl to PluginLoader/Registry
* Able to load and partially validate the plugins.json file
* More work on the plugin loader
* Break the world, move next gen stuff to plugin/
* Be sure to require base cli in bundled plugins
* Move test file
* Revert changes to v1 plugin, so we can have a separate one
* Checkpoint commit
* Move v2 plugin work to v2 area
* Move plugins v1 code into an isolated directory
* rubocop fixes
* Rip out the stuff about a user-dir config file, just use a plugin file
* Two psuedocode test file
* Working base API, moock plugin type, and loader.
* Adjust load path to be more welcoming
* Silence circular depencency warning, which was breaking a unit test
* Linting
* Fix plugin type registry, add tests to cover
* Feedback from Jerry
Signed-off-by: Clinton Wolfe <clintoncwolfe@gmail.com>
2018-08-16 22:16:32 +00:00
|
|
|
|
def inspec_with_env(commandline, env = {})
|
2018-11-16 22:03:40 +00:00
|
|
|
|
inspec(commandline, assemble_env_prefix(env))
|
|
|
|
|
end
|
|
|
|
|
|
2019-01-10 01:14:42 +00:00
|
|
|
|
# This version allows additional options.
|
2018-11-16 22:03:40 +00:00
|
|
|
|
# @param String command_line Invocation, without the word 'inspec'
|
|
|
|
|
# @param Hash opts Additonal options, see below
|
|
|
|
|
# :env Hash A hash of environment vars to expose to the invocation.
|
|
|
|
|
# :prefix String A string to prefix to the invocation. Prefix + env + invocation is the order.
|
|
|
|
|
# :cwd String A directory to change to. Implemented as 'cd CWD && ' + prefix
|
|
|
|
|
# :lock Boolean Default false. If false, add `--no-create-lockfile`.
|
|
|
|
|
# :json Boolean Default false. If true, add `--reporter json` and parse the output, which is stored in payload.json .
|
|
|
|
|
# :tmpdir Boolean default true. If true, wrap execution in a Dir.tmpdir block. Use pre_run and post_run to trigger actions.
|
|
|
|
|
# :pre_run: Proc(tmp_dir_path) - optional setup block.
|
|
|
|
|
# tmp_dir will exist and be empty.
|
|
|
|
|
# :post_run: Proc(FuncTestRunResult, tmp_dir_path) - optional result capture block.
|
|
|
|
|
# run_result will be populated, but you can add more to the ostruct .payload
|
|
|
|
|
# tmp_dir will still exist (for a moment!)
|
|
|
|
|
# @return FuncTestRunResult. Includes attrs exit_status, stderr, stdout, payload (an openstruct which may be used in many ways)
|
2019-01-12 00:42:46 +00:00
|
|
|
|
def run_inspec_process(command_line, opts = {})
|
2018-11-16 22:03:40 +00:00
|
|
|
|
raise 'Do not use tmpdir and cwd in the same invocation' if opts[:cwd] && opts[:tmpdir]
|
|
|
|
|
prefix = opts[:cwd] ? 'cd ' + opts[:cwd] + ' && ' : ''
|
|
|
|
|
prefix += opts[:prefix] || ''
|
|
|
|
|
prefix += assemble_env_prefix(opts[:env])
|
|
|
|
|
command_line += ' --reporter json ' if opts[:json] && command_line =~ /\bexec\b/
|
|
|
|
|
command_line += ' --no-create-lockfile ' if (!opts[:lock]) && command_line =~ /\bexec\b/
|
|
|
|
|
|
|
|
|
|
run_result = nil
|
|
|
|
|
if opts[:tmpdir]
|
|
|
|
|
Dir.mktmpdir do |tmp_dir|
|
|
|
|
|
opts[:pre_run].call(tmp_dir) if opts[:pre_run]
|
|
|
|
|
# Do NOT Dir.chdir here - chdir / pwd is per-process, and we are in the
|
|
|
|
|
# test harness process, which will be multithreaded because we parallelize the tests.
|
|
|
|
|
# Instead, make the spawned process change dirs using a cd prefix.
|
|
|
|
|
prefix = 'cd ' + tmp_dir + ' && ' + prefix
|
|
|
|
|
run_result = inspec(command_line, prefix)
|
|
|
|
|
opts[:post_run].call(run_result, tmp_dir) if opts[:post_run]
|
|
|
|
|
end
|
2018-09-12 22:04:16 +00:00
|
|
|
|
else
|
2018-11-16 22:03:40 +00:00
|
|
|
|
run_result = inspec(command_line, prefix)
|
2018-09-12 22:04:16 +00:00
|
|
|
|
end
|
2018-11-16 22:03:40 +00:00
|
|
|
|
|
2019-04-24 15:37:43 +00:00
|
|
|
|
if opts[:ignore_rspec_deprecations]
|
|
|
|
|
# RSpec keeps issuing a deprecation count to stdout when .should is called explicitly
|
|
|
|
|
# See https://github.com/inspec/inspec/pull/3560
|
|
|
|
|
run_result.stdout.sub!("\n1 deprecation warning total\n", '')
|
|
|
|
|
end
|
|
|
|
|
|
2018-11-16 22:03:40 +00:00
|
|
|
|
if opts[:json]
|
|
|
|
|
begin
|
|
|
|
|
run_result.payload.json = JSON.parse(run_result.stdout)
|
|
|
|
|
rescue JSON::ParserError => e
|
|
|
|
|
run_result.payload.json = {}
|
|
|
|
|
run_result.payload.json_error = e
|
|
|
|
|
end
|
|
|
|
|
end
|
|
|
|
|
|
|
|
|
|
run_result
|
Plugins API v2: Loader, Base API, and Test Harness (#3278)
* Functional tests for userdir option
* Accepts --config-dir CLI option
* Actually loads a config file from the config dir, more cases to test
* Able to load config and verify contents from config-dir
* Functional tests to ensure precedence for config options
* Enable setting config dir via env var
* .inspec, not .inspec.d
* Begin converting PluginCtl to PluginLoader/Registry
* Able to load and partially validate the plugins.json file
* More work on the plugin loader
* Break the world, move next gen stuff to plugin/
* Be sure to require base cli in bundled plugins
* Move test file
* Revert changes to v1 plugin, so we can have a separate one
* Checkpoint commit
* Move v2 plugin work to v2 area
* Move plugins v1 code into an isolated directory
* rubocop fixes
* Rip out the stuff about a user-dir config file, just use a plugin file
* Two psuedocode test file
* Working base API, moock plugin type, and loader.
* Adjust load path to be more welcoming
* Silence circular depencency warning, which was breaking a unit test
* Linting
* Fix plugin type registry, add tests to cover
* Feedback from Jerry
Signed-off-by: Clinton Wolfe <clintoncwolfe@gmail.com>
2018-08-16 22:16:32 +00:00
|
|
|
|
end
|
|
|
|
|
|
2018-11-16 22:03:40 +00:00
|
|
|
|
|
2018-12-12 16:44:16 +00:00
|
|
|
|
|
2017-06-12 12:01:26 +00:00
|
|
|
|
# Copy all examples to a temporary directory for functional tests.
|
|
|
|
|
# You can provide an optional directory which will be handed to your
|
|
|
|
|
# test block with its absolute path. If nothing is provided you will
|
|
|
|
|
# get the path of the examples directory in the tmp environment.
|
|
|
|
|
#
|
|
|
|
|
# @param dir = nil [String] optional directory you want to test
|
|
|
|
|
# @param &block [Type] actual test block
|
|
|
|
|
def prepare_examples(dir = nil, &block)
|
|
|
|
|
Dir.mktmpdir do |tmpdir|
|
2019-03-18 15:04:02 +00:00
|
|
|
|
FileUtils.cp_r(examples_path, tmpdir)
|
|
|
|
|
bn = File.basename(examples_path)
|
2017-06-12 12:01:26 +00:00
|
|
|
|
block.call(File.join(tmpdir, bn, dir.to_s))
|
|
|
|
|
end
|
|
|
|
|
end
|
2018-11-16 22:03:40 +00:00
|
|
|
|
|
|
|
|
|
private
|
|
|
|
|
def assemble_env_prefix(env = {})
|
|
|
|
|
if is_windows?
|
|
|
|
|
env_prefix = env.to_a.map { |assignment| "set #{assignment[0]}=#{assignment[1]}" }.join('&& ')
|
2019-01-10 01:16:44 +00:00
|
|
|
|
env_prefix += '&& ' unless env_prefix.empty?
|
2018-11-16 22:03:40 +00:00
|
|
|
|
else
|
|
|
|
|
env_prefix = env.to_a.map { |assignment| "#{assignment[0]}=#{assignment[1]}" }.join(' ')
|
|
|
|
|
env_prefix += ' '
|
|
|
|
|
end
|
|
|
|
|
env_prefix
|
|
|
|
|
end
|
|
|
|
|
end
|
|
|
|
|
|
|
|
|
|
#=========================================================================================#
|
|
|
|
|
# Plugin Support
|
|
|
|
|
#=========================================================================================#
|
|
|
|
|
module PluginFunctionalHelper
|
|
|
|
|
include FunctionalHelper
|
|
|
|
|
|
|
|
|
|
def run_inspec_with_plugin(command, opts)
|
|
|
|
|
pre = Proc.new do |tmp_dir|
|
|
|
|
|
content = JSON.generate(__make_plugin_file_data_structure_with_path(opts[:plugin_path]))
|
|
|
|
|
File.write(File.join(tmp_dir, 'plugins.json'), content)
|
|
|
|
|
end
|
|
|
|
|
|
|
|
|
|
opts.merge!({
|
|
|
|
|
pre_run: pre,
|
|
|
|
|
tmpdir: true,
|
|
|
|
|
json: true,
|
|
|
|
|
env: {
|
|
|
|
|
"INSPEC_CONFIG_DIR" => '.' # We're in tmpdir
|
|
|
|
|
}
|
|
|
|
|
})
|
|
|
|
|
run_inspec_process(command, opts)
|
|
|
|
|
end
|
|
|
|
|
|
|
|
|
|
def __make_plugin_file_data_structure_with_path(path)
|
|
|
|
|
# TODO: dry this up, refs #3350
|
|
|
|
|
plugin_name = File.basename(path, '.rb')
|
|
|
|
|
data = __make_empty_plugin_file_data_structure
|
|
|
|
|
data['plugins'] << {
|
|
|
|
|
'name' => plugin_name,
|
|
|
|
|
'installation_type' => 'path',
|
|
|
|
|
'installation_path' => path,
|
|
|
|
|
}
|
|
|
|
|
data
|
|
|
|
|
end
|
|
|
|
|
|
|
|
|
|
def __make_empty_plugin_file_data_structure
|
|
|
|
|
# TODO: dry this up, refs #3350
|
|
|
|
|
{
|
|
|
|
|
'plugins_config_version' => '1.0.0',
|
|
|
|
|
'plugins' => [],
|
|
|
|
|
}
|
|
|
|
|
end
|
2016-03-25 00:31:19 +00:00
|
|
|
|
end
|