Mirrors/inspec
2
0
Fork 0
mirror of https://github.com/inspec/inspec synced 2024-11-23 13:13:22 +00:00
Code Issues Projects Releases Packages Wiki Activity

Merge pull request #4865 from inspec/jh/schema-improvements

Browse source 
Jh/schema improvements
This commit is contained in:
Miah Johnson 2020-02-10 18:38:28 -08:00 committed by GitHub
commit 7bb0bb2188
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
19 changed files with 813 additions and 120 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
inspec-core.gemspec
View file

@ -26,7 +26,7 @@ Gem::Specification.new do |spec|
spec.add_dependency "chef-telemetry", "~> 1.0"
spec.add_dependency "license-acceptance", ">= 0.2.13", "< 2.0"
spec.add_dependency "thor", ">= 0.20", "< 2.0"
spec.add_dependency "json-schema", "~> 2.8"
spec.add_dependency "json_schemer", "~> 0.2.1"
spec.add_dependency "method_source", "~> 0.8"
spec.add_dependency "rubyzip", "~> 1.2", ">= 1.2.2"
spec.add_dependency "rspec", "~> 3.9"

6
lib/inspec/cli.rb
View file

@ -377,12 +377,12 @@ class Inspec::InspecCLI < Inspec::BaseCLI
desc "schema NAME", "print the JSON schema", hide: true
def schema(name)
require "inspec/schema"
require "inspec/schema/output_schema"
puts Inspec::Schema.json(name)
puts Inspec::Schema::OutputSchema.json(name)
rescue StandardError => e
puts e
puts "Valid schemas are #{Inspec::Schema.names.join(", ")}"
puts "Valid schemas are #{Inspec::Schema::OutputSchema.names.join(", ")}"
end
desc "version", "prints the version of this tool"

17
lib/inspec/schema/README.md Normal file
View file

@ -0,0 +1,17 @@
# Schema Generation
These files handle the generation of JSON schema's for inspec output as yielded by `inspec schema <schema-name>`.
This schema is in the JSON Schema 4th revision spec.
## Structure
Data/schema structures shared between all outputs are located in `primitives.rb`
The files `exec_json_min.rb`, `exec_json.rb`, and `profile_json.rb` contain JSON structures for their specific output types, respectively.
The output schema is "flat" in terms of schema structure, with each "type" defined in the JSONs `definitions` block and referred to via $ref.
The logic for this is handled via the `SchemaType` class in `primitives.rb`.
All relevant dependencies are included recursively via the `dependencies` property of that class, constructed using the `build_definitions` method in `output_schema.rb`.
## Reasoning and validation
In general the first iteration of this updated schema was made by examining the output of all of the profiles in the tests directory.
All of the profiles generated by those tests successfully validate against this schema.
However, in light of potential missed properties that aren't covered by existing tests, the schema was left very non-strict.
More specifically, "additionalProperties" was left as True on all schema objects.

128
lib/inspec/schema/exec_json.rb Normal file
View file

@ -0,0 +1,128 @@
require "inspec/schema/primitives"
# These type occur only when running "inspec exec --reporter json <file>".
module Inspec
module Schema
module ExecJson
# Represents a label and description, to provide human-readable info about a control
CONTROL_DESCRIPTION = Primitives::SchemaType.new("Control Description", {
"type" => "object",
"additionalProperties" => true,
"required" => %w{label data},
"properties" => {
"label" => Primitives::STRING,
"data" => Primitives::STRING,
},
}, [])
# Lists the potential values for a control result
CONTROL_RESULT_STATUS = Primitives::SchemaType.new("Control Result Status", {
"type" => "string",
"enum" => %w{passed failed skipped error},
}, [])
# Represents the statistics/result of a control"s execution
CONTROL_RESULT = Primitives::SchemaType.new("Control Result", {
"type" => "object",
"additionalProperties" => true,
"required" => %w{code_desc run_time start_time},
"properties" => {
"status" => CONTROL_RESULT_STATUS.ref,
"code_desc" => Primitives::STRING,
"run_time" => Primitives::NUMBER,
"start_time" => Primitives::STRING,
# All optional
"resource" => Primitives::STRING,
"message" => Primitives::STRING,
"skip_message" => Primitives::STRING,
"exception" => Primitives::STRING,
"backtrace" => {
"anyOf" => [
Primitives.array(Primitives::STRING),
Primitives::NULL,
],
},
},
}, [CONTROL_RESULT_STATUS])
# Represents a control produced
CONTROL = Primitives::SchemaType.new("Exec JSON Control", {
"type" => "object",
"additionalProperties" => true,
"required" => %w{id title desc impact refs tags code source_location results},
"properties" => {
"id" => Primitives.desc(Primitives::STRING, "The ID of this control"),
"title" => { "type" => %w{string null} }, # Nullable string
"desc" => { "type" => %w{string null} },
"descriptions" => Primitives.array(CONTROL_DESCRIPTION.ref),
"impact" => Primitives::IMPACT,
"refs" => Primitives.array(Primitives::REFERENCE.ref),
"tags" => Primitives::TAGS,
"code" => Primitives.desc(Primitives::STRING, "The raw source code of the control. Note that if this is an overlay control, it does not include the underlying source code"),
"source_location" => Primitives::SOURCE_LOCATION.ref,
"results" => Primitives.desc(Primitives.array(CONTROL_RESULT.ref), %q{
A list of all results of the controls describe blocks.
For instance, if in the controls code we had the following:
describe sshd_config do
its('Port') { should cmp 22 }
end
The result of this block as a ControlResult would be appended to the results list.
}),
},
}, [CONTROL_DESCRIPTION, Primitives::REFERENCE, Primitives::SOURCE_LOCATION, CONTROL_RESULT])
# Based loosely on https://www.inspec.io/docs/reference/profiles/ as of July 3, 2019
# However, concessions were made to the reality of current reporters, specifically
# with how description is omitted and version/inspec_version aren't as advertised online
PROFILE = Primitives::SchemaType.new("Exec JSON Profile", {
"type" => "object",
"additionalProperties" => true,
"required" => %w{name sha256 supports attributes groups controls},
# Name is mandatory in inspec.yml.
# supports, controls, groups, and attributes are always present, even if empty
# sha256, status, skip_message
"properties" => {
# These are provided in inspec.yml
"name" => Primitives::STRING,
"title" => Primitives::STRING,
"maintainer" => Primitives::STRING,
"copyright" => Primitives::STRING,
"copyright_email" => Primitives::STRING,
"depends" => Primitives.array(Primitives::DEPENDENCY.ref),
"parent_profile" => Primitives::STRING,
"license" => Primitives::STRING,
"summary" => Primitives::STRING,
"version" => Primitives::STRING,
"supports" => Primitives.array(Primitives::SUPPORT.ref),
"description" => Primitives::STRING,
"inspec_version" => Primitives::STRING,
# These are generated at runtime, and all except skip_message are guaranteed
"sha256" => Primitives::STRING,
"status" => Primitives::STRING,
"skip_message" => Primitives::STRING, # If skipped, why
"controls" => Primitives.array(CONTROL.ref),
"groups" => Primitives.array(Primitives::CONTROL_GROUP.ref),
"attributes" => Primitives.array(Primitives::INPUT),
},
}, [CONTROL, Primitives::CONTROL_GROUP, Primitives::DEPENDENCY, Primitives::SUPPORT])
# Result of exec json. Top level value
# TODO: Include the format of top level controls. This was omitted for lack of sufficient examples
OUTPUT = Primitives::SchemaType.new("Exec JSON Output", {
"type" => "object",
"additionalProperties" => true,
"required" => %w{platform profiles statistics version},
"properties" => {
"platform" => Primitives::PLATFORM.ref,
"profiles" => Primitives.array(PROFILE.ref),
"statistics" => Primitives::STATISTICS.ref,
"version" => Primitives::STRING,
},
}, [Primitives::PLATFORM, PROFILE, Primitives::STATISTICS])
end
end
end

40
lib/inspec/schema/exec_json_min.rb Normal file
View file

@ -0,0 +1,40 @@
require "inspec/schema/primitives"
# These type occur only when running "exec --reporter json-min <file>".
module Inspec
module Schema
module ExecJsonMin
# Represents a subset of the information about a control, designed for conciseness
CONTROL = Primitives::SchemaType.new("Exec JSON-MIN Control", {
"type" => "object",
"additionalProperties" => true,
"required" => %w{id profile_id profile_sha256 status code_desc},
"properties" => {
"id" => Primitives::STRING,
"profile_id" => { "type" => %w{string null} },
"profile_sha256" => Primitives::STRING,
"status" => Primitives::STRING,
"code_desc" => Primitives::STRING,
"skip_message" => Primitives::STRING,
"resource" => Primitives::STRING,
"message" => Primitives::STRING,
"exception" => Primitives::STRING,
"backtrace" => Primitives.array(Primitives::STRING),
},
}, [])
# Result of exec jsonmin. Top level value
OUTPUT = Primitives::SchemaType.new("Exec JSON-MIN output", {
"type" => "object",
"additionalProperties" => true,
"required" => %w{statistics controls version},
"properties" => {
"statistics" => Primitives::STATISTICS.ref,
"version" => Primitives::STRING,
"controls" => Primitives.array(CONTROL.ref),
},
}, [Primitives::STATISTICS, CONTROL])
end
end
end

53
lib/inspec/schema/output_schema.rb Normal file
View file

@ -0,0 +1,53 @@
require "json"
require "inspec/schema/primitives"
require "inspec/schema/exec_json"
require "inspec/schema/exec_json_min"
require "inspec/schema/profile_json"
module Inspec
module Schema
module OutputSchema
# Build our definitions
def self.build_definitions(schema_type)
{
"definitions" => schema_type.all_depends.map { |t| [t.ref_name, t.body] }.to_h,
}
end
# Helper function to automatically bundle a type with its dependencies
def self.finalize(schema_type)
schema_type.body.merge(OutputSchema.build_definitions(schema_type))
end
# using a proc here so we can lazy load it when we need
PLATFORMS = lambda do
require "train"
Train.create("mock").connection
Train::Platforms.export
end
LIST = {
"profile-json" => OutputSchema.finalize(Schema::ProfileJson::PROFILE),
"exec-json" => OutputSchema.finalize(Schema::ExecJson::OUTPUT),
"exec-jsonmin" => OutputSchema.finalize(Schema::ExecJsonMin::OUTPUT),
"platforms" => PLATFORMS,
}.freeze
def self.names
LIST.keys
end
def self.json(name)
if !LIST.key?(name)
raise("Cannot find schema #{name.inspect}.")
elsif LIST[name].is_a?(Proc)
v = LIST[name].call
else
v = LIST[name]
end
JSON.dump(v)
end
end
end
end

266
lib/inspec/schema/primitives.rb Normal file
View file

@ -0,0 +1,266 @@
require "set"
# These elements are shared between more than one output type
module Inspec
module Schema
module Primitives
######################### Establish simple helpers for this schema ########################################
# Use this function to easily make described types
def self.desc(obj, description)
obj.merge({ "description" => description })
end
# Use this function to easily make an array of a type
def self.array(of_type)
{
"type" => "array",
"items" => of_type,
}
end
# This function performs some simple validation on schemas, to catch obvious missed requirements
def self.validate_schema(schema)
return if schema["type"] != "object"
raise "Objects in schema must have a \"required\" property, even if it is empty." unless schema.key?("required")
return if schema["required"].empty?
raise "An object with required properties must have a properties hash." unless schema.key?("properties")
return if Set.new(schema["required"]) <= Set.new(schema["properties"].keys)
raise "Not all required properties are present."
end
# Use this class to quickly add/use object types to/in a definition block
class SchemaType
attr_accessor :name, :depends
def initialize(name, body, dependencies)
# Validate the schema
Primitives.validate_schema(body)
# The title of the type
@name = name
# The body of the type
@body = body
# What SchemaType[]s it depends on. In essence, any thing that you .ref in the body
@depends = Set.new(dependencies)
end
# Produce this schema types generated body.
# Use to actually define the ref!
def body
@body.merge({
"title" => @name,
})
end
# Formats this to have a JSON pointer compatible title
def ref_name
@name.gsub(/\s+/, "_")
end
# Yields this type as a json schema ref
def ref
{ "$ref" => "#/definitions/#{ref_name}" }
end
# Recursively acquire all depends for this schema. Return them sorted by name
def all_depends
result = @depends
# Fetch all from children
@depends.each do |nested_type|
# Yes, converting back to set here does some duplicate sorting.
# But here, performance really isn't our concern.
result += Set.new(nested_type.all_depends)
end
# Return the results as a sorted array
Array(result).sort_by(&:name)
end
end
######################### Establish basic type shorthands for this schema ########################################
# These three are essentially primitives, used as shorthand
OBJECT = { "type" => "object", "additionalProperties" => true }.freeze
NUMBER = { "type" => "number" }.freeze
STRING = { "type" => "string" }.freeze
NULL = { "type" => "null" }.freeze
# We might eventually enforce string format stuff on this
URL = { "type" => "string" }.freeze
# A controls tags can have any number of properties, and any sorts of values
TAGS = { "type" => "object", "additionalProperties" => true }.freeze
# Attributes/Inputs specify the inputs to a profile.
INPUT = { "type" => "object", "additionalProperties" => true }.freeze
# Within a control file, impacts can be numeric 0-1 or string in [none,low,medium,high,critical]
# However, when they are output, the string types are automatically converted as follows:
# none -> 0 to 0.01
# low -> 0.01 to 0.4
# medium -> 0.4 to 0.7
# high -> 0.7 to 0.9
# Critical -> 0.9 to 1.0 (inclusive)
IMPACT = {
"type" => "number",
"minimum" => 0.0,
"maximum" => 1.0,
}.freeze
# A status for a control
STATUS = {
"type" => "string",
"enum" => %w{passed failed skipped error},
}.freeze
######################### Establish inspec types common to several schemas helpers for this schema #######################################
# We use "title" to name the type.
# and "description" (via the describe function) to describe its particular usage
# Summary item containing run statistics about a subset of the controls
STATISTIC_ITEM = SchemaType.new("Statistic Block", {
"type" => "object",
"additionalProperties" => true,
"required" => ["total"],
"properties" => {
"total" => desc(NUMBER, "Total number of controls (in this category) for this inspec execution."),
},
}, [])
# Bundles several results statistics, representing distinct groups of controls
STATISTIC_GROUPING = SchemaType.new("Statistic Hash", {
"type" => "object",
"additionalProperties" => true,
"required" => [],
"properties" => {
"passed" => STATISTIC_ITEM.ref,
"skipped" => STATISTIC_ITEM.ref,
"failed" => STATISTIC_ITEM.ref,
},
}, [STATISTIC_ITEM])
# Contains statistics of an exec run.
STATISTICS = SchemaType.new("Statistics", {
"type" => "object",
"additionalProperties" => true,
"required" => ["duration"],
"properties" => {
"duration" => desc(NUMBER, "How long (in seconds) this inspec exec ran for."),
"controls" => desc(STATISTIC_GROUPING.ref, "Breakdowns of control statistics by result"),
},
}, [STATISTIC_GROUPING])
# Profile dependencies
DEPENDENCY = SchemaType.new("Dependency", {
"type" => "object",
"additionalProperties" => true, # Weird stuff shows up here sometimes
"required" => [], # Mysteriously even in a run profile we can have no status
"properties" => {
"name" => STRING,
"url" => URL,
"branch" => STRING,
"path" => STRING,
"skip_message" => STRING,
"status" => STRING,
"git" => URL,
"supermarket" => STRING,
"compliance" => STRING,
},
}, [])
# Represents the platform the test was run on
PLATFORM = SchemaType.new("Platform", {
"type" => "object",
"additionalProperties" => true,
"required" => %w{name release},
"properties" => {
"name" => desc(STRING, "The name of the platform this was run on."),
"release" => desc(STRING, "The version of the platform this was run on."),
"target_id" => STRING,
},
}, [])
# Explains what software ran the inspec profile/made this file. Typically inspec but could in theory be a different software
GENERATOR = SchemaType.new("Generator", {
"type" => "object",
"additionalProperties" => true,
"required" => %w{name version},
"properties" => {
"name" => desc(STRING, "The name of the software that generated this report."),
"version" => desc(STRING, "The version of the software that generated this report."),
},
}, [])
# Occurs from "exec --reporter json" and "inspec json"
# Denotes what file this control comes from, and where within
SOURCE_LOCATION = SchemaType.new("Source Location", {
"type" => "object",
"additionalProperties" => true,
"properties" => {
"ref" => desc(STRING, "Path to the file that this statement originates from"),
"line" => desc(NUMBER, "The line at which this statement is located in the file"),
},
"required" => %w{ref line},
}, [])
# References an external document
REFERENCE = SchemaType.new("Reference", {
# Needs at least one of title (ref), url, or uri.
"anyOf" => [
{
"type" => "object",
"required" => ["ref"],
"properties" => { "ref" => STRING },
},
{
"type" => "object",
"required" => ["url"],
"properties" => { "url" => STRING },
},
{
"type" => "object",
"required" => ["uri"],
"properties" => { "uri" => STRING },
},
# I am of the opinion that this is just an error in the codebase itself. See profiles/wrapper-override to uncover new mysteries!
{
"type" => "object",
"required" => ["ref"],
"properties" => { "ref" => array(OBJECT) },
},
],
}, [])
# Represents a group of controls within a profile/.rb file
CONTROL_GROUP = SchemaType.new("Control Group", {
"type" => "object",
"additionalProperties" => true,
"required" => %w{id controls},
"properties" => {
"id" => desc(STRING, "The unique identifier of the group"),
"title" => desc({ type: %w{string null} }, "The name of the group"),
"controls" => desc(array(STRING), "The control IDs in this group"),
},
}, [])
# Occurs from "inspec exec --reporter json" and "inspec json"
# Represents a platfrom or group of platforms that this profile supports
SUPPORT = SchemaType.new("Supported Platform", {
"type" => "object",
"additionalProperties" => true, # NOTE: This should really be false, and inspec should validate profiles to ensure people don't make dumb mistakes like platform_family
"required" => [],
"properties" => {
"platform-family" => STRING,
"platform-name" => STRING,
"platform" => STRING,
"release" => STRING,
# os-* supports are being deprecated
"os-family" => STRING,
"os-name" => STRING,
},
}, [])
end
end
end

60
lib/inspec/schema/profile_json.rb Normal file
View file

@ -0,0 +1,60 @@
require "inspec/schema/primitives"
# These type occur only when running "inspec json <file>".
module Inspec
module Schema
module ProfileJson
# Represents descriptions. Can have any string => string pairing
CONTROL_DESCRIPTIONS = Primitives::SchemaType.new("Profile JSON Control Descriptions", {
"type" => "object",
"aditionalProperties" => Primitives::STRING,
"required" => [],
}, [])
# Represents a control that hasn't been run
# Differs slightly from a normal control, in that it lacks results, and its descriptions are different
CONTROL = Primitives::SchemaType.new("Profile JSON Control", {
"type" => "object",
"additionalProperties" => true,
"required" => %w{id title desc impact tags code},
"properties" => {
"id" => Primitives.desc(Primitives::STRING, "The ID of this control"),
"title" => { "type" => %w{string null} },
"desc" => { "type" => %w{string null} },
"descriptions" => CONTROL_DESCRIPTIONS.ref,
"impact" => Primitives::IMPACT,
"refs" => Primitives.array(Primitives::REFERENCE.ref),
"tags" => Primitives::TAGS,
"code" => Primitives.desc(Primitives::STRING, "The raw source code of the control. Note that if this is an overlay control, it does not include the underlying source code"),
"source_location" => Primitives::SOURCE_LOCATION.ref,
},
}, [CONTROL_DESCRIPTIONS, Primitives::REFERENCE, Primitives::SOURCE_LOCATION])
# A profile that has not been run.
PROFILE = Primitives::SchemaType.new("Profile JSON Profile", {
"type" => "object",
"additionalProperties" => true, # Anything in the yaml will be put in here. LTTODO: Make this stricter!
"required" => %w{name supports controls groups sha256},
"properties" => {
"name" => Primitives::STRING,
"supports" => Primitives.array(Primitives::SUPPORT.ref),
"controls" => Primitives.array(CONTROL.ref),
"groups" => Primitives.array(Primitives::CONTROL_GROUP.ref),
"inputs" => Primitives.array(Primitives::INPUT),
"sha256" => Primitives::STRING,
"status" => Primitives::STRING,
"generator" => Primitives::GENERATOR.ref,
"version" => Primitives::STRING,
# Other properties possible in inspec docs, but that aren"t guaranteed
"title" => Primitives::STRING,
"maintainer" => Primitives::STRING,
"copyright" => Primitives::STRING,
"copyright_email" => Primitives::STRING,
"depends" => Primitives.array(Primitives::DEPENDENCY.ref), # Can have depends, but NOT a parentprofile
},
}, [Primitives::SUPPORT, CONTROL, Primitives::CONTROL_GROUP, Primitives::DEPENDENCY, Primitives::GENERATOR])
end
end
end

35
test/fixtures/profiles/old-examples/profile/controls/example.rb vendored
View file

@ -1,35 +0,0 @@
# copyright: 2016, Chef Software, Inc.
title 'Example Config Checks'
# To pass the test, create the following file
# ```bash
# mkdir -p /tmp/example
# cat <<EOF > /tmp/example/config.yaml
# version: '1.0'
# EOF
# ```
control 'example-1.0' do
impact 'critical'
title 'Verify the version number of Example'
desc 'An optional description...'
tag 'example'
ref 'Example Requirements 1.0', uri: 'http://...'
# Test using the custom example_config InSpec resource
# Find the resource content here: ../libraries/
describe example_config do
it { should exist }
its('version') { should eq('1.0') }
its('file_size') { should <= 20 }
its('comma_count') { should eq 0 }
end
# Test the version again to showcase variables
g = example_config
g_path = g.file_path
g_version = g.version
describe file(g_path) do
its('content') { should match g_version }
end
end

10
test/fixtures/profiles/old-examples/profile/controls/minimal.rb vendored Normal file
View file

@ -0,0 +1,10 @@
title 'Minimal control'
# you add controls here
control 'minimalist' do # A unique ID for this control
impact 0.7 # The criticality, if this control fails.
describe file('/') do # The actual test
it { should be_directory }
end
end

5
test/functional/helper.rb
View file

@ -25,6 +25,11 @@ module FunctionalHelper
let(:mock_path) { File.join(repo_path, "test", "fixtures") }
let(:profile_path) { File.join(mock_path, "profiles") }
let(:examples_path) { File.join(profile_path, "old-examples") }
let(:integration_test_path) { File.join(repo_path, "test", "integration", "default") }
let(:all_profiles) { Dir.glob("#{profile_path}/**/inspec.yml") }
let(:all_profile_folders) { all_profiles.map { |path| File.dirname(path) } }
let(:complete_profile) { "#{profile_path}/complete-profile" }
let(:example_profile) { File.join(examples_path, "profile") }
let(:meta_profile) { File.join(examples_path, "meta-profile") }
let(:example_control) { File.join(example_profile, "controls", "example-tmp.rb") }

8
test/functional/inheritance_test.rb
View file

@ -55,7 +55,7 @@ describe "example inheritance profile" do
s = out.stdout
hm = JSON.load(s)
_(hm["name"]).must_equal "inheritance"
_(hm["controls"].length).must_equal 5
_(hm["controls"].length).must_equal 6
assert_exit_code 0, out
end
@ -66,15 +66,17 @@ describe "example inheritance profile" do
s = out.stdout
hm = JSON.load(s)
_(hm["name"]).must_equal "inheritance"
_(hm["controls"].length).must_equal 5
_(hm["controls"].length).must_equal 6
assert_exit_code 0, out
end
it "can execute a profile inheritance" do
# TODO: the inheritence profile uses here fails on windows.
skip_windows!
out = inspec("exec " + path + " --reporter json --no-create-lockfile --input-file " + input_file)
_(out.stderr).must_equal ""
_(JSON.load(out.stdout)).must_be_kind_of Hash
assert_exit_code 101, out
assert_exit_code 0, out
end
end

4
test/functional/inspec_exec_automate_test.rb
View file

@ -16,7 +16,7 @@ describe "inspec exec automate" do
end
let(:invocation) do
"exec #{example_profile} --config #{config_path}"
"exec #{complete_profile} --config #{config_path}"
end
let(:run_result) { run_inspec_process(invocation) }
@ -95,7 +95,7 @@ describe "inspec exec automate" do
_(json["passthrough"].keys.sort).must_equal %w{another_tramp_datum projects}
_(json["passthrough"]["projects"]).must_equal %w{alpha beta}
assert_exit_code 101, run_result
assert_exit_code 0, run_result
end
end
end

67
test/functional/inspec_exec_json_test.rb
View file

@ -1,6 +1,5 @@
require "functional/helper"
require "json-schema"
require "inspec/schema"
require "json_schemer"
describe "inspec exec with json formatter" do
include FunctionalHelper
@ -11,32 +10,70 @@ describe "inspec exec with json formatter" do
it "can execute a simple file and validate the json schema" do
out = inspec("exec " + example_control + " --reporter json --no-create-lockfile")
data = JSON.parse(out.stdout)
_(JSON::Validator.validate(schema, data)).wont_equal false
sout = inspec("schema exec-json")
schema = JSONSchemer.schema(sout.stdout)
_(schema.validate(data).to_a).must_equal []
_(out.stderr).must_be_empty
skip_windows!
assert_exit_code 0, out
end
it "can execute a profile and validate the json schema" do
out = inspec("exec " + complete_profile + " --reporter json --no-create-lockfile")
data = JSON.parse(out.stdout)
sout = inspec("schema exec-json")
schema = JSONSchemer.schema(sout.stdout)
_(schema.validate(data).to_a).must_equal []
_(out.stderr).must_equal ""
assert_exit_code 0, out
end
it "can execute a profile and validate the json schema" do
out = inspec("exec " + example_profile + " --reporter json --no-create-lockfile")
it "can execute a simple file while using end of options after reporter cli option" do
out = inspec("exec --no-create-lockfile --reporter json -- " + example_control)
_(out.stderr).must_equal ""
_(out.exit_status).must_equal 0
data = JSON.parse(out.stdout)
_(JSON::Validator.validate(schema, data)).wont_equal false
sout = inspec("schema exec-json")
schema = JSONSchemer.schema(sout.stdout)
_(schema.validate(data).to_a).must_equal []
_(out.stderr).must_equal ""
assert_exit_code 101, out
skip_windows!
assert_exit_code 0, out
end
it "can execute a profile and validate the json schema with target_id" do
out = inspec("exec " + example_profile + " --reporter json --no-create-lockfile --target-id 1d3e399f-4d71-4863-ac54-84d437fbc444")
out = inspec("exec " + complete_profile + " --reporter json --no-create-lockfile --target-id 1d3e399f-4d71-4863-ac54-84d437fbc444")
data = JSON.parse(out.stdout)
_(data["platform"]["target_id"]).must_equal "1d3e399f-4d71-4863-ac54-84d437fbc444"
_(JSON::Validator.validate(schema, data)).wont_equal false
sout = inspec("schema exec-json")
schema = JSONSchemer.schema(sout.stdout)
_(schema.validate(data).to_a).must_equal []
_(out.stderr).must_equal ""
assert_exit_code 101, out
assert_exit_code 0, out
end
it "properly validates all (valid) unit tests against the schema" do
schema = JSONSchemer.schema(JSON.parse(inspec("schema exec-json").stdout))
all_profile_folders.first(1).each do |folder|
begin
out = inspec("exec " + folder + " --reporter json --no-create-lockfile")
# Ensure it parses properly
out = JSON.parse(out.stdout)
failures = schema.validate(out).to_a
_(failures).must_equal []
rescue JSON::ParserError
# We don't actually care about these; cannot validate if parsing fails!
nil
end
end
end
it "does not report skipped dependent profiles" do
@ -110,7 +147,6 @@ describe "inspec exec with json formatter" do
let(:controls) { profile["controls"] }
let(:ex1) { controls.find { |x| x["id"] == "tmp-1.0" } }
let(:ex2) { controls.find { |x| x["id"] =~ /generated/ } }
let(:ex3) { profile["controls"].find { |x| x["id"] == "example-1.0" } }
let(:check_result) do
ex3["results"].find { |x| x["resource"] == "example_config" }
end
@ -121,7 +157,7 @@ describe "inspec exec with json formatter" do
end
it "maps impact symbols to numbers" do
_(ex3["impact"]).must_equal 0.9
_(ex1["impact"]).must_equal 0.7
end
it "has all the metadata" do
@ -148,12 +184,12 @@ describe "inspec exec with json formatter" do
{ "id" => "controls/example-tmp.rb",
"title" => "/ profile",
"controls" => ["tmp-1.0", key] },
{ "id" => "controls/example.rb",
"title" => "Example Config Checks",
"controls" => ["example-1.0"] },
{ "id" => "controls/meta.rb",
"title" => "SSH Server Configuration",
"controls" => ["ssh-1"] },
{ "id" => "controls/minimal.rb",
"title" => "Minimal control",
"controls" => ["minimalist"] },
])
end
@ -168,7 +204,6 @@ describe "inspec exec with json formatter" do
it "has results for every control" do
_(ex1["results"].length).must_equal 1
_(ex2["results"].length).must_equal 1
_(ex3["results"].length).must_equal 2
end
it "has the right result for tmp-1.0" do

63
test/functional/inspec_exec_jsonmin_test.rb
View file

@ -1,27 +1,57 @@
require "functional/helper"
require "json-schema"
require "inspec/schema"
require "json_schemer"
describe "inspec exec" do
include FunctionalHelper
parallelize_me!
let(:out) { inspec("exec " + example_profile + " --reporter json-min --no-create-lockfile") }
let(:out) { inspec("exec " + complete_profile + " --reporter json-min --no-create-lockfile") }
let(:json) { JSON.load(out.stdout) }
it "can execute a profile with the mini json formatter and validate its schema" do
data = JSON.parse(out.stdout)
sout = Inspec::Schema.json("exec-jsonmin")
schema = JSON.parse(sout)
_(JSON::Validator.validate(schema, data)).wont_equal false
sout = inspec("schema exec-jsonmin")
schema = JSONSchemer.schema(sout.stdout)
_(schema.validate(data).to_a).must_equal []
_(out.stderr).must_equal ""
assert_exit_code 101, out
assert_exit_code 0, out
end
it "does not contain any duplicate results with describe.one" do
it "can execute a simple file with the mini json formatter and validate its schema" do
out = inspec("exec " + example_control + " --reporter json-min --no-create-lockfile")
_(out.stderr).must_equal ""
_(out.exit_status).must_equal 0
data = JSON.parse(out.stdout)
sout = inspec("schema exec-jsonmin")
schema = JSONSchemer.schema(sout.stdout)
_(schema.validate(data).to_a).must_equal []
_(out.stderr).must_equal ""
skip_windows!
assert_exit_code 0, out
end
it "properly validates all (valid) unit tests against the schema" do
schema = JSONSchemer.schema(JSON.parse(inspec("schema exec-jsonmin").stdout))
all_profile_folders.first(1).each do |folder|
begin
out = inspec("exec " + folder + " --reporter json-min --no-create-lockfile")
# Ensure it parses properly; discard the result
out = JSON.parse(out.stdout)
failures = schema.validate(out).to_a
_(failures).must_equal []
rescue JSON::ParserError
# We don't actually care about these; cannot validate if parsing fails!
nil
end
end
end
it "does not contain any dupilcate results with describe.one" do
out = inspec("shell -c 'describe.one do describe 1 do it { should cmp 2 } end end' --reporter=json-min")
data = JSON.parse(out.stdout)
_(data["controls"].length).must_equal 1
@ -34,17 +64,15 @@ describe "inspec exec" do
describe "execute a profile with mini json formatting" do
let(:controls) { json["controls"] }
let(:ex1) { controls.find { |x| x["id"] == "tmp-1.0" } }
let(:ex2) { controls.find { |x| x["id"] =~ /generated/ } }
let(:ex3) { controls.find { |x| x["id"] == "example-1.0" } }
let(:ex1) { controls.find { |x| x["id"] == "test01" } }
before do
# doesn't make sense on windows TODO: change the profile so it does?
skip if windows?
end
it "must have 5 examples" do
_(json["controls"].length).must_equal 5
it "must have 1 example" do
_(json["controls"].length).must_equal 1
end
it "has an id" do
@ -56,19 +84,12 @@ describe "inspec exec" do
end
it "has a code_desc" do
_(ex1["code_desc"]).must_equal "File / is expected to be directory"
_(ex1["code_desc"]).must_equal "Host example.com is expected to be resolvable"
_(controls.find { |ex| !ex.key? "code_desc" }).must_be :nil?
end
it "has a status" do
skip_windows!
_(ex1["status"]).must_equal "passed"
_(ex3["status"]).must_equal "skipped"
end
it "has a skip_message" do
_(ex1["skip_message"]).must_be :nil?
_(ex3["skip_message"]).must_equal "Can't find file `/tmp/example/config.yaml`"
end
end

14
test/functional/inspec_exec_junit_test.rb
View file

@ -20,7 +20,7 @@ describe "inspec exec with junit formatter" do
end
it "can execute the profile with the junit formatter" do
out = inspec("exec " + example_profile + " --reporter junit --no-create-lockfile")
out = inspec("exec " + complete_profile + " --reporter junit --no-create-lockfile")
# TODO: _never_ use rexml. Anything else is guaranteed faster
doc = REXML::Document.new(out.stdout)
@ -28,7 +28,7 @@ describe "inspec exec with junit formatter" do
_(out.stderr).must_equal ""
assert_exit_code 101, out
assert_exit_code 0, out
end
describe "execute a profile with junit formatting" do
@ -42,12 +42,12 @@ describe "inspec exec with junit formatter" do
describe "the test suite" do
let(:suite) { doc.elements.to_a("//testsuites/testsuite").first }
it "must have 5 testcase children" do
_(suite.elements.to_a("//testcase").length).must_equal 5
it "must have 6 testcase children" do
_(suite.elements.to_a("//testcase").length).must_equal 4
end
it "has the tests attribute with 5 total tests" do
_(suite.attribute("tests").value).must_equal "5"
_(suite.attribute("tests").value).must_equal "4"
end
it "has the failures attribute with 0 total tests" do
@ -56,7 +56,7 @@ describe "inspec exec with junit formatter" do
end
it 'has 2 elements named "File / should be directory"' do
_(REXML::XPath.match(suite, "//testcase[@name='File / is expected to be directory']").length).must_equal 2
_(REXML::XPath.match(suite, "//testcase[@name='File / is expected to be directory']").length).must_equal 3
end
describe 'the testcase named "example_config Can\'t find file ..."' do
@ -64,10 +64,12 @@ describe "inspec exec with junit formatter" do
let(:first_example_test) { example_yml_tests.first }
it "should be unique" do
skip
_(example_yml_tests.length).must_equal 1
end
it "should be skipped" do
skip
if is_windows?
_(first_example_test.elements.to_a("//skipped").length).must_equal 2
else

54
test/functional/inspec_exec_test.rb
View file

@ -41,26 +41,14 @@ describe "inspec exec" do
end
it "can execute the profile" do
inspec("exec " + example_profile + " --no-create-lockfile")
inspec("exec " + complete_profile + " --no-create-lockfile")
_(stdout).must_include " ✔ tmp-1.0: Create / directory\n"
_(stdout).must_include "
↺ example-1.0: Verify the version number of Example (1 skipped)
↺ Can't find file `/tmp/example/config.yaml`
"
if is_windows?
_(stdout).must_include " ↺ ssh-1: Allow only SSH Protocol 2\n"
_(stdout).must_include "\nProfile Summary: 1 successful control, 0 control failures, 2 controls skipped\n"
_(stdout).must_include "\nTest Summary: 3 successful, 0 failures, 2 skipped\n"
else
_(stdout).must_include " ✔ ssh-1: Allow only SSH Protocol 2\n"
_(stdout).must_include "\nProfile Summary: 2 successful controls, 0 control failures, 1 control skipped\n"
_(stdout).must_include "\nTest Summary: 4 successful, 0 failures, 1 skipped\n"
end
_(stdout).must_include "Host example.com"
_(stdout).must_include "1 successful control, "\
"0 control failures, 0 controls skipped"
_(stderr).must_be_empty
_(stderr).must_equal ""
assert_exit_code 101, out
assert_exit_code 0, out
end
it "executes a minimum metadata-only profile" do
@ -82,14 +70,14 @@ Test Summary: 0 successful, 0 failures, 0 skipped
it "can execute the profile and write to directory" do
outpath = Dir.tmpdir
inspec("exec #{example_profile} --no-create-lockfile --reporter json:#{outpath}/foo/bar/test.json")
inspec("exec #{complete_profile} --no-create-lockfile --reporter json:#{outpath}/foo/bar/test.json")
_(File.exist?("#{outpath}/foo/bar/test.json")).must_equal true
_(File.stat("#{outpath}/foo/bar/test.json").size).must_be :>, 0
_(stderr).must_equal ""
assert_exit_code 101, out
assert_exit_code 0, out
end
it "can execute --help after exec command" do
@ -123,13 +111,13 @@ Test Summary: 0 successful, 0 failures, 0 skipped
end
it "can execute the profile with a target_id passthrough" do
inspec("exec #{example_profile} --no-create-lockfile --target-id 1d3e399f-4d71-4863-ac54-84d437fbc444")
inspec("exec #{complete_profile} --no-create-lockfile --target-id 1d3e399f-4d71-4863-ac54-84d437fbc444")
_(stdout).must_include "Target ID: 1d3e399f-4d71-4863-ac54-84d437fbc444"
_(stderr).must_equal ""
assert_exit_code 101, out
assert_exit_code 0, out
end
it "executes a metadata-only profile" do
@ -228,15 +216,17 @@ Test Summary: 0 successful, 0 failures, 0 skipped
it "does not vendor profiles when using the a local path dependecy" do
Dir.mktmpdir do |tmpdir|
command = "exec " + inheritance_profile + " --no-create-lockfile"
command = "exec " + inheritance_profile + " --no-create-lockfile " \
"--input-file=#{examples_path}/profile-attribute.yml"
inspec_with_env(command, INSPEC_CONFIG_DIR: tmpdir)
if is_windows?
_(stdout).must_include "Profile Summary: 0 successful controls, 0 control failures, 2 controls skipped\n"
_(stdout).must_include "Test Summary: 2 successful, 1 failure, 3 skipped\n"
_(stdout).must_include "No tests executed."
assert_exit_code 1, out
else
_(stdout).must_include "Profile Summary: 1 successful control, 0 control failures, 1 control skipped\n"
_(stdout).must_include "Test Summary: 3 successful, 1 failure, 2 skipped\n"
_(stdout).must_include "Profile Summary: 2 successful controls, 0 control failures, 0 controls skipped\n"
_(stdout).must_include "Test Summary: 5 successful, 0 failures, 0 skipped\n"
assert_exit_code 0, out
end
cache_dir = File.join(tmpdir, "cache")
@ -244,8 +234,6 @@ Test Summary: 0 successful, 0 failures, 0 skipped
_(Dir.glob(File.join(cache_dir, "**", "*"))).must_be_empty
_(stderr).must_equal ""
assert_exit_code 100, out
end
end
@ -525,7 +513,7 @@ Test Summary: 2 successful, 0 failures, 0 skipped\n"
describe "when --password is used" do
it "raises an exception if no password is provided" do
inspec("exec " + example_profile + " --password")
inspec("exec " + complete_profile + " --password")
_(stderr).must_include "Please provide a value for --password. For example: --password=hello."
@ -535,7 +523,7 @@ Test Summary: 2 successful, 0 failures, 0 skipped\n"
describe "when --sudo-password is used" do
it "raises an exception if no sudo password is provided" do
inspec("exec " + example_profile + " --sudo-password")
inspec("exec " + complete_profile + " --sudo-password")
_(stderr).must_include "Please provide a value for --sudo-password. For example: --sudo-password=hello."
@ -545,7 +533,7 @@ Test Summary: 2 successful, 0 failures, 0 skipped\n"
describe "when --bastion-host and --proxy_command is used" do
it "raises an exception when both flags are provided" do
inspec("exec " + example_profile + " -t ssh://dummy@dummy --password dummy --proxy_command dummy --bastion_host dummy")
inspec("exec " + complete_profile + " -t ssh://dummy@dummy --password dummy --proxy_command dummy --bastion_host dummy")
_(stderr).must_include "Client error, can't connect to 'ssh' backend: Only one of proxy_command or bastion_host needs to be specified"
@ -555,7 +543,7 @@ Test Summary: 2 successful, 0 failures, 0 skipped\n"
describe "when --winrm-transport is used" do
it "raises an exception when an invalid transport is given" do
inspec("exec " + example_profile + " -t winrm://administrator@dummy --password dummy --winrm-transport nonesuch")
inspec("exec " + complete_profile + " -t winrm://administrator@dummy --password dummy --winrm-transport nonesuch")
_(stderr).must_include "Client error, can't connect to 'winrm' backend: Unsupported transport type: :nonesuch\n"

30
test/functional/inspec_json_profile_test.rb
View file

@ -1,5 +1,6 @@
require "functional/helper"
require "mixlib/shellout"
require "json_schemer"
describe "inspec json" do
include FunctionalHelper
@ -178,4 +179,33 @@ describe "inspec json" do
assert_exit_code 0, out
end
end
it "can format a profile and validate the json schema" do
out = inspec("json " + example_profile)
data = JSON.parse(out.stdout)
sout = inspec("schema profile-json")
schema = JSONSchemer.schema(sout.stdout)
_(schema.validate(data).to_a).must_equal []
_(out.stderr).must_equal ""
assert_exit_code 0, out
end
it "properly validates all (valid) unit tests against the schema" do
schema = JSONSchemer.schema(JSON.parse(inspec("schema profile-json").stdout))
all_profile_folders.first(1).each do |folder|
begin
out = inspec("json " + folder)
# Ensure it parses properly; discard the result
out = JSON.parse(out.stdout)
failures = schema.validate(out).to_a
_(failures).must_equal []
rescue JSON::ParserError
# We don't actually care about these; cannot validate if parsing fails!
nil
end
end
end
end

71
test/unit/schema/primivites_test.rb Normal file
View file

@ -0,0 +1,71 @@
# Tests that the components of the schema generation framework are behaving properly
require "helper"
require "inspec/schema/primitives"
describe Inspec::Schema::Primitives do
describe "title construction" do
let(:schema_alpha) do
Inspec::Schema::Primitives::SchemaType.new("Alpha", {
"type" => "string",
}, [])
end
let(:schema_beta) do
Inspec::Schema::Primitives::SchemaType.new("Beta", {
"type" => "object",
"required" => [],
"properties" => {
"param1" => { "type" => "number" },
},
}, [])
end
# Omega nests alpha and beta
let(:schema_omega) do
Inspec::Schema::Primitives::SchemaType.new("Omega", {
"type" => "object",
"required" => ["my_beta"],
"properties" => {
"my_alpha" => schema_alpha.ref,
"my_beta" => schema_beta.ref,
},
}, [schema_alpha, schema_beta])
end
it "should add the title to schema bodies" do
_(schema_alpha.body["title"]).must_equal "Alpha"
_(schema_beta.body["title"]).must_equal "Beta"
_(schema_omega.body["title"]).must_equal "Omega"
end
it "should properly generate ref keys" do
skip "Test Unimplemented"
end
end
describe "property validation" do
it "detects if an object does not define required properties" do
_ {
Inspec::Schema::Primitives::SchemaType.new("Test1", {
"type" => "object",
"properties" => {
"hello" => { "type" => "string" },
},
}, [])
}.must_raise RuntimeError
end
it "detects if a required property is missing" do
_ {
Inspec::Schema::Primitives::SchemaType.new("Test2", {
"type" => "object",
"required" => %w{alpha beta},
"properties" => {
"alpha" => { "type" => "number" },
"omega" => { "type" => "number" },
},
}, [])
}.must_raise RuntimeError
end
end
end