Mirrors/inspec
2
0
Fork 0
mirror of https://github.com/inspec/inspec synced 2024-11-10 07:04:15 +00:00
Code Issues Projects Releases Packages Wiki Activity

Revision of schema to JSON Schema draft 7

Browse source 
Signed-off-by: Jacob Henry <jchenry@mitre.org>
This commit is contained in:
Jacob Henry 2019-07-24 14:10:15 -04:00 committed by Ryan Davis
parent c3a37ee126
commit 724f82237b
17 changed files with 793 additions and 51 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
inspec-core.gemspec
View file

@ -26,7 +26,7 @@ Gem::Specification.new do |spec|
spec.add_dependency "chef-telemetry", "~> 1.0"
spec.add_dependency "license-acceptance", ">= 0.2.13", "< 2.0"
spec.add_dependency "thor", ">= 0.20", "< 2.0"
spec.add_dependency "json-schema", "~> 2.8"
spec.add_dependency "json_schemer", "~> 0.2.1"
spec.add_dependency "method_source", "~> 0.8"
spec.add_dependency "rubyzip", "~> 1.2", ">= 1.2.2"
spec.add_dependency "rspec", "~> 3.9"

6
lib/inspec/cli.rb
View file

@ -377,12 +377,12 @@ class Inspec::InspecCLI < Inspec::BaseCLI
desc "schema NAME", "print the JSON schema", hide: true
def schema(name)
require "inspec/schema"
require "inspec/schema/output_schema"
puts Inspec::Schema.json(name)
puts Inspec::Schema::OutputSchema.json(name)
rescue StandardError => e
puts e
puts "Valid schemas are #{Inspec::Schema.names.join(", ")}"
puts "Valid schemas are #{Inspec::Schema::OutputSchema.names.join(", ")}"
end
desc "version", "prints the version of this tool"

17
lib/inspec/schema/README.md Normal file
View file

@ -0,0 +1,17 @@
# Schema Generation
These files handle the generation of JSON schema's for inspec output as yielded by `inspec schema <schema-name>`.
This schema is in the JSON Schema 4th revision spec.
## Structure
Data/schema structures shared between all outputs are located in `primitives.rb`
The files `exec_json_min.rb`, `exec_json.rb`, and `profile_json.rb` contain JSON structures for their specific output types, respectively.
The output schema is "flat" in terms of schema structure, with each "type" defined in the JSONs `definitions` block and referred to via $ref.
The logic for this is handled via the `SchemaType` class in `primitives.rb`.
All relevant dependencies are included recursively via the `dependencies` property of that class, constructed using the `build_definitions` method in `output_schema.rb`.
## Reasoning and validation
In general the first iteration of this updated schema was made by examining the output of all of the profiles in the tests directory.
All of the profiles generated by those tests successfully validate against this schema.
However, in light of potential missed properties that aren't covered by existing tests, the schema was left very non-strict.
More specifically, "additionalProperties" was left as True on all schema objects.

128
lib/inspec/schema/exec_json.rb Normal file
View file

@ -0,0 +1,128 @@
require "inspec/schema/primitives"
# These type occur only when running "inspec exec --reporter json <file>".
module Inspec
module Schema
module ExecJson
# Represents a label and description, to provide human-readable info about a control
CONTROL_DESCRIPTION = Primitives::SchemaType.new("Control Description", {
"type" => "object",
"additionalProperties" => true,
"required" => %w{label data},
"properties" => {
"label" => Primitives::STRING,
"data" => Primitives::STRING,
},
}, [])
# Lists the potential values for a control result
CONTROL_RESULT_STATUS = Primitives::SchemaType.new("Control Result Status", {
"type" => "string",
"enum" => %w{passed failed skipped error},
}, [])
# Represents the statistics/result of a control"s execution
CONTROL_RESULT = Primitives::SchemaType.new("Control Result", {
"type" => "object",
"additionalProperties" => true,
"required" => %w{code_desc run_time start_time},
"properties" => {
"status" => CONTROL_RESULT_STATUS.ref,
"code_desc" => Primitives::STRING,
"run_time" => Primitives::NUMBER,
"start_time" => Primitives::STRING,
# All optional
"resource" => Primitives::STRING,
"message" => Primitives::STRING,
"skip_message" => Primitives::STRING,
"exception" => Primitives::STRING,
"backtrace" => {
"anyOf" => [
Primitives.array(Primitives::STRING),
Primitives::NULL,
],
},
},
}, [CONTROL_RESULT_STATUS])
# Represents a control produced
CONTROL = Primitives::SchemaType.new("Exec JSON Control", {
"type" => "object",
"additionalProperties" => true,
"required" => %w{id title desc impact refs tags code source_location results},
"properties" => {
"id" => Primitives.desc(Primitives::STRING, "The ID of this control"),
"title" => { "type" => %w{string null} }, # Nullable string
"desc" => { "type" => %w{string null} },
"descriptions" => Primitives.array(CONTROL_DESCRIPTION.ref),
"impact" => Primitives::IMPACT,
"refs" => Primitives.array(Primitives::REFERENCE.ref),
"tags" => Primitives::TAGS,
"code" => Primitives.desc(Primitives::STRING, "The raw source code of the control. Note that if this is an overlay control, it does not include the underlying source code"),
"source_location" => Primitives::SOURCE_LOCATION.ref,
"results" => Primitives.desc(Primitives.array(CONTROL_RESULT.ref), %q{
A list of all results of the controls describe blocks.
For instance, if in the controls code we had the following:
describe sshd_config do
its('Port') { should cmp 22 }
end
The result of this block as a ControlResult would be appended to the results list.
}),
},
}, [CONTROL_DESCRIPTION, Primitives::REFERENCE, Primitives::SOURCE_LOCATION, CONTROL_RESULT])
# Based loosely on https://www.inspec.io/docs/reference/profiles/ as of July 3, 2019
# However, concessions were made to the reality of current reporters, specifically
# with how description is omitted and version/inspec_version aren't as advertised online
PROFILE = Primitives::SchemaType.new("Exec JSON Profile", {
"type" => "object",
"additionalProperties" => true,
"required" => %w{name sha256 supports attributes groups controls},
# Name is mandatory in inspec.yml.
# supports, controls, groups, and attributes are always present, even if empty
# sha256, status, skip_message
"properties" => {
# These are provided in inspec.yml
"name" => Primitives::STRING,
"title" => Primitives::STRING,
"maintainer" => Primitives::STRING,
"copyright" => Primitives::STRING,
"copyright_email" => Primitives::STRING,
"depends" => Primitives.array(Primitives::DEPENDENCY.ref),
"parent_profile" => Primitives::STRING,
"license" => Primitives::STRING,
"summary" => Primitives::STRING,
"version" => Primitives::STRING,
"supports" => Primitives.array(Primitives::SUPPORT.ref),
"description" => Primitives::STRING,
"inspec_version" => Primitives::STRING,
# These are generated at runtime, and all except skip_message are guaranteed
"sha256" => Primitives::STRING,
"status" => Primitives::STRING,
"skip_message" => Primitives::STRING, # If skipped, why
"controls" => Primitives.array(CONTROL.ref),
"groups" => Primitives.array(Primitives::CONTROL_GROUP.ref),
"attributes" => Primitives.array(Primitives::INPUT),
},
}, [CONTROL, Primitives::CONTROL_GROUP, Primitives::DEPENDENCY, Primitives::SUPPORT])
# Result of exec json. Top level value
# TODO: Include the format of top level controls. This was omitted for lack of sufficient examples
OUTPUT = Primitives::SchemaType.new("Exec JSON Output", {
"type" => "object",
"additionalProperties" => true,
"required" => %w{platform profiles statistics version},
"properties" => {
"platform" => Primitives::PLATFORM.ref,
"profiles" => Primitives.array(PROFILE.ref),
"statistics" => Primitives::STATISTICS.ref,
"version" => Primitives::STRING,
},
}, [Primitives::PLATFORM, PROFILE, Primitives::STATISTICS])
end
end
end

40
lib/inspec/schema/exec_json_min.rb Normal file
View file

@ -0,0 +1,40 @@
require "inspec/schema/primitives"
# These type occur only when running "exec --reporter json-min <file>".
module Inspec
module Schema
module ExecJsonMin
# Represents a subset of the information about a control, designed for conciseness
CONTROL = Primitives::SchemaType.new("Exec JSON-MIN Control", {
"type" => "object",
"additionalProperties" => true,
"required" => %w{id profile_id profile_sha256 status code_desc},
"properties" => {
"id" => Primitives::STRING,
"profile_id" => { "type" => %w{string null} },
"profile_sha256" => Primitives::STRING,
"status" => Primitives::STRING,
"code_desc" => Primitives::STRING,
"skip_message" => Primitives::STRING,
"resource" => Primitives::STRING,
"message" => Primitives::STRING,
"exception" => Primitives::STRING,
"backtrace" => Primitives.array(Primitives::STRING),
},
}, [])
# Result of exec jsonmin. Top level value
OUTPUT = Primitives::SchemaType.new("Exec JSON-MIN output", {
"type" => "object",
"additionalProperties" => true,
"required" => %w{statistics controls version},
"properties" => {
"statistics" => Primitives::STATISTICS.ref,
"version" => Primitives::STRING,
"controls" => Primitives.array(CONTROL.ref),
},
}, [Primitives::STATISTICS, CONTROL])
end
end
end

53
lib/inspec/schema/output_schema.rb Normal file
View file

@ -0,0 +1,53 @@
require "json"
require "inspec/schema/primitives"
require "inspec/schema/exec_json"
require "inspec/schema/exec_json_min"
require "inspec/schema/profile_json"
module Inspec
module Schema
module OutputSchema
# Build our definitions
def self.build_definitions(schema_type)
{
"definitions" => schema_type.all_depends.map { |t| [t.ref_name, t.body] }.to_h,
}
end
# Helper function to automatically bundle a type with its dependencies
def self.finalize(schema_type)
schema_type.body.merge(OutputSchema.build_definitions(schema_type))
end
# using a proc here so we can lazy load it when we need
PLATFORMS = lambda do
require "train"
Train.create("mock").connection
Train::Platforms.export
end
LIST = {
"profile-json" => OutputSchema.finalize(Schema::ProfileJson::PROFILE),
"exec-json" => OutputSchema.finalize(Schema::ExecJson::OUTPUT),
"exec-jsonmin" => OutputSchema.finalize(Schema::ExecJsonMin::OUTPUT),
"platforms" => PLATFORMS,
}.freeze
def self.names
LIST.keys
end
def self.json(name)
if !LIST.key?(name)
raise("Cannot find schema #{name.inspect}.")
elsif LIST[name].is_a?(Proc)
v = LIST[name].call
else
v = LIST[name]
end
JSON.dump(v)
end
end
end
end

265
lib/inspec/schema/primitives.rb Normal file
View file

@ -0,0 +1,265 @@
require "set"
# These elements are shared between more than one output type
module Inspec
module Schema
module Primitives
######################### Establish simple helpers for this schema ########################################
# Use this function to easily make described types
def self.desc(obj, description)
obj.merge({ "description" => description })
end
# Use this function to easily make an array of a type
def self.array(of_type)
{
"type" => "array",
"items" => of_type,
}
end
# This function performs some simple validation on schemas, to catch obvious missed requirements
def self.validate_schema(schema)
return if schema["type"] != "object"
raise "Objects in schema must have a \"required\" property, even if it is empty." unless schema.key?("required")
return if schema["required"].empty?
raise "An object with required properties must have a properties hash." unless schema.key?("properties")
return if Set.new(schema["required"]) <= Set.new(schema["properties"].keys)
raise "Not all required properties are present."
end
# Use this class to quickly add/use object types to/in a definition block
class SchemaType
attr_accessor :name, :depends
def initialize(name, body, dependencies)
# Validate the schema
Primitives.validate_schema(body)
# The title of the type
@name = name
# The body of the type
@body = body
# What SchemaType[]s it depends on. In essence, any thing that you .ref in the body
@depends = Set.new(dependencies)
end
# Produce this schema types generated body.
# Use to actually define the ref!
def body
@body.merge({
"title" => @name,
})
end
# Formats this to have a JSON pointer compatible title
def ref_name
@name.gsub(/\s+/, "_")
end
# Yields this type as a json schema ref
def ref
{ "$ref" => "#/definitions/#{ref_name}" }
end
# Recursively acquire all depends for this schema. Return them sorted by name
def all_depends
result = @depends
# Fetch all from children
@depends.each do |nested_type|
# Yes, converting back to set here does some duplicate sorting.
# But here, performance really isn't our concern.
result += Set.new(nested_type.all_depends)
end
# Return the results as a sorted array
Array(result).sort_by { |type| type.name }
end
end
######################### Establish basic type shorthands for this schema ########################################
# These three are essentially primitives, used as shorthand
OBJECT = { "type" => "object", "additionalProperties" => true }.freeze
NUMBER = { "type" => "number" }.freeze
STRING = { "type" => "string" }.freeze
NULL = { "type" => "null" }.freeze
# We might eventually enforce string format stuff on this
URL = { "type" => "string" }.freeze
# A controls tags can have any number of properties, and any sorts of values
TAGS = { "type" => "object", "additionalProperties" => true }.freeze
# Attributes/Inputs specify the inputs to a profile.
INPUT = { "type" => "object", "additionalProperties" => true }.freeze
# Within a control file, impacts can be numeric 0-1 or string in [none,low,medium,high,critical]
# However, when they are output, the string types are automatically converted as follows:
# none -> 0 to 0.01
# low -> 0.01 to 0.4
# medium -> 0.4 to 0.7
# high -> 0.7 to 0.9
# Critical -> 0.9 to 1.0 (inclusive)
IMPACT = {
"type" => "number",
"minimum" => 0.0,
"maximum" => 1.0,
}.freeze
# A status for a control
STATUS = {
"type" => "string",
"enum" => %w{passed failed skipped error},
}.freeze
######################### Establish inspec types common to several schemas helpers for this schema #######################################
# We use "title" to name the type.
# and "description" (via the describe function) to describe its particular usage
# Summary item containing run statistics about a subset of the controls
STATISTIC_ITEM = SchemaType.new("Statistic Block", {
"type" => "object",
"additionalProperties" => true,
"required" => ["total"],
"properties" => {
"total" => desc(NUMBER, "Total number of controls (in this category) for this inspec execution."),
},
}, [])
# Bundles several results statistics, representing distinct groups of controls
STATISTIC_GROUPING = SchemaType.new("Statistic Hash", {
"type" => "object",
"additionalProperties" => true,
"required" => [],
"properties" => {
"passed" => STATISTIC_ITEM.ref,
"skipped" => STATISTIC_ITEM.ref,
"failed" => STATISTIC_ITEM.ref,
},
}, [STATISTIC_ITEM])
# Contains statistics of an exec run.
STATISTICS = SchemaType.new("Statistics", {
"type" => "object",
"additionalProperties" => true,
"required" => ["duration"],
"properties" => {
"duration" => desc(NUMBER, "How long (in seconds) this inspec exec ran for."),
"controls" => desc(STATISTIC_GROUPING.ref, "Breakdowns of control statistics by result"),
},
}, [STATISTIC_GROUPING])
# Profile dependencies
DEPENDENCY = SchemaType.new("Dependency", {
"type" => "object",
"additionalProperties" => true, # Weird stuff shows up here sometimes
"required" => [], # Mysteriously even in a run profile we can have no status
"properties" => {
"name" => STRING,
"url" => URL,
"branch" => STRING,
"path" => STRING,
"skip_message" => STRING,
"status" => STRING,
"git" => URL,
"supermarket" => STRING,
"compliance" => STRING,
},
}, [])
# Represents the platform the test was run on
PLATFORM = SchemaType.new("Platform", {
"type" => "object",
"additionalProperties" => true,
"required" => %w{name release},
"properties" => {
"name" => desc(STRING, "The name of the platform this was run on."),
"release" => desc(STRING, "The version of the platform this was run on."),
"target_id" => STRING,
},
}, [])
# Explains what software ran the inspec profile/made this file. Typically inspec but could in theory be a different software
GENERATOR = SchemaType.new("Generator", {
"type" => "object",
"additionalProperties" => true,
"required" => %w{name version},
"properties" => {
"name" => desc(STRING, "The name of the software that generated this report."),
"version" => desc(STRING, "The version of the software that generated this report."),
},
}, [])
# Occurs from "exec --reporter json" and "inspec json"
# Denotes what file this control comes from, and where within
SOURCE_LOCATION = SchemaType.new("Source Location", {
"type" => "object",
"additionalProperties" => true,
"properties" => {
"ref" => desc(STRING, "Path to the file that this statement originates from"),
"line" => desc(NUMBER, "The line at which this statement is located in the file"),
},
"required" => %w{ref line},
}, [])
# References an external document
REFERENCE = SchemaType.new("Reference", {
# Needs at least one of title (ref), url, or uri.
"anyOf" => [
{
"type" => "object",
"required" => ["ref"],
"properties" => { "ref" => STRING },
},
{
"type" => "object",
"required" => ["url"],
"properties" => { "url" => STRING },
},
{
"type" => "object",
"required" => ["uri"],
"properties" => { "uri" => STRING },
},
# I am of the opinion that this is just an error in the codebase itself. See profiles/wrapper-override to uncover new mysteries!
{
"type" => "object",
"required" => ["ref"],
"properties" => { "ref" => array(OBJECT) },
},
],
}, [])
# Represents a group of controls within a profile/.rb file
CONTROL_GROUP = SchemaType.new("Control Group", {
"type" => "object",
"additionalProperties" => true,
"required" => %w{id controls},
"properties" => {
"id" => desc(STRING, "The unique identifier of the group"),
"title" => desc({ type: %w{string null} }, "The name of the group"),
"controls" => desc(array(STRING), "The control IDs in this group"),
},
}, [])
# Occurs from "inspec exec --reporter json" and "inspec json"
# Represents a platfrom or group of platforms that this profile supports
SUPPORT = SchemaType.new("Supported Platform", {
"type" => "object",
"additionalProperties" => true, # NOTE: This should really be false, and inspec should validate profiles to ensure people don't make dumb mistakes like platform_family
"required" => [],
"properties" => {
"platform-family" => STRING,
"platform-name" => STRING,
"platform" => STRING,
"release" => STRING,
# os-* supports are being deprecated
"os-family" => STRING,
"os-name" => STRING,
},
}, [])
end
end
end

60
lib/inspec/schema/profile_json.rb Normal file
View file

@ -0,0 +1,60 @@
require "inspec/schema/primitives"
# These type occur only when running "inspec json <file>".
module Inspec
module Schema
module ProfileJson
# Represents descriptions. Can have any string => string pairing
CONTROL_DESCRIPTIONS = Primitives::SchemaType.new("Profile JSON Control Descriptions", {
"type" => "object",
"aditionalProperties" => Primitives::STRING,
"required" => [],
}, [])
# Represents a control that hasn't been run
# Differs slightly from a normal control, in that it lacks results, and its descriptions are different
CONTROL = Primitives::SchemaType.new("Profile JSON Control", {
"type" => "object",
"additionalProperties" => true,
"required" => %w{id title desc impact tags code},
"properties" => {
"id" => Primitives.desc(Primitives::STRING, "The ID of this control"),
"title" => { "type" => %w{string null} },
"desc" => { "type" => %w{string null} },
"descriptions" => CONTROL_DESCRIPTIONS.ref,
"impact" => Primitives::IMPACT,
"refs" => Primitives.array(Primitives::REFERENCE.ref),
"tags" => Primitives::TAGS,
"code" => Primitives.desc(Primitives::STRING, "The raw source code of the control. Note that if this is an overlay control, it does not include the underlying source code"),
"source_location" => Primitives::SOURCE_LOCATION.ref,
},
}, [CONTROL_DESCRIPTIONS, Primitives::REFERENCE, Primitives::SOURCE_LOCATION])
# A profile that has not been run.
PROFILE = Primitives::SchemaType.new("Profile JSON Profile", {
"type" => "object",
"additionalProperties" => true, # Anything in the yaml will be put in here. LTTODO: Make this stricter!
"required" => %w{name supports controls groups sha256},
"properties" => {
"name" => Primitives::STRING,
"supports" => Primitives.array(Primitives::SUPPORT.ref),
"controls" => Primitives.array(CONTROL.ref),
"groups" => Primitives.array(Primitives::CONTROL_GROUP.ref),
"inputs" => Primitives.array(Primitives::INPUT),
"sha256" => Primitives::STRING,
"status" => Primitives::STRING,
"generator" => Primitives::GENERATOR.ref,
"version" => Primitives::STRING,
# Other properties possible in inspec docs, but that aren"t guaranteed
"title" => Primitives::STRING,
"maintainer" => Primitives::STRING,
"copyright" => Primitives::STRING,
"copyright_email" => Primitives::STRING,
"depends" => Primitives.array(Primitives::DEPENDENCY.ref), # Can have depends, but NOT a parentprofile
},
}, [Primitives::SUPPORT, CONTROL, Primitives::CONTROL_GROUP, Primitives::DEPENDENCY, Primitives::GENERATOR])
end
end
end

4
test/functional/helper.rb
View file

@ -25,6 +25,10 @@ module FunctionalHelper
let(:mock_path) { File.join(repo_path, "test", "fixtures") }
let(:profile_path) { File.join(mock_path, "profiles") }
let(:examples_path) { File.join(profile_path, "old-examples") }
let(:integration_test_path) { File.join(repo_path, "test", "integration", "default") }
let(:all_profiles) { Dir.glob("**/inspec.yml", base: profile_path).map { |file| File.join(profile_path, file) } }
let(:all_profile_folders) { all_profiles.map { |path| File.dirname(path) } }
let(:example_profile) { File.join(examples_path, "profile") }
let(:meta_profile) { File.join(examples_path, "meta-profile") }
let(:example_control) { File.join(example_profile, "controls", "example-tmp.rb") }

8
test/functional/inheritance_test.rb
View file

@ -54,8 +54,8 @@ describe "example inheritance profile" do
_(out.stderr).must_equal ""
s = out.stdout
hm = JSON.load(s)
_(hm["name"]).must_equal "inheritance"
_(hm["controls"].length).must_equal 5
hm["name"].must_equal "inheritance"
hm["controls"].length.must_equal 6
assert_exit_code 0, out
end
@ -65,8 +65,8 @@ describe "example inheritance profile" do
_(out.stderr).must_equal ""
s = out.stdout
hm = JSON.load(s)
_(hm["name"]).must_equal "inheritance"
_(hm["controls"].length).must_equal 5
hm["name"].must_equal "inheritance"
hm["controls"].length.must_equal 6
assert_exit_code 0, out
end

76
test/functional/inspec_exec_json_test.rb
View file

@ -1,6 +1,5 @@
require "functional/helper"
require "json-schema"
require "inspec/schema"
require "json_schemer"
describe "inspec exec with json formatter" do
include FunctionalHelper
@ -11,34 +10,72 @@ describe "inspec exec with json formatter" do
it "can execute a simple file and validate the json schema" do
out = inspec("exec " + example_control + " --reporter json --no-create-lockfile")
data = JSON.parse(out.stdout)
_(JSON::Validator.validate(schema, data)).wont_equal false
sout = inspec("schema exec-json")
schema = JSONSchemer.schema(sout.stdout)
schema.validate(data).to_a.must_equal []
_(out.stderr).must_equal ""
out.stderr.must_equal ""
skip_windows!
assert_exit_code 0, out
end
it "can execute a profile and validate the json schema" do
out = inspec("exec " + example_profile + " --reporter json --no-create-lockfile")
data = JSON.parse(out.stdout)
_(JSON::Validator.validate(schema, data)).wont_equal false
sout = inspec("schema exec-json")
schema = JSONSchemer.schema(sout.stdout)
schema.validate(data).to_a.must_equal []
_(out.stderr).must_equal ""
out.stderr.must_equal ""
assert_exit_code 101, out
end
it "can execute a simple file while using end of options after reporter cli option" do
out = inspec("exec --no-create-lockfile --reporter json -- " + example_control)
out.stderr.must_equal ""
out.exit_status.must_equal 0
data = JSON.parse(out.stdout)
sout = inspec("schema exec-json")
schema = JSONSchemer.schema(sout.stdout)
schema.validate(data).to_a.must_equal []
out.stderr.must_equal ""
skip_windows!
assert_exit_code 0, out
end
it "can execute a profile and validate the json schema with target_id" do
out = inspec("exec " + example_profile + " --reporter json --no-create-lockfile --target-id 1d3e399f-4d71-4863-ac54-84d437fbc444")
data = JSON.parse(out.stdout)
_(data["platform"]["target_id"]).must_equal "1d3e399f-4d71-4863-ac54-84d437fbc444"
_(JSON::Validator.validate(schema, data)).wont_equal false
data["platform"]["target_id"].must_equal "1d3e399f-4d71-4863-ac54-84d437fbc444"
sout = inspec("schema exec-json")
schema = JSONSchemer.schema(sout.stdout)
schema.validate(data).to_a.must_equal []
_(out.stderr).must_equal ""
out.stderr.must_equal ""
assert_exit_code 101, out
end
it "properly validates all (valid) unit tests against the schema" do
schema = JSONSchemer.schema(JSON.parse(inspec("schema exec-json").stdout))
all_profile_folders.each do |folder|
begin
out = inspec("exec " + folder + " --reporter json --no-create-lockfile")
# Ensure it parses properly
out = JSON.parse(out.stdout)
failures = schema.validate(out).to_a
failures.must_equal []
rescue JSON::ParserError
# We don't actually care about these; cannot validate if parsing fails!
nil
end
end
end
it "does not report skipped dependent profiles" do
profile = File.join(profile_path, "unsupported_dependencies", "wrapper-profile")
out = inspec("exec " + profile + " --reporter json --no-create-lockfile")
@ -139,26 +176,23 @@ describe "inspec exec with json formatter" do
"license" => "Apache-2.0",
"summary" => "Demonstrates the use of InSpec Compliance Profile",
"version" => "1.0.0",
# No sense testing this
"sha256" => actual["sha256"],
"supports" => [{ "platform-family" => "unix" }, { "platform-family" => "windows" }],
"status" => "loaded",
"attributes" => [],
})
_(groups.sort_by { |x| x["id"] }).must_equal([
{ "id" => "controls/example-tmp.rb",
"title" => "/ profile",
"controls" => ["tmp-1.0", key] },
{ "id" => "controls/example.rb",
"title" => "Example Config Checks",
"controls" => ["example-1.0"] },
{ "id" => "controls/meta.rb",
"title" => "SSH Server Configuration",
"controls" => ["ssh-1"] },
groups.sort_by { |x| x["id"] }.must_equal([
{ "id" => "controls/example.rb", "title" => "/tmp profile", "controls" => ["tmp-1.0", key] },
{ "id" => "controls/gordon.rb", "title" => "Gordon Config Checks", "controls" => ["gordon-1.0"] },
{ "id" => "controls/meta.rb", "title" => "SSH Server Configuration", "controls" => ["ssh-1"] },
{ "id" => "controls/minimal.rb", "title" => "Minimal control", "controls" => ["minimalist"] },
])
end
it "must have 4 controls" do
_(controls.length).must_equal 4
it "must have 5 controls" do
controls.length.must_equal 5
end
it "has an id for every control" do

48
test/functional/inspec_exec_jsonmin_test.rb
View file

@ -1,6 +1,5 @@
require "functional/helper"
require "json-schema"
require "inspec/schema"
require "json_schemer"
describe "inspec exec" do
include FunctionalHelper
@ -12,16 +11,47 @@ describe "inspec exec" do
it "can execute a profile with the mini json formatter and validate its schema" do
data = JSON.parse(out.stdout)
sout = Inspec::Schema.json("exec-jsonmin")
schema = JSON.parse(sout)
_(JSON::Validator.validate(schema, data)).wont_equal false
sout = inspec("schema exec-jsonmin")
schema = JSONSchemer.schema(sout.stdout)
schema.validate(data).to_a.must_equal []
_(out.stderr).must_equal ""
out.stderr.must_equal ""
assert_exit_code 101, out
end
it "does not contain any duplicate results with describe.one" do
it "can execute a simple file with the mini json formatter and validate its schema" do
out = inspec("exec " + example_control + " --reporter json-min --no-create-lockfile")
out.stderr.must_equal ""
out.exit_status.must_equal 0
data = JSON.parse(out.stdout)
sout = inspec("schema exec-jsonmin")
schema = JSONSchemer.schema(sout.stdout)
schema.validate(data).to_a.must_equal []
out.stderr.must_equal ""
skip_windows!
assert_exit_code 0, out
end
it "properly validates all (valid) unit tests against the schema" do
schema = JSONSchemer.schema(JSON.parse(inspec("schema exec-jsonmin").stdout))
all_profile_folders.each do |folder|
begin
out = inspec("exec " + folder + " --reporter json-min --no-create-lockfile")
# Ensure it parses properly; discard the result
out = JSON.parse(out.stdout)
failures = schema.validate(out).to_a
failures.must_equal []
rescue JSON::ParserError
# We don't actually care about these; cannot validate if parsing fails!
nil
end
end
end
it "does not contain any dupilcate results with describe.one" do
out = inspec("shell -c 'describe.one do describe 1 do it { should cmp 2 } end end' --reporter=json-min")
data = JSON.parse(out.stdout)
_(data["controls"].length).must_equal 1
@ -43,8 +73,8 @@ describe "inspec exec" do
skip if windows?
end
it "must have 5 examples" do
_(json["controls"].length).must_equal 5
it "must have 6 examples" do
json["controls"].length.must_equal 6
end
it "has an id" do

6
test/functional/inspec_exec_junit_test.rb
View file

@ -42,12 +42,12 @@ describe "inspec exec with junit formatter" do
describe "the test suite" do
let(:suite) { doc.elements.to_a("//testsuites/testsuite").first }
it "must have 5 testcase children" do
_(suite.elements.to_a("//testcase").length).must_equal 5
it "must have 6 testcase children" do
suite.elements.to_a("//testcase").length.must_equal 6
end
it "has the tests attribute with 5 total tests" do
_(suite.attribute("tests").value).must_equal "5"
suite.attribute("tests").value.must_equal "6"
end
it "has the failures attribute with 0 total tests" do

13
test/functional/inspec_exec_test.rb
View file

@ -53,9 +53,9 @@ describe "inspec exec" do
_(stdout).must_include "\nProfile Summary: 1 successful control, 0 control failures, 2 controls skipped\n"
_(stdout).must_include "\nTest Summary: 3 successful, 0 failures, 2 skipped\n"
else
_(stdout).must_include " ✔ ssh-1: Allow only SSH Protocol 2\n"
_(stdout).must_include "\nProfile Summary: 2 successful controls, 0 control failures, 1 control skipped\n"
_(stdout).must_include "\nTest Summary: 4 successful, 0 failures, 1 skipped\n"
stdout.must_include "\e[38;5;41m ✔ ssh-1: Allow only SSH Protocol 2\e[0m\n"
stdout.must_include "\nProfile Summary: \e[38;5;41m3 successful controls\e[0m, 0 control failures, \e[38;5;247m1 control skipped\e[0m\n"
stdout.must_include "\nTest Summary: \e[38;5;41m5 successful\e[0m, 0 failures, \e[38;5;247m1 skipped\e[0m\n"
end
_(stderr).must_equal ""
@ -235,16 +235,15 @@ Test Summary: 0 successful, 0 failures, 0 skipped
_(stdout).must_include "Profile Summary: 0 successful controls, 0 control failures, 2 controls skipped\n"
_(stdout).must_include "Test Summary: 2 successful, 1 failure, 3 skipped\n"
else
_(stdout).must_include "Profile Summary: 1 successful control, 0 control failures, 1 control skipped\n"
_(stdout).must_include "Test Summary: 3 successful, 1 failure, 2 skipped\n"
stdout.must_include "Profile Summary: \e[38;5;41m2 successful controls\e[0m, 0 control failures, \e[38;5;247m1 control skipped\e[0m\n"
stdout.must_include "Test Summary: \e[38;5;41m4 successful\e[0m, \e[38;5;9m1 failure\e[0m, \e[38;5;247m2 skipped\e[0m\n"
end
cache_dir = File.join(tmpdir, "cache")
_(Dir.exist?(cache_dir)).must_equal true
_(Dir.glob(File.join(cache_dir, "**", "*"))).must_be_empty
_(stderr).must_equal ""
stderr.must_equal ""
assert_exit_code 100, out
end
end

36
test/functional/inspec_json_profile_test.rb
View file

@ -1,5 +1,6 @@
require "functional/helper"
require "mixlib/shellout"
require "json_schemer"
describe "inspec json" do
include FunctionalHelper
@ -50,7 +51,7 @@ describe "inspec json" do
end
it "has controls" do
_(json["controls"].length).must_equal 4
json["controls"].length.must_equal 5
end
describe "a control" do
@ -106,8 +107,8 @@ describe "inspec json" do
out = inspec("json " + example_profile + " --output " + dst.path)
hm = JSON.load(File.read(dst.path))
_(hm["name"]).must_equal "profile"
_(hm["controls"].length).must_equal 4
hm["name"].must_equal "profile"
hm["controls"].length.must_equal 5
_(out.stderr).must_equal ""
@ -178,4 +179,33 @@ describe "inspec json" do
assert_exit_code 0, out
end
end
it "can format a profile and validate the json schema" do
out = inspec("json " + example_profile)
data = JSON.parse(out.stdout)
sout = inspec("schema profile-json")
schema = JSONSchemer.schema(sout.stdout)
schema.validate(data).to_a.must_equal []
out.stderr.must_equal ""
assert_exit_code 0, out
end
it "properly validates all (valid) unit tests against the schema" do
schema = JSONSchemer.schema(JSON.parse(inspec("schema profile-json").stdout))
all_profile_folders.each do |folder|
begin
out = inspec("json " + folder)
# Ensure it parses properly; discard the result
out = JSON.parse(out.stdout)
failures = schema.validate(out).to_a
failures.must_equal []
rescue JSON::ParserError
# We don't actually care about these; cannot validate if parsing fails!
nil
end
end
end
end

10
test/unit/mock/profiles/old-examples/profile/controls/minimal.rb Normal file
View file

@ -0,0 +1,10 @@
title 'Minimal control'
# you add controls here
control 'minimalist' do # A unique ID for this control
impact 0.7 # The criticality, if this control fails.
describe file('/var') do # The actual test
it { should be_directory }
end
end

72
test/unit/schema/primivites_test.rb Normal file
View file

@ -0,0 +1,72 @@
# Tests that the components of the schema generation framework are behaving properly
require "helper"
require "inspec/schema/primitives"
describe Inspec::Schema::Primitives do
describe "title construction" do
let(:schema_alpha) do
Inspec::Schema::Primitives::SchemaType.new("Alpha", {
"type" => "string",
}, []) end
let(:schema_beta) do
Inspec::Schema::Primitives::SchemaType.new("Beta", {
"type" => "object",
"required" => [],
"properties" => {
"param1" => { "type" => "number" },
},
}, []) end
# Omega nests alpha and beta
let(:schema_omega) do
Inspec::Schema::Primitives::SchemaType.new("Omega", {
"type" => "object",
"required" => ["my_beta"],
"properties" => {
"my_alpha" => schema_alpha.ref,
"my_beta" => schema_beta.ref,
},
}, [schema_alpha, schema_beta]) end
it "should work" do
lambda { raise "oops" }.must_raise "oops"
end
it "should add the title to schema bodies" do
(schema_alpha.body["title"]).must_equal "Alpha"
(schema_beta.body["title"]).must_equal "Beta"
(schema_omega.body["title"]).must_equal "Omega"
end
it "should properly generate ref keys" do
# pass schema
end
end
describe "property validation" do
it "detects if an object does not define required properties" do
lambda {
Inspec::Schema::Primitives.SchemaType.new("Test1", {
"type" => "object",
"properties" => {
"hello" => { "type" => "string" },
},
}, [])
}.must_raise "Objects in schema must have a \"required\" property, even if it is empty"
end
it "detects if a required property is missing" do
lambda {
Inspec::Schema::Primitives.SchemaType.new("Test2", {
"type" => "object",
"required" => %w{alpha beta},
"properties" => {
"alpha" => { "type" => "number" },
"omega" => { "type" => "number" },
},
}, [])
}.must_raise "Property beta is required in schema Test2 but does not exist!"
end
end
end