add inspec json schema validation to functional tests

Signed-off-by: Dominik Richter <dominik.richter@gmail.com>

Gemfile

@@ -19,6 +19,7 @@ group :test do
   gem 'mocha', '~> 1.1'
   gem 'ruby-progressbar', '~> 1.8'
   gem 'webmock', '~> 2.3.2'
+  gem 'jsonschema', '~> 2.0.2'
 end
 
 group :integration do

lib/inspec/cli.rb

@@ -14,6 +14,7 @@ require 'inspec/base_cli'
 require 'inspec/plugins'
 require 'inspec/runner_mock'
 require 'inspec/env_printer'
+require 'inspec/schema'
 
 class Inspec::InspecCLI < Inspec::BaseCLI # rubocop:disable Metrics/ClassLength
   class_option :log_level, aliases: :l, type: :string,
@@ -221,6 +222,14 @@ class Inspec::InspecCLI < Inspec::BaseCLI # rubocop:disable Metrics/ClassLength
     pretty_handle_exception(e)
   end
 
+  desc 'schema NAME', 'print the JSON schema', hide: true
+  def schema(name)
+    puts Inspec::Schema.json(name)
+  rescue StandardError => e
+    puts e
+    puts "Valid schemas are #{Inspec::Schema.names.join(', ')}"
+  end
+
   desc 'version', 'prints the version of this tool'
   def version
     puts Inspec::VERSION

lib/inspec/schema.rb

@@ -0,0 +1,174 @@
+# encoding: utf-8
+require 'json'
+
+module Inspec
+  class Schema # rubocop:disable Metrics/ClassLength
+    STATISTICS = {
+      'type' => 'object',
+      'additionalProperties' => false,
+      'properties' => {
+        'duration' => { 'type' => 'number' },
+      },
+    }.freeze
+
+    # Tags are open right, with simple key-value associations and not restrictions
+    TAGS = { 'type' => 'object' }.freeze
+
+    RESULT = {
+      'type' => 'object',
+      'additionalProperties' => false,
+      'properties' => {
+        'status' => { 'type' => 'string' },
+        'code_desc' => { 'type' => 'string' },
+        'run_time' => { 'type' => 'number' },
+        'start_time' => { 'type' => 'string' },
+        'skip_message' => { 'type' => 'string', 'optional' => true },
+        'resource' => { 'type' => 'string', 'optional' => true },
+      },
+    }.freeze
+
+    REF = {
+      'type' => 'object',
+      'additionalProperties' => false,
+      'properties' => {
+        'ref' => { 'type' => 'string' },
+        # TODO: One of these needs to be deprecated
+        'uri' => { 'type' => 'string', 'optional' => true },
+        'url' => { 'type' => 'string', 'optional' => true },
+      },
+    }.freeze
+    REFS = { 'type' => 'array', 'items' => REF }.freeze
+
+    CONTROL = {
+      'type' => 'object',
+      'additionalProperties' => false,
+      'properties' => {
+        'id' => { 'type' => 'string' },
+        'title' => { 'type' => %w{string null} },
+        'desc' => { 'type' => %w{string null} },
+        'impact' => { 'type' => 'number' },
+        'refs' => REFS,
+        'tags' => TAGS,
+        'code' => { 'type' => 'string' },
+        'source_location' => {
+          'type' => 'object',
+          'properties' => {
+            'ref' => { 'type' => 'string' },
+            'line' => { 'type' => 'number' },
+          },
+        },
+        'results' => { 'type' => 'array', 'items' => RESULT },
+      },
+    }.freeze
+
+    SUPPORTS = {
+      'type' => 'object',
+      'additionalProperties' => false,
+      'properties' => {
+        'os-family' => { 'type' => 'string', 'optional' => true },
+      },
+    }.freeze
+
+    CONTROL_GROUP = {
+      'type' => 'object',
+      'additionalProperties' => false,
+      'properties' => {
+        'id' => { 'type' => 'string' },
+        'title' => { 'type' => 'string', 'optional' => true },
+        'controls' => { 'type' => 'array', 'items' => { 'type' => 'string' } },
+      },
+    }.freeze
+
+    PROFILE = {
+      'type' => 'object',
+      'additionalProperties' => false,
+      'properties' => {
+        'name' => { 'type' => 'string' },
+        'version' => { 'type' => 'string', 'optional' => true },
+
+        'title' => { 'type' => 'string', 'optional' => true },
+        'maintainer' => { 'type' => 'string', 'optional' => true },
+        'copyright' => { 'type' => 'string', 'optional' => true },
+        'copyright_email' => { 'type' => 'string', 'optional' => true },
+        'license' => { 'type' => 'string', 'optional' => true },
+        'summary' => { 'type' => 'string', 'optional' => true },
+
+        'supports' => {
+          'type' => 'array',
+          'items' => SUPPORTS,
+          'optional' => true,
+        },
+        'controls' => {
+          'type' => 'array',
+          'items' => CONTROL,
+        },
+        'groups' => {
+          'type' => 'array',
+          'items' => CONTROL_GROUP,
+        },
+        'attributes' => {
+          'type' => 'array',
+          # TODO: more detailed specification needed
+        },
+      },
+    }.freeze
+
+    EXEC_JSON = {
+      'type' => 'object',
+      'additionalProperties' => false,
+      'properties' => {
+        'profiles' => {
+          'type' => 'array',
+          'items' => PROFILE,
+        },
+        'statistics' => STATISTICS,
+        'version' => { 'type' => 'string' },
+
+        # DEPRECATED PROPERTIES!! These will be removed with the next major version bump
+        'controls' => 'array',
+        'other_checks' => 'array',
+      },
+    }.freeze
+
+    MIN_CONTROL = {
+      'type' => 'object',
+      'additionalProperties' => false,
+      'properties' => {
+        'id' => { 'type' => 'string' },
+        'profile_id' => { 'type' => %w{string null} },
+        'status' => { 'type' => 'string' },
+        'code_desc' => { 'type' => 'string' },
+        'skip_message' => { 'type' => 'string', 'optional' => true },
+        'resource' => { 'type' => 'string', 'optional' => true },
+      },
+    }.freeze
+
+    EXEC_JSONMIN = {
+      'type' => 'object',
+      'additionalProperties' => false,
+      'properties' => {
+        'statistics' => STATISTICS,
+        'version' => { 'type' => 'string' },
+        'controls' => {
+          'type' => 'array',
+          'items' => MIN_CONTROL,
+        },
+      },
+    }.freeze
+
+    LIST = {
+      'exec-json' => EXEC_JSON,
+      'exec-jsonmin' => EXEC_JSONMIN,
+    }.freeze
+
+    def self.names
+      LIST.keys
+    end
+
+    def self.json(name)
+      v = LIST[name] ||
+          raise("Cannot find schema #{name.inspect}.")
+      JSON.dump(v)
+    end
+  end
+end

lib/utils/schema/README.md

@@ -1,11 +0,0 @@
-# inspec schema
-
-JSON schema do verify InSpec's interface.
-
-## development
-
-All development is done on `schema.rb`. It will create all JSON schema files. Run it in this folder via:
-
-```
-ruby schema.rb
-```

lib/utils/schema/inspec.exec.full.json

@@ -1 +0,0 @@
-{"type":"object","properties":{"profiles":{"type":"array","items":{"type":"object","properties":{"title":{"type":"string"},"maintainer":{"type":"string"},"copyright":{"type":"string"},"copyright_email":{"type":"string"},"license":{"type":"string"},"summary":{"type":"string"},"version":{"type":"string"},"supports":{"type":"array","items":{"type":"object","properties":{"os-family":{"type":"string","optional":true}}}},"controls":{"type":"array","items":{"properties":{"id":{"type":"string"},"title":{"type":["string","null"]},"desc":{"type":["string","null"]},"impact":{"type":"number"},"refs":{"type":"array","items":{"type":"object","properties":{"url":{"type":"string","optional":true},"ref":{"type":"string"}}}},"tags":{"type":"object"},"code":{"type":"string"},"source_location":{"type":"object","properties":{"ref":{"type":"string"},"line":{"type":"number"}}},"results":{"type":"array","items":{"type":"object","properties":{"status":{"type":"string"},"code_desc":{"type":"string"},"run_time":{"type":"number"},"start_time":{"type":"string"}}}}}}}}}},"statistics":{"properties":{"duration":{"type":"number"}}},"version":{"type":"string"},"controls":"array","other_checks":"array"},"additionalProperties":false}

lib/utils/schema/inspec.exec.min.json

@@ -1 +0,0 @@
-{"type":"object","properties":{"statistics":{"properties":{"duration":{"type":"number"}}},"version":{"type":"string"},"controls":{"type":"array","items":{"type":"object","properties":{"id":{"type":"string"},"profile_id":{"type":"string"},"status":{"type":"string"},"code_desc":{"type":"string"},"skip_message":{"type":"string","optional":true},"resource":{"type":"string","optional":true}}}}}}

lib/utils/schema/schema.rb

@@ -1,127 +0,0 @@
-#!/usr/bin/env ruby
-require 'json'
-
-def generate(name, schema)
-  File.write(name, JSON.dump(schema))
-  puts "Schema created in #{name}"
-end
-
-statistics = {
-  'properties' => {
-    'duration' => { 'type' => 'number' },
-  },
-}
-
-tags = { 'type' => 'object' }
-
-result = {
-  'type' => 'object',
-  'properties' => {
-    'status' => { 'type' => 'string' },
-    'code_desc' => { 'type' => 'string' },
-    'run_time' => { 'type' => 'number' },
-    'start_time' => { 'type' => 'string' },
-  },
-}
-
-ref = {
-  'type' => 'object',
-  'properties' => {
-    'url' => { 'type' => 'string', 'optional' => true },
-    'ref' => { 'type' => 'string' },
-  },
-}
-refs = { 'type' => 'array', 'items' => ref }
-
-control = {
-  'properties' => {
-    'id' => { 'type' => 'string' },
-    'title' => { 'type' => %w{string null} },
-    'desc' => { 'type' => %w{string null} },
-    'impact' => { 'type' => 'number' },
-    'refs' => refs,
-    'tags' => tags,
-    'code' => { 'type' => 'string' },
-    'source_location' => {
-      'type' => 'object',
-      'properties' => {
-        'ref' => { 'type' => 'string' },
-        'line' => { 'type' => 'number' },
-      },
-    },
-    'results' => { 'type' => 'array', 'items' => result },
-  },
-}
-
-supports = {
-  'type' => 'object',
-  'properties' => {
-    'os-family' => { 'type' => 'string', 'optional' => true },
-  },
-}
-
-profile = {
-  'type' => 'object',
-  'properties' => {
-    'title' => { 'type' => 'string' },
-    'maintainer' => { 'type' => 'string' },
-    'copyright' => { 'type' => 'string' },
-    'copyright_email' => { 'type' => 'string' },
-    'license' => { 'type' => 'string' },
-    'summary' => { 'type' => 'string' },
-    'version' => { 'type' => 'string' },
-    'supports' => {
-      'type' => 'array',
-      'items' => supports,
-    },
-    'controls' => {
-      'type' => 'array',
-      'items' => control,
-    },
-  },
-}
-
-exec_full = {
-  'type' => 'object',
-  'properties' => {
-    'profiles' => {
-      'type' => 'array',
-      'items' => profile,
-    },
-    'statistics' => statistics,
-    'version' => { 'type' => 'string' },
-
-    # DEPRECATED PROPERTIES!! These will be removed with the next major version bump
-    'controls' => 'array',
-    'other_checks' => 'array',
-  },
-  'additionalProperties' => false,
-}
-
-generate('inspec.exec.full.json', exec_full)
-
-min_control = {
-  'type' => 'object',
-  'properties' => {
-    'id' => { 'type' => 'string' },
-    'profile_id' => { 'type' => 'string' },
-    'status' => { 'type' => 'string' },
-    'code_desc' => { 'type' => 'string' },
-    'skip_message' => { 'type' => 'string', 'optional' => true },
-    'resource' => { 'type' => 'string', 'optional' => true },
-  },
-}
-
-exec_min = {
-  'type' => 'object',
-  'properties' => {
-    'statistics' => statistics,
-    'version' => { 'type' => 'string' },
-    'controls' => {
-      'type' => 'array',
-      'items' => min_control,
-    },
-  },
-}
-
-generate('inspec.exec.min.json', exec_min)

test/functional/inspec_exec_json_test.rb

@@ -3,22 +3,29 @@
 # author: Christoph Hartmann
 
 require 'functional/helper'
+require 'jsonschema'
 
 describe 'inspec exec with json formatter' do
   include FunctionalHelper
 
-  it 'can execute a simple file with the json formatter' do
+  it 'can execute a simple file and validate the json schema' do
     out = inspec('exec ' + example_control + ' --format json --no-create-lockfile')
     out.stderr.must_equal ''
     out.exit_status.must_equal 0
-    JSON.load(out.stdout).must_be_kind_of Hash
+    data = JSON.parse(out.stdout)
+    sout = inspec('schema exec-json')
+    schema = JSON.parse(sout.stdout)
+    JSON::Schema.validate(data, schema)
   end
 
-  it 'can execute the profile with the json formatter' do
+  it 'can execute a profile and validate the json schema' do
     out = inspec('exec ' + example_profile + ' --format json --no-create-lockfile')
     out.stderr.must_equal ''
     out.exit_status.must_equal 0
-    JSON.load(out.stdout).must_be_kind_of Hash
+    data = JSON.parse(out.stdout)
+    sout = inspec('schema exec-json')
+    schema = JSON.parse(sout.stdout)
+    JSON::Schema.validate(data, schema)
   end
 
   describe 'execute a profile with json formatting' do

test/functional/inspec_exec_jsonmin_test.rb

@@ -3,22 +3,29 @@
 # author: Christoph Hartmann
 
 require 'functional/helper'
+require 'jsonschema'
 
 describe 'inspec exec' do
   include FunctionalHelper
 
-  it 'can execute the profile with the mini json formatter' do
+  it 'can execute a profile with the mini json formatter and validate its schema' do
     out = inspec('exec ' + example_profile + ' --format json-min --no-create-lockfile')
     out.stderr.must_equal ''
     out.exit_status.must_equal 0
-    JSON.load(out.stdout).must_be_kind_of Hash
+    data = JSON.parse(out.stdout)
+    sout = inspec('schema exec-jsonmin')
+    schema = JSON.parse(sout.stdout)
+    JSON::Schema.validate(data, schema)
   end
 
-  it 'can execute a simple file with the mini json formatter' do
+  it 'can execute a simple file with the mini json formatter and validate its schema' do
     out = inspec('exec ' + example_control + ' --format json-min --no-create-lockfile')
     out.stderr.must_equal ''
     out.exit_status.must_equal 0
-    JSON.load(out.stdout).must_be_kind_of Hash
+    data = JSON.parse(out.stdout)
+    sout = inspec('schema exec-jsonmin')
+    schema = JSON.parse(sout.stdout)
+    JSON::Schema.validate(data, schema)
   end
 
   describe 'execute a profile with mini json formatting' do