Add windows functional tests (#3385)

* Add windows functional tests.
* Fix tests for 2012 server.
* Fix windows build script
* Add more functional tests for windows.
* Update comment with TODO.

Signed-off-by: Jared Quick <jquick@chef.io>

Rakefile

@@ -75,6 +75,24 @@ namespace :test do
     t.ruby_opts = ['--dev'] if defined?(JRUBY_VERSION)
   end
 
+  # Functional tests on Windows are a WIP. Currently only some
+  # functional test files are supported.
+  # TODO: Ensure all functional tests are ran on Windows
+  Rake::TestTask.new(:'functional:windows') do |t|
+    t.libs << 'test'
+    t.test_files = [
+      'test/functional/inspec_exec_test.rb',
+      'test/functional/inspec_exec_json_test.rb',
+      'test/functional/inspec_detect_test.rb',
+      'test/functional/inspec_compliance_test.rb',
+      'test/functional/inspec_check_test.rb',
+      'test/functional/filter_table_test.rb',
+    ]
+    t.warning = true
+    t.verbose = true
+    t.ruby_opts = ['--dev'] if defined?(JRUBY_VERSION)
+  end
+
   task :resources do
     tests = Dir['test/resource/*_test.rb']
     return if tests.empty?

appveyor.yml

@@ -13,7 +13,6 @@ skip_tags: true
 branches:
   only:
     - master
-    - release-2.0
 
 cache:
   - vendor/bundle -> appveyor.yml
@@ -30,7 +29,19 @@ install:
 
 build_script:
   - bundle install --path=vendor/bundle --without integration tools maintenance deploy
+  - if not exist C:\tmp mkdir C:\tmp
 
 test_script:
   - SET SPEC_OPTS=--format progress
   - bundle exec rake
+
+for:
+  -
+    matrix:
+      only:
+        - ruby_version: "25"
+    test_script:
+      - SET SPEC_OPTS=--format progress
+      - bundle exec rake
+      - bundle exec rake test:functional:windows
+

examples/inheritance/inspec.yml

@@ -7,7 +7,8 @@ license: Apache-2.0
 summary: Demonstrates the use of InSpec profile inheritance
 version: 1.0.0
 supports:
-  - os-family: unix
+  - platform-family: unix
+  - platform-family: windows
 depends:
   - name: profile
     path: ../profile

examples/profile/controls/meta.rb

@@ -28,6 +28,8 @@ control 'ssh-1' do
   ref 'DISA-RHEL6-SG - Section 9.2.1', url: 'http://iasecontent.disa.mil/stigs/zip/Jan2016/U_RedHat_6_V1R10_STIG.zip'
   ref 'http://people.redhat.com/swells/scap-security-guide/RHEL/6/output/ssg-centos6-guide-C2S.html'
 
+  only_if { platform.in_family?('unix') }
+
   describe file('/bin/sh') do
     it { should be_owned_by 'root' }
   end

examples/profile/inspec.yml

@@ -7,4 +7,5 @@ license: Apache-2.0
 summary: Demonstrates the use of InSpec Compliance Profile
 version: 1.0.0
 supports:
-  - os-family: unix
+  - platform-family: unix
+  - platform-family: windows

lib/inspec/base_cli.rb

@@ -140,7 +140,7 @@ module Inspec
       if opts['reporter'].is_a?(Array)
         reports = {}
         opts['reporter'].each do |report|
-          reporter_name, target = report.split(':')
+          reporter_name, target = report.split(':', 2)
           if target.nil? || target.strip == '-'
             reports[reporter_name] = { 'stdout' => true }
           else

lib/resources/mysql_session.rb

@@ -72,6 +72,7 @@ module Inspec::Resources
 
     def init_fallback
       # support debian mysql administration login
+      return if inspec.platform.in_family?('windows')
       debian = inspec.command('test -f /etc/mysql/debian.cnf && cat /etc/mysql/debian.cnf').stdout
       return if debian.empty?
 

test/functional/helper.rb

@@ -3,6 +3,7 @@
 # author: Christoph Hartmann
 
 require 'helper'
+require 'rbconfig'
 
 require 'minitest/hell'
 class Minitest::Test
@@ -38,8 +39,32 @@ module FunctionalHelper
     TMP_CACHE[res.path] = res
   }
 
+  def convert_windows_output(text)
+    text = text.force_encoding("UTF-8")
+    text.gsub!("[PASS]", '✔')
+    text.gsub!("\033[0;1;32m", "\033[38;5;41m")
+    text.gsub!("[SKIP]", '↺')
+    text.gsub!("\033[0;37m", "\033[38;5;247m")
+    text.gsub!("[FAIL]", '×')
+    text.gsub!("\033[0;1;31m", "\033[38;5;9m")
+  end
+
+  def is_windows?
+    RbConfig::CONFIG['host_os'] =~ /mswin|mingw|cygwin/
+  end
+
   def inspec(commandline, prefix = nil)
-    CMD.run_command("#{prefix} #{exec_inspec} #{commandline}")
+    if is_windows?
+      result = CMD.run_command("cmd /C \"#{prefix} bundle exec ruby #{exec_inspec} #{commandline}\"")
+      result.stdout.encode!(universal_newline: true)
+      result.stderr.encode!(universal_newline: true)
+      convert_windows_output(result.stdout)
+      # remove the CLIXML header trash in windows
+      result.stderr.gsub!("#< CLIXML\n", '')
+      result
+    else
+      CMD.run_command("#{prefix} #{exec_inspec} #{commandline}")
+    end
   end
 
   def inspec_with_env(commandline, env = {})
@@ -47,8 +72,14 @@ module FunctionalHelper
     # single param for the command line.
     # TODO: what is the intent of using Train here?
     # HACK: glue together env vars
-    env_prefix = env.to_a.map { |assignment| "#{assignment[0]}=#{assignment[1]}" }.join(' ')
-    CMD.run_command("#{env_prefix} #{exec_inspec} #{commandline}")
+    if is_windows?
+      env_prefix = env.to_a.map { |assignment| "set #{assignment[0]}=#{assignment[1]}" }.join('&& ')
+      env_prefix += '&& '
+    else
+      env_prefix = env.to_a.map { |assignment| "#{assignment[0]}=#{assignment[1]}" }.join(' ')
+      env_prefix += ' '
+    end
+    inspec(commandline, env_prefix)
   end
 
   # Copy all examples to a temporary directory for functional tests.

test/functional/inspec_exec_json_test.rb

@@ -69,8 +69,8 @@ describe 'inspec exec with json formatter' do
         "license" => "Apache-2.0",
         "summary" => "Demonstrates the use of InSpec Compliance Profile",
         "version" => "1.0.0",
-        "sha256" => "57709d3a3d5cd06f4179be7e6fbe254c09e3af25ce274e474d52623e34487cc4",
-        "supports" => [{"platform-family" => "unix"}],
+        "sha256" => "e694e87e4a6cc989621a3da1efc285dc41aba95beeece8a1274a1ded0f4a6f32",
+        "supports" => [{"platform-family" => "unix"}, {"platform-family"=>"windows"}],
         "attributes" => []
       })
 

test/functional/inspec_exec_test.rb

@@ -12,14 +12,20 @@ describe 'inspec exec' do
     out.stderr.must_equal ''
     out.exit_status.must_equal 101
     stdout = out.stdout.force_encoding(Encoding::UTF_8)
-    stdout.must_include "\e[38;5;41m  ✔  ssh-1: Allow only SSH Protocol 2\e[0m\n"
     stdout.must_include "\e[38;5;41m  ✔  tmp-1.0: Create /tmp directory\e[0m\n"
     stdout.must_include "
 \e[38;5;247m  ↺  gordon-1.0: Verify the version number of Gordon (1 skipped)\e[0m
 \e[38;5;247m     ↺  Can't find file `/tmp/gordon/config.yaml`\e[0m
 "
-    stdout.must_include "\nProfile Summary: \e[38;5;41m2 successful controls\e[0m, 0 control failures, \e[38;5;247m1 control skipped\e[0m\n"
-    stdout.must_include "\nTest Summary: \e[38;5;41m4 successful\e[0m, 0 failures, \e[38;5;247m1 skipped\e[0m\n"
+    if is_windows?
+      stdout.must_include "\e[38;5;247m  ↺  ssh-1: Allow only SSH Protocol 2\e[0m\n"
+      stdout.must_include "\nProfile Summary: \e[38;5;41m1 successful control\e[0m, 0 control failures, \e[38;5;247m2 controls skipped\e[0m\n"
+      stdout.must_include "\nTest Summary: \e[38;5;41m3 successful\e[0m, 0 failures, \e[38;5;247m2 skipped\e[0m\n"
+    else
+      stdout.must_include "\e[38;5;41m  ✔  ssh-1: Allow only SSH Protocol 2\e[0m\n"
+      stdout.must_include "\nProfile Summary: \e[38;5;41m2 successful controls\e[0m, 0 control failures, \e[38;5;247m1 control skipped\e[0m\n"
+      stdout.must_include "\nTest Summary: \e[38;5;41m4 successful\e[0m, 0 failures, \e[38;5;247m1 skipped\e[0m\n"
+    end
   end
 
   it 'executes a minimum metadata-only profile' do
@@ -110,9 +116,9 @@ Test Summary: 0 successful, 0 failures, 0 skipped
   end
 
   it 'executes only specified controls when selecting failing controls by regex' do
-    out = inspec('exec ' + File.join(profile_path, 'filter_table') + ' --no-create-lockfile --controls \'/^(2943|2370)_fail/\'')
+    out = inspec('exec ' + File.join(profile_path, 'filter_table') + ' --no-create-lockfile --controls \'/^2943_fail/\'')
     out.exit_status.must_equal 100
-    out.stdout.force_encoding(Encoding::UTF_8).must_include "Profile Summary: 0 successful controls, \e[38;5;9m2 control failures\e[0m, 0 controls skipped"
+    out.stdout.force_encoding(Encoding::UTF_8).must_include "Profile Summary: 0 successful controls, \e[38;5;9m1 control failure\e[0m, 0 controls skipped"
   end
 
 
@@ -138,8 +144,13 @@ Test Summary: 0 successful, 0 failures, 0 skipped
       out = inspec_with_env(command, INSPEC_CONFIG_DIR: tmpdir)
       out.stderr.must_equal ''
       out.exit_status.must_equal 100
-      out.stdout.must_include "Profile Summary: \e[38;5;41m1 successful control\e[0m, 0 control failures, \e[38;5;247m1 control skipped\e[0m\n"
-      out.stdout.must_include "Test Summary: \e[38;5;41m3 successful\e[0m, \e[38;5;9m1 failure\e[0m, \e[38;5;247m2 skipped\e[0m\n"
+      if is_windows?
+        out.stdout.must_include "Profile Summary: 0 successful controls, 0 control failures, \e[38;5;247m2 controls skipped\e[0m\n"
+        out.stdout.must_include "Test Summary: \e[38;5;41m2 successful\e[0m, \e[38;5;9m1 failure\e[0m, \e[38;5;247m3 skipped\e[0m\n"
+      else
+        out.stdout.must_include "Profile Summary: \e[38;5;41m1 successful control\e[0m, 0 control failures, \e[38;5;247m1 control skipped\e[0m\n"
+        out.stdout.must_include "Test Summary: \e[38;5;41m3 successful\e[0m, \e[38;5;9m1 failure\e[0m, \e[38;5;247m2 skipped\e[0m\n"
+      end
       cache_dir = File.join(tmpdir, 'cache')
       Dir.exist?(cache_dir).must_equal true
       Dir.glob(File.join(cache_dir, '**', '*')).must_be_empty
@@ -265,8 +276,11 @@ Test Summary: \e[38;5;41m2 successful\e[0m, 0 failures, 0 skipped\n"
       out.stdout.force_encoding(Encoding::UTF_8).must_include "×  7 should cmp >= 9\n"
       out.stdout.force_encoding(Encoding::UTF_8).must_include "×  7 should not cmp == /^\\d$/\n"
       out.stdout.force_encoding(Encoding::UTF_8).must_include "✔  7 should cmp == \"7\""
-      out.stdout.force_encoding(Encoding::UTF_8).must_include "  expected: \"01147\"
-          got: \"01777\"\n"
+      if is_windows?
+        out.stdout.force_encoding(Encoding::UTF_8).must_include "  expected: \"01147\"\n          got: \"040755\"\n"
+      else
+        out.stdout.force_encoding(Encoding::UTF_8).must_include "  expected: \"01147\"\n          got: \"01777\"\n"
+      end
     end
   end
 
@@ -275,7 +289,7 @@ Test Summary: \e[38;5;41m2 successful\e[0m, 0 failures, 0 skipped\n"
 
     it 'should print all the results' do
       out.stdout.force_encoding(Encoding::UTF_8).must_include "×  tmp-1.0: Create /tmp directory (1 failed)\e[0m"
-      out.stdout.force_encoding(Encoding::UTF_8).must_include "×  should not be directory"
+      out.stdout.force_encoding(Encoding::UTF_8).must_include "×  cmp-1.0: Using the cmp matcher for numbers (2 failed)"
       out.stdout.force_encoding(Encoding::UTF_8).must_include "×  undefined method `should_nota'"
       out.stdout.force_encoding(Encoding::UTF_8).must_include "×  should not be directory\n     expected `File /tmp.directory?` to return false, got true\e[0m"
       out.stdout.force_encoding(Encoding::UTF_8).must_include "✔  profiled-1: Create /tmp directory (profile d)"
@@ -320,12 +334,20 @@ Test Summary: \e[38;5;41m2 successful\e[0m, 0 failures, 0 skipped\n"
   describe 'when using profiles on the supermarket' do
     it 'can run supermarket profiles directly from the command line' do
       out = inspec("exec supermarket://nathenharvey/tmp-compliance-profile --no-create-lockfile")
-      out.stdout.force_encoding(Encoding::UTF_8).must_include "Profile Summary: \e[38;5;41m2 successful controls\e[0m, 0 control failures, 0 controls skipped\n"
+      if is_windows?
+        out.stdout.force_encoding(Encoding::UTF_8).must_include "Profile Summary: \e[38;5;41m1 successful control\e[0m, \e[38;5;9m1 control failure\e[0m, 0 controls skipped\n"
+      else
+        out.stdout.force_encoding(Encoding::UTF_8).must_include "Profile Summary: \e[38;5;41m2 successful controls\e[0m, 0 control failures, 0 controls skipped\n"
+      end
     end
 
     it 'can run supermarket profiles from inspec.yml' do
       out = inspec("exec #{File.join(profile_path, 'supermarket-dep')} --no-create-lockfile")
-      out.stdout.force_encoding(Encoding::UTF_8).must_include "Profile Summary: \e[38;5;41m2 successful controls\e[0m, 0 control failures, 0 controls skipped\n"
+      if is_windows?
+        out.stdout.force_encoding(Encoding::UTF_8).must_include "Profile Summary: \e[38;5;41m1 successful control\e[0m, \e[38;5;9m1 control failure\e[0m, 0 controls skipped\n"
+      else
+        out.stdout.force_encoding(Encoding::UTF_8).must_include "Profile Summary: \e[38;5;41m2 successful controls\e[0m, 0 control failures, 0 controls skipped\n"
+      end
     end
   end
 
@@ -414,9 +436,13 @@ Test Summary: \e[38;5;41m2 successful\e[0m, 0 failures, 0 skipped\n"
     let(:child_control) { child_profile['controls'].select { |c| c['title'] == 'Profile 1 - Control 2-updated' }.first }
     let(:override) { controls.select { |c| c['title'] == 'Profile 1 - Control 2-updated' }.first }
 
-    it 'completes the run with failed controls but no exception' do
+    it 'completes the run with parent control overrides' do
       out.stderr.must_be_empty
-      out.exit_status.must_equal 0
+      if is_windows?
+        out.exit_status.must_equal 100
+      else
+        out.exit_status.must_equal 0
+      end
       controls.count.must_equal 2
 
       # check for json override

test/unit/mock/profiles/complete-metadata/inspec.yml

@@ -6,5 +6,6 @@ copyright: left
 summary: nothing
 license: Apache-2.0
 supports:
-  - linux
-  - darwin
+  - platform-family: linux
+  - platform-family: darwin
+  - platform-family: windows

test/unit/mock/profiles/library/inspec.yml

@@ -7,5 +7,6 @@ license: Apache-2.0
 summary: Testing stub
 version: 1.0.0
 supports:
-  - linux
-  - darwin
+  - platform_family: linux
+  - platform_family: darwin
+  - platform_family: windows

test/unit/mock/profiles/profile-support-skip/controls/example.rb

@@ -1,4 +1,4 @@
 # encoding: utf-8
 # copyright: 2015, The Authors
 
-include_controls 'windows-only'
+include_controls 'unsupported_inspec'

test/unit/mock/profiles/profile-support-skip/inspec.yml

@@ -7,5 +7,5 @@ license: Apache-2.0
 summary: An InSpec Compliance Profile
 version: 0.1.0
 depends:
-  - name: windows-only
-    path: '../windows-only'
+  - name: unsupported_inspec
+    path: '../unsupported_inspec'

test/unit/mock/profiles/unsupported_inspec/controls/unsupported.rb

@@ -0,0 +1,11 @@
+control 'unsupported control' do
+  describe 'this should be skipped' do
+    it { should eq 'false' }
+  end
+end
+
+control 'unsupported control 2' do
+  describe 'this should also be skipped' do
+    it { should eq 'false' }
+  end
+end