Create appvayor workers for Windows functional tests. (#3397)

* Add more windows functional tests and create some appvayor workers.
* Fix uuid test for automate

Signed-off-by: Jared Quick <jquick@chef.io>

Rakefile

@@ -81,19 +81,18 @@ namespace :test do
     t.ruby_opts = ['--dev'] if defined?(JRUBY_VERSION)
   end
 
-  # Functional tests on Windows are a WIP. Currently only some
+  # Functional tests on Windows take a bit to run. This
-  # functional test files are supported.
+  # optionally takes a env to breake the tests up into 3 workers.
-  # TODO: Ensure all functional tests are ran on Windows
+  Rake::TestTask.new(:'functional:windows') do |t, args|
-  Rake::TestTask.new(:'functional:windows') do |t|
+    files = Dir.glob('test/functional/*_test.rb').sort
+    if ENV['WORKER_NUMBER']
+      count = (files.count / 3).abs+1
+      start = (ENV['WORKER_NUMBER'].to_i - 1) * count
+      files = files[start..start+count-1]
+    end
+
     t.libs << 'test'
-    t.test_files = [
+    t.test_files = files
-      'test/functional/inspec_exec_test.rb',
-      'test/functional/inspec_exec_json_test.rb',
-      'test/functional/inspec_detect_test.rb',
-      'test/functional/inspec_vendor_test.rb',
-      'test/functional/inspec_check_test.rb',
-      'test/functional/filter_table_test.rb',
-    ]
     t.warning = true
     t.verbose = true
     t.ruby_opts = ['--dev'] if defined?(JRUBY_VERSION)

appveyor.yml

@@ -4,9 +4,18 @@ platform:
 
 environment:
   matrix:
-    - ruby_version: "23"
+    - name: unit-tests-ruby-2.3.3
-    - ruby_version: "24"
+      ruby_version: "23"
-    - ruby_version: "25"
+    - name: unit-tests-ruby-2.4.4
+      ruby_version: "24"
+    - name: unit-tests-ruby-2.5.1
+      ruby_version: "25"
+    - name: functional-tests-1
+      ruby_version: "25"
+    - name: functional-tests-2
+      ruby_version: "25"
+    - name: functional-tests-3
+      ruby_version: "25"
 clone_folder: c:\projects\inspec
 clone_depth: 1
 
@@ -54,9 +63,24 @@ for:
   -
     matrix:
       only:
-        - ruby_version: "25"
+        - name: functional-tests-1
     test_script:
       - SET SPEC_OPTS=--format progress
-      - bundle exec rake
+      - SET WORKER_NUMBER=1
+      - bundle exec rake test:functional:windows
+  -
+    matrix:
+      only:
+        - name: functional-tests-2
+    test_script:
+      - SET SPEC_OPTS=--format progress
+      - SET WORKER_NUMBER=2
+      - bundle exec rake test:functional:windows
+  -
+    matrix:
+      only:
+        - name: functional-tests-3
+    test_script:
+      - SET SPEC_OPTS=--format progress
+      - SET WORKER_NUMBER=3
       - bundle exec rake test:functional:windows
-

lib/inspec/runner.rb

@@ -238,9 +238,11 @@ module Inspec
 
       # Load local profile dependencies. This is used in inspec shell
       # to provide access to local profiles that add resources.
-      @depends
+      @depends.each do |dep|
-        .map { |x| Inspec::Profile.for_path(x, { profile_context: ctx }) }
+        # support for windows paths
-        .each(&:load_libraries)
+        dep = dep.tr('\\', '/')
+        Inspec::Profile.for_path(dep, { profile_context: ctx }).load_libraries
+      end
 
       ctx.load(command)
     end

test/functional/gitfetcher_test.rb

@@ -18,8 +18,10 @@ describe 'profiles with git-based dependencies' do
     Dir.chdir(@git_dep_dir) do
       CMD.run_command("git init")
       CMD.run_command("git add .")
-      CMD.run_command("git commit -m 'initial commit' --no-gpg-sign")
+      CMD.run_command("git config user.name \"test\"")
-      CMD.run_command("git commit -m 'another commit' --allow-empty --no-gpg-sign")
+      CMD.run_command("git config user.email \"test@yahoo.com\"")
+      CMD.run_command("git commit -m \"initial commit\" --no-gpg-sign")
+      CMD.run_command("git commit -m \"another commit\" --allow-empty --no-gpg-sign")
       CMD.run_command("git tag antag")
     end
 

test/functional/helper.rb

@@ -15,7 +15,12 @@ class Module
 end
 
 module FunctionalHelper
-  let(:repo_path) { File.expand_path(File.join( __FILE__, '..', '..', '..')) }
+  let(:repo_path) do
+    path = File.expand_path(File.join( __FILE__, '..', '..', '..'))
+    # fix for vagrant repo pathing
+    path.gsub!('//vboxsrv', 'C:') if is_windows?
+    path
+  end
   let(:exec_inspec) { File.join(repo_path, 'bin', 'inspec') }
   let(:mock_path) { File.join(repo_path, 'test', 'unit', 'mock') }
   let(:profile_path) { File.join(mock_path, 'profiles') }
@@ -50,10 +55,14 @@ module FunctionalHelper
     text.gsub!("\033[0;1;31m", "\033[38;5;9m")
   end
 
-  def is_windows?
+  def self.is_windows?
     RbConfig::CONFIG['host_os'] =~ /mswin|mingw|cygwin/
   end
 
+  def is_windows?
+    FunctionalHelper.is_windows?
+  end
+
   def inspec(commandline, prefix = nil)
     if is_windows?
       result = CMD.run_command("cmd /C \"#{prefix} bundle exec ruby #{exec_inspec} #{commandline}\"")

test/functional/inspec_archive_test.rb

@@ -7,6 +7,7 @@ require 'tmpdir'
 
 describe 'inspec archive' do
   include FunctionalHelper
+  let(:auto_dst) { File.expand_path(File.join(repo_path, 'profile-1.0.0.tar.gz')) }
 
   it 'archive is successful' do
     prepare_examples('profile') do |dir|
@@ -21,7 +22,7 @@ describe 'inspec archive' do
     prepare_examples('profile') do |dir|
       out = inspec('archive ' + dir + ' --output ' + dst.path)
       out.stderr.must_equal ''
-      out.stdout.must_include 'Generate archive '+ dst.path
+      out.stdout.must_include 'Generate archive ' + dst.path
       out.stdout.must_include 'Finished archive generation.'
       out.exit_status.must_equal 0
       File.exist?(dst.path).must_equal true
@@ -30,7 +31,6 @@ describe 'inspec archive' do
 
   it 'auto-archives when no --output is given' do
     prepare_examples('profile') do |dir|
-      auto_dst = File.join(repo_path, 'profile-1.0.0.tar.gz')
       out = inspec('archive ' + dir + ' --overwrite')
       out.stderr.must_equal ''
       out.stdout.must_include 'Generate archive ' + auto_dst
@@ -42,7 +42,7 @@ describe 'inspec archive' do
 
   it 'archive on invalid archive' do
     Dir.tmpdir do |target_dir|
-      out = inspec('archive #{target_dir} --output ' + dst.path)
+      out = inspec("archive #{target_dir} --output " + dst.path)
       out.stderr.must_include "Don't understand inspec profile in \"#{target_dir}\""
       out.exit_status.must_equal 1
       File.exist?(dst.path).must_equal false

test/functional/inspec_artifact_test.rb

@@ -0,0 +1,47 @@
+# encoding: utf-8
+
+require 'fileutils'
+require 'functional/helper'
+require 'securerandom'
+
+describe 'inspec exec' do
+  include FunctionalHelper
+
+  it 'can generate keys' do
+    prepare_examples do |dir|
+      unique_key_name = SecureRandom.uuid()
+      out = inspec("artifact generate --keyname #{unique_key_name}", "cd #{dir} && ")
+      out.exit_status.must_equal 0
+
+      stdout = out.stdout.force_encoding(Encoding::UTF_8)
+      stdout.must_include 'Generating private key'
+      stdout.must_include 'Generating public key'
+    end
+  end
+
+  it 'can sign, verify and install a signed profile' do
+    # The arcive install commands do not currently support windows and
+    # use specific linux extract tar commands. Since artifact is still
+    # experimental we are skipping it for now.
+    return if is_windows?
+    prepare_examples do |dir|
+      unique_key_name = SecureRandom.uuid()
+      install_dir = File.join(dir, SecureRandom.uuid())
+      profile = File.join(dir, 'profile')
+      FileUtils.mkdir(install_dir)
+
+      out = inspec("artifact generate --keyname #{unique_key_name}", "cd #{dir} &&")
+      out.exit_status.must_equal 0
+
+      out = inspec("artifact sign-profile --profile #{profile} --keyname #{unique_key_name}", "cd #{dir} &&")
+      out.exit_status.must_equal 0
+
+      out = inspec("artifact install-profile --infile profile-1.0.0.iaf --destdir #{install_dir}", "cd #{dir} &&")
+      out.exit_status.must_equal 0
+
+      out.stdout.force_encoding(Encoding::UTF_8).must_include "Installing to #{install_dir}"
+      Dir.entries(install_dir).join.must_include "inspec.yml"
+    end
+  end
+
+end

test/functional/inspec_exec_automate_test.rb

@@ -15,7 +15,8 @@ describe 'inspec exec automate' do
             "stdout" : false,
             "url" : "https://fake_url_a2.com/data-collector/v0/",
             "token" : "faketoken123",
-            "insecure" : true
+            "insecure" : true,
+            "node_uuid" : "test123"
             }
         }
     }

test/functional/inspec_exec_junit_test.rb

@@ -59,7 +59,11 @@ describe 'inspec exec with junit formatter' do
         end
 
         it 'should be skipped' do
-          first_gordon_test.elements.to_a('//skipped').length.must_equal 1
+          if is_windows?
+            first_gordon_test.elements.to_a('//skipped').length.must_equal 2
+          else
+            first_gordon_test.elements.to_a('//skipped').length.must_equal 1
+          end
         end
       end
     end

test/functional/inspec_json_profile_test.rb

@@ -115,16 +115,16 @@ describe 'inspec json' do
   describe 'json test for pax header archives' do
     let(:profile_tgz) { File.join(Dir.mktmpdir, "pax-profile-test.tar.gz") }
 
-    before do
+    it 'successfully reads a pax-formatted tar file' do
+      # TODO: this needs updated to also support windows taring
+      return if is_windows?
       files = Dir.glob("#{example_profile}/**/*").delete_if { |x| !File.file?(x) }
       relatives = files.map { |e| Pathname.new(e).relative_path_from(Pathname.new(example_profile)).to_s }
 
       cmd = Mixlib::ShellOut.new("tar --format=pax -czf #{profile_tgz} #{relatives.join(' ')}", cwd: example_profile)
       cmd.run_command
       cmd.error!
-    end
 
-    it 'successfully reads a pax-formatted tar file' do
       out = inspec("json #{profile_tgz}")
       out.exit_status.must_equal 0
     end

test/functional/inspec_shell_test.rb

@@ -8,7 +8,8 @@ describe 'inspec shell tests' do
   describe 'cmd' do
     def do_shell_c(code, exit_status, json = false, stderr = '')
       json_suffix = " --reporter 'json'" if json
-      out = inspec("shell -c '#{code.tr('\'','\\\'')}'#{json_suffix}")
+      command = "shell -c '#{code.tr('\'','\\\'')}'#{json_suffix}"
+      out = inspec(command)
       out.stderr.must_equal stderr
       out.exit_status.must_equal exit_status
       out
@@ -47,12 +48,16 @@ describe 'inspec shell tests' do
     end
 
     it 'can run arbitrary ruby (json output)' do
+      # You cannot have a pipe in a windows command line
+      return if is_windows?
       out = do_shell_c('x = [1,2,3].inject(0) {|a,v| a + v*v}; x+10', 0, true)
       j = JSON.load(out.stdout)
       j.must_equal 24  # 1^2 + 2^2 + 3^2 + 10
     end
 
     it 'can run arbitrary ruby' do
+      # You cannot have a pipe in a windows command line
+      return if is_windows?
       out = do_shell_c('x = [1,2,3].inject(0) {|a,v| a + v*v}; x+10', 0)
       out.stdout.must_equal "24\n"
     end
@@ -131,108 +136,112 @@ describe 'inspec shell tests' do
     end
   end
 
-  describe 'shell' do
+  # Pry does not support STDIN from windows currently. Skipping these for now.
-    def do_shell(code, exit_status = 0, stderr = '')
+  unless FunctionalHelper.is_windows?
-      cmd = "echo '#{code.tr('\'','\\\'')}' | #{exec_inspec} shell"
-      out = CMD.run_command(cmd)
-      #out.stderr.must_equal stderr
-      out.exit_status.must_equal exit_status
-      out
-    end
 
-    it 'loads a dependency' do
+    describe 'shell' do
-      cmd = "echo 'gordon_config' | #{exec_inspec} shell --depends #{example_profile}"
+      def do_shell(code, exit_status = 0, stderr = '')
-      res = CMD.run_command(cmd)
+        cmd = "echo '#{code.tr('\'','\\\'')}' | #{exec_inspec} shell"
-      res.exit_status.must_equal 0
+        out = CMD.run_command(cmd)
-      res.stdout.must_include "=> gordon_config"
+        #out.stderr.must_equal stderr
-    end
+        out.exit_status.must_equal exit_status
+        out
+      end
 
-    it 'displays the target device information for the user without requiring the help command' do
+      it 'loads a dependency' do
-      out = do_shell('1+1')
+        cmd = "echo 'gordon_config' | #{exec_inspec} shell --depends #{example_profile}"
-      out.stdout.must_include 'You are currently running on:'
+        res = CMD.run_command(cmd)
-    end
+        res.exit_status.must_equal 0
+        res.stdout.must_include "=> gordon_config"
+      end
 
-    it 'provides a help command' do
+      it 'displays the target device information for the user without requiring the help command' do
-      out = do_shell('help')
+        out = do_shell('1+1')
-      out.stdout.must_include 'Available commands:'
+        out.stdout.must_include 'You are currently running on:'
-      out.stdout.must_include 'You are currently running on:'
+      end
-    end
 
-    it 'provides resource help' do
+      it 'provides a help command' do
-      out = do_shell('help file')
+        out = do_shell('help')
-      out.stdout.must_include 'Use the file InSpec audit resource'
+        out.stdout.must_include 'Available commands:'
-    end
+        out.stdout.must_include 'You are currently running on:'
+      end
 
-    it 'provides helpful feedback if an invalid resource is provided' do
+      it 'provides resource help' do
-      out = do_shell('help not_a_valid_resource')
+        out = do_shell('help file')
-      out.stdout.must_include 'The resource not_a_valid_resource does not exist.'
+        out.stdout.must_include 'Use the file InSpec audit resource'
-    end
+      end
 
-    it 'provides a list of resources' do
+      it 'provides helpful feedback if an invalid resource is provided' do
-      out = do_shell('help resources')
+        out = do_shell('help not_a_valid_resource')
-      out.stdout.must_include ' - command'
+        out.stdout.must_include 'The resource not_a_valid_resource does not exist.'
-      out.stdout.must_include ' - file'
+      end
-      out.stdout.must_include ' - sshd_config'
-    end
 
-    it 'provides matchers help' do
+      it 'provides a list of resources' do
-      out = do_shell('help matchers')
+        out = do_shell('help resources')
-      out.stdout.must_include 'For more examples, see: https://www.inspec.io/docs/reference/matchers/'
+        out.stdout.must_include ' - command'
-    end
+        out.stdout.must_include ' - file'
+        out.stdout.must_include ' - sshd_config'
+      end
 
-    it 'provides empty example help' do
+      it 'provides matchers help' do
-      out = do_shell('help file')
+        out = do_shell('help matchers')
-      out.stdout.must_include 'Name'
+        out.stdout.must_include 'For more examples, see: https://www.inspec.io/docs/reference/matchers/'
-      out.stdout.must_include 'Description'
+      end
-      out.stdout.must_include 'Example'
-      out.stdout.must_include 'Web Reference'
-    end
 
-    it 'exposes all resources' do
+      it 'provides empty example help' do
-      out = do_shell('os')
+        out = do_shell('help file')
-      out.stdout.must_match(/\=> .*Operating.* .*System.* .*Detection/)
+        out.stdout.must_include 'Name'
-    end
+        out.stdout.must_include 'Description'
+        out.stdout.must_include 'Example'
+        out.stdout.must_include 'Web Reference'
+      end
 
-    it 'can run ruby expressions' do
+      it 'exposes all resources' do
-      x = rand
+        out = do_shell('os')
-      y = rand
+        out.stdout.must_match(/\=> .*Operating.* .*System.* .*Detection/)
-      out = do_shell("#{x} + #{y}")
+      end
-      out.stdout.must_include "#{x+y}"
-    end
 
-    it 'can run arbitrary ruby' do
+      it 'can run ruby expressions' do
-      out = do_shell('x = [1,2,3].inject(0) {|a,v| a + v*v}; x+10')
+        x = rand
-      out.stdout.must_include "24"
+        y = rand
-    end
+        out = do_shell("#{x} + #{y}")
+        out.stdout.must_include "#{x+y}"
+      end
 
-    it 'runs anonymous tests that succeed' do
+      it 'can run arbitrary ruby' do
-      out = do_shell("describe file(\"#{__FILE__}\") do it { should exist } end")
+        out = do_shell('x = [1,2,3].inject(0) {|a,v| a + v*v}; x+10')
-      out.stdout.must_include '1 successful'
+        out.stdout.must_include "24"
-      out.stdout.must_include '0 failures'
+      end
-    end
 
-    it 'runs anonymous tests that fail' do
+      it 'runs anonymous tests that succeed' do
-      out = do_shell("describe file(\"foo/bar/baz\") do it { should exist } end")
+        out = do_shell("describe file(\"#{__FILE__}\") do it { should exist } end")
-      out.stdout.must_include '0 successful'
+        out.stdout.must_include '1 successful'
-      out.stdout.must_include '1 failure'
+        out.stdout.must_include '0 failures'
-    end
+      end
 
-    it 'runs controls with tests' do
+      it 'runs anonymous tests that fail' do
-      out = do_shell("control \"test\" do describe file(\"#{__FILE__}\") do it { should exist } end end")
+        out = do_shell("describe file(\"foo/bar/baz\") do it { should exist } end")
-      out.stdout.must_include '1 successful'
+        out.stdout.must_include '0 successful'
-      out.stdout.must_include '0 failures'
+        out.stdout.must_include '1 failure'
-    end
+      end
 
-    it 'runs controls with multiple tests' do
+      it 'runs controls with tests' do
-      out = do_shell("control \"test\" do describe file(\"#{__FILE__}\") do it { should exist } end; describe file(\"foo/bar/baz\") do it { should exist } end end")
+        out = do_shell("control \"test\" do describe file(\"#{__FILE__}\") do it { should exist } end end")
-      out.stdout.must_include '0 successful'
+        out.stdout.must_include '1 successful'
-      out.stdout.must_include '1 failure'
+        out.stdout.must_include '0 failures'
-    end
+      end
 
-    it 'reruns controls when redefined' do
+      it 'runs controls with multiple tests' do
-      out = do_shell("control \"test\" do describe file(\"#{__FILE__}\") do it { should exist } end end\ncontrol \"test\" do describe file(\"foo/bar/baz\") do it { should exist } end end")
+        out = do_shell("control \"test\" do describe file(\"#{__FILE__}\") do it { should exist } end; describe file(\"foo/bar/baz\") do it { should exist } end end")
-      out.stdout.must_include '1 successful'
+        out.stdout.must_include '0 successful'
-      out.stdout.must_include '1 failure'
+        out.stdout.must_include '1 failure'
+      end
+
+      it 'reruns controls when redefined' do
+        out = do_shell("control \"test\" do describe file(\"#{__FILE__}\") do it { should exist } end end\ncontrol \"test\" do describe file(\"foo/bar/baz\") do it { should exist } end end")
+        out.stdout.must_include '1 successful'
+        out.stdout.must_include '1 failure'
+      end
     end
   end
 end

test/functional/inspec_supermakert_test.rb

@@ -19,7 +19,11 @@ describe 'inspec supermakert' do
   end
 
   it 'supermarket exec' do
-    out = inspec('supermarket exec dev-sec/ssh-baseline')
+    if is_windows?
+      out = inspec('supermarket exec dev-sec/windows-patch-baseline')
+    else
+      out = inspec('supermarket exec dev-sec/ssh-baseline')
+    end
     out.exit_status.wont_equal 1
     out.stderr.must_equal ''
     out.stdout.must_include 'Profile Summary'