run integration tests in docker

.kitchen.ec2.yml

@@ -12,6 +12,13 @@ transport:
   username: ec2-user
   ssh_key: <%= ENV['EC2_SSH_KEY_PATH'] %>
 
+provisioner:
+  name: chef_solo
+
+verifier:
+  name: inspec
+  sudo: true
+
 platforms:
   - name: windows-2008
     driver:
@@ -73,3 +80,9 @@ platforms:
   - name: aws-linux
     driver:
       image_id: ami-daaeaec7
+
+suites:
+  - name: default
+    run_list:
+      - recipe[os_prepare]
+    attributes:

.kitchen.vagrant.yml

@@ -0,0 +1,45 @@
+---
+driver:
+  name: vagrant
+
+provisioner:
+  name: chef_solo
+
+verifier:
+  name: inspec
+  sudo: true
+
+platforms:
+  - name: centos-7.1
+  - name: centos-6.7
+  - name: centos-6.7-i386
+  - name: centos-5.11
+  - name: centos-5.11-i386
+  - name: debian-6.0.10
+  - name: debian-6.0.10-i386
+  - name: debian-7.8
+  - name: debian-7.8-i386
+  - name: debian-8.1
+  - name: debian-8.1-i386
+  - name: fedora-21
+  - name: fedora-21-i386
+  - name: fedora-22
+  - name: freebsd-9.3
+  - name: freebsd-10.2
+  - name: opensuse-13.2-x86_64
+  - name: opensuse-13.2-i386
+  - name: ubuntu-14.04
+  - name: ubuntu-14.04-i386
+  - name: ubuntu-12.04
+  - name: ubuntu-12.04-i386
+  - name: ubuntu-10.04
+  - name: ubuntu-10.04-i386
+  - name: mint-17.2-cinnamon
+    driver_config:
+      box: artem-sidorenko/mint-17.2-cinnamon
+
+suites:
+  - name: default
+    run_list:
+      - recipe[os_prepare]
+    attributes:

.kitchen.yml

@@ -1,45 +1,80 @@
 ---
 driver:
-  name: vagrant
+  name: dokken
+  chef_version: 12.5.1
+  privileged: true # because Docker and SystemD/Upstart
+
+transport:
+  name: dokken
 
 provisioner:
-  name: chef_solo
+  name: dokken
 
 verifier:
   name: inspec
   sudo: true
 
 platforms:
-  - name: centos-7.1
-  - name: centos-6.7
-  - name: centos-6.7-i386
-  - name: centos-5.11
-  - name: centos-5.11-i386
-  - name: debian-6.0.10
-  - name: debian-6.0.10-i386
-  - name: debian-7.8
-  - name: debian-7.8-i386
-  - name: debian-8.1
-  - name: debian-8.1-i386
-  - name: fedora-21
-  - name: fedora-21-i386
-  - name: fedora-22
-  - name: freebsd-9.3
-  - name: freebsd-10.2
-  - name: opensuse-13.2-x86_64
-  - name: opensuse-13.2-i386
-  - name: ubuntu-14.04
-  - name: ubuntu-14.04-i386
-  - name: ubuntu-12.04
-  - name: ubuntu-12.04-i386
-  - name: ubuntu-10.04
-  - name: ubuntu-10.04-i386
-  - name: mint-17.2-cinnamon
-    driver_config:
-      box: artem-sidorenko/mint-17.2-cinnamon
+- name: ubuntu-12.04
+  driver:
+    image: ubuntu:12.04
+- name: ubuntu-14.04
+  driver:
+    image: ubuntu:14.04
+- name: ubuntu-15.10
+  driver:
+    image: ubuntu:15.10
+    pid_one_command: /bin/systemd
+- name: ubuntu-16.04
+  driver:
+    image: ubuntu:16.04
+    intermediate_instructions:
+    - RUN /usr/bin/apt-get update
+    pid_one_command: /bin/systemd
+- name: centos-6.6
+  driver:
+    image: centos:6.6
+- name: centos-6.7
+  driver:
+    image: centos:6.7
+    intermediate_instructions:
+      - RUN yum install -y initscripts
+- name: centos-7
+  driver:
+    image: centos:7
+    pid_one_command: /usr/lib/systemd/systemd
+- name: oracle-6.6
+  driver:
+    image: oraclelinux:6.6
+- name: oracle-6.7
+  driver:
+    image: oraclelinux:6.7
+- name: oracle-7.1
+  driver:
+    image: oraclelinux:7.1
+    pid_one_command: /usr/lib/systemd/systemd
+- name: debian-7
+  driver:
+    image: debian:7
+    intermediate_instructions:
+    - RUN /usr/bin/apt-get update
+    - RUN /usr/bin/apt-get install -y procps
+- name: debian-8
+  driver:
+    image: debian:8
+    intermediate_instructions:
+    - RUN /usr/bin/apt-get update
+    - RUN /usr/bin/apt-get install -y procps
+    pid_one_command: /bin/systemd
 
 suites:
-  - name: default
-    run_list:
-      - recipe[os_prepare]
-    attributes:
+- name: default
+  run_list:
+  - recipe[apt]
+  - recipe[yum]
+  - recipe[ssh-hardening]
+  - recipe[os_prepare]
+  attributes:
+    osprepare:
+      docker: true
+      application: false

.travis.yml

@@ -17,6 +17,12 @@ matrix:
     script: bundle exec rake lint test test:resources config=test/test.yaml
   - rvm: 2.2
     script: N=5 bundle exec rake test:functional test:resources config=test/test-extra.yaml
+  - rvm: 2.2
+    bundler_args: "--without guard tools"
+    script: DOCKER=true bundle exec rake test:integration OS='default-ubuntu-1204'
+  - rvm: 2.2
+    bundler_args: "--without guard tools"
+    script: DOCKER=true bundle exec rake test:integration OS='default-ubuntu-1604'
 deploy:
   provider: rubygems
   on:

Berksfile

@@ -4,3 +4,4 @@ source 'https://supermarket.chef.io'
 cookbook 'apt'
 cookbook 'os_prepare', path: './test/cookbooks/os_prepare'
 cookbook 'runit', github: 'hw-cookbooks/runit'
+cookbook 'ssh-hardening', git: 'https://github.com/dev-sec/chef-ssh-hardening.git'

Gemfile

@@ -23,6 +23,7 @@ group :integration do
   gem 'kitchen-vagrant'
   gem 'kitchen-inspec', '0.12.5'
   gem 'kitchen-ec2'
+  gem 'kitchen-dokken'
 end
 
 group :tools do

Rakefile

@@ -49,10 +49,10 @@ namespace :test do
     sh(Gem.ruby, 'test/docker_test.rb', *tests)
   end
 
-  task :vm do
-    concurrency = ENV['CONCURRENCY'] || 4
-    path = File.join(File.dirname(__FILE__), 'test', 'integration')
-    sh('sh', '-c', "cd #{path} && bundle exec kitchen test -c #{concurrency} -t .")
+  task :integration do
+    concurrency = ENV['CONCURRENCY'] || 1
+    os = ENV['OS'] || ''
+    sh('sh', '-c', "bundle exec kitchen test -c #{concurrency} #{os}")
   end
 
   task :ssh, [:target] do |_t, args|

test/cookbooks/os_prepare/attributes/default.rb

@@ -0,0 +1,2 @@
+default['osprepare']['docker'] = false
+default['osprepare']['application'] = true

test/cookbooks/os_prepare/recipes/default.rb

@@ -7,19 +7,21 @@
 
 # basic tests
 include_recipe('os_prepare::file')
-include_recipe('os_prepare::mount')
+include_recipe('os_prepare::mount') unless node['osprepare']['docker']
 include_recipe('os_prepare::service')
 include_recipe('os_prepare::package')
 include_recipe('os_prepare::registry_key')
-include_recipe('os_prepare::iptables')
+include_recipe('os_prepare::iptables') unless node['osprepare']['docker']
+
+# config file parsing
+include_recipe('os_prepare::json_yaml_csv_ini')
 
 # configure repos, eg. nginx
 include_recipe('os_prepare::apt')
 
 # application configuration
-include_recipe('os_prepare::postgres')
-include_recipe('os_prepare::auditctl')
-include_recipe('os_prepare::apache')
-
-# config file parsing
-include_recipe('os_prepare::json_yaml_csv_ini')
+if node['osprepare']['application']
+  include_recipe('os_prepare::postgres')
+  include_recipe('os_prepare::auditctl') unless node['osprepare']['docker']
+  include_recipe('os_prepare::apache')
+end

test/integration/default/apache_conf_spec.rb

@@ -1,15 +1,16 @@
 # encoding: utf-8
 
-if os.linux?
+# TODO: do not run those tests on docker yet
+return if ENV['DOCKER']
+return unless os.linux?
 
-  # direct access to params of apache conf
-  describe apache_conf do
-    its('LogLevel') { should eq 'warn' }
-    its('MaxKeepAliveRequests') { should eq '100' }
-  end
-
-  # only read one param
-  describe apache_conf.params('LogLevel') do
-    it { should include 'warn' }
-  end
+# direct access to params of apache conf
+describe apache_conf do
+  its('LogLevel') { should cmp 'warn' }
+  its('MaxKeepAliveRequests') { should cmp 100 }
+end
+
+# only read one param
+describe apache_conf.params('LogLevel') do
+  it { should include 'warn' }
 end

test/integration/default/compare_matcher_spec.rb

@@ -11,12 +11,12 @@ if os.linux?
     its('Port') { should cmp 22.0 }
     its('Port') { should_not cmp 22.1 }
 
-    its('LogLevel') { should eq 'INFO' }
-    its('LogLevel') { should_not eq 'info'}
+    its('LogLevel') { should eq 'VERBOSE' }
+    its('LogLevel') { should_not eq 'verbose'}
 
-    its('LogLevel') { should cmp 'INFO' }
-    its('LogLevel') { should cmp 'info' }
-    its('LogLevel') { should cmp 'InfO' }
+    its('LogLevel') { should cmp 'VERBOSE' }
+    its('LogLevel') { should cmp 'verbose' }
+    its('LogLevel') { should cmp 'VerBose' }
   end
 
   describe passwd.passwords.uniq do

test/integration/default/etc_group_spec.rb

@@ -1,5 +1,9 @@
 # encoding: utf-8
 
+# TODO: do not run those tests on docker yet
+return if ENV['DOCKER']
+
+# lets define our own group
 root_group = 'root'
 
 if os[:family] == 'aix'

test/integration/default/file_spec.rb

@@ -1,5 +1,8 @@
 # encoding: utf-8
 
+# TODO: do not run those tests on docker yet
+return if ENV['DOCKER']
+
 if os[:family] == 'freebsd'
   filedata = {
     user: 'root',
@@ -56,8 +59,8 @@ if os.unix?
     # it { should have_mode }
     its('mode') { should eq 00765 }
     it { should be_mode 00765 }
-    its('mode') { should cmp '0765' }
-    its('mode') { should_not cmp '0777' }
+    its('mode') { should cmp 0765 }
+    its('mode') { should_not cmp 0777 }
 
     it { should be_readable }
     it { should be_readable.by('owner') }

test/integration/default/iptables_spec.rb

@@ -1,5 +1,8 @@
 # encoding: utf-8
 
+# TODO: do not run those tests on docker yet
+return if ENV['DOCKER']
+
 case os[:family]
 when 'ubuntu', 'fedora'
   describe iptables do

test/integration/default/kernel_module_spec.rb

@@ -1,17 +1,18 @@
 # encoding: utf-8
 
+# TODO: do not run those tests on docker yet
+return if ENV['DOCKER']
+return unless os.linux?
+
 # Test kernel modules on all linux systems
-if os.linux?
-
-  describe kernel_module('video') do
-    it { should be_loaded }
-  end
-
-  describe kernel_module('bridge') do
-    it { should_not be_loaded }
-  end
-
-  describe kernel_module('dhcp') do
-    it { should_not be_loaded }
-  end
+describe kernel_module('video') do
+  it { should be_loaded }
+end
+
+describe kernel_module('bridge') do
+  it { should_not be_loaded }
+end
+
+describe kernel_module('dhcp') do
+  it { should_not be_loaded }
 end

test/integration/default/kernel_parameter_spec.rb

@@ -1,5 +1,8 @@
 # encoding: utf-8
 
+# TODO: do not run those tests on docker yet
+return if ENV['DOCKER']
+
 # prepare values
 if ['ubuntu', 'centos', 'fedora', 'opensuse', 'debian'].include?(os[:family])
   test_values = {

test/integration/default/mount_spec.rb

@@ -1,12 +1,14 @@
 # encoding: utf-8
 
-if os.linux?
-  # instead of `.with` or `.only_with` we recommend to use the `mount` resource
-  describe mount '/mnt/iso-disk' do
-    it { should be_mounted }
-    its('count') { should eq  1 }
-    its('device') { should eq '/tmp/empty.iso' }
-    its('type') { should eq  'iso9660' }
-    its('options') { should eq  ['ro'] }
-  end
+# TODO: do not run those tests on docker yet
+return if ENV['DOCKER']
+return unless os.linux?
+
+# instead of `.with` or `.only_with` we recommend to use the `mount` resource
+describe mount '/mnt/iso-disk' do
+  it { should be_mounted }
+  its('count') { should eq  1 }
+  its('device') { should eq '/tmp/empty.iso' }
+  its('type') { should eq  'iso9660' }
+  its('options') { should eq  ['ro'] }
 end

test/integration/default/port_spec.rb

@@ -1,5 +1,8 @@
 # encoding: utf-8
 
+# TODO: do not run those tests on docker yet
+return if ENV['DOCKER']
+
 # check that ssh runs
 if os.unix?
   describe port(22) do

test/integration/default/postgres_session_spec.rb

@@ -1,5 +1,8 @@
 # encoding: utf-8
 
+# TODO: do not run those tests on docker yet
+return if ENV['DOCKER']
+
 # postgres-server is installed on these platforms
 if ['ubuntu', 'centos'].include? os['family']
   postgres = postgres_session('postgres', 'inspec')

test/integration/default/powershell_spec.rb

@@ -1,4 +1,7 @@
 # encoding: utf-8
+
+return unless os.windows?
+
 script = <<-EOH
   Write-Output 'hello'
 EOH

test/integration/default/registry_key_spec.rb

@@ -1,67 +1,66 @@
 # encoding: utf-8
 
-if os.windows?
+return unless os.windows?
 
-  describe registry_key('HKLM\System\Test') do
-    it { should exist }
-    it { should have_value('test') }
-    it { should have_property('binary value', :binary) }
-    it { should have_property('Binary value', :binary) }
-    it { should have_property('string value') }
-    it { should have_property('String value') }
-    it { should have_property('dword value', :dword) }
-    it { should have_property_value('multistring value', :multi_string, ['test', 'multi','string','data']) }
-    it { should have_property_value('Multistring Value', :multi_string, ['test', 'multi','string','data']) }
-    it { should have_property_value('qword value', :qword, 0) }
-    it { should have_property_value('Qword value', :qword, 0) }
-    it { should have_property_value('binary value', :binary, 'dfa0f066') }
-    it { should have_property_value('Binary value', :binary, 'dfa0f066') }
-  end
-
-  # serverspec compatability
-  describe windows_registry_key('HKLM\System\Test') do
-    it { should exist }
-    it { should have_value('test') }
-    it { should have_property('string value') }
-    it { should have_property('binary value', :type_binary) }
-    it { should have_property('dword value', :type_dword) }
-    it { should have_property_value('multistring value', :type_multistring, ['test', 'multi','string','data']) }
-    it { should have_property_value('qword value', :type_qword, 0) }
-    it { should have_property_value('binary value', :type_binary, 'dfa0f066') }
-  end
-
-  describe registry_key('HKLM\Software\Policies\Microsoft\Windows\EventLog\System') do
-    it { should exist }
-    its('MaxSize') { should_not eq nil }
-  end
-
-  describe registry_key('HKLM\System\CurrentControlSet\Control\Session Manager') do
-    it { should exist }
-    it { should_not have_property_value('SafeDllSearchMode', :type_dword, 0) }
-    # case-insensitive test
-    it { should_not have_property_value('safedllsearchmode', :type_dword, 0) }
-  end
-
-  describe registry_key('HKLM\System\CurrentControlSet\Services\LanManServer\Parameters') do
-    it { should exist }
-    its('NullSessionShares') { should eq [''] }
-  end
-
-  describe registry_key('HKLM\Software\Policies\Microsoft\Internet Explorer\Main') do
-    it { should exist }
-    its('Isolation64Bit') { should eq 1 }
-    # check that its is case-insensitive
-    its('isolation64bit') { should eq 1 }
-  end
-
-  describe registry_key('HKLM\SOFTWARE\Policies\Microsoft\Windows NT\Terminal Services') do
-    it { should exist }
-    its('MinEncryptionLevel') { should eq 3 }
-  end
-
-  describe registry_key('HKLM\System\CurrentControlSet\Control\Lsa\MSV1_0') do
-    it { should exist }
-    its('NTLMMinServerSec') { should eq 537_395_200 }
-    its('NtlmMinServerSec') { should eq 537_395_200 }
-  end
+describe registry_key('HKLM\System\Test') do
+  it { should exist }
+  it { should have_value('test') }
+  it { should have_property('binary value', :binary) }
+  it { should have_property('Binary value', :binary) }
+  it { should have_property('string value') }
+  it { should have_property('String value') }
+  it { should have_property('dword value', :dword) }
+  it { should have_property_value('multistring value', :multi_string, ['test', 'multi','string','data']) }
+  it { should have_property_value('Multistring Value', :multi_string, ['test', 'multi','string','data']) }
+  it { should have_property_value('qword value', :qword, 0) }
+  it { should have_property_value('Qword value', :qword, 0) }
+  it { should have_property_value('binary value', :binary, 'dfa0f066') }
+  it { should have_property_value('Binary value', :binary, 'dfa0f066') }
+end
+
+# serverspec compatability
+describe windows_registry_key('HKLM\System\Test') do
+  it { should exist }
+  it { should have_value('test') }
+  it { should have_property('string value') }
+  it { should have_property('binary value', :type_binary) }
+  it { should have_property('dword value', :type_dword) }
+  it { should have_property_value('multistring value', :type_multistring, ['test', 'multi','string','data']) }
+  it { should have_property_value('qword value', :type_qword, 0) }
+  it { should have_property_value('binary value', :type_binary, 'dfa0f066') }
+end
+
+describe registry_key('HKLM\Software\Policies\Microsoft\Windows\EventLog\System') do
+  it { should exist }
+  its('MaxSize') { should_not eq nil }
+end
+
+describe registry_key('HKLM\System\CurrentControlSet\Control\Session Manager') do
+  it { should exist }
+  it { should_not have_property_value('SafeDllSearchMode', :type_dword, 0) }
+  # case-insensitive test
+  it { should_not have_property_value('safedllsearchmode', :type_dword, 0) }
+end
+
+describe registry_key('HKLM\System\CurrentControlSet\Services\LanManServer\Parameters') do
+  it { should exist }
+  its('NullSessionShares') { should eq [''] }
+end
+
+describe registry_key('HKLM\Software\Policies\Microsoft\Internet Explorer\Main') do
+  it { should exist }
+  its('Isolation64Bit') { should eq 1 }
+  # check that its is case-insensitive
+  its('isolation64bit') { should eq 1 }
+end
+
+describe registry_key('HKLM\SOFTWARE\Policies\Microsoft\Windows NT\Terminal Services') do
+  it { should exist }
+  its('MinEncryptionLevel') { should eq 3 }
+end
+
+describe registry_key('HKLM\System\CurrentControlSet\Control\Lsa\MSV1_0') do
+  it { should exist }
+  its('NTLMMinServerSec') { should eq 537_395_200 }
+  its('NtlmMinServerSec') { should eq 537_395_200 }
 end

test/integration/default/secpol_spec.rb

@@ -1,8 +1,8 @@
 # encoding: utf-8
 
-if os.windows?
-  describe security_policy do
-    its('EnableAdminAccount') { should eq 1 }
-    its('EnableGuestAccount') { should eq 0 }
-  end
+return unless os.windows?
+
+describe security_policy do
+  its('EnableAdminAccount') { should eq 1 }
+  its('EnableGuestAccount') { should eq 0 }
 end

test/integration/default/service_spec.rb

@@ -1,5 +1,8 @@
 # encoding: utf-8
 
+# TODO: do not run those tests on docker yet
+return if ENV['DOCKER']
+
 # based on operating system we select the available service
 if ['centos', 'fedora', 'freebsd', 'opensuse'].include?(os[:family])
   # CentOS, Fedora
@@ -38,8 +41,8 @@ describe service(available_service) do
 end
 
 # extra test for ubuntu upstart with systemv service
-if os[:family] == 'ubuntu'
-  describe service('ntp') do
+if os[:family] == 'ubuntu' && os[:release] == '12.04'
+  describe upstart_service('ssh') do
     it { should be_enabled }
     it { should be_installed }
     it { should be_running }

test/integration/default/user_spec.rb

@@ -85,7 +85,8 @@ else
 
       # check that the user is part of the groups
       if k.to_s == 'groups'
-        its(k) { should include v }
+        # TODO: do not run those tests on docker yet
+        its(k) { should include v } unless ENV['DOCKER']
       # default eq comparison
       else
         its(k) { should eq v }

test/integration/default/vbscript_spec.rb

@@ -1,5 +1,7 @@
 # encoding: utf-8
 
+return unless os.windows?
+
 # script that may have multiple lines
 vbscript = <<-EOH
   WScript.Echo "hello"

test/integration/default/wmi_spec.rb

@@ -1,5 +1,7 @@
 # encoding: utf-8
 
+return unless os.windows?
+
 # Get-WmiObject win32_service
 # Get-WmiObject -class win32_service
 # returns an array of service objects