Various small fixes/adjustments to the integration tests for AWS and Azure (#2745)

* Fix formatting of iam user integration tests by placing them in controls
* Fix subnet AZ test by making it an attribute; can't hardcode it
* Fix VPC ID fixture export for subnet testing
* Rename Azure integration tasks to match AWS and allow on-demand attribute dump

Signed-off-by: Clinton Wolfe <clintoncwolfe@gmail.com>

.gitignore

@@ -32,4 +32,4 @@ profile-1.0.0.tar.gz
 .terraform/
 terraform.tfstate*
 terraform.tfstate.backup
-
+inspec-azure.plan

Rakefile

@@ -151,15 +151,17 @@ namespace :test do
 
   namespace :azure do
     # Specify the directory for the integration tests
-    integration_dir = 'test/integration/azure'
+    integration_dir = File.join(project_dir, 'test', 'integration', 'azure')
+    attribute_file = File.join(integration_dir, '.attribute.yml')
 
-    task :init_workspace do
-      # Initialize terraform workspace
-      sh("cd #{integration_dir}/build/ && terraform init")
-    end
-
-    task :setup_integration_tests do
+    task :setup, :tf_workspace do |t, args|
+      tf_workspace = args[:tf_workspace] || ENV['INSPEC_TERRAFORM_ENV']
+      abort("You must either call the top-level test:azure task, or set the INSPEC_TERRAFORM_ENV variable.") unless tf_workspace
       puts '----> Setup'
+      sh("cd #{integration_dir}/build/ && terraform init")
+      sh("cd #{integration_dir}/build/ && terraform workspace new #{tf_workspace}")
+
+      # Generate Azure crendentials
       creds = Train.create('azure').connection.connect
 
       # Determine the storage account name and the admin password
@@ -170,35 +172,72 @@ namespace :test do
       suffix = sa_name[0..3]
 
       # Create the plan that can be applied to Azure
-      cmd = format("cd %s/build/ && terraform plan -var 'subscription_id=%s' -var 'client_id=%s' -var 'client_secret=%s' -var 'tenant_id=%s' -var 'storage_account_name=%s' -var 'admin_password=%s' -var 'suffix=%s' -out inspec-azure.plan", integration_dir, creds[:subscription_id], creds[:client_id], creds[:client_secret], creds[:tenant_id], sa_name, admin_password, suffix)
+      cmd  = ""
+      cmd += "cd #{integration_dir}/build/ && terraform plan -out inspec-azure.plan"
+      cmd += " -var 'subscription_id=#{creds[:subscription_id]}' "
+      cmd += " -var 'client_id=#{creds[:client_id]}' "
+      cmd += " -var 'client_secret=#{creds[:client_secret]}' "
+      cmd += " -var 'tenant_id=#{creds[:tenant_id]}' "
+      cmd += " -var 'storage_account_name=#{sa_name}' "
+      cmd += " -var 'admin_password=#{admin_password}' "
+      cmd += " -var 'suffix=#{suffix}' "
       sh(cmd)
 
       # Apply the plan on Azure
-      cmd = format("cd %s/build/ && terraform apply inspec-azure.plan", integration_dir)
+      cmd = "cd #{integration_dir}/build/ && terraform apply inspec-azure.plan"
       sh(cmd)
+
+      # Dump TF outputs to InSpec attributes file
+      Rake::Task["test:azure:dump_attrs"].execute
     end
 
-    task :run_integration_tests do
+    task :"dump_attrs" do
+      sh("cd #{integration_dir}/build/ && terraform output > #{attribute_file}")
+        raw_output = File.read(attribute_file)
+        yaml_output = raw_output.gsub(" = ", " : ")
+        File.open(attribute_file, "w") {|file| file.puts yaml_output}
+      end
+
+    task :run do
       puts '----> Run'
       sh("bundle exec inspec exec #{integration_dir}/verify -t azure://1e0b427a-d58b-494e-ae4f-ee558463ebbf")
     end
 
-    task :cleanup_integration_tests do
+    task :cleanup, :tf_workspace do |t, args|
+      tf_workspace = args[:tf_workspace] || ENV['INSPEC_TERRAFORM_ENV']
+      abort("You must either call the top-level test:azure task, or set the INSPEC_TERRAFORM_ENV variable.") unless tf_workspace
       puts '----> Cleanup'
+
       creds = Train.create('azure').connection.connect
 
-      cmd = format("cd %s/build/ && terraform destroy -force -var 'subscription_id=%s' -var 'client_id=%s' -var 'client_secret=%s' -var 'tenant_id=%s' -var 'admin_password=dummy' -var 'storage_account_name=dummy' -var 'suffix=dummy'", integration_dir, creds[:subscription_id], creds[:client_id], creds[:client_secret], creds[:tenant_id])
+      cmd  = ""
+      cmd += "cd #{integration_dir}/build/ && terraform destroy -force "
+      cmd += " -var 'subscription_id=#{creds[:subscription_id]}' "
+      cmd += " -var 'client_id=#{creds[:client_id]}' "
+      cmd += " -var 'client_secret=#{creds[:client_secret]}' "
+      cmd += " -var 'tenant_id=#{creds[:tenant_id]}' "
+      cmd += " -var 'storage_account_name=dummy' "
+      cmd += " -var 'admin_password=dummy' "
+      cmd += " -var 'suffix=dummy' "
+
       sh(cmd)
+
+      sh("cd #{integration_dir}/build/ && terraform workspace select default")
+      sh("cd #{integration_dir}/build && terraform workspace delete #{tf_workspace}")
     end
   end
 
   desc "Perform Azure Integration Tests"
   task :azure do
-    Rake::Task['test:azure:init_workspace'].execute
-    Rake::Task['test:azure:cleanup_integration_tests'].execute
-    Rake::Task['test:azure:setup_integration_tests'].execute
-    Rake::Task['test:azure:run_integration_tests'].execute
-    Rake::Task['test:azure:cleanup_integration_tests'].execute
+    tf_workspace = ENV['INSPEC_TERRAFORM_ENV'] || prompt("Please enter a workspace for your integration tests to run in: ")
+    begin
+      Rake::Task["test:azure:setup"].execute({:tf_workspace => tf_workspace})
+      Rake::Task["test:azure:run"].execute
+    rescue
+      abort("Integration testing has failed")
+    ensure
+      Rake::Task["test:azure:cleanup"].execute({:tf_workspace => tf_workspace})
+    end
   end
 end
 

test/integration/aws/default/build/ec2.tf

@@ -198,6 +198,10 @@ output "ec2_security_group_alpha_group_id" {
   value = "${aws_security_group.alpha.id}"
 }
 
+output "ec2_security_group_alpha_group_name" {
+  value = "${aws_security_group.alpha.name}"
+}
+
 #============================================================#
 #                      VPC Subnets
 #============================================================#
@@ -206,10 +210,16 @@ resource "aws_subnet" "subnet_01" {
   cidr_block = "172.31.96.0/20"
 }
 
-output "ec2_default_vpc_subnet_01_id" {
+# Re-output any VPC ID for subnet listings
+output "subnet_vpc_id" {
+  # Use the default VPC since it is gaurenteed
+  value = "${data.aws_vpc.default.id}"
+}
+
+output "subnet_01_id" {
   value = "${aws_subnet.subnet_01.id}"
 }
 
-output "ec2_security_group_alpha_group_name" {
-  value = "${aws_security_group.alpha.name}"
+output "subnet_01_az" {
+  value = "${aws_subnet.subnet_01.availability_zone}"
 }

test/integration/aws/default/verify/controls/aws_iam_user.rb

@@ -14,33 +14,38 @@ fixtures = {}
 end
 
 #-------------------  Recall / Miss -------------------#
-describe aws_iam_user(username: fixtures['iam_user_recall_hit']) do
-  it { should exist }
-end
-
-describe aws_iam_user(username: fixtures['iam_user_recall_miss']) do
-  it { should_not exist }
-end
-
-#------------- Property - has_mfa_enabled -------------#
-
-# TODO: fixture and test for has_mfa_enabled
-
-describe aws_iam_user(username: fixtures['iam_user_no_mfa_enabled']) do
-  it { should_not have_mfa_enabled }
-  it { should_not have_console_password } # TODO: this is working by accident, we should have a dedicated fixture
-end
-
-#---------- Property - has_console_password -----------#
-
-describe aws_iam_user(username: fixtures['iam_user_has_console_password']) do
-  it { should have_console_password }
-end
-
-#------------- Property - access_keys -------------#
-
-aws_iam_user(username: fixtures['iam_user_with_access_key']).access_keys.each { |access_key|
-  describe access_key do
-   its('status') { should eq 'Active' }
+control "aws_iam_user recall" do
+  describe aws_iam_user(username: fixtures['iam_user_recall_hit']) do
+    it { should exist }
   end
-}
+
+  describe aws_iam_user(username: fixtures['iam_user_recall_miss']) do
+    it { should_not exist }
+  end
+end
+
+control "aws_iam_user properties" do
+  #------------- Property - has_mfa_enabled -------------#
+
+  # TODO: fixture and test for has_mfa_enabled
+
+  describe aws_iam_user(username: fixtures['iam_user_no_mfa_enabled']) do
+    it { should_not have_mfa_enabled }
+    it { should_not have_console_password } # TODO: this is working by accident, we should have a dedicated fixture
+  end
+
+  #---------- Property - has_console_password -----------#
+
+  describe aws_iam_user(username: fixtures['iam_user_has_console_password']) do
+    it { should have_console_password }
+  end
+
+  #------------- Property - access_keys -------------#
+
+  aws_iam_user(username: fixtures['iam_user_with_access_key']).access_keys.each { |access_key|
+    describe access_key.access_key_id do
+      subject { access_key }
+     its('status') { should eq 'Active' }
+    end
+  }
+end

test/integration/aws/default/verify/controls/aws_iam_users.rb

@@ -1,4 +1,7 @@
-describe aws_iam_users.where(has_console_password?: true)
-                      .where(has_mfa_enabled?: false) do
-  it { should exist }
-end
+
+control "aws_iam_users filtering" do
+  describe aws_iam_users.where(has_console_password?: true)
+                        .where(has_mfa_enabled?: false) do
+    it { should exist }
+  end
+end

test/integration/aws/default/verify/controls/aws_subnet.rb

@@ -2,6 +2,7 @@ fixtures = {}
 [
   'ec2_security_group_default_vpc_id',
   'ec2_default_vpc_subnet_01_id',
+  'subnet_01_az',
 ].each do |fixture_name|
   fixtures[fixture_name] = attribute(
     fixture_name,
@@ -32,7 +33,7 @@ control "aws_subnet properties of subnet_01" do
     its('subnet_id') { should eq fixtures['ec2_default_vpc_subnet_01_id'] }
     its('cidr_block') { should eq '172.31.96.0/20' }
     its('available_ip_address_count') { should eq 4091 }
-    its('availability_zone') { should eq 'us-east-1c' }
+    its('availability_zone') { should eq fixtures['subnet_01_az'] }
     its('ipv_6_cidr_block_association_set') { should eq [] }
   end
 end

test/integration/aws/default/verify/controls/aws_subnets.rb

@@ -1,7 +1,7 @@
 fixtures = {}
 [
-  'ec2_security_group_default_vpc_id',
-  'ec2_default_vpc_subnet_id',
+  'subnet_01_id',
+  'subnet_vpc_id',
 ].each do |fixture_name|
   fixtures[fixture_name] = attribute(
     fixture_name,
@@ -14,12 +14,12 @@ control "aws_subnets recall" do
   all_subnets = aws_subnets
   
   # You should be able to get a specific subnet given its id
-  describe all_subnets.where(subnet_id: fixtures['ec2_default_vpc_subnet_id']) do
+  describe all_subnets.where(subnet_id: fixtures['subnet_01_id']) do
     it { should exist }
   end
 
   # You should be able to get subnets given a vpc_id
-  describe all_subnets.where(vpc_id: fixtures['ec2_security_group_default_vpc_id']) do
+  describe all_subnets.where(vpc_id: fixtures['subnet_vpc_id']) do
     it { should exist }
   end
 
@@ -32,17 +32,17 @@ control "aws_subnets recall" do
   end
 end
 
-control "aws_subnets properties of default VPC subnet" do
+control "aws_subnets properties by subnet id" do
   # you should be able to test the cidr_block of a subnet
-  describe aws_subnets.where(subnet_id: fixtures['ec2_default_vpc_subnet_id']) do
+  describe aws_subnets.where(subnet_id: fixtures['subnet_01_id']) do
     its('cidr_blocks') { should include '172.31.96.0/20' }
     its('states') { should_not include 'pending' }
   end
 end
 
-control "aws_subnets properties of default VPC" do
+control "aws_subnets properties by vpc_id" do
   # you should be able to test the cidr_block of a subnet
-  describe aws_subnets.where(vpc_id: fixtures['ec2_security_group_default_vpc_id']) do
+  describe aws_subnets.where(vpc_id: fixtures['subnet_vpc_id']) do
     its('cidr_blocks') { should include '172.31.96.0/20' }
     its('states') { should include 'available' }
   end

test/integration/azure/build/azure.tf

@@ -15,6 +15,11 @@ variable "location" {
   default = "West Europe"
 }
 
+# Output the sub ID so the fixture system has something to chew on
+output "subscription_id" {
+  value = "${var.subscription_id}"
+}
+
 # Configure the Azure RM provider
 provider "azurerm" {
   subscription_id = "${var.subscription_id}"