AWS API Pagination fixes (#2762)

* Add pagination support to aws_iam_groups
* Add pagination support to aws_iam_policy
* Add pagination to aws_iam_policies
* Adds pagination to aws_iam_access_keys
* Adds pagination to aws_kms_keys

Signed-off-by: Clinton Wolfe <clintoncwolfe@gmail.com>

lib/resources/aws/aws_iam_access_keys.rb

@@ -85,9 +85,14 @@ class AwsIamAccessKeys < Inspec.resource(1)
             # Swallow - a miss on search results should return an empty table
           end
         else
-          # TODO: pagination check and resume
+          pagination_opts = {}
-          iam_client.list_users.users.each do |info|
+          loop do
-            user_details[info.user_name] = info
+            api_result = iam_client.list_users(pagination_opts)
+            api_result.users.each do |info|
+              user_details[info.user_name] = info
+            end
+            break unless api_result.is_truncated
+            pagination_opts[:marker] = api_result.marker
           end
         end
 

lib/resources/aws/aws_iam_groups.rb

@@ -29,7 +29,14 @@ class AwsIamGroups < Inspec.resource(1)
 
   def fetch_from_api
     backend = BackendFactory.create(inspec_runner)
-    @table = backend.list_groups.to_h[:groups]
+    @table = []
+    pagination_opts = {}
+    loop do
+      api_result = backend.list_groups(pagination_opts)
+      @table += api_result.groups.map(&:to_h)
+      pagination_opts = { marker: api_result.marker }
+      break unless api_result.is_truncated
+    end
   end
 
   class Backend

lib/resources/aws/aws_iam_policies.rb

@@ -30,7 +30,14 @@ class AwsIamPolicies < Inspec.resource(1)
 
   def fetch_from_api
     backend = BackendFactory.create(inspec_runner)
-    @table = backend.list_policies({}).to_h[:policies]
+    @table = []
+    pagination_opts = {}
+    loop do
+      api_result = backend.list_policies(pagination_opts)
+      @table += api_result.policies.map(&:to_h)
+      pagination_opts = { marker: api_result.marker }
+      break unless api_result.is_truncated
+    end
   end
 
   class Backend

lib/resources/aws/aws_iam_policy.rb

@@ -70,18 +70,24 @@ class AwsIamPolicy < Inspec.resource(1)
   def fetch_from_api
     backend = BackendFactory.create(inspec_runner)
 
-    criteria = { max_items: 1000 } # maxItems max value is 1000
+    policy = nil
-    resp = backend.list_policies(criteria)
+    pagination_opts = { max_items: 1000 }
-    @policy = resp.policies.detect do |policy|
+    loop do
-      policy.policy_name == @policy_name
+      api_result = backend.list_policies(pagination_opts)
+      policy = api_result.policies.detect do |p|
+        p.policy_name == @policy_name
+      end
+      break if policy # Found it!
+      break unless api_result.is_truncated # Not found and no more results
+      pagination_opts[:marker] = api_result.marker
     end
 
-    @exists = !@policy.nil?
+    @exists = !policy.nil?
 
     return unless @exists
-    @arn = @policy[:arn]
+    @arn = policy[:arn]
-    @default_version_id = @policy[:default_version_id]
+    @default_version_id = policy[:default_version_id]
-    @attachment_count = @policy[:attachment_count]
+    @attachment_count = policy[:attachment_count]
   end
 
   def fetch_attached_entities

lib/resources/aws/aws_kms_keys.rb

@@ -30,7 +30,14 @@ class AwsKmsKeys < Inspec.resource(1)
 
   def fetch_from_api
     backend = BackendFactory.create(inspec_runner)
-    @table = backend.list_keys({ limit: 1000 }).to_h[:keys] # max value for limit is 1000
+    @table = []
+    pagination_opts = { limit: 1000 }
+    loop do
+      api_result = backend.list_keys(pagination_opts)
+      @table += api_result.keys.map(&:to_h)
+      break unless api_result.truncated
+      pagination_opts = { marker: api_result.next_marker }
+    end
   end
 
   class Backend