http resource: properly execute tests on remote target (#2209)

Currently, the http resource always executes locally, even when scanning
a remote machine with `--target` which leads to undesireable behavior.

This change adds the ability to remotely execute tests with curl. This
behavior is currently opt-in with the `enable_remote_worker` flag, but
will become the default behavior in InSpec 2.0. Deprecation warnings
are emitted if the user is scanning a remote target but has not opted
in to the new behavior.

Signed-off-by: Adam Leff <adam@leff.co>

docs/resources/http.md.erb

@@ -6,9 +6,12 @@ title: About the http Resource
 
 Use the `http` InSpec audit resource to test an http endpoint.
 
-<p class="warning">Currently, this resource always executes on the host on which <code>inspec exec</code> is run, even if you use the <code>--target</code> option to remotely scan a different host.<br>
+<p class="warning">In InSpec 1.40 and earlier, this resource always executes on the host on which <code>inspec exec</code> is run, even if you use the <code>--target</code> option to remotely scan a different host.<br>
 <br>
-This will be corrected in a future version of InSpec. New InSpec releases are posted in the <a href="https://discourse.chef.io/c/chef-release" target="_blank">Release Announcements Category in Discourse</a>.</p>
+Beginning with InSpec 1.41, you can enable the ability to have the HTTP test execute on the remote target, provided <code>curl</code> is available. See the "Local vs. Remote" section below.<br>
+<br>
+Executing the HTTP test on the remote target will be the default behavior in InSpec 2.0.
+</p>
 
 <br>
 
@@ -35,6 +38,15 @@ where
 * `ssl_verify` may be specified to enable or disable verification of SSL certificates (default to `true`)
 
 <br>
+## Local vs. Remote
+
+Beginning with InSpec 1.41, you can enable the ability to have the HTTP test execute on the remote target:
+
+    describe http('http://www.example.com', enable_remote_worker: true) do
+      its('body') { should cmp 'awesome' }
+    end
+
+In InSpec 2.0, the HTTP test will automatically execute remotely whenever InSpec is testing a remote node.
 
 ## Examples
 
@@ -63,10 +75,6 @@ For example, a service is listening on default http port can be tested like this
 
 <br>
 
-## Matchers
-
-This InSpec audit resource has the following matchers. For a full list of available matchers please visit our [matchers page](https://www.inspec.io/docs/reference/matchers/).
-
 ### body
 
 The `body` matcher tests body content of http response:

lib/inspec/backend.rb

@@ -17,6 +17,14 @@ module Inspec
         Inspec::VERSION
       end
 
+      # Determine whether the connection/transport is a local connection
+      # Useful for resources to modify behavior as necessary, such as using
+      # the Ruby stdlib for a better experience.
+      def local_transport?
+        return false unless defined?(Train::Transports::Local)
+        backend.is_a?(Train::Transports::Local::Connection)
+      end
+
       # Ruby internal for printing a nice name for this class
       def to_s
         'Inspec::Backend::Class'

lib/resources/http.rb

@@ -22,51 +22,201 @@ module Inspec::Resources
         its('Content-Length') { should cmp 258 }
         its('Content-Type') { should cmp 'text/html; charset=UTF-8' }
       end
+
+      # properly execute the HTTP call on the scanned machine instead of the
+      # machine executing InSpec. This will be the default behavior in InSpec 2.0.
+      describe http('http://localhost:8080', enable_remote_worker: true) do
+        its('body') { should cmp 'local web server on target machine' }
+      end
     "
 
     def initialize(url, opts = {})
       @url = url
-      @method = opts.fetch(:method, 'GET')
+      @opts = opts
-      @params = opts.fetch(:params, nil)
+
-      @auth = opts.fetch(:auth, {})
+      if use_remote_worker?
-      @headers = opts.fetch(:headers, {})
+        return skip_resource 'curl is not available on the target machine' unless inspec.command('curl').exist?
-      @data = opts.fetch(:data, nil)
+        @worker = Worker::Remote.new(inspec, http_method, url, opts)
-      @open_timeout = opts.fetch(:open_timeout, 60)
+      else
-      @read_timeout = opts.fetch(:read_timeout, 60)
+        @worker = Worker::Local.new(http_method, url, opts)
-      @ssl_verify = opts.fetch(:ssl_verify, true)
+      end
     end
 
     def status
-      response.status
+      @worker.status
-    end
-
-    def body
-      response.body
     end
 
     def headers
-      Hashie::Mash.new(response.headers.to_h)
+      Hashie::Mash.new(@worker.response_headers)
+    end
+
+    def body
+      @worker.body
+    end
+
+    def http_method
+      @opts.fetch(:method, 'GET')
     end
 
     def to_s
-      "http #{@method} on #{@url}"
+      "http #{http_method} on #{@url}"
     end
 
     private
 
-    def response
+    def use_remote_worker?
-      return @response if @response
+      return false if inspec.local_transport?
-      conn = Faraday.new url: @url, headers: @headers, params: @params, ssl: { verify: @ssl_verify }
+      return true if @opts[:enable_remote_worker]
 
-      # set basic authentication
+      warn "[DEPRECATION] #{self} will execute locally instead of the target machine. To execute remotely, add `enable_remote_worker: true`."
-      conn.basic_auth @auth[:user], @auth[:pass] unless @auth.empty?
+      warn '[DEPRECATION] `enable_remote_worker: true` will be the default behavior in InSpec 2.0.'
+      false
+    end
 
-      # set default timeout
+    class Worker
-      conn.options.timeout      = @read_timeout  # open/read timeout in seconds
+      class Base
-      conn.options.open_timeout = @open_timeout  # connection open timeout in seconds
+        attr_reader :http_method, :opts, :url
 
-      @response = conn.send(@method.downcase) do |req|
+        def initialize(http_method, url, opts)
-        req.body = @data
+          @http_method = http_method
+          @url = url
+          @opts = opts
+        end
+
+        private
+
+        def params
+          opts.fetch(:params, nil)
+        end
+
+        def username
+          opts.fetch(:auth, {})[:user]
+        end
+
+        def password
+          opts.fetch(:auth, {})[:pass]
+        end
+
+        def request_headers
+          opts.fetch(:headers, {})
+        end
+
+        def request_body
+          opts[:data]
+        end
+
+        def open_timeout
+          opts.fetch(:open_timeout, 60)
+        end
+
+        def read_timeout
+          opts.fetch(:read_timeout, 60)
+        end
+
+        def ssl_verify?
+          opts.fetch(:ssl_verify, true)
+        end
+      end
+
+      class Local < Base
+        def status
+          response.status
+        end
+
+        def body
+          response.body
+        end
+
+        def response_headers
+          response.headers.to_h
+        end
+
+        private
+
+        def response
+          return @response if @response
+          conn = Faraday.new url: url, headers: request_headers, params: params, ssl: { verify: ssl_verify? }
+
+          # set basic authentication
+          conn.basic_auth username, password unless username.nil? || password.nil?
+
+          # set default timeout
+          conn.options.timeout      = read_timeout  # open/read timeout in seconds
+          conn.options.open_timeout = open_timeout  # connection open timeout in seconds
+
+          @response = conn.send(http_method.downcase) do |req|
+            req.body = request_body
+          end
+        end
+      end
+
+      class Remote < Base
+        attr_reader :inspec
+
+        def initialize(inspec, http_method, url, opts)
+          @inspec = inspec
+          super(http_method, url, opts)
+        end
+
+        def status
+          run_curl
+          @status
+        end
+
+        def body
+          run_curl
+          @body
+        end
+
+        def response_headers
+          run_curl
+          @response_headers
+        end
+
+        private
+
+        def run_curl
+          return if @ran_curl
+
+          response = inspec.command(curl_command).stdout
+          @ran_curl = true
+          return if response.nil?
+
+          # strip any carriage returns to normalize output
+          response.delete!("\r")
+
+          # split the prelude (status line and headers) and the body
+          prelude, @body = response.split("\n\n", 2)
+          prelude = prelude.lines
+
+          # grab the status off of the first line of the prelude
+          status_line = prelude.shift
+          @status = status_line.split(' ', 3)[1].to_i
+
+          # parse the rest of the prelude which will be all the HTTP headers
+          @response_headers = {}
+          prelude.each do |line|
+            line.strip!
+            key, value = line.split(':', 2)
+            @response_headers[key] = value.strip
+          end
+        end
+
+        def curl_command
+          cmd = ["curl -i -X #{http_method}"]
+          cmd << "--connect-timeout #{open_timeout}"
+          cmd << "--user \'#{username}:#{password}\'" unless username.nil? || password.nil?
+          cmd << '--insecure' unless ssl_verify?
+          cmd << "--data #{Shellwords.shellescape(request_body)}" unless request_body.nil?
+
+          request_headers.each do |k, v|
+            cmd << "-H '#{k}=#{v}'"
+          end
+
+          cmd << "'#{url}'"
+
+          cmd.join(' ')
+        end
       end
     end
   end

test/helper.rb

@@ -417,6 +417,12 @@ class MockLoader
       '/usr/sbin/service sshd status' => empty.call,
       '/sbin/service sshd status' => empty.call,
       'type "lsof"' => empty.call,
+
+      # http resource - remote worker'
+      "curl -i -X GET --connect-timeout 60 'http://www.example.com'" => cmd.call('http-remote-no-options'),
+      "curl -i -X GET --connect-timeout 60 --user 'user:pass' 'http://www.example.com'" => cmd.call('http-remote-basic-auth'),
+      '2bdc8826b66efa554bdebd8cc5f3eaf7bfba5ada36adc7904a6b178d331395ea' => cmd.call('http-remote-post'),
+      "curl -i -X GET --connect-timeout 60 -H 'accept=application/json' -H 'foo=bar' 'http://www.example.com'" => cmd.call('http-remote-headers'),
     }
     @backend
   end

test/unit/mock/cmd/http-remote-basic-auth

@@ -0,0 +1,15 @@
+HTTP/1.1 200 OK
+Date: Tue, 03 Oct 2017 20:30:08 GMT
+Expires: -1
+Cache-Control: private, max-age=0
+Content-Type: text/html; charset=ISO-8859-1
+P3P: CP="This is not a P3P policy! See https://www.google.com/support/accounts/answer/151657?hl=en for more info."
+Server: gws
+X-XSS-Protection: 1; mode=block
+X-Frame-Options: SAMEORIGIN
+Set-Cookie: NID=113=kNR6MIUK3vNjVH3KvQqIjfnDLLPHJ96wmC_z643weEFQ6Cfq0B2iUYqxzQk5pKVoAkbL8ZxKFvvM9v55qiNFXH_O655WuuUxPshmlSIM5xpCSH0xy09SnIQJVi0l7eKY; expires=Wed, 04-Apr-2018 20:30:08 GMT; path=/; domain=.google.com; HttpOnly
+Accept-Ranges: none
+Vary: Accept-Encoding
+Transfer-Encoding: chunked
+
+auth ok

test/unit/mock/cmd/http-remote-headers

@@ -0,0 +1,16 @@
+HTTP/1.1 200 OK
+Date: Tue, 03 Oct 2017 20:30:08 GMT
+Expires: -1
+Cache-Control: private, max-age=0
+Content-Type: text/html; charset=ISO-8859-1
+P3P: CP="This is not a P3P policy! See https://www.google.com/support/accounts/answer/151657?hl=en for more info."
+Server: gws
+X-XSS-Protection: 1; mode=block
+X-Frame-Options: SAMEORIGIN
+Set-Cookie: NID=113=kNR6MIUK3vNjVH3KvQqIjfnDLLPHJ96wmC_z643weEFQ6Cfq0B2iUYqxzQk5pKVoAkbL8ZxKFvvM9v55qiNFXH_O655WuuUxPshmlSIM5xpCSH0xy09SnIQJVi0l7eKY; expires=Wed, 04-Apr-2018 20:30:08 GMT; path=/; domain=.google.com; HttpOnly
+Accept-Ranges: none
+Vary: Accept-Encoding
+Transfer-Encoding: chunked
+mock: ok
+
+headers ok

test/unit/mock/cmd/http-remote-no-options

@@ -0,0 +1,15 @@
+HTTP/1.1 200 OK
+Date: Tue, 03 Oct 2017 20:30:08 GMT
+Expires: -1
+Cache-Control: private, max-age=0
+Content-Type: text/html; charset=ISO-8859-1
+P3P: CP="This is not a P3P policy! See https://www.google.com/support/accounts/answer/151657?hl=en for more info."
+Server: gws
+X-XSS-Protection: 1; mode=block
+X-Frame-Options: SAMEORIGIN
+Set-Cookie: NID=113=kNR6MIUK3vNjVH3KvQqIjfnDLLPHJ96wmC_z643weEFQ6Cfq0B2iUYqxzQk5pKVoAkbL8ZxKFvvM9v55qiNFXH_O655WuuUxPshmlSIM5xpCSH0xy09SnIQJVi0l7eKY; expires=Wed, 04-Apr-2018 20:30:08 GMT; path=/; domain=.google.com; HttpOnly
+Accept-Ranges: none
+Vary: Accept-Encoding
+Transfer-Encoding: chunked
+
+no options

test/unit/mock/cmd/http-remote-post

@@ -0,0 +1,15 @@
+HTTP/1.1 200 OK
+Date: Tue, 03 Oct 2017 20:30:08 GMT
+Expires: -1
+Cache-Control: private, max-age=0
+Content-Type: text/html; charset=ISO-8859-1
+P3P: CP="This is not a P3P policy! See https://www.google.com/support/accounts/answer/151657?hl=en for more info."
+Server: gws
+X-XSS-Protection: 1; mode=block
+X-Frame-Options: SAMEORIGIN
+Set-Cookie: NID=113=kNR6MIUK3vNjVH3KvQqIjfnDLLPHJ96wmC_z643weEFQ6Cfq0B2iUYqxzQk5pKVoAkbL8ZxKFvvM9v55qiNFXH_O655WuuUxPshmlSIM5xpCSH0xy09SnIQJVi0l7eKY; expires=Wed, 04-Apr-2018 20:30:08 GMT; path=/; domain=.google.com; HttpOnly
+Accept-Ranges: none
+Vary: Accept-Encoding
+Transfer-Encoding: chunked
+
+post ok

test/unit/resources/http_test.rb

@@ -5,58 +5,109 @@ require 'helper'
 require 'inspec/resource'
 
 describe 'Inspec::Resources::Http' do
-  it 'verify simple http' do
+  describe 'InSpec::Resources::Http::Worker::Local' do
-    stub_request(:get, "www.example.com").to_return(status: 200, body: 'pong')
+    let(:domain)      { 'www.example.com' }
+    let(:http_method) { 'GET' }
+    let(:opts)        { {} }
+    let(:worker)      { Inspec::Resources::Http::Worker::Local.new(http_method, "http://#{domain}", opts) }
 
-    resource = load_resource('http', 'http://www.example.com')
+    describe 'simple HTTP request with no options' do
-    _(resource.status).must_equal 200
+      it 'returns correct data' do
-    _(resource.body).must_equal 'pong'
+        stub_request(:get, domain).to_return(status: 200, body: 'pong')
+
+        _(worker.status).must_equal 200
+        _(worker.body).must_equal 'pong'
+      end
+    end
+
+    describe 'request with basic auth' do
+      let(:opts) { { auth: { user: 'user', pass: 'pass' } } }
+
+      it 'returns correct data' do
+        stub_request(:get, domain).with(basic_auth: ['user', 'pass']).to_return(status: 200, body: 'auth ok')
+
+        _(worker.status).must_equal 200
+        _(worker.body).must_equal 'auth ok'
+      end
+    end
+
+    describe 'POST request with data' do
+      let(:http_method) { 'POST'}
+      let(:opts)        { { data: {a: '1', b: 'five'} } }
+
+      it 'returns correct data' do
+        stub_request(:post, domain).with(body: {a: '1', b: 'five'}).to_return(status: 200, body: 'post ok')
+
+        _(worker.status).must_equal 200
+        _(worker.body).must_equal 'post ok'
+      end
+    end
+
+    describe 'with request headers' do
+      let(:opts) { { headers: { 'accept' => 'application/json' } } }
+
+      it 'returns correct data' do
+        stub_request(:get, domain).with(headers: {'accept' => 'application/json'}).to_return(status: 200, body: 'headers ok', headers: {'mock' => 'ok'})
+
+        _(worker.status).must_equal 200
+        _(worker.body).must_equal 'headers ok'
+        _(worker.response_headers['mock']).must_equal 'ok'
+      end
+    end
+
+    describe 'with params' do
+      let(:opts) { { params: { a: 'b' } } }
+
+      it 'returns correct data' do
+        stub_request(:get, domain).with(query: {a: 'b'}).to_return(status: 200, body: 'params ok')
+
+        _(worker.status).must_equal 200
+        _(worker.body).must_equal 'params ok'
+      end
+    end
   end
 
-  it 'verify http with basic auth' do
+  describe 'Inspec::Resource::Http::Worker::Remote' do
-    stub_request(:get, "www.example.com").with(basic_auth: ['user', 'pass']).to_return(status: 200, body: 'auth ok')
+    let(:backend)     { MockLoader.new.backend }
+    let(:http_method) { 'GET' }
+    let(:url)         { 'http://www.example.com' }
+    let(:opts)        { {} }
+    let(:worker)      { Inspec::Resources::Http::Worker::Remote.new(backend, http_method, url, opts)}
 
-    resource = load_resource('http', 'http://www.example.com', auth: { user: 'user',pass: 'pass'})
+    describe 'simple HTTP request with no options' do
-    _(resource.status).must_equal 200
+      it 'returns correct data' do
-    _(resource.body).must_equal 'auth ok'
+        _(worker.status).must_equal 200
-  end
+        _(worker.body).must_equal 'no options'
+      end
+    end
 
-  it 'verify http post with data' do
+    describe 'request with basic auth' do
-    stub_request(:post, "www.example.com").with(body: {data: {a: '1', b: 'five'}}).to_return(status: 200, body: 'post ok')
+      let(:opts) { { auth: { user: 'user', pass: 'pass' } } }
 
-    resource = load_resource('http', 'http://www.example.com',
+      it 'returns correct data' do
-                             method: 'POST',
+        _(worker.status).must_equal 200
-                             data: '{"data":{"a":"1","b":"five"}}',
+        _(worker.body).must_equal 'auth ok'
-                             headers: {'content-type' => 'application/json'})
+      end
-    _(resource.status).must_equal 200
+    end
-    _(resource.body).must_equal 'post ok'
-  end
 
-  it 'verify http headers' do
+    describe 'POST request with data' do
-    stub_request(:post, "www.example.com").with(headers: {'content-type' => 'application/json'}).to_return(status: 200, body: 'headers ok', headers: {'mock' => 'ok'})
+      let(:http_method) { 'POST'}
+      let(:opts)        { { data: {a: '1', b: 'five'} } }
 
-    resource = load_resource('http', 'http://www.example.com',
+      it 'returns correct data' do
-                             method: 'POST',
+        _(worker.status).must_equal 200
-                             data: '{"data":{"a":"1","b":"five"}}',
+        _(worker.body).must_equal 'post ok'
-                             headers: {'content-type' => 'application/json'})
+      end
-    _(resource.status).must_equal 200
+    end
-    _(resource.body).must_equal 'headers ok'
-    _(resource.headers.mock).must_equal 'ok'
-  end
 
-  it 'verify http with params' do
+    describe 'with request headers' do
-    stub_request(:get, "www.example.com").with(query: {a: 'b'}).to_return(status: 200, body: 'params ok')
+      let(:opts) { { headers: { 'accept' => 'application/json', 'foo' => 'bar' } } }
 
-    resource = load_resource('http', 'http://www.example.com', params: {a: 'b'})
+      it 'returns correct data' do
-    _(resource.status).must_equal 200
+        _(worker.status).must_equal 200
-    _(resource.body).must_equal 'params ok'
+        _(worker.body).must_equal 'headers ok'
-  end
+        _(worker.response_headers['mock']).must_equal 'ok'
-
+      end
-  it 'verify http with timeouts' do
+    end
-    stub_request(:get, "www.example.com").to_return(status: 200, body: 'params ok')
-
-    resource = load_resource('http', 'http://www.example.com', open_timeout: 10, read_timeout: 10)
-    _(resource.instance_variable_get(:@open_timeout)).must_equal 10
-    _(resource.instance_variable_get(:@read_timeout)).must_equal 10
   end
 end