Merge pull request #5871 from inspec/vasundhara/support-profile-gem-dependency

CFINSPEC-13:  Adds support for specifying gem dependencies in the metadata file of InSpec Profile.
This commit is contained in:
Clinton Wolfe 2022-03-09 19:44:05 -05:00 committed by GitHub
commit 4a63319840
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
24 changed files with 608 additions and 0 deletions

View file

@ -285,6 +285,8 @@ This subcommand has the following additional options:
* ``--attrs=one two three``
Legacy name for --input-file - deprecated.
* ``--auto-install-gems``
Auto installs gem dependencies of the profile or resource pack.
* ``-b``, ``--backend=BACKEND``
Choose a backend: local, ssh, winrm, docker.
* ``--backend-cache``, ``--no-backend-cache``

View file

@ -63,6 +63,7 @@ Each profile must have an `inspec.yml` file that defines the following informati
- Use `supports` to specify a list of supported platform targets.
- Use `depends` to define a list of profiles on which this profile depends.
- Use `inputs` to define a list of inputs you can use in your controls.
- Use `gem_dependencies` to specify a list of profile gem dependencies that is required to be installed for the profile to function correctly.
`name` is required; all other profile settings are optional. For example:
@ -80,6 +81,9 @@ supports:
depends:
- name: profile
path: ../path/to/profile
gem_dependencies:
- name: "gem-name"
version: ">= 2.0.0"
inspec_version: "~> 2.1"
```
@ -294,6 +298,18 @@ depends:
- name: linux
compliance: base/linux
```
## Gem Dependencies
Any profile with ruby gem dependencies that need to be installed can be specified using the `gem_dependencies` settings in the `inspec.yml` metadata file.
For example, if you required any ruby library in a custom resource that needs a specific gem to be installed, then you can specify those gems in the metadata file. Chef InSpec will prompt to install the gems to `~/.inspec/gems` when you run your profile the first time. To skip the prompt and automatically install, pass the `--auto-install-gems` option to `inspec exec`.
```YAML
gem_dependencies:
- name: "mongo"
version: ">= 2.3.12"
```
## Vendoring Dependencies

View file

@ -28,3 +28,6 @@ require "inspec/base_cli"
require "inspec/fetcher"
require "inspec/source_reader"
require "inspec/resource"
require "inspec/dependency_loader"
require "inspec/dependency_installer"

View file

@ -145,6 +145,8 @@ module Inspec
desc: "Folder which contains referenced profiles."
option :vendor_cache, type: :string,
desc: "Use the given path for caching dependencies. (default: ~/.inspec/cache)"
option :auto_install_gems, type: :boolean, default: false,
desc: "Auto installs gem dependencies of the profile or resource pack."
end
def self.supermarket_options

View file

@ -201,6 +201,12 @@ class Inspec::InspecCLI < Inspec::BaseCLI
vendor_deps(path, vendor_options)
profile = Inspec::Profile.for_target(path, o)
gem_deps = profile.metadata.gem_dependencies + \
profile.locked_dependencies.list.map { |_k, v| v.profile.metadata.gem_dependencies }.flatten
unless gem_deps.empty?
o[:logger].warn "Archiving a profile that contains gem dependencies, but InSpec cannot package gems with the profile! Please archive your ~/.inspec/gems directory separately."
end
result = profile.check
if result && !o[:ignore_errors] == false

View file

@ -0,0 +1,74 @@
# This class will install the gem depedencies for profiles etc.
# The basic things which is required to install dependencies is the gem path where gem needs to be installed
# and the list of gems needs to be installed.
require "rubygems/remote_fetcher"
require "forwardable" unless defined?(Forwardable)
module Inspec
class DependencyInstaller
extend Forwardable
attr_reader :gem_path, :requested_gems, :dependency_loader
def_delegator :dependency_loader, :inspec_gem_path
def initialize(gem_path = nil, requested_gems = [])
@dependency_loader = Inspec::DependencyLoader.new
@gem_path = gem_path || inspec_gem_path
@requested_gems = requested_gems
end
def install
requested_gems.each do |requested_gem|
version = requested_gem[:version].nil? ? "> 0" : requested_gem[:version]
install_from_remote_gems(requested_gem[:name], { version: version })
end
end
private
def install_from_remote_gems(requested_gem_name, opts)
version = opts[:version].split(",")
begin
gem_dependency = Gem::Dependency.new(requested_gem_name, version || "> 0")
# BestSet is rubygems.org API + indexing, APISet is for custom sources
sources = if opts[:source]
Gem::Resolver::APISet.new(URI.join(opts[:source] + "/api/v1/dependencies"))
else
Gem::Resolver::BestSet.new
end
install_gem_to_gems_dir(gem_dependency, [sources], opts[:update_mode])
rescue Gem::RemoteFetcher::FetchError => gem_ex
ex = Inspec::GemDependencyInstallError.new(gem_ex.message)
ex.gem_name = requested_gem_name
raise ex
rescue Gem::Requirement::BadRequirementError => gem_ex
ex = Inspec::GemDependencyInstallError.new(gem_ex.message)
ex.gem_name = requested_gem_name
raise "Unparseable gem dependency '#{version}' for '#{ex.gem_name}'"
end
end
def install_gem_to_gems_dir(gem_dependency, extra_request_sets = [], update_mode = false)
# Solve the dependency (that is, find a way to install the new gem and anything it needs)
request_set = Gem::RequestSet.new(gem_dependency)
begin
solution = request_set.resolve
rescue Gem::UnsatisfiableDependencyError => gem_ex
ex = Inspec::GemDependencyInstallError.new(gem_ex.message)
ex.gem_name = gem_dependency.name
raise ex
end
# OK, perform the installation.
# Ignore deps here, because any needed deps should already be baked into gem_dependency
request_set.install_into(gem_path, true, ignore_dependencies: true, document: [])
# Locate the GemVersion for the new dependency and return it
solution.detect { |g| g.name == gem_dependency.name }.version
end
end
end

View file

@ -0,0 +1,97 @@
# This class will load the gem depedencies for profiles etc.
# The basic things which is required to load gem dependencies is the path from which gems needs to be loaded
# and the list of gems needs to be loaded.
module Inspec
class DependencyLoader
attr_accessor :gem_path, :gem_list
# initializes the dependency_loader
def initialize(gem_path = nil, gem_list = [])
@gem_path = gem_path || inspec_gem_path
@gem_list = gem_list
end
def load
Gem.path << gem_path
Gem.refresh
gem_list.each do |gem_data|
version = gem_data[:version].nil? ? "> 0" : gem_data[:version]
activate_gem_dependency(gem_data[:name], version)
end
end
def inspec_gem_path
self.class.inspec_gem_path
end
def self.inspec_gem_path
require "rbconfig" unless defined?(RbConfig)
ruby_abi_version = RbConfig::CONFIG["ruby_version"]
# TODO: why are we installing under the api directory for plugins?
base_dir = Inspec.config_dir
base_dir = File.realpath base_dir if File.exist? base_dir
File.join(base_dir, "gems", ruby_abi_version)
end
# Lists all gems found in the inspec_gem_path.
# @return [Array[Gem::Specification]] Specs of all gems found.
def list_managed_gems
Dir.glob(File.join(gem_path, "specifications", "*.gemspec")).map { |p| Gem::Specification.load(p) }
end
def list_installed_gems
list_managed_gems
end
def gem_installed?(name)
list_installed_gems.any? { |spec| spec.name == name }
end
def gem_version_installed?(name, version)
list_installed_gems.any? { |s| s.name == name && Gem::Requirement.new(version.split(",")) =~ s.version }
end
private
def activate_gem_dependency(name, version_constraint = "> 0")
version_constraint = version_constraint.split(",")
gem_deps = [Gem::Dependency.new(name.to_s, version_constraint)]
managed_gem_set = Gem::Resolver::VendorSet.new
# Note: There is an issue in resolving gem dependency.
# This block resolves that issue partially.
# But this will still fail for the gems which don't have the .gemspec file.
# TODO: Find the solution to resolve gem dependencies that work for the unpackaged gems which don't have the .gemspec file.
list_managed_gems.each do |spec|
unless Dir["#{spec.gem_dir}/*.gemspec"].empty?
managed_gem_set.add_vendor_gem(spec.name, spec.gem_dir)
end
end
# TODO: Next two lines merge our managed gems with the other gems available
# in our "local universe" - which may be the system, or it could be in a Bundler microcosm,
# or rbenv, etc. Do we want to merge that, though?
distrib_gem_set = Gem::Resolver::CurrentSet.new
installed_gem_set = Gem::Resolver.compose_sets(managed_gem_set, distrib_gem_set)
# So, given what we need, and what we have available, what activations are needed?
resolver = Gem::Resolver.new(gem_deps, installed_gem_set)
begin
solution = resolver.resolve
rescue Gem::UnsatisfiableDependencyError => gem_ex
# If you broke your install, or downgraded to a plugin with a bad gemspec, you could get here.
ex = Inspec::GemDependencyLoadError.new(gem_ex.message)
raise ex
end
solution.each do |activation_request|
next if activation_request.full_spec.activated?
activation_request.full_spec.activate
# TODO: If we are under Bundler, inform it that we loaded a gem
end
end
end
end

View file

@ -15,4 +15,11 @@ module Inspec
class ConfigError::Invalid < ConfigError; end
class UserInteractionRequired < Error; end
class GemDependencyLoadError < Error; end
class GemDependencyInstallError < Error
attr_accessor :gem_name
attr_accessor :version
end
end

View file

@ -51,6 +51,10 @@ module Inspec
params[:depends] || []
end
def gem_dependencies
params[:gem_dependencies] || []
end
def supports(sth, version = nil)
# Ignore supports with metadata.rb. This file is legacy and the way it
# it handles `supports` deprecated. A deprecation warning will be printed
@ -109,6 +113,33 @@ module Inspec
warnings.push("License '#{params[:license]}' needs to be in SPDX format or marked as 'Proprietary'. See https://spdx.org/licenses/.")
end
# If gem_dependencies is set, it must be an array of hashes with keys name and optional version
unless params[:gem_dependencies].nil?
list = params[:gem_dependencies]
if list.is_a?(Array) && list.all? { |e| e.is_a? Hash }
list.each do |entry|
errors.push("gem_dependencies entries must all have a 'name' field") unless entry.key?(:name)
if entry[:version]
orig = entry[:version]
begin
# Split on commas as we may have a complex dep
orig.split(",").map { |c| Gem::Requirement.parse(c) }
rescue Gem::Requirement::BadRequirementError
errors.push "Unparseable gem dependency '#{orig}' for #{entry[:name]}"
rescue Inspec::GemDependencyInstallError => e
errors.push e.message
end
end
extra = (entry.keys - %i{name version})
unless extra.empty?
warnings.push "Unknown gem_dependencies key(s) #{extra.join(",")} seen for entry '#{entry[:name]}'"
end
end
else
errors.push("gem_dependencies must be a List of Hashes")
end
end
[errors, warnings]
end

View file

@ -13,6 +13,8 @@ require "inspec/dependencies/cache"
require "inspec/dependencies/lockfile"
require "inspec/dependencies/dependency_set"
require "inspec/utils/json_profile_summary"
require "inspec/dependency_loader"
require "inspec/dependency_installer"
module Inspec
class Profile
@ -378,6 +380,66 @@ module Inspec
@runner_context
end
def collect_gem_dependencies(profile_context)
gem_dependencies = []
all_profiles = []
profile_context.dependencies.list.values.each do |requirement|
all_profiles << requirement.profile
end
all_profiles << self
all_profiles.each do |profile|
gem_dependencies << profile.metadata.gem_dependencies unless profile.metadata.gem_dependencies.empty?
end
gem_dependencies.flatten.uniq
end
# Loads the required gems specified in the Profile's metadata file from default inspec gems path i.e. ~/.inspec/gems
# else installs and loads them.
def load_gem_dependencies
gem_dependencies = collect_gem_dependencies(load_libraries)
gem_dependencies.each do |gem_data|
dependency_loader = DependencyLoader.new
if dependency_loader.gem_version_installed?(gem_data[:name], gem_data[:version]) ||
dependency_loader.gem_installed?(gem_data[:name])
load_gem_dependency(gem_data)
else
if Inspec::Config.cached[:auto_install_gems]
install_gem_dependency(gem_data)
load_gem_dependency(gem_data)
else
ui = Inspec::UI.new
gem_dependencies.each { |gem_dependency| ui.list_item("#{gem_dependency[:name]} #{gem_dependency[:version]}") }
choice = ui.prompt.select("Would you like to install profile gem dependencies listed above?", %w{Yes No})
if choice == "Yes"
Inspec::Config.cached[:auto_install_gems] = true
load_gem_dependencies
else
ui.error "Unable to resolve above listed profile gem dependencies."
Inspec::UI.new.exit(:gem_dependency_load_error)
end
end
end
end
end
# Requires gem_data as argument.
# gem_dta example: { name: "gem_name", version: "0.0.1"}
def load_gem_dependency(gem_data)
dependency_loader = DependencyLoader.new(nil, [gem_data])
dependency_loader.load
rescue Inspec::GemDependencyLoadError => e
raise e.message
end
# Requires gem_data as argument.
# gem_dta example: { name: "gem_name", version: "0.0.1"}
def install_gem_dependency(gem_data)
gem_dependency = DependencyInstaller.new(nil, [gem_data])
gem_dependency.install
rescue Inspec::GemDependencyInstallError => e
raise e.message
end
def to_s
"Inspec::Profile<#{name}>"
end
@ -774,6 +836,7 @@ module Inspec
end
def load_checks_params(params)
load_gem_dependencies
load_libraries
tests = collect_tests
params[:controls] = controls = {}

View file

@ -105,6 +105,7 @@ module Inspec
write_lockfile(profile) if @create_lockfile
profile.locked_dependencies
profile.load_gem_dependencies
profile_context = profile.load_libraries
profile_context.dependencies.list.values.each do |requirement|

View file

@ -30,6 +30,7 @@ module Inspec
EXIT_USAGE_ERROR = 1
EXIT_PLUGIN_ERROR = 2
EXIT_FATAL_DEPRECATION = 3
EXIT_GEM_DEPENDENCY_LOAD_ERROR = 4
EXIT_LICENSE_NOT_ACCEPTED = 172
EXIT_FAILED_TESTS = 100
EXIT_SKIPPED_TESTS = 101

View file

@ -0,0 +1,3 @@
# Example InSpec Profile
This example shows the implementation of an InSpec profile.

View file

@ -0,0 +1,9 @@
# copyright: 2018, The Authors
# you add controls here
include_controls "profile-with-gem-dependency"
control "tmp-1.1" do # A unique ID for this control
describe file('/') do
it { should be_directory }
end
end

View file

@ -0,0 +1,13 @@
name: profile-with-dependent-gem-dependency
title: InSpec Profile
maintainer: The Authors
copyright: The Authors
copyright_email: you@example.com
license: Apache-2.0
summary: An InSpec Compliance Profile
version: 0.1.0
supports:
platform: os
depends:
- name: profile-with-gem-dependency
path: ../profile-with-gem-dependency

View file

@ -0,0 +1,11 @@
require "money"
control "tmp-1.0" do
Money.rounding_mode = BigDecimal::ROUND_HALF_UP
m = Money.from_cents(1000, "USD")
cents = m.cents
describe cents do
it { should eq 1000 }
end
end

View file

@ -0,0 +1,13 @@
name: profile-with-gem-dependency
title: InSpec Profile
maintainer: The Authors
copyright: The Authors
copyright_email: you@example.com
license: Apache-2.0
summary: An InSpec Compliance Profile
version: 0.1.0
supports:
platform: os
gem_dependencies:
- name: money
version: ">= 6.16.0"

View file

@ -0,0 +1,18 @@
# copyright: 2018, The Authors
title "sample section"
# you can also use plain tests
describe file("/tmp") do
it { should be_directory }
end
# you add controls here
control "tmp-1.0" do # A unique ID for this control
impact 0.7 # The criticality, if this control fails.
title "Create /tmp directory" # A human-readable title
desc "An optional description..."
describe file("/tmp") do # The actual test
it { should be_directory }
end
end

View file

@ -0,0 +1,13 @@
name: profile-with-illformed-gem-depedency
title: InSpec Profile
maintainer: The Authors
copyright: The Authors
copyright_email: you@example.com
license: Apache-2.0
summary: An InSpec Compliance Profile
version: 0.1.0
supports:
platform: os
gem_dependencies:
name: "mongo"
version: "+ 2.3.12"

View file

@ -0,0 +1,51 @@
require "functional/helper"
describe "profile with gem dependencies" do
include FunctionalHelper
let(:gem_dependency_profiles_path) { File.join(profile_path, "profile-with-gem-dependency") }
let(:config_dir_path) { File.expand_path "test/fixtures/config_dirs" }
let(:depdent_profile_gem_dependency) { File.join(profile_path, "profile-with-dependent-gem-dependency") }
let(:ruby_abi_version) { RbConfig::CONFIG["ruby_version"] }
let(:illformatted_gem_dependncy) { File.join(profile_path, "profile-with-illformed-gem-depedency") }
def reset_globals
ENV["HOME"] = Dir.home
end
before(:each) do
reset_globals
ENV["HOME"] = File.join(config_dir_path, "profile_gems")
end
after do
reset_globals
if config_dir_path
Dir.glob(File.join(config_dir_path, "profile_gems")).each do |path|
next if path.end_with? ".gitkeep"
FileUtils.rm_rf(path)
end
end
end
it "installs the gem dependencies and load them if --auto-install-gems is provided." do
out = inspec_with_env("exec #{gem_dependency_profiles_path} --no-create-lockfile --auto-install-gems")
_(out.stderr).must_equal ""
_(File.directory?(File.join(config_dir_path, "profile_gems", ".inspec/gems/#{ruby_abi_version}/gems"))).must_equal true
assert_exit_code 0, out
end
it "installs the gem dependencies in dendent profile and load them if --auto-install-gems is provided." do
out = inspec_with_env("exec #{depdent_profile_gem_dependency} --no-create-lockfile --auto-install-gems")
_(out.stderr).must_equal ""
_(File.directory?(File.join(config_dir_path, "profile_gems", ".inspec/gems/#{ruby_abi_version}/gems"))).must_equal true
assert_exit_code 0, out
end
it "raises error for illformated gem dependencies found in the meta data file" do
out = inspec_with_env("exec #{illformatted_gem_dependncy} --no-create-lockfile --auto-install-gems")
_(out.stderr).must_include "Unparseable gem dependency '[\"+ 2.3.12\"]' for 'mongo'"
assert_exit_code 1, out
end
end

View file

@ -0,0 +1,60 @@
require "helper"
require "inspec/dependency_loader"
describe "dependency_loader" do
let(:config_dir_path) { File.expand_path "test/fixtures/config_dirs" }
let(:gem_list) { [{ name: "inspec-test-fixture", version: "0.1.0" }] }
def reset_globals
ENV["HOME"] = Dir.home
end
before(:each) do
reset_globals
ENV["HOME"] = File.join(config_dir_path, "test-fixture-1-float/gems/2.7.0")
end
after(:each) do
reset_globals
end
let(:gem_path) { [ENV["HOME"]] }
let(:dependency_loader) { Inspec::DependencyLoader.new(gem_path, gem_list) }
describe "load" do
it "loads the gem dependency if already installed on the given gem path." do
result = dependency_loader.load
_(result).must_equal gem_list
end
it "raises error if the gem dependency not exist on the given gem path." do
dependency_loader.gem_list = [{ name: "test_gem", version: "0.0.1" }]
err = _ { dependency_loader.load }.must_raise Inspec::GemDependencyLoadError
_(err.message).must_equal "Unable to resolve dependency: user requested \'test_gem (= 0.0.1)\'"
end
end
describe "gem_installed?" do
it "returns the list of gems installed if gem already installed" do
result = dependency_loader.gem_installed?("inspec-test-fixture")
_(result).must_equal true
end
it "returns nil if specified gem is not already installed." do
result = dependency_loader.gem_installed?("test_gem")
_(result).must_equal false
end
end
describe "gem_version_installed?" do
it "returns the list of gems installed if gem with specified version is already installed" do
result = dependency_loader.gem_version_installed?("inspec-test-fixture", "0.1.0")
_(result).must_equal true
end
it "returns nil if gem with specified version is not already installed." do
result = dependency_loader.gem_version_installed?("test_gem", "0.0.1")
_(result).must_equal false
end
end
end

View file

@ -38,6 +38,9 @@ describe "metadata with supported operating systems" do
url: "https://artifactory.com/artifactory/example-repo-local/inspec/0.4.1.tar.gz"
username: <%= ENV['USERNAME'] %>
password: <%= ENV['API_KEY'] %>
gem_dependencies:
- name: "test"
version: "1.0.0"
EOF
ENV["USERNAME"] = "dummy_user"
ENV["API_KEY"] = "dummy_pass"
@ -48,6 +51,8 @@ EOF
_(res.params[:depends][0][:url]).must_equal "https://artifactory.com/artifactory/example-repo-local/inspec/0.4.1.tar.gz"
_(res.params[:depends][0][:username]).must_equal "dummy_user"
_(res.params[:depends][0][:password]).must_equal "dummy_pass"
_(res.params[:gem_dependencies][0][:name]).must_equal "test"
_(res.params[:gem_dependencies][0][:version]).must_equal "1.0.0"
end
it "finalizes a loaded metadata via Profile ID" do
@ -253,3 +258,112 @@ EOF
end
end
end
describe "metadata validation" do
let(:logger) { Minitest::Mock.new }
let(:empty_options) { {} }
let(:backend) { MockLoader.new(:ubuntu).backend }
def gem_dep_check(gem_deps)
data = <<~EOF
name: dummy
title: InSpec Profile
version: 0.1.0
maintainer: human@example.com
summary: A test profile
description: A test profile
copyright: The Authors
copyright_email: you@example.com
license: Apache-2.0
#{gem_deps}
EOF
md = Inspec::Metadata.from_yaml("mock", data, nil)
Inspec::Metadata.finalize(md, "mock", empty_options)
md.valid
end
it "validates a well-formed but versionless gem dep" do
data = <<~EOF
gem_dependencies:
- name: money
- name: ordinal_array
EOF
err, wrn = gem_dep_check(data)
_(err).must_be_empty
_(wrn).must_be_empty
end
it "validates a complex versioned gem dep" do
data = <<~EOF
gem_dependencies:
- name: money
version: "~>6.10, >= 5.0.0"
- name: ordinal_array
EOF
err, wrn = gem_dep_check(data)
_(err).must_be_empty
_(wrn).must_be_empty
end
it "invalidates a malformed gem_dependencies section that is not an array" do
data = <<~EOF
gem_dependencies:
name: "test"
version: "1.0.0"
EOF
err, wrn = gem_dep_check(data)
_(err.count).must_equal 1
_(err[0]).must_match(/gem_dependencies must be a List of Hashes/)
_(wrn).must_be_empty
end
it "invalidates a malformed gem_dependencies section that is not an array of hashes" do
data = <<~EOF
gem_dependencies:
- A
- B
- C
EOF
err, wrn = gem_dep_check(data)
_(err.count).must_equal 1
_(err[0]).must_match(/gem_dependencies must be a List of Hashes/)
_(wrn).must_be_empty
end
it "invalidates a malformed gem_dependencies section that is missing the name key" do
data = <<~EOF
gem_dependencies:
- potAto: potAHto
EOF
err, wrn = gem_dep_check(data)
_(err.count).must_equal 1
_(err[0]).must_match(/gem_dependencies entries must all have a 'name' field/)
_(wrn.count).must_equal 1
_(wrn[0]).must_match(/Unknown gem_dependencies key\(s\) potAto seen for entry ''/)
end
it "invalidates a malformed gem_dependencies section that has a malformed version constraint" do
data = <<~EOF
gem_dependencies:
- name: money
version: lots
EOF
err, wrn = gem_dep_check(data)
_(err.count).must_equal 1
_(err[0]).must_match(/Unparseable gem dependency 'lots' for money/)
_(wrn).must_be_empty
end
it "invalidates a malformed gem_dependencies section that has extra keys" do
data = <<~EOF
gem_dependencies:
- name: money
versi0n: " >= 0"
EOF
err, wrn = gem_dep_check(data)
_(wrn.count).must_equal 1
_(wrn[0]).must_match(/Unknown gem_dependencies key\(s\) versi0n seen for entry 'money'/)
_(err).must_be_empty
end
end