Fix for executing git profiles with default branch not just master

Signed-off-by: Nikita Mathur <nikita.mathur@chef.io>
This commit is contained in:
Nikita Mathur 2021-03-22 18:20:37 +05:30
parent 8c93d81df4
commit d673e840a3
5 changed files with 218 additions and 101 deletions

View file

@ -1,7 +1,7 @@
require "tmpdir" unless defined?(Dir.mktmpdir)
require "fileutils" unless defined?(FileUtils)
require "mixlib/shellout" unless defined?(Mixlib::ShellOut)
require "inspec/log"
require 'tmpdir' unless defined?(Dir.mktmpdir)
require 'fileutils' unless defined?(FileUtils)
require 'mixlib/shellout' unless defined?(Mixlib::ShellOut)
require 'inspec/log'
module Inspec::Fetcher
#
@ -24,12 +24,12 @@ module Inspec::Fetcher
# omnibus source for hints.
#
class Git < Inspec.fetcher(1)
name "git"
name 'git'
priority 200
def self.resolve(target, opts = {})
if target.is_a?(String)
new(target, opts) if target.start_with?("git@") || target.end_with?(".git")
new(target, opts) if target.start_with?('git@') || target.end_with?('.git')
elsif target.respond_to?(:has_key?) && target.key?(:git)
new(target[:git], opts.merge(target))
end
@ -62,7 +62,6 @@ module Inspec::Fetcher
def fetch(destination_path)
@repo_directory = destination_path # Might be the cache, or vendoring, or something else
FileUtils.mkdir_p(destination_path) unless Dir.exist?(destination_path)
if cloned?
checkout
else
@ -73,7 +72,7 @@ module Inspec::Fetcher
else
Inspec::Log.debug("Checkout of #{resolved_ref} successful. " \
"Moving checkout to #{destination_path}")
FileUtils.cp_r(working_dir + "/.", destination_path)
FileUtils.cp_r(working_dir + '/.', destination_path)
end
end
end
@ -99,7 +98,7 @@ module Inspec::Fetcher
def cache_key
return resolved_ref unless @relative_path
OpenSSL::Digest.hexdigest("SHA256", resolved_ref + @relative_path)
OpenSSL::Digest.hexdigest('SHA256', resolved_ref + @relative_path)
end
def archive_path
@ -113,7 +112,7 @@ module Inspec::Fetcher
end
def update_from_opts(opts)
%i{branch tag ref}.map { |opt_name| update_ivar_from_opt(opt_name, opts) }.any?
%i[branch tag ref].map { |opt_name| update_ivar_from_opt(opt_name, opts) }.any?
end
private
@ -126,18 +125,26 @@ module Inspec::Fetcher
elsif @tag
resolve_ref(@tag)
else
resolve_ref("master")
resolve_ref(default_ref)
end
end
def default_ref
shellout("git remote show #{@remote_url} | grep 'HEAD branch' | cut -d ':' -f 2").stdout&.strip
end
def resolve_ref(ref_name)
command_string = "git ls-remote \"#{@remote_url}\" \"#{ref_name}*\""
cmd = shellout(command_string)
raise(Inspec::FetcherFailure, "Profile git dependency failed for #{@remote_url} - error running '#{command_string}': #{cmd.stderr}") unless cmd.exitstatus == 0
unless cmd.exitstatus == 0
raise(Inspec::FetcherFailure,
"Profile git dependency failed for #{@remote_url} - error running '#{command_string}': #{cmd.stderr}")
end
ref = parse_ls_remote(cmd.stdout, ref_name)
unless ref
raise Inspec::FetcherFailure, "Profile git dependency failed - unable to resolve #{ref_name} to a specific git commit for #{@remote_url}"
raise Inspec::FetcherFailure,
"Profile git dependency failed - unable to resolve #{ref_name} to a specific git commit for #{@remote_url}"
end
ref
@ -176,7 +183,7 @@ module Inspec::Fetcher
end
def cloned?
File.directory?(File.join(@repo_directory, ".git"))
File.directory?(File.join(@repo_directory, '.git'))
end
def clone(dir = @repo_directory)
@ -195,7 +202,8 @@ module Inspec::Fetcher
cmd.error!
cmd.status
rescue Errno::ENOENT
raise Inspec::FetcherFailure, "Profile git dependency failed for #{@remote_url} - to use git sources, you must have git installed."
raise Inspec::FetcherFailure,
"Profile git dependency failed for #{@remote_url} - to use git sources, you must have git installed."
end
def shellout(cmd, opts = {})
@ -203,12 +211,12 @@ module Inspec::Fetcher
cmd = Mixlib::ShellOut.new(cmd, opts)
cmd.run_command
Inspec::Log.debug("External command: completed with exit status: #{cmd.exitstatus}")
Inspec::Log.debug("External command: STDOUT BEGIN")
Inspec::Log.debug('External command: STDOUT BEGIN')
Inspec::Log.debug(cmd.stdout)
Inspec::Log.debug("External command: STDOUT END")
Inspec::Log.debug("External command: STDERR BEGIN")
Inspec::Log.debug('External command: STDOUT END')
Inspec::Log.debug('External command: STDERR BEGIN')
Inspec::Log.debug(cmd.stderr)
Inspec::Log.debug("External command: STDERR END")
Inspec::Log.debug('External command: STDERR END')
cmd
end
end

View file

@ -0,0 +1 @@
include_controls("default-main")

View file

@ -0,0 +1,12 @@
name: git-profile-default-main
maintainer: InSpec Team
license: Apache-2.0
summary: An inspec test profile which executes remote git profile.
version: 0.1.0
supports:
platform: os
depends:
- name: default-main
git: https://github.com/inspec/inspec-test-profile-default-main
branch: main
version: 0.1.0

View file

@ -1,15 +1,34 @@
require "functional/helper"
require "fileutils"
require "tmpdir"
require 'functional/helper'
require 'fileutils'
require 'tmpdir'
describe "running profiles with git-based dependencies" do
describe 'running profiles with git-based dependencies' do
include FunctionalHelper
let(:git_profiles) { "#{profile_path}/git-fetcher" }
attr_accessor :out
def inspec(commandline, prefix = nil)
@stdout = @stderr = nil
self.out = super
end
def stdout
@stdout ||= out.stdout
.force_encoding(Encoding::UTF_8)
.gsub(/\e\[(\d+)(;\d+)*m/, "") # strip ANSI color codes
end
def stderr
@stderr ||= out.stderr
.force_encoding(Encoding::UTF_8)
.gsub(/\e\[(\d+)(;\d+)*m/, "") # strip ANSI color codes
end
#======================================================================#
# Git Repo Setup
#======================================================================#
fixture_repos = %w{basic-local git-repo-01}
fixture_repos = %w[basic-local git-repo-01]
before(:all) do
skip_windows! # Right now, this is due to symlinking, break executes on L24 <nickchecked>
@ -56,21 +75,21 @@ describe "running profiles with git-based dependencies" do
assert_json_controls_passing(run_result)
# Should know about the top-level profile and the child profile
assert_equal expected_profiles, (@json["profiles"].map { |p| p["name"] })
assert_equal expected_profiles, (@json['profiles'].map { |p| p['name'] })
controls = @json["profiles"].map { |p| p["controls"] }.flatten.map { |c| c["id"] }.uniq
controls = @json['profiles'].map { |p| p['controls'] }.flatten.map { |c| c['id'] }.uniq
# Should have controls from the top-level and included child profile
expected_controls.each { |control| assert_includes controls, control }
# should not have controls from the profile defined at the top of the repo of the child profile
refute_includes controls, "red-dye"
refute_includes controls, 'red-dye'
end
#======================================================================#
# Basic Git Fetching
#======================================================================#
describe "running a profile with a basic local dependency" do
it "should work on a local checkout" do
describe 'running a profile with a basic local dependency' do
it 'should work on a local checkout' do
run_result = run_inspec_process("exec #{git_profiles}/basic-local", json: true)
assert_empty run_result.stderr
assert_json_controls_passing(run_result)
@ -90,24 +109,24 @@ describe "running profiles with git-based dependencies" do
#======================================================================#
#------------ Happy Cases for Relative Path Support -------------------#
describe "running a profile with a shallow relative path dependency" do
it "should find the relative path profile and execute exactly those controls" do
assert_relative_fetch_works("relative-shallow", %w{relative-shallow child-01}, %w{top-level-01 child-01})
describe 'running a profile with a shallow relative path dependency' do
it 'should find the relative path profile and execute exactly those controls' do
assert_relative_fetch_works('relative-shallow', %w[relative-shallow child-01], %w[top-level-01 child-01])
end
end
describe "running a profile with a deep relative path dependency" do
it "should find the relative path profile and execute exactly those controls" do
assert_relative_fetch_works("relative-deep", %w{relative-deep child-02}, %w{relative-deep-01 child-02})
describe 'running a profile with a deep relative path dependency' do
it 'should find the relative path profile and execute exactly those controls' do
assert_relative_fetch_works('relative-deep', %w[relative-deep child-02], %w[relative-deep-01 child-02])
end
end
describe "running a profile with a combination of relative path dependencies" do
it "should find the relative path profiles and execute exactly those controls" do
describe 'running a profile with a combination of relative path dependencies' do
it 'should find the relative path profiles and execute exactly those controls' do
assert_relative_fetch_works(
"relative-combo",
%w{relative-combo child-01 child-02},
%w{relative-combo-01 child-01 child-02}
'relative-combo',
%w[relative-combo child-01 child-02],
%w[relative-combo-01 child-01 child-02]
)
end
end
@ -115,30 +134,45 @@ describe "running profiles with git-based dependencies" do
#------------ Edge Cases for Relative Path Support -------------------#
describe "running a profile with an '' relative path dependency" do
it "should find the top-level profile in the git-referenced child profile and execute that" do
assert_relative_fetch_works("relative-empty", %w{relative-empty basic-local}, %w{relative-empty-01 basic-local-01})
it 'should find the top-level profile in the git-referenced child profile and execute that' do
assert_relative_fetch_works('relative-empty', %w[relative-empty basic-local],
%w[relative-empty-01 basic-local-01])
end
end
describe "running a profile with an ./ relative path dependency" do
it "should find the top-level profile in the git-referenced child profile and execute that" do
assert_relative_fetch_works("relative-dot-slash", %w{relative-dot-slash basic-local}, %w{relative-dot-slash-01 basic-local-01})
describe 'running a profile with an ./ relative path dependency' do
it 'should find the top-level profile in the git-referenced child profile and execute that' do
assert_relative_fetch_works('relative-dot-slash', %w[relative-dot-slash basic-local],
%w[relative-dot-slash-01 basic-local-01])
end
end
describe "running a profile with a relative path dependency that does not exist" do
it "should fail gracefully" do
describe 'running a profile with a relative path dependency that does not exist' do
it 'should fail gracefully' do
run_result = run_inspec_process("exec #{git_profiles}/relative-nonesuch")
assert_empty run_result.stdout
refute_includes run_result.stderr, "Errno::ENOENT" # No ugly file missing error
refute_includes run_result.stderr, 'Errno::ENOENT' # No ugly file missing error
assert_equal 1, run_result.stderr.lines.count # Not a giant stacktrace
# Spot check important parts of the message
assert_includes run_result.stderr, "Cannot find relative path"
assert_includes run_result.stderr, "no/such/path" # the actual missing path
assert_includes run_result.stderr, "profile in git repo"
assert_includes run_result.stderr, 'Cannot find relative path'
assert_includes run_result.stderr, 'no/such/path' # the actual missing path
assert_includes run_result.stderr, 'profile in git repo'
# The containing git repo (the only identifier the user will have)
assert_includes run_result.stderr, "test/fixtures/profiles/git-fetcher/git-repo-01"
assert_includes run_result.stderr, 'test/fixtures/profiles/git-fetcher/git-repo-01'
assert_exit_code(1, run_result) # General user error
end
end
#------------ Happy Case for default branch GIT fetching -------------------#
describe 'running a remote GIT profile' do
it 'should use default HEAD branch' do
inspec("exec #{git_profiles}/git-repo-default-main")
assert_empty stderr
assert_includes stdout, 'Profile: InSpec Profile (default-main)'
assert_includes stdout, "Profile Summary: 1 successful control, 0 control failures, 0 controls skipped\n"
assert_includes stdout, "Test Summary: 2 successful, 0 failures, 0 skipped\n"
assert_exit_code 0, out
end
end
end

View file

@ -1,42 +1,42 @@
require "helper"
require "inspec/fetcher"
require 'helper'
require 'inspec/fetcher'
describe Inspec::Fetcher::Git do
let(:fetcher) { Inspec::Fetcher::Git }
it "registers with the fetchers registry" do
it 'registers with the fetchers registry' do
reg = Inspec::Fetcher::Registry.registry
_(reg["git"]).must_equal fetcher
_(reg['git']).must_equal fetcher
end
it "handles sources beginning with `git@`" do
f = fetcher.resolve("git@github.com:foo/bar")
it 'handles sources beginning with `git@`' do
f = fetcher.resolve('git@github.com:foo/bar')
_(f).wont_be_nil
_(f).must_be_kind_of Inspec::Fetcher::Git
end
it "handles sources ending with `.git`" do
f = fetcher.resolve("https://github.com/foo/bar.git")
it 'handles sources ending with `.git`' do
f = fetcher.resolve('https://github.com/foo/bar.git')
_(f).wont_be_nil
_(f).must_be_kind_of Inspec::Fetcher::Git
end
it "handles sources specified by a :git key" do
f = fetcher.resolve({ git: "https://example.com/foo.gi" })
it 'handles sources specified by a :git key' do
f = fetcher.resolve({ git: 'https://example.com/foo.gi' })
_(f).wont_be_nil
_(f).must_be_kind_of Inspec::Fetcher::Git
end
describe "when given a valid repository" do
let(:git_dep_dir) { "test-directory" }
let(:git_master_ref) { "bf4d5774f02d24155bfc34b5897d22785a304cfa" }
let(:git_branch_ref) { "b979579e5fc8edb72511fe5d2a1230dede71eff7" }
let(:git_tag_ref) { "efc85d89ee9d5798ca93ee95db0c711b99061590" }
describe 'when given a valid repository' do
let(:git_dep_dir) { 'test-directory' }
let(:git_master_ref) { 'bf4d5774f02d24155bfc34b5897d22785a304cfa' }
let(:git_branch_ref) { 'b979579e5fc8edb72511fe5d2a1230dede71eff7' }
let(:git_tag_ref) { 'efc85d89ee9d5798ca93ee95db0c711b99061590' }
let(:git_output) do
out = mock
out.stubs(:stdout).returns("")
out.stubs(:stdout).returns('')
out.stubs(:exitstatus).returns(0)
out.stubs(:stderr).returns("")
out.stubs(:stderr).returns('')
out.stubs(:status).returns(true)
out.stubs(:error!).returns(false)
out.stubs(:run_command).returns(true)
@ -54,7 +54,17 @@ efc85d89ee9d5798ca93ee95db0c711b99061590\trefs/tags/antag^{}
be002c56b0806ea40aabf7a2b742c41182336198\trefs/tags/anothertag
a7729ce65636d6d8b80159dd5dd7a40fdb6f2501\trefs/tags/anothertag^{}\n")
out.stubs(:exitstatus).returns(0)
out.stubs(:stderr).returns("")
out.stubs(:stderr).returns('')
out.stubs(:error!).returns(false)
out.stubs(:run_command).returns(true)
out
end
let(:git_remote_head_master) do
out = mock
out.stubs(:stdout).returns("master\n")
out.stubs(:exitstatus).returns(0)
out.stubs(:stderr).returns('')
out.stubs(:error!).returns(false)
out.stubs(:run_command).returns(true)
out
@ -62,74 +72,126 @@ a7729ce65636d6d8b80159dd5dd7a40fdb6f2501\trefs/tags/anothertag^{}\n")
before do
# git fetcher likes to make directories, let's stub that for every test
Dir.stubs(:mktmpdir).yields("test-tmp-dir")
File.stubs(:directory?).with("fetchpath/.git").returns(false)
Dir.stubs(:mktmpdir).yields('test-tmp-dir')
File.stubs(:directory?).with('fetchpath/.git').returns(false)
FileUtils.stubs(:mkdir_p)
end
def expect_ls_remote(ref)
Mixlib::ShellOut.expects(:new).with("git ls-remote \"#{git_dep_dir}\" \"#{ref}*\"", {}).returns(git_ls_remote_output)
def expect_git_remote_head_master(remote_url)
Mixlib::ShellOut.expects(:new).with("git remote show #{remote_url} | grep 'HEAD branch' | cut -d ':' -f 2",
{}).returns(git_remote_head_master)
end
def expect_checkout(ref, at = "test-tmp-dir")
def expect_ls_remote(ref)
Mixlib::ShellOut.expects(:new).with("git ls-remote \"#{git_dep_dir}\" \"#{ref}*\"",
{}).returns(git_ls_remote_output)
end
def expect_checkout(ref, at = 'test-tmp-dir')
Mixlib::ShellOut.expects(:new).with("git checkout #{ref}", { cwd: at }).returns(git_output)
end
def expect_clone
Mixlib::ShellOut.expects(:new).with("git clone #{git_dep_dir} ./", { cwd: "test-tmp-dir" }).returns(git_output)
Mixlib::ShellOut.expects(:new).with("git clone #{git_dep_dir} ./", { cwd: 'test-tmp-dir' }).returns(git_output)
end
def expect_mv_into_place
FileUtils.expects(:cp_r).with("test-tmp-dir/.", "fetchpath")
FileUtils.expects(:cp_r).with('test-tmp-dir/.', 'fetchpath')
end
it "resolves to the revision of master by default" do
expect_ls_remote("master")
it 'resolves to the revision of master when head branch master' do
expect_git_remote_head_master(git_dep_dir)
expect_ls_remote('master')
result = fetcher.resolve({ git: git_dep_dir })
_(result.resolved_source).must_equal({ git: git_dep_dir, ref: git_master_ref })
end
it "can resolve a tag" do
expect_ls_remote("antag")
result = fetcher.resolve({ git: git_dep_dir, tag: "antag" })
it 'can resolve a tag' do
expect_ls_remote('antag')
result = fetcher.resolve({ git: git_dep_dir, tag: 'antag' })
_(result.resolved_source).must_equal({ git: git_dep_dir, ref: git_tag_ref })
end
it "can resolve a branch" do
expect_ls_remote("somebranch")
result = fetcher.resolve({ git: git_dep_dir, branch: "somebranch" })
it 'can resolve a branch' do
expect_ls_remote('somebranch')
result = fetcher.resolve({ git: git_dep_dir, branch: 'somebranch' })
_(result.resolved_source).must_equal({ git: git_dep_dir, ref: git_branch_ref })
end
it "assumes the ref you gave it is the thing you want" do
result = fetcher.resolve({ git: git_dep_dir, ref: "a_test_ref" })
_(result.resolved_source).must_equal({ git: git_dep_dir, ref: "a_test_ref" })
it 'assumes the ref you gave it is the thing you want' do
result = fetcher.resolve({ git: git_dep_dir, ref: 'a_test_ref' })
_(result.resolved_source).must_equal({ git: git_dep_dir, ref: 'a_test_ref' })
end
it "fetches to the given location" do
expect_ls_remote("master")
it 'fetches to the given location' do
expect_git_remote_head_master(git_dep_dir)
expect_ls_remote('master')
expect_clone
expect_checkout(git_master_ref)
expect_mv_into_place
result = fetcher.resolve({ git: git_dep_dir })
result.fetch("fetchpath")
result.fetch('fetchpath')
end
it "doesn't refetch an already cloned repo" do
File.expects(:directory?).with("fetchpath/.git").at_least_once.returns(true)
expect_ls_remote("master")
expect_checkout(git_master_ref, "fetchpath")
File.expects(:directory?).with('fetchpath/.git').at_least_once.returns(true)
expect_git_remote_head_master(git_dep_dir)
expect_ls_remote('master')
expect_checkout(git_master_ref, 'fetchpath')
result = fetcher.resolve({ git: git_dep_dir })
result.fetch("fetchpath")
result.fetch('fetchpath')
end
it "returns the repo_path that we fetched to as the archive_path" do
File.expects(:directory?).with("fetchpath/.git").at_least_once.returns(true)
expect_ls_remote("master")
expect_checkout(git_master_ref, "fetchpath")
it 'returns the repo_path that we fetched to as the archive_path' do
File.expects(:directory?).with('fetchpath/.git').at_least_once.returns(true)
expect_git_remote_head_master(git_dep_dir)
expect_ls_remote('master')
expect_checkout(git_master_ref, 'fetchpath')
result = fetcher.resolve({ git: git_dep_dir })
result.fetch("fetchpath")
_(result.archive_path).must_equal "fetchpath"
result.fetch('fetchpath')
_(result.archive_path).must_equal 'fetchpath'
end
end
describe 'when given a repository with default branch main' do
let(:git_default_main) { 'inspec-test-profile-default-main' }
let(:git_main_ref) { '69220a2ba6a3b276184f328e69a953e83e283323' }
let(:git_remote_head_main) do
out = mock
out.stubs(:stdout).returns("main\n")
out.stubs(:exitstatus).returns(0)
out.stubs(:stderr).returns('')
out.stubs(:error!).returns(false)
out.stubs(:run_command).returns(true)
out
end
let(:git_ls_remote_output_for_main) do
out = mock
out.stubs(:stdout).returns("69220a2ba6a3b276184f328e69a953e83e283323\trefs/heads/main")
out.stubs(:exitstatus).returns(0)
out.stubs(:stderr).returns('')
out.stubs(:error!).returns(false)
out.stubs(:run_command).returns(true)
out
end
def expect_git_remote_head_main(remote_url)
Mixlib::ShellOut.expects(:new).with("git remote show #{remote_url} | grep 'HEAD branch' | cut -d ':' -f 2",
{}).returns(git_remote_head_main)
end
def expect_ls_remote(ref)
Mixlib::ShellOut.expects(:new).with("git ls-remote \"#{git_default_main}\" \"#{ref}*\"",
{}).returns(git_ls_remote_output_for_main)
end
it 'resolves to the revision of main when head branch main' do
expect_git_remote_head_main(git_default_main)
expect_ls_remote('main')
result = fetcher.resolve({ git: git_default_main })
_(result.resolved_source).must_equal({ git: git_default_main, ref: git_main_ref })
end
end
end