Relative Path Support for the Git Fetcher (#4217)

Relative Path Support for the Git Fetcher
This commit is contained in:
Clinton Wolfe 2019-07-30 14:37:54 -04:00 committed by GitHub
commit 100e96db1c
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
74 changed files with 420 additions and 65 deletions

View file

@ -8,6 +8,9 @@ checks:
identical-code:
config:
threshold: 40 # Unfortunately, we have a lot of duplicate code in places like lib/inspec/control_eval_context.rb
method-complexity: # 'Cognitive Complexity' in the UI
config:
threshold: 10 # 5 Default 5 is really tight
plugins:
fixme:
enabled: true

View file

@ -212,9 +212,7 @@ depends:
### git
A `git` setting specifies a profile that is located in a git repository, with optional settings for branch, tag, commit, and version. The source location is translated into a URL upon resolution. This type of dependency supports version constraints via semantic versioning as git tags.
For example:
A `git` setting specifies a profile that is located in a git repository, with optional settings for branch, tag, commit, version, and relative_path. The source location is translated into a URL upon resolution. This type of dependency supports version constraints via semantic versioning as git tags.
```YAML
depends:
@ -224,6 +222,7 @@ depends:
tag: desired_version
commit: pinned_commit
version: semver_via_tags
relative_path: relative/optional/path/to/profile
```
### supermarket

View file

@ -39,28 +39,65 @@ module Fetchers
@branch = opts[:branch]
@tag = opts[:tag]
@ref = opts[:ref]
@remote_url = remote_url
@remote_url = expand_local_path(remote_url)
@repo_directory = nil
@relative_path = opts[:relative_path] if opts[:relative_path] && !opts[:relative_path].empty?
end
def fetch(dir)
@repo_directory = dir
FileUtils.mkdir_p(dir) unless Dir.exist?(dir)
def expand_local_path(url_or_file_path)
# This paths to local on-disk repos, not relative paths within repos.
# This is especially needed with testing.
# We could try to do something clever with URI
# processing, but then again, if you passed a relative path
# to an on-disk repo, you probably expect it to exist.
return url_or_file_path unless File.exist?(url_or_file_path)
# It's important to expand this path, because it may be specified
# locally in the metadata files, and when we clone, we will be
# in a temp dir.
File.expand_path(url_or_file_path)
end
def fetch(destination_path)
@repo_directory = destination_path # Might be the cache, or vendoring, or something else
FileUtils.mkdir_p(destination_path) unless Dir.exist?(destination_path)
if cloned?
checkout
else
Dir.mktmpdir do |tmpdir|
checkout(tmpdir)
Inspec::Log.debug("Checkout of #{resolved_ref} successful. Moving checkout to #{dir}")
FileUtils.cp_r(tmpdir + "/.", @repo_directory)
Dir.mktmpdir do |working_dir|
checkout(working_dir)
if @relative_path
perform_relative_path_fetch(destination_path, working_dir)
else
Inspec::Log.debug("Checkout of #{resolved_ref} successful. " \
"Moving checkout to #{destination_path}")
FileUtils.cp_r(working_dir + "/.", destination_path)
end
end
end
@repo_directory
end
def perform_relative_path_fetch(destination_path, working_dir)
Inspec::Log.debug("Checkout of #{resolved_ref} successful. " \
"Moving #{@relative_path} to #{destination_path}")
unless File.exist?("#{working_dir}/#{@relative_path}")
# Cleanup the destination path - otherwise we'll have an empty dir
# in the cache, which is enough to confuse the cache reader
# This is a courtesy, assuming we're writing to the cache; if we're
# vendoring to something more complex, don't bother.
FileUtils.rmdir(destination_path) if Dir.empty?(destination_path)
raise ArgumentError, "Cannot find relative path '#{@relative_path}' " \
"within profile in git repo specified by '#{@remote_url}'"
end
FileUtils.cp_r("#{working_dir}/#{@relative_path}", destination_path)
end
def cache_key
resolved_ref
return resolved_ref unless @relative_path
OpenSSL::Digest::SHA256.hexdigest(resolved_ref + @relative_path)
end
def archive_path
@ -68,7 +105,9 @@ module Fetchers
end
def resolved_source
{ git: @remote_url, ref: resolved_ref }
source = { git: @remote_url, ref: resolved_ref }
source[:relative_path] = @relative_path if @relative_path
source
end
private

View file

@ -0,0 +1,129 @@
require "functional/helper"
require "fileutils"
require "tmpdir"
describe "running profiles with git-based dependencies" do
include FunctionalHelper
let(:git_profiles) { "#{profile_path}/git-fetcher" }
#======================================================================#
# Git Repo Setup
#======================================================================#
fixture_repos = %w{basic-local git-repo-01}
before(:all) do
skip_windows! # Right now, this is due to symlinking
# We need a git repo for some of the profile test fixtures,
# but we can't store those directly in git.
# Here, one approach is to store the .git/ directory under a
# different name and then symlink to its proper name.
fixture_repos.each do |profile_name|
link_src = "#{git_profiles}/#{profile_name}/git-fixture"
link_dst = "#{git_profiles}/#{profile_name}/.git"
FileUtils.ln_sf(link_src, link_dst) # -f to tolerate existing links created during manual testing
end
end
after(:all) do
fixture_repos.each do |profile_name|
link = "#{git_profiles}/#{profile_name}/.git"
FileUtils.rm(link)
end
end
#======================================================================#
# Custom Local Assertions
#======================================================================#
def assert_relative_fetch_works(profile_name, expected_profiles, expected_controls)
run_result = run_inspec_process("exec #{git_profiles}/#{profile_name}", json: true)
assert_empty run_result.stderr
run_result.must_have_all_controls_passing
# Should know about the top-level profile and the child profile
assert_equal expected_profiles, (run_result.payload.json["profiles"].map { |p| p["name"] })
controls = run_result.payload.json["profiles"].map { |p| p["controls"] }.flatten.map { |c| c["id"] }.uniq
# Should have controls from the top-level and included child profile
expected_controls.each { |control| assert_includes controls, control }
# should not have controls from the profile defined at the top of the repo of the child profile
refute_includes controls, "red-dye"
end
#======================================================================#
# Basic Git Fetching
#======================================================================#
describe "running a profile with a basic local dependency" do
it "should work on a local checkout" do
run_result = run_inspec_process("exec #{git_profiles}/basic-local", json: true)
assert_empty run_result.stderr
run_result.must_have_all_controls_passing
end
end
# describe "running a profile with a basic remote dependency"
# TODO: move private SSH+git test from inspec_exec_test to here
#======================================================================#
# Revision Selection
#======================================================================#
# NOTE: test branch, rev, and tag capabilities are (lighty) tested in unit tests
#======================================================================#
# Relative Path Support
#======================================================================#
#------------ Happy Cases for Relative Path Support -------------------#
describe "running a profile with a shallow relative path dependency" do
it "should find the relative path profile and execute exactly those controls" do
assert_relative_fetch_works("relative-shallow", %w{relative-shallow child-01}, %w{top-level-01 child-01})
end
end
describe "running a profile with a deep relative path dependency" do
it "should find the relative path profile and execute exactly those controls" do
assert_relative_fetch_works("relative-deep", %w{relative-deep child-02}, %w{relative-deep-01 child-02})
end
end
describe "running a profile with a combination of relative path dependencies" do
it "should find the relative path profiles and execute exactly those controls" do
assert_relative_fetch_works(
"relative-combo",
%w{relative-combo child-01 child-02},
%w{relative-combo-01 child-01 child-02}
)
end
end
#------------ Edge Cases for Relative Path Support -------------------#
describe "running a profile with an '' relative path dependency" do
it "should find the top-level profile in the git-referenced child profile and execute that" do
assert_relative_fetch_works("relative-empty", %w{relative-empty basic-local}, %w{relative-empty-01 basic-local-01})
end
end
describe "running a profile with an ./ relative path dependency" do
it "should find the top-level profile in the git-referenced child profile and execute that" do
assert_relative_fetch_works("relative-dot-slash", %w{relative-dot-slash basic-local}, %w{relative-dot-slash-01 basic-local-01})
end
end
describe "running a profile with a relative path dependency that does not exist" do
it "should fail gracefully" do
run_result = run_inspec_process("exec #{git_profiles}/relative-nonesuch")
assert_empty run_result.stdout
refute_includes run_result.stderr, "Errno::ENOENT" # No ugly file missing error
assert_equal 1, run_result.stderr.lines.count # Not a giant stacktrace
# Spot check important parts of the message
assert_includes run_result.stderr, "Cannot find relative path"
assert_includes run_result.stderr, "no/such/path" # the actual missing path
assert_includes run_result.stderr, "profile in git repo"
# The containing git repo (the only identifier the user will have)
assert_includes run_result.stderr, "test/unit/mock/profiles/git-fetcher/git-repo-01"
assert_exit_code(1, run_result) # General user error
end
end
end

View file

@ -1,51 +0,0 @@
require "functional/helper"
require "fileutils"
require "tmpdir"
require "yaml"
describe "profiles with git-based dependencies" do
include FunctionalHelper
before(:all) do
@tmpdir = Dir.mktmpdir
@profile_dir = File.join(@tmpdir, "test-profile")
@git_dep_dir = File.join(@tmpdir, "git-dep")
Dir.chdir(@tmpdir) do
inspec("init profile git-dep")
inspec("init profile test-profile")
end
Dir.chdir(@git_dep_dir) do
CMD.run_command("git init")
CMD.run_command("git add .")
CMD.run_command("git config user.name \"test\"")
CMD.run_command("git config user.email \"test@yahoo.com\"")
CMD.run_command("git commit -m \"initial commit\" --no-gpg-sign")
CMD.run_command("git commit -m \"another commit\" --allow-empty --no-gpg-sign")
CMD.run_command("git tag antag")
end
inspec_yml = YAML.load(File.read(File.join(@profile_dir, "inspec.yml")))
inspec_yml["depends"] = [
{
"name" => "git-dep",
"git" => @git_dep_dir,
"tag" => "antag",
},
]
File.write(File.join(@profile_dir, "inspec.yml"), YAML.dump(inspec_yml))
end
after(:all) do
FileUtils.rm_rf(@tmpdir)
end
it "executes a profile with a git based dependency" do
out = inspec("exec #{@profile_dir} --no-create-lockfile")
skip_windows!
out.stderr.must_equal ""
assert_exit_code 0, out
end
end

View file

@ -69,7 +69,7 @@ describe "example inheritance profile" do
end
it "can vendor profile dependencies from git" do
git_depends_path = File.join(profile_path, "git-depends")
git_depends_path = File.join(profile_path, "git-fetcher", "basic")
Dir.mktmpdir do |tmpdir|
FileUtils.cp_r(git_depends_path + "/.", tmpdir)

View file

@ -0,0 +1 @@
This is a git repo used as a test fixture. Because we cannot directly store the .git/ directory, it is stored as git-fixture/, and a symlink is created at test runtime.

View file

@ -0,0 +1,5 @@
control 'basic-local-01' do
describe 'always-pass' do
it { should cmp 'always-pass'}
end
end

View file

@ -0,0 +1 @@
Add change only on one branch

View file

@ -0,0 +1 @@
ref: refs/heads/master

View file

@ -0,0 +1,10 @@
[core]
repositoryformatversion = 0
filemode = true
bare = false
logallrefupdates = true
ignorecase = true
precomposeunicode = true
[user]
name = test user
email = test@test.org

View file

@ -0,0 +1 @@
Unnamed repository; edit this file 'description' to name the repository.

View file

@ -0,0 +1,6 @@
# git ls-files --others --exclude-from=.git/info/exclude
# Lines that start with '#' are comments.
# For a project mostly in C, the following would be a good set of
# exclude patterns (uncomment them if you want to use them):
# *.[oa]
# *~

View file

@ -0,0 +1,4 @@
0000000000000000000000000000000000000000 0e7d2b9c2c5a1372341e36febceab86558439149 test user <test@test.org> 1560485536 -0400 commit (initial): Initial commit
0e7d2b9c2c5a1372341e36febceab86558439149 0e7d2b9c2c5a1372341e36febceab86558439149 test user <test@test.org> 1560485563 -0400 checkout: moving from master to test-branch
0e7d2b9c2c5a1372341e36febceab86558439149 54d0671d3e2c4a28865a0ecc98863859bd4d7475 test user <test@test.org> 1560485674 -0400 commit: Add change only on one branch
54d0671d3e2c4a28865a0ecc98863859bd4d7475 0e7d2b9c2c5a1372341e36febceab86558439149 test user <test@test.org> 1560485682 -0400 checkout: moving from test-branch to master

View file

@ -0,0 +1 @@
0000000000000000000000000000000000000000 0e7d2b9c2c5a1372341e36febceab86558439149 test user <test@test.org> 1560485536 -0400 commit (initial): Initial commit

View file

@ -0,0 +1,2 @@
0000000000000000000000000000000000000000 0e7d2b9c2c5a1372341e36febceab86558439149 test user <test@test.org> 1560485563 -0400 branch: Created from HEAD
0e7d2b9c2c5a1372341e36febceab86558439149 54d0671d3e2c4a28865a0ecc98863859bd4d7475 test user <test@test.org> 1560485674 -0400 commit: Add change only on one branch

View file

@ -0,0 +1,4 @@
x­<>±
Ã0 D;û+nó$h²ô{$Ùƒj+¥„Òo
]
;Ü-÷Ž'æyºœÔû6Ü<36>…£)™+<1B>§Œâ (5t4©ÈlwÞƒVŽøl@Ûð@,~³½®ßÐó¸×^Ò;é·e&ï¶E2¸ëòWé kI

View file

@ -0,0 +1,2 @@
xU±
€0 <05>ûoëT¨Ÿ”¦ ÑHSÿÝ.7Ý]M˜cœX·ÞTàYå Ê$!ÎY<1D>q«©À“œtYØÉìs@í¸aÁëþ<C3AB>ž±—-»Á {¥#

View file

@ -0,0 +1,4 @@
x<>Q
Â0DýÎ)öJ²É&)ˆèQͶl+iúáím½<6D>0<æcfx™¦gÄxjUrÈŽ°3¥I,Qo°”XÏ\<£³A½S•¹<E280A2>P0wŒLÉØ€Ö±¾—ÌrôDÑÙθ­<C2A5>K…&kƒm•
×ÃÞ\:ÜÀ<C39C>×.ÎÚi­ø7­íÑ?JêQ
ð˜æA`™_Ÿ»ö_5Í<ª/{fHJ

View file

@ -0,0 +1 @@
0e7d2b9c2c5a1372341e36febceab86558439149

View file

@ -0,0 +1 @@
54d0671d3e2c4a28865a0ecc98863859bd4d7475

View file

@ -0,0 +1 @@
0e7d2b9c2c5a1372341e36febceab86558439149

View file

@ -0,0 +1,8 @@
name: basic-local
title: basic-local
license: Apache-2.0
summary: A profile to be executed as a local git checkout
version: 0.1.0
supports:
platform: os

View file

@ -0,0 +1 @@
This is a git repo used as a test fixture. Because we cannot directly store the .git/ directory, it is stored as git-fixture/, and a symlink is created at test runtime.

View file

@ -0,0 +1,5 @@
control 'child-01' do
describe 'always-pass' do
it { should cmp 'always-pass'}
end
end

View file

@ -0,0 +1,8 @@
name: child-01
title: child-01
license: Apache-2.0
summary: A profile to be included via git and a relative path
version: 0.1.0
supports:
platform: os

View file

@ -0,0 +1,11 @@
# This control would only appear if it were accidentally included
control 'red-dye' do
# In rural areas where diesel fuel is sold for
# agricultrual purposes and is exempt from taxation (as part
# of farm subsidies), tractor diesel fuel is dyed red. Highway
# patrol officers check for red dye in the fuel when they
# stop grain hauling trucks, as it is a form of tax fraud.
describe 'truck fuel color' do
it { should cmp 'clear' }
end
end

View file

@ -0,0 +1,5 @@
control 'child-02' do
describe 'always-pass' do
it { should cmp 'always-pass'}
end
end

View file

@ -0,0 +1,8 @@
name: child-02
title: child-02
license: Apache-2.0
summary: A profile to be included via git and a deep relative path
version: 0.1.0
supports:
platform: os

View file

@ -0,0 +1 @@
Add child-02

View file

@ -0,0 +1 @@
ref: refs/heads/master

View file

@ -0,0 +1,10 @@
[core]
repositoryformatversion = 0
filemode = true
bare = false
logallrefupdates = true
ignorecase = true
precomposeunicode = true
[user]
name = InSpec Test Fixture
email = inspec@chef.io

View file

@ -0,0 +1 @@
Unnamed repository; edit this file 'description' to name the repository.

View file

@ -0,0 +1,6 @@
# git ls-files --others --exclude-from=.git/info/exclude
# Lines that start with '#' are comments.
# For a project mostly in C, the following would be a good set of
# exclude patterns (uncomment them if you want to use them):
# *.[oa]
# *~

View file

@ -0,0 +1,2 @@
0000000000000000000000000000000000000000 65dcdc3491c0c138538379f404f2517d968282a7 InSpec Test Fixture <inspec@chef.io> 1554036073 -0400 commit (initial): Add child-01 profile
65dcdc3491c0c138538379f404f2517d968282a7 36f4c4ecd2b870c0d0869c14a3e41dd506e5a887 InSpec Test Fixture <inspec@chef.io> 1554036109 -0400 commit: Add child-02

View file

@ -0,0 +1,2 @@
0000000000000000000000000000000000000000 65dcdc3491c0c138538379f404f2517d968282a7 InSpec Test Fixture <inspec@chef.io> 1554036073 -0400 commit (initial): Add child-01 profile
65dcdc3491c0c138538379f404f2517d968282a7 36f4c4ecd2b870c0d0869c14a3e41dd506e5a887 InSpec Test Fixture <inspec@chef.io> 1554036109 -0400 commit: Add child-02

View file

@ -0,0 +1,3 @@
x²▌K
1 @]В╧─▓~2mAD7┌k╫@MR╕═32VПЬ┼GpШЮ=оВ{КЮ╪_УER■)╧4╦▄Б■ё( √р5х5Б═╣▓y■E╖ Ш░-#[÷х's ╙#%ъ√+я■WГNсЫ║ }v8╤w-
ш6=©pоёжM⌡w`┴Заb├5Dц©к╝ЪЗФ <╤⌡╛я≥╩-H8

View file

@ -0,0 +1,3 @@
xMŒA
ƒ0E»Î)þºhv=ʘŒu É I¼}¥«.ßãñÖ¤+æçëQ(³GØ%ÅaZ\—žþ9IàÒnó6
;Ë8¹väLõº¬ê&‰Ñ+CJHGäˆSé AˆÌ†Ê‰ºœ £¾»“k-Ó8ÿŽfZ{ó°»Û´fmÎ}í7+

View file

@ -0,0 +1 @@
xMЊ1Г SуЉэЂ-;Jє<е зppОтыX©RОh4[Ц луq«TШ#$ЙqZVgbщџі®г2ЇF!с7ЋRЁ.‡Цu—М0ЕЖђтВ[ T#ќ3™њЊFЬЙ}€VЏe^іЦґЫрhW·k/:њых®5f

View file

@ -0,0 +1 @@
36f4c4ecd2b870c0d0869c14a3e41dd506e5a887

View file

@ -0,0 +1,2 @@
name: red-dye
description: A profile designed to detect if controls are accidentally included

View file

@ -0,0 +1,8 @@
include_controls 'child-01'
include_controls 'child-02'
control 'relative-combo-01' do
describe 'always-pass' do
it { should cmp 'always-pass'}
end
end

View file

@ -0,0 +1,14 @@
name: relative-combo
title: relative-combo
license: Apache-2.0
summary: A profile that uses git to depend on profiles in deep subdirs
version: 0.1.0
supports:
platform: os
depends:
- name: child-01
git: test/unit/mock/profiles/git-fetcher/git-repo-01
relative_path: child-01
- name: child-02
git: test/unit/mock/profiles/git-fetcher/git-repo-01
relative_path: deeper-path/in/repo/child-02

View file

@ -0,0 +1,7 @@
include_controls 'child-02'
control 'relative-deep-01' do
describe 'always-pass' do
it { should cmp 'always-pass'}
end
end

View file

@ -0,0 +1,11 @@
name: relative-deep
title: releative-deep
license: Apache-2.0
summary: A profile that uses git to depend on profiles in deep subdirs
version: 0.1.0
supports:
platform: os
depends:
- name: child-02
git: test/unit/mock/profiles/git-fetcher/git-repo-01
relative_path: deeper-path/in/repo/child-02

View file

@ -0,0 +1,7 @@
include_controls 'basic-local'
control 'relative-dot-slash-01' do
describe 'always-pass' do
it { should cmp 'always-pass'}
end
end

View file

@ -0,0 +1,11 @@
name: relative-dot-slash
title: relative-dot-slash
license: Apache-2.0
summary: A profile that depends on profiles using an ./ path
version: 0.1.0
supports:
platform: os
depends:
- name: basic-local
git: test/unit/mock/profiles/git-fetcher/basic-local
relative_path: ./

View file

@ -0,0 +1,7 @@
include_controls 'basic-local'
control 'relative-empty-01' do
describe 'always-pass' do
it { should cmp 'always-pass'}
end
end

View file

@ -0,0 +1,11 @@
name: relative-empty
title: relative-empty
license: Apache-2.0
summary: A profile that depends on profiles using an empty-string path
version: 0.1.0
supports:
platform: os
depends:
- name: basic-local
git: test/unit/mock/profiles/git-fetcher/basic-local
relative_path: ''

View file

@ -0,0 +1,7 @@
include_controls 'none-such-01'
control 'relative-nonesuch-01' do
describe 'always-pass' do
it { should cmp 'always-pass'}
end
end

View file

@ -0,0 +1,11 @@
name: relative-nonesuch
title: relative-nonesuch
license: Apache-2.0
summary: A profile that depends on a profile with a bad relative path
version: 0.1.0
supports:
platform: os
depends:
- name: none-such-01
git: test/unit/mock/profiles/git-fetcher/git-repo-01
relative_path: no/such/path

View file

@ -0,0 +1,7 @@
include_controls 'child-01'
control 'top-level-01' do
describe 'always-pass' do
it { should cmp 'always-pass'}
end
end

View file

@ -0,0 +1,11 @@
name: relative-shallow
title: relative-shallow
license: Apache-2.0
summary: A profile that depends on profiles that are stored in a repo in subdirs
version: 0.1.0
supports:
platform: os
depends:
- name: child-01
git: test/unit/mock/profiles/git-fetcher/git-repo-01
relative_path: child-01