Merge pull request #4480 from inspec/zenspider/isolate

Added test:isolate task that runs tests isolated but in parallel.

.expeditor/buildkite/verify.sh

@@ -16,4 +16,4 @@ echo "--- bundle install"
 bundle install --jobs=7 --retry=3 --without tools maintenance deploy
 
 echo "+++ bundle exec rake"
-bundle exec rake
+bundle exec rake ${RAKE_TASK:-}

.expeditor/verify.pipeline.yml

@@ -33,6 +33,14 @@ steps:
       docker:
         image: ruby:2.6-stretch
 
+- label: isolated-tests-ruby-2.6
+  command:
+    - RAKE_TASK=test:isolated /workdir/.expeditor/buildkite/verify.sh
+  expeditor:
+    executor:
+      docker:
+        image: ruby:2.6-stretch
+
 - label: run-tests-ruby-2.6-windows
   command:
     - /workdir/.expeditor/buildkite/verify.ps1

Rakefile

@@ -86,16 +86,77 @@ namespace :test do
     puts missing.sort
   end
 
-  task :isolated do
-    failures = Dir[*GLOBS]
-    failures.reject! do |file|
-      system(Gem.ruby, "-Ilib:test", file)
+  # rubocop:disable Style/BlockDelimiters,Layout/ExtraSpacing,Lint/AssignmentInCondition
+
+  def n_threads_run(n_workers, jobs)
+    queue = Queue.new
+
+    jobs.each do |job|
+      queue << job
     end
 
-    unless failures.empty?
-      puts "These test files failed:\n"
-      puts failures
-      raise "broken tests..."
+    n_workers.times.map {
+      queue << nil            # 1 quit value per thread
+      Thread.new do
+        while job = queue.pop # go until quit value
+          yield job
+        end
+      end
+    }.each(&:join)
+  end
+
+  task :isolated do
+    require "fileutils"
+    require "thread"
+
+    # Only needed for local runs, not CI?
+    FileUtils.rm_rf File.expand_path "~/.inspec"
+    FileUtils.rm_rf File.expand_path "~/.chef"
+
+    # 3 seems to be the magic number... (tho not by that much)
+    bad, good, n = {}, [], (ENV.delete("N") || 3).to_i
+    t0 = Time.now
+
+    srand 42
+    tests = Dir[*GLOBS].sort
+
+    n_threads_run n, tests do |path|
+      output = `bundle exec ruby -Ilib:test #{path} 2>&1`
+
+      if $?.success?
+        $stderr.print "."
+        good << path
+      else
+        $stderr.print "x"
+        bad[path] = output
+      end
+    end
+
+    puts "done"
+    puts "Ran in %d seconds" % [ Time.now - t0 ]
+
+    unless good.empty?
+      puts
+      puts "# Good tests:"
+      good.sort.each do |path|
+        puts path
+      end
+    end
+
+    unless bad.empty?
+      puts
+      puts "# Bad tests:"
+      bad.keys.each do |path|
+        puts path
+      end
+      puts
+      puts "# Bad Test Output:"
+      bad.each do |path, output|
+        puts
+        puts "# #{path}:"
+        puts output
+      end
+      exit 1
     end
   end