Added test:knapsack task (draft).

This requires an external artifact to be generated and stored in
BK (which we're close to having), but once we have that this runs much
better than test:parallel does because it evens out all the work
across the workers by file run time.

Until we have artifacts cached, this is useless.

Signed-off-by: Ryan Davis <zenspider@chef.io>
This commit is contained in:
Ryan Davis 2019-10-17 11:46:49 -07:00
parent 6afd9b77a5
commit 9d4bb53598
2 changed files with 328 additions and 0 deletions

View file

@ -156,6 +156,51 @@ namespace :test do
end
task parallel: [:accept_license] # given isolated being green, why is this needed?
task :knapsack do
require "json"
n = (ENV["K"] || 4).to_i
t0 = Time.now
lock = Mutex.new
path = "knapsack_minitest_report.json"
data = JSON.load File.read path
jobs = data.bin_by_values(n)
out = Queue.new
n_threads_run n, jobs do |job|
lock.synchronize do
warn "Running #{job.size} files in a thread"
end
t1 = Time.now
output = `bundle exec minitest #{job.join " "} 2>&1`
t2 = Time.now - t1
lock.synchronize do
if $?.success?
warn "Finished #{job.size} files successfully in %d seconds" % [t2]
else
passed = false
warn "Finished #{job.size} files with failures in %d seconds" % [t2]
end
end
out << output
end
puts "done"
puts
out.length.times do
puts out.shift
puts
end
puts "Ran in %d seconds" % [ Time.now - t0 ]
exit 1 unless passed
end
task :isolated do
require "fileutils"
@ -523,3 +568,26 @@ task :release_docker do
puts "--> #{cmd}"
sh("sh", "-c", cmd)
end
class Hash
##
# Bin a hash's keys into +n+ sub-collections that are subdivided as
# close to the same as possible via the values of the hash.
#
# { key => num, ... } => [[key, ...], ...]
def bin_by_values(n)
bins = Array.new(n) { [0] } # [ [ sum, key...], ...]
# rubocop:disable Layout/SpaceAroundOperators
sort_by { |k, v| [-v, k] }.each do |k, v|
smallest = bins.first # always first
smallest[0] += v # add to sum
smallest << k # add to collection
bins.sort_by!(&:first) # re-sort
end
bins.map { |a| a.drop 1 }
end
end

View file

@ -0,0 +1,260 @@
{
"test/unit/resources/http_test.rb": 1.2293035984039307,
"test/unit/resources/kernel_parameter_test.rb": 0.12483358383178711,
"test/unit/utils/deprecation_test.rb": 0.0019278526306152344,
"test/unit/file_provider_test.rb": 0.0969085693359375,
"test/unit/resources/etc_fstab_test.rb": 0.7208030223846436,
"test/unit/config_test.rb": 0.11157989501953125,
"test/unit/bundles/inspec-supermarket/api_test.rb": 0.04485940933227539,
"test/functional/logging_test.rb": 15.343752145767212,
"test/unit/resources/json_test.rb": 2.2913084030151367,
"test/unit/resources/wmi_test.rb": 0.27242064476013184,
"lib/plugins/inspec-habitat/test/unit/profile_test.rb": 0.11534738540649414,
"test/unit/dependencies/requirement_test.rb": 0.003537893295288086,
"test/unit/resources/aws_iam_role_test.rb": 0.004545927047729492,
"test/unit/reporters/cli_test.rb": 0.010650873184204102,
"test/unit/resources/docker_service_test.rb": 1.0367393493652344,
"test/unit/plugin/v1/resource_test.rb": 0.34064602851867676,
"test/unit/inputs/input_test.rb": 0.004008293151855469,
"test/unit/resources/aws_iam_access_key_test.rb": 0.010415792465209961,
"test/unit/resources/aws_cloudtrail_trail_test.rb": 0.009546518325805664,
"test/unit/inputs/events_test.rb": 0.0012328624725341797,
"lib/plugins/inspec-compliance/test/unit/target_test.rb": 0.0058438777923583984,
"test/unit/plugin/v2/api_cli_test.rb": 9.608268737792969e-05,
"test/unit/resources/ip6tables_test.rb": 0.4582860469818115,
"test/unit/dsl/objects_test.rb": 0.009767293930053711,
"test/unit/resources/aws_s3_bucket_object_test.rb": 0.0062007904052734375,
"test/unit/plugin/v2/api_input_test.rb": 0.00015211105346679688,
"test/unit/profiles/profile_test.rb": 0.13086366653442383,
"test/unit/resources/apt_test.rb": 0.923189640045166,
"test/unit/resources/docker_image_test.rb": 0.8074831962585449,
"test/unit/resources/aws_iam_root_user_test.rb": 0.0005619525909423828,
"test/unit/resources/host_test.rb": 1.369344711303711,
"test/unit/resources/auditd_conf_test.rb": 0.1235358715057373,
"test/unit/plugin/v2/plugin_conf_test.rb": 0.012718915939331055,
"test/unit/reporters/json_automate_test.rb": 0.003318309783935547,
"test/unit/resources/elasticsearch_test.rb": 2.740802526473999,
"test/unit/utils/filter_table_test.rb": 0.01719069480895996,
"test/unit/ui_test.rb": 0.12977838516235352,
"test/unit/resources/file_test.rb": 0.699383020401001,
"test/unit/resources/aws_eks_cluster_test.rb": 0.022138118743896484,
"test/unit/resources/postgres_session_test.rb": 0.21983027458190918,
"test/unit/resources/postgres_conf_test.rb": 0.26375484466552734,
"test/unit/resources/aws_s3_buckets_test.rb": 0.002189159393310547,
"test/unit/utils/filter_array_test.rb": 0.0003178119659423828,
"test/unit/resources/aws_s3_bucket_test.rb": 0.009132862091064453,
"test/unit/resources/aws_billing_reports_test.rb": 0.00447535514831543,
"test/unit/resources/aws_iam_access_keys_test.rb": 0.012114763259887695,
"test/functional/license_test.rb": 24.622792720794678,
"test/unit/profiles/profile_context_test.rb": 6.343690395355225,
"test/unit/resources/x509_certificate_test.rb": 1.5553324222564697,
"test/unit/resources/aws_ec2_instances_test.rb": 0.0027618408203125,
"test/unit/backend_test.rb": 0.44538426399230957,
"test/unit/resources/aws_sns_topics_test.rb": 0.0012202262878417969,
"lib/plugins/inspec-compliance/test/functional/inspec_compliance_test.rb": 12.158408164978027,
"test/unit/resources/package_test.rb": 1.923675537109375,
"test/unit/resources/aws_kms_keys_test.rb": 0.0036127567291259766,
"test/unit/runner_test.rb": 0.5668489933013916,
"test/unit/resources/etc_hosts_allow_deny_test.rb": 0.0008852481842041016,
"test/unit/reporters/automate_test.rb": 0.0023496150970458984,
"test/unit/resources/aws_subnet_test.rb": 0.008584260940551758,
"test/unit/resources/aws_sns_subscription_test.rb": 0.009505271911621094,
"test/unit/resources/aws_flow_log_test.rb": 0.005797147750854492,
"test/unit/resources/firewalld_test.rb": 0.001439809799194336,
"test/unit/reporters/yaml_test.rb": 0.001329183578491211,
"test/unit/resources/aws_rds_instance_test.rb": 0.0025510787963867188,
"test/unit/resources/packages_test.rb": 1.2467107772827148,
"test/unit/resources/aws_config_delivery_channel_test.rb": 0.053925514221191406,
"test/unit/source_readers/flat_test.rb": 0.0030901432037353516,
"test/unit/resources/ssh_conf_test.rb": 0.5001556873321533,
"test/unit/utils/find_files_test.rb": 0.0073697566986083984,
"test/unit/fetchers/url_test.rb": 0.03256821632385254,
"test/unit/resources/windows_feature_test.rb": 0.4632289409637451,
"test/functional/inspec_exec_automate_test.rb": 4.509188890457153,
"test/functional/inspec_artifact_test.rb": 8.120951890945435,
"test/functional/git_fetcher_test.rb": 15.92367959022522,
"test/unit/resources/aws_cloudwatch_log_metric_filter_test.rb": 0.0050504207611083984,
"test/unit/resources/vbscript_test.rb": 0.1094200611114502,
"test/unit/resources/rabbitmq_conf_test.rb": 0.11553096771240234,
"test/unit/resources/aws_sqs_queue_test.rb": 0.009199142456054688,
"test/unit/plugin/v2/api_base_test.rb": 0.0023660659790039062,
"test/unit/resources/postgres_ident_conf_test.rb": 0.000438690185546875,
"test/unit/reporters/json_min_test.rb": 0.0014014244079589844,
"test/unit/resources/kernel_module_test.rb": 1.9860725402832031,
"test/unit/inputs/validation_test.rb": 0.004639625549316406,
"test/unit/resources/aws_billing_report_test.rb": 0.008073091506958008,
"test/unit/resources/aws_config_recorder_test.rb": 0.005926609039306641,
"test/unit/resources/aws_iam_policies_test.rb": 0.002453327178955078,
"test/unit/resources/shadow_test.rb": 2.8242545127868652,
"test/unit/resources/aws_iam_user_test.rb": 0.019710063934326172,
"test/unit/profiles/profile_resource_exceptions_test.rb": 0.09007620811462402,
"test/unit/resources/sys_info_test.rb": 0.40329813957214355,
"test/unit/resources/xml_test.rb": 1.1623187065124512,
"test/unit/resources/mount_test.rb": 0.32556962966918945,
"test/unit/resources/mssql_session_test.rb": 0.9317989349365234,
"test/unit/fetchers/mock_test.rb": 0.0003528594970703125,
"test/unit/resources/interface_test.rb": 0.6857821941375732,
"test/unit/resources/aws_ebs_volumes_test.rb": 0.0028533935546875,
"lib/plugins/inspec-compliance/test/unit/api_test.rb": 0.012469768524169922,
"lib/plugins/inspec-plugin-manager-cli/test/functional/list_test.rb": 12.33847975730896,
"test/unit/resources/aws_elbs_test.rb": 0.0114898681640625,
"test/unit/fetchers/local_test.rb": 0.0002498626708984375,
"test/unit/utils/simpleconfig_test.rb": 0.0006554126739501953,
"test/unit/resources/command_test.rb": 1.308288335800171,
"test/unit/resources/aws_kms_key_test.rb": 0.010614633560180664,
"test/unit/resources/nginx_test.rb": 2.356570243835449,
"test/unit/resources/ini_test.rb": 0.0962381362915039,
"test/unit/resources/aws_iam_groups_test.rb": 0.0017852783203125,
"test/unit/plugin/v2/api_dsl_test.rb": 0.00039839744567871094,
"test/unit/resources/nginx_conf_test.rb": 3.4967095851898193,
"test/unit/resources/aws_cloudwatch_alarm_test.rb": 0.004189491271972656,
"test/unit/resources/aws_security_groups_test.rb": 0.004458427429199219,
"test/unit/resources/groups_test.rb": 1.5904195308685303,
"test/unit/utils/nginx_parser_test.rb": 0.04525184631347656,
"test/unit/resources/passwd_test.rb": 1.9739344120025635,
"test/unit/inputs/input_registry_test.rb": 0.006102800369262695,
"test/unit/plugin/v2/loader_test.rb": 2.8721730709075928,
"test/functional/filter_table_test.rb": 16.349918842315674,
"test/unit/resources/yum_test.rb": 0.234299898147583,
"test/unit/resources/toml_test.rb": 0.5408849716186523,
"test/unit/profiles/metadata_test.rb": 1.7872352600097656,
"test/unit/resources/etc_hosts_test.rb": 0.4144096374511719,
"test/unit/resources/bond_test.rb": 0.13292813301086426,
"test/unit/resources/aws_iam_policy_test.rb": 0.08157873153686523,
"test/unit/resources/parse_config_test.rb": 0.2272486686706543,
"test/unit/dsl/other_keywords_test.rb": 0.05983424186706543,
"test/unit/resources/aws_security_group_test.rb": 0.01898026466369629,
"test/unit/reporters/json_test.rb": 0.0036225318908691406,
"lib/plugins/inspec-plugin-manager-cli/test/functional/help_test.rb": 4.16137170791626,
"test/unit/resources/xinetd_conf_test.rb": 1.594449758529663,
"test/unit/resources/apache_conf_test.rb": 0.2027909755706787,
"test/unit/utils/object_traversal_test.rb": 0.0005295276641845703,
"test/unit/resources/virtualization_test.rb": 0.23914575576782227,
"test/unit/resources/security_policy_test.rb": 0.3184833526611328,
"test/unit/resources/crontab_test.rb": 1.989016056060791,
"lib/plugins/inspec-plugin-manager-cli/test/functional/update_test.rb": 13.025645017623901,
"test/unit/resources/aws_ec2_instance_test.rb": 0.0004534721374511719,
"test/unit/resources/aws_iam_group_test.rb": 0.004161357879638672,
"test/functional/inspec_test.rb": 17.965163707733154,
"test/unit/fetchers/fetchers_test.rb": 0.002389192581176758,
"test/unit/resources/aws_ecs_cluster_test.rb": 0.009258508682250977,
"test/unit/resources/user_test.rb": 1.248793363571167,
"test/unit/resources/aws_sns_topic_test.rb": 0.005607128143310547,
"test/functional/inheritance_test.rb": 17.1094651222229,
"test/unit/utils/telemetry/global_methods_test.rb": 0.0005459785461425781,
"test/unit/resources/mysql_conf_test.rb": 0.12889981269836426,
"test/unit/resources/aws_elb_test.rb": 0.013537406921386719,
"test/unit/plugin/v1/plugin_test.rb": 0.0041332244873046875,
"test/unit/plugin/v2/installer_test.rb": 9.516033411026001,
"test/unit/resources/aws_vpcs_test.rb": 0.007269382476806641,
"test/unit/dependencies/lockfile_test.rb": 0.0012049674987792969,
"test/unit/resources/aws_subnets_test.rb": 0.0039005279541015625,
"test/unit/resources/auditd_test.rb": 1.437373161315918,
"test/unit/impact_test.rb": 0.0007719993591308594,
"test/functional/inspec_supermarket_test.rb": 8.00983214378357,
"test/unit/profiles/control_eval_context_test.rb": 0.0387730598449707,
"test/unit/source_readers/inspec_test.rb": 0.024047136306762695,
"test/unit/fetchers/source_reader_test.rb": 9.989738464355469e-05,
"test/unit/resources/aws_route_tables_test.rb": 0.0018529891967773438,
"test/unit/resources/iptables_test.rb": 0.41360974311828613,
"test/unit/resources/grub_conf_test.rb": 0.7245750427246094,
"lib/plugins/inspec-artifact/test/functional/inspec_artifact_test.rb": 9.679787874221802,
"test/unit/resources/docker_test.rb": 0.9794678688049316,
"test/unit/resources/iis_site_test.rb": 0.08705711364746094,
"test/unit/utils/erlang_parser_test.rb": 0.053453683853149414,
"test/unit/reporters/junit_test.rb": 0.0008678436279296875,
"test/unit/resources/aws_cloudtrail_trails_test.rb": 0.003133058547973633,
"test/unit/resources/aws_iam_users_test.rb": 0.006739616394042969,
"test/unit/cached_fetcher_test.rb": 0.0016078948974609375,
"test/unit/matchers/matchers_test.rb": 0.017473459243774414,
"test/unit/resources/pip_test.rb": 0.2528543472290039,
"test/unit/resources/registry_key_test.rb": 0.527400016784668,
"test/functional/inspec_schema_test.rb": 1.8518378734588623,
"test/unit/resources/bridge_test.rb": 0.45383429527282715,
"test/unit/resources/zfs_dataset_test.rb": 0.10688591003417969,
"test/unit/fetchers/git_test.rb": 0.0060863494873046875,
"test/unit/resources/aws_vpc_test.rb": 0.010143756866455078,
"test/unit/resources/postfix_conf_test.rb": 0.23322129249572754,
"test/unit/resources/dh_params_test.rb": 0.5277886390686035,
"test/unit/resources/audit_policy_test.rb": 0.12401008605957031,
"test/unit/resources/aws_route_table_test.rb": 0.002173185348510742,
"test/unit/dsl/control_test.rb": 0.04923248291015625,
"test/unit/utils/bsd_mount_parser_test.rb": 0.00013256072998046875,
"test/unit/plugin/v2/back_compat_test.rb": 5.4836273193359375e-05,
"test/unit/resources/oracledb_session_test.rb": 0.22482538223266602,
"test/unit/resources/docker_plugin_test.rb": 0.4926760196685791,
"test/unit/resources/oneget_test.rb": 0.4584338665008545,
"test/unit/utils/telemetry/data_series_test.rb": 0.00033402442932128906,
"test/unit/utils/solaris_netstat_parser_test.rb": 0.0008101463317871094,
"lib/plugins/inspec-plugin-manager-cli/test/functional/uninstall_test.rb": 10.793046712875366,
"test/unit/utils/database_helpers_test.rb": 0.000102996826171875,
"lib/plugins/inspec-init/test/functional/inspec_init_plugin_test.rb": 6.031000852584839,
"lib/plugins/inspec-compliance/test/unit/api/login_test.rb": 0.009882926940917969,
"test/unit/resources/inetd_conf_test.rb": 0.12271404266357422,
"test/unit/resources/windows_task_test.rb": 0.2999277114868164,
"test/unit/resources/aws_ebs_volume_test.rb": 0.00044083595275878906,
"lib/plugins/inspec-plugin-manager-cli/test/unit/plugin_def_test.rb": 0.21892976760864258,
"test/unit/shell_detector_test.rb": 0.0022830963134765625,
"test/unit/resources/mysql_session_test.rb": 0.24007129669189453,
"test/functional/inspec_detect_test.rb": 8.632289171218872,
"test/unit/resources/filesystem_test.rb": 0.3572375774383545,
"test/unit/profiles/library_eval_context_test.rb": 0.006806850433349609,
"test/unit/resources/group_test.rb": 1.6048390865325928,
"test/unit/resources/aws_iam_password_policy_test.rb": 0.0004677772521972656,
"test/unit/resources/csv_test.rb": 0.6396923065185547,
"test/unit/resources/platform_test.rb": 1.347423791885376,
"test/unit/dsl/describe_test.rb": 0.0013093948364257812,
"test/unit/resource_supports/aws/aws_resource_mixin_test.rb": 0.0005450248718261719,
"test/unit/resources/os_test.rb": 0.5878362655639648,
"test/unit/resources/npm_test.rb": 0.19912981986999512,
"test/unit/base_cli_test.rb": 0.0010166168212890625,
"test/unit/resources/windows_hotfix_test.rb": 0.2955143451690674,
"test/unit/dependencies/resolver_test.rb": 0.0014138221740722656,
"test/unit/utils/telemetry/collector_test.rb": 0.00037789344787597656,
"lib/plugins/inspec-init/test/functional/inspec_init_profile_test.rb": 15.174067974090576,
"test/unit/resources/aide_conf_test.rb": 0.0003757476806640625,
"test/unit/reporters/base_test.rb": 0.0002818107604980469,
"test/unit/resources/service_test.rb": 4.78145170211792,
"test/unit/resources/processes_test.rb": 2.2988150119781494,
"test/unit/resources/postgres_hba_conf_test.rb": 0.0004925727844238281,
"lib/plugins/inspec-plugin-manager-cli/test/unit/cli_args_test.rb": 0.008144140243530273,
"test/unit/resources/iis_app_pool_test.rb": 0.11042523384094238,
"test/unit/resources/gem_test.rb": 0.8691647052764893,
"test/unit/utils/passwd_parser_test.rb": 0.00017714500427246094,
"test/unit/resources/port_linuxports_test.rb": 0.0002651214599609375,
"test/unit/resources/chocolatey_package_test.rb": 0.23300385475158691,
"lib/plugins/inspec-habitat/test/functional/inspec_habitat_test.rb": 4.041239023208618,
"test/unit/resources/etc_group_test.rb": 1.0811245441436768,
"test/unit/resources/login_def_test.rb": 0.10575509071350098,
"test/unit/resources/powershell_test.rb": 0.49767374992370605,
"test/unit/resources/os_env_test.rb": 0.20701098442077637,
"test/unit/resources/bash_test.rb": 0.42038869857788086,
"test/unit/resources/docker_container_test.rb": 0.7616477012634277,
"test/unit/resources/cpan_test.rb": 0.2529270648956299,
"test/unit/resources/key_rsa_test.rb": 0.26465725898742676,
"test/unit/resources/limits_conf_test.rb": 0.12478280067443848,
"test/unit/resources/ntp_conf_test.rb": 0.12037992477416992,
"test/unit/resources/port_test.rb": 3.487647533416748,
"test/unit/resources/zfs_pool_test.rb": 0.13034319877624512,
"test/unit/resources/ssl_test.rb": 0.9716839790344238,
"test/unit/resources/security_identifier_test.rb": 0.8614554405212402,
"test/unit/resources/cran_test.rb": 0.20490002632141113,
"test/functional/inspec_report_test.rb": 0.1719961166381836,
"test/unit/resources/ksh_test.rb": 0.5069341659545898,
"test/unit/resources/yaml_test.rb": 0.9822304248809814,
"test/functional/inspec_exec_test.rb": 141.8236107826233,
"test/functional/inputs_test.rb": 50.06088328361511,
"test/functional/inspec_check_test.rb": 26.29991340637207,
"test/functional/inspec_exec_junit_test.rb": 20.06471037864685,
"test/functional/waivers_test.rb": 31.40618085861206,
"test/functional/inspec_exec_json_test.rb": 40.68711543083191,
"test/functional/ui_test.rb": 30.272067070007324,
"test/functional/plugins_test.rb": 48.22778129577637,
"test/functional/inspec_json_profile_test.rb": 45.76652765274048,
"test/functional/inspec_archive_test.rb": 18.76940131187439,
"lib/plugins/inspec-plugin-manager-cli/test/functional/install_test.rb": 87.8083438873291,
"test/functional/inspec_vendor_test.rb": 70.06204652786255,
"test/functional/inspec_shell_test.rb": 83.80282974243164,
"test/functional/inspec_exec_jsonmin_test.rb": 24.284127950668335,
"lib/plugins/inspec-plugin-manager-cli/test/functional/search_test.rb": 57.54038906097412
}