diff --git a/hack/bench/plot.gp b/hack/bench/plot.gp new file mode 100644 index 000000000..28da33d70 --- /dev/null +++ b/hack/bench/plot.gp @@ -0,0 +1,10 @@ +set terminal png size 800,600 +set output "hack/bench/versions.png" + +set title "User Time vs. Version" +set xlabel "Version" +set ylabel "Average User Time (s)" + +set xtics rotate by -45 + +plot "hack/bench/plot.txt" using 2:xtic(1) with linespoints linestyle 1 notitle diff --git a/hack/bench/plot.sh b/hack/bench/plot.sh new file mode 100755 index 000000000..6ef222a59 --- /dev/null +++ b/hack/bench/plot.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +if [ $# -ne 2 ]; then + echo "Usage: $0 " + exit 1 +fi + +# Get the number of versions back to test from command line argument +num_versions="$2" + +test_repo="$1" + +bash hack/bench/versions.sh $test_repo $num_versions | tee hack/bench/plot.txt + +gnuplot hack/bench/plot.gp diff --git a/hack/bench/plot.txt b/hack/bench/plot.txt new file mode 100644 index 000000000..51f5c750f --- /dev/null +++ b/hack/bench/plot.txt @@ -0,0 +1,24 @@ +v3.33.0: 1.402 +v3.32.2: 1.298 +v3.32.1: 1.332 +v3.32.0: 1.348 +v3.31.6: 2.470 +v3.31.5: 2.462 +v3.31.4: 2.460 +v3.31.3: 2.418 +v3.31.2: 1.384 +v3.31.1: 1.344 +v3.31.0: 1.354 +v3.30.0: 1.392 +v3.29.1: 1.382 +v3.29.0: 1.340 +v3.28.7: 1.380 +v3.28.6: 1.308 +v3.28.5: 2.596 +v3.28.4: 2.554 +v3.28.3: 2.582 +v3.28.1: 2.578 +v3.28.2: 2.566 +v3.28.0: 2.552 +v3.27.1: 2.574 +v3.26.0: 2.538 diff --git a/hack/bench/versions.png b/hack/bench/versions.png new file mode 100644 index 000000000..8618e081e Binary files /dev/null and b/hack/bench/versions.png differ diff --git a/hack/bench/versions.sh b/hack/bench/versions.sh new file mode 100644 index 000000000..586fdb4c8 --- /dev/null +++ b/hack/bench/versions.sh @@ -0,0 +1,80 @@ +#!/bin/bash + +if [ $# -ne 2 ]; then + echo "Usage: $0 " + exit 1 +fi + +# Get the number of versions back to test from command line argument +num_versions="$2" + +test_repo="$1" + +num_iterations=5 + +# Create a temporary folder to clone the repository +repo_tmp=$(mktemp -d) +# Set up a trap to remove the temporary folder on exit or failure +trap "rm -rf $repo_tmp" EXIT +# Clone the test repository to a temporary folder +git clone --quiet "$test_repo" $repo_tmp + + +# Get list of git tags, sorted from newest to oldest +tags=$(git tag --sort=-creatordate) + +# Counter to keep track of number of tags checked out +count=0 + + +# Loop over tags and checkout each one in turn, up to the specified number of versions +for tag in $tags +do + if [[ $count -eq $num_versions ]]; then + break + fi + + # Skip RC tags + if [[ $tag == *"rc"* ]]; then + continue + fi + + # Skip alpha tags + if [[ $tag == *"alpha"* ]]; then + continue + fi + + # Use git checkout with the quiet flag to suppress output + git checkout $tag --quiet + + # Run make install with suppressed output + make install > /dev/null + + # Initialize the variable to store the sum of user times + user_time_sum=0 + + # Run each iteration 5 times and calculate the average user time + for i in {1..$num_iterations} + do + # Run trufflehog with suppressed output and capture user time with /usr/bin/time + tmpfile=$(mktemp) + /usr/bin/time -o $tmpfile trufflehog git "file://$repo_tmp" --no-verification --no-update >/dev/null 2>&1 + time_output=$(cat $tmpfile) + rm $tmpfile + + # Extract the user time from the output + user_time=$(echo $time_output | awk '{print $3}') + + # Add the user time to the sum + user_time_sum=$(echo "$user_time_sum + $user_time" | bc) + done + + # Calculate the average user time + average_user_time=$(echo "scale=3; $user_time_sum / $num_iterations" | bc) + + # Print the average user time output for this iteration in the specified format + echo "$tag: $average_user_time" + + # Increment the counter + count=$((count+1)) +done