mirror of
https://github.com/sharkdp/bat
synced 2024-11-27 14:20:45 +00:00
Parallelize syntax regression tests
The syntax highlighting regression tests can be trivially parallelized. On my notebook (8 core), this results in a 3.9x speedup.
This commit is contained in:
David Peter
David Peter
parent
5143f3ad43
commit
44a332c1c4
1 changed files with 70 additions and 58 deletions
|
|
@ -6,7 +6,7 @@ import sys
|
||||||
import os.path as path
|
import os.path as path
|
||||||
import os
|
import os
|
||||||
import argparse
|
import argparse
|
||||||
|
from multiprocessing import Pool
|
||||||
|
|
||||||
BAT_OPTIONS = [
|
BAT_OPTIONS = [
|
||||||
"--no-config",
|
"--no-config",
|
||||||
|
|
@ -38,68 +38,79 @@ def get_options(source):
|
||||||
return options
|
return options
|
||||||
|
|
||||||
|
|
||||||
|
def create_highlighted_version(args):
|
||||||
|
output_basepath, source = args
|
||||||
|
env = os.environ.copy()
|
||||||
|
env.pop("BAT_CACHE_PATH", None)
|
||||||
|
env.pop("BAT_CONFIG_DIR", None)
|
||||||
|
env.pop("BAT_CONFIG_PATH", None)
|
||||||
|
env.pop("BAT_OPTS", None)
|
||||||
|
env.pop("BAT_PAGER", None)
|
||||||
|
env.pop("BAT_STYLE", None)
|
||||||
|
env.pop("BAT_TABS", None)
|
||||||
|
env.pop("BAT_THEME", None)
|
||||||
|
env.pop("NO_COLOR", None)
|
||||||
|
env.pop("PAGER", None)
|
||||||
|
env["COLORTERM"] = "truecolor" # make sure to output 24bit colors
|
||||||
|
|
||||||
|
source_dirname = path.basename(path.dirname(source))
|
||||||
|
source_filename = path.basename(source)
|
||||||
|
|
||||||
|
if source_filename in SKIP_FILENAMES:
|
||||||
|
return
|
||||||
|
|
||||||
|
bat_output = subprocess.check_output(
|
||||||
|
["bat"] + get_options(source) + [source],
|
||||||
|
stderr=subprocess.PIPE,
|
||||||
|
env=env,
|
||||||
|
)
|
||||||
|
|
||||||
|
output_dir = path.join(output_basepath, source_dirname)
|
||||||
|
output_path = path.join(output_dir, source_filename)
|
||||||
|
|
||||||
|
os.makedirs(output_dir, exist_ok=True)
|
||||||
|
|
||||||
|
with open(output_path, "wb") as output_file:
|
||||||
|
output_file.write(bat_output)
|
||||||
|
|
||||||
|
print("Created '{}'".format(output_path))
|
||||||
|
|
||||||
|
|
||||||
def create_highlighted_versions(output_basepath):
|
def create_highlighted_versions(output_basepath):
|
||||||
root = os.path.dirname(os.path.abspath(__file__))
|
root = os.path.dirname(os.path.abspath(__file__))
|
||||||
sources = path.join(root, "source", "*")
|
source_paths = path.join(root, "source", "*")
|
||||||
|
|
||||||
for source in glob.glob(path.join(sources, "*")) + glob.glob(
|
sources = []
|
||||||
path.join(sources, ".*")
|
for source in glob.glob(path.join(source_paths, "*")) + glob.glob(
|
||||||
|
path.join(source_paths, ".*")
|
||||||
):
|
):
|
||||||
try:
|
sources.append((output_basepath, source))
|
||||||
env = os.environ.copy()
|
|
||||||
env.pop("BAT_CACHE_PATH", None)
|
|
||||||
env.pop("BAT_CONFIG_DIR", None)
|
|
||||||
env.pop("BAT_CONFIG_PATH", None)
|
|
||||||
env.pop("BAT_OPTS", None)
|
|
||||||
env.pop("BAT_PAGER", None)
|
|
||||||
env.pop("BAT_STYLE", None)
|
|
||||||
env.pop("BAT_TABS", None)
|
|
||||||
env.pop("BAT_THEME", None)
|
|
||||||
env.pop("NO_COLOR", None)
|
|
||||||
env.pop("PAGER", None)
|
|
||||||
env["COLORTERM"] = "truecolor" # make sure to output 24bit colors
|
|
||||||
|
|
||||||
source_dirname = path.basename(path.dirname(source))
|
try:
|
||||||
source_filename = path.basename(source)
|
with Pool() as p:
|
||||||
|
p.map(create_highlighted_version, sources)
|
||||||
|
except subprocess.CalledProcessError as err:
|
||||||
|
print(
|
||||||
|
"=== Error: Could not highlight source file '{}".format(source),
|
||||||
|
file=sys.stderr,
|
||||||
|
)
|
||||||
|
print(
|
||||||
|
"=== bat stdout:\n{}".format(err.stdout.decode("utf-8")),
|
||||||
|
file=sys.stderr,
|
||||||
|
)
|
||||||
|
print(
|
||||||
|
"=== bat stderr:\n{}".format(err.stderr.decode("utf-8")),
|
||||||
|
file=sys.stderr,
|
||||||
|
)
|
||||||
|
return False
|
||||||
|
except FileNotFoundError:
|
||||||
|
print(
|
||||||
|
"Error: Could not execute 'bat'. Please make sure that the executable "
|
||||||
|
"is available on the PATH."
|
||||||
|
)
|
||||||
|
return False
|
||||||
|
|
||||||
if source_filename in SKIP_FILENAMES:
|
return True
|
||||||
continue
|
|
||||||
|
|
||||||
bat_output = subprocess.check_output(
|
|
||||||
["bat"] + get_options(source) + [source],
|
|
||||||
stderr=subprocess.PIPE,
|
|
||||||
env=env,
|
|
||||||
)
|
|
||||||
|
|
||||||
output_dir = path.join(output_basepath, source_dirname)
|
|
||||||
output_path = path.join(output_dir, source_filename)
|
|
||||||
|
|
||||||
os.makedirs(output_dir, exist_ok=True)
|
|
||||||
|
|
||||||
with open(output_path, "wb") as output_file:
|
|
||||||
output_file.write(bat_output)
|
|
||||||
|
|
||||||
print("Created '{}'".format(output_path))
|
|
||||||
except subprocess.CalledProcessError as err:
|
|
||||||
print(
|
|
||||||
"=== Error: Could not highlight source file '{}".format(source),
|
|
||||||
file=sys.stderr,
|
|
||||||
)
|
|
||||||
print(
|
|
||||||
"=== bat stdout:\n{}".format(err.stdout.decode("utf-8")),
|
|
||||||
file=sys.stderr,
|
|
||||||
)
|
|
||||||
print(
|
|
||||||
"=== bat stderr:\n{}".format(err.stderr.decode("utf-8")),
|
|
||||||
file=sys.stderr,
|
|
||||||
)
|
|
||||||
sys.exit(1)
|
|
||||||
except FileNotFoundError:
|
|
||||||
print(
|
|
||||||
"Error: Could not execute 'bat'. Please make sure that the executable "
|
|
||||||
"is available on the PATH."
|
|
||||||
)
|
|
||||||
sys.exit(1)
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
|
|
@ -118,4 +129,5 @@ if __name__ == "__main__":
|
||||||
|
|
||||||
args = parser.parse_args()
|
args = parser.parse_args()
|
||||||
|
|
||||||
create_highlighted_versions(output_basepath=args.output)
|
if not create_highlighted_versions(output_basepath=args.output):
|
||||||
|
sys.exit(1)
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue