From d2c9ed3e45aeab034b0fc34b76f04fb4e98588b9 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Mon, 20 Nov 2023 16:29:06 +0200 Subject: [PATCH] Initial work on optimization commands The playbook can now optimize itself based on the enabled components in for all hosts in the inventory (`just optimize`) or for a specific host (`just optimize-for-host HOSTNAME`). The optimized playbook will have: - fewer requirements (fewer roles need to be installed by `just roles`) - a shorter and quicker to evaluate `group_vars/mash_servers` file - a `setup.yml` file which includes less roles Running the playbook optimized is still work in progress. There still probably exist various role dependencies in the group-vars file, etc. The `optimize-reset` command aims to restore your playbook to a non-optimized state, which should work as before (and not experience bugs). The playbook takes care to notice of changes to the various files in `templates/` (`setup.yml`, `requirements.yml`, `group_vars_mash_servers`) and update your optimized or non-optimized copies that are derived from these templates. To do this, it keeps `.srchash` files in the `run/` directory. When it notices a change in the source file's hash (by comparing to the `.srchash` file), it will update you to the new template. Optimization state is stored in a file in `run/` as well (`optimization-vars-files.state`). Should the playbook notice changes in the source `template/` files, it should update you and re-optimize using the same parameters as before (read from the state file). --- .editorconfig | 2 +- .gitignore | 11 +- bin/optimize.py | 161 ++++++++++++++++++ group_vars/.gitkeep | 0 justfile | 106 ++++++++++-- run/.gitkeep | 0 .../group_vars_mash_servers | 4 +- .../requirements.yml | 0 setup.all.yml => templates/setup.yml | 0 9 files changed, 265 insertions(+), 19 deletions(-) create mode 100644 bin/optimize.py create mode 100644 group_vars/.gitkeep create mode 100644 run/.gitkeep rename group_vars/mash_servers_all => templates/group_vars_mash_servers (99%) rename requirements.all.yml => templates/requirements.yml (100%) rename setup.all.yml => templates/setup.yml (100%) diff --git a/.editorconfig b/.editorconfig index 8628502..6bcb2dd 100644 --- a/.editorconfig +++ b/.editorconfig @@ -27,7 +27,7 @@ indent_size = 4 indent_style = space indent_size = 2 -[group_vars/mash_servers_all] +[templates/group_vars_mash_servers] indent_style = space indent_size = 2 diff --git a/.gitignore b/.gitignore index 9210eec..e748acc 100644 --- a/.gitignore +++ b/.gitignore @@ -11,10 +11,11 @@ .DS_Store /requirements.yml -/requirements.yml.srchash - /setup.yml -/setup.yml.srchash - /group_vars/mash_servers -/group_vars/mash_servers.srchash + +/run/* +!/run/.gitkeep + +/group_vars/* +!/group_vars/.gitkeep diff --git a/bin/optimize.py b/bin/optimize.py new file mode 100644 index 0000000..24570c0 --- /dev/null +++ b/bin/optimize.py @@ -0,0 +1,161 @@ +import argparse +import regex +import sys +import yaml + +parser = argparse.ArgumentParser(description='Optimizes the playbook based on enabled components found in vars.yml files') +parser.add_argument('--vars-paths', help='Path to vars.yml configuration files to process', required=True) +parser.add_argument('--src-requirements-yml-path', help='Path to source requirements.yml file with all role definitions', required=True) +parser.add_argument('--src-setup-yml-path', help='Path to source setup.yml file', required=True) +parser.add_argument('--src-group-vars-yml-path', help='Path to source group vars file', required=True) +parser.add_argument('--dst-requirements-yml-path', help='Path to destination requirements.yml file, where role definitions will be saved', required=True) +parser.add_argument('--dst-setup-yml-path', help='Path to destination setup.yml file', required=True) +parser.add_argument('--dst-group-vars-yml-path', help='Path to destination group vars file', required=True) + +args = parser.parse_args() + +def load_combined_variable_names_from_files(vars_yml_file_paths): + variable_names = set({}) + for vars_path in vars_yml_file_paths: + with open(vars_path, 'r') as file: + yaml_data = yaml.safe_load(file) + + variable_names = variable_names | set(yaml_data.keys()) + return variable_names + +def load_yaml_file(path): + with open(path, 'r') as file: + return yaml.safe_load(file) + +def is_role_definition_in_use(role_definition, used_variable_names): + for variable_name in used_variable_names: + if 'activation_prefix' in role_definition: + if role_definition['activation_prefix'] == '': + # Special value indicating "always activate". + # We don't really need this dedicated if, but it's more obvious with it. + return True + if variable_name.startswith(role_definition['activation_prefix']): + return True + return False + +def write_yaml_to_file(definitions, path): + with open(path, 'w') as file: + yaml.dump(definitions, file) + +def read_file(path): + with open(path, 'r') as file: + return file.read() + +def write_to_file(contents, path): + with open(path, 'w') as file: + file.write(contents) + +# Matches the beginning of role-specific blocks. +# Example: `# role-specific:playbook_help` +regex_role_specific_block_start = regex.compile('^\s*#\s*role-specific\:\s*([^\s]+)$') + +# Matches the end of role-specific blocks. +# Example: `# /role-specific:playbook_help` +regex_role_specific_block_end = regex.compile('^\s*#\s*/role-specific\:\s*([^\s]+)$') + +def process_file_contents(file_name, enabled_role_names, known_role_names): + contents = read_file(file_name) + + lines_preserved = [] + role_specific_stack = [] + + for line_number, line in enumerate(contents.split("\n")): + # Stage 1: looking for a role-specific starting block + start_role_matches = regex_role_specific_block_start.match(line) + if start_role_matches is not None: + role_name = start_role_matches.group(1) + if role_name not in known_role_names: + raise Exception('Found start block for role {0} on line {1} in file {2}, but it is not a known role name found among: {3}'.format( + role_name, + line_number, + file_name, + known_role_names, + )) + role_specific_stack.append(role_name) + continue + + # Stage 2: looking for role-specific closing blocks + end_role_matches = regex_role_specific_block_end.match(line) + if end_role_matches is not None: + role_name = end_role_matches.group(1) + if role_name not in known_role_names: + raise Exception('Found end block for role {0} on line {1} in file {2}, but it is not a known role name found among: {3}'.format( + role_name, + line_number, + file_name, + known_role_names, + )) + + if len(role_specific_stack) == 0: + raise Exception('Found end block for role {0} on line {1} in file {2}, but there is no opening statement for it'.format( + role_name, + line_number, + file_name, + )) + + last_role_name = role_specific_stack[len(role_specific_stack) - 1] + if role_name != last_role_name: + raise Exception('Found end block for role {0} on line {1} in file {2}, but the last starting block was for role {3}'.format( + role_name, + line_number, + file_name, + last_role_name, + )) + + role_specific_stack.pop() + + continue + + # Stage 3: regular line + all_roles_allowed = True + for role_name in role_specific_stack: + if role_name not in enabled_role_names: + all_roles_allowed = False + break + + if all_roles_allowed: + lines_preserved.append(line) + + if len(role_specific_stack) != 0: + raise Exception('Expected one or more closing block for role-specific tags in file {0}: {1}'.format(file_name, role_specific_stack)) + + lines_final = [] + sequential_blank_lines_count = 0 + for line in lines_preserved: + if line != "": + lines_final.append(line) + sequential_blank_lines_count = 0 + continue + + if sequential_blank_lines_count <= 1: + lines_final.append(line) + sequential_blank_lines_count += 1 + continue + + return "\n".join(lines_final) + +vars_paths = args.vars_paths.split(' ') +used_variable_names = load_combined_variable_names_from_files(vars_paths) + +all_role_definitions = load_yaml_file(args.src_requirements_yml_path) + +enabled_role_definitions = [] +for role_definition in all_role_definitions: + if is_role_definition_in_use(role_definition, used_variable_names): + enabled_role_definitions.append(role_definition) + +write_yaml_to_file(enabled_role_definitions, args.dst_requirements_yml_path) + +known_role_names = tuple(map(lambda definition: definition['name'], all_role_definitions)) +enabled_role_names = tuple(map(lambda definition: definition['name'], enabled_role_definitions)) + +setup_yml_processed = process_file_contents(args.src_setup_yml_path, enabled_role_names, known_role_names) +write_to_file(setup_yml_processed, args.dst_setup_yml_path) + +group_vars_yml_processed = process_file_contents(args.src_group_vars_yml_path, enabled_role_names, known_role_names) +write_to_file(group_vars_yml_processed, args.dst_group_vars_yml_path) diff --git a/group_vars/.gitkeep b/group_vars/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/justfile b/justfile index 0be5774..fe5ea2d 100644 --- a/justfile +++ b/justfile @@ -2,19 +2,75 @@ default: @just --list --justfile {{ justfile() }} +run_directory_path := justfile_directory() + "/run" +templates_directory_path := justfile_directory() + "/templates" +optimization_vars_files_file_path := run_directory_path + "/optimization-vars-files.state" + # Pulls external Ansible roles -roles: requirements-yml +roles: _requirements-yml #!/usr/bin/env sh if [ -x "$(command -v agru)" ]; then agru -r {{ justfile_directory() }}/requirements.yml else rm -rf roles/galaxy - ansible-galaxy install -r requirements.yml -p roles/galaxy/ --force + ansible-galaxy install -r {{ justfile_directory() }}/requirements.yml -p roles/galaxy/ --force fi +# Optimizes the playbook based on stored configuration (vars.yml paths) +optimize-restore: + #!/usr/bin/env sh + if [ -f "$optimization_vars_files_file_path" ]; then + just --justfile {{ justfile() }} \ + _optimize-for-var-paths \ + $(cat $optimization_vars_files_file_path) + else + echo "Cannot restore optimization state from a file ($optimization_vars_files_file_path), because it doesn't exist" + exit 1 + fi + +# Clears optimizations and resets the playbook to a non-optimized state +optimize-reset: && _clean_template_derived_files + #!/usr/bin/env sh + rm -f {{ run_directory_path }}/*.srchash + rm -f {{ optimization_vars_files_file_path }} + +# Optimizes the playbook based on the enabled components for all hosts in the inventory +optimize inventory_path='inventory': _reconfigure-for-all-hosts + +_reconfigure-for-all-hosts inventory_path='inventory': + #!/usr/bin/env sh + just --justfile {{ justfile() }} \ + _optimize-for-var-paths \ + $(find {{ inventory_path }}/host_vars/ -maxdepth 2 -name '*.yml' -exec readlink -f {} \;) + +# Optimizes the playbook based on the enabled components for a single host +optimize-for-host hostname inventory_path='inventory': + #!/usr/bin/env sh + just --justfile {{ justfile() }} \ + _optimize-for-var-paths \ + $(find {{ inventory_path }}/host_vars/{{ hostname }} -maxdepth 1 -name '*.yml' -exec readlink -f {} \;) + +# Optimizes the playbook based on the enabled components found in the given vars.yml files +_optimize-for-var-paths +PATHS: + #!/usr/bin/env sh + echo '{{ PATHS }}' > {{ optimization_vars_files_file_path }} + + just --justfile {{ justfile() }} _save_hash_for_file {{ templates_directory_path }}/requirements.yml {{ justfile_directory() }}/requirements.yml + just --justfile {{ justfile() }} _save_hash_for_file {{ templates_directory_path }}/setup.yml {{ justfile_directory() }}/setup.yml + just --justfile {{ justfile() }} _save_hash_for_file {{ templates_directory_path }}/group_vars_mash_servers {{ justfile_directory() }}/group_vars/mash_servers + + /usr/bin/env python {{ justfile_directory() }}/bin/optimize.py \ + --vars-paths='{{ PATHS }}' \ + --src-requirements-yml-path={{ templates_directory_path }}/requirements.yml \ + --dst-requirements-yml-path={{ justfile_directory() }}/requirements.yml \ + --src-setup-yml-path={{ templates_directory_path }}/setup.yml \ + --dst-setup-yml-path={{ justfile_directory() }}/setup.yml \ + --src-group-vars-yml-path={{ templates_directory_path }}/group_vars_mash_servers \ + --dst-group-vars-yml-path={{ justfile_directory() }}/group_vars/mash_servers + # Updates requirements.yml if there are any new tags available. Requires agru update: && opml - @agru -r {{ justfile_directory() }}/requirements.all.yml -u + @agru -r {{ templates_directory_path }}/requirements.yml -u # Runs ansible-lint against all roles in the playbook lint: @@ -39,7 +95,7 @@ install-service service *extra_args: setup-all *extra_args: (run-tags "setup-all,start" extra_args) # Runs the playbook with the given list of arguments -run +extra_args: requirements-yml setup-yml group-vars-mash-servers +run +extra_args: _requirements-yml _setup-yml _group-vars-mash-servers ansible-playbook -i inventory/hosts setup.yml {{ extra_args }} # Runs the playbook with the given list of comma-separated tags and optional arguments @@ -61,20 +117,21 @@ stop-group group *extra_args: @just --justfile {{ justfile() }} run-tags stop-group --extra-vars="group={{ group }}" {{ extra_args }} # Prepares the requirements.yml file -requirements-yml: - @just --justfile {{ justfile() }} _ensure_file_prepared {{ justfile_directory() }}/requirements.all.yml {{ justfile_directory() }}/requirements.yml +_requirements-yml: + @just --justfile {{ justfile() }} _ensure_file_prepared {{ templates_directory_path }}/requirements.yml {{ justfile_directory() }}/requirements.yml # Prepares the setup.yml file -setup-yml: - @just --justfile {{ justfile() }} _ensure_file_prepared {{ justfile_directory() }}/setup.all.yml {{ justfile_directory() }}/setup.yml +_setup-yml: + @just --justfile {{ justfile() }} _ensure_file_prepared {{ templates_directory_path }}/setup.yml {{ justfile_directory() }}/setup.yml # Prepares the group_vars/mash_servers file -group-vars-mash-servers: - @just --justfile {{ justfile() }} _ensure_file_prepared {{ justfile_directory() }}/group_vars/mash_servers_all {{ justfile_directory() }}/group_vars/mash_servers +_group-vars-mash-servers: + @just --justfile {{ justfile() }} _ensure_file_prepared {{ templates_directory_path }}/group_vars_mash_servers {{ justfile_directory() }}/group_vars/mash_servers _ensure_file_prepared src_path dst_path: #!/usr/bin/env sh - hash_path={{ dst_path }}.srchash + dst_file_name=$(basename "{{ dst_path }}") + hash_path={{ run_directory_path }}"/"$dst_file_name".srchash" src_hash=$(md5sum {{ src_path }} | cut -d ' ' -f 1) if [ ! -f "{{ dst_path }}" ] || [ ! -f "$hash_path" ]; then @@ -86,5 +143,32 @@ _ensure_file_prepared src_path dst_path: if [ "$current_hash" != "$src_hash" ]; then cp {{ src_path }} {{ dst_path }} echo $src_hash > $hash_path + + if [ -f "$optimization_vars_files_file_path" ]; then + just --justfile {{ justfile() }} \ + _optimize-for-var-paths \ + $(cat $optimization_vars_files_file_path) + fi fi fi + +_save_hash_for_file src_path dst_path: + #!/usr/bin/env sh + dst_file_name=$(basename "{{ dst_path }}") + hash_path={{ run_directory_path }}"/"$dst_file_name".srchash" + src_hash=$(md5sum {{ src_path }} | cut -d ' ' -f 1) + echo $src_hash > $hash_path + +_clean_template_derived_files: + #!/usr/bin/env sh + if [ -f "{{ justfile_directory() }}/requirements.yml" ]; then + rm {{ justfile_directory() }}/requirements.yml + fi + + if [ -f "{{ justfile_directory() }}/setup.yml" ]; then + rm {{ justfile_directory() }}/setup.yml + fi + + if [ -f "{{ justfile_directory() }}/group_vars/mash_servers" ]; then + rm {{ justfile_directory() }}/group_vars/mash_servers + fi diff --git a/run/.gitkeep b/run/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/group_vars/mash_servers_all b/templates/group_vars_mash_servers similarity index 99% rename from group_vars/mash_servers_all rename to templates/group_vars_mash_servers index 32f1a00..3e74bff 100644 --- a/group_vars/mash_servers_all +++ b/templates/group_vars_mash_servers @@ -274,7 +274,7 @@ mash_playbook_devture_systemd_service_manager_services_list_auto_itemized: # /role-specific:gotosocial # role-specific:grafana - - |- + - |- {{ ({'name': (grafana_identifier + '.service'), 'priority': 2000, 'groups': ['mash', 'grafana']} if grafana_enabled else omit) }} # /role-specific:grafana @@ -370,7 +370,7 @@ mash_playbook_devture_systemd_service_manager_services_list_auto_itemized: # /role-specific:n8n # role-specific:navidrome - - |- + - |- {{ ({'name': (navidrome_identifier + '.service'), 'priority': 2000, 'groups': ['mash', 'navidrome']} if navidrome_enabled else omit) }} # /role-specific:navidrome diff --git a/requirements.all.yml b/templates/requirements.yml similarity index 100% rename from requirements.all.yml rename to templates/requirements.yml diff --git a/setup.all.yml b/templates/setup.yml similarity index 100% rename from setup.all.yml rename to templates/setup.yml