From 4e7c396717c052bc300812e7fb9d4aa6a917348f Mon Sep 17 00:00:00 2001 From: meisnate12 Date: Thu, 30 Nov 2023 17:08:14 -0500 Subject: [PATCH] [34] added run_order setting --- VERSION | 2 +- config/config.yml.template | 1 + modules/config.py | 26 ++++++-- modules/library.py | 1 + plex_meta_manager.py | 119 ++++++++++++++++++------------------- 5 files changed, 81 insertions(+), 68 deletions(-) diff --git a/VERSION b/VERSION index 27a30319..70e0fd49 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.19.1-develop33 +1.19.1-develop34 diff --git a/config/config.yml.template b/config/config.yml.template index 54fa3e18..a522f228 100644 --- a/config/config.yml.template +++ b/config/config.yml.template @@ -33,6 +33,7 @@ playlist_files: libraries: Movies, TV Shows # list of libraries that you want the PMM Defaults playlists to look at # see the wiki for how to use local files, folders, URLs, or files from git settings: + run_order: metadata, overlays, operations cache: true cache_expiration: 60 asset_directory: config/assets diff --git a/modules/config.py b/modules/config.py index b3012d10..bdc7ad6f 100644 --- a/modules/config.py +++ b/modules/config.py @@ -33,6 +33,11 @@ from retrying import retry logger = util.logger mediastingers_url = "https://raw.githubusercontent.com/meisnate12/PMM-Mediastingers/master/stingers.yml" +run_order_options = { + "metadata": "Represents Collection and Metadata Updates", + "overlays": "Represents Overlay Updates", + "operations": "Represents Operations Updates" +} sync_modes = {"append": "Only Add Items to the Collection or Playlist", "sync": "Add & Remove Items from the Collection or Playlist"} imdb_label_options = { "none": "Add IMDb Parental Labels for None, Mild, Moderate, or Severe", @@ -153,6 +158,7 @@ class ConfigFile: self.collection_only = attrs["collection_only"] if "collection_only" in attrs else False self.operations_only = attrs["operations_only"] if "operations_only" in attrs else False self.overlays_only = attrs["overlays_only"] if "overlays_only" in attrs else False + self.libraries_first = attrs["libraries_first"] if "libraries_first" in attrs else False self.env_plex_url = attrs["plex_url"] if "plex_url" in attrs else "" self.env_plex_token = attrs["plex_token"] if "plex_token" in attrs else "" current_time = datetime.now() @@ -341,9 +347,13 @@ class ConfigFile: elif var_type == "path": if os.path.exists(os.path.abspath(data[attribute])): return data[attribute] else: message = f"Path {os.path.abspath(data[attribute])} does not exist" - elif var_type == "list": return util.get_list(data[attribute], split=False) - elif var_type == "comma_list": return util.get_list(data[attribute]) - elif var_type == "int_list": return util.get_list(data[attribute], int_list=True) + elif var_type in ["list", "comma_list", "int_list"]: + output_list = list(set(util.get_list(data[attribute], lower=var_type != "int_list", split=var_type != "list", int_list=var_type == "int_list"))) + failed_items = [o for o in output_list if o not in test_list] if test_list else [] + if failed_items: + message = f"{text}: {', '.join(failed_items)} is an invalid input" + else: + return output_list elif var_type == "list_path": temp_list = [] warning_message = "" @@ -390,7 +400,10 @@ class ConfigFile: logger.warning(options) return default + default_run = ["overlays", "operations", "metadata"] if self.libraries_first else ["metadata", "overlays", "operations"] + self.general = { + "run_order": check_for_attribute(self.data, "run_order", parent="settings", var_type="comma_list", test_list=run_order_options, default=default_run), "cache": check_for_attribute(self.data, "cache", parent="settings", var_type="bool", default=True), "cache_expiration": check_for_attribute(self.data, "cache_expiration", parent="settings", var_type="int", default=60, int_min=1), "asset_directory": check_for_attribute(self.data, "asset_directory", parent="settings", var_type="list_path", default_is_none=True), @@ -421,7 +434,7 @@ class ConfigFile: "save_report": check_for_attribute(self.data, "save_report", parent="settings", var_type="bool", default=False), "tvdb_language": check_for_attribute(self.data, "tvdb_language", parent="settings", default="default"), "ignore_ids": check_for_attribute(self.data, "ignore_ids", parent="settings", var_type="int_list", default_is_none=True), - "ignore_imdb_ids": check_for_attribute(self.data, "ignore_imdb_ids", parent="settings", var_type="list", default_is_none=True), + "ignore_imdb_ids": check_for_attribute(self.data, "ignore_imdb_ids", parent="settings", var_type="comma_list", default_is_none=True), "playlist_sync_to_users": check_for_attribute(self.data, "playlist_sync_to_users", parent="settings", default="all", default_is_none=True), "playlist_exclude_users": check_for_attribute(self.data, "playlist_exclude_users", parent="settings", default_is_none=True), "playlist_report": check_for_attribute(self.data, "playlist_report", parent="settings", var_type="bool", default=True), @@ -722,7 +735,8 @@ class ConfigFile: logger.info("") logger.info(f"Connecting to {display_name} Library...") - params["asset_directory"] = check_for_attribute(lib, "asset_directory", parent="settings", var_type="list_path", default=self.general["asset_directory"], default_is_none=True, save=False) + params["run_order"] = check_for_attribute(lib, "run_order", parent="settings", var_type="comma_list", default=self.general["run_order"], do_print=False, save=False) + params["asset_directory"] = check_for_attribute(lib, "asset_directory", parent="settings", var_type="list_path", default=self.general["asset_directory"], default_is_none=True, do_print=False, save=False) params["asset_folders"] = check_for_attribute(lib, "asset_folders", parent="settings", var_type="bool", default=self.general["asset_folders"], do_print=False, save=False) params["asset_depth"] = check_for_attribute(lib, "asset_depth", parent="settings", var_type="int", default=self.general["asset_depth"], do_print=False, save=False) params["sync_mode"] = check_for_attribute(lib, "sync_mode", parent="settings", test_list=sync_modes, default=self.general["sync_mode"], do_print=False, save=False) @@ -749,7 +763,7 @@ class ConfigFile: params["delete_not_scheduled"] = check_for_attribute(lib, "delete_not_scheduled", parent="settings", var_type="bool", default=self.general["delete_not_scheduled"], do_print=False, save=False) params["ignore_ids"] = check_for_attribute(lib, "ignore_ids", parent="settings", var_type="int_list", default_is_none=True, do_print=False, save=False) params["ignore_ids"].extend([i for i in self.general["ignore_ids"] if i not in params["ignore_ids"]]) - params["ignore_imdb_ids"] = check_for_attribute(lib, "ignore_imdb_ids", parent="settings", var_type="list", default_is_none=True, do_print=False, save=False) + params["ignore_imdb_ids"] = check_for_attribute(lib, "ignore_imdb_ids", parent="settings", var_type="comma_list", default_is_none=True, do_print=False, save=False) params["ignore_imdb_ids"].extend([i for i in self.general["ignore_imdb_ids"] if i not in params["ignore_imdb_ids"]]) params["changes_webhooks"] = check_for_attribute(lib, "changes", parent="webhooks", var_type="list", default=self.webhooks["changes"], do_print=False, save=False, default_is_none=True) params["report_path"] = None diff --git a/modules/library.py b/modules/library.py index b8565cd7..ee4d0b95 100644 --- a/modules/library.py +++ b/modules/library.py @@ -54,6 +54,7 @@ class Library(ABC): self.overlay_backup = os.path.join(self.overlay_folder, f"{self.mapping_name} Original Posters") self.report_path = params["report_path"] if params["report_path"] else os.path.join(self.default_dir, f"{self.mapping_name}_report.yml") self.report_data = {} + self.run_order = params["run_order"] self.asset_folders = params["asset_folders"] self.create_asset_folders = params["create_asset_folders"] self.dimensional_asset_rename = params["dimensional_asset_rename"] diff --git a/plex_meta_manager.py b/plex_meta_manager.py index 42ddaaa8..cdc02043 100644 --- a/plex_meta_manager.py +++ b/plex_meta_manager.py @@ -38,7 +38,7 @@ arguments = { "run-collections": {"args": ["rc", "cl", "collection", "collections", "run-collection"], "type": "str", "help": "Process only specified collections (pipe-separated list '|')"}, "run-libraries": {"args": ["rl", "l", "library", "libraries", "run-library"], "type": "str", "help": "Process only specified libraries (pipe-separated list '|')"}, "run-metadata-files": {"args": ["rm", "m", "metadata", "metadata-files"], "type": "str", "help": "Process only specified Metadata files (pipe-separated list '|')"}, - "libraries-first": {"args": ["lf", "library-first"], "type": "bool", "help": "Run library operations before collections"}, + "libraries-first": {"args": ["lf", "library-first"], "type": "bool", "help": argparse.SUPPRESS}, "ignore-schedules": {"args": "is", "type": "bool", "help": "Run ignoring collection schedules"}, "ignore-ghost": {"args": "ig", "type": "bool", "help": "Run ignoring ghost logging"}, "delete-collections": {"args": ["dc", "delete", "delete-collection"], "type": "bool", "help": "Deletes all Collections in the Plex Library before running"}, @@ -269,13 +269,15 @@ def start(attrs): attrs["playlist_only"] = run_args["playlists-only"] attrs["operations_only"] = run_args["operations-only"] attrs["overlays_only"] = run_args["overlays-only"] + attrs["libraries_first"] = run_args["libraries-first"] attrs["plex_url"] = plex_url attrs["plex_token"] = plex_token logger.separator(debug=True) logger.debug(f"Run Command: {run_arg}") for akey, adata in arguments.items(): - ext = '"' if adata["type"] == "str" and run_args[akey] not in [None, "None"] else "" - logger.debug(f"--{akey} (PMM_{akey.upper()}): {ext}{run_args[akey]}{ext}") + if isinstance(adata["help"], str): + ext = '"' if adata["type"] == "str" and run_args[akey] not in [None, "None"] else "" + logger.debug(f"--{akey} (PMM_{akey.upper()}): {ext}{run_args[akey]}{ext}") logger.debug("") if secret_args: logger.debug("PMM Secrets Read:") @@ -501,6 +503,7 @@ def run_libraries(config): logger.debug("") logger.debug(f"Library Name: {library.name}") + logger.debug(f"Run Order: {', '.join(library.run_order)}") logger.debug(f"Folder Name: {library.mapping_name}") for ad in library.asset_directory: logger.debug(f"Asset Directory: {ad}") @@ -561,9 +564,8 @@ def run_libraries(config): time_start = datetime.now() temp_items = None list_key = None - expired = None if config.Cache: - list_key, expired = config.Cache.query_list_cache("library", library.mapping_name, 1) + list_key, _ = config.Cache.query_list_cache("library", library.mapping_name, 1) if not temp_items: temp_items = library.cache_items() @@ -576,65 +578,60 @@ def run_libraries(config): library.map_guids(temp_items) library_status[library.name]["Library Loading and Mapping"] = str(datetime.now() - time_start).split('.')[0] - def run_operations_and_overlays(): - if not run_args["tests"] and not run_args["collections-only"] and not run_args["playlists-only"] and not config.requested_metadata_files: - if not run_args["overlays-only"] and library.library_operation: + for run_type in library.run_order: + if run_type == "metadata": + if not run_args["operations-only"] and not run_args["overlays-only"] and not run_args["playlists-only"]: + time_start = datetime.now() + for images in library.images_files: + images_name = images.get_file_name() + if config.requested_metadata_files and images_name not in config.requested_metadata_files: + logger.info("") + logger.separator(f"Skipping {images_name} Images File") + continue + logger.info("") + logger.separator(f"Running {images_name} Images File\n{images.path}") + if not run_args["tests"] and not run_args["resume"] and not run_args["collections-only"]: + try: + images.update_metadata() + except Failed as e: + library.notify(e) + logger.error(e) + library_status[library.name]["Library Images Files"] = str(datetime.now() - time_start).split('.')[0] + + time_start = datetime.now() + for metadata in library.metadata_files: + metadata_name = metadata.get_file_name() + if config.requested_metadata_files and metadata_name not in config.requested_metadata_files: + logger.info("") + logger.separator(f"Skipping {metadata_name} Metadata File") + continue + logger.info("") + logger.separator(f"Running {metadata_name} Metadata File\n{metadata.path}") + if not run_args["tests"] and not run_args["resume"] and not run_args["collections-only"]: + try: + metadata.update_metadata() + except Failed as e: + library.notify(e) + logger.error(e) + collections_to_run = metadata.get_collections(config.requested_collections) + if run_args["resume"] and run_args["resume"] not in collections_to_run: + logger.info("") + logger.warning(f"Collection: {run_args['resume']} not in Metadata File: {metadata.path}") + continue + if collections_to_run: + logger.info("") + logger.separator(f"{'Test ' if run_args['tests'] else ''}Collections") + # logger.remove_library_handler(library.mapping_name) + run_collection(config, library, metadata, collections_to_run) + # logger.re_add_library_handler(library.mapping_name) + library_status[library.name]["Library Metadata Files"] = str(datetime.now() - time_start).split('.')[0] + elif run_type == "overlays": + if not run_args["tests"] and not run_args["collections-only"] and not run_args["playlists-only"] and not config.requested_metadata_files and not run_args["overlays-only"] and library.library_operation: library_status[library.name]["Library Operations"] = library.Operations.run_operations() - if not run_args["operations-only"] and (library.overlay_files or library.remove_overlays): + elif run_type == "operations": + if not run_args["tests"] and not run_args["collections-only"] and not run_args["playlists-only"] and not config.requested_metadata_files and not run_args["operations-only"] and (library.overlay_files or library.remove_overlays): library_status[library.name]["Library Overlays"] = library.Overlays.run_overlays() - if run_args["libraries-first"]: - run_operations_and_overlays() - - if not run_args["operations-only"] and not run_args["overlays-only"] and not run_args["playlists-only"]: - time_start = datetime.now() - for images in library.images_files: - images_name = images.get_file_name() - if config.requested_metadata_files and images_name not in config.requested_metadata_files: - logger.info("") - logger.separator(f"Skipping {images_name} Images File") - continue - logger.info("") - logger.separator(f"Running {images_name} Images File\n{images.path}") - if not run_args["tests"] and not run_args["resume"] and not run_args["collections-only"]: - try: - images.update_metadata() - except Failed as e: - library.notify(e) - logger.error(e) - library_status[library.name]["Library Images Files"] = str(datetime.now() - time_start).split('.')[0] - - time_start = datetime.now() - for metadata in library.metadata_files: - metadata_name = metadata.get_file_name() - if config.requested_metadata_files and metadata_name not in config.requested_metadata_files: - logger.info("") - logger.separator(f"Skipping {metadata_name} Metadata File") - continue - logger.info("") - logger.separator(f"Running {metadata_name} Metadata File\n{metadata.path}") - if not run_args["tests"] and not run_args["resume"] and not run_args["collections-only"]: - try: - metadata.update_metadata() - except Failed as e: - library.notify(e) - logger.error(e) - collections_to_run = metadata.get_collections(config.requested_collections) - if run_args["resume"] and run_args["resume"] not in collections_to_run: - logger.info("") - logger.warning(f"Collection: {run_args['resume']} not in Metadata File: {metadata.path}") - continue - if collections_to_run: - logger.info("") - logger.separator(f"{'Test ' if run_args['tests'] else ''}Collections") - #logger.remove_library_handler(library.mapping_name) - run_collection(config, library, metadata, collections_to_run) - #logger.re_add_library_handler(library.mapping_name) - library_status[library.name]["Library Metadata Files"] = str(datetime.now() - time_start).split('.')[0] - - if not run_args["libraries-first"]: - run_operations_and_overlays() - #logger.remove_library_handler(library.mapping_name) except Exception as e: library.notify(e)