From 596f9bbbd050d7d4d84a375e7607983b1a9376aa Mon Sep 17 00:00:00 2001 From: meisnate12 Date: Mon, 24 Jul 2023 10:03:55 -0400 Subject: [PATCH] [70] merge everything --- CHANGELOG | 36 +++++++++++++++++++++++++++----- VERSION | 2 +- modules/config.py | 9 ++++---- modules/github.py | 49 ++++++++++++++++++++++++-------------------- modules/plex.py | 2 +- plex_meta_manager.py | 15 +++++++++++--- requirements.txt | 2 +- 7 files changed, 77 insertions(+), 38 deletions(-) diff --git a/CHANGELOG b/CHANGELOG index 2c72b317..ac118174 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,22 +1,48 @@ # Requirements Update (requirements will need to be reinstalled) +Updated GitPython requirement to 3.1.32 +Updated lxml requirement to 4.9.3 +Updated pathvalidate requirement to 3.1.0 +Updated pillow requirement to 10.0.0 +Updated PlexAPI requirement to 4.14.0 +Updated psutil requirement to 5.9.5 +Updated requests requirement to 2.31.0 +Updated ruamel.yaml requirement to 0.17.32 Updated schedule requirement to 1.2.0 Updated tmdbapis requirement to 1.2.0 -Updated requests requirement to 2.29.0 # New Features -Added `tmdb_vote_average` as a [Filter](https://metamanager.wiki/en/latest/metadata/filters.html#id7). thanks @tenshiak -Added `tmdb_birthday` [Detail](https://metamanager.wiki/en/latest/metadata/details/metadata.html) which controls if a collection is visible based on a `tmdb_person`'s birthday. +Add `tmdb_vote_average` as a [Filter](https://metamanager.wiki/en/latest/metadata/filters.html#id7). thanks @tenshiak +Add `tmdb_birthday` [Detail](https://metamanager.wiki/en/latest/metadata/details/metadata.html) which controls if a collection is visible based on a `tmdb_person`'s birthday. +Add `match` [attribute](https://metamanager.wiki/en/latest/metadata/metadata/movie.html#matching-movies) to match movies within Plex to items within the Metadata file. +Add `delete_collections_named` to [Shared Collection Variables](https://metamanager.wiki/en/latest/defaults/collection_variables.html) +Add support for (GitHub Personal Access Tokens)[https://metamanager.wiki/en/latest/config/github.html] in config files. thanks @chazlarson # New Defaults Features New Default Translations of Danish, Dutch, Italian, Norwegian Bokmål, Portuguese (Brazil), and Spanish. -New [Network](https://metamanager.wiki/en/latest/defaults/overlays/network.html) Default Overlay. +New [Network](https://metamanager.wiki/en/latest/defaults/overlays/network.html) Default Overlay. thanks @bullmoose20, @JohnFawkes and @arial-Z New Content Rating US [Movies](https://metamanager.wiki/en/latest/defaults/overlays/content_rating_us_movie.html)/[Shows](https://metamanager.wiki/en/latest/defaults/overlays/content_rating_us_show.html) and [Content Rating UK](https://metamanager.wiki/en/latest/defaults/overlays/content_rating_uk.html) Default Overlay. Add `file_poster`, `file_poster_<>`, `url_background`, `url_background_<>`, `file_background`, `file_background_<>`, `sort_prefix`, `sort_title`, and `name_mapping` as [Collection Variables](https://metamanager.wiki/en/latest/defaults/collection_variables.html). -Added new Separator Color Styles. +Add new Separator Color Styles. +Standardized overlay sizes for `- pmm: networks` and `- pmm: streaming` +Rebrand HBO Max to Max. +Add additional studios and networks +Add Heritage Months to [Seasonal Defaults File](https://metamanager.wiki/en/latest/defaults/movie/seasonal.html) # Bug Fixes +Fixes #1496 Fixes issue with IMDb Collections not building due to HTML structure changes. +Fixes #1445 Fixes FlixPatrol failed to parse error for most users. Fixes #1379 Fixes a Docstring. +Fixes #1478 Fixes issue with percent sign not appearing on some rating overlays. +Fixes #1490 Fixes incorrect language code for Philippines in Languages Defaults file. +Fixes #1378 Fixes incorrect default value for starting attribute in "Year" collections. +Fixes #1379 #1380 #1472 Fixes various docs issues and typos +Fixes #1518 `radarr_remove_by_tag` and `sonarr_remove_by_tag` being validated as bool instead of strings. thanks @chazlarson +Fixes #1531 Fixes secrets not being redacted from Run Command in log files +Fixes issue with validating filenames as part of `dimensional_asset_rename`. thanks @chazlarson +Fixes an issue where renamed collections could be incorrectly deleted. +Fixes bug when defining users for `sync_to_users` for Playlists. +Fixes bug where using mdb or metacritic as rating source on Ratings Overlay would not work correctly. Fixes bug where TMDb Filters using a modifier were ignored. Fixes bug with `mal_search` when using the `sfw` attribute Fixes bug where sorting a collection would fail if the collection name had a `,` in it. diff --git a/VERSION b/VERSION index 87bf9c4f..4946a2f8 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.19.0-develop69 +1.19.0-develop70 diff --git a/modules/config.py b/modules/config.py index bd9fd577..87394865 100644 --- a/modules/config.py +++ b/modules/config.py @@ -296,10 +296,9 @@ class ConfigFile: check_next(d) else: for secret, secret_value in self.secrets.items(): - if f"<<{secret}>>" in str(next_data): - return str(next_data).replace(f"<<{secret}>>", secret_value) - elif f"<<{secret.upper()}>>" in str(next_data): - return str(next_data).replace(f"<<{secret.upper()}>>", secret_value) + for test in [secret, secret.upper()]: + if f"<<{test}>>" in str(next_data): + return str(next_data).replace(f"<<{test}>>", secret_value) return next_data if self.secrets: check_next(self.data) @@ -453,7 +452,7 @@ class ConfigFile: self.Cache = Cache(self.config_path, self.general["cache_expiration"]) else: self.Cache = None - self.GitHub = GitHub(self) + self.GitHub = GitHub(self, {"token": check_for_attribute(self.data, "token", parent="github", default_is_none=True)}) logger.separator() diff --git a/modules/github.py b/modules/github.py index fd0a0267..80b5c66b 100644 --- a/modules/github.py +++ b/modules/github.py @@ -10,14 +10,11 @@ pmm_base = f"{base_url}/repos/meisnate12/Plex-Meta-Manager" configs_raw_url = f"{raw_url}/meisnate12/Plex-Meta-Manager-Configs" class GitHub: - def __init__(self, config): + def __init__(self, config, params): self.config = config - self.headers = None - try: - self.token = config.data["github"]["token"] - self.headers = {'Authorization': 'token ' + self.token} - except: - self.token = None + self.token = params["token"] + logger.secret(self.token) + self.headers = {"Authorization": f"token {self.token}"} if self.token else None self.images_raw_url = f"{raw_url}/meisnate12/PMM-Image-Sets/master/sets/" self.translation_url = f"{raw_url}/meisnate12/PMM-Translations/master/defaults/" self._configs_url = None @@ -25,35 +22,43 @@ class GitHub: self._translation_keys = [] self._translations = {} + def _requests(self, url, err_msg=None, json=True, params=None): + response = self.config.get(url, headers=self.headers, params=params) + if not err_msg: + err_msg = f"URL Not Found: {url}" + if response.status_code >= 400: + raise Failed(f"Git Error: {err_msg}") + if json: + try: + return response.json() + except ValueError: + logger.error(str(response.content)) + raise + return response + def get_top_tree(self, repo): if not str(repo).startswith("/"): repo = f"/{repo}" if not str(repo).endswith("/"): repo = f"{repo}/" - response = self.config.get(f"{base_url}/repos{repo}commits", headers=self.headers) - if response.status_code >= 400: - raise Failed(f"Git Error: No repo found at https://github.com{repo}") - return self.get_tree(response.json()[0]["commit"]["tree"]["url"]), repo + data = self._requests(f"{base_url}/repos{repo}commits", f"No repo found at https://github.com{repo}") + return self.get_tree(data[0]["commit"]["tree"]["url"]), repo def get_tree(self, tree_url): - response = self.config.get(tree_url) - if response.status_code >= 400: - raise Failed(f"Git Error: No tree found at {tree_url}") - return {i["path"]: i for i in response.json()["tree"]} + return {i["path"]: i for i in self._requests(tree_url, f"No tree found at {tree_url}")["tree"]} def latest_release_notes(self): - response = self.config.get_json(f"{pmm_base}/releases/latest", headers=self.headers) - return response["body"] + return self._requests(f"{pmm_base}/releases/latest")["body"] def get_commits(self, dev_version, nightly=False): - master_sha = self.config.get_json(f"{pmm_base}/commits/master", headers=self.headers)["sha"] - response = self.config.get_json(f"{pmm_base}/commits", headers=self.headers, params={"sha": "nightly" if nightly else "develop"}) + master_sha = self._requests(f"{pmm_base}/commits/master")["sha"] + response = self._requests(f"{pmm_base}/commits", params={"sha": "nightly" if nightly else "develop"}) commits = [] for commit in response: if commit["sha"] == master_sha: break message = commit["commit"]["message"] - match = re.match("^\\[(\\d)\\]", message) + match = re.match(r"^\[(\d)]", message) if match and int(match.group(1)) <= dev_version: break commits.append(message) @@ -63,7 +68,7 @@ class GitHub: def config_tags(self): if not self._config_tags: try: - self._config_tags = [r["ref"][11:] for r in self.config.get_json(f"{pmm_base}-Configs/git/refs/tags", headers=self.headers)] + self._config_tags = [r["ref"][11:] for r in self._requests(f"{pmm_base}-Configs/git/refs/tags")] except TypeError: pass return self._config_tags @@ -89,7 +94,7 @@ class GitHub: def translation_yaml(self, translation_key): if translation_key not in self._translations: url = f"{self.translation_url}{translation_key}.yml" - yaml = util.YAML(input_data=self.config.get(url, headers=self.headers).content).data + yaml = util.YAML(input_data=self._requests(url, json=False).content).data output = {"collections": {}, "key_names": {}, "variables": {}} for k in output: if k in yaml: diff --git a/modules/plex.py b/modules/plex.py index 03f7a928..bc6fac1c 100644 --- a/modules/plex.py +++ b/modules/plex.py @@ -1405,7 +1405,7 @@ class Plex(Library): if is_top_level and self.asset_folders and self.dimensional_asset_rename and (not poster or not background): for file in util.glob_filter(os.path.join(item_asset_directory, "*.*")): - if file.lower().endswith((".png", ".jpg", ".jpeg", "webp")) and not re.match(r"s\d+e\d+|season\d+", file.lower()): + if file.lower().endswith((".png", ".jpg", ".jpeg", "webp")) and not re.match(r"s\d+e\d+|season\d+", os.path.basename(file).lower()): try: with Image.open(file) as image: _w, _h = image.size diff --git a/plex_meta_manager.py b/plex_meta_manager.py index 133f886e..3599906c 100644 --- a/plex_meta_manager.py +++ b/plex_meta_manager.py @@ -95,7 +95,6 @@ except ImportError: env_version = get_arg("BRANCH_NAME", "master") is_docker = get_arg("PMM_DOCKER", False, arg_bool=True) is_linuxserver = get_arg("PMM_LINUXSERVER", False, arg_bool=True) -run_arg = " ".join([f'"{s}"' if " " in s else s for s in sys.argv[:]]) config_file = get_arg("PMM_CONFIG", args.config) times = get_arg("PMM_TIME", args.times) run = get_arg("PMM_RUN", args.run, arg_bool=True) @@ -130,7 +129,7 @@ plex_url = None plex_token = None i = 0 while i < len(unknown): - test_var = str(unknown[i]).lower() + test_var = str(unknown[i]).lower().replace("_", "-") if test_var.startswith("--pmm-") or test_var in ["-pu", "--plex-url", "-pt", "--plex-token"]: if test_var in ["-pu", "--plex-url"]: plex_url = str(unknown[i + 1]) @@ -144,9 +143,15 @@ while i < len(unknown): plex_url = get_arg("PMM_PLEX_URL", plex_url) plex_token = get_arg("PMM_PLEX_TOKEN", plex_token) +env_secrets = [] for env_name, env_data in os.environ.items(): if str(env_name).upper().startswith("PMM_") and str(env_name).upper() not in static_envs: - secret_args[str(env_name).lower()[4:]] = env_data + secret_args[str(env_name).lower()[4:].replace("_", "-")] = env_data + +run_arg = " ".join([f'"{s}"' if " " in s else s for s in sys.argv[:]]) +for _, v in secret_args.items(): + if v in run_arg: + run_arg.replace(v, "(redacted)") if collections: collection_only = True @@ -287,6 +292,10 @@ def start(attrs): logger.debug(f"--debug (PMM_DEBUG): {debug}") logger.debug(f"--trace (PMM_TRACE): {trace}") logger.debug("") + logger.debug("PMM Secrets Read:") + for sec in secret_args: + logger.debug(f"--pmm-{sec} (PMM_{sec.upper().replace('-', '_')}): (redacted)") + logger.debug("") logger.separator(f"Starting {start_type}Run") config = None stats = {"created": 0, "modified": 0, "deleted": 0, "added": 0, "unchanged": 0, "removed": 0, "radarr": 0, "sonarr": 0, "names": []} diff --git a/requirements.txt b/requirements.txt index 2e3693f8..852ca25f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,7 @@ arrapi==1.4.2 GitPython==3.1.32 lxml==4.9.3 num2words==0.5.12 -pathvalidate==3.0.0 +pathvalidate==3.1.0 pillow==10.0.0 PlexAPI==4.14.0 psutil==5.9.5