From 866ce010cc9f0c6756b4b36b733aa5e166b2a6a5 Mon Sep 17 00:00:00 2001 From: meisnate12 Date: Wed, 29 May 2024 16:56:56 -0400 Subject: [PATCH] [1] Update to using Build Numbers (#2081) --- .../{develop.yml => docker-develop.yml} | 0 .../{latest.yml => docker-latest.yml} | 0 .../{nightly.yml => docker-nightly.yml} | 0 .github/workflows/docker-version.yml | 97 ++++++++++++ .github/workflows/version.yml | 121 +++++---------- VERSION | 2 +- kometa.py | 24 +-- modules/config.py | 2 +- modules/github.py | 4 +- modules/request.py | 138 ++++++++++-------- modules/webhooks.py | 10 +- 11 files changed, 235 insertions(+), 163 deletions(-) rename .github/workflows/{develop.yml => docker-develop.yml} (100%) rename .github/workflows/{latest.yml => docker-latest.yml} (100%) rename .github/workflows/{nightly.yml => docker-nightly.yml} (100%) create mode 100644 .github/workflows/docker-version.yml diff --git a/.github/workflows/develop.yml b/.github/workflows/docker-develop.yml similarity index 100% rename from .github/workflows/develop.yml rename to .github/workflows/docker-develop.yml diff --git a/.github/workflows/latest.yml b/.github/workflows/docker-latest.yml similarity index 100% rename from .github/workflows/latest.yml rename to .github/workflows/docker-latest.yml diff --git a/.github/workflows/nightly.yml b/.github/workflows/docker-nightly.yml similarity index 100% rename from .github/workflows/nightly.yml rename to .github/workflows/docker-nightly.yml diff --git a/.github/workflows/docker-version.yml b/.github/workflows/docker-version.yml new file mode 100644 index 00000000..83ae75f2 --- /dev/null +++ b/.github/workflows/docker-version.yml @@ -0,0 +1,97 @@ +name: Docker Version Release + +on: + create: + tags: + - v* + +jobs: + + docker-build: + if: ${{ startsWith(github.ref, 'refs/tags/v1') || startsWith(github.ref, 'refs/tags/v2') }} + runs-on: ubuntu-latest + steps: + + - name: Check Out Repo + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Login to Docker Hub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKER_HUB_USERNAME }} + password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} + + - name: Set up QEMU + uses: docker/setup-qemu-action@master + with: + platforms: all + + - name: Set up Docker Buildx + id: buildx + uses: docker/setup-buildx-action@v3 + + - name: Get the version + id: get_version + run: echo "VERSION=${GITHUB_REF/refs\/tags\//}" >> $GITHUB_OUTPUT + + - name: Build and Push + id: docker_build + uses: docker/build-push-action@v5 + with: + context: ./ + file: ./Dockerfile + platforms: linux/amd64,linux/arm64,linux/arm/v7 + push: true + tags: kometateam/kometa:${{ steps.get_version.outputs.VERSION }} + cache-from: type=gha + cache-to: type=gha,mode=max + + - name: Discord Success Notification + uses: Kometa-Team/discord-notifications@master + if: success() + with: + webhook_id_token: ${{ secrets.BUILD_WEBHOOK }} + title: "${{ vars.REPO_NAME }} ${{ steps.get_version.outputs.VERSION }}: ${{ vars.TEXT_SUCCESS }}" + url: https://github.com/Kometa-Team/${{ vars.REPO_NAME }}/actions/runs/${{ github.run_id }} + color: ${{ vars.COLOR_SUCCESS }} + username: ${{ vars.BOT_NAME }} + avatar_url: ${{ vars.BOT_IMAGE }} + author: ${{ vars.DOCKER_NAME }} + author_icon_url: ${{ vars.DOCKER_IMAGE }} + + - name: Discord Failure Notification + uses: Kometa-Team/discord-notifications@master + if: failure() + with: + webhook_id_token: ${{ secrets.BUILD_WEBHOOK }} + message: ${{ vars.BUILD_FAILURE_ROLE }} + title: "${{ vars.REPO_NAME }} ${{ steps.get_version.outputs.VERSION }}: ${{ vars.TEXT_FAILURE }}" + url: https://github.com/Kometa-Team/${{ vars.REPO_NAME }}/actions/runs/${{ github.run_id }} + color: ${{ vars.COLOR_FAILURE }} + username: ${{ vars.BOT_NAME }} + avatar_url: ${{ vars.BOT_IMAGE }} + author: ${{ vars.DOCKER_NAME }} + author_icon_url: ${{ vars.DOCKER_IMAGE }} + + - name: Checkout Configs Repo + uses: actions/checkout@v4 + with: + repository: Kometa-Team/Community-Configs + token: ${{ secrets.PAT }} + path: kometaconfigs + + - name: Create & Push Tag + working-directory: ./kometaconfigs + run: | + git tag ${{ steps.get_version.outputs.VERSION }} + git push origin ${{ steps.get_version.outputs.VERSION }} + + - name: Create release + id: create_release + uses: softprops/action-gh-release@v2 + with: + body_path: CHANGELOG + token: ${{ secrets.PAT }} + tag_name: ${{ steps.get_version.outputs.VERSION }} \ No newline at end of file diff --git a/.github/workflows/version.yml b/.github/workflows/version.yml index 83ae75f2..38c18da9 100644 --- a/.github/workflows/version.yml +++ b/.github/workflows/version.yml @@ -1,97 +1,50 @@ -name: Docker Version Release +name: Update Build Version on: - create: - tags: - - v* + pull_request: + branches: [nightly] + types: [closed] jobs: - - docker-build: - if: ${{ startsWith(github.ref, 'refs/tags/v1') || startsWith(github.ref, 'refs/tags/v2') }} + increment-version: runs-on: ubuntu-latest + if: github.base_ref == 'nightly' && github.event.pull_request.merged steps: + - name: Create App Token + uses: actions/create-github-app-token@v1 + id: app-token + with: + app-id: ${{ vars.APP_ID }} + private-key: ${{ secrets.APP_TOKEN }} + - name: Check Out Repo uses: actions/checkout@v4 with: + token: ${{ steps.app-token.outputs.token }} + ref: nightly fetch-depth: 0 - - name: Login to Docker Hub - uses: docker/login-action@v3 - with: - username: ${{ secrets.DOCKER_HUB_USERNAME }} - password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} - - - name: Set up QEMU - uses: docker/setup-qemu-action@master - with: - platforms: all - - - name: Set up Docker Buildx - id: buildx - uses: docker/setup-buildx-action@v3 - - - name: Get the version - id: get_version - run: echo "VERSION=${GITHUB_REF/refs\/tags\//}" >> $GITHUB_OUTPUT - - - name: Build and Push - id: docker_build - uses: docker/build-push-action@v5 - with: - context: ./ - file: ./Dockerfile - platforms: linux/amd64,linux/arm64,linux/arm/v7 - push: true - tags: kometateam/kometa:${{ steps.get_version.outputs.VERSION }} - cache-from: type=gha - cache-to: type=gha,mode=max - - - name: Discord Success Notification - uses: Kometa-Team/discord-notifications@master - if: success() - with: - webhook_id_token: ${{ secrets.BUILD_WEBHOOK }} - title: "${{ vars.REPO_NAME }} ${{ steps.get_version.outputs.VERSION }}: ${{ vars.TEXT_SUCCESS }}" - url: https://github.com/Kometa-Team/${{ vars.REPO_NAME }}/actions/runs/${{ github.run_id }} - color: ${{ vars.COLOR_SUCCESS }} - username: ${{ vars.BOT_NAME }} - avatar_url: ${{ vars.BOT_IMAGE }} - author: ${{ vars.DOCKER_NAME }} - author_icon_url: ${{ vars.DOCKER_IMAGE }} - - - name: Discord Failure Notification - uses: Kometa-Team/discord-notifications@master - if: failure() - with: - webhook_id_token: ${{ secrets.BUILD_WEBHOOK }} - message: ${{ vars.BUILD_FAILURE_ROLE }} - title: "${{ vars.REPO_NAME }} ${{ steps.get_version.outputs.VERSION }}: ${{ vars.TEXT_FAILURE }}" - url: https://github.com/Kometa-Team/${{ vars.REPO_NAME }}/actions/runs/${{ github.run_id }} - color: ${{ vars.COLOR_FAILURE }} - username: ${{ vars.BOT_NAME }} - avatar_url: ${{ vars.BOT_IMAGE }} - author: ${{ vars.DOCKER_NAME }} - author_icon_url: ${{ vars.DOCKER_IMAGE }} - - - name: Checkout Configs Repo - uses: actions/checkout@v4 - with: - repository: Kometa-Team/Community-Configs - token: ${{ secrets.PAT }} - path: kometaconfigs - - - name: Create & Push Tag - working-directory: ./kometaconfigs + - name: Update VERSION File run: | - git tag ${{ steps.get_version.outputs.VERSION }} - git push origin ${{ steps.get_version.outputs.VERSION }} - - - name: Create release - id: create_release - uses: softprops/action-gh-release@v2 - with: - body_path: CHANGELOG - token: ${{ secrets.PAT }} - tag_name: ${{ steps.get_version.outputs.VERSION }} \ No newline at end of file + value=$(cat VERSION) + old_msg=$(git log -1 HEAD --pretty=format:%s) + + if [[ "$value" == *"-"* ]]; then + build_value="$((${value#*-build} + 1))" + new_value="${value%-build*}-build${build_value}" + new_msg="[${build_value}] ${old_msg}" + elif [[ "$value" == *"!"* ]]; then + new_value="${value%!*}" + new_msg="[${new_value}] ${old_msg}" + else + new_value="${value%-build*}-build1" + new_msg="[1] ${old_msg}" + fi + + echo "$new_value" > "VERSION" + git config --local user.email "action@github.com" + git config --local user.name "GitHub Action" + git add VERSION + git commit -m "${new_msg}" --amend + git push origin nightly --force-with-lease diff --git a/VERSION b/VERSION index e9105543..322153f9 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.0.1-develop30 +-build1 diff --git a/kometa.py b/kometa.py index 15cb26f5..58aa35c5 100644 --- a/kometa.py +++ b/kometa.py @@ -133,7 +133,7 @@ for arg_key, arg_data in arguments.items(): final_vars = [f"KOMETA_{arg_key.replace('-', '_').upper()}"] + [f"KOMETA_{a.replace('-', '_').upper()}" for a in temp_args if len(a) > 2] run_args[arg_key] = get_env(final_vars, getattr(args, arg_key.replace("-", "_")), arg_bool=arg_data["type"] == "bool", arg_int=arg_data["type"] == "int") -env_version = get_env("BRANCH_NAME", "master") +env_branch = get_env("BRANCH_NAME", "master") is_docker = get_env("KOMETA_DOCKER", False, arg_bool=True) is_linuxserver = get_env("KOMETA_LINUXSERVER", False, arg_bool=True) is_lxml = get_env("KOMETA_LXML", False, arg_bool=True) @@ -205,7 +205,7 @@ from modules import util util.logger = logger from modules.builder import CollectionBuilder from modules.config import ConfigFile -from modules.request import Requests, parse_version +from modules.request import Requests from modules.util import Failed, FilterFailed, NonExisting, NotScheduled, Deleted def my_except_hook(exctype, value, tb): @@ -225,12 +225,12 @@ def new_send(*send_args, **kwargs): requests.Session.send = new_send -file_version = ("Unknown", "Unknown", 0) +local_version = "Unknown" with open(os.path.join(os.path.dirname(os.path.abspath(__file__)), "VERSION")) as handle: for line in handle.readlines(): line = line.strip() if len(line) > 0: - file_version = parse_version(line) + local_version = line break uuid_file = os.path.join(default_dir, "UUID") @@ -271,13 +271,13 @@ def start(attrs): elif is_linuxserver: system_ver = "Linuxserver" elif is_docker: - system_ver = "Docker" + system_ver = f"Docker: {env_branch}" else: system_ver = f"Python {platform.python_version()}" - my_requests = Requests(file_version, env_version, git_branch, verify_ssl=False if run_args["no-verify-ssl"] else True) - logger.info(f" Version: {my_requests.version[0]} ({system_ver}){f' (Git: {git_branch})' if git_branch else ''}") - if my_requests.new_version: - logger.info(f" Newest Version: {my_requests.new_version}") + my_requests = Requests(local_version, env_branch, git_branch, verify_ssl=False if run_args["no-verify-ssl"] else True) + logger.info(f" Version: {my_requests.local} ({system_ver}){f' (Git: {git_branch})' if git_branch else ''}") + if my_requests.newest: + logger.info(f" Newest Version: {my_requests.newest}") logger.info(f" Platform: {platform.platform()}") logger.info(f" Total Memory: {round(psutil.virtual_memory().total / (1024.0 ** 3))} GB") logger.info(f" Available Memory: {round(psutil.virtual_memory().available / (1024.0 ** 3))} GB") @@ -347,9 +347,9 @@ def start(attrs): except Failed as e: logger.stacktrace() logger.error(f"Webhooks Error: {e}") - version_line = f"Version: {my_requests.version[0]}" - if my_requests.new_version: - version_line = f"{version_line} Newest Version: {my_requests.new_version}" + version_line = f"Version: {my_requests.local}" + if my_requests.newest: + version_line = f"{version_line} Newest Version: {my_requests.newest}" try: log_data = {} no_overlays = [] diff --git a/modules/config.py b/modules/config.py index 1f6e268d..a556eaaa 100644 --- a/modules/config.py +++ b/modules/config.py @@ -579,7 +579,7 @@ class ConfigFile: try: self.Webhooks.start_time_hooks(self.start_time) if self.Requests.has_new_version(): - self.Webhooks.version_hooks(self.Requests.version, self.Requests.latest_version) + self.Webhooks.version_hooks() except Failed as e: logger.stacktrace() logger.error(f"Webhooks Error: {e}") diff --git a/modules/github.py b/modules/github.py index 0d739282..e4ae3a69 100644 --- a/modules/github.py +++ b/modules/github.py @@ -77,8 +77,8 @@ class GitHub: def configs_url(self): if self._configs_url is None: self._configs_url = f"{configs_raw_url}/master/" - if self.requests.version[1] in self.config_tags and (self.requests.latest_version[1] != self.requests.version[1] or self.requests.branch == "master"): - self._configs_url = f"{configs_raw_url}/v{self.requests.version[1]}/" + if self.requests.local.main in self.config_tags and (self.requests.latest.main != self.requests.local.main or self.requests.branch == "master"): + self._configs_url = f"{configs_raw_url}/v{self.requests.local.main}/" return self._configs_url @property diff --git a/modules/request.py b/modules/request.py index eefb8dbd..48c53894 100644 --- a/modules/request.py +++ b/modules/request.py @@ -24,12 +24,6 @@ def get_header(headers, header, language): } -def parse_version(version, text="develop"): - version = version.replace("develop", text) - split_version = version.split(f"-{text}") - return version, split_version[0], int(split_version[1]) if len(split_version) > 1 else 0 - - def quote(data): return parse.quote(str(data)) @@ -45,23 +39,38 @@ def parse_qs(data): def urlparse(data): return parse.urlparse(str(data)) +class Version: + def __init__(self, version_string="Unknown"): + self.full = version_string.replace("develop", "build") + version_parts = self.full.split("-build") + self.main = version_parts[0] + self.build = int(version_parts[1]) if len(version_parts) > 1 else 0 + + def __bool__(self): + return self.full != "Unknown" + + def __repr__(self): + return str(self) + + def __str__(self): + return self.full + class Requests: - def __init__(self, file_version, env_version, git_branch, verify_ssl=True): - self.file_version = file_version - self.env_version = env_version + def __init__(self, local, env_branch, git_branch, verify_ssl=True): + self.local = Version(local) + self.env_branch = env_branch self.git_branch = git_branch self.image_content_types = ["image/png", "image/jpeg", "image/webp"] - self.nightly_version = None - self.develop_version = None - self.master_version = None + self._nightly = None + self._develop = None + self._master = None + self._branch = None + self._latest = None + self._newest = None self.session = self.create_session() self.global_ssl = verify_ssl if not self.global_ssl: self.no_verify_ssl() - self.branch = self.guess_branch() - self.version = (self.file_version[0].replace("develop", self.branch), self.file_version[1].replace("develop", self.branch), self.file_version[2]) - self.latest_version = self.current_version(self.version, branch=self.branch) - self.new_version = self.latest_version[0] if self.latest_version and (self.version[1] != self.latest_version[1] or (self.version[2] and self.version[2] < self.latest_version[2])) else None def create_session(self, verify_ssl=True): session = requests.Session() @@ -77,9 +86,6 @@ class Requests: import urllib3 urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) - def has_new_version(self): - return self.version[0] != "Unknown" and self.latest_version[0] != "Unknown" and self.version[1] != self.latest_version[1] or (self.version[2] and self.version[2] < self.latest_version[2]) - def download_image(self, title, image_url, download_directory, is_poster=True, filename=None): response = self.get_image(image_url) new_image = os.path.join(download_directory, f"{filename}") if filename else download_directory @@ -158,54 +164,70 @@ class Requests: def post(self, url, data=None, json=None, headers=None, header=None, language=None): return self.session.post(url, data=data, json=json, headers=get_header(headers, header, language)) - def guess_branch(self): - if self.git_branch: - return self.git_branch - elif self.env_version: - return self.env_version - elif self.file_version[2] > 0: - dev_version = self.get_develop() - if self.file_version[1] != dev_version[1] or self.file_version[2] <= dev_version[2]: - return "develop" + def has_new_version(self): + return self.local and self.latest and self.local.main != self.latest.main or (self.local.build and self.local.build < self.latest.build) + + @property + def branch(self): + if self._branch is None: + if self.git_branch in ["develop", "nightly"]: + self._branch = self.git_branch + elif self.env_branch in ["develop", "nightly"]: + self._branch = self.env_branch + elif self.local.build > 0: + if self.local.main != self.develop.main or self.local.build <= self.develop.build: + self._branch = "develop" + else: + self._branch = "nightly" else: - return "nightly" - else: - return "master" + self._branch = "master" + return self._branch - def current_version(self, version, branch=None): - if branch == "develop": - return self.get_develop() - elif branch: - return self.get_nightly() - elif version[2] > 0: - new_version = self.get_develop() - if version[1] != new_version[1] or new_version[2] >= version[2]: - return new_version - return self.get_nightly() - else: - return self.get_master() + @property + def latest(self): + if self._latest is None: + if self.branch == "develop": + self._latest = self.develop + elif self.branch == "nightly": + self._latest = self.nightly + elif self.local.build > 0: + if self.local.main != self.develop.main or self.develop.build >= self.local.build: + self._latest = self.develop + self._latest = self.nightly + else: + self._latest = self.master + return self._latest - def get_nightly(self): - if self.nightly_version is None: - self.nightly_version = self.get_version("nightly") - return self.nightly_version + @property + def newest(self): + if self._newest is None: + self._newest = self.latest if self.latest and (self.local.main != self.latest.main or (self.local.build and self.local.build < self.latest.build)) else None + return self._newest - def get_develop(self): - if self.develop_version is None: - self.develop_version = self.get_version("develop") - return self.develop_version + @property + def master(self): + if self._master is None: + self._master = self._version("master") + return self._master - def get_master(self): - if self.master_version is None: - self.master_version = self.get_version("master") - return self.master_version + @property + def develop(self): + if self._develop is None: + self._develop = self._version("develop") + return self._develop - def get_version(self, level): + @property + def nightly(self): + if self._nightly is None: + self._nightly = self._version("nightly") + return self._nightly + + def _version(self, level): try: url = f"https://raw.githubusercontent.com/Kometa-Team/Kometa/{level}/VERSION" - return parse_version(self.get(url).content.decode().strip(), text=level) + return Version(self.get(url).content.decode().strip()) except ConnectionError: - return "Unknown", "Unknown", 0 + return Version() class YAML: diff --git a/modules/webhooks.py b/modules/webhooks.py index 653b78ce..c055e5a1 100644 --- a/modules/webhooks.py +++ b/modules/webhooks.py @@ -78,14 +78,14 @@ class Webhooks: if self.run_start_webhooks: self._request(self.run_start_webhooks, {"event": "run_start", "start_time": start_time.strftime("%Y-%m-%d %H:%M:%S")}) - def version_hooks(self, version, latest_version): + def version_hooks(self): if self.version_webhooks: notes = None - if version[1] != latest_version[1]: + if self.requests.local.main != self.requests.latest.main: notes = self.config.GitHub.latest_release_notes() - elif version[2] and version[2] < latest_version[2]: - notes = self.config.GitHub.get_commits(version[2], nightly=self.requests.branch == "nightly") - self._request(self.version_webhooks, {"event": "version", "current": version[0], "latest": latest_version[0], "notes": notes}) + elif self.requests.local.build and self.requests.local.build < self.requests.latest.build: + notes = self.config.GitHub.get_commits(self.requests.local.build, nightly=self.requests.branch == "nightly") + self._request(self.version_webhooks, {"event": "version", "current": str(self.requests.local), "latest": str(self.requests.latest), "notes": notes}) def end_time_hooks(self, start_time, end_time, run_time, stats): if self.run_end_webhooks: