mirror of
https://github.com/mother-of-all-self-hosting/mash-playbook
synced 2024-11-10 06:14:17 +00:00
Merge pull request #84 from mother-of-all-self-hosting/opml
add OPML generator script
This commit is contained in:
Slavi Pantaleev
GitHub
commit
c15093f1a8
3 changed files with 206 additions and 1 deletions
157
bin/feeds.py
Normal file
157
bin/feeds.py
Normal file
|
|
@ -0,0 +1,157 @@
|
|||
import os
|
||||
import sys
|
||||
import argparse
|
||||
from urllib.parse import urlparse
|
||||
import xml.etree.ElementTree as ET
|
||||
|
||||
parser = argparse.ArgumentParser(description='Extracts release feeds from roles')
|
||||
parser.add_argument('root_dir', help='Root dir which to traverse recursively for defaults/main.yml roles files')
|
||||
parser.add_argument('action', help='Pass "check" to list roles with missing feeds or "dump" to dump an OPML file')
|
||||
args = parser.parse_args()
|
||||
if args.action not in ['check', 'dump', 'hookshot']:
|
||||
sys.exit('Error: possible arguments are "check" or "dump"')
|
||||
|
||||
excluded_paths = [
|
||||
# appservice-kakaotalk defines a Project URL, but that Gitea repository does not have an Atom/RSS feed.
|
||||
# It doesn't have any tags anyway.
|
||||
'./upstream/roles/custom/matrix-bridge-appservice-kakaotalk/defaults',
|
||||
]
|
||||
project_source_url_str = '# Project source code URL:'
|
||||
|
||||
def get_roles_files_from_dir(root_dir):
|
||||
file_paths = []
|
||||
for dir_name, sub_dur_list, file_list in os.walk(root_dir):
|
||||
for file_name in file_list:
|
||||
if not dir_name.endswith('defaults') or file_name != 'main.yml':
|
||||
continue
|
||||
if dir_name in excluded_paths:
|
||||
continue
|
||||
file_paths.append(os.path.join(dir_name, file_name))
|
||||
return file_paths
|
||||
|
||||
def get_git_repos_from_files(file_paths, break_on_missing_repos=False):
|
||||
git_repos = {}
|
||||
missing_repos = []
|
||||
|
||||
for file in file_paths:
|
||||
file_lines = open(file, 'r').readlines()
|
||||
found_project_repo = False
|
||||
for line in file_lines:
|
||||
project_repo_val = ''
|
||||
if project_source_url_str in line:
|
||||
# extract the value from a line like this:
|
||||
# Project source code URL: https://github.com/mautrix/signal
|
||||
project_repo_val = line.split(project_source_url_str)[1].strip()
|
||||
if not validate_url(project_repo_val):
|
||||
print('Invalid url for line ', line)
|
||||
break
|
||||
if project_repo_val != '':
|
||||
if file not in git_repos:
|
||||
git_repos[file] = []
|
||||
|
||||
git_repos[file].append(project_repo_val)
|
||||
found_project_repo = True
|
||||
|
||||
if not found_project_repo:
|
||||
missing_repos.append(file)
|
||||
|
||||
if break_on_missing_repos and len(missing_repos) > 0:
|
||||
print('Missing `{0}` comment for:\n{1}'.format(project_source_url_str, '\n'.join(missing_repos)))
|
||||
|
||||
return git_repos
|
||||
|
||||
def validate_url(text):
|
||||
if text == '':
|
||||
return False
|
||||
try:
|
||||
result = urlparse(text)
|
||||
return all([result.scheme, result.netloc])
|
||||
except:
|
||||
return False
|
||||
|
||||
|
||||
def format_feeds_from_git_repos(git_repos):
|
||||
feeds = {
|
||||
'ansible': {
|
||||
'text': 'ansible',
|
||||
'title': 'ansible',
|
||||
'type': 'rss',
|
||||
'htmlUrl': 'https://pypi.org/project/ansible/#history',
|
||||
'xmlUrl': 'https://pypi.org/rss/project/ansible/releases.xml'
|
||||
},
|
||||
'ansible-core': {
|
||||
'text': 'ansible-core',
|
||||
'title': 'ansible-core',
|
||||
'type': 'rss',
|
||||
'htmlUrl': 'https://pypi.org/project/ansible-core/#history',
|
||||
'xmlUrl': 'https://pypi.org/rss/project/ansible-core/releases.xml'
|
||||
}
|
||||
}
|
||||
for role, git_repos in git_repos.items():
|
||||
for idx, git_repo in enumerate(git_repos):
|
||||
if 'github' in git_repo:
|
||||
atomFilePath = git_repo.replace('.git', '') + '/releases.atom'
|
||||
elif ('gitlab' in git_repo or 'mau.dev' in git_repo):
|
||||
atomFilePath = git_repo.replace('.git', '') + '/-/tags?format=atom'
|
||||
elif 'git.zx2c4.com' in git_repo:
|
||||
atomFilePath = git_repo + '/atom/'
|
||||
else:
|
||||
print('Unrecognized git repository: %s' % git_repo)
|
||||
continue
|
||||
|
||||
role_name = role.split('/')[4]
|
||||
if role_name == 'defaults':
|
||||
role_name = role.split('/')[3]
|
||||
role_name = role_name.removeprefix('matrix-bot-').removeprefix('matrix-bridge-').removeprefix('matrix-client-').removeprefix('matrix-')
|
||||
if idx > 0:
|
||||
# there is more than 1 project source code for this role
|
||||
role_name += '-' + str(idx+1)
|
||||
|
||||
feeds[role_name] = {
|
||||
'text': role_name,
|
||||
'title': role_name,
|
||||
'type': 'rss',
|
||||
'htmlUrl': git_repo,
|
||||
'xmlUrl': atomFilePath
|
||||
}
|
||||
|
||||
feeds = {key: val for key, val in sorted(feeds.items(), key = lambda item: item[0])}
|
||||
return feeds
|
||||
|
||||
def dump_opml_file_from_feeds(feeds):
|
||||
tree = ET.ElementTree('tree')
|
||||
|
||||
opml = ET.Element('opml', {'version': '1.0'})
|
||||
head = ET.SubElement(opml, 'head')
|
||||
|
||||
title = ET.SubElement(head, 'title')
|
||||
title.text = 'Release feeds for roles'
|
||||
|
||||
body = ET.SubElement(opml, 'body')
|
||||
for role, feed_dict in feeds.items():
|
||||
outline = ET.SubElement(body, 'outline', feed_dict)
|
||||
|
||||
ET.indent(opml)
|
||||
tree._setroot(opml)
|
||||
file_name = 'releases.opml'
|
||||
tree.write(file_name, encoding = 'UTF-8', xml_declaration = True)
|
||||
print('Generated %s' % file_name)
|
||||
|
||||
def dump_hookshot_commands(feeds):
|
||||
file_name = 'releases.hookshot.txt'
|
||||
f = open(file_name, 'w')
|
||||
for role, feed_dict in feeds.items():
|
||||
f.write('!hookshot feed %s %s\n' % (feed_dict['xmlUrl'], role))
|
||||
f.close()
|
||||
print('Generated %s' % file_name)
|
||||
|
||||
if __name__ == '__main__':
|
||||
file_paths = get_roles_files_from_dir(root_dir=args.root_dir)
|
||||
break_on_missing = args.action == 'check'
|
||||
git_repos = get_git_repos_from_files(file_paths=file_paths, break_on_missing_repos=break_on_missing)
|
||||
feeds = format_feeds_from_git_repos(git_repos)
|
||||
|
||||
if args.action == 'dump':
|
||||
dump_opml_file_from_feeds(feeds)
|
||||
if args.action == 'hookshot':
|
||||
dump_hookshot_commands(feeds)
|
||||
7
justfile
7
justfile
|
|
@ -13,13 +13,18 @@ roles:
|
|||
fi
|
||||
|
||||
# Updates requirements.yml if there are any new tags available. Requires agru
|
||||
update:
|
||||
update: && opml
|
||||
@agru -u
|
||||
|
||||
# Runs ansible-lint against all roles in the playbook
|
||||
lint:
|
||||
ansible-lint
|
||||
|
||||
# dumps an OPML file with extracted git feeds for roles
|
||||
opml:
|
||||
@echo "generating opml..."
|
||||
@python bin/feeds.py . dump
|
||||
|
||||
# Runs the playbook with --tags=install-all,start and optional arguments
|
||||
install-all *extra_args: (run-tags "install-all,start" extra_args)
|
||||
|
||||
|
|
|
|||
43
releases.opml
Normal file
43
releases.opml
Normal file
|
|
@ -0,0 +1,43 @@
|
|||
<?xml version='1.0' encoding='UTF-8'?>
|
||||
<opml version="1.0">
|
||||
<head>
|
||||
<title>Release feeds for roles</title>
|
||||
</head>
|
||||
<body>
|
||||
<outline text="adguard_home" title="adguard_home" type="rss" htmlUrl="https://github.com/AdguardTeam/AdguardHome" xmlUrl="https://github.com/AdguardTeam/AdguardHome/releases.atom" />
|
||||
<outline text="ansible" title="ansible" type="rss" htmlUrl="https://pypi.org/project/ansible/#history" xmlUrl="https://pypi.org/rss/project/ansible/releases.xml" />
|
||||
<outline text="ansible-core" title="ansible-core" type="rss" htmlUrl="https://pypi.org/project/ansible-core/#history" xmlUrl="https://pypi.org/rss/project/ansible-core/releases.xml" />
|
||||
<outline text="appsmith" title="appsmith" type="rss" htmlUrl="https://github.com/appsmithorg/appsmith" xmlUrl="https://github.com/appsmithorg/appsmith/releases.atom" />
|
||||
<outline text="backup_borg" title="backup_borg" type="rss" htmlUrl="https://gitlab.com/etke.cc/borgmatic" xmlUrl="https://gitlab.com/etke.cc/borgmatic/-/tags?format=atom" />
|
||||
<outline text="com.devture.ansible.role.container_socket_proxy" title="com.devture.ansible.role.container_socket_proxy" type="rss" htmlUrl="https://github.com/Tecnativa/docker-socket-proxy" xmlUrl="https://github.com/Tecnativa/docker-socket-proxy/releases.atom" />
|
||||
<outline text="com.devture.ansible.role.postgres" title="com.devture.ansible.role.postgres" type="rss" htmlUrl="https://github.com/postgres/postgres" xmlUrl="https://github.com/postgres/postgres/releases.atom" />
|
||||
<outline text="com.devture.ansible.role.postgres_backup" title="com.devture.ansible.role.postgres_backup" type="rss" htmlUrl="https://github.com/prodrigestivill/docker-postgres-backup-local" xmlUrl="https://github.com/prodrigestivill/docker-postgres-backup-local/releases.atom" />
|
||||
<outline text="com.devture.ansible.role.traefik" title="com.devture.ansible.role.traefik" type="rss" htmlUrl="https://github.com/traefik/traefik" xmlUrl="https://github.com/traefik/traefik/releases.atom" />
|
||||
<outline text="com.devture.ansible.role.woodpecker_ci_agent" title="com.devture.ansible.role.woodpecker_ci_agent" type="rss" htmlUrl="https://github.com/woodpecker-ci/woodpecker" xmlUrl="https://github.com/woodpecker-ci/woodpecker/releases.atom" />
|
||||
<outline text="com.devture.ansible.role.woodpecker_ci_server" title="com.devture.ansible.role.woodpecker_ci_server" type="rss" htmlUrl="https://github.com/woodpecker-ci/woodpecker" xmlUrl="https://github.com/woodpecker-ci/woodpecker/releases.atom" />
|
||||
<outline text="focalboard" title="focalboard" type="rss" htmlUrl="https://github.com/mattermost/focalboard" xmlUrl="https://github.com/mattermost/focalboard/releases.atom" />
|
||||
<outline text="grafana" title="grafana" type="rss" htmlUrl="https://github.com/grafana/grafana" xmlUrl="https://github.com/grafana/grafana/releases.atom" />
|
||||
<outline text="healthchecks" title="healthchecks" type="rss" htmlUrl="https://github.com/healthchecks/healthchecks" xmlUrl="https://github.com/healthchecks/healthchecks/releases.atom" />
|
||||
<outline text="infisical" title="infisical" type="rss" htmlUrl="https://github.com/Infisical/infisical" xmlUrl="https://github.com/Infisical/infisical/releases.atom" />
|
||||
<outline text="jitsi" title="jitsi" type="rss" htmlUrl="https://github.com/jitsi/docker-jitsi-meet" xmlUrl="https://github.com/jitsi/docker-jitsi-meet/releases.atom" />
|
||||
<outline text="keycloak" title="keycloak" type="rss" htmlUrl="https://github.com/keycloak/keycloak" xmlUrl="https://github.com/keycloak/keycloak/releases.atom" />
|
||||
<outline text="lago" title="lago" type="rss" htmlUrl="https://github.com/lago/lago" xmlUrl="https://github.com/lago/lago/releases.atom" />
|
||||
<outline text="linkding" title="linkding" type="rss" htmlUrl="https://github.com/sissbruecker/linkding" xmlUrl="https://github.com/sissbruecker/linkding/releases.atom" />
|
||||
<outline text="miniflux" title="miniflux" type="rss" htmlUrl="https://github.com/miniflux/v2" xmlUrl="https://github.com/miniflux/v2/releases.atom" />
|
||||
<outline text="mongodb" title="mongodb" type="rss" htmlUrl="https://github.com/mongodb/mongo" xmlUrl="https://github.com/mongodb/mongo/releases.atom" />
|
||||
<outline text="n8n" title="n8n" type="rss" htmlUrl="https://github.com/n8n-io/n8n" xmlUrl="https://github.com/n8n-io/n8n/releases.atom" />
|
||||
<outline text="navidrome" title="navidrome" type="rss" htmlUrl="https://github.com/navidrome/navidrome" xmlUrl="https://github.com/navidrome/navidrome/releases.atom" />
|
||||
<outline text="netbox" title="netbox" type="rss" htmlUrl="https://github.com/netbox-community/netbox-docker/" xmlUrl="https://github.com/netbox-community/netbox-docker//releases.atom" />
|
||||
<outline text="owncast" title="owncast" type="rss" htmlUrl="https://github.com/owncast/owncast" xmlUrl="https://github.com/owncast/owncast/releases.atom" />
|
||||
<outline text="prometheus" title="prometheus" type="rss" htmlUrl="https://github.com/prometheus/prometheus" xmlUrl="https://github.com/prometheus/prometheus/releases.atom" />
|
||||
<outline text="prometheus_blackbox_exporter" title="prometheus_blackbox_exporter" type="rss" htmlUrl="https://github.com/prometheus/blackbox_exporter" xmlUrl="https://github.com/prometheus/blackbox_exporter/releases.atom" />
|
||||
<outline text="prometheus_node_exporter" title="prometheus_node_exporter" type="rss" htmlUrl="https://github.com/prometheus/node_exporter" xmlUrl="https://github.com/prometheus/node_exporter/releases.atom" />
|
||||
<outline text="prometheus_postgres_exporter" title="prometheus_postgres_exporter" type="rss" htmlUrl="https://github.com/prometheus-community/postgres_exporter" xmlUrl="https://github.com/prometheus-community/postgres_exporter/releases.atom" />
|
||||
<outline text="radicale" title="radicale" type="rss" htmlUrl="https://github.com/tomsquest/docker-radicale" xmlUrl="https://github.com/tomsquest/docker-radicale/releases.atom" />
|
||||
<outline text="redis" title="redis" type="rss" htmlUrl="https://github.com/redis/redis" xmlUrl="https://github.com/redis/redis/releases.atom" />
|
||||
<outline text="redmine" title="redmine" type="rss" htmlUrl="https://github.com/redmine/redmine" xmlUrl="https://github.com/redmine/redmine/releases.atom" />
|
||||
<outline text="soft_serve" title="soft_serve" type="rss" htmlUrl="https://github.com/charmbracelet/soft-serve" xmlUrl="https://github.com/charmbracelet/soft-serve/releases.atom" />
|
||||
<outline text="uptime_kuma" title="uptime_kuma" type="rss" htmlUrl="https://github.com/louislam/uptime-kuma" xmlUrl="https://github.com/louislam/uptime-kuma/releases.atom" />
|
||||
<outline text="wg_easy" title="wg_easy" type="rss" htmlUrl="https://github.com/WeeJeWel/wg-easy" xmlUrl="https://github.com/WeeJeWel/wg-easy/releases.atom" />
|
||||
</body>
|
||||
</opml>
|
||||
Loading…
Reference in a new issue