From 897ca333205cc2f4f64e97c2dc4120f98542cc9b Mon Sep 17 00:00:00 2001 From: Evelyn Alicke Date: Tue, 27 Jun 2023 14:28:22 +0200 Subject: [PATCH] feat(rclone-proxy): initial implementation --- playbooks/rclone_serve_restic.yml | 22 +++++++ roles/rclone_serve_restic/README.md | 22 +++++++ .../defaults/container.yml | 64 +++++++++++++++++++ roles/rclone_serve_restic/defaults/main.yml | 28 ++++++++ roles/rclone_serve_restic/handlers/main.yml | 8 +++ roles/rclone_serve_restic/tasks/main.yml | 53 +++++++++++++++ roles/rclone_serve_restic/templates/to-ini.j2 | 6 ++ roles/rclone_serve_restic/vars/main.yml | 15 +++++ 8 files changed, 218 insertions(+) create mode 100644 playbooks/rclone_serve_restic.yml create mode 100644 roles/rclone_serve_restic/README.md create mode 100644 roles/rclone_serve_restic/defaults/container.yml create mode 100644 roles/rclone_serve_restic/defaults/main.yml create mode 100644 roles/rclone_serve_restic/handlers/main.yml create mode 100644 roles/rclone_serve_restic/tasks/main.yml create mode 100644 roles/rclone_serve_restic/templates/to-ini.j2 create mode 100644 roles/rclone_serve_restic/vars/main.yml diff --git a/playbooks/rclone_serve_restic.yml b/playbooks/rclone_serve_restic.yml new file mode 100644 index 0000000..351cfd9 --- /dev/null +++ b/playbooks/rclone_serve_restic.yml @@ -0,0 +1,22 @@ +--- +- hosts: '{{ "rclone_serve_restic_target_hosts" | default("rclone_serve_restic") }}' + become: true + roles: + - role: rclone_serve_restic + + vars: + routing_service_name: rclone_serve_restic + routing_server_port: "{{ rclone_serve_restic_container_binds.port }}" + routing_traffic_public_rule: >- + Host(`{{ famedly_instance_domain }}`) + && PathPrefix(`/restic/`) + routing_traffic_public_middlewares: + - stripprefix + routing_traffic_metrics_rule: Path(`/restic/metrics`) + routing_traffic_metrics_middlewares: + - stripprefix + famedly_traefik_middlewares: + stripprefix: + stripprefix: + prefix: /restic + rclone_serve_restic_container_labels: "{{ famedly_traefik_labels_flat }}" diff --git a/roles/rclone_serve_restic/README.md b/roles/rclone_serve_restic/README.md new file mode 100644 index 0000000..2433c5f --- /dev/null +++ b/roles/rclone_serve_restic/README.md @@ -0,0 +1,22 @@ +# `famedly.base.rclone_serve_restic` ansible role for multi-tenancy append-only backups to s3 using rclone + +## Required Options + +This role supports a single s3 bucket as a backend, you can set the required options like so +``` yml +rclone_serve_restic_backend_config: + endpoint: "top secret" + access_key_id: "middle secret" + secret_access_key: "bottom secret" + + # you can just overwrite the defaults with the following + type: s3 + provider: minio + env_auth: false + region: home-sweet-home + acl: private + +``` + +You also NEED to write secrets to the htpasswd file yourself, or else it will be exposed WITHOUT AUTHENTICATION!! +The file is read from the location set in `rclone_serve_restic_htpasswd_file` diff --git a/roles/rclone_serve_restic/defaults/container.yml b/roles/rclone_serve_restic/defaults/container.yml new file mode 100644 index 0000000..520065a --- /dev/null +++ b/roles/rclone_serve_restic/defaults/container.yml @@ -0,0 +1,64 @@ +--- + +rclone_serve_restic_container_image_reference: >- + {{ + rclone_serve_restic_container_image_repository + ~ ':' + ~ rclone_serve_restic_container_image_tag | default(rclone_serve_restic_version) + }} + +rclone_serve_restic_container_image_repository: >- + {{ + [ + ( + container_registries[rclone_serve_restic_container_image_registry] + | default(rclone_serve_restic_container_image_registry) + ), + rclone_serve_restic_container_image_namespace | default(omit), + rclone_serve_restic_container_image_name + ] | join('/') + + }} + +rclone_serve_restic_container_image_registry: "docker.io" +rclone_serve_restic_container_image_namespace: "rclone" +rclone_serve_restic_container_image_name: "rclone" +# rclone_serve_restic_container_image_tag: + +rclone_serve_restic_container_restart_policy: unless-stopped + +rclone_serve_restic_container_base_labels: + version: "{{ rclone_serve_restic_version }}" + +rclone_serve_restic_container_labels: >- + {{ rclone_serve_restic_container_base_labels + | combine(rclone_serve_restic_container_extra_labels | default({})) }} + +rclone_serve_restic_container_binds: + port: 8000 + internal: + address: '[::1]' + port: 8080 + +rclone_serve_restic_container_ports: + - "{{ rclone_serve_restic_container_bind.port ~ ':' + ~ rclone_serve_restic_container_bind.internal.port }}" + +rclone_serve_restic_entry_opts_default: + addr: "{{ rclone_serve_restic_container_bind.internal | join(':', attribute=value) }}" + config: "{{ rclone_serve_restic_config_file }}" + private-repos: "true" + append-only: "true" + htpasswd: "{{ rclone_serve_restic_htpasswd_file }}" + +rclone_serve_restic_entry_opts_merged: >- + {{ rclone_serve_restic_entry_opts + | default([]) + | combine(rclone_serve_restic_entry_opts_default) }} + +rclone_serve_restic_container_volumes: + - "{{ rclone_config_path ~ ':' ~ rclone_config_path }}" +rclone_serve_restic_container_env: {} +rclone_serve_restic_container_name: 'rclone-serve-restic' + +rclone_serve_restic_container_force_pull: false diff --git a/roles/rclone_serve_restic/defaults/main.yml b/roles/rclone_serve_restic/defaults/main.yml new file mode 100644 index 0000000..e0e6afa --- /dev/null +++ b/roles/rclone_serve_restic/defaults/main.yml @@ -0,0 +1,28 @@ +--- + +rclone_serve_restic_user: "rclone" + +rclone_serve_restic_version: 1.62 + +rclone_serve_restic_config_path: "/opt/rclone-serve-restic/" + +rclone_serve_restic_entry_opts: [] # allows you to specify extra options passed to the process + +rclone_serve_restic_config_file: "{{ rclone_serve_restic_config_path ~ 'rclone_serve_restic.conf' }}" +rclone_serve_restic_htpasswd_file: "{{ rclone_serve_restic_config_path ~ 'rclone_serve_restic.htpasswd' }}" +rclone_serve_restic_htpasswd_scheme: "bcrypt" + +rclone_serve_restic_default_backend: + type: s3 + provider: Scaleway + env_auth: false + region: nl-ams + acl: private + +rclone_serve_restic_backend_config: + endpoint: "{{ rclone_serve_restic_s3.endpoint }}" + access_key_id: "{{ rclone_serve_restic_s3.key_id }}" + secret_access_key: "{{ rclone_serve_restic_s3.secret }}" + +rclone_serve_restic_backend_config_merged: + - default: rclone_serve_restic_backend_config | combine(rclone_serve_restic_default_backend) diff --git a/roles/rclone_serve_restic/handlers/main.yml b/roles/rclone_serve_restic/handlers/main.yml new file mode 100644 index 0000000..7c07e4d --- /dev/null +++ b/roles/rclone_serve_restic/handlers/main.yml @@ -0,0 +1,8 @@ +--- + +- name: Restart rclone serve restic container + listen: container-rclone-serve-restic-restart + community.docker.docker_container: + name: "{{ rclone_serve_restic_container_name }}" + state: started + restart: true diff --git a/roles/rclone_serve_restic/tasks/main.yml b/roles/rclone_serve_restic/tasks/main.yml new file mode 100644 index 0000000..8138a86 --- /dev/null +++ b/roles/rclone_serve_restic/tasks/main.yml @@ -0,0 +1,53 @@ +--- + +- name: Ensure user is present + ansible.builtin.user: + name: "{{ rclone_serve_restic_user }}" + state: present + system: true + register: rclone_serve_restic_user_res + +- name: Ensure config directory is present + ansible.builtin.file: + path: "{{ rclone_serve_restic_config_path }}" + state: directory + mode: '0640' + owner: "{{ rclone_serve_restic_user_res.uid }}" + group: "{{ rclone_serve_restic_user_res.group }}" + +- name: Ensure config is present + ansible.builtin.template: + src: to-ini.j2 + dest: "{{ rclone_serve_restic_config_file }}" + mode: '0600' + owner: "{{ rclone_serve_restic_user_res.uid }}" + group: "{{ rclone_serve_restic_user_res.group }}" + vars: + ini: "{{ rclone_serve_restic_backend_config_merged }}" + +- name: Ensure container image is present locally + docker_image: + name: "{{ rclone_serve_restic_container_image }}" + source: pull + state: present + force_source: "{{ rclone_serve_restic_container_force_pull }}" + register: rclone_serve_restic_container_image_pulled + until: rclone_serve_restic_container_image_pulled is success + retries: 10 + delay: 5 + +- name: Ensure container is started + community.docker.docker_container: + image: "{{ rclone_serve_restic_container_image }}" + name: "{{ rclone_serve_restic_container_name }}" + state: started + restart_policy: "{{ rclone_serve_restic_container_restart_policy | default(omit) }}" + user: "{{ rclone_serve_restic_user_res.uid ~ ':' ~ rclone_serve_restic_user_res.group }}" + volumes: "{{ rclone_serve_restic_container_volumes | default(omit) }}" + ports: "{{ rclone_serve_restic_container_ports | default(omit) }}" + env: "{{ rclone_serve_restic_container_env | default(omit) }}" + labels: "{{ rclone_serve_restic_container_labels_merged | default(omit) }}" + entrypoint: "{{ rclone_serve_restic_container_entry | default(omit) }}" + etc_hosts: "{{ rclone_serve_restic_container_etc_hosts | default(omit) }}" + networks: "{{ rclone_serve_restic_container_networks | default(omit) }}" + purge_networks: "{{ rclone_serve_restic_container_purge_networks | default(omit) }}" diff --git a/roles/rclone_serve_restic/templates/to-ini.j2 b/roles/rclone_serve_restic/templates/to-ini.j2 new file mode 100644 index 0000000..cacab66 --- /dev/null +++ b/roles/rclone_serve_restic/templates/to-ini.j2 @@ -0,0 +1,6 @@ +{% for section in ini %} +[{{ section }}] +{% for (key, value) in section %} + {{ key ~ '=' ~ value }} +{% endfor %} +{% endfor %} diff --git a/roles/rclone_serve_restic/vars/main.yml b/roles/rclone_serve_restic/vars/main.yml new file mode 100644 index 0000000..c608677 --- /dev/null +++ b/roles/rclone_serve_restic/vars/main.yml @@ -0,0 +1,15 @@ +--- + +# this should NEVER be overwritten. instead, overwrite _entry_opts_merged +rclone_serve_restic_container_entry: > + {% set args = [] %} + {% for opt in rclone_serve_restic_entry_opts_merged %} + {% set (opt,value) = opt %} + {% if value not "false" %} # if value is false, then treat as valueless arg to be absent + {% do args.append({{ '--' ~ opt | replace('_','-') }}) %} + {% if value not "true" %} # if the value is true, then treat as valueless arg to be present + {% do args.append({{ value }}) %} + {% endif %} + {% endif %} + {% endfor %} + rclone serve restic {{ args | join(' ') ~ ' ' ~ (rclone_serve_restic_backend | default ('default')) }}