feat(rclone-proxy): initial implementation

playbooks/rclone_serve_restic.yml

@@ -0,0 +1,22 @@
+---
+- hosts: '{{ "rclone_serve_restic_target_hosts" | default("rclone_serve_restic") }}'
+  become: true
+  roles:
+    - role: rclone_serve_restic
+
+  vars:
+    routing_service_name: rclone_serve_restic
+    routing_server_port: "{{ rclone_serve_restic_container_binds.port }}"
+    routing_traffic_public_rule: >-
+      Host(`{{ famedly_instance_domain }}`)
+      && PathPrefix(`/restic/`)
+    routing_traffic_public_middlewares:
+      - stripprefix
+    routing_traffic_metrics_rule: Path(`/restic/metrics`)
+    routing_traffic_metrics_middlewares:
+      - stripprefix
+    famedly_traefik_middlewares:
+      stripprefix:
+        stripprefix:
+          prefix: /restic
+    rclone_serve_restic_container_labels: "{{ famedly_traefik_labels_flat }}"

roles/rclone_serve_restic/README.md

@@ -0,0 +1,22 @@
+# `famedly.base.rclone_serve_restic` ansible role for multi-tenancy append-only backups to s3 using rclone
+
+## Required Options
+
+This role supports a single s3 bucket as a backend, you can set the required options like so
+``` yml
+rclone_serve_restic_backend_config:
+  endpoint: "top secret"
+  access_key_id: "middle secret"
+  secret_access_key: "bottom secret"
+
+  # you can just overwrite the defaults with the following
+  type: s3
+  provider: minio
+  env_auth: false
+  region: home-sweet-home
+  acl: private
+
+```
+
+You also NEED to write secrets to the htpasswd file yourself, or else it will be exposed WITHOUT AUTHENTICATION!!
+The file is read from the location set in `rclone_serve_restic_htpasswd_file`

roles/rclone_serve_restic/defaults/container.yml

@@ -0,0 +1,64 @@
+---
+
+rclone_serve_restic_container_image_reference: >-
+  {{
+    rclone_serve_restic_container_image_repository
+    ~ ':'
+    ~ rclone_serve_restic_container_image_tag | default(rclone_serve_restic_version)
+  }}
+
+rclone_serve_restic_container_image_repository: >-
+  {{
+    [
+    (
+      container_registries[rclone_serve_restic_container_image_registry]
+      | default(rclone_serve_restic_container_image_registry)
+    ),
+    rclone_serve_restic_container_image_namespace | default(omit),
+    rclone_serve_restic_container_image_name
+    ] | join('/')
+
+  }}
+
+rclone_serve_restic_container_image_registry: "docker.io"
+rclone_serve_restic_container_image_namespace: "rclone"
+rclone_serve_restic_container_image_name: "rclone"
+# rclone_serve_restic_container_image_tag:
+
+rclone_serve_restic_container_restart_policy: unless-stopped
+
+rclone_serve_restic_container_base_labels:
+  version: "{{ rclone_serve_restic_version }}"
+
+rclone_serve_restic_container_labels: >-
+  {{ rclone_serve_restic_container_base_labels
+  | combine(rclone_serve_restic_container_extra_labels | default({})) }}
+
+rclone_serve_restic_container_binds:
+  port: 8000
+  internal:
+    address: '[::1]'
+    port: 8080
+
+rclone_serve_restic_container_ports:
+  - "{{ rclone_serve_restic_container_bind.port ~ ':'
+      ~ rclone_serve_restic_container_bind.internal.port }}"
+
+rclone_serve_restic_entry_opts_default:
+  addr: "{{ rclone_serve_restic_container_bind.internal | join(':', attribute=value) }}"
+  config: "{{ rclone_serve_restic_config_file }}"
+  private-repos: "true"
+  append-only: "true"
+  htpasswd: "{{ rclone_serve_restic_htpasswd_file }}"
+
+rclone_serve_restic_entry_opts_merged: >-
+  {{ rclone_serve_restic_entry_opts
+  | default([])
+  | combine(rclone_serve_restic_entry_opts_default) }}
+
+rclone_serve_restic_container_volumes:
+  - "{{ rclone_config_path ~ ':' ~ rclone_config_path }}"
+rclone_serve_restic_container_env: {}
+rclone_serve_restic_container_name: 'rclone-serve-restic'
+
+rclone_serve_restic_container_force_pull: false

roles/rclone_serve_restic/defaults/main.yml

@@ -0,0 +1,28 @@
+---
+
+rclone_serve_restic_user: "rclone"
+
+rclone_serve_restic_version: 1.62
+
+rclone_serve_restic_config_path: "/opt/rclone-serve-restic/"
+
+rclone_serve_restic_entry_opts: []    # allows you to specify extra options passed to the process
+
+rclone_serve_restic_config_file: "{{ rclone_serve_restic_config_path ~ 'rclone_serve_restic.conf' }}"
+rclone_serve_restic_htpasswd_file: "{{ rclone_serve_restic_config_path ~ 'rclone_serve_restic.htpasswd' }}"
+rclone_serve_restic_htpasswd_scheme: "bcrypt"
+
+rclone_serve_restic_default_backend:
+  type: s3
+  provider: Scaleway
+  env_auth: false
+  region: nl-ams
+  acl: private
+
+rclone_serve_restic_backend_config:
+  endpoint: "{{ rclone_serve_restic_s3.endpoint }}"
+  access_key_id: "{{ rclone_serve_restic_s3.key_id }}"
+  secret_access_key: "{{ rclone_serve_restic_s3.secret }}"
+
+rclone_serve_restic_backend_config_merged:
+  - default: rclone_serve_restic_backend_config | combine(rclone_serve_restic_default_backend)

roles/rclone_serve_restic/handlers/main.yml

@@ -0,0 +1,8 @@
+---
+
+- name: Restart rclone serve restic container
+  listen: container-rclone-serve-restic-restart
+  community.docker.docker_container:
+    name: "{{ rclone_serve_restic_container_name }}"
+    state: started
+    restart: true

roles/rclone_serve_restic/tasks/main.yml

@@ -0,0 +1,53 @@
+---
+
+- name: Ensure user is present
+  ansible.builtin.user:
+    name: "{{ rclone_serve_restic_user }}"
+    state: present
+    system: true
+  register: rclone_serve_restic_user_res
+
+- name: Ensure config directory is present
+  ansible.builtin.file:
+    path: "{{ rclone_serve_restic_config_path }}"
+    state: directory
+    mode: '0640'
+    owner: "{{ rclone_serve_restic_user_res.uid }}"
+    group: "{{ rclone_serve_restic_user_res.group }}"
+
+- name: Ensure config is present
+  ansible.builtin.template:
+    src: to-ini.j2
+    dest: "{{ rclone_serve_restic_config_file }}"
+    mode: '0600'
+    owner: "{{ rclone_serve_restic_user_res.uid }}"
+    group: "{{ rclone_serve_restic_user_res.group }}"
+  vars:
+    ini: "{{ rclone_serve_restic_backend_config_merged }}"
+
+- name: Ensure container image is present locally
+  docker_image:
+    name: "{{ rclone_serve_restic_container_image }}"
+    source: pull
+    state: present
+    force_source: "{{ rclone_serve_restic_container_force_pull }}"
+  register: rclone_serve_restic_container_image_pulled
+  until: rclone_serve_restic_container_image_pulled is success
+  retries: 10
+  delay: 5
+
+- name: Ensure container is started
+  community.docker.docker_container:
+    image: "{{ rclone_serve_restic_container_image }}"
+    name: "{{ rclone_serve_restic_container_name }}"
+    state: started
+    restart_policy: "{{ rclone_serve_restic_container_restart_policy | default(omit) }}"
+    user: "{{ rclone_serve_restic_user_res.uid ~ ':' ~ rclone_serve_restic_user_res.group }}"
+    volumes: "{{ rclone_serve_restic_container_volumes | default(omit) }}"
+    ports: "{{ rclone_serve_restic_container_ports | default(omit) }}"
+    env: "{{ rclone_serve_restic_container_env | default(omit) }}"
+    labels: "{{ rclone_serve_restic_container_labels_merged | default(omit) }}"
+    entrypoint: "{{ rclone_serve_restic_container_entry | default(omit) }}"
+    etc_hosts: "{{ rclone_serve_restic_container_etc_hosts | default(omit) }}"
+    networks: "{{ rclone_serve_restic_container_networks | default(omit) }}"
+    purge_networks: "{{ rclone_serve_restic_container_purge_networks | default(omit) }}"

roles/rclone_serve_restic/templates/to-ini.j2

@@ -0,0 +1,6 @@
+{% for section in ini %}
+[{{ section }}]
+{% for (key, value) in section %}
+    {{ key  ~ '=' ~ value }}
+{% endfor %}
+{% endfor %}

roles/rclone_serve_restic/vars/main.yml

@@ -0,0 +1,15 @@
+---
+
+# this should NEVER be overwritten. instead, overwrite _entry_opts_merged
+rclone_serve_restic_container_entry: >
+  {% set args = [] %}
+  {% for opt in rclone_serve_restic_entry_opts_merged %}
+    {% set (opt,value) = opt %}
+    {% if value not "false" %}    # if value is false, then treat as valueless arg to be absent
+      {% do args.append({{ '--' ~ opt | replace('_','-') }}) %}
+      {% if value not "true" %}     # if the value is true, then treat as valueless arg to be present
+        {% do args.append({{ value }}) %}
+      {% endif %}
+    {% endif %}
+  {% endfor %}
+  rclone serve restic {{ args | join(' ') ~ ' ' ~ (rclone_serve_restic_backend | default ('default')) }}