feat(rclone_serve): initial implementation

Co-Authored-By: Lars Kaiser <lars@kaiser.yt>

CODEOWNERS

@@ -3,6 +3,7 @@
 /roles/dns/ @jdreichmann @famedly/infrastructure
 /roles/hostname/ @jdreichmann @famedly/infrastructure
 /roles/ldap/ @jcgruenhage @jdreichmann
+/roles/rclone_serve @evlli @lrsksr @famedly/infrastructure
 /roles/redis/ @jdreichmann @famedly/infrastructure
 /roles/ssh/ @jdreichmann @famedly/infrastructure
 /roles/user/ @jcgruenhage @famedly/infrastructure

README.md

@@ -14,8 +14,9 @@ to build services on.
 - [`roles/dropbear_luks_unlock`](roles/dropbear_luks_unlock/README.md) for setting up dropbear to unlock LUKS volumes using a SSH connection at boot
 - [`roles/hostname`](roles/hostname/README.md) for setting `/etc/hostname` and `/etc/hosts`
 - [`roles/ldap`](roles/ldap/README.md) to deploy openldap in a docker container
-- [`roles/restic`](roles/restic/README.md) to configure backups using restic controlled by systemd
+- [`roles/rclone_serve`](roles/rclone_serve/README.md) to deploy rclone serve in a docker container
 - [`roles/redis`](roles/redis/README.md) to deploy redis in a docker container
+- [`roles/restic`](roles/restic/README.md) to configure backups using restic controlled by systemd
 - [`roles/ssh`](roles/ssh/README.md) for SSH hardening
 - [`roles/user`](roles/user/README.md) for creating user accounts with SSH keys deployed
 

galaxy.yml

@@ -12,6 +12,8 @@ authors:
   - "Lukas Lihotzki <l.lihotzki@famedly.com>"
   - "Vincent Wilke"
 description: "Common functionality needed on hosts to build complex services on."
+dependencies:
+  "sivel.toiletwater": ">=0.0.15"
 license: ["AGPL-3.0-only"]
 build_ignore: ["*.tar.gz"]
 repository: "https://github.com/famedly/ansible-collection-base"

playbooks/rclone_serve.yml

@@ -0,0 +1,5 @@
+---
+- hosts: "{{ rclone_serve_hosts | default('rclone_serve') }}"
+  become: true
+  roles:
+    - role: rclone_serve

roles/rclone_serve/README.md

@@ -0,0 +1,55 @@
+# `famedly.base.rclone_serve` ansible role deploying rclone in a docker container
+
+This role is intended to have rclone with the `serve` command permanently deployed inside a docker container to
+translate from the supported protocols (eg. restic) to the supported backend storages (eg. Scaleway S3).
+
+## Role Variables
+For configuration consult the [official documentation](https://rclone.org/commands/rclone_serve/).
+This role does not verify your config or flags as there are too many combinations.
+Check the container logs for error messages.
+By default the server will listen on all container addresses on port 8080.
+
+### Required Variables
+- `rclone_serve_protocol` has to be set to one of the supported protocols.
+- `rclone_serve_backend_config` is a dict that contains the configuration of the storage backend.
+  See the example playbook below
+
+### Optional Variables
+Use the `rclone_serve_flags` dict for adding or overriding default command line flags like so:
+
+```yaml
+rclone_serve_flags:
+  addr: "172.35.1.1:8076"
+  htpasswd: "{{ rclone_serve_htpasswd_file }}"
+  private-repos:
+  append-only:
+```
+
+For more variables see default/main.yml file.
+
+## Dependencies
+- sivel.toiletwater collection for ini templating
+- Docker
+
+## Example Playbook
+```yaml
+- hosts: ["my_rclone_host"]
+  become: true
+  roles:
+    - role: rclone_serve
+  vars:
+    rclone_serve_protocol: "restic"
+    rclone_serve_backend_config:
+      type: "s3"
+      provider: "Scaleway"
+      env_auth: false
+      endpoint: "s3.nl-ams.scw.cloud"
+      access_key_id: "SCWXXXXXXXXXXXXXX"
+      secret_access_key: "1111111-2222-3333-44444-55555555555555"
+      region: "nl-ams"
+    rclone_serve_flags:
+      addr: "172.35.1.1:8076"
+      htpasswd: "{{ rclone_serve_htpasswd_file }}"
+      private-repos:
+      append-only:
+```

roles/rclone_serve/defaults/main.yml

@@ -0,0 +1,41 @@
+---
+
+rclone_serve_role_action: "install"
+
+rclone_serve_version: "1.64.2"
+rclone_serve_user: "rclone"
+rclone_serve_base_path: "/opt/rclone-serve/"
+
+rclone_serve_protocol: ~
+rclone_serve_backend_config: ~
+rclone_serve_flags: ~
+
+rclone_serve_config_file: "{{ rclone_serve_base_path ~ 'rclone-backend.conf' }}"
+
+rclone_serve_container_name: 'rclone-serve'
+rclone_serve_container_env: {}
+rclone_serve_container_ports: []
+rclone_serve_container_labels: {}
+rclone_serve_container_volumes: []
+rclone_serve_container_restart_policy: "unless-stopped"
+rclone_serve_container_force_pull: false
+
+rclone_serve_container_image_reference: >-
+  {{
+    rclone_serve_container_image_repository
+    + ':'
+    + rclone_serve_container_image_tag | default(rclone_serve_version)
+  }}
+rclone_serve_container_image_repository: >-
+  {{
+    (
+      container_registries[rclone_serve_container_image_registry]
+      | default(rclone_serve_container_image_registry)
+    )
+    + '/'
+    + rclone_serve_container_image_namespace | default('')
+    + rclone_serve_container_image_name
+  }}
+rclone_serve_container_image_registry: "docker.io"
+rclone_serve_container_image_namespace: "rclone/"
+rclone_serve_container_image_name: "rclone"

roles/rclone_serve/handlers/main.yml

@@ -0,0 +1,8 @@
+---
+
+- name: Restart rclone-serve container
+  listen: container-rclone-serve-restart
+  community.docker.docker_container:
+    name: "{{ rclone_serve_container_name }}"
+    state: started
+    restart: true

roles/rclone_serve/tasks/install.yml

@@ -0,0 +1,58 @@
+---
+
+- name: Assert that required variables are defined
+  ansible.builtin.assert:
+    that:
+      - rclone_serve_protocol != None
+      - rclone_serve_backend_config != None
+
+- name: Ensure user is present
+  ansible.builtin.user:
+    name: "{{ rclone_serve_user }}"
+    state: "present"
+    system: true
+  register: rclone_serve_user_res
+
+- name: Ensure config directory is present
+  ansible.builtin.file:
+    path: "{{ rclone_serve_base_path }}"
+    state: "directory"
+    mode: "755"
+    owner: "{{ rclone_serve_user_res.uid }}"
+    group: "{{ rclone_serve_user_res.group }}"
+
+- name: Ensure config is present
+  ansible.builtin.template:
+    src: "rclone-backend.conf.j2"
+    dest: "{{ rclone_serve_config_file }}"
+    mode: "0600"
+    owner: "{{ rclone_serve_user_res.uid }}"
+    group: "{{ rclone_serve_user_res.group }}"
+  notify: container-rclone-serve-restart
+
+- name: Ensure container image is present locally
+  docker_image:
+    name: "{{ rclone_serve_container_image_reference }}"
+    source: "pull"
+    state: "present"
+    force_source: "{{ rclone_serve_container_force_pull }}"
+  register: rclone_serve_container_image_pulled
+  until: rclone_serve_container_image_pulled is success
+  retries: 10
+  delay: 5
+
+- name: Ensure container is started
+  community.docker.docker_container:
+    image: "{{ rclone_serve_container_image_reference }}"
+    name: "{{ rclone_serve_container_name }}"
+    state: "started"
+    restart_policy: "{{ rclone_serve_container_restart_policy | default(omit) }}"
+    user: "{{ rclone_serve_user_res.uid ~ ':' ~ rclone_serve_user_res.group }}"
+    volumes: "{{ rclone_serve_container_volumes_merged }}"
+    ports: "{{ rclone_serve_container_ports }}"
+    env: "{{ rclone_serve_container_env | default(omit) }}"
+    labels: "{{ rclone_serve_container_labels_merged | default(omit) }}"
+    command: "{{ rclone_serve_container_command }}"
+    etc_hosts: "{{ rclone_serve_container_etc_hosts | default(omit) }}"
+    networks: "{{ rclone_serve_container_networks | default(omit) }}"
+    purge_networks: "{{ rclone_serve_container_purge_networks | default(omit) }}"

roles/rclone_serve/tasks/main.yml

@@ -0,0 +1,4 @@
+---
+
+- name: Include tasks for {{ rclone_serve_role_action }}
+  include_tasks: "{{ rclone_serve_role_action }}.yml"

roles/rclone_serve/tasks/remove.yml

@@ -0,0 +1,16 @@
+---
+
+- name: Ensure container is absent
+  community.docker.docker_container:
+    name: "{{ rclone_serve_container_name }}"
+    state: "absent"
+
+- name: Ensure config directory is absent
+  ansible.builtin.file:
+    path: "{{ rclone_serve_base_path }}"
+    state: "absent"
+
+- name: Ensure user is absent
+  ansible.builtin.user:
+    name: "{{ rclone_serve_user }}"
+    state: "absent"

roles/rclone_serve/templates/rclone-backend.conf.j2

@@ -0,0 +1,2 @@
+{{ 'Managed by ansible' | comment('plain', prefix='#####', postfix='#####') }}
+{{ rclone_serve_backends | sivel.toiletwater.to_ini }}

roles/rclone_serve/vars/main.yml

@@ -0,0 +1,38 @@
+---
+
+rclone_serve_backends:
+  default: "{{ rclone_serve_backend_config }}"
+
+rclone_serve_flags_base:
+  addr: ":8080"
+  config: "{{ rclone_serve_config_file }}"
+rclone_serve_flags_merged: >-
+  {{
+    rclone_serve_flags_base
+    | combine(rclone_serve_flags)
+  }}
+
+rclone_serve_container_volumes_base:
+  - "{{ rclone_serve_base_path }}:{{ rclone_serve_base_path }}:Z"
+rclone_serve_container_volumes_merged: "{{ rclone_serve_container_volumes_base + rclone_serve_container_volumes }}"
+
+rclone_serve_container_labels_base:
+  version: "{{ rclone_serve_version }}"
+rclone_serve_container_labels_merged: >-
+  {{ rclone_serve_container_labels_base
+  | combine(rclone_serve_container_labels | default({})) }}
+
+# this should NEVER be overwritten. instead, overwrite rclone_serve_flags_merged
+rclone_serve_container_command: >-
+  {% set rclone_flags = [] %}
+    {%- for rclone_flag in rclone_serve_flags_merged | dict2items -%}
+      {%- if not rclone_flag.value -%}
+        {%- do rclone_flags.append('--' ~ rclone_flag.key) -%}
+      {%- elif rclone_flag.value -%}
+        {%- do rclone_flags.append('--' ~ rclone_flag.key ~ ' ' ~ rclone_flag.value) -%}
+      {%- endif -%}
+    {%- endfor -%}
+  serve
+  {{ rclone_serve_protocol }}
+  default:
+  {{ rclone_flags | join(' ') }}