From 740799de2147a672e3e11bf88fe34c759641ac64 Mon Sep 17 00:00:00 2001 From: Evelyn Alicke Date: Tue, 27 Jun 2023 14:28:22 +0200 Subject: [PATCH] feat(rclone_serve): initial implementation Co-Authored-By: Lars Kaiser --- CODEOWNERS | 1 + README.md | 3 +- galaxy.yml | 2 + playbooks/rclone_serve.yml | 5 ++ roles/rclone_serve/README.md | 55 ++++++++++++++++++ roles/rclone_serve/defaults/main.yml | 41 +++++++++++++ roles/rclone_serve/handlers/main.yml | 8 +++ roles/rclone_serve/tasks/install.yml | 58 +++++++++++++++++++ roles/rclone_serve/tasks/main.yml | 4 ++ roles/rclone_serve/tasks/remove.yml | 16 +++++ .../templates/rclone-backend.conf.j2 | 2 + roles/rclone_serve/vars/main.yml | 38 ++++++++++++ 12 files changed, 232 insertions(+), 1 deletion(-) create mode 100644 playbooks/rclone_serve.yml create mode 100644 roles/rclone_serve/README.md create mode 100644 roles/rclone_serve/defaults/main.yml create mode 100644 roles/rclone_serve/handlers/main.yml create mode 100644 roles/rclone_serve/tasks/install.yml create mode 100644 roles/rclone_serve/tasks/main.yml create mode 100644 roles/rclone_serve/tasks/remove.yml create mode 100644 roles/rclone_serve/templates/rclone-backend.conf.j2 create mode 100644 roles/rclone_serve/vars/main.yml diff --git a/CODEOWNERS b/CODEOWNERS index 2be23a4..1aa74ec 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -3,6 +3,7 @@ /roles/dns/ @jdreichmann @famedly/infrastructure /roles/hostname/ @jdreichmann @famedly/infrastructure /roles/ldap/ @jcgruenhage @jdreichmann +/roles/rclone_serve @evlli @lrsksr @famedly/infrastructure /roles/redis/ @jdreichmann @famedly/infrastructure /roles/ssh/ @jdreichmann @famedly/infrastructure /roles/user/ @jcgruenhage @famedly/infrastructure diff --git a/README.md b/README.md index 99bb947..1225457 100644 --- a/README.md +++ b/README.md @@ -14,8 +14,9 @@ to build services on. - [`roles/dropbear_luks_unlock`](roles/dropbear_luks_unlock/README.md) for setting up dropbear to unlock LUKS volumes using a SSH connection at boot - [`roles/hostname`](roles/hostname/README.md) for setting `/etc/hostname` and `/etc/hosts` - [`roles/ldap`](roles/ldap/README.md) to deploy openldap in a docker container -- [`roles/restic`](roles/restic/README.md) to configure backups using restic controlled by systemd +- [`roles/rclone_serve`](roles/rclone_serve/README.md) to deploy rclone serve in a docker container - [`roles/redis`](roles/redis/README.md) to deploy redis in a docker container +- [`roles/restic`](roles/restic/README.md) to configure backups using restic controlled by systemd - [`roles/ssh`](roles/ssh/README.md) for SSH hardening - [`roles/user`](roles/user/README.md) for creating user accounts with SSH keys deployed diff --git a/galaxy.yml b/galaxy.yml index ab79f36..c5286e2 100644 --- a/galaxy.yml +++ b/galaxy.yml @@ -12,6 +12,8 @@ authors: - "Lukas Lihotzki " - "Vincent Wilke" description: "Common functionality needed on hosts to build complex services on." +dependencies: + "sivel.toiletwater": ">=0.0.15" license: ["AGPL-3.0-only"] build_ignore: ["*.tar.gz"] repository: "https://github.com/famedly/ansible-collection-base" diff --git a/playbooks/rclone_serve.yml b/playbooks/rclone_serve.yml new file mode 100644 index 0000000..3864878 --- /dev/null +++ b/playbooks/rclone_serve.yml @@ -0,0 +1,5 @@ +--- +- hosts: "{{ rclone_serve_hosts | default('rclone_serve') }}" + become: true + roles: + - role: rclone_serve diff --git a/roles/rclone_serve/README.md b/roles/rclone_serve/README.md new file mode 100644 index 0000000..4827a62 --- /dev/null +++ b/roles/rclone_serve/README.md @@ -0,0 +1,55 @@ +# `famedly.base.rclone_serve` ansible role deploying rclone in a docker container + +This role is intended to have rclone with the `serve` command permanently deployed inside a docker container to +translate from the supported protocols (eg. restic) to the supported backend storages (eg. Scaleway S3). + +## Role Variables +For configuration consult the [official documentation](https://rclone.org/commands/rclone_serve/). +This role does not verify your config or flags as there are too many combinations. +Check the container logs for error messages. +By default the server will listen on all container addresses on port 8080. + +### Required Variables +- `rclone_serve_protocol` has to be set to one of the supported protocols. +- `rclone_serve_backend_config` is a dict that contains the configuration of the storage backend. + See the example playbook below + +### Optional Variables +Use the `rclone_serve_flags` dict for adding or overriding default command line flags like so: + +```yaml +rclone_serve_flags: + addr: "172.35.1.1:8076" + htpasswd: "{{ rclone_serve_htpasswd_file }}" + private-repos: + append-only: +``` + +For more variables see default/main.yml file. + +## Dependencies +- sivel.toiletwater collection for ini templating +- Docker + +## Example Playbook +```yaml +- hosts: ["my_rclone_host"] + become: true + roles: + - role: rclone_serve + vars: + rclone_serve_protocol: "restic" + rclone_serve_backend_config: + type: "s3" + provider: "Scaleway" + env_auth: false + endpoint: "s3.nl-ams.scw.cloud" + access_key_id: "SCWXXXXXXXXXXXXXX" + secret_access_key: "1111111-2222-3333-44444-55555555555555" + region: "nl-ams" + rclone_serve_flags: + addr: "172.35.1.1:8076" + htpasswd: "{{ rclone_serve_htpasswd_file }}" + private-repos: + append-only: +``` diff --git a/roles/rclone_serve/defaults/main.yml b/roles/rclone_serve/defaults/main.yml new file mode 100644 index 0000000..f9a9864 --- /dev/null +++ b/roles/rclone_serve/defaults/main.yml @@ -0,0 +1,41 @@ +--- + +rclone_serve_role_action: "install" + +rclone_serve_version: "1.64.2" +rclone_serve_user: "rclone" +rclone_serve_base_path: "/opt/rclone-serve/" + +rclone_serve_protocol: ~ +rclone_serve_backend_config: ~ +rclone_serve_flags: ~ + +rclone_serve_config_file: "{{ rclone_serve_base_path ~ 'rclone-backend.conf' }}" + +rclone_serve_container_name: 'rclone-serve' +rclone_serve_container_env: {} +rclone_serve_container_ports: [] +rclone_serve_container_labels: {} +rclone_serve_container_volumes: [] +rclone_serve_container_restart_policy: "unless-stopped" +rclone_serve_container_force_pull: false + +rclone_serve_container_image_reference: >- + {{ + rclone_serve_container_image_repository + + ':' + + rclone_serve_container_image_tag | default(rclone_serve_version) + }} +rclone_serve_container_image_repository: >- + {{ + ( + container_registries[rclone_serve_container_image_registry] + | default(rclone_serve_container_image_registry) + ) + + '/' + + rclone_serve_container_image_namespace | default('') + + rclone_serve_container_image_name + }} +rclone_serve_container_image_registry: "docker.io" +rclone_serve_container_image_namespace: "rclone/" +rclone_serve_container_image_name: "rclone" diff --git a/roles/rclone_serve/handlers/main.yml b/roles/rclone_serve/handlers/main.yml new file mode 100644 index 0000000..2764384 --- /dev/null +++ b/roles/rclone_serve/handlers/main.yml @@ -0,0 +1,8 @@ +--- + +- name: Restart rclone-serve container + listen: container-rclone-serve-restart + community.docker.docker_container: + name: "{{ rclone_serve_container_name }}" + state: started + restart: true diff --git a/roles/rclone_serve/tasks/install.yml b/roles/rclone_serve/tasks/install.yml new file mode 100644 index 0000000..b33b206 --- /dev/null +++ b/roles/rclone_serve/tasks/install.yml @@ -0,0 +1,58 @@ +--- + +- name: Assert that required variables are defined + ansible.builtin.assert: + that: + - rclone_serve_protocol != None + - rclone_serve_backend_config != None + +- name: Ensure user is present + ansible.builtin.user: + name: "{{ rclone_serve_user }}" + state: "present" + system: true + register: rclone_serve_user_res + +- name: Ensure config directory is present + ansible.builtin.file: + path: "{{ rclone_serve_base_path }}" + state: "directory" + mode: "755" + owner: "{{ rclone_serve_user_res.uid }}" + group: "{{ rclone_serve_user_res.group }}" + +- name: Ensure config is present + ansible.builtin.template: + src: "rclone-backend.conf.j2" + dest: "{{ rclone_serve_config_file }}" + mode: "0600" + owner: "{{ rclone_serve_user_res.uid }}" + group: "{{ rclone_serve_user_res.group }}" + notify: container-rclone-serve-restart + +- name: Ensure container image is present locally + docker_image: + name: "{{ rclone_serve_container_image_reference }}" + source: "pull" + state: "present" + force_source: "{{ rclone_serve_container_force_pull }}" + register: rclone_serve_container_image_pulled + until: rclone_serve_container_image_pulled is success + retries: 10 + delay: 5 + +- name: Ensure container is started + community.docker.docker_container: + image: "{{ rclone_serve_container_image_reference }}" + name: "{{ rclone_serve_container_name }}" + state: "started" + restart_policy: "{{ rclone_serve_container_restart_policy | default(omit) }}" + user: "{{ rclone_serve_user_res.uid ~ ':' ~ rclone_serve_user_res.group }}" + volumes: "{{ rclone_serve_container_volumes_merged }}" + ports: "{{ rclone_serve_container_ports }}" + env: "{{ rclone_serve_container_env | default(omit) }}" + labels: "{{ rclone_serve_container_labels_merged | default(omit) }}" + command: "{{ rclone_serve_container_command }}" + etc_hosts: "{{ rclone_serve_container_etc_hosts | default(omit) }}" + networks: "{{ rclone_serve_container_networks | default(omit) }}" + purge_networks: "{{ rclone_serve_container_purge_networks | default(omit) }}" diff --git a/roles/rclone_serve/tasks/main.yml b/roles/rclone_serve/tasks/main.yml new file mode 100644 index 0000000..1006a58 --- /dev/null +++ b/roles/rclone_serve/tasks/main.yml @@ -0,0 +1,4 @@ +--- + +- name: Include tasks for {{ rclone_serve_role_action }} + include_tasks: "{{ rclone_serve_role_action }}.yml" diff --git a/roles/rclone_serve/tasks/remove.yml b/roles/rclone_serve/tasks/remove.yml new file mode 100644 index 0000000..8062499 --- /dev/null +++ b/roles/rclone_serve/tasks/remove.yml @@ -0,0 +1,16 @@ +--- + +- name: Ensure container is absent + community.docker.docker_container: + name: "{{ rclone_serve_container_name }}" + state: "absent" + +- name: Ensure config directory is absent + ansible.builtin.file: + path: "{{ rclone_serve_base_path }}" + state: "absent" + +- name: Ensure user is absent + ansible.builtin.user: + name: "{{ rclone_serve_user }}" + state: "absent" diff --git a/roles/rclone_serve/templates/rclone-backend.conf.j2 b/roles/rclone_serve/templates/rclone-backend.conf.j2 new file mode 100644 index 0000000..38d9baa --- /dev/null +++ b/roles/rclone_serve/templates/rclone-backend.conf.j2 @@ -0,0 +1,2 @@ +{{ 'Managed by ansible' | comment('plain', prefix='#####', postfix='#####') }} +{{ rclone_serve_backends | sivel.toiletwater.to_ini }} diff --git a/roles/rclone_serve/vars/main.yml b/roles/rclone_serve/vars/main.yml new file mode 100644 index 0000000..0cc0eee --- /dev/null +++ b/roles/rclone_serve/vars/main.yml @@ -0,0 +1,38 @@ +--- + +rclone_serve_backends: + default: "{{ rclone_serve_backend_config }}" + +rclone_serve_flags_base: + addr: ":8080" + config: "{{ rclone_serve_config_file }}" +rclone_serve_flags_merged: >- + {{ + rclone_serve_flags_base + | combine(rclone_serve_flags) + }} + +rclone_serve_container_volumes_base: + - "{{ rclone_serve_base_path }}:{{ rclone_serve_base_path }}:Z" +rclone_serve_container_volumes_merged: "{{ rclone_serve_container_volumes_base + rclone_serve_container_volumes }}" + +rclone_serve_container_labels_base: + version: "{{ rclone_serve_version }}" +rclone_serve_container_labels_merged: >- + {{ rclone_serve_container_labels_base + | combine(rclone_serve_container_labels | default({})) }} + +# this should NEVER be overwritten. instead, overwrite rclone_serve_flags_merged +rclone_serve_container_command: >- + {% set rclone_flags = [] %} + {%- for rclone_flag in rclone_serve_flags_merged | dict2items -%} + {%- if not rclone_flag.value -%} + {%- do rclone_flags.append('--' ~ rclone_flag.key) -%} + {%- elif rclone_flag.value -%} + {%- do rclone_flags.append('--' ~ rclone_flag.key ~ ' ' ~ rclone_flag.value) -%} + {%- endif -%} + {%- endfor -%} + serve + {{ rclone_serve_protocol }} + default: + {{ rclone_flags | join(' ') }}