feat(prometheus): Add shutdown timeout variable (#220)

In order to allow Prometheus more time to shutdown, especially if
`memory-snapshot-on-shutdown` is used, override the default 90s systemd
shutdown timeout. Otherwise systemd will SIGKILL Prometheus.
* Use 10min shutdown timeout by default.

Signed-off-by: SuperQ <superq@gmail.com>

roles/prometheus/defaults/main.yml

@@ -226,3 +226,5 @@ prometheus_alert_rules:  # noqa yaml[line-length]  # noqa line-length
     labels:
       severity: warning
 # yamllint enable rule:line-length
+
+prometheus_stop_timeout: '600s'

roles/prometheus/meta/argument_specs.yml

@@ -150,3 +150,8 @@ argument_specs:
         default:
           - "prometheus/targets/*.yml"
           - "prometheus/targets/*.json"
+      prometheus_stop_timeout:
+        description:
+          - "How long to wait for Prometheus to shutdown. This is passed as a systemd TimeoutStopSec time spec."
+        type: "str"
+        default: "600s"

roles/prometheus/molecule/alternative/molecule.yml

@@ -87,3 +87,4 @@ provisioner:
               - target_label: __address__
                 replacement: 127.0.0.1:9115  # Blackbox exporter.
         version: 2.25.2
+        prometheus_stop_timeout: 1min

roles/prometheus/molecule/alternative/tests/test_alternative.py

@@ -42,6 +42,8 @@ def test_files(host, files):
      "enable-feature=promql-at-modifier"),
     ("/etc/systemd/system/prometheus.service",
      "enable-feature=remote-write-receiver"),
+    ("/etc/systemd/system/prometheus.service",
+     "TimeoutStopSec=1min"),
 ])
 def test_file_contents(host, file, content):
     f = host.file(file)

roles/prometheus/templates/prometheus.service.j2

@@ -85,6 +85,7 @@ Environment="HTTP_PROXY={{ http_proxy }}"{% if https_proxy is defined %} "HTTPS_
 
 SyslogIdentifier=prometheus
 Restart=always
+TimeoutStopSec={{ prometheus_stop_timeout }}
 
 [Install]
 WantedBy=multi-user.target