Add option to disable bookmark URL validation (#57)

* Add option for disabled bookmark URL validation (#36)

* Add options documentation (#36)

.env.sample

@@ -1,3 +1,9 @@
+# Docker container name
 LD_CONTAINER_NAME=linkding
+# Port on the host system that the application should be published on
 LD_HOST_PORT=9090
+# Directory on the host system that should be mounted as data dir into the Docker container
 LD_HOST_DATA_DIR=./data
+
+# Option to disable URL validation for bookmarks completely
+LD_DISABLE_URL_VALIDATION=False

Options.md

@@ -0,0 +1,38 @@
+# Options
+
+This document lists the options that linkding can be configured with and explains how to use them in the individual install scenarios.
+
+## Using options
+
+### Docker
+
+Options are passed as environment variables to the Docker container by using the `-e` argument when using `docker run`. For example:
+
+```
+docker run --name linkding -p 9090:9090 -d -e LD_DISABLE_URL_VALIDATION=True sissbruecker/linkding:latest
+```
+
+For multiple options, use one `-e` argument per option.
+
+### Docker-compose
+
+For docker-compose options are configured using an `.env` file. 
+Follow the docker-compose setup in the README and copy `.env.sample` to `.env`. Then modify the options in `.env`.
+
+### Manual setup
+
+All options need to be defined as environment variables in the environment that linkding runs in.
+
+## List of options
+
+### `LD_DISABLE_URL_VALIDATION`
+
+Values: `True`, `False` | Default = `False`
+
+Completely disables URL validation for bookmarks. This can be useful if you intend to store non fully qualified domain name URLs, such as network paths, or you want to store URLs that use another protocol than `http` or `https`.
+
+### `LD_REQUEST_TIMEOUT`
+
+Values: `Integer` as seconds | Default = `60`
+
+Configures the request timeout in the uwsgi application server. This can be useful if you want to import a bookmark file with a high number of bookmarks and run into request timeouts.

README.md

@@ -38,7 +38,7 @@ If everything completed successfully the application should now be running and c
 
 If you are using a Linux system you can use the following [shell script](https://github.com/sissbruecker/linkding/blob/master/install-linkding.sh) for an automated setup. The script does basically everything described above, but also handles updating an existing container to a new application version (technically replaces the existing container with a new container built from a newer image, while mounting the same data folder).
 
-The script can be configured using using shell variables - for more details have a look at the script itself.
+The script can be configured using shell variables - for more details have a look at the script itself.
 
 ###  Docker-compose setup
 
@@ -67,6 +67,10 @@ The command will prompt you for a secure password. After the command has complet
 
 If you can not or don't want to use Docker you can install the application manually on your server. To do so you can basically follow the steps from the *Development* section below while cross-referencing the `Dockerfile` and `bootstrap.sh` on how to make the application production-ready.
 
+### Options
+
+Check the [options document](Options.md) on how to configure your linkding installation.
+
 ### Hosting
 
 The application runs in a web-server called [uWSGI](https://uwsgi-docs.readthedocs.io/en/latest/) that is production-ready and that you can expose to the web. If you don't know how to configure your server to expose the application to the web there are several more steps involved. I can not support support the process here, but I can give some pointers on what to search for:
@@ -93,11 +97,7 @@ The application provides a REST API that can be used by 3rd party applications t
 The default timeout for requests is 60 seconds, after which the application server will cancel the request and return the above error.
 Depending on the system that the application runs on, and the number of bookmarks that need to be imported, the import may take longer than the default 60 seconds.
 
-To increase the timeout you can provide a custom timeout to the Docker container using the `LD_REQUEST_TIMEOUT` environment variable:
-
-```
-docker run --name linkding -p 9090:9090 -e LD_REQUEST_TIMEOUT=180 -d sissbruecker/linkding:latest
-```
+To increase the timeout you can configure the [`LD_REQUEST_TIMEOUT` option](Options.md#LD_REQUEST_TIMEOUT).
 
 Note that any proxy servers that you are running in front of linkding may have their own timeout settings, which are not affected by the variable.
 

bookmarks/migrations/0005_auto_20210103_1212.py

@@ -0,0 +1,19 @@
+# Generated by Django 2.2.13 on 2021-01-03 12:12
+
+import bookmarks.validators
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('bookmarks', '0004_auto_20200926_1028'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='bookmark',
+            name='url',
+            field=models.CharField(max_length=2048, validators=[bookmarks.validators.BookmarkURLValidator()]),
+        ),
+    ]

bookmarks/models.py

@@ -5,6 +5,7 @@ from django.contrib.auth import get_user_model
 from django.db import models
 
 from bookmarks.utils import unique
+from bookmarks.validators import BookmarkURLValidator
 
 
 class Tag(models.Model):
@@ -32,7 +33,7 @@ def build_tag_string(tag_names: List[str], delimiter: str = ','):
 
 
 class Bookmark(models.Model):
-    url = models.URLField(max_length=2048)
+    url = models.CharField(max_length=2048, validators=[BookmarkURLValidator()])
     title = models.CharField(max_length=512, blank=True)
     description = models.TextField(blank=True)
     website_title = models.CharField(max_length=512, blank=True, null=True)
@@ -76,7 +77,7 @@ class Bookmark(models.Model):
 
 class BookmarkForm(forms.ModelForm):
     # Use URLField for URL
-    url = forms.URLField()
+    url = forms.CharField(validators=[BookmarkURLValidator()])
     tag_string = forms.CharField(required=False)
     # Do not require title and description in form as we fill these automatically if they are empty
     title = forms.CharField(max_length=512,

bookmarks/tests/test_bookmark_validation.py

@@ -0,0 +1,90 @@
+import datetime
+
+from django.contrib.auth import get_user_model
+from django.core.exceptions import ValidationError
+from django.test import TestCase, override_settings
+
+from bookmarks.models import BookmarkForm, Bookmark
+
+User = get_user_model()
+
+ENABLED_URL_VALIDATION_TEST_CASES = [
+    ('thisisnotavalidurl', False),
+    ('http://domain', False),
+    ('unknownscheme://domain.com', False),
+    ('http://domain.com', True),
+    ('http://www.domain.com', True),
+    ('https://domain.com', True),
+    ('https://www.domain.com', True),
+]
+
+DISABLED_URL_VALIDATION_TEST_CASES = [
+    ('thisisnotavalidurl', True),
+    ('http://domain', True),
+    ('unknownscheme://domain.com', True),
+    ('http://domain.com', True),
+    ('http://www.domain.com', True),
+    ('https://domain.com', True),
+    ('https://www.domain.com', True),
+]
+
+
+class BookmarkValidationTestCase(TestCase):
+
+    def setUp(self) -> None:
+        self.user = User.objects.create_user('testuser', 'test@example.com', 'password123')
+
+    @override_settings(LD_DISABLE_URL_VALIDATION=False)
+    def test_bookmark_model_should_validate_url_if_not_disabled_in_settings(self):
+        self._run_bookmark_model_url_validity_checks(ENABLED_URL_VALIDATION_TEST_CASES)
+
+    @override_settings(LD_DISABLE_URL_VALIDATION=True)
+    def test_bookmark_model_should_not_validate_url_if_disabled_in_settings(self):
+        self._run_bookmark_model_url_validity_checks(DISABLED_URL_VALIDATION_TEST_CASES)
+
+    def test_bookmark_form_should_validate_required_fields(self):
+        form = BookmarkForm(data={'url': ''})
+
+        self.assertEqual(len(form.errors), 1)
+        self.assertIn('required', str(form.errors))
+
+        form = BookmarkForm(data={'url': None})
+
+        self.assertEqual(len(form.errors), 1)
+        self.assertIn('required', str(form.errors))
+
+    @override_settings(LD_DISABLE_URL_VALIDATION=False)
+    def test_bookmark_form_should_validate_url_if_not_disabled_in_settings(self):
+        self._run_bookmark_form_url_validity_checks(ENABLED_URL_VALIDATION_TEST_CASES)
+
+    @override_settings(LD_DISABLE_URL_VALIDATION=True)
+    def test_bookmark_form_should_not_validate_url_if_disabled_in_settings(self):
+        self._run_bookmark_form_url_validity_checks(DISABLED_URL_VALIDATION_TEST_CASES)
+
+    def _run_bookmark_model_url_validity_checks(self, cases):
+        for case in cases:
+            url, expectation = case
+            bookmark = Bookmark(
+                url=url,
+                date_added=datetime.datetime.now(),
+                date_modified=datetime.datetime.now(),
+                owner=self.user
+            )
+
+            try:
+                bookmark.full_clean()
+                self.assertTrue(expectation, 'Did not expect validation error')
+            except ValidationError as e:
+                self.assertFalse(expectation, 'Expected validation error')
+                self.assertTrue('url' in e.message_dict, 'Expected URL validation to fail')
+
+    def _run_bookmark_form_url_validity_checks(self, cases):
+        for case in cases:
+            url, expectation = case
+            form = BookmarkForm(data={'url': url})
+
+            if expectation:
+                self.assertEqual(len(form.errors), 0)
+            else:
+                self.assertEqual(len(form.errors), 1)
+                self.assertIn('Enter a valid URL', str(form.errors))

bookmarks/validators.py

@@ -0,0 +1,14 @@
+from django.conf import settings
+from django.core import validators
+
+
+class BookmarkURLValidator(validators.URLValidator):
+    """
+    Extends default Django URLValidator and cancels validation if it is disabled in settings.
+    This allows to switch URL validation on/off dynamically which helps with testing
+    """
+    def __call__(self, value):
+        if settings.LD_DISABLE_URL_VALIDATION:
+            return
+
+        super().__call__(value)

siteroot/settings/base.py

@@ -160,3 +160,6 @@ REST_FRAMEWORK = {
 
 # Registration switch
 ALLOW_REGISTRATION = False
+
+# URL validation flag
+LD_DISABLE_URL_VALIDATION = os.getenv('LD_DISABLE_URL_VALIDATION', False) in (True, 'True', '1')