Fix case-insensitive search for unicode characters in SQLite (#520)

Dockerfile

@@ -33,13 +33,39 @@ RUN mkdir /opt/venv && \
     /opt/venv/bin/pip install -Ur requirements.txt
 
 
+FROM python-base AS compile-icu
+RUN apt-get update && apt-get -y install libicu-dev libsqlite3-dev wget unzip
+WORKDIR /etc/linkding
+
+# Defines SQLite version
+# Since this is only needed for downloading the header files this probably
+# doesn't need to be up-to-date, assuming the SQLite APIs used by the ICU
+# extension do not change
+ARG SQLITE_RELEASE_YEAR=2023
+ARG SQLITE_RELEASE=3430000
+
+# Compile the ICU extension needed for case-insensitive search and ordering
+# with SQLite. This does:
+# - Download SQLite amalgamation for header files
+# - Download ICU extension source file
+# - Compile ICU extension
+RUN wget https://www.sqlite.org/${SQLITE_RELEASE_YEAR}/sqlite-amalgamation-${SQLITE_RELEASE}.zip && \
+    unzip sqlite-amalgamation-${SQLITE_RELEASE}.zip && \
+    cp sqlite-amalgamation-${SQLITE_RELEASE}/sqlite3.h ./sqlite3.h && \
+    cp sqlite-amalgamation-${SQLITE_RELEASE}/sqlite3ext.h ./sqlite3ext.h && \
+    wget https://www.sqlite.org/src/raw/ext/icu/icu.c?name=91c021c7e3e8bbba286960810fa303295c622e323567b2e6def4ce58e4466e60 -O icu.c && \
+    gcc -fPIC -shared icu.c `pkg-config --libs --cflags icu-uc icu-io` -o libicu.so
+
+
 FROM python:3.10.6-slim-buster as final
-RUN apt-get update && apt-get -y install mime-support libpq-dev curl
+RUN apt-get update && apt-get -y install mime-support libpq-dev libicu-dev curl
 WORKDIR /etc/linkding
 # copy prod dependencies
 COPY --from=prod-deps /opt/venv /opt/venv
 # copy output from build stage
 COPY --from=python-build /etc/linkding/static static/
+# copy compiled icu extension
+COPY --from=compile-icu /etc/linkding/libicu.so libicu.so
 # copy application code
 COPY . .
 # Expose uwsgi server at port 9090

bookmarks/signals.py

@@ -1,8 +1,31 @@
+from os import path
+
 from django.contrib.auth import user_logged_in
+from django.db.backends.signals import connection_created
 from django.dispatch import receiver
+
 from bookmarks.services import tasks
 
+icu_extension_path = './libicu.so'
+icu_extension_exists = path.exists(icu_extension_path)
+
 
 @receiver(user_logged_in)
 def user_logged_in(sender, request, user, **kwargs):
     tasks.schedule_bookmarks_without_snapshots(user)
+
+
+@receiver(connection_created)
+def extend_sqlite(connection=None, **kwargs):
+    # Load ICU extension into Sqlite connection to support case-insensitive
+    # comparisons with unicode characters
+    if connection.vendor == 'sqlite' and icu_extension_exists:
+        connection.connection.enable_load_extension(True)
+        connection.connection.load_extension('./libicu')
+
+        with connection.cursor() as cursor:
+            # Load an ICU collation for case-insensitive ordering.
+            # The first param can be a specific locale, it seems that not
+            # providing one will use a default collation from the ICU project
+            # that works reasonably for multiple languages
+            cursor.execute("SELECT icu_load_collation('', 'ICU');")