Management Command Database Locking

This commit is contained in:
AJ Slater 2022-02-10 11:44:21 -08:00 committed by GitHub
parent 3fc4cfe46d
commit 870b48dfcd
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 57 additions and 3 deletions

View File

@ -44,15 +44,14 @@ jobs:
python-version: ['3.7', '3.8', '3.9', '3.10']
django-version: ['2.2', '3.2', '4.0']
xapian-version: ['1.4.18']
filelock-version: ['3.4.2']
exclude:
# Django added python 3.10 support in 3.2.9
- python-version: '3.10'
django-version: '2.2'
xapian-version: '1.4.18'
# Django dropped python 3.7 support in 4.0
- python-version: '3.7'
django-version: '4.0'
xapian-version: '1.4.18'
steps:
- name: Set up Python ${{ matrix.python-version }}
@ -74,7 +73,7 @@ jobs:
- name: Install Django and other Python dependencies
run: |
python -m pip install --upgrade pip
pip install django~=${{ matrix.django-version }} coveralls xapian*.whl
pip install django~=${{ matrix.django-version }} filelock~=${{ matrix.filelock-version }} coveralls xapian*.whl
- name: Checkout django-haystack
uses: actions/checkout@v2

View File

@ -6,6 +6,8 @@ Unreleased
----------
- Dropped support for Python 3.6.
- Fixed DatabaseLocked errors when running management commands with
multiple workers.
v3.0.1 (2021-11-12)
-------------------

View File

@ -92,6 +92,8 @@ The backend has the following optional settings:
See `here <http://xapian.org/docs/apidoc/html/classXapian_1_1QueryParser.html#ac7dc3b55b6083bd3ff98fc8b2726c8fd>`__ for
more information about the different strategies.
- ``HAYSTACK_XAPIAN_USE_LOCKFILE``: Use a lockfile to prevent database locking errors when running management commands with multiple workers.
Defaults to `True`.
Testing
-------

View File

@ -1,2 +1,3 @@
Django>=2.2
Django-Haystack>=3.0
filelock>=3.4

View File

@ -28,5 +28,6 @@ setup(
install_requires=[
'django>=2.2',
'django-haystack>=2.8.0',
'filelock>=3.4',
]
)

View File

@ -1,3 +1,5 @@
import sys
from io import StringIO
from unittest import TestCase
from django.core.management import call_command
@ -82,3 +84,20 @@ class ManagementCommandTestCase(HaystackBackendTestCase, TestCase):
# … but remove does:
call_command("update_index", remove=True, verbosity=0)
self.verify_indexed_document_count(self.NUM_BLOG_ENTRIES - 3)
def test_multiprocessing(self):
self.verify_indexed_document_count(0)
old_stderr = sys.stderr
sys.stderr = StringIO()
call_command(
"update_index",
verbosity=2,
workers=10,
batchsize=2,
)
err = sys.stderr.getvalue()
sys.stderr = old_stderr
print(err)
self.assertNotIn("xapian.DatabaseLockError", err)
self.verify_indexed_documents()

View File

@ -1,5 +1,6 @@
import datetime
import pickle
from pathlib import Path
import os
import re
import shutil
@ -8,6 +9,8 @@ import sys
from django.conf import settings
from django.core.exceptions import ImproperlyConfigured
from filelock import FileLock
from haystack import connections
from haystack.backends import BaseEngine, BaseSearchBackend, BaseSearchQuery, SearchNode, log_query
from haystack.constants import ID, DJANGO_ID, DJANGO_CT, DEFAULT_OPERATOR
@ -73,6 +76,24 @@ INTEGER_FORMAT = '%012d'
# texts with positional information
TERMPOS_DISTANCE = 100
def filelocked(func):
"""Decorator to wrap a XapianSearchBackend method in a filelock."""
def wrapper(self, *args, **kwargs):
"""Run the function inside a lock."""
if self.path == MEMORY_DB_NAME or not self.use_lockfile:
func(self, *args, **kwargs)
else:
lockfile = Path(self.filelock.lock_file)
lockfile.parent.mkdir(parents=True, exist_ok=True)
lockfile.touch()
with self.filelock:
func(self, *args, **kwargs)
return wrapper
class InvalidIndexError(HaystackError):
"""Raised when an index can not be opened."""
pass
@ -168,6 +189,9 @@ class XapianSearchBackend(BaseSearchBackend):
Also sets the stemming language to be used to `language`.
"""
self.use_lockfile = bool(
getattr(settings, 'HAYSTACK_XAPIAN_USE_LOCKFILE', True)
)
super().__init__(connection_alias, **connection_options)
if not 'PATH' in connection_options:
@ -182,6 +206,10 @@ class XapianSearchBackend(BaseSearchBackend):
except FileExistsError:
pass
if self.use_lockfile:
lockfile = Path(self.path) / "lockfile"
self.filelock = FileLock(lockfile)
self.flags = connection_options.get('FLAGS', DEFAULT_XAPIAN_FLAGS)
self.language = getattr(settings, 'HAYSTACK_XAPIAN_LANGUAGE', 'english')
@ -225,6 +253,7 @@ class XapianSearchBackend(BaseSearchBackend):
self._update_cache()
return self._columns
@filelocked
def update(self, index, iterable, commit=True):
"""
Updates the `index` with any objects in `iterable` by adding/updating
@ -476,6 +505,7 @@ class XapianSearchBackend(BaseSearchBackend):
finally:
database.close()
@filelocked
def remove(self, obj, commit=True):
"""
Remove indexes for `obj` from the database.