Management Command Database Locking
This commit is contained in:
parent
3fc4cfe46d
commit
870b48dfcd
|
@ -44,15 +44,14 @@ jobs:
|
||||||
python-version: ['3.7', '3.8', '3.9', '3.10']
|
python-version: ['3.7', '3.8', '3.9', '3.10']
|
||||||
django-version: ['2.2', '3.2', '4.0']
|
django-version: ['2.2', '3.2', '4.0']
|
||||||
xapian-version: ['1.4.18']
|
xapian-version: ['1.4.18']
|
||||||
|
filelock-version: ['3.4.2']
|
||||||
exclude:
|
exclude:
|
||||||
# Django added python 3.10 support in 3.2.9
|
# Django added python 3.10 support in 3.2.9
|
||||||
- python-version: '3.10'
|
- python-version: '3.10'
|
||||||
django-version: '2.2'
|
django-version: '2.2'
|
||||||
xapian-version: '1.4.18'
|
|
||||||
# Django dropped python 3.7 support in 4.0
|
# Django dropped python 3.7 support in 4.0
|
||||||
- python-version: '3.7'
|
- python-version: '3.7'
|
||||||
django-version: '4.0'
|
django-version: '4.0'
|
||||||
xapian-version: '1.4.18'
|
|
||||||
|
|
||||||
steps:
|
steps:
|
||||||
- name: Set up Python ${{ matrix.python-version }}
|
- name: Set up Python ${{ matrix.python-version }}
|
||||||
|
@ -74,7 +73,7 @@ jobs:
|
||||||
- name: Install Django and other Python dependencies
|
- name: Install Django and other Python dependencies
|
||||||
run: |
|
run: |
|
||||||
python -m pip install --upgrade pip
|
python -m pip install --upgrade pip
|
||||||
pip install django~=${{ matrix.django-version }} coveralls xapian*.whl
|
pip install django~=${{ matrix.django-version }} filelock~=${{ matrix.filelock-version }} coveralls xapian*.whl
|
||||||
|
|
||||||
- name: Checkout django-haystack
|
- name: Checkout django-haystack
|
||||||
uses: actions/checkout@v2
|
uses: actions/checkout@v2
|
||||||
|
|
|
@ -6,6 +6,8 @@ Unreleased
|
||||||
----------
|
----------
|
||||||
|
|
||||||
- Dropped support for Python 3.6.
|
- Dropped support for Python 3.6.
|
||||||
|
- Fixed DatabaseLocked errors when running management commands with
|
||||||
|
multiple workers.
|
||||||
|
|
||||||
v3.0.1 (2021-11-12)
|
v3.0.1 (2021-11-12)
|
||||||
-------------------
|
-------------------
|
||||||
|
|
|
@ -92,6 +92,8 @@ The backend has the following optional settings:
|
||||||
See `here <http://xapian.org/docs/apidoc/html/classXapian_1_1QueryParser.html#ac7dc3b55b6083bd3ff98fc8b2726c8fd>`__ for
|
See `here <http://xapian.org/docs/apidoc/html/classXapian_1_1QueryParser.html#ac7dc3b55b6083bd3ff98fc8b2726c8fd>`__ for
|
||||||
more information about the different strategies.
|
more information about the different strategies.
|
||||||
|
|
||||||
|
- ``HAYSTACK_XAPIAN_USE_LOCKFILE``: Use a lockfile to prevent database locking errors when running management commands with multiple workers.
|
||||||
|
Defaults to `True`.
|
||||||
|
|
||||||
Testing
|
Testing
|
||||||
-------
|
-------
|
||||||
|
|
|
@ -1,2 +1,3 @@
|
||||||
Django>=2.2
|
Django>=2.2
|
||||||
Django-Haystack>=3.0
|
Django-Haystack>=3.0
|
||||||
|
filelock>=3.4
|
||||||
|
|
1
setup.py
1
setup.py
|
@ -28,5 +28,6 @@ setup(
|
||||||
install_requires=[
|
install_requires=[
|
||||||
'django>=2.2',
|
'django>=2.2',
|
||||||
'django-haystack>=2.8.0',
|
'django-haystack>=2.8.0',
|
||||||
|
'filelock>=3.4',
|
||||||
]
|
]
|
||||||
)
|
)
|
||||||
|
|
|
@ -1,3 +1,5 @@
|
||||||
|
import sys
|
||||||
|
from io import StringIO
|
||||||
from unittest import TestCase
|
from unittest import TestCase
|
||||||
|
|
||||||
from django.core.management import call_command
|
from django.core.management import call_command
|
||||||
|
@ -82,3 +84,20 @@ class ManagementCommandTestCase(HaystackBackendTestCase, TestCase):
|
||||||
# … but remove does:
|
# … but remove does:
|
||||||
call_command("update_index", remove=True, verbosity=0)
|
call_command("update_index", remove=True, verbosity=0)
|
||||||
self.verify_indexed_document_count(self.NUM_BLOG_ENTRIES - 3)
|
self.verify_indexed_document_count(self.NUM_BLOG_ENTRIES - 3)
|
||||||
|
|
||||||
|
def test_multiprocessing(self):
|
||||||
|
self.verify_indexed_document_count(0)
|
||||||
|
|
||||||
|
old_stderr = sys.stderr
|
||||||
|
sys.stderr = StringIO()
|
||||||
|
call_command(
|
||||||
|
"update_index",
|
||||||
|
verbosity=2,
|
||||||
|
workers=10,
|
||||||
|
batchsize=2,
|
||||||
|
)
|
||||||
|
err = sys.stderr.getvalue()
|
||||||
|
sys.stderr = old_stderr
|
||||||
|
print(err)
|
||||||
|
self.assertNotIn("xapian.DatabaseLockError", err)
|
||||||
|
self.verify_indexed_documents()
|
||||||
|
|
|
@ -1,5 +1,6 @@
|
||||||
import datetime
|
import datetime
|
||||||
import pickle
|
import pickle
|
||||||
|
from pathlib import Path
|
||||||
import os
|
import os
|
||||||
import re
|
import re
|
||||||
import shutil
|
import shutil
|
||||||
|
@ -8,6 +9,8 @@ import sys
|
||||||
from django.conf import settings
|
from django.conf import settings
|
||||||
from django.core.exceptions import ImproperlyConfigured
|
from django.core.exceptions import ImproperlyConfigured
|
||||||
|
|
||||||
|
from filelock import FileLock
|
||||||
|
|
||||||
from haystack import connections
|
from haystack import connections
|
||||||
from haystack.backends import BaseEngine, BaseSearchBackend, BaseSearchQuery, SearchNode, log_query
|
from haystack.backends import BaseEngine, BaseSearchBackend, BaseSearchQuery, SearchNode, log_query
|
||||||
from haystack.constants import ID, DJANGO_ID, DJANGO_CT, DEFAULT_OPERATOR
|
from haystack.constants import ID, DJANGO_ID, DJANGO_CT, DEFAULT_OPERATOR
|
||||||
|
@ -73,6 +76,24 @@ INTEGER_FORMAT = '%012d'
|
||||||
# texts with positional information
|
# texts with positional information
|
||||||
TERMPOS_DISTANCE = 100
|
TERMPOS_DISTANCE = 100
|
||||||
|
|
||||||
|
|
||||||
|
def filelocked(func):
|
||||||
|
"""Decorator to wrap a XapianSearchBackend method in a filelock."""
|
||||||
|
|
||||||
|
def wrapper(self, *args, **kwargs):
|
||||||
|
"""Run the function inside a lock."""
|
||||||
|
if self.path == MEMORY_DB_NAME or not self.use_lockfile:
|
||||||
|
func(self, *args, **kwargs)
|
||||||
|
else:
|
||||||
|
lockfile = Path(self.filelock.lock_file)
|
||||||
|
lockfile.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
lockfile.touch()
|
||||||
|
with self.filelock:
|
||||||
|
func(self, *args, **kwargs)
|
||||||
|
|
||||||
|
return wrapper
|
||||||
|
|
||||||
|
|
||||||
class InvalidIndexError(HaystackError):
|
class InvalidIndexError(HaystackError):
|
||||||
"""Raised when an index can not be opened."""
|
"""Raised when an index can not be opened."""
|
||||||
pass
|
pass
|
||||||
|
@ -168,6 +189,9 @@ class XapianSearchBackend(BaseSearchBackend):
|
||||||
|
|
||||||
Also sets the stemming language to be used to `language`.
|
Also sets the stemming language to be used to `language`.
|
||||||
"""
|
"""
|
||||||
|
self.use_lockfile = bool(
|
||||||
|
getattr(settings, 'HAYSTACK_XAPIAN_USE_LOCKFILE', True)
|
||||||
|
)
|
||||||
super().__init__(connection_alias, **connection_options)
|
super().__init__(connection_alias, **connection_options)
|
||||||
|
|
||||||
if not 'PATH' in connection_options:
|
if not 'PATH' in connection_options:
|
||||||
|
@ -182,6 +206,10 @@ class XapianSearchBackend(BaseSearchBackend):
|
||||||
except FileExistsError:
|
except FileExistsError:
|
||||||
pass
|
pass
|
||||||
|
|
||||||
|
if self.use_lockfile:
|
||||||
|
lockfile = Path(self.path) / "lockfile"
|
||||||
|
self.filelock = FileLock(lockfile)
|
||||||
|
|
||||||
self.flags = connection_options.get('FLAGS', DEFAULT_XAPIAN_FLAGS)
|
self.flags = connection_options.get('FLAGS', DEFAULT_XAPIAN_FLAGS)
|
||||||
self.language = getattr(settings, 'HAYSTACK_XAPIAN_LANGUAGE', 'english')
|
self.language = getattr(settings, 'HAYSTACK_XAPIAN_LANGUAGE', 'english')
|
||||||
|
|
||||||
|
@ -225,6 +253,7 @@ class XapianSearchBackend(BaseSearchBackend):
|
||||||
self._update_cache()
|
self._update_cache()
|
||||||
return self._columns
|
return self._columns
|
||||||
|
|
||||||
|
@filelocked
|
||||||
def update(self, index, iterable, commit=True):
|
def update(self, index, iterable, commit=True):
|
||||||
"""
|
"""
|
||||||
Updates the `index` with any objects in `iterable` by adding/updating
|
Updates the `index` with any objects in `iterable` by adding/updating
|
||||||
|
@ -476,6 +505,7 @@ class XapianSearchBackend(BaseSearchBackend):
|
||||||
finally:
|
finally:
|
||||||
database.close()
|
database.close()
|
||||||
|
|
||||||
|
@filelocked
|
||||||
def remove(self, obj, commit=True):
|
def remove(self, obj, commit=True):
|
||||||
"""
|
"""
|
||||||
Remove indexes for `obj` from the database.
|
Remove indexes for `obj` from the database.
|
||||||
|
|
Loading…
Reference in New Issue