Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
135 changes: 134 additions & 1 deletion mock_tests/test_batch.py
Original file line number Diff line number Diff line change
@@ -1,10 +1,20 @@
from typing import AsyncGenerator, Generator, List
import threading
import uuid as uuid_package
from decimal import Decimal
from typing import AsyncGenerator, Dict, Generator, List

import grpc
import pytest
import pytest_asyncio
import weaviate
from weaviate.collections.batch.async_ import _BatchBaseAsync
from weaviate.collections.batch.base import _BatchDataWrapper
from weaviate.collections.batch.grpc_batch import _BatchGRPC
from weaviate.collections.batch.sync import _BatchBaseSync
from weaviate.collections.classes.batch import BatchObject, _BatchObject
from weaviate.exceptions import WeaviateInsertManyAllFailedError
from weaviate.proto.v1 import batch_pb2, weaviate_pb2_grpc
from weaviate.util import _ServerVersion
from .conftest import MOCK_IP, MOCK_PORT, MOCK_PORT_GRPC, mock_class, HTTPServer

HOW_MANY = 1000
Expand Down Expand Up @@ -160,3 +170,126 @@ def test_ssb_stream_reports_has_errors(
batch.add_object({"name": f"Object {i}"})
assert len(failed_object_stream.batch.failed_objects) == 2
assert failed_object_stream.batch.results.objs.has_errors


# --- regression tests for #2266: one unencodable object must not fail the batch ---


def _internal_obj(i: int, bad: bool = False) -> _BatchObject:
props: Dict[str, object] = {"name": f"Object {i}"}
if bad:
# a Decimal property cannot be encoded into a protobuf Struct on main
props["price"] = Decimal("1.5")
return _BatchObject(
collection="Test",
properties=props, # type: ignore[arg-type]
references={},
uuid=str(uuid_package.uuid4()),
vector=None,
tenant=None,
index=i,
)


class _StubConnection:
def __init__(self, errors: Dict[int, str] | None = None) -> None:
self.received: int | None = None
self._errors = errors or {}

def grpc_batch_objects(self, request, timeout, max_retries): # noqa: ANN001, ANN202
self.received = len(request.objects)
return self._errors


def test_grpc_objects_one_unencodable_object_does_not_fail_the_batch() -> None:
grpc_batch = _BatchGRPC(_ServerVersion.from_string("1.40.0"), None, None)
objs = [_internal_obj(i, bad=(i == 37)) for i in range(100)]
conn = _StubConnection()
res = grpc_batch.objects(connection=conn, objects=objs, timeout=10, max_retries=0) # type: ignore[arg-type]
assert conn.received == 99
assert len(res.uuids) == 99
assert list(res.errors.keys()) == [37]
assert str(objs[37].uuid) in res.errors[37].message
assert res.has_errors


def test_grpc_objects_server_error_is_translated_past_an_encoding_failure() -> None:
# object 0 fails to encode, so the server's error at request position 0
# belongs to the caller's object 1, not object 0
grpc_batch = _BatchGRPC(_ServerVersion.from_string("1.40.0"), None, None)
objs = [_internal_obj(i, bad=(i == 0)) for i in range(3)]
conn = _StubConnection(errors={0: "server error"})
res = grpc_batch.objects(connection=conn, objects=objs, timeout=10, max_retries=0) # type: ignore[arg-type]
assert conn.received == 2
assert sorted(res.errors.keys()) == [0, 1]
assert res.errors[1].message == "server error"
assert list(res.uuids.keys()) == [2]


def test_grpc_objects_all_unencodable_raises_all_failed() -> None:
grpc_batch = _BatchGRPC(_ServerVersion.from_string("1.40.0"), None, None)
objs = [_internal_obj(i, bad=True) for i in range(3)]
conn = _StubConnection()
with pytest.raises(WeaviateInsertManyAllFailedError):
grpc_batch.objects(connection=conn, objects=objs, timeout=10, max_retries=0) # type: ignore[arg-type]
assert conn.received is None


def _batch_object(i: int, bad: bool = False) -> BatchObject:
props: Dict[str, object] = {"name": f"Object {i}"}
if bad:
props["price"] = Decimal("1.5")
return BatchObject(
collection="Test",
properties=props, # type: ignore[arg-type]
references=None,
uuid=uuid_package.uuid4(),
vector=None,
tenant=None,
index=i,
)


def test_sync_stream_skips_unencodable_object_and_records_error() -> None:
batch = object.__new__(_BatchBaseSync)
batch._BatchBaseSync__batch_grpc = _BatchGRPC( # type: ignore[attr-defined]
_ServerVersion.from_string("1.40.0"), None, None
)
objs = [_batch_object(i, bad=(i == 1)) for i in range(3)]
batch._BatchBaseSync__objs_cache = {str(o.uuid): o for o in objs} # type: ignore[attr-defined]
batch._BatchBaseSync__uuid_lookup = {str(o.uuid) for o in objs} # type: ignore[attr-defined]
batch._BatchBaseSync__objs_cache_lock = threading.Lock() # type: ignore[attr-defined]
batch._BatchBaseSync__uuid_lookup_lock = threading.Lock() # type: ignore[attr-defined]
batch._BatchBaseSync__results_lock = threading.Lock() # type: ignore[attr-defined]
batch._BatchBaseSync__results_for_wrapper = _BatchDataWrapper() # type: ignore[attr-defined]

requests = list(batch._BatchBaseSync__generate_stream_requests(objs, [])) # type: ignore[attr-defined]
sent = [o for req in requests for o in req.proto.data.objects.values]
assert len(sent) == 2

wrapper = batch._BatchBaseSync__results_for_wrapper # type: ignore[attr-defined]
assert len(wrapper.failed_objects) == 1
assert wrapper.failed_objects[0].object_.index == 1
assert str(objs[1].uuid) in wrapper.failed_objects[0].message
assert str(objs[1].uuid) not in batch._BatchBaseSync__objs_cache # type: ignore[attr-defined]
assert str(objs[1].uuid) not in batch._BatchBaseSync__uuid_lookup # type: ignore[attr-defined]


def test_async_stream_skips_unencodable_object_and_records_error() -> None:
batch = object.__new__(_BatchBaseAsync)
batch._BatchBaseAsync__batch_grpc = _BatchGRPC( # type: ignore[attr-defined]
_ServerVersion.from_string("1.40.0"), None, None
)
objs = [_batch_object(i, bad=(i == 1)) for i in range(3)]
batch._BatchBaseAsync__objs_cache = {str(o.uuid): o for o in objs} # type: ignore[attr-defined]
batch._BatchBaseAsync__uuid_lookup = {str(o.uuid) for o in objs} # type: ignore[attr-defined]
batch._BatchBaseAsync__results_for_wrapper = _BatchDataWrapper() # type: ignore[attr-defined]

requests = list(batch._BatchBaseAsync__generate_stream_requests(objs, [])) # type: ignore[attr-defined]
sent = [o for req in requests for o in req.proto.data.objects.values]
assert len(sent) == 2

wrapper = batch._BatchBaseAsync__results_for_wrapper # type: ignore[attr-defined]
assert len(wrapper.failed_objects) == 1
assert wrapper.failed_objects[0].object_.index == 1
assert str(objs[1].uuid) not in batch._BatchBaseAsync__objs_cache # type: ignore[attr-defined]
38 changes: 37 additions & 1 deletion weaviate/collections/batch/async_.py
Original file line number Diff line number Diff line change
Expand Up @@ -278,6 +278,36 @@ async def __loop(self) -> None:
return
await asyncio.sleep(refresh_time)

def __record_encoding_error(self, object_: BatchObject, e: Exception) -> None:
"""Record one object's client-side encoding failure as its own error.

The object is dropped from the stream, but every other object is still
sent: previously this exception killed the background send loop and
silently lost every object that had not been sent yet. This generator
is consumed synchronously by the batch loop, so no other task can run
while these collections are mutated here.
"""
err = ErrorObject(
message=(
f"Failed to encode object with uuid {object_.uuid} for batch insertion: {e!r}"
),
object_=object_,
)
self.__objs_cache.pop(str(object_.uuid), None)
self.__uuid_lookup.discard(str(object_.uuid))
self.__results_for_wrapper.results.objs += BatchObjectReturn(
_all_responses=[err],
errors={object_.index: err},
)
self.__results_for_wrapper.failed_objects.append(err)
logger.error(
{
"error": err.message,
"object": str(object_.uuid),
"action": "use {client,collection}.batch.failed_objects to access this error",
}
)

def __generate_stream_requests(
self,
objects: List[BatchObject],
Expand All @@ -293,7 +323,11 @@ def request_maker():

uuids, beacons = set(), set()
for object_ in objects:
obj = self.__batch_grpc.grpc_object(object_._to_internal())
try:
obj = self.__batch_grpc.grpc_object(object_._to_internal())
except Exception as e:
self.__record_encoding_error(object_, e)
continue
obj_size = obj.ByteSize() + per_object_overhead

if obj_size > self.__batch_grpc.grpc_max_msg_size:
Expand Down Expand Up @@ -407,6 +441,8 @@ async def __recv(self) -> None:
try:
async with self.__objs_cache_lock:
cached = self.__objs_cache.pop(error.uuid)
async with self.__uuid_lookup_lock:
self.__uuid_lookup.discard(error.uuid)
except KeyError:
continue
err = ErrorObject(
Expand Down
59 changes: 49 additions & 10 deletions weaviate/collections/batch/grpc_batch.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
Mapping,
Optional,
Sequence,
Tuple,
Union,
cast,
)
Expand Down Expand Up @@ -94,6 +95,29 @@ def grpc_object(self, obj: _BatchObject) -> batch_pb2.BatchObject:
def grpc_objects(self, objects: List[_BatchObject]) -> List[batch_pb2.BatchObject]:
return [self.grpc_object(obj) for obj in objects]

def __encode_objects(
self, objects: List[_BatchObject]
) -> Tuple[List[batch_pb2.BatchObject], List[int], Dict[int, str]]:
"""Encode objects one by one so one unencodable object cannot fail the whole batch.

Returns the encoded objects, the original position of each encoded object, and
an error message keyed by original position for every object that failed to encode.
"""
encoded: List[batch_pb2.BatchObject] = []
positions: List[int] = []
errors: Dict[int, str] = {}
for pos, obj in enumerate(objects):
try:
encoded_obj = self.grpc_object(obj)
except Exception as e:
errors[pos] = (
f"Failed to encode object with uuid {obj.uuid} for batch insertion: {e!r}"
)
else:
positions.append(pos)
encoded.append(encoded_obj)
return encoded, positions, errors

def grpc_reference(self, reference: _BatchReference) -> batch_pb2.BatchReference:
ref = BatchReference._from_internal(reference)
return batch_pb2.BatchReference(
Expand Down Expand Up @@ -127,44 +151,59 @@ def objects(
timeout: The timeout in seconds for the request.
max_retries: The maximum number of retries in case of a failure.
"""
weaviate_objs = self.grpc_objects(objects)
weaviate_objs, positions, encoding_errors = self.__encode_objects(objects)
start = time.time()

if len(weaviate_objs) == 0:
# Escape sequence (backslash) not allowed in expression portion of f-string prior to Python 3.12: pylance
raise WeaviateInsertManyAllFailedError(
"Here is the set of all errors: {}".format(
"\n".join(err for err in set(encoding_errors.values()))
)
)

def resp(errors: Dict[int, str]) -> BatchObjectReturn:
if len(errors) == len(weaviate_objs):
# `errors` is keyed by position within the request, so translate those
# positions back to the caller's objects and merge them with the
# client-side encoding errors recorded before the request was sent
merged_errors: Dict[int, str] = dict(encoding_errors)
for sent_idx, message in errors.items():
merged_errors[positions[sent_idx]] = message

if len(merged_errors) == len(objects):
# Escape sequence (backslash) not allowed in expression portion of f-string prior to Python 3.12: pylance
raise WeaviateInsertManyAllFailedError(
"Here is the set of all errors: {}".format(
"\n".join(err for err in set(errors.values()))
"\n".join(err for err in set(merged_errors.values()))
)
)

elapsed_time = time.time() - start
all_responses: List[Union[uuid_package.UUID, ErrorObject]] = cast(
List[Union[uuid_package.UUID, ErrorObject]],
list(range(len(weaviate_objs))),
list(range(len(objects))),
)
return_success: Dict[int, uuid_package.UUID] = {}
return_errors: Dict[int, ErrorObject] = {}
for idx, weav_obj in enumerate(weaviate_objs):
obj = objects[idx]
if idx in errors:
sent_positions = {pos: sent_idx for sent_idx, pos in enumerate(positions)}
for idx, obj in enumerate(objects):
if idx in merged_errors:
error = ErrorObject(
errors[idx],
merged_errors[idx],
BatchObject._from_internal(obj),
original_uuid=obj.uuid,
)
return_errors[obj.index] = error
all_responses[idx] = error
else:
success = uuid_package.UUID(weav_obj.uuid)
success = uuid_package.UUID(weaviate_objs[sent_positions[idx]].uuid)
return_success[obj.index] = success
all_responses[idx] = success

return BatchObjectReturn(
uuids=return_success,
errors=return_errors,
has_errors=len(errors) > 0,
has_errors=len(merged_errors) > 0,
_all_responses=all_responses,
elapsed_seconds=elapsed_time,
)
Expand Down
39 changes: 38 additions & 1 deletion weaviate/collections/batch/sync.py
Original file line number Diff line number Diff line change
Expand Up @@ -221,6 +221,37 @@ def __loop(self) -> None:
return
time.sleep(refresh_time)

def __record_encoding_error(self, object_: BatchObject, e: Exception) -> None:
"""Record one object's client-side encoding failure as its own error.

The object is dropped from the stream, but every other object is still
sent: previously this exception killed the background send loop and
silently lost every object that had not been sent yet.
"""
err = ErrorObject(
message=(
f"Failed to encode object with uuid {object_.uuid} for batch insertion: {e!r}"
),
object_=object_,
)
with self.__objs_cache_lock:
self.__objs_cache.pop(str(object_.uuid), None)
with self.__uuid_lookup_lock:
self.__uuid_lookup.discard(str(object_.uuid))
with self.__results_lock:
self.__results_for_wrapper.results.objs += BatchObjectReturn(
_all_responses=[err],
errors={object_.index: err},
)
self.__results_for_wrapper.failed_objects.append(err)
logger.error(
{
"error": err.message,
"object": str(object_.uuid),
"action": "use {client,collection}.batch.failed_objects to access this error",
}
)

def __generate_stream_requests(
self,
objects: List[BatchObject],
Expand All @@ -236,7 +267,11 @@ def request_maker():

uuids, beacons = set(), set()
for object_ in objects:
obj = self.__batch_grpc.grpc_object(object_._to_internal())
try:
obj = self.__batch_grpc.grpc_object(object_._to_internal())
except Exception as e:
self.__record_encoding_error(object_, e)
continue
obj_size = obj.ByteSize() + per_object_overhead

if obj_size > self.__batch_grpc.grpc_max_msg_size:
Expand Down Expand Up @@ -356,6 +391,8 @@ def __recv(self) -> None:
try:
with self.__objs_cache_lock:
cached = self.__objs_cache.pop(error.uuid)
with self.__uuid_lookup_lock:
self.__uuid_lookup.discard(error.uuid)
except KeyError:
continue
err = ErrorObject(
Expand Down
Loading