Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
37 changes: 37 additions & 0 deletions integration/test_batch_v4.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
import concurrent.futures
import uuid
from dataclasses import dataclass
from decimal import Decimal
from typing import Callable, Generator, List, Optional, Protocol, Tuple

import pytest
Expand All @@ -25,6 +26,7 @@
_CrossReference,
)
from weaviate.collections.classes.tenants import Tenant
from weaviate.exceptions import WeaviateInvalidInputError
from weaviate.types import UUID, VECTORS

UUID1 = uuid.UUID("806827e0-2b31-43ca-9269-24fa95a221f9")
Expand Down Expand Up @@ -217,6 +219,41 @@ def test_add_object(
assert len(objs) == 1


def test_add_objects_with_decimal(client_factory: ClientFactory) -> None:
client, name = client_factory()
collection = client.collections.use(name)
collection.config.add_property(Property(name="price", data_type=DataType.NUMBER))
with client.batch.fixed_size() as batch:
for i in range(99):
batch.add_object(collection=name, properties={"price": float(i)})
batch.add_object(collection=name, properties={"price": Decimal("9.99")})
assert len(client.batch.failed_objects) == 0
assert len(collection) == 100

collection.data.insert_many([{"price": Decimal("1.5")}])
uid = collection.data.insert({"price": Decimal("2.5")})
assert len(collection) == 102
obj = collection.query.fetch_object_by_id(uid)
assert obj is not None
assert obj.properties["price"] == 2.5


def test_add_objects_with_unsupported_type(client_factory: ClientFactory) -> None:
client, name = client_factory()
collection = client.collections.use(name)
collection.config.add_property(Property(name="price", data_type=DataType.NUMBER))
with client.batch.fixed_size() as batch:
for i in range(99):
batch.add_object(collection=name, properties={"price": float(i)})
batch.add_object(collection=name, properties={"price": object()})
failed = client.batch.failed_objects
assert len(failed) == 100
assert "Cannot serialize value of type" in failed[0].message

with pytest.raises(WeaviateInvalidInputError, match="Cannot serialize value of type"):
collection.data.insert_many([{"price": object()}])


@pytest.mark.parametrize("from_object_uuid", [UUID1, str(UUID2), UUID3.hex])
@pytest.mark.parametrize("to_object_uuid", [UUID4.hex, UUID5, str(UUID6)])
@pytest.mark.parametrize("to_object_collection", [False, True])
Expand Down
54 changes: 54 additions & 0 deletions mock_tests/test_batch.py
Original file line number Diff line number Diff line change
@@ -1,9 +1,12 @@
import time
from decimal import Decimal
from typing import AsyncGenerator, Generator, List

import grpc
import pytest
import pytest_asyncio
import weaviate
from weaviate.exceptions import WeaviateInvalidInputError
from weaviate.proto.v1 import batch_pb2, weaviate_pb2_grpc
from .conftest import MOCK_IP, MOCK_PORT, MOCK_PORT_GRPC, mock_class, HTTPServer

Expand Down Expand Up @@ -160,3 +163,54 @@ def test_ssb_stream_reports_has_errors(
batch.add_object({"name": f"Object {i}"})
assert len(failed_object_stream.batch.failed_objects) == 2
assert failed_object_stream.batch.results.objs.has_errors


class MockRecordingWeaviateService(weaviate_pb2_grpc.WeaviateServicer):
def __init__(self) -> None:
self.objects: List[batch_pb2.BatchObject] = []

def BatchStream(
self,
request_iterator: Generator[batch_pb2.BatchStreamRequest, None, None],
context: grpc.ServicerContext,
) -> Generator[batch_pb2.BatchStreamReply, None, None]:
yield batch_pb2.BatchStreamReply(started=batch_pb2.BatchStreamReply.Started())
for request in request_iterator:
if request.HasField("data"):
self.objects.extend(request.data.objects.values)
uuids = [obj.uuid for obj in request.data.objects.values]
yield batch_pb2.BatchStreamReply(acks=batch_pb2.BatchStreamReply.Acks(uuids=uuids))
if request.HasField("stop"):
return


@pytest.fixture(scope="function")
def recording_stream(
canceled_stream_client: weaviate.WeaviateClient, start_grpc_server: grpc.Server
):
service = MockRecordingWeaviateService()
weaviate_pb2_grpc.add_WeaviateServicer_to_server(service, start_grpc_server)
return canceled_stream_client.collections.use(mock_class["class"]), service


def test_ssb_stream_converts_decimal(
recording_stream: tuple[weaviate.collections.Collection, MockRecordingWeaviateService],
) -> None:
collection, service = recording_stream
with collection.batch.stream() as batch:
batch.add_object({"name": "Object 1", "price": Decimal("9.99")})
assert len(service.objects) == 1
assert service.objects[0].properties.non_ref_properties["price"] == 9.99


def test_ssb_stream_raises_for_unsupported_type(
recording_stream: tuple[weaviate.collections.Collection, MockRecordingWeaviateService],
) -> None:
collection, service = recording_stream
with pytest.raises(WeaviateInvalidInputError, match="Cannot serialize value of type"):
with collection.batch.stream() as batch:
batch.add_object({"name": "Object 1", "price": object()})
for _ in range(100):
time.sleep(0.1)
batch.add_object({"name": "Object 2"})
assert len(service.objects) == 0
95 changes: 93 additions & 2 deletions test/collection/test_batch.py
Original file line number Diff line number Diff line change
@@ -1,9 +1,13 @@
import struct
import uuid
from decimal import Decimal
from typing import Any, Dict

import numpy as np
import pytest

from weaviate.collections.batch.base import _RateLimitedBatching
from weaviate.collections.batch.grpc_batch import _validate_props
from weaviate.collections.batch.grpc_batch import _BatchGRPC, _validate_props
from weaviate.collections.classes.batch import (
MAX_STORED_RESULTS,
BatchObject,
Expand All @@ -12,8 +16,11 @@
BatchReferenceReturn,
ErrorObject,
ErrorReference,
_BatchObject,
)
from weaviate.exceptions import WeaviateInsertInvalidPropertyError
from weaviate.exceptions import WeaviateInsertInvalidPropertyError, WeaviateInvalidInputError
from weaviate.proto.v1 import batch_pb2
from weaviate.util import _ServerVersion


def _error_object(index: int) -> ErrorObject:
Expand Down Expand Up @@ -141,3 +148,87 @@ def test_validate_props_raises_for_top_level_vector() -> None:
def test_validate_props_raises_for_nested_vector() -> None:
with pytest.raises(WeaviateInsertInvalidPropertyError):
_validate_props({"vector": [0.1, 0.2]}, nested=True)


def _grpc_object(properties: Dict[str, Any]) -> batch_pb2.BatchObject:
grpc = _BatchGRPC(_ServerVersion(1, 40, 0), None, None)
return grpc.grpc_object(
_BatchObject(
collection="Test",
properties=properties,
references=None,
uuid=None,
vector=None,
tenant=None,
index=0,
)
)


def _number_array(obj: batch_pb2.BatchObject, name: str) -> list:
for prop in obj.properties.number_array_properties:
if prop.prop_name == name:
return list(struct.unpack(f"{len(prop.values_bytes) // 8}d", prop.values_bytes))
raise AssertionError(f"no number array named {name}")


@pytest.mark.parametrize(
"value",
[Decimal("9.99"), np.float32(9.99)],
)
def test_grpc_object_converts_scalar_numbers(value: Any) -> None:
obj = _grpc_object({"price": value})
assert obj.properties.non_ref_properties["price"] == pytest.approx(9.99, rel=1e-6)


def test_grpc_object_converts_numpy_int() -> None:
obj = _grpc_object({"count": np.int64(3)})
assert obj.properties.non_ref_properties["count"] == 3


def test_grpc_object_converts_nested_decimal() -> None:
obj = _grpc_object({"item": {"price": Decimal("9.99")}})
nested = obj.properties.object_properties[0]
assert nested.prop_name == "item"
assert nested.value.non_ref_properties["price"] == 9.99


@pytest.mark.parametrize(
"value",
[[Decimal("1.5"), 2.5], [1.5, Decimal("2.5")], [Decimal("1.5"), Decimal("2.5")]],
)
def test_grpc_object_converts_decimals_in_lists(value: Any) -> None:
assert _number_array(_grpc_object({"prices": value}), "prices") == [1.5, 2.5]


def test_grpc_object_converts_numpy_ints_in_lists() -> None:
obj = _grpc_object({"counts": [np.int64(1), np.int64(2)]})
assert list(obj.properties.int_array_properties[0].values) == [1, 2]


def test_grpc_object_accepts_tuple_of_uuids() -> None:
obj = _grpc_object(
{
"ids": (
uuid.UUID("11111111-1111-1111-1111-111111111111"),
uuid.UUID("22222222-2222-2222-2222-222222222222"),
)
}
)
values = obj.properties.non_ref_properties["ids"]
assert list(values) == [
"11111111-1111-1111-1111-111111111111",
"22222222-2222-2222-2222-222222222222",
]


@pytest.mark.parametrize("value", [Decimal("NaN"), Decimal("1e400"), [1.5, Decimal("Infinity")]])
def test_grpc_object_raises_for_non_finite_numbers(value: Any) -> None:
with pytest.raises(WeaviateInvalidInputError, match="non-finite"):
_grpc_object({"price": value})


@pytest.mark.parametrize("value", [object(), {"inner": object()}, [object(), "a"]])
def test_grpc_object_raises_for_unsupported_types(value: Any) -> None:
with pytest.raises(WeaviateInvalidInputError, match="Cannot serialize value of type"):
_grpc_object({"value": value})
15 changes: 11 additions & 4 deletions weaviate/collections/batch/grpc_batch.py
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@
)
from weaviate.proto.v1 import base_pb2, batch_pb2
from weaviate.types import VECTORS
from weaviate.util import _datetime_to_string, _ServerVersion
from weaviate.util import _datetime_to_string, _ServerVersion, _to_python_number


class _BatchGRPC(_BaseGRPC):
Expand Down Expand Up @@ -276,6 +276,12 @@ def __translate_properties_from_python_to_grpc(
raise WeaviateInvalidInputError(f"Invalid reference: {ref}")

for key, entry in data.items():
if isinstance(entry, list):
entry = [_to_python_number(v) for v in entry]
elif isinstance(entry, tuple):
entry = tuple(_to_python_number(v) for v in entry)
else:
entry = _to_python_number(entry)
if isinstance(entry, dict):
parsed = self.__translate_properties_from_python_to_grpc(entry, {}, nested=True)
object_properties.append(
Expand Down Expand Up @@ -374,7 +380,8 @@ def _serialize_primitive(value: Any) -> Any:
return str(value)
if isinstance(value, datetime.datetime):
return _datetime_to_string(value)
if isinstance(value, list):
if isinstance(value, (list, tuple)):
return [_serialize_primitive(val) for val in value]

return value
if value is None or isinstance(value, (str, bool, int, float, dict)):
return value
raise WeaviateInvalidInputError(f"Cannot serialize value of type {type(value)} to Weaviate.")
Comment thread
steven-asearcy marked this conversation as resolved.
Resolved
2 changes: 2 additions & 0 deletions weaviate/collections/classes/types.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
import datetime
import uuid as uuid_package
from decimal import Decimal
from typing import Any, Dict, Mapping, Optional, Sequence, Type, Union, get_origin

from pydantic import BaseModel, ConfigDict, Field
Expand Down Expand Up @@ -62,6 +63,7 @@ class _PhoneNumber(_PhoneNumberBase):
bool, # boolean
int, # int
float, # number
Decimal, # number
datetime.datetime, # date
uuid_package.UUID, # uuid
GeoCoordinate, # geoCoordinates
Expand Down
3 changes: 2 additions & 1 deletion weaviate/collections/data/executor.py
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,7 @@
from weaviate.exceptions import WeaviateInvalidInputError
from weaviate.logger import logger
from weaviate.types import BEACON, UUID, VECTORS
from weaviate.util import _datetime_to_string, _get_vector_v4
from weaviate.util import _datetime_to_string, _get_vector_v4, _to_python_number
from weaviate.validator import _validate_input, _ValidateArgument


Expand Down Expand Up @@ -680,6 +680,7 @@ def __serialize_refs(self, refs: ReferenceInputs) -> Dict[str, Any]:
}

def __serialize_primitive(self, value: WeaviateField) -> Any:
value = _to_python_number(value)
if isinstance(value, str) or isinstance(value, int) or isinstance(value, float):
return value
if isinstance(value, uuid_package.UUID):
Expand Down
16 changes: 16 additions & 0 deletions weaviate/util.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,10 +5,13 @@
import functools
import io
import json
import math
import numbers
import os
import re
import uuid as uuid_lib
import warnings as _warnings
from decimal import Decimal
from pathlib import Path
from typing import Any, Callable, Dict, Generator, List, Optional, Sequence, Tuple, Union, cast
from urllib.parse import quote
Expand Down Expand Up @@ -756,6 +759,19 @@ def _decode_json_response_list(
raise UnexpectedStatusCodeError(location, response)


def _to_python_number(value: Any) -> Any:
if isinstance(value, (bool, int, float)):
return value
if isinstance(value, numbers.Integral):
return int(value)
if isinstance(value, (numbers.Real, Decimal)):
converted = float(value)
if not math.isfinite(converted):
raise WeaviateInvalidInputError(f"Cannot send non-finite number {value!r} to Weaviate.")
return converted
return value


def _datetime_to_string(value: TIME) -> str:
if value.tzinfo is None:
_Warnings.datetime_insertion_with_no_specified_timezone(value)
Expand Down
Loading