diff --git a/be/test/core/data_type_serde/data_type_serde_arrow_test.cpp b/be/test/core/data_type_serde/data_type_serde_arrow_test.cpp index c32d552788de97..902a55bf1851be 100644 --- a/be/test/core/data_type_serde/data_type_serde_arrow_test.cpp +++ b/be/test/core/data_type_serde/data_type_serde_arrow_test.cpp @@ -957,9 +957,8 @@ TEST(DataTypeSerDeArrowTest, PlainArrowConverterDoesNotInferIcebergUuid) { TEST(DataTypeSerDeArrowTest, IcebergVariantExtensionAndParquetSchema) { auto make_variant_column = []() { - JsonStringToVariantEncoder encoder({.max_json_key_length = 1024, - .throw_on_invalid_json = true, - .check_duplicate_json_path = false}); + JsonStringToVariantEncoder encoder( + {.max_json_key_length = 1024, .throw_on_invalid_json = true}); for (std::string_view json : {std::string_view {R"({"name":"doris","n":1})"}, std::string_view {"null"}, std::string_view {R"([1,true,"x"])"}}) { @@ -1054,9 +1053,8 @@ TEST(DataTypeSerDeArrowTest, IcebergVariantExtensionAndParquetSchema) { TEST(DataTypeSerDeArrowTest, NestedIcebergVariantExtensionsAndParquetSchema) { auto make_variant_column = [](std::initializer_list json_values) { - JsonStringToVariantEncoder encoder({.max_json_key_length = 1024, - .throw_on_invalid_json = true, - .check_duplicate_json_path = false}); + JsonStringToVariantEncoder encoder( + {.max_json_key_length = 1024, .throw_on_invalid_json = true}); for (std::string_view json : json_values) { encoder.add_json({json.data(), json.size()}); } diff --git a/be/test/core/data_type_serde/data_type_variant_v2_serde_jsonb_test.cpp b/be/test/core/data_type_serde/data_type_variant_v2_serde_jsonb_test.cpp index 5cecf9d9cefa00..a88aeec2d48e42 100644 --- a/be/test/core/data_type_serde/data_type_variant_v2_serde_jsonb_test.cpp +++ b/be/test/core/data_type_serde/data_type_variant_v2_serde_jsonb_test.cpp @@ -112,9 +112,8 @@ std::string json_at(const DataTypeVariantV2SerDe& serde, const IColumn& column, } ColumnVariantV2::MutablePtr encoded_json(std::initializer_list rows) { - JsonStringToVariantEncoder encoder({.max_json_key_length = 1024, - .throw_on_invalid_json = true, - .check_duplicate_json_path = false}); + JsonStringToVariantEncoder encoder( + {.max_json_key_length = 1024, .throw_on_invalid_json = true}); for (std::string_view row : rows) { encoder.add_json({row.data(), row.size()}); } diff --git a/be/test/exprs/variant_v2_execution_test.cpp b/be/test/exprs/variant_v2_execution_test.cpp index e3db965fc222cb..75d045cf4b4e7b 100644 --- a/be/test/exprs/variant_v2_execution_test.cpp +++ b/be/test/exprs/variant_v2_execution_test.cpp @@ -49,9 +49,8 @@ TEST(VariantV2ExecutionTest, TypePropertiesDoNotAffectEquality) { TEST(VariantV2ExecutionTest, CanonicalComparison) { auto parse = [](std::initializer_list rows) { - JsonStringToVariantEncoder encoder({.max_json_key_length = 255, - .throw_on_invalid_json = true, - .check_duplicate_json_path = false}); + JsonStringToVariantEncoder encoder( + {.max_json_key_length = 255, .throw_on_invalid_json = true}); for (auto json : rows) { encoder.add_json({json.data(), json.size()}); } diff --git a/be/test/format/transformer/vparquet_writer_test.cpp b/be/test/format/transformer/vparquet_writer_test.cpp index c4b05cf045ef32..acab031b79620d 100644 --- a/be/test/format/transformer/vparquet_writer_test.cpp +++ b/be/test/format/transformer/vparquet_writer_test.cpp @@ -195,9 +195,8 @@ TEST_F(VParquetWriterTest, WritesIcebergVariantAndCollectsLogicalMetrics) { options, &schema_json, *schema); ASSERT_TRUE(transformer.open().ok()); - JsonStringToVariantEncoder encoder({.max_json_key_length = 1024, - .throw_on_invalid_json = true, - .check_duplicate_json_path = false}); + JsonStringToVariantEncoder encoder( + {.max_json_key_length = 1024, .throw_on_invalid_json = true}); for (std::string_view json : {std::string_view {R"({"name":"doris","n":1})"}, std::string_view {"null"}, std::string_view {R"([1,2,3])"}}) { encoder.add_json({json.data(), json.size()}); @@ -282,9 +281,8 @@ TEST_F(VParquetWriterTest, WritesNestedIcebergVariant) { options, &schema_json, *schema); ASSERT_TRUE(transformer.open().ok()); - JsonStringToVariantEncoder encoder({.max_json_key_length = 1024, - .throw_on_invalid_json = true, - .check_duplicate_json_path = false}); + JsonStringToVariantEncoder encoder( + {.max_json_key_length = 1024, .throw_on_invalid_json = true}); for (std::string_view json : {std::string_view {R"({"kind":"nested"})"}, std::string_view {"null"}, std::string_view {"{}"}, std::string_view {"7"}}) { diff --git a/regression-test/data/external_table_p0/paimon/write/test_paimon_write_variant_errors.out b/regression-test/data/external_table_p0/paimon/write/test_paimon_write_variant_errors.out index f2333c25116c9f..0c20b7ea6c5315 100644 --- a/regression-test/data/external_table_p0/paimon/write/test_paimon_write_variant_errors.out +++ b/regression-test/data/external_table_p0/paimon/write/test_paimon_write_variant_errors.out @@ -1,7 +1,7 @@ -- This file is automatically generated. You should know what you did if you want to edit this -- !variant_after_errors -- 20 false true {"recovered":true} -21 false \N not-json +21 true \N \N 22 false \N {"typed":"string"} 23 false \N 7 24 false \N values-string diff --git a/regression-test/suites/external_table_p0/paimon/write/test_paimon_write_variant_errors.groovy b/regression-test/suites/external_table_p0/paimon/write/test_paimon_write_variant_errors.groovy index cde2e529c5e50a..a4e18f3a0e47cf 100644 --- a/regression-test/suites/external_table_p0/paimon/write/test_paimon_write_variant_errors.groovy +++ b/regression-test/suites/external_table_p0/paimon/write/test_paimon_write_variant_errors.groovy @@ -202,8 +202,7 @@ suite("test_paimon_write_variant_errors", "p0,external,paimon,nonConcurrent") { row.collect { value -> value == null ? null : value.toString() } }) - // Invalid JSON is preserved as a Variant string unless the global - // throw-on-invalid-JSON option is enabled. + // try_parse_to_variant returns NULL for text that is not a JSON document. order_qt_variant_after_errors """ SELECT id, payload IS NULL, CAST(payload['recovered'] AS BOOLEAN), diff --git a/regression-test/suites/variant_p0/test_variant_json_functions.groovy b/regression-test/suites/variant_p0/test_variant_json_functions.groovy index 458733170301a2..171f9282e0a8e9 100644 --- a/regression-test/suites/variant_p0/test_variant_json_functions.groovy +++ b/regression-test/suites/variant_p0/test_variant_json_functions.groovy @@ -23,6 +23,7 @@ suite("test_variant_json_functions", "p0") { DUPLICATE KEY(k) DISTRIBUTED BY HASH(k) BUCKETS 1 PROPERTIES ("replication_num" = "1") """ + // parse_to_variant only accepts JSON documents, so the Variant string row is a JSON string. sql """ INSERT INTO test_variant_json_functions SELECT k, parse_to_variant(doc) FROM ( @@ -30,7 +31,7 @@ suite("test_variant_json_functions", "p0") { UNION ALL SELECT 2, '{"a":2,"s":"yz","b":{"c":[3]},"n":{"m":true}}' UNION ALL SELECT 3, '[1,"two",{"three":3}]' UNION ALL SELECT 4, '42' - UNION ALL SELECT 5, 'abc' + UNION ALL SELECT 5, '"abc"' UNION ALL SELECT 6, NULL) t """