From 4f2bceaf18c4c92178f1b9fed9a31e302d1e8565 Mon Sep 17 00:00:00 2001 From: Shivadarshan Date: Fri, 13 Nov 2026 23:47:38 +0530 Subject: [PATCH] GH-3095: Follow the spec's LIST rules for the Avro schema parquet-cli derives --- .../org/apache/parquet/cli/util/Schemas.java | 10 +++- .../org/apache/parquet/cli/ReadCliTest.java | 47 +++++++++++++++ .../org/apache/parquet/cli/SchemaCliTest.java | 58 +++++++++++++++++++ 3 files changed, 114 insertions(+), 1 deletion(-) diff --git a/parquet-cli/src/main/java/org/apache/parquet/cli/util/Schemas.java b/parquet-cli/src/main/java/org/apache/parquet/cli/util/Schemas.java index 9478d783bc..8bfb0d0dee 100644 --- a/parquet-cli/src/main/java/org/apache/parquet/cli/util/Schemas.java +++ b/parquet-cli/src/main/java/org/apache/parquet/cli/util/Schemas.java @@ -82,7 +82,15 @@ public static Schema fromParquet(Configuration conf, URI location) throws IOExce if (schemaString != null) { return new Schema.Parser().parse(schemaString); } else { - return new AvroSchemaConverter(conf) + // Without an Avro schema, the file was likely not written by parquet-avro, so apply the LIST + // backward-compatibility rules of the Parquet spec: a repeated group with a single field is a + // synthetic layer around the element unless it is named "array" or "_tuple". + Configuration converterConf = conf; + if (conf.get(AvroSchemaConverter.ADD_LIST_ELEMENT_RECORDS) == null) { + converterConf = new Configuration(conf); + converterConf.setBoolean(AvroSchemaConverter.ADD_LIST_ELEMENT_RECORDS, false); + } + return new AvroSchemaConverter(converterConf) .convert(footer.getFileMetaData().getSchema()); } } diff --git a/parquet-cli/src/test/java/org/apache/parquet/cli/ReadCliTest.java b/parquet-cli/src/test/java/org/apache/parquet/cli/ReadCliTest.java index cfe3e38695..42a7feb680 100644 --- a/parquet-cli/src/test/java/org/apache/parquet/cli/ReadCliTest.java +++ b/parquet-cli/src/test/java/org/apache/parquet/cli/ReadCliTest.java @@ -25,6 +25,8 @@ import java.io.IOException; import java.nio.file.Path; import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.stream.IntStream; @@ -105,6 +107,51 @@ void scanRejectsColumnMissingFromInputSchema() throws Exception { assertThat(result.output()).contains("Cannot find field", "missing_field"); } + @Test + void catReadsStandardListsWithoutAvroSchema() throws Exception { + // Written without an Avro schema in the file metadata, like files from Spark or PyArrow + Path path = getTempFolder().toPath().resolve("lists.parquet"); + MessageType schema = Types.buildMessage() + .required(PrimitiveTypeName.INT32) + .named("id") + .optionalList() + .optionalElement(PrimitiveTypeName.BINARY) + .as(LogicalTypeAnnotation.stringType()) + .named("tags") + .named("record"); + List> tags = Arrays.asList(List.of("a", "b"), List.of(), null, Arrays.asList("c", null)); + SimpleGroupFactory groups = new SimpleGroupFactory(schema); + try (ParquetWriter writer = ExampleParquetWriter.builder(new org.apache.hadoop.fs.Path(path.toUri())) + .withType(schema) + .build()) { + for (int i = 0; i < tags.size(); i++) { + Group record = groups.newGroup().append("id", i); + if (tags.get(i) != null) { + Group list = record.addGroup("tags"); + for (String tag : tags.get(i)) { + Group element = list.addGroup("list"); + if (tag != null) { + element.append("element", tag); + } + } + } + writer.write(record); + } + } + + CliResult result = cli("cat", path.toString()); + + assertThat(result.exitCode()).as(result.output()).isZero(); + List> expected = new ArrayList<>(); + for (int i = 0; i < tags.size(); i++) { + Map record = new HashMap<>(); + record.put("id", i); + record.put("tags", tags.get(i)); + expected.add(record); + } + assertThat(readRecords(result.output())).containsExactlyElementsOf(expected); + } + private Path writeRecords(String name, int firstId, int count) throws IOException { Path path = getTempFolder().toPath().resolve(name); MessageType schema = Types.buildMessage() diff --git a/parquet-cli/src/test/java/org/apache/parquet/cli/SchemaCliTest.java b/parquet-cli/src/test/java/org/apache/parquet/cli/SchemaCliTest.java index 9188d40fd6..9d214ad2ea 100644 --- a/parquet-cli/src/test/java/org/apache/parquet/cli/SchemaCliTest.java +++ b/parquet-cli/src/test/java/org/apache/parquet/cli/SchemaCliTest.java @@ -24,6 +24,13 @@ import java.nio.file.Files; import java.nio.file.Path; import org.apache.avro.Schema; +import org.apache.parquet.example.data.Group; +import org.apache.parquet.hadoop.ParquetWriter; +import org.apache.parquet.hadoop.example.ExampleParquetWriter; +import org.apache.parquet.schema.LogicalTypeAnnotation; +import org.apache.parquet.schema.MessageType; +import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName; +import org.apache.parquet.schema.Types; import org.junit.jupiter.api.Test; class SchemaCliTest extends CliTestBase { @@ -64,6 +71,57 @@ void printsAvroSchemaByDefault() throws Exception { .isEqualTo("date"); } + @Test + void printsStandardListsWithoutAvroSchemaAsArrays() throws Exception { + CliResult result = cli("schema", writeStandardListFile().toString()); + + assertThat(result.exitCode()).as(result.output()).isZero(); + Schema tags = listSchema(result.output(), "tags"); + assertThat(tags.getType()).isEqualTo(Schema.Type.ARRAY); + assertThat(tags.getElementType().toString()).isEqualTo("[\"null\",\"string\"]"); + } + + @Test + void keepsExplicitAddListElementRecordsSetting() throws Exception { + CliResult result = cli( + "-D", + "parquet.avro.add-list-element-records=true", + "schema", + writeStandardListFile().toString()); + + assertThat(result.exitCode()).as(result.output()).isZero(); + Schema element = listSchema(result.output(), "tags").getElementType(); + assertThat(element.getType()).isEqualTo(Schema.Type.RECORD); + assertThat(element.getName()).isEqualTo("list"); + assertThat(element.getFields()).extracting(Schema.Field::name).containsExactly("element"); + } + + // Written without an Avro schema in the file metadata, like files from Spark or PyArrow + private Path writeStandardListFile() throws Exception { + Path input = getTempFolder().toPath().resolve("lists.parquet"); + MessageType parquetSchema = Types.buildMessage() + .optionalList() + .optionalElement(PrimitiveTypeName.BINARY) + .as(LogicalTypeAnnotation.stringType()) + .named("tags") + .named("record"); + try (ParquetWriter writer = ExampleParquetWriter.builder(new org.apache.hadoop.fs.Path(input.toUri())) + .withType(parquetSchema) + .build()) { + // only the schema is needed + } + return input; + } + + private static Schema listSchema(String avroSchema, String field) { + return new Schema.Parser() + .parse(avroSchema) + .getField(field) + .schema() + .getTypes() + .get(1); + } + @Test void printsParquetSchemaForPathContainingSpaces() throws Exception { Path input = getTempFolder().toPath().resolve("input with spaces.parquet");