diff --git a/pyiceberg/table/inspect.py b/pyiceberg/table/inspect.py index e580b66dc1..e89ff01dd9 100644 --- a/pyiceberg/table/inspect.py +++ b/pyiceberg/table/inspect.py @@ -272,7 +272,7 @@ def _readable_metrics_struct(bound_type: PrimitiveType) -> pa.StructType: "partition": partition_record_dict, "record_count": entry.data_file.record_count, "file_size_in_bytes": entry.data_file.file_size_in_bytes, - "column_sizes": dict(entry.data_file.column_sizes), + "column_sizes": dict(entry.data_file.column_sizes or {}), "value_counts": dict(entry.data_file.value_counts or {}), "null_value_counts": dict(entry.data_file.null_value_counts or {}), "nan_value_counts": dict(entry.data_file.nan_value_counts or {}), diff --git a/tests/table/test_inspect.py b/tests/table/test_inspect.py index 2404eadc74..a6e53168dd 100644 --- a/tests/table/test_inspect.py +++ b/tests/table/test_inspect.py @@ -15,21 +15,23 @@ # specific language governing permissions and limitations # under the License. +import os from pathlib import PosixPath from typing import Any import pyarrow as pa +import pyarrow.parquet as pq import pytest from pyiceberg.conversions import to_bytes -from pyiceberg.manifest import DataFile, DataFileContent +from pyiceberg.manifest import DataFile, DataFileContent, FileFormat from pyiceberg.partitioning import PartitionField, PartitionSpec from pyiceberg.schema import Schema from pyiceberg.table.inspect import InspectTable, _readable_bound from pyiceberg.table.snapshots import Snapshot from pyiceberg.transforms import IdentityTransform from pyiceberg.typedef import Record -from pyiceberg.types import NestedField, StringType +from pyiceberg.types import LongType, NestedField, StringType from tests.catalog.test_base import InMemoryCatalog @@ -76,6 +78,35 @@ def test_inspect_entries_and_files_render_null_bound(catalog: InMemoryCatalog) - assert files_metrics["upper_bound"] is None +def test_inspect_entries_and_files_with_data_file_without_metrics(catalog: InMemoryCatalog, tmp_path: PosixPath) -> None: + # column_sizes and the other metric maps are optional in a data file + schema = Schema(NestedField(1, "x", LongType(), required=False)) + tbl = catalog.create_table("default.no_metrics", schema) + file_path = f"{tmp_path.absolute().as_posix()}/a.parquet" + pq.write_table(pa.table({"x": [1]}), file_path) + data_file = DataFile.from_args( + content=DataFileContent.DATA, + file_path=file_path, + file_format=FileFormat.PARQUET, + partition=Record(), + record_count=1, + file_size_in_bytes=os.path.getsize(file_path), + ) + with tbl.transaction() as tx: + with tx.update_snapshot().fast_append() as fast_append: + fast_append.append_data_file(data_file) + + entries = tbl.inspect.entries().to_pylist() + assert len(entries) == 1 + for metric in ("column_sizes", "value_counts", "null_value_counts", "nan_value_counts"): + assert entries[0]["data_file"][metric] == [] + assert entries[0]["readable_metrics"]["x"]["column_size"] is None + + files = tbl.inspect.files().to_pylist() + assert len(files) == 1 + assert files[0]["column_sizes"] is None + + @pytest.mark.parametrize("newest_first", [False, True]) def test_partitions_last_updated_uses_latest_snapshot_regardless_of_order(newest_first: bool) -> None: # Manifest entries are visited in manifest order, which is not chronological, so the