diff --git a/pyiceberg/conversions.py b/pyiceberg/conversions.py index fde1885443..410551e035 100644 --- a/pyiceberg/conversions.py +++ b/pyiceberg/conversions.py @@ -90,6 +90,10 @@ _FLOAT_STRUCT = Struct(" Callable: # type: ignore """Handle cases where partition values are `None` or "__HIVE_DEFAULT_PARTITION__". @@ -352,14 +356,28 @@ def _(_: PrimitiveType, b: bytes) -> int: @from_bytes.register(TimeType) -@from_bytes.register(TimestampType) @from_bytes.register(TimestamptzType) -@from_bytes.register(TimestampNanoType) @from_bytes.register(TimestamptzNanoType) def _(_: PrimitiveType, b: bytes) -> int: return _LONG_STRUCT.unpack(b)[0] +@from_bytes.register(TimestampType) +def _(_: TimestampType, b: bytes) -> int: + if len(b) < 8: + # If the length is 4 bytes, it is a date promoted to timestamp, stored as days from the epoch + return _INT_STRUCT.unpack(b)[0] * _MICROS_PER_DAY + return _LONG_STRUCT.unpack(b)[0] + + +@from_bytes.register(TimestampNanoType) +def _(_: TimestampNanoType, b: bytes) -> int: + if len(b) < 8: + # If the length is 4 bytes, it is a date promoted to timestamp_ns, stored as days from the epoch + return _INT_STRUCT.unpack(b)[0] * _NANOS_PER_DAY + return _LONG_STRUCT.unpack(b)[0] + + @from_bytes.register(LongType) def _(_: PrimitiveType, b: bytes) -> int: if len(b) < 8: diff --git a/tests/expressions/test_evaluator.py b/tests/expressions/test_evaluator.py index bba4156e99..ee7265d7b3 100644 --- a/tests/expressions/test_evaluator.py +++ b/tests/expressions/test_evaluator.py @@ -17,11 +17,14 @@ # pylint:disable=redefined-outer-name from collections.abc import Iterator, Mapping from concurrent.futures import ThreadPoolExecutor +from datetime import date, datetime from threading import Event from typing import Any +import pyarrow as pa import pytest +from pyiceberg.catalog import Catalog from pyiceberg.conversions import to_bytes from pyiceberg.expressions import ( And, @@ -56,6 +59,7 @@ from pyiceberg.schema import Schema from pyiceberg.typedef import Record from pyiceberg.types import ( + DateType, DoubleType, FloatType, IcebergType, @@ -64,6 +68,7 @@ NestedField, PrimitiveType, StringType, + TimestampType, ) INT_MIN_VALUE = 30 @@ -1831,6 +1836,33 @@ def test_strict_integer_not_in(strict_data_file_schema: Schema, strict_data_file assert not should_read, "Should not match: no_nulls field does not have bounds" +def test_plan_and_inspect_after_date_to_timestamp_promotion(catalog: Catalog) -> None: + # PyIceberg cannot write v3 metadata yet (#1551), so the promotion is simulated on v2 metadata. The bound + # decoder does not depend on the format version, and the data files keep their 4-byte day counts either way. + catalog.create_namespace("default") + tbl = catalog.create_table("default.date_promotion", Schema(NestedField(1, "d", DateType(), required=False))) + tbl.append(pa.table({"d": pa.array([date(2020, 1, 1), date(2021, 6, 15)], pa.date32())})) + tbl.append(pa.table({"d": pa.array([date(2023, 3, 1), date(2024, 2, 29)], pa.date32())})) + paths = {row["readable_metrics"]["d"]["lower_bound"]: row["file_path"] for row in tbl.inspect.files().to_pylist()} + file_a, file_b = paths[date(2020, 1, 1)], paths[date(2023, 3, 1)] + + # date -> timestamp is a v3 promotion that promote() does not implement, so the incompatible change is allowed + with tbl.update_schema(allow_incompatible_changes=True) as update: + update.update_column("d", field_type=TimestampType()) + + for row_filter, expected_files in [ + # file A's latest date is 2021-06-15, file B's earliest date is 2023-03-01 + (GreaterThan("d", "2022-01-01T00:00:00"), [file_b]), + (LessThan("d", "2021-01-01T00:00:00"), [file_a]), + ]: + planned = tbl.scan(row_filter=row_filter).plan_files() + assert [task.file.file_path for task in planned] == expected_files + + bounds = {row["file_path"]: row["readable_metrics"]["d"] for row in tbl.inspect.files().to_pylist()} + assert (bounds[file_a]["lower_bound"], bounds[file_a]["upper_bound"]) == (datetime(2020, 1, 1), datetime(2021, 6, 15)) + assert (bounds[file_b]["lower_bound"], bounds[file_b]["upper_bound"]) == (datetime(2023, 3, 1), datetime(2024, 2, 29)) + + @pytest.mark.parametrize( "file_type, evolved_type, lower_bound, upper_bound, op, lit, expected", [ @@ -1840,6 +1872,9 @@ def test_strict_integer_not_in(strict_data_file_schema: Schema, strict_data_file # Float -> Double (FloatType(), DoubleType(), 30.0, 79.0, GreaterThan, 100.0, ROWS_CANNOT_MATCH), (FloatType(), DoubleType(), 30.0, 79.0, LessThan, 50.0, ROWS_MIGHT_MATCH), + # Date -> Timestamp, including a day count before the epoch + (DateType(), TimestampType(), 30, 79, GreaterThan, "2022-01-01T00:00:00", ROWS_CANNOT_MATCH), + (DateType(), TimestampType(), -10, 79, LessThan, "1969-12-31T00:00:00", ROWS_MIGHT_MATCH), ], ) def test_inclusive_metrics_eval_bounds_after_promotion( @@ -1880,6 +1915,8 @@ def test_inclusive_metrics_eval_bounds_after_promotion( (FloatType(), DoubleType(), 30.0, 79.0, GreaterThan, 100.0, ROWS_MIGHT_NOT_MATCH), (FloatType(), DoubleType(), 30.0, 79.0, LessThan, 100.0, ROWS_MUST_MATCH), (FloatType(), DoubleType(), 30.0, 79.0, LessThan, 20.0, ROWS_MIGHT_NOT_MATCH), + # Date -> Timestamp + (DateType(), TimestampType(), 30, 79, GreaterThan, "1970-01-15T00:00:00", ROWS_MUST_MATCH), ], ) def test_strict_metrics_eval_bounds_after_promotion( diff --git a/tests/test_conversions.py b/tests/test_conversions.py index ec14e87d52..1651d2ae3d 100644 --- a/tests/test_conversions.py +++ b/tests/test_conversions.py @@ -299,6 +299,31 @@ def test_from_bytes(primitive_type: PrimitiveType, b: bytes, result: Any) -> Non assert conversions.from_bytes(primitive_type, b) == result +def test_from_bytes_date_bound_promoted_to_timestamp() -> None: + # A bound written before a date -> timestamp promotion keeps its 4-byte day count + cases: list[tuple[PrimitiveType, bytes, int]] = [ + (TimestampType(), struct.pack(" None: + # A bound can be wider than the data, so the scaled value stays exact outside the int64 range + cases: list[tuple[PrimitiveType, int, int]] = [ + (TimestampNanoType(), 106752, 9223372800000000000), + (TimestampNanoType(), -106752, -9223372800000000000), + ] + for primitive_type, days, expected in cases: + assert conversions.from_bytes(primitive_type, struct.pack("