Source code for maxframe.liteframe.core

# Copyright 1999-2026 Alibaba Group Holding Ltd.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from typing import Dict

import pandas as pd
import pyarrow as pa

try:
    import polars as pl
except ImportError:
    pl = None

from maxframe.core import (
    HasShapeTileable,
    HasShapeTileableData,
    OutputType,
    is_build_mode,
    register_output_types,
)
from maxframe.core.entity.executable import _ExecuteAndFetchMixin
from maxframe.core.entity.utils import fill_chunk_slices, refresh_tileable_shape
from maxframe.lib.compat import cached_property
from maxframe.lib.dtypes_extension.blob import (
    is_polars_blob_type,
    polars_to_pandas_with_blobs,
)
from maxframe.serialization.serializables import (
    AnyField,
    DictField,
    FieldTypes,
    Int64Field,
    ListField,
    ReferenceField,
    Serializable,
    SeriesField,
)
from maxframe.tensor.datasource.from_liteframe import from_liteframe
from maxframe.utils import wrap_arrow_dtype
from maxframe.utils.polars import (
    has_pl_object_type,
    pl_get_ext_type_metadata,
    pl_get_ext_type_name_and_storage,
    pl_make_ext_or_object_dtype,
)

RANGE_COL_NAME = "__lf_range_index__"


def build_frame_from_ranges(range_columns_as_ranges):
    """Build a polars DataFrame from virtual range column metadata.

    Parameters
    ----------
    range_columns_as_ranges : dict[str, range]
        Mapping from column name to Python ``range`` object.

    Returns
    -------
    pl.DataFrame
        A DataFrame with one Int64 column per range entry. Empty if
        ``range_columns_as_ranges`` is falsy.
    """
    if not range_columns_as_ranges:
        return pl.DataFrame()
    data = {}
    for name, rng in range_columns_as_ranges.items():
        # Build from a Python list rather than ``pl.int_range`` (added in
        # polars 0.20.4) so the minimum supported polars (0.18.0) works.
        data[name] = pl.Series(name, list(rng), dtype=pl.Int64)
    return pl.DataFrame(data)


class RangeInfo(Serializable):
    start = Int64Field("start", default=None)
    stop = Int64Field("stop", default=None)
    step = Int64Field("step", default=None)


class FrameMetadata(Serializable):
    range_columns = DictField(
        "range_columns",
        key_type=FieldTypes.string,
        value_type=FieldTypes.reference(RangeInfo),
        default=None,
    )
    hidden_columns = ListField(
        "hidden_columns",
        FieldTypes.string,
        default=None,
    )


class LiteFrameResult(Serializable):
    """Result of a LiteFrame chunk computation.

    ``_frame`` holds the physical Polars frame (DataFrame or LazyFrame).
    When all output columns are virtual range columns (no physical data),
    ``_frame`` is ``None`` — the frame is materialized on demand from
    ``range_columns`` by :meth:`materialize`.

    Parameters
    ----------
    _frame : pl.DataFrame, pl.LazyFrame, or None
        Physical Polars frame, or ``None`` for all-virtual-range output.
    range_columns : dict[str, tuple[int, int, int]] or None
        Virtual range column metadata: name -> (start, stop, step).
    """

    _frame = AnyField("_frame")  # pl.DataFrame, pl.LazyFrame, or None
    range_columns = DictField(
        "range_columns",
        key_type=FieldTypes.string,
        value_type=FieldTypes.tuple,  # (start, stop, step)
        default=None,
    )

    @property
    def frame(self) -> "pl.DataFrame | pl.LazyFrame | None":
        """The underlying polars object, or None for all-virtual-range output."""
        return self._frame

    @property
    def range_columns_as_ranges(self) -> Dict[str, range]:
        """Convert stored (start, stop, step) tuples to Python range objects."""
        if not self.range_columns:
            return {}
        return {k: range(v[0], v[1], v[2]) for k, v in self.range_columns.items()}

    def get(self) -> "pl.DataFrame | pl.LazyFrame | None":
        """Get the polars object for computation.

        Returns ``None`` when ``_frame`` is ``None`` (all-virtual-range
        projection). Callers that need a physical frame should use
        :meth:`materialize` instead.
        """
        return self._frame

    def materialize(self) -> "pl.DataFrame | pl.LazyFrame":
        """Return a physical polars frame, materializing from range_columns if needed.

        When ``_frame`` is ``None``, builds a DataFrame from the virtual
        range columns. This is used by consumers that need a concrete
        polars object (e.g. to_odps, to_pandas, filter, sort).
        """
        if self._frame is not None:
            return self._frame
        return build_frame_from_ranges(self.range_columns_as_ranges)

    def consolidate(self) -> "pl.DataFrame | None":
        """Collect LazyFrame to DataFrame in-place and return it.

        If the frame is already a DataFrame, returns it as-is.
        If ``_frame`` is ``None``, returns ``None`` without
        materializing — use :meth:`materialize` for that.

        Note: For virtual range columns (when _frame is None), the caller
        should handle memory size estimation separately using range_columns.
        """
        if self._frame is not None and pl and isinstance(self._frame, pl.LazyFrame):
            self._frame = self._frame.collect()
        return self._frame

    def __getstate__(self):
        """Consolidate LazyFrame -> DataFrame before serialization.
        Called by Ray when sending between workers.

        Polars pl.Object columns do not survive Ray pickle — Python
        objects are replaced with raw C pointer bytes.  This covers
        pl.Object directly, pl.BaseExtension(Object), and blob
        extension types (which may have Binary storage after Arrow
        round-trip).

        To prevent corruption, all object-type column values are
        extracted into plain Python lists and stored alongside dtype
        metadata.  Pickle handles them correctly because objects like
        ODPSBlob/SolidBlob implement __reduce__.
        """
        frame = self.consolidate()

        object_data = None
        if pl and isinstance(frame, pl.DataFrame):
            for col in frame.columns:
                if not has_pl_object_type(frame[col].dtype):
                    continue
                if object_data is None:
                    object_data = {}
                ext_name, storage_str = pl_get_ext_type_name_and_storage(
                    frame[col].dtype
                )
                metadata = pl_get_ext_type_metadata(frame[col].dtype)
                object_data[col] = [
                    ext_name,
                    storage_str,
                    metadata,
                    frame[col].to_list(),
                ]
                # Replace with a null placeholder so the DataFrame
                # itself survives pickle without corruption.
                frame = frame.with_columns(
                    pl.Series(col, [None] * len(frame), dtype=pl.Binary)
                )

        state = {
            "_frame": frame,
            "range_columns": self.range_columns,
            "_object_data": object_data,
        }
        return state

    def __setstate__(self, state):
        self._frame = state["_frame"]
        self.range_columns = state.get("range_columns")

        object_data = state.get("_object_data")
        if object_data and pl and isinstance(self._frame, pl.DataFrame):
            for col, (ext_name, storage_str, metadata, values) in object_data.items():
                dtype = pl_make_ext_or_object_dtype(ext_name, storage_str, metadata)
                self._frame = self._frame.with_columns(
                    pl.Series(col, values, dtype=dtype)
                )


class LiteFrameData(HasShapeTileableData):
    __slots__ = ("_accessors", "__dict__")
    type_name = "LiteFrame"

    _physical_dtypes = SeriesField("_physical_dtypes")
    frame_metadata = ReferenceField("frame_metadata", FrameMetadata, default=None)

    def __init__(
        self, op=None, shape=None, physical_dtypes=None, frame_metadata=None, **kw
    ):
        super().__init__(
            _op=op,
            _shape=shape,
            _physical_dtypes=physical_dtypes,
            frame_metadata=frame_metadata,
            **kw,
        )
        self._accessors = dict()

    def __on_deserialize__(self):
        super().__on_deserialize__()
        self._accessors = dict()
        self.__dict__.pop("dtypes", None)
        self.__dict__.pop("columns", None)

    @cached_property
    def columns(self):
        cols = []
        if self.frame_metadata and self.frame_metadata.range_columns:
            cols.extend(self.frame_metadata.range_columns.keys())
        hidden = (
            set(self.frame_metadata.hidden_columns)
            if (self.frame_metadata and self.frame_metadata.hidden_columns)
            else set()
        )
        cols.extend(c for c in self._physical_dtypes.index if c not in hidden)
        return cols

    @cached_property
    def dtypes(self):
        hidden = (
            set(self.frame_metadata.hidden_columns)
            if (self.frame_metadata and self.frame_metadata.hidden_columns)
            else set()
        )
        result = (
            self._physical_dtypes.drop(list(hidden), errors="ignore")
            if hidden
            else self._physical_dtypes
        )
        if self.frame_metadata and self.frame_metadata.range_columns:
            range_dtypes = pd.Series(
                [wrap_arrow_dtype(pa.int64())] * len(self.frame_metadata.range_columns),
                index=list(self.frame_metadata.range_columns.keys()),
            )
            result = pd.concat([range_dtypes, result])
        return result

    @property
    def params(self):
        return {
            "shape": self.shape,
            "physical_dtypes": self._physical_dtypes,
            "frame_metadata": self.frame_metadata,
        }

    @property
    def _hidden_columns(self):
        if self.frame_metadata and self.frame_metadata.hidden_columns:
            return set(self.frame_metadata.hidden_columns)
        return set()

    @params.setter
    def params(self, new_params):
        params = new_params.copy()
        new_shape = params.pop("shape", None)
        if new_shape is not None:
            self._shape = new_shape
        physical_dtypes = params.pop("physical_dtypes", None)
        if physical_dtypes is not None:
            self._physical_dtypes = physical_dtypes
            self.__dict__.pop("dtypes", None)
            self.__dict__.pop("columns", None)
        frame_metadata = params.pop("frame_metadata", None)
        if frame_metadata is not None:
            self.frame_metadata = frame_metadata
            self.__dict__.pop("dtypes", None)
            self.__dict__.pop("columns", None)
        if params:
            raise TypeError(f"Unknown params: {list(params)}")

    def refresh_params(self):
        refresh_tileable_shape(self)
        fill_chunk_slices(self)

    def _to_str(self, representation=False):
        if is_build_mode() or len(self._executed_sessions) == 0:
            if representation:
                return (
                    f"{self.type_name} <op={type(self._op).__name__}, key={self.key}>"
                )
            else:
                return f"{self.type_name}(op={type(self._op).__name__})"
        else:
            data = self._fetch(session=self._executed_sessions[-1])
            if isinstance(data, LiteFrameResult):
                data = data.frame
            if pl and isinstance(data, pl.LazyFrame):
                data = data.collect()
            return repr(data) if representation else str(data)

    def __str__(self):
        return self._to_str(representation=False)

    def __repr__(self):
        return self._to_str(representation=True)

    def to_tensor(self, dtype=None):
        """Convert the LiteFrame to a MaxFrame tensor.

        All columns are combined into a single homogeneous tensor. When
        ``dtype`` is not provided, the result dtype is the common
        promotion of all column dtypes.

        Parameters
        ----------
        dtype : numpy.dtype, optional
            Target dtype for the resulting tensor. If None, the dtype is
            inferred from the column dtypes.

        Returns
        -------
        Tensor
            A MaxFrame tensor containing the LiteFrame data.

        See Also
        --------
        LiteFrame.from_tensor : Create a LiteFrame from a MaxFrame tensor.
        LiteFrame.values : Return the LiteFrame as a tensor.
        """
        return from_liteframe(self, dtype=dtype)

    def __maxframe_tensor__(self, dtype=None, order="K"):
        return self.to_tensor().astype(dtype=dtype, order=order, copy=False)

    @property
    def values(self):
        """Return the LiteFrame as a tensor.

        The LiteFrame data is converted to a homogeneous MaxFrame tensor
        whose dtype is the common promotion of all column dtypes.

        Returns
        -------
        Tensor
            A MaxFrame tensor containing the LiteFrame data.

        See Also
        --------
        LiteFrame.to_tensor : Convert the LiteFrame to a MaxFrame tensor.
        """
        return self.to_tensor()


class LiteFrameToPandasMixin(_ExecuteAndFetchMixin):
    __slots__ = ()

    def to_pandas(self, session=None, **kw):
        """Convert the LiteFrame to a pandas DataFrame.

        Executes the lazy LiteFrame and collects the results into a
        pandas DataFrame. Blob columns are preserved with their
        appropriate external blob dtypes.

        Parameters
        ----------
        session : Session, optional
            Session to use for execution. If None, the default session
            is used.
        **kw
            Additional keyword arguments forwarded to the execution
            backend.

        Returns
        -------
        pandas.DataFrame
            A pandas DataFrame containing the materialized results.
        """
        result = self._execute_and_fetch(session=session, **kw)
        if isinstance(result, LiteFrameResult):
            # Materialize virtual range columns (when _frame is None) and
            # convert through the blob-aware bridge so blob columns come
            # back as ExternalBlobDtype / AbstractExternalBlob values.
            result = result.materialize()
        if pl and isinstance(result, pl.LazyFrame):
            result = result.collect()
        if pl and isinstance(result, pl.DataFrame):
            # Only route through the blob-aware converter when blob columns
            # are present; otherwise plain to_pandas() preserves numpy
            # dtypes (int64, not int64[pyarrow]) as the public contract.
            if any(is_polars_blob_type(result[c].dtype) for c in result.columns):
                return polars_to_pandas_with_blobs(result)
            return result.to_pandas()
        return result


class LiteFrame(LiteFrameToPandasMixin, HasShapeTileable):
    __slots__ = ("_cache",)
    _allow_data_type_ = (LiteFrameData,)
    type_name = "LiteFrame"

    def __class_getitem__(cls, item):
        from maxframe.liteframe.functional.type_infer import LiteFrameType

        return LiteFrameType.from_getitem_args(item)

    @property
    def columns(self):
        """Return the column labels of the LiteFrame.

        Returns
        -------
        list
            A list of column name strings.
        """
        return self._data.columns

    @property
    def dtypes(self):
        """Return the dtype of each column.

        Returns
        -------
        pandas.Series
            A pandas Series of dtypes indexed by column name.
        """
        return self._data.dtypes

    @property
    def ndim(self):
        """Return the number of dimensions of the LiteFrame.

        A LiteFrame is always 2-dimensional (rows and columns).

        Returns
        -------
        int
            Always returns 2.
        """
        return 2

    @property
    def frame_metadata(self):
        return self._data.frame_metadata

    @property
    def _physical_dtypes(self):
        return self._data._physical_dtypes

    @property
    def _hidden_columns(self):
        return self._data._hidden_columns

    def __hash__(self):
        return super().__hash__()

[docs] def head(self, n=5): """Return the first ``n`` rows. Parameters ---------- n : int, default 5 Number of rows to select. Returns ------- LiteFrame A new LiteFrame containing the first ``n`` rows. Examples -------- >>> import maxframe.liteframe as mlf >>> lf = mlf.LiteFrame({'a': [1, 2, 3, 4, 5]}) >>> lf.head(3).execute() a 0 1 1 2 2 3 """ from maxframe.liteframe.operators.indexing import LiteFrameIlocGetItem op = LiteFrameIlocGetItem(indexes=[slice(0, int(n)), slice(None)]) return op(self)
[docs] def to_tensor(self, dtype=None): """Convert the LiteFrame to a MaxFrame tensor. All columns are combined into a single homogeneous tensor. When ``dtype`` is not provided, the result dtype is the common promotion of all column dtypes. Parameters ---------- dtype : numpy.dtype, optional Target dtype for the resulting tensor. If None, the dtype is inferred from the column dtypes. Returns ------- Tensor A MaxFrame tensor containing the LiteFrame data. See Also -------- LiteFrame.from_tensor : Create a LiteFrame from a MaxFrame tensor. LiteFrame.values : Return the LiteFrame as a tensor. """ return self._data.to_tensor(dtype=dtype)
def __maxframe_tensor__(self, dtype=None, order="K"): return self._data.__maxframe_tensor__(dtype=dtype, order=order) @property def values(self): """Return the LiteFrame as a tensor. The LiteFrame data is converted to a homogeneous MaxFrame tensor whose dtype is the common promotion of all column dtypes. Returns ------- Tensor A MaxFrame tensor containing the LiteFrame data. See Also -------- LiteFrame.to_tensor : Convert the LiteFrame to a MaxFrame tensor. """ return self.to_tensor() LITEFRAME_TYPE = (LiteFrame, LiteFrameData) register_output_types(OutputType.liteframe, LITEFRAME_TYPE)