mxiUdZddlmZddlZddlZddlZddlZddlZddlm Z m Z m Z m Z ddl mZmZmZmZddlmZmZmZddlmZddlmZmZmZmZmZmZmZm Z m!Z!m"Z"dd l#m$Z$m%Z%m&Z&m'Z'dd l(m)Z)m*Z*m+Z+m,Z,m-Z-m.Z.m/Z/m0Z0m1Z1m2Z2m3Z3erddl4Z5ddl6Z7dd l8m9Z9m:Z:m;Z;dd lm?Z?e&jeAZBdeCd<dZDejdZFdeCd<dZGdeCd<dZHdeCd<dZIdeCd<dZJdeCd<dZKdeCd<dZLdeCd<dZMdeCd <d!ZNdeCd"<d#ZOdeCd$<d%ZPdeCd&<d'ZQdeCd(<d)ZRdeCd*<d+ZSdeCd,<d-ZTdeCd.<d/ZUdeCd0<d1ZVdeCd2<d3ZWdeCd4<d5ZXdeCd6<d7ZYdeCd8<d9ZZdeCd:<d;Z[deCd<<d=Z\deCd><d?Z]deCd@<dAZ^deCdB<dCZ_deCdD<dEZ`deCdF<edGdHIZae"GdJdKeZbGdLdMeeaZcGdNdOeZdGdPdQeZeeeaeceazedzeezZfdReCdS<e dTdUdVdWdXdYdZeed[ebedeee)dfZgdReCd\<Gd]d^eZhdd_Zidd`ZjddaZkddbZlddcZmdddZnddeZoddfZpddgZqddhZrddiZsddjZtddkZuddlZvddmZwddnZxddoZyddpZzddqZ{ddrZ|ddsZ} d ddtZ~dduZddvZddwZeDdxf ddyZddzZdd{Zdd|Zdd}ZeDf dd~ZddZ d ddZddZ d ddZddZddZddZ ddZy)z8A bunch of useful utilities for dealing with dataframes.) annotationsN)ChainMapUserDictUserListdeque) ItemsViewIterableMappingSequence)EnumEnumMetaauto)MappingProxyType) TYPE_CHECKINGAnyFinalProtocol TypeAlias TypeGuardTypeVarUnioncastruntime_checkable)configerrorslogger string_util) CustomDictdump_pydantic_sequencehas_callable_attris_custom_dictis_dataclass_instance is_list_like is_namedtupleis_pydantic_modelis_sequence_of_pydantic_modelsis_typeis_version_less_than) DataFrameIndexSeries) _iLocIndexer)Stylerr_LOGGERi'z ^pandas.*$_PANDAS_DATA_OBJECT_TYPE_REz.dask.dataframe.dask_expr._collection.DataFrame_DASK_DATAFRAMEz+dask.dataframe.dask_expr._collection.Series _DASK_SERIESz*dask.dataframe.dask_expr._collection.Index _DASK_INDEXzdask.dataframe.core.DataFrame_DASK_DATAFRAME_LEGACYzdask.dataframe.core.Series_DASK_SERIES_LEGACYzdask.dataframe.core.Index_DASK_INDEX_LEGACYz_duckdb.DuckDBPyRelation_DUCKDB_RELATIONzduckdb.duckdb.DuckDBPyRelation_DUCKDB_RELATION_LEGACYz modin.pandas.dataframe.DataFrame_MODIN_DF_TYPE_STRzmodin.pandas.series.Series_MODIN_SERIES_TYPE_STRzpandas.io.formats.style.Styler_PANDAS_STYLER_TYPE_STRz polars.dataframe.frame.DataFrame_POLARS_DATAFRAMEz polars.lazyframe.frame.LazyFrame_POLARS_LAZYFRAMEzpolars.series.series.Series_POLARS_SERIESzpyspark.sql.dataframe.DataFrame_PYSPARK_DF_TYPE_STRz'pyspark.sql.connect.dataframe.DataFrame_PYSPARK_CONNECT_DF_TYPE_STRzray.data.dataset.Dataset _RAY_DATASETz$ray.data.dataset.MaterializedDataset_RAY_MATERIALIZED_DATASETz3snowflake.snowpark.modin.pandas.dataframe.DataFrame_SNOWPANDAS_DF_TYPE_STRz6snowflake.snowpark.modin.plugin.extensions.index.Index_SNOWPANDAS_INDEX_TYPE_STRz-snowflake.snowpark.modin.pandas.series.Series_SNOWPANDAS_SERIES_TYPE_STRzsnowflake.snowpark.row.Row_SNOWPARK_DF_ROW_TYPE_STRz&snowflake.snowpark.dataframe.DataFrame_SNOWPARK_DF_TYPE_STRzsnowflake.snowpark.table.Table_SNOWPARK_TABLE_TYPE_STRzxarray.core.dataset.Dataset_XARRAY_DATASET_TYPE_STRzxarray.core.dataarray.DataArray_XARRAY_DATA_ARRAY_TYPE_STRV_coT) covariantc8eZdZdZe ddZdddZd dZy) DBAPICursora`Protocol for DBAPI 2.0 Cursor objects (PEP 249). This is a simplified version of the DBAPI Cursor protocol that only contains the methods that are relevant or used for our DB API Integration. Specification: https://peps.python.org/pep-0249/ Inspired by: https://github.com/python/typeshed/blob/main/stdlib/_typeshed/dbapi.pyi cyNselfs O/home/htdocs/ttos/venv/lib/python3.12/site-packages/streamlit/dataframe_util.py descriptionzDBAPICursor.descriptionxs  cyrOrP)rRsizes rS fetchmanyzDBAPICursor.fetchmanyrUcyrOrPrQs rSfetchallzDBAPICursor.fetchallrYrUN)returnzdSequence[tuple[str, Any | None, int | None, int | None, int | None, int | None, bool | None]] | None).)rWintr\Sequence[Sequence[Any]])r\r^)__name__ __module__ __qualname____doc__propertyrTrXr[rPrUrSrMrMls.    L6rUrMc"eZdZdZeddZy)DataFrameGenericAliasatTechnically not a GenericAlias, but serves the same purpose in OptionSequence below, in that it is a type which admits DataFrame, but is generic. This allows OptionSequence to be a fully generic type, significantly increasing its usefulness. We can't use types.GenericAlias, as it is only available from python>=3.9, and isn't easily back-ported. cyrOrPrQs rSiloczDataFrameGenericAlias.ilocs$'rUN)r\r,)r_r`rarbrcrgrPrUrSreres''rUreceZdZdZddZy)PandasCompatiblezFProtocol for Pandas compatible objects that have a `to_pandas` method.cyrOrPrQs rS to_pandaszPandasCompatible.to_pandasrYrUN)r\zDataFrame | Series[Any])r_r`rarbrkrPrUrSririsP7rUriceZdZdZddZy)DataframeInterchangeCompatiblezProtocol for objects support the dataframe-interchange protocol. https://data-apis.org/dataframe-protocol/latest/index.html cyrOrP)rR allow_copys rS __dataframe__z,DataframeInterchangeCompatible.__dataframe__rYrUN)roboolr\r)r_r`rarbrprPrUrSrmrms  :rUrmrOptionSequencer) Series[Any]r-z Index[Any]pa.Tablezpa.Arrayznp.ndarray[Any, np.dtype[Any]]zMapping[Any, Any]DataceZdZdZeZeZeZeZeZ eZ eZ eZ eZ eZeZeZeZeZeZeZeZeZeZeZeZeZeZeZeZeZeZeZ eZ!eZ"eZ#eZ$eZ%y) DataFormatz7DataFormat is used to determine the format of the data.N)&r_r`rarbrUNKNOWNEMPTYCOLUMN_INDEX_MAPPINGCOLUMN_SERIES_MAPPINGCOLUMN_VALUE_MAPPING DASK_OBJECT DBAPI_CURSORDUCKDB_RELATIONKEY_VALUE_DICTLIST_OF_RECORDS LIST_OF_ROWSLIST_OF_VALUES MODIN_OBJECT NUMPY_LIST NUMPY_MATRIX PANDAS_ARRAYPANDAS_DATAFRAME PANDAS_INDEX PANDAS_SERIES PANDAS_STYLERPOLARS_DATAFRAMEPOLARS_LAZYFRAME POLARS_SERIES PYARROW_ARRAY PYARROW_TABLEPYSPARK_OBJECT RAY_DATASET SET_OF_VALUESSNOWPANDAS_OBJECTSNOWPARK_OBJECTTUPLE_OF_VALUESXARRAY_DATASETXARRAY_DATA_ARRAYrPrUrSrwrwsAfG FE6 F6&K6LfOVNfO6LVN6LJ6L6Lv6LFMFMvvFMFMFMVN&KFMfOfOVNrUrwc6ddl}t|j|S)aReturn True if the current Pyarrow version is less than the input version. Parameters ---------- v : str Version string, e.g. "0.25.0" Returns ------- bool Raises ------ InvalidVersion If the version strings are not valid. rN)pyarrowr( __version__)vpas rSis_pyarrow_version_less_thanrs&  22rUc6ddl}t|j|S)aReturn True if the current Pandas version is less than the input version. Parameters ---------- v : str Version string, e.g. "0.25.0" Returns ------- bool Raises ------ InvalidVersion If the version strings are not valid. rN)pandasr(r)rpds rSis_pandas_version_less_thanrs$  22rUc|/t|ttttt t tfryt|tjtjtjtjtjtjtj tj"tj$tj&tj(tj*tj,tj.tj0tj2tj4tj6tj8tj:tj<tj>hvS)zTrue if the object is a dataframe-like object. This does not include basic collection types like list, dict, tuple, etc. F) isinstancetuplesetstrbytesr]floatrqdetermine_data_formatrwr{r}r~rrrrrrrrrrrrrrrrrrrobjs rSis_dataframe_likers {juc3sE4&PQ  %((##    ####      !!$$""!!$$-* rUct|xs}t|xspt|xsct|xsVt |xsIt |xs<t |xs/t|xs"t|xstj|S)aTrue if the object is one of the supported unevaluated data objects. Currently supported objects are: - Snowpark DataFrame / Table - PySpark DataFrame - Modin DataFrame / Series - Snowpandas DataFrame / Series / Index - Dask DataFrame / Series / Index - Ray Dataset - Polars LazyFrame - Generator functions - DB API 2.0 Cursor (PEP 249) - DuckDB Relation (Relational API) Unevaluated means that the data is not yet in the local memory. Unevaluated data objects are treated differently from other data objects by only requesting a subset of the data instead of loading all data into th memory ) is_snowpark_data_objectis_pyspark_data_objectis_snowpandas_data_objectis_modin_data_objectis_ray_datasetis_polars_lazyframeis_dask_objectis_duckdb_relationis_dbapi_cursorinspectisgeneratorfunctionrs rSis_unevaluated_data_objectrCs( $ , !# & , $S ) ,  $ , #  , s # , #  , c " , 3  ,  & &s + rUc"t|tS)zLTrue if obj is a Pandas object (e.g. DataFrame, Series, Index, Styler, ...).)r'r/rs rSis_pandas_data_objectrd 33 44rUcFt|txst|tS)z-True if obj is a Snowpark DataFrame or Table.)r'rGrFrs rSrris 30 1 XWSBW5XXrUct|txr4t|dkDxr$t|dtxrt |ddS)z4True if obj is a list of snowflake.snowpark.row.Row.ras_dict)rlistlenr'rEr rs rSis_snowpark_row_listrnsL 3 1 HqL 1 CF5 6 1 c!fi 0 rUcbt|txst|txr t|dS)z6True if obj is a PySpark or PySpark Connect dataframe.toPandas)r'r>r?r rs rSrrxs0 )*Xgc;W.X - C ,-rUct|txsXt|txsFt|txs4t|txs"t|t xst|t S)z2True if obj is a Dask DataFrame, Series, or Index.)r'r0r3r1r4r2r5rs rSrrsg _% , 3. / , 3 % , 3+ , , 3 $  , 3* + rUcFt|txst|tS)z,True if obj is of Modin Dataframe or Series.)r'r8r9rs rSrrs 3* + SwsB=M=M(8(8 91QQqTB  9SW ||DNN+?@'|R ::a=0 0 "&{'B'BCW'X&YZFF K&&D!||d;sS[[];<<%d+ << 6t <= ={+||DNN,-- $0 ' 0E 9&&$$33D9!,w||~9'9""4(! LL*46BVW X  ::a=0 0 "&{'B'BCW'X&YZAA K&&$!!",,/FA/F"GHH$U__h?@!",,tDz":;;d!$,,.11T!$,,.11T"!+"4"4T":;;$#3X>?CT D"$t*--!",,t"455Y :<   >0G8   dD !$$Z0 0)$/ 0 0 0**##':,/     sZ$e5e ee ,e% ee% g(.)g# f1" g#+g(1f: 6-g##g(cj t|}ddl}|j }|j ||j}|j||jtd|jjS#t$r!}tjd|Yd}~d}~wwxYw)zSerialize pyarrow.Table to Arrow IPC bytes. Parameters ---------- table : pyarrow.Table A table to convert. Returns ------- bytes The serialized Arrow IPC bytes. zMRecursion error while truncating Arrow table. This is not supposed to happen.exc_infoNrr) _maybe_truncate_tableRecursionErrorr.warningrBufferOutputStreamRecordBatchStreamWriterschema write_tableclosergetvalue to_pybytes)tableerrrsinkwriters rS"convert_arrow_table_to_arrow_bytesr9s  %e, !2 "D 'R ' 'ell ;F u LLN 335 66%    "    s B B2B--B2cLddl} |jj|}t|S#|j|j|j f$rQ}t jd|t|}|jj|}Yd}~t|Sd}~wwxYw)zSerialize pandas.DataFrame to Arrow IPC bytes. Parameters ---------- df : pandas.DataFrame A dataframe to convert. Returns ------- bytes The serialized Arrow IPC bytes. rNzSerialization of dataframe to Arrow table was unsuccessful. Applying automatic fixes for column types to make the dataframe Arrow-compatible.r)) rTable from_pandasArrowTypeError ArrowInvalidArrowNotImplementedErrorr.info#fix_arrow_incompatible_column_typesr9)rrr5r&s rS convert_pandas_df_to_arrow_bytesrB7s )$$R( .e 44   r0K0K L)    1 4$$R( -e 44)s,&B#=BB#cbddl}|j|}td|jS)aConvert Arrow bytes (IPC format) to pandas.DataFrame. Using this function in production needs to make sure that the pyarrow version >= 14.0.1, because of a critical security vulnerability in pyarrow < 14.0.1. Parameters ---------- source : bytes A bytes object to convert. Returns ------- pandas.DataFrame The converted dataframe. rNr))rRecordBatchStreamReaderr read_pandas)sourcerreaders rS convert_arrow_bytes_to_pandas_dfrHTs/" 'R ' ' /F  V//1 22rUcPddlm}|jj|y)z[Show a message to the user with important information about the processed dataset. r)get_dg_singleton_instanceN)$streamlit.delta_generator_singletonsrJmain_dgcaption)msgrJs rSr r ksO''//4rUczddl}t||jr t|St ||}t |S)adTry to convert different formats to Arrow IPC format (bytes). This method tries to directly convert the input data to Arrow bytes for some supported formats, but falls back to conversion to a Pandas DataFrame and then to Arrow bytes. Parameters ---------- data : dataframe-, array-, or collections-like object The data to convert to Arrow bytes. max_unevaluated_rows: int If unevaluated data is detected this func will evaluate it, taking max_unevaluated_rows, defaults to 10k. Returns ------- bytes The serialized Arrow IPC bytes. rN)rrr;r9r'rB)rr rrs rSconvert_anything_to_arrow_bytesrPts:2$!1$77 't-A BB +B //rUcf|gSt|ttttfr|gSt|t r,|Dcgc] }t|tr |j n|"c}St|tr#tdt|jSt|r t|sttd|S t|d}|jrgStdt|jdddfj!Scc}w#t"j$$r|gcYSwxYw)aTry to convert different formats to a list. If the input is a dataframe-like object, we just select the first column to iterate over. Non sequence-like objects and scalar types, will just be wrapped into a list. Parameters ---------- obj : dataframe-, array-, or collections-like object The object to convert to a list. Returns ------- list The converted list. N list[V_co]zIterable[V_co]T)r!r)rrr]rrqr rr rrkeysr#rr'emptyrgto_listrr)rmemberrs rSconvert_anything_to_listrWs" { #S%./u #x SVV 63 7 VCVV#wL$sxxz"233C!5c!:D)3/00 0F}}  lDad);)C)C)E$FG 'W0  ' 'u s%DD3DD0/D0c tjdrhttjddz}t|jdz}|j}|dkDr||kDrt j |||z z}t jtt|t j||z dzz ||dzz |dz d}|jd|}t||xsd||z zS|rtj|j}tj|j|z}||k(r-t|j}t|j|z}td |d |d |S) aExperimental feature to automatically truncate tables that are larger than the maximum allowed message size. It needs to be enabled via the server.enableArrowTruncation config option. Parameters ---------- table : pyarrow.Table A table to truncate. truncated_rows : int or None The number of rows that have been truncated so far. This is used by the recursion logic to keep track of the total number of truncated rows. zserver.enableArrowTruncationzserver.maxMessageSizeg.Arg?g{Gz?ru⚠️ Showing z out of z# rows due to data size limitations.)r get_optionr]nbytesnum_rowsmathceilfloormaxminslicer+rr rr ) r5truncated_rowsmax_message_size table_size table_rows targeted_rows sliced_tabledisplayed_rows total_rowss rSr+r+s&78v001HICOP/0 ^^ >j+;;!IIj4Dz4Q&RSM!JJ& J4NRV3V(WW"j4&78"Q  M!;;q-8L(~2zM7QR  (88HN$44U^^n5TUJ+"%U^^!4 .!@A "!.!1*F55  LrUclddlm}m}m}|jj dvryt |jdvry|jdk(rj||d}|dvry|d k(rVt|dk(s t|d sytd |jd}||r||st|tryy y ) zYReturn True if the column type is known to cause issues during Arrow conversion. r)r is_dict_liker#)r%T> period[B] period[N] period[U] period[ns] period[us]geometryobjectr)z mixed-integercomplexrrgzDataFrameGenericAlias[Any]F) rrrlr#dtypekindrrhasattrrrgr frozenset)columnrrlr# inferred_type first_values rS is_colum_type_arrow_incompatibler|sIH || 6<<   ||x$F48    G #6{awvv'>;VDII!LK!- ,k95 rUcddl}d}|xs |jD]:}t||s||j}||j d||<<|sgt |j |jsGt|j r2||j}|j j d|_||S|S)aFix column types that are not supported by Arrow table. This includes mixed types (e.g. mix of integers and strings) as well as complex numbers (complex128 type). These types will cause errors during conversion of the dataframe to an Arrow table. It is fixed by converting all values of the column to strings This is sufficient for displaying the data on the frontend. Parameters ---------- df : pandas.DataFrame A dataframe to fix. selected_columns: List[str] or None A list of columns to fix. If None, all columns are evaluated. Returns ------- The fixed dataframe. rNstring)rrr|rastyperr MultiIndex)rselected_columnsrdf_copycols rSrArAZs.!%G-2::4 +BsG 4'')c7>>(3GCL 4   HH MM  -RXX 6 ?ggiG1 )71r1rUc ddl}ddl}ddl}|tjSt ||j rtjSt ||jr8t|jdk(rtjStjSt ||jrtjSt ||jrtj St ||j"rtj$St ||j&rtj(St+|rtj,St ||j.j0j2rtj4St7|rtj8St;|rtj<St?|rtj@StC|rtjDStG|rtjHStK|rtjLStO|rtjPStS|rtjTStW|rtjXSt[|rtj\St_|s ta|rtjbSte|rtjfSti|rtjjSt |tltntpfs,ts|s!tu|stw|s ty|rtjzSt |t|t~frtjSt |ttttfrt|rVt |trtjSt |ttfrtjStjStt|}t |ts ty|rtjSt |ttttfrtjStjSt |ttfr|stjzSt|dkDrtt|j}t |trtjSt |ttfrtjSt ||j"rtjStjzStjSt|rtjStjS)zDetermine the data format of the input data. Parameters ---------- input_data : Any The input data to determine the data format of. Returns ------- DataFormat The data format of the input data. rNr)Trrrrwryrr)rrrrrrr;rArrayrr+rr*rrrrrrrrrrrrrrrrrrrrrrrrrrr}rrrrrrr~rrrr"r$r!r%rrrrrrrrxrrrrnextiterrrr valuesrzr|r{r#rx) input_datar"rr first_elementr{s rSrrs*bll+****bjj) z A %(( (&&&*bhh''''*bhh''''*bii('''*bhh'&&& #'''*bff//>>?&&& #''':&***:&***J'&&& ,+++j)((($(((J'+++j!%%%j!%%%z*.B:.N)))*%)))z"&&&  x!1 2  ! ,  $ * % Z ((((*y)45&&&*tUC;< z **e,!111*sI&67!///,, ,T*-. mT *.? .N-- - mdE3 %B C** *&   % Jw 0,, , z?Q tJ$5$5$789K+t,!666+e}5!666+ryy1!777,, ,    j !(((   rUcddl}ddl}|j|j|j|j gdj S)a Unify all missing values in a DataFrame to None. Pandas uses a variety of values to represent missing values, including np.nan, NaT, None, and pd.NA. This function replaces all of these values with None, which is the only missing value type that is supported by all data rN)rrreplaceNANaTnan infer_objects)rr"rs rS_unify_missing_valuesrs9 ::ruubffbff-t 4 B B DDrUct|jdk7r"tdt|jd|jdddfS)zConvert a Pandas DataFrame to a Pandas Series by selecting the first column. Raises ------ ValueError If the DataFrame has more than one column. r6DataFrame is expected to have a single column but has .Nr)rrrrg)rs rS_pandas_df_to_seriesrsK 2::!DS_DUUV W   771a4=rUc |tjtjtjtjtj tj tjtjtjtjtjtjtjh vr|S|tjk(rCddl}|j r|j"dS|j$dddfj'S|tj(k(r2ddl}|j r|j"dS|j'S|tj*k(rddl}|j.j1|S|tj2k(r(ddl}|j4j1t7|S|tj8k(r t7|S|tj:tj<hvrddl}|j1|S|tj@k(rddl}|j1t7|S|tjBk(rddl"}|jFjI|S|tjJk(r(ddl"}|jLjOt7|S|tjPk(rtS|jUdS|tjVk(r'tS|j'jYS|tjZk(rtS|jUdS|tj\k(rtS|jUdS|tj^k(r|jUdS|tj`tjbtjdfvrtS|}g}tg|jhdk(r"|j$dddfjY}n:tg|jhdk\r"tkd tg|jhd |tjbk(r tm|S|tjdk(r to|S|S|tjpk(r:tS|}|j riS|j$dddfjUStkd |) aConvert a Pandas DataFrame to the specified data format. Parameters ---------- df : pd.DataFrame The dataframe to convert. data_format : DataFormat The data format to convert to. Returns ------- pd.DataFrame, pd.Series, pyarrow.Table, np.ndarray, xarray.Dataset, xarray.DataArray, polars.Dataframe, polars.Series, list, set, tuple, or dict. The converted dataframe. rNrecordsrrrseriesrrrzUnsupported input data format: )9rwryr}r~rrrrrrrrrrrrrTrrgto_numpyrrrr;r<rrrrrrpolarsrrxarrayDatasetrr DataArray from_seriesrrrrtolistrzr|r{rrrrrrrrr)r data_formatr"rplxr return_lists rS convert_pandas_df_to_data_formatr!s<""##  !!$$"" j+++ !#zrzz!}Fbggadm.D.D.FFj--- "zrzz!};bkkm;j...xx##B''j...xx##$8$<==j...#B''z22J4O4OPP~~b!!j...~~22677j///zz((,,j222||''(r?r@rArBrCrDrErFrGrHrIrJrMrerirmrrrurwrrrrrrrrrrrrrrrrrrrrrrrrrr'r9rBrHr rPrWr+r|rArrrrrPrUrSrsH?" ;;BB%%"   :9    //1."""8,,!%/RZZ %>U>IIC eCA UA!@?9U97E74%4!AA>E> <