mxi} ZddlZddlZddlZddlZddlmZddlm Z ddlm Z m Z  ddl m Z ejjZd)dZdZejj'dddgd Zejj'd ej*d d d fdZdZejj'dddgdZdZdZejj'dddgdZdZdZejj'dddgdZdZejj'dddgdZ ejjBdZ"ejjBdZ#ejj'dejHejJddejHejJdejJdejJddkDgd Z&d!Z'd"Z(d#Z)d$Z*d%Z+d&Z,d'Z-d(Z.y#e$rdZ YwxYw)*N)tobytes) ArrowInvalidArrowNotImplementedErrorcDddlm}|tj|S)Nr)_get_udf_context)pyarrow._computerpadefault_memory_pool) batch_lengthrs S/home/htdocs/ttos/venv/lib/python3.12/site-packages/pyarrow/tests/test_substrait.pymock_udf_contextr &s1 B224l CCctjjt||}tj j ||j5}|j|ddd|S#1swY|SxYw)N)schema) ospathjoinstrr ipcRecordBatchFileWriterr write_table)tmpdir file_nametablerwriters r _write_dummy_data_to_diskr+s_ 77<<F Y /D  % %d5<< % @"F5!" K" Ks A11A; use_threadsTFcd}d}tjgdgdg}t|||}t|j dt j |j}tjj|}tj||}|j} |jdg| jdgk(sJy)Nax { "version": { "major": 9999 }, "relations": [ {"rel": { "read": { "base_schema": { "struct": { "types": [ {"i64": {}} ] }, "names": [ "foo" ] }, "local_files": { "items": [ { "uri_file": "FILENAME_PLACEHOLDER", "arrow": {} } ] } } }} ] } zread_data.arrowfoonamesFILENAME_PLACEHOLDERrr rrrreplacepathlibPathas_uri _substrait_parse_json_plan substrait run_queryread_allselect rrsubstrait_queryrrrquerybufreaderres_tbs r test_run_serialized_queryr;2sO<"I HHo&ug 6E $VY >D O++ T 2 9 9 ;= >E -- ( ( /C  + >F __ F << FMM5'$: :: :rr7sbuffersbytesr c|t|tjtfsIdt |d}t j t|5tj|dddyd}t j t|5tj|dddy#1swYyxYw#1swYyxYw)Nz)Expected 'pyarrow.Buffer' or bytes, got ''matchz1ParseFromZeroCopyStream failed for substrait.Plan) isinstancer Bufferbytestypepytestraises TypeErrorr1r2r)rr7msgs r test_run_query_input_typesrH`s ebii/ 09$u+aH ]]9C 0 '    & ' >C |3 /#E"##  '##s B&B2&B/2B;cd}tjjt|}d}t j t |5tj|dddy#1swYyxYw)Nz2 { "relations": [ ] } zPlan has no relationsr>) r r/r0rrDrErr1r2)r7r8 exec_messages r test_invalid_planrKpsY E -- ( ( 8C*L |< 8!C !!!s A''A0cd}d}tjgdgdg}t|||}t|j dt j |j}tjjt|}tj||}|j} |jdg| jdgk(sJy)Na { "version": { "major": 9999 }, "relations": [ {"rel": { "read": { "base_schema": { "struct": { "types": [ {"i64": {}} ] }, "names": [ "bar" ] }, "local_files": { "items": [ { "uri_file": "FILENAME_PLACEHOLDER", "arrow": {}, "metadata" : { "created_by" : {}, } } ] } } }} ] } zbinary_json_data.arrowrbarr&r(r)r*r5s r (test_binary_conversion_with_json_optionsrN}sOB)I HHo&ug 6E $VY >D O++ T 2 9 9 ;= >E -- ( ( 8C  + >F __ F << FMM5'$: :: :rcF|d|}|D]}|j|syy)N#TF)endswith)fnsext_filefn_namesuffixfns r has_functionrWs5z7) $F ;;v  rc|tjj}t|ddsJt|ddsJy)Nzfunctions_arithmetic.yamladdsum)r r/get_supported_functionsrW)supported_functionss r test_get_supported_functionsr]sK--??A +3U << < +3U << .table_providersP/0 0 1X X% %%  1X  56 6ra { "version": { "major": 9999 }, "relations": [ {"rel": { "read": { "base_schema": { "struct": { "types": [ {"i64": {}} ] }, "names": [ "x" ] }, "namedTable": { "names": ["t1"] } } }} ] } rlr) r Table from_pydictrfieldint64r/r0rr1r2r3) rrlr6r8r9r:rirjrks @@@r test_named_tablerrs88''i(89L88''i(89Lyy"((3 345H 7O2 -- ( ()A BC \\ # # N $EF __ F \ !! !rc<tjjdgdifd}d}tjj t |}d}t jt|5tj||dddy#1swYyxYw)Nr_r`cH|s td|ddk(rStdNrcrrdrfrgr'_rjs r rlz;test_named_table_invalid_table_name..table_provider0/0 0 1X  56 6ra { "version": { "major": 9999 }, "relations": [ {"rel": { "read": { "base_schema": { "struct": { "types": [ {"i64": {}} ] }, "names": [ "x" ] }, "namedTable": { "names": ["t3"] } } }} ] } zInvalid NamedTable Sourcer>rl) r rnror/r0rrDrErr1r2)rlr6r8rJrjs @r #test_named_table_invalid_table_namerzs88''i(89L7O2 -- ( ()A BC.L |< 8@C?@@@s 1BBcRtjjdgdifd}d}t|}tjj t|}d}t jt|5tj||dddy#1swYyxYw)Nr_r`cH|s td|ddk(rStdrurgrvs r rlz4test_named_table_empty_names..table_providerrxra { "version": { "major": 9999 }, "relations": [ {"rel": { "read": { "base_schema": { "struct": { "types": [ {"i64": {}} ] }, "names": [ "x" ] }, "namedTable": { "names": [] } } }} ] } z!names for NamedTable not providedr>ry) r rnrorr/r0rDrErr1r2)rlr6r7r8rJrjs @r test_named_table_empty_namesr}s88''i(89L7O0 O $E -- ( ( 8C6L |< 8@C?@@@s <BB&c ` tjjdgdi fd}d}tjj |}tj j |||}|j}|\}} jdd|td d} || k(sJy) Nr_r`cH|s td|ddk(rStdrurgr'rw test_tables r rlz.test_udf_via_substrait..table_providerJ0/0 0 1X  56 6rs { "extensionUris": [ { "extensionUriAnchor": 1 }, { "extensionUriAnchor": 2, "uri": "urn:arrow:substrait_simple_extension_function" } ], "extensions": [ { "extensionFunction": { "extensionUriReference": 2, "functionAnchor": 1, "name": "y=x+1" } } ], "relations": [ { "root": { "input": { "project": { "common": { "emit": { "outputMapping": [ 1, 2, ] } }, "input": { "read": { "baseSchema": { "names": [ "t", ], "struct": { "types": [ { "i64": { "nullability": "NULLABILITY_REQUIRED" } }, ], "nullability": "NULLABILITY_REQUIRED" } }, "namedTable": { "names": [ "t1" ] } } }, "expressions": [ { "selection": { "directReference": { "structField": {} }, "rootReference": {} } }, { "scalarFunction": { "functionReference": 1, "outputType": { "i64": { "nullability": "NULLABILITY_NULLABLE" } }, "arguments": [ { "value": { "selection": { "directReference": { "structField": {} }, "rootReference": {} } } } ] } } ] } }, "names": [ "x", "y", ] } } ] } rmr y ) r rnror/r0r1r2r3 add_columnr ) unary_func_fixturerrlr6r8r9r:functionname expected_tbrs @r test_udf_via_substraitrFs%%sI&67J7cOJ -- ( ( 9C \\ # # N $EF __ F'NHd''3jo1/0K [  rc|tjjdgdifd}d}tjj |}t j tj5}tjj||ddddtjvsJy#1swY#xYw)Nr_r`cH|s td|ddk(rStdrurgrs r rlz=test_udf_via_substrait_wrong_udf_name..table_providerrrs { "extensionUris": [ { "extensionUriAnchor": 1 }, { "extensionUriAnchor": 2, "uri": "urn:arrow:substrait_simple_extension_function" } ], "extensions": [ { "extensionFunction": { "extensionUriReference": 2, "functionAnchor": 1, "name": "wrong_udf_name" } } ], "relations": [ { "root": { "input": { "project": { "common": { "emit": { "outputMapping": [ 1, 2, ] } }, "input": { "read": { "baseSchema": { "names": [ "t", ], "struct": { "types": [ { "i64": { "nullability": "NULLABILITY_REQUIRED" } }, ], "nullability": "NULLABILITY_REQUIRED" } }, "namedTable": { "names": [ "t1" ] } } }, "expressions": [ { "selection": { "directReference": { "structField": {} }, "rootReference": {} } }, { "scalarFunction": { "functionReference": 1, "outputType": { "i64": { "nullability": "NULLABILITY_NULLABLE" } }, "arguments": [ { "value": { "selection": { "directReference": { "structField": {} }, "rootReference": {} } } } ] } } ] } }, "names": [ "x", "y", ] } } ] } ryzNo function registered) r rnror/r0rDrE ArrowKeyErrorr1r2rvalue)rlr6r8excinfors @r %test_udf_via_substrait_wrong_udf_namers%%sI&67J7cOJ -- ( ( 9C r'' (CG s>BC #s7=='9 99 9CCs ."B22B;cbtjjdgdifd}d}tjj t |}tj j|||}|j}tjjdgdi}||k(sJy)Nr_r`cSN)r'rin_tables r rlz/test_output_field_names..table_provider<sra { "version": { "major": 9999 }, "relations": [ { "root": { "input": { "read": { "base_schema": { "struct": { "types": [{"i64": {}}] }, "names": ["x"] }, "namedTable": { "names": ["t1"] } } }, "names": ["out"] } } ] } rmout) r rnror/r0rr1r2r3)rrlr6r8r9r:expectedrs @r test_output_field_namesr8sxx##S)$45HO4 -- ( ()A BC \\ # # N $EF __ Fxx##UI$67H X  rcbtjjgdgdgddfd}d}tjj |}tj j ||d}|j}tjjd d gd d gd }||k(sJy)Nr r r!r!)r r!r"r#)?rrr)kv1v2cSrrrs r rlz7test_scalar_aggregate_udf_basic..table_providerk rs { "extensionUris": [ { "extensionUriAnchor": 1, "uri": "urn:arrow:substrait_simple_extension_function" }, ], "extensions": [ { "extensionFunction": { "extensionUriReference": 1, "functionAnchor": 1, "name": "sum_mean" } } ], "relations": [ { "root": { "input": { "extensionSingle": { "common": { "emit": { "outputMapping": [ 0, 1 ] } }, "input": { "read": { "baseSchema": { "names": [ "k", "v1", "v2", ], "struct": { "types": [ { "i64": { "nullability": "NULLABILITY_REQUIRED" } }, { "i64": { "nullability": "NULLABILITY_NULLABLE" } }, { "fp64": { "nullability": "NULLABILITY_NULLABLE" } } ], "nullability": "NULLABILITY_REQUIRED" } }, "namedTable": { "names": ["t1"] } } }, "detail": { "@type": "/arrow.substrait_ext.SegmentedAggregateRel", "segmentKeys": [ { "directReference": { "structField": {} }, "rootReference": {} } ], "measures": [ { "measure": { "functionReference": 1, "phase": "AGGREGATION_PHASE_INITIAL_TO_RESULT", "outputType": { "fp64": { "nullability": "NULLABILITY_NULLABLE" } }, "arguments": [ { "value": { "selection": { "directReference": { "structField": { "field": 1 } }, "rootReference": {} } } }, { "value": { "selection": { "directReference": { "structField": { "field": 2 } }, "rootReference": {} } } } ] } } ] } } }, "names": [ "k", "v_avg" ] } } ], } Frmr r!g@g@)rv_avgr rnror/r0r1r2r3varargs_agg_func_fixturerlr6r8r9r:rrs @r test_scalar_aggregate_udf_basicrcs%%,# %J |Oz -- ( ( 9C \\ # # N$?F __ F((&&Vs(K [  rcntjjgdgdgdgddfd}d}tjj |}tj j ||d }|j}tjjgd gd gd d }||k(sJy)N)r r r r r!r!r!r!)r rrr rr rr )r r!r"r#r$ra)rrrrg@g@g@g@)trrrcSrrrs r rlz5test_hash_aggregate_udf_basic..table_providerrrs  { "extensionUris": [ { "extensionUriAnchor": 1, "uri": "urn:arrow:substrait_simple_extension_function" }, ], "extensions": [ { "extensionFunction": { "extensionUriReference": 1, "functionAnchor": 1, "name": "sum_mean" } } ], "relations": [ { "root": { "input": { "extensionSingle": { "common": { "emit": { "outputMapping": [ 0, 1, 2 ] } }, "input": { "read": { "baseSchema": { "names": [ "t", "k", "v1", "v2", ], "struct": { "types": [ { "i64": { "nullability": "NULLABILITY_REQUIRED" } }, { "i64": { "nullability": "NULLABILITY_REQUIRED" } }, { "i64": { "nullability": "NULLABILITY_NULLABLE" } }, { "fp64": { "nullability": "NULLABILITY_NULLABLE" } } ], "nullability": "NULLABILITY_REQUIRED" } }, "namedTable": { "names": ["t1"] } } }, "detail": { "@type": "/arrow.substrait_ext.SegmentedAggregateRel", "groupingKeys": [ { "directReference": { "structField": { "field": 1 } }, "rootReference": {} } ], "segmentKeys": [ { "directReference": { "structField": {} }, "rootReference": {} } ], "measures": [ { "measure": { "functionReference": 1, "phase": "AGGREGATION_PHASE_INITIAL_TO_RESULT", "outputType": { "fp64": { "nullability": "NULLABILITY_NULLABLE" } }, "arguments": [ { "value": { "selection": { "directReference": { "structField": { "field": 2 } }, "rootReference": {} } } }, { "value": { "selection": { "directReference": { "structField": { "field": 3 } }, "rootReference": {} } } } ] } } ] } } }, "names": [ "t", "k", "v_avg" ] } } ], } Frmr)r rrr ) @rg"@g&@)rrrrrs @r test_hash_aggregate_udf_basicrs%%&&'7 9JNO^ -- ( ( 9C \\ # # N$?F __ F((&&  &(K [  rexprr_rr2ctjtjdtjtjdtjg}tjj |gdg|}tjj |}||jk(sJt|jdk(sJd|jvsJy)Nr_r test_exprr ) r rrpint32r1serialize_expressionsdeserialize_expressionslen expressions)rrr8returneds r test_serializing_expressionsrs YY bhhj! bhhj!F ,, , ,dVk]F KC||33C8H X__ $$ $ x## $ )) ) (.. .. .rc `tjddftjddftjddftjdftjdftj dfd}tj |jDcgc]\}\}}tj|| c}}}fd}|jD];\}\}}|tj|tj|| k(=ycc}}}w) Nsrmsns test_strings test_string) time_seconds time_millis time_nanos date_millis large_string large_binaryctjj|gdg}tjj|}|jk(sJyNr)r r1rrr)rr8rrs r check_round_tripz3test_arrow_specific_types..check_round_tripsEll00$+O<<77<(((r)rC) r time32time64date64rrritemsrppcscalar)fieldsrtyprwrvalrs @r test_arrow_specific_typesrs3+ $+yy* Q'*M:*N; FYY&,,.QQhsAs+Q RF) #LLNEjsC$299Ss+CCDERs)#D) c tjtjdtjtjdtjtjdtj tj tjtjdtjtj tjgtjtjdtjtjdtjtjdtjtjdtjgfd}D] }|| y)N binary_view string_view dictionaryreectjtj|j}tj j |gdg}tj j|}|jk(sJyr) ris_nullrprr r1rrr)rprr8r alt_schemars r check_one_wayz/test_arrow_one_way_types..check_one_ways`zz"((5::./ll00$+O<<77<X__,,,r) r rrprrrrstringrun_end_encodedbinary)rrprrs @@r test_arrow_one_way_typesrs YY HH]BNN$4 5 HH]BNN$4 5 HH\2==RYY[#I J HHUB..rxxz299;G H  F HH]BIIK 0 HH]BIIK 0 HH\299; / HHUBIIK (  J- erctjtjdtjtjdtjg}t j t jdd}t j t jdd}t jt5}tjj|gg|ddddtjvsJt jt5}tjj|gddg|ddddt|jvsJt jt5}tjj|gdg|dddd t|jvsJy#1swYxYw#1swYxYw#1swY;xYw) Nr_rrzzneed to have the same lengthr%rMrzNo match for FieldRef) r rrprrequalrDrE ValueErrorr1rrr)rrbad_exprrs r test_invalid_expression_ser_desrss YY bhhj! bhhj!F 88BHHSM1 %Dxx q)H z "?g **D62v>? )S-? ?? ? z "Kg **D6E5>6JK )S-? ?? ? z "Ig **H:xHI "c'--&8 88 8??KKIIs$#G,%G $G)GG&)G2ctjtjdtjtjdtjg}t j t jddt j t jdt jdg}tj j|ddg|}tj j|}||jk(sJt|jdk(sJt j t jddt j t jdt jdg}t|jdt|dk(sJt|jdt|dk(sJy) Nr_rrfirstsecondr!rr ) r rrprrrr1rrrrr)rexprsr8r norm_exprss r %test_serializing_multiple_expressionsrsY YY bhhj! bhhj!FXXbhhsmQ '"((3-#)O PE ,, , ,UWh4G PC||33C8H X__ $$ $ x## $ )) )((288A;*BHHRXXa["((1+,NOJ x##G, -Z]1C CC C x##H- .#jm2D DD Drc<tjtjdtjtjdtjg}t j t jdd}t j t jdd}|j |}tjj|}||jk(sJt|jdk(sJt|jdt|k(sJtjj||gddg|}tjt5}tj j#|dddd tj$vsJtjj|gd g|}tj j#|}t|t|k(sJy#1swY~xYw) Nr_rrrr expressionrrzcontained multiple expressions weirdname)r rrprrr to_substraitr1rrrrrrDrEr Expressionfrom_substraitr)rr expr_normr8rrexpr2s r test_serializing_with_computer s YY bhhj! bhhj!F 88BHHSM1 %D!a(I   F #C||33C8H X__ $$ $ x## $ )) ) x##L1 2c)n DD D ,, , ,dD\GX;NPV WC z "*g $$S)* +s7==/A AA A ,, , ,dVk]F KC MM ( ( -E u:Y '' '**s 5 HHctjtjdtjg}t j d}t j d}t j ||g}tjt5tjj|dg|dddtjj|dg|d}tjj|}||jk(sJt|jdk(sJt|jdt|dk(sJy#1swYxYw) Nr_rr#rT)allow_arrow_extensionsr r)r rrpuint32rr shift_leftrDrErr1rrrrr)rabrr8rs r test_serializing_udfsr$s#YY biik"F " A ! A ]]1a !E / 0D **56(FCD ,, , , x - >C||33C8H X__ $$ $ x## $ )) ) x##F+ ,E!H == =DDs #EE#cd}tjtjdtjtjdtjg}tj j |}||k(sJtj j|}|j|k(sJtj j |}||k(sJtj j |j}||k(sJtj j|j}|j|k(sJy)Ns x y * br_r) r rrprrr1deserialize_schemaserialize_schemarr)substrait_schemaexpected_schemararrow_substrait_schemas r test_serializing_schemar:s Vii bhhj! biik"!O||../?@H h && &\\::8D ! ( (,< << <||../EFH h && &||../E/L/LMH h && &||334J4U4UVH ??o -- -rcGdd}d}tjjj||}t |j dk(sJd|j vsJd|j vsJy)NceZdZdZdZy)7test_bound_expression_from_Message..FakeMessagec||_yrr)selfrs r __init__z@test_bound_expression_from_Message..FakeMessage.__init__Rs DIrc|jSrr)rs r SerializeToStringzItest_bound_expression_from_Message..FakeMessage.SerializeToStringUs 99 rN)__name__ __module__ __qualname__rrrrr FakeMessagerQs   rr sj  project_release  project_version"0 project_version project_release : br!project_releaseproject_version)r r1BoundExpressionsrrr)r messagers r "test_bound_expression_from_MessagerPsvAG LL ) ) 8 8W9M NE u  !Q && &  1 1 11 1  1 1 11 1r)r)/rr,rDpyarrowr pyarrow.computecomputer pyarrow.librrrpyarrow.substraitr1 ImportErrormark pytestmarkr r parametrizer; py_bufferrHrKrNrWr]rrrzr}rrrnumpyrrrrprrrrrrrrrrrr rsm$  >) [[ " " D u 6*;7*;Z<2<< #:Ha"HI #J # !u 6,;7,;b<u 6-"7-"`'@T'@Tu 6x!7x!vs:lu 6'7'TQ!Q!hg!g!T BHHXRXXc]A BHHXRXXc]HBHHSM* BHHSMB" /  /E(89( E (6>,.,2e!IsHH*)H*