Skip to content

Commit

Permalink
Added additional test
Browse files Browse the repository at this point in the history
  • Loading branch information
malhotrashivam committed Nov 6, 2023
1 parent 4786d17 commit 382c021
Showing 1 changed file with 26 additions and 0 deletions.
26 changes: 26 additions & 0 deletions py/server/tests/test_parquet.py
Original file line number Diff line number Diff line change
Expand Up @@ -391,6 +391,32 @@ def test_dates_and_time(self):
self.assert_table_equals(dh_table.select(["someDateColumn", "someTimeColumn"]),
from_disk_pandas.select(["someDateColumn", "someTimeColumn"]))

def test_time_with_different_units(self):
""" Test that we can write and read time columns with different units """
dh_table = empty_table(20000).update(formulas=[
"someTimeColumn = i % 10 == 0 ? null : java.time.LocalTime.of(i%24, i%60, (i+10)%60)"
])
write(dh_table, "data_from_dh.parquet")
table = pyarrow.parquet.read_table('data_from_dh.parquet')

def time_test_helper(pa_table, new_schema, dest):
# Write the provided pyarrow table type-casted to the new schema
pyarrow.parquet.write_table(pa_table.cast(new_schema), dest)
from_disk = read(dest)
df_from_disk = to_pandas(from_disk)
original_df = pa_table.to_pandas()
# Compare the dataframes as strings
print((df_from_disk.astype(str) == original_df.astype(str)).all().values.all())

# Test for nanoseconds, microseconds, and milliseconds
schema_nsec = table.schema.set(0, pyarrow.field('someTimeColumn', pyarrow.time64('ns')))
time_test_helper(table, schema_nsec, "data_from_pq_nsec.parquet")

schema_usec = table.schema.set(0, pyarrow.field('someTimeColumn', pyarrow.time64('us')))
time_test_helper(table, schema_usec, "data_from_pq_usec.parquet")

schema_msec = table.schema.set(0, pyarrow.field('someTimeColumn', pyarrow.time32('ms')))
time_test_helper(table, schema_msec, "data_from_pq_msec.parquet")

if __name__ == '__main__':
unittest.main()

0 comments on commit 382c021

Please sign in to comment.