Skip to content

Commit

Permalink
Npgsql: Use EnableDynamicJson to unlock better container type mappings
Browse files Browse the repository at this point in the history 
Vanilla Npgsql provides good enough support to handle CrateDB's ARRAY
and OBJECT types better than just plain strings. This patch demonstrates
type mappings to native .NET `List` and `Dictionary` types, as well as
type mappings to custom .NET POCO types.
  • Loading branch information
@amotl
amotl committed Dec 19, 2024
1 parent b6c71b1 commit be0835d
Show file tree
Hide file tree
Showing 4 changed files with 141 additions and 53 deletions.
22 changes: 22 additions & 0 deletions by-language/csharp-npgsql/BasicPoco.cs
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
namespace demo;

public class BasicPoco
{

public string? name { get; set; }
public int? age { get; set; }

public override bool Equals(object obj)
{
var other = (BasicPoco) obj;
return name == other.name && age == other.age;
}

public override int GetHashCode()
{
return base.GetHashCode();
}

public override string ToString() => "Name: " + name + " Age: " + age;

}
12 changes: 9 additions & 3 deletions by-language/csharp-npgsql/DemoProgram.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,12 +18,18 @@ await Parser.Default.ParseArguments<Options>(args)
var connString = $"Host={options.Host};Port={options.Port};SSL Mode={options.SslMode};" +
$"Username={options.Username};Password={options.Password};Database={options.Database}";
Console.WriteLine($"Connecting to {connString}\n");
await using var conn = new NpgsqlConnection(connString);
conn.Open();

var dataSourceBuilder = new NpgsqlDataSourceBuilder(connString);
dataSourceBuilder.EnableDynamicJson();
await using var dataSource = dataSourceBuilder.Build();
await using var conn = dataSource.OpenConnection();

await DatabaseWorkloads.SystemQueryExample(conn);
await DatabaseWorkloads.BasicConversationExample(conn);
await DatabaseWorkloads.UnnestExample(conn);
await DatabaseWorkloadsMore.AllTypesExample(conn);
await DatabaseWorkloadsMore.AllTypesNativeExample(conn);
await DatabaseWorkloadsMore.ObjectPocoExample(conn);
await DatabaseWorkloadsMore.ArrayPocoExample(conn);
conn.Close();
});

Expand Down
80 changes: 53 additions & 27 deletions by-language/csharp-npgsql/DemoTypes.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -57,9 +57,9 @@ public class AllTypesRecord
public class DatabaseWorkloadsMore
{

public static async Task<DataTable> AllTypesExample(NpgsqlConnection conn)
public static async Task<DataTable> AllTypesNativeExample(NpgsqlConnection conn)
{
Console.WriteLine("Running AllTypesExample");
Console.WriteLine("Running AllTypesNativeExample");

// Submit DDL, create database schema.
await using (var cmd = new NpgsqlCommand("DROP TABLE IF EXISTS testdrive.example", conn))
Expand Down Expand Up @@ -154,11 +154,8 @@ INSERT INTO testdrive.example (
cmd.Parameters.AddWithValue("timestamp_tz", "1970-01-02T00:00:00+01:00");
cmd.Parameters.AddWithValue("timestamp_notz", "1970-01-02T00:00:00");
cmd.Parameters.AddWithValue("ip", "127.0.0.1");
cmd.Parameters.AddWithValue("array", new List<string>{"foo", "bar"});
// FIXME: System.NotSupportedException: Cannot resolve 'hstore' to a fully qualified datatype name. The datatype was not found in the current database info.
// https://github.com/crate/zk/issues/26
// cmd.Parameters.AddWithValue("object", new Dictionary<string, string>(){{"foo", "bar"}});
cmd.Parameters.AddWithValue("object", """{"foo": "bar"}""");
cmd.Parameters.AddWithValue("array", NpgsqlDbType.Json, new List<string>{"foo", "bar"});
cmd.Parameters.AddWithValue("object", NpgsqlDbType.Json, new Dictionary<string, string>{{"foo", "bar"}});
cmd.Parameters.AddWithValue("geopoint", new List<double>{85.43, 66.23});
// TODO: Check if `GEO_SHAPE` types can be represented by real .NET or Npgsql data types.
cmd.Parameters.AddWithValue("geoshape", "POLYGON ((5 5, 10 5, 10 10, 5 10, 5 5))");
Expand All @@ -185,20 +182,23 @@ INSERT INTO testdrive.example (

}

public static async Task<DataTable> ContainerTypesExample(NpgsqlConnection conn)
public static async Task ProvisionPoco(NpgsqlConnection conn)
{
Console.WriteLine("Running AllTypesExample");
/***
* Verify Npgsql POCO mapping with CrateDB.
* https://www.npgsql.org/doc/types/json.html#poco-mapping
*/
Console.WriteLine("Running ProvisionPoco");

// Submit DDL, create database schema.
await using (var cmd = new NpgsqlCommand("DROP TABLE IF EXISTS testdrive.container", conn))
await using (var cmd = new NpgsqlCommand("DROP TABLE IF EXISTS testdrive.poco", conn))
{
cmd.ExecuteNonQuery();
}

await using (var cmd = new NpgsqlCommand("""
CREATE TABLE testdrive.container (
-- Container types
"array" ARRAY(STRING),
CREATE TABLE testdrive.poco (
"array" ARRAY(OBJECT(DYNAMIC)),
"object" OBJECT(DYNAMIC)
);
""", conn))
Expand All @@ -208,7 +208,7 @@ CREATE TABLE testdrive.container (

// Insert single data point.
await using (var cmd = new NpgsqlCommand("""
INSERT INTO testdrive.container (
INSERT INTO testdrive.poco (
"array",
"object"
) VALUES (
Expand All @@ -217,32 +217,58 @@ INSERT INTO testdrive.container (
);
""", conn))
{
Console.WriteLine(cmd);
// FIXME: While doing conversations with ARRAY types works natively,
// it doesn't work for OBJECT types.
// Yet, they can be submitted as STRING in JSON format.
cmd.Parameters.AddWithValue("array", new List<string>{"foo", "bar"});
cmd.Parameters.AddWithValue("object", """{"foo": "bar"}""");
cmd.Parameters.AddWithValue("object", NpgsqlDbType.Json, new BasicPoco { name = "Hotzenplotz" });
cmd.Parameters.AddWithValue("array", NpgsqlDbType.Json, new List<BasicPoco>
{
new BasicPoco { name = "Hotzenplotz" },
new BasicPoco { name = "Petrosilius", age = 42 },
});
cmd.ExecuteNonQuery();
}

// Flush data.
await using (var cmd = new NpgsqlCommand("REFRESH TABLE testdrive.container", conn))
await using (var cmd = new NpgsqlCommand("REFRESH TABLE testdrive.poco", conn))
{
cmd.ExecuteNonQuery();
}

}

public static async Task<BasicPoco> ObjectPocoExample(NpgsqlConnection conn)
{
Console.WriteLine("Running ObjectPocoExample");

// Provision data.
await ProvisionPoco(conn);

// Query back data.
await using (var cmd = new NpgsqlCommand("SELECT * FROM testdrive.container", conn))
await using (var cmd = new NpgsqlCommand("SELECT * FROM testdrive.poco", conn))
await using (var reader = cmd.ExecuteReader())
{
var dataTable = new DataTable();
dataTable.Load(reader);
var payload = JsonConvert.SerializeObject(dataTable);
Console.WriteLine(payload);
return (DataTable) dataTable;
reader.Read();
var obj = reader.GetFieldValue<BasicPoco>("object");
Console.WriteLine(obj);
return obj;
}
}

public static async Task<List<BasicPoco>> ArrayPocoExample(NpgsqlConnection conn)
{
Console.WriteLine("Running ArrayPocoExample");

// Provision data.
await ProvisionPoco(conn);

// Query back data.
await using (var cmd = new NpgsqlCommand("SELECT * FROM testdrive.poco", conn))
await using (var reader = cmd.ExecuteReader())
{
reader.Read();
var obj = reader.GetFieldValue<List<BasicPoco>>("array");
Console.WriteLine(obj[0]);
Console.WriteLine(obj[1]);
return obj;
}
}

}
Expand Down
80 changes: 57 additions & 23 deletions by-language/csharp-npgsql/tests/DemoProgramTest.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,8 +21,11 @@ public DatabaseFixture()
CRATEDB_DSN = $"Host=localhost;Port=5432;Username=crate;Password=;Database=testdrive";
}
Console.WriteLine($"Connecting to {CRATEDB_DSN}\n");
Db = new NpgsqlConnection(CRATEDB_DSN);
Db.Open();

var dataSourceBuilder = new NpgsqlDataSourceBuilder(CRATEDB_DSN);
dataSourceBuilder.EnableDynamicJson();
using var dataSource = dataSourceBuilder.Build();
Db = dataSource.OpenConnection();
}

public void Dispose()
Expand Down Expand Up @@ -83,12 +86,12 @@ public async Task TestUnnestExample()
}

[Fact]
public async Task TestAllTypesExample()
public async Task TestAllTypesNativeExample()
{
var conn = fixture.Db;

// Invoke database workload.
var task = DatabaseWorkloadsMore.AllTypesExample(conn);
// Provision data.
var task = DatabaseWorkloadsMore.AllTypesNativeExample(conn);
var dt = await task.WaitAsync(TimeSpan.FromSeconds(0.5));

// Check results.
Expand All @@ -112,12 +115,18 @@ public async Task TestAllTypesExample()
Assert.Equal("127.0.0.1", row["ip"]);

// Container types
// FIXME: While doing conversations with ARRAY types works natively,
// it doesn't work for OBJECT types.
// Yet, they can be submitted as STRING in JSON format.
Assert.Equal(new List<string>{"foo", "bar"}, row["array"]);
Assert.Equal("""{"foo":"bar"}""", row["object"]);

// Note: While it works on the ingress side to communicate `Dictionary` types,
// this kind of equality check does not work on the egress side,
// presenting an error that indicates a different internal representation,
// or a programming error ;].
//
// Expected: [["foo"] = "bar"]
// Actual: {"foo":"bar"}
// Assert.Equal(new Dictionary<string, string>{{"foo", "bar"}}, row["object"]);

// Geospatial types
// TODO: Unlock native data types?
// GEO_POINT and GEO_SHAPE types can be marshalled back and forth using STRING.
Expand All @@ -135,29 +144,21 @@ public async Task TestContainerTypesExample()
{
var conn = fixture.Db;

// Invoke database workload.
var task = DatabaseWorkloadsMore.ContainerTypesExample(conn);
var dt = await task.WaitAsync(TimeSpan.FromSeconds(0.5));

// Check results.
var row = dt.Rows[0];
// FIXME: While doing conversations with ARRAY types works natively,
// it doesn't work for OBJECT types.
// Yet, they can be submitted as STRING in JSON format.
Assert.Equal(new List<string>{"foo", "bar"}, row["array"]);
Assert.Equal("""{"foo":"bar"}""", row["object"]);
// Provision data.
var task = DatabaseWorkloadsMore.AllTypesNativeExample(conn);
await task.WaitAsync(TimeSpan.FromSeconds(0.5));

// Run a special query indexing into ARRAY types.
await using (var cmd = new NpgsqlCommand("""SELECT "array[2]" AS foo FROM testdrive.container""", conn))
// Run an SQL query indexing into ARRAY types.
await using (var cmd = new NpgsqlCommand("""SELECT "array[2]" AS foo FROM testdrive.example""", conn))
await using (var reader = cmd.ExecuteReader())
{
var dataTable = new DataTable();
dataTable.Load(reader);
Assert.Equal("bar", dataTable.Rows[0]["foo"]);
}

// Run a special query indexing into OBJECT types.
await using (var cmd = new NpgsqlCommand("""SELECT "object['foo']" AS foo FROM testdrive.container""", conn))
// Run an SQL query indexing into OBJECT types.
await using (var cmd = new NpgsqlCommand("""SELECT "object['foo']" AS foo FROM testdrive.example""", conn))
await using (var reader = cmd.ExecuteReader())
{
var dataTable = new DataTable();
Expand All @@ -167,5 +168,38 @@ public async Task TestContainerTypesExample()

}

[Fact]
public async Task TestObjectPocoExample()
{
var conn = fixture.Db;

// Invoke database workload.
var task = DatabaseWorkloadsMore.ObjectPocoExample(conn);
var obj = await task.WaitAsync(TimeSpan.FromSeconds(0.5));

// Validate the outcome.
Assert.Equal(new BasicPoco { name = "Hotzenplotz" }, obj);

}

[Fact]
public async Task TestArrayPocoExample()
{
var conn = fixture.Db;

// Invoke database workload.
var task = DatabaseWorkloadsMore.ArrayPocoExample(conn);
var obj = await task.WaitAsync(TimeSpan.FromSeconds(0.5));

// Validate the outcome.
var reference = new List<BasicPoco>
{
new BasicPoco { name = "Hotzenplotz" },
new BasicPoco { name = "Petrosilius", age = 42 },
};
Assert.Equal(reference, obj);

}

}
}

0 comments on commit be0835d

Please sign in to comment.