Skip to content

Commit

Permalink
feat: export presence (#328)
Browse files Browse the repository at this point in the history 
* feat(presence): add venom test

* feat(presence): implement flag in row export

* feat(presence): fix venom test

* feat(presence): wip! do not extract value

* feat(presence): wip! do not extract value

* feat(presence): do not extract value

* feat(export): fix presence for oracle
  • Loading branch information
@adrienaury
adrienaury authored Nov 21, 2024
1 parent 23f2239 commit 57cffdb
Show file tree
Hide file tree
Showing 12 changed files with 162 additions and 57 deletions.
1 change: 1 addition & 0 deletions CHANGELOG.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ Types of changes
- `Added` possibility to filter columns via `select` property in ingress descriptors
- `Added` commands `set-parent-select` and `set-child-select` to `lino id`
- `Added` `import: no` option for columns in `tables.yaml` configuration
- `Added` `export: presence` option for columns in `tables.yaml` configuration

## [3.0.2]

Expand Down
2 changes: 1 addition & 1 deletion internal/infra/analyse/sql_datasource.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -134,7 +134,7 @@ func (ds *SQLDataSource) Open() error {
// Get WHERE Clause query
sqlWhere, values := commonsql.GetWhereSQLAndValues(map[string]any{}, ds.where, ds.dialect)

sql := ds.dialect.Select(ds.table, ds.schema, sqlWhere, false, ds.column)
sql := ds.dialect.Select(ds.table, ds.schema, sqlWhere, false, commonsql.ColumnExportDefinition{Name: ds.column})

// If log level is more than debug level, this function will log all SQL Query
commonsql.LogSQLQuery(sql, values, ds.dialect)
Expand Down
17 changes: 15 additions & 2 deletions internal/infra/commonsql/dialect.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,19 @@ import (
"github.com/rs/zerolog/log"
)

type ColumnExportDefinition struct {
Name string
OnlyPresence bool
}

func Names(columns []ColumnExportDefinition) []string {
us := make([]string, len(columns))
for i := range columns {
us[i] = columns[i].Name
}
return us
}

type Dialect interface {
// Placeholder format variable in query
Placeholder(int) string
Expand All @@ -35,9 +48,9 @@ type Dialect interface {
// Where clause
Where(string) string
// Select clause
Select(tableName string, schemaName string, where string, distinct bool, columns ...string) string
Select(tableName string, schemaName string, where string, distinct bool, columns ...ColumnExportDefinition) string
// SelectLimit clause
SelectLimit(tableName string, schemaName string, where string, distinct bool, limit uint, columns ...string) string
SelectLimit(tableName string, schemaName string, where string, distinct bool, limit uint, columns ...ColumnExportDefinition) string
// Quote identifier
Quote(id string) string

Expand Down
32 changes: 22 additions & 10 deletions internal/infra/commonsql/dialect_db2.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,7 @@ func (db2 Db2Dialect) Where(where string) string {
}

// Select clause
func (db2 Db2Dialect) Select(tableName string, schemaName string, where string, distinct bool, columns ...string) string {
func (db2 Db2Dialect) Select(tableName string, schemaName string, where string, distinct bool, columns ...ColumnExportDefinition) string {
var query strings.Builder

query.WriteString("SELECT ")
Expand All @@ -62,11 +62,15 @@ func (db2 Db2Dialect) Select(tableName string, schemaName string, where string,
query.WriteString("DISTINCT ")
}

if len(columns) > 0 {
for i := range columns {
columns[i] = db2.Quote(columns[i])
if names := Names(columns); len(names) > 0 {
for i := range names {
if columns[i].OnlyPresence {
names[i] = db2.selectPresence(names[i])
} else {
names[i] = db2.Quote(names[i])
}
}
query.WriteString(strings.Join(columns, ", "))
query.WriteString(strings.Join(names, ", "))
} else {
query.WriteRune('*')
}
Expand All @@ -80,7 +84,7 @@ func (db2 Db2Dialect) Select(tableName string, schemaName string, where string,
}

// SelectLimit clause
func (db2 Db2Dialect) SelectLimit(tableName string, schemaName string, where string, distinct bool, limit uint, columns ...string) string {
func (db2 Db2Dialect) SelectLimit(tableName string, schemaName string, where string, distinct bool, limit uint, columns ...ColumnExportDefinition) string {
var query strings.Builder

query.WriteString("SELECT ")
Expand All @@ -89,11 +93,15 @@ func (db2 Db2Dialect) SelectLimit(tableName string, schemaName string, where str
query.WriteString("DISTINCT ")
}

if len(columns) > 0 {
for i := range columns {
columns[i] = db2.Quote(columns[i])
if names := Names(columns); len(names) > 0 {
for i := range names {
if columns[i].OnlyPresence {
names[i] = db2.selectPresence(names[i])
} else {
names[i] = db2.Quote(names[i])
}
}
query.WriteString(strings.Join(columns, ", "))
query.WriteString(strings.Join(names, ", "))
} else {
query.WriteRune('*')
}
Expand Down Expand Up @@ -123,3 +131,7 @@ func (db2 Db2Dialect) Quote(id string) string {
func (db2 Db2Dialect) CreateSelect(sel string, where string, limit string, columns string, from string) string {
return fmt.Sprintf("%s %s %s %s %s", sel, columns, from, where, limit)
}

func (db2 Db2Dialect) selectPresence(column string) string {
return fmt.Sprintf("CASE WHEN %s IS NOT NULL THEN 1 ELSE NULL END AS %s", db2.Quote(column), db2.Quote(column))
}
32 changes: 22 additions & 10 deletions internal/infra/commonsql/dialect_mariadb.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,7 @@ func (pd MariadbDialect) Where(where string) string {
}

// Select clause
func (pd MariadbDialect) Select(tableName string, schemaName string, where string, distinct bool, columns ...string) string {
func (pd MariadbDialect) Select(tableName string, schemaName string, where string, distinct bool, columns ...ColumnExportDefinition) string {
var query strings.Builder

query.WriteString("SELECT ")
Expand All @@ -62,11 +62,15 @@ func (pd MariadbDialect) Select(tableName string, schemaName string, where strin
query.WriteString("DISTINCT ")
}

if len(columns) > 0 {
for i := range columns {
columns[i] = pd.Quote(columns[i])
if names := Names(columns); len(names) > 0 {
for i := range names {
if columns[i].OnlyPresence {
names[i] = pd.selectPresence(names[i])
} else {
names[i] = pd.Quote(names[i])
}
}
query.WriteString(strings.Join(columns, ", "))
query.WriteString(strings.Join(names, ", "))
} else {
query.WriteRune('*')
}
Expand All @@ -80,7 +84,7 @@ func (pd MariadbDialect) Select(tableName string, schemaName string, where strin
}

// SelectLimit clause
func (pd MariadbDialect) SelectLimit(tableName string, schemaName string, where string, distinct bool, limit uint, columns ...string) string {
func (pd MariadbDialect) SelectLimit(tableName string, schemaName string, where string, distinct bool, limit uint, columns ...ColumnExportDefinition) string {
var query strings.Builder

query.WriteString("SELECT ")
Expand All @@ -89,11 +93,15 @@ func (pd MariadbDialect) SelectLimit(tableName string, schemaName string, where
query.WriteString("DISTINCT ")
}

if len(columns) > 0 {
for i := range columns {
columns[i] = pd.Quote(columns[i])
if names := Names(columns); len(names) > 0 {
for i := range names {
if columns[i].OnlyPresence {
names[i] = pd.selectPresence(names[i])
} else {
names[i] = pd.Quote(names[i])
}
}
query.WriteString(strings.Join(columns, ", "))
query.WriteString(strings.Join(names, ", "))
} else {
query.WriteRune('*')
}
Expand Down Expand Up @@ -123,3 +131,7 @@ func (sd MariadbDialect) Quote(id string) string {
func (sd MariadbDialect) CreateSelect(sel string, where string, limit string, columns string, from string) string {
return fmt.Sprintf("%s %s %s %s %s", sel, columns, from, where, limit)
}

func (sd MariadbDialect) selectPresence(column string) string {
return fmt.Sprintf("CASE WHEN %s IS NOT NULL THEN 'TRUE' ELSE NULL END AS %s", sd.Quote(column), sd.Quote(column))
}
32 changes: 22 additions & 10 deletions internal/infra/commonsql/dialect_oracle.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,7 @@ func (od OracleDialect) Where(where string) string {
}

// Select clause
func (od OracleDialect) Select(tableName string, schemaName string, where string, distinct bool, columns ...string) string {
func (od OracleDialect) Select(tableName string, schemaName string, where string, distinct bool, columns ...ColumnExportDefinition) string {
var query strings.Builder

query.WriteString("SELECT ")
Expand All @@ -62,11 +62,15 @@ func (od OracleDialect) Select(tableName string, schemaName string, where string
query.WriteString("DISTINCT ")
}

if len(columns) > 0 {
for i := range columns {
columns[i] = od.Quote(columns[i])
if names := Names(columns); len(names) > 0 {
for i := range names {
if columns[i].OnlyPresence {
names[i] = od.selectPresence(names[i])
} else {
names[i] = od.Quote(names[i])
}
}
query.WriteString(strings.Join(columns, ", "))
query.WriteString(strings.Join(names, ", "))
} else {
query.WriteRune('*')
}
Expand All @@ -80,7 +84,7 @@ func (od OracleDialect) Select(tableName string, schemaName string, where string
}

// SelectLimit clause
func (od OracleDialect) SelectLimit(tableName string, schemaName string, where string, distinct bool, limit uint, columns ...string) string {
func (od OracleDialect) SelectLimit(tableName string, schemaName string, where string, distinct bool, limit uint, columns ...ColumnExportDefinition) string {
var query strings.Builder

query.WriteString("SELECT ")
Expand All @@ -89,11 +93,15 @@ func (od OracleDialect) SelectLimit(tableName string, schemaName string, where s
query.WriteString("DISTINCT ")
}

if len(columns) > 0 {
for i := range columns {
columns[i] = od.Quote(columns[i])
if names := Names(columns); len(names) > 0 {
for i := range names {
if columns[i].OnlyPresence {
names[i] = od.selectPresence(names[i])
} else {
names[i] = od.Quote(names[i])
}
}
query.WriteString(strings.Join(columns, ", "))
query.WriteString(strings.Join(names, ", "))
} else {
query.WriteRune('*')
}
Expand Down Expand Up @@ -123,3 +131,7 @@ func (od OracleDialect) Quote(id string) string {
func (od OracleDialect) CreateSelect(sel string, where string, limit string, columns string, from string) string {
return fmt.Sprintf("%s %s %s %s %s", sel, columns, from, where, limit)
}

func (od OracleDialect) selectPresence(column string) string {
return fmt.Sprintf("CASE WHEN %s IS NOT NULL THEN 'TRUE' ELSE NULL END AS %s", od.Quote(column), od.Quote(column))
}
30 changes: 21 additions & 9 deletions internal/infra/commonsql/dialect_postgres.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,7 @@ func (pgd PostgresDialect) Where(where string) string {
}

// Select clause
func (pgd PostgresDialect) Select(tableName string, schemaName string, where string, distinct bool, columns ...string) string {
func (pgd PostgresDialect) Select(tableName string, schemaName string, where string, distinct bool, columns ...ColumnExportDefinition) string {
var query strings.Builder

query.WriteString("SELECT ")
Expand All @@ -62,11 +62,15 @@ func (pgd PostgresDialect) Select(tableName string, schemaName string, where str
query.WriteString("DISTINCT ")
}

if len(columns) > 0 {
if names := Names(columns); len(names) > 0 {
for i := range columns {
columns[i] = pgd.Quote(columns[i])
if columns[i].OnlyPresence {
names[i] = pgd.selectPresence(names[i])
} else {
names[i] = pgd.Quote(names[i])
}
}
query.WriteString(strings.Join(columns, ", "))
query.WriteString(strings.Join(names, ", "))
} else {
query.WriteRune('*')
}
Expand All @@ -80,7 +84,7 @@ func (pgd PostgresDialect) Select(tableName string, schemaName string, where str
}

// SelectLimit clause
func (pgd PostgresDialect) SelectLimit(tableName string, schemaName string, where string, distinct bool, limit uint, columns ...string) string {
func (pgd PostgresDialect) SelectLimit(tableName string, schemaName string, where string, distinct bool, limit uint, columns ...ColumnExportDefinition) string {
var query strings.Builder

query.WriteString("SELECT ")
Expand All @@ -89,11 +93,15 @@ func (pgd PostgresDialect) SelectLimit(tableName string, schemaName string, wher
query.WriteString("DISTINCT ")
}

if len(columns) > 0 {
for i := range columns {
columns[i] = pgd.Quote(columns[i])
if names := Names(columns); len(names) > 0 {
for i := range names {
if columns[i].OnlyPresence {
names[i] = pgd.selectPresence(names[i])
} else {
names[i] = pgd.Quote(names[i])
}
}
query.WriteString(strings.Join(columns, ", "))
query.WriteString(strings.Join(names, ", "))
} else {
query.WriteRune('*')
}
Expand Down Expand Up @@ -123,3 +131,7 @@ func (pgd PostgresDialect) Quote(id string) string {
func (pgd PostgresDialect) CreateSelect(sel string, where string, limit string, columns string, from string) string {
return fmt.Sprintf("%s %s %s %s %s", sel, columns, from, where, limit)
}

func (pgd PostgresDialect) selectPresence(column string) string {
return fmt.Sprintf("CASE WHEN (%s IS NOT NULL) THEN TRUE ELSE NULL END AS %s", pgd.Quote(column), pgd.Quote(column))
}
28 changes: 20 additions & 8 deletions internal/infra/commonsql/dialect_sqlserver.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,7 @@ func (sd SQLServerDialect) Where(where string) string {
}

// Select clause
func (sd SQLServerDialect) Select(tableName string, schemaName string, where string, distinct bool, columns ...string) string {
func (sd SQLServerDialect) Select(tableName string, schemaName string, where string, distinct bool, columns ...ColumnExportDefinition) string {
var query strings.Builder

query.WriteString("SELECT ")
Expand All @@ -63,11 +63,15 @@ func (sd SQLServerDialect) Select(tableName string, schemaName string, where str
query.WriteString("DISTINCT ")
}

if len(columns) > 0 {
if names := Names(columns); len(names) > 0 {
for i := range columns {
columns[i] = sd.Quote(columns[i])
if columns[i].OnlyPresence {
names[i] = sd.selectPresence(names[i])
} else {
names[i] = sd.Quote(names[i])
}
}
query.WriteString(strings.Join(columns, ", "))
query.WriteString(strings.Join(names, ", "))
} else {
query.WriteRune('*')
}
Expand All @@ -81,7 +85,7 @@ func (sd SQLServerDialect) Select(tableName string, schemaName string, where str
}

// SelectLimit clause
func (sd SQLServerDialect) SelectLimit(tableName string, schemaName string, where string, distinct bool, limit uint, columns ...string) string {
func (sd SQLServerDialect) SelectLimit(tableName string, schemaName string, where string, distinct bool, limit uint, columns ...ColumnExportDefinition) string {
var query strings.Builder

query.WriteString("SELECT ")
Expand All @@ -93,11 +97,15 @@ func (sd SQLServerDialect) SelectLimit(tableName string, schemaName string, wher
query.WriteString(sd.Limit(limit))
query.WriteRune(' ')

if len(columns) > 0 {
if names := Names(columns); len(names) > 0 {
for i := range columns {
columns[i] = sd.Quote(columns[i])
if columns[i].OnlyPresence {
names[i] = sd.selectPresence(names[i])
} else {
names[i] = sd.Quote(names[i])
}
}
query.WriteString(strings.Join(columns, ", "))
query.WriteString(strings.Join(names, ", "))
} else {
query.WriteRune('*')
}
Expand Down Expand Up @@ -125,3 +133,7 @@ func (sd SQLServerDialect) Quote(id string) string {
func (sd SQLServerDialect) CreateSelect(sel string, where string, limit string, columns string, from string) string {
return fmt.Sprintf("%s %s %s %s %s", sel, limit, columns, from, where)
}

func (sd SQLServerDialect) selectPresence(column string) string {
return fmt.Sprintf("CASE WHEN %s IS NOT NULL THEN 1 ELSE NULL END AS %s", sd.Quote(column), sd.Quote(column))
}
Loading

0 comments on commit 57cffdb

Please sign in to comment.