docs: update changelog and readme

CGI-FR · Oct 17, 2023 · 66b56fb · 66b56fb
1 parent 1af78c4
commit 66b56fb
Show file tree

Hide file tree

Showing 5 changed files with 114 additions and 13 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -14,8 +14,12 @@ Types of changes
 - `Fixed` for any bug fixes.
 - `Security` in case of vulnerabilities.
 
+## [2.5.0]
+
+- `Added` command `analyse` to extract metrics from the database in YAML format.
+
 ## [2.4.0]
- 
+
 - `Added` go-ora driver for oracle in replacement of old driver (remove technical prerequisite to install Oracle Instant Client)
 
 ## [2.3.0]

diff --git a/README.md b/README.md
@@ -1,10 +1,10 @@
 ![GitHub Workflow Status](https://img.shields.io/github/actions/workflow/status/CGI-FR/LINO/ci.yml?branch=main)
- [![Go Report Card](https://goreportcard.com/badge/github.com/cgi-fr/pimo)](https://goreportcard.com/report/github.com/cgi-fr/lino)
- ![GitHub all releases](https://img.shields.io/github/downloads/CGI-FR/LINO/total)
- ![GitHub](https://img.shields.io/github/license/CGI-FR/LINO)
- ![GitHub Repo stars](https://img.shields.io/github/stars/CGI-FR/LINO)
- ![GitHub go.mod Go version](https://img.shields.io/github/go-mod/go-version/CGI-FR/LINO)
- ![GitHub release (latest by date)](https://img.shields.io/github/v/release/CGI-FR/LINO)
+[![Go Report Card](https://goreportcard.com/badge/github.com/cgi-fr/pimo)](https://goreportcard.com/report/github.com/cgi-fr/lino)
+![GitHub all releases](https://img.shields.io/github/downloads/CGI-FR/LINO/total)
+![GitHub](https://img.shields.io/github/license/CGI-FR/LINO)
+![GitHub Repo stars](https://img.shields.io/github/stars/CGI-FR/LINO)
+![GitHub go.mod Go version](https://img.shields.io/github/go-mod/go-version/CGI-FR/LINO)
+![GitHub release (latest by date)](https://img.shields.io/github/v/release/CGI-FR/LINO)
 
 # LINO : Large Input, Narrow Output
 
@@ -287,11 +287,106 @@ lino push update source --table actor <<<'{"actor_id":998,"last_name":"CHASE","_
 
 The `__usingpk__` field can also be used with an ingress descriptor at any level in the data. The name of this field can be changed to another value with the `--using-pk-field` flag.
 
-### Interaction with other tools
+## Analyse
+
+Use the `lino analyse <data_connector_alias>` command to extract metrics from the database in YAML format.
+
+Only tables and columns explicitly listed in the tables.yaml file will be analysed.
+
+Example result :
+
+```yaml
+database: source
+tables:
+    - name: first_name
+      columns:
+        - name: actor
+          type: string
+          concept: ""
+          constraint: []
+          confidential: null
+          mainMetric:
+            count: 200
+            empty: 0
+            unique: 128
+            sample:
+                - WALTER
+                - MAE
+                - LAURENCE
+                - GREG
+                - ALEC
+          stringMetric:
+            mostFrequentLen:
+                - length: 4
+                  freq: 0.235
+                  sample:
+                    - GARY
+                    - ALAN
+                    - ADAM
+                    - JEFF
+                    - GINA
+                - length: 5
+                  freq: 0.215
+                  sample:
+                    - REESE
+                    - MILLA
+                    - SALMA
+                    - RALPH
+                    - SUSAN
+                - length: 7
+                  freq: 0.16
+                  sample:
+                    - OLYMPIA
+                    - KIRSTEN
+                    - MATTHEW
+                    - RICHARD
+                    - KIRSTEN
+                - length: 6
+                  freq: 0.14
+                  sample:
+                    - WHOOPI
+                    - WALTER
+                    - SANDRA
+                    - WHOOPI
+                    - JOHNNY
+                - length: 3
+                  freq: 0.12
+                  sample:
+                    - BOB
+                    - BEN
+                    - KIM
+                    - BOB
+                    - TOM
+            leastFrequentLen:
+                - length: 11
+                  freq: 0.01
+                  sample:
+                    - CHRISTOPHER
+                - length: 9
+                  freq: 0.02
+                  sample:
+                    - CHRISTIAN
+                    - SYLVESTER
+                - length: 2
+                  freq: 0.02
+                  sample:
+                    - AL
+                    - ED
+                - length: 8
+                  freq: 0.08
+                  sample:
+                    - JULIANNE
+                    - LAURENCE
+                    - JULIANNE
+                    - SCARLETT
+                    - LAURENCE
+```
+
+## Interaction with other tools
 
 **LINO** respect the UNIX philosophy and use standards input an output to share data with others tools.
 
-## MongoDB storage
+### MongoDB storage
 
 Data set could be store in mongoDB easily with the `mongoimport` tool:
 
@@ -304,11 +399,12 @@ and reload later to a database :
 ```bash
 $ mongoexport --db myproject --collection customer | lino push customer --jdbc jdbc:oracle:thin:scott/tiger@target:1721:xe
 ```
-## Miller `mlr`
+
+### Miller `mlr`
 
 `mlr` tool can be used to format json lines into another tabular format (csv, markdown table, ...).
 
-## jq
+### jq
 
 `jq` tool can be piped with the **LINO** output to prettify it.
 

diff --git a/internal/app/analyse/cli.go b/internal/app/analyse/cli.go
@@ -68,15 +68,13 @@ func NewCommand(fullName string, err *os.File, out *os.File, in *os.File) *cobra
 			}
 
 			extractor, e1 := getExtractor(dataConnector, out)
-
 			if e1 != nil {
 				fmt.Fprintln(err, e1.Error())
 				os.Exit(1)
 			}
 
 			analyser := analyserFactory.New(out)
 			e2 := analyse.Do(dataSource, extractor, analyser)
-
 			if e2 != nil {
 				fmt.Fprintf(err, "analyse failed '%s'", dataConnector)
 				fmt.Fprintln(err)

diff --git a/internal/infra/analyse/sql_datasource.go b/internal/infra/analyse/sql_datasource.go
@@ -47,6 +47,8 @@ type SQLDataSource struct {
 func (ds *SQLDataSource) ExtractValues(tableName string, columnName string) ([]interface{}, error) {
 	result := []interface{}{}
 
+	log.Trace().Str("tablename", tableName).Str("columnname", columnName).Msg("extract values")
+
 	err := ds.Open()
 	if err != nil {
 		log.Error().Err(err).Msg("Connection failed")

diff --git a/pkg/analyse/driver.go b/pkg/analyse/driver.go
@@ -66,6 +66,7 @@ func (ci *ColumnIterator) Next() bool {
 			return true
 		}
 	}
+
 	return false
 }
-Original file line number
+Diff line change
@@ Expand Up / @@ -66,6 +66,7 @@ func (ci *ColumnIterator) Next() bool { @@
     			return true
     		}
     	}
     	return false
     }
@@ Expand Down @@