diff --git a/CHANGELOG.md b/CHANGELOG.md index 220737f1..aa667cc8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,8 +14,12 @@ Types of changes - `Fixed` for any bug fixes. - `Security` in case of vulnerabilities. +## [2.5.0] + +- `Added` command `analyse` to extract metrics from the database in YAML format. + ## [2.4.0] - + - `Added` go-ora driver for oracle in replacement of old driver (remove technical prerequisite to install Oracle Instant Client) ## [2.3.0] diff --git a/README.md b/README.md index 9fd6ba3d..ad3996db 100755 --- a/README.md +++ b/README.md @@ -1,10 +1,10 @@ ![GitHub Workflow Status](https://img.shields.io/github/actions/workflow/status/CGI-FR/LINO/ci.yml?branch=main) - [![Go Report Card](https://goreportcard.com/badge/github.com/cgi-fr/pimo)](https://goreportcard.com/report/github.com/cgi-fr/lino) - ![GitHub all releases](https://img.shields.io/github/downloads/CGI-FR/LINO/total) - ![GitHub](https://img.shields.io/github/license/CGI-FR/LINO) - ![GitHub Repo stars](https://img.shields.io/github/stars/CGI-FR/LINO) - ![GitHub go.mod Go version](https://img.shields.io/github/go-mod/go-version/CGI-FR/LINO) - ![GitHub release (latest by date)](https://img.shields.io/github/v/release/CGI-FR/LINO) +[![Go Report Card](https://goreportcard.com/badge/github.com/cgi-fr/pimo)](https://goreportcard.com/report/github.com/cgi-fr/lino) +![GitHub all releases](https://img.shields.io/github/downloads/CGI-FR/LINO/total) +![GitHub](https://img.shields.io/github/license/CGI-FR/LINO) +![GitHub Repo stars](https://img.shields.io/github/stars/CGI-FR/LINO) +![GitHub go.mod Go version](https://img.shields.io/github/go-mod/go-version/CGI-FR/LINO) +![GitHub release (latest by date)](https://img.shields.io/github/v/release/CGI-FR/LINO) # LINO : Large Input, Narrow Output @@ -287,11 +287,106 @@ lino push update source --table actor <<<'{"actor_id":998,"last_name":"CHASE","_ The `__usingpk__` field can also be used with an ingress descriptor at any level in the data. The name of this field can be changed to another value with the `--using-pk-field` flag. -### Interaction with other tools +## Analyse + +Use the `lino analyse ` command to extract metrics from the database in YAML format. + +Only tables and columns explicitly listed in the tables.yaml file will be analysed. + +Example result : + +```yaml +database: source +tables: + - name: first_name + columns: + - name: actor + type: string + concept: "" + constraint: [] + confidential: null + mainMetric: + count: 200 + empty: 0 + unique: 128 + sample: + - WALTER + - MAE + - LAURENCE + - GREG + - ALEC + stringMetric: + mostFrequentLen: + - length: 4 + freq: 0.235 + sample: + - GARY + - ALAN + - ADAM + - JEFF + - GINA + - length: 5 + freq: 0.215 + sample: + - REESE + - MILLA + - SALMA + - RALPH + - SUSAN + - length: 7 + freq: 0.16 + sample: + - OLYMPIA + - KIRSTEN + - MATTHEW + - RICHARD + - KIRSTEN + - length: 6 + freq: 0.14 + sample: + - WHOOPI + - WALTER + - SANDRA + - WHOOPI + - JOHNNY + - length: 3 + freq: 0.12 + sample: + - BOB + - BEN + - KIM + - BOB + - TOM + leastFrequentLen: + - length: 11 + freq: 0.01 + sample: + - CHRISTOPHER + - length: 9 + freq: 0.02 + sample: + - CHRISTIAN + - SYLVESTER + - length: 2 + freq: 0.02 + sample: + - AL + - ED + - length: 8 + freq: 0.08 + sample: + - JULIANNE + - LAURENCE + - JULIANNE + - SCARLETT + - LAURENCE +``` + +## Interaction with other tools **LINO** respect the UNIX philosophy and use standards input an output to share data with others tools. -## MongoDB storage +### MongoDB storage Data set could be store in mongoDB easily with the `mongoimport` tool: @@ -304,11 +399,12 @@ and reload later to a database : ```bash $ mongoexport --db myproject --collection customer | lino push customer --jdbc jdbc:oracle:thin:scott/tiger@target:1721:xe ``` -## Miller `mlr` + +### Miller `mlr` `mlr` tool can be used to format json lines into another tabular format (csv, markdown table, ...). -## jq +### jq `jq` tool can be piped with the **LINO** output to prettify it. diff --git a/cmd/lino/dep_analyse.go b/cmd/lino/dep_analyse.go new file mode 100644 index 00000000..49727854 --- /dev/null +++ b/cmd/lino/dep_analyse.go @@ -0,0 +1,37 @@ +// Copyright (C) 2021 CGI France +// +// This file is part of LINO. +// +// LINO is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// LINO is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with LINO. If not, see . + +package main + +import ( + infra "github.com/cgi-fr/lino/internal/infra/analyse" + domain "github.com/cgi-fr/lino/pkg/analyse" +) + +func analyseDataSourceFactory() map[string]domain.ExtractorFactory { + return map[string]domain.ExtractorFactory{ + "postgres": infra.NewSQLExtractorFactory(), + "godror": infra.NewSQLExtractorFactory(), + "godror-raw": infra.NewSQLExtractorFactory(), + "mysql": infra.NewSQLExtractorFactory(), + "db2": infra.NewSQLExtractorFactory(), + } +} + +func analyserFactory() domain.AnalyserFactory { + return infra.RimoAnalyserFactory{} +} diff --git a/cmd/lino/main.go b/cmd/lino/main.go index 92fcb74e..ba3b8db5 100755 --- a/cmd/lino/main.go +++ b/cmd/lino/main.go @@ -27,6 +27,7 @@ import ( "text/template" over "github.com/adrienaury/zeromdc" + "github.com/cgi-fr/lino/internal/app/analyse" "github.com/cgi-fr/lino/internal/app/dataconnector" "github.com/cgi-fr/lino/internal/app/http" "github.com/cgi-fr/lino/internal/app/id" @@ -158,6 +159,7 @@ func init() { rootCmd.AddCommand(pull.NewCommand("lino", os.Stderr, os.Stdout, os.Stdin)) rootCmd.AddCommand(push.NewCommand("lino", os.Stderr, os.Stdout, os.Stdin)) rootCmd.AddCommand(http.NewCommand("lino", os.Stderr, os.Stdout, os.Stdin)) + rootCmd.AddCommand(analyse.NewCommand("lino", os.Stderr, os.Stdout, os.Stdin)) } func initConfig() { @@ -202,6 +204,7 @@ func initConfig() { zerolog.SetGlobalLevel(zerolog.Disabled) } + analyse.Inject(tableStorage(), dataconnectorStorage(), analyseDataSourceFactory(), analyserFactory()) dataconnector.Inject(dataconnectorStorage(), dataPingerFactory()) relation.Inject(dataconnectorStorage(), relationStorage(), relationExtractorFactory()) table.Inject(dataconnectorStorage(), tableStorage(), tableExtractorFactory()) diff --git a/go.mod b/go.mod index c4070eac..7dcfff30 100644 --- a/go.mod +++ b/go.mod @@ -6,6 +6,7 @@ require ( github.com/adrienaury/zeromdc v0.0.0-20221116212822-6a366c26ee61 github.com/awalterschulze/gographviz v2.0.3+incompatible github.com/cgi-fr/jsonline v0.5.0 + github.com/cgi-fr/rimo v0.2.0 github.com/docker/docker-credential-helpers v0.8.0 github.com/go-sql-driver/mysql v1.7.1 github.com/gorilla/mux v1.8.0 @@ -27,17 +28,22 @@ require ( require ( github.com/davecgh/go-spew v1.1.1 // indirect - github.com/google/go-cmp v0.5.8 // indirect + github.com/google/go-cmp v0.5.9 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect + github.com/hexops/valast v1.4.4 // indirect + github.com/iancoleman/orderedmap v0.3.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/invopop/jsonschema v0.7.0 // indirect github.com/klauspost/compress v1.16.0 // indirect - github.com/kr/pretty v0.3.0 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-sqlite3 v2.0.3+incompatible // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/spf13/pflag v1.0.5 // indirect github.com/stretchr/objx v0.5.0 // indirect + golang.org/x/mod v0.12.0 // indirect golang.org/x/sys v0.12.0 // indirect + golang.org/x/tools v0.12.0 // indirect google.golang.org/protobuf v1.28.1 // indirect gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect + mvdan.cc/gofumpt v0.5.0 // indirect ) diff --git a/go.sum b/go.sum index 66f3ccc3..6e4e0c22 100644 --- a/go.sum +++ b/go.sum @@ -62,6 +62,8 @@ github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cgi-fr/jsonline v0.5.0 h1:UFLDJauppXLXWlUXFgAKqlZaKt0g1gRq3tLJUj95ohY= github.com/cgi-fr/jsonline v0.5.0/go.mod h1:2eo1zPtPXeGiGCEI+Y2m0GYlQgmRikVeQOwLwOZtWXQ= +github.com/cgi-fr/rimo v0.2.0 h1:dzX8Xscmj/zPQjVdz8TfdH7a/g0y6hKlrOsNPLk/28Y= +github.com/cgi-fr/rimo v0.2.0/go.mod h1:Q/eTKXMKJ3m5v8EdHSbVA0LrBykepBYYxJjclkMAJr8= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= @@ -76,7 +78,6 @@ github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSV github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= -github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -92,6 +93,7 @@ github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.m github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= +github.com/frankban/quicktest v1.14.4 h1:g2rn0vABPOOXmZUj+vbmUp0lPoXEMuhTpIluN0XL9UY= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/fsnotify/fsnotify v1.5.1/go.mod h1:T3375wBYaZdLLcVNkcVbzGHY7f1l/uK5T5Ai1i3InKU= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= @@ -166,8 +168,8 @@ github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg= -github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= @@ -230,6 +232,13 @@ github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2p github.com/hashicorp/memberlist v0.2.2/go.mod h1:MS2lj3INKhZjWNqd3N0m3J+Jxf3DAOnAH9VT3Sh9MUE= github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= github.com/hashicorp/serf v0.9.5/go.mod h1:UWDWwZeL5cuWDJdl0C6wrvrUwEqtQ4ZKBKKENpqIUyk= +github.com/hexops/autogold v0.8.1 h1:wvyd/bAJ+Dy+DcE09BoLk6r4Fa5R5W+O+GUzmR985WM= +github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM= +github.com/hexops/valast v1.4.4 h1:rETyycw+/L2ZVJHHNxEBgh8KUn+87WugH9MxcEv9PGs= +github.com/hexops/valast v1.4.4/go.mod h1:Jcy1pNH7LNraVaAZDLyv21hHg2WBv9Nf9FL6fGxU7o4= +github.com/iancoleman/orderedmap v0.0.0-20190318233801-ac98e3ecb4b0/go.mod h1:N0Wam8K1arqPXNWjMo21EXnBPOPp36vB07FNRdD2geA= +github.com/iancoleman/orderedmap v0.3.0 h1:5cbR2grmZR/DiVt+VJopEhtVs9YGInGIxAoMJn+Ichc= +github.com/iancoleman/orderedmap v0.3.0/go.mod h1:XuLcCUkdL5owUCQeF2Ue9uuw1EptkJDkXXS7VoV7XGE= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ibmdb/go_ibm_db v0.4.4 h1:zeTY8nc7sIWZDHVEPTiI2IC9z8qJz1BAYGEneHdVDgg= @@ -237,6 +246,8 @@ github.com/ibmdb/go_ibm_db v0.4.4/go.mod h1:nl5aUh1IzBVExcqYXaZLApaq8RUvTEph3VP4 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/invopop/jsonschema v0.7.0 h1:2vgQcBz1n256N+FpX3Jq7Y17AjYt46Ig3zIWyy770So= +github.com/invopop/jsonschema v0.7.0/go.mod h1:O9uiLokuu0+MGFlyiaqtWxwqJm41/+8Nj0lD7A36YH0= github.com/jmoiron/sqlx v1.3.5 h1:vFFPA71p1o5gAeqtEAwLU4dnX2napprKtHr7PYIcN3g= github.com/jmoiron/sqlx v1.3.5/go.mod h1:nRVWtLre0KfCLJvgxzCsLVMogSvQ1zNJtpYr2Ccp0mQ= github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= @@ -254,12 +265,10 @@ github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= -github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= -github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= -github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/leodido/go-urn v1.2.0 h1:hpXL4XnriNwQ/ABnpepYM/1vCLWNDfUNts8dX3xTG6Y= github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII= github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= @@ -316,8 +325,7 @@ github.com/posener/complete v1.2.3/go.mod h1:WZIdtGGp+qx0sLrYKtIRAruyNpv6hFCicSg github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/rogpeppe/go-internal v1.6.1 h1:/FiVV8dS/e+YqF2JvO3yXRFbBLTIuSDkuC7aBOAvL+k= -github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= +github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= github.com/rs/xid v1.3.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= @@ -353,6 +361,7 @@ github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.3.1-0.20190311161405-34c6fa2dc709/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= @@ -432,6 +441,8 @@ golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.12.0 h1:rmsUpXtvNzj340zd98LZ4KntptpfRHwpFOHG188oHXc= +golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -497,6 +508,7 @@ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -629,6 +641,8 @@ golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.12.0 h1:YW6HUoUmYBpwSgyaGaZq1fHjrBjX1rlpZ54T6mu2kss= +golang.org/x/tools v0.12.0/go.mod h1:Sc0INKfu04TlqNoRA1hgpFZbhYXHPr4V5DzpSBTPqQM= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -785,6 +799,8 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +mvdan.cc/gofumpt v0.5.0 h1:0EQ+Z56k8tXjj/6TQD25BFNKQXpCvT0rnansIc7Ug5E= +mvdan.cc/gofumpt v0.5.0/go.mod h1:HBeVDtMKRZpXyxFciAirzdKklDlGu8aAy1wEbH5Y9js= nhooyr.io/websocket v1.8.7 h1:usjR2uOr/zjjkVMy0lW+PPohFok7PCow5sDjLgX4P4g= nhooyr.io/websocket v1.8.7/go.mod h1:B70DZP8IakI65RVQ51MsWP/8jndNma26DVA/nFSCgW0= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= diff --git a/internal/app/analyse/cli.go b/internal/app/analyse/cli.go new file mode 100644 index 00000000..a4d18855 --- /dev/null +++ b/internal/app/analyse/cli.go @@ -0,0 +1,126 @@ +// Copyright (C) 2021 CGI France +// +// This file is part of LINO. +// +// LINO is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// LINO is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with LINO. If not, see . + +package analyse + +import ( + "fmt" + "io" + "os" + + "github.com/cgi-fr/lino/internal/app/urlbuilder" + ianalyse "github.com/cgi-fr/lino/internal/infra/analyse" + "github.com/cgi-fr/lino/pkg/analyse" + "github.com/cgi-fr/lino/pkg/dataconnector" + "github.com/cgi-fr/lino/pkg/table" + "github.com/spf13/cobra" +) + +var ( + tableStorage table.Storage + dataconnectorStorage dataconnector.Storage + extractorFactories map[string]analyse.ExtractorFactory + analyserFactory analyse.AnalyserFactory +) + +// Inject dependencies +func Inject( + ts table.Storage, + dbas dataconnector.Storage, + dsf map[string]analyse.ExtractorFactory, + a analyse.AnalyserFactory, +) { + tableStorage = ts + dataconnectorStorage = dbas + extractorFactories = dsf + analyserFactory = a +} + +// NewCommand implements the cli analyse command +func NewCommand(fullName string, err *os.File, out *os.File, in *os.File) *cobra.Command { + cmd := &cobra.Command{ + Use: "analyse", + Short: "Analyse database content", + Long: "", + Example: fmt.Sprintf(" %[1]s", fullName), + Aliases: []string{"rimo"}, + Args: cobra.ExactArgs(1), + Run: func(cmd *cobra.Command, args []string) { + dataConnector := args[0] + dataSource, e0 := getDatasource(dataConnector) + if e0 != nil { + fmt.Fprintln(err, e0.Error()) + os.Exit(1) + } + + extractor, e1 := getExtractor(dataConnector, out) + if e1 != nil { + fmt.Fprintln(err, e1.Error()) + os.Exit(1) + } + + analyser := analyserFactory.New(out) + e2 := analyse.Do(dataSource, extractor, analyser) + if e2 != nil { + fmt.Fprintf(err, "analyse failed '%s'", dataConnector) + fmt.Fprintln(err) + os.Exit(5) + } + }, + } + cmd.SetOut(out) + cmd.SetErr(err) + cmd.SetIn(in) + return cmd +} + +func getExtractor(dataconnectorName string, out io.Writer) (analyse.Extractor, error) { + alias, e1 := dataconnector.Get(dataconnectorStorage, dataconnectorName) + if e1 != nil { + return nil, e1 + } + if alias == nil { + return nil, fmt.Errorf("Data Connector %s not found", dataconnectorName) + } + + u := urlbuilder.BuildURL(alias, out) + + datasourceFactory, ok := extractorFactories[u.UnaliasedDriver] + if !ok { + return nil, fmt.Errorf("no extractor found for database type") + } + + return datasourceFactory.New(u.URL.String(), alias.Schema), nil +} + +func getDatasource(dataconnectorName string) (analyse.DataSource, error) { + result := map[string][]string{} + tables, err := tableStorage.List() + if err != nil { + return nil, err + } + + for _, table := range tables { + columns := []string{} + for _, column := range table.Columns { + columns = append(columns, column.Name) + } + result[table.Name] = columns + } + + return ianalyse.NewMapDataSource(dataconnectorName, result), nil +} diff --git a/internal/infra/analyse/datasource.go b/internal/infra/analyse/datasource.go new file mode 100644 index 00000000..d972ccf6 --- /dev/null +++ b/internal/infra/analyse/datasource.go @@ -0,0 +1,31 @@ +package analyse + +import "github.com/cgi-fr/lino/pkg/analyse" + +func NewMapDataSource(name string, tables map[string][]string) analyse.DataSource { + return &MapDataSource{name: name, tables: tables} +} + +type MapDataSource struct { + tables map[string][]string + name string +} + +// ListColumn implements analyse.DataSource +func (ds *MapDataSource) ListColumn(tableName string) []string { + return ds.tables[tableName] +} + +// ListTables implements analyse.DataSource +func (ds *MapDataSource) ListTables() []string { + result := []string{} + for table := range ds.tables { + result = append(result, table) + } + return result +} + +// Name implements analyse.DataSource +func (ds *MapDataSource) Name() string { + return ds.name +} diff --git a/internal/infra/analyse/rimo_analyser.go b/internal/infra/analyse/rimo_analyser.go new file mode 100644 index 00000000..e9fd8d8a --- /dev/null +++ b/internal/infra/analyse/rimo_analyser.go @@ -0,0 +1,69 @@ +// Copyright (C) 2021 CGI France +// +// This file is part of LINO. +// +// LINO is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// LINO is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with LINO. If not, see . + +package analyse + +import ( + "fmt" + "io" + + "gopkg.in/yaml.v3" + + "github.com/cgi-fr/lino/pkg/analyse" + "github.com/cgi-fr/rimo/pkg/model" + "github.com/cgi-fr/rimo/pkg/rimo" +) + +type RimoAnalyserFactory struct{} + +func (r RimoAnalyserFactory) New(out io.Writer) analyse.Analyser { + return RimoAnalyser{ + writer: &YAMLWriter{output: out}, + } +} + +type RimoAnalyser struct { + writer rimo.Writer +} + +func (ra RimoAnalyser) Analyse(ds *analyse.ColumnIterator) error { + return rimo.AnalyseBase(ds, ra) +} + +func (ra RimoAnalyser) Export(base *model.Base) error { + return ra.writer.Export(base) +} + +// YAML Writter interface + +type YAMLWriter struct { + output io.Writer +} + +// Write a YAML file from RIMO base at outputPath. +func (w *YAMLWriter) Export(base *model.Base) error { + // Encode Base to YAML. + encoder := yaml.NewEncoder(w.output) + defer encoder.Close() + + err := encoder.Encode(base) + if err != nil { + return fmt.Errorf("failed to encode Base to YAML: %w", err) + } + + return nil +} diff --git a/internal/infra/analyse/sql_datasource.go b/internal/infra/analyse/sql_datasource.go new file mode 100644 index 00000000..ce14083e --- /dev/null +++ b/internal/infra/analyse/sql_datasource.go @@ -0,0 +1,106 @@ +// Copyright (C) 2021 CGI France +// +// This file is part of LINO. +// +// LINO is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// LINO is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with LINO. If not, see . + +package analyse + +import ( + "database/sql" + "fmt" + + "github.com/cgi-fr/lino/pkg/analyse" + "github.com/jmoiron/sqlx" + "github.com/rs/zerolog/log" + "github.com/xo/dburl" +) + +// SQLExtractorFactory exposes methods to create new Postgres pullers. +type SQLExtractorFactory struct{} + +// NewSQLExtractorFactory creates a new postgres datasource factory. +func NewSQLExtractorFactory() *SQLExtractorFactory { + return &SQLExtractorFactory{} +} + +// SQLDataSource to read in the analyse process. +type SQLDataSource struct { + url string + schema string + dbx *sqlx.DB + db *sql.DB +} + +// ExtractValues implements analyse.DataSource +func (ds *SQLDataSource) ExtractValues(tableName string, columnName string) ([]interface{}, error) { + result := []interface{}{} + + log.Trace().Str("tablename", tableName).Str("columnname", columnName).Msg("extract values") + + err := ds.Open() + if err != nil { + log.Error().Err(err).Msg("Connection failed") + return result, err + } + defer ds.db.Close() + + cursor, err := ds.db.Query(fmt.Sprintf("select %s from %s", columnName, tableName)) + if err != nil { + log.Error().Err(err).Msg("SQL select failed") + return result, err + } + for cursor.Next() { + var value interface{} + err = cursor.Scan(&value) + if err != nil { + log.Error().Err(err).Msg("SQL scan failed") + return result, err + } + + result = append(result, value) + } + return result, nil +} + +func (e *SQLExtractorFactory) New(url string, schema string) analyse.Extractor { + return &SQLDataSource{ + url: url, + schema: schema, + } +} + +// Open a connection to the SQL DB +func (ds *SQLDataSource) Open() error { + db, err := dburl.Open(ds.url) + if err != nil { + return err + } + + ds.db = db + + u, err := dburl.Parse(ds.url) + if err != nil { + return err + } + + ds.dbx = sqlx.NewDb(db, u.UnaliasedDriver) + + err = ds.dbx.Ping() + if err != nil { + return err + } + + return nil +} diff --git a/pkg/analyse/driven.go b/pkg/analyse/driven.go new file mode 100644 index 00000000..ef521958 --- /dev/null +++ b/pkg/analyse/driven.go @@ -0,0 +1,47 @@ +// Copyright (C) 2021 CGI France +// +// This file is part of LINO. +// +// LINO is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// LINO is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with LINO. If not, see . + +package analyse + +import "io" + +type ExtractorFactory interface { + New(url string, schema string) Extractor +} + +type DataSource interface { + Name() string + ListTables() []string + ListColumn(tableName string) []string +} + +type Extractor interface { + ExtractValues(tableName string, columnName string) ([]interface{}, error) +} + +type AnalyserFactory interface { + New(io.Writer) Analyser +} + +// Analyser is the provider of statistics analyse +type Analyser interface { + Analyse(ds *ColumnIterator) error +} + +type DataSourceFactory interface { + New() DataSource +} diff --git a/pkg/analyse/driver.go b/pkg/analyse/driver.go new file mode 100644 index 00000000..bba2a3d0 --- /dev/null +++ b/pkg/analyse/driver.go @@ -0,0 +1,77 @@ +// Copyright (C) 2021 CGI France +// +// This file is part of LINO. +// +// LINO is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// LINO is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with LINO. If not, see . + +package analyse + +// Do performe statistics on datasource. +func Do(ds DataSource, ex Extractor, analyser Analyser) error { + iterator := NewColumnIterator(ds, ex) + return analyser.Analyse(iterator) +} + +type ColumnIterator struct { + tables []string + column []string + DataSource + Extractor +} + +func NewColumnIterator(ds DataSource, ex Extractor) *ColumnIterator { + return &ColumnIterator{ + tables: []string{}, + column: []string{}, + DataSource: ds, + Extractor: ex, + } +} + +func (ci *ColumnIterator) BaseName() string { return ci.Name() } + +// Next return true if there is more column to iterate over. +func (ci *ColumnIterator) Next() bool { + if len(ci.tables) == 0 { + ci.tables = ci.ListTables() + if len(ci.tables) == 0 { + return false + } + ci.column = ci.ListColumn(ci.tables[0]) + if len(ci.column) > 0 { + return true + } + } + + if len(ci.column) > 1 { + ci.column = ci.column[1:] + return true + } + + for len(ci.tables) > 1 { + ci.tables = ci.tables[1:] + ci.column = ci.DataSource.ListColumn(ci.tables[0]) + if len(ci.column) > 0 { + return true + } + } + + return false +} + +// Value return the column content. +func (ci *ColumnIterator) Value() ([]interface{}, string, string, error) { + values, err := ci.ExtractValues(ci.tables[0], ci.column[0]) + return values, ci.tables[0], ci.column[0], err +} diff --git a/pkg/analyse/driver_test.go b/pkg/analyse/driver_test.go new file mode 100644 index 00000000..3f790fec --- /dev/null +++ b/pkg/analyse/driver_test.go @@ -0,0 +1,106 @@ +// Copyright (C) 2021 CGI France +// +// This file is part of LINO. +// +// LINO is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// LINO is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with LINO. If not, see . + +package analyse_test + +import ( + "fmt" + "testing" + + "github.com/cgi-fr/lino/pkg/analyse" + "github.com/cgi-fr/rimo/pkg/model" + "github.com/cgi-fr/rimo/pkg/rimo" + "github.com/stretchr/testify/assert" +) + +type rimoAnalyser struct { + writer rimo.Writer +} + +func (ra rimoAnalyser) Analyse(ds *analyse.ColumnIterator) error { + return rimo.AnalyseBase(ds, ra) +} + +func (ra rimoAnalyser) Export(base *model.Base) error { + return ra.writer.Export(base) +} + +type testDataSource struct{} + +func (tds *testDataSource) Name() string { + return "TestBase" +} + +func (tds *testDataSource) ListTables() []string { + return []string{"table1", "table2"} +} + +func (tds *testDataSource) ListColumn(tableName string) []string { + return []string{"col1", "col2"} +} + +type testExtractor struct{} + +func (tds *testExtractor) ExtractValues(tableName string, columnName string) ([]interface{}, error) { + return []interface{}{1., 2., 3., 4., 5.}, nil +} + +type testWriter struct { + result *model.Base +} + +func (tw *testWriter) Export(report *model.Base) error { + tw.result = report + return nil +} + +func TestAnalyseShouldNotReturnError(t *testing.T) { + t.Parallel() + dataSource := &testDataSource{} + extractor := &testExtractor{} + writer := &testWriter{} + analyser := rimoAnalyser{writer} + + err := analyse.Do(dataSource, extractor, analyser) + + assert.Nil(t, err) + assert.NotNil(t, writer.result) + assert.Equal(t, "TestBase", writer.result.Name) + assert.Equal(t, 2, len(writer.result.Tables)) + assert.Equal(t, 5, writer.result.Tables[0].Columns[0].MainMetric.Count) +} + +func TestColumnIteratorNext(t *testing.T) { + t.Parallel() + + dataSource := &testDataSource{} + extractor := &testExtractor{} + + iterator := analyse.NewColumnIterator(dataSource, extractor) + + for table := 1; table < 3; table++ { + for c := 1; c < 3; c++ { + assert.True(t, iterator.Next()) + _, tableName, columnName, err := iterator.Value() + assert.Nil(t, err) + assert.Equal(t, fmt.Sprintf("table%d", table), tableName) + assert.Equal(t, fmt.Sprintf("col%d", c), columnName) + } + } + + assert.False(t, iterator.Next()) +} diff --git a/pkg/table/driven.go b/pkg/table/driven.go index c062b084..77e04835 100755 --- a/pkg/table/driven.go +++ b/pkg/table/driven.go @@ -28,7 +28,7 @@ type Extractor interface { Count(tableName string) (int, *Error) } -// Storage allows to store and retrieve Relations objects. +// Storage allows to store and retrieve Tables objects. type Storage interface { List() ([]Table, *Error) Store(tables []Table) *Error diff --git a/pkg/table/driver.go b/pkg/table/driver.go index 3aa618df..5a212de6 100755 --- a/pkg/table/driver.go +++ b/pkg/table/driver.go @@ -45,6 +45,7 @@ func Count(s Storage, e Extractor) ([]TableCount, *Error) { if err != nil { return nil, err } + result = append(result, TableCount{table, count}) } return result, nil