From d3de04b7bac853fcea4811ab03065839f8e7878f Mon Sep 17 00:00:00 2001 From: andy Date: Sat, 28 Sep 2024 22:51:02 +0800 Subject: [PATCH] support import && export --- README.md | 51 +++++++++++++++++++++++++++++++++++++-------------- go.mod | 7 +++---- go.sum | 16 ++++++---------- 3 files changed, 46 insertions(+), 28 deletions(-) diff --git a/README.md b/README.md index ddb5186..ff8ae81 100644 --- a/README.md +++ b/README.md @@ -38,7 +38,7 @@ elastics: # elasticsearch clusters password: "" tasks: # tasks which is executed orderly. - - name: task1 # task name + - name: full-sync-task # task name source_es: es5 # source elasticsearch cluster name which is defined in elastics config target_es: es8 # target elasticsearch cluster name which is defined in elastics config index_pairs: # index multiple pairs which is used to sync data from source index to target_index @@ -57,39 +57,62 @@ tasks: # tasks which is executed orderly. parallelism: 12 # parallelism which is used to sync data in parallel index pairs, default is 12. - - name: task2 + - name: increment-sync-task source_es: es5 target_es: es8 index_pairs: - source_index: "sample_hello" target_index: "sample_hello" - action: sync - force: true + action: sync_diff - - name: task3 + - name: compare-task source_es: es5 target_es: es6 index_pairs: - source_index: "sample_hello" target_index: "sample_hello" - - slice_size: 5 # search with slice number which is the parallelism size, default is 20. - scroll_size: 5000 # scroll size which is used to scroll data from source index, default is 10000. - buffer_count: 3000 # buffer count which is used to buffer data to sync, default is 100000. - compare_parallelism: 5 # compare parallelism which is used to compare data in parallel index pairs, default is 20. - scroll_time: 10 # scroll time which is used to scroll data from source index, default is 10 minutes. - parallelism: 12 # parallelism which is used to sync data in parallel index pairs, default is 12. action: compare - - name: task3 + - name: copy-index source_es: es5 target_es: es6 index_pairs: - source_index: "sample_hello" target_index: "sample_hello" - action: sync_diff + action: copy_index + + - name: copy-index-task + source_es: es5 + target_es: es6 + index_pairs: + - + source_index: "sample_hello" + target_index: "sample_hello" + action: copy_index + + - name: import-task + source_es: es5 + target_es: es6 + index_pattern: "test_*" # import test prefix index data from index data file. + index_file_root: "C:/Users/andy/Documents" # index file root which is used to import index data. + index_pairs: + - + index: "sample_hello" # import data from index file dir to sample_hello index. + index_file_dir: "C:/Users/andy/Documents/abc" + action: import + + - name: export-task + source_es: es5 + target_es: es6 + index_pattern: "test_*" # export test prefix index data to file. + index_file_root: "C:/Users/andy/Documents" # index file root which is used to export index data. + index_pairs: + - + index: "sample_hello" # export sample_hello index data to index file dir. + index_file_dir: "C:/Users/andy/Documents/abc" + action: export ``` You can communicate more directly with author through the Knowledge Planet. diff --git a/go.mod b/go.mod index 8d3b173..7f7c717 100644 --- a/go.mod +++ b/go.mod @@ -3,12 +3,13 @@ module github.com/CharellKing/ela go 1.23.0 require ( - github.com/CharellKing/ela-lib v1.0.22 + github.com/CharellKing/ela-lib v1.0.30 github.com/spf13/viper v1.19.0 ) require ( github.com/alitto/pond v1.9.1 // indirect + github.com/bytedance/gopkg v0.1.1 // indirect github.com/elastic/elastic-transport-go/v8 v8.6.0 // indirect github.com/elastic/go-elasticsearch/v5 v5.6.1 // indirect github.com/elastic/go-elasticsearch/v6 v6.8.10 // indirect @@ -18,12 +19,10 @@ require ( github.com/go-logr/logr v1.4.1 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/google/uuid v1.6.0 // indirect - github.com/gosuri/uilive v0.0.4 // indirect - github.com/gosuri/uiprogress v0.0.1 // indirect + github.com/hashicorp/go-version v1.7.0 // indirect github.com/hashicorp/hcl v1.0.0 // indirect github.com/jinzhu/copier v0.4.0 // indirect github.com/magiconair/properties v1.8.7 // indirect - github.com/mattn/go-isatty v0.0.19 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/pelletier/go-toml/v2 v2.2.2 // indirect github.com/pkg/errors v0.9.1 // indirect diff --git a/go.sum b/go.sum index 65f1b12..ede229e 100644 --- a/go.sum +++ b/go.sum @@ -1,10 +1,11 @@ -github.com/CharellKing/ela-lib v1.0.21 h1:uEodd8k/Oc8WBoz9DcoEo8BDvTg2IOcpgGiflJrzJTA= -github.com/CharellKing/ela-lib v1.0.21/go.mod h1:f0LBjns3zok0ejHTgmMncP5fAKH61sPSwEGegVcW9NI= -github.com/CharellKing/ela-lib v1.0.22/go.mod h1:f0LBjns3zok0ejHTgmMncP5fAKH61sPSwEGegVcW9NI= +github.com/CharellKing/ela-lib v1.0.30 h1:E6r+uJdwD2XYdZ+freVkUyTtxw0tIjeoylSTE53WR44= +github.com/CharellKing/ela-lib v1.0.30/go.mod h1:6AbIwbjxtUHawusGSbKEYJCTUQQ4vJlHDBEWThH98qs= github.com/alitto/pond v1.9.1 h1:OfCpIrMyrWJpn34f647DcFmUxjK8+7Nu3eoVN/WTP+o= github.com/alitto/pond v1.9.1/go.mod h1:xQn3P/sHTYcU/1BR3i86IGIrilcrGC2LiS+E2+CJWsI= github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 h1:DDGfHa7BWjL4YnC6+E63dPcxHo2sUxDIu8g3QgEJdRY= github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4= +github.com/bytedance/gopkg v0.1.1 h1:3azzgSkiaw79u24a+w9arfH8OfnQQ4MHUt9lJFREEaE= +github.com/bytedance/gopkg v0.1.1/go.mod h1:576VvJ+eJgyCzdjS+c4+77QF3p7ubbtiKARP3TxducM= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= @@ -32,10 +33,8 @@ github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/gosuri/uilive v0.0.4 h1:hUEBpQDj8D8jXgtCdBu7sWsy5sbW/5GhuO8KBwJ2jyY= -github.com/gosuri/uilive v0.0.4/go.mod h1:V/epo5LjjlDE5RJUcqx8dbw+zc93y5Ya3yg8tfZ74VI= -github.com/gosuri/uiprogress v0.0.1 h1:0kpv/XY/qTmFWl/SkaJykZXrBBzwwadmW8fRb7RJSxw= -github.com/gosuri/uiprogress v0.0.1/go.mod h1:C1RTYn4Sc7iEyf6j8ft5dyoZ4212h8G1ol9QQluh5+0= +github.com/hashicorp/go-version v1.7.0 h1:5tqGy27NaOTB8yJKUZELlFAS/LTKJkrmONwQKeRZfjY= +github.com/hashicorp/go-version v1.7.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/jinzhu/copier v0.4.0 h1:w3ciUoD19shMCRargcpm0cm91ytaBhDvuRpz1ODO/U8= @@ -46,8 +45,6 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= -github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA= -github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= @@ -107,7 +104,6 @@ go.uber.org/multierr v1.9.0/go.mod h1:X2jQV1h+kxSjClGpnseKVIxpmcjrj7MNnI0bnlfKTV golang.org/x/exp v0.0.0-20230905200255-921286631fa9 h1:GoHiUyI/Tp2nVkLI2mCxVkOjsbSXD66ic0XW0js0R9g= golang.org/x/exp v0.0.0-20230905200255-921286631fa9/go.mod h1:S2oDrQGGwySpoQPVqRShND87VCbxmc6bL1Yd2oYrm6k= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y= golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4=