support import && export

README.md

@@ -38,7 +38,7 @@ elastics:             # elasticsearch clusters
     password: ""
 
 tasks:  # tasks which is executed orderly.
-  - name: task1 # task name
+  - name: full-sync-task # task name
     source_es: es5  # source elasticsearch cluster name which is defined in elastics config
     target_es: es8  # target elasticsearch cluster name which is defined in elastics config 
     index_pairs:    # index multiple pairs which is used to sync data from source index to target_index
@@ -57,39 +57,62 @@ tasks:  # tasks which is executed orderly.
     parallelism: 12   # parallelism which is used to sync data in parallel index pairs, default is 12.
 
 
-  - name: task2
+  - name: increment-sync-task
     source_es: es5
     target_es: es8
     index_pairs:
       - source_index: "sample_hello"
         target_index: "sample_hello"
-    action: sync
-    force: true
+    action: sync_diff
 
-  - name: task3
+  - name: compare-task
     source_es: es5
     target_es: es6
     index_pairs:
       -
         source_index: "sample_hello"
         target_index: "sample_hello"
-
-    slice_size: 5 # search with slice number which is the parallelism size, default is 20.
-    scroll_size: 5000 # scroll size which is used to scroll data from source index, default is 10000.
-    buffer_count: 3000 # buffer count which is used to buffer data to sync, default is 100000.
-    compare_parallelism: 5  # compare parallelism which is used to compare data in parallel index pairs, default is 20.
-    scroll_time: 10   # scroll time which is used to scroll data from source index, default is 10 minutes.
-    parallelism: 12   # parallelism which is used to sync data in parallel index pairs, default is 12.
     action: compare
 
-  - name: task3
+  - name: copy-index
     source_es: es5
     target_es: es6
     index_pairs:
       -
         source_index: "sample_hello"
         target_index: "sample_hello"
-    action: sync_diff
+    action: copy_index
+
+  - name: copy-index-task
+    source_es: es5
+    target_es: es6
+    index_pairs:
+      -
+        source_index: "sample_hello"
+        target_index: "sample_hello"
+    action: copy_index
+
+  - name: import-task
+    source_es: es5
+    target_es: es6
+    index_pattern: "test_*" # import test prefix index data from index data file.
+    index_file_root: "C:/Users/andy/Documents" # index file root which is used to import index data.
+    index_pairs:
+      -
+        index: "sample_hello" # import data from index file dir to sample_hello index.
+        index_file_dir: "C:/Users/andy/Documents/abc"
+    action: import
+
+  - name: export-task
+    source_es: es5
+    target_es: es6
+    index_pattern: "test_*" # export test prefix index data to file.
+    index_file_root: "C:/Users/andy/Documents" # index file root which is used to export index data.
+    index_pairs:
+      -
+        index: "sample_hello"  # export sample_hello index data to index file dir.
+        index_file_dir: "C:/Users/andy/Documents/abc"
+    action: export
 ```
 
 You can communicate more directly with author through the Knowledge Planet.

go.mod

@@ -3,12 +3,13 @@ module github.com/CharellKing/ela
 go 1.23.0
 
 require (
-	github.com/CharellKing/ela-lib v1.0.22
+	github.com/CharellKing/ela-lib v1.0.30
 	github.com/spf13/viper v1.19.0
 )
 
 require (
 	github.com/alitto/pond v1.9.1 // indirect
+	github.com/bytedance/gopkg v0.1.1 // indirect
 	github.com/elastic/elastic-transport-go/v8 v8.6.0 // indirect
 	github.com/elastic/go-elasticsearch/v5 v5.6.1 // indirect
 	github.com/elastic/go-elasticsearch/v6 v6.8.10 // indirect
@@ -18,12 +19,10 @@ require (
 	github.com/go-logr/logr v1.4.1 // indirect
 	github.com/go-logr/stdr v1.2.2 // indirect
 	github.com/google/uuid v1.6.0 // indirect
-	github.com/gosuri/uilive v0.0.4 // indirect
-	github.com/gosuri/uiprogress v0.0.1 // indirect
+	github.com/hashicorp/go-version v1.7.0 // indirect
 	github.com/hashicorp/hcl v1.0.0 // indirect
 	github.com/jinzhu/copier v0.4.0 // indirect
 	github.com/magiconair/properties v1.8.7 // indirect
-	github.com/mattn/go-isatty v0.0.19 // indirect
 	github.com/mitchellh/mapstructure v1.5.0 // indirect
 	github.com/pelletier/go-toml/v2 v2.2.2 // indirect
 	github.com/pkg/errors v0.9.1 // indirect

go.sum

@@ -1,10 +1,11 @@
-github.com/CharellKing/ela-lib v1.0.21 h1:uEodd8k/Oc8WBoz9DcoEo8BDvTg2IOcpgGiflJrzJTA=
-github.com/CharellKing/ela-lib v1.0.21/go.mod h1:f0LBjns3zok0ejHTgmMncP5fAKH61sPSwEGegVcW9NI=
-github.com/CharellKing/ela-lib v1.0.22/go.mod h1:f0LBjns3zok0ejHTgmMncP5fAKH61sPSwEGegVcW9NI=
+github.com/CharellKing/ela-lib v1.0.30 h1:E6r+uJdwD2XYdZ+freVkUyTtxw0tIjeoylSTE53WR44=
+github.com/CharellKing/ela-lib v1.0.30/go.mod h1:6AbIwbjxtUHawusGSbKEYJCTUQQ4vJlHDBEWThH98qs=
 github.com/alitto/pond v1.9.1 h1:OfCpIrMyrWJpn34f647DcFmUxjK8+7Nu3eoVN/WTP+o=
 github.com/alitto/pond v1.9.1/go.mod h1:xQn3P/sHTYcU/1BR3i86IGIrilcrGC2LiS+E2+CJWsI=
 github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 h1:DDGfHa7BWjL4YnC6+E63dPcxHo2sUxDIu8g3QgEJdRY=
 github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4=
+github.com/bytedance/gopkg v0.1.1 h1:3azzgSkiaw79u24a+w9arfH8OfnQQ4MHUt9lJFREEaE=
+github.com/bytedance/gopkg v0.1.1/go.mod h1:576VvJ+eJgyCzdjS+c4+77QF3p7ubbtiKARP3TxducM=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM=
@@ -32,10 +33,8 @@ github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
 github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
 github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
 github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
-github.com/gosuri/uilive v0.0.4 h1:hUEBpQDj8D8jXgtCdBu7sWsy5sbW/5GhuO8KBwJ2jyY=
-github.com/gosuri/uilive v0.0.4/go.mod h1:V/epo5LjjlDE5RJUcqx8dbw+zc93y5Ya3yg8tfZ74VI=
-github.com/gosuri/uiprogress v0.0.1 h1:0kpv/XY/qTmFWl/SkaJykZXrBBzwwadmW8fRb7RJSxw=
-github.com/gosuri/uiprogress v0.0.1/go.mod h1:C1RTYn4Sc7iEyf6j8ft5dyoZ4212h8G1ol9QQluh5+0=
+github.com/hashicorp/go-version v1.7.0 h1:5tqGy27NaOTB8yJKUZELlFAS/LTKJkrmONwQKeRZfjY=
+github.com/hashicorp/go-version v1.7.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
 github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4=
 github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
 github.com/jinzhu/copier v0.4.0 h1:w3ciUoD19shMCRargcpm0cm91ytaBhDvuRpz1ODO/U8=
@@ -46,8 +45,6 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
 github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY=
 github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0=
-github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA=
-github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
 github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY=
 github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
 github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM=
@@ -107,7 +104,6 @@ go.uber.org/multierr v1.9.0/go.mod h1:X2jQV1h+kxSjClGpnseKVIxpmcjrj7MNnI0bnlfKTV
 golang.org/x/exp v0.0.0-20230905200255-921286631fa9 h1:GoHiUyI/Tp2nVkLI2mCxVkOjsbSXD66ic0XW0js0R9g=
 golang.org/x/exp v0.0.0-20230905200255-921286631fa9/go.mod h1:S2oDrQGGwySpoQPVqRShND87VCbxmc6bL1Yd2oYrm6k=
 golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y=
 golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4=