diff --git a/content/404.html b/content/404.html
index 6dfca1529d655..4ea5d98d3b7b9 100644
--- a/content/404.html
+++ b/content/404.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/404/index.html b/content/404/index.html
index 376664e22b70a..32b40379636f3 100644
--- a/content/404/index.html
+++ b/content/404/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/assets/js/02e54e09.a7c8fe63.js b/content/assets/js/02e54e09.dc70936f.js
similarity index 95%
rename from content/assets/js/02e54e09.a7c8fe63.js
rename to content/assets/js/02e54e09.dc70936f.js
index 04a952db63e39..7d061ddbcbeb9 100644
--- a/content/assets/js/02e54e09.a7c8fe63.js
+++ b/content/assets/js/02e54e09.dc70936f.js
@@ -1 +1 @@
-"use strict";(self.webpackChunkhudi=self.webpackChunkhudi||[]).push([[34691],{28372:(e,t,i)=>{i.r(t),i.d(t,{assets:()=>o,contentTitle:()=>l,default:()=>u,frontMatter:()=>a,metadata:()=>n,toc:()=>c});const n=JSON.parse('{"id":"clustering","title":"Clustering","description":"Background","source":"@site/docs/clustering.md","sourceDirName":".","slug":"/clustering","permalink":"/docs/next/clustering","draft":false,"unlisted":false,"editUrl":"https://github.com/apache/hudi/tree/asf-site/website/docs/clustering.md","tags":[],"version":"current","frontMatter":{"title":"Clustering","summary":"In this page, we describe async compaction in Hudi.","toc":true,"last_modified_at":null},"sidebar":"docs","previous":{"title":"Compaction","permalink":"/docs/next/compaction"},"next":{"title":"Indexing","permalink":"/docs/next/metadata_indexing"}}');var s=i(74848),r=i(28453);const a={title:"Clustering",summary:"In this page, we describe async compaction in Hudi.",toc:!0,last_modified_at:null},l=void 0,o={},c=[{value:"Background",id:"background",level:2},{value:"How is compaction different from clustering?",id:"how-is-compaction-different-from-clustering",level:2},{value:"Clustering Architecture",id:"clustering-architecture",level:2},{value:"Overall, there are 2 steps to clustering",id:"overall-there-are-2-steps-to-clustering",level:3},{value:"Schedule clustering",id:"schedule-clustering",level:3},{value:"Execute clustering",id:"execute-clustering",level:3},{value:"Clustering Usecases",id:"clustering-usecases",level:2},{value:"Batching small files",id:"batching-small-files",level:3},{value:"Cluster by sort key",id:"cluster-by-sort-key",level:3},{value:"Clustering Strategies",id:"clustering-strategies",level:2},{value:"Plan Strategy",id:"plan-strategy",level:3},{value:"Size-based clustering strategies",id:"size-based-clustering-strategies",level:4},{value:"SparkSingleFileSortPlanStrategy",id:"sparksinglefilesortplanstrategy",level:4},{value:"SparkConsistentBucketClusteringPlanStrategy",id:"sparkconsistentbucketclusteringplanstrategy",level:4},{value:"Execution Strategy",id:"execution-strategy",level:3},{value:"Update Strategy",id:"update-strategy",level:3},{value:"Inline clustering",id:"inline-clustering",level:2},{value:"Async Clustering",id:"async-clustering",level:2},{value:"Setup Asynchronous Clustering",id:"setup-asynchronous-clustering",level:2},{value:"HoodieClusteringJob",id:"hoodieclusteringjob",level:3},{value:"HoodieStreamer",id:"hoodiestreamer",level:3},{value:"Spark Structured Streaming",id:"spark-structured-streaming",level:3},{value:"Java Client",id:"java-client",level:2},{value:"Related Resources",id:"related-resources",level:2}];function d(e){const t={a:"a",admonition:"admonition",code:"code",em:"em",h2:"h2",h3:"h3",h4:"h4",img:"img",li:"li",ol:"ol",p:"p",pre:"pre",strong:"strong",table:"table",tbody:"tbody",td:"td",th:"th",thead:"thead",tr:"tr",ul:"ul",...(0,r.R)(),...e.components};return(0,s.jsxs)(s.Fragment,{children:[(0,s.jsx)(t.h2,{id:"background",children:"Background"}),"\n",(0,s.jsxs)(t.p,{children:["Apache Hudi brings stream processing to big data, providing fresh data while being an order of magnitude efficient over traditional batch processing. In a data lake/warehouse, one of the key trade-offs is between ingestion speed and query performance. Data ingestion typically prefers small files to improve parallelism and make data available to queries as soon as possible. However, query performance degrades poorly with a lot of small files. Also, during ingestion, data is typically co-located based on arrival time. However, the query engines perform better when the data frequently queried is co-located together. In most architectures each of these systems tend to add optimizations independently to improve performance which hits limitations due to un-optimized data layouts. This doc introduces a new kind of table service called clustering ",(0,s.jsx)(t.a,{href:"https://cwiki.apache.org/confluence/display/HUDI/RFC+-+19+Clustering+data+for+freshness+and+query+performance",children:"[RFC-19]"})," to reorganize data for improved query performance without compromising on ingestion speed."]}),"\n",(0,s.jsx)(t.h2,{id:"how-is-compaction-different-from-clustering",children:"How is compaction different from clustering?"}),"\n",(0,s.jsxs)(t.p,{children:["Hudi is modeled like a log-structured storage engine with multiple versions of the data.\nParticularly, ",(0,s.jsx)(t.a,{href:"/docs/table_types#merge-on-read-table",children:"Merge-On-Read"}),"\ntables in Hudi store data using a combination of base file in columnar format and row-based delta logs that contain\nupdates. Compaction is a way to merge the delta logs with base files to produce the latest file slices with the most\nrecent snapshot of data. Compaction helps to keep the query performance in check (larger delta log files would incur\nlonger merge times on query side). On the other hand, clustering is a data layout optimization technique. One can stitch\ntogether small files into larger files using clustering. Additionally, data can be clustered by sort key so that queries\ncan take advantage of data locality."]}),"\n",(0,s.jsx)(t.h2,{id:"clustering-architecture",children:"Clustering Architecture"}),"\n",(0,s.jsxs)(t.p,{children:["At a high level, Hudi provides different operations such as insert/upsert/bulk_insert through it\u2019s write client API to be able to write data to a Hudi table. To be able to choose a trade-off between file size and ingestion speed, Hudi provides a knob ",(0,s.jsx)(t.code,{children:"hoodie.parquet.small.file.limit"})," to be able to configure the smallest allowable file size. Users are able to configure the small file ",(0,s.jsx)(t.a,{href:"https://hudi.apache.org/docs/configurations/#hoodieparquetsmallfilelimit",children:"soft limit"})," to ",(0,s.jsx)(t.code,{children:"0"})," to force new data to go into a new set of filegroups or set it to a higher value to ensure new data gets \u201cpadded\u201d to existing files until it meets that limit that adds to ingestion latencies."]}),"\n",(0,s.jsx)(t.p,{children:"To be able to support an architecture that allows for fast ingestion without compromising query performance, we have introduced a \u2018clustering\u2019 service to rewrite the data to optimize Hudi data lake file layout."}),"\n",(0,s.jsx)(t.p,{children:"Clustering table service can run asynchronously or synchronously adding a new action type called \u201cREPLACE\u201d, that will mark the clustering action in the Hudi metadata timeline."}),"\n",(0,s.jsx)(t.h3,{id:"overall-there-are-2-steps-to-clustering",children:"Overall, there are 2 steps to clustering"}),"\n",(0,s.jsxs)(t.ol,{children:["\n",(0,s.jsx)(t.li,{children:"Scheduling clustering: Create a clustering plan using a pluggable clustering strategy."}),"\n",(0,s.jsx)(t.li,{children:"Execute clustering: Process the plan using an execution strategy to create new files and replace old files."}),"\n"]}),"\n",(0,s.jsx)(t.h3,{id:"schedule-clustering",children:"Schedule clustering"}),"\n",(0,s.jsx)(t.p,{children:"Following steps are followed to schedule clustering."}),"\n",(0,s.jsxs)(t.ol,{children:["\n",(0,s.jsx)(t.li,{children:"Identify files that are eligible for clustering: Depending on the clustering strategy chosen, the scheduling logic will identify the files eligible for clustering."}),"\n",(0,s.jsx)(t.li,{children:"Group files that are eligible for clustering based on specific criteria. Each group is expected to have data size in multiples of \u2018targetFileSize\u2019. Grouping is done as part of \u2018strategy\u2019 defined in the plan. Additionally, there is an option to put a cap on group size to improve parallelism and avoid shuffling large amounts of data."}),"\n",(0,s.jsxs)(t.li,{children:["Finally, the clustering plan is saved to the timeline in an avro ",(0,s.jsx)(t.a,{href:"https://github.com/apache/hudi/blob/master/hudi-common/src/main/avro/HoodieClusteringPlan.avsc",children:"metadata format"}),"."]}),"\n"]}),"\n",(0,s.jsx)(t.h3,{id:"execute-clustering",children:"Execute clustering"}),"\n",(0,s.jsxs)(t.ol,{children:["\n",(0,s.jsx)(t.li,{children:"Read the clustering plan and get the \u2018clusteringGroups\u2019 that mark the file groups that need to be clustered."}),"\n",(0,s.jsx)(t.li,{children:"For each group, we instantiate appropriate strategy class with strategyParams (example: sortColumns) and apply that strategy to rewrite the data."}),"\n",(0,s.jsxs)(t.li,{children:["Create a \u201cREPLACE\u201d commit and update the metadata in ",(0,s.jsx)(t.a,{href:"https://github.com/apache/hudi/blob/master/hudi-common/src/main/java/org/apache/hudi/common/model/HoodieReplaceCommitMetadata.java",children:"HoodieReplaceCommitMetadata"}),"."]}),"\n"]}),"\n",(0,s.jsx)(t.p,{children:"Clustering Service builds on Hudi\u2019s MVCC based design to allow for writers to continue to insert new data while clustering action runs in the background to reformat data layout, ensuring snapshot isolation between concurrent readers and writers."}),"\n",(0,s.jsx)(t.p,{children:"NOTE: Clustering can only be scheduled for tables / partitions not receiving any concurrent updates. In the future, concurrent updates use-case will be supported as well."}),"\n",(0,s.jsxs)(t.p,{children:[(0,s.jsx)(t.img,{alt:"Clustering example",src:i(7004).A+"",width:"6827",height:"3334"}),"\n",(0,s.jsx)(t.em,{children:"Figure: Illustrating query performance improvements by clustering"})]}),"\n",(0,s.jsx)(t.h2,{id:"clustering-usecases",children:"Clustering Usecases"}),"\n",(0,s.jsx)(t.h3,{id:"batching-small-files",children:"Batching small files"}),"\n",(0,s.jsx)(t.p,{children:"As mentioned in the intro, streaming ingestion generally results in smaller files in your data lake. But having a lot of\nsuch small files could lead to higher query latency. From our experience supporting community users, there are quite a\nfew users who are using Hudi just for small file handling capabilities. So, you could employ clustering to batch a lot\nof such small files into larger ones."}),"\n",(0,s.jsx)(t.p,{children:(0,s.jsx)(t.img,{alt:"Batching small files",src:i(67491).A+"",width:"3100",height:"1620"})}),"\n",(0,s.jsx)(t.h3,{id:"cluster-by-sort-key",children:"Cluster by sort key"}),"\n",(0,s.jsx)(t.p,{children:"Another classic problem in data lake is the arrival time vs event time problem. Generally you write data based on\narrival time, while query predicates do not sit well with it. With clustering, you can re-write your data by sorting\nbased on query predicates and so, your data skipping will be very efficient and your query can ignore scanning a lot of\nunnecessary data."}),"\n",(0,s.jsx)(t.p,{children:(0,s.jsx)(t.img,{alt:"Batching small files",src:i(14438).A+"",width:"5003",height:"1464"})}),"\n",(0,s.jsx)(t.h2,{id:"clustering-strategies",children:"Clustering Strategies"}),"\n",(0,s.jsx)(t.p,{children:"On a high level, clustering creates a plan based on a configurable strategy, groups eligible files based on specific\ncriteria and then executes the plan. As mentioned before, clustering plan as well as execution depends on configurable\nstrategy. These strategies can be broadly classified into three types: clustering plan strategy, execution strategy and\nupdate strategy."}),"\n",(0,s.jsx)(t.h3,{id:"plan-strategy",children:"Plan Strategy"}),"\n",(0,s.jsxs)(t.p,{children:["This strategy comes into play while creating clustering plan. It helps to decide what file groups should be clustered\nand how many output file groups should the clustering produce. Note that these strategies are easily pluggable using the\nconfig ",(0,s.jsx)(t.a,{href:"/docs/configurations#hoodieclusteringplanstrategyclass",children:"hoodie.clustering.plan.strategy.class"}),"."]}),"\n",(0,s.jsx)(t.p,{children:"Different plan strategies are as follows:"}),"\n",(0,s.jsx)(t.h4,{id:"size-based-clustering-strategies",children:"Size-based clustering strategies"}),"\n",(0,s.jsxs)(t.p,{children:["This strategy creates clustering groups based on max size allowed per group. Also, it excludes files that are greater\nthan the small file limit from the clustering plan. Available strategies depending on write client\nare: ",(0,s.jsx)(t.code,{children:"SparkSizeBasedClusteringPlanStrategy"}),", ",(0,s.jsx)(t.code,{children:"FlinkSizeBasedClusteringPlanStrategy"}),"\nand ",(0,s.jsx)(t.code,{children:"JavaSizeBasedClusteringPlanStrategy"}),". Furthermore, Hudi provides flexibility to include or exclude partitions for\nclustering, tune the file size limits, maximum number of output groups. Please refer to ",(0,s.jsx)(t.a,{href:"https://hudi.apache.org/docs/next/configurations/#hoodieclusteringplanstrategysmallfilelimit",children:"hoodie.clustering.plan.strategy.small.file.limit"}),"\n, ",(0,s.jsx)(t.a,{href:"https://hudi.apache.org/docs/next/configurations/#hoodieclusteringplanstrategymaxnumgroups",children:"hoodie.clustering.plan.strategy.max.num.groups"}),", ",(0,s.jsx)(t.a,{href:"https://hudi.apache.org/docs/next/configurations/#hoodieclusteringplanstrategymaxbytespergroup",children:"hoodie.clustering.plan.strategy.max.bytes.per.group"}),"\n, ",(0,s.jsx)(t.a,{href:"https://hudi.apache.org/docs/next/configurations/#hoodieclusteringplanstrategytargetfilemaxbytes",children:"hoodie.clustering.plan.strategy.target.file.max.bytes"})," for more details."]}),"\n",(0,s.jsxs)(t.table,{children:[(0,s.jsx)(t.thead,{children:(0,s.jsxs)(t.tr,{children:[(0,s.jsx)(t.th,{children:"Config Name"}),(0,s.jsx)(t.th,{children:"Default"}),(0,s.jsx)(t.th,{children:"Description"})]})}),(0,s.jsxs)(t.tbody,{children:[(0,s.jsxs)(t.tr,{children:[(0,s.jsx)(t.td,{children:"hoodie.clustering.plan.strategy.partition.selected"}),(0,s.jsxs)(t.td,{children:["N/A ",(0,s.jsx)(t.strong,{children:"(Required)"})]}),(0,s.jsxs)(t.td,{children:["Comma separated list of partitions to run clustering",(0,s.jsx)("br",{}),(0,s.jsx)("br",{}),(0,s.jsx)(t.code,{children:"Config Param: PARTITION_SELECTED"}),(0,s.jsx)("br",{}),(0,s.jsx)(t.code,{children:"Since Version: 0.11.0"})]})]}),(0,s.jsxs)(t.tr,{children:[(0,s.jsx)(t.td,{children:"hoodie.clustering.plan.strategy.partition.regex.pattern"}),(0,s.jsxs)(t.td,{children:["N/A ",(0,s.jsx)(t.strong,{children:"(Required)"})]}),(0,s.jsxs)(t.td,{children:["Filter clustering partitions that matched regex pattern",(0,s.jsx)("br",{}),(0,s.jsx)("br",{}),(0,s.jsx)(t.code,{children:"Config Param: PARTITION_REGEX_PATTERN"}),(0,s.jsx)("br",{}),(0,s.jsx)(t.code,{children:"Since Version: 0.11.0"})]})]}),(0,s.jsxs)(t.tr,{children:[(0,s.jsx)(t.td,{children:"hoodie.clustering.plan.partition.filter.mode"}),(0,s.jsx)(t.td,{children:"NONE (Optional)"}),(0,s.jsxs)(t.td,{children:["Partition filter mode used in the creation of clustering plan. Possible values:",(0,s.jsx)("br",{}),(0,s.jsxs)("ul",{children:[(0,s.jsxs)("li",{children:[(0,s.jsx)(t.code,{children:"NONE"}),": Do not filter partitions. The clustering plan will include all partitions that have clustering candidates."]}),(0,s.jsxs)("li",{children:[(0,s.jsx)(t.code,{children:"RECENT_DAYS"}),": This filter assumes that your data is partitioned by date. The clustering plan will only include partitions from K days ago to N days ago, where K >= N. K is determined by ",(0,s.jsx)(t.code,{children:"hoodie.clustering.plan.strategy.daybased.lookback.partitions"})," and N is determined by ",(0,s.jsx)(t.code,{children:"hoodie.clustering.plan.strategy.daybased.skipfromlatest.partitions"}),"."]}),(0,s.jsxs)("li",{children:[(0,s.jsx)(t.code,{children:"SELECTED_PARTITIONS"}),": The clustering plan will include only partition paths with names that sort within the inclusive range [",(0,s.jsx)(t.code,{children:"hoodie.clustering.plan.strategy.cluster.begin.partition"}),", ",(0,s.jsx)(t.code,{children:"hoodie.clustering.plan.strategy.cluster.end.partition"}),"]."]}),(0,s.jsxs)("li",{children:[(0,s.jsx)(t.code,{children:"DAY_ROLLING"}),": To determine the partitions in the clustering plan, the eligible partitions will be sorted in ascending order. Each partition will have an index i in that list. The clustering plan will only contain partitions such that i mod 24 = H, where H is the current hour of the day (from 0 to 23)."]})]}),(0,s.jsx)("br",{}),(0,s.jsx)(t.code,{children:"Config Param: PLAN_PARTITION_FILTER_MODE_NAME"}),(0,s.jsx)("br",{}),(0,s.jsx)(t.code,{children:"Since Version: 0.11.0"})]})]})]})]}),"\n",(0,s.jsx)(t.h4,{id:"sparksinglefilesortplanstrategy",children:"SparkSingleFileSortPlanStrategy"}),"\n",(0,s.jsxs)(t.p,{children:["In this strategy, clustering group for each partition is built in the same way as ",(0,s.jsx)(t.code,{children:"SparkSizeBasedClusteringPlanStrategy"}),"\n. The difference is that the output group is 1 and file group id remains the same,\nwhile ",(0,s.jsx)(t.code,{children:"SparkSizeBasedClusteringPlanStrategy"})," can create multiple file groups with newer fileIds."]}),"\n",(0,s.jsx)(t.h4,{id:"sparkconsistentbucketclusteringplanstrategy",children:"SparkConsistentBucketClusteringPlanStrategy"}),"\n",(0,s.jsx)(t.p,{children:"This strategy is specifically used for consistent bucket index. This will be leveraged to expand your bucket index (from\nstatic partitioning to dynamic). Typically, users don\u2019t need to use this strategy. Hudi internally uses this for\ndynamically expanding the buckets for bucket index datasets."}),"\n",(0,s.jsx)(t.admonition,{title:"The latter two strategies are applicable only for the Spark engine.",type:"note"}),"\n",(0,s.jsx)(t.h3,{id:"execution-strategy",children:"Execution Strategy"}),"\n",(0,s.jsxs)(t.p,{children:["After building the clustering groups in the planning phase, Hudi applies execution strategy, for each group, primarily\nbased on sort columns and size. The strategy can be specified using the\nconfig ",(0,s.jsx)(t.a,{href:"/docs/configurations/#hoodieclusteringexecutionstrategyclass",children:"hoodie.clustering.execution.strategy.class"}),". By\ndefault, Hudi sorts the file groups in the plan by the specified columns, while meeting the configured target file\nsizes."]}),"\n",(0,s.jsxs)(t.table,{children:[(0,s.jsx)(t.thead,{children:(0,s.jsxs)(t.tr,{children:[(0,s.jsx)(t.th,{children:"Config Name"}),(0,s.jsx)(t.th,{children:"Default"}),(0,s.jsx)(t.th,{children:"Description"})]})}),(0,s.jsx)(t.tbody,{children:(0,s.jsxs)(t.tr,{children:[(0,s.jsx)(t.td,{children:"hoodie.clustering.execution.strategy.class"}),(0,s.jsx)(t.td,{children:"org.apache.hudi.client.clustering.run.strategy.SparkSortAndSizeExecutionStrategy (Optional)"}),(0,s.jsxs)(t.td,{children:["Config to provide a strategy class (subclass of RunClusteringStrategy) to define how the clustering plan is executed. By default, we sort the file groups in th plan by the specified columns, while meeting the configured target file sizes.",(0,s.jsx)("br",{}),(0,s.jsx)("br",{}),(0,s.jsx)(t.code,{children:"Config Param: EXECUTION_STRATEGY_CLASS_NAME"}),(0,s.jsx)("br",{}),(0,s.jsx)(t.code,{children:"Since Version: 0.7.0"})]})]})})]}),"\n",(0,s.jsx)(t.p,{children:"The available strategies are as follows:"}),"\n",(0,s.jsxs)(t.ol,{children:["\n",(0,s.jsxs)(t.li,{children:[(0,s.jsx)(t.code,{children:"SPARK_SORT_AND_SIZE_EXECUTION_STRATEGY"}),": Uses bulk_insert to re-write data from input file groups.","\n",(0,s.jsxs)(t.ol,{children:["\n",(0,s.jsxs)(t.li,{children:["Set ",(0,s.jsx)(t.code,{children:"hoodie.clustering.execution.strategy.class"}),"\nto ",(0,s.jsx)(t.code,{children:"org.apache.hudi.client.clustering.run.strategy.SparkSortAndSizeExecutionStrategy"}),"."]}),"\n",(0,s.jsxs)(t.li,{children:[(0,s.jsx)(t.code,{children:"hoodie.clustering.plan.strategy.sort.columns"}),": Columns to sort the data while clustering. This goes in\nconjunction with layout optimization strategies depending on your query predicates. One can set comma separated\nlist of columns that needs to be sorted in this config."]}),"\n"]}),"\n"]}),"\n",(0,s.jsxs)(t.li,{children:[(0,s.jsx)(t.code,{children:"JAVA_SORT_AND_SIZE_EXECUTION_STRATEGY"}),": Similar to ",(0,s.jsx)(t.code,{children:"SPARK_SORT_AND_SIZE_EXECUTION_STRATEGY"}),", for the Java and Flink\nengines. Set ",(0,s.jsx)(t.code,{children:"hoodie.clustering.execution.strategy.class"}),"\nto ",(0,s.jsx)(t.code,{children:"org.apache.hudi.client.clustering.run.strategy.JavaSortAndSizeExecutionStrategy"}),"."]}),"\n",(0,s.jsxs)(t.li,{children:[(0,s.jsx)(t.code,{children:"SPARK_CONSISTENT_BUCKET_EXECUTION_STRATEGY"}),": As the name implies, this is applicable to dynamically expand\nconsistent bucket index and only applicable to the Spark engine. Set ",(0,s.jsx)(t.code,{children:"hoodie.clustering.execution.strategy.class"}),"\nto ",(0,s.jsx)(t.code,{children:"org.apache.hudi.client.clustering.run.strategy.SparkConsistentBucketClusteringExecutionStrategy"}),"."]}),"\n"]}),"\n",(0,s.jsx)(t.h3,{id:"update-strategy",children:"Update Strategy"}),"\n",(0,s.jsxs)(t.p,{children:["Currently, clustering can only be scheduled for tables/partitions not receiving any concurrent updates. By default,\nthe config for update strategy - ",(0,s.jsx)(t.a,{href:"/docs/configurations/#hoodieclusteringupdatesstrategy",children:(0,s.jsx)(t.code,{children:"hoodie.clustering.updates.strategy"})})," is set to ",(0,s.jsx)(t.em,{children:(0,s.jsx)(t.strong,{children:"SparkRejectUpdateStrategy"})}),". If some file group has updates during clustering then it will reject updates and throw an\nexception. However, in some use-cases updates are very sparse and do not touch most file groups. The default strategy to\nsimply reject updates does not seem fair. In such use-cases, users can set the config to ",(0,s.jsx)(t.em,{children:(0,s.jsx)(t.strong,{children:"SparkAllowUpdateStrategy"})}),"."]}),"\n",(0,s.jsxs)(t.p,{children:["We discussed the critical strategy configurations. All other configurations related to clustering are\nlisted ",(0,s.jsx)(t.a,{href:"/docs/configurations/#Clustering-Configs",children:"here"}),". Out of this list, a few configurations that will be very useful\nfor inline or async clustering are shown below with code samples."]}),"\n",(0,s.jsx)(t.h2,{id:"inline-clustering",children:"Inline clustering"}),"\n",(0,s.jsx)(t.p,{children:"Inline clustering happens synchronously with the regular ingestion writer or as part of the data ingestion pipeline. This means the next round of ingestion cannot proceed until the clustering is complete With inline clustering, Hudi will schedule, plan clustering operations after each commit is completed and execute the clustering plans after it\u2019s created. This is the simplest deployment model to run because it\u2019s easier to manage than running different asynchronous Spark jobs. This mode is supported on Spark Datasource, Flink, Spark-SQL and DeltaStreamer in a sync-once mode."}),"\n",(0,s.jsxs)(t.p,{children:["For this deployment mode, please enable and set: ",(0,s.jsx)(t.code,{children:"hoodie.clustering.inline"})]}),"\n",(0,s.jsxs)(t.p,{children:["To choose how often clustering is triggered, also set: ",(0,s.jsx)(t.code,{children:"hoodie.clustering.inline.max.commits"}),"."]}),"\n",(0,s.jsx)(t.p,{children:"Inline clustering can be setup easily using spark dataframe options.\nSee sample below:"}),"\n",(0,s.jsx)(t.pre,{children:(0,s.jsx)(t.code,{className:"language-scala",children:'import org.apache.hudi.QuickstartUtils._\nimport scala.collection.JavaConversions._\nimport org.apache.spark.sql.SaveMode._\nimport org.apache.hudi.DataSourceReadOptions._\nimport org.apache.hudi.DataSourceWriteOptions._\nimport org.apache.hudi.config.HoodieWriteConfig._\n\n\nval df = //generate data frame\ndf.write.format("org.apache.hudi").\n options(getQuickstartWriteConfigs).\n option("hoodie.datasource.write.precombine.field", "ts").\n option("hoodie.datasource.write.recordkey.field", "uuid").\n option("hoodie.datasource.write.partitionpath.field", "partitionpath").\n option("hoodie.table.name", "tableName").\n option("hoodie.parquet.small.file.limit", "0").\n option("hoodie.clustering.inline", "true").\n option("hoodie.clustering.inline.max.commits", "4").\n option("hoodie.clustering.plan.strategy.target.file.max.bytes", "1073741824").\n option("hoodie.clustering.plan.strategy.small.file.limit", "629145600").\n option("hoodie.clustering.plan.strategy.sort.columns", "column1,column2"). //optional, if sorting is needed as part of rewriting data\n mode(Append).\n save("dfs://location");\n'})}),"\n",(0,s.jsx)(t.h2,{id:"async-clustering",children:"Async Clustering"}),"\n",(0,s.jsx)(t.p,{children:"Async clustering runs the clustering table service in the background without blocking the regular ingestions writers. There are three different ways to deploy an asynchronous clustering process:"}),"\n",(0,s.jsxs)(t.ul,{children:["\n",(0,s.jsxs)(t.li,{children:[(0,s.jsx)(t.strong,{children:"Asynchronous execution within the same process"}),": In this deployment mode, Hudi will schedule and plan the clustering operations after each commit is completed as part of the ingestion pipeline. Separately, Hudi spins up another thread within the same job and executes the clustering table service. This is supported by Spark Streaming, Flink and DeltaStreamer in continuous mode. For this deployment mode, please enable ",(0,s.jsx)(t.code,{children:"hoodie.clustering.async.enabled"})," and ",(0,s.jsx)(t.code,{children:"hoodie.clustering.async.max.commits\u200b"}),"."]}),"\n",(0,s.jsxs)(t.li,{children:[(0,s.jsx)(t.strong,{children:"Asynchronous scheduling and execution by a separate process"}),": In this deployment mode, the application will write data to a Hudi table as part of the ingestion pipeline. A separate clustering job will schedule, plan and execute the clustering operation. By running a different job for the clustering operation, it rebalances how Hudi uses compute resources: fewer compute resources are needed for the ingestion, which makes ingestion latency stable, and an independent set of compute resources are reserved for the clustering process. Please configure the lock providers for the concurrency control among all jobs (both writer and table service jobs). In general, configure lock providers when there are two different jobs or two different processes occurring. All writers support this deployment model. For this deployment mode, no clustering configs should be set for the ingestion writer."]}),"\n",(0,s.jsxs)(t.li,{children:[(0,s.jsx)(t.strong,{children:"Scheduling inline and executing async"}),": In this deployment mode, the application ingests data and schedules the clustering in one job; in another, the application executes the clustering plan. The supported writers (see below) won\u2019t be blocked from ingesting data. If the metadata table is enabled, a lock provider is not needed. However, if the metadata table is enabled, please ensure all jobs have the lock providers configured for concurrency control. All writers support this deployment option. For this deployment mode, please enable, ",(0,s.jsx)(t.code,{children:"hoodie.clustering.schedule.inline"})," and ",(0,s.jsx)(t.code,{children:"hoodie.clustering.async.enabled"}),"."]}),"\n"]}),"\n",(0,s.jsxs)(t.p,{children:["Hudi supports ",(0,s.jsx)(t.a,{href:"https://hudi.apache.org/docs/concurrency_control#enabling-multi-writing",children:"multi-writers"})," which provides\nsnapshot isolation between multiple table services, thus allowing writers to continue with ingestion while clustering\nruns in the background."]}),"\n",(0,s.jsxs)(t.table,{children:[(0,s.jsx)(t.thead,{children:(0,s.jsxs)(t.tr,{children:[(0,s.jsx)(t.th,{children:"Config Name"}),(0,s.jsx)(t.th,{children:"Default"}),(0,s.jsx)(t.th,{children:"Description"})]})}),(0,s.jsxs)(t.tbody,{children:[(0,s.jsxs)(t.tr,{children:[(0,s.jsx)(t.td,{children:"hoodie.clustering.async.enabled"}),(0,s.jsx)(t.td,{children:"false (Optional)"}),(0,s.jsxs)(t.td,{children:["Enable running of clustering service, asynchronously as inserts happen on the table.",(0,s.jsx)("br",{}),(0,s.jsx)("br",{}),(0,s.jsx)(t.code,{children:"Config Param: ASYNC_CLUSTERING_ENABLE"}),(0,s.jsx)("br",{}),(0,s.jsx)(t.code,{children:"Since Version: 0.7.0"})]})]}),(0,s.jsxs)(t.tr,{children:[(0,s.jsx)(t.td,{children:"hoodie.clustering.async.max.commits"}),(0,s.jsx)(t.td,{children:"4 (Optional)"}),(0,s.jsxs)(t.td,{children:["Config to control frequency of async clustering",(0,s.jsx)("br",{}),(0,s.jsx)("br",{}),(0,s.jsx)(t.code,{children:"Config Param: ASYNC_CLUSTERING_MAX_COMMITS"}),(0,s.jsx)("br",{}),(0,s.jsx)(t.code,{children:"Since Version: 0.9.0"})]})]})]})]}),"\n",(0,s.jsx)(t.h2,{id:"setup-asynchronous-clustering",children:"Setup Asynchronous Clustering"}),"\n",(0,s.jsxs)(t.p,{children:["Users can leverage ",(0,s.jsx)(t.a,{href:"https://cwiki.apache.org/confluence/display/HUDI/RFC+-+19+Clustering+data+for+freshness+and+query+performance#RFC19Clusteringdataforfreshnessandqueryperformance-SetupforAsyncclusteringJob",children:"HoodieClusteringJob"}),"\nto setup 2-step asynchronous clustering."]}),"\n",(0,s.jsx)(t.h3,{id:"hoodieclusteringjob",children:"HoodieClusteringJob"}),"\n",(0,s.jsxs)(t.p,{children:["By specifying the ",(0,s.jsx)(t.code,{children:"scheduleAndExecute"})," mode both schedule as well as clustering can be achieved in the same step.\nThe appropriate mode can be specified using ",(0,s.jsx)(t.code,{children:"-mode"})," or ",(0,s.jsx)(t.code,{children:"-m"})," option. There are three modes:"]}),"\n",(0,s.jsxs)(t.ol,{children:["\n",(0,s.jsxs)(t.li,{children:[(0,s.jsx)(t.code,{children:"schedule"}),": Make a clustering plan. This gives an instant which can be passed in execute mode."]}),"\n",(0,s.jsxs)(t.li,{children:[(0,s.jsx)(t.code,{children:"execute"}),": Execute a clustering plan at a particular instant. If no instant-time is specified, HoodieClusteringJob will execute for the earliest instant on the Hudi timeline."]}),"\n",(0,s.jsxs)(t.li,{children:[(0,s.jsx)(t.code,{children:"scheduleAndExecute"}),": Make a clustering plan first and execute that plan immediately."]}),"\n"]}),"\n",(0,s.jsx)(t.p,{children:"Note that to run this job while the original writer is still running, please enable multi-writing:"}),"\n",(0,s.jsx)(t.pre,{children:(0,s.jsx)(t.code,{children:"hoodie.write.concurrency.mode=optimistic_concurrency_control\nhoodie.write.lock.provider=org.apache.hudi.client.transaction.lock.ZookeeperBasedLockProvider\n"})}),"\n",(0,s.jsx)(t.p,{children:"A sample spark-submit command to setup HoodieClusteringJob is as below:"}),"\n",(0,s.jsx)(t.pre,{children:(0,s.jsx)(t.code,{className:"language-bash",children:'spark-submit \\\n--jars "packaging/hudi-utilities-slim-bundle/target/hudi-utilities-slim-bundle_2.12-1.0.0.jar,packaging/hudi-spark-bundle/target/hudi-spark3.5-bundle_2.12-1.0.0.jar" \\\n--class org.apache.hudi.utilities.HoodieClusteringJob \\\n/path/to/hudi-utilities-slim-bundle/target/hudi-utilities-slim-bundle_2.12-1.0.0.jar \\\n--props /path/to/config/clusteringjob.properties \\\n--mode scheduleAndExecute \\\n--base-path /path/to/hudi_table/basePath \\\n--table-name hudi_table_schedule_clustering \\\n--spark-memory 1g\n'})}),"\n",(0,s.jsxs)(t.p,{children:["A sample ",(0,s.jsx)(t.code,{children:"clusteringjob.properties"})," file:"]}),"\n",(0,s.jsx)(t.pre,{children:(0,s.jsx)(t.code,{children:"hoodie.clustering.async.enabled=true\nhoodie.clustering.async.max.commits=4\nhoodie.clustering.plan.strategy.target.file.max.bytes=1073741824\nhoodie.clustering.plan.strategy.small.file.limit=629145600\nhoodie.clustering.execution.strategy.class=org.apache.hudi.client.clustering.run.strategy.SparkSortAndSizeExecutionStrategy\nhoodie.clustering.plan.strategy.sort.columns=column1,column2\n"})}),"\n",(0,s.jsx)(t.h3,{id:"hoodiestreamer",children:"HoodieStreamer"}),"\n",(0,s.jsxs)(t.p,{children:["This brings us to our users' favorite utility in Hudi. Now, we can trigger asynchronous clustering with Hudi Streamer.\nJust set the ",(0,s.jsx)(t.code,{children:"hoodie.clustering.async.enabled"})," config to true and specify other clustering config in properties file\nwhose location can be pased as ",(0,s.jsx)(t.code,{children:"\u2014props"})," when starting the Hudi Streamer (just like in the case of HoodieClusteringJob)."]}),"\n",(0,s.jsx)(t.p,{children:"A sample spark-submit command to setup HoodieStreamer is as below:"}),"\n",(0,s.jsx)(t.pre,{children:(0,s.jsx)(t.code,{className:"language-bash",children:'spark-submit \\\n--jars "packaging/hudi-utilities-slim-bundle/target/hudi-utilities-slim-bundle_2.12-1.0.0.jar,packaging/hudi-spark-bundle/target/hudi-spark3.5-bundle_2.12-1.0.0.jar" \\\n--class org.apache.hudi.utilities.streamer.HoodieStreamer \\\n/path/to/hudi-utilities-slim-bundle/target/hudi-utilities-slim-bundle_2.12-1.0.0.jar \\\n--props /path/to/config/clustering_kafka.properties \\\n--schemaprovider-class org.apache.hudi.utilities.schema.SchemaRegistryProvider \\\n--source-class org.apache.hudi.utilities.sources.AvroKafkaSource \\\n--source-ordering-field impresssiontime \\\n--table-type COPY_ON_WRITE \\\n--target-base-path /path/to/hudi_table/basePath \\\n--target-table impressions_cow_cluster \\\n--op INSERT \\\n--hoodie-conf hoodie.clustering.async.enabled=true \\\n--continuous\n'})}),"\n",(0,s.jsx)(t.h3,{id:"spark-structured-streaming",children:"Spark Structured Streaming"}),"\n",(0,s.jsx)(t.p,{children:"We can also enable asynchronous clustering with Spark structured streaming sink as shown below."}),"\n",(0,s.jsx)(t.pre,{children:(0,s.jsx)(t.code,{className:"language-scala",children:'val commonOpts = Map(\n "hoodie.insert.shuffle.parallelism" -> "4",\n "hoodie.upsert.shuffle.parallelism" -> "4",\n "hoodie.datasource.write.recordkey.field" -> "_row_key",\n "hoodie.datasource.write.partitionpath.field" -> "partition",\n "hoodie.datasource.write.precombine.field" -> "timestamp",\n "hoodie.table.name" -> "hoodie_test"\n)\n\ndef getAsyncClusteringOpts(isAsyncClustering: String, \n clusteringNumCommit: String, \n executionStrategy: String):Map[String, String] = {\n commonOpts + (DataSourceWriteOptions.ASYNC_CLUSTERING_ENABLE.key -> isAsyncClustering,\n HoodieClusteringConfig.ASYNC_CLUSTERING_MAX_COMMITS.key -> clusteringNumCommit,\n HoodieClusteringConfig.EXECUTION_STRATEGY_CLASS_NAME.key -> executionStrategy\n )\n}\n\ndef initStreamingWriteFuture(hudiOptions: Map[String, String]): Future[Unit] = {\n val streamingInput = // define the source of streaming\n Future {\n println("streaming starting")\n streamingInput\n .writeStream\n .format("org.apache.hudi")\n .options(hudiOptions)\n .option("checkpointLocation", basePath + "/checkpoint")\n .mode(Append)\n .start()\n .awaitTermination(10000)\n println("streaming ends")\n }\n}\n\ndef structuredStreamingWithClustering(): Unit = {\n val df = //generate data frame\n val hudiOptions = getClusteringOpts("true", "1", "org.apache.hudi.client.clustering.run.strategy.SparkSortAndSizeExecutionStrategy")\n val f1 = initStreamingWriteFuture(hudiOptions)\n Await.result(f1, Duration.Inf)\n}\n'})}),"\n",(0,s.jsx)(t.h2,{id:"java-client",children:"Java Client"}),"\n",(0,s.jsxs)(t.p,{children:["Clustering is also supported via Java client. Plan strategy ",(0,s.jsx)(t.code,{children:"org.apache.hudi.client.clustering.plan.strategy.JavaSizeBasedClusteringPlanStrategy"}),"\nand execution strategy ",(0,s.jsx)(t.code,{children:"org.apache.hudi.client.clustering.run.strategy.JavaSortAndSizeExecutionStrategy"})," are supported\nout-of-the-box. Note that as of now only linear sort is supported in Java execution strategy."]}),"\n",(0,s.jsx)(t.h2,{id:"related-resources",children:"Related Resources"}),"\n",(0,s.jsx)("h3",{children:"Videos"}),"\n",(0,s.jsxs)(t.ul,{children:["\n",(0,s.jsx)(t.li,{children:(0,s.jsx)(t.a,{href:"https://www.youtube.com/watch?v=R_sm4wlGXuE",children:"Understanding Clustering in Apache Hudi and the Benefits of Asynchronous Clustering"})}),"\n"]})]})}function u(e={}){const{wrapper:t}={...(0,r.R)(),...e.components};return t?(0,s.jsx)(t,{...e,children:(0,s.jsx)(d,{...e})}):d(e)}},7004:(e,t,i)=>{i.d(t,{A:()=>n});const n=i.p+"assets/images/clustering1_new-d67c9e691d235b140f7c80d68400f425.png"},67491:(e,t,i)=>{i.d(t,{A:()=>n});const n=i.p+"assets/images/clustering2_new-0837c07b6db44ab75873633f0eab2e2c.png"},14438:(e,t,i)=>{i.d(t,{A:()=>n});const n=i.p+"assets/images/clustering_3-8bc286ab44c48137f8409b5c342a7207.png"},28453:(e,t,i)=>{i.d(t,{R:()=>a,x:()=>l});var n=i(96540);const s={},r=n.createContext(s);function a(e){const t=n.useContext(r);return n.useMemo((function(){return"function"==typeof e?e(t):{...t,...e}}),[t,e])}function l(e){let t;return t=e.disableParentContext?"function"==typeof e.components?e.components(s):e.components||s:a(e.components),n.createElement(r.Provider,{value:t},e.children)}}}]);
\ No newline at end of file
+"use strict";(self.webpackChunkhudi=self.webpackChunkhudi||[]).push([[34691],{28372:(e,t,i)=>{i.r(t),i.d(t,{assets:()=>o,contentTitle:()=>l,default:()=>u,frontMatter:()=>a,metadata:()=>n,toc:()=>c});const n=JSON.parse('{"id":"clustering","title":"Clustering","description":"Background","source":"@site/docs/clustering.md","sourceDirName":".","slug":"/clustering","permalink":"/docs/next/clustering","draft":false,"unlisted":false,"editUrl":"https://github.com/apache/hudi/tree/asf-site/website/docs/clustering.md","tags":[],"version":"current","frontMatter":{"title":"Clustering","summary":"In this page, we describe async compaction in Hudi.","toc":true,"last_modified_at":null},"sidebar":"docs","previous":{"title":"Compaction","permalink":"/docs/next/compaction"},"next":{"title":"Indexing","permalink":"/docs/next/metadata_indexing"}}');var s=i(74848),r=i(28453);const a={title:"Clustering",summary:"In this page, we describe async compaction in Hudi.",toc:!0,last_modified_at:null},l=void 0,o={},c=[{value:"Background",id:"background",level:2},{value:"How is compaction different from clustering?",id:"how-is-compaction-different-from-clustering",level:2},{value:"Clustering Architecture",id:"clustering-architecture",level:2},{value:"Overall, there are 2 steps to clustering",id:"overall-there-are-2-steps-to-clustering",level:3},{value:"Schedule clustering",id:"schedule-clustering",level:3},{value:"Execute clustering",id:"execute-clustering",level:3},{value:"Clustering Usecases",id:"clustering-usecases",level:2},{value:"Batching small files",id:"batching-small-files",level:3},{value:"Cluster by sort key",id:"cluster-by-sort-key",level:3},{value:"Clustering Strategies",id:"clustering-strategies",level:2},{value:"Plan Strategy",id:"plan-strategy",level:3},{value:"Size-based clustering strategies",id:"size-based-clustering-strategies",level:4},{value:"SparkSingleFileSortPlanStrategy",id:"sparksinglefilesortplanstrategy",level:4},{value:"SparkConsistentBucketClusteringPlanStrategy",id:"sparkconsistentbucketclusteringplanstrategy",level:4},{value:"Execution Strategy",id:"execution-strategy",level:3},{value:"Update Strategy",id:"update-strategy",level:3},{value:"Inline clustering",id:"inline-clustering",level:2},{value:"Async Clustering",id:"async-clustering",level:2},{value:"Setup Asynchronous Clustering",id:"setup-asynchronous-clustering",level:2},{value:"HoodieClusteringJob",id:"hoodieclusteringjob",level:3},{value:"HoodieStreamer",id:"hoodiestreamer",level:3},{value:"Spark Structured Streaming",id:"spark-structured-streaming",level:3},{value:"Java Client",id:"java-client",level:2},{value:"Related Resources",id:"related-resources",level:2}];function d(e){const t={a:"a",admonition:"admonition",code:"code",em:"em",h2:"h2",h3:"h3",h4:"h4",img:"img",li:"li",ol:"ol",p:"p",pre:"pre",strong:"strong",table:"table",tbody:"tbody",td:"td",th:"th",thead:"thead",tr:"tr",ul:"ul",...(0,r.R)(),...e.components};return(0,s.jsxs)(s.Fragment,{children:[(0,s.jsx)(t.h2,{id:"background",children:"Background"}),"\n",(0,s.jsxs)(t.p,{children:["Apache Hudi brings stream processing to big data, providing fresh data while being an order of magnitude efficient over traditional batch processing. In a data lake/warehouse, one of the key trade-offs is between ingestion speed and query performance. Data ingestion typically prefers small files to improve parallelism and make data available to queries as soon as possible. However, query performance degrades poorly with a lot of small files. Also, during ingestion, data is typically co-located based on arrival time. However, the query engines perform better when the data frequently queried is co-located together. In most architectures each of these systems tend to add optimizations independently to improve performance which hits limitations due to un-optimized data layouts. This doc introduces a new kind of table service called clustering ",(0,s.jsx)(t.a,{href:"https://cwiki.apache.org/confluence/display/HUDI/RFC+-+19+Clustering+data+for+freshness+and+query+performance",children:"[RFC-19]"})," to reorganize data for improved query performance without compromising on ingestion speed."]}),"\n",(0,s.jsx)(t.h2,{id:"how-is-compaction-different-from-clustering",children:"How is compaction different from clustering?"}),"\n",(0,s.jsxs)(t.p,{children:["Hudi is modeled like a log-structured storage engine with multiple versions of the data.\nParticularly, ",(0,s.jsx)(t.a,{href:"/docs/table_types#merge-on-read-table",children:"Merge-On-Read"}),"\ntables in Hudi store data using a combination of base file in columnar format and row-based delta logs that contain\nupdates. Compaction is a way to merge the delta logs with base files to produce the latest file slices with the most\nrecent snapshot of data. Compaction helps to keep the query performance in check (larger delta log files would incur\nlonger merge times on query side). On the other hand, clustering is a data layout optimization technique. One can stitch\ntogether small files into larger files using clustering. Additionally, data can be clustered by sort key so that queries\ncan take advantage of data locality."]}),"\n",(0,s.jsx)(t.h2,{id:"clustering-architecture",children:"Clustering Architecture"}),"\n",(0,s.jsxs)(t.p,{children:["At a high level, Hudi provides different operations such as insert/upsert/bulk_insert through it\u2019s write client API to be able to write data to a Hudi table. To be able to choose a trade-off between file size and ingestion speed, Hudi provides a knob ",(0,s.jsx)(t.code,{children:"hoodie.parquet.small.file.limit"})," to be able to configure the smallest allowable file size. Users are able to configure the small file ",(0,s.jsx)(t.a,{href:"https://hudi.apache.org/docs/configurations/#hoodieparquetsmallfilelimit",children:"soft limit"})," to ",(0,s.jsx)(t.code,{children:"0"})," to force new data to go into a new set of filegroups or set it to a higher value to ensure new data gets \u201cpadded\u201d to existing files until it meets that limit that adds to ingestion latencies."]}),"\n",(0,s.jsx)(t.p,{children:"To be able to support an architecture that allows for fast ingestion without compromising query performance, we have introduced a \u2018clustering\u2019 service to rewrite the data to optimize Hudi data lake file layout."}),"\n",(0,s.jsx)(t.p,{children:"Clustering table service can run asynchronously or synchronously adding a new action type called \u201cREPLACE\u201d, that will mark the clustering action in the Hudi metadata timeline."}),"\n",(0,s.jsx)(t.h3,{id:"overall-there-are-2-steps-to-clustering",children:"Overall, there are 2 steps to clustering"}),"\n",(0,s.jsxs)(t.ol,{children:["\n",(0,s.jsx)(t.li,{children:"Scheduling clustering: Create a clustering plan using a pluggable clustering strategy."}),"\n",(0,s.jsx)(t.li,{children:"Execute clustering: Process the plan using an execution strategy to create new files and replace old files."}),"\n"]}),"\n",(0,s.jsx)(t.h3,{id:"schedule-clustering",children:"Schedule clustering"}),"\n",(0,s.jsx)(t.p,{children:"Following steps are followed to schedule clustering."}),"\n",(0,s.jsxs)(t.ol,{children:["\n",(0,s.jsx)(t.li,{children:"Identify files that are eligible for clustering: Depending on the clustering strategy chosen, the scheduling logic will identify the files eligible for clustering."}),"\n",(0,s.jsx)(t.li,{children:"Group files that are eligible for clustering based on specific criteria. Each group is expected to have data size in multiples of \u2018targetFileSize\u2019. Grouping is done as part of \u2018strategy\u2019 defined in the plan. Additionally, there is an option to put a cap on group size to improve parallelism and avoid shuffling large amounts of data."}),"\n",(0,s.jsxs)(t.li,{children:["Finally, the clustering plan is saved to the timeline in an avro ",(0,s.jsx)(t.a,{href:"https://github.com/apache/hudi/blob/master/hudi-common/src/main/avro/HoodieClusteringPlan.avsc",children:"metadata format"}),"."]}),"\n"]}),"\n",(0,s.jsx)(t.h3,{id:"execute-clustering",children:"Execute clustering"}),"\n",(0,s.jsxs)(t.ol,{children:["\n",(0,s.jsx)(t.li,{children:"Read the clustering plan and get the \u2018clusteringGroups\u2019 that mark the file groups that need to be clustered."}),"\n",(0,s.jsx)(t.li,{children:"For each group, we instantiate appropriate strategy class with strategyParams (example: sortColumns) and apply that strategy to rewrite the data."}),"\n",(0,s.jsxs)(t.li,{children:["Create a \u201cREPLACE\u201d commit and update the metadata in ",(0,s.jsx)(t.a,{href:"https://github.com/apache/hudi/blob/master/hudi-common/src/main/java/org/apache/hudi/common/model/HoodieReplaceCommitMetadata.java",children:"HoodieReplaceCommitMetadata"}),"."]}),"\n"]}),"\n",(0,s.jsx)(t.p,{children:"Clustering Service builds on Hudi\u2019s MVCC based design to allow for writers to continue to insert new data while clustering action runs in the background to reformat data layout, ensuring snapshot isolation between concurrent readers and writers."}),"\n",(0,s.jsx)(t.p,{children:"NOTE: Clustering can only be scheduled for tables / partitions not receiving any concurrent updates. In the future, concurrent updates use-case will be supported as well."}),"\n",(0,s.jsxs)(t.p,{children:[(0,s.jsx)(t.img,{alt:"Clustering example",src:i(7004).A+"",width:"6827",height:"3334"}),"\n",(0,s.jsx)(t.em,{children:"Figure: Illustrating query performance improvements by clustering"})]}),"\n",(0,s.jsx)(t.h2,{id:"clustering-usecases",children:"Clustering Usecases"}),"\n",(0,s.jsx)(t.h3,{id:"batching-small-files",children:"Batching small files"}),"\n",(0,s.jsx)(t.p,{children:"As mentioned in the intro, streaming ingestion generally results in smaller files in your data lake. But having a lot of\nsuch small files could lead to higher query latency. From our experience supporting community users, there are quite a\nfew users who are using Hudi just for small file handling capabilities. So, you could employ clustering to batch a lot\nof such small files into larger ones."}),"\n",(0,s.jsx)(t.p,{children:(0,s.jsx)(t.img,{alt:"Batching small files",src:i(67491).A+"",width:"3100",height:"1620"})}),"\n",(0,s.jsx)(t.h3,{id:"cluster-by-sort-key",children:"Cluster by sort key"}),"\n",(0,s.jsx)(t.p,{children:"Another classic problem in data lake is the arrival time vs event time problem. Generally you write data based on\narrival time, while query predicates do not sit well with it. With clustering, you can re-write your data by sorting\nbased on query predicates and so, your data skipping will be very efficient and your query can ignore scanning a lot of\nunnecessary data."}),"\n",(0,s.jsx)(t.p,{children:(0,s.jsx)(t.img,{alt:"Batching small files",src:i(14438).A+"",width:"5003",height:"1464"})}),"\n",(0,s.jsx)(t.h2,{id:"clustering-strategies",children:"Clustering Strategies"}),"\n",(0,s.jsx)(t.p,{children:"On a high level, clustering creates a plan based on a configurable strategy, groups eligible files based on specific\ncriteria and then executes the plan. As mentioned before, clustering plan as well as execution depends on configurable\nstrategy. These strategies can be broadly classified into three types: clustering plan strategy, execution strategy and\nupdate strategy."}),"\n",(0,s.jsx)(t.h3,{id:"plan-strategy",children:"Plan Strategy"}),"\n",(0,s.jsxs)(t.p,{children:["This strategy comes into play while creating clustering plan. It helps to decide what file groups should be clustered\nand how many output file groups should the clustering produce. Note that these strategies are easily pluggable using the\nconfig ",(0,s.jsx)(t.a,{href:"/docs/configurations#hoodieclusteringplanstrategyclass",children:"hoodie.clustering.plan.strategy.class"}),"."]}),"\n",(0,s.jsx)(t.p,{children:"Different plan strategies are as follows:"}),"\n",(0,s.jsx)(t.h4,{id:"size-based-clustering-strategies",children:"Size-based clustering strategies"}),"\n",(0,s.jsxs)(t.p,{children:["This strategy creates clustering groups based on max size allowed per group. Also, it excludes files that are greater\nthan the small file limit from the clustering plan. Available strategies depending on write client\nare: ",(0,s.jsx)(t.code,{children:"SparkSizeBasedClusteringPlanStrategy"}),", ",(0,s.jsx)(t.code,{children:"FlinkSizeBasedClusteringPlanStrategy"}),"\nand ",(0,s.jsx)(t.code,{children:"JavaSizeBasedClusteringPlanStrategy"}),". Furthermore, Hudi provides flexibility to include or exclude partitions for\nclustering, tune the file size limits, maximum number of output groups. Please refer to ",(0,s.jsx)(t.a,{href:"https://hudi.apache.org/docs/next/configurations/#hoodieclusteringplanstrategysmallfilelimit",children:"hoodie.clustering.plan.strategy.small.file.limit"}),"\n, ",(0,s.jsx)(t.a,{href:"https://hudi.apache.org/docs/next/configurations/#hoodieclusteringplanstrategymaxnumgroups",children:"hoodie.clustering.plan.strategy.max.num.groups"}),", ",(0,s.jsx)(t.a,{href:"https://hudi.apache.org/docs/next/configurations/#hoodieclusteringplanstrategymaxbytespergroup",children:"hoodie.clustering.plan.strategy.max.bytes.per.group"}),"\n, ",(0,s.jsx)(t.a,{href:"https://hudi.apache.org/docs/next/configurations/#hoodieclusteringplanstrategytargetfilemaxbytes",children:"hoodie.clustering.plan.strategy.target.file.max.bytes"})," for more details."]}),"\n",(0,s.jsxs)(t.table,{children:[(0,s.jsx)(t.thead,{children:(0,s.jsxs)(t.tr,{children:[(0,s.jsx)(t.th,{children:"Config Name"}),(0,s.jsx)(t.th,{children:"Default"}),(0,s.jsx)(t.th,{children:"Description"})]})}),(0,s.jsxs)(t.tbody,{children:[(0,s.jsxs)(t.tr,{children:[(0,s.jsx)(t.td,{children:"hoodie.clustering.plan.strategy.partition.selected"}),(0,s.jsxs)(t.td,{children:["N/A ",(0,s.jsx)(t.strong,{children:"(Required)"})]}),(0,s.jsxs)(t.td,{children:["Comma separated list of partitions to run clustering",(0,s.jsx)("br",{}),(0,s.jsx)("br",{}),(0,s.jsx)(t.code,{children:"Config Param: PARTITION_SELECTED"}),(0,s.jsx)("br",{}),(0,s.jsx)(t.code,{children:"Since Version: 0.11.0"})]})]}),(0,s.jsxs)(t.tr,{children:[(0,s.jsx)(t.td,{children:"hoodie.clustering.plan.strategy.partition.regex.pattern"}),(0,s.jsxs)(t.td,{children:["N/A ",(0,s.jsx)(t.strong,{children:"(Required)"})]}),(0,s.jsxs)(t.td,{children:["Filter clustering partitions that matched regex pattern",(0,s.jsx)("br",{}),(0,s.jsx)("br",{}),(0,s.jsx)(t.code,{children:"Config Param: PARTITION_REGEX_PATTERN"}),(0,s.jsx)("br",{}),(0,s.jsx)(t.code,{children:"Since Version: 0.11.0"})]})]}),(0,s.jsxs)(t.tr,{children:[(0,s.jsx)(t.td,{children:"hoodie.clustering.plan.partition.filter.mode"}),(0,s.jsx)(t.td,{children:"NONE (Optional)"}),(0,s.jsxs)(t.td,{children:["Partition filter mode used in the creation of clustering plan. Possible values:",(0,s.jsx)("br",{}),(0,s.jsxs)("ul",{children:[(0,s.jsxs)("li",{children:[(0,s.jsx)(t.code,{children:"NONE"}),": Do not filter partitions. The clustering plan will include all partitions that have clustering candidates."]}),(0,s.jsxs)("li",{children:[(0,s.jsx)(t.code,{children:"RECENT_DAYS"}),": This filter assumes that your data is partitioned by date. The clustering plan will only include partitions from K days ago to N days ago, where K >= N. K is determined by ",(0,s.jsx)(t.code,{children:"hoodie.clustering.plan.strategy.daybased.lookback.partitions"})," and N is determined by ",(0,s.jsx)(t.code,{children:"hoodie.clustering.plan.strategy.daybased.skipfromlatest.partitions"}),"."]}),(0,s.jsxs)("li",{children:[(0,s.jsx)(t.code,{children:"SELECTED_PARTITIONS"}),": The clustering plan will include only partition paths with names that sort within the inclusive range [",(0,s.jsx)(t.code,{children:"hoodie.clustering.plan.strategy.cluster.begin.partition"}),", ",(0,s.jsx)(t.code,{children:"hoodie.clustering.plan.strategy.cluster.end.partition"}),"]."]}),(0,s.jsxs)("li",{children:[(0,s.jsx)(t.code,{children:"DAY_ROLLING"}),": To determine the partitions in the clustering plan, the eligible partitions will be sorted in ascending order. Each partition will have an index i in that list. The clustering plan will only contain partitions such that i mod 24 = H, where H is the current hour of the day (from 0 to 23)."]})]}),(0,s.jsx)("br",{}),(0,s.jsx)(t.code,{children:"Config Param: PLAN_PARTITION_FILTER_MODE_NAME"}),(0,s.jsx)("br",{}),(0,s.jsx)(t.code,{children:"Since Version: 0.11.0"})]})]})]})]}),"\n",(0,s.jsx)(t.h4,{id:"sparksinglefilesortplanstrategy",children:"SparkSingleFileSortPlanStrategy"}),"\n",(0,s.jsxs)(t.p,{children:["In this strategy, clustering group for each partition is built in the same way as ",(0,s.jsx)(t.code,{children:"SparkSizeBasedClusteringPlanStrategy"}),"\n. The difference is that the output group is 1 and file group id remains the same,\nwhile ",(0,s.jsx)(t.code,{children:"SparkSizeBasedClusteringPlanStrategy"})," can create multiple file groups with newer fileIds."]}),"\n",(0,s.jsx)(t.h4,{id:"sparkconsistentbucketclusteringplanstrategy",children:"SparkConsistentBucketClusteringPlanStrategy"}),"\n",(0,s.jsx)(t.p,{children:"This strategy is specifically used for consistent bucket index. This will be leveraged to expand your bucket index (from\nstatic partitioning to dynamic). Typically, users don\u2019t need to use this strategy. Hudi internally uses this for\ndynamically expanding the buckets for bucket index datasets."}),"\n",(0,s.jsx)(t.admonition,{title:"The latter two strategies are applicable only for the Spark engine.",type:"note"}),"\n",(0,s.jsx)(t.h3,{id:"execution-strategy",children:"Execution Strategy"}),"\n",(0,s.jsxs)(t.p,{children:["After building the clustering groups in the planning phase, Hudi applies execution strategy, for each group, primarily\nbased on sort columns and size. The strategy can be specified using the\nconfig ",(0,s.jsx)(t.a,{href:"/docs/configurations/#hoodieclusteringexecutionstrategyclass",children:"hoodie.clustering.execution.strategy.class"}),". By\ndefault, Hudi sorts the file groups in the plan by the specified columns, while meeting the configured target file\nsizes."]}),"\n",(0,s.jsxs)(t.table,{children:[(0,s.jsx)(t.thead,{children:(0,s.jsxs)(t.tr,{children:[(0,s.jsx)(t.th,{children:"Config Name"}),(0,s.jsx)(t.th,{children:"Default"}),(0,s.jsx)(t.th,{children:"Description"})]})}),(0,s.jsx)(t.tbody,{children:(0,s.jsxs)(t.tr,{children:[(0,s.jsx)(t.td,{children:"hoodie.clustering.execution.strategy.class"}),(0,s.jsx)(t.td,{children:"org.apache.hudi.client.clustering.run.strategy.SparkSortAndSizeExecutionStrategy (Optional)"}),(0,s.jsxs)(t.td,{children:["Config to provide a strategy class (subclass of RunClusteringStrategy) to define how the clustering plan is executed. By default, we sort the file groups in th plan by the specified columns, while meeting the configured target file sizes.",(0,s.jsx)("br",{}),(0,s.jsx)("br",{}),(0,s.jsx)(t.code,{children:"Config Param: EXECUTION_STRATEGY_CLASS_NAME"}),(0,s.jsx)("br",{}),(0,s.jsx)(t.code,{children:"Since Version: 0.7.0"})]})]})})]}),"\n",(0,s.jsx)(t.p,{children:"The available strategies are as follows:"}),"\n",(0,s.jsxs)(t.ol,{children:["\n",(0,s.jsxs)(t.li,{children:[(0,s.jsx)(t.code,{children:"SPARK_SORT_AND_SIZE_EXECUTION_STRATEGY"}),": Uses bulk_insert to re-write data from input file groups.","\n",(0,s.jsxs)(t.ol,{children:["\n",(0,s.jsxs)(t.li,{children:["Set ",(0,s.jsx)(t.code,{children:"hoodie.clustering.execution.strategy.class"}),"\nto ",(0,s.jsx)(t.code,{children:"org.apache.hudi.client.clustering.run.strategy.SparkSortAndSizeExecutionStrategy"}),"."]}),"\n",(0,s.jsxs)(t.li,{children:[(0,s.jsx)(t.code,{children:"hoodie.clustering.plan.strategy.sort.columns"}),": Columns to sort the data while clustering. This goes in\nconjunction with layout optimization strategies depending on your query predicates. One can set comma separated\nlist of columns that needs to be sorted in this config."]}),"\n"]}),"\n"]}),"\n",(0,s.jsxs)(t.li,{children:[(0,s.jsx)(t.code,{children:"JAVA_SORT_AND_SIZE_EXECUTION_STRATEGY"}),": Similar to ",(0,s.jsx)(t.code,{children:"SPARK_SORT_AND_SIZE_EXECUTION_STRATEGY"}),", for the Java and Flink\nengines. Set ",(0,s.jsx)(t.code,{children:"hoodie.clustering.execution.strategy.class"}),"\nto ",(0,s.jsx)(t.code,{children:"org.apache.hudi.client.clustering.run.strategy.JavaSortAndSizeExecutionStrategy"}),"."]}),"\n",(0,s.jsxs)(t.li,{children:[(0,s.jsx)(t.code,{children:"SPARK_CONSISTENT_BUCKET_EXECUTION_STRATEGY"}),": As the name implies, this is applicable to dynamically expand\nconsistent bucket index and only applicable to the Spark engine. Set ",(0,s.jsx)(t.code,{children:"hoodie.clustering.execution.strategy.class"}),"\nto ",(0,s.jsx)(t.code,{children:"org.apache.hudi.client.clustering.run.strategy.SparkConsistentBucketClusteringExecutionStrategy"}),"."]}),"\n"]}),"\n",(0,s.jsx)(t.h3,{id:"update-strategy",children:"Update Strategy"}),"\n",(0,s.jsxs)(t.p,{children:["Currently, clustering can only be scheduled for tables/partitions not receiving any concurrent updates. By default,\nthe config for update strategy - ",(0,s.jsx)(t.a,{href:"/docs/configurations/#hoodieclusteringupdatesstrategy",children:(0,s.jsx)(t.code,{children:"hoodie.clustering.updates.strategy"})})," is set to ",(0,s.jsx)(t.em,{children:(0,s.jsx)(t.strong,{children:"SparkRejectUpdateStrategy"})}),". If some file group has updates during clustering then it will reject updates and throw an\nexception. However, in some use-cases updates are very sparse and do not touch most file groups. The default strategy to\nsimply reject updates does not seem fair. In such use-cases, users can set the config to ",(0,s.jsx)(t.em,{children:(0,s.jsx)(t.strong,{children:"SparkAllowUpdateStrategy"})}),"."]}),"\n",(0,s.jsxs)(t.p,{children:["We discussed the critical strategy configurations. All other configurations related to clustering are\nlisted ",(0,s.jsx)(t.a,{href:"/docs/configurations/#Clustering-Configs",children:"here"}),". Out of this list, a few configurations that will be very useful\nfor inline or async clustering are shown below with code samples."]}),"\n",(0,s.jsx)(t.h2,{id:"inline-clustering",children:"Inline clustering"}),"\n",(0,s.jsx)(t.p,{children:"Inline clustering happens synchronously with the regular ingestion writer or as part of the data ingestion pipeline. This means the next round of ingestion cannot proceed until the clustering is complete With inline clustering, Hudi will schedule, plan clustering operations after each commit is completed and execute the clustering plans after it\u2019s created. This is the simplest deployment model to run because it\u2019s easier to manage than running different asynchronous Spark jobs. This mode is supported on Spark Datasource, Flink, Spark-SQL and DeltaStreamer in a sync-once mode."}),"\n",(0,s.jsxs)(t.p,{children:["For this deployment mode, please enable and set: ",(0,s.jsx)(t.code,{children:"hoodie.clustering.inline"})]}),"\n",(0,s.jsxs)(t.p,{children:["To choose how often clustering is triggered, also set: ",(0,s.jsx)(t.code,{children:"hoodie.clustering.inline.max.commits"}),"."]}),"\n",(0,s.jsx)(t.p,{children:"Inline clustering can be setup easily using spark dataframe options.\nSee sample below:"}),"\n",(0,s.jsx)(t.pre,{children:(0,s.jsx)(t.code,{className:"language-scala",children:'import org.apache.hudi.QuickstartUtils._\nimport scala.collection.JavaConversions._\nimport org.apache.spark.sql.SaveMode._\nimport org.apache.hudi.DataSourceReadOptions._\nimport org.apache.hudi.DataSourceWriteOptions._\nimport org.apache.hudi.config.HoodieWriteConfig._\n\n\nval df = //generate data frame\ndf.write.format("org.apache.hudi").\n options(getQuickstartWriteConfigs).\n option("hoodie.datasource.write.precombine.field", "ts").\n option("hoodie.datasource.write.recordkey.field", "uuid").\n option("hoodie.datasource.write.partitionpath.field", "partitionpath").\n option("hoodie.table.name", "tableName").\n option("hoodie.parquet.small.file.limit", "0").\n option("hoodie.clustering.inline", "true").\n option("hoodie.clustering.inline.max.commits", "4").\n option("hoodie.clustering.plan.strategy.target.file.max.bytes", "1073741824").\n option("hoodie.clustering.plan.strategy.small.file.limit", "629145600").\n option("hoodie.clustering.plan.strategy.sort.columns", "column1,column2"). //optional, if sorting is needed as part of rewriting data\n mode(Append).\n save("dfs://location");\n'})}),"\n",(0,s.jsx)(t.h2,{id:"async-clustering",children:"Async Clustering"}),"\n",(0,s.jsx)(t.p,{children:"Async clustering runs the clustering table service in the background without blocking the regular ingestions writers. There are three different ways to deploy an asynchronous clustering process:"}),"\n",(0,s.jsxs)(t.ul,{children:["\n",(0,s.jsxs)(t.li,{children:[(0,s.jsx)(t.strong,{children:"Asynchronous execution within the same process"}),": In this deployment mode, Hudi will schedule and plan the clustering operations after each commit is completed as part of the ingestion pipeline. Separately, Hudi spins up another thread within the same job and executes the clustering table service. This is supported by Spark Streaming, Flink and DeltaStreamer in continuous mode. For this deployment mode, please enable ",(0,s.jsx)(t.code,{children:"hoodie.clustering.async.enabled"})," and ",(0,s.jsx)(t.code,{children:"hoodie.clustering.async.max.commits\u200b"}),"."]}),"\n",(0,s.jsxs)(t.li,{children:[(0,s.jsx)(t.strong,{children:"Asynchronous scheduling and execution by a separate process"}),": In this deployment mode, the application will write data to a Hudi table as part of the ingestion pipeline. A separate clustering job will schedule, plan and execute the clustering operation. By running a different job for the clustering operation, it rebalances how Hudi uses compute resources: fewer compute resources are needed for the ingestion, which makes ingestion latency stable, and an independent set of compute resources are reserved for the clustering process. Please configure the lock providers for the concurrency control among all jobs (both writer and table service jobs). In general, configure lock providers when there are two different jobs or two different processes occurring. All writers support this deployment model. For this deployment mode, no clustering configs should be set for the ingestion writer."]}),"\n",(0,s.jsxs)(t.li,{children:[(0,s.jsx)(t.strong,{children:"Scheduling inline and executing async"}),": In this deployment mode, the application ingests data and schedules the clustering in one job; in another, the application executes the clustering plan. The supported writers (see below) won\u2019t be blocked from ingesting data. If the metadata table is enabled, a lock provider is not needed. However, if the metadata table is enabled, please ensure all jobs have the lock providers configured for concurrency control. All writers support this deployment option. For this deployment mode, please enable, ",(0,s.jsx)(t.code,{children:"hoodie.clustering.schedule.inline"})," and ",(0,s.jsx)(t.code,{children:"hoodie.clustering.async.enabled"}),"."]}),"\n"]}),"\n",(0,s.jsxs)(t.p,{children:["Hudi supports ",(0,s.jsx)(t.a,{href:"https://hudi.apache.org/docs/concurrency_control#enabling-multi-writing",children:"multi-writers"})," which provides\nsnapshot isolation between multiple table services, thus allowing writers to continue with ingestion while clustering\nruns in the background."]}),"\n",(0,s.jsxs)(t.table,{children:[(0,s.jsx)(t.thead,{children:(0,s.jsxs)(t.tr,{children:[(0,s.jsx)(t.th,{children:"Config Name"}),(0,s.jsx)(t.th,{children:"Default"}),(0,s.jsx)(t.th,{children:"Description"})]})}),(0,s.jsxs)(t.tbody,{children:[(0,s.jsxs)(t.tr,{children:[(0,s.jsx)(t.td,{children:"hoodie.clustering.async.enabled"}),(0,s.jsx)(t.td,{children:"false (Optional)"}),(0,s.jsxs)(t.td,{children:["Enable running of clustering service, asynchronously as inserts happen on the table.",(0,s.jsx)("br",{}),(0,s.jsx)("br",{}),(0,s.jsx)(t.code,{children:"Config Param: ASYNC_CLUSTERING_ENABLE"}),(0,s.jsx)("br",{}),(0,s.jsx)(t.code,{children:"Since Version: 0.7.0"})]})]}),(0,s.jsxs)(t.tr,{children:[(0,s.jsx)(t.td,{children:"hoodie.clustering.async.max.commits"}),(0,s.jsx)(t.td,{children:"4 (Optional)"}),(0,s.jsxs)(t.td,{children:["Config to control frequency of async clustering",(0,s.jsx)("br",{}),(0,s.jsx)("br",{}),(0,s.jsx)(t.code,{children:"Config Param: ASYNC_CLUSTERING_MAX_COMMITS"}),(0,s.jsx)("br",{}),(0,s.jsx)(t.code,{children:"Since Version: 0.9.0"})]})]})]})]}),"\n",(0,s.jsx)(t.h2,{id:"setup-asynchronous-clustering",children:"Setup Asynchronous Clustering"}),"\n",(0,s.jsxs)(t.p,{children:["Users can leverage ",(0,s.jsx)(t.a,{href:"https://cwiki.apache.org/confluence/display/HUDI/RFC+-+19+Clustering+data+for+freshness+and+query+performance#RFC19Clusteringdataforfreshnessandqueryperformance-SetupforAsyncclusteringJob",children:"HoodieClusteringJob"}),"\nto setup 2-step asynchronous clustering."]}),"\n",(0,s.jsx)(t.h3,{id:"hoodieclusteringjob",children:"HoodieClusteringJob"}),"\n",(0,s.jsxs)(t.p,{children:["By specifying the ",(0,s.jsx)(t.code,{children:"scheduleAndExecute"})," mode both schedule as well as clustering can be achieved in the same step.\nThe appropriate mode can be specified using ",(0,s.jsx)(t.code,{children:"-mode"})," or ",(0,s.jsx)(t.code,{children:"-m"})," option. There are three modes:"]}),"\n",(0,s.jsxs)(t.ol,{children:["\n",(0,s.jsxs)(t.li,{children:[(0,s.jsx)(t.code,{children:"schedule"}),": Make a clustering plan. This gives an instant which can be passed in execute mode."]}),"\n",(0,s.jsxs)(t.li,{children:[(0,s.jsx)(t.code,{children:"execute"}),": Execute a clustering plan at a particular instant. If no instant-time is specified, HoodieClusteringJob will execute for the earliest instant on the Hudi timeline."]}),"\n",(0,s.jsxs)(t.li,{children:[(0,s.jsx)(t.code,{children:"scheduleAndExecute"}),": Make a clustering plan first and execute that plan immediately."]}),"\n"]}),"\n",(0,s.jsx)(t.p,{children:"Note that to run this job while the original writer is still running, please enable multi-writing:"}),"\n",(0,s.jsx)(t.pre,{children:(0,s.jsx)(t.code,{children:"hoodie.write.concurrency.mode=optimistic_concurrency_control\nhoodie.write.lock.provider=org.apache.hudi.client.transaction.lock.ZookeeperBasedLockProvider\n"})}),"\n",(0,s.jsx)(t.p,{children:"A sample spark-submit command to setup HoodieClusteringJob is as below:"}),"\n",(0,s.jsx)(t.pre,{children:(0,s.jsx)(t.code,{className:"language-bash",children:'spark-submit \\\n--jars "packaging/hudi-utilities-slim-bundle/target/hudi-utilities-slim-bundle_2.12-1.0.0.jar,packaging/hudi-spark-bundle/target/hudi-spark3.5-bundle_2.12-1.0.0.jar" \\\n--class org.apache.hudi.utilities.HoodieClusteringJob \\\n/path/to/hudi-utilities-slim-bundle/target/hudi-utilities-slim-bundle_2.12-1.0.0.jar \\\n--props /path/to/config/clusteringjob.properties \\\n--mode scheduleAndExecute \\\n--base-path /path/to/hudi_table/basePath \\\n--table-name hudi_table_schedule_clustering \\\n--spark-memory 1g\n'})}),"\n",(0,s.jsxs)(t.p,{children:["A sample ",(0,s.jsx)(t.code,{children:"clusteringjob.properties"})," file:"]}),"\n",(0,s.jsx)(t.pre,{children:(0,s.jsx)(t.code,{children:"hoodie.clustering.async.enabled=true\nhoodie.clustering.async.max.commits=4\nhoodie.clustering.plan.strategy.target.file.max.bytes=1073741824\nhoodie.clustering.plan.strategy.small.file.limit=629145600\nhoodie.clustering.execution.strategy.class=org.apache.hudi.client.clustering.run.strategy.SparkSortAndSizeExecutionStrategy\nhoodie.clustering.plan.strategy.sort.columns=column1,column2\n"})}),"\n",(0,s.jsx)(t.h3,{id:"hoodiestreamer",children:"HoodieStreamer"}),"\n",(0,s.jsxs)(t.p,{children:["This brings us to our users' favorite utility in Hudi. Now, we can trigger asynchronous clustering with Hudi Streamer.\nJust set the ",(0,s.jsx)(t.code,{children:"hoodie.clustering.async.enabled"})," config to true and specify other clustering config in properties file\nwhose location can be pased as ",(0,s.jsx)(t.code,{children:"\u2014props"})," when starting the Hudi Streamer (just like in the case of HoodieClusteringJob)."]}),"\n",(0,s.jsx)(t.p,{children:"A sample spark-submit command to setup HoodieStreamer is as below:"}),"\n",(0,s.jsx)(t.pre,{children:(0,s.jsx)(t.code,{className:"language-bash",children:'spark-submit \\\n--jars "packaging/hudi-utilities-slim-bundle/target/hudi-utilities-slim-bundle_2.12-1.0.0.jar,packaging/hudi-spark-bundle/target/hudi-spark3.5-bundle_2.12-1.0.0.jar" \\\n--class org.apache.hudi.utilities.streamer.HoodieStreamer \\\n/path/to/hudi-utilities-slim-bundle/target/hudi-utilities-slim-bundle_2.12-1.0.0.jar \\\n--props /path/to/config/clustering_kafka.properties \\\n--schemaprovider-class org.apache.hudi.utilities.schema.SchemaRegistryProvider \\\n--source-class org.apache.hudi.utilities.sources.AvroKafkaSource \\\n--source-ordering-field impresssiontime \\\n--table-type COPY_ON_WRITE \\\n--target-base-path /path/to/hudi_table/basePath \\\n--target-table impressions_cow_cluster \\\n--op INSERT \\\n--hoodie-conf hoodie.clustering.async.enabled=true \\\n--continuous\n'})}),"\n",(0,s.jsx)(t.h3,{id:"spark-structured-streaming",children:"Spark Structured Streaming"}),"\n",(0,s.jsx)(t.p,{children:"We can also enable asynchronous clustering with Spark structured streaming sink as shown below."}),"\n",(0,s.jsx)(t.pre,{children:(0,s.jsx)(t.code,{className:"language-scala",children:'val commonOpts = Map(\n "hoodie.insert.shuffle.parallelism" -> "4",\n "hoodie.upsert.shuffle.parallelism" -> "4",\n "hoodie.datasource.write.recordkey.field" -> "_row_key",\n "hoodie.datasource.write.partitionpath.field" -> "partition",\n "hoodie.datasource.write.precombine.field" -> "timestamp",\n "hoodie.table.name" -> "hoodie_test"\n)\n\ndef getAsyncClusteringOpts(isAsyncClustering: String, \n clusteringNumCommit: String, \n executionStrategy: String):Map[String, String] = {\n commonOpts + (DataSourceWriteOptions.ASYNC_CLUSTERING_ENABLE.key -> isAsyncClustering,\n HoodieClusteringConfig.ASYNC_CLUSTERING_MAX_COMMITS.key -> clusteringNumCommit,\n HoodieClusteringConfig.EXECUTION_STRATEGY_CLASS_NAME.key -> executionStrategy\n )\n}\n\ndef initStreamingWriteFuture(hudiOptions: Map[String, String]): Future[Unit] = {\n val streamingInput = // define the source of streaming\n Future {\n println("streaming starting")\n streamingInput\n .writeStream\n .format("org.apache.hudi")\n .options(hudiOptions)\n .option("checkpointLocation", basePath + "/checkpoint")\n .mode(Append)\n .start()\n .awaitTermination(10000)\n println("streaming ends")\n }\n}\n\ndef structuredStreamingWithClustering(): Unit = {\n val df = //generate data frame\n val hudiOptions = getClusteringOpts("true", "1", "org.apache.hudi.client.clustering.run.strategy.SparkSortAndSizeExecutionStrategy")\n val f1 = initStreamingWriteFuture(hudiOptions)\n Await.result(f1, Duration.Inf)\n}\n'})}),"\n",(0,s.jsx)(t.h2,{id:"java-client",children:"Java Client"}),"\n",(0,s.jsxs)(t.p,{children:["Clustering is also supported via Java client. Plan strategy ",(0,s.jsx)(t.code,{children:"org.apache.hudi.client.clustering.plan.strategy.JavaSizeBasedClusteringPlanStrategy"}),"\nand execution strategy ",(0,s.jsx)(t.code,{children:"org.apache.hudi.client.clustering.run.strategy.JavaSortAndSizeExecutionStrategy"})," are supported\nout-of-the-box. Note that as of now only linear sort is supported in Java execution strategy."]}),"\n",(0,s.jsx)(t.h2,{id:"related-resources",children:"Related Resources"}),"\n",(0,s.jsxs)(t.p,{children:[(0,s.jsx)("h3",{children:"Blogs"}),"\n",(0,s.jsx)(t.a,{href:"https://www.onehouse.ai/blog/apachehudi-z-order-and-hilbert-space-filling-curves",children:"Apache Hudi Z-Order and Hilbert Space Filling Curves"}),"\n",(0,s.jsx)(t.a,{href:"https://medium.com/apache-hudi-blogs/hudi-z-order-and-hilbert-space-filling-curves-68fa28bffaf0",children:"Hudi Z-Order and Hilbert Space-filling Curves"})]}),"\n",(0,s.jsx)("h3",{children:"Videos"}),"\n",(0,s.jsxs)(t.ul,{children:["\n",(0,s.jsx)(t.li,{children:(0,s.jsx)(t.a,{href:"https://www.youtube.com/watch?v=R_sm4wlGXuE",children:"Understanding Clustering in Apache Hudi and the Benefits of Asynchronous Clustering"})}),"\n"]})]})}function u(e={}){const{wrapper:t}={...(0,r.R)(),...e.components};return t?(0,s.jsx)(t,{...e,children:(0,s.jsx)(d,{...e})}):d(e)}},7004:(e,t,i)=>{i.d(t,{A:()=>n});const n=i.p+"assets/images/clustering1_new-d67c9e691d235b140f7c80d68400f425.png"},67491:(e,t,i)=>{i.d(t,{A:()=>n});const n=i.p+"assets/images/clustering2_new-0837c07b6db44ab75873633f0eab2e2c.png"},14438:(e,t,i)=>{i.d(t,{A:()=>n});const n=i.p+"assets/images/clustering_3-8bc286ab44c48137f8409b5c342a7207.png"},28453:(e,t,i)=>{i.d(t,{R:()=>a,x:()=>l});var n=i(96540);const s={},r=n.createContext(s);function a(e){const t=n.useContext(r);return n.useMemo((function(){return"function"==typeof e?e(t):{...t,...e}}),[t,e])}function l(e){let t;return t=e.disableParentContext?"function"==typeof e.components?e.components(s):e.components||s:a(e.components),n.createElement(r.Provider,{value:t},e.children)}}}]);
\ No newline at end of file
diff --git a/content/assets/js/10b6d210.af5d2223.js b/content/assets/js/10b6d210.af5d2223.js
deleted file mode 100644
index 2bd069901324c..0000000000000
--- a/content/assets/js/10b6d210.af5d2223.js
+++ /dev/null
@@ -1 +0,0 @@
-"use strict";(self.webpackChunkhudi=self.webpackChunkhudi||[]).push([[18757],{78367:(e,i,t)=>{t.r(i),t.d(i,{assets:()=>c,contentTitle:()=>r,default:()=>h,frontMatter:()=>s,metadata:()=>n,toc:()=>l});const n=JSON.parse('{"id":"compaction","title":"Compaction","description":"Background","source":"@site/docs/compaction.md","sourceDirName":".","slug":"/compaction","permalink":"/docs/next/compaction","draft":false,"unlisted":false,"editUrl":"https://github.com/apache/hudi/tree/asf-site/website/docs/compaction.md","tags":[],"version":"current","frontMatter":{"title":"Compaction","summary":"In this page, we describe async compaction in Hudi.","toc":true,"toc_min_heading_level":2,"toc_max_heading_level":4,"last_modified_at":null},"sidebar":"docs","previous":{"title":"Cleaning","permalink":"/docs/next/cleaning"},"next":{"title":"Clustering","permalink":"/docs/next/clustering"}}');var o=t(74848),a=t(28453);const s={title:"Compaction",summary:"In this page, we describe async compaction in Hudi.",toc:!0,toc_min_heading_level:2,toc_max_heading_level:4,last_modified_at:null},r=void 0,c={},l=[{value:"Background",id:"background",level:2},{value:"Why MOR tables need compaction?",id:"why-mor-tables-need-compaction",level:3},{value:"Compaction Architecture",id:"compaction-architecture",level:2},{value:"Strategies in Compaction Scheduling",id:"strategies-in-compaction-scheduling",level:3},{value:"Trigger Strategies",id:"trigger-strategies",level:4},{value:"Compaction Strategies",id:"compaction-strategies",level:4},{value:"Ways to trigger Compaction",id:"ways-to-trigger-compaction",level:2},{value:"Inline",id:"inline",level:3},{value:"Async & Offline Compaction models",id:"async--offline-compaction-models",level:3},{value:"Async execution within the same process",id:"async-execution-within-the-same-process",level:4},{value:"Spark Structured Streaming",id:"spark-structured-streaming",level:5},{value:"Hudi Streamer Continuous Mode",id:"hudi-streamer-continuous-mode",level:5},{value:"Scheduling and Execution by a separate process",id:"scheduling-and-execution-by-a-separate-process",level:4},{value:"Scheduling inline and executing async",id:"scheduling-inline-and-executing-async",level:4},{value:"Hudi Compactor Utility",id:"hudi-compactor-utility",level:4},{value:"Hudi CLI",id:"hudi-cli",level:4},{value:"Flink Offline Compaction",id:"flink-offline-compaction",level:4},{value:"Options",id:"options",level:4}];function d(e){const i={a:"a",admonition:"admonition",br:"br",code:"code",em:"em",h2:"h2",h3:"h3",h4:"h4",h5:"h5",img:"img",li:"li",p:"p",pre:"pre",strong:"strong",table:"table",tbody:"tbody",td:"td",th:"th",thead:"thead",tr:"tr",ul:"ul",...(0,a.R)(),...e.components};return(0,o.jsxs)(o.Fragment,{children:[(0,o.jsx)(i.h2,{id:"background",children:"Background"}),"\n",(0,o.jsx)(i.p,{children:"Compaction is a table service employed by Hudi specifically in Merge On Read(MOR) tables to merge updates from row-based log\nfiles to the corresponding columnar-based base file periodically to produce a new version of the base file. Compaction is\nnot applicable to Copy On Write(COW) tables and only applies to MOR tables."}),"\n",(0,o.jsx)(i.h3,{id:"why-mor-tables-need-compaction",children:"Why MOR tables need compaction?"}),"\n",(0,o.jsxs)(i.p,{children:["To understand the significance of compaction in MOR tables, it is helpful to understand the MOR table layout first. In Hudi,\ndata is organized in terms of ",(0,o.jsx)(i.a,{href:"https://hudi.apache.org/docs/file_layouts/",children:"file groups"}),". Each file group in a MOR table\nconsists of a base file and one or more log files. Typically, during writes, inserts are stored in the base file, and updates\nare appended to log files."]}),"\n",(0,o.jsxs)(i.p,{children:[(0,o.jsx)(i.img,{alt:"mor_table_file_layout",src:t(67587).A+"",width:"6528",height:"2450"}),"\n",(0,o.jsx)(i.em,{children:"Figure: MOR table file layout showing different file groups with base data file and log files"})]}),"\n",(0,o.jsx)(i.p,{children:"During the compaction process, updates from the log files are merged with the base file to form a new version of the\nbase file as shown below. Since MOR is designed to be write-optimized, on new writes, after index tagging is complete,\nHudi appends the records pertaining to each file groups as log blocks in log files. There is no synchronous merge\nhappening during write, resulting in a lower write amplification and better write latency. In contrast, on new writes to a\nCOW table, Hudi combines the new writes with the older base file to produce a new version of the base file resulting in\na higher write amplification and higher write latencies."}),"\n",(0,o.jsxs)(i.p,{children:[(0,o.jsx)(i.img,{alt:"mor_table_file_layout",src:t(56040).A+"",width:"5081",height:"3148"}),"\n",(0,o.jsx)(i.em,{children:"Figure: Compaction on a given file group"})]}),"\n",(0,o.jsx)(i.p,{children:"While serving the read query(snapshot read), for each file group, records in base file and all its corresponding log\nfiles are merged together and served. And hence the read latency for MOR snapshot query might be higher compared to\nCOW table since there is no merge involved in case of COW at read time. Compaction takes care of merging the updates from\nlog files with the base file at regular intervals to bound the growth of log files and to ensure the read latencies do not\nspike up."}),"\n",(0,o.jsx)(i.h2,{id:"compaction-architecture",children:"Compaction Architecture"}),"\n",(0,o.jsx)(i.p,{children:"There are two steps to compaction."}),"\n",(0,o.jsxs)(i.ul,{children:["\n",(0,o.jsxs)(i.li,{children:[(0,o.jsx)(i.em,{children:(0,o.jsx)(i.strong,{children:"Compaction Scheduling"})}),": In this step, Hudi scans the partitions and selects file slices to be compacted. A compaction\nplan is finally written to Hudi timeline."]}),"\n",(0,o.jsxs)(i.li,{children:[(0,o.jsx)(i.em,{children:(0,o.jsx)(i.strong,{children:"Compaction Execution"})}),": In this step the compaction plan is read and file slices are compacted."]}),"\n"]}),"\n",(0,o.jsx)(i.h3,{id:"strategies-in-compaction-scheduling",children:"Strategies in Compaction Scheduling"}),"\n",(0,o.jsx)(i.p,{children:"There are two strategies involved in scheduling the compaction:"}),"\n",(0,o.jsxs)(i.ul,{children:["\n",(0,o.jsx)(i.li,{children:"Trigger Strategy: Determines how often to trigger scheduling of the compaction."}),"\n",(0,o.jsx)(i.li,{children:"Compaction Strategy: Determines which file groups to compact."}),"\n"]}),"\n",(0,o.jsx)(i.p,{children:"Hudi provides various options for both these strategies as discussed below."}),"\n",(0,o.jsx)(i.h4,{id:"trigger-strategies",children:"Trigger Strategies"}),"\n",(0,o.jsxs)(i.table,{children:[(0,o.jsx)(i.thead,{children:(0,o.jsxs)(i.tr,{children:[(0,o.jsx)(i.th,{children:"Config Name"}),(0,o.jsx)(i.th,{children:"Default"}),(0,o.jsx)(i.th,{children:"Description"})]})}),(0,o.jsxs)(i.tbody,{children:[(0,o.jsxs)(i.tr,{children:[(0,o.jsx)(i.td,{children:"hoodie.compact.inline.trigger.strategy"}),(0,o.jsx)(i.td,{children:"NUM_COMMITS (Optional)"}),(0,o.jsxs)(i.td,{children:["org.apache.hudi.table.action.compact.CompactionTriggerStrategy: Controls when compaction is scheduled.",(0,o.jsx)("br",{}),(0,o.jsx)(i.code,{children:"Config Param: INLINE_COMPACT_TRIGGER_STRATEGY"})," ",(0,o.jsx)("br",{})]})]}),(0,o.jsxs)(i.tr,{children:[(0,o.jsx)(i.td,{children:(0,o.jsxs)("ul",{children:[(0,o.jsxs)("li",{children:[(0,o.jsx)(i.code,{children:"NUM_COMMITS"}),": triggers compaction when there are at least N delta commits after last completed compaction."]}),(0,o.jsxs)("li",{children:[(0,o.jsx)(i.code,{children:"NUM_COMMITS_AFTER_LAST_REQUEST"}),": triggers compaction when there are at least N delta commits after last completed or requested compaction."]}),(0,o.jsxs)("li",{children:[(0,o.jsx)(i.code,{children:"TIME_ELAPSED"}),": triggers compaction after N seconds since last compaction."]}),(0,o.jsxs)("li",{children:[(0,o.jsx)(i.code,{children:"NUM_AND_TIME"}),": triggers compaction when both there are at least N delta commits and N seconds elapsed (both must be satisfied) after last completed compaction."]}),(0,o.jsxs)("li",{children:[(0,o.jsx)(i.code,{children:"NUM_OR_TIME"}),": triggers compaction when both there are at least N delta commits or N seconds elapsed (either condition is satisfied) after last completed compaction."]})]})}),(0,o.jsx)(i.td,{}),(0,o.jsx)(i.td,{})]})]})]}),"\n",(0,o.jsx)(i.h4,{id:"compaction-strategies",children:"Compaction Strategies"}),"\n",(0,o.jsxs)(i.table,{children:[(0,o.jsx)(i.thead,{children:(0,o.jsxs)(i.tr,{children:[(0,o.jsx)(i.th,{children:"Config Name"}),(0,o.jsx)(i.th,{children:"Default"}),(0,o.jsx)(i.th,{children:"Description"})]})}),(0,o.jsx)(i.tbody,{children:(0,o.jsxs)(i.tr,{children:[(0,o.jsx)(i.td,{children:"hoodie.compaction.strategy"}),(0,o.jsx)(i.td,{children:"org.apache.hudi.table.action.compact.strategy.LogFileSizeBasedCompactionStrategy (Optional)"}),(0,o.jsxs)(i.td,{children:["Compaction strategy decides which file groups are picked up for compaction during each compaction run. By default. Hudi picks the log file with most accumulated unmerged data. ",(0,o.jsx)("br",{}),(0,o.jsx)("br",{}),(0,o.jsx)(i.code,{children:"Config Param: COMPACTION_STRATEGY"})]})]})})]}),"\n",(0,o.jsxs)(i.p,{children:["Available Strategies (Provide the full package name when using the strategy): ",(0,o.jsxs)("ul",{children:[(0,o.jsxs)("li",{children:[(0,o.jsx)(i.code,{children:"LogFileNumBasedCompactionStrategy"}),":\norders the compactions based on the total log files count, filters the file group with log files count greater than the\nthreshold and limits the compactions within a configured IO bound."]}),(0,o.jsxs)("li",{children:[(0,o.jsx)(i.code,{children:"LogFileSizeBasedCompactionStrategy"}),": orders\nthe compactions based on the total log files size, filters the file group which log files size is greater than the\nthreshold and limits the compactions within a configured IO bound."]}),(0,o.jsxs)("li",{children:[(0,o.jsx)(i.code,{children:"BoundedIOCompactionStrategy"}),": CompactionStrategy\nwhich looks at total IO to be done for the compaction (read + write) and limits the list of compactions to be under a\nconfigured limit on the IO."]}),(0,o.jsxs)("li",{children:[(0,o.jsx)(i.code,{children:"BoundedPartitionAwareCompactionStrategy"}),":This"," strategy ensures that the last N partitions\nare picked up even if there are later partitions created for the table. lastNPartitions is defined as the N partitions before\nthe currentDate. currentDay = 2018/01/01 The table has partitions for 2018/02/02 and 2018/03/03 beyond the currentDay This\nstrategy will pick up the following partitions for compaction : (2018/01/01, allPartitionsInRange[(2018/01/01 - lastNPartitions)\nto 2018/01/01), 2018/02/02, 2018/03/03)"]}),(0,o.jsxs)("li",{children:[(0,o.jsx)(i.code,{children:"DayBasedCompactionStrategy"}),":This"," strategy orders compactions in reverse\norder of creation of Hive Partitions. It helps to compact data in latest partitions first and then older capped at the\nTotal_IO allowed."]}),(0,o.jsxs)("li",{children:[(0,o.jsx)(i.code,{children:"UnBoundedCompactionStrategy"}),": UnBoundedCompactionStrategy will not change ordering or filter\nany compaction. It is a pass-through and will compact all the base files which has a log file. This usually means\nno-intelligence on compaction."]}),(0,o.jsxs)("li",{children:[(0,o.jsx)(i.code,{children:"UnBoundedPartitionAwareCompactionStrategy"}),":UnBoundedPartitionAwareCompactionStrategy"," is a custom UnBounded Strategy. This will filter all the partitions that\nare eligible to be compacted by a {@link BoundedPartitionAwareCompactionStrategy} and return the result. This is done\nso that a long running UnBoundedPartitionAwareCompactionStrategy does not step over partitions in a shorter running\nBoundedPartitionAwareCompactionStrategy. Essentially, this is an inverse of the partitions chosen in\nBoundedPartitionAwareCompactionStrategy"]})]})]}),"\n",(0,o.jsx)(i.admonition,{type:"note",children:(0,o.jsxs)(i.p,{children:["Please refer to ",(0,o.jsx)(i.a,{href:"https://hudi.apache.org/docs/next/configurations#Compaction-Configs",children:"advanced configs"})," for more details."]})}),"\n",(0,o.jsx)(i.h2,{id:"ways-to-trigger-compaction",children:"Ways to trigger Compaction"}),"\n",(0,o.jsx)(i.h3,{id:"inline",children:"Inline"}),"\n",(0,o.jsx)(i.p,{children:"By default, compaction is run asynchronously."}),"\n",(0,o.jsx)(i.p,{children:"If latency of ingesting records is important for you, you are most likely using Merge-On-Read tables.\nMerge-On-Read tables store data using a combination of columnar (e.g parquet) + row based (e.g avro) file formats.\nUpdates are logged to delta files & later compacted to produce new versions of columnar files.\nTo improve ingestion latency, Async Compaction is the default configuration."}),"\n",(0,o.jsx)(i.p,{children:"If immediate read performance of a new commit is important for you, or you want simplicity of not managing separate compaction jobs,\nyou may want synchronous inline compaction, which means that as a commit is written it is also compacted by the same job."}),"\n",(0,o.jsxs)(i.p,{children:["For this deployment mode, please use ",(0,o.jsx)(i.code,{children:"hoodie.compact.inline = true"})," for Spark Datasource and Spark SQL writers. For\nHoodieStreamer sync once mode inline compaction can be achieved by passing the flag ",(0,o.jsx)(i.code,{children:"--disable-compaction"})," (Meaning to\ndisable async compaction). Further in HoodieStreamer when both\ningestion and compaction is running in the same spark context, you can use resource allocation configuration\nin Hudi Streamer CLI such as (",(0,o.jsx)(i.code,{children:"--delta-sync-scheduling-weight"}),",\n",(0,o.jsx)(i.code,{children:"--compact-scheduling-weight"}),", ",(0,o.jsx)(i.code,{children:"--delta-sync-scheduling-minshare"}),", and ",(0,o.jsx)(i.code,{children:"--compact-scheduling-minshare"}),")\nto control executor allocation between ingestion and compaction."]}),"\n",(0,o.jsx)(i.h3,{id:"async--offline-compaction-models",children:"Async & Offline Compaction models"}),"\n",(0,o.jsx)(i.p,{children:"There are a couple of ways here to trigger compaction ."}),"\n",(0,o.jsx)(i.h4,{id:"async-execution-within-the-same-process",children:"Async execution within the same process"}),"\n",(0,o.jsx)(i.p,{children:"In streaming ingestion write models like HoodieStreamer\ncontinuous mode, Flink and Spark Streaming, async compaction is enabled by default and runs alongside without blocking\nregular ingestion."}),"\n",(0,o.jsx)(i.h5,{id:"spark-structured-streaming",children:"Spark Structured Streaming"}),"\n",(0,o.jsx)(i.p,{children:"Compactions are scheduled and executed asynchronously inside the\nstreaming job.Here is an example snippet in java"}),"\n",(0,o.jsx)(i.pre,{children:(0,o.jsx)(i.code,{className:"language-properties",children:'import org.apache.hudi.DataSourceWriteOptions;\nimport org.apache.hudi.HoodieDataSourceHelpers;\nimport org.apache.hudi.config.HoodieCompactionConfig;\nimport org.apache.hudi.config.HoodieWriteConfig;\n\nimport org.apache.spark.sql.streaming.OutputMode;\nimport org.apache.spark.sql.streaming.ProcessingTime;\n\n\n DataStreamWriter writer = streamingInput.writeStream().format("org.apache.hudi")\n .option("hoodie.datasource.write.operation", operationType)\n .option("hoodie.datasource.write.table.type", tableType)\n .option("hoodie.datasource.write.recordkey.field", "_row_key")\n .option("hoodie.datasource.write.partitionpath.field", "partition")\n .option("hoodie.datasource.write.precombine.field"(), "timestamp")\n .option("hoodie.compact.inline.max.delta.commits", "10")\n .option("hoodie.datasource.compaction.async.enable", "true")\n .option("hoodie.table.name", tableName).option("checkpointLocation", checkpointLocation)\n .outputMode(OutputMode.Append());\n writer.trigger(new ProcessingTime(30000)).start(tablePath);\n'})}),"\n",(0,o.jsx)(i.h5,{id:"hudi-streamer-continuous-mode",children:"Hudi Streamer Continuous Mode"}),"\n",(0,o.jsxs)(i.p,{children:["Hudi Streamer provides continuous ingestion mode where a single long running spark application",(0,o.jsx)(i.br,{}),"\n","ingests data to Hudi table continuously from upstream sources. In this mode, Hudi supports managing asynchronous\ncompactions. Here is an example snippet for running in continuous mode with async compactions"]}),"\n",(0,o.jsx)(i.pre,{children:(0,o.jsx)(i.code,{className:"language-properties",children:"spark-submit --packages org.apache.hudi:hudi-utilities-slim-bundle_2.12:1.0.0,org.apache.hudi:hudi-spark3.5-bundle_2.12:1.0.0 \\\n--class org.apache.hudi.utilities.streamer.HoodieStreamer \\\n--table-type MERGE_ON_READ \\\n--target-base-path \\\n--target-table \\\n--source-class org.apache.hudi.utilities.sources.JsonDFSSource \\\n--source-ordering-field ts \\\n--props /path/to/source.properties \\\n--continous\n"})}),"\n",(0,o.jsx)(i.h4,{id:"scheduling-and-execution-by-a-separate-process",children:"Scheduling and Execution by a separate process"}),"\n",(0,o.jsxs)(i.p,{children:["For some use cases with long running table services, instead of having the regular writes block, users have the option to run\nboth steps of the compaction (",(0,o.jsx)(i.a,{href:"#compaction-architecture",children:"scheduling and execution"}),") offline in a separate process altogether.\nThis allows for regular writers to not bother about these compaction steps and allows users to provide more resources for\nthe compaction job as needed."]}),"\n",(0,o.jsx)(i.admonition,{type:"note",children:(0,o.jsx)(i.p,{children:"This model needs a lock provider configured for all jobs - the regular writer as well as the offline compaction job."})}),"\n",(0,o.jsx)(i.h4,{id:"scheduling-inline-and-executing-async",children:"Scheduling inline and executing async"}),"\n",(0,o.jsx)(i.p,{children:"In this model, it is possible for a Spark Datasource writer or a Flink job to just schedule the compaction inline ( that\nwill serialize the compaction plan in the timeline but will not execute it). And then a separate utility like\nHudiCompactor or HoodieFlinkCompactor can take care of periodically executing the compaction plan."}),"\n",(0,o.jsx)(i.admonition,{type:"note",children:(0,o.jsxs)(i.p,{children:["This model may need a lock provider ",(0,o.jsx)(i.strong,{children:"if"})," metadata table is enabled."]})}),"\n",(0,o.jsx)(i.h4,{id:"hudi-compactor-utility",children:"Hudi Compactor Utility"}),"\n",(0,o.jsxs)(i.p,{children:["Hudi provides a standalone tool to execute specific compactions asynchronously. Below is an example and you can read more in the ",(0,o.jsx)(i.a,{href:"/docs/cli#compactions",children:"deployment guide"}),"\nThe compactor utility allows to do scheduling and execution of compaction."]}),"\n",(0,o.jsx)(i.p,{children:"Example:"}),"\n",(0,o.jsx)(i.pre,{children:(0,o.jsx)(i.code,{className:"language-properties",children:"spark-submit --packages org.apache.hudi:hudi-utilities-slim-bundle_2.12:1.0.0,org.apache.hudi:hudi-spark3.5-bundle_2.12:1.0.0 \\\n--class org.apache.hudi.utilities.HoodieCompactor \\\n--base-path \\\n--table-name \\\n--schema-file \\\n--instant-time \n"})}),"\n",(0,o.jsxs)(i.p,{children:["Note, the ",(0,o.jsx)(i.code,{children:"instant-time"})," parameter is now optional for the Hudi Compactor Utility. If using the utility without ",(0,o.jsx)(i.code,{children:"--instant time"}),",\nthe spark-submit will execute the earliest scheduled compaction on the Hudi timeline."]}),"\n",(0,o.jsx)(i.h4,{id:"hudi-cli",children:"Hudi CLI"}),"\n",(0,o.jsxs)(i.p,{children:["Hudi CLI is yet another way to execute specific compactions asynchronously. Here is an example and you can read more in the ",(0,o.jsx)(i.a,{href:"/docs/cli#compactions",children:"deployment guide"})]}),"\n",(0,o.jsx)(i.p,{children:"Example:"}),"\n",(0,o.jsx)(i.pre,{children:(0,o.jsx)(i.code,{className:"language-properties",children:"hudi:trips->compaction run --tableName --parallelism --compactionInstant \n...\n"})}),"\n",(0,o.jsx)(i.h4,{id:"flink-offline-compaction",children:"Flink Offline Compaction"}),"\n",(0,o.jsxs)(i.p,{children:["Offline compaction needs to submit the Flink task on the command line. The program entry is as follows: ",(0,o.jsx)(i.code,{children:"hudi-flink-bundle_2.11-0.9.0-SNAPSHOT.jar"})," :\n",(0,o.jsx)(i.code,{children:"org.apache.hudi.sink.compact.HoodieFlinkCompactor"})]}),"\n",(0,o.jsx)(i.pre,{children:(0,o.jsx)(i.code,{className:"language-bash",children:"# Command line\n./bin/flink run -c org.apache.hudi.sink.compact.HoodieFlinkCompactor lib/hudi-flink-bundle_2.11-0.9.0.jar --path hdfs://xxx:9000/table\n"})}),"\n",(0,o.jsx)(i.h4,{id:"options",children:"Options"}),"\n",(0,o.jsxs)(i.table,{children:[(0,o.jsx)(i.thead,{children:(0,o.jsxs)(i.tr,{children:[(0,o.jsx)(i.th,{children:"Option Name"}),(0,o.jsx)(i.th,{children:"Default"}),(0,o.jsx)(i.th,{children:"Description"})]})}),(0,o.jsxs)(i.tbody,{children:[(0,o.jsxs)(i.tr,{children:[(0,o.jsx)(i.td,{children:(0,o.jsx)(i.code,{children:"--path"})}),(0,o.jsx)(i.td,{children:(0,o.jsx)(i.code,{children:"n/a **(Required)**"})}),(0,o.jsx)(i.td,{children:"The path where the target table is stored on Hudi"})]}),(0,o.jsxs)(i.tr,{children:[(0,o.jsx)(i.td,{children:(0,o.jsx)(i.code,{children:"--compaction-max-memory"})}),(0,o.jsxs)(i.td,{children:[(0,o.jsx)(i.code,{children:"100"})," (Optional)"]}),(0,o.jsx)(i.td,{children:"The index map size of log data during compaction, 100 MB by default. If you have enough memory, you can turn up this parameter"})]}),(0,o.jsxs)(i.tr,{children:[(0,o.jsx)(i.td,{children:(0,o.jsx)(i.code,{children:"--schedule"})}),(0,o.jsxs)(i.td,{children:[(0,o.jsx)(i.code,{children:"false"})," (Optional)"]}),(0,o.jsx)(i.td,{children:"whether to execute the operation of scheduling compaction plan. When the write process is still writing\uff0c turning on this parameter have a risk of losing data. Therefore, it must be ensured that there are no write tasks currently writing data to this table when this parameter is turned on"})]}),(0,o.jsxs)(i.tr,{children:[(0,o.jsx)(i.td,{children:(0,o.jsx)(i.code,{children:"--seq"})}),(0,o.jsxs)(i.td,{children:[(0,o.jsx)(i.code,{children:"LIFO"})," (Optional)"]}),(0,o.jsxs)(i.td,{children:["The order in which compaction tasks are executed. Executing from the latest compaction plan by default. ",(0,o.jsx)(i.code,{children:"LIFO"}),": executing from the latest plan. ",(0,o.jsx)(i.code,{children:"FIFO"}),": executing from the oldest plan."]})]}),(0,o.jsxs)(i.tr,{children:[(0,o.jsx)(i.td,{children:(0,o.jsx)(i.code,{children:"--service"})}),(0,o.jsxs)(i.td,{children:[(0,o.jsx)(i.code,{children:"false"})," (Optional)"]}),(0,o.jsx)(i.td,{children:"Whether to start a monitoring service that checks and schedules new compaction task in configured interval."})]}),(0,o.jsxs)(i.tr,{children:[(0,o.jsx)(i.td,{children:(0,o.jsx)(i.code,{children:"--min-compaction-interval-seconds"})}),(0,o.jsxs)(i.td,{children:[(0,o.jsx)(i.code,{children:"600(s)"})," (optional)"]}),(0,o.jsx)(i.td,{children:"The checking interval for service mode, by default 10 minutes."})]})]})]})]})}function h(e={}){const{wrapper:i}={...(0,a.R)(),...e.components};return i?(0,o.jsx)(i,{...e,children:(0,o.jsx)(d,{...e})}):d(e)}},67587:(e,i,t)=>{t.d(i,{A:()=>n});const n=t.p+"assets/images/hudi_mor_file_layout-643f9f7fda5aa0d532682af27fe3e42c.jpg"},56040:(e,i,t)=>{t.d(i,{A:()=>n});const n=t.p+"assets/images/hudi_mor_file_layout_post_compaction-9f10af785d4927dc3d66303dac5bc7ba.jpg"},28453:(e,i,t)=>{t.d(i,{R:()=>s,x:()=>r});var n=t(96540);const o={},a=n.createContext(o);function s(e){const i=n.useContext(a);return n.useMemo((function(){return"function"==typeof e?e(i):{...i,...e}}),[i,e])}function r(e){let i;return i=e.disableParentContext?"function"==typeof e.components?e.components(o):e.components||o:s(e.components),n.createElement(a.Provider,{value:i},e.children)}}}]);
\ No newline at end of file
diff --git a/content/assets/js/10b6d210.bc03ff70.js b/content/assets/js/10b6d210.bc03ff70.js
new file mode 100644
index 0000000000000..06d277b2e3d9e
--- /dev/null
+++ b/content/assets/js/10b6d210.bc03ff70.js
@@ -0,0 +1 @@
+"use strict";(self.webpackChunkhudi=self.webpackChunkhudi||[]).push([[18757],{78367:(e,i,t)=>{t.r(i),t.d(i,{assets:()=>c,contentTitle:()=>r,default:()=>h,frontMatter:()=>s,metadata:()=>n,toc:()=>l});const n=JSON.parse('{"id":"compaction","title":"Compaction","description":"Background","source":"@site/docs/compaction.md","sourceDirName":".","slug":"/compaction","permalink":"/docs/next/compaction","draft":false,"unlisted":false,"editUrl":"https://github.com/apache/hudi/tree/asf-site/website/docs/compaction.md","tags":[],"version":"current","frontMatter":{"title":"Compaction","summary":"In this page, we describe async compaction in Hudi.","toc":true,"toc_min_heading_level":2,"toc_max_heading_level":4,"last_modified_at":null},"sidebar":"docs","previous":{"title":"Cleaning","permalink":"/docs/next/cleaning"},"next":{"title":"Clustering","permalink":"/docs/next/clustering"}}');var o=t(74848),a=t(28453);const s={title:"Compaction",summary:"In this page, we describe async compaction in Hudi.",toc:!0,toc_min_heading_level:2,toc_max_heading_level:4,last_modified_at:null},r=void 0,c={},l=[{value:"Background",id:"background",level:2},{value:"Why MOR tables need compaction?",id:"why-mor-tables-need-compaction",level:3},{value:"Compaction Architecture",id:"compaction-architecture",level:2},{value:"Strategies in Compaction Scheduling",id:"strategies-in-compaction-scheduling",level:3},{value:"Trigger Strategies",id:"trigger-strategies",level:4},{value:"Compaction Strategies",id:"compaction-strategies",level:4},{value:"Ways to trigger Compaction",id:"ways-to-trigger-compaction",level:2},{value:"Inline",id:"inline",level:3},{value:"Async & Offline Compaction models",id:"async--offline-compaction-models",level:3},{value:"Async execution within the same process",id:"async-execution-within-the-same-process",level:4},{value:"Spark Structured Streaming",id:"spark-structured-streaming",level:5},{value:"Hudi Streamer Continuous Mode",id:"hudi-streamer-continuous-mode",level:5},{value:"Scheduling and Execution by a separate process",id:"scheduling-and-execution-by-a-separate-process",level:4},{value:"Scheduling inline and executing async",id:"scheduling-inline-and-executing-async",level:4},{value:"Hudi Compactor Utility",id:"hudi-compactor-utility",level:4},{value:"Hudi CLI",id:"hudi-cli",level:4},{value:"Flink Offline Compaction",id:"flink-offline-compaction",level:4},{value:"Options",id:"options",level:4},{value:"Related Resources",id:"related-resources",level:2}];function d(e){const i={a:"a",admonition:"admonition",br:"br",code:"code",em:"em",h2:"h2",h3:"h3",h4:"h4",h5:"h5",img:"img",li:"li",p:"p",pre:"pre",strong:"strong",table:"table",tbody:"tbody",td:"td",th:"th",thead:"thead",tr:"tr",ul:"ul",...(0,a.R)(),...e.components};return(0,o.jsxs)(o.Fragment,{children:[(0,o.jsx)(i.h2,{id:"background",children:"Background"}),"\n",(0,o.jsx)(i.p,{children:"Compaction is a table service employed by Hudi specifically in Merge On Read(MOR) tables to merge updates from row-based log\nfiles to the corresponding columnar-based base file periodically to produce a new version of the base file. Compaction is\nnot applicable to Copy On Write(COW) tables and only applies to MOR tables."}),"\n",(0,o.jsx)(i.h3,{id:"why-mor-tables-need-compaction",children:"Why MOR tables need compaction?"}),"\n",(0,o.jsxs)(i.p,{children:["To understand the significance of compaction in MOR tables, it is helpful to understand the MOR table layout first. In Hudi,\ndata is organized in terms of ",(0,o.jsx)(i.a,{href:"https://hudi.apache.org/docs/file_layouts/",children:"file groups"}),". Each file group in a MOR table\nconsists of a base file and one or more log files. Typically, during writes, inserts are stored in the base file, and updates\nare appended to log files."]}),"\n",(0,o.jsxs)(i.p,{children:[(0,o.jsx)(i.img,{alt:"mor_table_file_layout",src:t(67587).A+"",width:"6528",height:"2450"}),"\n",(0,o.jsx)(i.em,{children:"Figure: MOR table file layout showing different file groups with base data file and log files"})]}),"\n",(0,o.jsx)(i.p,{children:"During the compaction process, updates from the log files are merged with the base file to form a new version of the\nbase file as shown below. Since MOR is designed to be write-optimized, on new writes, after index tagging is complete,\nHudi appends the records pertaining to each file groups as log blocks in log files. There is no synchronous merge\nhappening during write, resulting in a lower write amplification and better write latency. In contrast, on new writes to a\nCOW table, Hudi combines the new writes with the older base file to produce a new version of the base file resulting in\na higher write amplification and higher write latencies."}),"\n",(0,o.jsxs)(i.p,{children:[(0,o.jsx)(i.img,{alt:"mor_table_file_layout",src:t(56040).A+"",width:"5081",height:"3148"}),"\n",(0,o.jsx)(i.em,{children:"Figure: Compaction on a given file group"})]}),"\n",(0,o.jsx)(i.p,{children:"While serving the read query(snapshot read), for each file group, records in base file and all its corresponding log\nfiles are merged together and served. And hence the read latency for MOR snapshot query might be higher compared to\nCOW table since there is no merge involved in case of COW at read time. Compaction takes care of merging the updates from\nlog files with the base file at regular intervals to bound the growth of log files and to ensure the read latencies do not\nspike up."}),"\n",(0,o.jsx)(i.h2,{id:"compaction-architecture",children:"Compaction Architecture"}),"\n",(0,o.jsx)(i.p,{children:"There are two steps to compaction."}),"\n",(0,o.jsxs)(i.ul,{children:["\n",(0,o.jsxs)(i.li,{children:[(0,o.jsx)(i.em,{children:(0,o.jsx)(i.strong,{children:"Compaction Scheduling"})}),": In this step, Hudi scans the partitions and selects file slices to be compacted. A compaction\nplan is finally written to Hudi timeline."]}),"\n",(0,o.jsxs)(i.li,{children:[(0,o.jsx)(i.em,{children:(0,o.jsx)(i.strong,{children:"Compaction Execution"})}),": In this step the compaction plan is read and file slices are compacted."]}),"\n"]}),"\n",(0,o.jsx)(i.h3,{id:"strategies-in-compaction-scheduling",children:"Strategies in Compaction Scheduling"}),"\n",(0,o.jsx)(i.p,{children:"There are two strategies involved in scheduling the compaction:"}),"\n",(0,o.jsxs)(i.ul,{children:["\n",(0,o.jsx)(i.li,{children:"Trigger Strategy: Determines how often to trigger scheduling of the compaction."}),"\n",(0,o.jsx)(i.li,{children:"Compaction Strategy: Determines which file groups to compact."}),"\n"]}),"\n",(0,o.jsx)(i.p,{children:"Hudi provides various options for both these strategies as discussed below."}),"\n",(0,o.jsx)(i.h4,{id:"trigger-strategies",children:"Trigger Strategies"}),"\n",(0,o.jsxs)(i.table,{children:[(0,o.jsx)(i.thead,{children:(0,o.jsxs)(i.tr,{children:[(0,o.jsx)(i.th,{children:"Config Name"}),(0,o.jsx)(i.th,{children:"Default"}),(0,o.jsx)(i.th,{children:"Description"})]})}),(0,o.jsxs)(i.tbody,{children:[(0,o.jsxs)(i.tr,{children:[(0,o.jsx)(i.td,{children:"hoodie.compact.inline.trigger.strategy"}),(0,o.jsx)(i.td,{children:"NUM_COMMITS (Optional)"}),(0,o.jsxs)(i.td,{children:["org.apache.hudi.table.action.compact.CompactionTriggerStrategy: Controls when compaction is scheduled.",(0,o.jsx)("br",{}),(0,o.jsx)(i.code,{children:"Config Param: INLINE_COMPACT_TRIGGER_STRATEGY"})," ",(0,o.jsx)("br",{})]})]}),(0,o.jsxs)(i.tr,{children:[(0,o.jsx)(i.td,{children:(0,o.jsxs)("ul",{children:[(0,o.jsxs)("li",{children:[(0,o.jsx)(i.code,{children:"NUM_COMMITS"}),": triggers compaction when there are at least N delta commits after last completed compaction."]}),(0,o.jsxs)("li",{children:[(0,o.jsx)(i.code,{children:"NUM_COMMITS_AFTER_LAST_REQUEST"}),": triggers compaction when there are at least N delta commits after last completed or requested compaction."]}),(0,o.jsxs)("li",{children:[(0,o.jsx)(i.code,{children:"TIME_ELAPSED"}),": triggers compaction after N seconds since last compaction."]}),(0,o.jsxs)("li",{children:[(0,o.jsx)(i.code,{children:"NUM_AND_TIME"}),": triggers compaction when both there are at least N delta commits and N seconds elapsed (both must be satisfied) after last completed compaction."]}),(0,o.jsxs)("li",{children:[(0,o.jsx)(i.code,{children:"NUM_OR_TIME"}),": triggers compaction when both there are at least N delta commits or N seconds elapsed (either condition is satisfied) after last completed compaction."]})]})}),(0,o.jsx)(i.td,{}),(0,o.jsx)(i.td,{})]})]})]}),"\n",(0,o.jsx)(i.h4,{id:"compaction-strategies",children:"Compaction Strategies"}),"\n",(0,o.jsxs)(i.table,{children:[(0,o.jsx)(i.thead,{children:(0,o.jsxs)(i.tr,{children:[(0,o.jsx)(i.th,{children:"Config Name"}),(0,o.jsx)(i.th,{children:"Default"}),(0,o.jsx)(i.th,{children:"Description"})]})}),(0,o.jsx)(i.tbody,{children:(0,o.jsxs)(i.tr,{children:[(0,o.jsx)(i.td,{children:"hoodie.compaction.strategy"}),(0,o.jsx)(i.td,{children:"org.apache.hudi.table.action.compact.strategy.LogFileSizeBasedCompactionStrategy (Optional)"}),(0,o.jsxs)(i.td,{children:["Compaction strategy decides which file groups are picked up for compaction during each compaction run. By default. Hudi picks the log file with most accumulated unmerged data. ",(0,o.jsx)("br",{}),(0,o.jsx)("br",{}),(0,o.jsx)(i.code,{children:"Config Param: COMPACTION_STRATEGY"})]})]})})]}),"\n",(0,o.jsxs)(i.p,{children:["Available Strategies (Provide the full package name when using the strategy): ",(0,o.jsxs)("ul",{children:[(0,o.jsxs)("li",{children:[(0,o.jsx)(i.code,{children:"LogFileNumBasedCompactionStrategy"}),":\norders the compactions based on the total log files count, filters the file group with log files count greater than the\nthreshold and limits the compactions within a configured IO bound."]}),(0,o.jsxs)("li",{children:[(0,o.jsx)(i.code,{children:"LogFileSizeBasedCompactionStrategy"}),": orders\nthe compactions based on the total log files size, filters the file group which log files size is greater than the\nthreshold and limits the compactions within a configured IO bound."]}),(0,o.jsxs)("li",{children:[(0,o.jsx)(i.code,{children:"BoundedIOCompactionStrategy"}),": CompactionStrategy\nwhich looks at total IO to be done for the compaction (read + write) and limits the list of compactions to be under a\nconfigured limit on the IO."]}),(0,o.jsxs)("li",{children:[(0,o.jsx)(i.code,{children:"BoundedPartitionAwareCompactionStrategy"}),":This"," strategy ensures that the last N partitions\nare picked up even if there are later partitions created for the table. lastNPartitions is defined as the N partitions before\nthe currentDate. currentDay = 2018/01/01 The table has partitions for 2018/02/02 and 2018/03/03 beyond the currentDay This\nstrategy will pick up the following partitions for compaction : (2018/01/01, allPartitionsInRange[(2018/01/01 - lastNPartitions)\nto 2018/01/01), 2018/02/02, 2018/03/03)"]}),(0,o.jsxs)("li",{children:[(0,o.jsx)(i.code,{children:"DayBasedCompactionStrategy"}),":This"," strategy orders compactions in reverse\norder of creation of Hive Partitions. It helps to compact data in latest partitions first and then older capped at the\nTotal_IO allowed."]}),(0,o.jsxs)("li",{children:[(0,o.jsx)(i.code,{children:"UnBoundedCompactionStrategy"}),": UnBoundedCompactionStrategy will not change ordering or filter\nany compaction. It is a pass-through and will compact all the base files which has a log file. This usually means\nno-intelligence on compaction."]}),(0,o.jsxs)("li",{children:[(0,o.jsx)(i.code,{children:"UnBoundedPartitionAwareCompactionStrategy"}),":UnBoundedPartitionAwareCompactionStrategy"," is a custom UnBounded Strategy. This will filter all the partitions that\nare eligible to be compacted by a {@link BoundedPartitionAwareCompactionStrategy} and return the result. This is done\nso that a long running UnBoundedPartitionAwareCompactionStrategy does not step over partitions in a shorter running\nBoundedPartitionAwareCompactionStrategy. Essentially, this is an inverse of the partitions chosen in\nBoundedPartitionAwareCompactionStrategy"]})]})]}),"\n",(0,o.jsx)(i.admonition,{type:"note",children:(0,o.jsxs)(i.p,{children:["Please refer to ",(0,o.jsx)(i.a,{href:"https://hudi.apache.org/docs/next/configurations#Compaction-Configs",children:"advanced configs"})," for more details."]})}),"\n",(0,o.jsx)(i.h2,{id:"ways-to-trigger-compaction",children:"Ways to trigger Compaction"}),"\n",(0,o.jsx)(i.h3,{id:"inline",children:"Inline"}),"\n",(0,o.jsx)(i.p,{children:"By default, compaction is run asynchronously."}),"\n",(0,o.jsx)(i.p,{children:"If latency of ingesting records is important for you, you are most likely using Merge-On-Read tables.\nMerge-On-Read tables store data using a combination of columnar (e.g parquet) + row based (e.g avro) file formats.\nUpdates are logged to delta files & later compacted to produce new versions of columnar files.\nTo improve ingestion latency, Async Compaction is the default configuration."}),"\n",(0,o.jsx)(i.p,{children:"If immediate read performance of a new commit is important for you, or you want simplicity of not managing separate compaction jobs,\nyou may want synchronous inline compaction, which means that as a commit is written it is also compacted by the same job."}),"\n",(0,o.jsxs)(i.p,{children:["For this deployment mode, please use ",(0,o.jsx)(i.code,{children:"hoodie.compact.inline = true"})," for Spark Datasource and Spark SQL writers. For\nHoodieStreamer sync once mode inline compaction can be achieved by passing the flag ",(0,o.jsx)(i.code,{children:"--disable-compaction"})," (Meaning to\ndisable async compaction). Further in HoodieStreamer when both\ningestion and compaction is running in the same spark context, you can use resource allocation configuration\nin Hudi Streamer CLI such as (",(0,o.jsx)(i.code,{children:"--delta-sync-scheduling-weight"}),",\n",(0,o.jsx)(i.code,{children:"--compact-scheduling-weight"}),", ",(0,o.jsx)(i.code,{children:"--delta-sync-scheduling-minshare"}),", and ",(0,o.jsx)(i.code,{children:"--compact-scheduling-minshare"}),")\nto control executor allocation between ingestion and compaction."]}),"\n",(0,o.jsx)(i.h3,{id:"async--offline-compaction-models",children:"Async & Offline Compaction models"}),"\n",(0,o.jsx)(i.p,{children:"There are a couple of ways here to trigger compaction ."}),"\n",(0,o.jsx)(i.h4,{id:"async-execution-within-the-same-process",children:"Async execution within the same process"}),"\n",(0,o.jsx)(i.p,{children:"In streaming ingestion write models like HoodieStreamer\ncontinuous mode, Flink and Spark Streaming, async compaction is enabled by default and runs alongside without blocking\nregular ingestion."}),"\n",(0,o.jsx)(i.h5,{id:"spark-structured-streaming",children:"Spark Structured Streaming"}),"\n",(0,o.jsx)(i.p,{children:"Compactions are scheduled and executed asynchronously inside the\nstreaming job.Here is an example snippet in java"}),"\n",(0,o.jsx)(i.pre,{children:(0,o.jsx)(i.code,{className:"language-properties",children:'import org.apache.hudi.DataSourceWriteOptions;\nimport org.apache.hudi.HoodieDataSourceHelpers;\nimport org.apache.hudi.config.HoodieCompactionConfig;\nimport org.apache.hudi.config.HoodieWriteConfig;\n\nimport org.apache.spark.sql.streaming.OutputMode;\nimport org.apache.spark.sql.streaming.ProcessingTime;\n\n\n DataStreamWriter writer = streamingInput.writeStream().format("org.apache.hudi")\n .option("hoodie.datasource.write.operation", operationType)\n .option("hoodie.datasource.write.table.type", tableType)\n .option("hoodie.datasource.write.recordkey.field", "_row_key")\n .option("hoodie.datasource.write.partitionpath.field", "partition")\n .option("hoodie.datasource.write.precombine.field"(), "timestamp")\n .option("hoodie.compact.inline.max.delta.commits", "10")\n .option("hoodie.datasource.compaction.async.enable", "true")\n .option("hoodie.table.name", tableName).option("checkpointLocation", checkpointLocation)\n .outputMode(OutputMode.Append());\n writer.trigger(new ProcessingTime(30000)).start(tablePath);\n'})}),"\n",(0,o.jsx)(i.h5,{id:"hudi-streamer-continuous-mode",children:"Hudi Streamer Continuous Mode"}),"\n",(0,o.jsxs)(i.p,{children:["Hudi Streamer provides continuous ingestion mode where a single long running spark application",(0,o.jsx)(i.br,{}),"\n","ingests data to Hudi table continuously from upstream sources. In this mode, Hudi supports managing asynchronous\ncompactions. Here is an example snippet for running in continuous mode with async compactions"]}),"\n",(0,o.jsx)(i.pre,{children:(0,o.jsx)(i.code,{className:"language-properties",children:"spark-submit --packages org.apache.hudi:hudi-utilities-slim-bundle_2.12:1.0.0,org.apache.hudi:hudi-spark3.5-bundle_2.12:1.0.0 \\\n--class org.apache.hudi.utilities.streamer.HoodieStreamer \\\n--table-type MERGE_ON_READ \\\n--target-base-path \\\n--target-table \\\n--source-class org.apache.hudi.utilities.sources.JsonDFSSource \\\n--source-ordering-field ts \\\n--props /path/to/source.properties \\\n--continous\n"})}),"\n",(0,o.jsx)(i.h4,{id:"scheduling-and-execution-by-a-separate-process",children:"Scheduling and Execution by a separate process"}),"\n",(0,o.jsxs)(i.p,{children:["For some use cases with long running table services, instead of having the regular writes block, users have the option to run\nboth steps of the compaction (",(0,o.jsx)(i.a,{href:"#compaction-architecture",children:"scheduling and execution"}),") offline in a separate process altogether.\nThis allows for regular writers to not bother about these compaction steps and allows users to provide more resources for\nthe compaction job as needed."]}),"\n",(0,o.jsx)(i.admonition,{type:"note",children:(0,o.jsx)(i.p,{children:"This model needs a lock provider configured for all jobs - the regular writer as well as the offline compaction job."})}),"\n",(0,o.jsx)(i.h4,{id:"scheduling-inline-and-executing-async",children:"Scheduling inline and executing async"}),"\n",(0,o.jsx)(i.p,{children:"In this model, it is possible for a Spark Datasource writer or a Flink job to just schedule the compaction inline ( that\nwill serialize the compaction plan in the timeline but will not execute it). And then a separate utility like\nHudiCompactor or HoodieFlinkCompactor can take care of periodically executing the compaction plan."}),"\n",(0,o.jsx)(i.admonition,{type:"note",children:(0,o.jsxs)(i.p,{children:["This model may need a lock provider ",(0,o.jsx)(i.strong,{children:"if"})," metadata table is enabled."]})}),"\n",(0,o.jsx)(i.h4,{id:"hudi-compactor-utility",children:"Hudi Compactor Utility"}),"\n",(0,o.jsxs)(i.p,{children:["Hudi provides a standalone tool to execute specific compactions asynchronously. Below is an example and you can read more in the ",(0,o.jsx)(i.a,{href:"/docs/cli#compactions",children:"deployment guide"}),"\nThe compactor utility allows to do scheduling and execution of compaction."]}),"\n",(0,o.jsx)(i.p,{children:"Example:"}),"\n",(0,o.jsx)(i.pre,{children:(0,o.jsx)(i.code,{className:"language-properties",children:"spark-submit --packages org.apache.hudi:hudi-utilities-slim-bundle_2.12:1.0.0,org.apache.hudi:hudi-spark3.5-bundle_2.12:1.0.0 \\\n--class org.apache.hudi.utilities.HoodieCompactor \\\n--base-path \\\n--table-name \\\n--schema-file \\\n--instant-time \n"})}),"\n",(0,o.jsxs)(i.p,{children:["Note, the ",(0,o.jsx)(i.code,{children:"instant-time"})," parameter is now optional for the Hudi Compactor Utility. If using the utility without ",(0,o.jsx)(i.code,{children:"--instant time"}),",\nthe spark-submit will execute the earliest scheduled compaction on the Hudi timeline."]}),"\n",(0,o.jsx)(i.h4,{id:"hudi-cli",children:"Hudi CLI"}),"\n",(0,o.jsxs)(i.p,{children:["Hudi CLI is yet another way to execute specific compactions asynchronously. Here is an example and you can read more in the ",(0,o.jsx)(i.a,{href:"/docs/cli#compactions",children:"deployment guide"})]}),"\n",(0,o.jsx)(i.p,{children:"Example:"}),"\n",(0,o.jsx)(i.pre,{children:(0,o.jsx)(i.code,{className:"language-properties",children:"hudi:trips->compaction run --tableName --parallelism --compactionInstant \n...\n"})}),"\n",(0,o.jsx)(i.h4,{id:"flink-offline-compaction",children:"Flink Offline Compaction"}),"\n",(0,o.jsxs)(i.p,{children:["Offline compaction needs to submit the Flink task on the command line. The program entry is as follows: ",(0,o.jsx)(i.code,{children:"hudi-flink-bundle_2.11-0.9.0-SNAPSHOT.jar"})," :\n",(0,o.jsx)(i.code,{children:"org.apache.hudi.sink.compact.HoodieFlinkCompactor"})]}),"\n",(0,o.jsx)(i.pre,{children:(0,o.jsx)(i.code,{className:"language-bash",children:"# Command line\n./bin/flink run -c org.apache.hudi.sink.compact.HoodieFlinkCompactor lib/hudi-flink-bundle_2.11-0.9.0.jar --path hdfs://xxx:9000/table\n"})}),"\n",(0,o.jsx)(i.h4,{id:"options",children:"Options"}),"\n",(0,o.jsxs)(i.table,{children:[(0,o.jsx)(i.thead,{children:(0,o.jsxs)(i.tr,{children:[(0,o.jsx)(i.th,{children:"Option Name"}),(0,o.jsx)(i.th,{children:"Default"}),(0,o.jsx)(i.th,{children:"Description"})]})}),(0,o.jsxs)(i.tbody,{children:[(0,o.jsxs)(i.tr,{children:[(0,o.jsx)(i.td,{children:(0,o.jsx)(i.code,{children:"--path"})}),(0,o.jsx)(i.td,{children:(0,o.jsx)(i.code,{children:"n/a **(Required)**"})}),(0,o.jsx)(i.td,{children:"The path where the target table is stored on Hudi"})]}),(0,o.jsxs)(i.tr,{children:[(0,o.jsx)(i.td,{children:(0,o.jsx)(i.code,{children:"--compaction-max-memory"})}),(0,o.jsxs)(i.td,{children:[(0,o.jsx)(i.code,{children:"100"})," (Optional)"]}),(0,o.jsx)(i.td,{children:"The index map size of log data during compaction, 100 MB by default. If you have enough memory, you can turn up this parameter"})]}),(0,o.jsxs)(i.tr,{children:[(0,o.jsx)(i.td,{children:(0,o.jsx)(i.code,{children:"--schedule"})}),(0,o.jsxs)(i.td,{children:[(0,o.jsx)(i.code,{children:"false"})," (Optional)"]}),(0,o.jsx)(i.td,{children:"whether to execute the operation of scheduling compaction plan. When the write process is still writing\uff0c turning on this parameter have a risk of losing data. Therefore, it must be ensured that there are no write tasks currently writing data to this table when this parameter is turned on"})]}),(0,o.jsxs)(i.tr,{children:[(0,o.jsx)(i.td,{children:(0,o.jsx)(i.code,{children:"--seq"})}),(0,o.jsxs)(i.td,{children:[(0,o.jsx)(i.code,{children:"LIFO"})," (Optional)"]}),(0,o.jsxs)(i.td,{children:["The order in which compaction tasks are executed. Executing from the latest compaction plan by default. ",(0,o.jsx)(i.code,{children:"LIFO"}),": executing from the latest plan. ",(0,o.jsx)(i.code,{children:"FIFO"}),": executing from the oldest plan."]})]}),(0,o.jsxs)(i.tr,{children:[(0,o.jsx)(i.td,{children:(0,o.jsx)(i.code,{children:"--service"})}),(0,o.jsxs)(i.td,{children:[(0,o.jsx)(i.code,{children:"false"})," (Optional)"]}),(0,o.jsx)(i.td,{children:"Whether to start a monitoring service that checks and schedules new compaction task in configured interval."})]}),(0,o.jsxs)(i.tr,{children:[(0,o.jsx)(i.td,{children:(0,o.jsx)(i.code,{children:"--min-compaction-interval-seconds"})}),(0,o.jsxs)(i.td,{children:[(0,o.jsx)(i.code,{children:"600(s)"})," (optional)"]}),(0,o.jsx)(i.td,{children:"The checking interval for service mode, by default 10 minutes."})]})]})]}),"\n",(0,o.jsx)(i.h2,{id:"related-resources",children:"Related Resources"}),"\n",(0,o.jsxs)(i.p,{children:[(0,o.jsx)("h3",{children:"Blogs"}),"\n",(0,o.jsx)(i.a,{href:"https://medium.com/@simpsons/apache-hudi-compaction-6e6383790234",children:"Apache Hudi Compaction"}),"\n",(0,o.jsx)(i.a,{href:"https://medium.com/@simpsons/standalone-hoodiecompactor-utility-890198e4c539",children:"Standalone HoodieCompactor Utility"})]})]})}function h(e={}){const{wrapper:i}={...(0,a.R)(),...e.components};return i?(0,o.jsx)(i,{...e,children:(0,o.jsx)(d,{...e})}):d(e)}},67587:(e,i,t)=>{t.d(i,{A:()=>n});const n=t.p+"assets/images/hudi_mor_file_layout-643f9f7fda5aa0d532682af27fe3e42c.jpg"},56040:(e,i,t)=>{t.d(i,{A:()=>n});const n=t.p+"assets/images/hudi_mor_file_layout_post_compaction-9f10af785d4927dc3d66303dac5bc7ba.jpg"},28453:(e,i,t)=>{t.d(i,{R:()=>s,x:()=>r});var n=t(96540);const o={},a=n.createContext(o);function s(e){const i=n.useContext(a);return n.useMemo((function(){return"function"==typeof e?e(i):{...i,...e}}),[i,e])}function r(e){let i;return i=e.disableParentContext?"function"==typeof e.components?e.components(o):e.components||o:s(e.components),n.createElement(a.Provider,{value:i},e.children)}}}]);
\ No newline at end of file
diff --git a/content/assets/js/1a20bc57.7e53ee45.js b/content/assets/js/1a20bc57.7e53ee45.js
deleted file mode 100644
index 8db1f3ee1cc7b..0000000000000
--- a/content/assets/js/1a20bc57.7e53ee45.js
+++ /dev/null
@@ -1 +0,0 @@
-"use strict";(self.webpackChunkhudi=self.webpackChunkhudi||[]).push([[59114],{46768:(e,_,n)=>{n.r(_),n.d(_,{assets:()=>l,contentTitle:()=>s,default:()=>d,frontMatter:()=>o,metadata:()=>t,toc:()=>r});const t=JSON.parse('{"id":"cli","title":"CLI","description":"Local set up","source":"@site/docs/cli.md","sourceDirName":".","slug":"/cli","permalink":"/docs/next/cli","draft":false,"unlisted":false,"editUrl":"https://github.com/apache/hudi/tree/asf-site/website/docs/cli.md","tags":[],"version":"current","frontMatter":{"title":"CLI","keywords":["hudi","cli"],"last_modified_at":"2021-08-18T19:59:57.000Z"},"sidebar":"docs","previous":{"title":"SQL Procedures","permalink":"/docs/next/procedures"},"next":{"title":"Metrics","permalink":"/docs/next/metrics"}}');var i=n(74848),a=n(28453);const o={title:"CLI",keywords:["hudi","cli"],last_modified_at:new Date("2021-08-18T19:59:57.000Z")},s=void 0,l={},r=[{value:"Local set up",id:"local-set-up",level:3},{value:"Hudi CLI Bundle setup",id:"hudi-cli-bundle-setup",level:3},{value:"Base path",id:"base-path",level:3},{value:"Using Hudi-cli in S3",id:"using-hudi-cli-in-s3",level:3},{value:"Note: These AWS jar versions below are specific to Spark 3.2.0",id:"note-these-aws-jar-versions-below-are-specific-to-spark-320",level:4},{value:"Using hudi-cli on Google Dataproc",id:"using-hudi-cli-on-google-dataproc",level:3},{value:"Connect to a Kerberized cluster",id:"connect-to-a-kerberized-cluster",level:2},{value:"Using hudi-cli",id:"using-hudi-cli",level:2},{value:"Inspecting Commits",id:"inspecting-commits",level:3},{value:"Drilling Down to a specific Commit",id:"drilling-down-to-a-specific-commit",level:3},{value:"FileSystem View",id:"filesystem-view",level:3},{value:"Statistics",id:"statistics",level:3},{value:"Archived Commits",id:"archived-commits",level:3},{value:"Compactions",id:"compactions",level:3},{value:"Validate Compaction",id:"validate-compaction",level:3},{value:"Unscheduling Compaction",id:"unscheduling-compaction",level:3},{value:"Repair Compaction",id:"repair-compaction",level:3},{value:"Savepoint and Restore",id:"savepoint-and-restore",level:3},{value:"Upgrade and Downgrade Table",id:"upgrade-and-downgrade-table",level:3},{value:"Change Hudi Table Type",id:"change-hudi-table-type",level:3}];function c(e){const _={a:"a",admonition:"admonition",br:"br",code:"code",h2:"h2",h3:"h3",h4:"h4",li:"li",ol:"ol",p:"p",pre:"pre",strong:"strong",table:"table",tbody:"tbody",td:"td",th:"th",thead:"thead",tr:"tr",...(0,a.R)(),...e.components};return(0,i.jsxs)(i.Fragment,{children:[(0,i.jsx)(_.h3,{id:"local-set-up",children:"Local set up"}),"\n",(0,i.jsxs)(_.p,{children:["Once hudi has been built, the shell can be fired by via ",(0,i.jsx)(_.code,{children:"cd hudi-cli && ./hudi-cli.sh"}),"."]}),"\n",(0,i.jsx)(_.h3,{id:"hudi-cli-bundle-setup",children:"Hudi CLI Bundle setup"}),"\n",(0,i.jsxs)(_.p,{children:["In release ",(0,i.jsx)(_.code,{children:"0.13.0"})," we have now added another way of launching the ",(0,i.jsx)(_.code,{children:"hudi cli"}),", which is using the ",(0,i.jsx)(_.code,{children:"hudi-cli-bundle"}),"."]}),"\n",(0,i.jsxs)(_.p,{children:["There are a couple of requirements when using this approach such as having ",(0,i.jsx)(_.code,{children:"spark"})," installed locally on your machine.\nIt is required to use a spark distribution with hadoop dependencies packaged such as ",(0,i.jsx)(_.code,{children:"spark-3.3.1-bin-hadoop2.tgz"})," from ",(0,i.jsx)(_.a,{href:"https://archive.apache.org/dist/spark/",children:"https://archive.apache.org/dist/spark/"}),".\nWe also recommend you set an env variable ",(0,i.jsx)(_.code,{children:"$SPARK_HOME"})," to the path of where spark is installed on your machine.\nOne important thing to note is that the ",(0,i.jsx)(_.code,{children:"hudi-spark-bundle"})," should also be present when using the ",(0,i.jsx)(_.code,{children:"hudi-cli-bundle"}),".",(0,i.jsx)(_.br,{}),"\n","To provide the locations of these bundle jars you can set them in your shell like so:\n",(0,i.jsx)(_.code,{children:"export CLI_BUNDLE_JAR="})," , ",(0,i.jsx)(_.code,{children:"export SPARK_BUNDLE_JAR="}),"."]}),"\n",(0,i.jsx)(_.p,{children:"For steps see below if you are not compiling the project and downloading the jars:"}),"\n",(0,i.jsxs)(_.ol,{children:["\n",(0,i.jsx)(_.li,{children:"Create an empty folder as a new directory"}),"\n",(0,i.jsx)(_.li,{children:"Copy the hudi-cli-bundle jars and hudi-spark*-bundle jars to this directory"}),"\n",(0,i.jsx)(_.li,{children:"Copy the following script and folder to this directory"}),"\n"]}),"\n",(0,i.jsx)(_.pre,{children:(0,i.jsx)(_.code,{children:"packaging/hudi-cli-bundle/hudi-cli-with-bundle.sh\npackaging/hudi-cli-bundle/conf . the `conf` folder should be in this directory.\n"})}),"\n",(0,i.jsxs)(_.ol,{start:"4",children:["\n",(0,i.jsx)(_.li,{children:"Start Hudi CLI shell with environment variables set"}),"\n"]}),"\n",(0,i.jsx)(_.pre,{children:(0,i.jsx)(_.code,{children:"export SPARK_HOME=\nexport CLI_BUNDLE_JAR=\nexport SPARK_BUNDLE_JAR=\n\n./hudi-cli-with-bundle.sh\n\n"})}),"\n",(0,i.jsx)(_.h3,{id:"base-path",children:"Base path"}),"\n",(0,i.jsxs)(_.p,{children:["A hudi table resides on DFS, in a location referred to as the ",(0,i.jsx)(_.code,{children:"basePath"})," and\nwe would need this location in order to connect to a Hudi table. Hudi library effectively manages this table internally, using ",(0,i.jsx)(_.code,{children:".hoodie"})," subfolder to track all metadata."]}),"\n",(0,i.jsx)(_.h3,{id:"using-hudi-cli-in-s3",children:"Using Hudi-cli in S3"}),"\n",(0,i.jsxs)(_.p,{children:["If you are using hudi that comes packaged with AWS EMR, you can find instructions to use hudi-cli ",(0,i.jsx)(_.a,{href:"https://docs.aws.amazon.com/emr/latest/ReleaseGuide/emr-hudi-cli.html",children:"here"}),".\nIf you are not using EMR, or would like to use latest hudi-cli from master, you can follow the below steps to access S3 dataset in your local environment (laptop)."]}),"\n",(0,i.jsx)(_.p,{children:"Build Hudi with corresponding Spark version, for eg, -Dspark3.1.x"}),"\n",(0,i.jsx)(_.p,{children:"Set the following environment variables."}),"\n",(0,i.jsx)(_.pre,{children:(0,i.jsx)(_.code,{children:"export AWS_REGION=us-east-2\nexport AWS_ACCESS_KEY_ID=\nexport AWS_SECRET_ACCESS_KEY=\nexport SPARK_HOME=\n"})}),"\n",(0,i.jsx)(_.p,{children:"Ensure you set the SPARK_HOME to your local spark home compatible to compiled hudi spark version above."}),"\n",(0,i.jsx)(_.p,{children:"Apart from these, we might need to add aws jars to class path so that accessing S3 is feasible from local.\nWe need two jars, namely, aws-java-sdk-bundle jar and hadoop-aws jar which you can find online.\nFor eg:"}),"\n",(0,i.jsx)(_.pre,{children:(0,i.jsx)(_.code,{children:"wget https://repo1.maven.org/maven2/org/apache/hadoop/hadoop-aws/3.2.0/hadoop-aws-3.2.0.jar -o /lib/spark-3.2.0-bin-hadoop3.2/jars/hadoop-aws-3.2.0.jar\nwget https://repo1.maven.org/maven2/com/amazonaws/aws-java-sdk-bundle/1.11.375/aws-java-sdk-bundle-1.11.375.jar -o /lib/spark-3.2.0-bin-hadoop3.2/jars/aws-java-sdk-bundle-1.11.375.jar\n"})}),"\n",(0,i.jsx)(_.h4,{id:"note-these-aws-jar-versions-below-are-specific-to-spark-320",children:"Note: These AWS jar versions below are specific to Spark 3.2.0"}),"\n",(0,i.jsx)(_.pre,{children:(0,i.jsx)(_.code,{children:"export CLIENT_JAR=/lib/spark-3.2.0-bin-hadoop3.2/jars/aws-java-sdk-bundle-1.12.48.jar:/lib/spark-3.2.0-bin-hadoop3.2/jars/hadoop-aws-3.3.1.jar\n"})}),"\n",(0,i.jsx)(_.p,{children:"Once these are set, you are good to launch hudi-cli and access S3 dataset."}),"\n",(0,i.jsx)(_.pre,{children:(0,i.jsx)(_.code,{children:"./hudi-cli/hudi-cli.sh\n"})}),"\n",(0,i.jsx)(_.h3,{id:"using-hudi-cli-on-google-dataproc",children:"Using hudi-cli on Google Dataproc"}),"\n",(0,i.jsxs)(_.p,{children:[(0,i.jsx)(_.a,{href:"https://cloud.google.com/dataproc",children:"Dataproc"})," is Google's managed service for running Apache Hadoop, Apache Spark,\nApache Flink, Presto and many other frameworks, including Hudi. If you want to run the Hudi CLI on a Dataproc node\nwhich has not been launched with Hudi support enabled, you can use the steps below:"]}),"\n",(0,i.jsx)(_.p,{children:"These steps use Hudi version 0.13.0. If you want to use a different version you will have to edit the below commands\nappropriately:"}),"\n",(0,i.jsxs)(_.ol,{children:["\n",(0,i.jsx)(_.li,{children:"Once you've started the Dataproc cluster, you can ssh into it as follows:"}),"\n"]}),"\n",(0,i.jsx)(_.pre,{children:(0,i.jsx)(_.code,{children:'$ gcloud compute ssh --zone "YOUR_ZONE" "HOSTNAME_OF_MASTER_NODE" --project "YOUR_PROJECT"\n'})}),"\n",(0,i.jsxs)(_.ol,{start:"2",children:["\n",(0,i.jsx)(_.li,{children:"Download the Hudi CLI bundle"}),"\n"]}),"\n",(0,i.jsx)(_.pre,{children:(0,i.jsx)(_.code,{children:"wget https://repo1.maven.org/maven2/org/apache/hudi/hudi-cli-bundle_2.12/0.13.0/hudi-cli-bundle_2.12-0.13.0.jar \n"})}),"\n",(0,i.jsxs)(_.ol,{start:"3",children:["\n",(0,i.jsx)(_.li,{children:"Download the Hudi Spark bundle"}),"\n"]}),"\n",(0,i.jsx)(_.pre,{children:(0,i.jsx)(_.code,{children:"wget https://repo1.maven.org/maven2/org/apache/hudi/hudi-spark-bundle_2.12/0.13.0/hudi-spark-bundle_2.12-0.13.0.jar\n"})}),"\n",(0,i.jsxs)(_.ol,{start:"4",children:["\n",(0,i.jsx)(_.li,{children:"Download the shell script that launches Hudi CLI bundle"}),"\n"]}),"\n",(0,i.jsx)(_.pre,{children:(0,i.jsx)(_.code,{children:"wget https://raw.githubusercontent.com/apache/hudi/release-0.13.0/packaging/hudi-cli-bundle/hudi-cli-with-bundle.sh\n"})}),"\n",(0,i.jsxs)(_.ol,{start:"5",children:["\n",(0,i.jsx)(_.li,{children:"Launch Hudi CLI bundle with appropriate environment variables as follows:"}),"\n"]}),"\n",(0,i.jsx)(_.pre,{children:(0,i.jsx)(_.code,{children:"CLIENT_JAR=$DATAPROC_DIR/lib/gcs-connector.jar CLI_BUNDLE_JAR=hudi-cli-bundle_2.12-0.13.0.jar SPARK_BUNDLE_JAR=hudi-spark-bundle_2.12-0.13.0.jar ./hudi-cli-with-bundle.sh \n"})}),"\n",(0,i.jsxs)(_.ol,{start:"6",children:["\n",(0,i.jsxs)(_.li,{children:["\n",(0,i.jsxs)(_.p,{children:["hudi->connect --path gs://path_to_some_table",(0,i.jsx)(_.br,{}),"\n","Metadata for table some_table loaded"]}),"\n"]}),"\n",(0,i.jsxs)(_.li,{children:["\n",(0,i.jsxs)(_.p,{children:["hudi:some_table->commits show --limit 5",(0,i.jsx)(_.br,{}),"\n","This command should show the recent commits, if the above steps work correctly."]}),"\n"]}),"\n"]}),"\n",(0,i.jsx)(_.h2,{id:"connect-to-a-kerberized-cluster",children:"Connect to a Kerberized cluster"}),"\n",(0,i.jsxs)(_.p,{children:["Before connecting to a Kerberized cluster, you can use ",(0,i.jsx)(_.strong,{children:"kerberos kinit"})," command. Following is the usage of this command."]}),"\n",(0,i.jsx)(_.pre,{children:(0,i.jsx)(_.code,{className:"language-shell",children:"hudi->help kerberos kinit\nNAME\n kerberos kinit - Perform Kerberos authentication\n\nSYNOPSIS\n kerberos kinit --krb5conf String [--principal String] [--keytab String]\n\nOPTIONS\n --krb5conf String\n Path to krb5.conf\n [Optional, default = /etc/krb5.conf]\n\n --principal String\n Kerberos principal\n [Mandatory]\n\n --keytab String\n Path to keytab\n [Mandatory]\n"})}),"\n",(0,i.jsx)(_.p,{children:"For example:"}),"\n",(0,i.jsx)(_.pre,{children:(0,i.jsx)(_.code,{className:"language-shell",children:"hudi->kerberos kinit --principal user/host@DOMAIN --keytab /etc/security/keytabs/user.keytab\nPerform Kerberos authentication\nParameters:\n--krb5conf: /etc/krb5.conf\n--principal: user/host@DOMAIN\n--keytab: /etc/security/keytabs/user.keytab\nKerberos current user: user/host@DOMAIN (auth:KERBEROS)\nKerberos login user: user/host@DOMAIN (auth:KERBEROS)\nKerberos authentication success\n"})}),"\n",(0,i.jsx)(_.p,{children:'If you see "Kerberos authentication success" in the command output, it means Kerberos authentication has been successful.'}),"\n",(0,i.jsx)(_.h2,{id:"using-hudi-cli",children:"Using hudi-cli"}),"\n",(0,i.jsx)(_.p,{children:"To initialize a hudi table, use the following command."}),"\n",(0,i.jsx)(_.pre,{children:(0,i.jsx)(_.code,{className:"language-java",children:"===================================================================\n* ___ ___ *\n* /\\__\\ ___ /\\ \\ ___ *\n* / / / /\\__\\ / \\ \\ /\\ \\ *\n* / /__/ / / / / /\\ \\ \\ \\ \\ \\ *\n* / \\ \\ ___ / / / / / \\ \\__\\ / \\__\\ *\n* / /\\ \\ /\\__\\ / /__/ ___ / /__/ \\ |__| / /\\/__/ *\n* \\/ \\ \\/ / / \\ \\ \\ /\\__\\ \\ \\ \\ / / / /\\/ / / *\n* \\ / / \\ \\ / / / \\ \\ / / / \\ /__/ *\n* / / / \\ \\/ / / \\ \\/ / / \\ \\__\\ *\n* / / / \\ / / \\ / / \\/__/ *\n* \\/__/ \\/__/ \\/__/ Apache Hudi CLI *\n* *\n===================================================================\n\nhudi->create --path /user/hive/warehouse/table1 --tableName hoodie_table_1 --tableType COPY_ON_WRITE\n.....\n"})}),"\n",(0,i.jsx)(_.p,{children:"To see the description of hudi table, use the command:"}),"\n",(0,i.jsx)(_.pre,{children:(0,i.jsx)(_.code,{className:"language-java",children:"hudi:hoodie_table_1->desc\n18/09/06 15:57:19 INFO timeline.HoodieActiveTimeline: Loaded instants []\n _________________________________________________________\n | Property | Value |\n |========================================================|\n | basePath | ... |\n | metaPath | ... |\n | fileSystem | hdfs |\n | hoodie.table.name | hoodie_table_1 |\n | hoodie.table.type | COPY_ON_WRITE |\n | hoodie.archivelog.folder| |\n"})}),"\n",(0,i.jsx)(_.p,{children:"Following is a sample command to connect to a Hudi table contains uber trips."}),"\n",(0,i.jsx)(_.pre,{children:(0,i.jsx)(_.code,{className:"language-java",children:"hudi:trips->connect --path /app/uber/trips\n\n16/10/05 23:20:37 INFO model.HoodieTableMetadata: All commits :HoodieCommits{commitList=[20161002045850, 20161002052915, 20161002055918, 20161002065317, 20161002075932, 20161002082904, 20161002085949, 20161002092936, 20161002105903, 20161002112938, 20161002123005, 20161002133002, 20161002155940, 20161002165924, 20161002172907, 20161002175905, 20161002190016, 20161002192954, 20161002195925, 20161002205935, 20161002215928, 20161002222938, 20161002225915, 20161002232906, 20161003003028, 20161003005958, 20161003012936, 20161003022924, 20161003025859, 20161003032854, 20161003042930, 20161003052911, 20161003055907, 20161003062946, 20161003065927, 20161003075924, 20161003082926, 20161003085925, 20161003092909, 20161003100010, 20161003102913, 20161003105850, 20161003112910, 20161003115851, 20161003122929, 20161003132931, 20161003142952, 20161003145856, 20161003152953, 20161003155912, 20161003162922, 20161003165852, 20161003172923, 20161003175923, 20161003195931, 20161003210118, 20161003212919, 20161003215928, 20161003223000, 20161003225858, 20161004003042, 20161004011345, 20161004015235, 20161004022234, 20161004063001, 20161004072402, 20161004074436, 20161004080224, 20161004082928, 20161004085857, 20161004105922, 20161004122927, 20161004142929, 20161004163026, 20161004175925, 20161004194411, 20161004203202, 20161004211210, 20161004214115, 20161004220437, 20161004223020, 20161004225321, 20161004231431, 20161004233643, 20161005010227, 20161005015927, 20161005022911, 20161005032958, 20161005035939, 20161005052904, 20161005070028, 20161005074429, 20161005081318, 20161005083455, 20161005085921, 20161005092901, 20161005095936, 20161005120158, 20161005123418, 20161005125911, 20161005133107, 20161005155908, 20161005163517, 20161005165855, 20161005180127, 20161005184226, 20161005191051, 20161005193234, 20161005203112, 20161005205920, 20161005212949, 20161005223034, 20161005225920]}\nMetadata for table trips loaded\n"})}),"\n",(0,i.jsx)(_.p,{children:"Once connected to the table, a lot of other commands become available. The shell has contextual autocomplete help (press TAB) and below is a list of all commands, few of which are reviewed in this section"}),"\n",(0,i.jsx)(_.pre,{children:(0,i.jsx)(_.code,{className:"language-shell",children:"hudi:trips->help\n* ! - Allows execution of operating system (OS) commands\n* // - Inline comment markers (start of line only)\n* ; - Inline comment markers (start of line only)\n* bootstrap index showmapping - Show bootstrap index mapping\n* bootstrap index showpartitions - Show bootstrap indexed partitions\n* bootstrap run - Run a bootstrap action for current Hudi table\n* clean showpartitions - Show partition level details of a clean\n* cleans refresh - Refresh table metadata\n* cleans run - run clean\n* cleans show - Show the cleans\n* clear - Clears the console\n* cls - Clears the console\n* clustering run - Run Clustering\n* clustering schedule - Schedule Clustering\n* clustering scheduleAndExecute - Run Clustering. Make a cluster plan first and execute that plan immediately\n* commit rollback - Rollback a commit\n* commits compare - Compare commits with another Hoodie table\n* commit show_write_stats - Show write stats of a commit\n* commit showfiles - Show file level details of a commit\n* commit showpartitions - Show partition level details of a commit\n* commits refresh - Refresh table metadata\n* commits show - Show the commits\n* commits showarchived - Show the archived commits\n* commits sync - Sync commits with another Hoodie table\n* compaction repair - Renames the files to make them consistent with the timeline as dictated by Hoodie metadata. Use when compaction unschedule fails partially.\n* compaction run - Run Compaction for given instant time\n* compaction schedule - Schedule Compaction\n* compaction scheduleAndExecute - Schedule compaction plan and execute this plan\n* compaction show - Shows compaction details for a specific compaction instant\n* compaction showarchived - Shows compaction details for a specific compaction instant\n* compactions show all - Shows all compactions that are in active timeline\n* compactions showarchived - Shows compaction details for specified time window\n* compaction unschedule - Unschedule Compaction\n* compaction unscheduleFileId - UnSchedule Compaction for a fileId\n* compaction validate - Validate Compaction\n* connect - Connect to a hoodie table\n* create - Create a hoodie table if not present\n* date - Displays the local date and time\n* desc - Describe Hoodie Table properties\n* downgrade table - Downgrades a table\n* exit - Exits the shell\n* export instants - Export Instants and their metadata from the Timeline\n* fetch table schema - Fetches latest table schema\n* hdfsparquetimport - Imports Parquet table to a hoodie table\n* help - List all commands usage\n* marker delete - Delete the marker\n* metadata create - Create the Metadata Table if it does not exist\n* metadata delete - Remove the Metadata Table\n* metadata init - Update the metadata table from commits since the creation\n* metadata list-files - Print a list of all files in a partition from the metadata\n* metadata list-partitions - List all partitions from metadata\n* metadata refresh - Refresh table metadata\n* metadata set - Set options for Metadata Table\n* metadata stats - Print stats about the metadata\n* metadata validate-files - Validate all files in all partitions from the metadata\n* quit - Exits the shell\n* refresh - Refresh table metadata\n* repair addpartitionmeta - Add partition metadata to a table, if not present\n* repair corrupted clean files - repair corrupted clean files\n* repair deduplicate - De-duplicate a partition path contains duplicates & produce repaired files to replace with\n* repair migrate-partition-meta - Migrate all partition meta file currently stored in text format to be stored in base file format. See HoodieTableConfig#PARTITION_METAFILE_USE_DATA_FORMAT.\n* repair overwrite-hoodie-props - Overwrite hoodie.properties with provided file. Risky operation. Proceed with caution!\n* savepoint create - Savepoint a commit\n* savepoint delete - Delete the savepoint\n* savepoint rollback - Savepoint a commit\n* savepoints refresh - Refresh table metadata\n* savepoints show - Show the savepoints\n* script - Parses the specified resource file and executes its commands\n* set - Set spark launcher env to cli\n* show archived commits - Read commits from archived files and show details\n* show archived commit stats - Read commits from archived files and show details\n* show env - Show spark launcher env by key\n* show envs all - Show spark launcher envs\n* show fsview all - Show entire file-system view\n* show fsview latest - Show latest file-system view\n* show logfile metadata - Read commit metadata from log files\n* show logfile records - Read records from log files\n* show rollback - Show details of a rollback instant\n* show rollbacks - List all rollback instants\n* stats filesizes - File Sizes. Display summary stats on sizes of files\n* stats wa - Write Amplification. Ratio of how many records were upserted to how many records were actually written\n* sync validate - Validate the sync by counting the number of records\n* system properties - Shows the shell's properties\n* table delete-configs - Delete the supplied table configs from the table.\n* table recover-configs - Recover table configs, from update/delete that failed midway.\n* table update-configs - Update the table configs with configs with provided file.\n* temp_delete - Delete view name\n* temp_query - query against created temp view\n* temp delete - Delete view name\n* temp query - query against created temp view\n* temps_show - Show all views name\n* temps show - Show all views name\n* upgrade table - Upgrades a table\n* utils loadClass - Load a class\n* version - Displays shell version\n\nhudi:trips->\n"})}),"\n",(0,i.jsx)(_.h3,{id:"inspecting-commits",children:"Inspecting Commits"}),"\n",(0,i.jsxs)(_.p,{children:["The task of upserting or inserting a batch of incoming records is known as a ",(0,i.jsx)(_.strong,{children:"commit"})," in Hudi. A commit provides basic atomicity guarantees such that only committed data is available for querying.\nEach commit has a monotonically increasing string/number called the ",(0,i.jsx)(_.strong,{children:"commit number"}),". Typically, this is the time at which we started the commit."]}),"\n",(0,i.jsx)(_.p,{children:"To view some basic information about the last 10 commits,"}),"\n",(0,i.jsx)(_.pre,{children:(0,i.jsx)(_.code,{className:"language-java",children:'hudi:trips->commits show --sortBy "Total Bytes Written" --desc true --limit 10\n ________________________________________________________________________________________________________________________________________________________________________\n | CommitTime | Total Bytes Written| Total Files Added| Total Files Updated| Total Partitions Written| Total Records Written| Total Update Records Written| Total Errors|\n |=======================================================================================================================================================================|\n ....\n ....\n ....\n'})}),"\n",(0,i.jsx)(_.p,{children:"At the start of each write, Hudi also writes a .inflight commit to the .hoodie folder. You can use the timestamp there to estimate how long the commit has been inflight"}),"\n",(0,i.jsx)(_.pre,{children:(0,i.jsx)(_.code,{className:"language-java",children:"$ hdfs dfs -ls /app/uber/trips/.hoodie/*.inflight\n-rw-r--r-- 3 vinoth supergroup 321984 2016-10-05 23:18 /app/uber/trips/.hoodie/20161005225920.inflight\n"})}),"\n",(0,i.jsx)(_.h3,{id:"drilling-down-to-a-specific-commit",children:"Drilling Down to a specific Commit"}),"\n",(0,i.jsx)(_.p,{children:"To understand how the writes spread across specific partiions,"}),"\n",(0,i.jsx)(_.pre,{children:(0,i.jsx)(_.code,{className:"language-java",children:'hudi:trips->commit showpartitions --commit 20161005165855 --sortBy "Total Bytes Written" --desc true --limit 10\n __________________________________________________________________________________________________________________________________________\n | Partition Path| Total Files Added| Total Files Updated| Total Records Inserted| Total Records Updated| Total Bytes Written| Total Errors|\n |=========================================================================================================================================|\n ....\n ....\n'})}),"\n",(0,i.jsx)(_.p,{children:"If you need file level granularity , we can do the following"}),"\n",(0,i.jsx)(_.pre,{children:(0,i.jsx)(_.code,{className:"language-java",children:'hudi:trips->commit showfiles --commit 20161005165855 --sortBy "Partition Path"\n ________________________________________________________________________________________________________________________________________________________\n | Partition Path| File ID | Previous Commit| Total Records Updated| Total Records Written| Total Bytes Written| Total Errors|\n |=======================================================================================================================================================|\n ....\n ....\n'})}),"\n",(0,i.jsx)(_.h3,{id:"filesystem-view",children:"FileSystem View"}),"\n",(0,i.jsx)(_.p,{children:"Hudi views each partition as a collection of file-groups with each file-group containing a list of file-slices in commit order (See concepts).\nThe below commands allow users to view the file-slices for a data-set."}),"\n",(0,i.jsx)(_.pre,{children:(0,i.jsx)(_.code,{className:"language-java",children:'hudi:stock_ticks_mor->show fsview all\n ....\n _______________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________\n | Partition | FileId | Base-Instant | Data-File | Data-File Size| Num Delta Files| Total Delta File Size| Delta Files |\n |==============================================================================================================================================================================================================================================================================================================================================================================================================|\n | 2018/08/31| 111415c3-f26d-4639-86c8-f9956f245ac3| 20181002180759| hdfs://namenode:8020/user/hive/warehouse/stock_ticks_mor/2018/08/31/111415c3-f26d-4639-86c8-f9956f245ac3_0_20181002180759.parquet| 432.5 KB | 1 | 20.8 KB | [HoodieLogFile {hdfs://namenode:8020/user/hive/warehouse/stock_ticks_mor/2018/08/31/.111415c3-f26d-4639-86c8-f9956f245ac3_20181002180759.log.1}]|\n\n\n\nhudi:stock_ticks_mor->show fsview latest --partitionPath "2018/08/31"\n ......\n __________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________\n | Partition | FileId | Base-Instant | Data-File | Data-File Size| Num Delta Files| Total Delta Size| Delta Size - compaction scheduled| Delta Size - compaction unscheduled| Delta To Base Ratio - compaction scheduled| Delta To Base Ratio - compaction unscheduled| Delta Files - compaction scheduled | Delta Files - compaction unscheduled|\n |=================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================|\n | 2018/08/31| 111415c3-f26d-4639-86c8-f9956f245ac3| 20181002180759| hdfs://namenode:8020/user/hive/warehouse/stock_ticks_mor/2018/08/31/111415c3-f26d-4639-86c8-f9956f245ac3_0_20181002180759.parquet| 432.5 KB | 1 | 20.8 KB | 20.8 KB | 0.0 B | 0.0 B | 0.0 B | [HoodieLogFile {hdfs://namenode:8020/user/hive/warehouse/stock_ticks_mor/2018/08/31/.111415c3-f26d-4639-86c8-f9956f245ac3_20181002180759.log.1}]| [] |\n\n'})}),"\n",(0,i.jsx)(_.h3,{id:"statistics",children:"Statistics"}),"\n",(0,i.jsx)(_.p,{children:"Since Hudi directly manages file sizes for DFS table, it might be good to get an overall picture"}),"\n",(0,i.jsx)(_.pre,{children:(0,i.jsx)(_.code,{className:"language-java",children:'hudi:trips->stats filesizes --partitionPath 2016/09/01 --sortBy "95th" --desc true --limit 10\n ________________________________________________________________________________________________\n | CommitTime | Min | 10th | 50th | avg | 95th | Max | NumFiles| StdDev |\n |===============================================================================================|\n | | 93.9 MB | 93.9 MB | 93.9 MB | 93.9 MB | 93.9 MB | 93.9 MB | 2 | 2.3 KB |\n ....\n ....\n'})}),"\n",(0,i.jsx)(_.p,{children:"In case of Hudi write taking much longer, it might be good to see the write amplification for any sudden increases"}),"\n",(0,i.jsx)(_.pre,{children:(0,i.jsx)(_.code,{className:"language-java",children:"hudi:trips->stats wa\n __________________________________________________________________________\n | CommitTime | Total Upserted| Total Written| Write Amplifiation Factor|\n |=========================================================================|\n ....\n ....\n"})}),"\n",(0,i.jsx)(_.h3,{id:"archived-commits",children:"Archived Commits"}),"\n",(0,i.jsx)(_.p,{children:"In order to limit the amount of growth of .commit files on DFS, Hudi archives older .commit files (with due respect to the cleaner policy) into a commits.archived file.\nThis is a sequence file that contains a mapping from commitNumber => json with raw information about the commit (same that is nicely rolled up above)."}),"\n",(0,i.jsx)(_.h3,{id:"compactions",children:"Compactions"}),"\n",(0,i.jsx)(_.p,{children:"To get an idea of the lag between compaction and writer applications, use the below command to list down all\npending compactions."}),"\n",(0,i.jsx)(_.pre,{children:(0,i.jsx)(_.code,{className:"language-java",children:"hudi:trips->compactions show all\n ___________________________________________________________________\n | Compaction Instant Time| State | Total FileIds to be Compacted|\n |==================================================================|\n | | REQUESTED| 35 |\n | | INFLIGHT | 27 |\n"})}),"\n",(0,i.jsx)(_.p,{children:"To inspect a specific compaction plan, use"}),"\n",(0,i.jsx)(_.pre,{children:(0,i.jsx)(_.code,{className:"language-java",children:"hudi:trips->compaction show --instant \n _________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________\n | Partition Path| File Id | Base Instant | Data File Path | Total Delta Files| getMetrics |\n |================================================================================================================================================================================================================================================\n | 2018/07/17 | | | viewfs://ns-default/.../../UUID_.parquet | 1 | {TOTAL_LOG_FILES=1.0, TOTAL_IO_READ_MB=1230.0, TOTAL_LOG_FILES_SIZE=2.51255751E8, TOTAL_IO_WRITE_MB=991.0, TOTAL_IO_MB=2221.0}|\n\n"})}),"\n",(0,i.jsx)(_.p,{children:"To manually schedule or run a compaction, use the below command. This command uses spark launcher to perform compaction\noperations."}),"\n",(0,i.jsxs)(_.p,{children:[(0,i.jsx)(_.strong,{children:"NOTE:"})," Make sure no other application is scheduling compaction for this table concurrently\n{: .notice--info}"]}),"\n",(0,i.jsx)(_.pre,{children:(0,i.jsx)(_.code,{className:"language-java",children:"hudi:trips->help compaction schedule\nKeyword: compaction schedule\nDescription: Schedule Compaction\n Keyword: sparkMemory\n Help: Spark executor memory\n Mandatory: false\n Default if specified: '__NULL__'\n Default if unspecified: '1G'\n\n* compaction schedule - Schedule Compaction\n"})}),"\n",(0,i.jsx)(_.pre,{children:(0,i.jsx)(_.code,{className:"language-java",children:"hudi:trips->help compaction run\nKeyword: compaction run\nDescription: Run Compaction for given instant time\n Keyword: tableName\n Help: Table name\n Mandatory: true\n Default if specified: '__NULL__'\n Default if unspecified: '__NULL__'\n\n Keyword: parallelism\n Help: Parallelism for hoodie compaction\n Mandatory: true\n Default if specified: '__NULL__'\n Default if unspecified: '__NULL__'\n\n Keyword: schemaFilePath\n Help: Path for Avro schema file\n Mandatory: true\n Default if specified: '__NULL__'\n Default if unspecified: '__NULL__'\n\n Keyword: sparkMemory\n Help: Spark executor memory\n Mandatory: true\n Default if specified: '__NULL__'\n Default if unspecified: '__NULL__'\n\n Keyword: retry\n Help: Number of retries\n Mandatory: true\n Default if specified: '__NULL__'\n Default if unspecified: '__NULL__'\n\n Keyword: compactionInstant\n Help: Base path for the target hoodie table\n Mandatory: true\n Default if specified: '__NULL__'\n Default if unspecified: '__NULL__'\n\n* compaction run - Run Compaction for given instant time\n"})}),"\n",(0,i.jsx)(_.h3,{id:"validate-compaction",children:"Validate Compaction"}),"\n",(0,i.jsx)(_.p,{children:"Validating a compaction plan : Check if all the files necessary for compactions are present and are valid"}),"\n",(0,i.jsx)(_.pre,{children:(0,i.jsx)(_.code,{className:"language-java",children:"hudi:stock_ticks_mor->compaction validate --instant 20181005222611\n...\n\n COMPACTION PLAN VALID\n\n ___________________________________________________________________________________________________________________________________________________________________________________________________________________________\n | File Id | Base Instant Time| Base Data File | Num Delta Files| Valid| Error|\n |==========================================================================================================================================================================================================================|\n | 05320e98-9a57-4c38-b809-a6beaaeb36bd| 20181005222445 | hdfs://namenode:8020/user/hive/warehouse/stock_ticks_mor/2018/08/31/05320e98-9a57-4c38-b809-a6beaaeb36bd_0_20181005222445.parquet| 1 | true | |\n\n\n\nhudi:stock_ticks_mor->compaction validate --instant 20181005222601\n\n COMPACTION PLAN INVALID\n\n _______________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________\n | File Id | Base Instant Time| Base Data File | Num Delta Files| Valid| Error |\n |=====================================================================================================================================================================================================================================================================================================|\n | 05320e98-9a57-4c38-b809-a6beaaeb36bd| 20181005222445 | hdfs://namenode:8020/user/hive/warehouse/stock_ticks_mor/2018/08/31/05320e98-9a57-4c38-b809-a6beaaeb36bd_0_20181005222445.parquet| 1 | false| All log files specified in compaction operation is not present. Missing .... |\n"})}),"\n",(0,i.jsxs)(_.p,{children:[(0,i.jsx)(_.strong,{children:"NOTE:"})," The following commands must be executed without any other writer/ingestion application running.\n{: .notice--warning}"]}),"\n",(0,i.jsx)(_.p,{children:"Sometimes, it becomes necessary to remove a fileId from a compaction-plan inorder to speed-up or unblock compaction\noperation. Any new log-files that happened on this file after the compaction got scheduled will be safely renamed\nso that are preserved. Hudi provides the following CLI to support it"}),"\n",(0,i.jsx)(_.h3,{id:"unscheduling-compaction",children:"Unscheduling Compaction"}),"\n",(0,i.jsx)(_.pre,{children:(0,i.jsx)(_.code,{className:"language-java",children:"hudi:trips->compaction unscheduleFileId --fileId \n....\nNo File renames needed to unschedule file from pending compaction. Operation successful.\n"})}),"\n",(0,i.jsx)(_.p,{children:"In other cases, an entire compaction plan needs to be reverted. This is supported by the following CLI"}),"\n",(0,i.jsx)(_.pre,{children:(0,i.jsx)(_.code,{className:"language-java",children:"hudi:trips->compaction unschedule --instant \n.....\nNo File renames needed to unschedule pending compaction. Operation successful.\n"})}),"\n",(0,i.jsx)(_.h3,{id:"repair-compaction",children:"Repair Compaction"}),"\n",(0,i.jsxs)(_.p,{children:["The above compaction unscheduling operations could sometimes fail partially (e",":g"," -> DFS temporarily unavailable). With\npartial failures, the compaction operation could become inconsistent with the state of file-slices. When you run\n",(0,i.jsx)(_.code,{children:"compaction validate"}),", you can notice invalid compaction operations if there is one. In these cases, the repair\ncommand comes to the rescue, it will rearrange the file-slices so that there is no loss and the file-slices are\nconsistent with the compaction plan"]}),"\n",(0,i.jsx)(_.pre,{children:(0,i.jsx)(_.code,{className:"language-java",children:"hudi:stock_ticks_mor->compaction repair --instant 20181005222611\n......\nCompaction successfully repaired\n.....\n"})}),"\n",(0,i.jsx)(_.h3,{id:"savepoint-and-restore",children:"Savepoint and Restore"}),"\n",(0,i.jsxs)(_.p,{children:['As the name suggest, "savepoint" saves the table as of the commit time, so that it lets you restore the table to this\nsavepoint at a later point in time if need be. You can read more about savepoints and restore ',(0,i.jsx)(_.a,{href:"disaster_recovery",children:"here"})]}),"\n",(0,i.jsx)(_.p,{children:"To trigger savepoint for a hudi table"}),"\n",(0,i.jsx)(_.pre,{children:(0,i.jsx)(_.code,{className:"language-java",children:"connect --path /tmp/hudi_trips_cow/\ncommits show\nset --conf SPARK_HOME=\nsavepoint create --commit 20220128160245447 --sparkMaster local[2]\n"})}),"\n",(0,i.jsx)(_.p,{children:"To restore the table to one of the savepointed commit:"}),"\n",(0,i.jsx)(_.pre,{children:(0,i.jsx)(_.code,{className:"language-java",children:"connect --path /tmp/hudi_trips_cow/\ncommits show\nset --conf SPARK_HOME=\nsavepoints show\n\u2554\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2557\n\u2551 SavepointTime \u2551\n\u2560\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2563\n\u2551 20220128160245447 \u2551\n\u255a\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u255d\nsavepoint rollback --savepoint 20220128160245447 --sparkMaster local[2]\n"})}),"\n",(0,i.jsx)(_.h3,{id:"upgrade-and-downgrade-table",children:"Upgrade and Downgrade Table"}),"\n",(0,i.jsx)(_.p,{children:"In case the user needs to downgrade the version of Hudi library used, the Hudi table needs to be manually downgraded\non the newer version of Hudi CLI before library downgrade. To downgrade a Hudi table through CLI, user needs to specify\nthe target Hudi table version as follows:"}),"\n",(0,i.jsx)(_.pre,{children:(0,i.jsx)(_.code,{className:"language-shell",children:"connect --path \ndowngrade table --toVersion \n"})}),"\n",(0,i.jsx)(_.p,{children:"The following table shows the Hudi table versions corresponding to the Hudi release versions:"}),"\n",(0,i.jsxs)(_.table,{children:[(0,i.jsx)(_.thead,{children:(0,i.jsxs)(_.tr,{children:[(0,i.jsx)(_.th,{style:{textAlign:"left"},children:"Hudi Table Version"}),(0,i.jsx)(_.th,{style:{textAlign:"left"},children:"Hudi Release Version(s)"})]})}),(0,i.jsxs)(_.tbody,{children:[(0,i.jsxs)(_.tr,{children:[(0,i.jsxs)(_.td,{style:{textAlign:"left"},children:[(0,i.jsx)(_.code,{children:"FIVE"})," or ",(0,i.jsx)(_.code,{children:"5"})]}),(0,i.jsx)(_.td,{style:{textAlign:"left"},children:"0.12.x"})]}),(0,i.jsxs)(_.tr,{children:[(0,i.jsxs)(_.td,{style:{textAlign:"left"},children:[(0,i.jsx)(_.code,{children:"FOUR"})," or ",(0,i.jsx)(_.code,{children:"4"})]}),(0,i.jsx)(_.td,{style:{textAlign:"left"},children:"0.11.x"})]}),(0,i.jsxs)(_.tr,{children:[(0,i.jsxs)(_.td,{style:{textAlign:"left"},children:[(0,i.jsx)(_.code,{children:"THREE"})," or ",(0,i.jsx)(_.code,{children:"3"})]}),(0,i.jsx)(_.td,{style:{textAlign:"left"},children:"0.10.x"})]}),(0,i.jsxs)(_.tr,{children:[(0,i.jsxs)(_.td,{style:{textAlign:"left"},children:[(0,i.jsx)(_.code,{children:"TWO"})," or ",(0,i.jsx)(_.code,{children:"2"})]}),(0,i.jsx)(_.td,{style:{textAlign:"left"},children:"0.9.x"})]}),(0,i.jsxs)(_.tr,{children:[(0,i.jsxs)(_.td,{style:{textAlign:"left"},children:[(0,i.jsx)(_.code,{children:"ONE"})," or ",(0,i.jsx)(_.code,{children:"1"})]}),(0,i.jsx)(_.td,{style:{textAlign:"left"},children:"0.6.x - 0.8.x"})]}),(0,i.jsxs)(_.tr,{children:[(0,i.jsxs)(_.td,{style:{textAlign:"left"},children:[(0,i.jsx)(_.code,{children:"ZERO"})," or ",(0,i.jsx)(_.code,{children:"0"})]}),(0,i.jsx)(_.td,{style:{textAlign:"left"},children:"0.5.x and below"})]})]})]}),"\n",(0,i.jsxs)(_.p,{children:["For example, to downgrade a table from version ",(0,i.jsx)(_.code,{children:"FIVE"}),"(",(0,i.jsx)(_.code,{children:"5"}),") (current version) to ",(0,i.jsx)(_.code,{children:"TWO"}),"(",(0,i.jsx)(_.code,{children:"2"}),"), you should run (use proper Spark master based\non your environment)"]}),"\n",(0,i.jsx)(_.pre,{children:(0,i.jsx)(_.code,{className:"language-shell",children:"downgrade table --toVersion TWO --sparkMaster local[2]\n"})}),"\n",(0,i.jsx)(_.p,{children:"or"}),"\n",(0,i.jsx)(_.pre,{children:(0,i.jsx)(_.code,{className:"language-shell",children:"downgrade table --toVersion 2 --sparkMaster local[2]\n"})}),"\n",(0,i.jsxs)(_.p,{children:["You can verify the table version by looking at the ",(0,i.jsx)(_.code,{children:"hoodie.table.version"})," property in ",(0,i.jsx)(_.code,{children:".hoodie/hoodie.properties"})," under\nthe table path:"]}),"\n",(0,i.jsx)(_.pre,{children:(0,i.jsx)(_.code,{className:"language-properties",children:"hoodie.table.version=2\n"})}),"\n",(0,i.jsx)(_.p,{children:"Hudi CLI also provides the ability to manually upgrade a Hudi table. To upgrade a Hudi table through CLI:"}),"\n",(0,i.jsx)(_.pre,{children:(0,i.jsx)(_.code,{className:"language-shell",children:"upgrade table --toVersion \n"})}),"\n",(0,i.jsxs)(_.admonition,{type:"note",children:[(0,i.jsxs)(_.p,{children:["Table upgrade is automatically handled by the Hudi write client in different deployment modes such as DeltaStreamer\nafter upgrading the Hudi library so that the user does not have to do manual upgrade. Such automatic table upgrade\nis the ",(0,i.jsx)(_.strong,{children:"recommended"})," way in general, instead of using ",(0,i.jsx)(_.code,{children:"upgrade"})," CLI command."]}),(0,i.jsx)(_.p,{children:'Table upgrade from table version ONE to TWO requires key generator related configs such as\n"hoodie.datasource.write.recordkey.field", which is only available when user configures the write job. So the table\nupgrade from version ONE to TWO through CLI is not supported, and user should rely on the automatic upgrade in the write\nclient instead.'})]}),"\n",(0,i.jsx)(_.p,{children:"You may also run the upgrade command without specifying the target version. In such a case, the latest table version\ncorresponding to the library release version is used:"}),"\n",(0,i.jsx)(_.pre,{children:(0,i.jsx)(_.code,{className:"language-shell",children:"upgrade table\n"})}),"\n",(0,i.jsx)(_.h3,{id:"change-hudi-table-type",children:"Change Hudi Table Type"}),"\n",(0,i.jsx)(_.p,{children:"There are cases we want to change the hudi table type. For example, change COW table to MOR for more efficient and\nlower latency ingestion; change MOR to COW for better read performance and compatibility with downstream engines.\nSo we offer the table command to perform this modification conveniently."}),"\n",(0,i.jsxs)(_.p,{children:["Changing ",(0,i.jsx)(_.strong,{children:"COW to MOR"}),", we can simply modify the ",(0,i.jsx)(_.code,{children:"hoodie.table.type"})," in ",(0,i.jsx)(_.code,{children:"hoodie.properties"})," to MERGE_ON_READ."]}),"\n",(0,i.jsxs)(_.p,{children:["While changing ",(0,i.jsx)(_.strong,{children:"MOR to COW"}),", we must make sure all the log files are compacted before modifying the table type,\nor it will cause data loss."]}),"\n",(0,i.jsx)(_.pre,{children:(0,i.jsx)(_.code,{className:"language-shell",children:"connect --path \ntable change-table-type \n"})}),"\n",(0,i.jsxs)(_.p,{children:["The parameter ",(0,i.jsx)(_.code,{children:"target_table_type"})," candidates are below:"]}),"\n",(0,i.jsxs)(_.table,{children:[(0,i.jsx)(_.thead,{children:(0,i.jsxs)(_.tr,{children:[(0,i.jsx)(_.th,{style:{textAlign:"left"},children:"target table type"}),(0,i.jsx)(_.th,{style:{textAlign:"left"},children:"comment"})]})}),(0,i.jsxs)(_.tbody,{children:[(0,i.jsxs)(_.tr,{children:[(0,i.jsx)(_.td,{style:{textAlign:"left"},children:"MOR"}),(0,i.jsx)(_.td,{style:{textAlign:"left"},children:"Change COW table to MERGE_ON_READ."})]}),(0,i.jsxs)(_.tr,{children:[(0,i.jsx)(_.td,{style:{textAlign:"left"},children:"COW"}),(0,i.jsxs)(_.td,{style:{textAlign:"left"},children:["Change MOR table to COPY_ON_WRITE. ",(0,i.jsx)("br",{}),"By default, changing to COW will ",(0,i.jsx)(_.strong,{children:"execute all pending compactions"})," and ",(0,i.jsx)(_.strong,{children:"perform a full compaction"})," if any log file left. Set ",(0,i.jsx)(_.code,{children:"--enable-compaction=false"})," will disable the default compaction. ",(0,i.jsx)("br",{}),"There are params can be set for the compaction operation:",(0,i.jsx)("br",{}),(0,i.jsx)(_.code,{children:"--parallelism"}),": Default ",(0,i.jsx)(_.code,{children:"3"}),". Parallelism for hoodie compaction",(0,i.jsx)("br",{}),(0,i.jsx)(_.code,{children:"--sparkMaster"}),": Default ",(0,i.jsx)(_.code,{children:"local"}),". Spark Master",(0,i.jsx)("br",{}),(0,i.jsx)(_.code,{children:"--sparkMemory"}),": Default ",(0,i.jsx)(_.code,{children:"4G"}),". Spark executor memory",(0,i.jsx)("br",{}),(0,i.jsx)(_.code,{children:"--retry"}),": Default ",(0,i.jsx)(_.code,{children:"1"}),". Number of retries",(0,i.jsx)("br",{}),(0,i.jsx)(_.code,{children:"--propsFilePath"}),": Default ",(0,i.jsx)(_.code,{children:" "}),". path to properties file on localfs or dfs with configurations for hoodie client for compacting",(0,i.jsx)("br",{}),(0,i.jsx)(_.code,{children:"--hoodieConfigs"}),": Default ",(0,i.jsx)(_.code,{children:" "}),". Any configuration that can be set in the properties file can be passed here in the form of an array"]})]})]})]}),"\n",(0,i.jsx)(_.p,{children:"Example below is changing MOR table to COW:"}),"\n",(0,i.jsx)(_.pre,{children:(0,i.jsx)(_.code,{className:"language-shell",children:"connect --path /var/dataset/test_table_mor2cow\ndesc\n\u2554\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2564\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2557\n\u2551 Property \u2502 Value \u2551\n\u2560\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u256a\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2563\n\u2551 basePath \u2502 /var/dataset/test_table_mor2cow \u2551\n\u255f\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2562\n\u2551 metaPath \u2502 /var/dataset/test_table_mor2cow/.hoodie \u2551\n\u255f\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2562\n\u2551 fileSystem \u2502 file \u2551\n\u255f\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2562\n\u2551 hoodie.table.name \u2502 test_table \u2551\n\u255f\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2562\n\u2551 hoodie.compaction.record.merger.strategy \u2502 eeb8d96f-b1e4-49fd-bbf8-28ac514178e5 \u2551\n\u255f\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2562\n\u2551 hoodie.table.metadata.partitions \u2502 files \u2551\n\u255f\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2562\n\u2551 hoodie.table.type \u2502 MERGE_ON_READ \u2551\n\u255f\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2562\n\u2551 hoodie.table.metadata.partitions.inflight \u2502 \u2551\n\u255f\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2562\n\u2551 hoodie.archivelog.folder \u2502 archived \u2551\n\u255f\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2562\n\u2551 hoodie.timeline.layout.version \u2502 1 \u2551\n\u255f\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2562\n\u2551 hoodie.table.checksum \u2502 2702201862 \u2551\n\u255f\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2562\n\u2551 hoodie.compaction.payload.type \u2502 HOODIE_AVRO \u2551\n\u255f\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2562\n\u2551 hoodie.table.version \u2502 6 \u2551\n\u255f\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2562\n\u2551 hoodie.datasource.write.drop.partition.columns \u2502 false \u2551\n\u255a\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2567\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u255d\n\ntable change-table-type COW\n\u2554\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2564\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2564\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2557\n\u2551 Property \u2502 Old Value \u2502 New Value \u2551\n\u2560\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u256a\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u256a\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2563\n\u2551 hoodie.archivelog.folder \u2502 archived \u2502 archived \u2551\n\u255f\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2562\n\u2551 hoodie.compaction.payload.type \u2502 HOODIE_AVRO \u2502 HOODIE_AVRO \u2551\n\u255f\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2562\n\u2551 hoodie.compaction.record.merger.strategy \u2502 eeb8d96f-b1e4-49fd-bbf8-28ac514178e5 \u2502 eeb8d96f-b1e4-49fd-bbf8-28ac514178e5 \u2551\n\u255f\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2562\n\u2551 hoodie.datasource.write.drop.partition.columns \u2502 false \u2502 false \u2551\n\u255f\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2562\n\u2551 hoodie.table.checksum \u2502 2702201862 \u2502 2702201862 \u2551\n\u255f\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2562\n\u2551 hoodie.table.metadata.partitions \u2502 files \u2502 files \u2551\n\u255f\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2562\n\u2551 hoodie.table.metadata.partitions.inflight \u2502 \u2502 \u2551\n\u255f\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2562\n\u2551 hoodie.table.name \u2502 test_table \u2502 test_table \u2551\n\u255f\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2562\n\u2551 hoodie.table.type \u2502 MERGE_ON_READ \u2502 COPY_ON_WRITE \u2551\n\u255f\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2562\n\u2551 hoodie.table.version \u2502 6 \u2502 6 \u2551\n\u255f\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2562\n\u2551 hoodie.timeline.layout.version \u2502 1 \u2502 1 \u2551\n\u255a\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2567\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2567\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u255d\n"})})]})}function d(e={}){const{wrapper:_}={...(0,a.R)(),...e.components};return _?(0,i.jsx)(_,{...e,children:(0,i.jsx)(c,{...e})}):c(e)}},28453:(e,_,n)=>{n.d(_,{R:()=>o,x:()=>s});var t=n(96540);const i={},a=t.createContext(i);function o(e){const _=t.useContext(a);return t.useMemo((function(){return"function"==typeof e?e(_):{..._,...e}}),[_,e])}function s(e){let _;return _=e.disableParentContext?"function"==typeof e.components?e.components(i):e.components||i:o(e.components),t.createElement(a.Provider,{value:_},e.children)}}}]);
\ No newline at end of file
diff --git a/content/assets/js/1a20bc57.83de7b83.js b/content/assets/js/1a20bc57.83de7b83.js
new file mode 100644
index 0000000000000..f02bd056618f3
--- /dev/null
+++ b/content/assets/js/1a20bc57.83de7b83.js
@@ -0,0 +1 @@
+"use strict";(self.webpackChunkhudi=self.webpackChunkhudi||[]).push([[59114],{46768:(e,_,n)=>{n.r(_),n.d(_,{assets:()=>l,contentTitle:()=>s,default:()=>d,frontMatter:()=>o,metadata:()=>t,toc:()=>r});const t=JSON.parse('{"id":"cli","title":"CLI","description":"Local set up","source":"@site/docs/cli.md","sourceDirName":".","slug":"/cli","permalink":"/docs/next/cli","draft":false,"unlisted":false,"editUrl":"https://github.com/apache/hudi/tree/asf-site/website/docs/cli.md","tags":[],"version":"current","frontMatter":{"title":"CLI","keywords":["hudi","cli"],"last_modified_at":"2021-08-18T19:59:57.000Z"},"sidebar":"docs","previous":{"title":"SQL Procedures","permalink":"/docs/next/procedures"},"next":{"title":"Metrics","permalink":"/docs/next/metrics"}}');var i=n(74848),a=n(28453);const o={title:"CLI",keywords:["hudi","cli"],last_modified_at:new Date("2021-08-18T19:59:57.000Z")},s=void 0,l={},r=[{value:"Local set up",id:"local-set-up",level:3},{value:"Hudi CLI Bundle setup",id:"hudi-cli-bundle-setup",level:3},{value:"Base path",id:"base-path",level:3},{value:"Using Hudi-cli in S3",id:"using-hudi-cli-in-s3",level:3},{value:"Note: These AWS jar versions below are specific to Spark 3.2.0",id:"note-these-aws-jar-versions-below-are-specific-to-spark-320",level:4},{value:"Using hudi-cli on Google Dataproc",id:"using-hudi-cli-on-google-dataproc",level:3},{value:"Connect to a Kerberized cluster",id:"connect-to-a-kerberized-cluster",level:2},{value:"Using hudi-cli",id:"using-hudi-cli",level:2},{value:"Inspecting Commits",id:"inspecting-commits",level:3},{value:"Drilling Down to a specific Commit",id:"drilling-down-to-a-specific-commit",level:3},{value:"FileSystem View",id:"filesystem-view",level:3},{value:"Statistics",id:"statistics",level:3},{value:"Archived Commits",id:"archived-commits",level:3},{value:"Compactions",id:"compactions",level:3},{value:"Validate Compaction",id:"validate-compaction",level:3},{value:"Unscheduling Compaction",id:"unscheduling-compaction",level:3},{value:"Repair Compaction",id:"repair-compaction",level:3},{value:"Savepoint and Restore",id:"savepoint-and-restore",level:3},{value:"Upgrade and Downgrade Table",id:"upgrade-and-downgrade-table",level:3},{value:"Change Hudi Table Type",id:"change-hudi-table-type",level:3},{value:"Related Resources",id:"related-resources",level:2}];function c(e){const _={a:"a",admonition:"admonition",br:"br",code:"code",h2:"h2",h3:"h3",h4:"h4",li:"li",ol:"ol",p:"p",pre:"pre",strong:"strong",table:"table",tbody:"tbody",td:"td",th:"th",thead:"thead",tr:"tr",ul:"ul",...(0,a.R)(),...e.components};return(0,i.jsxs)(i.Fragment,{children:[(0,i.jsx)(_.h3,{id:"local-set-up",children:"Local set up"}),"\n",(0,i.jsxs)(_.p,{children:["Once hudi has been built, the shell can be fired by via ",(0,i.jsx)(_.code,{children:"cd hudi-cli && ./hudi-cli.sh"}),"."]}),"\n",(0,i.jsx)(_.h3,{id:"hudi-cli-bundle-setup",children:"Hudi CLI Bundle setup"}),"\n",(0,i.jsxs)(_.p,{children:["In release ",(0,i.jsx)(_.code,{children:"0.13.0"})," we have now added another way of launching the ",(0,i.jsx)(_.code,{children:"hudi cli"}),", which is using the ",(0,i.jsx)(_.code,{children:"hudi-cli-bundle"}),"."]}),"\n",(0,i.jsxs)(_.p,{children:["There are a couple of requirements when using this approach such as having ",(0,i.jsx)(_.code,{children:"spark"})," installed locally on your machine.\nIt is required to use a spark distribution with hadoop dependencies packaged such as ",(0,i.jsx)(_.code,{children:"spark-3.3.1-bin-hadoop2.tgz"})," from ",(0,i.jsx)(_.a,{href:"https://archive.apache.org/dist/spark/",children:"https://archive.apache.org/dist/spark/"}),".\nWe also recommend you set an env variable ",(0,i.jsx)(_.code,{children:"$SPARK_HOME"})," to the path of where spark is installed on your machine.\nOne important thing to note is that the ",(0,i.jsx)(_.code,{children:"hudi-spark-bundle"})," should also be present when using the ",(0,i.jsx)(_.code,{children:"hudi-cli-bundle"}),".",(0,i.jsx)(_.br,{}),"\n","To provide the locations of these bundle jars you can set them in your shell like so:\n",(0,i.jsx)(_.code,{children:"export CLI_BUNDLE_JAR="})," , ",(0,i.jsx)(_.code,{children:"export SPARK_BUNDLE_JAR="}),"."]}),"\n",(0,i.jsx)(_.p,{children:"For steps see below if you are not compiling the project and downloading the jars:"}),"\n",(0,i.jsxs)(_.ol,{children:["\n",(0,i.jsx)(_.li,{children:"Create an empty folder as a new directory"}),"\n",(0,i.jsx)(_.li,{children:"Copy the hudi-cli-bundle jars and hudi-spark*-bundle jars to this directory"}),"\n",(0,i.jsx)(_.li,{children:"Copy the following script and folder to this directory"}),"\n"]}),"\n",(0,i.jsx)(_.pre,{children:(0,i.jsx)(_.code,{children:"packaging/hudi-cli-bundle/hudi-cli-with-bundle.sh\npackaging/hudi-cli-bundle/conf . the `conf` folder should be in this directory.\n"})}),"\n",(0,i.jsxs)(_.ol,{start:"4",children:["\n",(0,i.jsx)(_.li,{children:"Start Hudi CLI shell with environment variables set"}),"\n"]}),"\n",(0,i.jsx)(_.pre,{children:(0,i.jsx)(_.code,{children:"export SPARK_HOME=\nexport CLI_BUNDLE_JAR=\nexport SPARK_BUNDLE_JAR=\n\n./hudi-cli-with-bundle.sh\n\n"})}),"\n",(0,i.jsx)(_.h3,{id:"base-path",children:"Base path"}),"\n",(0,i.jsxs)(_.p,{children:["A hudi table resides on DFS, in a location referred to as the ",(0,i.jsx)(_.code,{children:"basePath"})," and\nwe would need this location in order to connect to a Hudi table. Hudi library effectively manages this table internally, using ",(0,i.jsx)(_.code,{children:".hoodie"})," subfolder to track all metadata."]}),"\n",(0,i.jsx)(_.h3,{id:"using-hudi-cli-in-s3",children:"Using Hudi-cli in S3"}),"\n",(0,i.jsxs)(_.p,{children:["If you are using hudi that comes packaged with AWS EMR, you can find instructions to use hudi-cli ",(0,i.jsx)(_.a,{href:"https://docs.aws.amazon.com/emr/latest/ReleaseGuide/emr-hudi-cli.html",children:"here"}),".\nIf you are not using EMR, or would like to use latest hudi-cli from master, you can follow the below steps to access S3 dataset in your local environment (laptop)."]}),"\n",(0,i.jsx)(_.p,{children:"Build Hudi with corresponding Spark version, for eg, -Dspark3.1.x"}),"\n",(0,i.jsx)(_.p,{children:"Set the following environment variables."}),"\n",(0,i.jsx)(_.pre,{children:(0,i.jsx)(_.code,{children:"export AWS_REGION=us-east-2\nexport AWS_ACCESS_KEY_ID=\nexport AWS_SECRET_ACCESS_KEY=\nexport SPARK_HOME=\n"})}),"\n",(0,i.jsx)(_.p,{children:"Ensure you set the SPARK_HOME to your local spark home compatible to compiled hudi spark version above."}),"\n",(0,i.jsx)(_.p,{children:"Apart from these, we might need to add aws jars to class path so that accessing S3 is feasible from local.\nWe need two jars, namely, aws-java-sdk-bundle jar and hadoop-aws jar which you can find online.\nFor eg:"}),"\n",(0,i.jsx)(_.pre,{children:(0,i.jsx)(_.code,{children:"wget https://repo1.maven.org/maven2/org/apache/hadoop/hadoop-aws/3.2.0/hadoop-aws-3.2.0.jar -o /lib/spark-3.2.0-bin-hadoop3.2/jars/hadoop-aws-3.2.0.jar\nwget https://repo1.maven.org/maven2/com/amazonaws/aws-java-sdk-bundle/1.11.375/aws-java-sdk-bundle-1.11.375.jar -o /lib/spark-3.2.0-bin-hadoop3.2/jars/aws-java-sdk-bundle-1.11.375.jar\n"})}),"\n",(0,i.jsx)(_.h4,{id:"note-these-aws-jar-versions-below-are-specific-to-spark-320",children:"Note: These AWS jar versions below are specific to Spark 3.2.0"}),"\n",(0,i.jsx)(_.pre,{children:(0,i.jsx)(_.code,{children:"export CLIENT_JAR=/lib/spark-3.2.0-bin-hadoop3.2/jars/aws-java-sdk-bundle-1.12.48.jar:/lib/spark-3.2.0-bin-hadoop3.2/jars/hadoop-aws-3.3.1.jar\n"})}),"\n",(0,i.jsx)(_.p,{children:"Once these are set, you are good to launch hudi-cli and access S3 dataset."}),"\n",(0,i.jsx)(_.pre,{children:(0,i.jsx)(_.code,{children:"./hudi-cli/hudi-cli.sh\n"})}),"\n",(0,i.jsx)(_.h3,{id:"using-hudi-cli-on-google-dataproc",children:"Using hudi-cli on Google Dataproc"}),"\n",(0,i.jsxs)(_.p,{children:[(0,i.jsx)(_.a,{href:"https://cloud.google.com/dataproc",children:"Dataproc"})," is Google's managed service for running Apache Hadoop, Apache Spark,\nApache Flink, Presto and many other frameworks, including Hudi. If you want to run the Hudi CLI on a Dataproc node\nwhich has not been launched with Hudi support enabled, you can use the steps below:"]}),"\n",(0,i.jsx)(_.p,{children:"These steps use Hudi version 0.13.0. If you want to use a different version you will have to edit the below commands\nappropriately:"}),"\n",(0,i.jsxs)(_.ol,{children:["\n",(0,i.jsx)(_.li,{children:"Once you've started the Dataproc cluster, you can ssh into it as follows:"}),"\n"]}),"\n",(0,i.jsx)(_.pre,{children:(0,i.jsx)(_.code,{children:'$ gcloud compute ssh --zone "YOUR_ZONE" "HOSTNAME_OF_MASTER_NODE" --project "YOUR_PROJECT"\n'})}),"\n",(0,i.jsxs)(_.ol,{start:"2",children:["\n",(0,i.jsx)(_.li,{children:"Download the Hudi CLI bundle"}),"\n"]}),"\n",(0,i.jsx)(_.pre,{children:(0,i.jsx)(_.code,{children:"wget https://repo1.maven.org/maven2/org/apache/hudi/hudi-cli-bundle_2.12/0.13.0/hudi-cli-bundle_2.12-0.13.0.jar \n"})}),"\n",(0,i.jsxs)(_.ol,{start:"3",children:["\n",(0,i.jsx)(_.li,{children:"Download the Hudi Spark bundle"}),"\n"]}),"\n",(0,i.jsx)(_.pre,{children:(0,i.jsx)(_.code,{children:"wget https://repo1.maven.org/maven2/org/apache/hudi/hudi-spark-bundle_2.12/0.13.0/hudi-spark-bundle_2.12-0.13.0.jar\n"})}),"\n",(0,i.jsxs)(_.ol,{start:"4",children:["\n",(0,i.jsx)(_.li,{children:"Download the shell script that launches Hudi CLI bundle"}),"\n"]}),"\n",(0,i.jsx)(_.pre,{children:(0,i.jsx)(_.code,{children:"wget https://raw.githubusercontent.com/apache/hudi/release-0.13.0/packaging/hudi-cli-bundle/hudi-cli-with-bundle.sh\n"})}),"\n",(0,i.jsxs)(_.ol,{start:"5",children:["\n",(0,i.jsx)(_.li,{children:"Launch Hudi CLI bundle with appropriate environment variables as follows:"}),"\n"]}),"\n",(0,i.jsx)(_.pre,{children:(0,i.jsx)(_.code,{children:"CLIENT_JAR=$DATAPROC_DIR/lib/gcs-connector.jar CLI_BUNDLE_JAR=hudi-cli-bundle_2.12-0.13.0.jar SPARK_BUNDLE_JAR=hudi-spark-bundle_2.12-0.13.0.jar ./hudi-cli-with-bundle.sh \n"})}),"\n",(0,i.jsxs)(_.ol,{start:"6",children:["\n",(0,i.jsxs)(_.li,{children:["\n",(0,i.jsxs)(_.p,{children:["hudi->connect --path gs://path_to_some_table",(0,i.jsx)(_.br,{}),"\n","Metadata for table some_table loaded"]}),"\n"]}),"\n",(0,i.jsxs)(_.li,{children:["\n",(0,i.jsxs)(_.p,{children:["hudi:some_table->commits show --limit 5",(0,i.jsx)(_.br,{}),"\n","This command should show the recent commits, if the above steps work correctly."]}),"\n"]}),"\n"]}),"\n",(0,i.jsx)(_.h2,{id:"connect-to-a-kerberized-cluster",children:"Connect to a Kerberized cluster"}),"\n",(0,i.jsxs)(_.p,{children:["Before connecting to a Kerberized cluster, you can use ",(0,i.jsx)(_.strong,{children:"kerberos kinit"})," command. Following is the usage of this command."]}),"\n",(0,i.jsx)(_.pre,{children:(0,i.jsx)(_.code,{className:"language-shell",children:"hudi->help kerberos kinit\nNAME\n kerberos kinit - Perform Kerberos authentication\n\nSYNOPSIS\n kerberos kinit --krb5conf String [--principal String] [--keytab String]\n\nOPTIONS\n --krb5conf String\n Path to krb5.conf\n [Optional, default = /etc/krb5.conf]\n\n --principal String\n Kerberos principal\n [Mandatory]\n\n --keytab String\n Path to keytab\n [Mandatory]\n"})}),"\n",(0,i.jsx)(_.p,{children:"For example:"}),"\n",(0,i.jsx)(_.pre,{children:(0,i.jsx)(_.code,{className:"language-shell",children:"hudi->kerberos kinit --principal user/host@DOMAIN --keytab /etc/security/keytabs/user.keytab\nPerform Kerberos authentication\nParameters:\n--krb5conf: /etc/krb5.conf\n--principal: user/host@DOMAIN\n--keytab: /etc/security/keytabs/user.keytab\nKerberos current user: user/host@DOMAIN (auth:KERBEROS)\nKerberos login user: user/host@DOMAIN (auth:KERBEROS)\nKerberos authentication success\n"})}),"\n",(0,i.jsx)(_.p,{children:'If you see "Kerberos authentication success" in the command output, it means Kerberos authentication has been successful.'}),"\n",(0,i.jsx)(_.h2,{id:"using-hudi-cli",children:"Using hudi-cli"}),"\n",(0,i.jsx)(_.p,{children:"To initialize a hudi table, use the following command."}),"\n",(0,i.jsx)(_.pre,{children:(0,i.jsx)(_.code,{className:"language-java",children:"===================================================================\n* ___ ___ *\n* /\\__\\ ___ /\\ \\ ___ *\n* / / / /\\__\\ / \\ \\ /\\ \\ *\n* / /__/ / / / / /\\ \\ \\ \\ \\ \\ *\n* / \\ \\ ___ / / / / / \\ \\__\\ / \\__\\ *\n* / /\\ \\ /\\__\\ / /__/ ___ / /__/ \\ |__| / /\\/__/ *\n* \\/ \\ \\/ / / \\ \\ \\ /\\__\\ \\ \\ \\ / / / /\\/ / / *\n* \\ / / \\ \\ / / / \\ \\ / / / \\ /__/ *\n* / / / \\ \\/ / / \\ \\/ / / \\ \\__\\ *\n* / / / \\ / / \\ / / \\/__/ *\n* \\/__/ \\/__/ \\/__/ Apache Hudi CLI *\n* *\n===================================================================\n\nhudi->create --path /user/hive/warehouse/table1 --tableName hoodie_table_1 --tableType COPY_ON_WRITE\n.....\n"})}),"\n",(0,i.jsx)(_.p,{children:"To see the description of hudi table, use the command:"}),"\n",(0,i.jsx)(_.pre,{children:(0,i.jsx)(_.code,{className:"language-java",children:"hudi:hoodie_table_1->desc\n18/09/06 15:57:19 INFO timeline.HoodieActiveTimeline: Loaded instants []\n _________________________________________________________\n | Property | Value |\n |========================================================|\n | basePath | ... |\n | metaPath | ... |\n | fileSystem | hdfs |\n | hoodie.table.name | hoodie_table_1 |\n | hoodie.table.type | COPY_ON_WRITE |\n | hoodie.archivelog.folder| |\n"})}),"\n",(0,i.jsx)(_.p,{children:"Following is a sample command to connect to a Hudi table contains uber trips."}),"\n",(0,i.jsx)(_.pre,{children:(0,i.jsx)(_.code,{className:"language-java",children:"hudi:trips->connect --path /app/uber/trips\n\n16/10/05 23:20:37 INFO model.HoodieTableMetadata: All commits :HoodieCommits{commitList=[20161002045850, 20161002052915, 20161002055918, 20161002065317, 20161002075932, 20161002082904, 20161002085949, 20161002092936, 20161002105903, 20161002112938, 20161002123005, 20161002133002, 20161002155940, 20161002165924, 20161002172907, 20161002175905, 20161002190016, 20161002192954, 20161002195925, 20161002205935, 20161002215928, 20161002222938, 20161002225915, 20161002232906, 20161003003028, 20161003005958, 20161003012936, 20161003022924, 20161003025859, 20161003032854, 20161003042930, 20161003052911, 20161003055907, 20161003062946, 20161003065927, 20161003075924, 20161003082926, 20161003085925, 20161003092909, 20161003100010, 20161003102913, 20161003105850, 20161003112910, 20161003115851, 20161003122929, 20161003132931, 20161003142952, 20161003145856, 20161003152953, 20161003155912, 20161003162922, 20161003165852, 20161003172923, 20161003175923, 20161003195931, 20161003210118, 20161003212919, 20161003215928, 20161003223000, 20161003225858, 20161004003042, 20161004011345, 20161004015235, 20161004022234, 20161004063001, 20161004072402, 20161004074436, 20161004080224, 20161004082928, 20161004085857, 20161004105922, 20161004122927, 20161004142929, 20161004163026, 20161004175925, 20161004194411, 20161004203202, 20161004211210, 20161004214115, 20161004220437, 20161004223020, 20161004225321, 20161004231431, 20161004233643, 20161005010227, 20161005015927, 20161005022911, 20161005032958, 20161005035939, 20161005052904, 20161005070028, 20161005074429, 20161005081318, 20161005083455, 20161005085921, 20161005092901, 20161005095936, 20161005120158, 20161005123418, 20161005125911, 20161005133107, 20161005155908, 20161005163517, 20161005165855, 20161005180127, 20161005184226, 20161005191051, 20161005193234, 20161005203112, 20161005205920, 20161005212949, 20161005223034, 20161005225920]}\nMetadata for table trips loaded\n"})}),"\n",(0,i.jsx)(_.p,{children:"Once connected to the table, a lot of other commands become available. The shell has contextual autocomplete help (press TAB) and below is a list of all commands, few of which are reviewed in this section"}),"\n",(0,i.jsx)(_.pre,{children:(0,i.jsx)(_.code,{className:"language-shell",children:"hudi:trips->help\n* ! - Allows execution of operating system (OS) commands\n* // - Inline comment markers (start of line only)\n* ; - Inline comment markers (start of line only)\n* bootstrap index showmapping - Show bootstrap index mapping\n* bootstrap index showpartitions - Show bootstrap indexed partitions\n* bootstrap run - Run a bootstrap action for current Hudi table\n* clean showpartitions - Show partition level details of a clean\n* cleans refresh - Refresh table metadata\n* cleans run - run clean\n* cleans show - Show the cleans\n* clear - Clears the console\n* cls - Clears the console\n* clustering run - Run Clustering\n* clustering schedule - Schedule Clustering\n* clustering scheduleAndExecute - Run Clustering. Make a cluster plan first and execute that plan immediately\n* commit rollback - Rollback a commit\n* commits compare - Compare commits with another Hoodie table\n* commit show_write_stats - Show write stats of a commit\n* commit showfiles - Show file level details of a commit\n* commit showpartitions - Show partition level details of a commit\n* commits refresh - Refresh table metadata\n* commits show - Show the commits\n* commits showarchived - Show the archived commits\n* commits sync - Sync commits with another Hoodie table\n* compaction repair - Renames the files to make them consistent with the timeline as dictated by Hoodie metadata. Use when compaction unschedule fails partially.\n* compaction run - Run Compaction for given instant time\n* compaction schedule - Schedule Compaction\n* compaction scheduleAndExecute - Schedule compaction plan and execute this plan\n* compaction show - Shows compaction details for a specific compaction instant\n* compaction showarchived - Shows compaction details for a specific compaction instant\n* compactions show all - Shows all compactions that are in active timeline\n* compactions showarchived - Shows compaction details for specified time window\n* compaction unschedule - Unschedule Compaction\n* compaction unscheduleFileId - UnSchedule Compaction for a fileId\n* compaction validate - Validate Compaction\n* connect - Connect to a hoodie table\n* create - Create a hoodie table if not present\n* date - Displays the local date and time\n* desc - Describe Hoodie Table properties\n* downgrade table - Downgrades a table\n* exit - Exits the shell\n* export instants - Export Instants and their metadata from the Timeline\n* fetch table schema - Fetches latest table schema\n* hdfsparquetimport - Imports Parquet table to a hoodie table\n* help - List all commands usage\n* marker delete - Delete the marker\n* metadata create - Create the Metadata Table if it does not exist\n* metadata delete - Remove the Metadata Table\n* metadata init - Update the metadata table from commits since the creation\n* metadata list-files - Print a list of all files in a partition from the metadata\n* metadata list-partitions - List all partitions from metadata\n* metadata refresh - Refresh table metadata\n* metadata set - Set options for Metadata Table\n* metadata stats - Print stats about the metadata\n* metadata validate-files - Validate all files in all partitions from the metadata\n* quit - Exits the shell\n* refresh - Refresh table metadata\n* repair addpartitionmeta - Add partition metadata to a table, if not present\n* repair corrupted clean files - repair corrupted clean files\n* repair deduplicate - De-duplicate a partition path contains duplicates & produce repaired files to replace with\n* repair migrate-partition-meta - Migrate all partition meta file currently stored in text format to be stored in base file format. See HoodieTableConfig#PARTITION_METAFILE_USE_DATA_FORMAT.\n* repair overwrite-hoodie-props - Overwrite hoodie.properties with provided file. Risky operation. Proceed with caution!\n* savepoint create - Savepoint a commit\n* savepoint delete - Delete the savepoint\n* savepoint rollback - Savepoint a commit\n* savepoints refresh - Refresh table metadata\n* savepoints show - Show the savepoints\n* script - Parses the specified resource file and executes its commands\n* set - Set spark launcher env to cli\n* show archived commits - Read commits from archived files and show details\n* show archived commit stats - Read commits from archived files and show details\n* show env - Show spark launcher env by key\n* show envs all - Show spark launcher envs\n* show fsview all - Show entire file-system view\n* show fsview latest - Show latest file-system view\n* show logfile metadata - Read commit metadata from log files\n* show logfile records - Read records from log files\n* show rollback - Show details of a rollback instant\n* show rollbacks - List all rollback instants\n* stats filesizes - File Sizes. Display summary stats on sizes of files\n* stats wa - Write Amplification. Ratio of how many records were upserted to how many records were actually written\n* sync validate - Validate the sync by counting the number of records\n* system properties - Shows the shell's properties\n* table delete-configs - Delete the supplied table configs from the table.\n* table recover-configs - Recover table configs, from update/delete that failed midway.\n* table update-configs - Update the table configs with configs with provided file.\n* temp_delete - Delete view name\n* temp_query - query against created temp view\n* temp delete - Delete view name\n* temp query - query against created temp view\n* temps_show - Show all views name\n* temps show - Show all views name\n* upgrade table - Upgrades a table\n* utils loadClass - Load a class\n* version - Displays shell version\n\nhudi:trips->\n"})}),"\n",(0,i.jsx)(_.h3,{id:"inspecting-commits",children:"Inspecting Commits"}),"\n",(0,i.jsxs)(_.p,{children:["The task of upserting or inserting a batch of incoming records is known as a ",(0,i.jsx)(_.strong,{children:"commit"})," in Hudi. A commit provides basic atomicity guarantees such that only committed data is available for querying.\nEach commit has a monotonically increasing string/number called the ",(0,i.jsx)(_.strong,{children:"commit number"}),". Typically, this is the time at which we started the commit."]}),"\n",(0,i.jsx)(_.p,{children:"To view some basic information about the last 10 commits,"}),"\n",(0,i.jsx)(_.pre,{children:(0,i.jsx)(_.code,{className:"language-java",children:'hudi:trips->commits show --sortBy "Total Bytes Written" --desc true --limit 10\n ________________________________________________________________________________________________________________________________________________________________________\n | CommitTime | Total Bytes Written| Total Files Added| Total Files Updated| Total Partitions Written| Total Records Written| Total Update Records Written| Total Errors|\n |=======================================================================================================================================================================|\n ....\n ....\n ....\n'})}),"\n",(0,i.jsx)(_.p,{children:"At the start of each write, Hudi also writes a .inflight commit to the .hoodie folder. You can use the timestamp there to estimate how long the commit has been inflight"}),"\n",(0,i.jsx)(_.pre,{children:(0,i.jsx)(_.code,{className:"language-java",children:"$ hdfs dfs -ls /app/uber/trips/.hoodie/*.inflight\n-rw-r--r-- 3 vinoth supergroup 321984 2016-10-05 23:18 /app/uber/trips/.hoodie/20161005225920.inflight\n"})}),"\n",(0,i.jsx)(_.h3,{id:"drilling-down-to-a-specific-commit",children:"Drilling Down to a specific Commit"}),"\n",(0,i.jsx)(_.p,{children:"To understand how the writes spread across specific partiions,"}),"\n",(0,i.jsx)(_.pre,{children:(0,i.jsx)(_.code,{className:"language-java",children:'hudi:trips->commit showpartitions --commit 20161005165855 --sortBy "Total Bytes Written" --desc true --limit 10\n __________________________________________________________________________________________________________________________________________\n | Partition Path| Total Files Added| Total Files Updated| Total Records Inserted| Total Records Updated| Total Bytes Written| Total Errors|\n |=========================================================================================================================================|\n ....\n ....\n'})}),"\n",(0,i.jsx)(_.p,{children:"If you need file level granularity , we can do the following"}),"\n",(0,i.jsx)(_.pre,{children:(0,i.jsx)(_.code,{className:"language-java",children:'hudi:trips->commit showfiles --commit 20161005165855 --sortBy "Partition Path"\n ________________________________________________________________________________________________________________________________________________________\n | Partition Path| File ID | Previous Commit| Total Records Updated| Total Records Written| Total Bytes Written| Total Errors|\n |=======================================================================================================================================================|\n ....\n ....\n'})}),"\n",(0,i.jsx)(_.h3,{id:"filesystem-view",children:"FileSystem View"}),"\n",(0,i.jsx)(_.p,{children:"Hudi views each partition as a collection of file-groups with each file-group containing a list of file-slices in commit order (See concepts).\nThe below commands allow users to view the file-slices for a data-set."}),"\n",(0,i.jsx)(_.pre,{children:(0,i.jsx)(_.code,{className:"language-java",children:'hudi:stock_ticks_mor->show fsview all\n ....\n _______________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________\n | Partition | FileId | Base-Instant | Data-File | Data-File Size| Num Delta Files| Total Delta File Size| Delta Files |\n |==============================================================================================================================================================================================================================================================================================================================================================================================================|\n | 2018/08/31| 111415c3-f26d-4639-86c8-f9956f245ac3| 20181002180759| hdfs://namenode:8020/user/hive/warehouse/stock_ticks_mor/2018/08/31/111415c3-f26d-4639-86c8-f9956f245ac3_0_20181002180759.parquet| 432.5 KB | 1 | 20.8 KB | [HoodieLogFile {hdfs://namenode:8020/user/hive/warehouse/stock_ticks_mor/2018/08/31/.111415c3-f26d-4639-86c8-f9956f245ac3_20181002180759.log.1}]|\n\n\n\nhudi:stock_ticks_mor->show fsview latest --partitionPath "2018/08/31"\n ......\n __________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________\n | Partition | FileId | Base-Instant | Data-File | Data-File Size| Num Delta Files| Total Delta Size| Delta Size - compaction scheduled| Delta Size - compaction unscheduled| Delta To Base Ratio - compaction scheduled| Delta To Base Ratio - compaction unscheduled| Delta Files - compaction scheduled | Delta Files - compaction unscheduled|\n |=================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================|\n | 2018/08/31| 111415c3-f26d-4639-86c8-f9956f245ac3| 20181002180759| hdfs://namenode:8020/user/hive/warehouse/stock_ticks_mor/2018/08/31/111415c3-f26d-4639-86c8-f9956f245ac3_0_20181002180759.parquet| 432.5 KB | 1 | 20.8 KB | 20.8 KB | 0.0 B | 0.0 B | 0.0 B | [HoodieLogFile {hdfs://namenode:8020/user/hive/warehouse/stock_ticks_mor/2018/08/31/.111415c3-f26d-4639-86c8-f9956f245ac3_20181002180759.log.1}]| [] |\n\n'})}),"\n",(0,i.jsx)(_.h3,{id:"statistics",children:"Statistics"}),"\n",(0,i.jsx)(_.p,{children:"Since Hudi directly manages file sizes for DFS table, it might be good to get an overall picture"}),"\n",(0,i.jsx)(_.pre,{children:(0,i.jsx)(_.code,{className:"language-java",children:'hudi:trips->stats filesizes --partitionPath 2016/09/01 --sortBy "95th" --desc true --limit 10\n ________________________________________________________________________________________________\n | CommitTime | Min | 10th | 50th | avg | 95th | Max | NumFiles| StdDev |\n |===============================================================================================|\n | | 93.9 MB | 93.9 MB | 93.9 MB | 93.9 MB | 93.9 MB | 93.9 MB | 2 | 2.3 KB |\n ....\n ....\n'})}),"\n",(0,i.jsx)(_.p,{children:"In case of Hudi write taking much longer, it might be good to see the write amplification for any sudden increases"}),"\n",(0,i.jsx)(_.pre,{children:(0,i.jsx)(_.code,{className:"language-java",children:"hudi:trips->stats wa\n __________________________________________________________________________\n | CommitTime | Total Upserted| Total Written| Write Amplifiation Factor|\n |=========================================================================|\n ....\n ....\n"})}),"\n",(0,i.jsx)(_.h3,{id:"archived-commits",children:"Archived Commits"}),"\n",(0,i.jsx)(_.p,{children:"In order to limit the amount of growth of .commit files on DFS, Hudi archives older .commit files (with due respect to the cleaner policy) into a commits.archived file.\nThis is a sequence file that contains a mapping from commitNumber => json with raw information about the commit (same that is nicely rolled up above)."}),"\n",(0,i.jsx)(_.h3,{id:"compactions",children:"Compactions"}),"\n",(0,i.jsx)(_.p,{children:"To get an idea of the lag between compaction and writer applications, use the below command to list down all\npending compactions."}),"\n",(0,i.jsx)(_.pre,{children:(0,i.jsx)(_.code,{className:"language-java",children:"hudi:trips->compactions show all\n ___________________________________________________________________\n | Compaction Instant Time| State | Total FileIds to be Compacted|\n |==================================================================|\n | | REQUESTED| 35 |\n | | INFLIGHT | 27 |\n"})}),"\n",(0,i.jsx)(_.p,{children:"To inspect a specific compaction plan, use"}),"\n",(0,i.jsx)(_.pre,{children:(0,i.jsx)(_.code,{className:"language-java",children:"hudi:trips->compaction show --instant \n _________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________\n | Partition Path| File Id | Base Instant | Data File Path | Total Delta Files| getMetrics |\n |================================================================================================================================================================================================================================================\n | 2018/07/17 | | | viewfs://ns-default/.../../UUID_.parquet | 1 | {TOTAL_LOG_FILES=1.0, TOTAL_IO_READ_MB=1230.0, TOTAL_LOG_FILES_SIZE=2.51255751E8, TOTAL_IO_WRITE_MB=991.0, TOTAL_IO_MB=2221.0}|\n\n"})}),"\n",(0,i.jsx)(_.p,{children:"To manually schedule or run a compaction, use the below command. This command uses spark launcher to perform compaction\noperations."}),"\n",(0,i.jsxs)(_.p,{children:[(0,i.jsx)(_.strong,{children:"NOTE:"})," Make sure no other application is scheduling compaction for this table concurrently\n{: .notice--info}"]}),"\n",(0,i.jsx)(_.pre,{children:(0,i.jsx)(_.code,{className:"language-java",children:"hudi:trips->help compaction schedule\nKeyword: compaction schedule\nDescription: Schedule Compaction\n Keyword: sparkMemory\n Help: Spark executor memory\n Mandatory: false\n Default if specified: '__NULL__'\n Default if unspecified: '1G'\n\n* compaction schedule - Schedule Compaction\n"})}),"\n",(0,i.jsx)(_.pre,{children:(0,i.jsx)(_.code,{className:"language-java",children:"hudi:trips->help compaction run\nKeyword: compaction run\nDescription: Run Compaction for given instant time\n Keyword: tableName\n Help: Table name\n Mandatory: true\n Default if specified: '__NULL__'\n Default if unspecified: '__NULL__'\n\n Keyword: parallelism\n Help: Parallelism for hoodie compaction\n Mandatory: true\n Default if specified: '__NULL__'\n Default if unspecified: '__NULL__'\n\n Keyword: schemaFilePath\n Help: Path for Avro schema file\n Mandatory: true\n Default if specified: '__NULL__'\n Default if unspecified: '__NULL__'\n\n Keyword: sparkMemory\n Help: Spark executor memory\n Mandatory: true\n Default if specified: '__NULL__'\n Default if unspecified: '__NULL__'\n\n Keyword: retry\n Help: Number of retries\n Mandatory: true\n Default if specified: '__NULL__'\n Default if unspecified: '__NULL__'\n\n Keyword: compactionInstant\n Help: Base path for the target hoodie table\n Mandatory: true\n Default if specified: '__NULL__'\n Default if unspecified: '__NULL__'\n\n* compaction run - Run Compaction for given instant time\n"})}),"\n",(0,i.jsx)(_.h3,{id:"validate-compaction",children:"Validate Compaction"}),"\n",(0,i.jsx)(_.p,{children:"Validating a compaction plan : Check if all the files necessary for compactions are present and are valid"}),"\n",(0,i.jsx)(_.pre,{children:(0,i.jsx)(_.code,{className:"language-java",children:"hudi:stock_ticks_mor->compaction validate --instant 20181005222611\n...\n\n COMPACTION PLAN VALID\n\n ___________________________________________________________________________________________________________________________________________________________________________________________________________________________\n | File Id | Base Instant Time| Base Data File | Num Delta Files| Valid| Error|\n |==========================================================================================================================================================================================================================|\n | 05320e98-9a57-4c38-b809-a6beaaeb36bd| 20181005222445 | hdfs://namenode:8020/user/hive/warehouse/stock_ticks_mor/2018/08/31/05320e98-9a57-4c38-b809-a6beaaeb36bd_0_20181005222445.parquet| 1 | true | |\n\n\n\nhudi:stock_ticks_mor->compaction validate --instant 20181005222601\n\n COMPACTION PLAN INVALID\n\n _______________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________\n | File Id | Base Instant Time| Base Data File | Num Delta Files| Valid| Error |\n |=====================================================================================================================================================================================================================================================================================================|\n | 05320e98-9a57-4c38-b809-a6beaaeb36bd| 20181005222445 | hdfs://namenode:8020/user/hive/warehouse/stock_ticks_mor/2018/08/31/05320e98-9a57-4c38-b809-a6beaaeb36bd_0_20181005222445.parquet| 1 | false| All log files specified in compaction operation is not present. Missing .... |\n"})}),"\n",(0,i.jsxs)(_.p,{children:[(0,i.jsx)(_.strong,{children:"NOTE:"})," The following commands must be executed without any other writer/ingestion application running.\n{: .notice--warning}"]}),"\n",(0,i.jsx)(_.p,{children:"Sometimes, it becomes necessary to remove a fileId from a compaction-plan inorder to speed-up or unblock compaction\noperation. Any new log-files that happened on this file after the compaction got scheduled will be safely renamed\nso that are preserved. Hudi provides the following CLI to support it"}),"\n",(0,i.jsx)(_.h3,{id:"unscheduling-compaction",children:"Unscheduling Compaction"}),"\n",(0,i.jsx)(_.pre,{children:(0,i.jsx)(_.code,{className:"language-java",children:"hudi:trips->compaction unscheduleFileId --fileId \n....\nNo File renames needed to unschedule file from pending compaction. Operation successful.\n"})}),"\n",(0,i.jsx)(_.p,{children:"In other cases, an entire compaction plan needs to be reverted. This is supported by the following CLI"}),"\n",(0,i.jsx)(_.pre,{children:(0,i.jsx)(_.code,{className:"language-java",children:"hudi:trips->compaction unschedule --instant \n.....\nNo File renames needed to unschedule pending compaction. Operation successful.\n"})}),"\n",(0,i.jsx)(_.h3,{id:"repair-compaction",children:"Repair Compaction"}),"\n",(0,i.jsxs)(_.p,{children:["The above compaction unscheduling operations could sometimes fail partially (e",":g"," -> DFS temporarily unavailable). With\npartial failures, the compaction operation could become inconsistent with the state of file-slices. When you run\n",(0,i.jsx)(_.code,{children:"compaction validate"}),", you can notice invalid compaction operations if there is one. In these cases, the repair\ncommand comes to the rescue, it will rearrange the file-slices so that there is no loss and the file-slices are\nconsistent with the compaction plan"]}),"\n",(0,i.jsx)(_.pre,{children:(0,i.jsx)(_.code,{className:"language-java",children:"hudi:stock_ticks_mor->compaction repair --instant 20181005222611\n......\nCompaction successfully repaired\n.....\n"})}),"\n",(0,i.jsx)(_.h3,{id:"savepoint-and-restore",children:"Savepoint and Restore"}),"\n",(0,i.jsxs)(_.p,{children:['As the name suggest, "savepoint" saves the table as of the commit time, so that it lets you restore the table to this\nsavepoint at a later point in time if need be. You can read more about savepoints and restore ',(0,i.jsx)(_.a,{href:"disaster_recovery",children:"here"})]}),"\n",(0,i.jsx)(_.p,{children:"To trigger savepoint for a hudi table"}),"\n",(0,i.jsx)(_.pre,{children:(0,i.jsx)(_.code,{className:"language-java",children:"connect --path /tmp/hudi_trips_cow/\ncommits show\nset --conf SPARK_HOME=\nsavepoint create --commit 20220128160245447 --sparkMaster local[2]\n"})}),"\n",(0,i.jsx)(_.p,{children:"To restore the table to one of the savepointed commit:"}),"\n",(0,i.jsx)(_.pre,{children:(0,i.jsx)(_.code,{className:"language-java",children:"connect --path /tmp/hudi_trips_cow/\ncommits show\nset --conf SPARK_HOME=\nsavepoints show\n\u2554\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2557\n\u2551 SavepointTime \u2551\n\u2560\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2563\n\u2551 20220128160245447 \u2551\n\u255a\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u255d\nsavepoint rollback --savepoint 20220128160245447 --sparkMaster local[2]\n"})}),"\n",(0,i.jsx)(_.h3,{id:"upgrade-and-downgrade-table",children:"Upgrade and Downgrade Table"}),"\n",(0,i.jsx)(_.p,{children:"In case the user needs to downgrade the version of Hudi library used, the Hudi table needs to be manually downgraded\non the newer version of Hudi CLI before library downgrade. To downgrade a Hudi table through CLI, user needs to specify\nthe target Hudi table version as follows:"}),"\n",(0,i.jsx)(_.pre,{children:(0,i.jsx)(_.code,{className:"language-shell",children:"connect --path \ndowngrade table --toVersion \n"})}),"\n",(0,i.jsx)(_.p,{children:"The following table shows the Hudi table versions corresponding to the Hudi release versions:"}),"\n",(0,i.jsxs)(_.table,{children:[(0,i.jsx)(_.thead,{children:(0,i.jsxs)(_.tr,{children:[(0,i.jsx)(_.th,{style:{textAlign:"left"},children:"Hudi Table Version"}),(0,i.jsx)(_.th,{style:{textAlign:"left"},children:"Hudi Release Version(s)"})]})}),(0,i.jsxs)(_.tbody,{children:[(0,i.jsxs)(_.tr,{children:[(0,i.jsxs)(_.td,{style:{textAlign:"left"},children:[(0,i.jsx)(_.code,{children:"FIVE"})," or ",(0,i.jsx)(_.code,{children:"5"})]}),(0,i.jsx)(_.td,{style:{textAlign:"left"},children:"0.12.x"})]}),(0,i.jsxs)(_.tr,{children:[(0,i.jsxs)(_.td,{style:{textAlign:"left"},children:[(0,i.jsx)(_.code,{children:"FOUR"})," or ",(0,i.jsx)(_.code,{children:"4"})]}),(0,i.jsx)(_.td,{style:{textAlign:"left"},children:"0.11.x"})]}),(0,i.jsxs)(_.tr,{children:[(0,i.jsxs)(_.td,{style:{textAlign:"left"},children:[(0,i.jsx)(_.code,{children:"THREE"})," or ",(0,i.jsx)(_.code,{children:"3"})]}),(0,i.jsx)(_.td,{style:{textAlign:"left"},children:"0.10.x"})]}),(0,i.jsxs)(_.tr,{children:[(0,i.jsxs)(_.td,{style:{textAlign:"left"},children:[(0,i.jsx)(_.code,{children:"TWO"})," or ",(0,i.jsx)(_.code,{children:"2"})]}),(0,i.jsx)(_.td,{style:{textAlign:"left"},children:"0.9.x"})]}),(0,i.jsxs)(_.tr,{children:[(0,i.jsxs)(_.td,{style:{textAlign:"left"},children:[(0,i.jsx)(_.code,{children:"ONE"})," or ",(0,i.jsx)(_.code,{children:"1"})]}),(0,i.jsx)(_.td,{style:{textAlign:"left"},children:"0.6.x - 0.8.x"})]}),(0,i.jsxs)(_.tr,{children:[(0,i.jsxs)(_.td,{style:{textAlign:"left"},children:[(0,i.jsx)(_.code,{children:"ZERO"})," or ",(0,i.jsx)(_.code,{children:"0"})]}),(0,i.jsx)(_.td,{style:{textAlign:"left"},children:"0.5.x and below"})]})]})]}),"\n",(0,i.jsxs)(_.p,{children:["For example, to downgrade a table from version ",(0,i.jsx)(_.code,{children:"FIVE"}),"(",(0,i.jsx)(_.code,{children:"5"}),") (current version) to ",(0,i.jsx)(_.code,{children:"TWO"}),"(",(0,i.jsx)(_.code,{children:"2"}),"), you should run (use proper Spark master based\non your environment)"]}),"\n",(0,i.jsx)(_.pre,{children:(0,i.jsx)(_.code,{className:"language-shell",children:"downgrade table --toVersion TWO --sparkMaster local[2]\n"})}),"\n",(0,i.jsx)(_.p,{children:"or"}),"\n",(0,i.jsx)(_.pre,{children:(0,i.jsx)(_.code,{className:"language-shell",children:"downgrade table --toVersion 2 --sparkMaster local[2]\n"})}),"\n",(0,i.jsxs)(_.p,{children:["You can verify the table version by looking at the ",(0,i.jsx)(_.code,{children:"hoodie.table.version"})," property in ",(0,i.jsx)(_.code,{children:".hoodie/hoodie.properties"})," under\nthe table path:"]}),"\n",(0,i.jsx)(_.pre,{children:(0,i.jsx)(_.code,{className:"language-properties",children:"hoodie.table.version=2\n"})}),"\n",(0,i.jsx)(_.p,{children:"Hudi CLI also provides the ability to manually upgrade a Hudi table. To upgrade a Hudi table through CLI:"}),"\n",(0,i.jsx)(_.pre,{children:(0,i.jsx)(_.code,{className:"language-shell",children:"upgrade table --toVersion \n"})}),"\n",(0,i.jsxs)(_.admonition,{type:"note",children:[(0,i.jsxs)(_.p,{children:["Table upgrade is automatically handled by the Hudi write client in different deployment modes such as DeltaStreamer\nafter upgrading the Hudi library so that the user does not have to do manual upgrade. Such automatic table upgrade\nis the ",(0,i.jsx)(_.strong,{children:"recommended"})," way in general, instead of using ",(0,i.jsx)(_.code,{children:"upgrade"})," CLI command."]}),(0,i.jsx)(_.p,{children:'Table upgrade from table version ONE to TWO requires key generator related configs such as\n"hoodie.datasource.write.recordkey.field", which is only available when user configures the write job. So the table\nupgrade from version ONE to TWO through CLI is not supported, and user should rely on the automatic upgrade in the write\nclient instead.'})]}),"\n",(0,i.jsx)(_.p,{children:"You may also run the upgrade command without specifying the target version. In such a case, the latest table version\ncorresponding to the library release version is used:"}),"\n",(0,i.jsx)(_.pre,{children:(0,i.jsx)(_.code,{className:"language-shell",children:"upgrade table\n"})}),"\n",(0,i.jsx)(_.h3,{id:"change-hudi-table-type",children:"Change Hudi Table Type"}),"\n",(0,i.jsx)(_.p,{children:"There are cases we want to change the hudi table type. For example, change COW table to MOR for more efficient and\nlower latency ingestion; change MOR to COW for better read performance and compatibility with downstream engines.\nSo we offer the table command to perform this modification conveniently."}),"\n",(0,i.jsxs)(_.p,{children:["Changing ",(0,i.jsx)(_.strong,{children:"COW to MOR"}),", we can simply modify the ",(0,i.jsx)(_.code,{children:"hoodie.table.type"})," in ",(0,i.jsx)(_.code,{children:"hoodie.properties"})," to MERGE_ON_READ."]}),"\n",(0,i.jsxs)(_.p,{children:["While changing ",(0,i.jsx)(_.strong,{children:"MOR to COW"}),", we must make sure all the log files are compacted before modifying the table type,\nor it will cause data loss."]}),"\n",(0,i.jsx)(_.pre,{children:(0,i.jsx)(_.code,{className:"language-shell",children:"connect --path \ntable change-table-type \n"})}),"\n",(0,i.jsxs)(_.p,{children:["The parameter ",(0,i.jsx)(_.code,{children:"target_table_type"})," candidates are below:"]}),"\n",(0,i.jsxs)(_.table,{children:[(0,i.jsx)(_.thead,{children:(0,i.jsxs)(_.tr,{children:[(0,i.jsx)(_.th,{style:{textAlign:"left"},children:"target table type"}),(0,i.jsx)(_.th,{style:{textAlign:"left"},children:"comment"})]})}),(0,i.jsxs)(_.tbody,{children:[(0,i.jsxs)(_.tr,{children:[(0,i.jsx)(_.td,{style:{textAlign:"left"},children:"MOR"}),(0,i.jsx)(_.td,{style:{textAlign:"left"},children:"Change COW table to MERGE_ON_READ."})]}),(0,i.jsxs)(_.tr,{children:[(0,i.jsx)(_.td,{style:{textAlign:"left"},children:"COW"}),(0,i.jsxs)(_.td,{style:{textAlign:"left"},children:["Change MOR table to COPY_ON_WRITE. ",(0,i.jsx)("br",{}),"By default, changing to COW will ",(0,i.jsx)(_.strong,{children:"execute all pending compactions"})," and ",(0,i.jsx)(_.strong,{children:"perform a full compaction"})," if any log file left. Set ",(0,i.jsx)(_.code,{children:"--enable-compaction=false"})," will disable the default compaction. ",(0,i.jsx)("br",{}),"There are params can be set for the compaction operation:",(0,i.jsx)("br",{}),(0,i.jsx)(_.code,{children:"--parallelism"}),": Default ",(0,i.jsx)(_.code,{children:"3"}),". Parallelism for hoodie compaction",(0,i.jsx)("br",{}),(0,i.jsx)(_.code,{children:"--sparkMaster"}),": Default ",(0,i.jsx)(_.code,{children:"local"}),". Spark Master",(0,i.jsx)("br",{}),(0,i.jsx)(_.code,{children:"--sparkMemory"}),": Default ",(0,i.jsx)(_.code,{children:"4G"}),". Spark executor memory",(0,i.jsx)("br",{}),(0,i.jsx)(_.code,{children:"--retry"}),": Default ",(0,i.jsx)(_.code,{children:"1"}),". Number of retries",(0,i.jsx)("br",{}),(0,i.jsx)(_.code,{children:"--propsFilePath"}),": Default ",(0,i.jsx)(_.code,{children:" "}),". path to properties file on localfs or dfs with configurations for hoodie client for compacting",(0,i.jsx)("br",{}),(0,i.jsx)(_.code,{children:"--hoodieConfigs"}),": Default ",(0,i.jsx)(_.code,{children:" "}),". Any configuration that can be set in the properties file can be passed here in the form of an array"]})]})]})]}),"\n",(0,i.jsx)(_.p,{children:"Example below is changing MOR table to COW:"}),"\n",(0,i.jsx)(_.pre,{children:(0,i.jsx)(_.code,{className:"language-shell",children:"connect --path /var/dataset/test_table_mor2cow\ndesc\n\u2554\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2564\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2557\n\u2551 Property \u2502 Value \u2551\n\u2560\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u256a\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2563\n\u2551 basePath \u2502 /var/dataset/test_table_mor2cow \u2551\n\u255f\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2562\n\u2551 metaPath \u2502 /var/dataset/test_table_mor2cow/.hoodie \u2551\n\u255f\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2562\n\u2551 fileSystem \u2502 file \u2551\n\u255f\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2562\n\u2551 hoodie.table.name \u2502 test_table \u2551\n\u255f\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2562\n\u2551 hoodie.compaction.record.merger.strategy \u2502 eeb8d96f-b1e4-49fd-bbf8-28ac514178e5 \u2551\n\u255f\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2562\n\u2551 hoodie.table.metadata.partitions \u2502 files \u2551\n\u255f\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2562\n\u2551 hoodie.table.type \u2502 MERGE_ON_READ \u2551\n\u255f\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2562\n\u2551 hoodie.table.metadata.partitions.inflight \u2502 \u2551\n\u255f\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2562\n\u2551 hoodie.archivelog.folder \u2502 archived \u2551\n\u255f\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2562\n\u2551 hoodie.timeline.layout.version \u2502 1 \u2551\n\u255f\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2562\n\u2551 hoodie.table.checksum \u2502 2702201862 \u2551\n\u255f\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2562\n\u2551 hoodie.compaction.payload.type \u2502 HOODIE_AVRO \u2551\n\u255f\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2562\n\u2551 hoodie.table.version \u2502 6 \u2551\n\u255f\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2562\n\u2551 hoodie.datasource.write.drop.partition.columns \u2502 false \u2551\n\u255a\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2567\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u255d\n\ntable change-table-type COW\n\u2554\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2564\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2564\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2557\n\u2551 Property \u2502 Old Value \u2502 New Value \u2551\n\u2560\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u256a\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u256a\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2563\n\u2551 hoodie.archivelog.folder \u2502 archived \u2502 archived \u2551\n\u255f\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2562\n\u2551 hoodie.compaction.payload.type \u2502 HOODIE_AVRO \u2502 HOODIE_AVRO \u2551\n\u255f\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2562\n\u2551 hoodie.compaction.record.merger.strategy \u2502 eeb8d96f-b1e4-49fd-bbf8-28ac514178e5 \u2502 eeb8d96f-b1e4-49fd-bbf8-28ac514178e5 \u2551\n\u255f\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2562\n\u2551 hoodie.datasource.write.drop.partition.columns \u2502 false \u2502 false \u2551\n\u255f\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2562\n\u2551 hoodie.table.checksum \u2502 2702201862 \u2502 2702201862 \u2551\n\u255f\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2562\n\u2551 hoodie.table.metadata.partitions \u2502 files \u2502 files \u2551\n\u255f\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2562\n\u2551 hoodie.table.metadata.partitions.inflight \u2502 \u2502 \u2551\n\u255f\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2562\n\u2551 hoodie.table.name \u2502 test_table \u2502 test_table \u2551\n\u255f\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2562\n\u2551 hoodie.table.type \u2502 MERGE_ON_READ \u2502 COPY_ON_WRITE \u2551\n\u255f\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2562\n\u2551 hoodie.table.version \u2502 6 \u2502 6 \u2551\n\u255f\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2562\n\u2551 hoodie.timeline.layout.version \u2502 1 \u2502 1 \u2551\n\u255a\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2567\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2567\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u255d\n"})}),"\n",(0,i.jsx)(_.h2,{id:"related-resources",children:"Related Resources"}),"\n",(0,i.jsx)("h3",{children:"Blogs"}),"\n",(0,i.jsxs)(_.ul,{children:["\n",(0,i.jsx)(_.li,{children:(0,i.jsx)(_.a,{href:"https://www.onehouse.ai/blog/getting-started-manage-your-hudi-tables-with-the-admin-hudi-cli-tool",children:"Getting Started: Manage your Hudi tables with the admin Hudi-CLI tool"})}),"\n"]})]})}function d(e={}){const{wrapper:_}={...(0,a.R)(),...e.components};return _?(0,i.jsx)(_,{...e,children:(0,i.jsx)(c,{...e})}):c(e)}},28453:(e,_,n)=>{n.d(_,{R:()=>o,x:()=>s});var t=n(96540);const i={},a=t.createContext(i);function o(e){const _=t.useContext(a);return t.useMemo((function(){return"function"==typeof e?e(_):{..._,...e}}),[_,e])}function s(e){let _;return _=e.disableParentContext?"function"==typeof e.components?e.components(i):e.components||i:o(e.components),t.createElement(a.Provider,{value:_},e.children)}}}]);
\ No newline at end of file
diff --git a/content/assets/js/2263a65b.8b774927.js b/content/assets/js/2263a65b.00a479c5.js
similarity index 89%
rename from content/assets/js/2263a65b.8b774927.js
rename to content/assets/js/2263a65b.00a479c5.js
index e7f03c91300ee..905d0f7dbe852 100644
--- a/content/assets/js/2263a65b.8b774927.js
+++ b/content/assets/js/2263a65b.00a479c5.js
@@ -1 +1 @@
-"use strict";(self.webpackChunkhudi=self.webpackChunkhudi||[]).push([[99502],{8946:(e,a,t)=>{t.r(a),t.d(a,{assets:()=>s,contentTitle:()=>l,default:()=>u,frontMatter:()=>n,metadata:()=>i,toc:()=>d});const i=JSON.parse('{"id":"precommit_validator","title":"Data Quality","description":"Data quality refers to the overall accuracy, completeness, consistency, and validity of data. Ensuring data quality is vital for accurate analysis and reporting, as well as for compliance with regulations and maintaining trust in your organization\'s data infrastructure.","source":"@site/docs/precommit_validator.md","sourceDirName":".","slug":"/precommit_validator","permalink":"/docs/next/precommit_validator","draft":false,"unlisted":false,"editUrl":"https://github.com/apache/hudi/tree/asf-site/website/docs/precommit_validator.md","tags":[],"version":"current","frontMatter":{"title":"Data Quality","keywords":["hudi","quality","expectations","pre-commit validator"]},"sidebar":"docs","previous":{"title":"Exporter","permalink":"/docs/next/snapshot_exporter"},"next":{"title":"Post-commit Callback","permalink":"/docs/next/platform_services_post_commit_callback"}}');var o=t(74848),r=t(28453);const n={title:"Data Quality",keywords:["hudi","quality","expectations","pre-commit validator"]},l=void 0,s={},d=[{value:"SQL Query Single Result",id:"sql-query-single-result",level:2},{value:"SQL Query Equality",id:"sql-query-equality",level:2},{value:"SQL Query Inequality",id:"sql-query-inequality",level:2},{value:"Extend Custom Validator",id:"extend-custom-validator",level:2},{value:"Additional Monitoring with Notifications",id:"additional-monitoring-with-notifications",level:2},{value:"Related Resources",id:"related-resources",level:2}];function c(e){const a={a:"a",code:"code",h2:"h2",li:"li",p:"p",pre:"pre",strong:"strong",ul:"ul",...(0,r.R)(),...e.components};return(0,o.jsxs)(o.Fragment,{children:[(0,o.jsx)(a.p,{children:"Data quality refers to the overall accuracy, completeness, consistency, and validity of data. Ensuring data quality is vital for accurate analysis and reporting, as well as for compliance with regulations and maintaining trust in your organization's data infrastructure."}),"\n",(0,o.jsxs)(a.p,{children:["Hudi offers ",(0,o.jsx)(a.strong,{children:"Pre-Commit Validators"})," that allow you to ensure that your data meets certain data quality expectations as you are writing with Hudi Streamer or Spark Datasource writers."]}),"\n",(0,o.jsxs)(a.p,{children:["To configure pre-commit validators, use this setting ",(0,o.jsx)(a.code,{children:"hoodie.precommit.validators="}),"."]}),"\n",(0,o.jsx)(a.p,{children:"Example:"}),"\n",(0,o.jsx)(a.pre,{children:(0,o.jsx)(a.code,{className:"language-scala",children:'spark.write.format("hudi")\n .option("hoodie.precommit.validators", "org.apache.hudi.client.validator.SqlQueryEqualityPreCommitValidator")\n'})}),"\n",(0,o.jsx)(a.p,{children:"Today you can use any of these validators and even have the flexibility to extend your own:"}),"\n",(0,o.jsx)(a.h2,{id:"sql-query-single-result",children:"SQL Query Single Result"}),"\n",(0,o.jsx)(a.p,{children:(0,o.jsx)(a.a,{href:"https://github.com/apache/hudi/blob/bf5a52e51bbeaa089995335a0a4c55884792e505/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/validator/SqlQuerySingleResultPreCommitValidator.java",children:"org.apache.hudi.client.validator.SqlQuerySingleResultPreCommitValidator"})}),"\n",(0,o.jsx)(a.p,{children:"The SQL Query Single Result validator can be used to validate that a query on the table results in a specific value. This validator allows you to run a SQL query and abort the commit if it does not match the expected output."}),"\n",(0,o.jsxs)(a.p,{children:["Multiple queries can be separated by ",(0,o.jsx)(a.code,{children:";"})," delimiter. Include the expected result as part of the query separated by ",(0,o.jsx)(a.code,{children:"#"}),"."]}),"\n",(0,o.jsxs)(a.p,{children:["Syntax: ",(0,o.jsx)(a.code,{children:"query1#result1;query2#result2"})]}),"\n",(0,o.jsx)(a.p,{children:"Example:"}),"\n",(0,o.jsx)(a.pre,{children:(0,o.jsx)(a.code,{className:"language-scala",children:'// In this example, we set up a validator that expects there is no row with `col` column as `null`\n\nimport org.apache.hudi.config.HoodiePreCommitValidatorConfig._\n\ndf.write.format("hudi").mode(Overwrite).\n option("hoodie.table.name", tableName).\n option("hoodie.precommit.validators", "org.apache.hudi.client.validator.SqlQuerySingleResultPreCommitValidator").\n option("hoodie.precommit.validators.single.value.sql.queries", "select count(*) from where col is null#0").\n save(basePath)\n'})}),"\n",(0,o.jsx)(a.h2,{id:"sql-query-equality",children:"SQL Query Equality"}),"\n",(0,o.jsx)(a.p,{children:(0,o.jsx)(a.a,{href:"https://github.com/apache/hudi/blob/bf5a52e51bbeaa089995335a0a4c55884792e505/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/validator/SqlQueryEqualityPreCommitValidator.java",children:"org.apache.hudi.client.validator.SqlQueryEqualityPreCommitValidator"})}),"\n",(0,o.jsx)(a.p,{children:"The SQL Query Equality validator runs a query before ingesting the data, then runs the same query after ingesting the data and confirms that both outputs match. This allows you to validate for equality of rows before and after the commit."}),"\n",(0,o.jsx)(a.p,{children:"This validator is useful when you want to verify that your query does not change a specific subset of the data. Some examples:"}),"\n",(0,o.jsxs)(a.ul,{children:["\n",(0,o.jsx)(a.li,{children:"Validate that the number of null fields is the same before and after your query"}),"\n",(0,o.jsx)(a.li,{children:"Validate that there are no duplicate records after your query runs"}),"\n",(0,o.jsx)(a.li,{children:"Validate that you are only updating the data, and no inserts slip through"}),"\n"]}),"\n",(0,o.jsx)(a.p,{children:"Example:"}),"\n",(0,o.jsx)(a.pre,{children:(0,o.jsx)(a.code,{className:"language-scala",children:'// In this example, we set up a validator that expects no change of null rows with the new commit\n\nimport org.apache.hudi.config.HoodiePreCommitValidatorConfig._\n\ndf.write.format("hudi").mode(Overwrite).\n option("hoodie.table.name", tableName).\n option("hoodie.precommit.validators", "org.apache.hudi.client.validator.SqlQueryEqualityPreCommitValidator").\n option("hoodie.precommit.validators.equality.sql.queries", "select count(*) from where col is null").\n save(basePath)\n'})}),"\n",(0,o.jsx)(a.h2,{id:"sql-query-inequality",children:"SQL Query Inequality"}),"\n",(0,o.jsx)(a.p,{children:(0,o.jsx)(a.a,{href:"https://github.com/apache/hudi/blob/bf5a52e51bbeaa089995335a0a4c55884792e505/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/validator/SqlQueryInequalityPreCommitValidator.java",children:"org.apache.hudi.client.validator.SqlQueryInequalityPreCommitValidator"})}),"\n",(0,o.jsx)(a.p,{children:"The SQL Query Inquality validator runs a query before ingesting the data, then runs the same query after ingesting the data and confirms that both outputs DO NOT match. This allows you to confirm changes in the rows before and after the commit."}),"\n",(0,o.jsx)(a.p,{children:"Example:"}),"\n",(0,o.jsx)(a.pre,{children:(0,o.jsx)(a.code,{className:"language-scala",children:'// In this example, we set up a validator that expects a change of null rows with the new commit\n\nimport org.apache.hudi.config.HoodiePreCommitValidatorConfig._\n\ndf.write.format("hudi").mode(Overwrite).\n option("hoodie.table.name", tableName).\n option("hoodie.precommit.validators", "org.apache.hudi.client.validator.SqlQueryInequalityPreCommitValidator").\n option("hoodie.precommit.validators.inequality.sql.queries", "select count(*) from where col is null").\n save(basePath)\n'})}),"\n",(0,o.jsx)(a.h2,{id:"extend-custom-validator",children:"Extend Custom Validator"}),"\n",(0,o.jsxs)(a.p,{children:["Users can also provide their own implementations by extending the abstract class ",(0,o.jsx)(a.a,{href:"https://github.com/apache/hudi/blob/bf5a52e51bbeaa089995335a0a4c55884792e505/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/validator/SparkPreCommitValidator.java",children:"SparkPreCommitValidator"}),"\nand overriding this method"]}),"\n",(0,o.jsx)(a.pre,{children:(0,o.jsx)(a.code,{className:"language-java",children:"void validateRecordsBeforeAndAfter(Dataset before, \n Dataset after, \n Set partitionsAffected)\n"})}),"\n",(0,o.jsx)(a.h2,{id:"additional-monitoring-with-notifications",children:"Additional Monitoring with Notifications"}),"\n",(0,o.jsxs)(a.p,{children:["Hudi offers a ",(0,o.jsx)(a.a,{href:"platform_services_post_commit_callback",children:"commit notification service"})," that can be configured to trigger notifications about write commits."]}),"\n",(0,o.jsx)(a.p,{children:"The commit notification service can be combined with pre-commit validators to send a notification when a commit fails a validation. This is possible by passing details about the validation as a custom value to the HTTP endpoint."}),"\n",(0,o.jsx)(a.h2,{id:"related-resources",children:"Related Resources"}),"\n",(0,o.jsx)("h3",{children:"Videos"}),"\n",(0,o.jsxs)(a.ul,{children:["\n",(0,o.jsx)(a.li,{children:(0,o.jsx)(a.a,{href:"https://www.youtube.com/watch?v=KNzs9dj_Btc",children:"Learn About Apache Hudi Pre Commit Validator with Hands on Lab"})}),"\n"]})]})}function u(e={}){const{wrapper:a}={...(0,r.R)(),...e.components};return a?(0,o.jsx)(a,{...e,children:(0,o.jsx)(c,{...e})}):c(e)}},28453:(e,a,t)=>{t.d(a,{R:()=>n,x:()=>l});var i=t(96540);const o={},r=i.createContext(o);function n(e){const a=i.useContext(r);return i.useMemo((function(){return"function"==typeof e?e(a):{...a,...e}}),[a,e])}function l(e){let a;return a=e.disableParentContext?"function"==typeof e.components?e.components(o):e.components||o:n(e.components),i.createElement(r.Provider,{value:a},e.children)}}}]);
\ No newline at end of file
+"use strict";(self.webpackChunkhudi=self.webpackChunkhudi||[]).push([[99502],{8946:(e,a,t)=>{t.r(a),t.d(a,{assets:()=>s,contentTitle:()=>l,default:()=>u,frontMatter:()=>n,metadata:()=>i,toc:()=>d});const i=JSON.parse('{"id":"precommit_validator","title":"Data Quality","description":"Data quality refers to the overall accuracy, completeness, consistency, and validity of data. Ensuring data quality is vital for accurate analysis and reporting, as well as for compliance with regulations and maintaining trust in your organization\'s data infrastructure.","source":"@site/docs/precommit_validator.md","sourceDirName":".","slug":"/precommit_validator","permalink":"/docs/next/precommit_validator","draft":false,"unlisted":false,"editUrl":"https://github.com/apache/hudi/tree/asf-site/website/docs/precommit_validator.md","tags":[],"version":"current","frontMatter":{"title":"Data Quality","keywords":["hudi","quality","expectations","pre-commit validator"]},"sidebar":"docs","previous":{"title":"Exporter","permalink":"/docs/next/snapshot_exporter"},"next":{"title":"Post-commit Callback","permalink":"/docs/next/platform_services_post_commit_callback"}}');var o=t(74848),r=t(28453);const n={title:"Data Quality",keywords:["hudi","quality","expectations","pre-commit validator"]},l=void 0,s={},d=[{value:"SQL Query Single Result",id:"sql-query-single-result",level:2},{value:"SQL Query Equality",id:"sql-query-equality",level:2},{value:"SQL Query Inequality",id:"sql-query-inequality",level:2},{value:"Extend Custom Validator",id:"extend-custom-validator",level:2},{value:"Additional Monitoring with Notifications",id:"additional-monitoring-with-notifications",level:2},{value:"Related Resources",id:"related-resources",level:2}];function c(e){const a={a:"a",code:"code",h2:"h2",li:"li",p:"p",pre:"pre",strong:"strong",ul:"ul",...(0,r.R)(),...e.components};return(0,o.jsxs)(o.Fragment,{children:[(0,o.jsx)(a.p,{children:"Data quality refers to the overall accuracy, completeness, consistency, and validity of data. Ensuring data quality is vital for accurate analysis and reporting, as well as for compliance with regulations and maintaining trust in your organization's data infrastructure."}),"\n",(0,o.jsxs)(a.p,{children:["Hudi offers ",(0,o.jsx)(a.strong,{children:"Pre-Commit Validators"})," that allow you to ensure that your data meets certain data quality expectations as you are writing with Hudi Streamer or Spark Datasource writers."]}),"\n",(0,o.jsxs)(a.p,{children:["To configure pre-commit validators, use this setting ",(0,o.jsx)(a.code,{children:"hoodie.precommit.validators="}),"."]}),"\n",(0,o.jsx)(a.p,{children:"Example:"}),"\n",(0,o.jsx)(a.pre,{children:(0,o.jsx)(a.code,{className:"language-scala",children:'spark.write.format("hudi")\n .option("hoodie.precommit.validators", "org.apache.hudi.client.validator.SqlQueryEqualityPreCommitValidator")\n'})}),"\n",(0,o.jsx)(a.p,{children:"Today you can use any of these validators and even have the flexibility to extend your own:"}),"\n",(0,o.jsx)(a.h2,{id:"sql-query-single-result",children:"SQL Query Single Result"}),"\n",(0,o.jsx)(a.p,{children:(0,o.jsx)(a.a,{href:"https://github.com/apache/hudi/blob/bf5a52e51bbeaa089995335a0a4c55884792e505/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/validator/SqlQuerySingleResultPreCommitValidator.java",children:"org.apache.hudi.client.validator.SqlQuerySingleResultPreCommitValidator"})}),"\n",(0,o.jsx)(a.p,{children:"The SQL Query Single Result validator can be used to validate that a query on the table results in a specific value. This validator allows you to run a SQL query and abort the commit if it does not match the expected output."}),"\n",(0,o.jsxs)(a.p,{children:["Multiple queries can be separated by ",(0,o.jsx)(a.code,{children:";"})," delimiter. Include the expected result as part of the query separated by ",(0,o.jsx)(a.code,{children:"#"}),"."]}),"\n",(0,o.jsxs)(a.p,{children:["Syntax: ",(0,o.jsx)(a.code,{children:"query1#result1;query2#result2"})]}),"\n",(0,o.jsx)(a.p,{children:"Example:"}),"\n",(0,o.jsx)(a.pre,{children:(0,o.jsx)(a.code,{className:"language-scala",children:'// In this example, we set up a validator that expects there is no row with `col` column as `null`\n\nimport org.apache.hudi.config.HoodiePreCommitValidatorConfig._\n\ndf.write.format("hudi").mode(Overwrite).\n option("hoodie.table.name", tableName).\n option("hoodie.precommit.validators", "org.apache.hudi.client.validator.SqlQuerySingleResultPreCommitValidator").\n option("hoodie.precommit.validators.single.value.sql.queries", "select count(*) from where col is null#0").\n save(basePath)\n'})}),"\n",(0,o.jsx)(a.h2,{id:"sql-query-equality",children:"SQL Query Equality"}),"\n",(0,o.jsx)(a.p,{children:(0,o.jsx)(a.a,{href:"https://github.com/apache/hudi/blob/bf5a52e51bbeaa089995335a0a4c55884792e505/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/validator/SqlQueryEqualityPreCommitValidator.java",children:"org.apache.hudi.client.validator.SqlQueryEqualityPreCommitValidator"})}),"\n",(0,o.jsx)(a.p,{children:"The SQL Query Equality validator runs a query before ingesting the data, then runs the same query after ingesting the data and confirms that both outputs match. This allows you to validate for equality of rows before and after the commit."}),"\n",(0,o.jsx)(a.p,{children:"This validator is useful when you want to verify that your query does not change a specific subset of the data. Some examples:"}),"\n",(0,o.jsxs)(a.ul,{children:["\n",(0,o.jsx)(a.li,{children:"Validate that the number of null fields is the same before and after your query"}),"\n",(0,o.jsx)(a.li,{children:"Validate that there are no duplicate records after your query runs"}),"\n",(0,o.jsx)(a.li,{children:"Validate that you are only updating the data, and no inserts slip through"}),"\n"]}),"\n",(0,o.jsx)(a.p,{children:"Example:"}),"\n",(0,o.jsx)(a.pre,{children:(0,o.jsx)(a.code,{className:"language-scala",children:'// In this example, we set up a validator that expects no change of null rows with the new commit\n\nimport org.apache.hudi.config.HoodiePreCommitValidatorConfig._\n\ndf.write.format("hudi").mode(Overwrite).\n option("hoodie.table.name", tableName).\n option("hoodie.precommit.validators", "org.apache.hudi.client.validator.SqlQueryEqualityPreCommitValidator").\n option("hoodie.precommit.validators.equality.sql.queries", "select count(*) from where col is null").\n save(basePath)\n'})}),"\n",(0,o.jsx)(a.h2,{id:"sql-query-inequality",children:"SQL Query Inequality"}),"\n",(0,o.jsx)(a.p,{children:(0,o.jsx)(a.a,{href:"https://github.com/apache/hudi/blob/bf5a52e51bbeaa089995335a0a4c55884792e505/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/validator/SqlQueryInequalityPreCommitValidator.java",children:"org.apache.hudi.client.validator.SqlQueryInequalityPreCommitValidator"})}),"\n",(0,o.jsx)(a.p,{children:"The SQL Query Inquality validator runs a query before ingesting the data, then runs the same query after ingesting the data and confirms that both outputs DO NOT match. This allows you to confirm changes in the rows before and after the commit."}),"\n",(0,o.jsx)(a.p,{children:"Example:"}),"\n",(0,o.jsx)(a.pre,{children:(0,o.jsx)(a.code,{className:"language-scala",children:'// In this example, we set up a validator that expects a change of null rows with the new commit\n\nimport org.apache.hudi.config.HoodiePreCommitValidatorConfig._\n\ndf.write.format("hudi").mode(Overwrite).\n option("hoodie.table.name", tableName).\n option("hoodie.precommit.validators", "org.apache.hudi.client.validator.SqlQueryInequalityPreCommitValidator").\n option("hoodie.precommit.validators.inequality.sql.queries", "select count(*) from where col is null").\n save(basePath)\n'})}),"\n",(0,o.jsx)(a.h2,{id:"extend-custom-validator",children:"Extend Custom Validator"}),"\n",(0,o.jsxs)(a.p,{children:["Users can also provide their own implementations by extending the abstract class ",(0,o.jsx)(a.a,{href:"https://github.com/apache/hudi/blob/bf5a52e51bbeaa089995335a0a4c55884792e505/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/validator/SparkPreCommitValidator.java",children:"SparkPreCommitValidator"}),"\nand overriding this method"]}),"\n",(0,o.jsx)(a.pre,{children:(0,o.jsx)(a.code,{className:"language-java",children:"void validateRecordsBeforeAndAfter(Dataset before, \n Dataset after, \n Set partitionsAffected)\n"})}),"\n",(0,o.jsx)(a.h2,{id:"additional-monitoring-with-notifications",children:"Additional Monitoring with Notifications"}),"\n",(0,o.jsxs)(a.p,{children:["Hudi offers a ",(0,o.jsx)(a.a,{href:"platform_services_post_commit_callback",children:"commit notification service"})," that can be configured to trigger notifications about write commits."]}),"\n",(0,o.jsx)(a.p,{children:"The commit notification service can be combined with pre-commit validators to send a notification when a commit fails a validation. This is possible by passing details about the validation as a custom value to the HTTP endpoint."}),"\n",(0,o.jsx)(a.h2,{id:"related-resources",children:"Related Resources"}),"\n",(0,o.jsx)("h3",{children:"Blogs"}),"\n",(0,o.jsxs)(a.ul,{children:["\n",(0,o.jsx)(a.li,{children:(0,o.jsx)(a.a,{href:"https://www.onehouse.ai/blog/apply-pre-commit-validation-for-data-quality-in-apache-hudi",children:"Apply Pre-Commit Validation for Data Quality in Apache Hudi"})}),"\n"]}),"\n",(0,o.jsx)("h3",{children:"Videos"}),"\n",(0,o.jsxs)(a.ul,{children:["\n",(0,o.jsx)(a.li,{children:(0,o.jsx)(a.a,{href:"https://www.youtube.com/watch?v=KNzs9dj_Btc",children:"Learn About Apache Hudi Pre Commit Validator with Hands on Lab"})}),"\n"]})]})}function u(e={}){const{wrapper:a}={...(0,r.R)(),...e.components};return a?(0,o.jsx)(a,{...e,children:(0,o.jsx)(c,{...e})}):c(e)}},28453:(e,a,t)=>{t.d(a,{R:()=>n,x:()=>l});var i=t(96540);const o={},r=i.createContext(o);function n(e){const a=i.useContext(r);return i.useMemo((function(){return"function"==typeof e?e(a):{...a,...e}}),[a,e])}function l(e){let a;return a=e.disableParentContext?"function"==typeof e.components?e.components(o):e.components||o:n(e.components),i.createElement(r.Provider,{value:a},e.children)}}}]);
\ No newline at end of file
diff --git a/content/assets/js/45a5cd1f.789c1bc4.js b/content/assets/js/45a5cd1f.789c1bc4.js
deleted file mode 100644
index ea3701a0a13a1..0000000000000
--- a/content/assets/js/45a5cd1f.789c1bc4.js
+++ /dev/null
@@ -1 +0,0 @@
-"use strict";(self.webpackChunkhudi=self.webpackChunkhudi||[]).push([[94e3],{27515:(e,t,i)=>{i.r(t),i.d(t,{assets:()=>l,contentTitle:()=>r,default:()=>h,frontMatter:()=>o,metadata:()=>n,toc:()=>d});const n=JSON.parse('{"id":"concepts","title":"Concepts","description":"Apache Hudi (pronounced \u201cHudi\u201d) provides the following streaming primitives over hadoop compatible storages","source":"@site/docs/concepts.md","sourceDirName":".","slug":"/concepts","permalink":"/docs/next/concepts","draft":false,"unlisted":false,"editUrl":"https://github.com/apache/hudi/tree/asf-site/website/docs/concepts.md","tags":[],"version":"current","frontMatter":{"version":"0.6.0","title":"Concepts","keywords":["hudi","design","table","queries","timeline"],"summary":"Here we introduce some basic concepts & give a broad technical overview of Hudi","toc":true,"last_modified_at":"2019-12-30T19:59:57.000Z"}}');var s=i(74848),a=i(28453);const o={version:"0.6.0",title:"Concepts",keywords:["hudi","design","table","queries","timeline"],summary:"Here we introduce some basic concepts & give a broad technical overview of Hudi",toc:!0,last_modified_at:new Date("2019-12-30T19:59:57.000Z")},r=void 0,l={},d=[{value:"Timeline",id:"timeline",level:2},{value:"File management",id:"file-management",level:2},{value:"Index",id:"index",level:2},{value:"Table Types & Queries",id:"table-types--queries",level:2},{value:"Table Types",id:"table-types",level:3},{value:"Query types",id:"query-types",level:3},{value:"Copy On Write Table",id:"copy-on-write-table",level:2},{value:"Merge On Read Table",id:"merge-on-read-table",level:2}];function c(e){const t={a:"a",code:"code",h2:"h2",h3:"h3",li:"li",p:"p",strong:"strong",table:"table",tbody:"tbody",td:"td",th:"th",thead:"thead",tr:"tr",ul:"ul",...(0,a.R)(),...e.components};return(0,s.jsxs)(s.Fragment,{children:[(0,s.jsx)(t.p,{children:"Apache Hudi (pronounced \u201cHudi\u201d) provides the following streaming primitives over hadoop compatible storages"}),"\n",(0,s.jsxs)(t.ul,{children:["\n",(0,s.jsx)(t.li,{children:"Update/Delete Records (how do I change records in a table?)"}),"\n",(0,s.jsx)(t.li,{children:"Change Streams (how do I fetch records that changed?)"}),"\n"]}),"\n",(0,s.jsx)(t.p,{children:"In this section, we will discuss key concepts & terminologies that are important to understand, to be able to effectively use these primitives."}),"\n",(0,s.jsx)(t.h2,{id:"timeline",children:"Timeline"}),"\n",(0,s.jsxs)(t.p,{children:["At its core, Hudi maintains a ",(0,s.jsx)(t.code,{children:"timeline"})," of all actions performed on the table at different ",(0,s.jsx)(t.code,{children:"instants"})," of time that helps provide instantaneous views of the table,\nwhile also efficiently supporting retrieval of data in the order of arrival. A Hudi instant consists of the following components"]}),"\n",(0,s.jsxs)(t.ul,{children:["\n",(0,s.jsxs)(t.li,{children:[(0,s.jsx)(t.code,{children:"Instant action"})," : Type of action performed on the table"]}),"\n",(0,s.jsxs)(t.li,{children:[(0,s.jsx)(t.code,{children:"Instant time"})," : Instant time is typically a timestamp (e.g: 20190117010349), which monotonically increases in the order of action's begin time."]}),"\n",(0,s.jsxs)(t.li,{children:[(0,s.jsx)(t.code,{children:"state"})," : current state of the instant"]}),"\n"]}),"\n",(0,s.jsx)(t.p,{children:"Hudi guarantees that the actions performed on the timeline are atomic & timeline consistent based on the instant time."}),"\n",(0,s.jsx)(t.p,{children:"Key actions performed include"}),"\n",(0,s.jsxs)(t.ul,{children:["\n",(0,s.jsxs)(t.li,{children:[(0,s.jsx)(t.code,{children:"COMMITS"})," - A commit denotes an ",(0,s.jsx)(t.strong,{children:"atomic write"})," of a batch of records into a table."]}),"\n",(0,s.jsxs)(t.li,{children:[(0,s.jsx)(t.code,{children:"CLEANS"})," - Background activity that gets rid of older versions of files in the table, that are no longer needed."]}),"\n",(0,s.jsxs)(t.li,{children:[(0,s.jsx)(t.code,{children:"DELTA_COMMIT"})," - A delta commit refers to an ",(0,s.jsx)(t.strong,{children:"atomic write"})," of a batch of records into a MergeOnRead type table, where some/all of the data could be just written to delta logs."]}),"\n",(0,s.jsxs)(t.li,{children:[(0,s.jsx)(t.code,{children:"COMPACTION"})," - Background activity to reconcile differential data structures within Hudi e.g: moving updates from row based log files to columnar formats. Internally, compaction manifests as a special commit on the timeline"]}),"\n",(0,s.jsxs)(t.li,{children:[(0,s.jsx)(t.code,{children:"ROLLBACK"})," - Indicates that a commit/delta commit was unsuccessful & rolled back, removing any partial files produced during such a write"]}),"\n",(0,s.jsxs)(t.li,{children:[(0,s.jsx)(t.code,{children:"SAVEPOINT"}),' - Marks certain file groups as "saved", such that cleaner will not delete them. It helps restore the table to a point on the timeline, in case of disaster/data recovery scenarios.']}),"\n"]}),"\n",(0,s.jsx)(t.p,{children:"Any given instant can be\nin one of the following states"}),"\n",(0,s.jsxs)(t.ul,{children:["\n",(0,s.jsxs)(t.li,{children:[(0,s.jsx)(t.code,{children:"REQUESTED"})," - Denotes an action has been scheduled, but has not initiated"]}),"\n",(0,s.jsxs)(t.li,{children:[(0,s.jsx)(t.code,{children:"INFLIGHT"})," - Denotes that the action is currently being performed"]}),"\n",(0,s.jsxs)(t.li,{children:[(0,s.jsx)(t.code,{children:"COMPLETED"})," - Denotes completion of an action on the timeline"]}),"\n"]}),"\n",(0,s.jsx)("figure",{children:(0,s.jsx)("img",{className:"docimage",src:"/assets/images/hudi_timeline.png",alt:"hudi_timeline.png"})}),"\n",(0,s.jsxs)(t.p,{children:["Example above shows upserts happenings between 10:00 and 10:20 on a Hudi table, roughly every 5 mins, leaving commit metadata on the Hudi timeline, along\nwith other background cleaning/compactions. One key observation to make is that the commit time indicates the ",(0,s.jsx)(t.code,{children:"arrival time"})," of the data (10:20AM), while the actual data\norganization reflects the actual time or ",(0,s.jsx)(t.code,{children:"event time"}),", the data was intended for (hourly buckets from 07:00). These are two key concepts when reasoning about tradeoffs between latency and completeness of data."]}),"\n",(0,s.jsx)(t.p,{children:"When there is late arriving data (data intended for 9:00 arriving >1 hr late at 10:20), we can see the upsert producing new data into even older time buckets/folders.\nWith the help of the timeline, an incremental query attempting to get all new data that was committed successfully since 10:00 hours, is able to very efficiently consume\nonly the changed files without say scanning all the time buckets > 07:00."}),"\n",(0,s.jsx)(t.h2,{id:"file-management",children:"File management"}),"\n",(0,s.jsxs)(t.p,{children:["Hudi organizes a table into a directory structure under a ",(0,s.jsx)(t.code,{children:"basepath"})," on DFS. Table is broken up into partitions, which are folders containing data files for that partition,\nvery similar to Hive tables. Each partition is uniquely identified by its ",(0,s.jsx)(t.code,{children:"partitionpath"}),", which is relative to the basepath."]}),"\n",(0,s.jsxs)(t.p,{children:["Within each partition, files are organized into ",(0,s.jsx)(t.code,{children:"file groups"}),", uniquely identified by a ",(0,s.jsx)(t.code,{children:"file id"}),". Each file group contains several\n",(0,s.jsx)(t.code,{children:"file slices"}),", where each slice contains a base file (",(0,s.jsx)(t.code,{children:"*.parquet"}),") produced at a certain commit/compaction instant time,\nalong with set of log files (",(0,s.jsx)(t.code,{children:"*.log.*"}),") that contain inserts/updates to the base file since the base file was produced.\nHudi adopts a MVCC design, where compaction action merges logs and base files to produce new file slices and cleaning action gets rid of\nunused/older file slices to reclaim space on DFS."]}),"\n",(0,s.jsx)(t.h2,{id:"index",children:"Index"}),"\n",(0,s.jsx)(t.p,{children:"Hudi provides efficient upserts, by mapping a given hoodie key (record key + partition path) consistently to a file id, via an indexing mechanism.\nThis mapping between record key and file group/file id, never changes once the first version of a record has been written to a file. In short, the\nmapped file group contains all versions of a group of records."}),"\n",(0,s.jsx)(t.h2,{id:"table-types--queries",children:"Table Types & Queries"}),"\n",(0,s.jsxs)(t.p,{children:["Hudi table types define how data is indexed & laid out on the DFS and how the above primitives and timeline activities are implemented on top of such organization (i.e how data is written).\nIn turn, ",(0,s.jsx)(t.code,{children:"query types"})," define how the underlying data is exposed to the queries (i.e how data is read)."]}),"\n",(0,s.jsxs)(t.table,{children:[(0,s.jsx)(t.thead,{children:(0,s.jsxs)(t.tr,{children:[(0,s.jsx)(t.th,{children:"Table Type"}),(0,s.jsx)(t.th,{children:"Supported Query types"})]})}),(0,s.jsxs)(t.tbody,{children:[(0,s.jsxs)(t.tr,{children:[(0,s.jsx)(t.td,{children:"Copy On Write"}),(0,s.jsx)(t.td,{children:"Snapshot Queries + Incremental Queries"})]}),(0,s.jsxs)(t.tr,{children:[(0,s.jsx)(t.td,{children:"Merge On Read"}),(0,s.jsx)(t.td,{children:"Snapshot Queries + Incremental Queries + Read Optimized Queries"})]})]})]}),"\n",(0,s.jsx)(t.h3,{id:"table-types",children:"Table Types"}),"\n",(0,s.jsx)(t.p,{children:"Hudi supports the following table types."}),"\n",(0,s.jsxs)(t.ul,{children:["\n",(0,s.jsxs)(t.li,{children:[(0,s.jsx)(t.a,{href:"#copy-on-write-table",children:"Copy On Write"})," : Stores data using exclusively columnar file formats (e.g parquet). Updates simply version & rewrite the files by performing a synchronous merge during write."]}),"\n",(0,s.jsxs)(t.li,{children:[(0,s.jsx)(t.a,{href:"#merge-on-read-table",children:"Merge On Read"})," : Stores data using a combination of columnar (e.g parquet) + row based (e.g avro) file formats. Updates are logged to delta files & later compacted to produce new versions of columnar files synchronously or asynchronously."]}),"\n"]}),"\n",(0,s.jsx)(t.p,{children:"Following table summarizes the trade-offs between these two table types"}),"\n",(0,s.jsxs)(t.table,{children:[(0,s.jsx)(t.thead,{children:(0,s.jsxs)(t.tr,{children:[(0,s.jsx)(t.th,{children:"Trade-off"}),(0,s.jsx)(t.th,{children:"CopyOnWrite"}),(0,s.jsx)(t.th,{children:"MergeOnRead"})]})}),(0,s.jsxs)(t.tbody,{children:[(0,s.jsxs)(t.tr,{children:[(0,s.jsx)(t.td,{children:"Data Latency"}),(0,s.jsx)(t.td,{children:"Higher"}),(0,s.jsx)(t.td,{children:"Lower"})]}),(0,s.jsxs)(t.tr,{children:[(0,s.jsx)(t.td,{children:"Update cost (I/O)"}),(0,s.jsx)(t.td,{children:"Higher (rewrite entire parquet)"}),(0,s.jsx)(t.td,{children:"Lower (append to delta log)"})]}),(0,s.jsxs)(t.tr,{children:[(0,s.jsx)(t.td,{children:"Parquet File Size"}),(0,s.jsx)(t.td,{children:"Smaller (high update(I/0) cost)"}),(0,s.jsx)(t.td,{children:"Larger (low update cost)"})]}),(0,s.jsxs)(t.tr,{children:[(0,s.jsx)(t.td,{children:"Write Amplification"}),(0,s.jsx)(t.td,{children:"Higher"}),(0,s.jsx)(t.td,{children:"Lower (depending on compaction strategy)"})]})]})]}),"\n",(0,s.jsx)(t.h3,{id:"query-types",children:"Query types"}),"\n",(0,s.jsx)(t.p,{children:"Hudi supports the following query types"}),"\n",(0,s.jsxs)(t.ul,{children:["\n",(0,s.jsxs)(t.li,{children:[(0,s.jsx)(t.strong,{children:"Snapshot Queries"})," : Queries see the latest snapshot of the table as of a given commit or compaction action. In case of merge on read table, it exposes near-real time data(few mins) by merging\nthe base and delta files of the latest file slice on-the-fly. For copy on write table, it provides a drop-in replacement for existing parquet tables, while providing upsert/delete and other write side features."]}),"\n",(0,s.jsxs)(t.li,{children:[(0,s.jsx)(t.strong,{children:"Incremental Queries"})," : Queries only see new data written to the table, since a given commit/compaction. This effectively provides change streams to enable incremental data pipelines."]}),"\n",(0,s.jsxs)(t.li,{children:[(0,s.jsx)(t.strong,{children:"Read Optimized Queries"})," : Queries see the latest snapshot of table as of a given commit/compaction action. Exposes only the base/columnar files in latest file slices and guarantees the\nsame columnar query performance compared to a non-hudi columnar table."]}),"\n"]}),"\n",(0,s.jsx)(t.p,{children:"Following table summarizes the trade-offs between the different query types."}),"\n",(0,s.jsxs)(t.table,{children:[(0,s.jsx)(t.thead,{children:(0,s.jsxs)(t.tr,{children:[(0,s.jsx)(t.th,{children:"Trade-off"}),(0,s.jsx)(t.th,{children:"Snapshot"}),(0,s.jsx)(t.th,{children:"Read Optimized"})]})}),(0,s.jsxs)(t.tbody,{children:[(0,s.jsxs)(t.tr,{children:[(0,s.jsx)(t.td,{children:"Data Latency"}),(0,s.jsx)(t.td,{children:"Lower"}),(0,s.jsx)(t.td,{children:"Higher"})]}),(0,s.jsxs)(t.tr,{children:[(0,s.jsx)(t.td,{children:"Query Latency"}),(0,s.jsx)(t.td,{children:"Higher (merge base / columnar file + row based delta / log files)"}),(0,s.jsx)(t.td,{children:"Lower (raw base / columnar file performance)"})]})]})]}),"\n",(0,s.jsx)(t.h2,{id:"copy-on-write-table",children:"Copy On Write Table"}),"\n",(0,s.jsx)(t.p,{children:"File slices in Copy-On-Write table only contain the base/columnar file and each commit produces new versions of base files.\nIn other words, we implicitly compact on every commit, such that only columnar data exists. As a result, the write amplification\n(number of bytes written for 1 byte of incoming data) is much higher, where read amplification is zero.\nThis is a much desired property for analytical workloads, which is predominantly read-heavy."}),"\n",(0,s.jsx)(t.p,{children:"Following illustrates how this works conceptually, when data written into copy-on-write table and two queries running on top of it."}),"\n",(0,s.jsx)("figure",{children:(0,s.jsx)("img",{className:"docimage",src:"/assets/images/hudi_cow.png",alt:"hudi_cow.png"})}),"\n",(0,s.jsxs)(t.p,{children:["As data gets written, updates to existing file groups produce a new slice for that file group stamped with the commit instant time,\nwhile inserts allocate a new file group and write its first slice for that file group. These file slices and their commit instant times are color coded above.\nSQL queries running against such a table (eg: ",(0,s.jsx)(t.code,{children:"select count(*)"})," counting the total records in that partition), first checks the timeline for the latest commit\nand filters all but latest file slices of each file group. As you can see, an old query does not see the current inflight commit's files color coded in pink,\nbut a new query starting after the commit picks up the new data. Thus queries are immune to any write failures/partial writes and only run on committed data."]}),"\n",(0,s.jsx)(t.p,{children:"The intention of copy on write table, is to fundamentally improve how tables are managed today through"}),"\n",(0,s.jsxs)(t.ul,{children:["\n",(0,s.jsx)(t.li,{children:"First class support for atomically updating data at file-level, instead of rewriting whole tables/partitions"}),"\n",(0,s.jsx)(t.li,{children:"Ability to incremental consume changes, as opposed to wasteful scans or fumbling with heuristics"}),"\n",(0,s.jsx)(t.li,{children:"Tight control of file sizes to keep query performance excellent (small files hurt query performance considerably)."}),"\n"]}),"\n",(0,s.jsx)(t.h2,{id:"merge-on-read-table",children:"Merge On Read Table"}),"\n",(0,s.jsx)(t.p,{children:"Merge on read table is a superset of copy on write, in the sense it still supports read optimized queries of the table by exposing only the base/columnar files in latest file slices.\nAdditionally, it stores incoming upserts for each file group, onto a row based delta log, to support snapshot queries by applying the delta log,\nonto the latest version of each file id on-the-fly during query time. Thus, this table type attempts to balance read and write amplification intelligently, to provide near real-time data.\nThe most significant change here, would be to the compactor, which now carefully chooses which delta log files need to be compacted onto\ntheir columnar base file, to keep the query performance in check (larger delta log files would incur longer merge times with merge data on query side)"}),"\n",(0,s.jsx)(t.p,{children:"Following illustrates how the table works, and shows two types of queries - snapshot query and read optimized query."}),"\n",(0,s.jsx)("figure",{children:(0,s.jsx)("img",{className:"docimage",src:"/assets/images/hudi_mor.png",alt:"hudi_mor.png"})}),"\n",(0,s.jsx)(t.p,{children:"There are lot of interesting things happening in this example, which bring out the subtleties in the approach."}),"\n",(0,s.jsxs)(t.ul,{children:["\n",(0,s.jsx)(t.li,{children:"We now have commits every 1 minute or so, something we could not do in the other table type."}),"\n",(0,s.jsx)(t.li,{children:"Within each file id group, now there is an delta log file, which holds incoming updates to the records already present in the base columnar files. In the example, the delta log files hold\nall the data from 10:05 to 10:10. The base columnar files are still versioned with the commit, as before.\nThus, if one were to simply look at base files alone, then the table layout looks exactly like a copy on write table."}),"\n",(0,s.jsx)(t.li,{children:"A periodic compaction process reconciles these changes from the delta log and produces a new version of base file, just like what happened at 10:05 in the example."}),"\n",(0,s.jsx)(t.li,{children:"There are two ways of querying the same underlying table: Read Optimized query and Snapshot query, depending on whether we chose query performance or freshness of data."}),"\n",(0,s.jsx)(t.li,{children:"The semantics around when data from a commit is available to a query changes in a subtle way for a read optimized query. Note, that such a query\nrunning at 10:10, wont see data after 10:05 above, while a snapshot query always sees the freshest data."}),"\n",(0,s.jsx)(t.li,{children:"When we trigger compaction & what it decides to compact hold all the key to solving these hard problems. By implementing a compacting\nstrategy, where we aggressively compact the latest partitions compared to older partitions, we could ensure the read optimized queries see data\npublished within X minutes in a consistent fashion."}),"\n"]}),"\n",(0,s.jsx)(t.p,{children:"The intention of merge on read table is to enable near real-time processing directly on top of DFS, as opposed to copying\ndata out to specialized systems, which may not be able to handle the data volume. There are also a few secondary side benefits to\nthis table such as reduced write amplification by avoiding synchronous merge of data, i.e, the amount of data written per 1 bytes of data in a batch"})]})}function h(e={}){const{wrapper:t}={...(0,a.R)(),...e.components};return t?(0,s.jsx)(t,{...e,children:(0,s.jsx)(c,{...e})}):c(e)}},28453:(e,t,i)=>{i.d(t,{R:()=>o,x:()=>r});var n=i(96540);const s={},a=n.createContext(s);function o(e){const t=n.useContext(a);return n.useMemo((function(){return"function"==typeof e?e(t):{...t,...e}}),[t,e])}function r(e){let t;return t=e.disableParentContext?"function"==typeof e.components?e.components(s):e.components||s:o(e.components),n.createElement(a.Provider,{value:t},e.children)}}}]);
\ No newline at end of file
diff --git a/content/assets/js/45a5cd1f.85a8b76e.js b/content/assets/js/45a5cd1f.85a8b76e.js
new file mode 100644
index 0000000000000..83ed0fc809c71
--- /dev/null
+++ b/content/assets/js/45a5cd1f.85a8b76e.js
@@ -0,0 +1 @@
+"use strict";(self.webpackChunkhudi=self.webpackChunkhudi||[]).push([[94e3],{27515:(e,t,i)=>{i.r(t),i.d(t,{assets:()=>l,contentTitle:()=>r,default:()=>h,frontMatter:()=>o,metadata:()=>n,toc:()=>d});const n=JSON.parse('{"id":"concepts","title":"Concepts","description":"Apache Hudi (pronounced \u201cHudi\u201d) provides the following streaming primitives over hadoop compatible storages","source":"@site/docs/concepts.md","sourceDirName":".","slug":"/concepts","permalink":"/docs/next/concepts","draft":false,"unlisted":false,"editUrl":"https://github.com/apache/hudi/tree/asf-site/website/docs/concepts.md","tags":[],"version":"current","frontMatter":{"version":"0.6.0","title":"Concepts","keywords":["hudi","design","table","queries","timeline"],"summary":"Here we introduce some basic concepts & give a broad technical overview of Hudi","toc":true,"last_modified_at":"2019-12-30T19:59:57.000Z"}}');var s=i(74848),a=i(28453);const o={version:"0.6.0",title:"Concepts",keywords:["hudi","design","table","queries","timeline"],summary:"Here we introduce some basic concepts & give a broad technical overview of Hudi",toc:!0,last_modified_at:new Date("2019-12-30T19:59:57.000Z")},r=void 0,l={},d=[{value:"Timeline",id:"timeline",level:2},{value:"File management",id:"file-management",level:2},{value:"Index",id:"index",level:2},{value:"Table Types & Queries",id:"table-types--queries",level:2},{value:"Table Types",id:"table-types",level:3},{value:"Query types",id:"query-types",level:3},{value:"Copy On Write Table",id:"copy-on-write-table",level:2},{value:"Merge On Read Table",id:"merge-on-read-table",level:2},{value:"Related Resources",id:"related-resources",level:2}];function c(e){const t={a:"a",code:"code",h2:"h2",h3:"h3",li:"li",p:"p",strong:"strong",table:"table",tbody:"tbody",td:"td",th:"th",thead:"thead",tr:"tr",ul:"ul",...(0,a.R)(),...e.components};return(0,s.jsxs)(s.Fragment,{children:[(0,s.jsx)(t.p,{children:"Apache Hudi (pronounced \u201cHudi\u201d) provides the following streaming primitives over hadoop compatible storages"}),"\n",(0,s.jsxs)(t.ul,{children:["\n",(0,s.jsx)(t.li,{children:"Update/Delete Records (how do I change records in a table?)"}),"\n",(0,s.jsx)(t.li,{children:"Change Streams (how do I fetch records that changed?)"}),"\n"]}),"\n",(0,s.jsx)(t.p,{children:"In this section, we will discuss key concepts & terminologies that are important to understand, to be able to effectively use these primitives."}),"\n",(0,s.jsx)(t.h2,{id:"timeline",children:"Timeline"}),"\n",(0,s.jsxs)(t.p,{children:["At its core, Hudi maintains a ",(0,s.jsx)(t.code,{children:"timeline"})," of all actions performed on the table at different ",(0,s.jsx)(t.code,{children:"instants"})," of time that helps provide instantaneous views of the table,\nwhile also efficiently supporting retrieval of data in the order of arrival. A Hudi instant consists of the following components"]}),"\n",(0,s.jsxs)(t.ul,{children:["\n",(0,s.jsxs)(t.li,{children:[(0,s.jsx)(t.code,{children:"Instant action"})," : Type of action performed on the table"]}),"\n",(0,s.jsxs)(t.li,{children:[(0,s.jsx)(t.code,{children:"Instant time"})," : Instant time is typically a timestamp (e.g: 20190117010349), which monotonically increases in the order of action's begin time."]}),"\n",(0,s.jsxs)(t.li,{children:[(0,s.jsx)(t.code,{children:"state"})," : current state of the instant"]}),"\n"]}),"\n",(0,s.jsx)(t.p,{children:"Hudi guarantees that the actions performed on the timeline are atomic & timeline consistent based on the instant time."}),"\n",(0,s.jsx)(t.p,{children:"Key actions performed include"}),"\n",(0,s.jsxs)(t.ul,{children:["\n",(0,s.jsxs)(t.li,{children:[(0,s.jsx)(t.code,{children:"COMMITS"})," - A commit denotes an ",(0,s.jsx)(t.strong,{children:"atomic write"})," of a batch of records into a table."]}),"\n",(0,s.jsxs)(t.li,{children:[(0,s.jsx)(t.code,{children:"CLEANS"})," - Background activity that gets rid of older versions of files in the table, that are no longer needed."]}),"\n",(0,s.jsxs)(t.li,{children:[(0,s.jsx)(t.code,{children:"DELTA_COMMIT"})," - A delta commit refers to an ",(0,s.jsx)(t.strong,{children:"atomic write"})," of a batch of records into a MergeOnRead type table, where some/all of the data could be just written to delta logs."]}),"\n",(0,s.jsxs)(t.li,{children:[(0,s.jsx)(t.code,{children:"COMPACTION"})," - Background activity to reconcile differential data structures within Hudi e.g: moving updates from row based log files to columnar formats. Internally, compaction manifests as a special commit on the timeline"]}),"\n",(0,s.jsxs)(t.li,{children:[(0,s.jsx)(t.code,{children:"ROLLBACK"})," - Indicates that a commit/delta commit was unsuccessful & rolled back, removing any partial files produced during such a write"]}),"\n",(0,s.jsxs)(t.li,{children:[(0,s.jsx)(t.code,{children:"SAVEPOINT"}),' - Marks certain file groups as "saved", such that cleaner will not delete them. It helps restore the table to a point on the timeline, in case of disaster/data recovery scenarios.']}),"\n"]}),"\n",(0,s.jsx)(t.p,{children:"Any given instant can be\nin one of the following states"}),"\n",(0,s.jsxs)(t.ul,{children:["\n",(0,s.jsxs)(t.li,{children:[(0,s.jsx)(t.code,{children:"REQUESTED"})," - Denotes an action has been scheduled, but has not initiated"]}),"\n",(0,s.jsxs)(t.li,{children:[(0,s.jsx)(t.code,{children:"INFLIGHT"})," - Denotes that the action is currently being performed"]}),"\n",(0,s.jsxs)(t.li,{children:[(0,s.jsx)(t.code,{children:"COMPLETED"})," - Denotes completion of an action on the timeline"]}),"\n"]}),"\n",(0,s.jsx)("figure",{children:(0,s.jsx)("img",{className:"docimage",src:"/assets/images/hudi_timeline.png",alt:"hudi_timeline.png"})}),"\n",(0,s.jsxs)(t.p,{children:["Example above shows upserts happenings between 10:00 and 10:20 on a Hudi table, roughly every 5 mins, leaving commit metadata on the Hudi timeline, along\nwith other background cleaning/compactions. One key observation to make is that the commit time indicates the ",(0,s.jsx)(t.code,{children:"arrival time"})," of the data (10:20AM), while the actual data\norganization reflects the actual time or ",(0,s.jsx)(t.code,{children:"event time"}),", the data was intended for (hourly buckets from 07:00). These are two key concepts when reasoning about tradeoffs between latency and completeness of data."]}),"\n",(0,s.jsx)(t.p,{children:"When there is late arriving data (data intended for 9:00 arriving >1 hr late at 10:20), we can see the upsert producing new data into even older time buckets/folders.\nWith the help of the timeline, an incremental query attempting to get all new data that was committed successfully since 10:00 hours, is able to very efficiently consume\nonly the changed files without say scanning all the time buckets > 07:00."}),"\n",(0,s.jsx)(t.h2,{id:"file-management",children:"File management"}),"\n",(0,s.jsxs)(t.p,{children:["Hudi organizes a table into a directory structure under a ",(0,s.jsx)(t.code,{children:"basepath"})," on DFS. Table is broken up into partitions, which are folders containing data files for that partition,\nvery similar to Hive tables. Each partition is uniquely identified by its ",(0,s.jsx)(t.code,{children:"partitionpath"}),", which is relative to the basepath."]}),"\n",(0,s.jsxs)(t.p,{children:["Within each partition, files are organized into ",(0,s.jsx)(t.code,{children:"file groups"}),", uniquely identified by a ",(0,s.jsx)(t.code,{children:"file id"}),". Each file group contains several\n",(0,s.jsx)(t.code,{children:"file slices"}),", where each slice contains a base file (",(0,s.jsx)(t.code,{children:"*.parquet"}),") produced at a certain commit/compaction instant time,\nalong with set of log files (",(0,s.jsx)(t.code,{children:"*.log.*"}),") that contain inserts/updates to the base file since the base file was produced.\nHudi adopts a MVCC design, where compaction action merges logs and base files to produce new file slices and cleaning action gets rid of\nunused/older file slices to reclaim space on DFS."]}),"\n",(0,s.jsx)(t.h2,{id:"index",children:"Index"}),"\n",(0,s.jsx)(t.p,{children:"Hudi provides efficient upserts, by mapping a given hoodie key (record key + partition path) consistently to a file id, via an indexing mechanism.\nThis mapping between record key and file group/file id, never changes once the first version of a record has been written to a file. In short, the\nmapped file group contains all versions of a group of records."}),"\n",(0,s.jsx)(t.h2,{id:"table-types--queries",children:"Table Types & Queries"}),"\n",(0,s.jsxs)(t.p,{children:["Hudi table types define how data is indexed & laid out on the DFS and how the above primitives and timeline activities are implemented on top of such organization (i.e how data is written).\nIn turn, ",(0,s.jsx)(t.code,{children:"query types"})," define how the underlying data is exposed to the queries (i.e how data is read)."]}),"\n",(0,s.jsxs)(t.table,{children:[(0,s.jsx)(t.thead,{children:(0,s.jsxs)(t.tr,{children:[(0,s.jsx)(t.th,{children:"Table Type"}),(0,s.jsx)(t.th,{children:"Supported Query types"})]})}),(0,s.jsxs)(t.tbody,{children:[(0,s.jsxs)(t.tr,{children:[(0,s.jsx)(t.td,{children:"Copy On Write"}),(0,s.jsx)(t.td,{children:"Snapshot Queries + Incremental Queries"})]}),(0,s.jsxs)(t.tr,{children:[(0,s.jsx)(t.td,{children:"Merge On Read"}),(0,s.jsx)(t.td,{children:"Snapshot Queries + Incremental Queries + Read Optimized Queries"})]})]})]}),"\n",(0,s.jsx)(t.h3,{id:"table-types",children:"Table Types"}),"\n",(0,s.jsx)(t.p,{children:"Hudi supports the following table types."}),"\n",(0,s.jsxs)(t.ul,{children:["\n",(0,s.jsxs)(t.li,{children:[(0,s.jsx)(t.a,{href:"#copy-on-write-table",children:"Copy On Write"})," : Stores data using exclusively columnar file formats (e.g parquet). Updates simply version & rewrite the files by performing a synchronous merge during write."]}),"\n",(0,s.jsxs)(t.li,{children:[(0,s.jsx)(t.a,{href:"#merge-on-read-table",children:"Merge On Read"})," : Stores data using a combination of columnar (e.g parquet) + row based (e.g avro) file formats. Updates are logged to delta files & later compacted to produce new versions of columnar files synchronously or asynchronously."]}),"\n"]}),"\n",(0,s.jsx)(t.p,{children:"Following table summarizes the trade-offs between these two table types"}),"\n",(0,s.jsxs)(t.table,{children:[(0,s.jsx)(t.thead,{children:(0,s.jsxs)(t.tr,{children:[(0,s.jsx)(t.th,{children:"Trade-off"}),(0,s.jsx)(t.th,{children:"CopyOnWrite"}),(0,s.jsx)(t.th,{children:"MergeOnRead"})]})}),(0,s.jsxs)(t.tbody,{children:[(0,s.jsxs)(t.tr,{children:[(0,s.jsx)(t.td,{children:"Data Latency"}),(0,s.jsx)(t.td,{children:"Higher"}),(0,s.jsx)(t.td,{children:"Lower"})]}),(0,s.jsxs)(t.tr,{children:[(0,s.jsx)(t.td,{children:"Update cost (I/O)"}),(0,s.jsx)(t.td,{children:"Higher (rewrite entire parquet)"}),(0,s.jsx)(t.td,{children:"Lower (append to delta log)"})]}),(0,s.jsxs)(t.tr,{children:[(0,s.jsx)(t.td,{children:"Parquet File Size"}),(0,s.jsx)(t.td,{children:"Smaller (high update(I/0) cost)"}),(0,s.jsx)(t.td,{children:"Larger (low update cost)"})]}),(0,s.jsxs)(t.tr,{children:[(0,s.jsx)(t.td,{children:"Write Amplification"}),(0,s.jsx)(t.td,{children:"Higher"}),(0,s.jsx)(t.td,{children:"Lower (depending on compaction strategy)"})]})]})]}),"\n",(0,s.jsx)(t.h3,{id:"query-types",children:"Query types"}),"\n",(0,s.jsx)(t.p,{children:"Hudi supports the following query types"}),"\n",(0,s.jsxs)(t.ul,{children:["\n",(0,s.jsxs)(t.li,{children:[(0,s.jsx)(t.strong,{children:"Snapshot Queries"})," : Queries see the latest snapshot of the table as of a given commit or compaction action. In case of merge on read table, it exposes near-real time data(few mins) by merging\nthe base and delta files of the latest file slice on-the-fly. For copy on write table, it provides a drop-in replacement for existing parquet tables, while providing upsert/delete and other write side features."]}),"\n",(0,s.jsxs)(t.li,{children:[(0,s.jsx)(t.strong,{children:"Incremental Queries"})," : Queries only see new data written to the table, since a given commit/compaction. This effectively provides change streams to enable incremental data pipelines."]}),"\n",(0,s.jsxs)(t.li,{children:[(0,s.jsx)(t.strong,{children:"Read Optimized Queries"})," : Queries see the latest snapshot of table as of a given commit/compaction action. Exposes only the base/columnar files in latest file slices and guarantees the\nsame columnar query performance compared to a non-hudi columnar table."]}),"\n"]}),"\n",(0,s.jsx)(t.p,{children:"Following table summarizes the trade-offs between the different query types."}),"\n",(0,s.jsxs)(t.table,{children:[(0,s.jsx)(t.thead,{children:(0,s.jsxs)(t.tr,{children:[(0,s.jsx)(t.th,{children:"Trade-off"}),(0,s.jsx)(t.th,{children:"Snapshot"}),(0,s.jsx)(t.th,{children:"Read Optimized"})]})}),(0,s.jsxs)(t.tbody,{children:[(0,s.jsxs)(t.tr,{children:[(0,s.jsx)(t.td,{children:"Data Latency"}),(0,s.jsx)(t.td,{children:"Lower"}),(0,s.jsx)(t.td,{children:"Higher"})]}),(0,s.jsxs)(t.tr,{children:[(0,s.jsx)(t.td,{children:"Query Latency"}),(0,s.jsx)(t.td,{children:"Higher (merge base / columnar file + row based delta / log files)"}),(0,s.jsx)(t.td,{children:"Lower (raw base / columnar file performance)"})]})]})]}),"\n",(0,s.jsx)(t.h2,{id:"copy-on-write-table",children:"Copy On Write Table"}),"\n",(0,s.jsx)(t.p,{children:"File slices in Copy-On-Write table only contain the base/columnar file and each commit produces new versions of base files.\nIn other words, we implicitly compact on every commit, such that only columnar data exists. As a result, the write amplification\n(number of bytes written for 1 byte of incoming data) is much higher, where read amplification is zero.\nThis is a much desired property for analytical workloads, which is predominantly read-heavy."}),"\n",(0,s.jsx)(t.p,{children:"Following illustrates how this works conceptually, when data written into copy-on-write table and two queries running on top of it."}),"\n",(0,s.jsx)("figure",{children:(0,s.jsx)("img",{className:"docimage",src:"/assets/images/hudi_cow.png",alt:"hudi_cow.png"})}),"\n",(0,s.jsxs)(t.p,{children:["As data gets written, updates to existing file groups produce a new slice for that file group stamped with the commit instant time,\nwhile inserts allocate a new file group and write its first slice for that file group. These file slices and their commit instant times are color coded above.\nSQL queries running against such a table (eg: ",(0,s.jsx)(t.code,{children:"select count(*)"})," counting the total records in that partition), first checks the timeline for the latest commit\nand filters all but latest file slices of each file group. As you can see, an old query does not see the current inflight commit's files color coded in pink,\nbut a new query starting after the commit picks up the new data. Thus queries are immune to any write failures/partial writes and only run on committed data."]}),"\n",(0,s.jsx)(t.p,{children:"The intention of copy on write table, is to fundamentally improve how tables are managed today through"}),"\n",(0,s.jsxs)(t.ul,{children:["\n",(0,s.jsx)(t.li,{children:"First class support for atomically updating data at file-level, instead of rewriting whole tables/partitions"}),"\n",(0,s.jsx)(t.li,{children:"Ability to incremental consume changes, as opposed to wasteful scans or fumbling with heuristics"}),"\n",(0,s.jsx)(t.li,{children:"Tight control of file sizes to keep query performance excellent (small files hurt query performance considerably)."}),"\n"]}),"\n",(0,s.jsx)(t.h2,{id:"merge-on-read-table",children:"Merge On Read Table"}),"\n",(0,s.jsx)(t.p,{children:"Merge on read table is a superset of copy on write, in the sense it still supports read optimized queries of the table by exposing only the base/columnar files in latest file slices.\nAdditionally, it stores incoming upserts for each file group, onto a row based delta log, to support snapshot queries by applying the delta log,\nonto the latest version of each file id on-the-fly during query time. Thus, this table type attempts to balance read and write amplification intelligently, to provide near real-time data.\nThe most significant change here, would be to the compactor, which now carefully chooses which delta log files need to be compacted onto\ntheir columnar base file, to keep the query performance in check (larger delta log files would incur longer merge times with merge data on query side)"}),"\n",(0,s.jsx)(t.p,{children:"Following illustrates how the table works, and shows two types of queries - snapshot query and read optimized query."}),"\n",(0,s.jsx)("figure",{children:(0,s.jsx)("img",{className:"docimage",src:"/assets/images/hudi_mor.png",alt:"hudi_mor.png"})}),"\n",(0,s.jsx)(t.p,{children:"There are lot of interesting things happening in this example, which bring out the subtleties in the approach."}),"\n",(0,s.jsxs)(t.ul,{children:["\n",(0,s.jsx)(t.li,{children:"We now have commits every 1 minute or so, something we could not do in the other table type."}),"\n",(0,s.jsx)(t.li,{children:"Within each file id group, now there is an delta log file, which holds incoming updates to the records already present in the base columnar files. In the example, the delta log files hold\nall the data from 10:05 to 10:10. The base columnar files are still versioned with the commit, as before.\nThus, if one were to simply look at base files alone, then the table layout looks exactly like a copy on write table."}),"\n",(0,s.jsx)(t.li,{children:"A periodic compaction process reconciles these changes from the delta log and produces a new version of base file, just like what happened at 10:05 in the example."}),"\n",(0,s.jsx)(t.li,{children:"There are two ways of querying the same underlying table: Read Optimized query and Snapshot query, depending on whether we chose query performance or freshness of data."}),"\n",(0,s.jsx)(t.li,{children:"The semantics around when data from a commit is available to a query changes in a subtle way for a read optimized query. Note, that such a query\nrunning at 10:10, wont see data after 10:05 above, while a snapshot query always sees the freshest data."}),"\n",(0,s.jsx)(t.li,{children:"When we trigger compaction & what it decides to compact hold all the key to solving these hard problems. By implementing a compacting\nstrategy, where we aggressively compact the latest partitions compared to older partitions, we could ensure the read optimized queries see data\npublished within X minutes in a consistent fashion."}),"\n"]}),"\n",(0,s.jsx)(t.p,{children:"The intention of merge on read table is to enable near real-time processing directly on top of DFS, as opposed to copying\ndata out to specialized systems, which may not be able to handle the data volume. There are also a few secondary side benefits to\nthis table such as reduced write amplification by avoiding synchronous merge of data, i.e, the amount of data written per 1 bytes of data in a batch"}),"\n",(0,s.jsx)(t.h2,{id:"related-resources",children:"Related Resources"}),"\n",(0,s.jsx)("h3",{children:"Blogs"}),"\n",(0,s.jsxs)(t.ul,{children:["\n",(0,s.jsx)(t.li,{children:(0,s.jsx)(t.a,{href:"https://www.onehouse.ai/blog/comparing-apache-hudis-mor-and-cow-tables-use-cases-from-uber-and-shopee",children:"Comparing Apache Hudi's MOR and COW Tables: Use Cases from Uber and Shopee"})}),"\n",(0,s.jsx)(t.li,{children:(0,s.jsx)(t.a,{href:"https://www.onehouse.ai/blog/hudi-metafields-demystified",children:"Hudi Metafields demystified"})}),"\n",(0,s.jsx)(t.li,{children:(0,s.jsx)(t.a,{href:"https://medium.com/@simpsons/file-naming-conventions-in-apache-hudi-cd1cdd95f5e7",children:"File Naming conventions in Apache Hudi"})}),"\n"]})]})}function h(e={}){const{wrapper:t}={...(0,a.R)(),...e.components};return t?(0,s.jsx)(t,{...e,children:(0,s.jsx)(c,{...e})}):c(e)}},28453:(e,t,i)=>{i.d(t,{R:()=>o,x:()=>r});var n=i(96540);const s={},a=n.createContext(s);function o(e){const t=n.useContext(a);return n.useMemo((function(){return"function"==typeof e?e(t):{...t,...e}}),[t,e])}function r(e){let t;return t=e.disableParentContext?"function"==typeof e.components?e.components(s):e.components||s:o(e.components),n.createElement(a.Provider,{value:t},e.children)}}}]);
\ No newline at end of file
diff --git a/content/assets/js/491d56e0.6e8fb4cd.js b/content/assets/js/491d56e0.6e8fb4cd.js
new file mode 100644
index 0000000000000..3b77cb0bd134f
--- /dev/null
+++ b/content/assets/js/491d56e0.6e8fb4cd.js
@@ -0,0 +1 @@
+"use strict";(self.webpackChunkhudi=self.webpackChunkhudi||[]).push([[72907],{48105:(e,r,i)=>{i.r(r),i.d(r,{assets:()=>d,contentTitle:()=>s,default:()=>h,frontMatter:()=>a,metadata:()=>n,toc:()=>c});const n=JSON.parse('{"id":"record_merger","title":"Record Mergers","description":"Hudi handles mutations to records and streaming data, as we briefly touched upon in timeline ordering section.","source":"@site/docs/record_merger.md","sourceDirName":".","slug":"/record_merger","permalink":"/docs/next/record_merger","draft":false,"unlisted":false,"editUrl":"https://github.com/apache/hudi/tree/asf-site/website/docs/record_merger.md","tags":[],"version":"current","frontMatter":{"title":"Record Mergers","keywords":["hudi","merge","upsert","precombine"],"toc":true,"toc_min_heading_level":2,"toc_max_heading_level":4},"sidebar":"docs","previous":{"title":"Key Generation","permalink":"/docs/next/key_generation"},"next":{"title":"Table Metadata","permalink":"/docs/next/metadata"}}');var o=i(74848),t=i(28453);const a={title:"Record Mergers",keywords:["hudi","merge","upsert","precombine"],toc:!0,toc_min_heading_level:2,toc_max_heading_level:4},s=void 0,d={},c=[{value:"COMMIT_TIME_ORDERING",id:"commit_time_ordering",level:3},{value:"EVENT_TIME_ORDERING",id:"event_time_ordering",level:3},{value:"CUSTOM",id:"custom",level:3},{value:"Record Merge Configs",id:"record-merge-configs",level:3},{value:"Record Payloads",id:"record-payloads",level:3},{value:"OverwriteWithLatestAvroPayload",id:"overwritewithlatestavropayload",level:4},{value:"DefaultHoodieRecordPayload",id:"defaulthoodierecordpayload",level:4},{value:"EventTimeAvroPayload",id:"eventtimeavropayload",level:4},{value:"OverwriteNonDefaultsWithLatestAvroPayload",id:"overwritenondefaultswithlatestavropayload",level:4},{value:"PartialUpdateAvroPayload",id:"partialupdateavropayload",level:4},{value:"Configs",id:"configs",level:4},{value:"Related Resources",id:"related-resources",level:2}];function l(e){const r={a:"a",admonition:"admonition",code:"code",em:"em",h2:"h2",h3:"h3",h4:"h4",li:"li",p:"p",pre:"pre",strong:"strong",table:"table",tbody:"tbody",td:"td",th:"th",thead:"thead",tr:"tr",ul:"ul",...(0,t.R)(),...e.components};return(0,o.jsxs)(o.Fragment,{children:[(0,o.jsxs)(r.p,{children:["Hudi handles mutations to records and streaming data, as we briefly touched upon in ",(0,o.jsx)(r.a,{href:"timeline#ordering-of-actions",children:"timeline ordering"})," section.\nTo provide users full-fledged support for stream processing, Hudi goes all the way making the storage engine and the underlying storage format\nunderstand how to merge changes to the same record key, that may arrive even in different order at different times. With the rise of mobile applications\nand IoT, these scenarios have become the normal than an exception. For e.g. a social networking application uploading user events several hours after they happened,\nwhen the user connects to WiFi networks."]}),"\n",(0,o.jsx)(r.p,{children:"To achieve this, Hudi supports merge modes, which define how the base and log files are ordered in a file slice and further how different records with\nthe same record key within that file slice are merged consistently to produce the same deterministic results for snapshot queries, writers and table services. Specifically,\nthere are three merge modes supported as a table-level configuration, invoked in the following places."}),"\n",(0,o.jsxs)(r.ul,{children:["\n",(0,o.jsxs)(r.li,{children:["\n",(0,o.jsxs)(r.p,{children:[(0,o.jsx)(r.strong,{children:"(writing)"})," Combining multiple change records for the same record key while reading input data during writes. This is an optional optimization that\nreduces the number of records written to log files to improve query and write performance subsequently."]}),"\n"]}),"\n",(0,o.jsxs)(r.li,{children:["\n",(0,o.jsxs)(r.p,{children:[(0,o.jsx)(r.strong,{children:"(writing)"})," Merging final change record (partial/full update/delete) against existing record in storage for CoW tables."]}),"\n"]}),"\n",(0,o.jsxs)(r.li,{children:["\n",(0,o.jsxs)(r.p,{children:[(0,o.jsx)(r.strong,{children:"(compaction)"})," Compaction service merges all change records in log files against base files, respecting the merge mode."]}),"\n"]}),"\n",(0,o.jsxs)(r.li,{children:["\n",(0,o.jsxs)(r.p,{children:[(0,o.jsx)(r.strong,{children:"(query)"})," Merging change records in log files, after filtering/projections against base file for MoR table queries."]}),"\n"]}),"\n"]}),"\n",(0,o.jsx)(r.p,{children:"Note that the merge mode should not be altered once the table is created to avoid inconsistent behavior due to compaction producing\ndifferent merge results when switching between the modes."}),"\n",(0,o.jsx)(r.h3,{id:"commit_time_ordering",children:"COMMIT_TIME_ORDERING"}),"\n",(0,o.jsx)(r.p,{children:"Here, we expect the input records to arrive in strict order such that arrival order is same as their\ndelta commit order on the table. Merging simply picks the record belonging to the latest write as the merged result. In relational data mode speak,\nthis provides overwrite semantics aligned with serializable writes on the timeline."}),"\n",(0,o.jsx)("figure",{children:(0,o.jsx)("img",{className:"docimage",src:i(38526).A,alt:"upsert_path.png"})}),"\n",(0,o.jsx)(r.p,{children:"In the example above, the writer process consumes a database change log, expected to be in strict order of a logical sequence number (lsn)\nthat denotes the ordering of the writes in the upstream database."}),"\n",(0,o.jsx)(r.h3,{id:"event_time_ordering",children:"EVENT_TIME_ORDERING"}),"\n",(0,o.jsxs)(r.p,{children:["This is the default merge mode. While commit time ordering provides a well-understood standard behavior, it's hardly sufficient. The commit time is unrelated to the actual\nordering of data that a user may care about and strict ordering of input in complex distributed systems is difficult to achieve.\nWith event time ordering, the merging picks the record with the highest value on a user specified ",(0,o.jsx)(r.em,{children:(0,o.jsx)(r.strong,{children:"ordering or precombine field"})})," as the merged result."]}),"\n",(0,o.jsx)("figure",{children:(0,o.jsx)("img",{className:"docimage",src:i(57159).A,alt:"upsert_path.png"})}),"\n",(0,o.jsxs)(r.p,{children:['In the example above, two microservices product change records about orders at different times, that can arrive out-of-order. As color coded,\nthis can lead to application-level inconsistent states in the table if simply merged in commit time order like a cancelled order being re-created or\na paid order moved back to just created state expecting payment again. Event time ordering helps by ignoring older state changes that arrive late and\navoiding order status from "jumping back" in time. Combined with ',(0,o.jsx)(r.a,{href:"concurrency_control#non-blocking-concurrency-control-mode",children:"non-blocking concurrency control"}),",\nthis provides a very powerful way for processing such data streams efficiently and correctly."]}),"\n",(0,o.jsx)(r.h3,{id:"custom",children:"CUSTOM"}),"\n",(0,o.jsx)(r.p,{children:'In some cases, even more control and customization may be needed. Extending the same example above, the two microservices could be updating two different\nset of columns "order_info" and "payment_info", along with order state. The merge logic is then expected to not only resolve the correct status, but merge\norder_info from the record in created state, into the record in cancelled state that already has payment_info fields populated with reasons payment failed.\nSuch reconciliation provide a simple denormalized data model for downstream consumption where queries (for e.g. fraud detection) can simply filter fields\nacross order_info and payment_info without costly self-join on each access.'}),"\n",(0,o.jsxs)(r.p,{children:["Hudi allows authoring of cross-language custom record mergers on top of a standard record merger API, that supports full and partial merges. The java APIs\nare sketched below at a high-level. It simply takes older/newer records in engine native formats and produces a merged record or returns empty to skip them entirely (e.g. soft deletes).\nRecord merger is configured using a ",(0,o.jsx)(r.code,{children:"hoodie.write.record.merge.strategy.id"})," write config whose value is an uuid, that is taken by the writer to persist in the table config, and is expected to be returned by ",(0,o.jsx)(r.code,{children:"getMergingStrategy()"}),"\nmethod below. Using this mechanism, Hudi can automatically deduce the record merger to use for the table across different language/engine runtimes."]}),"\n",(0,o.jsx)(r.pre,{children:(0,o.jsx)(r.code,{className:"language-Java",children:"interface HoodieRecordMerger {\n\n Option> merge(HoodieRecord older, Schema oldSchema, \n HoodieRecord newer, Schema newSchema, \n TypedProperties props) {\n ...\n }\n\n Option> partialMerge(HoodieRecord older, Schema oldSchema, \n HoodieRecord newer, Schema newSchema, \n Schema readerSchema, TypedProperties props) {\n ...\n }\n \n HoodieRecordType getRecordType() {...}\n \n String getMergingStrategy(); {...}\n}\n"})}),"\n",(0,o.jsx)(r.h3,{id:"record-merge-configs",children:"Record Merge Configs"}),"\n",(0,o.jsx)(r.p,{children:"The record merge mode and optional record merge strategy ID and custom merge implementation classes can be specified using the below configs."}),"\n",(0,o.jsxs)(r.table,{children:[(0,o.jsx)(r.thead,{children:(0,o.jsxs)(r.tr,{children:[(0,o.jsx)(r.th,{children:"Config Name"}),(0,o.jsx)(r.th,{children:"Default"}),(0,o.jsx)(r.th,{children:"Description"})]})}),(0,o.jsxs)(r.tbody,{children:[(0,o.jsxs)(r.tr,{children:[(0,o.jsx)(r.td,{children:"hoodie.write.record.merge.mode"}),(0,o.jsx)(r.td,{children:"EVENT_TIME_ORDERING"}),(0,o.jsxs)(r.td,{children:["Determines the logic of merging different records with the same record key. Valid values: (1) ",(0,o.jsx)(r.code,{children:"COMMIT_TIME_ORDERING"}),": use commit time to merge records, i.e., the record from later commit overwrites the earlier record with the same key. (2) ",(0,o.jsx)(r.code,{children:"EVENT_TIME_ORDERING"})," (default): use event time as the ordering to merge records, i.e., the record with the larger event time overwrites the record with the smaller event time on the same key, regardless of commit time. The event time or preCombine field needs to be specified by the user. (3) ",(0,o.jsx)(r.code,{children:"CUSTOM"}),": use custom merging logic specified by the user.",(0,o.jsx)("br",{}),(0,o.jsx)(r.code,{children:"Config Param: RECORD_MERGE_MODE"}),(0,o.jsx)("br",{}),(0,o.jsx)(r.code,{children:"Since Version: 1.0.0"})]})]}),(0,o.jsxs)(r.tr,{children:[(0,o.jsx)(r.td,{children:"hoodie.write.record.merge.strategy.id"}),(0,o.jsx)(r.td,{children:"N/A (Optional)"}),(0,o.jsxs)(r.td,{children:["ID of record merge strategy. When you specify this config, you also need to specify ",(0,o.jsx)(r.code,{children:"hoodie.write.record.merge.custom.implementation.classes"}),". Hudi picks the ",(0,o.jsx)(r.code,{children:"HoodieRecordMerger"})," implementation class from the list of classes in ",(0,o.jsx)(r.code,{children:"hoodie.write.record.merge.custom.implementation.classes"})," that has the specified merge strategy ID.",(0,o.jsx)("br",{}),(0,o.jsx)(r.code,{children:"Config Param: RECORD_MERGE_STRATEGY_ID"}),(0,o.jsx)("br",{}),(0,o.jsx)(r.code,{children:"Since Version: 0.13.0"})]})]}),(0,o.jsxs)(r.tr,{children:[(0,o.jsx)(r.td,{children:"hoodie.write.record.merge.custom.implementation.classes"}),(0,o.jsx)(r.td,{children:"N/A (Optional)"}),(0,o.jsxs)(r.td,{children:["List of ",(0,o.jsx)(r.code,{children:"HoodieRecordMerger"})," implementations constituting Hudi's merging strategy based on the engine used. Hudi picks the ",(0,o.jsx)(r.code,{children:"HoodieRecordMerger"})," implementation class from this list based on the specified ",(0,o.jsx)(r.code,{children:"hoodie.write.record.merge.strategy.id"}),".",(0,o.jsx)("br",{}),(0,o.jsx)(r.code,{children:"Config Param: RECORD_MERGE_IMPL_CLASSES"}),(0,o.jsx)("br",{}),(0,o.jsx)(r.code,{children:"Since Version: 0.13.0"})]})]})]})]}),"\n",(0,o.jsx)(r.h3,{id:"record-payloads",children:"Record Payloads"}),"\n",(0,o.jsx)(r.admonition,{type:"caution",children:(0,o.jsx)(r.p,{children:"Going forward, we recommend users to migrate and use the record merger APIs and not write new payload implementations."})}),"\n",(0,o.jsx)(r.p,{children:"Record payload is an older abstraction/API for achieving similar record-level merge capabilities. While record payloads were very useful and popular,\nit had drawbacks like lower performance due to conversion of engine native record formats to Apache Avro for merging and lack of cross-language support.\nAs we shall see below, Hudi provides out-of-box support for different payloads for different use cases. Hudi implements fallback from\nrecord merger APIs to payload APIs internally, to provide backwards compatibility for existing payload implementations."}),"\n",(0,o.jsx)(r.h4,{id:"overwritewithlatestavropayload",children:"OverwriteWithLatestAvroPayload"}),"\n",(0,o.jsx)(r.pre,{children:(0,o.jsx)(r.code,{className:"language-scala",children:"hoodie.datasource.write.payload.class=org.apache.hudi.common.model.OverwriteWithLatestAvroPayload\n"})}),"\n",(0,o.jsxs)(r.p,{children:["This is the default record payload implementation. It picks the record with the greatest value (determined by calling\n",(0,o.jsx)(r.code,{children:".compareTo()"})," on the value of precombine key) to break ties and simply picks the latest record while merging. This gives\nlatest-write-wins style semantics."]}),"\n",(0,o.jsx)(r.h4,{id:"defaulthoodierecordpayload",children:"DefaultHoodieRecordPayload"}),"\n",(0,o.jsx)(r.pre,{children:(0,o.jsx)(r.code,{className:"language-scala",children:"hoodie.datasource.write.payload.class=org.apache.hudi.common.model.DefaultHoodieRecordPayload\n"})}),"\n",(0,o.jsxs)(r.p,{children:["While ",(0,o.jsx)(r.code,{children:"OverwriteWithLatestAvroPayload"})," precombines based on an ordering field and picks the latest record while merging,\n",(0,o.jsx)(r.code,{children:"DefaultHoodieRecordPayload"})," honors the ordering field for both precombinig and merging. Let's understand the difference with an example:"]}),"\n",(0,o.jsxs)(r.p,{children:["Let's say the ordering field is ",(0,o.jsx)(r.code,{children:"ts"})," and record key is ",(0,o.jsx)(r.code,{children:"id"})," and schema is:"]}),"\n",(0,o.jsx)(r.pre,{children:(0,o.jsx)(r.code,{children:'{\n [\n {"name":"id","type":"string"},\n {"name":"ts","type":"long"},\n {"name":"name","type":"string"},\n {"name":"price","type":"string"}\n ]\n}\n'})}),"\n",(0,o.jsx)(r.p,{children:"Current record in storage:"}),"\n",(0,o.jsx)(r.pre,{children:(0,o.jsx)(r.code,{children:" id ts name price\n 1 2 name_2 price_2\n"})}),"\n",(0,o.jsx)(r.p,{children:"Incoming record:"}),"\n",(0,o.jsx)(r.pre,{children:(0,o.jsx)(r.code,{children:" id ts name price\n 1 1 name_1 price_1\n"})}),"\n",(0,o.jsxs)(r.p,{children:["Result data after merging using ",(0,o.jsx)(r.code,{children:"OverwriteWithLatestAvroPayload"})," (latest-write-wins):"]}),"\n",(0,o.jsx)(r.pre,{children:(0,o.jsx)(r.code,{children:" id ts name price\n 1 1 name_1 price_1\n"})}),"\n",(0,o.jsxs)(r.p,{children:["Result data after merging using ",(0,o.jsx)(r.code,{children:"DefaultHoodieRecordPayload"})," (always honors ordering field):"]}),"\n",(0,o.jsx)(r.pre,{children:(0,o.jsx)(r.code,{children:" id ts name price\n 1 2 name_2 price_2\n"})}),"\n",(0,o.jsx)(r.h4,{id:"eventtimeavropayload",children:"EventTimeAvroPayload"}),"\n",(0,o.jsx)(r.pre,{children:(0,o.jsx)(r.code,{className:"language-scala",children:"hoodie.datasource.write.payload.class=org.apache.hudi.common.model.EventTimeAvroPayload\n"})}),"\n",(0,o.jsxs)(r.p,{children:["This is the default record payload for Flink based writing. Some use cases require merging records by event time and\nthus event time plays the role of an ordering field. This payload is particularly useful in the case of late-arriving data.\nFor such use cases, users need to set the ",(0,o.jsx)(r.a,{href:"/docs/configurations#RECORD_PAYLOAD",children:"payload event time field"})," configuration."]}),"\n",(0,o.jsx)(r.h4,{id:"overwritenondefaultswithlatestavropayload",children:"OverwriteNonDefaultsWithLatestAvroPayload"}),"\n",(0,o.jsx)(r.pre,{children:(0,o.jsx)(r.code,{className:"language-scala",children:"hoodie.datasource.write.payload.class=org.apache.hudi.common.model.OverwriteNonDefaultsWithLatestAvroPayload\n"})}),"\n",(0,o.jsxs)(r.p,{children:["This payload is quite similar to ",(0,o.jsx)(r.code,{children:"OverwriteWithLatestAvroPayload"})," with slight difference while merging records. For\nprecombining, just like ",(0,o.jsx)(r.code,{children:"OverwriteWithLatestAvroPayload"}),", it picks the latest record for a key, based on an ordering\nfield. While merging, it overwrites the existing record on storage only for the specified ",(0,o.jsx)(r.strong,{children:"fields that don't equal\ndefault value"})," for that field."]}),"\n",(0,o.jsx)(r.h4,{id:"partialupdateavropayload",children:"PartialUpdateAvroPayload"}),"\n",(0,o.jsx)(r.pre,{children:(0,o.jsx)(r.code,{className:"language-scala",children:"hoodie.datasource.write.payload.class=org.apache.hudi.common.model.PartialUpdateAvroPayload\n"})}),"\n",(0,o.jsxs)(r.p,{children:["This payload supports partial update. Typically, once the merge step resolves which record to pick, then the record on\nstorage is fully replaced by the resolved record. But, in some cases, the requirement is to update only certain fields\nand not replace the whole record. This is called partial update. ",(0,o.jsx)(r.code,{children:"PartialUpdateAvroPayload"})," provides out-of-box support\nfor such use cases. To illustrate the point, let us look at a simple example:"]}),"\n",(0,o.jsxs)(r.p,{children:["Let's say the ordering field is ",(0,o.jsx)(r.code,{children:"ts"})," and record key is ",(0,o.jsx)(r.code,{children:"id"})," and schema is:"]}),"\n",(0,o.jsx)(r.pre,{children:(0,o.jsx)(r.code,{children:'{\n [\n {"name":"id","type":"string"},\n {"name":"ts","type":"long"},\n {"name":"name","type":"string"},\n {"name":"price","type":"string"}\n ]\n}\n'})}),"\n",(0,o.jsx)(r.p,{children:"Current record in storage:"}),"\n",(0,o.jsx)(r.pre,{children:(0,o.jsx)(r.code,{children:" id ts name price\n 1 2 name_1 null\n"})}),"\n",(0,o.jsx)(r.p,{children:"Incoming record:"}),"\n",(0,o.jsx)(r.pre,{children:(0,o.jsx)(r.code,{children:" id ts name price\n 1 1 null price_1\n"})}),"\n",(0,o.jsxs)(r.p,{children:["Result data after merging using ",(0,o.jsx)(r.code,{children:"PartialUpdateAvroPayload"}),":"]}),"\n",(0,o.jsx)(r.pre,{children:(0,o.jsx)(r.code,{children:" id ts name price\n 1 2 name_1 price_1\n"})}),"\n",(0,o.jsx)(r.h4,{id:"configs",children:"Configs"}),"\n",(0,o.jsxs)(r.p,{children:["Payload class can be specified using the below configs. For more advanced configs refer ",(0,o.jsx)(r.a,{href:"https://hudi.apache.org/docs/configurations#RECORD_PAYLOAD",children:"here"})]}),"\n",(0,o.jsx)(r.p,{children:(0,o.jsx)(r.strong,{children:"Spark based configs:"})}),"\n",(0,o.jsxs)(r.table,{children:[(0,o.jsx)(r.thead,{children:(0,o.jsxs)(r.tr,{children:[(0,o.jsx)(r.th,{children:"Config Name"}),(0,o.jsx)(r.th,{children:"Default"}),(0,o.jsx)(r.th,{children:"Description"})]})}),(0,o.jsx)(r.tbody,{children:(0,o.jsxs)(r.tr,{children:[(0,o.jsx)(r.td,{children:"hoodie.datasource.write.payload.class"}),(0,o.jsx)(r.td,{children:"org.apache.hudi.common.model.OverwriteWithLatestAvroPayload (Optional)"}),(0,o.jsxs)(r.td,{children:["Payload class used. Override this, if you like to roll your own merge logic, when upserting/inserting. This will render any value set for PRECOMBINE_FIELD_OPT_VAL in-effective",(0,o.jsx)("br",{}),(0,o.jsx)("br",{}),(0,o.jsx)(r.code,{children:"Config Param: WRITE_PAYLOAD_CLASS_NAME"})]})]})})]}),"\n",(0,o.jsx)(r.p,{children:(0,o.jsx)(r.strong,{children:"Flink based configs:"})}),"\n",(0,o.jsxs)(r.table,{children:[(0,o.jsx)(r.thead,{children:(0,o.jsxs)(r.tr,{children:[(0,o.jsx)(r.th,{children:"Config Name"}),(0,o.jsx)(r.th,{children:"Default"}),(0,o.jsx)(r.th,{children:"Description"})]})}),(0,o.jsx)(r.tbody,{children:(0,o.jsxs)(r.tr,{children:[(0,o.jsx)(r.td,{children:"payload.class"}),(0,o.jsx)(r.td,{children:"org.apache.hudi.common.model.EventTimeAvroPayload (Optional)"}),(0,o.jsxs)(r.td,{children:["Payload class used. Override this, if you like to roll your own merge logic, when upserting/inserting. This will render any value set for the option in-effective",(0,o.jsx)("br",{}),(0,o.jsx)("br",{})," ",(0,o.jsx)(r.code,{children:"Config Param: PAYLOAD_CLASS_NAME"})]})]})})]}),"\n",(0,o.jsxs)(r.p,{children:["There are also quite a few other implementations. Developers may be interested in looking at the hierarchy of ",(0,o.jsx)(r.code,{children:"HoodieRecordPayload"})," interface. For\nexample, ",(0,o.jsx)(r.a,{href:"https://github.com/apache/hudi/blob/e76dd102bcaf8aec5a932e7277ccdbfd73ce1a32/hudi-common/src/main/java/org/apache/hudi/common/model/debezium/MySqlDebeziumAvroPayload.java",children:(0,o.jsx)(r.code,{children:"MySqlDebeziumAvroPayload"})})," and ",(0,o.jsx)(r.a,{href:"https://github.com/apache/hudi/blob/e76dd102bcaf8aec5a932e7277ccdbfd73ce1a32/hudi-common/src/main/java/org/apache/hudi/common/model/debezium/PostgresDebeziumAvroPayload.java",children:(0,o.jsx)(r.code,{children:"PostgresDebeziumAvroPayload"})})," provides support for seamlessly applying changes\ncaptured via Debezium for MySQL and PostgresDB. ",(0,o.jsx)(r.a,{href:"https://github.com/apache/hudi/blob/e76dd102bcaf8aec5a932e7277ccdbfd73ce1a32/hudi-common/src/main/java/org/apache/hudi/common/model/AWSDmsAvroPayload.java",children:(0,o.jsx)(r.code,{children:"AWSDmsAvroPayload"})})," provides support for applying changes captured via Amazon Database Migration Service onto S3.\nFor full configurations, go ",(0,o.jsx)(r.a,{href:"/docs/configurations#RECORD_PAYLOAD",children:"here"})," and please check out ",(0,o.jsx)(r.a,{href:"faq_writing_tables/#can-i-implement-my-own-logic-for-how-input-records-are-merged-with-record-on-storage",children:"this FAQ"})," if you want to implement your own custom payloads."]}),"\n",(0,o.jsx)(r.h2,{id:"related-resources",children:"Related Resources"}),"\n",(0,o.jsx)("h3",{children:"Blogs"}),"\n",(0,o.jsxs)(r.ul,{children:["\n",(0,o.jsx)(r.li,{children:(0,o.jsx)(r.a,{href:"https://medium.com/@simpsons/how-to-define-your-own-merge-logic-with-apache-hudi-622ee5ccab1e",children:"How to define your own merge logic with Apache Hudi"})}),"\n"]})]})}function h(e={}){const{wrapper:r}={...(0,t.R)(),...e.components};return r?(0,o.jsx)(r,{...e,children:(0,o.jsx)(l,{...e})}):l(e)}},38526:(e,r,i)=>{i.d(r,{A:()=>n});const n=i.p+"assets/images/commit-time-ordering-merge-mode-e9b6af3dcdb508053202617218f3ffe6.png"},57159:(e,r,i)=>{i.d(r,{A:()=>n});const n=i.p+"assets/images/event-time-ordering-merge-mode-c8164e035840388bf4290fa81ac6262a.png"},28453:(e,r,i)=>{i.d(r,{R:()=>a,x:()=>s});var n=i(96540);const o={},t=n.createContext(o);function a(e){const r=n.useContext(t);return n.useMemo((function(){return"function"==typeof e?e(r):{...r,...e}}),[r,e])}function s(e){let r;return r=e.disableParentContext?"function"==typeof e.components?e.components(o):e.components||o:a(e.components),n.createElement(t.Provider,{value:r},e.children)}}}]);
\ No newline at end of file
diff --git a/content/assets/js/491d56e0.8897656e.js b/content/assets/js/491d56e0.8897656e.js
deleted file mode 100644
index 6e93ff68aa610..0000000000000
--- a/content/assets/js/491d56e0.8897656e.js
+++ /dev/null
@@ -1 +0,0 @@
-"use strict";(self.webpackChunkhudi=self.webpackChunkhudi||[]).push([[72907],{48105:(e,r,i)=>{i.r(r),i.d(r,{assets:()=>d,contentTitle:()=>s,default:()=>h,frontMatter:()=>a,metadata:()=>n,toc:()=>c});const n=JSON.parse('{"id":"record_merger","title":"Record Mergers","description":"Hudi handles mutations to records and streaming data, as we briefly touched upon in timeline ordering section.","source":"@site/docs/record_merger.md","sourceDirName":".","slug":"/record_merger","permalink":"/docs/next/record_merger","draft":false,"unlisted":false,"editUrl":"https://github.com/apache/hudi/tree/asf-site/website/docs/record_merger.md","tags":[],"version":"current","frontMatter":{"title":"Record Mergers","keywords":["hudi","merge","upsert","precombine"],"toc":true,"toc_min_heading_level":2,"toc_max_heading_level":4},"sidebar":"docs","previous":{"title":"Key Generation","permalink":"/docs/next/key_generation"},"next":{"title":"Table Metadata","permalink":"/docs/next/metadata"}}');var o=i(74848),t=i(28453);const a={title:"Record Mergers",keywords:["hudi","merge","upsert","precombine"],toc:!0,toc_min_heading_level:2,toc_max_heading_level:4},s=void 0,d={},c=[{value:"COMMIT_TIME_ORDERING",id:"commit_time_ordering",level:3},{value:"EVENT_TIME_ORDERING",id:"event_time_ordering",level:3},{value:"CUSTOM",id:"custom",level:3},{value:"Record Merge Configs",id:"record-merge-configs",level:3},{value:"Record Payloads",id:"record-payloads",level:3},{value:"OverwriteWithLatestAvroPayload",id:"overwritewithlatestavropayload",level:4},{value:"DefaultHoodieRecordPayload",id:"defaulthoodierecordpayload",level:4},{value:"EventTimeAvroPayload",id:"eventtimeavropayload",level:4},{value:"OverwriteNonDefaultsWithLatestAvroPayload",id:"overwritenondefaultswithlatestavropayload",level:4},{value:"PartialUpdateAvroPayload",id:"partialupdateavropayload",level:4},{value:"Configs",id:"configs",level:4}];function l(e){const r={a:"a",admonition:"admonition",code:"code",em:"em",h3:"h3",h4:"h4",li:"li",p:"p",pre:"pre",strong:"strong",table:"table",tbody:"tbody",td:"td",th:"th",thead:"thead",tr:"tr",ul:"ul",...(0,t.R)(),...e.components};return(0,o.jsxs)(o.Fragment,{children:[(0,o.jsxs)(r.p,{children:["Hudi handles mutations to records and streaming data, as we briefly touched upon in ",(0,o.jsx)(r.a,{href:"timeline#ordering-of-actions",children:"timeline ordering"})," section.\nTo provide users full-fledged support for stream processing, Hudi goes all the way making the storage engine and the underlying storage format\nunderstand how to merge changes to the same record key, that may arrive even in different order at different times. With the rise of mobile applications\nand IoT, these scenarios have become the normal than an exception. For e.g. a social networking application uploading user events several hours after they happened,\nwhen the user connects to WiFi networks."]}),"\n",(0,o.jsx)(r.p,{children:"To achieve this, Hudi supports merge modes, which define how the base and log files are ordered in a file slice and further how different records with\nthe same record key within that file slice are merged consistently to produce the same deterministic results for snapshot queries, writers and table services. Specifically,\nthere are three merge modes supported as a table-level configuration, invoked in the following places."}),"\n",(0,o.jsxs)(r.ul,{children:["\n",(0,o.jsxs)(r.li,{children:["\n",(0,o.jsxs)(r.p,{children:[(0,o.jsx)(r.strong,{children:"(writing)"})," Combining multiple change records for the same record key while reading input data during writes. This is an optional optimization that\nreduces the number of records written to log files to improve query and write performance subsequently."]}),"\n"]}),"\n",(0,o.jsxs)(r.li,{children:["\n",(0,o.jsxs)(r.p,{children:[(0,o.jsx)(r.strong,{children:"(writing)"})," Merging final change record (partial/full update/delete) against existing record in storage for CoW tables."]}),"\n"]}),"\n",(0,o.jsxs)(r.li,{children:["\n",(0,o.jsxs)(r.p,{children:[(0,o.jsx)(r.strong,{children:"(compaction)"})," Compaction service merges all change records in log files against base files, respecting the merge mode."]}),"\n"]}),"\n",(0,o.jsxs)(r.li,{children:["\n",(0,o.jsxs)(r.p,{children:[(0,o.jsx)(r.strong,{children:"(query)"})," Merging change records in log files, after filtering/projections against base file for MoR table queries."]}),"\n"]}),"\n"]}),"\n",(0,o.jsx)(r.p,{children:"Note that the merge mode should not be altered once the table is created to avoid inconsistent behavior due to compaction producing\ndifferent merge results when switching between the modes."}),"\n",(0,o.jsx)(r.h3,{id:"commit_time_ordering",children:"COMMIT_TIME_ORDERING"}),"\n",(0,o.jsx)(r.p,{children:"Here, we expect the input records to arrive in strict order such that arrival order is same as their\ndelta commit order on the table. Merging simply picks the record belonging to the latest write as the merged result. In relational data mode speak,\nthis provides overwrite semantics aligned with serializable writes on the timeline."}),"\n",(0,o.jsx)("figure",{children:(0,o.jsx)("img",{className:"docimage",src:i(38526).A,alt:"upsert_path.png"})}),"\n",(0,o.jsx)(r.p,{children:"In the example above, the writer process consumes a database change log, expected to be in strict order of a logical sequence number (lsn)\nthat denotes the ordering of the writes in the upstream database."}),"\n",(0,o.jsx)(r.h3,{id:"event_time_ordering",children:"EVENT_TIME_ORDERING"}),"\n",(0,o.jsxs)(r.p,{children:["This is the default merge mode. While commit time ordering provides a well-understood standard behavior, it's hardly sufficient. The commit time is unrelated to the actual\nordering of data that a user may care about and strict ordering of input in complex distributed systems is difficult to achieve.\nWith event time ordering, the merging picks the record with the highest value on a user specified ",(0,o.jsx)(r.em,{children:(0,o.jsx)(r.strong,{children:"ordering or precombine field"})})," as the merged result."]}),"\n",(0,o.jsx)("figure",{children:(0,o.jsx)("img",{className:"docimage",src:i(57159).A,alt:"upsert_path.png"})}),"\n",(0,o.jsxs)(r.p,{children:['In the example above, two microservices product change records about orders at different times, that can arrive out-of-order. As color coded,\nthis can lead to application-level inconsistent states in the table if simply merged in commit time order like a cancelled order being re-created or\na paid order moved back to just created state expecting payment again. Event time ordering helps by ignoring older state changes that arrive late and\navoiding order status from "jumping back" in time. Combined with ',(0,o.jsx)(r.a,{href:"concurrency_control#non-blocking-concurrency-control-mode",children:"non-blocking concurrency control"}),",\nthis provides a very powerful way for processing such data streams efficiently and correctly."]}),"\n",(0,o.jsx)(r.h3,{id:"custom",children:"CUSTOM"}),"\n",(0,o.jsx)(r.p,{children:'In some cases, even more control and customization may be needed. Extending the same example above, the two microservices could be updating two different\nset of columns "order_info" and "payment_info", along with order state. The merge logic is then expected to not only resolve the correct status, but merge\norder_info from the record in created state, into the record in cancelled state that already has payment_info fields populated with reasons payment failed.\nSuch reconciliation provide a simple denormalized data model for downstream consumption where queries (for e.g. fraud detection) can simply filter fields\nacross order_info and payment_info without costly self-join on each access.'}),"\n",(0,o.jsxs)(r.p,{children:["Hudi allows authoring of cross-language custom record mergers on top of a standard record merger API, that supports full and partial merges. The java APIs\nare sketched below at a high-level. It simply takes older/newer records in engine native formats and produces a merged record or returns empty to skip them entirely (e.g. soft deletes).\nRecord merger is configured using a ",(0,o.jsx)(r.code,{children:"hoodie.write.record.merge.strategy.id"})," write config whose value is an uuid, that is taken by the writer to persist in the table config, and is expected to be returned by ",(0,o.jsx)(r.code,{children:"getMergingStrategy()"}),"\nmethod below. Using this mechanism, Hudi can automatically deduce the record merger to use for the table across different language/engine runtimes."]}),"\n",(0,o.jsx)(r.pre,{children:(0,o.jsx)(r.code,{className:"language-Java",children:"interface HoodieRecordMerger {\n\n Option> merge(HoodieRecord older, Schema oldSchema, \n HoodieRecord newer, Schema newSchema, \n TypedProperties props) {\n ...\n }\n\n Option> partialMerge(HoodieRecord older, Schema oldSchema, \n HoodieRecord newer, Schema newSchema, \n Schema readerSchema, TypedProperties props) {\n ...\n }\n \n HoodieRecordType getRecordType() {...}\n \n String getMergingStrategy(); {...}\n}\n"})}),"\n",(0,o.jsx)(r.h3,{id:"record-merge-configs",children:"Record Merge Configs"}),"\n",(0,o.jsx)(r.p,{children:"The record merge mode and optional record merge strategy ID and custom merge implementation classes can be specified using the below configs."}),"\n",(0,o.jsxs)(r.table,{children:[(0,o.jsx)(r.thead,{children:(0,o.jsxs)(r.tr,{children:[(0,o.jsx)(r.th,{children:"Config Name"}),(0,o.jsx)(r.th,{children:"Default"}),(0,o.jsx)(r.th,{children:"Description"})]})}),(0,o.jsxs)(r.tbody,{children:[(0,o.jsxs)(r.tr,{children:[(0,o.jsx)(r.td,{children:"hoodie.write.record.merge.mode"}),(0,o.jsx)(r.td,{children:"EVENT_TIME_ORDERING"}),(0,o.jsxs)(r.td,{children:["Determines the logic of merging different records with the same record key. Valid values: (1) ",(0,o.jsx)(r.code,{children:"COMMIT_TIME_ORDERING"}),": use commit time to merge records, i.e., the record from later commit overwrites the earlier record with the same key. (2) ",(0,o.jsx)(r.code,{children:"EVENT_TIME_ORDERING"})," (default): use event time as the ordering to merge records, i.e., the record with the larger event time overwrites the record with the smaller event time on the same key, regardless of commit time. The event time or preCombine field needs to be specified by the user. (3) ",(0,o.jsx)(r.code,{children:"CUSTOM"}),": use custom merging logic specified by the user.",(0,o.jsx)("br",{}),(0,o.jsx)(r.code,{children:"Config Param: RECORD_MERGE_MODE"}),(0,o.jsx)("br",{}),(0,o.jsx)(r.code,{children:"Since Version: 1.0.0"})]})]}),(0,o.jsxs)(r.tr,{children:[(0,o.jsx)(r.td,{children:"hoodie.write.record.merge.strategy.id"}),(0,o.jsx)(r.td,{children:"N/A (Optional)"}),(0,o.jsxs)(r.td,{children:["ID of record merge strategy. When you specify this config, you also need to specify ",(0,o.jsx)(r.code,{children:"hoodie.write.record.merge.custom.implementation.classes"}),". Hudi picks the ",(0,o.jsx)(r.code,{children:"HoodieRecordMerger"})," implementation class from the list of classes in ",(0,o.jsx)(r.code,{children:"hoodie.write.record.merge.custom.implementation.classes"})," that has the specified merge strategy ID.",(0,o.jsx)("br",{}),(0,o.jsx)(r.code,{children:"Config Param: RECORD_MERGE_STRATEGY_ID"}),(0,o.jsx)("br",{}),(0,o.jsx)(r.code,{children:"Since Version: 0.13.0"})]})]}),(0,o.jsxs)(r.tr,{children:[(0,o.jsx)(r.td,{children:"hoodie.write.record.merge.custom.implementation.classes"}),(0,o.jsx)(r.td,{children:"N/A (Optional)"}),(0,o.jsxs)(r.td,{children:["List of ",(0,o.jsx)(r.code,{children:"HoodieRecordMerger"})," implementations constituting Hudi's merging strategy based on the engine used. Hudi picks the ",(0,o.jsx)(r.code,{children:"HoodieRecordMerger"})," implementation class from this list based on the specified ",(0,o.jsx)(r.code,{children:"hoodie.write.record.merge.strategy.id"}),".",(0,o.jsx)("br",{}),(0,o.jsx)(r.code,{children:"Config Param: RECORD_MERGE_IMPL_CLASSES"}),(0,o.jsx)("br",{}),(0,o.jsx)(r.code,{children:"Since Version: 0.13.0"})]})]})]})]}),"\n",(0,o.jsx)(r.h3,{id:"record-payloads",children:"Record Payloads"}),"\n",(0,o.jsx)(r.admonition,{type:"caution",children:(0,o.jsx)(r.p,{children:"Going forward, we recommend users to migrate and use the record merger APIs and not write new payload implementations."})}),"\n",(0,o.jsx)(r.p,{children:"Record payload is an older abstraction/API for achieving similar record-level merge capabilities. While record payloads were very useful and popular,\nit had drawbacks like lower performance due to conversion of engine native record formats to Apache Avro for merging and lack of cross-language support.\nAs we shall see below, Hudi provides out-of-box support for different payloads for different use cases. Hudi implements fallback from\nrecord merger APIs to payload APIs internally, to provide backwards compatibility for existing payload implementations."}),"\n",(0,o.jsx)(r.h4,{id:"overwritewithlatestavropayload",children:"OverwriteWithLatestAvroPayload"}),"\n",(0,o.jsx)(r.pre,{children:(0,o.jsx)(r.code,{className:"language-scala",children:"hoodie.datasource.write.payload.class=org.apache.hudi.common.model.OverwriteWithLatestAvroPayload\n"})}),"\n",(0,o.jsxs)(r.p,{children:["This is the default record payload implementation. It picks the record with the greatest value (determined by calling\n",(0,o.jsx)(r.code,{children:".compareTo()"})," on the value of precombine key) to break ties and simply picks the latest record while merging. This gives\nlatest-write-wins style semantics."]}),"\n",(0,o.jsx)(r.h4,{id:"defaulthoodierecordpayload",children:"DefaultHoodieRecordPayload"}),"\n",(0,o.jsx)(r.pre,{children:(0,o.jsx)(r.code,{className:"language-scala",children:"hoodie.datasource.write.payload.class=org.apache.hudi.common.model.DefaultHoodieRecordPayload\n"})}),"\n",(0,o.jsxs)(r.p,{children:["While ",(0,o.jsx)(r.code,{children:"OverwriteWithLatestAvroPayload"})," precombines based on an ordering field and picks the latest record while merging,\n",(0,o.jsx)(r.code,{children:"DefaultHoodieRecordPayload"})," honors the ordering field for both precombinig and merging. Let's understand the difference with an example:"]}),"\n",(0,o.jsxs)(r.p,{children:["Let's say the ordering field is ",(0,o.jsx)(r.code,{children:"ts"})," and record key is ",(0,o.jsx)(r.code,{children:"id"})," and schema is:"]}),"\n",(0,o.jsx)(r.pre,{children:(0,o.jsx)(r.code,{children:'{\n [\n {"name":"id","type":"string"},\n {"name":"ts","type":"long"},\n {"name":"name","type":"string"},\n {"name":"price","type":"string"}\n ]\n}\n'})}),"\n",(0,o.jsx)(r.p,{children:"Current record in storage:"}),"\n",(0,o.jsx)(r.pre,{children:(0,o.jsx)(r.code,{children:" id ts name price\n 1 2 name_2 price_2\n"})}),"\n",(0,o.jsx)(r.p,{children:"Incoming record:"}),"\n",(0,o.jsx)(r.pre,{children:(0,o.jsx)(r.code,{children:" id ts name price\n 1 1 name_1 price_1\n"})}),"\n",(0,o.jsxs)(r.p,{children:["Result data after merging using ",(0,o.jsx)(r.code,{children:"OverwriteWithLatestAvroPayload"})," (latest-write-wins):"]}),"\n",(0,o.jsx)(r.pre,{children:(0,o.jsx)(r.code,{children:" id ts name price\n 1 1 name_1 price_1\n"})}),"\n",(0,o.jsxs)(r.p,{children:["Result data after merging using ",(0,o.jsx)(r.code,{children:"DefaultHoodieRecordPayload"})," (always honors ordering field):"]}),"\n",(0,o.jsx)(r.pre,{children:(0,o.jsx)(r.code,{children:" id ts name price\n 1 2 name_2 price_2\n"})}),"\n",(0,o.jsx)(r.h4,{id:"eventtimeavropayload",children:"EventTimeAvroPayload"}),"\n",(0,o.jsx)(r.pre,{children:(0,o.jsx)(r.code,{className:"language-scala",children:"hoodie.datasource.write.payload.class=org.apache.hudi.common.model.EventTimeAvroPayload\n"})}),"\n",(0,o.jsxs)(r.p,{children:["This is the default record payload for Flink based writing. Some use cases require merging records by event time and\nthus event time plays the role of an ordering field. This payload is particularly useful in the case of late-arriving data.\nFor such use cases, users need to set the ",(0,o.jsx)(r.a,{href:"/docs/configurations#RECORD_PAYLOAD",children:"payload event time field"})," configuration."]}),"\n",(0,o.jsx)(r.h4,{id:"overwritenondefaultswithlatestavropayload",children:"OverwriteNonDefaultsWithLatestAvroPayload"}),"\n",(0,o.jsx)(r.pre,{children:(0,o.jsx)(r.code,{className:"language-scala",children:"hoodie.datasource.write.payload.class=org.apache.hudi.common.model.OverwriteNonDefaultsWithLatestAvroPayload\n"})}),"\n",(0,o.jsxs)(r.p,{children:["This payload is quite similar to ",(0,o.jsx)(r.code,{children:"OverwriteWithLatestAvroPayload"})," with slight difference while merging records. For\nprecombining, just like ",(0,o.jsx)(r.code,{children:"OverwriteWithLatestAvroPayload"}),", it picks the latest record for a key, based on an ordering\nfield. While merging, it overwrites the existing record on storage only for the specified ",(0,o.jsx)(r.strong,{children:"fields that don't equal\ndefault value"})," for that field."]}),"\n",(0,o.jsx)(r.h4,{id:"partialupdateavropayload",children:"PartialUpdateAvroPayload"}),"\n",(0,o.jsx)(r.pre,{children:(0,o.jsx)(r.code,{className:"language-scala",children:"hoodie.datasource.write.payload.class=org.apache.hudi.common.model.PartialUpdateAvroPayload\n"})}),"\n",(0,o.jsxs)(r.p,{children:["This payload supports partial update. Typically, once the merge step resolves which record to pick, then the record on\nstorage is fully replaced by the resolved record. But, in some cases, the requirement is to update only certain fields\nand not replace the whole record. This is called partial update. ",(0,o.jsx)(r.code,{children:"PartialUpdateAvroPayload"})," provides out-of-box support\nfor such use cases. To illustrate the point, let us look at a simple example:"]}),"\n",(0,o.jsxs)(r.p,{children:["Let's say the ordering field is ",(0,o.jsx)(r.code,{children:"ts"})," and record key is ",(0,o.jsx)(r.code,{children:"id"})," and schema is:"]}),"\n",(0,o.jsx)(r.pre,{children:(0,o.jsx)(r.code,{children:'{\n [\n {"name":"id","type":"string"},\n {"name":"ts","type":"long"},\n {"name":"name","type":"string"},\n {"name":"price","type":"string"}\n ]\n}\n'})}),"\n",(0,o.jsx)(r.p,{children:"Current record in storage:"}),"\n",(0,o.jsx)(r.pre,{children:(0,o.jsx)(r.code,{children:" id ts name price\n 1 2 name_1 null\n"})}),"\n",(0,o.jsx)(r.p,{children:"Incoming record:"}),"\n",(0,o.jsx)(r.pre,{children:(0,o.jsx)(r.code,{children:" id ts name price\n 1 1 null price_1\n"})}),"\n",(0,o.jsxs)(r.p,{children:["Result data after merging using ",(0,o.jsx)(r.code,{children:"PartialUpdateAvroPayload"}),":"]}),"\n",(0,o.jsx)(r.pre,{children:(0,o.jsx)(r.code,{children:" id ts name price\n 1 2 name_1 price_1\n"})}),"\n",(0,o.jsx)(r.h4,{id:"configs",children:"Configs"}),"\n",(0,o.jsxs)(r.p,{children:["Payload class can be specified using the below configs. For more advanced configs refer ",(0,o.jsx)(r.a,{href:"https://hudi.apache.org/docs/configurations#RECORD_PAYLOAD",children:"here"})]}),"\n",(0,o.jsx)(r.p,{children:(0,o.jsx)(r.strong,{children:"Spark based configs:"})}),"\n",(0,o.jsxs)(r.table,{children:[(0,o.jsx)(r.thead,{children:(0,o.jsxs)(r.tr,{children:[(0,o.jsx)(r.th,{children:"Config Name"}),(0,o.jsx)(r.th,{children:"Default"}),(0,o.jsx)(r.th,{children:"Description"})]})}),(0,o.jsx)(r.tbody,{children:(0,o.jsxs)(r.tr,{children:[(0,o.jsx)(r.td,{children:"hoodie.datasource.write.payload.class"}),(0,o.jsx)(r.td,{children:"org.apache.hudi.common.model.OverwriteWithLatestAvroPayload (Optional)"}),(0,o.jsxs)(r.td,{children:["Payload class used. Override this, if you like to roll your own merge logic, when upserting/inserting. This will render any value set for PRECOMBINE_FIELD_OPT_VAL in-effective",(0,o.jsx)("br",{}),(0,o.jsx)("br",{}),(0,o.jsx)(r.code,{children:"Config Param: WRITE_PAYLOAD_CLASS_NAME"})]})]})})]}),"\n",(0,o.jsx)(r.p,{children:(0,o.jsx)(r.strong,{children:"Flink based configs:"})}),"\n",(0,o.jsxs)(r.table,{children:[(0,o.jsx)(r.thead,{children:(0,o.jsxs)(r.tr,{children:[(0,o.jsx)(r.th,{children:"Config Name"}),(0,o.jsx)(r.th,{children:"Default"}),(0,o.jsx)(r.th,{children:"Description"})]})}),(0,o.jsx)(r.tbody,{children:(0,o.jsxs)(r.tr,{children:[(0,o.jsx)(r.td,{children:"payload.class"}),(0,o.jsx)(r.td,{children:"org.apache.hudi.common.model.EventTimeAvroPayload (Optional)"}),(0,o.jsxs)(r.td,{children:["Payload class used. Override this, if you like to roll your own merge logic, when upserting/inserting. This will render any value set for the option in-effective",(0,o.jsx)("br",{}),(0,o.jsx)("br",{})," ",(0,o.jsx)(r.code,{children:"Config Param: PAYLOAD_CLASS_NAME"})]})]})})]}),"\n",(0,o.jsxs)(r.p,{children:["There are also quite a few other implementations. Developers may be interested in looking at the hierarchy of ",(0,o.jsx)(r.code,{children:"HoodieRecordPayload"})," interface. For\nexample, ",(0,o.jsx)(r.a,{href:"https://github.com/apache/hudi/blob/e76dd102bcaf8aec5a932e7277ccdbfd73ce1a32/hudi-common/src/main/java/org/apache/hudi/common/model/debezium/MySqlDebeziumAvroPayload.java",children:(0,o.jsx)(r.code,{children:"MySqlDebeziumAvroPayload"})})," and ",(0,o.jsx)(r.a,{href:"https://github.com/apache/hudi/blob/e76dd102bcaf8aec5a932e7277ccdbfd73ce1a32/hudi-common/src/main/java/org/apache/hudi/common/model/debezium/PostgresDebeziumAvroPayload.java",children:(0,o.jsx)(r.code,{children:"PostgresDebeziumAvroPayload"})})," provides support for seamlessly applying changes\ncaptured via Debezium for MySQL and PostgresDB. ",(0,o.jsx)(r.a,{href:"https://github.com/apache/hudi/blob/e76dd102bcaf8aec5a932e7277ccdbfd73ce1a32/hudi-common/src/main/java/org/apache/hudi/common/model/AWSDmsAvroPayload.java",children:(0,o.jsx)(r.code,{children:"AWSDmsAvroPayload"})})," provides support for applying changes captured via Amazon Database Migration Service onto S3.\nFor full configurations, go ",(0,o.jsx)(r.a,{href:"/docs/configurations#RECORD_PAYLOAD",children:"here"})," and please check out ",(0,o.jsx)(r.a,{href:"faq_writing_tables/#can-i-implement-my-own-logic-for-how-input-records-are-merged-with-record-on-storage",children:"this FAQ"})," if you want to implement your own custom payloads."]})]})}function h(e={}){const{wrapper:r}={...(0,t.R)(),...e.components};return r?(0,o.jsx)(r,{...e,children:(0,o.jsx)(l,{...e})}):l(e)}},38526:(e,r,i)=>{i.d(r,{A:()=>n});const n=i.p+"assets/images/commit-time-ordering-merge-mode-e9b6af3dcdb508053202617218f3ffe6.png"},57159:(e,r,i)=>{i.d(r,{A:()=>n});const n=i.p+"assets/images/event-time-ordering-merge-mode-c8164e035840388bf4290fa81ac6262a.png"},28453:(e,r,i)=>{i.d(r,{R:()=>a,x:()=>s});var n=i(96540);const o={},t=n.createContext(o);function a(e){const r=n.useContext(t);return n.useMemo((function(){return"function"==typeof e?e(r):{...r,...e}}),[r,e])}function s(e){let r;return r=e.disableParentContext?"function"==typeof e.components?e.components(o):e.components||o:a(e.components),n.createElement(t.Provider,{value:r},e.children)}}}]);
\ No newline at end of file
diff --git a/content/assets/js/5733876a.1a5c8931.js b/content/assets/js/5733876a.1a5c8931.js
new file mode 100644
index 0000000000000..49da946c17646
--- /dev/null
+++ b/content/assets/js/5733876a.1a5c8931.js
@@ -0,0 +1 @@
+"use strict";(self.webpackChunkhudi=self.webpackChunkhudi||[]).push([[85973],{91855:(e,t,n)=>{n.r(t),n.d(t,{assets:()=>c,contentTitle:()=>u,default:()=>h,frontMatter:()=>l,metadata:()=>r,toc:()=>d});const r=JSON.parse('{"id":"writing_tables_streaming_writes","title":"Streaming Writes","description":"Spark Streaming","source":"@site/docs/writing_tables_streaming_writes.md","sourceDirName":".","slug":"/writing_tables_streaming_writes","permalink":"/docs/next/writing_tables_streaming_writes","draft":false,"unlisted":false,"editUrl":"https://github.com/apache/hudi/tree/asf-site/website/docs/writing_tables_streaming_writes.md","tags":[],"version":"current","frontMatter":{"title":"Streaming Writes","keywords":["hudi","spark","flink","streaming","processing"],"last_modified_at":"2024-03-13T19:59:57.000Z"},"sidebar":"docs","previous":{"title":"Batch Writes","permalink":"/docs/next/writing_data"},"next":{"title":"SQL Queries","permalink":"/docs/next/sql_queries"}}');var a=n(74848),i=n(28453),s=n(11470),o=n(19365);const l={title:"Streaming Writes",keywords:["hudi","spark","flink","streaming","processing"],last_modified_at:new Date("2024-03-13T19:59:57.000Z")},u=void 0,c={},d=[{value:"Spark Streaming",id:"spark-streaming",level:2},{value:"Related Resources",id:"related-resources",level:2}];function p(e){const t={a:"a",code:"code",h2:"h2",li:"li",p:"p",pre:"pre",ul:"ul",...(0,i.R)(),...e.components};return(0,a.jsxs)(a.Fragment,{children:[(0,a.jsx)(t.h2,{id:"spark-streaming",children:"Spark Streaming"}),"\n",(0,a.jsx)(t.p,{children:"You can write Hudi tables using spark's structured streaming."}),"\n",(0,a.jsxs)(s.A,{groupId:"programming-language",defaultValue:"python",values:[{label:"Scala",value:"scala"},{label:"Python",value:"python"}],children:[(0,a.jsx)(o.A,{value:"scala",children:(0,a.jsx)(t.pre,{children:(0,a.jsx)(t.code,{className:"language-scala",children:'// spark-shell\n// prepare to stream write to new table\nimport org.apache.spark.sql.streaming.Trigger\n\nval streamingTableName = "hudi_trips_cow_streaming"\nval baseStreamingPath = "file:///tmp/hudi_trips_cow_streaming"\nval checkpointLocation = "file:///tmp/checkpoints/hudi_trips_cow_streaming"\n\n// create streaming df\nval df = spark.readStream.\n format("hudi").\n load(basePath)\n\n// write stream to new hudi table\ndf.writeStream.format("hudi").\n options(getQuickstartWriteConfigs).\n option("hoodie.datasource.write.precombine.field", "ts").\n option("hoodie.datasource.write.recordkey.field", "uuid").\n option("hoodie.datasource.write.partitionpath.field", "partitionpath").\n option("hoodie.table.name", streamingTableName).\n outputMode("append").\n option("path", baseStreamingPath).\n option("checkpointLocation", checkpointLocation).\n trigger(Trigger.Once()).\n start()\n\n'})})}),(0,a.jsx)(o.A,{value:"python",children:(0,a.jsx)(t.pre,{children:(0,a.jsx)(t.code,{className:"language-python",children:"# pyspark\n# prepare to stream write to new table\nstreamingTableName = \"hudi_trips_cow_streaming\"\nbaseStreamingPath = \"file:///tmp/hudi_trips_cow_streaming\"\ncheckpointLocation = \"file:///tmp/checkpoints/hudi_trips_cow_streaming\"\n\nhudi_streaming_options = {\n 'hoodie.table.name': streamingTableName,\n 'hoodie.datasource.write.recordkey.field': 'uuid',\n 'hoodie.datasource.write.partitionpath.field': 'partitionpath',\n 'hoodie.datasource.write.table.name': streamingTableName,\n 'hoodie.datasource.write.operation': 'upsert',\n 'hoodie.datasource.write.precombine.field': 'ts',\n 'hoodie.upsert.shuffle.parallelism': 2,\n 'hoodie.insert.shuffle.parallelism': 2\n}\n\n# create streaming df\ndf = spark.readStream \n .format(\"hudi\") \n .load(basePath)\n\n# write stream to new hudi table\ndf.writeStream.format(\"hudi\") \n .options(**hudi_streaming_options) \n .outputMode(\"append\") \n .option(\"path\", baseStreamingPath) \n .option(\"checkpointLocation\", checkpointLocation) \n .trigger(once=True) \n .start()\n\n"})})})]}),"\n",(0,a.jsx)(t.h2,{id:"related-resources",children:"Related Resources"}),"\n",(0,a.jsx)("h3",{children:"Blogs"}),"\n",(0,a.jsxs)(t.ul,{children:["\n",(0,a.jsx)(t.li,{children:(0,a.jsx)(t.a,{href:"https://www.onehouse.ai/blog/intro-to-hudi-and-flink",children:"An Introduction to the Hudi and Flink Integration"})}),"\n",(0,a.jsx)(t.li,{children:(0,a.jsx)(t.a,{href:"https://medium.com/@simpsons/bulk-insert-sort-modes-with-apache-hudi-c781e77841bc",children:"Bulk Insert Sort Modes with Apache Hudi"})}),"\n"]})]})}function h(e={}){const{wrapper:t}={...(0,i.R)(),...e.components};return t?(0,a.jsx)(t,{...e,children:(0,a.jsx)(p,{...e})}):p(e)}},19365:(e,t,n)=>{n.d(t,{A:()=>s});n(96540);var r=n(34164);const a={tabItem:"tabItem_Ymn6"};var i=n(74848);function s(e){let{children:t,hidden:n,className:s}=e;return(0,i.jsx)("div",{role:"tabpanel",className:(0,r.A)(a.tabItem,s),hidden:n,children:t})}},11470:(e,t,n)=>{n.d(t,{A:()=>x});var r=n(96540),a=n(34164),i=n(23104),s=n(56347),o=n(205),l=n(57485),u=n(31682),c=n(70679);function d(e){return r.Children.toArray(e).filter((e=>"\n"!==e)).map((e=>{if(!e||(0,r.isValidElement)(e)&&function(e){const{props:t}=e;return!!t&&"object"==typeof t&&"value"in t}(e))return e;throw new Error(`Docusaurus error: Bad child <${"string"==typeof e.type?e.type:e.type.name}>: all children of the component should be , and every should have a unique "value" prop.`)}))?.filter(Boolean)??[]}function p(e){const{values:t,children:n}=e;return(0,r.useMemo)((()=>{const e=t??function(e){return d(e).map((e=>{let{props:{value:t,label:n,attributes:r,default:a}}=e;return{value:t,label:n,attributes:r,default:a}}))}(n);return function(e){const t=(0,u.XI)(e,((e,t)=>e.value===t.value));if(t.length>0)throw new Error(`Docusaurus error: Duplicate values "${t.map((e=>e.value)).join(", ")}" found in . Every value needs to be unique.`)}(e),e}),[t,n])}function h(e){let{value:t,tabValues:n}=e;return n.some((e=>e.value===t))}function m(e){let{queryString:t=!1,groupId:n}=e;const a=(0,s.W6)(),i=function(e){let{queryString:t=!1,groupId:n}=e;if("string"==typeof t)return t;if(!1===t)return null;if(!0===t&&!n)throw new Error('Docusaurus error: The component groupId prop is required if queryString=true, because this value is used as the search param name. You can also provide an explicit value such as queryString="my-search-param".');return n??null}({queryString:t,groupId:n});return[(0,l.aZ)(i),(0,r.useCallback)((e=>{if(!i)return;const t=new URLSearchParams(a.location.search);t.set(i,e),a.replace({...a.location,search:t.toString()})}),[i,a])]}function f(e){const{defaultValue:t,queryString:n=!1,groupId:a}=e,i=p(e),[s,l]=(0,r.useState)((()=>function(e){let{defaultValue:t,tabValues:n}=e;if(0===n.length)throw new Error("Docusaurus error: the component requires at least one children component");if(t){if(!h({value:t,tabValues:n}))throw new Error(`Docusaurus error: The has a defaultValue "${t}" but none of its children has the corresponding value. Available values are: ${n.map((e=>e.value)).join(", ")}. If you intend to show no default tab, use defaultValue={null} instead.`);return t}const r=n.find((e=>e.default))??n[0];if(!r)throw new Error("Unexpected error: 0 tabValues");return r.value}({defaultValue:t,tabValues:i}))),[u,d]=m({queryString:n,groupId:a}),[f,g]=function(e){let{groupId:t}=e;const n=function(e){return e?`docusaurus.tab.${e}`:null}(t),[a,i]=(0,c.Dv)(n);return[a,(0,r.useCallback)((e=>{n&&i.set(e)}),[n,i])]}({groupId:a}),b=(()=>{const e=u??f;return h({value:e,tabValues:i})?e:null})();(0,o.A)((()=>{b&&l(b)}),[b]);return{selectedValue:s,selectValue:(0,r.useCallback)((e=>{if(!h({value:e,tabValues:i}))throw new Error(`Can't select invalid tab value=${e}`);l(e),d(e),g(e)}),[d,g,i]),tabValues:i}}var g=n(92303);const b={tabList:"tabList__CuJ",tabItem:"tabItem_LNqP"};var w=n(74848);function v(e){let{className:t,block:n,selectedValue:r,selectValue:s,tabValues:o}=e;const l=[],{blockElementScrollPositionUntilNextRender:u}=(0,i.a_)(),c=e=>{const t=e.currentTarget,n=l.indexOf(t),a=o[n].value;a!==r&&(u(t),s(a))},d=e=>{let t=null;switch(e.key){case"Enter":c(e);break;case"ArrowRight":{const n=l.indexOf(e.currentTarget)+1;t=l[n]??l[0];break}case"ArrowLeft":{const n=l.indexOf(e.currentTarget)-1;t=l[n]??l[l.length-1];break}}t?.focus()};return(0,w.jsx)("ul",{role:"tablist","aria-orientation":"horizontal",className:(0,a.A)("tabs",{"tabs--block":n},t),children:o.map((e=>{let{value:t,label:n,attributes:i}=e;return(0,w.jsx)("li",{role:"tab",tabIndex:r===t?0:-1,"aria-selected":r===t,ref:e=>l.push(e),onKeyDown:d,onClick:c,...i,className:(0,a.A)("tabs__item",b.tabItem,i?.className,{"tabs__item--active":r===t}),children:n??t},t)}))})}function _(e){let{lazy:t,children:n,selectedValue:i}=e;const s=(Array.isArray(n)?n:[n]).filter(Boolean);if(t){const e=s.find((e=>e.props.value===i));return e?(0,r.cloneElement)(e,{className:(0,a.A)("margin-top--md",e.props.className)}):null}return(0,w.jsx)("div",{className:"margin-top--md",children:s.map(((e,t)=>(0,r.cloneElement)(e,{key:t,hidden:e.props.value!==i})))})}function k(e){const t=f(e);return(0,w.jsxs)("div",{className:(0,a.A)("tabs-container",b.tabList),children:[(0,w.jsx)(v,{...t,...e}),(0,w.jsx)(_,{...t,...e})]})}function x(e){const t=(0,g.A)();return(0,w.jsx)(k,{...e,children:d(e.children)},String(t))}},28453:(e,t,n)=>{n.d(t,{R:()=>s,x:()=>o});var r=n(96540);const a={},i=r.createContext(a);function s(e){const t=r.useContext(i);return r.useMemo((function(){return"function"==typeof e?e(t):{...t,...e}}),[t,e])}function o(e){let t;return t=e.disableParentContext?"function"==typeof e.components?e.components(a):e.components||a:s(e.components),r.createElement(i.Provider,{value:t},e.children)}}}]);
\ No newline at end of file
diff --git a/content/assets/js/5733876a.b1b5df36.js b/content/assets/js/5733876a.b1b5df36.js
deleted file mode 100644
index ded2a92ad9685..0000000000000
--- a/content/assets/js/5733876a.b1b5df36.js
+++ /dev/null
@@ -1 +0,0 @@
-"use strict";(self.webpackChunkhudi=self.webpackChunkhudi||[]).push([[85973],{91855:(e,t,a)=>{a.r(t),a.d(t,{assets:()=>c,contentTitle:()=>u,default:()=>m,frontMatter:()=>l,metadata:()=>r,toc:()=>d});const r=JSON.parse('{"id":"writing_tables_streaming_writes","title":"Streaming Writes","description":"Spark Streaming","source":"@site/docs/writing_tables_streaming_writes.md","sourceDirName":".","slug":"/writing_tables_streaming_writes","permalink":"/docs/next/writing_tables_streaming_writes","draft":false,"unlisted":false,"editUrl":"https://github.com/apache/hudi/tree/asf-site/website/docs/writing_tables_streaming_writes.md","tags":[],"version":"current","frontMatter":{"title":"Streaming Writes","keywords":["hudi","spark","flink","streaming","processing"],"last_modified_at":"2024-03-13T19:59:57.000Z"},"sidebar":"docs","previous":{"title":"Batch Writes","permalink":"/docs/next/writing_data"},"next":{"title":"SQL Queries","permalink":"/docs/next/sql_queries"}}');var n=a(74848),i=a(28453),s=a(11470),o=a(19365);const l={title:"Streaming Writes",keywords:["hudi","spark","flink","streaming","processing"],last_modified_at:new Date("2024-03-13T19:59:57.000Z")},u=void 0,c={},d=[{value:"Spark Streaming",id:"spark-streaming",level:2}];function p(e){const t={code:"code",h2:"h2",p:"p",pre:"pre",...(0,i.R)(),...e.components};return(0,n.jsxs)(n.Fragment,{children:[(0,n.jsx)(t.h2,{id:"spark-streaming",children:"Spark Streaming"}),"\n",(0,n.jsx)(t.p,{children:"You can write Hudi tables using spark's structured streaming."}),"\n",(0,n.jsxs)(s.A,{groupId:"programming-language",defaultValue:"python",values:[{label:"Scala",value:"scala"},{label:"Python",value:"python"}],children:[(0,n.jsx)(o.A,{value:"scala",children:(0,n.jsx)(t.pre,{children:(0,n.jsx)(t.code,{className:"language-scala",children:'// spark-shell\n// prepare to stream write to new table\nimport org.apache.spark.sql.streaming.Trigger\n\nval streamingTableName = "hudi_trips_cow_streaming"\nval baseStreamingPath = "file:///tmp/hudi_trips_cow_streaming"\nval checkpointLocation = "file:///tmp/checkpoints/hudi_trips_cow_streaming"\n\n// create streaming df\nval df = spark.readStream.\n format("hudi").\n load(basePath)\n\n// write stream to new hudi table\ndf.writeStream.format("hudi").\n options(getQuickstartWriteConfigs).\n option("hoodie.datasource.write.precombine.field", "ts").\n option("hoodie.datasource.write.recordkey.field", "uuid").\n option("hoodie.datasource.write.partitionpath.field", "partitionpath").\n option("hoodie.table.name", streamingTableName).\n outputMode("append").\n option("path", baseStreamingPath).\n option("checkpointLocation", checkpointLocation).\n trigger(Trigger.Once()).\n start()\n\n'})})}),(0,n.jsx)(o.A,{value:"python",children:(0,n.jsx)(t.pre,{children:(0,n.jsx)(t.code,{className:"language-python",children:"# pyspark\n# prepare to stream write to new table\nstreamingTableName = \"hudi_trips_cow_streaming\"\nbaseStreamingPath = \"file:///tmp/hudi_trips_cow_streaming\"\ncheckpointLocation = \"file:///tmp/checkpoints/hudi_trips_cow_streaming\"\n\nhudi_streaming_options = {\n 'hoodie.table.name': streamingTableName,\n 'hoodie.datasource.write.recordkey.field': 'uuid',\n 'hoodie.datasource.write.partitionpath.field': 'partitionpath',\n 'hoodie.datasource.write.table.name': streamingTableName,\n 'hoodie.datasource.write.operation': 'upsert',\n 'hoodie.datasource.write.precombine.field': 'ts',\n 'hoodie.upsert.shuffle.parallelism': 2,\n 'hoodie.insert.shuffle.parallelism': 2\n}\n\n# create streaming df\ndf = spark.readStream \n .format(\"hudi\") \n .load(basePath)\n\n# write stream to new hudi table\ndf.writeStream.format(\"hudi\") \n .options(**hudi_streaming_options) \n .outputMode(\"append\") \n .option(\"path\", baseStreamingPath) \n .option(\"checkpointLocation\", checkpointLocation) \n .trigger(once=True) \n .start()\n\n"})})})]})]})}function m(e={}){const{wrapper:t}={...(0,i.R)(),...e.components};return t?(0,n.jsx)(t,{...e,children:(0,n.jsx)(p,{...e})}):p(e)}},19365:(e,t,a)=>{a.d(t,{A:()=>s});a(96540);var r=a(34164);const n={tabItem:"tabItem_Ymn6"};var i=a(74848);function s(e){let{children:t,hidden:a,className:s}=e;return(0,i.jsx)("div",{role:"tabpanel",className:(0,r.A)(n.tabItem,s),hidden:a,children:t})}},11470:(e,t,a)=>{a.d(t,{A:()=>y});var r=a(96540),n=a(34164),i=a(23104),s=a(56347),o=a(205),l=a(57485),u=a(31682),c=a(70679);function d(e){return r.Children.toArray(e).filter((e=>"\n"!==e)).map((e=>{if(!e||(0,r.isValidElement)(e)&&function(e){const{props:t}=e;return!!t&&"object"==typeof t&&"value"in t}(e))return e;throw new Error(`Docusaurus error: Bad child <${"string"==typeof e.type?e.type:e.type.name}>: all children of the component should be , and every should have a unique "value" prop.`)}))?.filter(Boolean)??[]}function p(e){const{values:t,children:a}=e;return(0,r.useMemo)((()=>{const e=t??function(e){return d(e).map((e=>{let{props:{value:t,label:a,attributes:r,default:n}}=e;return{value:t,label:a,attributes:r,default:n}}))}(a);return function(e){const t=(0,u.XI)(e,((e,t)=>e.value===t.value));if(t.length>0)throw new Error(`Docusaurus error: Duplicate values "${t.map((e=>e.value)).join(", ")}" found in . Every value needs to be unique.`)}(e),e}),[t,a])}function m(e){let{value:t,tabValues:a}=e;return a.some((e=>e.value===t))}function h(e){let{queryString:t=!1,groupId:a}=e;const n=(0,s.W6)(),i=function(e){let{queryString:t=!1,groupId:a}=e;if("string"==typeof t)return t;if(!1===t)return null;if(!0===t&&!a)throw new Error('Docusaurus error: The component groupId prop is required if queryString=true, because this value is used as the search param name. You can also provide an explicit value such as queryString="my-search-param".');return a??null}({queryString:t,groupId:a});return[(0,l.aZ)(i),(0,r.useCallback)((e=>{if(!i)return;const t=new URLSearchParams(n.location.search);t.set(i,e),n.replace({...n.location,search:t.toString()})}),[i,n])]}function f(e){const{defaultValue:t,queryString:a=!1,groupId:n}=e,i=p(e),[s,l]=(0,r.useState)((()=>function(e){let{defaultValue:t,tabValues:a}=e;if(0===a.length)throw new Error("Docusaurus error: the component requires at least one children component");if(t){if(!m({value:t,tabValues:a}))throw new Error(`Docusaurus error: The has a defaultValue "${t}" but none of its children has the corresponding value. Available values are: ${a.map((e=>e.value)).join(", ")}. If you intend to show no default tab, use defaultValue={null} instead.`);return t}const r=a.find((e=>e.default))??a[0];if(!r)throw new Error("Unexpected error: 0 tabValues");return r.value}({defaultValue:t,tabValues:i}))),[u,d]=h({queryString:a,groupId:n}),[f,g]=function(e){let{groupId:t}=e;const a=function(e){return e?`docusaurus.tab.${e}`:null}(t),[n,i]=(0,c.Dv)(a);return[n,(0,r.useCallback)((e=>{a&&i.set(e)}),[a,i])]}({groupId:n}),b=(()=>{const e=u??f;return m({value:e,tabValues:i})?e:null})();(0,o.A)((()=>{b&&l(b)}),[b]);return{selectedValue:s,selectValue:(0,r.useCallback)((e=>{if(!m({value:e,tabValues:i}))throw new Error(`Can't select invalid tab value=${e}`);l(e),d(e),g(e)}),[d,g,i]),tabValues:i}}var g=a(92303);const b={tabList:"tabList__CuJ",tabItem:"tabItem_LNqP"};var w=a(74848);function v(e){let{className:t,block:a,selectedValue:r,selectValue:s,tabValues:o}=e;const l=[],{blockElementScrollPositionUntilNextRender:u}=(0,i.a_)(),c=e=>{const t=e.currentTarget,a=l.indexOf(t),n=o[a].value;n!==r&&(u(t),s(n))},d=e=>{let t=null;switch(e.key){case"Enter":c(e);break;case"ArrowRight":{const a=l.indexOf(e.currentTarget)+1;t=l[a]??l[0];break}case"ArrowLeft":{const a=l.indexOf(e.currentTarget)-1;t=l[a]??l[l.length-1];break}}t?.focus()};return(0,w.jsx)("ul",{role:"tablist","aria-orientation":"horizontal",className:(0,n.A)("tabs",{"tabs--block":a},t),children:o.map((e=>{let{value:t,label:a,attributes:i}=e;return(0,w.jsx)("li",{role:"tab",tabIndex:r===t?0:-1,"aria-selected":r===t,ref:e=>l.push(e),onKeyDown:d,onClick:c,...i,className:(0,n.A)("tabs__item",b.tabItem,i?.className,{"tabs__item--active":r===t}),children:a??t},t)}))})}function _(e){let{lazy:t,children:a,selectedValue:i}=e;const s=(Array.isArray(a)?a:[a]).filter(Boolean);if(t){const e=s.find((e=>e.props.value===i));return e?(0,r.cloneElement)(e,{className:(0,n.A)("margin-top--md",e.props.className)}):null}return(0,w.jsx)("div",{className:"margin-top--md",children:s.map(((e,t)=>(0,r.cloneElement)(e,{key:t,hidden:e.props.value!==i})))})}function k(e){const t=f(e);return(0,w.jsxs)("div",{className:(0,n.A)("tabs-container",b.tabList),children:[(0,w.jsx)(v,{...t,...e}),(0,w.jsx)(_,{...t,...e})]})}function y(e){const t=(0,g.A)();return(0,w.jsx)(k,{...e,children:d(e.children)},String(t))}},28453:(e,t,a)=>{a.d(t,{R:()=>s,x:()=>o});var r=a(96540);const n={},i=r.createContext(n);function s(e){const t=r.useContext(i);return r.useMemo((function(){return"function"==typeof e?e(t):{...t,...e}}),[t,e])}function o(e){let t;return t=e.disableParentContext?"function"==typeof e.components?e.components(n):e.components||n:s(e.components),r.createElement(i.Provider,{value:t},e.children)}}}]);
\ No newline at end of file
diff --git a/content/assets/js/5e1c183f.30c1c2e5.js b/content/assets/js/5e1c183f.30c1c2e5.js
new file mode 100644
index 0000000000000..52984c435f5fb
--- /dev/null
+++ b/content/assets/js/5e1c183f.30c1c2e5.js
@@ -0,0 +1 @@
+"use strict";(self.webpackChunkhudi=self.webpackChunkhudi||[]).push([[70253],{34886:(e,r,t)=>{t.r(r),t.d(r,{assets:()=>l,contentTitle:()=>o,default:()=>h,frontMatter:()=>s,metadata:()=>i,toc:()=>c});const i=JSON.parse('{"id":"markers","title":"Marker Mechanism","description":"Purpose of Markers","source":"@site/docs/markers.md","sourceDirName":".","slug":"/markers","permalink":"/docs/next/markers","draft":false,"unlisted":false,"editUrl":"https://github.com/apache/hudi/tree/asf-site/website/docs/markers.md","tags":[],"version":"current","frontMatter":{"title":"Marker Mechanism","toc":true},"sidebar":"docs","previous":{"title":"Auto Rollbacks","permalink":"/docs/next/rollbacks"},"next":{"title":"File Sizing","permalink":"/docs/next/file_sizing"}}');var a=t(74848),n=t(28453);const s={title:"Marker Mechanism",toc:!0},o=void 0,l={},c=[{value:"Purpose of Markers",id:"purpose-of-markers",level:2},{value:"Marker structure",id:"marker-structure",level:2},{value:"Marker Writing Options",id:"marker-writing-options",level:2},{value:"Direct Write Markers",id:"direct-write-markers",level:3},{value:"Timeline Server Markers (Default)",id:"timeline-server-markers-default",level:3},{value:"Marker Configuration Parameters",id:"marker-configuration-parameters",level:2},{value:"Related Resources",id:"related-resources",level:2}];function d(e){const r={a:"a",code:"code",h2:"h2",h3:"h3",img:"img",li:"li",p:"p",strong:"strong",table:"table",tbody:"tbody",td:"td",th:"th",thead:"thead",tr:"tr",ul:"ul",...(0,n.R)(),...e.components};return(0,a.jsxs)(a.Fragment,{children:[(0,a.jsx)(r.h2,{id:"purpose-of-markers",children:"Purpose of Markers"}),"\n",(0,a.jsx)(r.p,{children:"A write operation can fail before it completes, leaving partial or corrupt data files on storage. Markers are used to track\nand cleanup any partial or failed write operations. As a write operation begins, a marker is created indicating\nthat a file write is in progress. When the write commit succeeds, the marker is deleted. If a write operation fails part\nway through, a marker is left behind which indicates that the file is incomplete. Two important operations that use markers include:"}),"\n",(0,a.jsxs)(r.ul,{children:["\n",(0,a.jsxs)(r.li,{children:[(0,a.jsx)(r.strong,{children:"Removing duplicate/partial data files"}),":","\n",(0,a.jsxs)(r.ul,{children:["\n",(0,a.jsx)(r.li,{children:"In Spark, the Hudi write client delegates the data file writing to multiple executors. One executor can fail the task,\nleaving partial data files written, and Spark retries the task in this case until it succeeds."}),"\n",(0,a.jsx)(r.li,{children:"When speculative execution is enabled, there can also be multiple successful attempts at writing out the same data\ninto different files, only one of which is finally handed to the Spark driver process for committing.\nThe markers help efficiently identify the partial data files written, which contain duplicate data compared to the data\nfiles written by the successful trial later, and these duplicate data files are cleaned up when the commit is finalized."}),"\n"]}),"\n"]}),"\n",(0,a.jsxs)(r.li,{children:[(0,a.jsx)(r.strong,{children:"Rolling back failed commits"}),": If a write operation fails, the next write client will roll back the failed commit before proceeding with the new write. The rollback is done with the help of markers to identify the data files written as part of the failed commit."]}),"\n"]}),"\n",(0,a.jsx)(r.p,{children:"If we did not have markers to track the per-commit data files, we would have to list all files in the file system,\ncorrelate that with the files seen in timeline and then delete the ones that belong to partial write failures.\nAs you could imagine, this would be very costly in a very large installation of a datalake."}),"\n",(0,a.jsx)(r.h2,{id:"marker-structure",children:"Marker structure"}),"\n",(0,a.jsxs)(r.p,{children:["Each marker entry is composed of three parts, the data file name,\nthe marker extension (",(0,a.jsx)(r.code,{children:".marker"}),"), and the I/O operation created the file (",(0,a.jsx)(r.code,{children:"CREATE"})," - inserts, ",(0,a.jsx)(r.code,{children:"MERGE"})," - updates/deletes,\nor ",(0,a.jsx)(r.code,{children:"APPEND"})," - either). For example, the marker ",(0,a.jsx)(r.code,{children:"91245ce3-bb82-4f9f-969e-343364159174-0_140-579-0_20210820173605.parquet.marker.CREATE"})," indicates\nthat the corresponding data file is ",(0,a.jsx)(r.code,{children:"91245ce3-bb82-4f9f-969e-343364159174-0_140-579-0_20210820173605.parquet"})," and the I/O type is ",(0,a.jsx)(r.code,{children:"CREATE"}),"."]}),"\n",(0,a.jsx)(r.h2,{id:"marker-writing-options",children:"Marker Writing Options"}),"\n",(0,a.jsx)(r.p,{children:"There are two ways to write Markers:"}),"\n",(0,a.jsxs)(r.ul,{children:["\n",(0,a.jsx)(r.li,{children:"Directly writing markers to storage, which is a legacy configuration."}),"\n",(0,a.jsx)(r.li,{children:"Writing markers to the Timeline Server which batches marker requests before writing them to storage (Default). This option improves write performance of large files as described below."}),"\n"]}),"\n",(0,a.jsx)(r.h3,{id:"direct-write-markers",children:"Direct Write Markers"}),"\n",(0,a.jsxs)(r.p,{children:["Directly writing to storage creates a new marker file corresponding to each data file, with the marker filename as described above.\nThe marker file does not have any content, i.e., empty. Each marker file is written to storage in the same directory\nhierarchy, i.e., commit instant and partition path, under a temporary folder ",(0,a.jsx)(r.code,{children:".hoodie/.temp"})," under the base path of the Hudi table.\nFor example, the figure below shows one example of the marker files created and the corresponding data files when writing\ndata to the Hudi table. When getting or deleting all the marker file paths, the mechanism first lists all the paths\nunder the temporary folder, ",(0,a.jsx)(r.code,{children:".hoodie/.temp/"}),", and then does the operation."]}),"\n",(0,a.jsx)(r.p,{children:(0,a.jsx)(r.img,{alt:"An example of marker and data files in direct marker file mechanism",src:t(36995).A+"",width:"3440",height:"1444"})}),"\n",(0,a.jsxs)(r.p,{children:["While it's much efficient over scanning the entire table for uncommitted data files, as the number of data files to write\nincreases, so does the number of marker files to create. For large writes which need to write significant number of data\nfiles, e.g., 10K or more, this can create performance bottlenecks for cloud storage such as AWS S3. In AWS S3, each\nfile create and delete call triggers an HTTP request and there is ",(0,a.jsx)(r.a,{href:"https://docs.aws.amazon.com/AmazonS3/latest/userguide/optimizing-performance.html",children:"rate-limiting"}),"\non how many requests can be processed per second per prefix in a bucket. When the number of data files to write concurrently\nand the number of marker files is huge, the marker file operations could take up non-trivial time during the write operation,\nsometimes on the order of a few minutes or more."]}),"\n",(0,a.jsx)(r.h3,{id:"timeline-server-markers-default",children:"Timeline Server Markers (Default)"}),"\n",(0,a.jsx)(r.p,{children:"To address the performance bottleneck due to rate-limiting of AWS S3 explained above, we introduce a new marker mechanism\nleveraging the timeline server, which optimizes the marker-related latency for storage with non-trivial file I/O latency.\nIn the diagram below you can see the timeline-server-based marker mechanism delegates the marker creation and other marker-related\noperations from individual executors to the timeline server for centralized processing. The timeline server batches the\nmarker creation requests and writes the markers to a bounded set of files in the file system at configurable batch intervals (default 50ms).\nIn this way, the number of actual file operations and latency related to markers can be significantly reduced even with\na huge number of data files, leading to improved performance of large writes."}),"\n",(0,a.jsx)(r.p,{children:(0,a.jsx)(r.img,{alt:"Timeline-server-based marker mechanism",src:t(56542).A+"",width:"1200",height:"432"})}),"\n",(0,a.jsx)(r.p,{children:"Each marker creation request is handled asynchronously in the Javalin timeline server and queued before processing.\nFor every batch interval, the timeline server pulls the pending marker creation requests from the queue and\nwrites all markers to the next file in a round robin fashion. Inside the timeline server, such batch processing is\nmulti-threaded, designed and implemented to guarantee consistency and correctness. Both the batch interval and the batch\nconcurrency can be configured through the write options."}),"\n",(0,a.jsx)(r.p,{children:(0,a.jsx)(r.img,{alt:"Batched processing of marker creation requests",src:t(78660).A+"",width:"3184",height:"1168"})}),"\n",(0,a.jsx)(r.p,{children:"Note that the worker thread always checks whether the marker has already been created by comparing the marker name from\nthe request with the memory copy of all markers maintained at the timeline server. The underlying files storing the\nmarkers are only read upon the first marker request (lazy loading). The responses of requests are only sent back once the\nnew markers are flushed to the files, so that in the case of the timeline server failure, the timeline server can recover\nthe already created markers. These ensure consistency between storage and the in-memory copy, and improve the performance\nof processing marker requests."}),"\n",(0,a.jsxs)(r.p,{children:[(0,a.jsx)(r.strong,{children:"NOTE:"})," Timeline based markers are not yet supported for HDFS, however, users may barely notice performance challenges\nwith direct markers because the file system metadata is efficiently cached in memory and doesn't face the same rate-limiting as S3."]}),"\n",(0,a.jsx)(r.h2,{id:"marker-configuration-parameters",children:"Marker Configuration Parameters"}),"\n",(0,a.jsxs)(r.table,{children:[(0,a.jsx)(r.thead,{children:(0,a.jsxs)(r.tr,{children:[(0,a.jsx)(r.th,{children:"Property Name"}),(0,a.jsx)(r.th,{children:"Default"}),(0,a.jsx)(r.th,{style:{textAlign:"center"},children:"Meaning"})]})}),(0,a.jsxs)(r.tbody,{children:[(0,a.jsxs)(r.tr,{children:[(0,a.jsx)(r.td,{children:(0,a.jsx)(r.code,{children:"hoodie.write.markers.type"})}),(0,a.jsx)(r.td,{children:"timeline_server_based"}),(0,a.jsxs)(r.td,{style:{textAlign:"center"},children:["Marker type to use. Two modes are supported: (1) ",(0,a.jsx)(r.code,{children:"direct"}),": individual marker file corresponding to each data file is directly created by the executor; (2) ",(0,a.jsx)(r.code,{children:"timeline_server_based"}),": marker operations are all handled at the timeline service which serves as a proxy. New marker entries are batch processed and stored in a limited number of underlying files for efficiency."]})]}),(0,a.jsxs)(r.tr,{children:[(0,a.jsx)(r.td,{children:(0,a.jsx)(r.code,{children:"hoodie.markers.timeline_server_based.batch.num_threads"})}),(0,a.jsx)(r.td,{children:"20"}),(0,a.jsx)(r.td,{style:{textAlign:"center"},children:"Number of threads to use for batch processing marker creation requests at the timeline server."})]}),(0,a.jsxs)(r.tr,{children:[(0,a.jsx)(r.td,{children:(0,a.jsx)(r.code,{children:"hoodie.markers.timeline_server_based.batch.interval_ms"})}),(0,a.jsx)(r.td,{children:"50"}),(0,a.jsx)(r.td,{style:{textAlign:"center"},children:"The batch interval in milliseconds for marker creation batch processing."})]})]})]}),"\n",(0,a.jsx)(r.h2,{id:"related-resources",children:"Related Resources"}),"\n",(0,a.jsxs)(r.p,{children:[(0,a.jsx)("h3",{children:"Blogs"}),"\n",(0,a.jsx)(r.a,{href:"https://medium.com/@simpsons/timeline-server-in-apache-hudi-b5be25f85e47",children:"Timeline Server in Apache Hudi"})]})]})}function h(e={}){const{wrapper:r}={...(0,n.R)(),...e.components};return r?(0,a.jsx)(r,{...e,children:(0,a.jsx)(d,{...e})}):d(e)}},78660:(e,r,t)=>{t.d(r,{A:()=>i});const i=t.p+"assets/images/batched-marker-creation-e8455c544f3b11ceed810b663df59f7f.png"},36995:(e,r,t)=>{t.d(r,{A:()=>i});const i=t.p+"assets/images/direct-marker-file-mechanism-b97b82f80430598f1d6a9b96521bb1a0.png"},56542:(e,r,t)=>{t.d(r,{A:()=>i});const i=t.p+"assets/images/timeline-server-based-marker-mechanism-11d616800a7a241382c8a4ed647515a6.png"},28453:(e,r,t)=>{t.d(r,{R:()=>s,x:()=>o});var i=t(96540);const a={},n=i.createContext(a);function s(e){const r=i.useContext(n);return i.useMemo((function(){return"function"==typeof e?e(r):{...r,...e}}),[r,e])}function o(e){let r;return r=e.disableParentContext?"function"==typeof e.components?e.components(a):e.components||a:s(e.components),i.createElement(n.Provider,{value:r},e.children)}}}]);
\ No newline at end of file
diff --git a/content/assets/js/5e1c183f.f5ded396.js b/content/assets/js/5e1c183f.f5ded396.js
deleted file mode 100644
index d80b164e236b3..0000000000000
--- a/content/assets/js/5e1c183f.f5ded396.js
+++ /dev/null
@@ -1 +0,0 @@
-"use strict";(self.webpackChunkhudi=self.webpackChunkhudi||[]).push([[70253],{34886:(e,r,t)=>{t.r(r),t.d(r,{assets:()=>l,contentTitle:()=>o,default:()=>h,frontMatter:()=>s,metadata:()=>i,toc:()=>c});const i=JSON.parse('{"id":"markers","title":"Marker Mechanism","description":"Purpose of Markers","source":"@site/docs/markers.md","sourceDirName":".","slug":"/markers","permalink":"/docs/next/markers","draft":false,"unlisted":false,"editUrl":"https://github.com/apache/hudi/tree/asf-site/website/docs/markers.md","tags":[],"version":"current","frontMatter":{"title":"Marker Mechanism","toc":true},"sidebar":"docs","previous":{"title":"Auto Rollbacks","permalink":"/docs/next/rollbacks"},"next":{"title":"File Sizing","permalink":"/docs/next/file_sizing"}}');var a=t(74848),n=t(28453);const s={title:"Marker Mechanism",toc:!0},o=void 0,l={},c=[{value:"Purpose of Markers",id:"purpose-of-markers",level:2},{value:"Marker structure",id:"marker-structure",level:2},{value:"Marker Writing Options",id:"marker-writing-options",level:2},{value:"Direct Write Markers",id:"direct-write-markers",level:3},{value:"Timeline Server Markers (Default)",id:"timeline-server-markers-default",level:3},{value:"Marker Configuration Parameters",id:"marker-configuration-parameters",level:2}];function d(e){const r={a:"a",code:"code",h2:"h2",h3:"h3",img:"img",li:"li",p:"p",strong:"strong",table:"table",tbody:"tbody",td:"td",th:"th",thead:"thead",tr:"tr",ul:"ul",...(0,n.R)(),...e.components};return(0,a.jsxs)(a.Fragment,{children:[(0,a.jsx)(r.h2,{id:"purpose-of-markers",children:"Purpose of Markers"}),"\n",(0,a.jsx)(r.p,{children:"A write operation can fail before it completes, leaving partial or corrupt data files on storage. Markers are used to track\nand cleanup any partial or failed write operations. As a write operation begins, a marker is created indicating\nthat a file write is in progress. When the write commit succeeds, the marker is deleted. If a write operation fails part\nway through, a marker is left behind which indicates that the file is incomplete. Two important operations that use markers include:"}),"\n",(0,a.jsxs)(r.ul,{children:["\n",(0,a.jsxs)(r.li,{children:[(0,a.jsx)(r.strong,{children:"Removing duplicate/partial data files"}),":","\n",(0,a.jsxs)(r.ul,{children:["\n",(0,a.jsx)(r.li,{children:"In Spark, the Hudi write client delegates the data file writing to multiple executors. One executor can fail the task,\nleaving partial data files written, and Spark retries the task in this case until it succeeds."}),"\n",(0,a.jsx)(r.li,{children:"When speculative execution is enabled, there can also be multiple successful attempts at writing out the same data\ninto different files, only one of which is finally handed to the Spark driver process for committing.\nThe markers help efficiently identify the partial data files written, which contain duplicate data compared to the data\nfiles written by the successful trial later, and these duplicate data files are cleaned up when the commit is finalized."}),"\n"]}),"\n"]}),"\n",(0,a.jsxs)(r.li,{children:[(0,a.jsx)(r.strong,{children:"Rolling back failed commits"}),": If a write operation fails, the next write client will roll back the failed commit before proceeding with the new write. The rollback is done with the help of markers to identify the data files written as part of the failed commit."]}),"\n"]}),"\n",(0,a.jsx)(r.p,{children:"If we did not have markers to track the per-commit data files, we would have to list all files in the file system,\ncorrelate that with the files seen in timeline and then delete the ones that belong to partial write failures.\nAs you could imagine, this would be very costly in a very large installation of a datalake."}),"\n",(0,a.jsx)(r.h2,{id:"marker-structure",children:"Marker structure"}),"\n",(0,a.jsxs)(r.p,{children:["Each marker entry is composed of three parts, the data file name,\nthe marker extension (",(0,a.jsx)(r.code,{children:".marker"}),"), and the I/O operation created the file (",(0,a.jsx)(r.code,{children:"CREATE"})," - inserts, ",(0,a.jsx)(r.code,{children:"MERGE"})," - updates/deletes,\nor ",(0,a.jsx)(r.code,{children:"APPEND"})," - either). For example, the marker ",(0,a.jsx)(r.code,{children:"91245ce3-bb82-4f9f-969e-343364159174-0_140-579-0_20210820173605.parquet.marker.CREATE"})," indicates\nthat the corresponding data file is ",(0,a.jsx)(r.code,{children:"91245ce3-bb82-4f9f-969e-343364159174-0_140-579-0_20210820173605.parquet"})," and the I/O type is ",(0,a.jsx)(r.code,{children:"CREATE"}),"."]}),"\n",(0,a.jsx)(r.h2,{id:"marker-writing-options",children:"Marker Writing Options"}),"\n",(0,a.jsx)(r.p,{children:"There are two ways to write Markers:"}),"\n",(0,a.jsxs)(r.ul,{children:["\n",(0,a.jsx)(r.li,{children:"Directly writing markers to storage, which is a legacy configuration."}),"\n",(0,a.jsx)(r.li,{children:"Writing markers to the Timeline Server which batches marker requests before writing them to storage (Default). This option improves write performance of large files as described below."}),"\n"]}),"\n",(0,a.jsx)(r.h3,{id:"direct-write-markers",children:"Direct Write Markers"}),"\n",(0,a.jsxs)(r.p,{children:["Directly writing to storage creates a new marker file corresponding to each data file, with the marker filename as described above.\nThe marker file does not have any content, i.e., empty. Each marker file is written to storage in the same directory\nhierarchy, i.e., commit instant and partition path, under a temporary folder ",(0,a.jsx)(r.code,{children:".hoodie/.temp"})," under the base path of the Hudi table.\nFor example, the figure below shows one example of the marker files created and the corresponding data files when writing\ndata to the Hudi table. When getting or deleting all the marker file paths, the mechanism first lists all the paths\nunder the temporary folder, ",(0,a.jsx)(r.code,{children:".hoodie/.temp/"}),", and then does the operation."]}),"\n",(0,a.jsx)(r.p,{children:(0,a.jsx)(r.img,{alt:"An example of marker and data files in direct marker file mechanism",src:t(36995).A+"",width:"3440",height:"1444"})}),"\n",(0,a.jsxs)(r.p,{children:["While it's much efficient over scanning the entire table for uncommitted data files, as the number of data files to write\nincreases, so does the number of marker files to create. For large writes which need to write significant number of data\nfiles, e.g., 10K or more, this can create performance bottlenecks for cloud storage such as AWS S3. In AWS S3, each\nfile create and delete call triggers an HTTP request and there is ",(0,a.jsx)(r.a,{href:"https://docs.aws.amazon.com/AmazonS3/latest/userguide/optimizing-performance.html",children:"rate-limiting"}),"\non how many requests can be processed per second per prefix in a bucket. When the number of data files to write concurrently\nand the number of marker files is huge, the marker file operations could take up non-trivial time during the write operation,\nsometimes on the order of a few minutes or more."]}),"\n",(0,a.jsx)(r.h3,{id:"timeline-server-markers-default",children:"Timeline Server Markers (Default)"}),"\n",(0,a.jsx)(r.p,{children:"To address the performance bottleneck due to rate-limiting of AWS S3 explained above, we introduce a new marker mechanism\nleveraging the timeline server, which optimizes the marker-related latency for storage with non-trivial file I/O latency.\nIn the diagram below you can see the timeline-server-based marker mechanism delegates the marker creation and other marker-related\noperations from individual executors to the timeline server for centralized processing. The timeline server batches the\nmarker creation requests and writes the markers to a bounded set of files in the file system at configurable batch intervals (default 50ms).\nIn this way, the number of actual file operations and latency related to markers can be significantly reduced even with\na huge number of data files, leading to improved performance of large writes."}),"\n",(0,a.jsx)(r.p,{children:(0,a.jsx)(r.img,{alt:"Timeline-server-based marker mechanism",src:t(56542).A+"",width:"1200",height:"432"})}),"\n",(0,a.jsx)(r.p,{children:"Each marker creation request is handled asynchronously in the Javalin timeline server and queued before processing.\nFor every batch interval, the timeline server pulls the pending marker creation requests from the queue and\nwrites all markers to the next file in a round robin fashion. Inside the timeline server, such batch processing is\nmulti-threaded, designed and implemented to guarantee consistency and correctness. Both the batch interval and the batch\nconcurrency can be configured through the write options."}),"\n",(0,a.jsx)(r.p,{children:(0,a.jsx)(r.img,{alt:"Batched processing of marker creation requests",src:t(78660).A+"",width:"3184",height:"1168"})}),"\n",(0,a.jsx)(r.p,{children:"Note that the worker thread always checks whether the marker has already been created by comparing the marker name from\nthe request with the memory copy of all markers maintained at the timeline server. The underlying files storing the\nmarkers are only read upon the first marker request (lazy loading). The responses of requests are only sent back once the\nnew markers are flushed to the files, so that in the case of the timeline server failure, the timeline server can recover\nthe already created markers. These ensure consistency between storage and the in-memory copy, and improve the performance\nof processing marker requests."}),"\n",(0,a.jsxs)(r.p,{children:[(0,a.jsx)(r.strong,{children:"NOTE:"})," Timeline based markers are not yet supported for HDFS, however, users may barely notice performance challenges\nwith direct markers because the file system metadata is efficiently cached in memory and doesn't face the same rate-limiting as S3."]}),"\n",(0,a.jsx)(r.h2,{id:"marker-configuration-parameters",children:"Marker Configuration Parameters"}),"\n",(0,a.jsxs)(r.table,{children:[(0,a.jsx)(r.thead,{children:(0,a.jsxs)(r.tr,{children:[(0,a.jsx)(r.th,{children:"Property Name"}),(0,a.jsx)(r.th,{children:"Default"}),(0,a.jsx)(r.th,{style:{textAlign:"center"},children:"Meaning"})]})}),(0,a.jsxs)(r.tbody,{children:[(0,a.jsxs)(r.tr,{children:[(0,a.jsx)(r.td,{children:(0,a.jsx)(r.code,{children:"hoodie.write.markers.type"})}),(0,a.jsx)(r.td,{children:"timeline_server_based"}),(0,a.jsxs)(r.td,{style:{textAlign:"center"},children:["Marker type to use. Two modes are supported: (1) ",(0,a.jsx)(r.code,{children:"direct"}),": individual marker file corresponding to each data file is directly created by the executor; (2) ",(0,a.jsx)(r.code,{children:"timeline_server_based"}),": marker operations are all handled at the timeline service which serves as a proxy. New marker entries are batch processed and stored in a limited number of underlying files for efficiency."]})]}),(0,a.jsxs)(r.tr,{children:[(0,a.jsx)(r.td,{children:(0,a.jsx)(r.code,{children:"hoodie.markers.timeline_server_based.batch.num_threads"})}),(0,a.jsx)(r.td,{children:"20"}),(0,a.jsx)(r.td,{style:{textAlign:"center"},children:"Number of threads to use for batch processing marker creation requests at the timeline server."})]}),(0,a.jsxs)(r.tr,{children:[(0,a.jsx)(r.td,{children:(0,a.jsx)(r.code,{children:"hoodie.markers.timeline_server_based.batch.interval_ms"})}),(0,a.jsx)(r.td,{children:"50"}),(0,a.jsx)(r.td,{style:{textAlign:"center"},children:"The batch interval in milliseconds for marker creation batch processing."})]})]})]})]})}function h(e={}){const{wrapper:r}={...(0,n.R)(),...e.components};return r?(0,a.jsx)(r,{...e,children:(0,a.jsx)(d,{...e})}):d(e)}},78660:(e,r,t)=>{t.d(r,{A:()=>i});const i=t.p+"assets/images/batched-marker-creation-e8455c544f3b11ceed810b663df59f7f.png"},36995:(e,r,t)=>{t.d(r,{A:()=>i});const i=t.p+"assets/images/direct-marker-file-mechanism-b97b82f80430598f1d6a9b96521bb1a0.png"},56542:(e,r,t)=>{t.d(r,{A:()=>i});const i=t.p+"assets/images/timeline-server-based-marker-mechanism-11d616800a7a241382c8a4ed647515a6.png"},28453:(e,r,t)=>{t.d(r,{R:()=>s,x:()=>o});var i=t(96540);const a={},n=i.createContext(a);function s(e){const r=i.useContext(n);return i.useMemo((function(){return"function"==typeof e?e(r):{...r,...e}}),[r,e])}function o(e){let r;return r=e.disableParentContext?"function"==typeof e.components?e.components(a):e.components||a:s(e.components),i.createElement(n.Provider,{value:r},e.children)}}}]);
\ No newline at end of file
diff --git a/content/assets/js/782eae4d.0b1c0a0d.js b/content/assets/js/782eae4d.bd5caa0e.js
similarity index 95%
rename from content/assets/js/782eae4d.0b1c0a0d.js
rename to content/assets/js/782eae4d.bd5caa0e.js
index 833e3b02b5548..10cb27d2e7262 100644
--- a/content/assets/js/782eae4d.0b1c0a0d.js
+++ b/content/assets/js/782eae4d.bd5caa0e.js
@@ -1 +1 @@
-"use strict";(self.webpackChunkhudi=self.webpackChunkhudi||[]).push([[34543],{97568:(e,t,a)=>{a.r(t),a.d(t,{assets:()=>d,contentTitle:()=>l,default:()=>h,frontMatter:()=>r,metadata:()=>n,toc:()=>o});const n=JSON.parse('{"id":"metadata","title":"Table Metadata","description":"Hudi tracks metadata about a table to remove bottlenecks in achieving great read/write performance, specifically on cloud storage.","source":"@site/docs/metadata.md","sourceDirName":".","slug":"/metadata","permalink":"/docs/next/metadata","draft":false,"unlisted":false,"editUrl":"https://github.com/apache/hudi/tree/asf-site/website/docs/metadata.md","tags":[],"version":"current","frontMatter":{"title":"Table Metadata","keywords":["hudi","metadata","S3","GCS","file listings","statistics"]},"sidebar":"docs","previous":{"title":"Record Mergers","permalink":"/docs/next/record_merger"},"next":{"title":"Indexes","permalink":"/docs/next/indexes"}}');var i=a(74848),s=a(28453);const r={title:"Table Metadata",keywords:["hudi","metadata","S3","GCS","file listings","statistics"]},l=void 0,d={},o=[{value:"Metadata Table",id:"metadata-table",level:2},{value:"Types of table metadata",id:"types-of-table-metadata",level:2},{value:"Metadata Tracking on Writers",id:"metadata-tracking-on-writers",level:2},{value:"Leveraging metadata during queries",id:"leveraging-metadata-during-queries",level:2},{value:"files index",id:"files-index",level:3},{value:"column_stats index and data skipping",id:"column_stats-index-and-data-skipping",level:3},{value:"Concurrency Control for Metadata Table",id:"concurrency-control-for-metadata-table",level:2},{value:"Related Resources",id:"related-resources",level:2}];function c(e){const t={a:"a",admonition:"admonition",code:"code",em:"em",h2:"h2",h3:"h3",img:"img",li:"li",p:"p",strong:"strong",table:"table",tbody:"tbody",td:"td",th:"th",thead:"thead",tr:"tr",ul:"ul",...(0,s.R)(),...e.components};return(0,i.jsxs)(i.Fragment,{children:[(0,i.jsx)(t.p,{children:"Hudi tracks metadata about a table to remove bottlenecks in achieving great read/write performance, specifically on cloud storage."}),"\n",(0,i.jsxs)(t.ul,{children:["\n",(0,i.jsxs)(t.li,{children:["\n",(0,i.jsxs)(t.p,{children:[(0,i.jsx)(t.strong,{children:"Avoid list operations to obtain set of files in a table"}),": A fundamental need for any engine that wants to read or write Hudi tables is\nto know all the files/objects that are part of the table, by performing listing of table partitions/folders. Unlike many distributed file systems,\nsuch operation scales poorly on cloud storage taking few seconds or even many minutes on large tables. This is particularly amplified when tables\nare large and partitioned multiple levels deep. Hudi tracks the file listings so they are readily available for readers/writers without listing the folders\ncontaining the data files."]}),"\n"]}),"\n",(0,i.jsxs)(t.li,{children:["\n",(0,i.jsxs)(t.p,{children:[(0,i.jsx)(t.strong,{children:"Expose columns statistics for better query planning and faster queries"}),": Query engines rely on techniques such as partitioning and data skipping\nto cut down on the amount of irrelevant data scanned for query planning and execution. During query planning phase, file footer statistics like column value ranges,\nnull counts are read from all data files to determine if a particular file needs to be read to satisfy the query. This approach is expensive since reading\nfooters from all files can increase cloud storage API costs and even be subject to throttling issues for larger tables. Hudi enables relevant query predicates to\nbe efficiently evaluated on operate on column statistics without incurring these costs."]}),"\n"]}),"\n"]}),"\n",(0,i.jsx)(t.h2,{id:"metadata-table",children:"Metadata Table"}),"\n",(0,i.jsxs)(t.p,{children:["Hudi employs a special ",(0,i.jsx)(t.strong,{children:(0,i.jsx)(t.em,{children:"metadata table"})}),", within each table to provide these capabilities. The metadata table implemented as a single\ninternal Hudi Merge-On-Read table that hosts different types of table metadata in each partition. This is similar to common practices in databases where metadata\nis tracked using internal tables. This approach provides the following advantages."]}),"\n",(0,i.jsxs)(t.ul,{children:["\n",(0,i.jsxs)(t.li,{children:["\n",(0,i.jsxs)(t.p,{children:[(0,i.jsx)(t.strong,{children:"Scalable"}),": The table metadata must scale to large sizes as well (see ",(0,i.jsx)(t.a,{href:"https://vldb.org/pvldb/vol14/p3083-edara.pdf",children:"Big Metadata paper"})," from Google).\nDifferent types of indexes should be easily integrated to support various use cases with consistent management of metadata. By implementing metadata using the\nsame storage format and engine used for data, Hudi is able to scale to even TBs of metadata with built-in table services for managing metadata."]}),"\n"]}),"\n",(0,i.jsxs)(t.li,{children:["\n",(0,i.jsxs)(t.p,{children:[(0,i.jsx)(t.strong,{children:"Flexible"}),": The foundational framework for multi-modal indexing is built to enable and disable new indexes as needed. The\n",(0,i.jsx)(t.a,{href:"https://www.onehouse.ai/blog/asynchronous-indexing-using-hudi",children:"async indexing"})," protocol index building alongside regular writers without impacting the write latency."]}),"\n"]}),"\n",(0,i.jsxs)(t.li,{children:["\n",(0,i.jsxs)(t.p,{children:[(0,i.jsx)(t.strong,{children:"transactional updates"}),": Tables data, metadata and indexes must be upto-date and consistent with each other as writes happen or table services are performed. and table metadata must be always up-to-date and in sync with the data table.\nThe data and metadata table's timelines share a parent-child relationship, to ensure they are always in sync with each other. Furthermore, the MoR table storage helps absorb fast changes to metadata from streaming writes without requiring\nrewriting of all table metadata on each write."]}),"\n"]}),"\n",(0,i.jsxs)(t.li,{children:["\n",(0,i.jsxs)(t.p,{children:[(0,i.jsx)(t.strong,{children:"Fast lookups"}),": By employing a SSTable like base file format (HFile) in the metadata table, query engines are able to efficiently perform lookup scans for only specific parts of\nmetadata needed. For e.g. query accessing only 10 out of 100 columns in a table can read stats about only the 10 columns it's interested in, during down planning time and costs.\nFurther, these metadata can also be served via a centralized/embedded timeline server which caches the metadata, further reducing the latency of the lookup from executors."]}),"\n"]}),"\n"]}),"\n",(0,i.jsxs)(t.p,{children:[(0,i.jsx)(t.img,{alt:"Metadata Table Mechanics",src:a(73633).A+"",width:"1280",height:"720"}),"\n",(0,i.jsx)("p",{align:"center",children:"Figure: Mechanics for Metadata Table in Hudi"})]}),"\n",(0,i.jsx)(t.h2,{id:"types-of-table-metadata",children:"Types of table metadata"}),"\n",(0,i.jsx)(t.p,{children:"Following are the different types of metadata currently supported."}),"\n",(0,i.jsxs)(t.ul,{children:["\n",(0,i.jsxs)(t.li,{children:["\n",(0,i.jsxs)(t.p,{children:[(0,i.jsx)(t.em,{children:(0,i.jsx)(t.strong,{children:(0,i.jsx)(t.a,{href:"https://cwiki.apache.org/confluence/display/HUDI/RFC+-+15%3A+HUDI+File+Listing+Improvements",children:"files listings"})})}),":\nStored as ",(0,i.jsx)(t.em,{children:"files"})," partition in the metadata table. Contains file information such as file name, size, and active state\nfor each partition in the data table, along with list of all partitions in the table. Improves the files listing performance\nby avoiding direct storage calls such as ",(0,i.jsx)(t.em,{children:"exists, listStatus"})," and ",(0,i.jsx)(t.em,{children:"listFiles"})," on the data table."]}),"\n"]}),"\n",(0,i.jsxs)(t.li,{children:["\n",(0,i.jsxs)(t.p,{children:[(0,i.jsx)(t.em,{children:(0,i.jsx)(t.strong,{children:(0,i.jsx)(t.a,{href:"https://github.com/apache/hudi/blob/master/rfc/rfc-27/rfc-27.md",children:"column statistics"})})}),": Stored as ",(0,i.jsx)(t.em,{children:"column_stats"}),"\npartition in the metadata table. Contains the statistics for a set of tracked columns, such as min and max values, total values,\nnull counts, size, etc., for all data files and are used while serving queries with predicates matching interested\ncolumns. This is heavily used by techniques like ",(0,i.jsx)(t.a,{href:"https://www.onehouse.ai/blog/hudis-column-stats-index-and-data-skipping-feature-help-speed-up-queries-by-an-orders-of-magnitude",children:"data skipping"})," to speed up queries by orders of magnitude, by skipping\nirrelevant files."]}),"\n"]}),"\n",(0,i.jsxs)(t.li,{children:["\n",(0,i.jsxs)(t.p,{children:[(0,i.jsx)(t.em,{children:(0,i.jsx)(t.strong,{children:"Partition Statistics"})}),": Partition stats index aggregates statistics at the partition level for the columns tracked by\nthe column statistics for which it is enabled. This helps in efficient partition pruning by skipping entire folders very quickly,\neven without examining column statistics at the file level. The partition stats index is stored in ",(0,i.jsx)(t.em,{children:"partition_stats"})," partition in the metadata table.\nPartition stats index can be enabled using the following configs (note it is required to specify the columns for which stats should be aggregated)."]}),"\n"]}),"\n"]}),"\n",(0,i.jsxs)(t.p,{children:["To try out these features, refer to the ",(0,i.jsx)(t.a,{href:"sql_ddl#create-partition-stats-index",children:"SQL guide"}),"."]}),"\n",(0,i.jsx)(t.h2,{id:"metadata-tracking-on-writers",children:"Metadata Tracking on Writers"}),"\n",(0,i.jsxs)(t.p,{children:["Following are based basic configs that are needed to enable metadata tracking. For advanced configs please refer\n",(0,i.jsx)(t.a,{href:"configurations#Metadata-Configs",children:"here"}),"."]}),"\n",(0,i.jsxs)(t.table,{children:[(0,i.jsx)(t.thead,{children:(0,i.jsxs)(t.tr,{children:[(0,i.jsx)(t.th,{children:"Config Name"}),(0,i.jsx)(t.th,{children:"Default"}),(0,i.jsx)(t.th,{children:"Description"})]})}),(0,i.jsxs)(t.tbody,{children:[(0,i.jsxs)(t.tr,{children:[(0,i.jsx)(t.td,{children:"hoodie.metadata.enable"}),(0,i.jsx)(t.td,{children:"true (Optional) Enabled on the write side"}),(0,i.jsxs)(t.td,{children:["Enable the internal metadata table serving file listings. For 0.10.1 and prior releases, metadata table is disabled by default and needs to be explicitly enabled.",(0,i.jsx)("br",{}),(0,i.jsx)("br",{}),(0,i.jsx)(t.code,{children:"Config Param: ENABLE"}),(0,i.jsx)("br",{}),(0,i.jsx)(t.code,{children:"Since Version: 0.7.0"})]})]}),(0,i.jsxs)(t.tr,{children:[(0,i.jsx)(t.td,{children:"hoodie.metadata.index.column.stats.enable"}),(0,i.jsx)(t.td,{children:"false (Optional)"}),(0,i.jsxs)(t.td,{children:["Enable column statistics tracking of files under metadata table. When enabled, metadata table will have a partition to store the column ranges and will be used for pruning files during data skipping.",(0,i.jsx)("br",{}),(0,i.jsx)("br",{}),(0,i.jsx)(t.code,{children:"Config Param: ENABLE_METADATA_INDEX_COLUMN_STATS"}),(0,i.jsx)("br",{}),(0,i.jsx)(t.code,{children:"Since Version: 0.11.0"})]})]}),(0,i.jsxs)(t.tr,{children:[(0,i.jsx)(t.td,{children:"hoodie.metadata.index.column.stats.columns"}),(0,i.jsx)(t.td,{children:"all columns in the table"}),(0,i.jsx)(t.td,{children:"Comma separated list of columns to track column statistics on."})]}),(0,i.jsxs)(t.tr,{children:[(0,i.jsx)(t.td,{children:"hoodie.metadata.index.partition.stats.enable"}),(0,i.jsx)(t.td,{children:"false (Optional)"}),(0,i.jsx)(t.td,{children:"Enable the partition stats tracking, on the same columns tracked by column stats metadata."})]})]})]}),"\n",(0,i.jsxs)(t.p,{children:["For Flink, following are the basic configs of interest to enable metadata tracking. Please refer\n",(0,i.jsx)(t.a,{href:"https://hudi.apache.org/docs/next/configurations#Flink-Options",children:"here"})," for advanced configs"]}),"\n",(0,i.jsxs)(t.table,{children:[(0,i.jsx)(t.thead,{children:(0,i.jsxs)(t.tr,{children:[(0,i.jsx)(t.th,{children:"Config Name"}),(0,i.jsx)(t.th,{children:"Default"}),(0,i.jsx)(t.th,{children:"Description"})]})}),(0,i.jsx)(t.tbody,{children:(0,i.jsxs)(t.tr,{children:[(0,i.jsx)(t.td,{children:"metadata.enabled"}),(0,i.jsx)(t.td,{children:"true (Optional)"}),(0,i.jsxs)(t.td,{children:["Enable the internal metadata table which serves table metadata like level file listings, default enabled",(0,i.jsx)("br",{}),(0,i.jsx)("br",{})," ",(0,i.jsx)(t.code,{children:"Config Param: METADATA_ENABLED"})]})]})})]}),"\n",(0,i.jsx)(t.admonition,{type:"note",children:(0,i.jsx)(t.p,{children:"If you turn off the metadata table after enabling, be sure to wait for a few commits so that the metadata table is fully\ncleaned up, before re-enabling the metadata table again."})}),"\n",(0,i.jsx)(t.h2,{id:"leveraging-metadata-during-queries",children:"Leveraging metadata during queries"}),"\n",(0,i.jsx)(t.h3,{id:"files-index",children:"files index"}),"\n",(0,i.jsxs)(t.p,{children:["Metadata based listing using ",(0,i.jsx)(t.em,{children:"files_index"})," can be leveraged on the read side by setting appropriate configs/session properties\nfrom different engines as shown below:"]}),"\n",(0,i.jsxs)(t.table,{children:[(0,i.jsx)(t.thead,{children:(0,i.jsxs)(t.tr,{children:[(0,i.jsx)(t.th,{children:"Readers"}),(0,i.jsx)(t.th,{children:"Config"}),(0,i.jsx)(t.th,{children:"Description"})]})}),(0,i.jsxs)(t.tbody,{children:[(0,i.jsxs)(t.tr,{children:[(0,i.jsx)(t.td,{children:"Spark DataSource, Spark SQL, Strucured Streaming"}),(0,i.jsx)(t.td,{children:"hoodie.metadata.enable"}),(0,i.jsxs)(t.td,{children:["When set to ",(0,i.jsx)(t.code,{children:"true"})," enables use of the spark file index implementation for Hudi, that speeds up listing of large tables.",(0,i.jsx)("br",{})]})]}),(0,i.jsxs)(t.tr,{children:[(0,i.jsx)(t.td,{children:"Flink DataStream, Flink SQL"}),(0,i.jsx)(t.td,{children:"metadata.enabled"}),(0,i.jsxs)(t.td,{children:["When set to ",(0,i.jsx)(t.code,{children:"true"})," from DDL uses the internal metadata table to serves table metadata like level file listings"]})]}),(0,i.jsxs)(t.tr,{children:[(0,i.jsx)(t.td,{children:"Presto"}),(0,i.jsx)(t.td,{children:(0,i.jsx)(t.a,{href:"https://prestodb.io/docs/current/connector/hudi.html",children:"hudi.metadata-table-enabled"})}),(0,i.jsxs)(t.td,{children:["When set to ",(0,i.jsx)(t.code,{children:"true"})," fetches the list of file names and sizes from Hudi\u2019s metadata table rather than storage."]})]}),(0,i.jsxs)(t.tr,{children:[(0,i.jsx)(t.td,{children:"Trino"}),(0,i.jsx)(t.td,{children:"N/A"}),(0,i.jsxs)(t.td,{children:["Support for reading from the metadata table ",(0,i.jsx)(t.a,{href:"https://issues.apache.org/jira/browse/HUDI-7020",children:"has been dropped in Trino 419"}),"."]})]}),(0,i.jsxs)(t.tr,{children:[(0,i.jsx)(t.td,{children:"Athena"}),(0,i.jsx)(t.td,{children:(0,i.jsx)(t.a,{href:"https://docs.aws.amazon.com/athena/latest/ug/querying-hudi.html",children:"hudi.metadata-listing-enabled"})}),(0,i.jsxs)(t.td,{children:["When this table property is set to ",(0,i.jsx)(t.code,{children:"TRUE"})," enables the Hudi metadata table and the related file listing functionality"]})]})]})]}),"\n",(0,i.jsx)(t.h3,{id:"column_stats-index-and-data-skipping",children:"column_stats index and data skipping"}),"\n",(0,i.jsx)(t.p,{children:"Enabling metadata table and column stats index is a prerequisite to enabling data skipping capabilities. Following are the\ncorresponding configs across Spark and Flink readers."}),"\n",(0,i.jsxs)(t.table,{children:[(0,i.jsx)(t.thead,{children:(0,i.jsxs)(t.tr,{children:[(0,i.jsx)(t.th,{children:"Readers"}),(0,i.jsx)(t.th,{children:"Config"}),(0,i.jsx)(t.th,{children:"Description"})]})}),(0,i.jsxs)(t.tbody,{children:[(0,i.jsxs)(t.tr,{children:[(0,i.jsx)(t.td,{children:"Spark DataSource, Spark SQL, Strucured Streaming"}),(0,i.jsx)(t.td,{children:(0,i.jsxs)("ul",{children:[(0,i.jsx)("li",{children:(0,i.jsx)(t.code,{children:"hoodie.metadata.enable"})}),(0,i.jsx)("li",{children:(0,i.jsx)(t.code,{children:"hoodie.enable.data.skipping"})})]})}),(0,i.jsx)(t.td,{children:(0,i.jsxs)("ul",{children:[(0,i.jsxs)("li",{children:["When set to ",(0,i.jsx)(t.code,{children:"true"})," enables use of the spark file index implementation for Hudi, that speeds up listing of large tables."]}),(0,i.jsxs)("li",{children:["When set to ",(0,i.jsx)(t.code,{children:"true"})," enables data-skipping allowing queries to leverage indexes to reduce the search space by skipping over files ",(0,i.jsx)("br",{}),(0,i.jsx)(t.code,{children:"Config Param: ENABLE_DATA_SKIPPING"}),(0,i.jsx)("br",{}),(0,i.jsx)(t.code,{children:"Since Version: 0.10.0"})," ",(0,i.jsx)("br",{})]})]})})]}),(0,i.jsxs)(t.tr,{children:[(0,i.jsx)(t.td,{children:"Flink DataStream, Flink SQL"}),(0,i.jsx)(t.td,{children:(0,i.jsxs)("ul",{children:[(0,i.jsx)("li",{children:(0,i.jsx)(t.code,{children:"metadata.enabled"})}),(0,i.jsx)("li",{children:(0,i.jsx)(t.code,{children:"read.data.skipping.enabled"})})]})}),(0,i.jsx)(t.td,{children:(0,i.jsxs)("ul",{children:[(0,i.jsxs)("li",{children:[" When set to ",(0,i.jsx)(t.code,{children:"true"})," from DDL uses the internal metadata table to serves table metadata like level file listings"]}),(0,i.jsxs)("li",{children:["When set to ",(0,i.jsx)(t.code,{children:"true"})," enables data-skipping allowing queries to leverage indexes to reduce the search space byskipping over files"]})]})})]})]})]}),"\n",(0,i.jsx)(t.h2,{id:"concurrency-control-for-metadata-table",children:"Concurrency Control for Metadata Table"}),"\n",(0,i.jsxs)(t.p,{children:["To ensure that metadata table stays up to date and table metadata is tracked safely across concurrent write and\ntable operations, there are some additional considerations. If async table services are enabled for the table (i.e. running a separate compaction (",(0,i.jsx)(t.code,{children:"HoodieCompactor"}),") or\nclustering (",(0,i.jsx)(t.code,{children:"HoodieClusteringJob"}),") job), even with just a single writer, lock providers\nmust be configured. Please refer to ",(0,i.jsx)(t.a,{href:"concurrency_control",children:"concurrency control"})," for more details."]}),"\n",(0,i.jsxs)(t.p,{children:["Before enabling metadata table for the first time, all writers on the same table must and table services must be stopped.\nIf your current deployment model is ",(0,i.jsx)(t.a,{href:"concurrency_control#full-on-multi-writer--async-table-services",children:"multi-writer"})," along with a lock\nprovider and other required configs set for every writer as follows, there is no additional configuration required. You\ncan bring up the writers sequentially after stopping the writers for enabling metadata table. Applying the proper\nconfigurations to only a subset of writers or table services is unsafe and can lead to loss of data. So, please ensure you enable\nmetadata table across all writers."]}),"\n",(0,i.jsx)(t.h2,{id:"related-resources",children:"Related Resources"}),"\n",(0,i.jsx)("h3",{children:"Blogs"}),"\n",(0,i.jsxs)(t.ul,{children:["\n",(0,i.jsx)(t.li,{children:(0,i.jsx)(t.a,{href:"https://medium.com/@simpsons/table-service-deployment-models-in-apache-hudi-9cfa5a44addf",children:"Table service deployment models in Apache Hudi"})}),"\n",(0,i.jsx)(t.li,{children:(0,i.jsx)(t.a,{href:"https://www.onehouse.ai/blog/introducing-multi-modal-index-for-the-lakehouse-in-apache-hudi",children:"Multi Modal Indexing for the Data Lakehouse"})}),"\n"]})]})}function h(e={}){const{wrapper:t}={...(0,s.R)(),...e.components};return t?(0,i.jsx)(t,{...e,children:(0,i.jsx)(c,{...e})}):c(e)}},73633:(e,t,a)=>{a.d(t,{A:()=>n});const n=a.p+"assets/images/metadata_table_anim-4d6225cef4c2e0937805288146eaf6ad.gif"},28453:(e,t,a)=>{a.d(t,{R:()=>r,x:()=>l});var n=a(96540);const i={},s=n.createContext(i);function r(e){const t=n.useContext(s);return n.useMemo((function(){return"function"==typeof e?e(t):{...t,...e}}),[t,e])}function l(e){let t;return t=e.disableParentContext?"function"==typeof e.components?e.components(i):e.components||i:r(e.components),n.createElement(s.Provider,{value:t},e.children)}}}]);
\ No newline at end of file
+"use strict";(self.webpackChunkhudi=self.webpackChunkhudi||[]).push([[34543],{97568:(e,t,a)=>{a.r(t),a.d(t,{assets:()=>d,contentTitle:()=>l,default:()=>h,frontMatter:()=>r,metadata:()=>n,toc:()=>o});const n=JSON.parse('{"id":"metadata","title":"Table Metadata","description":"Hudi tracks metadata about a table to remove bottlenecks in achieving great read/write performance, specifically on cloud storage.","source":"@site/docs/metadata.md","sourceDirName":".","slug":"/metadata","permalink":"/docs/next/metadata","draft":false,"unlisted":false,"editUrl":"https://github.com/apache/hudi/tree/asf-site/website/docs/metadata.md","tags":[],"version":"current","frontMatter":{"title":"Table Metadata","keywords":["hudi","metadata","S3","GCS","file listings","statistics"]},"sidebar":"docs","previous":{"title":"Record Mergers","permalink":"/docs/next/record_merger"},"next":{"title":"Indexes","permalink":"/docs/next/indexes"}}');var i=a(74848),s=a(28453);const r={title:"Table Metadata",keywords:["hudi","metadata","S3","GCS","file listings","statistics"]},l=void 0,d={},o=[{value:"Metadata Table",id:"metadata-table",level:2},{value:"Types of table metadata",id:"types-of-table-metadata",level:2},{value:"Metadata Tracking on Writers",id:"metadata-tracking-on-writers",level:2},{value:"Leveraging metadata during queries",id:"leveraging-metadata-during-queries",level:2},{value:"files index",id:"files-index",level:3},{value:"column_stats index and data skipping",id:"column_stats-index-and-data-skipping",level:3},{value:"Concurrency Control for Metadata Table",id:"concurrency-control-for-metadata-table",level:2},{value:"Related Resources",id:"related-resources",level:2}];function c(e){const t={a:"a",admonition:"admonition",code:"code",em:"em",h2:"h2",h3:"h3",img:"img",li:"li",p:"p",strong:"strong",table:"table",tbody:"tbody",td:"td",th:"th",thead:"thead",tr:"tr",ul:"ul",...(0,s.R)(),...e.components};return(0,i.jsxs)(i.Fragment,{children:[(0,i.jsx)(t.p,{children:"Hudi tracks metadata about a table to remove bottlenecks in achieving great read/write performance, specifically on cloud storage."}),"\n",(0,i.jsxs)(t.ul,{children:["\n",(0,i.jsxs)(t.li,{children:["\n",(0,i.jsxs)(t.p,{children:[(0,i.jsx)(t.strong,{children:"Avoid list operations to obtain set of files in a table"}),": A fundamental need for any engine that wants to read or write Hudi tables is\nto know all the files/objects that are part of the table, by performing listing of table partitions/folders. Unlike many distributed file systems,\nsuch operation scales poorly on cloud storage taking few seconds or even many minutes on large tables. This is particularly amplified when tables\nare large and partitioned multiple levels deep. Hudi tracks the file listings so they are readily available for readers/writers without listing the folders\ncontaining the data files."]}),"\n"]}),"\n",(0,i.jsxs)(t.li,{children:["\n",(0,i.jsxs)(t.p,{children:[(0,i.jsx)(t.strong,{children:"Expose columns statistics for better query planning and faster queries"}),": Query engines rely on techniques such as partitioning and data skipping\nto cut down on the amount of irrelevant data scanned for query planning and execution. During query planning phase, file footer statistics like column value ranges,\nnull counts are read from all data files to determine if a particular file needs to be read to satisfy the query. This approach is expensive since reading\nfooters from all files can increase cloud storage API costs and even be subject to throttling issues for larger tables. Hudi enables relevant query predicates to\nbe efficiently evaluated on operate on column statistics without incurring these costs."]}),"\n"]}),"\n"]}),"\n",(0,i.jsx)(t.h2,{id:"metadata-table",children:"Metadata Table"}),"\n",(0,i.jsxs)(t.p,{children:["Hudi employs a special ",(0,i.jsx)(t.strong,{children:(0,i.jsx)(t.em,{children:"metadata table"})}),", within each table to provide these capabilities. The metadata table implemented as a single\ninternal Hudi Merge-On-Read table that hosts different types of table metadata in each partition. This is similar to common practices in databases where metadata\nis tracked using internal tables. This approach provides the following advantages."]}),"\n",(0,i.jsxs)(t.ul,{children:["\n",(0,i.jsxs)(t.li,{children:["\n",(0,i.jsxs)(t.p,{children:[(0,i.jsx)(t.strong,{children:"Scalable"}),": The table metadata must scale to large sizes as well (see ",(0,i.jsx)(t.a,{href:"https://vldb.org/pvldb/vol14/p3083-edara.pdf",children:"Big Metadata paper"})," from Google).\nDifferent types of indexes should be easily integrated to support various use cases with consistent management of metadata. By implementing metadata using the\nsame storage format and engine used for data, Hudi is able to scale to even TBs of metadata with built-in table services for managing metadata."]}),"\n"]}),"\n",(0,i.jsxs)(t.li,{children:["\n",(0,i.jsxs)(t.p,{children:[(0,i.jsx)(t.strong,{children:"Flexible"}),": The foundational framework for multi-modal indexing is built to enable and disable new indexes as needed. The\n",(0,i.jsx)(t.a,{href:"https://www.onehouse.ai/blog/asynchronous-indexing-using-hudi",children:"async indexing"})," protocol index building alongside regular writers without impacting the write latency."]}),"\n"]}),"\n",(0,i.jsxs)(t.li,{children:["\n",(0,i.jsxs)(t.p,{children:[(0,i.jsx)(t.strong,{children:"transactional updates"}),": Tables data, metadata and indexes must be upto-date and consistent with each other as writes happen or table services are performed. and table metadata must be always up-to-date and in sync with the data table.\nThe data and metadata table's timelines share a parent-child relationship, to ensure they are always in sync with each other. Furthermore, the MoR table storage helps absorb fast changes to metadata from streaming writes without requiring\nrewriting of all table metadata on each write."]}),"\n"]}),"\n",(0,i.jsxs)(t.li,{children:["\n",(0,i.jsxs)(t.p,{children:[(0,i.jsx)(t.strong,{children:"Fast lookups"}),": By employing a SSTable like base file format (HFile) in the metadata table, query engines are able to efficiently perform lookup scans for only specific parts of\nmetadata needed. For e.g. query accessing only 10 out of 100 columns in a table can read stats about only the 10 columns it's interested in, during down planning time and costs.\nFurther, these metadata can also be served via a centralized/embedded timeline server which caches the metadata, further reducing the latency of the lookup from executors."]}),"\n"]}),"\n"]}),"\n",(0,i.jsxs)(t.p,{children:[(0,i.jsx)(t.img,{alt:"Metadata Table Mechanics",src:a(73633).A+"",width:"1280",height:"720"}),"\n",(0,i.jsx)("p",{align:"center",children:"Figure: Mechanics for Metadata Table in Hudi"})]}),"\n",(0,i.jsx)(t.h2,{id:"types-of-table-metadata",children:"Types of table metadata"}),"\n",(0,i.jsx)(t.p,{children:"Following are the different types of metadata currently supported."}),"\n",(0,i.jsxs)(t.ul,{children:["\n",(0,i.jsxs)(t.li,{children:["\n",(0,i.jsxs)(t.p,{children:[(0,i.jsx)(t.em,{children:(0,i.jsx)(t.strong,{children:(0,i.jsx)(t.a,{href:"https://cwiki.apache.org/confluence/display/HUDI/RFC+-+15%3A+HUDI+File+Listing+Improvements",children:"files listings"})})}),":\nStored as ",(0,i.jsx)(t.em,{children:"files"})," partition in the metadata table. Contains file information such as file name, size, and active state\nfor each partition in the data table, along with list of all partitions in the table. Improves the files listing performance\nby avoiding direct storage calls such as ",(0,i.jsx)(t.em,{children:"exists, listStatus"})," and ",(0,i.jsx)(t.em,{children:"listFiles"})," on the data table."]}),"\n"]}),"\n",(0,i.jsxs)(t.li,{children:["\n",(0,i.jsxs)(t.p,{children:[(0,i.jsx)(t.em,{children:(0,i.jsx)(t.strong,{children:(0,i.jsx)(t.a,{href:"https://github.com/apache/hudi/blob/master/rfc/rfc-27/rfc-27.md",children:"column statistics"})})}),": Stored as ",(0,i.jsx)(t.em,{children:"column_stats"}),"\npartition in the metadata table. Contains the statistics for a set of tracked columns, such as min and max values, total values,\nnull counts, size, etc., for all data files and are used while serving queries with predicates matching interested\ncolumns. This is heavily used by techniques like ",(0,i.jsx)(t.a,{href:"https://www.onehouse.ai/blog/hudis-column-stats-index-and-data-skipping-feature-help-speed-up-queries-by-an-orders-of-magnitude",children:"data skipping"})," to speed up queries by orders of magnitude, by skipping\nirrelevant files."]}),"\n"]}),"\n",(0,i.jsxs)(t.li,{children:["\n",(0,i.jsxs)(t.p,{children:[(0,i.jsx)(t.em,{children:(0,i.jsx)(t.strong,{children:"Partition Statistics"})}),": Partition stats index aggregates statistics at the partition level for the columns tracked by\nthe column statistics for which it is enabled. This helps in efficient partition pruning by skipping entire folders very quickly,\neven without examining column statistics at the file level. The partition stats index is stored in ",(0,i.jsx)(t.em,{children:"partition_stats"})," partition in the metadata table.\nPartition stats index can be enabled using the following configs (note it is required to specify the columns for which stats should be aggregated)."]}),"\n"]}),"\n"]}),"\n",(0,i.jsxs)(t.p,{children:["To try out these features, refer to the ",(0,i.jsx)(t.a,{href:"sql_ddl#create-partition-stats-index",children:"SQL guide"}),"."]}),"\n",(0,i.jsx)(t.h2,{id:"metadata-tracking-on-writers",children:"Metadata Tracking on Writers"}),"\n",(0,i.jsxs)(t.p,{children:["Following are based basic configs that are needed to enable metadata tracking. For advanced configs please refer\n",(0,i.jsx)(t.a,{href:"configurations#Metadata-Configs",children:"here"}),"."]}),"\n",(0,i.jsxs)(t.table,{children:[(0,i.jsx)(t.thead,{children:(0,i.jsxs)(t.tr,{children:[(0,i.jsx)(t.th,{children:"Config Name"}),(0,i.jsx)(t.th,{children:"Default"}),(0,i.jsx)(t.th,{children:"Description"})]})}),(0,i.jsxs)(t.tbody,{children:[(0,i.jsxs)(t.tr,{children:[(0,i.jsx)(t.td,{children:"hoodie.metadata.enable"}),(0,i.jsx)(t.td,{children:"true (Optional) Enabled on the write side"}),(0,i.jsxs)(t.td,{children:["Enable the internal metadata table serving file listings. For 0.10.1 and prior releases, metadata table is disabled by default and needs to be explicitly enabled.",(0,i.jsx)("br",{}),(0,i.jsx)("br",{}),(0,i.jsx)(t.code,{children:"Config Param: ENABLE"}),(0,i.jsx)("br",{}),(0,i.jsx)(t.code,{children:"Since Version: 0.7.0"})]})]}),(0,i.jsxs)(t.tr,{children:[(0,i.jsx)(t.td,{children:"hoodie.metadata.index.column.stats.enable"}),(0,i.jsx)(t.td,{children:"false (Optional)"}),(0,i.jsxs)(t.td,{children:["Enable column statistics tracking of files under metadata table. When enabled, metadata table will have a partition to store the column ranges and will be used for pruning files during data skipping.",(0,i.jsx)("br",{}),(0,i.jsx)("br",{}),(0,i.jsx)(t.code,{children:"Config Param: ENABLE_METADATA_INDEX_COLUMN_STATS"}),(0,i.jsx)("br",{}),(0,i.jsx)(t.code,{children:"Since Version: 0.11.0"})]})]}),(0,i.jsxs)(t.tr,{children:[(0,i.jsx)(t.td,{children:"hoodie.metadata.index.column.stats.columns"}),(0,i.jsx)(t.td,{children:"all columns in the table"}),(0,i.jsx)(t.td,{children:"Comma separated list of columns to track column statistics on."})]}),(0,i.jsxs)(t.tr,{children:[(0,i.jsx)(t.td,{children:"hoodie.metadata.index.partition.stats.enable"}),(0,i.jsx)(t.td,{children:"false (Optional)"}),(0,i.jsx)(t.td,{children:"Enable the partition stats tracking, on the same columns tracked by column stats metadata."})]})]})]}),"\n",(0,i.jsxs)(t.p,{children:["For Flink, following are the basic configs of interest to enable metadata tracking. Please refer\n",(0,i.jsx)(t.a,{href:"https://hudi.apache.org/docs/next/configurations#Flink-Options",children:"here"})," for advanced configs"]}),"\n",(0,i.jsxs)(t.table,{children:[(0,i.jsx)(t.thead,{children:(0,i.jsxs)(t.tr,{children:[(0,i.jsx)(t.th,{children:"Config Name"}),(0,i.jsx)(t.th,{children:"Default"}),(0,i.jsx)(t.th,{children:"Description"})]})}),(0,i.jsx)(t.tbody,{children:(0,i.jsxs)(t.tr,{children:[(0,i.jsx)(t.td,{children:"metadata.enabled"}),(0,i.jsx)(t.td,{children:"true (Optional)"}),(0,i.jsxs)(t.td,{children:["Enable the internal metadata table which serves table metadata like level file listings, default enabled",(0,i.jsx)("br",{}),(0,i.jsx)("br",{})," ",(0,i.jsx)(t.code,{children:"Config Param: METADATA_ENABLED"})]})]})})]}),"\n",(0,i.jsx)(t.admonition,{type:"note",children:(0,i.jsx)(t.p,{children:"If you turn off the metadata table after enabling, be sure to wait for a few commits so that the metadata table is fully\ncleaned up, before re-enabling the metadata table again."})}),"\n",(0,i.jsx)(t.h2,{id:"leveraging-metadata-during-queries",children:"Leveraging metadata during queries"}),"\n",(0,i.jsx)(t.h3,{id:"files-index",children:"files index"}),"\n",(0,i.jsxs)(t.p,{children:["Metadata based listing using ",(0,i.jsx)(t.em,{children:"files_index"})," can be leveraged on the read side by setting appropriate configs/session properties\nfrom different engines as shown below:"]}),"\n",(0,i.jsxs)(t.table,{children:[(0,i.jsx)(t.thead,{children:(0,i.jsxs)(t.tr,{children:[(0,i.jsx)(t.th,{children:"Readers"}),(0,i.jsx)(t.th,{children:"Config"}),(0,i.jsx)(t.th,{children:"Description"})]})}),(0,i.jsxs)(t.tbody,{children:[(0,i.jsxs)(t.tr,{children:[(0,i.jsx)(t.td,{children:"Spark DataSource, Spark SQL, Strucured Streaming"}),(0,i.jsx)(t.td,{children:"hoodie.metadata.enable"}),(0,i.jsxs)(t.td,{children:["When set to ",(0,i.jsx)(t.code,{children:"true"})," enables use of the spark file index implementation for Hudi, that speeds up listing of large tables.",(0,i.jsx)("br",{})]})]}),(0,i.jsxs)(t.tr,{children:[(0,i.jsx)(t.td,{children:"Flink DataStream, Flink SQL"}),(0,i.jsx)(t.td,{children:"metadata.enabled"}),(0,i.jsxs)(t.td,{children:["When set to ",(0,i.jsx)(t.code,{children:"true"})," from DDL uses the internal metadata table to serves table metadata like level file listings"]})]}),(0,i.jsxs)(t.tr,{children:[(0,i.jsx)(t.td,{children:"Presto"}),(0,i.jsx)(t.td,{children:(0,i.jsx)(t.a,{href:"https://prestodb.io/docs/current/connector/hudi.html",children:"hudi.metadata-table-enabled"})}),(0,i.jsxs)(t.td,{children:["When set to ",(0,i.jsx)(t.code,{children:"true"})," fetches the list of file names and sizes from Hudi\u2019s metadata table rather than storage."]})]}),(0,i.jsxs)(t.tr,{children:[(0,i.jsx)(t.td,{children:"Trino"}),(0,i.jsx)(t.td,{children:"N/A"}),(0,i.jsxs)(t.td,{children:["Support for reading from the metadata table ",(0,i.jsx)(t.a,{href:"https://issues.apache.org/jira/browse/HUDI-7020",children:"has been dropped in Trino 419"}),"."]})]}),(0,i.jsxs)(t.tr,{children:[(0,i.jsx)(t.td,{children:"Athena"}),(0,i.jsx)(t.td,{children:(0,i.jsx)(t.a,{href:"https://docs.aws.amazon.com/athena/latest/ug/querying-hudi.html",children:"hudi.metadata-listing-enabled"})}),(0,i.jsxs)(t.td,{children:["When this table property is set to ",(0,i.jsx)(t.code,{children:"TRUE"})," enables the Hudi metadata table and the related file listing functionality"]})]})]})]}),"\n",(0,i.jsx)(t.h3,{id:"column_stats-index-and-data-skipping",children:"column_stats index and data skipping"}),"\n",(0,i.jsx)(t.p,{children:"Enabling metadata table and column stats index is a prerequisite to enabling data skipping capabilities. Following are the\ncorresponding configs across Spark and Flink readers."}),"\n",(0,i.jsxs)(t.table,{children:[(0,i.jsx)(t.thead,{children:(0,i.jsxs)(t.tr,{children:[(0,i.jsx)(t.th,{children:"Readers"}),(0,i.jsx)(t.th,{children:"Config"}),(0,i.jsx)(t.th,{children:"Description"})]})}),(0,i.jsxs)(t.tbody,{children:[(0,i.jsxs)(t.tr,{children:[(0,i.jsx)(t.td,{children:"Spark DataSource, Spark SQL, Strucured Streaming"}),(0,i.jsx)(t.td,{children:(0,i.jsxs)("ul",{children:[(0,i.jsx)("li",{children:(0,i.jsx)(t.code,{children:"hoodie.metadata.enable"})}),(0,i.jsx)("li",{children:(0,i.jsx)(t.code,{children:"hoodie.enable.data.skipping"})})]})}),(0,i.jsx)(t.td,{children:(0,i.jsxs)("ul",{children:[(0,i.jsxs)("li",{children:["When set to ",(0,i.jsx)(t.code,{children:"true"})," enables use of the spark file index implementation for Hudi, that speeds up listing of large tables."]}),(0,i.jsxs)("li",{children:["When set to ",(0,i.jsx)(t.code,{children:"true"})," enables data-skipping allowing queries to leverage indexes to reduce the search space by skipping over files ",(0,i.jsx)("br",{}),(0,i.jsx)(t.code,{children:"Config Param: ENABLE_DATA_SKIPPING"}),(0,i.jsx)("br",{}),(0,i.jsx)(t.code,{children:"Since Version: 0.10.0"})," ",(0,i.jsx)("br",{})]})]})})]}),(0,i.jsxs)(t.tr,{children:[(0,i.jsx)(t.td,{children:"Flink DataStream, Flink SQL"}),(0,i.jsx)(t.td,{children:(0,i.jsxs)("ul",{children:[(0,i.jsx)("li",{children:(0,i.jsx)(t.code,{children:"metadata.enabled"})}),(0,i.jsx)("li",{children:(0,i.jsx)(t.code,{children:"read.data.skipping.enabled"})})]})}),(0,i.jsx)(t.td,{children:(0,i.jsxs)("ul",{children:[(0,i.jsxs)("li",{children:[" When set to ",(0,i.jsx)(t.code,{children:"true"})," from DDL uses the internal metadata table to serves table metadata like level file listings"]}),(0,i.jsxs)("li",{children:["When set to ",(0,i.jsx)(t.code,{children:"true"})," enables data-skipping allowing queries to leverage indexes to reduce the search space byskipping over files"]})]})})]})]})]}),"\n",(0,i.jsx)(t.h2,{id:"concurrency-control-for-metadata-table",children:"Concurrency Control for Metadata Table"}),"\n",(0,i.jsxs)(t.p,{children:["To ensure that metadata table stays up to date and table metadata is tracked safely across concurrent write and\ntable operations, there are some additional considerations. If async table services are enabled for the table (i.e. running a separate compaction (",(0,i.jsx)(t.code,{children:"HoodieCompactor"}),") or\nclustering (",(0,i.jsx)(t.code,{children:"HoodieClusteringJob"}),") job), even with just a single writer, lock providers\nmust be configured. Please refer to ",(0,i.jsx)(t.a,{href:"concurrency_control",children:"concurrency control"})," for more details."]}),"\n",(0,i.jsxs)(t.p,{children:["Before enabling metadata table for the first time, all writers on the same table must and table services must be stopped.\nIf your current deployment model is ",(0,i.jsx)(t.a,{href:"concurrency_control#full-on-multi-writer--async-table-services",children:"multi-writer"})," along with a lock\nprovider and other required configs set for every writer as follows, there is no additional configuration required. You\ncan bring up the writers sequentially after stopping the writers for enabling metadata table. Applying the proper\nconfigurations to only a subset of writers or table services is unsafe and can lead to loss of data. So, please ensure you enable\nmetadata table across all writers."]}),"\n",(0,i.jsx)(t.h2,{id:"related-resources",children:"Related Resources"}),"\n",(0,i.jsx)("h3",{children:"Blogs"}),"\n",(0,i.jsxs)(t.ul,{children:["\n",(0,i.jsx)(t.li,{children:(0,i.jsx)(t.a,{href:"https://medium.com/@simpsons/table-service-deployment-models-in-apache-hudi-9cfa5a44addf",children:"Table service deployment models in Apache Hudi"})}),"\n",(0,i.jsx)(t.li,{children:(0,i.jsx)(t.a,{href:"https://www.onehouse.ai/blog/introducing-multi-modal-index-for-the-lakehouse-in-apache-hudi",children:"Multi Modal Indexing for the Data Lakehouse"})}),"\n",(0,i.jsx)(t.li,{children:(0,i.jsx)(t.a,{href:"https://www.onehouse.ai/blog/how-to-optimize-performance-for-your-open-data-lakehouse",children:"How to Optimize Performance for Your Open Data Lakehouse"})}),"\n"]})]})}function h(e={}){const{wrapper:t}={...(0,s.R)(),...e.components};return t?(0,i.jsx)(t,{...e,children:(0,i.jsx)(c,{...e})}):c(e)}},73633:(e,t,a)=>{a.d(t,{A:()=>n});const n=a.p+"assets/images/metadata_table_anim-4d6225cef4c2e0937805288146eaf6ad.gif"},28453:(e,t,a)=>{a.d(t,{R:()=>r,x:()=>l});var n=a(96540);const i={},s=n.createContext(i);function r(e){const t=n.useContext(s);return n.useMemo((function(){return"function"==typeof e?e(t):{...t,...e}}),[t,e])}function l(e){let t;return t=e.disableParentContext?"function"==typeof e.components?e.components(i):e.components||i:r(e.components),n.createElement(s.Provider,{value:t},e.children)}}}]);
\ No newline at end of file
diff --git a/content/assets/js/b93a682d.187bd928.js b/content/assets/js/b93a682d.56cd85f4.js
similarity index 89%
rename from content/assets/js/b93a682d.187bd928.js
rename to content/assets/js/b93a682d.56cd85f4.js
index 0101c2ac91947..cd585f6af79c8 100644
--- a/content/assets/js/b93a682d.187bd928.js
+++ b/content/assets/js/b93a682d.56cd85f4.js
@@ -1 +1 @@
-"use strict";(self.webpackChunkhudi=self.webpackChunkhudi||[]).push([[97476],{79280:(e,n,i)=>{i.r(n),i.d(n,{assets:()=>l,contentTitle:()=>r,default:()=>h,frontMatter:()=>t,metadata:()=>a,toc:()=>c});const a=JSON.parse('{"id":"cleaning","title":"Cleaning","description":"Background","source":"@site/docs/cleaning.md","sourceDirName":".","slug":"/cleaning","permalink":"/docs/next/cleaning","draft":false,"unlisted":false,"editUrl":"https://github.com/apache/hudi/tree/asf-site/website/docs/cleaning.md","tags":[],"version":"current","frontMatter":{"title":"Cleaning","toc":true,"toc_min_heading_level":2,"toc_max_heading_level":4},"sidebar":"docs","previous":{"title":"Streaming Reads","permalink":"/docs/next/reading_tables_streaming_reads"},"next":{"title":"Compaction","permalink":"/docs/next/compaction"}}');var s=i(74848),o=i(28453);const t={title:"Cleaning",toc:!0,toc_min_heading_level:2,toc_max_heading_level:4},r=void 0,l={},c=[{value:"Background",id:"background",level:2},{value:"Cleaning Retention Policies",id:"cleaning-retention-policies",level:3},{value:"Configs",id:"configs",level:3},{value:"Ways to trigger Cleaning",id:"ways-to-trigger-cleaning",level:3},{value:"Inline",id:"inline",level:4},{value:"Async",id:"async",level:4},{value:"Run independently",id:"run-independently",level:4},{value:"CLI",id:"cli",level:4},{value:"Related Resources",id:"related-resources",level:2}];function d(e){const n={a:"a",br:"br",code:"code",h2:"h2",h3:"h3",h4:"h4",li:"li",p:"p",pre:"pre",strong:"strong",table:"table",tbody:"tbody",td:"td",th:"th",thead:"thead",tr:"tr",ul:"ul",...(0,o.R)(),...e.components};return(0,s.jsxs)(s.Fragment,{children:[(0,s.jsx)(n.h2,{id:"background",children:"Background"}),"\n",(0,s.jsxs)(n.p,{children:["Cleaning is a table service employed by Hudi to reclaim space occupied by older versions of data and keep storage costs\nin check. Apache Hudi provides snapshot isolation between writers and readers by managing multiple versioned files with ",(0,s.jsx)(n.strong,{children:"MVCC"}),"\nconcurrency. These file versions provide history and enable time travel and rollbacks, but it is important to manage\nhow much history you keep to balance your costs. Cleaning service plays a crucial role in manging the tradeoff between\nretaining long history of data and the associated storage costs."]}),"\n",(0,s.jsxs)(n.p,{children:["Hudi enables ",(0,s.jsx)(n.a,{href:"/docs/configurations/#hoodiecleanautomatic",children:"Automatic Hudi cleaning"})," by default. Cleaning is invoked\nimmediately after each commit, to delete older file slices. It's recommended to leave this enabled to ensure metadata\nand data storage growth is bounded. Cleaner can also be scheduled after every few commits instead of after every commit by\nconfiguring ",(0,s.jsx)(n.a,{href:"https://hudi.apache.org/docs/configurations#hoodiecleanmaxcommits",children:"hoodie.clean.max.commits"}),"."]}),"\n",(0,s.jsx)(n.h3,{id:"cleaning-retention-policies",children:"Cleaning Retention Policies"}),"\n",(0,s.jsx)(n.p,{children:"When cleaning old files, you should be careful not to remove files that are being actively used by long running queries."}),"\n",(0,s.jsx)(n.p,{children:"For spark based:"}),"\n",(0,s.jsxs)(n.table,{children:[(0,s.jsx)(n.thead,{children:(0,s.jsxs)(n.tr,{children:[(0,s.jsx)(n.th,{children:"Config Name"}),(0,s.jsx)(n.th,{children:"Default"}),(0,s.jsx)(n.th,{children:"Description"})]})}),(0,s.jsx)(n.tbody,{children:(0,s.jsxs)(n.tr,{children:[(0,s.jsx)(n.td,{children:"hoodie.cleaner.policy"}),(0,s.jsx)(n.td,{children:"KEEP_LATEST_COMMITS (Optional)"}),(0,s.jsxs)(n.td,{children:["org.apache.hudi.common.model.HoodieCleaningPolicy: Cleaning policy to be used. ",(0,s.jsx)("br",{}),(0,s.jsx)("br",{}),(0,s.jsx)(n.code,{children:"Config Param: CLEANER_POLICY"})]})]})})]}),"\n",(0,s.jsxs)(n.p,{children:["The corresponding config for Flink based engine is ",(0,s.jsx)(n.a,{href:"https://hudi.apache.org/docs/configurations/#cleanpolicy",children:(0,s.jsx)(n.code,{children:"clean.policy"})}),"."]}),"\n",(0,s.jsx)(n.p,{children:"Hudi cleaner currently supports the below cleaning policies to keep a certain number of commits or file versions:"}),"\n",(0,s.jsxs)(n.ul,{children:["\n",(0,s.jsxs)(n.li,{children:["\n",(0,s.jsxs)(n.p,{children:[(0,s.jsx)(n.strong,{children:"KEEP_LATEST_COMMITS"}),": This is the default policy. This is a temporal cleaning policy that ensures the effect of\nhaving lookback into all the changes that happened in the last X commits. Suppose a writer is ingesting data\ninto a Hudi dataset every 30 minutes and the longest running query can take 5 hours to finish, then the user should\nretain atleast the last 10 commits. With such a configuration, we ensure that the oldest version of a file is kept on\ndisk for at least 5 hours, thereby preventing the longest running query from failing at any point in time. Incremental\ncleaning is also possible using this policy.\nNumber of commits to retain can be configured by ",(0,s.jsx)(n.a,{href:"https://analytics.google.com/analytics/web/#/p300324801/reports/intelligenthome",children:(0,s.jsx)(n.code,{children:"hoodie.cleaner.commits.retained"})}),".\nThe corresponding Flink related config is ",(0,s.jsx)(n.a,{href:"https://hudi.apache.org/docs/configurations/#cleanretain_commits",children:(0,s.jsx)(n.code,{children:"clean.retain_commits"})}),"."]}),"\n"]}),"\n",(0,s.jsxs)(n.li,{children:["\n",(0,s.jsxs)(n.p,{children:[(0,s.jsx)(n.strong,{children:"KEEP_LATEST_FILE_VERSIONS"}),": This policy has the effect of keeping N number of file versions irrespective of time.\nThis policy is useful when it is known how many MAX versions of the file does one want to keep at any given time.\nTo achieve the same behaviour as before of preventing long running queries from failing, one should do their calculations\nbased on data patterns. Alternatively, this policy is also useful if a user just wants to maintain 1 latest version of the file.\nNumber of file versions to retain can be configured by ",(0,s.jsx)(n.a,{href:"https://hudi.apache.org/docs/configurations/#hoodiecleanerfileversionsretained",children:(0,s.jsx)(n.code,{children:"hoodie.cleaner.fileversions.retained"})}),".\nThe corresponding Flink related config is ",(0,s.jsx)(n.a,{href:"https://hudi.apache.org/docs/configurations/#cleanretain_file_versions",children:(0,s.jsx)(n.code,{children:"clean.retain_file_versions"})}),"."]}),"\n"]}),"\n",(0,s.jsxs)(n.li,{children:["\n",(0,s.jsxs)(n.p,{children:[(0,s.jsx)(n.strong,{children:"KEEP_LATEST_BY_HOURS"}),": This policy clean up based on hours.It is simple and useful when knowing that you want to\nkeep files at any given time. Corresponding to commits with commit times older than the configured number of hours to\nbe retained are cleaned. Currently you can configure by parameter ",(0,s.jsx)(n.a,{href:"https://hudi.apache.org/docs/configurations/#hoodiecleanerhoursretained",children:(0,s.jsx)(n.code,{children:"hoodie.cleaner.hours.retained"})}),".\nThe corresponding Flink related config is ",(0,s.jsx)(n.a,{href:"https://hudi.apache.org/docs/configurations/#cleanretain_hours",children:(0,s.jsx)(n.code,{children:"clean.retain_hours"})}),"."]}),"\n"]}),"\n"]}),"\n",(0,s.jsx)(n.h3,{id:"configs",children:"Configs"}),"\n",(0,s.jsxs)(n.p,{children:["For details about all possible configurations and their default values see the ",(0,s.jsx)(n.a,{href:"https://hudi.apache.org/docs/next/configurations/#Clean-Configs",children:"configuration docs"}),".\nFor Flink related configs refer ",(0,s.jsx)(n.a,{href:"https://hudi.apache.org/docs/next/configurations/#FLINK_SQL",children:"here"}),"."]}),"\n",(0,s.jsx)(n.h3,{id:"ways-to-trigger-cleaning",children:"Ways to trigger Cleaning"}),"\n",(0,s.jsx)(n.h4,{id:"inline",children:"Inline"}),"\n",(0,s.jsxs)(n.p,{children:["By default, in Spark based writing, cleaning is run inline after every commit using the default policy of ",(0,s.jsx)(n.code,{children:"KEEP_LATEST_COMMITS"}),". It's recommended\nto keep this enabled, to ensure metadata and data storage growth is bounded. To enable this, users do not have to set any configs. Following are the relevant basic configs."]}),"\n",(0,s.jsxs)(n.table,{children:[(0,s.jsx)(n.thead,{children:(0,s.jsxs)(n.tr,{children:[(0,s.jsx)(n.th,{children:"Config Name"}),(0,s.jsx)(n.th,{children:"Default"}),(0,s.jsx)(n.th,{children:"Description"})]})}),(0,s.jsxs)(n.tbody,{children:[(0,s.jsxs)(n.tr,{children:[(0,s.jsx)(n.td,{children:"hoodie.clean.automatic"}),(0,s.jsx)(n.td,{children:"true (Optional)"}),(0,s.jsxs)(n.td,{children:["When enabled, the cleaner table service is invoked immediately after each commit, to delete older file slices. It's recommended to enable this, to ensure metadata and data storage growth is bounded.",(0,s.jsx)("br",{}),(0,s.jsx)("br",{}),(0,s.jsx)(n.code,{children:"Config Param: AUTO_CLEAN"})]})]}),(0,s.jsxs)(n.tr,{children:[(0,s.jsx)(n.td,{children:"hoodie.cleaner.commits.retained"}),(0,s.jsx)(n.td,{children:"10 (Optional)"}),(0,s.jsxs)(n.td,{children:["Number of commits to retain, without cleaning. This will be retained for num_of_commits * time_between_commits (scheduled). This also directly translates into how much data retention the table supports for incremental queries.",(0,s.jsx)("br",{}),(0,s.jsx)("br",{}),(0,s.jsx)(n.code,{children:"Config Param: CLEANER_COMMITS_RETAINED"})]})]})]})]}),"\n",(0,s.jsx)(n.h4,{id:"async",children:"Async"}),"\n",(0,s.jsxs)(n.p,{children:["In case you wish to run the cleaner service asynchronously along with writing, please enable the ",(0,s.jsx)(n.a,{href:"https://hudi.apache.org/docs/configurations#hoodiecleanasync",children:(0,s.jsx)(n.code,{children:"hoodie.clean.async"})})," as shown below:"]}),"\n",(0,s.jsx)(n.pre,{children:(0,s.jsx)(n.code,{className:"language-java",children:"hoodie.clean.automatic=true\nhoodie.clean.async=true\n"})}),"\n",(0,s.jsxs)(n.p,{children:["For Flink based writing, this is the default mode of cleaning. Please refer to ",(0,s.jsx)(n.a,{href:"https://hudi.apache.org/docs/configurations/#cleanasyncenabled",children:(0,s.jsx)(n.code,{children:"clean.async.enabled"})})," for details."]}),"\n",(0,s.jsx)(n.h4,{id:"run-independently",children:"Run independently"}),"\n",(0,s.jsx)(n.p,{children:"Hoodie Cleaner can also be run as a separate process. Following is the command for running the cleaner independently:"}),"\n",(0,s.jsx)(n.pre,{children:(0,s.jsx)(n.code,{children:'spark-submit --master local \\\n --packages org.apache.hudi:hudi-utilities-slim-bundle_2.12:1.0.0,org.apache.hudi:hudi-spark3.5-bundle_2.12:1.0.0 \\\n --class org.apache.hudi.utilities.HoodieCleaner `ls packaging/hudi-utilities-slim-bundle/target/hudi-utilities-slim-bundle-*.jar` --help\n Usage: [options]\n Options:\n --help, -h\n\n --hoodie-conf\n Any configuration that can be set in the properties file (using the CLI\n parameter "--props") can also be passed command line using this\n parameter. This can be repeated\n Default: []\n --props\n path to properties file on localfs or dfs, with configurations for\n hoodie client for cleaning\n --spark-master\n spark master to use.\n Default: local[2]\n * --target-base-path\n base path for the hoodie table to be cleaner.\n'})}),"\n",(0,s.jsxs)(n.p,{children:["Some examples to run the cleaner.",(0,s.jsx)(n.br,{}),"\n","Keep the latest 10 commits"]}),"\n",(0,s.jsx)(n.pre,{children:(0,s.jsx)(n.code,{children:"spark-submit --master local \\\n --packages org.apache.hudi:hudi-utilities-slim-bundle_2.12:1.0.0,org.apache.hudi:hudi-spark3.5-bundle_2.12:1.0.0 \\\n --class org.apache.hudi.utilities.HoodieCleaner `ls packaging/hudi-utilities-slim-bundle/target/hudi-utilities-slim-bundle-*.jar` \\\n --target-base-path /path/to/hoodie_table \\\n --hoodie-conf hoodie.cleaner.policy=KEEP_LATEST_COMMITS \\\n --hoodie-conf hoodie.cleaner.commits.retained=10 \\\n --hoodie-conf hoodie.cleaner.parallelism=200\n"})}),"\n",(0,s.jsx)(n.p,{children:"Keep the latest 3 file versions"}),"\n",(0,s.jsx)(n.pre,{children:(0,s.jsx)(n.code,{children:"spark-submit --master local \\\n --packages org.apache.hudi:hudi-utilities-slim-bundle_2.12:1.0.0,org.apache.hudi:hudi-spark3.5-bundle_2.12:1.0.0 \\\n --class org.apache.hudi.utilities.HoodieCleaner `ls packaging/hudi-utilities-slim-bundle/target/hudi-utilities-slim-bundle-*.jar` \\\n --hoodie-conf hoodie.cleaner.policy=KEEP_LATEST_FILE_VERSIONS \\\n --hoodie-conf hoodie.cleaner.fileversions.retained=3 \\\n --hoodie-conf hoodie.cleaner.parallelism=200\n"})}),"\n",(0,s.jsx)(n.p,{children:"Clean commits older than 24 hours"}),"\n",(0,s.jsx)(n.pre,{children:(0,s.jsx)(n.code,{children:"spark-submit --master local \\\n --packages org.apache.hudi:hudi-utilities-slim-bundle_2.12:1.0.0,org.apache.hudi:hudi-spark3.5-bundle_2.12:1.0.0 \\\n --class org.apache.hudi.utilities.HoodieCleaner `ls packaging/hudi-utilities-slim-bundle/target/hudi-utilities-slim-bundle-*.jar` \\\n --target-base-path /path/to/hoodie_table \\\n --hoodie-conf hoodie.cleaner.policy=KEEP_LATEST_BY_HOURS \\\n --hoodie-conf hoodie.cleaner.hours.retained=24 \\\n --hoodie-conf hoodie.cleaner.parallelism=200\n"})}),"\n",(0,s.jsxs)(n.p,{children:["Note: The parallelism takes the min value of number of partitions to clean and ",(0,s.jsx)(n.code,{children:"hoodie.cleaner.parallelism"}),"."]}),"\n",(0,s.jsx)(n.h4,{id:"cli",children:"CLI"}),"\n",(0,s.jsxs)(n.p,{children:["You can also use ",(0,s.jsx)(n.a,{href:"/docs/cli",children:"Hudi CLI"})," to run Hoodie Cleaner."]}),"\n",(0,s.jsx)(n.p,{children:"CLI provides the below commands for cleaner service:"}),"\n",(0,s.jsxs)(n.ul,{children:["\n",(0,s.jsx)(n.li,{children:(0,s.jsx)(n.code,{children:"cleans show"})}),"\n",(0,s.jsx)(n.li,{children:(0,s.jsx)(n.code,{children:"clean showpartitions"})}),"\n",(0,s.jsx)(n.li,{children:(0,s.jsx)(n.code,{children:"cleans run"})}),"\n"]}),"\n",(0,s.jsx)(n.p,{children:"Example of cleaner keeping the latest 10 commits"}),"\n",(0,s.jsx)(n.pre,{children:(0,s.jsx)(n.code,{children:"cleans run --sparkMaster local --hoodieConfigs hoodie.cleaner.policy=KEEP_LATEST_COMMITS hoodie.cleaner.commits.retained=3 hoodie.cleaner.parallelism=200\n"})}),"\n",(0,s.jsxs)(n.p,{children:["You can find more details and the relevant code for these commands in ",(0,s.jsx)(n.a,{href:"https://github.com/apache/hudi/blob/master/hudi-cli/src/main/java/org/apache/hudi/cli/commands/CleansCommand.java",children:(0,s.jsx)(n.code,{children:"org.apache.hudi.cli.commands.CleansCommand"})})," class."]}),"\n",(0,s.jsx)(n.h2,{id:"related-resources",children:"Related Resources"}),"\n",(0,s.jsx)("h3",{children:"Videos"}),"\n",(0,s.jsxs)(n.ul,{children:["\n",(0,s.jsx)(n.li,{children:(0,s.jsx)(n.a,{href:"https://youtu.be/mUvRhJDoO3w",children:"Cleaner Service: Save up to 40% on data lake storage costs | Hudi Labs"})}),"\n",(0,s.jsx)(n.li,{children:(0,s.jsx)(n.a,{href:"https://www.youtube.com/watch?v=CEzgFtmVjx4",children:"Efficient Data Lake Management with Apache Hudi Cleaner: Benefits of Scheduling Data Cleaning #1"})}),"\n",(0,s.jsx)(n.li,{children:(0,s.jsx)(n.a,{href:"https://www.youtube.com/watch?v=RbBF9Ys2GqM",children:"Efficient Data Lake Management with Apache Hudi Cleaner: Benefits of Scheduling Data Cleaning #2"})}),"\n"]})]})}function h(e={}){const{wrapper:n}={...(0,o.R)(),...e.components};return n?(0,s.jsx)(n,{...e,children:(0,s.jsx)(d,{...e})}):d(e)}},28453:(e,n,i)=>{i.d(n,{R:()=>t,x:()=>r});var a=i(96540);const s={},o=a.createContext(s);function t(e){const n=a.useContext(o);return a.useMemo((function(){return"function"==typeof e?e(n):{...n,...e}}),[n,e])}function r(e){let n;return n=e.disableParentContext?"function"==typeof e.components?e.components(s):e.components||s:t(e.components),a.createElement(o.Provider,{value:n},e.children)}}}]);
\ No newline at end of file
+"use strict";(self.webpackChunkhudi=self.webpackChunkhudi||[]).push([[97476],{79280:(e,n,i)=>{i.r(n),i.d(n,{assets:()=>r,contentTitle:()=>l,default:()=>h,frontMatter:()=>t,metadata:()=>a,toc:()=>c});const a=JSON.parse('{"id":"cleaning","title":"Cleaning","description":"Background","source":"@site/docs/cleaning.md","sourceDirName":".","slug":"/cleaning","permalink":"/docs/next/cleaning","draft":false,"unlisted":false,"editUrl":"https://github.com/apache/hudi/tree/asf-site/website/docs/cleaning.md","tags":[],"version":"current","frontMatter":{"title":"Cleaning","toc":true,"toc_min_heading_level":2,"toc_max_heading_level":4},"sidebar":"docs","previous":{"title":"Streaming Reads","permalink":"/docs/next/reading_tables_streaming_reads"},"next":{"title":"Compaction","permalink":"/docs/next/compaction"}}');var s=i(74848),o=i(28453);const t={title:"Cleaning",toc:!0,toc_min_heading_level:2,toc_max_heading_level:4},l=void 0,r={},c=[{value:"Background",id:"background",level:2},{value:"Cleaning Retention Policies",id:"cleaning-retention-policies",level:3},{value:"Configs",id:"configs",level:3},{value:"Ways to trigger Cleaning",id:"ways-to-trigger-cleaning",level:3},{value:"Inline",id:"inline",level:4},{value:"Async",id:"async",level:4},{value:"Run independently",id:"run-independently",level:4},{value:"CLI",id:"cli",level:4},{value:"Related Resources",id:"related-resources",level:2}];function d(e){const n={a:"a",br:"br",code:"code",h2:"h2",h3:"h3",h4:"h4",li:"li",p:"p",pre:"pre",strong:"strong",table:"table",tbody:"tbody",td:"td",th:"th",thead:"thead",tr:"tr",ul:"ul",...(0,o.R)(),...e.components};return(0,s.jsxs)(s.Fragment,{children:[(0,s.jsx)(n.h2,{id:"background",children:"Background"}),"\n",(0,s.jsxs)(n.p,{children:["Cleaning is a table service employed by Hudi to reclaim space occupied by older versions of data and keep storage costs\nin check. Apache Hudi provides snapshot isolation between writers and readers by managing multiple versioned files with ",(0,s.jsx)(n.strong,{children:"MVCC"}),"\nconcurrency. These file versions provide history and enable time travel and rollbacks, but it is important to manage\nhow much history you keep to balance your costs. Cleaning service plays a crucial role in manging the tradeoff between\nretaining long history of data and the associated storage costs."]}),"\n",(0,s.jsxs)(n.p,{children:["Hudi enables ",(0,s.jsx)(n.a,{href:"/docs/configurations/#hoodiecleanautomatic",children:"Automatic Hudi cleaning"})," by default. Cleaning is invoked\nimmediately after each commit, to delete older file slices. It's recommended to leave this enabled to ensure metadata\nand data storage growth is bounded. Cleaner can also be scheduled after every few commits instead of after every commit by\nconfiguring ",(0,s.jsx)(n.a,{href:"https://hudi.apache.org/docs/configurations#hoodiecleanmaxcommits",children:"hoodie.clean.max.commits"}),"."]}),"\n",(0,s.jsx)(n.h3,{id:"cleaning-retention-policies",children:"Cleaning Retention Policies"}),"\n",(0,s.jsx)(n.p,{children:"When cleaning old files, you should be careful not to remove files that are being actively used by long running queries."}),"\n",(0,s.jsx)(n.p,{children:"For spark based:"}),"\n",(0,s.jsxs)(n.table,{children:[(0,s.jsx)(n.thead,{children:(0,s.jsxs)(n.tr,{children:[(0,s.jsx)(n.th,{children:"Config Name"}),(0,s.jsx)(n.th,{children:"Default"}),(0,s.jsx)(n.th,{children:"Description"})]})}),(0,s.jsx)(n.tbody,{children:(0,s.jsxs)(n.tr,{children:[(0,s.jsx)(n.td,{children:"hoodie.cleaner.policy"}),(0,s.jsx)(n.td,{children:"KEEP_LATEST_COMMITS (Optional)"}),(0,s.jsxs)(n.td,{children:["org.apache.hudi.common.model.HoodieCleaningPolicy: Cleaning policy to be used. ",(0,s.jsx)("br",{}),(0,s.jsx)("br",{}),(0,s.jsx)(n.code,{children:"Config Param: CLEANER_POLICY"})]})]})})]}),"\n",(0,s.jsxs)(n.p,{children:["The corresponding config for Flink based engine is ",(0,s.jsx)(n.a,{href:"https://hudi.apache.org/docs/configurations/#cleanpolicy",children:(0,s.jsx)(n.code,{children:"clean.policy"})}),"."]}),"\n",(0,s.jsx)(n.p,{children:"Hudi cleaner currently supports the below cleaning policies to keep a certain number of commits or file versions:"}),"\n",(0,s.jsxs)(n.ul,{children:["\n",(0,s.jsxs)(n.li,{children:["\n",(0,s.jsxs)(n.p,{children:[(0,s.jsx)(n.strong,{children:"KEEP_LATEST_COMMITS"}),": This is the default policy. This is a temporal cleaning policy that ensures the effect of\nhaving lookback into all the changes that happened in the last X commits. Suppose a writer is ingesting data\ninto a Hudi dataset every 30 minutes and the longest running query can take 5 hours to finish, then the user should\nretain atleast the last 10 commits. With such a configuration, we ensure that the oldest version of a file is kept on\ndisk for at least 5 hours, thereby preventing the longest running query from failing at any point in time. Incremental\ncleaning is also possible using this policy.\nNumber of commits to retain can be configured by ",(0,s.jsx)(n.a,{href:"https://analytics.google.com/analytics/web/#/p300324801/reports/intelligenthome",children:(0,s.jsx)(n.code,{children:"hoodie.cleaner.commits.retained"})}),".\nThe corresponding Flink related config is ",(0,s.jsx)(n.a,{href:"https://hudi.apache.org/docs/configurations/#cleanretain_commits",children:(0,s.jsx)(n.code,{children:"clean.retain_commits"})}),"."]}),"\n"]}),"\n",(0,s.jsxs)(n.li,{children:["\n",(0,s.jsxs)(n.p,{children:[(0,s.jsx)(n.strong,{children:"KEEP_LATEST_FILE_VERSIONS"}),": This policy has the effect of keeping N number of file versions irrespective of time.\nThis policy is useful when it is known how many MAX versions of the file does one want to keep at any given time.\nTo achieve the same behaviour as before of preventing long running queries from failing, one should do their calculations\nbased on data patterns. Alternatively, this policy is also useful if a user just wants to maintain 1 latest version of the file.\nNumber of file versions to retain can be configured by ",(0,s.jsx)(n.a,{href:"https://hudi.apache.org/docs/configurations/#hoodiecleanerfileversionsretained",children:(0,s.jsx)(n.code,{children:"hoodie.cleaner.fileversions.retained"})}),".\nThe corresponding Flink related config is ",(0,s.jsx)(n.a,{href:"https://hudi.apache.org/docs/configurations/#cleanretain_file_versions",children:(0,s.jsx)(n.code,{children:"clean.retain_file_versions"})}),"."]}),"\n"]}),"\n",(0,s.jsxs)(n.li,{children:["\n",(0,s.jsxs)(n.p,{children:[(0,s.jsx)(n.strong,{children:"KEEP_LATEST_BY_HOURS"}),": This policy clean up based on hours.It is simple and useful when knowing that you want to\nkeep files at any given time. Corresponding to commits with commit times older than the configured number of hours to\nbe retained are cleaned. Currently you can configure by parameter ",(0,s.jsx)(n.a,{href:"https://hudi.apache.org/docs/configurations/#hoodiecleanerhoursretained",children:(0,s.jsx)(n.code,{children:"hoodie.cleaner.hours.retained"})}),".\nThe corresponding Flink related config is ",(0,s.jsx)(n.a,{href:"https://hudi.apache.org/docs/configurations/#cleanretain_hours",children:(0,s.jsx)(n.code,{children:"clean.retain_hours"})}),"."]}),"\n"]}),"\n"]}),"\n",(0,s.jsx)(n.h3,{id:"configs",children:"Configs"}),"\n",(0,s.jsxs)(n.p,{children:["For details about all possible configurations and their default values see the ",(0,s.jsx)(n.a,{href:"https://hudi.apache.org/docs/next/configurations/#Clean-Configs",children:"configuration docs"}),".\nFor Flink related configs refer ",(0,s.jsx)(n.a,{href:"https://hudi.apache.org/docs/next/configurations/#FLINK_SQL",children:"here"}),"."]}),"\n",(0,s.jsx)(n.h3,{id:"ways-to-trigger-cleaning",children:"Ways to trigger Cleaning"}),"\n",(0,s.jsx)(n.h4,{id:"inline",children:"Inline"}),"\n",(0,s.jsxs)(n.p,{children:["By default, in Spark based writing, cleaning is run inline after every commit using the default policy of ",(0,s.jsx)(n.code,{children:"KEEP_LATEST_COMMITS"}),". It's recommended\nto keep this enabled, to ensure metadata and data storage growth is bounded. To enable this, users do not have to set any configs. Following are the relevant basic configs."]}),"\n",(0,s.jsxs)(n.table,{children:[(0,s.jsx)(n.thead,{children:(0,s.jsxs)(n.tr,{children:[(0,s.jsx)(n.th,{children:"Config Name"}),(0,s.jsx)(n.th,{children:"Default"}),(0,s.jsx)(n.th,{children:"Description"})]})}),(0,s.jsxs)(n.tbody,{children:[(0,s.jsxs)(n.tr,{children:[(0,s.jsx)(n.td,{children:"hoodie.clean.automatic"}),(0,s.jsx)(n.td,{children:"true (Optional)"}),(0,s.jsxs)(n.td,{children:["When enabled, the cleaner table service is invoked immediately after each commit, to delete older file slices. It's recommended to enable this, to ensure metadata and data storage growth is bounded.",(0,s.jsx)("br",{}),(0,s.jsx)("br",{}),(0,s.jsx)(n.code,{children:"Config Param: AUTO_CLEAN"})]})]}),(0,s.jsxs)(n.tr,{children:[(0,s.jsx)(n.td,{children:"hoodie.cleaner.commits.retained"}),(0,s.jsx)(n.td,{children:"10 (Optional)"}),(0,s.jsxs)(n.td,{children:["Number of commits to retain, without cleaning. This will be retained for num_of_commits * time_between_commits (scheduled). This also directly translates into how much data retention the table supports for incremental queries.",(0,s.jsx)("br",{}),(0,s.jsx)("br",{}),(0,s.jsx)(n.code,{children:"Config Param: CLEANER_COMMITS_RETAINED"})]})]})]})]}),"\n",(0,s.jsx)(n.h4,{id:"async",children:"Async"}),"\n",(0,s.jsxs)(n.p,{children:["In case you wish to run the cleaner service asynchronously along with writing, please enable the ",(0,s.jsx)(n.a,{href:"https://hudi.apache.org/docs/configurations#hoodiecleanasync",children:(0,s.jsx)(n.code,{children:"hoodie.clean.async"})})," as shown below:"]}),"\n",(0,s.jsx)(n.pre,{children:(0,s.jsx)(n.code,{className:"language-java",children:"hoodie.clean.automatic=true\nhoodie.clean.async=true\n"})}),"\n",(0,s.jsxs)(n.p,{children:["For Flink based writing, this is the default mode of cleaning. Please refer to ",(0,s.jsx)(n.a,{href:"https://hudi.apache.org/docs/configurations/#cleanasyncenabled",children:(0,s.jsx)(n.code,{children:"clean.async.enabled"})})," for details."]}),"\n",(0,s.jsx)(n.h4,{id:"run-independently",children:"Run independently"}),"\n",(0,s.jsx)(n.p,{children:"Hoodie Cleaner can also be run as a separate process. Following is the command for running the cleaner independently:"}),"\n",(0,s.jsx)(n.pre,{children:(0,s.jsx)(n.code,{children:'spark-submit --master local \\\n --packages org.apache.hudi:hudi-utilities-slim-bundle_2.12:1.0.0,org.apache.hudi:hudi-spark3.5-bundle_2.12:1.0.0 \\\n --class org.apache.hudi.utilities.HoodieCleaner `ls packaging/hudi-utilities-slim-bundle/target/hudi-utilities-slim-bundle-*.jar` --help\n Usage: [options]\n Options:\n --help, -h\n\n --hoodie-conf\n Any configuration that can be set in the properties file (using the CLI\n parameter "--props") can also be passed command line using this\n parameter. This can be repeated\n Default: []\n --props\n path to properties file on localfs or dfs, with configurations for\n hoodie client for cleaning\n --spark-master\n spark master to use.\n Default: local[2]\n * --target-base-path\n base path for the hoodie table to be cleaner.\n'})}),"\n",(0,s.jsxs)(n.p,{children:["Some examples to run the cleaner.",(0,s.jsx)(n.br,{}),"\n","Keep the latest 10 commits"]}),"\n",(0,s.jsx)(n.pre,{children:(0,s.jsx)(n.code,{children:"spark-submit --master local \\\n --packages org.apache.hudi:hudi-utilities-slim-bundle_2.12:1.0.0,org.apache.hudi:hudi-spark3.5-bundle_2.12:1.0.0 \\\n --class org.apache.hudi.utilities.HoodieCleaner `ls packaging/hudi-utilities-slim-bundle/target/hudi-utilities-slim-bundle-*.jar` \\\n --target-base-path /path/to/hoodie_table \\\n --hoodie-conf hoodie.cleaner.policy=KEEP_LATEST_COMMITS \\\n --hoodie-conf hoodie.cleaner.commits.retained=10 \\\n --hoodie-conf hoodie.cleaner.parallelism=200\n"})}),"\n",(0,s.jsx)(n.p,{children:"Keep the latest 3 file versions"}),"\n",(0,s.jsx)(n.pre,{children:(0,s.jsx)(n.code,{children:"spark-submit --master local \\\n --packages org.apache.hudi:hudi-utilities-slim-bundle_2.12:1.0.0,org.apache.hudi:hudi-spark3.5-bundle_2.12:1.0.0 \\\n --class org.apache.hudi.utilities.HoodieCleaner `ls packaging/hudi-utilities-slim-bundle/target/hudi-utilities-slim-bundle-*.jar` \\\n --hoodie-conf hoodie.cleaner.policy=KEEP_LATEST_FILE_VERSIONS \\\n --hoodie-conf hoodie.cleaner.fileversions.retained=3 \\\n --hoodie-conf hoodie.cleaner.parallelism=200\n"})}),"\n",(0,s.jsx)(n.p,{children:"Clean commits older than 24 hours"}),"\n",(0,s.jsx)(n.pre,{children:(0,s.jsx)(n.code,{children:"spark-submit --master local \\\n --packages org.apache.hudi:hudi-utilities-slim-bundle_2.12:1.0.0,org.apache.hudi:hudi-spark3.5-bundle_2.12:1.0.0 \\\n --class org.apache.hudi.utilities.HoodieCleaner `ls packaging/hudi-utilities-slim-bundle/target/hudi-utilities-slim-bundle-*.jar` \\\n --target-base-path /path/to/hoodie_table \\\n --hoodie-conf hoodie.cleaner.policy=KEEP_LATEST_BY_HOURS \\\n --hoodie-conf hoodie.cleaner.hours.retained=24 \\\n --hoodie-conf hoodie.cleaner.parallelism=200\n"})}),"\n",(0,s.jsxs)(n.p,{children:["Note: The parallelism takes the min value of number of partitions to clean and ",(0,s.jsx)(n.code,{children:"hoodie.cleaner.parallelism"}),"."]}),"\n",(0,s.jsx)(n.h4,{id:"cli",children:"CLI"}),"\n",(0,s.jsxs)(n.p,{children:["You can also use ",(0,s.jsx)(n.a,{href:"/docs/cli",children:"Hudi CLI"})," to run Hoodie Cleaner."]}),"\n",(0,s.jsx)(n.p,{children:"CLI provides the below commands for cleaner service:"}),"\n",(0,s.jsxs)(n.ul,{children:["\n",(0,s.jsx)(n.li,{children:(0,s.jsx)(n.code,{children:"cleans show"})}),"\n",(0,s.jsx)(n.li,{children:(0,s.jsx)(n.code,{children:"clean showpartitions"})}),"\n",(0,s.jsx)(n.li,{children:(0,s.jsx)(n.code,{children:"cleans run"})}),"\n"]}),"\n",(0,s.jsx)(n.p,{children:"Example of cleaner keeping the latest 10 commits"}),"\n",(0,s.jsx)(n.pre,{children:(0,s.jsx)(n.code,{children:"cleans run --sparkMaster local --hoodieConfigs hoodie.cleaner.policy=KEEP_LATEST_COMMITS hoodie.cleaner.commits.retained=3 hoodie.cleaner.parallelism=200\n"})}),"\n",(0,s.jsxs)(n.p,{children:["You can find more details and the relevant code for these commands in ",(0,s.jsx)(n.a,{href:"https://github.com/apache/hudi/blob/master/hudi-cli/src/main/java/org/apache/hudi/cli/commands/CleansCommand.java",children:(0,s.jsx)(n.code,{children:"org.apache.hudi.cli.commands.CleansCommand"})})," class."]}),"\n",(0,s.jsx)(n.h2,{id:"related-resources",children:"Related Resources"}),"\n",(0,s.jsx)("h3",{children:"Blogs"}),"\n",(0,s.jsxs)(n.ul,{children:["\n",(0,s.jsx)(n.li,{children:(0,s.jsx)(n.a,{href:"https://medium.com/@simpsons/cleaner-and-archival-in-apache-hudi-9e15b08b2933",children:"Cleaner and Archival in Apache Hudi"})}),"\n"]}),"\n",(0,s.jsx)("h3",{children:"Videos"}),"\n",(0,s.jsxs)(n.ul,{children:["\n",(0,s.jsx)(n.li,{children:(0,s.jsx)(n.a,{href:"https://youtu.be/mUvRhJDoO3w",children:"Cleaner Service: Save up to 40% on data lake storage costs | Hudi Labs"})}),"\n",(0,s.jsx)(n.li,{children:(0,s.jsx)(n.a,{href:"https://www.youtube.com/watch?v=CEzgFtmVjx4",children:"Efficient Data Lake Management with Apache Hudi Cleaner: Benefits of Scheduling Data Cleaning #1"})}),"\n",(0,s.jsx)(n.li,{children:(0,s.jsx)(n.a,{href:"https://www.youtube.com/watch?v=RbBF9Ys2GqM",children:"Efficient Data Lake Management with Apache Hudi Cleaner: Benefits of Scheduling Data Cleaning #2"})}),"\n"]})]})}function h(e={}){const{wrapper:n}={...(0,o.R)(),...e.components};return n?(0,s.jsx)(n,{...e,children:(0,s.jsx)(d,{...e})}):d(e)}},28453:(e,n,i)=>{i.d(n,{R:()=>t,x:()=>l});var a=i(96540);const s={},o=a.createContext(s);function t(e){const n=a.useContext(o);return a.useMemo((function(){return"function"==typeof e?e(n):{...n,...e}}),[n,e])}function l(e){let n;return n=e.disableParentContext?"function"==typeof e.components?e.components(s):e.components||s:t(e.components),a.createElement(o.Provider,{value:n},e.children)}}}]);
\ No newline at end of file
diff --git a/content/assets/js/ba47c136.0223e146.js b/content/assets/js/ba47c136.a55b1584.js
similarity index 96%
rename from content/assets/js/ba47c136.0223e146.js
rename to content/assets/js/ba47c136.a55b1584.js
index d8950723b8d09..ea5336dc68bd9 100644
--- a/content/assets/js/ba47c136.0223e146.js
+++ b/content/assets/js/ba47c136.a55b1584.js
@@ -1 +1 @@
-"use strict";(self.webpackChunkhudi=self.webpackChunkhudi||[]).push([[97424],{90430:(e,t,i)=>{i.r(t),i.d(t,{assets:()=>o,contentTitle:()=>a,default:()=>c,frontMatter:()=>d,metadata:()=>r,toc:()=>l});const r=JSON.parse('{"id":"key_generation","title":"Key Generation","description":"Hudi needs some way to point to records in the table, so that base/log files can be merged efficiently for updates/deletes,","source":"@site/docs/key_generation.md","sourceDirName":".","slug":"/key_generation","permalink":"/docs/next/key_generation","draft":false,"unlisted":false,"editUrl":"https://github.com/apache/hudi/tree/asf-site/website/docs/key_generation.md","tags":[],"version":"current","frontMatter":{"title":"Key Generation","summary":"In this page, we describe key generation in Hudi.","toc":true,"last_modified_at":null},"sidebar":"docs","previous":{"title":"Table & Query Types","permalink":"/docs/next/table_types"},"next":{"title":"Record Mergers","permalink":"/docs/next/record_merger"}}');var n=i(74848),s=i(28453);const d={title:"Key Generation",summary:"In this page, we describe key generation in Hudi.",toc:!0,last_modified_at:null},a=void 0,o={},l=[{value:"Key Generators",id:"key-generators",level:2},{value:"SIMPLE",id:"simple",level:3},{value:"COMPLEX",id:"complex",level:3},{value:"NON_PARTITION",id:"non_partition",level:3},{value:"CUSTOM",id:"custom",level:3},{value:"TIMESTAMP",id:"timestamp",level:3},{value:"Timestamp is GMT",id:"timestamp-is-gmt",level:4},{value:"Timestamp is DATE_STRING",id:"timestamp-is-date_string",level:4},{value:"Scalar examples",id:"scalar-examples",level:4},{value:"ISO8601WithMsZ with Single Input format",id:"iso8601withmsz-with-single-input-format",level:4},{value:"ISO8601WithMsZ with Multiple Input formats",id:"iso8601withmsz-with-multiple-input-formats",level:4},{value:"ISO8601NoMs with offset using multiple input formats",id:"iso8601noms-with-offset-using-multiple-input-formats",level:4},{value:"Input as short date string and expect date in date format",id:"input-as-short-date-string-and-expect-date-in-date-format",level:4},{value:"Related Resources",id:"related-resources",level:2}];function h(e){const t={a:"a",code:"code",h2:"h2",h3:"h3",h4:"h4",li:"li",p:"p",pre:"pre",strong:"strong",table:"table",tbody:"tbody",td:"td",th:"th",thead:"thead",tr:"tr",ul:"ul",...(0,s.R)(),...e.components};return(0,n.jsxs)(n.Fragment,{children:[(0,n.jsx)(t.p,{children:"Hudi needs some way to point to records in the table, so that base/log files can be merged efficiently for updates/deletes,\nindex entries can reference these rows and records can move around within the table from clustering without side effects.\nIn fact, most databases adopt similar techniques. Every record in Hudi is uniquely identified a pair of record key and an optional\npartition path that can limit the scope of the key's uniqueness (non-global indexing). For tables with a global index, records are\nidentified by just the record key such that uniqueness is applied across partitions."}),"\n",(0,n.jsxs)(t.p,{children:["Using keys, Hudi can impose partition/table level uniqueness integrity constraint as well as enable fast updates and deletes on records. Record keys are materialized in a\nspecial ",(0,n.jsx)(t.code,{children:"_hoodie_record_key"})," field in the table, to ensure key uniqueness is maintained even when the record generation is changed\nduring the table's lifetime. Without materialization, there are no guarantees that the past data written for a new key is unique across the table."]}),"\n",(0,n.jsx)(t.p,{children:"Hudi offers many ways to generate record keys from the input data during writes."}),"\n",(0,n.jsxs)(t.ul,{children:["\n",(0,n.jsxs)(t.li,{children:["\n",(0,n.jsxs)(t.p,{children:["For Java client/Spark/Flink writers, Hudi provides built-in key generator classes (described below) as well as an ",(0,n.jsx)(t.a,{href:"https://github.com/apache/hudi/blob/master/hudi-common/src/main/java/org/apache/hudi/keygen/KeyGenerator.java",children:"interface"}),"\nto write custom implementations."]}),"\n"]}),"\n",(0,n.jsxs)(t.li,{children:["\n",(0,n.jsxs)(t.p,{children:["SQL engines offer options to pass in key fields and use ",(0,n.jsx)(t.code,{children:"PARTITIONED BY"})," clauses to control partitioning."]}),"\n"]}),"\n"]}),"\n",(0,n.jsx)(t.p,{children:"By default, Hudi auto-generates keys for INSERT, BULK_INSERT write operations, that are efficient\nfor compute, storage and read to meet the uniqueness requirements of the primary key. Auto generated keys are highly\ncompressible compared to UUIDs costing about $0.023 per GB in cloud storage and 3-10x computationally lighter to generate\nthan base64/uuid encoded keys."}),"\n",(0,n.jsx)(t.h2,{id:"key-generators",children:"Key Generators"}),"\n",(0,n.jsx)(t.p,{children:"Hudi provides several key generators out of the box for JVM users can use based on their need, while having a pluggable\ninterface for users to implement and use their own."}),"\n",(0,n.jsx)(t.p,{children:"Before diving into different types of key generators, let\u2019s go over some of the common configs relevant to key generators."}),"\n",(0,n.jsxs)(t.table,{children:[(0,n.jsx)(t.thead,{children:(0,n.jsxs)(t.tr,{children:[(0,n.jsx)(t.th,{children:"Config Name"}),(0,n.jsx)(t.th,{children:"Default"}),(0,n.jsx)(t.th,{children:"Description"})]})}),(0,n.jsxs)(t.tbody,{children:[(0,n.jsxs)(t.tr,{children:[(0,n.jsx)(t.td,{children:"hoodie.datasource.write.recordkey.field"}),(0,n.jsx)(t.td,{children:"N/A (Optional)"}),(0,n.jsxs)(t.td,{children:["Record key field. Value to be used as the ",(0,n.jsx)(t.code,{children:"recordKey"})," component of ",(0,n.jsx)(t.code,{children:"HoodieKey"}),". ",(0,n.jsxs)("ul",{children:[(0,n.jsxs)("li",{children:["When configured, actual value will be obtained by invoking .toString() on the field value. Nested fields can be specified using the dot notation eg: ",(0,n.jsx)(t.code,{children:"a.b.c"}),". "]}),(0,n.jsx)("li",{children:"When not configured record key will be automatically generated by Hudi. This feature is handy for use cases like log ingestion that do not have a naturally present record key."})]})," ",(0,n.jsx)("br",{}),(0,n.jsx)(t.code,{children:"Config Param: RECORDKEY_FIELD_NAME"})]})]}),(0,n.jsxs)(t.tr,{children:[(0,n.jsx)(t.td,{children:"hoodie.datasource.write.partitionpath.field"}),(0,n.jsx)(t.td,{children:"N/A (Optional)"}),(0,n.jsxs)(t.td,{children:["Partition path field. Value to be used at the partitionPath component of HoodieKey. This needs to be specified if a partitioned table is desired. Actual value obtained by invoking .toString()",(0,n.jsx)("br",{}),(0,n.jsx)(t.code,{children:"Config Param: PARTITIONPATH_FIELD_NAME"})]})]}),(0,n.jsxs)(t.tr,{children:[(0,n.jsx)(t.td,{children:"hoodie.datasource.write.keygenerator.type"}),(0,n.jsx)(t.td,{children:"SIMPLE"}),(0,n.jsxs)(t.td,{children:["String representing key generator type ",(0,n.jsx)("br",{}),(0,n.jsx)("br",{}),(0,n.jsx)(t.code,{children:"Config Param: KEYGENERATOR_TYPE"})]})]}),(0,n.jsxs)(t.tr,{children:[(0,n.jsx)(t.td,{children:"hoodie.datasource.write.keygenerator.class"}),(0,n.jsx)(t.td,{children:"N/A (Optional)"}),(0,n.jsxs)(t.td,{children:["Key generator class, that implements ",(0,n.jsx)(t.code,{children:"org.apache.hudi.keygen.KeyGenerator"})," extract a key out of incoming records. ",(0,n.jsxs)("ul",{children:[(0,n.jsx)("li",{children:"When set, the configured value takes precedence to be in effect and automatic inference is not triggered."}),(0,n.jsxs)("li",{children:["When not configured, if ",(0,n.jsx)(t.code,{children:"hoodie.datasource.write.keygenerator.type"})," is set, the configured value is used else automatic inference is triggered."]}),(0,n.jsx)("li",{children:"In case of auto generated record keys, if neither the key generator class nor type are configured, Hudi will also auto infer the partitioning. for eg, if partition field is not configured, hudi will assume its non-partitioned. "})]})," ",(0,n.jsx)("br",{}),(0,n.jsx)(t.code,{children:"Config Param: KEYGENERATOR_CLASS_NAME"})]})]}),(0,n.jsxs)(t.tr,{children:[(0,n.jsx)(t.td,{children:"hoodie.datasource.write.hive_style_partitioning"}),(0,n.jsx)(t.td,{children:"false (Optional)"}),(0,n.jsxs)(t.td,{children:["Flag to indicate whether to use Hive style partitioning. If set true, the names of partition folders follow = format. By default false (the names of partition folders are only partition values)",(0,n.jsx)("br",{}),(0,n.jsx)("br",{}),(0,n.jsx)(t.code,{children:"Config Param: HIVE_STYLE_PARTITIONING_ENABLE"})]})]}),(0,n.jsxs)(t.tr,{children:[(0,n.jsx)(t.td,{children:"hoodie.datasource.write.partitionpath.urlencode"}),(0,n.jsx)(t.td,{children:"false (Optional)"}),(0,n.jsxs)(t.td,{children:["Should we url encode the partition path value, before creating the folder structure.",(0,n.jsx)("br",{}),(0,n.jsx)("br",{}),(0,n.jsx)(t.code,{children:"Config Param: URL_ENCODE_PARTITIONING"})]})]})]})]}),"\n",(0,n.jsxs)(t.p,{children:["For all advanced configs refer ",(0,n.jsx)(t.a,{href:"https://hudi.apache.org/docs/next/configurations#KEY_GENERATOR",children:"here"}),"."]}),"\n",(0,n.jsx)(t.h3,{id:"simple",children:(0,n.jsx)(t.a,{href:"https://github.com/apache/hudi/blob/master/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/keygen/SimpleKeyGenerator.java",children:"SIMPLE"})}),"\n",(0,n.jsx)(t.p,{children:"This is the most commonly used option. Record key is generated from two fields from the schema, one for record key and one for partition path. Values are interpreted as is from dataframe and converted to string."}),"\n",(0,n.jsx)(t.h3,{id:"complex",children:(0,n.jsx)(t.a,{href:"https://github.com/apache/hudi/blob/master/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/keygen/ComplexKeyGenerator.java",children:"COMPLEX"})}),"\n",(0,n.jsxs)(t.p,{children:["Both record key and partition paths comprise one or more than one field by name(combination of multiple fields). Fields\nare expected to be comma separated in the config value. For example ",(0,n.jsx)(t.code,{children:'"Hoodie.datasource.write.recordkey.field" : \u201ccol1,col4\u201d'})]}),"\n",(0,n.jsx)(t.h3,{id:"non_partition",children:(0,n.jsx)(t.a,{href:"https://github.com/apache/hudi/blob/master/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/keygen/NonpartitionedKeyGenerator.java",children:"NON_PARTITION"})}),"\n",(0,n.jsx)(t.p,{children:"If your hudi dataset is not partitioned, you could use this \u201cNonpartitionedKeyGenerator\u201d which will return an empty\npartition for all records. In other words, all records go to the same partition (which is empty \u201c\u201d)"}),"\n",(0,n.jsx)(t.h3,{id:"custom",children:(0,n.jsx)(t.a,{href:"https://github.com/apache/hudi/blob/master/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/keygen/CustomKeyGenerator.java",children:"CUSTOM"})}),"\n",(0,n.jsx)(t.p,{children:"This is a generic implementation of KeyGenerator where users are able to leverage the benefits of SimpleKeyGenerator,\nComplexKeyGenerator and TimestampBasedKeyGenerator all at the same time. One can configure record key and partition\npaths as a single field or a combination of fields."}),"\n",(0,n.jsx)(t.pre,{children:(0,n.jsx)(t.code,{className:"language-java",children:"hoodie.datasource.write.recordkey.field\nhoodie.datasource.write.partitionpath.field\nhoodie.datasource.write.keygenerator.class=org.apache.hudi.keygen.CustomKeyGenerator\n"})}),"\n",(0,n.jsxs)(t.p,{children:["This keyGenerator is particularly useful if you want to define\ncomplex partition paths involving regular fields and timestamp based fields. It expects value for prop ",(0,n.jsx)(t.code,{children:'"hoodie.datasource.write.partitionpath.field"'}),'\nin a specific format. The format should be "field1',":PartitionKeyType1",",field2",":PartitionKeyType2",'..."']}),"\n",(0,n.jsxs)(t.p,{children:["The complete partition path is created as\n",(0,n.jsx)(t.code,{children:"/ "}),"\nand so on. Each partition key type could either be SIMPLE or TIMESTAMP."]}),"\n",(0,n.jsxs)(t.p,{children:["Example config value: ",(0,n.jsx)(t.code,{children:"\u201cfield_3:simple,field_5:timestamp\u201d"})]}),"\n",(0,n.jsx)(t.p,{children:"RecordKey config value is either single field incase of SimpleKeyGenerator or a comma separate field names if referring to ComplexKeyGenerator.\nExample:"}),"\n",(0,n.jsx)(t.pre,{children:(0,n.jsx)(t.code,{className:"language-java",children:"hoodie.datasource.write.recordkey.field=field1,field2\n"})}),"\n",(0,n.jsxs)(t.p,{children:["This will create your record key in the format ",(0,n.jsx)(t.code,{children:"field1:value1,field2:value2"})," and so on, otherwise you can specify only one field in case of simple record keys. ",(0,n.jsx)(t.code,{children:"CustomKeyGenerator"})," class defines an enum ",(0,n.jsx)(t.code,{children:"PartitionKeyType"})," for configuring partition paths. It can take two possible values - SIMPLE and TIMESTAMP.\nThe value for ",(0,n.jsx)(t.code,{children:"hoodie.datasource.write.partitionpath.field"})," property in case of partitioned tables needs to be provided in the format ",(0,n.jsx)(t.code,{children:"field1:PartitionKeyType1,field2:PartitionKeyType2"})," and so on. For example, if you want to create partition path using 2 fields ",(0,n.jsx)(t.code,{children:"country"})," and ",(0,n.jsx)(t.code,{children:"date"})," where the latter has timestamp based values and needs to be customised in a given format, you can specify the following"]}),"\n",(0,n.jsx)(t.pre,{children:(0,n.jsx)(t.code,{className:"language-java",children:"hoodie.datasource.write.partitionpath.field=country:SIMPLE,date:TIMESTAMP\n"})}),"\n",(0,n.jsxs)(t.p,{children:["This will create the partition path in the format ",(0,n.jsx)(t.code,{children:"/"})," or ",(0,n.jsx)(t.code,{children:"country=/date="})," depending on whether you want hive style partitioning or not."]}),"\n",(0,n.jsx)(t.h3,{id:"timestamp",children:(0,n.jsx)(t.a,{href:"https://github.com/apache/hudi/blob/master/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/keygen/TimestampBasedKeyGenerator.java",children:"TIMESTAMP"})}),"\n",(0,n.jsx)(t.p,{children:"This key generator relies on timestamps for the partition field. The field values are interpreted as timestamps\nand not just converted to string while generating partition path value for records. Record key is same as before where it is chosen by\nfield name. Users are expected to set few more configs to use this KeyGenerator."}),"\n",(0,n.jsx)(t.p,{children:"Configs to be set:"}),"\n",(0,n.jsxs)(t.table,{children:[(0,n.jsx)(t.thead,{children:(0,n.jsxs)(t.tr,{children:[(0,n.jsx)(t.th,{children:"Config Name"}),(0,n.jsx)(t.th,{children:"Default"}),(0,n.jsx)(t.th,{children:"Description"})]})}),(0,n.jsxs)(t.tbody,{children:[(0,n.jsxs)(t.tr,{children:[(0,n.jsx)(t.td,{children:"hoodie.keygen.timebased.timestamp.type"}),(0,n.jsxs)(t.td,{children:["N/A ",(0,n.jsx)(t.strong,{children:"(Required)"})]}),(0,n.jsx)(t.td,{children:"Required only when the key generator is TimestampBasedKeyGenerator. One of the timestamp types supported(UNIX_TIMESTAMP, DATE_STRING, MIXED, EPOCHMILLISECONDS, SCALAR)"})]}),(0,n.jsxs)(t.tr,{children:[(0,n.jsx)(t.td,{children:"hoodie.keygen.timebased.output.dateformat"}),(0,n.jsx)(t.td,{children:'"" (Optional)'}),(0,n.jsxs)(t.td,{children:["Output date format such as ",(0,n.jsx)(t.code,{children:"yyyy-MM-dd'T'HH:mm:ss.SSSZ"})]})]}),(0,n.jsxs)(t.tr,{children:[(0,n.jsx)(t.td,{children:"hoodie.keygen.timebased.timezone"}),(0,n.jsx)(t.td,{children:'"UTC" (Optional)'}),(0,n.jsxs)(t.td,{children:["Timezone of both input and output timestamp if they are the same, such as ",(0,n.jsx)(t.code,{children:"UTC"}),". Please use ",(0,n.jsx)(t.code,{children:"hoodie.keygen.timebased.input.timezone"})," and ",(0,n.jsx)(t.code,{children:"hoodie.keygen.timebased.output.timezone"})," instead if the input and output timezones are different."]})]}),(0,n.jsxs)(t.tr,{children:[(0,n.jsx)(t.td,{children:"hoodie.keygen.timebased.input.dateformat"}),(0,n.jsx)(t.td,{children:'"" (Optional)'}),(0,n.jsxs)(t.td,{children:["Input date format such as ",(0,n.jsx)(t.code,{children:"yyyy-MM-dd'T'HH:mm:ss.SSSZ"}),"."]})]})]})]}),"\n",(0,n.jsx)(t.p,{children:"Let's go over some example values for TimestampBasedKeyGenerator."}),"\n",(0,n.jsx)(t.h4,{id:"timestamp-is-gmt",children:"Timestamp is GMT"}),"\n",(0,n.jsxs)(t.table,{children:[(0,n.jsx)(t.thead,{children:(0,n.jsxs)(t.tr,{children:[(0,n.jsx)(t.th,{children:"Config Name"}),(0,n.jsx)(t.th,{children:"Value"})]})}),(0,n.jsxs)(t.tbody,{children:[(0,n.jsxs)(t.tr,{children:[(0,n.jsx)(t.td,{children:(0,n.jsx)(t.code,{children:"hoodie.streamer.keygen.timebased.timestamp.type"})}),(0,n.jsx)(t.td,{children:'"EPOCHMILLISECONDS"'})]}),(0,n.jsxs)(t.tr,{children:[(0,n.jsx)(t.td,{children:(0,n.jsx)(t.code,{children:"hoodie.streamer.keygen.timebased.output.dateformat"})}),(0,n.jsx)(t.td,{children:'"yyyy-MM-dd hh"'})]}),(0,n.jsxs)(t.tr,{children:[(0,n.jsx)(t.td,{children:(0,n.jsx)(t.code,{children:"hoodie.streamer.keygen.timebased.timezone"})}),(0,n.jsx)(t.td,{children:'"GMT+8:00"'})]})]})]}),"\n",(0,n.jsxs)(t.p,{children:["Input Field value: \u201c1578283932000L\u201d ",(0,n.jsx)("br",{}),"\nPartition path generated from key generator: \u201c2020-01-06 12\u201d"]}),"\n",(0,n.jsxs)(t.p,{children:["If input field value is null for some rows. ",(0,n.jsx)("br",{}),"\nPartition path generated from key generator: \u201c1970-01-01 08\u201d"]}),"\n",(0,n.jsx)(t.h4,{id:"timestamp-is-date_string",children:"Timestamp is DATE_STRING"}),"\n",(0,n.jsxs)(t.table,{children:[(0,n.jsx)(t.thead,{children:(0,n.jsxs)(t.tr,{children:[(0,n.jsx)(t.th,{children:"Config Name"}),(0,n.jsx)(t.th,{children:"Value"})]})}),(0,n.jsxs)(t.tbody,{children:[(0,n.jsxs)(t.tr,{children:[(0,n.jsx)(t.td,{children:(0,n.jsx)(t.code,{children:"hoodie.streamer.keygen.timebased.timestamp.type"})}),(0,n.jsx)(t.td,{children:'"DATE_STRING"'})]}),(0,n.jsxs)(t.tr,{children:[(0,n.jsx)(t.td,{children:(0,n.jsx)(t.code,{children:"hoodie.streamer.keygen.timebased.output.dateformat"})}),(0,n.jsx)(t.td,{children:'"yyyy-MM-dd hh"'})]}),(0,n.jsxs)(t.tr,{children:[(0,n.jsx)(t.td,{children:(0,n.jsx)(t.code,{children:"hoodie.streamer.keygen.timebased.timezone"})}),(0,n.jsx)(t.td,{children:'"GMT+8:00"'})]}),(0,n.jsxs)(t.tr,{children:[(0,n.jsx)(t.td,{children:(0,n.jsx)(t.code,{children:"hoodie.streamer.keygen.timebased.input.dateformat"})}),(0,n.jsxs)(t.td,{children:['"yyyy-MM-dd hh:mm',":ss",'"']})]})]})]}),"\n",(0,n.jsxs)(t.p,{children:["Input field value: \u201c2020-01-06 12:12:12\u201d ",(0,n.jsx)("br",{}),"\nPartition path generated from key generator: \u201c2020-01-06 12\u201d"]}),"\n",(0,n.jsxs)(t.p,{children:["If input field value is null for some rows. ",(0,n.jsx)("br",{}),"\nPartition path generated from key generator: \u201c1970-01-01 12:00:00\u201d"]}),"\n",(0,n.jsx)("br",{}),"\n",(0,n.jsx)(t.h4,{id:"scalar-examples",children:"Scalar examples"}),"\n",(0,n.jsxs)(t.table,{children:[(0,n.jsx)(t.thead,{children:(0,n.jsxs)(t.tr,{children:[(0,n.jsx)(t.th,{children:"Config Name"}),(0,n.jsx)(t.th,{children:"Value"})]})}),(0,n.jsxs)(t.tbody,{children:[(0,n.jsxs)(t.tr,{children:[(0,n.jsx)(t.td,{children:(0,n.jsx)(t.code,{children:"hoodie.streamer.keygen.timebased.timestamp.type"})}),(0,n.jsx)(t.td,{children:'"SCALAR"'})]}),(0,n.jsxs)(t.tr,{children:[(0,n.jsx)(t.td,{children:(0,n.jsx)(t.code,{children:"hoodie.streamer.keygen.timebased.output.dateformat"})}),(0,n.jsx)(t.td,{children:'"yyyy-MM-dd hh"'})]}),(0,n.jsxs)(t.tr,{children:[(0,n.jsx)(t.td,{children:(0,n.jsx)(t.code,{children:"hoodie.streamer.keygen.timebased.timezone"})}),(0,n.jsx)(t.td,{children:'"GMT"'})]}),(0,n.jsxs)(t.tr,{children:[(0,n.jsx)(t.td,{children:(0,n.jsx)(t.code,{children:"hoodie.streamer.keygen.timebased.timestamp.scalar.time.unit"})}),(0,n.jsx)(t.td,{children:'"days"'})]})]})]}),"\n",(0,n.jsxs)(t.p,{children:["Input field value: \u201c20000L\u201d ",(0,n.jsx)("br",{}),"\nPartition path generated from key generator: \u201c2024-10-04 12\u201d"]}),"\n",(0,n.jsxs)(t.p,{children:["If input field value is null. ",(0,n.jsx)("br",{}),"\nPartition path generated from key generator: \u201c1970-01-02 12\u201d"]}),"\n",(0,n.jsx)(t.h4,{id:"iso8601withmsz-with-single-input-format",children:"ISO8601WithMsZ with Single Input format"}),"\n",(0,n.jsxs)(t.table,{children:[(0,n.jsx)(t.thead,{children:(0,n.jsxs)(t.tr,{children:[(0,n.jsx)(t.th,{children:"Config Name"}),(0,n.jsx)(t.th,{children:"Value"})]})}),(0,n.jsxs)(t.tbody,{children:[(0,n.jsxs)(t.tr,{children:[(0,n.jsx)(t.td,{children:(0,n.jsx)(t.code,{children:"hoodie.streamer.keygen.timebased.timestamp.type"})}),(0,n.jsx)(t.td,{children:'"DATE_STRING"'})]}),(0,n.jsxs)(t.tr,{children:[(0,n.jsx)(t.td,{children:(0,n.jsx)(t.code,{children:"hoodie.streamer.keygen.timebased.input.dateformat"})}),(0,n.jsxs)(t.td,{children:["\"yyyy-MM-dd'T'HH:mm",":ss",'.SSSZ"']})]}),(0,n.jsxs)(t.tr,{children:[(0,n.jsx)(t.td,{children:(0,n.jsx)(t.code,{children:"hoodie.streamer.keygen.timebased.input.dateformat.list.delimiter.regex"})}),(0,n.jsx)(t.td,{children:'""'})]}),(0,n.jsxs)(t.tr,{children:[(0,n.jsx)(t.td,{children:(0,n.jsx)(t.code,{children:"hoodie.streamer.keygen.timebased.input.timezone"})}),(0,n.jsx)(t.td,{children:'""'})]}),(0,n.jsxs)(t.tr,{children:[(0,n.jsx)(t.td,{children:(0,n.jsx)(t.code,{children:"hoodie.streamer.keygen.timebased.output.dateformat"})}),(0,n.jsx)(t.td,{children:'"yyyyMMddHH"'})]}),(0,n.jsxs)(t.tr,{children:[(0,n.jsx)(t.td,{children:(0,n.jsx)(t.code,{children:"hoodie.streamer.keygen.timebased.output.timezone"})}),(0,n.jsx)(t.td,{children:'"GMT"'})]})]})]}),"\n",(0,n.jsxs)(t.p,{children:['Input field value: "2020-04-01T13:01:33.428Z" ',(0,n.jsx)("br",{}),'\nPartition path generated from key generator: "2020040113"']}),"\n",(0,n.jsx)(t.h4,{id:"iso8601withmsz-with-multiple-input-formats",children:"ISO8601WithMsZ with Multiple Input formats"}),"\n",(0,n.jsxs)(t.table,{children:[(0,n.jsx)(t.thead,{children:(0,n.jsxs)(t.tr,{children:[(0,n.jsx)(t.th,{children:"Config Name"}),(0,n.jsx)(t.th,{children:"Value"})]})}),(0,n.jsxs)(t.tbody,{children:[(0,n.jsxs)(t.tr,{children:[(0,n.jsx)(t.td,{children:(0,n.jsx)(t.code,{children:"hoodie.streamer.keygen.timebased.timestamp.type"})}),(0,n.jsx)(t.td,{children:'"DATE_STRING"'})]}),(0,n.jsxs)(t.tr,{children:[(0,n.jsx)(t.td,{children:(0,n.jsx)(t.code,{children:"hoodie.streamer.keygen.timebased.input.dateformat"})}),(0,n.jsxs)(t.td,{children:["\"yyyy-MM-dd'T'HH:mm",":ssZ",",yyyy-MM-dd'T'HH:mm",":ss",'.SSSZ"']})]}),(0,n.jsxs)(t.tr,{children:[(0,n.jsx)(t.td,{children:(0,n.jsx)(t.code,{children:"hoodie.streamer.keygen.timebased.input.dateformat.list.delimiter.regex"})}),(0,n.jsx)(t.td,{children:'""'})]}),(0,n.jsxs)(t.tr,{children:[(0,n.jsx)(t.td,{children:(0,n.jsx)(t.code,{children:"hoodie.streamer.keygen.timebased.input.timezone"})}),(0,n.jsx)(t.td,{children:'""'})]}),(0,n.jsxs)(t.tr,{children:[(0,n.jsx)(t.td,{children:(0,n.jsx)(t.code,{children:"hoodie.streamer.keygen.timebased.output.dateformat"})}),(0,n.jsx)(t.td,{children:'"yyyyMMddHH"'})]}),(0,n.jsxs)(t.tr,{children:[(0,n.jsx)(t.td,{children:(0,n.jsx)(t.code,{children:"hoodie.streamer.keygen.timebased.output.timezone"})}),(0,n.jsx)(t.td,{children:'"UTC"'})]})]})]}),"\n",(0,n.jsxs)(t.p,{children:['Input field value: "2020-04-01T13:01:33.428Z" ',(0,n.jsx)("br",{}),'\nPartition path generated from key generator: "2020040113"']}),"\n",(0,n.jsx)(t.h4,{id:"iso8601noms-with-offset-using-multiple-input-formats",children:"ISO8601NoMs with offset using multiple input formats"}),"\n",(0,n.jsxs)(t.table,{children:[(0,n.jsx)(t.thead,{children:(0,n.jsxs)(t.tr,{children:[(0,n.jsx)(t.th,{children:"Config Name"}),(0,n.jsx)(t.th,{children:"Value"})]})}),(0,n.jsxs)(t.tbody,{children:[(0,n.jsxs)(t.tr,{children:[(0,n.jsx)(t.td,{children:(0,n.jsx)(t.code,{children:"hoodie.streamer.keygen.timebased.timestamp.type"})}),(0,n.jsx)(t.td,{children:'"DATE_STRING"'})]}),(0,n.jsxs)(t.tr,{children:[(0,n.jsx)(t.td,{children:(0,n.jsx)(t.code,{children:"hoodie.streamer.keygen.timebased.input.dateformat"})}),(0,n.jsxs)(t.td,{children:["\"yyyy-MM-dd'T'HH:mm",":ssZ",",yyyy-MM-dd'T'HH:mm",":ss",'.SSSZ"']})]}),(0,n.jsxs)(t.tr,{children:[(0,n.jsx)(t.td,{children:(0,n.jsx)(t.code,{children:"hoodie.streamer.keygen.timebased.input.dateformat.list.delimiter.regex"})}),(0,n.jsx)(t.td,{children:'""'})]}),(0,n.jsxs)(t.tr,{children:[(0,n.jsx)(t.td,{children:(0,n.jsx)(t.code,{children:"hoodie.streamer.keygen.timebased.input.timezone"})}),(0,n.jsx)(t.td,{children:'""'})]}),(0,n.jsxs)(t.tr,{children:[(0,n.jsx)(t.td,{children:(0,n.jsx)(t.code,{children:"hoodie.streamer.keygen.timebased.output.dateformat"})}),(0,n.jsx)(t.td,{children:'"yyyyMMddHH"'})]}),(0,n.jsxs)(t.tr,{children:[(0,n.jsx)(t.td,{children:(0,n.jsx)(t.code,{children:"hoodie.streamer.keygen.timebased.output.timezone"})}),(0,n.jsx)(t.td,{children:'"UTC"'})]})]})]}),"\n",(0,n.jsxs)(t.p,{children:['Input field value: "2020-04-01T13:01:33-',(0,n.jsx)(t.strong,{children:"05:00"}),'" ',(0,n.jsx)("br",{}),'\nPartition path generated from key generator: "2020040118"']}),"\n",(0,n.jsx)(t.h4,{id:"input-as-short-date-string-and-expect-date-in-date-format",children:"Input as short date string and expect date in date format"}),"\n",(0,n.jsxs)(t.table,{children:[(0,n.jsx)(t.thead,{children:(0,n.jsxs)(t.tr,{children:[(0,n.jsx)(t.th,{children:"Config Name"}),(0,n.jsx)(t.th,{children:"Value"})]})}),(0,n.jsxs)(t.tbody,{children:[(0,n.jsxs)(t.tr,{children:[(0,n.jsx)(t.td,{children:(0,n.jsx)(t.code,{children:"hoodie.streamer.keygen.timebased.timestamp.type"})}),(0,n.jsx)(t.td,{children:'"DATE_STRING"'})]}),(0,n.jsxs)(t.tr,{children:[(0,n.jsx)(t.td,{children:(0,n.jsx)(t.code,{children:"hoodie.streamer.keygen.timebased.input.dateformat"})}),(0,n.jsxs)(t.td,{children:["\"yyyy-MM-dd'T'HH:mm",":ssZ",",yyyy-MM-dd'T'HH:mm",":ss",'.SSSZ,yyyyMMdd"']})]}),(0,n.jsxs)(t.tr,{children:[(0,n.jsx)(t.td,{children:(0,n.jsx)(t.code,{children:"hoodie.streamer.keygen.timebased.input.dateformat.list.delimiter.regex"})}),(0,n.jsx)(t.td,{children:'""'})]}),(0,n.jsxs)(t.tr,{children:[(0,n.jsx)(t.td,{children:(0,n.jsx)(t.code,{children:"hoodie.streamer.keygen.timebased.input.timezone"})}),(0,n.jsx)(t.td,{children:'"UTC"'})]}),(0,n.jsxs)(t.tr,{children:[(0,n.jsx)(t.td,{children:(0,n.jsx)(t.code,{children:"hoodie.streamer.keygen.timebased.output.dateformat"})}),(0,n.jsx)(t.td,{children:'"MM/dd/yyyy"'})]}),(0,n.jsxs)(t.tr,{children:[(0,n.jsx)(t.td,{children:(0,n.jsx)(t.code,{children:"hoodie.streamer.keygen.timebased.output.timezone"})}),(0,n.jsx)(t.td,{children:'"UTC"'})]})]})]}),"\n",(0,n.jsxs)(t.p,{children:['Input field value: "20200401" ',(0,n.jsx)("br",{}),'\nPartition path generated from key generator: "04/01/2020"']}),"\n",(0,n.jsx)(t.h2,{id:"related-resources",children:"Related Resources"}),"\n",(0,n.jsxs)(t.ul,{children:["\n",(0,n.jsx)(t.li,{children:(0,n.jsx)(t.a,{href:"https://www.onehouse.ai/blog/hudi-metafields-demystified",children:"Hudi metafields demystified"})}),"\n"]})]})}function c(e={}){const{wrapper:t}={...(0,s.R)(),...e.components};return t?(0,n.jsx)(t,{...e,children:(0,n.jsx)(h,{...e})}):h(e)}},28453:(e,t,i)=>{i.d(t,{R:()=>d,x:()=>a});var r=i(96540);const n={},s=r.createContext(n);function d(e){const t=r.useContext(s);return r.useMemo((function(){return"function"==typeof e?e(t):{...t,...e}}),[t,e])}function a(e){let t;return t=e.disableParentContext?"function"==typeof e.components?e.components(n):e.components||n:d(e.components),r.createElement(s.Provider,{value:t},e.children)}}}]);
\ No newline at end of file
+"use strict";(self.webpackChunkhudi=self.webpackChunkhudi||[]).push([[97424],{90430:(e,t,i)=>{i.r(t),i.d(t,{assets:()=>o,contentTitle:()=>a,default:()=>c,frontMatter:()=>d,metadata:()=>r,toc:()=>l});const r=JSON.parse('{"id":"key_generation","title":"Key Generation","description":"Hudi needs some way to point to records in the table, so that base/log files can be merged efficiently for updates/deletes,","source":"@site/docs/key_generation.md","sourceDirName":".","slug":"/key_generation","permalink":"/docs/next/key_generation","draft":false,"unlisted":false,"editUrl":"https://github.com/apache/hudi/tree/asf-site/website/docs/key_generation.md","tags":[],"version":"current","frontMatter":{"title":"Key Generation","summary":"In this page, we describe key generation in Hudi.","toc":true,"last_modified_at":null},"sidebar":"docs","previous":{"title":"Table & Query Types","permalink":"/docs/next/table_types"},"next":{"title":"Record Mergers","permalink":"/docs/next/record_merger"}}');var n=i(74848),s=i(28453);const d={title:"Key Generation",summary:"In this page, we describe key generation in Hudi.",toc:!0,last_modified_at:null},a=void 0,o={},l=[{value:"Key Generators",id:"key-generators",level:2},{value:"SIMPLE",id:"simple",level:3},{value:"COMPLEX",id:"complex",level:3},{value:"NON_PARTITION",id:"non_partition",level:3},{value:"CUSTOM",id:"custom",level:3},{value:"TIMESTAMP",id:"timestamp",level:3},{value:"Timestamp is GMT",id:"timestamp-is-gmt",level:4},{value:"Timestamp is DATE_STRING",id:"timestamp-is-date_string",level:4},{value:"Scalar examples",id:"scalar-examples",level:4},{value:"ISO8601WithMsZ with Single Input format",id:"iso8601withmsz-with-single-input-format",level:4},{value:"ISO8601WithMsZ with Multiple Input formats",id:"iso8601withmsz-with-multiple-input-formats",level:4},{value:"ISO8601NoMs with offset using multiple input formats",id:"iso8601noms-with-offset-using-multiple-input-formats",level:4},{value:"Input as short date string and expect date in date format",id:"input-as-short-date-string-and-expect-date-in-date-format",level:4},{value:"Related Resources",id:"related-resources",level:2}];function h(e){const t={a:"a",code:"code",h2:"h2",h3:"h3",h4:"h4",li:"li",p:"p",pre:"pre",strong:"strong",table:"table",tbody:"tbody",td:"td",th:"th",thead:"thead",tr:"tr",ul:"ul",...(0,s.R)(),...e.components};return(0,n.jsxs)(n.Fragment,{children:[(0,n.jsx)(t.p,{children:"Hudi needs some way to point to records in the table, so that base/log files can be merged efficiently for updates/deletes,\nindex entries can reference these rows and records can move around within the table from clustering without side effects.\nIn fact, most databases adopt similar techniques. Every record in Hudi is uniquely identified a pair of record key and an optional\npartition path that can limit the scope of the key's uniqueness (non-global indexing). For tables with a global index, records are\nidentified by just the record key such that uniqueness is applied across partitions."}),"\n",(0,n.jsxs)(t.p,{children:["Using keys, Hudi can impose partition/table level uniqueness integrity constraint as well as enable fast updates and deletes on records. Record keys are materialized in a\nspecial ",(0,n.jsx)(t.code,{children:"_hoodie_record_key"})," field in the table, to ensure key uniqueness is maintained even when the record generation is changed\nduring the table's lifetime. Without materialization, there are no guarantees that the past data written for a new key is unique across the table."]}),"\n",(0,n.jsx)(t.p,{children:"Hudi offers many ways to generate record keys from the input data during writes."}),"\n",(0,n.jsxs)(t.ul,{children:["\n",(0,n.jsxs)(t.li,{children:["\n",(0,n.jsxs)(t.p,{children:["For Java client/Spark/Flink writers, Hudi provides built-in key generator classes (described below) as well as an ",(0,n.jsx)(t.a,{href:"https://github.com/apache/hudi/blob/master/hudi-common/src/main/java/org/apache/hudi/keygen/KeyGenerator.java",children:"interface"}),"\nto write custom implementations."]}),"\n"]}),"\n",(0,n.jsxs)(t.li,{children:["\n",(0,n.jsxs)(t.p,{children:["SQL engines offer options to pass in key fields and use ",(0,n.jsx)(t.code,{children:"PARTITIONED BY"})," clauses to control partitioning."]}),"\n"]}),"\n"]}),"\n",(0,n.jsx)(t.p,{children:"By default, Hudi auto-generates keys for INSERT, BULK_INSERT write operations, that are efficient\nfor compute, storage and read to meet the uniqueness requirements of the primary key. Auto generated keys are highly\ncompressible compared to UUIDs costing about $0.023 per GB in cloud storage and 3-10x computationally lighter to generate\nthan base64/uuid encoded keys."}),"\n",(0,n.jsx)(t.h2,{id:"key-generators",children:"Key Generators"}),"\n",(0,n.jsx)(t.p,{children:"Hudi provides several key generators out of the box for JVM users can use based on their need, while having a pluggable\ninterface for users to implement and use their own."}),"\n",(0,n.jsx)(t.p,{children:"Before diving into different types of key generators, let\u2019s go over some of the common configs relevant to key generators."}),"\n",(0,n.jsxs)(t.table,{children:[(0,n.jsx)(t.thead,{children:(0,n.jsxs)(t.tr,{children:[(0,n.jsx)(t.th,{children:"Config Name"}),(0,n.jsx)(t.th,{children:"Default"}),(0,n.jsx)(t.th,{children:"Description"})]})}),(0,n.jsxs)(t.tbody,{children:[(0,n.jsxs)(t.tr,{children:[(0,n.jsx)(t.td,{children:"hoodie.datasource.write.recordkey.field"}),(0,n.jsx)(t.td,{children:"N/A (Optional)"}),(0,n.jsxs)(t.td,{children:["Record key field. Value to be used as the ",(0,n.jsx)(t.code,{children:"recordKey"})," component of ",(0,n.jsx)(t.code,{children:"HoodieKey"}),". ",(0,n.jsxs)("ul",{children:[(0,n.jsxs)("li",{children:["When configured, actual value will be obtained by invoking .toString() on the field value. Nested fields can be specified using the dot notation eg: ",(0,n.jsx)(t.code,{children:"a.b.c"}),". "]}),(0,n.jsx)("li",{children:"When not configured record key will be automatically generated by Hudi. This feature is handy for use cases like log ingestion that do not have a naturally present record key."})]})," ",(0,n.jsx)("br",{}),(0,n.jsx)(t.code,{children:"Config Param: RECORDKEY_FIELD_NAME"})]})]}),(0,n.jsxs)(t.tr,{children:[(0,n.jsx)(t.td,{children:"hoodie.datasource.write.partitionpath.field"}),(0,n.jsx)(t.td,{children:"N/A (Optional)"}),(0,n.jsxs)(t.td,{children:["Partition path field. Value to be used at the partitionPath component of HoodieKey. This needs to be specified if a partitioned table is desired. Actual value obtained by invoking .toString()",(0,n.jsx)("br",{}),(0,n.jsx)(t.code,{children:"Config Param: PARTITIONPATH_FIELD_NAME"})]})]}),(0,n.jsxs)(t.tr,{children:[(0,n.jsx)(t.td,{children:"hoodie.datasource.write.keygenerator.type"}),(0,n.jsx)(t.td,{children:"SIMPLE"}),(0,n.jsxs)(t.td,{children:["String representing key generator type ",(0,n.jsx)("br",{}),(0,n.jsx)("br",{}),(0,n.jsx)(t.code,{children:"Config Param: KEYGENERATOR_TYPE"})]})]}),(0,n.jsxs)(t.tr,{children:[(0,n.jsx)(t.td,{children:"hoodie.datasource.write.keygenerator.class"}),(0,n.jsx)(t.td,{children:"N/A (Optional)"}),(0,n.jsxs)(t.td,{children:["Key generator class, that implements ",(0,n.jsx)(t.code,{children:"org.apache.hudi.keygen.KeyGenerator"})," extract a key out of incoming records. ",(0,n.jsxs)("ul",{children:[(0,n.jsx)("li",{children:"When set, the configured value takes precedence to be in effect and automatic inference is not triggered."}),(0,n.jsxs)("li",{children:["When not configured, if ",(0,n.jsx)(t.code,{children:"hoodie.datasource.write.keygenerator.type"})," is set, the configured value is used else automatic inference is triggered."]}),(0,n.jsx)("li",{children:"In case of auto generated record keys, if neither the key generator class nor type are configured, Hudi will also auto infer the partitioning. for eg, if partition field is not configured, hudi will assume its non-partitioned. "})]})," ",(0,n.jsx)("br",{}),(0,n.jsx)(t.code,{children:"Config Param: KEYGENERATOR_CLASS_NAME"})]})]}),(0,n.jsxs)(t.tr,{children:[(0,n.jsx)(t.td,{children:"hoodie.datasource.write.hive_style_partitioning"}),(0,n.jsx)(t.td,{children:"false (Optional)"}),(0,n.jsxs)(t.td,{children:["Flag to indicate whether to use Hive style partitioning. If set true, the names of partition folders follow = format. By default false (the names of partition folders are only partition values)",(0,n.jsx)("br",{}),(0,n.jsx)("br",{}),(0,n.jsx)(t.code,{children:"Config Param: HIVE_STYLE_PARTITIONING_ENABLE"})]})]}),(0,n.jsxs)(t.tr,{children:[(0,n.jsx)(t.td,{children:"hoodie.datasource.write.partitionpath.urlencode"}),(0,n.jsx)(t.td,{children:"false (Optional)"}),(0,n.jsxs)(t.td,{children:["Should we url encode the partition path value, before creating the folder structure.",(0,n.jsx)("br",{}),(0,n.jsx)("br",{}),(0,n.jsx)(t.code,{children:"Config Param: URL_ENCODE_PARTITIONING"})]})]})]})]}),"\n",(0,n.jsxs)(t.p,{children:["For all advanced configs refer ",(0,n.jsx)(t.a,{href:"https://hudi.apache.org/docs/next/configurations#KEY_GENERATOR",children:"here"}),"."]}),"\n",(0,n.jsx)(t.h3,{id:"simple",children:(0,n.jsx)(t.a,{href:"https://github.com/apache/hudi/blob/master/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/keygen/SimpleKeyGenerator.java",children:"SIMPLE"})}),"\n",(0,n.jsx)(t.p,{children:"This is the most commonly used option. Record key is generated from two fields from the schema, one for record key and one for partition path. Values are interpreted as is from dataframe and converted to string."}),"\n",(0,n.jsx)(t.h3,{id:"complex",children:(0,n.jsx)(t.a,{href:"https://github.com/apache/hudi/blob/master/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/keygen/ComplexKeyGenerator.java",children:"COMPLEX"})}),"\n",(0,n.jsxs)(t.p,{children:["Both record key and partition paths comprise one or more than one field by name(combination of multiple fields). Fields\nare expected to be comma separated in the config value. For example ",(0,n.jsx)(t.code,{children:'"Hoodie.datasource.write.recordkey.field" : \u201ccol1,col4\u201d'})]}),"\n",(0,n.jsx)(t.h3,{id:"non_partition",children:(0,n.jsx)(t.a,{href:"https://github.com/apache/hudi/blob/master/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/keygen/NonpartitionedKeyGenerator.java",children:"NON_PARTITION"})}),"\n",(0,n.jsx)(t.p,{children:"If your hudi dataset is not partitioned, you could use this \u201cNonpartitionedKeyGenerator\u201d which will return an empty\npartition for all records. In other words, all records go to the same partition (which is empty \u201c\u201d)"}),"\n",(0,n.jsx)(t.h3,{id:"custom",children:(0,n.jsx)(t.a,{href:"https://github.com/apache/hudi/blob/master/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/keygen/CustomKeyGenerator.java",children:"CUSTOM"})}),"\n",(0,n.jsx)(t.p,{children:"This is a generic implementation of KeyGenerator where users are able to leverage the benefits of SimpleKeyGenerator,\nComplexKeyGenerator and TimestampBasedKeyGenerator all at the same time. One can configure record key and partition\npaths as a single field or a combination of fields."}),"\n",(0,n.jsx)(t.pre,{children:(0,n.jsx)(t.code,{className:"language-java",children:"hoodie.datasource.write.recordkey.field\nhoodie.datasource.write.partitionpath.field\nhoodie.datasource.write.keygenerator.class=org.apache.hudi.keygen.CustomKeyGenerator\n"})}),"\n",(0,n.jsxs)(t.p,{children:["This keyGenerator is particularly useful if you want to define\ncomplex partition paths involving regular fields and timestamp based fields. It expects value for prop ",(0,n.jsx)(t.code,{children:'"hoodie.datasource.write.partitionpath.field"'}),'\nin a specific format. The format should be "field1',":PartitionKeyType1",",field2",":PartitionKeyType2",'..."']}),"\n",(0,n.jsxs)(t.p,{children:["The complete partition path is created as\n",(0,n.jsx)(t.code,{children:"/ "}),"\nand so on. Each partition key type could either be SIMPLE or TIMESTAMP."]}),"\n",(0,n.jsxs)(t.p,{children:["Example config value: ",(0,n.jsx)(t.code,{children:"\u201cfield_3:simple,field_5:timestamp\u201d"})]}),"\n",(0,n.jsx)(t.p,{children:"RecordKey config value is either single field incase of SimpleKeyGenerator or a comma separate field names if referring to ComplexKeyGenerator.\nExample:"}),"\n",(0,n.jsx)(t.pre,{children:(0,n.jsx)(t.code,{className:"language-java",children:"hoodie.datasource.write.recordkey.field=field1,field2\n"})}),"\n",(0,n.jsxs)(t.p,{children:["This will create your record key in the format ",(0,n.jsx)(t.code,{children:"field1:value1,field2:value2"})," and so on, otherwise you can specify only one field in case of simple record keys. ",(0,n.jsx)(t.code,{children:"CustomKeyGenerator"})," class defines an enum ",(0,n.jsx)(t.code,{children:"PartitionKeyType"})," for configuring partition paths. It can take two possible values - SIMPLE and TIMESTAMP.\nThe value for ",(0,n.jsx)(t.code,{children:"hoodie.datasource.write.partitionpath.field"})," property in case of partitioned tables needs to be provided in the format ",(0,n.jsx)(t.code,{children:"field1:PartitionKeyType1,field2:PartitionKeyType2"})," and so on. For example, if you want to create partition path using 2 fields ",(0,n.jsx)(t.code,{children:"country"})," and ",(0,n.jsx)(t.code,{children:"date"})," where the latter has timestamp based values and needs to be customised in a given format, you can specify the following"]}),"\n",(0,n.jsx)(t.pre,{children:(0,n.jsx)(t.code,{className:"language-java",children:"hoodie.datasource.write.partitionpath.field=country:SIMPLE,date:TIMESTAMP\n"})}),"\n",(0,n.jsxs)(t.p,{children:["This will create the partition path in the format ",(0,n.jsx)(t.code,{children:"/"})," or ",(0,n.jsx)(t.code,{children:"country=/date="})," depending on whether you want hive style partitioning or not."]}),"\n",(0,n.jsx)(t.h3,{id:"timestamp",children:(0,n.jsx)(t.a,{href:"https://github.com/apache/hudi/blob/master/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/keygen/TimestampBasedKeyGenerator.java",children:"TIMESTAMP"})}),"\n",(0,n.jsx)(t.p,{children:"This key generator relies on timestamps for the partition field. The field values are interpreted as timestamps\nand not just converted to string while generating partition path value for records. Record key is same as before where it is chosen by\nfield name. Users are expected to set few more configs to use this KeyGenerator."}),"\n",(0,n.jsx)(t.p,{children:"Configs to be set:"}),"\n",(0,n.jsxs)(t.table,{children:[(0,n.jsx)(t.thead,{children:(0,n.jsxs)(t.tr,{children:[(0,n.jsx)(t.th,{children:"Config Name"}),(0,n.jsx)(t.th,{children:"Default"}),(0,n.jsx)(t.th,{children:"Description"})]})}),(0,n.jsxs)(t.tbody,{children:[(0,n.jsxs)(t.tr,{children:[(0,n.jsx)(t.td,{children:"hoodie.keygen.timebased.timestamp.type"}),(0,n.jsxs)(t.td,{children:["N/A ",(0,n.jsx)(t.strong,{children:"(Required)"})]}),(0,n.jsx)(t.td,{children:"Required only when the key generator is TimestampBasedKeyGenerator. One of the timestamp types supported(UNIX_TIMESTAMP, DATE_STRING, MIXED, EPOCHMILLISECONDS, SCALAR)"})]}),(0,n.jsxs)(t.tr,{children:[(0,n.jsx)(t.td,{children:"hoodie.keygen.timebased.output.dateformat"}),(0,n.jsx)(t.td,{children:'"" (Optional)'}),(0,n.jsxs)(t.td,{children:["Output date format such as ",(0,n.jsx)(t.code,{children:"yyyy-MM-dd'T'HH:mm:ss.SSSZ"})]})]}),(0,n.jsxs)(t.tr,{children:[(0,n.jsx)(t.td,{children:"hoodie.keygen.timebased.timezone"}),(0,n.jsx)(t.td,{children:'"UTC" (Optional)'}),(0,n.jsxs)(t.td,{children:["Timezone of both input and output timestamp if they are the same, such as ",(0,n.jsx)(t.code,{children:"UTC"}),". Please use ",(0,n.jsx)(t.code,{children:"hoodie.keygen.timebased.input.timezone"})," and ",(0,n.jsx)(t.code,{children:"hoodie.keygen.timebased.output.timezone"})," instead if the input and output timezones are different."]})]}),(0,n.jsxs)(t.tr,{children:[(0,n.jsx)(t.td,{children:"hoodie.keygen.timebased.input.dateformat"}),(0,n.jsx)(t.td,{children:'"" (Optional)'}),(0,n.jsxs)(t.td,{children:["Input date format such as ",(0,n.jsx)(t.code,{children:"yyyy-MM-dd'T'HH:mm:ss.SSSZ"}),"."]})]})]})]}),"\n",(0,n.jsx)(t.p,{children:"Let's go over some example values for TimestampBasedKeyGenerator."}),"\n",(0,n.jsx)(t.h4,{id:"timestamp-is-gmt",children:"Timestamp is GMT"}),"\n",(0,n.jsxs)(t.table,{children:[(0,n.jsx)(t.thead,{children:(0,n.jsxs)(t.tr,{children:[(0,n.jsx)(t.th,{children:"Config Name"}),(0,n.jsx)(t.th,{children:"Value"})]})}),(0,n.jsxs)(t.tbody,{children:[(0,n.jsxs)(t.tr,{children:[(0,n.jsx)(t.td,{children:(0,n.jsx)(t.code,{children:"hoodie.streamer.keygen.timebased.timestamp.type"})}),(0,n.jsx)(t.td,{children:'"EPOCHMILLISECONDS"'})]}),(0,n.jsxs)(t.tr,{children:[(0,n.jsx)(t.td,{children:(0,n.jsx)(t.code,{children:"hoodie.streamer.keygen.timebased.output.dateformat"})}),(0,n.jsx)(t.td,{children:'"yyyy-MM-dd hh"'})]}),(0,n.jsxs)(t.tr,{children:[(0,n.jsx)(t.td,{children:(0,n.jsx)(t.code,{children:"hoodie.streamer.keygen.timebased.timezone"})}),(0,n.jsx)(t.td,{children:'"GMT+8:00"'})]})]})]}),"\n",(0,n.jsxs)(t.p,{children:["Input Field value: \u201c1578283932000L\u201d ",(0,n.jsx)("br",{}),"\nPartition path generated from key generator: \u201c2020-01-06 12\u201d"]}),"\n",(0,n.jsxs)(t.p,{children:["If input field value is null for some rows. ",(0,n.jsx)("br",{}),"\nPartition path generated from key generator: \u201c1970-01-01 08\u201d"]}),"\n",(0,n.jsx)(t.h4,{id:"timestamp-is-date_string",children:"Timestamp is DATE_STRING"}),"\n",(0,n.jsxs)(t.table,{children:[(0,n.jsx)(t.thead,{children:(0,n.jsxs)(t.tr,{children:[(0,n.jsx)(t.th,{children:"Config Name"}),(0,n.jsx)(t.th,{children:"Value"})]})}),(0,n.jsxs)(t.tbody,{children:[(0,n.jsxs)(t.tr,{children:[(0,n.jsx)(t.td,{children:(0,n.jsx)(t.code,{children:"hoodie.streamer.keygen.timebased.timestamp.type"})}),(0,n.jsx)(t.td,{children:'"DATE_STRING"'})]}),(0,n.jsxs)(t.tr,{children:[(0,n.jsx)(t.td,{children:(0,n.jsx)(t.code,{children:"hoodie.streamer.keygen.timebased.output.dateformat"})}),(0,n.jsx)(t.td,{children:'"yyyy-MM-dd hh"'})]}),(0,n.jsxs)(t.tr,{children:[(0,n.jsx)(t.td,{children:(0,n.jsx)(t.code,{children:"hoodie.streamer.keygen.timebased.timezone"})}),(0,n.jsx)(t.td,{children:'"GMT+8:00"'})]}),(0,n.jsxs)(t.tr,{children:[(0,n.jsx)(t.td,{children:(0,n.jsx)(t.code,{children:"hoodie.streamer.keygen.timebased.input.dateformat"})}),(0,n.jsxs)(t.td,{children:['"yyyy-MM-dd hh:mm',":ss",'"']})]})]})]}),"\n",(0,n.jsxs)(t.p,{children:["Input field value: \u201c2020-01-06 12:12:12\u201d ",(0,n.jsx)("br",{}),"\nPartition path generated from key generator: \u201c2020-01-06 12\u201d"]}),"\n",(0,n.jsxs)(t.p,{children:["If input field value is null for some rows. ",(0,n.jsx)("br",{}),"\nPartition path generated from key generator: \u201c1970-01-01 12:00:00\u201d"]}),"\n",(0,n.jsx)("br",{}),"\n",(0,n.jsx)(t.h4,{id:"scalar-examples",children:"Scalar examples"}),"\n",(0,n.jsxs)(t.table,{children:[(0,n.jsx)(t.thead,{children:(0,n.jsxs)(t.tr,{children:[(0,n.jsx)(t.th,{children:"Config Name"}),(0,n.jsx)(t.th,{children:"Value"})]})}),(0,n.jsxs)(t.tbody,{children:[(0,n.jsxs)(t.tr,{children:[(0,n.jsx)(t.td,{children:(0,n.jsx)(t.code,{children:"hoodie.streamer.keygen.timebased.timestamp.type"})}),(0,n.jsx)(t.td,{children:'"SCALAR"'})]}),(0,n.jsxs)(t.tr,{children:[(0,n.jsx)(t.td,{children:(0,n.jsx)(t.code,{children:"hoodie.streamer.keygen.timebased.output.dateformat"})}),(0,n.jsx)(t.td,{children:'"yyyy-MM-dd hh"'})]}),(0,n.jsxs)(t.tr,{children:[(0,n.jsx)(t.td,{children:(0,n.jsx)(t.code,{children:"hoodie.streamer.keygen.timebased.timezone"})}),(0,n.jsx)(t.td,{children:'"GMT"'})]}),(0,n.jsxs)(t.tr,{children:[(0,n.jsx)(t.td,{children:(0,n.jsx)(t.code,{children:"hoodie.streamer.keygen.timebased.timestamp.scalar.time.unit"})}),(0,n.jsx)(t.td,{children:'"days"'})]})]})]}),"\n",(0,n.jsxs)(t.p,{children:["Input field value: \u201c20000L\u201d ",(0,n.jsx)("br",{}),"\nPartition path generated from key generator: \u201c2024-10-04 12\u201d"]}),"\n",(0,n.jsxs)(t.p,{children:["If input field value is null. ",(0,n.jsx)("br",{}),"\nPartition path generated from key generator: \u201c1970-01-02 12\u201d"]}),"\n",(0,n.jsx)(t.h4,{id:"iso8601withmsz-with-single-input-format",children:"ISO8601WithMsZ with Single Input format"}),"\n",(0,n.jsxs)(t.table,{children:[(0,n.jsx)(t.thead,{children:(0,n.jsxs)(t.tr,{children:[(0,n.jsx)(t.th,{children:"Config Name"}),(0,n.jsx)(t.th,{children:"Value"})]})}),(0,n.jsxs)(t.tbody,{children:[(0,n.jsxs)(t.tr,{children:[(0,n.jsx)(t.td,{children:(0,n.jsx)(t.code,{children:"hoodie.streamer.keygen.timebased.timestamp.type"})}),(0,n.jsx)(t.td,{children:'"DATE_STRING"'})]}),(0,n.jsxs)(t.tr,{children:[(0,n.jsx)(t.td,{children:(0,n.jsx)(t.code,{children:"hoodie.streamer.keygen.timebased.input.dateformat"})}),(0,n.jsxs)(t.td,{children:["\"yyyy-MM-dd'T'HH:mm",":ss",'.SSSZ"']})]}),(0,n.jsxs)(t.tr,{children:[(0,n.jsx)(t.td,{children:(0,n.jsx)(t.code,{children:"hoodie.streamer.keygen.timebased.input.dateformat.list.delimiter.regex"})}),(0,n.jsx)(t.td,{children:'""'})]}),(0,n.jsxs)(t.tr,{children:[(0,n.jsx)(t.td,{children:(0,n.jsx)(t.code,{children:"hoodie.streamer.keygen.timebased.input.timezone"})}),(0,n.jsx)(t.td,{children:'""'})]}),(0,n.jsxs)(t.tr,{children:[(0,n.jsx)(t.td,{children:(0,n.jsx)(t.code,{children:"hoodie.streamer.keygen.timebased.output.dateformat"})}),(0,n.jsx)(t.td,{children:'"yyyyMMddHH"'})]}),(0,n.jsxs)(t.tr,{children:[(0,n.jsx)(t.td,{children:(0,n.jsx)(t.code,{children:"hoodie.streamer.keygen.timebased.output.timezone"})}),(0,n.jsx)(t.td,{children:'"GMT"'})]})]})]}),"\n",(0,n.jsxs)(t.p,{children:['Input field value: "2020-04-01T13:01:33.428Z" ',(0,n.jsx)("br",{}),'\nPartition path generated from key generator: "2020040113"']}),"\n",(0,n.jsx)(t.h4,{id:"iso8601withmsz-with-multiple-input-formats",children:"ISO8601WithMsZ with Multiple Input formats"}),"\n",(0,n.jsxs)(t.table,{children:[(0,n.jsx)(t.thead,{children:(0,n.jsxs)(t.tr,{children:[(0,n.jsx)(t.th,{children:"Config Name"}),(0,n.jsx)(t.th,{children:"Value"})]})}),(0,n.jsxs)(t.tbody,{children:[(0,n.jsxs)(t.tr,{children:[(0,n.jsx)(t.td,{children:(0,n.jsx)(t.code,{children:"hoodie.streamer.keygen.timebased.timestamp.type"})}),(0,n.jsx)(t.td,{children:'"DATE_STRING"'})]}),(0,n.jsxs)(t.tr,{children:[(0,n.jsx)(t.td,{children:(0,n.jsx)(t.code,{children:"hoodie.streamer.keygen.timebased.input.dateformat"})}),(0,n.jsxs)(t.td,{children:["\"yyyy-MM-dd'T'HH:mm",":ssZ",",yyyy-MM-dd'T'HH:mm",":ss",'.SSSZ"']})]}),(0,n.jsxs)(t.tr,{children:[(0,n.jsx)(t.td,{children:(0,n.jsx)(t.code,{children:"hoodie.streamer.keygen.timebased.input.dateformat.list.delimiter.regex"})}),(0,n.jsx)(t.td,{children:'""'})]}),(0,n.jsxs)(t.tr,{children:[(0,n.jsx)(t.td,{children:(0,n.jsx)(t.code,{children:"hoodie.streamer.keygen.timebased.input.timezone"})}),(0,n.jsx)(t.td,{children:'""'})]}),(0,n.jsxs)(t.tr,{children:[(0,n.jsx)(t.td,{children:(0,n.jsx)(t.code,{children:"hoodie.streamer.keygen.timebased.output.dateformat"})}),(0,n.jsx)(t.td,{children:'"yyyyMMddHH"'})]}),(0,n.jsxs)(t.tr,{children:[(0,n.jsx)(t.td,{children:(0,n.jsx)(t.code,{children:"hoodie.streamer.keygen.timebased.output.timezone"})}),(0,n.jsx)(t.td,{children:'"UTC"'})]})]})]}),"\n",(0,n.jsxs)(t.p,{children:['Input field value: "2020-04-01T13:01:33.428Z" ',(0,n.jsx)("br",{}),'\nPartition path generated from key generator: "2020040113"']}),"\n",(0,n.jsx)(t.h4,{id:"iso8601noms-with-offset-using-multiple-input-formats",children:"ISO8601NoMs with offset using multiple input formats"}),"\n",(0,n.jsxs)(t.table,{children:[(0,n.jsx)(t.thead,{children:(0,n.jsxs)(t.tr,{children:[(0,n.jsx)(t.th,{children:"Config Name"}),(0,n.jsx)(t.th,{children:"Value"})]})}),(0,n.jsxs)(t.tbody,{children:[(0,n.jsxs)(t.tr,{children:[(0,n.jsx)(t.td,{children:(0,n.jsx)(t.code,{children:"hoodie.streamer.keygen.timebased.timestamp.type"})}),(0,n.jsx)(t.td,{children:'"DATE_STRING"'})]}),(0,n.jsxs)(t.tr,{children:[(0,n.jsx)(t.td,{children:(0,n.jsx)(t.code,{children:"hoodie.streamer.keygen.timebased.input.dateformat"})}),(0,n.jsxs)(t.td,{children:["\"yyyy-MM-dd'T'HH:mm",":ssZ",",yyyy-MM-dd'T'HH:mm",":ss",'.SSSZ"']})]}),(0,n.jsxs)(t.tr,{children:[(0,n.jsx)(t.td,{children:(0,n.jsx)(t.code,{children:"hoodie.streamer.keygen.timebased.input.dateformat.list.delimiter.regex"})}),(0,n.jsx)(t.td,{children:'""'})]}),(0,n.jsxs)(t.tr,{children:[(0,n.jsx)(t.td,{children:(0,n.jsx)(t.code,{children:"hoodie.streamer.keygen.timebased.input.timezone"})}),(0,n.jsx)(t.td,{children:'""'})]}),(0,n.jsxs)(t.tr,{children:[(0,n.jsx)(t.td,{children:(0,n.jsx)(t.code,{children:"hoodie.streamer.keygen.timebased.output.dateformat"})}),(0,n.jsx)(t.td,{children:'"yyyyMMddHH"'})]}),(0,n.jsxs)(t.tr,{children:[(0,n.jsx)(t.td,{children:(0,n.jsx)(t.code,{children:"hoodie.streamer.keygen.timebased.output.timezone"})}),(0,n.jsx)(t.td,{children:'"UTC"'})]})]})]}),"\n",(0,n.jsxs)(t.p,{children:['Input field value: "2020-04-01T13:01:33-',(0,n.jsx)(t.strong,{children:"05:00"}),'" ',(0,n.jsx)("br",{}),'\nPartition path generated from key generator: "2020040118"']}),"\n",(0,n.jsx)(t.h4,{id:"input-as-short-date-string-and-expect-date-in-date-format",children:"Input as short date string and expect date in date format"}),"\n",(0,n.jsxs)(t.table,{children:[(0,n.jsx)(t.thead,{children:(0,n.jsxs)(t.tr,{children:[(0,n.jsx)(t.th,{children:"Config Name"}),(0,n.jsx)(t.th,{children:"Value"})]})}),(0,n.jsxs)(t.tbody,{children:[(0,n.jsxs)(t.tr,{children:[(0,n.jsx)(t.td,{children:(0,n.jsx)(t.code,{children:"hoodie.streamer.keygen.timebased.timestamp.type"})}),(0,n.jsx)(t.td,{children:'"DATE_STRING"'})]}),(0,n.jsxs)(t.tr,{children:[(0,n.jsx)(t.td,{children:(0,n.jsx)(t.code,{children:"hoodie.streamer.keygen.timebased.input.dateformat"})}),(0,n.jsxs)(t.td,{children:["\"yyyy-MM-dd'T'HH:mm",":ssZ",",yyyy-MM-dd'T'HH:mm",":ss",'.SSSZ,yyyyMMdd"']})]}),(0,n.jsxs)(t.tr,{children:[(0,n.jsx)(t.td,{children:(0,n.jsx)(t.code,{children:"hoodie.streamer.keygen.timebased.input.dateformat.list.delimiter.regex"})}),(0,n.jsx)(t.td,{children:'""'})]}),(0,n.jsxs)(t.tr,{children:[(0,n.jsx)(t.td,{children:(0,n.jsx)(t.code,{children:"hoodie.streamer.keygen.timebased.input.timezone"})}),(0,n.jsx)(t.td,{children:'"UTC"'})]}),(0,n.jsxs)(t.tr,{children:[(0,n.jsx)(t.td,{children:(0,n.jsx)(t.code,{children:"hoodie.streamer.keygen.timebased.output.dateformat"})}),(0,n.jsx)(t.td,{children:'"MM/dd/yyyy"'})]}),(0,n.jsxs)(t.tr,{children:[(0,n.jsx)(t.td,{children:(0,n.jsx)(t.code,{children:"hoodie.streamer.keygen.timebased.output.timezone"})}),(0,n.jsx)(t.td,{children:'"UTC"'})]})]})]}),"\n",(0,n.jsxs)(t.p,{children:['Input field value: "20200401" ',(0,n.jsx)("br",{}),'\nPartition path generated from key generator: "04/01/2020"']}),"\n",(0,n.jsx)(t.h2,{id:"related-resources",children:"Related Resources"}),"\n",(0,n.jsx)("h3",{children:"Blogs"}),"\n",(0,n.jsxs)(t.ul,{children:["\n",(0,n.jsx)(t.li,{children:(0,n.jsx)(t.a,{href:"https://www.onehouse.ai/blog/hudi-metafields-demystified",children:"Hudi metafields demystified"})}),"\n",(0,n.jsx)(t.li,{children:(0,n.jsx)(t.a,{href:"https://medium.com/@simpsons/primary-key-and-partition-generators-with-apache-hudi-f0e4d71d9d26",children:"Primary key and Partition Generators with Apache Hudi"})}),"\n"]})]})}function c(e={}){const{wrapper:t}={...(0,s.R)(),...e.components};return t?(0,n.jsx)(t,{...e,children:(0,n.jsx)(h,{...e})}):h(e)}},28453:(e,t,i)=>{i.d(t,{R:()=>d,x:()=>a});var r=i(96540);const n={},s=r.createContext(n);function d(e){const t=r.useContext(s);return r.useMemo((function(){return"function"==typeof e?e(t):{...t,...e}}),[t,e])}function a(e){let t;return t=e.disableParentContext?"function"==typeof e.components?e.components(n):e.components||n:d(e.components),r.createElement(s.Provider,{value:t},e.children)}}}]);
\ No newline at end of file
diff --git a/content/assets/js/ea8366f8.49359600.js b/content/assets/js/ea8366f8.60a4a0cc.js
similarity index 98%
rename from content/assets/js/ea8366f8.49359600.js
rename to content/assets/js/ea8366f8.60a4a0cc.js
index 511a371eb8233..ba963690d05f3 100644
--- a/content/assets/js/ea8366f8.49359600.js
+++ b/content/assets/js/ea8366f8.60a4a0cc.js
@@ -1 +1 @@
-"use strict";(self.webpackChunkhudi=self.webpackChunkhudi||[]).push([[23700],{49188:(e,i,t)=>{t.r(i),t.d(i,{assets:()=>c,contentTitle:()=>a,default:()=>h,frontMatter:()=>s,metadata:()=>n,toc:()=>l});const n=JSON.parse('{"id":"concurrency_control","title":"Concurrency Control","description":"Concurrency control defines how different writers/readers/table services coordinate access to a Hudi table. Hudi ensures atomic writes, by way of publishing commits atomically to the timeline,","source":"@site/docs/concurrency_control.md","sourceDirName":".","slug":"/concurrency_control","permalink":"/docs/next/concurrency_control","draft":false,"unlisted":false,"editUrl":"https://github.com/apache/hudi/tree/asf-site/website/docs/concurrency_control.md","tags":[],"version":"current","frontMatter":{"title":"Concurrency Control","summary":"In this page, we will discuss how to perform concurrent writes to Hudi Tables.","toc":true,"toc_min_heading_level":2,"toc_max_heading_level":4,"last_modified_at":"2021-03-19T19:59:57.000Z"},"sidebar":"docs","previous":{"title":"Indexes","permalink":"/docs/next/indexes"},"next":{"title":"Schema Evolution","permalink":"/docs/next/schema_evolution"}}');var r=t(74848),o=t(28453);const s={title:"Concurrency Control",summary:"In this page, we will discuss how to perform concurrent writes to Hudi Tables.",toc:!0,toc_min_heading_level:2,toc_max_heading_level:4,last_modified_at:new Date("2021-03-19T19:59:57.000Z")},a=void 0,c={},l=[{value:"Distributed Locking",id:"distributed-locking",level:2},{value:"Zookeeper based",id:"zookeeper-based",level:4},{value:"HiveMetastore based",id:"hivemetastore-based",level:4},{value:"Amazon DynamoDB based",id:"amazon-dynamodb-based",level:4},{value:"FileSystem based (not for production use)",id:"filesystem-based-not-for-production-use",level:4},{value:"Simple Single writer + table services",id:"simple-single-writer--table-services",level:2},{value:"Inline table services",id:"inline-table-services",level:3},{value:"Async table services",id:"async-table-services",level:3},{value:"Full-on Multi-writer + Async table services",id:"full-on-multi-writer--async-table-services",level:2},{value:"Multi Writer Guarantees",id:"multi-writer-guarantees",level:4},{value:"Non-Blocking Concurrency Control",id:"non-blocking-concurrency-control",level:2},{value:"Early conflict Detection",id:"early-conflict-detection",level:2},{value:"Enabling Multi Writing",id:"enabling-multi-writing",level:2},{value:"Multi Writing via Hudi Streamer",id:"multi-writing-via-hudi-streamer",level:3},{value:"Multi Writing via Spark Datasource Writer",id:"multi-writing-via-spark-datasource-writer",level:3},{value:"Disabling Multi Writing",id:"disabling-multi-writing",level:2},{value:"OCC Best Practices",id:"occ-best-practices",level:2},{value:"Caveats",id:"caveats",level:2},{value:"Related Resources",id:"related-resources",level:2}];function d(e){const i={a:"a",admonition:"admonition",code:"code",em:"em",h2:"h2",h3:"h3",h4:"h4",li:"li",mdxAdmonitionTitle:"mdxAdmonitionTitle",ol:"ol",p:"p",pre:"pre",strong:"strong",table:"table",tbody:"tbody",td:"td",th:"th",thead:"thead",tr:"tr",ul:"ul",...(0,o.R)(),...e.components};return(0,r.jsxs)(r.Fragment,{children:[(0,r.jsxs)(i.p,{children:["Concurrency control defines how different writers/readers/table services coordinate access to a Hudi table. Hudi ensures atomic writes, by way of publishing commits atomically to the timeline,\nstamped with an instant time that denotes the time at which the action is deemed to have occurred. Unlike general purpose file version control, Hudi draws clear distinction between\nwriter processes that issue ",(0,r.jsx)(i.a,{href:"write_operations",children:"write operations"})," and table services that (re)write data/metadata to optimize/perform bookkeeping and\nreaders (that execute queries and read data)."]}),"\n",(0,r.jsx)(i.p,{children:"Hudi provides"}),"\n",(0,r.jsxs)(i.ul,{children:["\n",(0,r.jsxs)(i.li,{children:[(0,r.jsx)(i.strong,{children:"Snapshot isolation"})," between all three types of processes, meaning they all operate on a consistent snapshot of the table."]}),"\n",(0,r.jsxs)(i.li,{children:[(0,r.jsx)(i.strong,{children:"Optimistic concurrency control (OCC)"})," between writers to provide standard relational database semantics."]}),"\n",(0,r.jsxs)(i.li,{children:[(0,r.jsx)(i.strong,{children:"Multiversion Concurrency Control (MVCC)"})," based concurrency control between writers and table-services and between different table services."]}),"\n",(0,r.jsxs)(i.li,{children:[(0,r.jsx)(i.strong,{children:"Non-blocking Concurrency Control (NBCC)"})," between writers, to provide streaming semantics and avoiding live-locks/starvation between writers."]}),"\n"]}),"\n",(0,r.jsx)(i.p,{children:"In this section, we will discuss the different concurrency controls supported by Hudi and how they are leveraged to provide flexible deployment models for single and multiple writer scenarios.\nWe\u2019ll also describe ways to ingest data into a Hudi Table from multiple writers using different writers, like Hudi Streamer, Hudi datasource, Spark Structured Streaming and Spark SQL."}),"\n",(0,r.jsxs)(i.admonition,{type:"note",children:[(0,r.jsx)(i.p,{children:"If there is only one process performing writing AND async/inline table services on the table, you can\navoid the overhead of a distributed lock requirement by configuring the in process lock provider."}),(0,r.jsx)(i.pre,{children:(0,r.jsx)(i.code,{className:"language-properties",children:"hoodie.write.lock.provider=org.apache.hudi.client.transaction.lock.InProcessLockProvider\n"})})]}),"\n",(0,r.jsx)(i.h2,{id:"distributed-locking",children:"Distributed Locking"}),"\n",(0,r.jsxs)(i.p,{children:["A pre-requisite for distributed co-ordination in Hudi, like many other distributed database systems is a distributed lock provider, that different processes can use to plan, schedule and\nexecute actions on the Hudi timeline in a concurrent fashion. Locks are also used to ",(0,r.jsx)(i.a,{href:"timeline#truetime-generation",children:"generate TrueTime"}),", as discussed before."]}),"\n",(0,r.jsx)(i.p,{children:"External locking is typically used in conjunction with optimistic concurrency control\nbecause it provides a way to prevent conflicts that might occur when two or more transactions (commits in our case) attempt to modify the same resource concurrently.\nWhen a transaction attempts to modify a resource that is currently locked by another transaction, it must wait until the lock is released before proceeding."}),"\n",(0,r.jsx)(i.p,{children:"In case of multi-writing in Hudi, the locks are acquired on the Hudi table for a very short duration during specific phases (such as just before committing the writes or before scheduling table services) instead of locking for the entire span of time. This approach allows multiple writers to work on the same table simultaneously, increasing concurrency and avoids conflicts."}),"\n",(0,r.jsxs)(i.p,{children:["There are 4 different lock providers that require different configurations to be set. Please refer to comprehensive locking configs ",(0,r.jsx)(i.a,{href:"https://hudi.apache.org/docs/next/configurations#LOCK",children:"here"}),"."]}),"\n",(0,r.jsx)(i.h4,{id:"zookeeper-based",children:"Zookeeper based"}),"\n",(0,r.jsx)(i.pre,{children:(0,r.jsx)(i.code,{children:"hoodie.write.lock.provider=org.apache.hudi.client.transaction.lock.ZookeeperBasedLockProvider\n"})}),"\n",(0,r.jsx)(i.p,{children:"Following are the basic configs required to setup this lock provider:"}),"\n",(0,r.jsxs)(i.table,{children:[(0,r.jsx)(i.thead,{children:(0,r.jsxs)(i.tr,{children:[(0,r.jsx)(i.th,{children:"Config Name"}),(0,r.jsx)(i.th,{children:"Default"}),(0,r.jsx)(i.th,{children:"Description"})]})}),(0,r.jsxs)(i.tbody,{children:[(0,r.jsxs)(i.tr,{children:[(0,r.jsx)(i.td,{children:"hoodie.write.lock.zookeeper.base_path"}),(0,r.jsxs)(i.td,{children:["N/A ",(0,r.jsx)(i.strong,{children:"(Required)"})]}),(0,r.jsxs)(i.td,{children:["The base path on Zookeeper under which to create lock related ZNodes. This should be same for all concurrent writers to the same table",(0,r.jsx)("br",{}),(0,r.jsx)("br",{}),(0,r.jsx)(i.code,{children:"Config Param: ZK_BASE_PATH"}),(0,r.jsx)("br",{}),(0,r.jsx)(i.code,{children:"Since Version: 0.8.0"})]})]}),(0,r.jsxs)(i.tr,{children:[(0,r.jsx)(i.td,{children:"hoodie.write.lock.zookeeper.port"}),(0,r.jsxs)(i.td,{children:["N/A ",(0,r.jsx)(i.strong,{children:"(Required)"})]}),(0,r.jsxs)(i.td,{children:["Zookeeper port to connect to.",(0,r.jsx)("br",{}),(0,r.jsx)("br",{}),(0,r.jsx)(i.code,{children:"Config Param: ZK_PORT"}),(0,r.jsx)("br",{}),(0,r.jsx)(i.code,{children:"Since Version: 0.8.0"})]})]}),(0,r.jsxs)(i.tr,{children:[(0,r.jsx)(i.td,{children:"hoodie.write.lock.zookeeper.url"}),(0,r.jsxs)(i.td,{children:["N/A ",(0,r.jsx)(i.strong,{children:"(Required)"})]}),(0,r.jsxs)(i.td,{children:["Zookeeper URL to connect to.",(0,r.jsx)("br",{}),(0,r.jsx)("br",{}),(0,r.jsx)(i.code,{children:"Config Param: ZK_CONNECT_URL"}),(0,r.jsx)("br",{}),(0,r.jsx)(i.code,{children:"Since Version: 0.8.0"})]})]})]})]}),"\n",(0,r.jsx)(i.h4,{id:"hivemetastore-based",children:"HiveMetastore based"}),"\n",(0,r.jsx)(i.pre,{children:(0,r.jsx)(i.code,{children:"hoodie.write.lock.provider=org.apache.hudi.hive.transaction.lock.HiveMetastoreBasedLockProvider\n"})}),"\n",(0,r.jsx)(i.p,{children:"Following are the basic configs required to setup this lock provider:"}),"\n",(0,r.jsxs)(i.table,{children:[(0,r.jsx)(i.thead,{children:(0,r.jsxs)(i.tr,{children:[(0,r.jsx)(i.th,{children:"Config Name"}),(0,r.jsx)(i.th,{children:"Default"}),(0,r.jsx)(i.th,{children:"Description"})]})}),(0,r.jsxs)(i.tbody,{children:[(0,r.jsxs)(i.tr,{children:[(0,r.jsx)(i.td,{children:"hoodie.write.lock.hivemetastore.database"}),(0,r.jsxs)(i.td,{children:["N/A ",(0,r.jsx)(i.strong,{children:"(Required)"})]}),(0,r.jsxs)(i.td,{children:["For Hive based lock provider, the Hive database to acquire lock against",(0,r.jsx)("br",{}),(0,r.jsx)("br",{}),(0,r.jsx)(i.code,{children:"Config Param: HIVE_DATABASE_NAME"}),(0,r.jsx)("br",{}),(0,r.jsx)(i.code,{children:"Since Version: 0.8.0"})]})]}),(0,r.jsxs)(i.tr,{children:[(0,r.jsx)(i.td,{children:"hoodie.write.lock.hivemetastore.table"}),(0,r.jsxs)(i.td,{children:["N/A ",(0,r.jsx)(i.strong,{children:"(Required)"})]}),(0,r.jsxs)(i.td,{children:["For Hive based lock provider, the Hive table to acquire lock against",(0,r.jsx)("br",{}),(0,r.jsx)("br",{}),(0,r.jsx)(i.code,{children:"Config Param: HIVE_TABLE_NAME"}),(0,r.jsx)("br",{}),(0,r.jsx)(i.code,{children:"Since Version: 0.8.0"})]})]})]})]}),"\n",(0,r.jsx)(i.p,{children:(0,r.jsx)(i.code,{children:"The HiveMetastore URI's are picked up from the hadoop configuration file loaded during runtime."})}),"\n",(0,r.jsx)(i.h4,{id:"amazon-dynamodb-based",children:"Amazon DynamoDB based"}),"\n",(0,r.jsx)(i.pre,{children:(0,r.jsx)(i.code,{children:"hoodie.write.lock.provider=org.apache.hudi.aws.transaction.lock.DynamoDBBasedLockProvider\n"})}),"\n",(0,r.jsxs)(i.p,{children:["Amazon DynamoDB based lock provides a simple way to support multi writing across different clusters. You can refer to the\n",(0,r.jsx)(i.a,{href:"https://hudi.apache.org/docs/configurations#DynamoDB-based-Locks-Configurations",children:"DynamoDB based Locks Configurations"}),"\nsection for the details of each related configuration knob. Following are the basic configs required to setup this lock provider:"]}),"\n",(0,r.jsxs)(i.table,{children:[(0,r.jsx)(i.thead,{children:(0,r.jsxs)(i.tr,{children:[(0,r.jsx)(i.th,{children:"Config Name"}),(0,r.jsx)(i.th,{children:"Default"}),(0,r.jsx)(i.th,{children:"Description"})]})}),(0,r.jsx)(i.tbody,{children:(0,r.jsxs)(i.tr,{children:[(0,r.jsx)(i.td,{children:"hoodie.write.lock.dynamodb.endpoint_url"}),(0,r.jsxs)(i.td,{children:["N/A ",(0,r.jsx)(i.strong,{children:"(Required)"})]}),(0,r.jsxs)(i.td,{children:["For DynamoDB based lock provider, the url endpoint used for Amazon DynamoDB service. Useful for development with a local dynamodb instance.",(0,r.jsx)("br",{}),(0,r.jsx)("br",{}),(0,r.jsx)(i.code,{children:"Config Param: DYNAMODB_ENDPOINT_URL"}),(0,r.jsx)("br",{}),(0,r.jsx)(i.code,{children:"Since Version: 0.10.1"})]})]})})]}),"\n",(0,r.jsxs)(i.p,{children:["For advanced configs refer ",(0,r.jsx)(i.a,{href:"https://hudi.apache.org/docs/next/configurations#DynamoDB-based-Locks-Configurations",children:"here"})]}),"\n",(0,r.jsxs)(i.p,{children:["When using the DynamoDB-based lock provider, the name of the DynamoDB table acting as the lock table for Hudi is\nspecified by the config ",(0,r.jsx)(i.code,{children:"hoodie.write.lock.dynamodb.table"}),". This DynamoDB table is automatically created by Hudi, so you\ndon't have to create the table yourself. If you want to use an existing DynamoDB table, make sure that an attribute with\nthe name ",(0,r.jsx)(i.code,{children:"key"})," is present in the table. The ",(0,r.jsx)(i.code,{children:"key"})," attribute should be the partition key of the DynamoDB table. The\nconfig ",(0,r.jsx)(i.code,{children:"hoodie.write.lock.dynamodb.partition_key"})," specifies the value to put for the ",(0,r.jsx)(i.code,{children:"key"})," attribute (not the attribute\nname), which is used for the lock on the same table. By default, ",(0,r.jsx)(i.code,{children:"hoodie.write.lock.dynamodb.partition_key"})," is set to\nthe table name, so that multiple writers writing to the same table share the same lock. If you customize the name, make\nsure it's the same across multiple writers."]}),"\n",(0,r.jsx)(i.p,{children:"Also, to set up the credentials for accessing AWS resources, customers can pass the following props to Hudi jobs:"}),"\n",(0,r.jsx)(i.pre,{children:(0,r.jsx)(i.code,{children:"hoodie.aws.access.key\nhoodie.aws.secret.key\nhoodie.aws.session.token\n"})}),"\n",(0,r.jsxs)(i.p,{children:["If not configured, Hudi falls back to use ",(0,r.jsx)(i.a,{href:"https://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/auth/DefaultAWSCredentialsProviderChain.html",children:"DefaultAWSCredentialsProviderChain"}),"."]}),"\n",(0,r.jsx)(i.p,{children:"IAM policy for your service instance will need to add the following permissions:"}),"\n",(0,r.jsx)(i.pre,{children:(0,r.jsx)(i.code,{className:"language-json",children:'{\n "Sid":"DynamoDBLocksTable",\n "Effect": "Allow",\n "Action": [\n "dynamodb:CreateTable",\n "dynamodb:DeleteItem",\n "dynamodb:DescribeTable",\n "dynamodb:GetItem",\n "dynamodb:PutItem",\n "dynamodb:Scan",\n "dynamodb:UpdateItem"\n ],\n "Resource": "arn:${Partition}:dynamodb:${Region}:${Account}:table/${TableName}"\n}\n'})}),"\n",(0,r.jsxs)(i.ul,{children:["\n",(0,r.jsxs)(i.li,{children:[(0,r.jsx)(i.code,{children:"TableName"})," : same as ",(0,r.jsx)(i.code,{children:"hoodie.write.lock.dynamodb.partition_key"})]}),"\n",(0,r.jsxs)(i.li,{children:[(0,r.jsx)(i.code,{children:"Region"}),": same as ",(0,r.jsx)(i.code,{children:"hoodie.write.lock.dynamodb.region"})]}),"\n"]}),"\n",(0,r.jsx)(i.p,{children:"AWS SDK dependencies are not bundled with Hudi from v0.10.x and will need to be added to your classpath.\nAdd the following Maven packages (check the latest versions at time of install):"}),"\n",(0,r.jsx)(i.pre,{children:(0,r.jsx)(i.code,{children:"com.amazonaws:dynamodb-lock-client\ncom.amazonaws:aws-java-sdk-dynamodb\ncom.amazonaws:aws-java-sdk-core\n"})}),"\n",(0,r.jsx)(i.h4,{id:"filesystem-based-not-for-production-use",children:"FileSystem based (not for production use)"}),"\n",(0,r.jsx)(i.p,{children:"FileSystem based lock provider supports multiple writers cross different jobs/applications based on atomic create/delete operations of the underlying filesystem."}),"\n",(0,r.jsx)(i.pre,{children:(0,r.jsx)(i.code,{children:"hoodie.write.lock.provider=org.apache.hudi.client.transaction.lock.FileSystemBasedLockProvider\n"})}),"\n",(0,r.jsxs)(i.p,{children:["When using the FileSystem based lock provider, by default, the lock file will store into ",(0,r.jsx)(i.code,{children:"hoodie.base.path"}),"+",(0,r.jsx)(i.code,{children:"/.hoodie/lock"}),". You may use a custom folder to store the lock file by specifying ",(0,r.jsx)(i.code,{children:"hoodie.write.lock.filesystem.path"}),"."]}),"\n",(0,r.jsxs)(i.p,{children:["In case the lock cannot release during job crash, you can set ",(0,r.jsx)(i.code,{children:"hoodie.write.lock.filesystem.expire"})," (lock will never expire by default) to a desired expire time in minutes. You may also delete lock file manually in such situation."]}),"\n",(0,r.jsx)(i.admonition,{type:"note",children:(0,r.jsx)(i.p,{children:"FileSystem based lock provider is not supported with cloud storage like S3 or GCS."})}),"\n",(0,r.jsx)(i.h2,{id:"simple-single-writer--table-services",children:"Simple Single writer + table services"}),"\n",(0,r.jsx)(i.p,{children:"Data lakehouse pipelines tend to be predominantly single writer, with the most common need for distributed co-ordination on a table coming from table management. For e.g. a Apache Flink\njob producing fast writes into a table, requiring regular file-size management or cleaning. Hudi's storage engine and platform tools provide a lot of support for such common scenarios."}),"\n",(0,r.jsx)(i.h3,{id:"inline-table-services",children:"Inline table services"}),"\n",(0,r.jsx)(i.p,{children:"This is the simplest form of concurrency, meaning there is no concurrency at all in the write processes. In this model, Hudi eliminates the need for concurrency control and maximizes throughput by supporting these table services out-of-box and running inline after every write to the table. Execution plans are idempotent, persisted to the timeline and auto-recover from failures. For most simple use-cases, this means just writing is sufficient to get a well-managed table that needs no concurrency control."}),"\n",(0,r.jsxs)(i.p,{children:["There is no actual concurrent writing in this model. ",(0,r.jsx)(i.strong,{children:"MVCC"})," is leveraged to provide snapshot isolation guarantees between ingestion writer and multiple readers and also between multiple table service writers and readers. Writes to the table either from ingestion or from table services produce versioned data that are available to readers only after the writes are committed. Until then, readers can access only the previous version of the data."]}),"\n",(0,r.jsx)(i.p,{children:"A single writer with all table services such as cleaning, clustering, compaction, etc can be configured to be inline (such as Hudi Streamer sync-once mode and Spark Datasource with default configs) without any additional configs."}),"\n",(0,r.jsx)(i.h3,{id:"async-table-services",children:"Async table services"}),"\n",(0,r.jsxs)(i.p,{children:["Hudi provides the option of running the table services in an async fashion, where most of the heavy lifting (e.g actually rewriting the columnar data by compaction service) is done asynchronously. In this model, the async deployment eliminates any repeated wasteful retries and optimizes the table using clustering techniques while a single writer consumes the writes to the table without having to be blocked by such table services. This model avoids the need for taking an ",(0,r.jsx)(i.a,{href:"#external-locking-and-lock-providers",children:"external lock"})," to control concurrency and avoids the need to separately orchestrate and monitor offline table services jobs.."]}),"\n",(0,r.jsxs)(i.p,{children:["A single writer along with async table services runs in the same process. For example, you can have a Hudi Streamer in continuous mode write to a MOR table using async compaction; you can use Spark Streaming (where ",(0,r.jsx)(i.a,{href:"https://hudi.apache.org/docs/compaction",children:"compaction"})," is async by default), and you can use Flink streaming or your own job setup and enable async table services inside the same writer."]}),"\n",(0,r.jsxs)(i.p,{children:["Hudi leverages ",(0,r.jsx)(i.strong,{children:"MVCC"})," in this model to support running any number of table service jobs concurrently, without any concurrency conflict. This is made possible by ensuring Hudi 's ingestion writer and async table services coordinate among themselves to ensure no conflicts and avoid race conditions. The same single write guarantees described in Model A above can be achieved in this model as well.\nWith this model users don't need to spin up different spark jobs and manage the orchestration among it. For larger deployments, this model can ease the operational burden significantly while getting the table services running without blocking the writers."]}),"\n",(0,r.jsx)(i.p,{children:(0,r.jsx)(i.strong,{children:"Single Writer Guarantees"})}),"\n",(0,r.jsxs)(i.p,{children:["In this model, the following are the guarantees on ",(0,r.jsx)(i.a,{href:"https://hudi.apache.org/docs/write_operations/",children:"write operations"})," to expect:"]}),"\n",(0,r.jsxs)(i.ul,{children:["\n",(0,r.jsxs)(i.li,{children:[(0,r.jsx)(i.em,{children:"UPSERT Guarantee"}),": The target table will NEVER show duplicates."]}),"\n",(0,r.jsxs)(i.li,{children:[(0,r.jsx)(i.em,{children:"INSERT Guarantee"}),": The target table wilL NEVER have duplicates if dedup: ",(0,r.jsx)(i.a,{href:"https://hudi.apache.org/docs/configurations#hoodiedatasourcewriteinsertdropduplicates",children:(0,r.jsx)(i.code,{children:"hoodie.datasource.write.insert.drop.duplicates"})})," & ",(0,r.jsx)(i.a,{href:"https://hudi.apache.org/docs/configurations/#hoodiecombinebeforeinsert",children:(0,r.jsx)(i.code,{children:"hoodie.combine.before.insert"})}),", is enabled."]}),"\n",(0,r.jsxs)(i.li,{children:[(0,r.jsx)(i.em,{children:"BULK_INSERT Guarantee"}),": The target table will NEVER have duplicates if dedup: ",(0,r.jsx)(i.a,{href:"https://hudi.apache.org/docs/configurations#hoodiedatasourcewriteinsertdropduplicates",children:(0,r.jsx)(i.code,{children:"hoodie.datasource.write.insert.drop.duplicates"})})," & ",(0,r.jsx)(i.a,{href:"https://hudi.apache.org/docs/configurations/#hoodiecombinebeforeinsert",children:(0,r.jsx)(i.code,{children:"hoodie.combine.before.insert"})}),", is enabled."]}),"\n",(0,r.jsxs)(i.li,{children:[(0,r.jsx)(i.em,{children:"INCREMENTAL QUERY Guarantee"}),": Data consumption and checkpoints are NEVER out of order."]}),"\n"]}),"\n",(0,r.jsx)(i.h2,{id:"full-on-multi-writer--async-table-services",children:"Full-on Multi-writer + Async table services"}),"\n",(0,r.jsxs)(i.p,{children:["Hudi has introduced a new concurrency mode ",(0,r.jsx)(i.code,{children:"NON_BLOCKING_CONCURRENCY_CONTROL"}),", where unlike OCC, multiple writers can\noperate on the table with non-blocking conflict resolution. The writers can write into the same file group with the\nconflicts resolved automatically by the query reader and the compactor. The new concurrency mode is currently\navailable for preview in version 1.0.0-beta only. You can read more about it under section ",(0,r.jsx)(i.a,{href:"#model-c-multi-writer",children:"Model C: Multi-writer"}),"."]}),"\n",(0,r.jsx)(i.p,{children:"It is not always possible to serialize all write operations to a table (such as UPSERT, INSERT or DELETE) into the same write process and therefore, multi-writing capability may be required.\nIn multi-writing, disparate distributed processes run in parallel or overlapping time windows to write to the same table. In such cases, an external locking mechanism is a must to safely\ncoordinate concurrent accesses. Here are few different scenarios that would all fall under multi-writing:"}),"\n",(0,r.jsxs)(i.ul,{children:["\n",(0,r.jsxs)(i.li,{children:["Multiple ingestion writers to the same table",":For"," instance, two Spark Datasource writers working on different sets of partitions form a source kafka topic."]}),"\n",(0,r.jsx)(i.li,{children:"Multiple ingestion writers to the same table, including one writer with async table services: For example, a Hudi Streamer with async compaction for regular ingestion & a Spark Datasource writer for backfilling."}),"\n",(0,r.jsx)(i.li,{children:"A single ingestion writer and a separate compaction (HoodieCompactor) or clustering (HoodieClusteringJob) job apart from the ingestion writer: This is considered as multi-writing as they are not running in the same process."}),"\n"]}),"\n",(0,r.jsxs)(i.p,{children:["Hudi's concurrency model intelligently differentiates actual writing to the table from table services that manage or optimize the table. Hudi offers similar ",(0,r.jsx)(i.strong,{children:"optimistic concurrency control across multiple writers"}),", but ",(0,r.jsx)(i.strong,{children:"table services can still execute completely lock-free and async"})," as long as they run in the same process as one of the writers.\nFor multi-writing, Hudi leverages file level optimistic concurrency control(OCC). For example, when two writers write to non overlapping files, both writes are allowed to succeed. However, when the writes from different writers overlap (touch the same set of files), only one of them will succeed. Please note that this feature is currently experimental and requires external lock providers to acquire locks briefly at critical sections during the write. More on lock providers below."]}),"\n",(0,r.jsx)(i.h4,{id:"multi-writer-guarantees",children:"Multi Writer Guarantees"}),"\n",(0,r.jsx)(i.p,{children:"With multiple writers using OCC, these are the write guarantees to expect:"}),"\n",(0,r.jsxs)(i.ul,{children:["\n",(0,r.jsxs)(i.li,{children:[(0,r.jsx)(i.em,{children:"UPSERT Guarantee"}),": The target table will NEVER show duplicates."]}),"\n",(0,r.jsxs)(i.li,{children:[(0,r.jsx)(i.em,{children:"INSERT Guarantee"}),": The target table MIGHT have duplicates even if dedup is enabled."]}),"\n",(0,r.jsxs)(i.li,{children:[(0,r.jsx)(i.em,{children:"BULK_INSERT Guarantee"}),": The target table MIGHT have duplicates even if dedup is enabled."]}),"\n",(0,r.jsxs)(i.li,{children:[(0,r.jsx)(i.em,{children:"INCREMENTAL PULL Guarantee"}),": Data consumption and checkpoints are NEVER out of order. If there are inflight commits\n(due to multi-writing), incremental queries will not expose the completed commits following the inflight commits."]}),"\n"]}),"\n",(0,r.jsx)(i.h2,{id:"non-blocking-concurrency-control",children:"Non-Blocking Concurrency Control"}),"\n",(0,r.jsxs)(i.p,{children:[(0,r.jsx)(i.code,{children:"NON_BLOCKING_CONCURRENCY_CONTROL"}),", offers the same set of guarantees as mentioned in the case of OCC but without\nexplicit locks for serializing the writes. Lock is only needed for writing the commit metadata to the Hudi timeline. The\ncompletion time for the commits reflects the serialization order and file slicing is done based on completion time.\nMultiple writers can operate on the table with non-blocking conflict resolution. The writers can write into the same\nfile group with the conflicts resolved automatically by the query reader and the compactor. The new concurrency mode is\ncurrently available for preview in version 1.0.0-beta only with the caveat that conflict resolution is not supported yet\nbetween clustering and ingestion. It works for compaction and ingestion, and we can see an example of that with Flink\nwriters ",(0,r.jsx)(i.a,{href:"sql_dml#non-blocking-concurrency-control-experimental",children:"here"}),"."]}),"\n",(0,r.jsx)(i.admonition,{type:"note",children:(0,r.jsxs)(i.p,{children:[(0,r.jsx)(i.code,{children:"NON_BLOCKING_CONCURRENCY_CONTROL"})," between ingestion writer and table service writer is not yet supported for clustering.\nPlease use ",(0,r.jsx)(i.code,{children:"OPTIMISTIC_CONCURRENCY_CONTROL"})," for clustering."]})}),"\n",(0,r.jsx)(i.h2,{id:"early-conflict-detection",children:"Early conflict Detection"}),"\n",(0,r.jsx)(i.p,{children:"Multi writing using OCC allows multiple writers to concurrently write and atomically commit to the Hudi table if there is no overlapping data file to be written, to guarantee data consistency, integrity and correctness. Prior to 0.13.0 release, as the OCC (optimistic concurrency control) name suggests, each writer will optimistically proceed with ingestion and towards the end, just before committing will go about conflict resolution flow to deduce overlapping writes and abort one if need be. But this could result in lot of compute waste, since the aborted commit will have to retry from beginning. With 0.13.0, Hudi introduced early conflict deduction leveraging markers in hudi to deduce the conflicts eagerly and abort early in the write lifecyle instead of doing it in the end. For large scale deployments, this might avoid wasting lot o compute resources if there could be overlapping concurrent writers."}),"\n",(0,r.jsxs)(i.p,{children:["To improve the concurrency control, the ",(0,r.jsx)(i.a,{href:"https://hudi.apache.org/releases/release-0.13.0#early-conflict-detection-for-multi-writer",children:"0.13.0 release"})," introduced a new feature, early conflict detection in OCC, to detect the conflict during the data writing phase and abort the writing early on once a conflict is detected, using Hudi's marker mechanism. Hudi can now stop a conflicting writer much earlier because of the early conflict detection and release computing resources necessary to cluster, improving resource utilization."]}),"\n",(0,r.jsxs)(i.p,{children:["By default, this feature is turned off. To try this out, a user needs to set ",(0,r.jsx)(i.code,{children:"hoodie.write.concurrency.early.conflict.detection.enable"})," to true, when using OCC for concurrency control (Refer ",(0,r.jsx)(i.a,{href:"https://hudi.apache.org/docs/next/configurations#Write-Configurations-advanced-configs",children:"configs"})," page for all relevant configs)."]}),"\n",(0,r.jsx)(i.admonition,{type:"note",children:(0,r.jsxs)(i.p,{children:["Early conflict Detection in OCC is an ",(0,r.jsx)(i.strong,{children:"EXPERIMENTAL"})," feature"]})}),"\n",(0,r.jsx)(i.h2,{id:"enabling-multi-writing",children:"Enabling Multi Writing"}),"\n",(0,r.jsx)(i.p,{children:"The following properties are needed to be set appropriately to turn on optimistic concurrency control to achieve multi writing."}),"\n",(0,r.jsx)(i.pre,{children:(0,r.jsx)(i.code,{children:"hoodie.write.concurrency.mode=optimistic_concurrency_control\nhoodie.write.lock.provider=\nhoodie.cleaner.policy.failed.writes=LAZY\n"})}),"\n",(0,r.jsxs)(i.table,{children:[(0,r.jsx)(i.thead,{children:(0,r.jsxs)(i.tr,{children:[(0,r.jsx)(i.th,{children:"Config Name"}),(0,r.jsx)(i.th,{children:"Default"}),(0,r.jsx)(i.th,{children:"Description"})]})}),(0,r.jsxs)(i.tbody,{children:[(0,r.jsxs)(i.tr,{children:[(0,r.jsx)(i.td,{children:"hoodie.write.concurrency.mode"}),(0,r.jsx)(i.td,{children:"SINGLE_WRITER (Optional)"}),(0,r.jsxs)(i.td,{children:[(0,r.jsx)("u",{children:(0,r.jsx)(i.a,{href:"https://github.com/apache/hudi/blob/00ece7bce0a4a8d0019721a28049723821e01842/hudi-common/src/main/java/org/apache/hudi/common/model/WriteConcurrencyMode.java",children:"Concurrency modes"})})," for write operations.",(0,r.jsx)("br",{}),"Possible values:",(0,r.jsx)("br",{}),(0,r.jsxs)("ul",{children:[(0,r.jsxs)("li",{children:[(0,r.jsx)(i.code,{children:"SINGLE_WRITER"}),": Only one active writer to the table. Maximizes throughput."]}),(0,r.jsxs)("li",{children:[(0,r.jsx)(i.code,{children:"OPTIMISTIC_CONCURRENCY_CONTROL"}),": Multiple writers can operate on the table with lazy conflict resolution using locks. This means that only one writer succeeds if multiple writers write to the same file group."]}),(0,r.jsxs)("li",{children:[(0,r.jsx)(i.code,{children:"NON_BLOCKING_CONCURRENCY_CONTROL"}),": Multiple writers can operate on the table with non-blocking conflict resolution. The writers can write into the same file group with the conflicts resolved automatically by the query reader and the compactor."]})]}),(0,r.jsx)("br",{}),(0,r.jsx)(i.code,{children:"Config Param: WRITE_CONCURRENCY_MODE"})]})]}),(0,r.jsxs)(i.tr,{children:[(0,r.jsx)(i.td,{children:"hoodie.write.lock.provider"}),(0,r.jsx)(i.td,{children:"org.apache.hudi.client.transaction.lock.ZookeeperBasedLockProvider (Optional)"}),(0,r.jsxs)(i.td,{children:["Lock provider class name, user can provide their own implementation of LockProvider which should be subclass of org.apache.hudi.common.lock.LockProvider",(0,r.jsx)("br",{}),(0,r.jsx)("br",{}),(0,r.jsx)(i.code,{children:"Config Param: LOCK_PROVIDER_CLASS_NAME"}),(0,r.jsx)("br",{}),(0,r.jsx)(i.code,{children:"Since Version: 0.8.0"})]})]}),(0,r.jsxs)(i.tr,{children:[(0,r.jsx)(i.td,{children:"hoodie.cleaner.policy.failed.writes"}),(0,r.jsx)(i.td,{children:"EAGER (Optional)"}),(0,r.jsxs)(i.td,{children:["org.apache.hudi.common.model.HoodieFailedWritesCleaningPolicy: Policy that controls how to clean up failed writes. Hudi will delete any files written by failed writes to re-claim space. EAGER(default): Clean failed writes inline after every write operation. LAZY: Clean failed writes lazily after heartbeat timeout when the cleaning service runs. This policy is required when multi-writers are enabled. NEVER: Never clean failed writes.",(0,r.jsx)("br",{}),(0,r.jsx)("br",{}),(0,r.jsx)(i.code,{children:"Config Param: FAILED_WRITES_CLEANER_POLICY"})]})]})]})]}),"\n",(0,r.jsx)(i.h3,{id:"multi-writing-via-hudi-streamer",children:"Multi Writing via Hudi Streamer"}),"\n",(0,r.jsxs)(i.p,{children:["The ",(0,r.jsx)(i.code,{children:"HoodieStreamer"})," utility (part of hudi-utilities-slim-bundle) provides ways to ingest from different sources such as DFS or Kafka, with the following capabilities."]}),"\n",(0,r.jsx)(i.p,{children:"Using optimistic_concurrency_control via Hudi Streamer requires adding the above configs to the properties file that can be passed to the\njob. For example below, adding the configs to kafka-source.properties file and passing them to Hudi Streamer will enable optimistic concurrency.\nA Hudi Streamer job can then be triggered as follows:"}),"\n",(0,r.jsx)(i.pre,{children:(0,r.jsx)(i.code,{className:"language-java",children:'[hoodie]$ spark-submit \\\n --jars "packaging/hudi-utilities-slim-bundle/target/hudi-utilities-slim-bundle_2.12-1.0.0.jar,packaging/hudi-spark-bundle/target/hudi-spark3.5-bundle_2.12-1.0.0.jar" \\\n --class org.apache.hudi.utilities.streamer.HoodieStreamer `ls packaging/hudi-utilities-slim-bundle/target/hudi-utilities-slim-bundle-*.jar` \\\n --props file://${PWD}/hudi-utilities/src/test/resources/streamer-config/kafka-source.properties \\\n --schemaprovider-class org.apache.hudi.utilities.schema.SchemaRegistryProvider \\\n --source-class org.apache.hudi.utilities.sources.AvroKafkaSource \\\n --source-ordering-field impresssiontime \\\n --target-base-path file:\\/\\/\\/tmp/hudi-streamer-op \\ \n --target-table tableName \\\n --op BULK_INSERT\n'})}),"\n",(0,r.jsx)(i.h3,{id:"multi-writing-via-spark-datasource-writer",children:"Multi Writing via Spark Datasource Writer"}),"\n",(0,r.jsxs)(i.p,{children:["The ",(0,r.jsx)(i.code,{children:"hudi-spark"})," module offers the DataSource API to write (and read) a Spark DataFrame into a Hudi table."]}),"\n",(0,r.jsx)(i.p,{children:"Following is an example of how to use optimistic_concurrency_control via spark datasource"}),"\n",(0,r.jsx)(i.pre,{children:(0,r.jsx)(i.code,{className:"language-java",children:'inputDF.write.format("hudi")\n .options(getQuickstartWriteConfigs)\n .option("hoodie.datasource.write.precombine.field", "ts")\n .option("hoodie.cleaner.policy.failed.writes", "LAZY")\n .option("hoodie.write.concurrency.mode", "optimistic_concurrency_control")\n .option("hoodie.write.lock.zookeeper.url", "zookeeper")\n .option("hoodie.write.lock.zookeeper.port", "2181")\n .option("hoodie.write.lock.zookeeper.base_path", "/test")\n .option("hoodie.datasource.write.recordkey.field", "uuid")\n .option("hoodie.datasource.write.partitionpath.field", "partitionpath")\n .option("hoodie.table.name", tableName)\n .mode(Overwrite)\n .save(basePath)\n'})}),"\n",(0,r.jsx)(i.h2,{id:"disabling-multi-writing",children:"Disabling Multi Writing"}),"\n",(0,r.jsx)(i.p,{children:"Remove the following settings that were used to enable multi-writer or override with default values."}),"\n",(0,r.jsx)(i.pre,{children:(0,r.jsx)(i.code,{children:"hoodie.write.concurrency.mode=single_writer\nhoodie.cleaner.policy.failed.writes=EAGER\n"})}),"\n",(0,r.jsx)(i.h2,{id:"occ-best-practices",children:"OCC Best Practices"}),"\n",(0,r.jsx)(i.p,{children:"Concurrent Writing to Hudi tables requires acquiring a lock with one of the lock providers mentioned above. Due to several reasons you might want to configure retries to allow your application to acquire the lock."}),"\n",(0,r.jsxs)(i.ol,{children:["\n",(0,r.jsx)(i.li,{children:"Network connectivity or excessive load on servers increasing time for lock acquisition resulting in timeouts"}),"\n",(0,r.jsx)(i.li,{children:"Running a large number of concurrent jobs that are writing to the same hudi table can result in contention during lock acquisition can cause timeouts"}),"\n",(0,r.jsx)(i.li,{children:"In some scenarios of conflict resolution, Hudi commit operations might take upto 10's of seconds while the lock is being held. This can result in timeouts for other jobs waiting to acquire a lock."}),"\n"]}),"\n",(0,r.jsx)(i.p,{children:"Set the correct native lock provider client retries."}),"\n",(0,r.jsxs)(i.admonition,{type:"note",children:[(0,r.jsx)(i.mdxAdmonitionTitle,{}),(0,r.jsx)(i.p,{children:"Please note that sometimes these settings are set on the server once and all clients inherit the same configs. Please check your settings before enabling optimistic concurrency."})]}),"\n",(0,r.jsx)(i.pre,{children:(0,r.jsx)(i.code,{children:"hoodie.write.lock.wait_time_ms\nhoodie.write.lock.num_retries\n"})}),"\n",(0,r.jsx)(i.p,{children:"Set the correct hudi client retries for Zookeeper & HiveMetastore. This is useful in cases when native client retry settings cannot be changed. Please note that these retries will happen in addition to any native client retries that you may have set."}),"\n",(0,r.jsx)(i.pre,{children:(0,r.jsx)(i.code,{children:"hoodie.write.lock.client.wait_time_ms\nhoodie.write.lock.client.num_retries\n"})}),"\n",(0,r.jsx)(i.p,{children:(0,r.jsx)(i.em,{children:"Setting the right values for these depends on a case by case basis; some defaults have been provided for general cases."})}),"\n",(0,r.jsx)(i.h2,{id:"caveats",children:"Caveats"}),"\n",(0,r.jsxs)(i.p,{children:["If you are using the ",(0,r.jsx)(i.code,{children:"WriteClient"})," API, please note that multiple writes to the table need to be initiated from 2 different instances of the write client.\nIt is ",(0,r.jsx)(i.strong,{children:"NOT"})," recommended to use the same instance of the write client to perform multi writing."]}),"\n",(0,r.jsx)(i.h2,{id:"related-resources",children:"Related Resources"}),"\n",(0,r.jsx)("h3",{children:"Videos"}),"\n",(0,r.jsxs)(i.ul,{children:["\n",(0,r.jsx)(i.li,{children:(0,r.jsx)(i.a,{href:"https://youtu.be/JP0orl9_0yQ",children:"Hands on Lab with using DynamoDB as lock table for Apache Hudi Data Lakes"})}),"\n",(0,r.jsx)(i.li,{children:(0,r.jsx)(i.a,{href:"/blog/2024/12/06/non-blocking-concurrency-control",children:"Non Blocking Concurrency Control Flink Demo"})}),"\n"]})]})}function h(e={}){const{wrapper:i}={...(0,o.R)(),...e.components};return i?(0,r.jsx)(i,{...e,children:(0,r.jsx)(d,{...e})}):d(e)}},28453:(e,i,t)=>{t.d(i,{R:()=>s,x:()=>a});var n=t(96540);const r={},o=n.createContext(r);function s(e){const i=n.useContext(o);return n.useMemo((function(){return"function"==typeof e?e(i):{...i,...e}}),[i,e])}function a(e){let i;return i=e.disableParentContext?"function"==typeof e.components?e.components(r):e.components||r:s(e.components),n.createElement(o.Provider,{value:i},e.children)}}}]);
\ No newline at end of file
+"use strict";(self.webpackChunkhudi=self.webpackChunkhudi||[]).push([[23700],{49188:(e,i,t)=>{t.r(i),t.d(i,{assets:()=>c,contentTitle:()=>a,default:()=>h,frontMatter:()=>s,metadata:()=>n,toc:()=>l});const n=JSON.parse('{"id":"concurrency_control","title":"Concurrency Control","description":"Concurrency control defines how different writers/readers/table services coordinate access to a Hudi table. Hudi ensures atomic writes, by way of publishing commits atomically to the timeline,","source":"@site/docs/concurrency_control.md","sourceDirName":".","slug":"/concurrency_control","permalink":"/docs/next/concurrency_control","draft":false,"unlisted":false,"editUrl":"https://github.com/apache/hudi/tree/asf-site/website/docs/concurrency_control.md","tags":[],"version":"current","frontMatter":{"title":"Concurrency Control","summary":"In this page, we will discuss how to perform concurrent writes to Hudi Tables.","toc":true,"toc_min_heading_level":2,"toc_max_heading_level":4,"last_modified_at":"2021-03-19T19:59:57.000Z"},"sidebar":"docs","previous":{"title":"Indexes","permalink":"/docs/next/indexes"},"next":{"title":"Schema Evolution","permalink":"/docs/next/schema_evolution"}}');var r=t(74848),o=t(28453);const s={title:"Concurrency Control",summary:"In this page, we will discuss how to perform concurrent writes to Hudi Tables.",toc:!0,toc_min_heading_level:2,toc_max_heading_level:4,last_modified_at:new Date("2021-03-19T19:59:57.000Z")},a=void 0,c={},l=[{value:"Distributed Locking",id:"distributed-locking",level:2},{value:"Zookeeper based",id:"zookeeper-based",level:4},{value:"HiveMetastore based",id:"hivemetastore-based",level:4},{value:"Amazon DynamoDB based",id:"amazon-dynamodb-based",level:4},{value:"FileSystem based (not for production use)",id:"filesystem-based-not-for-production-use",level:4},{value:"Simple Single writer + table services",id:"simple-single-writer--table-services",level:2},{value:"Inline table services",id:"inline-table-services",level:3},{value:"Async table services",id:"async-table-services",level:3},{value:"Full-on Multi-writer + Async table services",id:"full-on-multi-writer--async-table-services",level:2},{value:"Multi Writer Guarantees",id:"multi-writer-guarantees",level:4},{value:"Non-Blocking Concurrency Control",id:"non-blocking-concurrency-control",level:2},{value:"Early conflict Detection",id:"early-conflict-detection",level:2},{value:"Enabling Multi Writing",id:"enabling-multi-writing",level:2},{value:"Multi Writing via Hudi Streamer",id:"multi-writing-via-hudi-streamer",level:3},{value:"Multi Writing via Spark Datasource Writer",id:"multi-writing-via-spark-datasource-writer",level:3},{value:"Disabling Multi Writing",id:"disabling-multi-writing",level:2},{value:"OCC Best Practices",id:"occ-best-practices",level:2},{value:"Caveats",id:"caveats",level:2},{value:"Related Resources",id:"related-resources",level:2}];function d(e){const i={a:"a",admonition:"admonition",code:"code",em:"em",h2:"h2",h3:"h3",h4:"h4",li:"li",mdxAdmonitionTitle:"mdxAdmonitionTitle",ol:"ol",p:"p",pre:"pre",strong:"strong",table:"table",tbody:"tbody",td:"td",th:"th",thead:"thead",tr:"tr",ul:"ul",...(0,o.R)(),...e.components};return(0,r.jsxs)(r.Fragment,{children:[(0,r.jsxs)(i.p,{children:["Concurrency control defines how different writers/readers/table services coordinate access to a Hudi table. Hudi ensures atomic writes, by way of publishing commits atomically to the timeline,\nstamped with an instant time that denotes the time at which the action is deemed to have occurred. Unlike general purpose file version control, Hudi draws clear distinction between\nwriter processes that issue ",(0,r.jsx)(i.a,{href:"write_operations",children:"write operations"})," and table services that (re)write data/metadata to optimize/perform bookkeeping and\nreaders (that execute queries and read data)."]}),"\n",(0,r.jsx)(i.p,{children:"Hudi provides"}),"\n",(0,r.jsxs)(i.ul,{children:["\n",(0,r.jsxs)(i.li,{children:[(0,r.jsx)(i.strong,{children:"Snapshot isolation"})," between all three types of processes, meaning they all operate on a consistent snapshot of the table."]}),"\n",(0,r.jsxs)(i.li,{children:[(0,r.jsx)(i.strong,{children:"Optimistic concurrency control (OCC)"})," between writers to provide standard relational database semantics."]}),"\n",(0,r.jsxs)(i.li,{children:[(0,r.jsx)(i.strong,{children:"Multiversion Concurrency Control (MVCC)"})," based concurrency control between writers and table-services and between different table services."]}),"\n",(0,r.jsxs)(i.li,{children:[(0,r.jsx)(i.strong,{children:"Non-blocking Concurrency Control (NBCC)"})," between writers, to provide streaming semantics and avoiding live-locks/starvation between writers."]}),"\n"]}),"\n",(0,r.jsx)(i.p,{children:"In this section, we will discuss the different concurrency controls supported by Hudi and how they are leveraged to provide flexible deployment models for single and multiple writer scenarios.\nWe\u2019ll also describe ways to ingest data into a Hudi Table from multiple writers using different writers, like Hudi Streamer, Hudi datasource, Spark Structured Streaming and Spark SQL."}),"\n",(0,r.jsxs)(i.admonition,{type:"note",children:[(0,r.jsx)(i.p,{children:"If there is only one process performing writing AND async/inline table services on the table, you can\navoid the overhead of a distributed lock requirement by configuring the in process lock provider."}),(0,r.jsx)(i.pre,{children:(0,r.jsx)(i.code,{className:"language-properties",children:"hoodie.write.lock.provider=org.apache.hudi.client.transaction.lock.InProcessLockProvider\n"})})]}),"\n",(0,r.jsx)(i.h2,{id:"distributed-locking",children:"Distributed Locking"}),"\n",(0,r.jsxs)(i.p,{children:["A pre-requisite for distributed co-ordination in Hudi, like many other distributed database systems is a distributed lock provider, that different processes can use to plan, schedule and\nexecute actions on the Hudi timeline in a concurrent fashion. Locks are also used to ",(0,r.jsx)(i.a,{href:"timeline#truetime-generation",children:"generate TrueTime"}),", as discussed before."]}),"\n",(0,r.jsx)(i.p,{children:"External locking is typically used in conjunction with optimistic concurrency control\nbecause it provides a way to prevent conflicts that might occur when two or more transactions (commits in our case) attempt to modify the same resource concurrently.\nWhen a transaction attempts to modify a resource that is currently locked by another transaction, it must wait until the lock is released before proceeding."}),"\n",(0,r.jsx)(i.p,{children:"In case of multi-writing in Hudi, the locks are acquired on the Hudi table for a very short duration during specific phases (such as just before committing the writes or before scheduling table services) instead of locking for the entire span of time. This approach allows multiple writers to work on the same table simultaneously, increasing concurrency and avoids conflicts."}),"\n",(0,r.jsxs)(i.p,{children:["There are 4 different lock providers that require different configurations to be set. Please refer to comprehensive locking configs ",(0,r.jsx)(i.a,{href:"https://hudi.apache.org/docs/next/configurations#LOCK",children:"here"}),"."]}),"\n",(0,r.jsx)(i.h4,{id:"zookeeper-based",children:"Zookeeper based"}),"\n",(0,r.jsx)(i.pre,{children:(0,r.jsx)(i.code,{children:"hoodie.write.lock.provider=org.apache.hudi.client.transaction.lock.ZookeeperBasedLockProvider\n"})}),"\n",(0,r.jsx)(i.p,{children:"Following are the basic configs required to setup this lock provider:"}),"\n",(0,r.jsxs)(i.table,{children:[(0,r.jsx)(i.thead,{children:(0,r.jsxs)(i.tr,{children:[(0,r.jsx)(i.th,{children:"Config Name"}),(0,r.jsx)(i.th,{children:"Default"}),(0,r.jsx)(i.th,{children:"Description"})]})}),(0,r.jsxs)(i.tbody,{children:[(0,r.jsxs)(i.tr,{children:[(0,r.jsx)(i.td,{children:"hoodie.write.lock.zookeeper.base_path"}),(0,r.jsxs)(i.td,{children:["N/A ",(0,r.jsx)(i.strong,{children:"(Required)"})]}),(0,r.jsxs)(i.td,{children:["The base path on Zookeeper under which to create lock related ZNodes. This should be same for all concurrent writers to the same table",(0,r.jsx)("br",{}),(0,r.jsx)("br",{}),(0,r.jsx)(i.code,{children:"Config Param: ZK_BASE_PATH"}),(0,r.jsx)("br",{}),(0,r.jsx)(i.code,{children:"Since Version: 0.8.0"})]})]}),(0,r.jsxs)(i.tr,{children:[(0,r.jsx)(i.td,{children:"hoodie.write.lock.zookeeper.port"}),(0,r.jsxs)(i.td,{children:["N/A ",(0,r.jsx)(i.strong,{children:"(Required)"})]}),(0,r.jsxs)(i.td,{children:["Zookeeper port to connect to.",(0,r.jsx)("br",{}),(0,r.jsx)("br",{}),(0,r.jsx)(i.code,{children:"Config Param: ZK_PORT"}),(0,r.jsx)("br",{}),(0,r.jsx)(i.code,{children:"Since Version: 0.8.0"})]})]}),(0,r.jsxs)(i.tr,{children:[(0,r.jsx)(i.td,{children:"hoodie.write.lock.zookeeper.url"}),(0,r.jsxs)(i.td,{children:["N/A ",(0,r.jsx)(i.strong,{children:"(Required)"})]}),(0,r.jsxs)(i.td,{children:["Zookeeper URL to connect to.",(0,r.jsx)("br",{}),(0,r.jsx)("br",{}),(0,r.jsx)(i.code,{children:"Config Param: ZK_CONNECT_URL"}),(0,r.jsx)("br",{}),(0,r.jsx)(i.code,{children:"Since Version: 0.8.0"})]})]})]})]}),"\n",(0,r.jsx)(i.h4,{id:"hivemetastore-based",children:"HiveMetastore based"}),"\n",(0,r.jsx)(i.pre,{children:(0,r.jsx)(i.code,{children:"hoodie.write.lock.provider=org.apache.hudi.hive.transaction.lock.HiveMetastoreBasedLockProvider\n"})}),"\n",(0,r.jsx)(i.p,{children:"Following are the basic configs required to setup this lock provider:"}),"\n",(0,r.jsxs)(i.table,{children:[(0,r.jsx)(i.thead,{children:(0,r.jsxs)(i.tr,{children:[(0,r.jsx)(i.th,{children:"Config Name"}),(0,r.jsx)(i.th,{children:"Default"}),(0,r.jsx)(i.th,{children:"Description"})]})}),(0,r.jsxs)(i.tbody,{children:[(0,r.jsxs)(i.tr,{children:[(0,r.jsx)(i.td,{children:"hoodie.write.lock.hivemetastore.database"}),(0,r.jsxs)(i.td,{children:["N/A ",(0,r.jsx)(i.strong,{children:"(Required)"})]}),(0,r.jsxs)(i.td,{children:["For Hive based lock provider, the Hive database to acquire lock against",(0,r.jsx)("br",{}),(0,r.jsx)("br",{}),(0,r.jsx)(i.code,{children:"Config Param: HIVE_DATABASE_NAME"}),(0,r.jsx)("br",{}),(0,r.jsx)(i.code,{children:"Since Version: 0.8.0"})]})]}),(0,r.jsxs)(i.tr,{children:[(0,r.jsx)(i.td,{children:"hoodie.write.lock.hivemetastore.table"}),(0,r.jsxs)(i.td,{children:["N/A ",(0,r.jsx)(i.strong,{children:"(Required)"})]}),(0,r.jsxs)(i.td,{children:["For Hive based lock provider, the Hive table to acquire lock against",(0,r.jsx)("br",{}),(0,r.jsx)("br",{}),(0,r.jsx)(i.code,{children:"Config Param: HIVE_TABLE_NAME"}),(0,r.jsx)("br",{}),(0,r.jsx)(i.code,{children:"Since Version: 0.8.0"})]})]})]})]}),"\n",(0,r.jsx)(i.p,{children:(0,r.jsx)(i.code,{children:"The HiveMetastore URI's are picked up from the hadoop configuration file loaded during runtime."})}),"\n",(0,r.jsx)(i.h4,{id:"amazon-dynamodb-based",children:"Amazon DynamoDB based"}),"\n",(0,r.jsx)(i.pre,{children:(0,r.jsx)(i.code,{children:"hoodie.write.lock.provider=org.apache.hudi.aws.transaction.lock.DynamoDBBasedLockProvider\n"})}),"\n",(0,r.jsxs)(i.p,{children:["Amazon DynamoDB based lock provides a simple way to support multi writing across different clusters. You can refer to the\n",(0,r.jsx)(i.a,{href:"https://hudi.apache.org/docs/configurations#DynamoDB-based-Locks-Configurations",children:"DynamoDB based Locks Configurations"}),"\nsection for the details of each related configuration knob. Following are the basic configs required to setup this lock provider:"]}),"\n",(0,r.jsxs)(i.table,{children:[(0,r.jsx)(i.thead,{children:(0,r.jsxs)(i.tr,{children:[(0,r.jsx)(i.th,{children:"Config Name"}),(0,r.jsx)(i.th,{children:"Default"}),(0,r.jsx)(i.th,{children:"Description"})]})}),(0,r.jsx)(i.tbody,{children:(0,r.jsxs)(i.tr,{children:[(0,r.jsx)(i.td,{children:"hoodie.write.lock.dynamodb.endpoint_url"}),(0,r.jsxs)(i.td,{children:["N/A ",(0,r.jsx)(i.strong,{children:"(Required)"})]}),(0,r.jsxs)(i.td,{children:["For DynamoDB based lock provider, the url endpoint used for Amazon DynamoDB service. Useful for development with a local dynamodb instance.",(0,r.jsx)("br",{}),(0,r.jsx)("br",{}),(0,r.jsx)(i.code,{children:"Config Param: DYNAMODB_ENDPOINT_URL"}),(0,r.jsx)("br",{}),(0,r.jsx)(i.code,{children:"Since Version: 0.10.1"})]})]})})]}),"\n",(0,r.jsxs)(i.p,{children:["For advanced configs refer ",(0,r.jsx)(i.a,{href:"https://hudi.apache.org/docs/next/configurations#DynamoDB-based-Locks-Configurations",children:"here"})]}),"\n",(0,r.jsxs)(i.p,{children:["When using the DynamoDB-based lock provider, the name of the DynamoDB table acting as the lock table for Hudi is\nspecified by the config ",(0,r.jsx)(i.code,{children:"hoodie.write.lock.dynamodb.table"}),". This DynamoDB table is automatically created by Hudi, so you\ndon't have to create the table yourself. If you want to use an existing DynamoDB table, make sure that an attribute with\nthe name ",(0,r.jsx)(i.code,{children:"key"})," is present in the table. The ",(0,r.jsx)(i.code,{children:"key"})," attribute should be the partition key of the DynamoDB table. The\nconfig ",(0,r.jsx)(i.code,{children:"hoodie.write.lock.dynamodb.partition_key"})," specifies the value to put for the ",(0,r.jsx)(i.code,{children:"key"})," attribute (not the attribute\nname), which is used for the lock on the same table. By default, ",(0,r.jsx)(i.code,{children:"hoodie.write.lock.dynamodb.partition_key"})," is set to\nthe table name, so that multiple writers writing to the same table share the same lock. If you customize the name, make\nsure it's the same across multiple writers."]}),"\n",(0,r.jsx)(i.p,{children:"Also, to set up the credentials for accessing AWS resources, customers can pass the following props to Hudi jobs:"}),"\n",(0,r.jsx)(i.pre,{children:(0,r.jsx)(i.code,{children:"hoodie.aws.access.key\nhoodie.aws.secret.key\nhoodie.aws.session.token\n"})}),"\n",(0,r.jsxs)(i.p,{children:["If not configured, Hudi falls back to use ",(0,r.jsx)(i.a,{href:"https://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/auth/DefaultAWSCredentialsProviderChain.html",children:"DefaultAWSCredentialsProviderChain"}),"."]}),"\n",(0,r.jsx)(i.p,{children:"IAM policy for your service instance will need to add the following permissions:"}),"\n",(0,r.jsx)(i.pre,{children:(0,r.jsx)(i.code,{className:"language-json",children:'{\n "Sid":"DynamoDBLocksTable",\n "Effect": "Allow",\n "Action": [\n "dynamodb:CreateTable",\n "dynamodb:DeleteItem",\n "dynamodb:DescribeTable",\n "dynamodb:GetItem",\n "dynamodb:PutItem",\n "dynamodb:Scan",\n "dynamodb:UpdateItem"\n ],\n "Resource": "arn:${Partition}:dynamodb:${Region}:${Account}:table/${TableName}"\n}\n'})}),"\n",(0,r.jsxs)(i.ul,{children:["\n",(0,r.jsxs)(i.li,{children:[(0,r.jsx)(i.code,{children:"TableName"})," : same as ",(0,r.jsx)(i.code,{children:"hoodie.write.lock.dynamodb.partition_key"})]}),"\n",(0,r.jsxs)(i.li,{children:[(0,r.jsx)(i.code,{children:"Region"}),": same as ",(0,r.jsx)(i.code,{children:"hoodie.write.lock.dynamodb.region"})]}),"\n"]}),"\n",(0,r.jsx)(i.p,{children:"AWS SDK dependencies are not bundled with Hudi from v0.10.x and will need to be added to your classpath.\nAdd the following Maven packages (check the latest versions at time of install):"}),"\n",(0,r.jsx)(i.pre,{children:(0,r.jsx)(i.code,{children:"com.amazonaws:dynamodb-lock-client\ncom.amazonaws:aws-java-sdk-dynamodb\ncom.amazonaws:aws-java-sdk-core\n"})}),"\n",(0,r.jsx)(i.h4,{id:"filesystem-based-not-for-production-use",children:"FileSystem based (not for production use)"}),"\n",(0,r.jsx)(i.p,{children:"FileSystem based lock provider supports multiple writers cross different jobs/applications based on atomic create/delete operations of the underlying filesystem."}),"\n",(0,r.jsx)(i.pre,{children:(0,r.jsx)(i.code,{children:"hoodie.write.lock.provider=org.apache.hudi.client.transaction.lock.FileSystemBasedLockProvider\n"})}),"\n",(0,r.jsxs)(i.p,{children:["When using the FileSystem based lock provider, by default, the lock file will store into ",(0,r.jsx)(i.code,{children:"hoodie.base.path"}),"+",(0,r.jsx)(i.code,{children:"/.hoodie/lock"}),". You may use a custom folder to store the lock file by specifying ",(0,r.jsx)(i.code,{children:"hoodie.write.lock.filesystem.path"}),"."]}),"\n",(0,r.jsxs)(i.p,{children:["In case the lock cannot release during job crash, you can set ",(0,r.jsx)(i.code,{children:"hoodie.write.lock.filesystem.expire"})," (lock will never expire by default) to a desired expire time in minutes. You may also delete lock file manually in such situation."]}),"\n",(0,r.jsx)(i.admonition,{type:"note",children:(0,r.jsx)(i.p,{children:"FileSystem based lock provider is not supported with cloud storage like S3 or GCS."})}),"\n",(0,r.jsx)(i.h2,{id:"simple-single-writer--table-services",children:"Simple Single writer + table services"}),"\n",(0,r.jsx)(i.p,{children:"Data lakehouse pipelines tend to be predominantly single writer, with the most common need for distributed co-ordination on a table coming from table management. For e.g. a Apache Flink\njob producing fast writes into a table, requiring regular file-size management or cleaning. Hudi's storage engine and platform tools provide a lot of support for such common scenarios."}),"\n",(0,r.jsx)(i.h3,{id:"inline-table-services",children:"Inline table services"}),"\n",(0,r.jsx)(i.p,{children:"This is the simplest form of concurrency, meaning there is no concurrency at all in the write processes. In this model, Hudi eliminates the need for concurrency control and maximizes throughput by supporting these table services out-of-box and running inline after every write to the table. Execution plans are idempotent, persisted to the timeline and auto-recover from failures. For most simple use-cases, this means just writing is sufficient to get a well-managed table that needs no concurrency control."}),"\n",(0,r.jsxs)(i.p,{children:["There is no actual concurrent writing in this model. ",(0,r.jsx)(i.strong,{children:"MVCC"})," is leveraged to provide snapshot isolation guarantees between ingestion writer and multiple readers and also between multiple table service writers and readers. Writes to the table either from ingestion or from table services produce versioned data that are available to readers only after the writes are committed. Until then, readers can access only the previous version of the data."]}),"\n",(0,r.jsx)(i.p,{children:"A single writer with all table services such as cleaning, clustering, compaction, etc can be configured to be inline (such as Hudi Streamer sync-once mode and Spark Datasource with default configs) without any additional configs."}),"\n",(0,r.jsx)(i.h3,{id:"async-table-services",children:"Async table services"}),"\n",(0,r.jsxs)(i.p,{children:["Hudi provides the option of running the table services in an async fashion, where most of the heavy lifting (e.g actually rewriting the columnar data by compaction service) is done asynchronously. In this model, the async deployment eliminates any repeated wasteful retries and optimizes the table using clustering techniques while a single writer consumes the writes to the table without having to be blocked by such table services. This model avoids the need for taking an ",(0,r.jsx)(i.a,{href:"#external-locking-and-lock-providers",children:"external lock"})," to control concurrency and avoids the need to separately orchestrate and monitor offline table services jobs.."]}),"\n",(0,r.jsxs)(i.p,{children:["A single writer along with async table services runs in the same process. For example, you can have a Hudi Streamer in continuous mode write to a MOR table using async compaction; you can use Spark Streaming (where ",(0,r.jsx)(i.a,{href:"https://hudi.apache.org/docs/compaction",children:"compaction"})," is async by default), and you can use Flink streaming or your own job setup and enable async table services inside the same writer."]}),"\n",(0,r.jsxs)(i.p,{children:["Hudi leverages ",(0,r.jsx)(i.strong,{children:"MVCC"})," in this model to support running any number of table service jobs concurrently, without any concurrency conflict. This is made possible by ensuring Hudi 's ingestion writer and async table services coordinate among themselves to ensure no conflicts and avoid race conditions. The same single write guarantees described in Model A above can be achieved in this model as well.\nWith this model users don't need to spin up different spark jobs and manage the orchestration among it. For larger deployments, this model can ease the operational burden significantly while getting the table services running without blocking the writers."]}),"\n",(0,r.jsx)(i.p,{children:(0,r.jsx)(i.strong,{children:"Single Writer Guarantees"})}),"\n",(0,r.jsxs)(i.p,{children:["In this model, the following are the guarantees on ",(0,r.jsx)(i.a,{href:"https://hudi.apache.org/docs/write_operations/",children:"write operations"})," to expect:"]}),"\n",(0,r.jsxs)(i.ul,{children:["\n",(0,r.jsxs)(i.li,{children:[(0,r.jsx)(i.em,{children:"UPSERT Guarantee"}),": The target table will NEVER show duplicates."]}),"\n",(0,r.jsxs)(i.li,{children:[(0,r.jsx)(i.em,{children:"INSERT Guarantee"}),": The target table wilL NEVER have duplicates if dedup: ",(0,r.jsx)(i.a,{href:"https://hudi.apache.org/docs/configurations#hoodiedatasourcewriteinsertdropduplicates",children:(0,r.jsx)(i.code,{children:"hoodie.datasource.write.insert.drop.duplicates"})})," & ",(0,r.jsx)(i.a,{href:"https://hudi.apache.org/docs/configurations/#hoodiecombinebeforeinsert",children:(0,r.jsx)(i.code,{children:"hoodie.combine.before.insert"})}),", is enabled."]}),"\n",(0,r.jsxs)(i.li,{children:[(0,r.jsx)(i.em,{children:"BULK_INSERT Guarantee"}),": The target table will NEVER have duplicates if dedup: ",(0,r.jsx)(i.a,{href:"https://hudi.apache.org/docs/configurations#hoodiedatasourcewriteinsertdropduplicates",children:(0,r.jsx)(i.code,{children:"hoodie.datasource.write.insert.drop.duplicates"})})," & ",(0,r.jsx)(i.a,{href:"https://hudi.apache.org/docs/configurations/#hoodiecombinebeforeinsert",children:(0,r.jsx)(i.code,{children:"hoodie.combine.before.insert"})}),", is enabled."]}),"\n",(0,r.jsxs)(i.li,{children:[(0,r.jsx)(i.em,{children:"INCREMENTAL QUERY Guarantee"}),": Data consumption and checkpoints are NEVER out of order."]}),"\n"]}),"\n",(0,r.jsx)(i.h2,{id:"full-on-multi-writer--async-table-services",children:"Full-on Multi-writer + Async table services"}),"\n",(0,r.jsxs)(i.p,{children:["Hudi has introduced a new concurrency mode ",(0,r.jsx)(i.code,{children:"NON_BLOCKING_CONCURRENCY_CONTROL"}),", where unlike OCC, multiple writers can\noperate on the table with non-blocking conflict resolution. The writers can write into the same file group with the\nconflicts resolved automatically by the query reader and the compactor. The new concurrency mode is currently\navailable for preview in version 1.0.0-beta only. You can read more about it under section ",(0,r.jsx)(i.a,{href:"#model-c-multi-writer",children:"Model C: Multi-writer"}),"."]}),"\n",(0,r.jsx)(i.p,{children:"It is not always possible to serialize all write operations to a table (such as UPSERT, INSERT or DELETE) into the same write process and therefore, multi-writing capability may be required.\nIn multi-writing, disparate distributed processes run in parallel or overlapping time windows to write to the same table. In such cases, an external locking mechanism is a must to safely\ncoordinate concurrent accesses. Here are few different scenarios that would all fall under multi-writing:"}),"\n",(0,r.jsxs)(i.ul,{children:["\n",(0,r.jsxs)(i.li,{children:["Multiple ingestion writers to the same table",":For"," instance, two Spark Datasource writers working on different sets of partitions form a source kafka topic."]}),"\n",(0,r.jsx)(i.li,{children:"Multiple ingestion writers to the same table, including one writer with async table services: For example, a Hudi Streamer with async compaction for regular ingestion & a Spark Datasource writer for backfilling."}),"\n",(0,r.jsx)(i.li,{children:"A single ingestion writer and a separate compaction (HoodieCompactor) or clustering (HoodieClusteringJob) job apart from the ingestion writer: This is considered as multi-writing as they are not running in the same process."}),"\n"]}),"\n",(0,r.jsxs)(i.p,{children:["Hudi's concurrency model intelligently differentiates actual writing to the table from table services that manage or optimize the table. Hudi offers similar ",(0,r.jsx)(i.strong,{children:"optimistic concurrency control across multiple writers"}),", but ",(0,r.jsx)(i.strong,{children:"table services can still execute completely lock-free and async"})," as long as they run in the same process as one of the writers.\nFor multi-writing, Hudi leverages file level optimistic concurrency control(OCC). For example, when two writers write to non overlapping files, both writes are allowed to succeed. However, when the writes from different writers overlap (touch the same set of files), only one of them will succeed. Please note that this feature is currently experimental and requires external lock providers to acquire locks briefly at critical sections during the write. More on lock providers below."]}),"\n",(0,r.jsx)(i.h4,{id:"multi-writer-guarantees",children:"Multi Writer Guarantees"}),"\n",(0,r.jsx)(i.p,{children:"With multiple writers using OCC, these are the write guarantees to expect:"}),"\n",(0,r.jsxs)(i.ul,{children:["\n",(0,r.jsxs)(i.li,{children:[(0,r.jsx)(i.em,{children:"UPSERT Guarantee"}),": The target table will NEVER show duplicates."]}),"\n",(0,r.jsxs)(i.li,{children:[(0,r.jsx)(i.em,{children:"INSERT Guarantee"}),": The target table MIGHT have duplicates even if dedup is enabled."]}),"\n",(0,r.jsxs)(i.li,{children:[(0,r.jsx)(i.em,{children:"BULK_INSERT Guarantee"}),": The target table MIGHT have duplicates even if dedup is enabled."]}),"\n",(0,r.jsxs)(i.li,{children:[(0,r.jsx)(i.em,{children:"INCREMENTAL PULL Guarantee"}),": Data consumption and checkpoints are NEVER out of order. If there are inflight commits\n(due to multi-writing), incremental queries will not expose the completed commits following the inflight commits."]}),"\n"]}),"\n",(0,r.jsx)(i.h2,{id:"non-blocking-concurrency-control",children:"Non-Blocking Concurrency Control"}),"\n",(0,r.jsxs)(i.p,{children:[(0,r.jsx)(i.code,{children:"NON_BLOCKING_CONCURRENCY_CONTROL"}),", offers the same set of guarantees as mentioned in the case of OCC but without\nexplicit locks for serializing the writes. Lock is only needed for writing the commit metadata to the Hudi timeline. The\ncompletion time for the commits reflects the serialization order and file slicing is done based on completion time.\nMultiple writers can operate on the table with non-blocking conflict resolution. The writers can write into the same\nfile group with the conflicts resolved automatically by the query reader and the compactor. The new concurrency mode is\ncurrently available for preview in version 1.0.0-beta only with the caveat that conflict resolution is not supported yet\nbetween clustering and ingestion. It works for compaction and ingestion, and we can see an example of that with Flink\nwriters ",(0,r.jsx)(i.a,{href:"sql_dml#non-blocking-concurrency-control-experimental",children:"here"}),"."]}),"\n",(0,r.jsx)(i.admonition,{type:"note",children:(0,r.jsxs)(i.p,{children:[(0,r.jsx)(i.code,{children:"NON_BLOCKING_CONCURRENCY_CONTROL"})," between ingestion writer and table service writer is not yet supported for clustering.\nPlease use ",(0,r.jsx)(i.code,{children:"OPTIMISTIC_CONCURRENCY_CONTROL"})," for clustering."]})}),"\n",(0,r.jsx)(i.h2,{id:"early-conflict-detection",children:"Early conflict Detection"}),"\n",(0,r.jsx)(i.p,{children:"Multi writing using OCC allows multiple writers to concurrently write and atomically commit to the Hudi table if there is no overlapping data file to be written, to guarantee data consistency, integrity and correctness. Prior to 0.13.0 release, as the OCC (optimistic concurrency control) name suggests, each writer will optimistically proceed with ingestion and towards the end, just before committing will go about conflict resolution flow to deduce overlapping writes and abort one if need be. But this could result in lot of compute waste, since the aborted commit will have to retry from beginning. With 0.13.0, Hudi introduced early conflict deduction leveraging markers in hudi to deduce the conflicts eagerly and abort early in the write lifecyle instead of doing it in the end. For large scale deployments, this might avoid wasting lot o compute resources if there could be overlapping concurrent writers."}),"\n",(0,r.jsxs)(i.p,{children:["To improve the concurrency control, the ",(0,r.jsx)(i.a,{href:"https://hudi.apache.org/releases/release-0.13.0#early-conflict-detection-for-multi-writer",children:"0.13.0 release"})," introduced a new feature, early conflict detection in OCC, to detect the conflict during the data writing phase and abort the writing early on once a conflict is detected, using Hudi's marker mechanism. Hudi can now stop a conflicting writer much earlier because of the early conflict detection and release computing resources necessary to cluster, improving resource utilization."]}),"\n",(0,r.jsxs)(i.p,{children:["By default, this feature is turned off. To try this out, a user needs to set ",(0,r.jsx)(i.code,{children:"hoodie.write.concurrency.early.conflict.detection.enable"})," to true, when using OCC for concurrency control (Refer ",(0,r.jsx)(i.a,{href:"https://hudi.apache.org/docs/next/configurations#Write-Configurations-advanced-configs",children:"configs"})," page for all relevant configs)."]}),"\n",(0,r.jsx)(i.admonition,{type:"note",children:(0,r.jsxs)(i.p,{children:["Early conflict Detection in OCC is an ",(0,r.jsx)(i.strong,{children:"EXPERIMENTAL"})," feature"]})}),"\n",(0,r.jsx)(i.h2,{id:"enabling-multi-writing",children:"Enabling Multi Writing"}),"\n",(0,r.jsx)(i.p,{children:"The following properties are needed to be set appropriately to turn on optimistic concurrency control to achieve multi writing."}),"\n",(0,r.jsx)(i.pre,{children:(0,r.jsx)(i.code,{children:"hoodie.write.concurrency.mode=optimistic_concurrency_control\nhoodie.write.lock.provider=\nhoodie.cleaner.policy.failed.writes=LAZY\n"})}),"\n",(0,r.jsxs)(i.table,{children:[(0,r.jsx)(i.thead,{children:(0,r.jsxs)(i.tr,{children:[(0,r.jsx)(i.th,{children:"Config Name"}),(0,r.jsx)(i.th,{children:"Default"}),(0,r.jsx)(i.th,{children:"Description"})]})}),(0,r.jsxs)(i.tbody,{children:[(0,r.jsxs)(i.tr,{children:[(0,r.jsx)(i.td,{children:"hoodie.write.concurrency.mode"}),(0,r.jsx)(i.td,{children:"SINGLE_WRITER (Optional)"}),(0,r.jsxs)(i.td,{children:[(0,r.jsx)("u",{children:(0,r.jsx)(i.a,{href:"https://github.com/apache/hudi/blob/00ece7bce0a4a8d0019721a28049723821e01842/hudi-common/src/main/java/org/apache/hudi/common/model/WriteConcurrencyMode.java",children:"Concurrency modes"})})," for write operations.",(0,r.jsx)("br",{}),"Possible values:",(0,r.jsx)("br",{}),(0,r.jsxs)("ul",{children:[(0,r.jsxs)("li",{children:[(0,r.jsx)(i.code,{children:"SINGLE_WRITER"}),": Only one active writer to the table. Maximizes throughput."]}),(0,r.jsxs)("li",{children:[(0,r.jsx)(i.code,{children:"OPTIMISTIC_CONCURRENCY_CONTROL"}),": Multiple writers can operate on the table with lazy conflict resolution using locks. This means that only one writer succeeds if multiple writers write to the same file group."]}),(0,r.jsxs)("li",{children:[(0,r.jsx)(i.code,{children:"NON_BLOCKING_CONCURRENCY_CONTROL"}),": Multiple writers can operate on the table with non-blocking conflict resolution. The writers can write into the same file group with the conflicts resolved automatically by the query reader and the compactor."]})]}),(0,r.jsx)("br",{}),(0,r.jsx)(i.code,{children:"Config Param: WRITE_CONCURRENCY_MODE"})]})]}),(0,r.jsxs)(i.tr,{children:[(0,r.jsx)(i.td,{children:"hoodie.write.lock.provider"}),(0,r.jsx)(i.td,{children:"org.apache.hudi.client.transaction.lock.ZookeeperBasedLockProvider (Optional)"}),(0,r.jsxs)(i.td,{children:["Lock provider class name, user can provide their own implementation of LockProvider which should be subclass of org.apache.hudi.common.lock.LockProvider",(0,r.jsx)("br",{}),(0,r.jsx)("br",{}),(0,r.jsx)(i.code,{children:"Config Param: LOCK_PROVIDER_CLASS_NAME"}),(0,r.jsx)("br",{}),(0,r.jsx)(i.code,{children:"Since Version: 0.8.0"})]})]}),(0,r.jsxs)(i.tr,{children:[(0,r.jsx)(i.td,{children:"hoodie.cleaner.policy.failed.writes"}),(0,r.jsx)(i.td,{children:"EAGER (Optional)"}),(0,r.jsxs)(i.td,{children:["org.apache.hudi.common.model.HoodieFailedWritesCleaningPolicy: Policy that controls how to clean up failed writes. Hudi will delete any files written by failed writes to re-claim space. EAGER(default): Clean failed writes inline after every write operation. LAZY: Clean failed writes lazily after heartbeat timeout when the cleaning service runs. This policy is required when multi-writers are enabled. NEVER: Never clean failed writes.",(0,r.jsx)("br",{}),(0,r.jsx)("br",{}),(0,r.jsx)(i.code,{children:"Config Param: FAILED_WRITES_CLEANER_POLICY"})]})]})]})]}),"\n",(0,r.jsx)(i.h3,{id:"multi-writing-via-hudi-streamer",children:"Multi Writing via Hudi Streamer"}),"\n",(0,r.jsxs)(i.p,{children:["The ",(0,r.jsx)(i.code,{children:"HoodieStreamer"})," utility (part of hudi-utilities-slim-bundle) provides ways to ingest from different sources such as DFS or Kafka, with the following capabilities."]}),"\n",(0,r.jsx)(i.p,{children:"Using optimistic_concurrency_control via Hudi Streamer requires adding the above configs to the properties file that can be passed to the\njob. For example below, adding the configs to kafka-source.properties file and passing them to Hudi Streamer will enable optimistic concurrency.\nA Hudi Streamer job can then be triggered as follows:"}),"\n",(0,r.jsx)(i.pre,{children:(0,r.jsx)(i.code,{className:"language-java",children:'[hoodie]$ spark-submit \\\n --jars "packaging/hudi-utilities-slim-bundle/target/hudi-utilities-slim-bundle_2.12-1.0.0.jar,packaging/hudi-spark-bundle/target/hudi-spark3.5-bundle_2.12-1.0.0.jar" \\\n --class org.apache.hudi.utilities.streamer.HoodieStreamer `ls packaging/hudi-utilities-slim-bundle/target/hudi-utilities-slim-bundle-*.jar` \\\n --props file://${PWD}/hudi-utilities/src/test/resources/streamer-config/kafka-source.properties \\\n --schemaprovider-class org.apache.hudi.utilities.schema.SchemaRegistryProvider \\\n --source-class org.apache.hudi.utilities.sources.AvroKafkaSource \\\n --source-ordering-field impresssiontime \\\n --target-base-path file:\\/\\/\\/tmp/hudi-streamer-op \\ \n --target-table tableName \\\n --op BULK_INSERT\n'})}),"\n",(0,r.jsx)(i.h3,{id:"multi-writing-via-spark-datasource-writer",children:"Multi Writing via Spark Datasource Writer"}),"\n",(0,r.jsxs)(i.p,{children:["The ",(0,r.jsx)(i.code,{children:"hudi-spark"})," module offers the DataSource API to write (and read) a Spark DataFrame into a Hudi table."]}),"\n",(0,r.jsx)(i.p,{children:"Following is an example of how to use optimistic_concurrency_control via spark datasource"}),"\n",(0,r.jsx)(i.pre,{children:(0,r.jsx)(i.code,{className:"language-java",children:'inputDF.write.format("hudi")\n .options(getQuickstartWriteConfigs)\n .option("hoodie.datasource.write.precombine.field", "ts")\n .option("hoodie.cleaner.policy.failed.writes", "LAZY")\n .option("hoodie.write.concurrency.mode", "optimistic_concurrency_control")\n .option("hoodie.write.lock.zookeeper.url", "zookeeper")\n .option("hoodie.write.lock.zookeeper.port", "2181")\n .option("hoodie.write.lock.zookeeper.base_path", "/test")\n .option("hoodie.datasource.write.recordkey.field", "uuid")\n .option("hoodie.datasource.write.partitionpath.field", "partitionpath")\n .option("hoodie.table.name", tableName)\n .mode(Overwrite)\n .save(basePath)\n'})}),"\n",(0,r.jsx)(i.h2,{id:"disabling-multi-writing",children:"Disabling Multi Writing"}),"\n",(0,r.jsx)(i.p,{children:"Remove the following settings that were used to enable multi-writer or override with default values."}),"\n",(0,r.jsx)(i.pre,{children:(0,r.jsx)(i.code,{children:"hoodie.write.concurrency.mode=single_writer\nhoodie.cleaner.policy.failed.writes=EAGER\n"})}),"\n",(0,r.jsx)(i.h2,{id:"occ-best-practices",children:"OCC Best Practices"}),"\n",(0,r.jsx)(i.p,{children:"Concurrent Writing to Hudi tables requires acquiring a lock with one of the lock providers mentioned above. Due to several reasons you might want to configure retries to allow your application to acquire the lock."}),"\n",(0,r.jsxs)(i.ol,{children:["\n",(0,r.jsx)(i.li,{children:"Network connectivity or excessive load on servers increasing time for lock acquisition resulting in timeouts"}),"\n",(0,r.jsx)(i.li,{children:"Running a large number of concurrent jobs that are writing to the same hudi table can result in contention during lock acquisition can cause timeouts"}),"\n",(0,r.jsx)(i.li,{children:"In some scenarios of conflict resolution, Hudi commit operations might take upto 10's of seconds while the lock is being held. This can result in timeouts for other jobs waiting to acquire a lock."}),"\n"]}),"\n",(0,r.jsx)(i.p,{children:"Set the correct native lock provider client retries."}),"\n",(0,r.jsxs)(i.admonition,{type:"note",children:[(0,r.jsx)(i.mdxAdmonitionTitle,{}),(0,r.jsx)(i.p,{children:"Please note that sometimes these settings are set on the server once and all clients inherit the same configs. Please check your settings before enabling optimistic concurrency."})]}),"\n",(0,r.jsx)(i.pre,{children:(0,r.jsx)(i.code,{children:"hoodie.write.lock.wait_time_ms\nhoodie.write.lock.num_retries\n"})}),"\n",(0,r.jsx)(i.p,{children:"Set the correct hudi client retries for Zookeeper & HiveMetastore. This is useful in cases when native client retry settings cannot be changed. Please note that these retries will happen in addition to any native client retries that you may have set."}),"\n",(0,r.jsx)(i.pre,{children:(0,r.jsx)(i.code,{children:"hoodie.write.lock.client.wait_time_ms\nhoodie.write.lock.client.num_retries\n"})}),"\n",(0,r.jsx)(i.p,{children:(0,r.jsx)(i.em,{children:"Setting the right values for these depends on a case by case basis; some defaults have been provided for general cases."})}),"\n",(0,r.jsx)(i.h2,{id:"caveats",children:"Caveats"}),"\n",(0,r.jsxs)(i.p,{children:["If you are using the ",(0,r.jsx)(i.code,{children:"WriteClient"})," API, please note that multiple writes to the table need to be initiated from 2 different instances of the write client.\nIt is ",(0,r.jsx)(i.strong,{children:"NOT"})," recommended to use the same instance of the write client to perform multi writing."]}),"\n",(0,r.jsx)(i.h2,{id:"related-resources",children:"Related Resources"}),"\n",(0,r.jsx)("h3",{children:"Blogs"}),"\n",(0,r.jsxs)(i.ul,{children:["\n",(0,r.jsx)(i.li,{children:(0,r.jsx)(i.a,{href:"https://www.onehouse.ai/blog/lakehouse-concurrency-control-are-we-too-optimistic",children:"Data Lakehouse Concurrency Control"})}),"\n",(0,r.jsx)(i.li,{children:(0,r.jsx)(i.a,{href:"https://medium.com/@simpsons/multi-writer-support-with-apache-hudi-e1b75dca29e6",children:"Multi-writer support with Apache Hudi"})}),"\n"]}),"\n",(0,r.jsx)("h3",{children:"Videos"}),"\n",(0,r.jsxs)(i.ul,{children:["\n",(0,r.jsx)(i.li,{children:(0,r.jsx)(i.a,{href:"https://youtu.be/JP0orl9_0yQ",children:"Hands on Lab with using DynamoDB as lock table for Apache Hudi Data Lakes"})}),"\n",(0,r.jsx)(i.li,{children:(0,r.jsx)(i.a,{href:"/blog/2024/12/06/non-blocking-concurrency-control",children:"Non Blocking Concurrency Control Flink Demo"})}),"\n"]})]})}function h(e={}){const{wrapper:i}={...(0,o.R)(),...e.components};return i?(0,r.jsx)(i,{...e,children:(0,r.jsx)(d,{...e})}):d(e)}},28453:(e,i,t)=>{t.d(i,{R:()=>s,x:()=>a});var n=t(96540);const r={},o=n.createContext(r);function s(e){const i=n.useContext(o);return n.useMemo((function(){return"function"==typeof e?e(i):{...i,...e}}),[i,e])}function a(e){let i;return i=e.disableParentContext?"function"==typeof e.components?e.components(r):e.components||r:s(e.components),n.createElement(o.Provider,{value:i},e.children)}}}]);
\ No newline at end of file
diff --git a/content/assets/js/ed47441b.65b131bc.js b/content/assets/js/ed47441b.65b131bc.js
new file mode 100644
index 0000000000000..575bf16644668
--- /dev/null
+++ b/content/assets/js/ed47441b.65b131bc.js
@@ -0,0 +1 @@
+"use strict";(self.webpackChunkhudi=self.webpackChunkhudi||[]).push([[71590],{29697:(e,n,t)=>{t.r(n),t.d(n,{assets:()=>d,contentTitle:()=>r,default:()=>u,frontMatter:()=>o,metadata:()=>s,toc:()=>c});const s=JSON.parse('{"id":"azure_hoodie","title":"Microsoft Azure","description":"In this page, we explain how to use Hudi on Microsoft Azure.","source":"@site/docs/azure_hoodie.md","sourceDirName":".","slug":"/azure_hoodie","permalink":"/docs/next/azure_hoodie","draft":false,"unlisted":false,"editUrl":"https://github.com/apache/hudi/tree/asf-site/website/docs/azure_hoodie.md","tags":[],"version":"current","frontMatter":{"title":"Microsoft Azure","keywords":["hudi","hive","azure","spark","presto"],"summary":"In this page, we go over how to configure Hudi with Azure filesystem.","last_modified_at":"2020-05-25T23:00:57.000Z"},"sidebar":"docs","previous":{"title":"Alibaba Cloud","permalink":"/docs/next/oss_hoodie"},"next":{"title":"Tencent Cloud","permalink":"/docs/next/cos_hoodie"}}');var a=t(74848),i=t(28453);const o={title:"Microsoft Azure",keywords:["hudi","hive","azure","spark","presto"],summary:"In this page, we go over how to configure Hudi with Azure filesystem.",last_modified_at:new Date("2020-05-25T23:00:57.000Z")},r=void 0,d={},c=[{value:"Disclaimer",id:"disclaimer",level:2},{value:"Supported Storage System",id:"supported-storage-system",level:2},{value:"Verified Combination of Spark and storage system",id:"verified-combination-of-spark-and-storage-system",level:2},{value:"HDInsight Spark2.4 on Azure Data Lake Storage Gen 2",id:"hdinsight-spark24-on-azure-data-lake-storage-gen-2",level:4},{value:"Databricks Spark2.4 on Azure Data Lake Storage Gen 2",id:"databricks-spark24-on-azure-data-lake-storage-gen-2",level:4},{value:"Related Resources",id:"related-resources",level:2}];function l(e){const n={a:"a",code:"code",h2:"h2",h4:"h4",li:"li",p:"p",pre:"pre",ul:"ul",...(0,i.R)(),...e.components};return(0,a.jsxs)(a.Fragment,{children:[(0,a.jsx)(n.p,{children:"In this page, we explain how to use Hudi on Microsoft Azure."}),"\n",(0,a.jsx)(n.h2,{id:"disclaimer",children:"Disclaimer"}),"\n",(0,a.jsx)(n.p,{children:"This page is maintained by the Hudi community.\nIf the information is inaccurate or you have additional information to add.\nPlease feel free to create a JIRA ticket. Contribution is highly appreciated."}),"\n",(0,a.jsx)(n.h2,{id:"supported-storage-system",children:"Supported Storage System"}),"\n",(0,a.jsx)(n.p,{children:"There are two storage systems support Hudi ."}),"\n",(0,a.jsxs)(n.ul,{children:["\n",(0,a.jsx)(n.li,{children:"Azure Blob Storage"}),"\n",(0,a.jsx)(n.li,{children:"Azure Data Lake Gen 2"}),"\n"]}),"\n",(0,a.jsx)(n.h2,{id:"verified-combination-of-spark-and-storage-system",children:"Verified Combination of Spark and storage system"}),"\n",(0,a.jsx)(n.h4,{id:"hdinsight-spark24-on-azure-data-lake-storage-gen-2",children:"HDInsight Spark2.4 on Azure Data Lake Storage Gen 2"}),"\n",(0,a.jsx)(n.p,{children:"This combination works out of the box. No extra config needed."}),"\n",(0,a.jsx)(n.h4,{id:"databricks-spark24-on-azure-data-lake-storage-gen-2",children:"Databricks Spark2.4 on Azure Data Lake Storage Gen 2"}),"\n",(0,a.jsxs)(n.ul,{children:["\n",(0,a.jsxs)(n.li,{children:["\n",(0,a.jsx)(n.p,{children:"Import Hudi jar to databricks workspace"}),"\n"]}),"\n",(0,a.jsxs)(n.li,{children:["\n",(0,a.jsx)(n.p,{children:"Mount the file system to dbutils."}),"\n",(0,a.jsx)(n.pre,{children:(0,a.jsx)(n.code,{className:"language-scala",children:'dbutils.fs.mount(\n source = "abfss://xxx@xxx.dfs.core.windows.net",\n mountPoint = "/mountpoint",\n extraConfigs = configs)\n'})}),"\n"]}),"\n",(0,a.jsxs)(n.li,{children:["\n",(0,a.jsx)(n.p,{children:"When writing Hudi dataset, use abfss URL"}),"\n",(0,a.jsx)(n.pre,{children:(0,a.jsx)(n.code,{className:"language-scala",children:'inputDF.write\n .format("org.apache.hudi")\n .options(opts)\n .mode(SaveMode.Append)\n .save("abfss://<>.dfs.core.windows.net/hudi-tables/customer")\n'})}),"\n"]}),"\n",(0,a.jsxs)(n.li,{children:["\n",(0,a.jsx)(n.p,{children:"When reading Hudi dataset, use the mounting point"}),"\n",(0,a.jsx)(n.pre,{children:(0,a.jsx)(n.code,{className:"language-scala",children:'spark.read\n .format("org.apache.hudi")\n .load("/mountpoint/hudi-tables/customer")\n'})}),"\n"]}),"\n"]}),"\n",(0,a.jsx)(n.h2,{id:"related-resources",children:"Related Resources"}),"\n",(0,a.jsx)("h3",{children:"Blogs"}),"\n",(0,a.jsxs)(n.ul,{children:["\n",(0,a.jsx)(n.li,{children:(0,a.jsx)(n.a,{href:"https://www.onehouse.ai/blog/how-to-use-apache-hudi-with-databricks",children:"How to use Apache Hudi with Databricks"})}),"\n"]})]})}function u(e={}){const{wrapper:n}={...(0,i.R)(),...e.components};return n?(0,a.jsx)(n,{...e,children:(0,a.jsx)(l,{...e})}):l(e)}},28453:(e,n,t)=>{t.d(n,{R:()=>o,x:()=>r});var s=t(96540);const a={},i=s.createContext(a);function o(e){const n=s.useContext(i);return s.useMemo((function(){return"function"==typeof e?e(n):{...n,...e}}),[n,e])}function r(e){let n;return n=e.disableParentContext?"function"==typeof e.components?e.components(a):e.components||a:o(e.components),s.createElement(i.Provider,{value:n},e.children)}}}]);
\ No newline at end of file
diff --git a/content/assets/js/ed47441b.c9f8c1ee.js b/content/assets/js/ed47441b.c9f8c1ee.js
deleted file mode 100644
index fe1bee36f6371..0000000000000
--- a/content/assets/js/ed47441b.c9f8c1ee.js
+++ /dev/null
@@ -1 +0,0 @@
-"use strict";(self.webpackChunkhudi=self.webpackChunkhudi||[]).push([[71590],{29697:(e,n,t)=>{t.r(n),t.d(n,{assets:()=>d,contentTitle:()=>r,default:()=>u,frontMatter:()=>o,metadata:()=>s,toc:()=>c});const s=JSON.parse('{"id":"azure_hoodie","title":"Microsoft Azure","description":"In this page, we explain how to use Hudi on Microsoft Azure.","source":"@site/docs/azure_hoodie.md","sourceDirName":".","slug":"/azure_hoodie","permalink":"/docs/next/azure_hoodie","draft":false,"unlisted":false,"editUrl":"https://github.com/apache/hudi/tree/asf-site/website/docs/azure_hoodie.md","tags":[],"version":"current","frontMatter":{"title":"Microsoft Azure","keywords":["hudi","hive","azure","spark","presto"],"summary":"In this page, we go over how to configure Hudi with Azure filesystem.","last_modified_at":"2020-05-25T23:00:57.000Z"},"sidebar":"docs","previous":{"title":"Alibaba Cloud","permalink":"/docs/next/oss_hoodie"},"next":{"title":"Tencent Cloud","permalink":"/docs/next/cos_hoodie"}}');var a=t(74848),i=t(28453);const o={title:"Microsoft Azure",keywords:["hudi","hive","azure","spark","presto"],summary:"In this page, we go over how to configure Hudi with Azure filesystem.",last_modified_at:new Date("2020-05-25T23:00:57.000Z")},r=void 0,d={},c=[{value:"Disclaimer",id:"disclaimer",level:2},{value:"Supported Storage System",id:"supported-storage-system",level:2},{value:"Verified Combination of Spark and storage system",id:"verified-combination-of-spark-and-storage-system",level:2},{value:"HDInsight Spark2.4 on Azure Data Lake Storage Gen 2",id:"hdinsight-spark24-on-azure-data-lake-storage-gen-2",level:4},{value:"Databricks Spark2.4 on Azure Data Lake Storage Gen 2",id:"databricks-spark24-on-azure-data-lake-storage-gen-2",level:4}];function l(e){const n={code:"code",h2:"h2",h4:"h4",li:"li",p:"p",pre:"pre",ul:"ul",...(0,i.R)(),...e.components};return(0,a.jsxs)(a.Fragment,{children:[(0,a.jsx)(n.p,{children:"In this page, we explain how to use Hudi on Microsoft Azure."}),"\n",(0,a.jsx)(n.h2,{id:"disclaimer",children:"Disclaimer"}),"\n",(0,a.jsx)(n.p,{children:"This page is maintained by the Hudi community.\nIf the information is inaccurate or you have additional information to add.\nPlease feel free to create a JIRA ticket. Contribution is highly appreciated."}),"\n",(0,a.jsx)(n.h2,{id:"supported-storage-system",children:"Supported Storage System"}),"\n",(0,a.jsx)(n.p,{children:"There are two storage systems support Hudi ."}),"\n",(0,a.jsxs)(n.ul,{children:["\n",(0,a.jsx)(n.li,{children:"Azure Blob Storage"}),"\n",(0,a.jsx)(n.li,{children:"Azure Data Lake Gen 2"}),"\n"]}),"\n",(0,a.jsx)(n.h2,{id:"verified-combination-of-spark-and-storage-system",children:"Verified Combination of Spark and storage system"}),"\n",(0,a.jsx)(n.h4,{id:"hdinsight-spark24-on-azure-data-lake-storage-gen-2",children:"HDInsight Spark2.4 on Azure Data Lake Storage Gen 2"}),"\n",(0,a.jsx)(n.p,{children:"This combination works out of the box. No extra config needed."}),"\n",(0,a.jsx)(n.h4,{id:"databricks-spark24-on-azure-data-lake-storage-gen-2",children:"Databricks Spark2.4 on Azure Data Lake Storage Gen 2"}),"\n",(0,a.jsxs)(n.ul,{children:["\n",(0,a.jsxs)(n.li,{children:["\n",(0,a.jsx)(n.p,{children:"Import Hudi jar to databricks workspace"}),"\n"]}),"\n",(0,a.jsxs)(n.li,{children:["\n",(0,a.jsx)(n.p,{children:"Mount the file system to dbutils."}),"\n",(0,a.jsx)(n.pre,{children:(0,a.jsx)(n.code,{className:"language-scala",children:'dbutils.fs.mount(\n source = "abfss://xxx@xxx.dfs.core.windows.net",\n mountPoint = "/mountpoint",\n extraConfigs = configs)\n'})}),"\n"]}),"\n",(0,a.jsxs)(n.li,{children:["\n",(0,a.jsx)(n.p,{children:"When writing Hudi dataset, use abfss URL"}),"\n",(0,a.jsx)(n.pre,{children:(0,a.jsx)(n.code,{className:"language-scala",children:'inputDF.write\n .format("org.apache.hudi")\n .options(opts)\n .mode(SaveMode.Append)\n .save("abfss://<>.dfs.core.windows.net/hudi-tables/customer")\n'})}),"\n"]}),"\n",(0,a.jsxs)(n.li,{children:["\n",(0,a.jsx)(n.p,{children:"When reading Hudi dataset, use the mounting point"}),"\n",(0,a.jsx)(n.pre,{children:(0,a.jsx)(n.code,{className:"language-scala",children:'spark.read\n .format("org.apache.hudi")\n .load("/mountpoint/hudi-tables/customer")\n'})}),"\n"]}),"\n"]})]})}function u(e={}){const{wrapper:n}={...(0,i.R)(),...e.components};return n?(0,a.jsx)(n,{...e,children:(0,a.jsx)(l,{...e})}):l(e)}},28453:(e,n,t)=>{t.d(n,{R:()=>o,x:()=>r});var s=t(96540);const a={},i=s.createContext(a);function o(e){const n=s.useContext(i);return s.useMemo((function(){return"function"==typeof e?e(n):{...n,...e}}),[n,e])}function r(e){let n;return n=e.disableParentContext?"function"==typeof e.components?e.components(a):e.components||a:o(e.components),s.createElement(i.Provider,{value:n},e.children)}}}]);
\ No newline at end of file
diff --git a/content/assets/js/edefc60b.660e3035.js b/content/assets/js/edefc60b.660e3035.js
new file mode 100644
index 0000000000000..2e53a96829af7
--- /dev/null
+++ b/content/assets/js/edefc60b.660e3035.js
@@ -0,0 +1 @@
+"use strict";(self.webpackChunkhudi=self.webpackChunkhudi||[]).push([[73624],{96496:(e,n,i)=>{i.r(n),i.d(n,{assets:()=>l,contentTitle:()=>r,default:()=>h,frontMatter:()=>o,metadata:()=>t,toc:()=>d});const t=JSON.parse('{"id":"performance","title":"Performance","description":"Optimized DFS Access","source":"@site/docs/performance.md","sourceDirName":".","slug":"/performance","permalink":"/docs/next/performance","draft":false,"unlisted":false,"editUrl":"https://github.com/apache/hudi/tree/asf-site/website/docs/performance.md","tags":[],"version":"current","frontMatter":{"title":"Performance","keywords":["hudi","index","storage","compaction","cleaning","implementation"],"toc":false,"last_modified_at":"2019-12-30T19:59:57.000Z"},"sidebar":"docs","previous":{"title":"Bootstrapping","permalink":"/docs/next/migration_guide"},"next":{"title":"Deployment","permalink":"/docs/next/deployment"}}');var a=i(74848),s=i(28453);const o={title:"Performance",keywords:["hudi","index","storage","compaction","cleaning","implementation"],toc:!1,last_modified_at:new Date("2019-12-30T19:59:57.000Z")},r=void 0,l={},d=[{value:"Optimized DFS Access",id:"optimized-dfs-access",level:2},{value:"Performance Optimizations",id:"performance-optimizations",level:2},{value:"Write Path",id:"write-path",level:3},{value:"Bulk Insert",id:"bulk-insert",level:4},{value:"Upserts",id:"upserts",level:4},{value:"Indexing",id:"indexing",level:4},{value:"Read Path",id:"read-path",level:3},{value:"Data Skipping",id:"data-skipping",level:4},{value:"Related Resources",id:"related-resources",level:2}];function c(e){const n={a:"a",admonition:"admonition",code:"code",em:"em",h2:"h2",h3:"h3",h4:"h4",li:"li",ol:"ol",p:"p",pre:"pre",strong:"strong",ul:"ul",...(0,s.R)(),...e.components};return(0,a.jsxs)(a.Fragment,{children:[(0,a.jsx)(n.h2,{id:"optimized-dfs-access",children:"Optimized DFS Access"}),"\n",(0,a.jsx)(n.p,{children:"Hudi also performs several key storage management functions on the data stored in a Hudi table. A key aspect of storing data on DFS is managing file sizes and counts\nand reclaiming storage space. For e.g HDFS is infamous for its handling of small files, which exerts memory/RPC pressure on the Name Node and can potentially destabilize\nthe entire cluster. In general, query engines provide much better performance on adequately sized columnar files, since they can effectively amortize cost of obtaining\ncolumn statistics etc. Even on some cloud data stores, there is often cost to listing directories with large number of small files."}),"\n",(0,a.jsx)(n.p,{children:"Here are some ways to efficiently manage the storage of your Hudi tables."}),"\n",(0,a.jsxs)(n.ul,{children:["\n",(0,a.jsxs)(n.li,{children:["The ",(0,a.jsx)(n.a,{href:"/docs/configurations/#hoodieparquetsmallfilelimit",children:"small file handling feature"})," in Hudi, profiles incoming workload\nand distributes inserts to existing file groups instead of creating new file groups, which can lead to small files."]}),"\n",(0,a.jsxs)(n.li,{children:["Cleaner can be ",(0,a.jsx)(n.a,{href:"/docs/configurations#hoodiecleanercommitsretained",children:"configured"})," to clean up older file slices, more or less aggressively depending on maximum time for queries to run & lookback needed for incremental pull"]}),"\n",(0,a.jsxs)(n.li,{children:["User can also tune the size of the ",(0,a.jsx)(n.a,{href:"/docs/configurations#hoodieparquetmaxfilesize",children:"base/parquet file"}),", ",(0,a.jsx)(n.a,{href:"/docs/configurations#hoodielogfilemaxsize",children:"log files"})," & expected ",(0,a.jsx)(n.a,{href:"/docs/configurations#hoodieparquetcompressionratio",children:"compression ratio"}),",\nsuch that sufficient number of inserts are grouped into the same file group, resulting in well sized base files ultimately."]}),"\n",(0,a.jsxs)(n.li,{children:["Intelligently tuning the ",(0,a.jsx)(n.a,{href:"/docs/configurations#hoodiebulkinsertshuffleparallelism",children:"bulk insert parallelism"}),", can again in nicely sized initial file groups. It is in fact critical to get this right, since the file groups\nonce created cannot be changed without re-clustering the table. Writes will simply expand given file groups with new updates/inserts as explained before."]}),"\n",(0,a.jsxs)(n.li,{children:["For workloads with heavy updates, the ",(0,a.jsx)(n.a,{href:"/docs/concepts#merge-on-read-table",children:"merge-on-read table"})," provides a nice mechanism for ingesting quickly into smaller files and then later merging them into larger base files via compaction."]}),"\n"]}),"\n",(0,a.jsx)(n.h2,{id:"performance-optimizations",children:"Performance Optimizations"}),"\n",(0,a.jsx)(n.p,{children:"In this section, we go over some real world performance numbers for Hudi upserts, incremental pull and compare them against\nthe conventional alternatives for achieving these tasks."}),"\n",(0,a.jsx)(n.h3,{id:"write-path",children:"Write Path"}),"\n",(0,a.jsx)(n.h4,{id:"bulk-insert",children:"Bulk Insert"}),"\n",(0,a.jsx)(n.p,{children:"Write configurations in Hudi are optimized for incremental upserts by default. In fact, the default write operation type is UPSERT as well.\nFor simple append-only use case to bulk load the data, following set of configurations are recommended for optimal writing:"}),"\n",(0,a.jsx)(n.pre,{children:(0,a.jsx)(n.code,{children:"-- Use \u201cbulk-insert\u201d write-operation instead of default \u201cupsert\u201d\nhoodie.datasource.write.operation = BULK_INSERT\n-- Disable populating meta columns and metadata, and enable virtual keys\nhoodie.populate.meta.fields = false\nhoodie.metadata.enable = false\n-- Enable snappy compression codec for lesser CPU cycles (but more storage overhead)\nhoodie.parquet.compression.codec = snappy\n"})}),"\n",(0,a.jsx)(n.p,{children:"For ingesting via spark-sql"}),"\n",(0,a.jsx)(n.pre,{children:(0,a.jsx)(n.code,{children:"-- Use \u201cbulk-insert\u201d write-operation instead of default \u201cupsert\u201d\nhoodie.sql.insert.mode = non-strict,\nhoodie.sql.bulk.insert.enable = true,\n-- Disable populating meta columns and metadata, and enable virtual keys\nhoodie.populate.meta.fields = false\nhoodie.metadata.enable = false\n-- Enable snappy compression codec for lesser CPU cycles (but more storage overhead)\nhoodie.parquet.compression.codec = snappy\n"})}),"\n",(0,a.jsxs)(n.p,{children:["We recently benchmarked Hudi against TPC-DS workload.\nPlease check out ",(0,a.jsx)(n.a,{href:"/blog/2022/06/29/Apache-Hudi-vs-Delta-Lake-transparent-tpc-ds-lakehouse-performance-benchmarks",children:"our blog"})," for more details."]}),"\n",(0,a.jsx)(n.h4,{id:"upserts",children:"Upserts"}),"\n",(0,a.jsx)(n.p,{children:"Following shows the speed up obtained for NoSQL database ingestion, from incrementally upserting on a Hudi table on the copy-on-write storage,\non 5 tables ranging from small to huge (as opposed to bulk loading the tables)"}),"\n",(0,a.jsx)("figure",{children:(0,a.jsx)("img",{className:"docimage",src:i(63490).A,alt:"hudi_upsert_perf1.png"})}),"\n",(0,a.jsx)(n.p,{children:"Given Hudi can build the table incrementally, it opens doors for also scheduling ingesting more frequently thus reducing latency, with\nsignificant savings on the overall compute cost."}),"\n",(0,a.jsx)("figure",{children:(0,a.jsx)("img",{className:"docimage",src:i(19577).A,alt:"hudi_upsert_perf2.png"})}),"\n",(0,a.jsxs)(n.p,{children:["Hudi upserts have been stress tested upto 4TB in a single commit across the t1 table.\nSee ",(0,a.jsx)(n.a,{href:"https://cwiki.apache.org/confluence/display/HUDI/Tuning+Guide",children:"here"})," for some tuning tips."]}),"\n",(0,a.jsx)(n.h4,{id:"indexing",children:"Indexing"}),"\n",(0,a.jsx)(n.p,{children:"In order to efficiently upsert data, Hudi needs to classify records in a write batch into inserts & updates (tagged with the file group\nit belongs to). In order to speed this operation, Hudi employs a pluggable index mechanism that stores a mapping between recordKey and\nthe file group id it belongs to. By default, Hudi uses a built in index that uses file ranges and bloom filters to accomplish this, with\nupto 10x speed up over a spark join to do the same."}),"\n",(0,a.jsxs)(n.p,{children:["Hudi provides best indexing performance when you model the recordKey to be monotonically increasing (e.g timestamp prefix), leading to range pruning filtering\nout a lot of files for comparison. Even for UUID based keys, there are ",(0,a.jsx)(n.a,{href:"https://www.percona.com/blog/2014/12/19/store-uuid-optimized-way/",children:"known techniques"})," to achieve this.\nFor e.g , with 100M timestamp prefixed keys (5% updates, 95% inserts) on a event table with 80B keys/3 partitions/11416 files/10TB data, Hudi index achieves a\n",(0,a.jsx)(n.strong,{children:"~7X (2880 secs vs 440 secs) speed up"})," over vanilla spark join. Even for a challenging workload like an '100% update' database ingestion workload spanning\n3.25B UUID keys/30 partitions/6180 files using 300 cores, Hudi indexing offers a ",(0,a.jsx)(n.strong,{children:"80-100% speedup"}),"."]}),"\n",(0,a.jsx)(n.h3,{id:"read-path",children:"Read Path"}),"\n",(0,a.jsx)(n.h4,{id:"data-skipping",children:"Data Skipping"}),"\n",(0,a.jsx)(n.p,{children:"Data Skipping is a technique (originally introduced in Hudi 0.10) that leverages metadata to very effectively prune the search space of a query,\nby eliminating files that cannot possibly contain data matching the query's filters. By maintaining this metadata in the internal Hudi metadata table,\nHudi avoids reading file footers to obtain this information, which can be costly for queries spanning tens of thousands of files."}),"\n",(0,a.jsxs)(n.p,{children:["Data Skipping leverages metadata table's ",(0,a.jsx)(n.code,{children:"col_stats"})," partition bearing column-level statistics (such as min-value, max-value, count of null-values in the column, etc)\nfor every file of the Hudi table. This then allows Hudi for every incoming query instead of enumerating every file in the table and reading its corresponding metadata\n(for ex, Parquet footers) for analysis whether it could contain any data matching the query filters, to simply do a query against a Column Stats Index\nin the Metadata Table (which in turn is a Hudi table itself) and within seconds (even for TBs scale tables, with 10s of thousands of files) obtain the list\nof ",(0,a.jsx)(n.em,{children:"all the files that might potentially contain the data"})," matching query's filters with crucial property that files that could be ruled out as not containing such data\n(based on their column-level statistics) will be stripped out. See ",(0,a.jsx)(n.a,{href:"https://github.com/apache/hudi/blob/master/rfc/rfc-27/rfc-27.md",children:"RFC-27"})," for detailed design."]}),"\n",(0,a.jsx)(n.p,{children:"Partitioning can be considered a coarse form of indexing and data skipping using the col_stats partition can be thought of as a range index, that databases use to identify potential\nblocks of data interesting to a query. Unlike partition pruning for tables using physical partitioning where records in the dataset are organized into a folder structure based\non some column's value, data skipping using col_stats delivers a logical/virtual partitioning."}),"\n",(0,a.jsx)(n.p,{children:"For very large tables (1Tb+, 10s of 1000s of files), Data skipping could"}),"\n",(0,a.jsxs)(n.ol,{children:["\n",(0,a.jsxs)(n.li,{children:["Substantially improve query execution runtime ",(0,a.jsx)(n.strong,{children:"10x"})," as compared to the same query on the same dataset but w/o Data Skipping enabled."]}),"\n",(0,a.jsx)(n.li,{children:"Help avoid hitting Cloud Storages throttling limits (for issuing too many requests, for ex, AWS limits # of requests / sec that could be issued based on the object's prefix which considerably complicates things for partitioned tables)"}),"\n"]}),"\n",(0,a.jsx)(n.p,{children:"To unlock the power of Data Skipping you will need to"}),"\n",(0,a.jsxs)(n.ol,{children:["\n",(0,a.jsxs)(n.li,{children:["Enable Metadata Table along with Column Stats Index on the ",(0,a.jsx)(n.em,{children:"write path"})," (See ",(0,a.jsx)(n.a,{href:"/docs/metadata_indexing",children:"Metadata Indexing"}),"), using ",(0,a.jsx)(n.code,{children:"hoodie.metadata.enable=true"})," (to enable Metadata Table on the write path, enabled by default)"]}),"\n",(0,a.jsxs)(n.li,{children:["Enable Data Skipping in your queries, using ",(0,a.jsx)(n.code,{children:"hoodie.metadata.index.column.stats.enable=true"})," (to enable Column Stats Index being populated on the write path, disabled by default)"]}),"\n"]}),"\n",(0,a.jsx)(n.admonition,{type:"note",children:(0,a.jsxs)(n.p,{children:["If you're planning on enabling Column Stats Index for already existing table, please check out the ",(0,a.jsx)(n.a,{href:"/docs/metadata_indexing",children:"Metadata Indexing"})," guide on how to build Metadata Table Indices (such as Column Stats Index) for existing tables."]})}),"\n",(0,a.jsxs)(n.p,{children:["To enable Data Skipping in your queries make sure to set following properties to ",(0,a.jsx)(n.code,{children:"true"})," (on the read path):"]}),"\n",(0,a.jsxs)(n.ul,{children:["\n",(0,a.jsxs)(n.li,{children:[(0,a.jsx)(n.code,{children:"hoodie.enable.data.skipping"})," (to control data skipping, enabled by default)"]}),"\n",(0,a.jsxs)(n.li,{children:[(0,a.jsx)(n.code,{children:"hoodie.metadata.enable"})," (to enable metadata table use on the read path, enabled by default)"]}),"\n",(0,a.jsxs)(n.li,{children:[(0,a.jsx)(n.code,{children:"hoodie.metadata.index.column.stats.enable"})," (to enable column stats index use on the read path)"]}),"\n"]}),"\n",(0,a.jsx)(n.h2,{id:"related-resources",children:"Related Resources"}),"\n",(0,a.jsx)("h3",{children:"Blogs"}),"\n",(0,a.jsxs)(n.ul,{children:["\n",(0,a.jsx)(n.li,{children:(0,a.jsx)(n.a,{href:"https://www.onehouse.ai/blog/hudis-column-stats-index-and-data-skipping-feature-help-speed-up-queries-by-an-orders-of-magnitude",children:"Hudi\u2019s Column Stats Index and Data Skipping feature help speed up queries by an orders of magnitude!"})}),"\n",(0,a.jsx)(n.li,{children:(0,a.jsx)(n.a,{href:"https://www.onehouse.ai/blog/top-3-things-you-can-do-to-get-fast-upsert-performance-in-apache-hudi",children:"Top 3 Things You Can Do to Get Fast Upsert Performance in Apache Hudi"})}),"\n"]})]})}function h(e={}){const{wrapper:n}={...(0,s.R)(),...e.components};return n?(0,a.jsx)(n,{...e,children:(0,a.jsx)(c,{...e})}):c(e)}},63490:(e,n,i)=>{i.d(n,{A:()=>t});const t=i.p+"assets/images/hudi_upsert_perf1-8f41921dacb5fb026f1e5457f8c47aa6.png"},19577:(e,n,i)=>{i.d(n,{A:()=>t});const t=i.p+"assets/images/hudi_upsert_perf2-d4bfcab4e9e8d942a02b712797ee2755.png"},28453:(e,n,i)=>{i.d(n,{R:()=>o,x:()=>r});var t=i(96540);const a={},s=t.createContext(a);function o(e){const n=t.useContext(s);return t.useMemo((function(){return"function"==typeof e?e(n):{...n,...e}}),[n,e])}function r(e){let n;return n=e.disableParentContext?"function"==typeof e.components?e.components(a):e.components||a:o(e.components),t.createElement(s.Provider,{value:n},e.children)}}}]);
\ No newline at end of file
diff --git a/content/assets/js/edefc60b.7161ba9d.js b/content/assets/js/edefc60b.7161ba9d.js
deleted file mode 100644
index 5a45151b5031c..0000000000000
--- a/content/assets/js/edefc60b.7161ba9d.js
+++ /dev/null
@@ -1 +0,0 @@
-"use strict";(self.webpackChunkhudi=self.webpackChunkhudi||[]).push([[73624],{96496:(e,n,i)=>{i.r(n),i.d(n,{assets:()=>l,contentTitle:()=>r,default:()=>h,frontMatter:()=>o,metadata:()=>t,toc:()=>d});const t=JSON.parse('{"id":"performance","title":"Performance","description":"Optimized DFS Access","source":"@site/docs/performance.md","sourceDirName":".","slug":"/performance","permalink":"/docs/next/performance","draft":false,"unlisted":false,"editUrl":"https://github.com/apache/hudi/tree/asf-site/website/docs/performance.md","tags":[],"version":"current","frontMatter":{"title":"Performance","keywords":["hudi","index","storage","compaction","cleaning","implementation"],"toc":false,"last_modified_at":"2019-12-30T19:59:57.000Z"},"sidebar":"docs","previous":{"title":"Bootstrapping","permalink":"/docs/next/migration_guide"},"next":{"title":"Deployment","permalink":"/docs/next/deployment"}}');var a=i(74848),s=i(28453);const o={title:"Performance",keywords:["hudi","index","storage","compaction","cleaning","implementation"],toc:!1,last_modified_at:new Date("2019-12-30T19:59:57.000Z")},r=void 0,l={},d=[{value:"Optimized DFS Access",id:"optimized-dfs-access",level:2},{value:"Performance Optimizations",id:"performance-optimizations",level:2},{value:"Write Path",id:"write-path",level:3},{value:"Bulk Insert",id:"bulk-insert",level:4},{value:"Upserts",id:"upserts",level:4},{value:"Indexing",id:"indexing",level:4},{value:"Read Path",id:"read-path",level:3},{value:"Data Skipping",id:"data-skipping",level:4}];function c(e){const n={a:"a",admonition:"admonition",code:"code",em:"em",h2:"h2",h3:"h3",h4:"h4",li:"li",ol:"ol",p:"p",pre:"pre",strong:"strong",ul:"ul",...(0,s.R)(),...e.components};return(0,a.jsxs)(a.Fragment,{children:[(0,a.jsx)(n.h2,{id:"optimized-dfs-access",children:"Optimized DFS Access"}),"\n",(0,a.jsx)(n.p,{children:"Hudi also performs several key storage management functions on the data stored in a Hudi table. A key aspect of storing data on DFS is managing file sizes and counts\nand reclaiming storage space. For e.g HDFS is infamous for its handling of small files, which exerts memory/RPC pressure on the Name Node and can potentially destabilize\nthe entire cluster. In general, query engines provide much better performance on adequately sized columnar files, since they can effectively amortize cost of obtaining\ncolumn statistics etc. Even on some cloud data stores, there is often cost to listing directories with large number of small files."}),"\n",(0,a.jsx)(n.p,{children:"Here are some ways to efficiently manage the storage of your Hudi tables."}),"\n",(0,a.jsxs)(n.ul,{children:["\n",(0,a.jsxs)(n.li,{children:["The ",(0,a.jsx)(n.a,{href:"/docs/configurations/#hoodieparquetsmallfilelimit",children:"small file handling feature"})," in Hudi, profiles incoming workload\nand distributes inserts to existing file groups instead of creating new file groups, which can lead to small files."]}),"\n",(0,a.jsxs)(n.li,{children:["Cleaner can be ",(0,a.jsx)(n.a,{href:"/docs/configurations#hoodiecleanercommitsretained",children:"configured"})," to clean up older file slices, more or less aggressively depending on maximum time for queries to run & lookback needed for incremental pull"]}),"\n",(0,a.jsxs)(n.li,{children:["User can also tune the size of the ",(0,a.jsx)(n.a,{href:"/docs/configurations#hoodieparquetmaxfilesize",children:"base/parquet file"}),", ",(0,a.jsx)(n.a,{href:"/docs/configurations#hoodielogfilemaxsize",children:"log files"})," & expected ",(0,a.jsx)(n.a,{href:"/docs/configurations#hoodieparquetcompressionratio",children:"compression ratio"}),",\nsuch that sufficient number of inserts are grouped into the same file group, resulting in well sized base files ultimately."]}),"\n",(0,a.jsxs)(n.li,{children:["Intelligently tuning the ",(0,a.jsx)(n.a,{href:"/docs/configurations#hoodiebulkinsertshuffleparallelism",children:"bulk insert parallelism"}),", can again in nicely sized initial file groups. It is in fact critical to get this right, since the file groups\nonce created cannot be changed without re-clustering the table. Writes will simply expand given file groups with new updates/inserts as explained before."]}),"\n",(0,a.jsxs)(n.li,{children:["For workloads with heavy updates, the ",(0,a.jsx)(n.a,{href:"/docs/concepts#merge-on-read-table",children:"merge-on-read table"})," provides a nice mechanism for ingesting quickly into smaller files and then later merging them into larger base files via compaction."]}),"\n"]}),"\n",(0,a.jsx)(n.h2,{id:"performance-optimizations",children:"Performance Optimizations"}),"\n",(0,a.jsx)(n.p,{children:"In this section, we go over some real world performance numbers for Hudi upserts, incremental pull and compare them against\nthe conventional alternatives for achieving these tasks."}),"\n",(0,a.jsx)(n.h3,{id:"write-path",children:"Write Path"}),"\n",(0,a.jsx)(n.h4,{id:"bulk-insert",children:"Bulk Insert"}),"\n",(0,a.jsx)(n.p,{children:"Write configurations in Hudi are optimized for incremental upserts by default. In fact, the default write operation type is UPSERT as well.\nFor simple append-only use case to bulk load the data, following set of configurations are recommended for optimal writing:"}),"\n",(0,a.jsx)(n.pre,{children:(0,a.jsx)(n.code,{children:"-- Use \u201cbulk-insert\u201d write-operation instead of default \u201cupsert\u201d\nhoodie.datasource.write.operation = BULK_INSERT\n-- Disable populating meta columns and metadata, and enable virtual keys\nhoodie.populate.meta.fields = false\nhoodie.metadata.enable = false\n-- Enable snappy compression codec for lesser CPU cycles (but more storage overhead)\nhoodie.parquet.compression.codec = snappy\n"})}),"\n",(0,a.jsx)(n.p,{children:"For ingesting via spark-sql"}),"\n",(0,a.jsx)(n.pre,{children:(0,a.jsx)(n.code,{children:"-- Use \u201cbulk-insert\u201d write-operation instead of default \u201cupsert\u201d\nhoodie.sql.insert.mode = non-strict,\nhoodie.sql.bulk.insert.enable = true,\n-- Disable populating meta columns and metadata, and enable virtual keys\nhoodie.populate.meta.fields = false\nhoodie.metadata.enable = false\n-- Enable snappy compression codec for lesser CPU cycles (but more storage overhead)\nhoodie.parquet.compression.codec = snappy\n"})}),"\n",(0,a.jsxs)(n.p,{children:["We recently benchmarked Hudi against TPC-DS workload.\nPlease check out ",(0,a.jsx)(n.a,{href:"/blog/2022/06/29/Apache-Hudi-vs-Delta-Lake-transparent-tpc-ds-lakehouse-performance-benchmarks",children:"our blog"})," for more details."]}),"\n",(0,a.jsx)(n.h4,{id:"upserts",children:"Upserts"}),"\n",(0,a.jsx)(n.p,{children:"Following shows the speed up obtained for NoSQL database ingestion, from incrementally upserting on a Hudi table on the copy-on-write storage,\non 5 tables ranging from small to huge (as opposed to bulk loading the tables)"}),"\n",(0,a.jsx)("figure",{children:(0,a.jsx)("img",{className:"docimage",src:i(63490).A,alt:"hudi_upsert_perf1.png"})}),"\n",(0,a.jsx)(n.p,{children:"Given Hudi can build the table incrementally, it opens doors for also scheduling ingesting more frequently thus reducing latency, with\nsignificant savings on the overall compute cost."}),"\n",(0,a.jsx)("figure",{children:(0,a.jsx)("img",{className:"docimage",src:i(19577).A,alt:"hudi_upsert_perf2.png"})}),"\n",(0,a.jsxs)(n.p,{children:["Hudi upserts have been stress tested upto 4TB in a single commit across the t1 table.\nSee ",(0,a.jsx)(n.a,{href:"https://cwiki.apache.org/confluence/display/HUDI/Tuning+Guide",children:"here"})," for some tuning tips."]}),"\n",(0,a.jsx)(n.h4,{id:"indexing",children:"Indexing"}),"\n",(0,a.jsx)(n.p,{children:"In order to efficiently upsert data, Hudi needs to classify records in a write batch into inserts & updates (tagged with the file group\nit belongs to). In order to speed this operation, Hudi employs a pluggable index mechanism that stores a mapping between recordKey and\nthe file group id it belongs to. By default, Hudi uses a built in index that uses file ranges and bloom filters to accomplish this, with\nupto 10x speed up over a spark join to do the same."}),"\n",(0,a.jsxs)(n.p,{children:["Hudi provides best indexing performance when you model the recordKey to be monotonically increasing (e.g timestamp prefix), leading to range pruning filtering\nout a lot of files for comparison. Even for UUID based keys, there are ",(0,a.jsx)(n.a,{href:"https://www.percona.com/blog/2014/12/19/store-uuid-optimized-way/",children:"known techniques"})," to achieve this.\nFor e.g , with 100M timestamp prefixed keys (5% updates, 95% inserts) on a event table with 80B keys/3 partitions/11416 files/10TB data, Hudi index achieves a\n",(0,a.jsx)(n.strong,{children:"~7X (2880 secs vs 440 secs) speed up"})," over vanilla spark join. Even for a challenging workload like an '100% update' database ingestion workload spanning\n3.25B UUID keys/30 partitions/6180 files using 300 cores, Hudi indexing offers a ",(0,a.jsx)(n.strong,{children:"80-100% speedup"}),"."]}),"\n",(0,a.jsx)(n.h3,{id:"read-path",children:"Read Path"}),"\n",(0,a.jsx)(n.h4,{id:"data-skipping",children:"Data Skipping"}),"\n",(0,a.jsx)(n.p,{children:"Data Skipping is a technique (originally introduced in Hudi 0.10) that leverages metadata to very effectively prune the search space of a query,\nby eliminating files that cannot possibly contain data matching the query's filters. By maintaining this metadata in the internal Hudi metadata table,\nHudi avoids reading file footers to obtain this information, which can be costly for queries spanning tens of thousands of files."}),"\n",(0,a.jsxs)(n.p,{children:["Data Skipping leverages metadata table's ",(0,a.jsx)(n.code,{children:"col_stats"})," partition bearing column-level statistics (such as min-value, max-value, count of null-values in the column, etc)\nfor every file of the Hudi table. This then allows Hudi for every incoming query instead of enumerating every file in the table and reading its corresponding metadata\n(for ex, Parquet footers) for analysis whether it could contain any data matching the query filters, to simply do a query against a Column Stats Index\nin the Metadata Table (which in turn is a Hudi table itself) and within seconds (even for TBs scale tables, with 10s of thousands of files) obtain the list\nof ",(0,a.jsx)(n.em,{children:"all the files that might potentially contain the data"})," matching query's filters with crucial property that files that could be ruled out as not containing such data\n(based on their column-level statistics) will be stripped out. See ",(0,a.jsx)(n.a,{href:"https://github.com/apache/hudi/blob/master/rfc/rfc-27/rfc-27.md",children:"RFC-27"})," for detailed design."]}),"\n",(0,a.jsx)(n.p,{children:"Partitioning can be considered a coarse form of indexing and data skipping using the col_stats partition can be thought of as a range index, that databases use to identify potential\nblocks of data interesting to a query. Unlike partition pruning for tables using physical partitioning where records in the dataset are organized into a folder structure based\non some column's value, data skipping using col_stats delivers a logical/virtual partitioning."}),"\n",(0,a.jsx)(n.p,{children:"For very large tables (1Tb+, 10s of 1000s of files), Data skipping could"}),"\n",(0,a.jsxs)(n.ol,{children:["\n",(0,a.jsxs)(n.li,{children:["Substantially improve query execution runtime ",(0,a.jsx)(n.strong,{children:"10x"})," as compared to the same query on the same dataset but w/o Data Skipping enabled."]}),"\n",(0,a.jsx)(n.li,{children:"Help avoid hitting Cloud Storages throttling limits (for issuing too many requests, for ex, AWS limits # of requests / sec that could be issued based on the object's prefix which considerably complicates things for partitioned tables)"}),"\n"]}),"\n",(0,a.jsx)(n.p,{children:"To unlock the power of Data Skipping you will need to"}),"\n",(0,a.jsxs)(n.ol,{children:["\n",(0,a.jsxs)(n.li,{children:["Enable Metadata Table along with Column Stats Index on the ",(0,a.jsx)(n.em,{children:"write path"})," (See ",(0,a.jsx)(n.a,{href:"/docs/metadata_indexing",children:"Metadata Indexing"}),"), using ",(0,a.jsx)(n.code,{children:"hoodie.metadata.enable=true"})," (to enable Metadata Table on the write path, enabled by default)"]}),"\n",(0,a.jsxs)(n.li,{children:["Enable Data Skipping in your queries, using ",(0,a.jsx)(n.code,{children:"hoodie.metadata.index.column.stats.enable=true"})," (to enable Column Stats Index being populated on the write path, disabled by default)"]}),"\n"]}),"\n",(0,a.jsx)(n.admonition,{type:"note",children:(0,a.jsxs)(n.p,{children:["If you're planning on enabling Column Stats Index for already existing table, please check out the ",(0,a.jsx)(n.a,{href:"/docs/metadata_indexing",children:"Metadata Indexing"})," guide on how to build Metadata Table Indices (such as Column Stats Index) for existing tables."]})}),"\n",(0,a.jsxs)(n.p,{children:["To enable Data Skipping in your queries make sure to set following properties to ",(0,a.jsx)(n.code,{children:"true"})," (on the read path):"]}),"\n",(0,a.jsxs)(n.ul,{children:["\n",(0,a.jsxs)(n.li,{children:[(0,a.jsx)(n.code,{children:"hoodie.enable.data.skipping"})," (to control data skipping, enabled by default)"]}),"\n",(0,a.jsxs)(n.li,{children:[(0,a.jsx)(n.code,{children:"hoodie.metadata.enable"})," (to enable metadata table use on the read path, enabled by default)"]}),"\n",(0,a.jsxs)(n.li,{children:[(0,a.jsx)(n.code,{children:"hoodie.metadata.index.column.stats.enable"})," (to enable column stats index use on the read path)"]}),"\n"]})]})}function h(e={}){const{wrapper:n}={...(0,s.R)(),...e.components};return n?(0,a.jsx)(n,{...e,children:(0,a.jsx)(c,{...e})}):c(e)}},63490:(e,n,i)=>{i.d(n,{A:()=>t});const t=i.p+"assets/images/hudi_upsert_perf1-8f41921dacb5fb026f1e5457f8c47aa6.png"},19577:(e,n,i)=>{i.d(n,{A:()=>t});const t=i.p+"assets/images/hudi_upsert_perf2-d4bfcab4e9e8d942a02b712797ee2755.png"},28453:(e,n,i)=>{i.d(n,{R:()=>o,x:()=>r});var t=i(96540);const a={},s=t.createContext(a);function o(e){const n=t.useContext(s);return t.useMemo((function(){return"function"==typeof e?e(n):{...n,...e}}),[n,e])}function r(e){let n;return n=e.disableParentContext?"function"==typeof e.components?e.components(a):e.components||a:o(e.components),t.createElement(s.Provider,{value:n},e.children)}}}]);
\ No newline at end of file
diff --git a/content/assets/js/effd3c61.65cd727e.js b/content/assets/js/effd3c61.65cd727e.js
new file mode 100644
index 0000000000000..236c482f244e1
--- /dev/null
+++ b/content/assets/js/effd3c61.65cd727e.js
@@ -0,0 +1 @@
+"use strict";(self.webpackChunkhudi=self.webpackChunkhudi||[]).push([[8889],{6989:(e,t,i)=>{i.r(t),i.d(t,{assets:()=>l,contentTitle:()=>o,default:()=>h,frontMatter:()=>r,metadata:()=>n,toc:()=>c});const n=JSON.parse('{"id":"timeline","title":"Timeline","description":"Changes to table state (writes, table services, schema changes, etc) are recorded as actions_** in the Hudi timeline_. The Hudi timeline is a log of all actions performed","source":"@site/docs/timeline.md","sourceDirName":".","slug":"/timeline","permalink":"/docs/next/timeline","draft":false,"unlisted":false,"editUrl":"https://github.com/apache/hudi/tree/asf-site/website/docs/timeline.md","tags":[],"version":"current","frontMatter":{"title":"Timeline","toc":true,"toc_min_heading_level":2,"toc_max_heading_level":4},"sidebar":"docs","previous":{"title":"Apache Hudi Stack","permalink":"/docs/next/hudi_stack"},"next":{"title":"Storage Layouts","permalink":"/docs/next/storage_layouts"}}');var s=i(74848),a=i(28453);const r={title:"Timeline",toc:!0,toc_min_heading_level:2,toc_max_heading_level:4},o=void 0,l={},c=[{value:"Action Types",id:"action-types",level:3},{value:"State Transitions",id:"state-transitions",level:3},{value:"TrueTime Generation",id:"truetime-generation",level:3},{value:"Ordering of Actions",id:"ordering-of-actions",level:3},{value:"Timeline Components",id:"timeline-components",level:3},{value:"Active Timeline",id:"active-timeline",level:4},{value:"LSM Timeline History",id:"lsm-timeline-history",level:4},{value:"Timeline Archival Configs",id:"timeline-archival-configs",level:3},{value:"Spark configs",id:"spark-configs",level:4},{value:"Flink Options",id:"flink-options",level:4},{value:"Related Resources",id:"related-resources",level:2}];function d(e){const t={a:"a",br:"br",code:"code",em:"em",h2:"h2",h3:"h3",h4:"h4",img:"img",li:"li",p:"p",pre:"pre",strong:"strong",table:"table",tbody:"tbody",td:"td",th:"th",thead:"thead",tr:"tr",ul:"ul",...(0,a.R)(),...e.components};return(0,s.jsxs)(s.Fragment,{children:[(0,s.jsxs)(t.p,{children:["Changes to table state (writes, table services, schema changes, etc) are recorded as ",(0,s.jsx)(t.strong,{children:(0,s.jsx)(t.em,{children:"actions"})})," in the Hudi ",(0,s.jsx)(t.strong,{children:(0,s.jsx)(t.em,{children:"timeline"})}),". The Hudi timeline is a log of all actions performed\non the table at different ",(0,s.jsx)(t.strong,{children:(0,s.jsx)(t.em,{children:"instants"})})," (points in time). It is a key component of Hudi's architecture, acting as a source of truth for the state of the table. All instant times\nused on the timeline follow ",(0,s.jsx)(t.a,{href:"https://research.google/pubs/spanner-truetime-and-the-cap-theorem/",children:"TrueTime"})," semantics, and are monotonically increasing globally across various\nprocesses involved. See TrueTime section below for more details."]}),"\n",(0,s.jsx)(t.p,{children:"Each action has the following attributes associated with it."}),"\n",(0,s.jsxs)(t.ul,{children:["\n",(0,s.jsxs)(t.li,{children:[(0,s.jsx)(t.strong,{children:"requested instant"})," : Instant time representing when the action was requested on the timeline and acts as the transaction id. An immutable plan for the action should be generated before the action is requested."]}),"\n",(0,s.jsxs)(t.li,{children:[(0,s.jsx)(t.strong,{children:"completed instant"})," : Instant time representing when the action was completed on the timeline. All relevant changes to table data/metadata should be made before the action is completed."]}),"\n",(0,s.jsxs)(t.li,{children:[(0,s.jsx)(t.strong,{children:"state"})," : state of the action. valid states are ",(0,s.jsx)(t.code,{children:"REQUESTED"}),", ",(0,s.jsx)(t.code,{children:"INFLIGHT"})," and ",(0,s.jsx)(t.code,{children:"COMPLETED"})," during an action's lifecycle."]}),"\n",(0,s.jsxs)(t.li,{children:[(0,s.jsx)(t.strong,{children:"type"})," : the kind of action performed. See below for full list of actions."]}),"\n"]}),"\n",(0,s.jsxs)(t.p,{children:[(0,s.jsx)(t.img,{alt:"Timeline actions",src:i(57050).A+"",width:"817",height:"424"}),"\n",(0,s.jsx)("p",{align:"center",children:"Figure: Actions in the timeline"})]}),"\n",(0,s.jsx)(t.h3,{id:"action-types",children:"Action Types"}),"\n",(0,s.jsx)(t.p,{children:"Following are the valid action types."}),"\n",(0,s.jsxs)(t.ul,{children:["\n",(0,s.jsxs)(t.li,{children:[(0,s.jsx)(t.strong,{children:"COMMIT"})," - Write operation denoting an atomic write of a batch of records into a base files in the table."]}),"\n",(0,s.jsxs)(t.li,{children:[(0,s.jsx)(t.strong,{children:"DELTA_COMMIT"})," - Write operation denoting an atomic write of a batch of records into merge-on-read type table, where some/all of the data could be just written to delta logs."]}),"\n",(0,s.jsxs)(t.li,{children:[(0,s.jsx)(t.strong,{children:"REPLACE_COMMIT"})," - Write operation that atomically replaces a set of file groups in the table with another. Used for implementing batch write operations like ",(0,s.jsx)(t.em,{children:"insert_overwrite"}),", ",(0,s.jsx)(t.em,{children:"delete_partition"})," etc, as well as table services\nlike clustering."]}),"\n",(0,s.jsxs)(t.li,{children:[(0,s.jsx)(t.strong,{children:"CLEANS"})," - Table service that removes older file slices that are no longer needed from the table, by deleting those files."]}),"\n",(0,s.jsxs)(t.li,{children:[(0,s.jsx)(t.strong,{children:"COMPACTION"})," - Table service to reconcile differential data between base and delta files, by merging delta files into base files."]}),"\n",(0,s.jsxs)(t.li,{children:[(0,s.jsx)(t.strong,{children:"LOGCOMPACTION"})," - Table service to merge multiple small log files into a bigger log file in the same file slice."]}),"\n",(0,s.jsxs)(t.li,{children:[(0,s.jsx)(t.strong,{children:"CLUSTERING"})," - Table service to rewrite existing file groups with optimized sort order or storage layouts, as new file groups in the table."]}),"\n",(0,s.jsxs)(t.li,{children:[(0,s.jsx)(t.strong,{children:"INDEXING"})," - Table service to build an index of a requested type on a column of the table, consistent with the state of the table at the completed instant in face of ongoing writes."]}),"\n",(0,s.jsxs)(t.li,{children:[(0,s.jsx)(t.strong,{children:"ROLLBACK"})," - Indicates that an unsuccessful write operation was rolled back, removing any partial/uncommitted files produced during such a write from storage."]}),"\n",(0,s.jsxs)(t.li,{children:[(0,s.jsx)(t.strong,{children:"SAVEPOINT"}),' - Marks certain file slices as "saved", such that cleaner will not delete them. It helps restore the table to a point on the timeline, in case of disaster/data recovery scenarios or perform time-travel queries as of those instants.']}),"\n",(0,s.jsxs)(t.li,{children:[(0,s.jsx)(t.strong,{children:"RESTORE"})," - Restores a table to a given savepoint on the timeline, in case of disaster/data recovery scenarios."]}),"\n"]}),"\n",(0,s.jsxs)(t.p,{children:["In some cases, action types in the completed state may be different from requested/inflight states, but still tracked by the same requested instant. For e.g. ",(0,s.jsx)(t.em,{children:"CLUSTERING"})," as in requested/inflight state,\nbecomes ",(0,s.jsx)(t.em,{children:"REPLACE_COMMIT"})," in completed state. Compactions complete as ",(0,s.jsx)(t.em,{children:"COMMIT"})," action on the timeline producing new base files. In general, multiple write operations from the storage engine\nmay map to the same action on the timeline."]}),"\n",(0,s.jsx)(t.h3,{id:"state-transitions",children:"State Transitions"}),"\n",(0,s.jsxs)(t.p,{children:["Actions go through state transitions on the timeline, with each transition recorded by a file of the pattern ",(0,s.jsx)(t.code,{children:".."}),"(for other states) or\n",(0,s.jsx)(t.code,{children:"_."})," (for COMPLETED state). Hudi guarantees that the state transitions are atomic and timeline consistent based on the instant time.\nAtomicity is achieved by relying on the atomic operations on the underlying storage (e.g. PUT calls to S3/Cloud Storage)."]}),"\n",(0,s.jsx)(t.p,{children:"Valid state transitions are as follows:"}),"\n",(0,s.jsxs)(t.ul,{children:["\n",(0,s.jsxs)(t.li,{children:[(0,s.jsx)(t.code,{children:"[ ] -> REQUESTED"})," - Denotes an action has been scheduled, but has not initiated by any process yet.\nNote that the process requesting the action can be different from the process that will perform/complete the action."]}),"\n",(0,s.jsxs)(t.li,{children:[(0,s.jsx)(t.code,{children:"REQUESTED -> INFLIGHT"})," - Denotes that the action is currently being performed by some process."]}),"\n",(0,s.jsxs)(t.li,{children:[(0,s.jsx)(t.code,{children:"INFLIGHT -> REQUESTED"})," or ",(0,s.jsx)(t.code,{children:"INFLIGHT -> INFLIGHT"})," - A process can safely fail many times while performing the action."]}),"\n",(0,s.jsxs)(t.li,{children:[(0,s.jsx)(t.code,{children:"INFLIGHT -> COMPLETED"})," - Denotes that the action has been completed successfully."]}),"\n"]}),"\n",(0,s.jsxs)(t.p,{children:["The current state of an action on the timeline is the highest state recorded for that action on the timeline, with states ordered as ",(0,s.jsx)(t.code,{children:"REQUESTED < INFLIGHT < COMPLETED"}),"."]}),"\n",(0,s.jsx)(t.h3,{id:"truetime-generation",children:"TrueTime Generation"}),"\n",(0,s.jsxs)(t.p,{children:["Time in distributed systems has been studied literally for ",(0,s.jsx)(t.a,{href:"https://lamport.azurewebsites.net/pubs/chandy.pdf",children:"decades"}),". Google Spanner\u2019s\n",(0,s.jsx)(t.a,{href:"https://research.google/pubs/spanner-truetime-and-the-cap-theorem/",children:"TrueTime"})," API addresses the challenges of managing time in distributed systems by providing a globally\nsynchronized clock with bounded uncertainty. Traditional systems struggle with clock drift and lack of a consistent timeline, but TrueTime ensures that all nodes operate with\na common notion of time, defined by a strict interval of uncertainty. This enables Spanner to achieve external consistency in distributed transactions, allowing it to assign\ntimestamps with confidence that no other operation in the past or future will conflict, solving age-old issues of clock synchronization and causality. Several OLTP databases\nlike Spanner, ",(0,s.jsx)(t.a,{href:"https://www.cockroachlabs.com/blog/living-without-atomic-clocks/",children:"CockroachDB"})," rely on TrueTime."]}),"\n",(0,s.jsx)(t.p,{children:"Hudi uses these semantics for instant times on the timeline, to provide unique monotonically increasing instant values. TrueTime can be generated by a single shared time generator\nprocess or by having each process generate its own time and waiting for time >= maximum expected clock drift across all processes within a distributed lock. Locking ensures only one\nprocess is generating time at a time and waiting ensures enough time passes such that any new time generated is guaranteed to be greater than the previous time."}),"\n",(0,s.jsxs)(t.p,{children:[(0,s.jsx)(t.img,{alt:"Timeline actions",src:i(94200).A+"",width:"704",height:"511"}),"\n",(0,s.jsx)("p",{align:"center",children:"Figure: TrueTime generation for processes A & B"})]}),"\n",(0,s.jsxs)(t.p,{children:["The figure above shows how time generated by process A and B are monotonically increasing, even though process B has a lower local clock than A at the start, by waiting for uncertainty window of x ms to pass.",(0,s.jsx)(t.br,{}),"\n","In fact, given Hudi targets transaction durations > 1 second, we can afford to operate with a much higher uncertainty bound (> 100ms) guaranteeing extremely high fidelity time generation."]}),"\n",(0,s.jsx)(t.h3,{id:"ordering-of-actions",children:"Ordering of Actions"}),"\n",(0,s.jsx)(t.p,{children:"Thus, actions appear on the timeline as an interval starting at the requested instant and ending at the completed instant. Such actions can be ordered by completion time to"}),"\n",(0,s.jsxs)(t.ul,{children:["\n",(0,s.jsxs)(t.li,{children:[(0,s.jsx)(t.strong,{children:"Commit time ordering"})," : To obtain serializable execution order of writes performed consistent with typical relational databases, the actions can be ordered by completed instant."]}),"\n",(0,s.jsxs)(t.li,{children:[(0,s.jsx)(t.strong,{children:"Event time ordering"}),": Data lakehouses ultimately deal with streams of data (CDC, events, slowly changing data etc), where ordering is dependent on business fields in\nthe data. In such cases, actions can be ordered by commit time, while the records themselves are further merged in order of a specified event time field."]}),"\n"]}),"\n",(0,s.jsx)(t.p,{children:"Hudi relies on ordering of requested instants of certain actions against completed instants of other actions, to implement non-blocking table service operations or concurrent streaming model\nwrites with event time ordering."}),"\n",(0,s.jsx)(t.h3,{id:"timeline-components",children:"Timeline Components"}),"\n",(0,s.jsx)(t.h4,{id:"active-timeline",children:"Active Timeline"}),"\n",(0,s.jsxs)(t.p,{children:["Hudi implements the timeline as a Log Structured Merge (",(0,s.jsx)(t.a,{href:"https://en.wikipedia.org/wiki/Log-structured_merge-tree",children:"LSM"}),") tree under the ",(0,s.jsx)(t.code,{children:".hoodie/timeline"})," directory. Unlike typical LSM implementations,\nthe memory component and the write-ahead-log are at once replaced by ",(0,s.jsx)(t.a,{href:"https://avro.apache.org/",children:"avro"})," serialized files containing individual actions (",(0,s.jsx)(t.strong,{children:(0,s.jsx)(t.em,{children:"active timeline"})}),") for high durability and inter-process co-ordination.\nAll actions on the Hudi table are created in the active timeline a new entry and periodically actions are archived from the active timeline to the LSM structure (timeline history).\nAs the name suggests active timeline is consulted all the time to build a consistent view of data and archiving completed actions ensures reads on the timeline does not incur unnecessary latencies\nas timeline grows. The key invariant around such archiving is that any side effects from completed/pending actions (e.g. uncommitted files) are removed from storage, before archiving them."]}),"\n",(0,s.jsx)(t.h4,{id:"lsm-timeline-history",children:"LSM Timeline History"}),"\n",(0,s.jsxs)(t.p,{children:["As mentioned above, active timeline has limited log history to be fast, while archived timeline is expensive to access\nduring reads or writes, especially with high write throughput. To overcome this limitation, Hudi introduced the LSM (\nlog-structured merge) tree based timeline. Completed actions, their plans and completion metadata are stored in a more\nscalable LSM tree based archived timeline organized in an ",(0,s.jsx)(t.strong,{children:(0,s.jsx)(t.em,{children:"history"})})," storage folder under the ",(0,s.jsx)(t.code,{children:".hoodie/timeline"})," metadata\npath. It consists of Apache Parquet files with action instant data and bookkeeping metadata files, in the following\nmanner."]}),"\n",(0,s.jsx)(t.pre,{children:(0,s.jsx)(t.code,{className:"language-bash",children:"/.hoodie/timeline/history/ \t\t\t\t\t\n\u251c\u2500\u2500 _version_ \t\t\t\t\t <-- stores the manifest version that is current\n\u251c\u2500\u2500 manifest_1 <-- manifests store list of files in timeline\n\u251c\u2500\u2500 manifest_2 <-- compactions, cleaning, writes produce new manifest files\n\u251c\u2500\u2500 ... \n\u251c\u2500\u2500 manifest_ <-- there can be many manifest files at any given time\n\u251c\u2500\u2500 __.parquet <-- files storing actual action details\n"})}),"\n",(0,s.jsx)(t.p,{children:"One can read more about the details of LSM timeline in Hudi 1.0 specs. To understand it better, here is an example."}),"\n",(0,s.jsx)("figure",{children:(0,s.jsx)("img",{className:"docimage",src:i(91140).A,alt:"lsm_tree.png"})}),"\n",(0,s.jsxs)(t.p,{children:["In the above figure, each level is a tree sorted by instant times. We can see that for a bunch of commits the metadata\nis stored in a parquet file. As and when more commits are accumulated, they get compacted and pushed down to lower level\nof the tree. Each new operation to the timeline yields a new snapshot version. The advantage of such a structure is that\nwe can keep the top level in memory if needed, and still load the remaining levels efficiently from the disk if we need to walk\nback longer history. The LSM timeline compaction frequency is controlled by",(0,s.jsx)(t.code,{children:"hoodie.timeline.compaction.batch.size"})," i.e.\nfor every ",(0,s.jsx)(t.em,{children:"N"})," parquet files in the current level, they are merged and flush as a compacted file in the next level."]}),"\n",(0,s.jsx)(t.h3,{id:"timeline-archival-configs",children:"Timeline Archival Configs"}),"\n",(0,s.jsx)(t.p,{children:"Basic configurations that control archival."}),"\n",(0,s.jsx)(t.h4,{id:"spark-configs",children:"Spark configs"}),"\n",(0,s.jsxs)(t.table,{children:[(0,s.jsx)(t.thead,{children:(0,s.jsxs)(t.tr,{children:[(0,s.jsx)(t.th,{children:"Config Name"}),(0,s.jsx)(t.th,{children:"Default"}),(0,s.jsx)(t.th,{children:"Description"})]})}),(0,s.jsxs)(t.tbody,{children:[(0,s.jsxs)(t.tr,{children:[(0,s.jsx)(t.td,{children:"hoodie.keep.max.commits"}),(0,s.jsx)(t.td,{children:"30 (Optional)"}),(0,s.jsx)(t.td,{children:"Archiving service moves older entries from timeline into an archived log after each write, to keep the metadata overhead constant, even as the table size grows. This config controls the maximum number of instants to retain in the active timeline."})]}),(0,s.jsxs)(t.tr,{children:[(0,s.jsx)(t.td,{children:"hoodie.keep.min.commits"}),(0,s.jsx)(t.td,{children:"20 (Optional)"}),(0,s.jsx)(t.td,{children:"Similar to hoodie.keep.max.commits, but controls the minimum number of instants to retain in the active timeline."})]}),(0,s.jsxs)(t.tr,{children:[(0,s.jsx)(t.td,{children:"hoodie.timeline.compaction.batch.size"}),(0,s.jsx)(t.td,{children:"10 (Optional)"}),(0,s.jsx)(t.td,{children:"Controls the number of parquet files to compact in a single compaction run at the current level of the LSM tree."})]})]})]}),"\n",(0,s.jsxs)(t.p,{children:["For more advanced configs refer ",(0,s.jsx)(t.a,{href:"https://hudi.apache.org/docs/next/configurations#Archival-Configs-advanced-configs",children:"here"}),"."]}),"\n",(0,s.jsx)(t.h4,{id:"flink-options",children:"Flink Options"}),"\n",(0,s.jsx)(t.p,{children:"Flink jobs using the SQL can be configured through the options in WITH clause. The actual datasource level configs are listed below."}),"\n",(0,s.jsxs)(t.table,{children:[(0,s.jsx)(t.thead,{children:(0,s.jsxs)(t.tr,{children:[(0,s.jsx)(t.th,{children:"Config Name"}),(0,s.jsx)(t.th,{children:"Default"}),(0,s.jsx)(t.th,{children:"Description"})]})}),(0,s.jsxs)(t.tbody,{children:[(0,s.jsxs)(t.tr,{children:[(0,s.jsx)(t.td,{children:"archive.max_commits"}),(0,s.jsx)(t.td,{children:"50 (Optional)"}),(0,s.jsxs)(t.td,{children:["Max number of commits to keep before archiving older commits into a sequential log, default 50",(0,s.jsx)("br",{}),(0,s.jsx)("br",{})," ",(0,s.jsx)(t.code,{children:"Config Param: ARCHIVE_MAX_COMMITS"})]})]}),(0,s.jsxs)(t.tr,{children:[(0,s.jsx)(t.td,{children:"archive.min_commits"}),(0,s.jsx)(t.td,{children:"40 (Optional)"}),(0,s.jsxs)(t.td,{children:["Min number of commits to keep before archiving older commits into a sequential log, default 40",(0,s.jsx)("br",{}),(0,s.jsx)("br",{})," ",(0,s.jsx)(t.code,{children:"Config Param: ARCHIVE_MIN_COMMITS"})]})]}),(0,s.jsxs)(t.tr,{children:[(0,s.jsx)(t.td,{children:"hoodie.timeline.compaction.batch.size"}),(0,s.jsx)(t.td,{children:"10 (Optional)"}),(0,s.jsx)(t.td,{children:"Controls the number of parquet files to compact in a single compaction run at the current level of the LSM tree."})]})]})]}),"\n",(0,s.jsxs)(t.p,{children:["Refer ",(0,s.jsx)(t.a,{href:"https://hudi.apache.org/docs/next/configurations#Flink-Options",children:"here"})," for more details."]}),"\n",(0,s.jsx)(t.h2,{id:"related-resources",children:"Related Resources"}),"\n",(0,s.jsx)("h3",{children:"Blogs"}),"\n",(0,s.jsxs)(t.ul,{children:["\n",(0,s.jsx)(t.li,{children:(0,s.jsx)(t.a,{href:"https://medium.com/@simpsons/hoodie-timeline-foundational-pillar-for-acid-transactions-be871399cbae",children:"Apache Hudi Timeline: Foundational pillar for ACID transactions"})}),"\n"]})]})}function h(e={}){const{wrapper:t}={...(0,a.R)(),...e.components};return t?(0,s.jsx)(t,{...e,children:(0,s.jsx)(d,{...e})}):d(e)}},91140:(e,t,i)=>{i.d(t,{A:()=>n});const n=i.p+"assets/images/lsm_tree-0a069798a1196c1c71330abcb7ff3581.png"},57050:(e,t,i)=>{i.d(t,{A:()=>n});const n=i.p+"assets/images/hudi-timeline-actions-e56d0d9fad5645d9910f2591ad7775de.png"},94200:(e,t,i)=>{i.d(t,{A:()=>n});const n=i.p+"assets/images/hudi-timeline-truetime-4cb47da19e5344580d5ebdcdce3d6cf2.png"},28453:(e,t,i)=>{i.d(t,{R:()=>r,x:()=>o});var n=i(96540);const s={},a=n.createContext(s);function r(e){const t=n.useContext(a);return n.useMemo((function(){return"function"==typeof e?e(t):{...t,...e}}),[t,e])}function o(e){let t;return t=e.disableParentContext?"function"==typeof e.components?e.components(s):e.components||s:r(e.components),n.createElement(a.Provider,{value:t},e.children)}}}]);
\ No newline at end of file
diff --git a/content/assets/js/effd3c61.86b477eb.js b/content/assets/js/effd3c61.86b477eb.js
deleted file mode 100644
index 0db4aeda8e20f..0000000000000
--- a/content/assets/js/effd3c61.86b477eb.js
+++ /dev/null
@@ -1 +0,0 @@
-"use strict";(self.webpackChunkhudi=self.webpackChunkhudi||[]).push([[8889],{6989:(e,t,i)=>{i.r(t),i.d(t,{assets:()=>l,contentTitle:()=>o,default:()=>h,frontMatter:()=>r,metadata:()=>n,toc:()=>c});const n=JSON.parse('{"id":"timeline","title":"Timeline","description":"Changes to table state (writes, table services, schema changes, etc) are recorded as actions_** in the Hudi timeline_. The Hudi timeline is a log of all actions performed","source":"@site/docs/timeline.md","sourceDirName":".","slug":"/timeline","permalink":"/docs/next/timeline","draft":false,"unlisted":false,"editUrl":"https://github.com/apache/hudi/tree/asf-site/website/docs/timeline.md","tags":[],"version":"current","frontMatter":{"title":"Timeline","toc":true,"toc_min_heading_level":2,"toc_max_heading_level":4},"sidebar":"docs","previous":{"title":"Apache Hudi Stack","permalink":"/docs/next/hudi_stack"},"next":{"title":"Storage Layouts","permalink":"/docs/next/storage_layouts"}}');var s=i(74848),a=i(28453);const r={title:"Timeline",toc:!0,toc_min_heading_level:2,toc_max_heading_level:4},o=void 0,l={},c=[{value:"Action Types",id:"action-types",level:3},{value:"State Transitions",id:"state-transitions",level:3},{value:"TrueTime Generation",id:"truetime-generation",level:3},{value:"Ordering of Actions",id:"ordering-of-actions",level:3},{value:"Timeline Components",id:"timeline-components",level:3},{value:"Active Timeline",id:"active-timeline",level:4},{value:"LSM Timeline History",id:"lsm-timeline-history",level:4},{value:"Timeline Archival Configs",id:"timeline-archival-configs",level:3},{value:"Spark configs",id:"spark-configs",level:4},{value:"Flink Options",id:"flink-options",level:4}];function d(e){const t={a:"a",br:"br",code:"code",em:"em",h3:"h3",h4:"h4",img:"img",li:"li",p:"p",pre:"pre",strong:"strong",table:"table",tbody:"tbody",td:"td",th:"th",thead:"thead",tr:"tr",ul:"ul",...(0,a.R)(),...e.components};return(0,s.jsxs)(s.Fragment,{children:[(0,s.jsxs)(t.p,{children:["Changes to table state (writes, table services, schema changes, etc) are recorded as ",(0,s.jsx)(t.strong,{children:(0,s.jsx)(t.em,{children:"actions"})})," in the Hudi ",(0,s.jsx)(t.strong,{children:(0,s.jsx)(t.em,{children:"timeline"})}),". The Hudi timeline is a log of all actions performed\non the table at different ",(0,s.jsx)(t.strong,{children:(0,s.jsx)(t.em,{children:"instants"})})," (points in time). It is a key component of Hudi's architecture, acting as a source of truth for the state of the table. All instant times\nused on the timeline follow ",(0,s.jsx)(t.a,{href:"https://research.google/pubs/spanner-truetime-and-the-cap-theorem/",children:"TrueTime"})," semantics, and are monotonically increasing globally across various\nprocesses involved. See TrueTime section below for more details."]}),"\n",(0,s.jsx)(t.p,{children:"Each action has the following attributes associated with it."}),"\n",(0,s.jsxs)(t.ul,{children:["\n",(0,s.jsxs)(t.li,{children:[(0,s.jsx)(t.strong,{children:"requested instant"})," : Instant time representing when the action was requested on the timeline and acts as the transaction id. An immutable plan for the action should be generated before the action is requested."]}),"\n",(0,s.jsxs)(t.li,{children:[(0,s.jsx)(t.strong,{children:"completed instant"})," : Instant time representing when the action was completed on the timeline. All relevant changes to table data/metadata should be made before the action is completed."]}),"\n",(0,s.jsxs)(t.li,{children:[(0,s.jsx)(t.strong,{children:"state"})," : state of the action. valid states are ",(0,s.jsx)(t.code,{children:"REQUESTED"}),", ",(0,s.jsx)(t.code,{children:"INFLIGHT"})," and ",(0,s.jsx)(t.code,{children:"COMPLETED"})," during an action's lifecycle."]}),"\n",(0,s.jsxs)(t.li,{children:[(0,s.jsx)(t.strong,{children:"type"})," : the kind of action performed. See below for full list of actions."]}),"\n"]}),"\n",(0,s.jsxs)(t.p,{children:[(0,s.jsx)(t.img,{alt:"Timeline actions",src:i(57050).A+"",width:"817",height:"424"}),"\n",(0,s.jsx)("p",{align:"center",children:"Figure: Actions in the timeline"})]}),"\n",(0,s.jsx)(t.h3,{id:"action-types",children:"Action Types"}),"\n",(0,s.jsx)(t.p,{children:"Following are the valid action types."}),"\n",(0,s.jsxs)(t.ul,{children:["\n",(0,s.jsxs)(t.li,{children:[(0,s.jsx)(t.strong,{children:"COMMIT"})," - Write operation denoting an atomic write of a batch of records into a base files in the table."]}),"\n",(0,s.jsxs)(t.li,{children:[(0,s.jsx)(t.strong,{children:"DELTA_COMMIT"})," - Write operation denoting an atomic write of a batch of records into merge-on-read type table, where some/all of the data could be just written to delta logs."]}),"\n",(0,s.jsxs)(t.li,{children:[(0,s.jsx)(t.strong,{children:"REPLACE_COMMIT"})," - Write operation that atomically replaces a set of file groups in the table with another. Used for implementing batch write operations like ",(0,s.jsx)(t.em,{children:"insert_overwrite"}),", ",(0,s.jsx)(t.em,{children:"delete_partition"})," etc, as well as table services\nlike clustering."]}),"\n",(0,s.jsxs)(t.li,{children:[(0,s.jsx)(t.strong,{children:"CLEANS"})," - Table service that removes older file slices that are no longer needed from the table, by deleting those files."]}),"\n",(0,s.jsxs)(t.li,{children:[(0,s.jsx)(t.strong,{children:"COMPACTION"})," - Table service to reconcile differential data between base and delta files, by merging delta files into base files."]}),"\n",(0,s.jsxs)(t.li,{children:[(0,s.jsx)(t.strong,{children:"LOGCOMPACTION"})," - Table service to merge multiple small log files into a bigger log file in the same file slice."]}),"\n",(0,s.jsxs)(t.li,{children:[(0,s.jsx)(t.strong,{children:"CLUSTERING"})," - Table service to rewrite existing file groups with optimized sort order or storage layouts, as new file groups in the table."]}),"\n",(0,s.jsxs)(t.li,{children:[(0,s.jsx)(t.strong,{children:"INDEXING"})," - Table service to build an index of a requested type on a column of the table, consistent with the state of the table at the completed instant in face of ongoing writes."]}),"\n",(0,s.jsxs)(t.li,{children:[(0,s.jsx)(t.strong,{children:"ROLLBACK"})," - Indicates that an unsuccessful write operation was rolled back, removing any partial/uncommitted files produced during such a write from storage."]}),"\n",(0,s.jsxs)(t.li,{children:[(0,s.jsx)(t.strong,{children:"SAVEPOINT"}),' - Marks certain file slices as "saved", such that cleaner will not delete them. It helps restore the table to a point on the timeline, in case of disaster/data recovery scenarios or perform time-travel queries as of those instants.']}),"\n",(0,s.jsxs)(t.li,{children:[(0,s.jsx)(t.strong,{children:"RESTORE"})," - Restores a table to a given savepoint on the timeline, in case of disaster/data recovery scenarios."]}),"\n"]}),"\n",(0,s.jsxs)(t.p,{children:["In some cases, action types in the completed state may be different from requested/inflight states, but still tracked by the same requested instant. For e.g. ",(0,s.jsx)(t.em,{children:"CLUSTERING"})," as in requested/inflight state,\nbecomes ",(0,s.jsx)(t.em,{children:"REPLACE_COMMIT"})," in completed state. Compactions complete as ",(0,s.jsx)(t.em,{children:"COMMIT"})," action on the timeline producing new base files. In general, multiple write operations from the storage engine\nmay map to the same action on the timeline."]}),"\n",(0,s.jsx)(t.h3,{id:"state-transitions",children:"State Transitions"}),"\n",(0,s.jsxs)(t.p,{children:["Actions go through state transitions on the timeline, with each transition recorded by a file of the pattern ",(0,s.jsx)(t.code,{children:".."}),"(for other states) or\n",(0,s.jsx)(t.code,{children:"_."})," (for COMPLETED state). Hudi guarantees that the state transitions are atomic and timeline consistent based on the instant time.\nAtomicity is achieved by relying on the atomic operations on the underlying storage (e.g. PUT calls to S3/Cloud Storage)."]}),"\n",(0,s.jsx)(t.p,{children:"Valid state transitions are as follows:"}),"\n",(0,s.jsxs)(t.ul,{children:["\n",(0,s.jsxs)(t.li,{children:[(0,s.jsx)(t.code,{children:"[ ] -> REQUESTED"})," - Denotes an action has been scheduled, but has not initiated by any process yet.\nNote that the process requesting the action can be different from the process that will perform/complete the action."]}),"\n",(0,s.jsxs)(t.li,{children:[(0,s.jsx)(t.code,{children:"REQUESTED -> INFLIGHT"})," - Denotes that the action is currently being performed by some process."]}),"\n",(0,s.jsxs)(t.li,{children:[(0,s.jsx)(t.code,{children:"INFLIGHT -> REQUESTED"})," or ",(0,s.jsx)(t.code,{children:"INFLIGHT -> INFLIGHT"})," - A process can safely fail many times while performing the action."]}),"\n",(0,s.jsxs)(t.li,{children:[(0,s.jsx)(t.code,{children:"INFLIGHT -> COMPLETED"})," - Denotes that the action has been completed successfully."]}),"\n"]}),"\n",(0,s.jsxs)(t.p,{children:["The current state of an action on the timeline is the highest state recorded for that action on the timeline, with states ordered as ",(0,s.jsx)(t.code,{children:"REQUESTED < INFLIGHT < COMPLETED"}),"."]}),"\n",(0,s.jsx)(t.h3,{id:"truetime-generation",children:"TrueTime Generation"}),"\n",(0,s.jsxs)(t.p,{children:["Time in distributed systems has been studied literally for ",(0,s.jsx)(t.a,{href:"https://lamport.azurewebsites.net/pubs/chandy.pdf",children:"decades"}),". Google Spanner\u2019s\n",(0,s.jsx)(t.a,{href:"https://research.google/pubs/spanner-truetime-and-the-cap-theorem/",children:"TrueTime"})," API addresses the challenges of managing time in distributed systems by providing a globally\nsynchronized clock with bounded uncertainty. Traditional systems struggle with clock drift and lack of a consistent timeline, but TrueTime ensures that all nodes operate with\na common notion of time, defined by a strict interval of uncertainty. This enables Spanner to achieve external consistency in distributed transactions, allowing it to assign\ntimestamps with confidence that no other operation in the past or future will conflict, solving age-old issues of clock synchronization and causality. Several OLTP databases\nlike Spanner, ",(0,s.jsx)(t.a,{href:"https://www.cockroachlabs.com/blog/living-without-atomic-clocks/",children:"CockroachDB"})," rely on TrueTime."]}),"\n",(0,s.jsx)(t.p,{children:"Hudi uses these semantics for instant times on the timeline, to provide unique monotonically increasing instant values. TrueTime can be generated by a single shared time generator\nprocess or by having each process generate its own time and waiting for time >= maximum expected clock drift across all processes within a distributed lock. Locking ensures only one\nprocess is generating time at a time and waiting ensures enough time passes such that any new time generated is guaranteed to be greater than the previous time."}),"\n",(0,s.jsxs)(t.p,{children:[(0,s.jsx)(t.img,{alt:"Timeline actions",src:i(94200).A+"",width:"704",height:"511"}),"\n",(0,s.jsx)("p",{align:"center",children:"Figure: TrueTime generation for processes A & B"})]}),"\n",(0,s.jsxs)(t.p,{children:["The figure above shows how time generated by process A and B are monotonically increasing, even though process B has a lower local clock than A at the start, by waiting for uncertainty window of x ms to pass.",(0,s.jsx)(t.br,{}),"\n","In fact, given Hudi targets transaction durations > 1 second, we can afford to operate with a much higher uncertainty bound (> 100ms) guaranteeing extremely high fidelity time generation."]}),"\n",(0,s.jsx)(t.h3,{id:"ordering-of-actions",children:"Ordering of Actions"}),"\n",(0,s.jsx)(t.p,{children:"Thus, actions appear on the timeline as an interval starting at the requested instant and ending at the completed instant. Such actions can be ordered by completion time to"}),"\n",(0,s.jsxs)(t.ul,{children:["\n",(0,s.jsxs)(t.li,{children:[(0,s.jsx)(t.strong,{children:"Commit time ordering"})," : To obtain serializable execution order of writes performed consistent with typical relational databases, the actions can be ordered by completed instant."]}),"\n",(0,s.jsxs)(t.li,{children:[(0,s.jsx)(t.strong,{children:"Event time ordering"}),": Data lakehouses ultimately deal with streams of data (CDC, events, slowly changing data etc), where ordering is dependent on business fields in\nthe data. In such cases, actions can be ordered by commit time, while the records themselves are further merged in order of a specified event time field."]}),"\n"]}),"\n",(0,s.jsx)(t.p,{children:"Hudi relies on ordering of requested instants of certain actions against completed instants of other actions, to implement non-blocking table service operations or concurrent streaming model\nwrites with event time ordering."}),"\n",(0,s.jsx)(t.h3,{id:"timeline-components",children:"Timeline Components"}),"\n",(0,s.jsx)(t.h4,{id:"active-timeline",children:"Active Timeline"}),"\n",(0,s.jsxs)(t.p,{children:["Hudi implements the timeline as a Log Structured Merge (",(0,s.jsx)(t.a,{href:"https://en.wikipedia.org/wiki/Log-structured_merge-tree",children:"LSM"}),") tree under the ",(0,s.jsx)(t.code,{children:".hoodie/timeline"})," directory. Unlike typical LSM implementations,\nthe memory component and the write-ahead-log are at once replaced by ",(0,s.jsx)(t.a,{href:"https://avro.apache.org/",children:"avro"})," serialized files containing individual actions (",(0,s.jsx)(t.strong,{children:(0,s.jsx)(t.em,{children:"active timeline"})}),") for high durability and inter-process co-ordination.\nAll actions on the Hudi table are created in the active timeline a new entry and periodically actions are archived from the active timeline to the LSM structure (timeline history).\nAs the name suggests active timeline is consulted all the time to build a consistent view of data and archiving completed actions ensures reads on the timeline does not incur unnecessary latencies\nas timeline grows. The key invariant around such archiving is that any side effects from completed/pending actions (e.g. uncommitted files) are removed from storage, before archiving them."]}),"\n",(0,s.jsx)(t.h4,{id:"lsm-timeline-history",children:"LSM Timeline History"}),"\n",(0,s.jsxs)(t.p,{children:["As mentioned above, active timeline has limited log history to be fast, while archived timeline is expensive to access\nduring reads or writes, especially with high write throughput. To overcome this limitation, Hudi introduced the LSM (\nlog-structured merge) tree based timeline. Completed actions, their plans and completion metadata are stored in a more\nscalable LSM tree based archived timeline organized in an ",(0,s.jsx)(t.strong,{children:(0,s.jsx)(t.em,{children:"history"})})," storage folder under the ",(0,s.jsx)(t.code,{children:".hoodie/timeline"})," metadata\npath. It consists of Apache Parquet files with action instant data and bookkeeping metadata files, in the following\nmanner."]}),"\n",(0,s.jsx)(t.pre,{children:(0,s.jsx)(t.code,{className:"language-bash",children:"/.hoodie/timeline/history/ \t\t\t\t\t\n\u251c\u2500\u2500 _version_ \t\t\t\t\t <-- stores the manifest version that is current\n\u251c\u2500\u2500 manifest_1 <-- manifests store list of files in timeline\n\u251c\u2500\u2500 manifest_2 <-- compactions, cleaning, writes produce new manifest files\n\u251c\u2500\u2500 ... \n\u251c\u2500\u2500 manifest_ <-- there can be many manifest files at any given time\n\u251c\u2500\u2500 __.parquet <-- files storing actual action details\n"})}),"\n",(0,s.jsx)(t.p,{children:"One can read more about the details of LSM timeline in Hudi 1.0 specs. To understand it better, here is an example."}),"\n",(0,s.jsx)("figure",{children:(0,s.jsx)("img",{className:"docimage",src:i(91140).A,alt:"lsm_tree.png"})}),"\n",(0,s.jsxs)(t.p,{children:["In the above figure, each level is a tree sorted by instant times. We can see that for a bunch of commits the metadata\nis stored in a parquet file. As and when more commits are accumulated, they get compacted and pushed down to lower level\nof the tree. Each new operation to the timeline yields a new snapshot version. The advantage of such a structure is that\nwe can keep the top level in memory if needed, and still load the remaining levels efficiently from the disk if we need to walk\nback longer history. The LSM timeline compaction frequency is controlled by",(0,s.jsx)(t.code,{children:"hoodie.timeline.compaction.batch.size"})," i.e.\nfor every ",(0,s.jsx)(t.em,{children:"N"})," parquet files in the current level, they are merged and flush as a compacted file in the next level."]}),"\n",(0,s.jsx)(t.h3,{id:"timeline-archival-configs",children:"Timeline Archival Configs"}),"\n",(0,s.jsx)(t.p,{children:"Basic configurations that control archival."}),"\n",(0,s.jsx)(t.h4,{id:"spark-configs",children:"Spark configs"}),"\n",(0,s.jsxs)(t.table,{children:[(0,s.jsx)(t.thead,{children:(0,s.jsxs)(t.tr,{children:[(0,s.jsx)(t.th,{children:"Config Name"}),(0,s.jsx)(t.th,{children:"Default"}),(0,s.jsx)(t.th,{children:"Description"})]})}),(0,s.jsxs)(t.tbody,{children:[(0,s.jsxs)(t.tr,{children:[(0,s.jsx)(t.td,{children:"hoodie.keep.max.commits"}),(0,s.jsx)(t.td,{children:"30 (Optional)"}),(0,s.jsx)(t.td,{children:"Archiving service moves older entries from timeline into an archived log after each write, to keep the metadata overhead constant, even as the table size grows. This config controls the maximum number of instants to retain in the active timeline."})]}),(0,s.jsxs)(t.tr,{children:[(0,s.jsx)(t.td,{children:"hoodie.keep.min.commits"}),(0,s.jsx)(t.td,{children:"20 (Optional)"}),(0,s.jsx)(t.td,{children:"Similar to hoodie.keep.max.commits, but controls the minimum number of instants to retain in the active timeline."})]}),(0,s.jsxs)(t.tr,{children:[(0,s.jsx)(t.td,{children:"hoodie.timeline.compaction.batch.size"}),(0,s.jsx)(t.td,{children:"10 (Optional)"}),(0,s.jsx)(t.td,{children:"Controls the number of parquet files to compact in a single compaction run at the current level of the LSM tree."})]})]})]}),"\n",(0,s.jsxs)(t.p,{children:["For more advanced configs refer ",(0,s.jsx)(t.a,{href:"https://hudi.apache.org/docs/next/configurations#Archival-Configs-advanced-configs",children:"here"}),"."]}),"\n",(0,s.jsx)(t.h4,{id:"flink-options",children:"Flink Options"}),"\n",(0,s.jsx)(t.p,{children:"Flink jobs using the SQL can be configured through the options in WITH clause. The actual datasource level configs are listed below."}),"\n",(0,s.jsxs)(t.table,{children:[(0,s.jsx)(t.thead,{children:(0,s.jsxs)(t.tr,{children:[(0,s.jsx)(t.th,{children:"Config Name"}),(0,s.jsx)(t.th,{children:"Default"}),(0,s.jsx)(t.th,{children:"Description"})]})}),(0,s.jsxs)(t.tbody,{children:[(0,s.jsxs)(t.tr,{children:[(0,s.jsx)(t.td,{children:"archive.max_commits"}),(0,s.jsx)(t.td,{children:"50 (Optional)"}),(0,s.jsxs)(t.td,{children:["Max number of commits to keep before archiving older commits into a sequential log, default 50",(0,s.jsx)("br",{}),(0,s.jsx)("br",{})," ",(0,s.jsx)(t.code,{children:"Config Param: ARCHIVE_MAX_COMMITS"})]})]}),(0,s.jsxs)(t.tr,{children:[(0,s.jsx)(t.td,{children:"archive.min_commits"}),(0,s.jsx)(t.td,{children:"40 (Optional)"}),(0,s.jsxs)(t.td,{children:["Min number of commits to keep before archiving older commits into a sequential log, default 40",(0,s.jsx)("br",{}),(0,s.jsx)("br",{})," ",(0,s.jsx)(t.code,{children:"Config Param: ARCHIVE_MIN_COMMITS"})]})]}),(0,s.jsxs)(t.tr,{children:[(0,s.jsx)(t.td,{children:"hoodie.timeline.compaction.batch.size"}),(0,s.jsx)(t.td,{children:"10 (Optional)"}),(0,s.jsx)(t.td,{children:"Controls the number of parquet files to compact in a single compaction run at the current level of the LSM tree."})]})]})]}),"\n",(0,s.jsxs)(t.p,{children:["Refer ",(0,s.jsx)(t.a,{href:"https://hudi.apache.org/docs/next/configurations#Flink-Options",children:"here"})," for more details."]})]})}function h(e={}){const{wrapper:t}={...(0,a.R)(),...e.components};return t?(0,s.jsx)(t,{...e,children:(0,s.jsx)(d,{...e})}):d(e)}},91140:(e,t,i)=>{i.d(t,{A:()=>n});const n=i.p+"assets/images/lsm_tree-0a069798a1196c1c71330abcb7ff3581.png"},57050:(e,t,i)=>{i.d(t,{A:()=>n});const n=i.p+"assets/images/hudi-timeline-actions-e56d0d9fad5645d9910f2591ad7775de.png"},94200:(e,t,i)=>{i.d(t,{A:()=>n});const n=i.p+"assets/images/hudi-timeline-truetime-4cb47da19e5344580d5ebdcdce3d6cf2.png"},28453:(e,t,i)=>{i.d(t,{R:()=>r,x:()=>o});var n=i(96540);const s={},a=n.createContext(s);function r(e){const t=n.useContext(a);return n.useMemo((function(){return"function"==typeof e?e(t):{...t,...e}}),[t,e])}function o(e){let t;return t=e.disableParentContext?"function"==typeof e.components?e.components(s):e.components||s:r(e.components),n.createElement(a.Provider,{value:t},e.children)}}}]);
\ No newline at end of file
diff --git a/content/assets/js/f67021ea.53f18aee.js b/content/assets/js/f67021ea.79ab4ac0.js
similarity index 94%
rename from content/assets/js/f67021ea.53f18aee.js
rename to content/assets/js/f67021ea.79ab4ac0.js
index d638d2af06db7..534927e560f12 100644
--- a/content/assets/js/f67021ea.53f18aee.js
+++ b/content/assets/js/f67021ea.79ab4ac0.js
@@ -1 +1 @@
-"use strict";(self.webpackChunkhudi=self.webpackChunkhudi||[]).push([[17987],{26947:(e,n,i)=>{i.r(n),i.d(n,{assets:()=>d,contentTitle:()=>r,default:()=>h,frontMatter:()=>o,metadata:()=>t,toc:()=>l});const t=JSON.parse('{"id":"indexes","title":"Indexes","description":"In databases, indexes are auxiliary data structures maintained to quickly locate records needed, without reading unnecessary data","source":"@site/docs/indexes.md","sourceDirName":".","slug":"/indexes","permalink":"/docs/next/indexes","draft":false,"unlisted":false,"editUrl":"https://github.com/apache/hudi/tree/asf-site/website/docs/indexes.md","tags":[],"version":"current","frontMatter":{"title":"Indexes","toc":true,"toc_min_heading_level":2,"toc_max_heading_level":4},"sidebar":"docs","previous":{"title":"Table Metadata","permalink":"/docs/next/metadata"},"next":{"title":"Concurrency Control","permalink":"/docs/next/concurrency_control"}}');var s=i(74848),a=i(28453);const o={title:"Indexes",toc:!0,toc_min_heading_level:2,toc_max_heading_level:4},r=void 0,d={},l=[{value:"Mapping keys to file groups",id:"mapping-keys-to-file-groups",level:2},{value:"Need for indexing",id:"need-for-indexing",level:2},{value:"Multi-modal Indexing",id:"multi-modal-indexing",level:2},{value:"Bloom Filters",id:"bloom-filters",level:3},{value:"Record Indexes",id:"record-indexes",level:3},{value:"Expression Index",id:"expression-index",level:3},{value:"Secondary Index",id:"secondary-index",level:3},{value:"Additional writer-side indexes",id:"additional-writer-side-indexes",level:2},{value:"Global and Non-Global Indexes",id:"global-and-non-global-indexes",level:3},{value:"Configs",id:"configs",level:3},{value:"Spark based configs",id:"spark-based-configs",level:4},{value:"Flink based configs",id:"flink-based-configs",level:4},{value:"Picking Indexing Strategies",id:"picking-indexing-strategies",level:3},{value:"Workload 1: Late arriving updates to fact tables",id:"workload-1-late-arriving-updates-to-fact-tables",level:4},{value:"Workload 2: De-Duplication in event tables",id:"workload-2-de-duplication-in-event-tables",level:4},{value:"Workload 3: Random updates/deletes to a dimension table",id:"workload-3-random-updatesdeletes-to-a-dimension-table",level:4},{value:"Related Resources",id:"related-resources",level:2}];function c(e){const n={a:"a",code:"code",em:"em",h2:"h2",h3:"h3",h4:"h4",img:"img",li:"li",p:"p",strong:"strong",table:"table",tbody:"tbody",td:"td",th:"th",thead:"thead",tr:"tr",ul:"ul",...(0,a.R)(),...e.components};return(0,s.jsxs)(s.Fragment,{children:[(0,s.jsxs)(n.p,{children:["In databases, indexes are auxiliary data structures maintained to quickly locate records needed, without reading unnecessary data\nfrom storage. Given that Hudi\u2019s design has been heavily optimized for handling mutable change streams, with different\nwrite patterns, Hudi considers ",(0,s.jsx)(n.a,{href:"#indexing",children:"indexing"})," as an integral part of its design and has uniquely supported\n",(0,s.jsx)(n.a,{href:"https://hudi.apache.org/blog/2020/11/11/hudi-indexing-mechanisms/",children:"indexing capabilities"})," from its inception, to speed\nup writes on the ",(0,s.jsx)(n.a,{href:"https://hudi.apache.org/blog/2024/07/11/what-is-a-data-lakehouse/",children:"data lakehouse"}),", while still providing\ncolumnar query performance."]}),"\n",(0,s.jsx)(n.h2,{id:"mapping-keys-to-file-groups",children:"Mapping keys to file groups"}),"\n",(0,s.jsx)(n.p,{children:"The most foundational index mechanism in Hudi tracks a mapping from a given key (record key + optionally partition path) consistently to a file id. Other types of indexes like secondary indexes,\nbuild on this foundation. This mapping between record key and file group/file id rarely changes once the first version of a record has been written to a file group.\nOnly clustering or cross-partition updates that are implemented as deletes + inserts remap the record key to a different file group. Even then, a given record key is associated with exactly one\nfile group at any completed instant on the timeline."}),"\n",(0,s.jsx)(n.h2,{id:"need-for-indexing",children:"Need for indexing"}),"\n",(0,s.jsxs)(n.p,{children:["For ",(0,s.jsx)(n.a,{href:"table_types#copy-on-write-table",children:"Copy-On-Write tables"}),", indexing enables fast upsert/delete operations, by avoiding the need to join against the entire dataset to determine which files to rewrite.\nFor ",(0,s.jsx)(n.a,{href:"table_types#merge-on-read-table",children:"Merge-On-Read tables"}),", indexing allows Hudi to bound the amount of change records any given base file needs to be merged against. Specifically, a given base file needs to merged\nonly against updates for records that are part of that base file."]}),"\n",(0,s.jsxs)(n.p,{children:[(0,s.jsx)(n.img,{alt:"Fact table",src:i(82301).A+"",width:"3994",height:"1704"}),"\n",(0,s.jsx)("p",{align:"center",children:"Figure: Comparison of merge cost for updates (dark blue blocks) against base files (light blue blocks)"})]}),"\n",(0,s.jsx)(n.p,{children:"In contrast,"}),"\n",(0,s.jsxs)(n.ul,{children:["\n",(0,s.jsxs)(n.li,{children:["Designs without an indexing component (e.g: ",(0,s.jsx)(n.a,{href:"https://cwiki.apache.org/confluence/display/Hive/Hive+Transactions",children:"Apache Hive/Apache Iceberg"}),") end up having to merge all the base files against all incoming updates/delete records\n(10-100x more ",(0,s.jsx)(n.a,{href:"table_types#comparison",children:"read amplification"}),")."]}),"\n",(0,s.jsx)(n.li,{children:"Designs that implement heavily write-optimized OLTP data structures like LSM trees do not require an indexing component. But they perform poorly scan heavy workloads\nagainst cloud storage making them unsuitable for serving analytical queries."}),"\n"]}),"\n",(0,s.jsx)(n.p,{children:"Hudi shines by achieving both great write performance and read performance, at the extra storage costs of an index, which can however unlock a lot more, as we explore below."}),"\n",(0,s.jsx)(n.h2,{id:"multi-modal-indexing",children:"Multi-modal Indexing"}),"\n",(0,s.jsxs)(n.p,{children:[(0,s.jsx)(n.a,{href:"https://www.onehouse.ai/blog/introducing-multi-modal-index-for-the-lakehouse-in-apache-hudi",children:"Multi-modal indexing"}),",\nintroduced in ",(0,s.jsx)(n.a,{href:"https://hudi.apache.org/releases/release-0.11.0/#multi-modal-index",children:"0.11.0 Hudi release"}),",\nis a re-imagination of what a general purpose indexing subsystem should look like for the lake. Multi-modal indexing is\nimplemented by enhancing the metadata table with the flexibility to extend to new index types as new partitions,\nalong with an ",(0,s.jsx)(n.a,{href:"https://hudi.apache.org/docs/metadata_indexing/#setup-async-indexing",children:"asynchronous index"})," building"]}),"\n",(0,s.jsxs)(n.p,{children:["Hudi supports a multi-modal index by augmenting the metadata table with the capability to incorporate new types of indexes, complemented by an\nasynchronous mechanism for ",(0,s.jsx)(n.a,{href:"metadata_indexing",children:"index construction"}),". This enhancement supports a range of indexes within\nthe ",(0,s.jsx)(n.a,{href:"metadata#metadata-table",children:"metadata table"}),", significantly improving the efficiency of both writing to and reading from the table."]}),"\n",(0,s.jsxs)(n.p,{children:[(0,s.jsx)(n.img,{alt:"Indexes",src:i(4390).A+"",width:"1124",height:"639"}),"\n",(0,s.jsx)("p",{align:"center",children:"Figure: Indexes in Hudi"})]}),"\n",(0,s.jsx)(n.h3,{id:"bloom-filters",children:"Bloom Filters"}),"\n",(0,s.jsxs)(n.p,{children:[(0,s.jsx)(n.a,{href:"https://github.com/apache/hudi/blob/46f41d186c6c84a6af2c54a907ff2736b6013e15/rfc/rfc-37/rfc-37.md",children:"Bloom filter"})," indexes as ",(0,s.jsx)(n.em,{children:"bloom_filter"})," partition in the metadata table.\nThis index employs range-based pruning on the minimum and maximum values of the record keys and bloom-filter-based lookups to tag incoming records. For large tables, this\ninvolves reading the footers of all matching data files for bloom filters, which can be expensive in the case of random\nupdates across the entire dataset. This index stores bloom filters of all data files centrally to avoid scanning the\nfooters directly from all data files."]}),"\n",(0,s.jsx)(n.h3,{id:"record-indexes",children:"Record Indexes"}),"\n",(0,s.jsxs)(n.p,{children:[(0,s.jsx)(n.a,{href:"https://cwiki.apache.org/confluence/display/HUDI/RFC-08++Record+level+indexing+mechanisms+for+Hudi+datasets",children:"Record indexes"})," as ",(0,s.jsx)(n.em,{children:"record_index"})," partition in the metadata table.\nContains the mapping of the record key to location. Record index is a global index, enforcing key uniqueness across all partitions in the table. This index aids in locating records faster than\nother existing indexes and can provide a speedup orders of magnitude faster in large deployments where index lookup dominates write latencies. To accommodate very high scales, it utilizes hash-based\nsharding of the key space. Additionally, when it comes to reading data, the index allows for point lookups significantly speeding up index mapping retrieval process."]}),"\n",(0,s.jsx)(n.h3,{id:"expression-index",children:"Expression Index"}),"\n",(0,s.jsxs)(n.p,{children:["An ",(0,s.jsx)(n.a,{href:"https://github.com/apache/hudi/blob/3789840be3d041cbcfc6b24786740210e4e6d6ac/rfc/rfc-63/rfc-63.md",children:"expression index"})," is an index on a function of a column. If a query has a predicate on a function of a column, the expression index can\nbe used to speed up the query. Expression index is stored in ",(0,s.jsx)(n.em,{children:"expr_index_"})," prefixed partitions (one for each\nexpression index) under metadata table. Expression index can be created using SQL syntax. Please checkout SQL DDL\ndocs ",(0,s.jsx)(n.a,{href:"sql_ddl#create-expression-index",children:"here"})," for more details."]}),"\n",(0,s.jsx)(n.h3,{id:"secondary-index",children:"Secondary Index"}),"\n",(0,s.jsxs)(n.p,{children:["Secondary indexes allow users to create indexes on columns that are not part of record key columns in Hudi tables (for\nrecord key fields, Hudi supports ",(0,s.jsx)(n.a,{href:"/blog/2023/11/01/record-level-index",children:"Record-level Index"}),". Secondary indexes\ncan be used to speed up queries with predicate on columns other than record key columns."]}),"\n",(0,s.jsx)(n.p,{children:"Following are configurations that control enabling index building and maintenance on the writer."}),"\n",(0,s.jsxs)(n.table,{children:[(0,s.jsx)(n.thead,{children:(0,s.jsxs)(n.tr,{children:[(0,s.jsx)(n.th,{children:"Config Name"}),(0,s.jsx)(n.th,{children:"Default"}),(0,s.jsx)(n.th,{children:"Description"})]})}),(0,s.jsxs)(n.tbody,{children:[(0,s.jsxs)(n.tr,{children:[(0,s.jsx)(n.td,{children:"hoodie.metadata.index.bloom.filter.enable"}),(0,s.jsx)(n.td,{children:"false (Optional)"}),(0,s.jsxs)(n.td,{children:["Enable indexing bloom filters of user data files under metadata table. When enabled, metadata table will have a partition to store the bloom filter index and will be used during the index lookups.",(0,s.jsx)("br",{}),(0,s.jsx)("br",{}),(0,s.jsx)(n.code,{children:"Config Param: ENABLE_METADATA_INDEX_BLOOM_FILTER"}),(0,s.jsx)("br",{}),(0,s.jsx)(n.code,{children:"Since Version: 0.11.0"})]})]}),(0,s.jsxs)(n.tr,{children:[(0,s.jsx)(n.td,{children:"hoodie.metadata.record.index.enable"}),(0,s.jsx)(n.td,{children:"false (Optional)"}),(0,s.jsxs)(n.td,{children:["Create the record Index within the metadata table",(0,s.jsx)("br",{}),(0,s.jsx)("br",{}),(0,s.jsx)(n.code,{children:"Config Param: RECORD_INDEX_ENABLE_PROP"}),(0,s.jsx)("br",{}),(0,s.jsx)(n.code,{children:"Since Version: 0.14.0"}),". This is a pre-requisite for secondary indexes or expression indexes on them."]})]})]})]}),"\n",(0,s.jsx)(n.h2,{id:"additional-writer-side-indexes",children:"Additional writer-side indexes"}),"\n",(0,s.jsx)(n.p,{children:"All the indexes discussed above are available both readers/writers using integration with metadata table. There are also indexing mechanisms\nimplemented by the storage engine, by efficiently reading/joining/processing incoming input records against information stored in base/log\nfiles themselves (e.g. bloom filters stored in parquet file footers) or intelligent data layout (e.g. bucket index)."}),"\n",(0,s.jsxs)(n.p,{children:["Currently, Hudi supports the following index types. Default is SIMPLE on Spark engine, and INMEMORY on Flink and Java\nengines. Writers can pick one of these options using ",(0,s.jsx)(n.code,{children:"hoodie.index.type"})," config option."]}),"\n",(0,s.jsxs)(n.ul,{children:["\n",(0,s.jsxs)(n.li,{children:["\n",(0,s.jsxs)(n.p,{children:[(0,s.jsx)(n.strong,{children:"SIMPLE (default for Spark engines)"}),": This is the standard index type for the Spark engine. It executes an efficient join of incoming records with keys retrieved from the table stored on disk. It requires keys to be partition-level unique so it can function correctly."]}),"\n"]}),"\n",(0,s.jsxs)(n.li,{children:["\n",(0,s.jsxs)(n.p,{children:[(0,s.jsx)(n.strong,{children:"RECORD_INDEX"})," : Use the record index from section above as the writer side index."]}),"\n"]}),"\n",(0,s.jsxs)(n.li,{children:["\n",(0,s.jsxs)(n.p,{children:[(0,s.jsx)(n.strong,{children:"BLOOM"}),": Uses bloom filters generated from record keys, with the option to further narrow down candidate files based on the ranges of the record keys. It requires keys to be partition-level unique so it can function correctly."]}),"\n"]}),"\n",(0,s.jsxs)(n.li,{children:["\n",(0,s.jsxs)(n.p,{children:[(0,s.jsx)(n.strong,{children:"GLOBAL_BLOOM"}),": Utilizes bloom filters created from record keys, and may also refine the selection of candidate files by using the ranges of record keys. It requires keys to be table/global-level unique so it can function correctly."]}),"\n"]}),"\n",(0,s.jsxs)(n.li,{children:["\n",(0,s.jsxs)(n.p,{children:[(0,s.jsx)(n.strong,{children:"GLOBAL_SIMPLE"}),": Performs a lean join of the incoming records against keys extracted from the table on storage. It requires keys to be table/global-level unique so it can function correctly."]}),"\n"]}),"\n",(0,s.jsxs)(n.li,{children:["\n",(0,s.jsxs)(n.p,{children:[(0,s.jsx)(n.strong,{children:"HBASE"}),": Mangages the index mapping through an external table in Apache HBase."]}),"\n"]}),"\n",(0,s.jsxs)(n.li,{children:["\n",(0,s.jsxs)(n.p,{children:[(0,s.jsx)(n.strong,{children:"INMEMORY (default for Flink and Java)"}),": Uses in-memory hashmap in Spark and Java engine and Flink in-memory state in Flink for indexing."]}),"\n"]}),"\n",(0,s.jsxs)(n.li,{children:["\n",(0,s.jsxs)(n.p,{children:[(0,s.jsx)(n.strong,{children:"BUCKET"}),": Utilizes bucket hashing to identify the file group that houses the records, which proves to be particularly advantageous on a large scale. To select the type of bucket engine\u2014that is, the method by which buckets are created\u2014use the ",(0,s.jsx)(n.code,{children:"hoodie.index.bucket.engine"})," configuration option."]}),"\n",(0,s.jsxs)(n.ul,{children:["\n",(0,s.jsxs)(n.li,{children:["\n",(0,s.jsxs)(n.p,{children:[(0,s.jsx)(n.strong,{children:"SIMPLE(default)"}),": This index employs a fixed number of buckets for file groups within each partition, which do not have the capacity to decrease or increase in size. It is applicable to both COW and MOR tables. Due to the unchangeable number of buckets and the design principle of mapping each bucket to a single file group, this indexing method may not be ideal for partitions with significant data skew."]}),"\n"]}),"\n",(0,s.jsxs)(n.li,{children:["\n",(0,s.jsxs)(n.p,{children:[(0,s.jsx)(n.strong,{children:"CONSISTENT_HASHING"}),": This index accommodates a dynamic number of buckets, with the capability for bucket resizing to ensure each bucket is sized appropriately. This addresses the issue of data skew in partitions with a high volume of data by allowing these partitions to be dynamically resized. As a result, partitions can have multiple reasonably sized buckets, unlike the fixed bucket count per partition seen in the SIMPLE bucket engine type. This feature is exclusively compatible with MOR tables."]}),"\n"]}),"\n"]}),"\n"]}),"\n",(0,s.jsxs)(n.li,{children:["\n",(0,s.jsxs)(n.p,{children:[(0,s.jsx)(n.strong,{children:"Bring your own implementation:"})," You can extend this ",(0,s.jsx)(n.a,{href:"https://github.com/apache/hudi/blob/master/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/index/HoodieIndex.java",children:"public API"}),"\nand supply a subclass of ",(0,s.jsx)(n.code,{children:"SparkHoodieIndex"})," (for Apache Spark writers) using ",(0,s.jsx)(n.code,{children:"hoodie.index.class"})," to implement custom indexing."]}),"\n"]}),"\n"]}),"\n",(0,s.jsx)(n.h3,{id:"global-and-non-global-indexes",children:"Global and Non-Global Indexes"}),"\n",(0,s.jsxs)(n.p,{children:["Another key aspect worth understanding is the difference between global and non-global indexes. Both bloom and simple index have\nglobal options - ",(0,s.jsx)(n.code,{children:"hoodie.index.type=GLOBAL_BLOOM"})," and ",(0,s.jsx)(n.code,{children:"hoodie.index.type=GLOBAL_SIMPLE"})," - respectively. Record index and\nHBase index are by nature a global index."]}),"\n",(0,s.jsxs)(n.ul,{children:["\n",(0,s.jsxs)(n.li,{children:[(0,s.jsx)(n.strong,{children:"Global index:"})," Global indexes enforce uniqueness of keys across all partitions of a table i.e guarantees that exactly\none record exists in the table for a given record key. Global indexes offer stronger guarantees, but the update/delete\ncost can still grow with size of the table ",(0,s.jsx)(n.code,{children:"O(size of table)"}),", since the record could belong to any partition in storage.\nIn case of non-global index, lookup involves file groups only for the matching partitions from the incoming records and\nso its not impacted by the total size of the table. These global indexes(GLOBAL_SIMPLE or GLOBAL_BLOOM), might be\nacceptable for decent sized tables, but for large tables, a newly added index (0.14.0) called Record Level Index (RLI),\ncan offer pretty good index lookup performance compared to other global indexes(GLOBAL_SIMPLE or GLOBAL_BLOOM) or\nHbase and also avoids the operational overhead of maintaining external systems."]}),"\n",(0,s.jsxs)(n.li,{children:[(0,s.jsx)(n.strong,{children:"Non Global index:"})," On the other hand, the default index implementations enforce this constraint only within a specific partition.\nAs one might imagine, non global indexes depends on the writer to provide the same consistent partition path for a given record key during update/delete,\nbut can deliver much better performance since the index lookup operation becomes ",(0,s.jsx)(n.code,{children:"O(number of records updated/deleted)"})," and\nscales well with write volume."]}),"\n"]}),"\n",(0,s.jsx)(n.h3,{id:"configs",children:"Configs"}),"\n",(0,s.jsx)(n.h4,{id:"spark-based-configs",children:"Spark based configs"}),"\n",(0,s.jsxs)(n.p,{children:["For Spark DataSource, Spark SQL, DeltaStreamer and Structured Streaming following are the key configs that control\nindexing behavior. Please refer to ",(0,s.jsx)(n.a,{href:"https://hudi.apache.org/docs/next/configurations#Common-Index-Configs-advanced-configs",children:"Advanced Configs"}),"\nfor more details. All these, support the index types mentioned ",(0,s.jsx)(n.a,{href:"#index-types-in-hudi",children:"above"}),"."]}),"\n",(0,s.jsxs)(n.table,{children:[(0,s.jsx)(n.thead,{children:(0,s.jsxs)(n.tr,{children:[(0,s.jsx)(n.th,{children:"Config Name"}),(0,s.jsx)(n.th,{children:"Default"}),(0,s.jsx)(n.th,{children:"Description"})]})}),(0,s.jsxs)(n.tbody,{children:[(0,s.jsxs)(n.tr,{children:[(0,s.jsx)(n.td,{children:"hoodie.index.type"}),(0,s.jsxs)(n.td,{children:["N/A ",(0,s.jsx)(n.strong,{children:"(Required)"})]}),(0,s.jsxs)(n.td,{children:["org.apache.hudi.index.HoodieIndex$IndexType: Determines how input records are indexed, i.e., looked up based on the key for the location in the existing table. Default is SIMPLE on Spark engine, and INMEMORY on Flink and Java engines. Possible Values: ",(0,s.jsx)("br",{})," ",(0,s.jsxs)("ul",{children:[(0,s.jsx)("li",{children:"BLOOM"}),(0,s.jsx)("li",{children:"GLOBAL_BLOOM"}),(0,s.jsx)("li",{children:"SIMPLE"}),(0,s.jsx)("li",{children:"GLOBAL_SIMPLE"}),(0,s.jsx)("li",{children:"HBASE"}),(0,s.jsx)("li",{children:"INMEMORY"}),(0,s.jsx)("li",{children:"FLINK_STATE"}),(0,s.jsx)("li",{children:"BUCKET"}),(0,s.jsx)("li",{children:"RECORD_INDEX"})]}),(0,s.jsx)("br",{}),(0,s.jsx)(n.code,{children:"Config Param: INDEX_TYPE"})]})]}),(0,s.jsxs)(n.tr,{children:[(0,s.jsx)(n.td,{children:"hoodie.index.bucket.engine"}),(0,s.jsx)(n.td,{children:"SIMPLE (Optional)"}),(0,s.jsxs)(n.td,{children:["org.apache.hudi.index.HoodieIndex$BucketIndexEngineType: Determines the type of bucketing or hashing to use when ",(0,s.jsx)(n.code,{children:"hoodie.index.type"})," is set to ",(0,s.jsx)(n.code,{children:"BUCKET"}),". Possible Values: ",(0,s.jsx)("br",{})," ",(0,s.jsxs)("ul",{children:[(0,s.jsx)("li",{children:"SIMPLE"}),(0,s.jsx)("li",{children:"CONSISTENT_HASHING"})]})," ",(0,s.jsx)("br",{}),(0,s.jsx)(n.code,{children:"Config Param: BUCKET_INDEX_ENGINE_TYPE"}),(0,s.jsx)("br",{}),(0,s.jsx)(n.code,{children:"Since Version: 0.11.0"})]})]}),(0,s.jsxs)(n.tr,{children:[(0,s.jsx)(n.td,{children:"hoodie.index.class"}),(0,s.jsx)(n.td,{children:"(Optional)"}),(0,s.jsxs)(n.td,{children:["Full path of user-defined index class and must be a subclass of HoodieIndex class. It will take precedence over the hoodie.index.type configuration if specified",(0,s.jsx)("br",{}),(0,s.jsx)("br",{}),(0,s.jsx)(n.code,{children:"Config Param: INDEX_CLASS_NAME"})]})]}),(0,s.jsxs)(n.tr,{children:[(0,s.jsx)(n.td,{children:"hoodie.bloom.index.update.partition.path"}),(0,s.jsx)(n.td,{children:"true (Optional)"}),(0,s.jsxs)(n.td,{children:["Only applies if index type is GLOBAL_BLOOM. When set to true, an update including the partition path of a record that already exists will result in inserting the incoming record into the new partition and deleting the original record in the old partition. When set to false, the original record will only be updated in the old partition, ignoring the new incoming partition if there is a mis-match between partition value for an incoming record with whats in storage.",(0,s.jsx)("br",{}),(0,s.jsx)("br",{}),(0,s.jsx)(n.code,{children:"Config Param: BLOOM_INDEX_UPDATE_PARTITION_PATH_ENABLE"})]})]}),(0,s.jsxs)(n.tr,{children:[(0,s.jsx)(n.td,{children:"hoodie.record.index.update.partition.path"}),(0,s.jsx)(n.td,{children:"false (Optional)"}),(0,s.jsxs)(n.td,{children:["Similar to Key: 'hoodie.bloom.index.update.partition.path' , Only applies if index type is RECORD_INDEX. When set to true, an update including the partition path of a record that already exists will result in inserting the incoming record into the new partition and deleting the original record in the old partition. When set to false, the original record will only be updated in the old partition, ignoring the new incoming partition if there is a mis-match between partition value for an incoming record with whats in storage. ",(0,s.jsx)("br",{}),(0,s.jsx)("br",{}),(0,s.jsx)(n.code,{children:"Config Param: RECORD_INDEX_UPDATE_PARTITION_PATH_ENABLE"})]})]}),(0,s.jsxs)(n.tr,{children:[(0,s.jsx)(n.td,{children:"hoodie.simple.index.update.partition.path"}),(0,s.jsx)(n.td,{children:"true (Optional)"}),(0,s.jsxs)(n.td,{children:["Similar to Key: 'hoodie.bloom.index.update.partition.path' , Only applies if index type is GLOBAL_SIMPLE. When set to true, an update including the partition path of a record that already exists will result in inserting the incoming record into the new partition and deleting the original record in the old partition. When set to false, the original record will only be updated in the old partition, ignoring the new incoming partition if there is a mis-match between partition value for an incoming record with whats in storage. ",(0,s.jsx)("br",{}),(0,s.jsx)("br",{}),(0,s.jsx)(n.code,{children:"Config Param: SIMPLE_INDEX_UPDATE_PARTITION_PATH_ENABLE"})]})]}),(0,s.jsxs)(n.tr,{children:[(0,s.jsx)(n.td,{children:"hoodie.hbase.index.update.partition.path"}),(0,s.jsx)(n.td,{children:"false (Optional)"}),(0,s.jsxs)(n.td,{children:["Only applies if index type is HBASE. When an already existing record is upserted to a new partition compared to whats in storage, this config when set, will delete old record in old partition and will insert it as new record in new partition.",(0,s.jsx)("br",{}),(0,s.jsx)("br",{}),(0,s.jsx)(n.code,{children:"Config Param: UPDATE_PARTITION_PATH_ENABLE"})]})]})]})]}),"\n",(0,s.jsx)(n.h4,{id:"flink-based-configs",children:"Flink based configs"}),"\n",(0,s.jsxs)(n.p,{children:["For Flink DataStream and Flink SQL only support Bucket Index and internal Flink state store backed in memory index.\nFollowing are the basic configs that control the indexing behavior. Please refer ",(0,s.jsx)(n.a,{href:"https://hudi.apache.org/docs/next/configurations#Flink-Options-advanced-configs",children:"here"}),"\nfor advanced configs."]}),"\n",(0,s.jsxs)(n.table,{children:[(0,s.jsx)(n.thead,{children:(0,s.jsxs)(n.tr,{children:[(0,s.jsx)(n.th,{children:"Config Name"}),(0,s.jsx)(n.th,{children:"Default"}),(0,s.jsx)(n.th,{children:"Description"})]})}),(0,s.jsxs)(n.tbody,{children:[(0,s.jsxs)(n.tr,{children:[(0,s.jsx)(n.td,{children:"index.type"}),(0,s.jsx)(n.td,{children:"FLINK_STATE (Optional)"}),(0,s.jsxs)(n.td,{children:["Index type of Flink write job, default is using state backed index. Possible values:",(0,s.jsx)("br",{})," ",(0,s.jsxs)("ul",{children:[(0,s.jsx)("li",{children:"FLINK_STATE"}),(0,s.jsx)("li",{children:"BUCKET"})]}),(0,s.jsx)("br",{})," ",(0,s.jsx)(n.code,{children:"Config Param: INDEX_TYPE"})]})]}),(0,s.jsxs)(n.tr,{children:[(0,s.jsx)(n.td,{children:"hoodie.index.bucket.engine"}),(0,s.jsx)(n.td,{children:"SIMPLE (Optional)"}),(0,s.jsxs)(n.td,{children:["org.apache.hudi.index.HoodieIndex$BucketIndexEngineType: Determines the type of bucketing or hashing to use when ",(0,s.jsx)(n.code,{children:"hoodie.index.type"})," is set to ",(0,s.jsx)(n.code,{children:"BUCKET"}),". Possible Values: ",(0,s.jsx)("br",{})," ",(0,s.jsxs)("ul",{children:[(0,s.jsx)("li",{children:"SIMPLE"}),(0,s.jsx)("li",{children:"CONSISTENT_HASHING"})]})]})]})]})]}),"\n",(0,s.jsx)(n.h3,{id:"picking-indexing-strategies",children:"Picking Indexing Strategies"}),"\n",(0,s.jsx)(n.p,{children:"Since data comes in at different volumes, velocity and has different access patterns, different indexes could be used for different workload types.\nLet\u2019s walk through some typical workload types and see how to leverage the right Hudi index for such use-cases.\nThis is based on our experience and you should diligently decide if the same strategies are best for your workloads."}),"\n",(0,s.jsx)(n.h4,{id:"workload-1-late-arriving-updates-to-fact-tables",children:"Workload 1: Late arriving updates to fact tables"}),"\n",(0,s.jsx)(n.p,{children:"Many companies store large volumes of transactional data in NoSQL data stores. For eg, trip tables in case of ride-sharing, buying and selling of shares,\norders in an e-commerce site. These tables are usually ever growing with random updates on most recent data with long tail updates going to older data, either\ndue to transactions settling at a later date/data corrections. In other words, most updates go into the latest partitions with few updates going to older ones."}),"\n",(0,s.jsxs)(n.p,{children:[(0,s.jsx)(n.img,{alt:"Fact table",src:i(50042).A+"",width:"3191",height:"1327"}),"\n",(0,s.jsx)("p",{align:"center",children:"Figure: Typical update pattern for Fact tables"})]}),"\n",(0,s.jsxs)(n.p,{children:["For such workloads, the ",(0,s.jsx)(n.code,{children:"BLOOM"})," index performs well, since index look-up will prune a lot of data files based on a well-sized bloom filter.\nAdditionally, if the keys can be constructed such that they have a certain ordering, the number of files to be compared is further reduced by range pruning.\nHudi constructs an interval tree with all the file key ranges and efficiently filters out the files that don't match any key ranges in the updates/deleted records."]}),"\n",(0,s.jsxs)(n.p,{children:["In order to efficiently compare incoming record keys against bloom filters i.e with minimal number of bloom filter reads and uniform distribution of work across\nthe executors, Hudi leverages caching of input records and employs a custom partitioner that can iron out data skews using statistics. At times, if the bloom filter\nfalse positive ratio is high, it could increase the amount of data shuffled to perform the lookup. Hudi supports dynamic bloom filters\n(enabled using ",(0,s.jsx)(n.code,{children:"hoodie.bloom.index.filter.type=DYNAMIC_V0"}),"), which adjusts its size based on the number of records stored in a given file to deliver the\nconfigured false positive ratio."]}),"\n",(0,s.jsx)(n.h4,{id:"workload-2-de-duplication-in-event-tables",children:"Workload 2: De-Duplication in event tables"}),"\n",(0,s.jsx)(n.p,{children:'Event Streaming is everywhere. Events coming from Apache Kafka or similar message bus are typically 10-100x the size of fact tables and often treat "time" (event\'s arrival time/processing\ntime) as a first class citizen. For eg, IoT event stream, click stream data, ad impressions etc. Inserts and updates only span the last few partitions as these are mostly append only data.\nGiven duplicate events can be introduced anywhere in the end-end pipeline, de-duplication before storing on the data lake is a common requirement.'}),"\n",(0,s.jsxs)(n.p,{children:[(0,s.jsx)(n.img,{alt:"Event table",src:i(39506).A+"",width:"6218",height:"2609"}),"\n",(0,s.jsx)("p",{align:"center",children:"Figure showing the spread of updates for Event table."})]}),"\n",(0,s.jsxs)(n.p,{children:["In general, this is a very challenging problem to solve at lower cost. Although, we could even employ a key value store to perform this de-duplication with HBASE index, the index storage\ncosts would grow linear with number of events and thus can be prohibitively expensive. In fact, ",(0,s.jsx)(n.code,{children:"BLOOM"})," index with range pruning is the optimal solution here. One can leverage the fact\nthat time is often a first class citizen and construct a key such as ",(0,s.jsx)(n.code,{children:"event_ts + event_id"})," such that the inserted records have monotonically increasing keys. This yields great returns\nby pruning large amounts of files even within the latest table partitions."]}),"\n",(0,s.jsx)(n.h4,{id:"workload-3-random-updatesdeletes-to-a-dimension-table",children:"Workload 3: Random updates/deletes to a dimension table"}),"\n",(0,s.jsx)(n.p,{children:"These types of tables usually contain high dimensional data and hold reference data e.g user profile, merchant information. These are high fidelity tables where the updates are often small but also spread\nacross a lot of partitions and data files ranging across the dataset from old to new. Often times, these tables are also un-partitioned, since there is also not a good way to partition these tables."}),"\n",(0,s.jsxs)(n.p,{children:[(0,s.jsx)(n.img,{alt:"Dimensions table",src:i(17429).A+"",width:"13365",height:"5602"}),"\n",(0,s.jsx)("p",{align:"center",children:"Figure showing the spread of updates for Dimensions table."})]}),"\n",(0,s.jsxs)(n.p,{children:["As discussed before, the ",(0,s.jsx)(n.code,{children:"BLOOM"})," index may not yield benefits if a good number of files cannot be pruned out by comparing ranges/filters. In such a random write workload, updates end up touching\nmost files within in the table and thus bloom filters will typically indicate a true positive for all files based on some incoming update. Consequently, we would end up comparing ranges/filter, only\nto finally check the incoming updates against all files. The ",(0,s.jsx)(n.code,{children:"SIMPLE"})," Index will be a better fit as it does not do any upfront pruning based, but directly joins with interested fields from every data file.\n",(0,s.jsx)(n.code,{children:"HBASE"})," index can be employed, if the operational overhead is acceptable and would provide much better lookup times for these tables."]}),"\n",(0,s.jsxs)(n.p,{children:["When using a global index, users should also consider setting ",(0,s.jsx)(n.code,{children:"hoodie.bloom.index.update.partition.path=true"})," or ",(0,s.jsx)(n.code,{children:"hoodie.simple.index.update.partition.path=true"})," to deal with cases where the\npartition path value could change due to an update e.g users table partitioned by home city; user relocates to a different city. These tables are also excellent candidates for the Merge-On-Read table type."]}),"\n",(0,s.jsx)(n.h2,{id:"related-resources",children:"Related Resources"}),"\n",(0,s.jsx)("h3",{children:"Videos"}),"\n",(0,s.jsxs)(n.ul,{children:["\n",(0,s.jsx)(n.li,{children:(0,s.jsx)(n.a,{href:"https://youtu.be/XlRvMFJ7g9c",children:"Global Bloom Index: Remove duplicates & guarantee uniquness - Hudi Labs"})}),"\n",(0,s.jsx)(n.li,{children:(0,s.jsx)(n.a,{href:"https://www.onehouse.ai/blog/introducing-multi-modal-index-for-the-lakehouse-in-apache-hudi",children:"Multi-Modal Index for the Lakehouse in Apache Hudi"})}),"\n"]})]})}function h(e={}){const{wrapper:n}={...(0,a.R)(),...e.components};return n?(0,s.jsx)(n,{...e,children:(0,s.jsx)(c,{...e})}):c(e)}},17429:(e,n,i)=>{i.d(n,{A:()=>t});const t=i.p+"assets/images/dimension-c1a4d25a9b59f1ae577b2159336b2a4e.png"},39506:(e,n,i)=>{i.d(n,{A:()=>t});const t=i.p+"assets/images/event_bus-0066b1fff4c3b67ef966404738e53e59.png"},50042:(e,n,i)=>{i.d(n,{A:()=>t});const t=i.p+"assets/images/nosql-bc8be272a92982296f05780fb60394ff.png"},82301:(e,n,i)=>{i.d(n,{A:()=>t});const t=i.p+"assets/images/with_without_index-c0808363df23ac1aba63bc81a68b6c8c.png"},4390:(e,n,i)=>{i.d(n,{A:()=>t});const t=i.p+"assets/images/hudi-stack-indexes-589506d411b969d14a9087633253a391.png"},28453:(e,n,i)=>{i.d(n,{R:()=>o,x:()=>r});var t=i(96540);const s={},a=t.createContext(s);function o(e){const n=t.useContext(a);return t.useMemo((function(){return"function"==typeof e?e(n):{...n,...e}}),[n,e])}function r(e){let n;return n=e.disableParentContext?"function"==typeof e.components?e.components(s):e.components||s:o(e.components),t.createElement(a.Provider,{value:n},e.children)}}}]);
\ No newline at end of file
+"use strict";(self.webpackChunkhudi=self.webpackChunkhudi||[]).push([[17987],{26947:(e,n,i)=>{i.r(n),i.d(n,{assets:()=>d,contentTitle:()=>r,default:()=>h,frontMatter:()=>o,metadata:()=>t,toc:()=>l});const t=JSON.parse('{"id":"indexes","title":"Indexes","description":"In databases, indexes are auxiliary data structures maintained to quickly locate records needed, without reading unnecessary data","source":"@site/docs/indexes.md","sourceDirName":".","slug":"/indexes","permalink":"/docs/next/indexes","draft":false,"unlisted":false,"editUrl":"https://github.com/apache/hudi/tree/asf-site/website/docs/indexes.md","tags":[],"version":"current","frontMatter":{"title":"Indexes","toc":true,"toc_min_heading_level":2,"toc_max_heading_level":4},"sidebar":"docs","previous":{"title":"Table Metadata","permalink":"/docs/next/metadata"},"next":{"title":"Concurrency Control","permalink":"/docs/next/concurrency_control"}}');var s=i(74848),a=i(28453);const o={title:"Indexes",toc:!0,toc_min_heading_level:2,toc_max_heading_level:4},r=void 0,d={},l=[{value:"Mapping keys to file groups",id:"mapping-keys-to-file-groups",level:2},{value:"Need for indexing",id:"need-for-indexing",level:2},{value:"Multi-modal Indexing",id:"multi-modal-indexing",level:2},{value:"Bloom Filters",id:"bloom-filters",level:3},{value:"Record Indexes",id:"record-indexes",level:3},{value:"Expression Index",id:"expression-index",level:3},{value:"Secondary Index",id:"secondary-index",level:3},{value:"Additional writer-side indexes",id:"additional-writer-side-indexes",level:2},{value:"Global and Non-Global Indexes",id:"global-and-non-global-indexes",level:3},{value:"Configs",id:"configs",level:3},{value:"Spark based configs",id:"spark-based-configs",level:4},{value:"Flink based configs",id:"flink-based-configs",level:4},{value:"Picking Indexing Strategies",id:"picking-indexing-strategies",level:3},{value:"Workload 1: Late arriving updates to fact tables",id:"workload-1-late-arriving-updates-to-fact-tables",level:4},{value:"Workload 2: De-Duplication in event tables",id:"workload-2-de-duplication-in-event-tables",level:4},{value:"Workload 3: Random updates/deletes to a dimension table",id:"workload-3-random-updatesdeletes-to-a-dimension-table",level:4},{value:"Related Resources",id:"related-resources",level:2}];function c(e){const n={a:"a",code:"code",em:"em",h2:"h2",h3:"h3",h4:"h4",img:"img",li:"li",p:"p",strong:"strong",table:"table",tbody:"tbody",td:"td",th:"th",thead:"thead",tr:"tr",ul:"ul",...(0,a.R)(),...e.components};return(0,s.jsxs)(s.Fragment,{children:[(0,s.jsxs)(n.p,{children:["In databases, indexes are auxiliary data structures maintained to quickly locate records needed, without reading unnecessary data\nfrom storage. Given that Hudi\u2019s design has been heavily optimized for handling mutable change streams, with different\nwrite patterns, Hudi considers ",(0,s.jsx)(n.a,{href:"#indexing",children:"indexing"})," as an integral part of its design and has uniquely supported\n",(0,s.jsx)(n.a,{href:"https://hudi.apache.org/blog/2020/11/11/hudi-indexing-mechanisms/",children:"indexing capabilities"})," from its inception, to speed\nup writes on the ",(0,s.jsx)(n.a,{href:"https://hudi.apache.org/blog/2024/07/11/what-is-a-data-lakehouse/",children:"data lakehouse"}),", while still providing\ncolumnar query performance."]}),"\n",(0,s.jsx)(n.h2,{id:"mapping-keys-to-file-groups",children:"Mapping keys to file groups"}),"\n",(0,s.jsx)(n.p,{children:"The most foundational index mechanism in Hudi tracks a mapping from a given key (record key + optionally partition path) consistently to a file id. Other types of indexes like secondary indexes,\nbuild on this foundation. This mapping between record key and file group/file id rarely changes once the first version of a record has been written to a file group.\nOnly clustering or cross-partition updates that are implemented as deletes + inserts remap the record key to a different file group. Even then, a given record key is associated with exactly one\nfile group at any completed instant on the timeline."}),"\n",(0,s.jsx)(n.h2,{id:"need-for-indexing",children:"Need for indexing"}),"\n",(0,s.jsxs)(n.p,{children:["For ",(0,s.jsx)(n.a,{href:"table_types#copy-on-write-table",children:"Copy-On-Write tables"}),", indexing enables fast upsert/delete operations, by avoiding the need to join against the entire dataset to determine which files to rewrite.\nFor ",(0,s.jsx)(n.a,{href:"table_types#merge-on-read-table",children:"Merge-On-Read tables"}),", indexing allows Hudi to bound the amount of change records any given base file needs to be merged against. Specifically, a given base file needs to merged\nonly against updates for records that are part of that base file."]}),"\n",(0,s.jsxs)(n.p,{children:[(0,s.jsx)(n.img,{alt:"Fact table",src:i(82301).A+"",width:"3994",height:"1704"}),"\n",(0,s.jsx)("p",{align:"center",children:"Figure: Comparison of merge cost for updates (dark blue blocks) against base files (light blue blocks)"})]}),"\n",(0,s.jsx)(n.p,{children:"In contrast,"}),"\n",(0,s.jsxs)(n.ul,{children:["\n",(0,s.jsxs)(n.li,{children:["Designs without an indexing component (e.g: ",(0,s.jsx)(n.a,{href:"https://cwiki.apache.org/confluence/display/Hive/Hive+Transactions",children:"Apache Hive/Apache Iceberg"}),") end up having to merge all the base files against all incoming updates/delete records\n(10-100x more ",(0,s.jsx)(n.a,{href:"table_types#comparison",children:"read amplification"}),")."]}),"\n",(0,s.jsx)(n.li,{children:"Designs that implement heavily write-optimized OLTP data structures like LSM trees do not require an indexing component. But they perform poorly scan heavy workloads\nagainst cloud storage making them unsuitable for serving analytical queries."}),"\n"]}),"\n",(0,s.jsx)(n.p,{children:"Hudi shines by achieving both great write performance and read performance, at the extra storage costs of an index, which can however unlock a lot more, as we explore below."}),"\n",(0,s.jsx)(n.h2,{id:"multi-modal-indexing",children:"Multi-modal Indexing"}),"\n",(0,s.jsxs)(n.p,{children:[(0,s.jsx)(n.a,{href:"https://www.onehouse.ai/blog/introducing-multi-modal-index-for-the-lakehouse-in-apache-hudi",children:"Multi-modal indexing"}),",\nintroduced in ",(0,s.jsx)(n.a,{href:"https://hudi.apache.org/releases/release-0.11.0/#multi-modal-index",children:"0.11.0 Hudi release"}),",\nis a re-imagination of what a general purpose indexing subsystem should look like for the lake. Multi-modal indexing is\nimplemented by enhancing the metadata table with the flexibility to extend to new index types as new partitions,\nalong with an ",(0,s.jsx)(n.a,{href:"https://hudi.apache.org/docs/metadata_indexing/#setup-async-indexing",children:"asynchronous index"})," building"]}),"\n",(0,s.jsxs)(n.p,{children:["Hudi supports a multi-modal index by augmenting the metadata table with the capability to incorporate new types of indexes, complemented by an\nasynchronous mechanism for ",(0,s.jsx)(n.a,{href:"metadata_indexing",children:"index construction"}),". This enhancement supports a range of indexes within\nthe ",(0,s.jsx)(n.a,{href:"metadata#metadata-table",children:"metadata table"}),", significantly improving the efficiency of both writing to and reading from the table."]}),"\n",(0,s.jsxs)(n.p,{children:[(0,s.jsx)(n.img,{alt:"Indexes",src:i(4390).A+"",width:"1124",height:"639"}),"\n",(0,s.jsx)("p",{align:"center",children:"Figure: Indexes in Hudi"})]}),"\n",(0,s.jsx)(n.h3,{id:"bloom-filters",children:"Bloom Filters"}),"\n",(0,s.jsxs)(n.p,{children:[(0,s.jsx)(n.a,{href:"https://github.com/apache/hudi/blob/46f41d186c6c84a6af2c54a907ff2736b6013e15/rfc/rfc-37/rfc-37.md",children:"Bloom filter"})," indexes as ",(0,s.jsx)(n.em,{children:"bloom_filter"})," partition in the metadata table.\nThis index employs range-based pruning on the minimum and maximum values of the record keys and bloom-filter-based lookups to tag incoming records. For large tables, this\ninvolves reading the footers of all matching data files for bloom filters, which can be expensive in the case of random\nupdates across the entire dataset. This index stores bloom filters of all data files centrally to avoid scanning the\nfooters directly from all data files."]}),"\n",(0,s.jsx)(n.h3,{id:"record-indexes",children:"Record Indexes"}),"\n",(0,s.jsxs)(n.p,{children:[(0,s.jsx)(n.a,{href:"https://cwiki.apache.org/confluence/display/HUDI/RFC-08++Record+level+indexing+mechanisms+for+Hudi+datasets",children:"Record indexes"})," as ",(0,s.jsx)(n.em,{children:"record_index"})," partition in the metadata table.\nContains the mapping of the record key to location. Record index is a global index, enforcing key uniqueness across all partitions in the table. This index aids in locating records faster than\nother existing indexes and can provide a speedup orders of magnitude faster in large deployments where index lookup dominates write latencies. To accommodate very high scales, it utilizes hash-based\nsharding of the key space. Additionally, when it comes to reading data, the index allows for point lookups significantly speeding up index mapping retrieval process."]}),"\n",(0,s.jsx)(n.h3,{id:"expression-index",children:"Expression Index"}),"\n",(0,s.jsxs)(n.p,{children:["An ",(0,s.jsx)(n.a,{href:"https://github.com/apache/hudi/blob/3789840be3d041cbcfc6b24786740210e4e6d6ac/rfc/rfc-63/rfc-63.md",children:"expression index"})," is an index on a function of a column. If a query has a predicate on a function of a column, the expression index can\nbe used to speed up the query. Expression index is stored in ",(0,s.jsx)(n.em,{children:"expr_index_"})," prefixed partitions (one for each\nexpression index) under metadata table. Expression index can be created using SQL syntax. Please checkout SQL DDL\ndocs ",(0,s.jsx)(n.a,{href:"sql_ddl#create-expression-index",children:"here"})," for more details."]}),"\n",(0,s.jsx)(n.h3,{id:"secondary-index",children:"Secondary Index"}),"\n",(0,s.jsxs)(n.p,{children:["Secondary indexes allow users to create indexes on columns that are not part of record key columns in Hudi tables (for\nrecord key fields, Hudi supports ",(0,s.jsx)(n.a,{href:"/blog/2023/11/01/record-level-index",children:"Record-level Index"}),". Secondary indexes\ncan be used to speed up queries with predicate on columns other than record key columns."]}),"\n",(0,s.jsx)(n.p,{children:"Following are configurations that control enabling index building and maintenance on the writer."}),"\n",(0,s.jsxs)(n.table,{children:[(0,s.jsx)(n.thead,{children:(0,s.jsxs)(n.tr,{children:[(0,s.jsx)(n.th,{children:"Config Name"}),(0,s.jsx)(n.th,{children:"Default"}),(0,s.jsx)(n.th,{children:"Description"})]})}),(0,s.jsxs)(n.tbody,{children:[(0,s.jsxs)(n.tr,{children:[(0,s.jsx)(n.td,{children:"hoodie.metadata.index.bloom.filter.enable"}),(0,s.jsx)(n.td,{children:"false (Optional)"}),(0,s.jsxs)(n.td,{children:["Enable indexing bloom filters of user data files under metadata table. When enabled, metadata table will have a partition to store the bloom filter index and will be used during the index lookups.",(0,s.jsx)("br",{}),(0,s.jsx)("br",{}),(0,s.jsx)(n.code,{children:"Config Param: ENABLE_METADATA_INDEX_BLOOM_FILTER"}),(0,s.jsx)("br",{}),(0,s.jsx)(n.code,{children:"Since Version: 0.11.0"})]})]}),(0,s.jsxs)(n.tr,{children:[(0,s.jsx)(n.td,{children:"hoodie.metadata.record.index.enable"}),(0,s.jsx)(n.td,{children:"false (Optional)"}),(0,s.jsxs)(n.td,{children:["Create the record Index within the metadata table",(0,s.jsx)("br",{}),(0,s.jsx)("br",{}),(0,s.jsx)(n.code,{children:"Config Param: RECORD_INDEX_ENABLE_PROP"}),(0,s.jsx)("br",{}),(0,s.jsx)(n.code,{children:"Since Version: 0.14.0"}),". This is a pre-requisite for secondary indexes or expression indexes on them."]})]})]})]}),"\n",(0,s.jsx)(n.h2,{id:"additional-writer-side-indexes",children:"Additional writer-side indexes"}),"\n",(0,s.jsx)(n.p,{children:"All the indexes discussed above are available both readers/writers using integration with metadata table. There are also indexing mechanisms\nimplemented by the storage engine, by efficiently reading/joining/processing incoming input records against information stored in base/log\nfiles themselves (e.g. bloom filters stored in parquet file footers) or intelligent data layout (e.g. bucket index)."}),"\n",(0,s.jsxs)(n.p,{children:["Currently, Hudi supports the following index types. Default is SIMPLE on Spark engine, and INMEMORY on Flink and Java\nengines. Writers can pick one of these options using ",(0,s.jsx)(n.code,{children:"hoodie.index.type"})," config option."]}),"\n",(0,s.jsxs)(n.ul,{children:["\n",(0,s.jsxs)(n.li,{children:["\n",(0,s.jsxs)(n.p,{children:[(0,s.jsx)(n.strong,{children:"SIMPLE (default for Spark engines)"}),": This is the standard index type for the Spark engine. It executes an efficient join of incoming records with keys retrieved from the table stored on disk. It requires keys to be partition-level unique so it can function correctly."]}),"\n"]}),"\n",(0,s.jsxs)(n.li,{children:["\n",(0,s.jsxs)(n.p,{children:[(0,s.jsx)(n.strong,{children:"RECORD_INDEX"})," : Use the record index from section above as the writer side index."]}),"\n"]}),"\n",(0,s.jsxs)(n.li,{children:["\n",(0,s.jsxs)(n.p,{children:[(0,s.jsx)(n.strong,{children:"BLOOM"}),": Uses bloom filters generated from record keys, with the option to further narrow down candidate files based on the ranges of the record keys. It requires keys to be partition-level unique so it can function correctly."]}),"\n"]}),"\n",(0,s.jsxs)(n.li,{children:["\n",(0,s.jsxs)(n.p,{children:[(0,s.jsx)(n.strong,{children:"GLOBAL_BLOOM"}),": Utilizes bloom filters created from record keys, and may also refine the selection of candidate files by using the ranges of record keys. It requires keys to be table/global-level unique so it can function correctly."]}),"\n"]}),"\n",(0,s.jsxs)(n.li,{children:["\n",(0,s.jsxs)(n.p,{children:[(0,s.jsx)(n.strong,{children:"GLOBAL_SIMPLE"}),": Performs a lean join of the incoming records against keys extracted from the table on storage. It requires keys to be table/global-level unique so it can function correctly."]}),"\n"]}),"\n",(0,s.jsxs)(n.li,{children:["\n",(0,s.jsxs)(n.p,{children:[(0,s.jsx)(n.strong,{children:"HBASE"}),": Mangages the index mapping through an external table in Apache HBase."]}),"\n"]}),"\n",(0,s.jsxs)(n.li,{children:["\n",(0,s.jsxs)(n.p,{children:[(0,s.jsx)(n.strong,{children:"INMEMORY (default for Flink and Java)"}),": Uses in-memory hashmap in Spark and Java engine and Flink in-memory state in Flink for indexing."]}),"\n"]}),"\n",(0,s.jsxs)(n.li,{children:["\n",(0,s.jsxs)(n.p,{children:[(0,s.jsx)(n.strong,{children:"BUCKET"}),": Utilizes bucket hashing to identify the file group that houses the records, which proves to be particularly advantageous on a large scale. To select the type of bucket engine\u2014that is, the method by which buckets are created\u2014use the ",(0,s.jsx)(n.code,{children:"hoodie.index.bucket.engine"})," configuration option."]}),"\n",(0,s.jsxs)(n.ul,{children:["\n",(0,s.jsxs)(n.li,{children:["\n",(0,s.jsxs)(n.p,{children:[(0,s.jsx)(n.strong,{children:"SIMPLE(default)"}),": This index employs a fixed number of buckets for file groups within each partition, which do not have the capacity to decrease or increase in size. It is applicable to both COW and MOR tables. Due to the unchangeable number of buckets and the design principle of mapping each bucket to a single file group, this indexing method may not be ideal for partitions with significant data skew."]}),"\n"]}),"\n",(0,s.jsxs)(n.li,{children:["\n",(0,s.jsxs)(n.p,{children:[(0,s.jsx)(n.strong,{children:"CONSISTENT_HASHING"}),": This index accommodates a dynamic number of buckets, with the capability for bucket resizing to ensure each bucket is sized appropriately. This addresses the issue of data skew in partitions with a high volume of data by allowing these partitions to be dynamically resized. As a result, partitions can have multiple reasonably sized buckets, unlike the fixed bucket count per partition seen in the SIMPLE bucket engine type. This feature is exclusively compatible with MOR tables."]}),"\n"]}),"\n"]}),"\n"]}),"\n",(0,s.jsxs)(n.li,{children:["\n",(0,s.jsxs)(n.p,{children:[(0,s.jsx)(n.strong,{children:"Bring your own implementation:"})," You can extend this ",(0,s.jsx)(n.a,{href:"https://github.com/apache/hudi/blob/master/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/index/HoodieIndex.java",children:"public API"}),"\nand supply a subclass of ",(0,s.jsx)(n.code,{children:"SparkHoodieIndex"})," (for Apache Spark writers) using ",(0,s.jsx)(n.code,{children:"hoodie.index.class"})," to implement custom indexing."]}),"\n"]}),"\n"]}),"\n",(0,s.jsx)(n.h3,{id:"global-and-non-global-indexes",children:"Global and Non-Global Indexes"}),"\n",(0,s.jsxs)(n.p,{children:["Another key aspect worth understanding is the difference between global and non-global indexes. Both bloom and simple index have\nglobal options - ",(0,s.jsx)(n.code,{children:"hoodie.index.type=GLOBAL_BLOOM"})," and ",(0,s.jsx)(n.code,{children:"hoodie.index.type=GLOBAL_SIMPLE"})," - respectively. Record index and\nHBase index are by nature a global index."]}),"\n",(0,s.jsxs)(n.ul,{children:["\n",(0,s.jsxs)(n.li,{children:[(0,s.jsx)(n.strong,{children:"Global index:"})," Global indexes enforce uniqueness of keys across all partitions of a table i.e guarantees that exactly\none record exists in the table for a given record key. Global indexes offer stronger guarantees, but the update/delete\ncost can still grow with size of the table ",(0,s.jsx)(n.code,{children:"O(size of table)"}),", since the record could belong to any partition in storage.\nIn case of non-global index, lookup involves file groups only for the matching partitions from the incoming records and\nso its not impacted by the total size of the table. These global indexes(GLOBAL_SIMPLE or GLOBAL_BLOOM), might be\nacceptable for decent sized tables, but for large tables, a newly added index (0.14.0) called Record Level Index (RLI),\ncan offer pretty good index lookup performance compared to other global indexes(GLOBAL_SIMPLE or GLOBAL_BLOOM) or\nHbase and also avoids the operational overhead of maintaining external systems."]}),"\n",(0,s.jsxs)(n.li,{children:[(0,s.jsx)(n.strong,{children:"Non Global index:"})," On the other hand, the default index implementations enforce this constraint only within a specific partition.\nAs one might imagine, non global indexes depends on the writer to provide the same consistent partition path for a given record key during update/delete,\nbut can deliver much better performance since the index lookup operation becomes ",(0,s.jsx)(n.code,{children:"O(number of records updated/deleted)"})," and\nscales well with write volume."]}),"\n"]}),"\n",(0,s.jsx)(n.h3,{id:"configs",children:"Configs"}),"\n",(0,s.jsx)(n.h4,{id:"spark-based-configs",children:"Spark based configs"}),"\n",(0,s.jsxs)(n.p,{children:["For Spark DataSource, Spark SQL, DeltaStreamer and Structured Streaming following are the key configs that control\nindexing behavior. Please refer to ",(0,s.jsx)(n.a,{href:"https://hudi.apache.org/docs/next/configurations#Common-Index-Configs-advanced-configs",children:"Advanced Configs"}),"\nfor more details. All these, support the index types mentioned ",(0,s.jsx)(n.a,{href:"#index-types-in-hudi",children:"above"}),"."]}),"\n",(0,s.jsxs)(n.table,{children:[(0,s.jsx)(n.thead,{children:(0,s.jsxs)(n.tr,{children:[(0,s.jsx)(n.th,{children:"Config Name"}),(0,s.jsx)(n.th,{children:"Default"}),(0,s.jsx)(n.th,{children:"Description"})]})}),(0,s.jsxs)(n.tbody,{children:[(0,s.jsxs)(n.tr,{children:[(0,s.jsx)(n.td,{children:"hoodie.index.type"}),(0,s.jsxs)(n.td,{children:["N/A ",(0,s.jsx)(n.strong,{children:"(Required)"})]}),(0,s.jsxs)(n.td,{children:["org.apache.hudi.index.HoodieIndex$IndexType: Determines how input records are indexed, i.e., looked up based on the key for the location in the existing table. Default is SIMPLE on Spark engine, and INMEMORY on Flink and Java engines. Possible Values: ",(0,s.jsx)("br",{})," ",(0,s.jsxs)("ul",{children:[(0,s.jsx)("li",{children:"BLOOM"}),(0,s.jsx)("li",{children:"GLOBAL_BLOOM"}),(0,s.jsx)("li",{children:"SIMPLE"}),(0,s.jsx)("li",{children:"GLOBAL_SIMPLE"}),(0,s.jsx)("li",{children:"HBASE"}),(0,s.jsx)("li",{children:"INMEMORY"}),(0,s.jsx)("li",{children:"FLINK_STATE"}),(0,s.jsx)("li",{children:"BUCKET"}),(0,s.jsx)("li",{children:"RECORD_INDEX"})]}),(0,s.jsx)("br",{}),(0,s.jsx)(n.code,{children:"Config Param: INDEX_TYPE"})]})]}),(0,s.jsxs)(n.tr,{children:[(0,s.jsx)(n.td,{children:"hoodie.index.bucket.engine"}),(0,s.jsx)(n.td,{children:"SIMPLE (Optional)"}),(0,s.jsxs)(n.td,{children:["org.apache.hudi.index.HoodieIndex$BucketIndexEngineType: Determines the type of bucketing or hashing to use when ",(0,s.jsx)(n.code,{children:"hoodie.index.type"})," is set to ",(0,s.jsx)(n.code,{children:"BUCKET"}),". Possible Values: ",(0,s.jsx)("br",{})," ",(0,s.jsxs)("ul",{children:[(0,s.jsx)("li",{children:"SIMPLE"}),(0,s.jsx)("li",{children:"CONSISTENT_HASHING"})]})," ",(0,s.jsx)("br",{}),(0,s.jsx)(n.code,{children:"Config Param: BUCKET_INDEX_ENGINE_TYPE"}),(0,s.jsx)("br",{}),(0,s.jsx)(n.code,{children:"Since Version: 0.11.0"})]})]}),(0,s.jsxs)(n.tr,{children:[(0,s.jsx)(n.td,{children:"hoodie.index.class"}),(0,s.jsx)(n.td,{children:"(Optional)"}),(0,s.jsxs)(n.td,{children:["Full path of user-defined index class and must be a subclass of HoodieIndex class. It will take precedence over the hoodie.index.type configuration if specified",(0,s.jsx)("br",{}),(0,s.jsx)("br",{}),(0,s.jsx)(n.code,{children:"Config Param: INDEX_CLASS_NAME"})]})]}),(0,s.jsxs)(n.tr,{children:[(0,s.jsx)(n.td,{children:"hoodie.bloom.index.update.partition.path"}),(0,s.jsx)(n.td,{children:"true (Optional)"}),(0,s.jsxs)(n.td,{children:["Only applies if index type is GLOBAL_BLOOM. When set to true, an update including the partition path of a record that already exists will result in inserting the incoming record into the new partition and deleting the original record in the old partition. When set to false, the original record will only be updated in the old partition, ignoring the new incoming partition if there is a mis-match between partition value for an incoming record with whats in storage.",(0,s.jsx)("br",{}),(0,s.jsx)("br",{}),(0,s.jsx)(n.code,{children:"Config Param: BLOOM_INDEX_UPDATE_PARTITION_PATH_ENABLE"})]})]}),(0,s.jsxs)(n.tr,{children:[(0,s.jsx)(n.td,{children:"hoodie.record.index.update.partition.path"}),(0,s.jsx)(n.td,{children:"false (Optional)"}),(0,s.jsxs)(n.td,{children:["Similar to Key: 'hoodie.bloom.index.update.partition.path' , Only applies if index type is RECORD_INDEX. When set to true, an update including the partition path of a record that already exists will result in inserting the incoming record into the new partition and deleting the original record in the old partition. When set to false, the original record will only be updated in the old partition, ignoring the new incoming partition if there is a mis-match between partition value for an incoming record with whats in storage. ",(0,s.jsx)("br",{}),(0,s.jsx)("br",{}),(0,s.jsx)(n.code,{children:"Config Param: RECORD_INDEX_UPDATE_PARTITION_PATH_ENABLE"})]})]}),(0,s.jsxs)(n.tr,{children:[(0,s.jsx)(n.td,{children:"hoodie.simple.index.update.partition.path"}),(0,s.jsx)(n.td,{children:"true (Optional)"}),(0,s.jsxs)(n.td,{children:["Similar to Key: 'hoodie.bloom.index.update.partition.path' , Only applies if index type is GLOBAL_SIMPLE. When set to true, an update including the partition path of a record that already exists will result in inserting the incoming record into the new partition and deleting the original record in the old partition. When set to false, the original record will only be updated in the old partition, ignoring the new incoming partition if there is a mis-match between partition value for an incoming record with whats in storage. ",(0,s.jsx)("br",{}),(0,s.jsx)("br",{}),(0,s.jsx)(n.code,{children:"Config Param: SIMPLE_INDEX_UPDATE_PARTITION_PATH_ENABLE"})]})]}),(0,s.jsxs)(n.tr,{children:[(0,s.jsx)(n.td,{children:"hoodie.hbase.index.update.partition.path"}),(0,s.jsx)(n.td,{children:"false (Optional)"}),(0,s.jsxs)(n.td,{children:["Only applies if index type is HBASE. When an already existing record is upserted to a new partition compared to whats in storage, this config when set, will delete old record in old partition and will insert it as new record in new partition.",(0,s.jsx)("br",{}),(0,s.jsx)("br",{}),(0,s.jsx)(n.code,{children:"Config Param: UPDATE_PARTITION_PATH_ENABLE"})]})]})]})]}),"\n",(0,s.jsx)(n.h4,{id:"flink-based-configs",children:"Flink based configs"}),"\n",(0,s.jsxs)(n.p,{children:["For Flink DataStream and Flink SQL only support Bucket Index and internal Flink state store backed in memory index.\nFollowing are the basic configs that control the indexing behavior. Please refer ",(0,s.jsx)(n.a,{href:"https://hudi.apache.org/docs/next/configurations#Flink-Options-advanced-configs",children:"here"}),"\nfor advanced configs."]}),"\n",(0,s.jsxs)(n.table,{children:[(0,s.jsx)(n.thead,{children:(0,s.jsxs)(n.tr,{children:[(0,s.jsx)(n.th,{children:"Config Name"}),(0,s.jsx)(n.th,{children:"Default"}),(0,s.jsx)(n.th,{children:"Description"})]})}),(0,s.jsxs)(n.tbody,{children:[(0,s.jsxs)(n.tr,{children:[(0,s.jsx)(n.td,{children:"index.type"}),(0,s.jsx)(n.td,{children:"FLINK_STATE (Optional)"}),(0,s.jsxs)(n.td,{children:["Index type of Flink write job, default is using state backed index. Possible values:",(0,s.jsx)("br",{})," ",(0,s.jsxs)("ul",{children:[(0,s.jsx)("li",{children:"FLINK_STATE"}),(0,s.jsx)("li",{children:"BUCKET"})]}),(0,s.jsx)("br",{})," ",(0,s.jsx)(n.code,{children:"Config Param: INDEX_TYPE"})]})]}),(0,s.jsxs)(n.tr,{children:[(0,s.jsx)(n.td,{children:"hoodie.index.bucket.engine"}),(0,s.jsx)(n.td,{children:"SIMPLE (Optional)"}),(0,s.jsxs)(n.td,{children:["org.apache.hudi.index.HoodieIndex$BucketIndexEngineType: Determines the type of bucketing or hashing to use when ",(0,s.jsx)(n.code,{children:"hoodie.index.type"})," is set to ",(0,s.jsx)(n.code,{children:"BUCKET"}),". Possible Values: ",(0,s.jsx)("br",{})," ",(0,s.jsxs)("ul",{children:[(0,s.jsx)("li",{children:"SIMPLE"}),(0,s.jsx)("li",{children:"CONSISTENT_HASHING"})]})]})]})]})]}),"\n",(0,s.jsx)(n.h3,{id:"picking-indexing-strategies",children:"Picking Indexing Strategies"}),"\n",(0,s.jsx)(n.p,{children:"Since data comes in at different volumes, velocity and has different access patterns, different indexes could be used for different workload types.\nLet\u2019s walk through some typical workload types and see how to leverage the right Hudi index for such use-cases.\nThis is based on our experience and you should diligently decide if the same strategies are best for your workloads."}),"\n",(0,s.jsx)(n.h4,{id:"workload-1-late-arriving-updates-to-fact-tables",children:"Workload 1: Late arriving updates to fact tables"}),"\n",(0,s.jsx)(n.p,{children:"Many companies store large volumes of transactional data in NoSQL data stores. For eg, trip tables in case of ride-sharing, buying and selling of shares,\norders in an e-commerce site. These tables are usually ever growing with random updates on most recent data with long tail updates going to older data, either\ndue to transactions settling at a later date/data corrections. In other words, most updates go into the latest partitions with few updates going to older ones."}),"\n",(0,s.jsxs)(n.p,{children:[(0,s.jsx)(n.img,{alt:"Fact table",src:i(50042).A+"",width:"3191",height:"1327"}),"\n",(0,s.jsx)("p",{align:"center",children:"Figure: Typical update pattern for Fact tables"})]}),"\n",(0,s.jsxs)(n.p,{children:["For such workloads, the ",(0,s.jsx)(n.code,{children:"BLOOM"})," index performs well, since index look-up will prune a lot of data files based on a well-sized bloom filter.\nAdditionally, if the keys can be constructed such that they have a certain ordering, the number of files to be compared is further reduced by range pruning.\nHudi constructs an interval tree with all the file key ranges and efficiently filters out the files that don't match any key ranges in the updates/deleted records."]}),"\n",(0,s.jsxs)(n.p,{children:["In order to efficiently compare incoming record keys against bloom filters i.e with minimal number of bloom filter reads and uniform distribution of work across\nthe executors, Hudi leverages caching of input records and employs a custom partitioner that can iron out data skews using statistics. At times, if the bloom filter\nfalse positive ratio is high, it could increase the amount of data shuffled to perform the lookup. Hudi supports dynamic bloom filters\n(enabled using ",(0,s.jsx)(n.code,{children:"hoodie.bloom.index.filter.type=DYNAMIC_V0"}),"), which adjusts its size based on the number of records stored in a given file to deliver the\nconfigured false positive ratio."]}),"\n",(0,s.jsx)(n.h4,{id:"workload-2-de-duplication-in-event-tables",children:"Workload 2: De-Duplication in event tables"}),"\n",(0,s.jsx)(n.p,{children:'Event Streaming is everywhere. Events coming from Apache Kafka or similar message bus are typically 10-100x the size of fact tables and often treat "time" (event\'s arrival time/processing\ntime) as a first class citizen. For eg, IoT event stream, click stream data, ad impressions etc. Inserts and updates only span the last few partitions as these are mostly append only data.\nGiven duplicate events can be introduced anywhere in the end-end pipeline, de-duplication before storing on the data lake is a common requirement.'}),"\n",(0,s.jsxs)(n.p,{children:[(0,s.jsx)(n.img,{alt:"Event table",src:i(39506).A+"",width:"6218",height:"2609"}),"\n",(0,s.jsx)("p",{align:"center",children:"Figure showing the spread of updates for Event table."})]}),"\n",(0,s.jsxs)(n.p,{children:["In general, this is a very challenging problem to solve at lower cost. Although, we could even employ a key value store to perform this de-duplication with HBASE index, the index storage\ncosts would grow linear with number of events and thus can be prohibitively expensive. In fact, ",(0,s.jsx)(n.code,{children:"BLOOM"})," index with range pruning is the optimal solution here. One can leverage the fact\nthat time is often a first class citizen and construct a key such as ",(0,s.jsx)(n.code,{children:"event_ts + event_id"})," such that the inserted records have monotonically increasing keys. This yields great returns\nby pruning large amounts of files even within the latest table partitions."]}),"\n",(0,s.jsx)(n.h4,{id:"workload-3-random-updatesdeletes-to-a-dimension-table",children:"Workload 3: Random updates/deletes to a dimension table"}),"\n",(0,s.jsx)(n.p,{children:"These types of tables usually contain high dimensional data and hold reference data e.g user profile, merchant information. These are high fidelity tables where the updates are often small but also spread\nacross a lot of partitions and data files ranging across the dataset from old to new. Often times, these tables are also un-partitioned, since there is also not a good way to partition these tables."}),"\n",(0,s.jsxs)(n.p,{children:[(0,s.jsx)(n.img,{alt:"Dimensions table",src:i(17429).A+"",width:"13365",height:"5602"}),"\n",(0,s.jsx)("p",{align:"center",children:"Figure showing the spread of updates for Dimensions table."})]}),"\n",(0,s.jsxs)(n.p,{children:["As discussed before, the ",(0,s.jsx)(n.code,{children:"BLOOM"})," index may not yield benefits if a good number of files cannot be pruned out by comparing ranges/filters. In such a random write workload, updates end up touching\nmost files within in the table and thus bloom filters will typically indicate a true positive for all files based on some incoming update. Consequently, we would end up comparing ranges/filter, only\nto finally check the incoming updates against all files. The ",(0,s.jsx)(n.code,{children:"SIMPLE"})," Index will be a better fit as it does not do any upfront pruning based, but directly joins with interested fields from every data file.\n",(0,s.jsx)(n.code,{children:"HBASE"})," index can be employed, if the operational overhead is acceptable and would provide much better lookup times for these tables."]}),"\n",(0,s.jsxs)(n.p,{children:["When using a global index, users should also consider setting ",(0,s.jsx)(n.code,{children:"hoodie.bloom.index.update.partition.path=true"})," or ",(0,s.jsx)(n.code,{children:"hoodie.simple.index.update.partition.path=true"})," to deal with cases where the\npartition path value could change due to an update e.g users table partitioned by home city; user relocates to a different city. These tables are also excellent candidates for the Merge-On-Read table type."]}),"\n",(0,s.jsx)(n.h2,{id:"related-resources",children:"Related Resources"}),"\n",(0,s.jsx)("h3",{children:"Blogs"}),"\n",(0,s.jsxs)(n.ul,{children:["\n",(0,s.jsx)(n.li,{children:(0,s.jsx)(n.a,{href:"https://medium.com/@simpsons/global-vs-non-global-index-in-apache-hudi-ac880b031cbc",children:"Global vs Non-global index in Apache Hudi"})}),"\n"]}),"\n",(0,s.jsx)("h3",{children:"Videos"}),"\n",(0,s.jsxs)(n.ul,{children:["\n",(0,s.jsx)(n.li,{children:(0,s.jsx)(n.a,{href:"https://youtu.be/XlRvMFJ7g9c",children:"Global Bloom Index: Remove duplicates & guarantee uniquness - Hudi Labs"})}),"\n",(0,s.jsx)(n.li,{children:(0,s.jsx)(n.a,{href:"https://www.onehouse.ai/blog/introducing-multi-modal-index-for-the-lakehouse-in-apache-hudi",children:"Multi-Modal Index for the Lakehouse in Apache Hudi"})}),"\n"]})]})}function h(e={}){const{wrapper:n}={...(0,a.R)(),...e.components};return n?(0,s.jsx)(n,{...e,children:(0,s.jsx)(c,{...e})}):c(e)}},17429:(e,n,i)=>{i.d(n,{A:()=>t});const t=i.p+"assets/images/dimension-c1a4d25a9b59f1ae577b2159336b2a4e.png"},39506:(e,n,i)=>{i.d(n,{A:()=>t});const t=i.p+"assets/images/event_bus-0066b1fff4c3b67ef966404738e53e59.png"},50042:(e,n,i)=>{i.d(n,{A:()=>t});const t=i.p+"assets/images/nosql-bc8be272a92982296f05780fb60394ff.png"},82301:(e,n,i)=>{i.d(n,{A:()=>t});const t=i.p+"assets/images/with_without_index-c0808363df23ac1aba63bc81a68b6c8c.png"},4390:(e,n,i)=>{i.d(n,{A:()=>t});const t=i.p+"assets/images/hudi-stack-indexes-589506d411b969d14a9087633253a391.png"},28453:(e,n,i)=>{i.d(n,{R:()=>o,x:()=>r});var t=i(96540);const s={},a=t.createContext(s);function o(e){const n=t.useContext(a);return t.useMemo((function(){return"function"==typeof e?e(n):{...n,...e}}),[n,e])}function r(e){let n;return n=e.disableParentContext?"function"==typeof e.components?e.components(s):e.components||s:o(e.components),t.createElement(a.Provider,{value:n},e.children)}}}]);
\ No newline at end of file
diff --git a/content/assets/js/runtime~main.352dbe58.js b/content/assets/js/runtime~main.ba01cd22.js
similarity index 99%
rename from content/assets/js/runtime~main.352dbe58.js
rename to content/assets/js/runtime~main.ba01cd22.js
index 901d8df46c154..3b5a4f8e23e05 100644
--- a/content/assets/js/runtime~main.352dbe58.js
+++ b/content/assets/js/runtime~main.ba01cd22.js
@@ -1 +1 @@
-(()=>{"use strict";var e,a,f,c,b,d={},t={};function r(e){var a=t[e];if(void 0!==a)return a.exports;var f=t[e]={exports:{}};return d[e].call(f.exports,f,f.exports,r),f.exports}r.m=d,e=[],r.O=(a,f,c,b)=>{if(!f){var d=1/0;for(i=0;i=b)&&Object.keys(r.O).every((e=>r.O[e](f[o])))?f.splice(o--,1):(t=!1,b0&&e[i-1][2]>b;i--)e[i]=e[i-1];e[i]=[f,c,b]},r.n=e=>{var a=e&&e.__esModule?()=>e.default:()=>e;return r.d(a,{a:a}),a},f=Object.getPrototypeOf?e=>Object.getPrototypeOf(e):e=>e.__proto__,r.t=function(e,c){if(1&c&&(e=this(e)),8&c)return e;if("object"==typeof e&&e){if(4&c&&e.__esModule)return e;if(16&c&&"function"==typeof e.then)return e}var b=Object.create(null);r.r(b);var d={};a=a||[null,f({}),f([]),f(f)];for(var t=2&c&&e;"object"==typeof t&&!~a.indexOf(t);t=f(t))Object.getOwnPropertyNames(t).forEach((a=>d[a]=()=>e[a]));return d.default=()=>e,r.d(b,d),b},r.d=(e,a)=>{for(var f in a)r.o(a,f)&&!r.o(e,f)&&Object.defineProperty(e,f,{enumerable:!0,get:a[f]})},r.f={},r.e=e=>Promise.all(Object.keys(r.f).reduce(((a,f)=>(r.f[f](e,a),a)),[])),r.u=e=>"assets/js/"+({16:"0583dad8",25:"a33c645e",28:"0da1f28a",39:"a978e5ab",48:"a674d2f7",60:"e9bd6ef2",67:"56169fa6",106:"6c6aeb2d",129:"275b5c51",159:"af8190e8",217:"b1f77b90",278:"a1a7cea0",292:"fcc9c1b0",323:"be9655ff",335:"43d16b11",354:"0b4da2f0",387:"19ac452e",437:"a168e2e0",569:"b4a99b76",614:"0a87a91b",805:"5c109a5d",822:"79cc09db",851:"8b10f6da",878:"e9038c17",910:"ce3da75c",917:"02b5547e",924:"9699c87d",940:"0c459987",1103:"4b935b7f",1107:"060b2af9",1119:"e3b4090c",1136:"ae09e9a1",1190:"b0127f30",1251:"ea1c8f1c",1284:"62a07427",1337:"9cab4fb3",1360:"eb8e7dea",1422:"562c897e",1442:"50f3bb83",1506:"4b25608f",1517:"da868c82",1550:"8425e24c",1635:"73c8d421",1656:"cc5200ca",1658:"7a51e13d",1784:"89dd9b54",1785:"269ce614",1791:"4b2ed2d0",1841:"92bb01f4",1846:"9f349e0c",1857:"d032d8fc",1882:"c28fec2b",1928:"6e47a3c2",1946:"45b94beb",1955:"7fbadf7c",1983:"b93dc3c9",2016:"3c15683d",2053:"7a3be72c",2121:"fdb4abad",2172:"ef5805c5",2214:"32aa7863",2225:"9781112c",2239:"2b154460",2256:"31863783",2262:"7d9ec3a3",2298:"d2a01f74",2385:"fb4f9065",2450:"3b3f1ad3",2472:"1b7e3feb",2540:"e8b490f7",2577:"b9e56da2",2638:"372d2263",2640:"dba6e014",2652:"9ed88412",2663:"9d0312da",2679:"7e9327af",2700:"5e58268b",2739:"4c5bb0e3",2748:"7c87f31d",2759:"d28b7dcd",2768:"ad32f2e8",2772:"48f24bab",2791:"ba2305d7",2839:"00cd9d46",2854:"093ec6b1",2867:"c03096de",2884:"8539bc43",2912:"404a423b",2932:"580eda40",2954:"6e7eafb1",3075:"a80470ee",3125:"32dcc326",3131:"d29911ee",3137:"4f7cd6bb",3144:"aed1d196",3179:"5c3bdea9",3303:"6be904db",3313:"a3e90044",3323:"b9073401",3471:"1c4da86a",3558:"7fb3fac1",3693:"b783cafb",3694:"5e85aa31",3722:"50cd3db4",3740:"dc1f5b39",3764:"394bb252",3847:"43d31177",3940:"260d7fd0",3965:"d18b6106",3968:"50494a26",4014:"c943b655",4028:"04ae9608",4040:"b4ced939",4060:"0bf991f2",4074:"817ed3cf",4117:"d772531a",4133:"b18f8924",4137:"1a47b5e7",4205:"0c12eeea",4269:"18ffe98c",4355:"1497bce1",4368:"25fe5fdb",4382:"56471e5f",4405:"1dcf0194",4467:"ff5a6f64",4514:"3e8d408e",4530:"1cb4ff83",4574:"d9f219f0",4591:"405284b3",4602:"9920fffc",4777:"43d0cc21",4790:"3b6474f3",4863:"a0c4b68b",4890:"7c52b6f4",4919:"4929a6fa",4949:"066b815f",4980:"006c64a3",5008:"6880fe95",5028:"f5c507c6",5029:"ca8c3af7",5065:"25ed107d",5193:"d0ff2798",5195:"8c7f4a45",5206:"4b9df2bb",5269:"627b1f39",5270:"6ff0c558",5289:"b3a451d2",5295:"bbc3d059",5304:"44fefb3e",5322:"22b4f68c",5334:"86642e8b",5338:"0e11b6db",5594:"0c74166d",5622:"fb863cab",5631:"32ba6a4c",5633:"a804bb8e",5647:"d03e0408",5679:"58c4a37c",5721:"42bda306",5806:"1adb4ab6",5834:"6a34bd67",5887:"b5960565",5888:"478a7123",5936:"4959f63e",6022:"959a592e",6054:"87c914ef",6090:"384f1d9f",6158:"0d706529",6170:"14ed99c2",6203:"0ba05648",6225:"baed6a8a",6289:"59ba4d58",6310:"14a73a68",6350:"ce3c8465",6368:"207e0bd3",6371:"4658250f",6372:"5c865244",6407:"47ddb104",6467:"3faff22d",6475:"316bb886",6552:"ae59a273",6559:"21b16c5b",6636:"f6a1f09d",6677:"ea2cf37a",6692:"0cca5757",6697:"17ace3c3",6709:"debf2b54",6739:"01ceb854",6761:"507d925b",6778:"9cb5abfe",6791:"ae0384e2",6823:"4f684eef",6897:"b52407fb",6930:"ac385e9e",6944:"52f660a4",7035:"e841f655",7071:"6e78b29b",7100:"e7c12d1f",7103:"9d67d055",7125:"b744ccc8",7167:"bfd461a8",7317:"b1b9fe16",7355:"96f48bd9",7444:"502758cc",7458:"8fdc7f76",7517:"4abd5d51",7566:"12a626af",7622:"9df106ad",7629:"2a74f6a7",7715:"06a1e3b2",7734:"d6b46894",7747:"79a2ad2d",7773:"869d1599",7802:"deab0229",7850:"68c81474",7907:"7ba20019",7953:"df4b657d",8003:"f85e4dd8",8070:"0480b142",8139:"addb1e87",8174:"464278e7",8177:"38122bdb",8183:"c7a3e7d6",8192:"934057b1",8209:"01a85c17",8289:"c286af15",8348:"09ff3d76",8354:"db9fbbab",8363:"24179e00",8367:"abd23114",8407:"7a049829",8413:"cd17745a",8417:"bbced460",8585:"497095d8",8619:"15c2f51a",8632:"22351c69",8645:"212a125d",8651:"80bd773e",8699:"8f08e203",8762:"1cd42d14",8766:"7a11cfa3",8840:"e161c803",8888:"3a6a48c9",8889:"effd3c61",8900:"eebb3a9f",8902:"dc8afb47",8965:"b6e28713",8978:"6d5644d6",9003:"91415840",9045:"7c9ee56f",9066:"4c492fb9",9076:"52b6ceb8",9127:"29c12ff9",9152:"17666b14",9189:"d4eb85a9",9195:"b2d2727d",9228:"d336ca2e",9257:"0687df23",9262:"29a0dcae",9351:"4c14260d",9379:"7fd1d1a0",9380:"409e52b6",9385:"8ea09047",9428:"cfb1206c",9432:"8cd4ff44",9446:"49b92117",9458:"762ed4b3",9460:"01964c8d",9471:"ebf12a38",9499:"953153ea",9516:"2fec12c7",9525:"bf1307fc",9570:"8eea179e",9573:"13ab27aa",9631:"79e67a34",9647:"5e95c892",9650:"747973bc",9676:"e4d28161",9702:"a0d28f55",9722:"c35662b8",9724:"46e7f252",9737:"0d37dd31",9790:"6a0b8fcc",9836:"2fd2285d",9860:"d6d00b14",9870:"36e3724e",9891:"8a5cd0c4",9907:"9c0f572d",9914:"06b7dbb6",9944:"9ee88b57",9986:"c782ecef",10030:"c2277200",10118:"f89af623",10181:"f7519df0",10198:"ce5d6b55",10238:"0eb44ec6",10285:"b69fdc1c",10354:"8c29db87",10472:"0ca8e4c5",10488:"dc8e8e39",10492:"40011a75",10535:"79c374cf",10545:"1ca50c03",10610:"b1051fbd",10637:"6a326c1d",10642:"936cf648",10653:"5bf02344",10688:"f71afd42",10727:"f53cf472",10767:"931768b8",10789:"10fb8f29",10884:"9009d9a7",10887:"ac65ed35",10907:"b964b0d3",10920:"42746bb6",10924:"f0c78ca3",10956:"726b1d2e",10978:"90db14b9",11052:"9f1f18ec",11058:"7c44e50a",11111:"2750cc84",11156:"f97de939",11275:"2e961a80",11289:"82b971d3",11299:"ac338f84",11326:"f9cb0cea",11355:"6ed20719",11508:"039c7621",11565:"3d026c66",11631:"d2a270f8",11684:"9aee6bed",11716:"8fa68fae",11725:"51e360fd",11745:"446810fe",11768:"4cc89b71",11782:"3f65fb56",11795:"43ff550e",11804:"b9168984",11806:"7cb0ba19",11842:"f630dd79",11855:"51e592eb",11869:"43c57b15",11905:"41b6dd58",11906:"757b41cf",11977:"608a85b6",12047:"d25544dd",12134:"088c57bd",12144:"5617941b",12186:"3deb4153",12316:"77a5f3f8",12318:"9b28f794",12378:"2592f25d",12393:"7d05d2dd",12395:"a0fb693a",12431:"61c30610",12452:"f4a568da",12477:"ac9a039d",12485:"3e8ec4bd",12513:"70570476",12535:"2ca30cc7",12542:"0bce5bfd",12608:"ae632d28",12613:"bb6acec0",12654:"98af85a1",12679:"755aa830",12686:"f81e2128",12763:"0166538a",12869:"236efcfc",12945:"bc561320",12995:"72ed5e19",13021:"9b078b3d",13060:"075d0b63",13063:"6695bce8",13152:"4d01f822",13203:"a5fd3751",13210:"326f3c2b",13236:"814487fc",13298:"be65306b",13321:"bb230ab4",13366:"c7690f1a",13374:"0c9b8a46",13375:"15be7f5e",13406:"cdd81d53",13419:"05f125a2",13465:"f7887fd4",13511:"3f9b0969",13514:"35f2b245",13522:"559e18a5",13525:"aeb9cb2b",13534:"a221d960",13535:"27aec4ea",13575:"1b3749bd",13580:"b7ffc82e",13626:"16444e78",13698:"e32c93f6",13733:"e4126631",13762:"35ace877",13803:"5005f1cd",13849:"fdf3b0ba",13918:"a586ac12",13943:"79c522c5",13982:"f4dd7c05",13996:"bb5bef2a",14003:"fe0a5e13",14027:"4fe2812d",14042:"6c6eb236",14043:"2da40935",14125:"c58d5881",14134:"88677a17",14136:"5ba7f3a0",14152:"0c792305",14163:"79f5ed7e",14179:"0466f8a1",14257:"973a6c2e",14259:"4d62164b",14261:"881e0310",14264:"15b03510",14317:"2d2f3863",14340:"02704d8d",14363:"33732613",14388:"4330e2c5",14399:"95a29cbe",14410:"262f048a",14433:"5a1ac6cd",14435:"a2218d2d",14496:"0dc66419",14536:"b812e3a7",14599:"04287605",14627:"6b0d4b59",14726:"cb290368",14750:"8049dc2a",14802:"4193b389",14822:"da9a6aef",14835:"3145e86d",14868:"9c9e6d14",14871:"281df6d1",14917:"b2f79aaf",14919:"68aea991",14943:"b634826e",15038:"e8bf575a",15078:"6773b5f7",15079:"461201e5",15144:"e6691bac",15165:"29fcaeb8",15186:"809a1b46",15220:"e0719818",15235:"246d116d",15272:"ee5dc595",15292:"7bd6096e",15316:"a0583d0f",15403:"bbbd3731",15407:"413b2e2c",15427:"63067f40",15473:"ad07d5ed",15501:"259868d4",15552:"1ff337bc",15607:"a309ba1a",15764:"0d11ee2a",15805:"65df1ffd",15857:"520bf372",15917:"04d2875d",15947:"f05cb9af",15955:"1ea67680",16017:"777c5b61",16032:"7f5a2db2",16100:"5685cb00",16209:"f0db470e",16231:"d05ef008",16319:"1839a246",16377:"8eca1d9c",16499:"58c0e7a1",16500:"b5eb2b1f",16622:"3d8e248c",16641:"4ccf094c",16747:"ab84da8d",16885:"7c3644b1",16942:"426a3c0f",16950:"bd7a3684",17009:"a3848d24",17033:"30e67995",17115:"3687929f",17119:"87557f51",17232:"bc36781c",17294:"82fc01c6",17332:"b4762a45",17368:"de821e02",17402:"b629aa71",17411:"496a5e3a",17441:"c5c8072f",17474:"f7750a15",17502:"dddd9631",17508:"80e60ab1",17530:"8c212f85",17581:"46e89fee",17626:"47f3a505",17681:"a2af438a",17709:"89514438",17739:"b1e22259",17805:"bc4a985b",17818:"2d9024ae",17860:"7cfc22c1",17867:"9130943e",17944:"36f96221",17987:"f67021ea",17988:"a3255c75",18006:"059d2658",18033:"c1fa975f",18103:"7707ed5e",18121:"3a2db09e",18146:"c15d9823",18350:"889bcd45",18351:"670426f7",18401:"17896441",18438:"7eea383c",18445:"762cb2d4",18520:"905fdac9",18620:"1bdcf2af",18645:"e798c07e",18738:"3b375bfc",18757:"10b6d210",18798:"6c76cc4e",18940:"6cc2587c",18950:"d75d009b",18998:"7487c019",19042:"378587af",19052:"6e547e58",19070:"6f878c05",19108:"6f3d983d",19109:"868e6cc1",19136:"9c3fccaa",19237:"e77e0240",19252:"d9884433",19267:"1354c4fd",19271:"345a8726",19369:"a2cfe627",19387:"9228ff94",19396:"c61ed569",19425:"efd9e5ca",19428:"4b6ad6c1",19474:"a3c850e8",19513:"3faa4dcb",19521:"7f5a4e86",19542:"3548485f",19664:"76b52ed8",19687:"547af68c",19711:"0e4a3360",19736:"9e46f997",19774:"3c6c92ad",19798:"83c71362",19801:"a5052667",19865:"7e53000f",19891:"389b06c3",19999:"99bce14a",20038:"0ef00d46",20103:"430ee71a",20160:"ecc82a52",20194:"69fce7d3",20229:"5aae4c30",20286:"f60a8e85",20325:"6b7b009f",20358:"a55cf427",20361:"784d2d5b",20405:"a170c4b2",20415:"839320c8",20424:"c97a4f9f",20434:"39c0c867",20491:"4c056f0f",20535:"ca16b0fe",20541:"2925f13d",20619:"eae39fd7",20676:"874c4928",20713:"ff781369",20812:"a827f20e",20857:"bd440596",20892:"1d99c39c",20924:"92328698",20931:"de69fdc1",20951:"5ef30e9e",21070:"e3b4fd8a",21077:"7ff678ac",21121:"c1b7702d",21177:"574aa417",21191:"53798754",21273:"58f98693",21275:"e737da10",21290:"fab95ca2",21303:"e4bea80c",21316:"266d37b9",21341:"b23dd436",21346:"b2c62a3d",21373:"50737734",21401:"5105c354",21417:"9db9c0e0",21448:"995d9cd2",21482:"5f404bc7",21524:"69f9ca18",21560:"6c888f35",21574:"263bc6da",21585:"d17a9646",21630:"aad02470",21635:"6224d640",21662:"b1722c22",21729:"ea9d899a",21764:"df2a910d",21805:"1926d3d7",21816:"fe2d6fd6",21837:"7ce47929",21874:"a54a9a31",21909:"d3de5519",21921:"a8a7f86a",22141:"625beb64",22313:"334d7764",22369:"83a16324",22385:"9a5e9795",22428:"47425d5b",22447:"f93d9ca9",22472:"494588b9",22483:"95d688dd",22499:"2923e8f3",22501:"ff15e2eb",22504:"61773850",22534:"6f37f43d",22595:"ca5d6ba8",22628:"a36f1f19",22640:"69375b61",22647:"c760fa6a",22673:"c699b29d",22709:"74c5eee6",22750:"f701b058",22816:"70db0e3d",22844:"d6ededeb",22911:"9a29e675",22916:"d49fdba3",22932:"0d5f3865",22973:"70a32eac",23002:"1fd06df8",23041:"07da43c6",23081:"ef1c4348",23083:"7050405a",23119:"39307ee6",23141:"a1c39b3f",23172:"b994507e",23332:"fe86f3f9",23388:"fa5f4abf",23438:"95d97af4",23475:"fdeab78a",23547:"a67ef186",23563:"f24f712d",23591:"53746e73",23600:"af9b00f1",23625:"996abb7f",23634:"59e9dc06",23637:"9c150758",23641:"35313843",23666:"cb16b650",23676:"d235b83f",23700:"ea8366f8",23765:"fac8c34b",23773:"2153fb85",23880:"3affb2e4",23938:"be354741",23998:"afe4349e",24022:"9c7497bb",24065:"5ef1cef9",24084:"4ed60a8a",24098:"89ad6a53",24126:"897856d4",24140:"5ce7213e",24172:"1d2c1565",24234:"3af3061a",24265:"7ebf6f1e",24340:"a4e075be",24366:"99d3870c",24373:"5cbd92da",24377:"b5b376a0",24381:"4066930f",24437:"77b23935",24475:"045bd6f5",24476:"6cf93e89",24704:"3befffe2",24728:"91c64349",24729:"931beacc",24741:"e2c09daf",24776:"a852ca5b",24815:"a68f650d",24826:"ed6ec893",24853:"7cfd7c3a",24866:"e0c51b98",24867:"b4972e92",24868:"c6301cab",24891:"f0709bee",24912:"8e01a36e",24919:"cf6129a8",24932:"51fff4d3",24970:"231cac3a",25013:"1e3d98df",25044:"9cd11b72",25067:"9531441e",25101:"8bd2257d",25130:"b277530c",25186:"96046d19",25273:"19531bab",25285:"1da16fcf",25335:"537936fe",25396:"d78b8cbc",25397:"4083b748",25426:"b973fe72",25452:"5ef8e026",25531:"af5c68f9",25532:"40ebd41e",25539:"7fb09d55",25578:"c7382291",25582:"65603bfa",25601:"7205cbcf",25778:"d56bdeaf",25781:"ba115cad",25845:"d2c3f17d",25869:"6d1cdd35",25871:"c9953bfc",25874:"17268e53",25959:"f7f19049",25987:"1645b95d",25993:"92db1d5a",26003:"90c5833e",26009:"20f9256f",26060:"7662a79c",26064:"2316ae30",26081:"de97f7b0",26183:"fd6c914a",26216:"4e3e1e17",26260:"477cc2f7",26305:"878dce9b",26333:"9579dcb9",26337:"65009e94",26414:"14fd0d1c",26548:"9a5789a7",26577:"8eca70a5",26627:"8353278b",26630:"ba839983",26632:"960d3eb8",26722:"1802ae9e",26724:"78bc6f25",26744:"21e535c3",26797:"4681e75f",26833:"e2fe7a08",26859:"04fb6378",26882:"958a296c",26900:"b3749361",27012:"aca5eee4",27020:"58714c61",27088:"1547c10f",27115:"e37692b4",27123:"c4d1e829",27179:"cbec0238",27212:"ce710151",27369:"f52b34fb",27388:"f7e205e5",27398:"a3983124",27444:"7a3be7e3",27473:"31ed8f07",27663:"9f8f3c52",27737:"6d701be3",27823:"f95764fc",27887:"b0d5c266",27921:"10ccea9f",27955:"2e7e1134",27995:"ee4b8f19",28020:"ecd026d3",28086:"2cc67ca6",28127:"859500dc",28147:"abde445c",28166:"8120957a",28169:"513b54fb",28178:"9adef974",28204:"758b9bc9",28235:"20a81622",28252:"b8d3e47d",28319:"ec410c33",28410:"89cbd1b8",28428:"b212a261",28435:"5b374304",28499:"261fe657",28627:"54ad050e",28679:"0871002b",28687:"ab48a39f",28701:"c3856c66",28782:"fd5c5192",28793:"379ac6de",28803:"8b6fbbb4",28834:"7ed2bc06",28853:"4c00d6b9",28889:"2a0b5233",28920:"e7ef6e16",28941:"8c5b1685",28945:"f382fc88",28993:"8e7128cd",29060:"9c7a2f87",29123:"763fa02a",29171:"6f3c73f2",29204:"d4836e14",29243:"d330b699",29337:"8a472aba",29408:"fbce6636",29414:"b92135a6",29421:"163a1bae",29452:"13171783",29457:"e4d4ec4e",29498:"da05f515",29523:"b2f58160",29526:"b45a54e9",29653:"13f72d6c",29679:"6e0a413f",29690:"b1633424",29698:"a642dcef",29708:"80b93830",29733:"da10e58f",29795:"e6557239",29820:"16381f20",29829:"d973177d",29845:"5d94dea0",29891:"26f5f607",29922:"5397b577",29950:"5de1c39a",29969:"332b14e0",30035:"87546ad3",30066:"6b836ac4",30154:"339988c0",30167:"22bf7f69",30262:"5320ed26",30265:"132be4b3",30317:"af953e30",30355:"4b678e99",30378:"5c5d3a2d",30396:"e4a43002",30410:"90ffbe81",30564:"407230da",30569:"c7165ecb",30579:"1ee0a129",30583:"775c5b43",30628:"6ee9522e",30654:"5d70257a",30790:"a7c3ebee",30827:"5814834c",30831:"ced554c7",30842:"d8412727",30848:"677757ca",30934:"44c28e7f",31004:"7dea1595",31016:"416fb909",31039:"71d5b1f8",31047:"3f9a993f",31125:"c3c8ddc4",31141:"be9442ac",31144:"fbf25af7",31220:"b8ab477f",31255:"5c81ce12",31257:"67f1ea1e",31272:"fa366c46",31393:"07f6d4d1",31465:"cc564cf4",31541:"1668cad9",31544:"c602d51d",31567:"8c4bf82a",31680:"b38306ed",31705:"05957343",31720:"0a70c05a",31726:"624fba40",31749:"974bc2fa",31791:"04eab06f",31797:"14a26e2a",31861:"a14af959",31862:"c791a767",31869:"2dcd9099",31957:"2e31e40f",31972:"4ef864ee",31996:"8dbbcff6",32005:"f0dc0726",32012:"9ba9284f",32039:"116b7e55",32183:"5110a7b7",32225:"eb05a324",32270:"53ba3577",32278:"149a2d9e",32335:"f9a4941d",32365:"fe345ee5",32391:"e1407f5b",32472:"c08ba4cb",32527:"e6c2d44b",32558:"40c3731f",32691:"798484e3",32728:"1b520177",32824:"943ce597",32825:"fdddbf66",32847:"8590fd3e",32875:"65511420",32949:"43764587",32960:"225cc45a",32993:"604cd7c5",33049:"97e51b1f",33065:"a172f6b1",33192:"ee003a92",33214:"2a422024",33234:"465994bd",33290:"7da68ccc",33303:"f5ff1a23",33350:"2d0c5b52",33458:"c16e0158",33517:"3ed70074",33524:"f81c13fc",33533:"65ea2434",33540:"fe7ac0b7",33559:"8f00bf20",33567:"6075be34",33572:"269d704a",33577:"df68b7b1",33612:"3f933ca5",33621:"33241236",33638:"d294bdb0",33643:"da7149d8",33659:"81a6b262",33724:"755872dc",33804:"1b31aebb",33806:"07399e71",33816:"45f2496e",33825:"f8a443f2",33854:"06758c2b",33881:"a7690409",33889:"e9c7c2b7",33999:"49aa1a16",34127:"18b5779a",34156:"f5c63bac",34179:"3f4396e6",34182:"80f4f5b6",34196:"13e6af21",34255:"e8a8dea2",34331:"164e61b9",34335:"6153f283",34388:"79fbf53e",34520:"538371ec",34528:"b038a3ec",34543:"782eae4d",34549:"0735e5be",34597:"40cd51eb",34621:"f30fab6f",34638:"852f3aa7",34662:"ef071c32",34691:"02e54e09",34706:"47a1bde9",34750:"a5fea1bd",34803:"e076fc92",34808:"2e6ef254",34810:"924dcf98",34837:"10fcd509",34847:"dc7f8f8b",34877:"f7702246",34904:"f44e1bcd",34918:"c9cac9d9",34970:"40dc6b06",35044:"5e2f1fff",35067:"47cb0ede",35088:"35788c78",35139:"6fe5390f",35144:"83829ea7",35179:"372475fb",35248:"2947aa63",35288:"0f3b9f0c",35325:"d42065ad",35329:"a7827f50",35379:"d9454955",35402:"976ab233",35532:"388d5845",35618:"97bb73fe",35730:"2b4cfa56",35733:"8fc868c2",35742:"aba21aa0",35766:"ff6f6072",35767:"3019fa66",35784:"d240c26d",35837:"dd61deac",35881:"8852ad6f",35922:"92928c02",35932:"a3142532",35960:"dfa4e294",35983:"b5a599e0",36009:"04afe417",36055:"59be0e8d",36067:"ceff0062",36113:"28f38ec5",36144:"307d712e",36192:"b60e28b6",36264:"8a759f03",36302:"eafd7a74",36379:"467ff4a0",36387:"e23d1069",36549:"5ea64d6c",36569:"1472dfa3",36594:"94a5efbc",36619:"2573355f",36640:"1084a32f",36672:"b6fb003b",36689:"77787075",36747:"a3015275",36790:"12935692",36851:"649faa0e",36885:"1007513a",37109:"d73e6b67",37115:"0b4fcfba",37147:"97da4333",37191:"d786420c",37204:"8f3b27f1",37206:"43dcd746",37209:"e34cac0e",37257:"37524cd6",37329:"554882a9",37366:"6013ac2e",37403:"32b44a8d",37486:"ce46a18e",37489:"6a293d29",37577:"e9705ef5",37578:"e17d733a",37617:"be2ede3f",37643:"a6aa9e1f",37673:"e2657c7c",37740:"2cfb265a",37799:"feab7dcb",37809:"5f87fd99",37827:"42545929",37839:"d5e29b01",37841:"9ad03801",37842:"cd060f40",37872:"88d0aec4",37954:"3f272b07",38001:"690cbdd8",38003:"8d086d51",38056:"baad6bf9",38107:"1efbb938",38158:"66372f10",38238:"2e9f383c",38348:"bbc1ba35",38370:"cf858c26",38385:"7c15a7a1",38416:"dcd7017c",38439:"9474fc0c",38449:"91335d19",38451:"f0fb7623",38468:"b4a84943",38600:"0f497157",38605:"6dc06ee1",38607:"7eaa135a",38700:"1bf4b05b",38767:"c4de5678",38786:"872a41d4",38816:"5ea125b2",38835:"6495713a",38839:"110e78c3",38888:"6f5eeb66",38974:"2c50796a",38999:"daa4f13d",39058:"53317765",39088:"5560f355",39110:"cb9e1f3d",39138:"755ebe0d",39142:"b5f01515",39153:"0b8a4e62",39192:"09b62bf8",39197:"484cbf74",39202:"8b795044",39208:"93f7a5ff",39230:"b992801d",39240:"90f157dc",39260:"47074836",39265:"b40642d4",39283:"b1043e58",39405:"cc6505da",39407:"995840bc",39417:"085b8128",39423:"a615dd65",39444:"0f65ccfb",39487:"9d833fad",39494:"b326207c",39496:"7b2a260d",39505:"782d926e",39518:"630bda65",39520:"441dded5",39555:"4a80e27a",39569:"06a8eab3",39587:"7d0e1375",39608:"97ae1fff",39712:"8f64cb7a",39715:"a68f0e64",39758:"07bef82a",39773:"fb3759c3",39777:"320da59d",39781:"3c2d0811",39840:"8fa753de",39887:"9b0c8626",39901:"d76c455f",39908:"09a52b17",39921:"6471fe03",39973:"d4a2e930",40065:"5f469a3b",40124:"bc817d28",40178:"0508dae2",40205:"edf7dc61",40245:"ecaa52af",40369:"3c36ce76",40463:"0cd1a20c",40464:"4d1df301",40555:"3415fffa",40606:"3e032765",40616:"12a5de9b",40651:"bf5126e1",40690:"dfc3c224",40697:"cc01a9d7",40755:"8393a516",40758:"1a7b33c3",40797:"d2110cc0",40827:"ddebc8bf",40847:"2662e92d",40907:"321dafc4",40919:"2884dc3d",40927:"881bebdd",40959:"ca0149b2",40999:"68c5632a",41030:"33d4a0d4",41125:"0fa5c29b",41135:"85ffab62",41159:"93340492",41260:"6c9200cf",41361:"247909f7",41455:"2bf5edd4",41461:"fd517388",41491:"c3f88c36",41504:"8cd118b0",41518:"bf9d9be8",41594:"da172daa",41640:"d407c2c6",41651:"e8e3bd30",41677:"9017a110",41739:"b9edf71b",41750:"bc38a1dd",41852:"29db9f25",41867:"78f0e259",41872:"58b3152e",41892:"dc5506b6",41907:"b7a56be4",41944:"14828316",41954:"7c49023e",42068:"c413ce11",42187:"295cb0c0",42196:"671726ea",42268:"89c8b6fe",42298:"3fa17fc2",42299:"1f979aba",42350:"6df57399",42368:"ffac4af0",42465:"b8212f90",42487:"203cc654",42518:"34825c6d",42566:"2b505997",42576:"b184a577",42596:"eafac744",42690:"165a93e5",42702:"28826a4b",42723:"d8ed3ccd",42794:"89f1622a",42799:"5314580d",42801:"bf3d61ec",42879:"aeca0a21",42881:"432e670c",42900:"c9d95cbd",42905:"145364e0",42911:"39c7a203",42920:"ebbd9274",42937:"a8c92c97",42942:"19028add",42995:"2812e3b9",43029:"e3477e52",43035:"2907d5eb",43040:"676c7c8e",43042:"bec2896b",43078:"50536743",43088:"341293cf",43113:"12164231",43195:"d831e3b2",43286:"fe9b6676",43352:"4c07ba53",43421:"f048922c",43432:"fbf82623",43445:"83fec35c",43461:"7b89671b",43512:"d319e9f2",43642:"4cd0ba1b",43677:"7117a8cd",43739:"752351f9",43868:"c09831a6",43922:"a8b66660",43980:"509aac2d",44008:"5a8b22a0",44022:"454b7ded",44040:"43d1bad2",44045:"894f5c0e",44118:"6503f568",44123:"e2fc22cf",44125:"ca2b0f7e",44129:"d653d7ed",44143:"19acc4ed",44290:"4f087d20",44326:"cfca4f26",44339:"0a91021f",44380:"8fd374dc",44405:"d363a80c",44419:"216bf65d",44489:"6fb01322",44492:"9882af90",44551:"9b2ae5ff",44567:"502906a9",44640:"20bfa334",44662:"d65b2c25",44682:"bdc69003",44719:"60165173",44840:"27077527",44876:"260982a7",44891:"30769210",44933:"f3d02b29",44939:"43f3c18b",44948:"c7a67184",44960:"12b957b7",44964:"8914fdfe",44994:"41af2803",44997:"6e34e766",45113:"55946737",45117:"86fe1f77",45146:"b528d4d0",45210:"49a37e79",45231:"7737fe61",45238:"c3f790da",45290:"db90ec2b",45323:"815e47ce",45339:"24d4b044",45460:"459dbf85",45500:"e85bde03",45541:"a722b43a",45561:"7dce0476",45640:"ef7891f5",45670:"c36346e3",45682:"3533dbd1",45919:"a2a2954f",45983:"9926d60d",45988:"95aba771",46001:"8cf74eb7",46008:"2e048b0f",46025:"1fb7523b",46039:"f724037a",46063:"67893f6a",46074:"f1087137",46156:"53cdeba4",46197:"8e4df3f0",46244:"ea511817",46296:"04b49851",46375:"e49bd305",46425:"00239e8e",46451:"4bb02a47",46472:"bcb734aa",46513:"9dfe5a97",46515:"c13f246c",46547:"c5427124",46578:"8012465a",46598:"8393d26f",46652:"5f0630da",46662:"4f48cd24",46754:"2b6a7ab0",46802:"8f07070c",46840:"b2e2853b",46850:"917a523c",46888:"db9d1243",46891:"85f8bce5",46905:"57a58e78",46911:"ff13b684",46917:"7b0a3d41",46921:"a28b89d0",46937:"37ffd066",46979:"a9afee85",47041:"bbb92300",47060:"26996ad6",47087:"a5c8a2aa",47106:"696d889c",47110:"1366dd9b",47127:"c648308f",47206:"4d2e69a6",47236:"131c6f6e",47247:"5048202c",47248:"c7e2144d",47297:"6111afa9",47433:"b9a767d8",47517:"09e5dcaa",47537:"df0967b2",47555:"0029660c",47622:"0c1ff092",47668:"b1899f7e",47693:"39a07549",47699:"17093d00",47730:"4edf53e5",47917:"6bb76d2c",48003:"7ad7688e",48023:"47f96c4a",48050:"ca713325",48063:"c17f9567",48071:"852ff8c6",48091:"0cc98403",48130:"f81c1134",48132:"c467a11b",48162:"519b5759",48167:"00021e3e",48188:"9d8965aa",48214:"bef59fc9",48285:"e672d61a",48295:"e2b4c006",48304:"189e3ac3",48310:"f6884a75",48355:"d8c836b4",48360:"959b45a2",48361:"712c16a5",48532:"2896ce7a",48544:"0a5f31be",48592:"c1506482",48613:"7a556257",48661:"ef4f3413",48673:"5db19d16",48733:"d009d47f",48768:"e054d919",48793:"b71be52f",48803:"79392e36",48842:"d0b9fc21",48883:"ce18dbde",49013:"9d9f8394",49046:"c68848f4",49055:"be94b749",49071:"97d17d75",49112:"3455c0d5",49174:"0ac5df82",49199:"7795e379",49226:"f2a06fea",49280:"6111bc4c",49305:"ad0a2b75",49350:"2417a912",49388:"8f1510f6",49461:"2019135a",49521:"49d63cc6",49569:"0d819826",49579:"e01bb6c7",49583:"8ec8c5c5",49587:"2cbb324b",49655:"b0ba51ed",49676:"2391d372",49783:"acf72eb8",49786:"5bd6f6db",49880:"3fc6d2b7",49991:"1d5cbb7b",50012:"40c88f7a",50095:"837c6843",50176:"756c7559",50178:"18dddf22",50183:"f2790563",50259:"ea981684",50261:"ec6a0daf",50293:"fc6a53b6",50361:"09a0b8d6",50414:"c59a4667",50428:"a33de290",50446:"eaebafad",50472:"a4e24d6c",50486:"313c653e",50507:"129c4e7a",50570:"c112d1b7",50584:"3fd21eb6",50642:"e964b083",50700:"75be343d",50703:"0914ee26",50747:"0d98f85b",50769:"a339c33e",50800:"b365f6bc",50819:"4251704e",50827:"aa04bdb6",50839:"8aa0dce2",50934:"e7b82dc0",51047:"4dd6f8d8",51049:"6bd6fe03",51060:"e1758f93",51090:"12aaf145",51102:"3304ac90",51116:"d366555a",51170:"5e85b509",51175:"c39359c5",51184:"90d97cfa",51229:"a06af3ed",51237:"85a6e3f4",51378:"0a8d92af",51412:"fe2deb8c",51420:"7a04b795",51438:"14515c80",51494:"65eb0274",51548:"45b02367",51578:"8447ad38",51584:"4c2a4e19",51589:"f762fff5",51636:"ba5671ab",51649:"6537c712",51665:"65d842b9",51694:"612cc46d",51711:"89bdbd96",51755:"b50c8022",51763:"9199f8bd",51769:"1bd7f091",51814:"91a39dd0",51828:"44e51e65",51837:"5c238f81",51878:"48b6c251",51890:"b5fb4df8",51937:"34689118",52036:"1d79e793",52042:"7f8ebea7",52093:"edea4b8a",52102:"95cc61bd",52132:"3498be82",52135:"d0ed28b7",52176:"81d19844",52207:"508f03f4",52218:"1562cf35",52235:"fadc68df",52250:"8ec5e058",52266:"3e088827",52267:"6818c2e9",52282:"be224150",52286:"8e1e17e5",52342:"0dd8c0ac",52408:"ab586159",52450:"2dc793da",52462:"3016c86b",52466:"8c67a0ff",52478:"cda1618e",52488:"ca57223f",52563:"f2da277e",52569:"7199ad43",52572:"a9417ee3",52612:"27f2e2a4",52634:"c4f5d8e4",52680:"2124517a",52711:"9e4087bc",52731:"c8f57d29",52770:"85afc7f5",52785:"25eae11f",52826:"ce319efa",52840:"9e1bed9d",52882:"fa2ae802",52895:"69fcecaa",52911:"4d4179b3",52912:"82c60778",52913:"4adafdbf",53074:"2f32a261",53132:"5de85215",53170:"aaf3ff15",53178:"71514a42",53201:"d670063b",53237:"79ab3378",53256:"fa713049",53266:"76a0389c",53288:"f8fe23f1",53314:"5440b68b",53449:"e2bce80a",53458:"63436e22",53492:"a45dd8f5",53540:"92f1a745",53619:"16640689",53629:"5a2d67ad",53633:"f6ef9721",53646:"6c9978fa",53670:"52ed38a1",53742:"77612fce",53771:"3a447539",53786:"e352cccc",53832:"7c8deb08",53860:"e623e42b",53861:"2a5e97be",53871:"94dd9534",53881:"e9a95c5e",53901:"a4da17e3",53925:"a8933372",53982:"d5f056f5",53990:"f72b3fe4",54097:"8c317219",54106:"93b4d825",54200:"f97d7d88",54240:"3a55b748",54246:"5ba9c4b5",54270:"e5c0ea66",54272:"f020ef51",54274:"b7bc328b",54277:"71e2c4b4",54399:"25aa47d2",54415:"c87c5e1b",54462:"0c3d0366",54487:"52676037",54500:"fb9e14c7",54540:"66193a96",54568:"633af835",54730:"d4522125",54749:"ff4fb41e",54759:"0f7d8652",54772:"13883961",54851:"a2de6851",54879:"a872c629",54881:"c46bba44",54927:"55f8cc28",54950:"0a893fdf",54951:"98bd463b",54992:"5bcffa9a",55004:"37d125cb",55100:"70d55848",55144:"152d0582",55167:"c1cb0a0b",55206:"aaa8a12d",55276:"c962ae4a",55302:"facc4cc2",55342:"ee44120f",55401:"2027fd18",55483:"342d2e6b",55526:"f7fa3381",55534:"ab685cdb",55559:"29cf75d4",55566:"c8861997",55579:"3a9c66ce",55616:"d5caed5f",55664:"a7511015",55668:"bbaa8144",55754:"069b26b8",55780:"d425e9d6",55836:"94b063ba",55929:"2cbaf53f",55942:"41adf405",55955:"194c8540",56028:"e1a6f6ca",56037:"69ef23f7",56062:"cde3e47b",56087:"c96d4604",56097:"096bca72",56122:"a7b4d0d7",56123:"602e2465",56133:"887c7539",56190:"5160d3b0",56256:"e028a908",56259:"75a12619",56297:"575869e5",56384:"fec58251",56408:"cbb5064a",56493:"858f1266",56510:"86a45dc2",56560:"c9c31477",56607:"609f9177",56648:"06eed450",56662:"beb32294",56688:"8dd3eb38",56723:"69e6c429",56725:"9a502a1c",56767:"2d57b909",56779:"6664c24a",56826:"eec5649b",56916:"ebc13825",56948:"1dbfdc18",56974:"39ed0ae4",57054:"97b1d23e",57119:"4d68fc5d",57141:"037241c6",57155:"23348138",57159:"e15f47bb",57165:"034f0308",57178:"2575da36",57182:"a2f498c0",57222:"8ee4a7d7",57245:"e3b05f38",57251:"779bba71",57283:"5e727800",57302:"55b89dea",57326:"3ed58e4a",57359:"841b525c",57361:"79cc2eba",57426:"533bed85",57529:"0b7a8a63",57547:"27211971",57581:"b20f9cb2",57594:"2b2d1be1",57614:"dd48b375",57647:"0ed7fb46",57730:"658997e4",57749:"c07b5739",57760:"73d3ccec",57762:"27b8ef72",57885:"f1691dde",58e3:"e2070fcf",58002:"31fce735",58079:"2655f170",58111:"fa4b529c",58144:"a2171a4d",58255:"400deb23",58259:"078daaf7",58301:"b296abb0",58351:"cbf83637",58359:"a5e62092",58393:"15ea2a5f",58428:"ec854811",58479:"2a882da6",58484:"36d29ed8",58492:"0ba9210d",58581:"f0e5f3ed",58594:"6bd22ece",58666:"a1e59af5",58667:"a36e07dd",58704:"7fde9a4c",58707:"6b6aadc5",58721:"9ffdcfdf",58780:"b7d1b016",58857:"bec552c1",58874:"030dfd2b",58880:"52e97b50",59004:"7dd8c933",59005:"39fca8ac",59009:"a169aa70",59020:"e84457bb",59053:"11b61292",59070:"c43c6f4f",59104:"c762272b",59114:"1a20bc57",59123:"17ffd4ff",59262:"97db775f",59326:"e2b886c9",59349:"156bd387",59475:"55d26943",59493:"150f4998",59521:"b2555332",59614:"c5c8b091",59680:"70cca634",59702:"c9b0e9c8",59732:"9873f053",59740:"ab90a7b7",59779:"28553a2f",59781:"0e7c02dc",59866:"03f08ad1",59873:"b37c8625",59874:"b62e8859",59923:"95789563",59940:"888f35af",59981:"03137a3d",60020:"cc3a93a6",60119:"66eb519d",60167:"152819f0",60185:"e04f784f",60187:"a52bfe45",60193:"5c430979",60198:"5c417b7f",60203:"5c5dd48c",60257:"3fa5d64e",60267:"3d3aadb0",60294:"93d54c10",60315:"42d452f1",60445:"f83acba0",60510:"821320eb",60565:"492bd0ed",60613:"3523854b",60625:"7a82ef89",60676:"244c7b0a",60696:"d163928b",60725:"73e80d5d",60762:"e9d18558",60765:"0614dce3",60819:"c316a3d7",60830:"2df3fdca",60837:"d2436a2b",60861:"c9d25191",60884:"709e1a02",60901:"00b58b18",60908:"fe53d354",60953:"3e85731b",60965:"4d18bf1b",61027:"b1506a50",61114:"127f988d",61228:"4521c19b",61235:"a7456010",61265:"84536aab",61300:"31eb78c6",61345:"f12e5474",61378:"c90911b0",61416:"a1e3441b",61481:"b0f00280",61559:"6b206066",61584:"55b57e06",61627:"35190cab",61652:"495a3878",61702:"faa24029",61721:"4f3239be",61734:"ad2a3b86",61740:"bc80aebf",61786:"aa1329f2",61844:"14da6f38",61886:"124bf435",61939:"88937603",61950:"820ef6f8",61959:"9df3c51e",61991:"a6972a3c",62015:"5cc68a8f",62037:"f1b1ae9c",62061:"2638eb20",62111:"d2289dcb",62120:"cf1cd5da",62138:"1a4e3797",62147:"8d364c04",62149:"5955b5ee",62162:"22c027ab",62182:"0e7e04d8",62188:"1e38d1de",62254:"9b695413",62264:"cb69dde4",62400:"8041f857",62431:"856bc38c",62456:"3e9695b6",62469:"f448ea15",62501:"564f4b07",62549:"cca423c7",62568:"abfb0eb9",62635:"61b91652",62639:"0f378f0f",62641:"58728d91",62672:"fcaa2a90",62691:"d9175e87",62703:"345b88a3",62712:"f9a8c23e",62714:"1769198d",62827:"4e78ea4f",62845:"ae5c2262",62867:"07853d90",62875:"f1096413",62927:"c03ef713",62953:"7a0d5539",62957:"e5c6728d",62975:"baa95a18",63005:"b0635fce",63064:"754efc6e",63075:"37844666",63124:"d4bcf69a",63151:"93f4c2fc",63181:"29fd9202",63228:"34c77011",63262:"df8bdaca",63291:"2f1bf685",63298:"259b7e50",63317:"7dd3b2a7",63330:"60b75cbd",63332:"67474760",63367:"44bdb747",63382:"5f87f962",63387:"9780a9d6",63420:"04ea9cef",63430:"465af2b7",63455:"f30739cc",63515:"a3041c66",63521:"58f053e0",63553:"1fd2ca4a",63576:"b9795b3d",63627:"43a56d47",63646:"47298c37",63682:"f82e3d74",63700:"eadd254f",63739:"81a5dcd4",63779:"c2750739",63795:"a4974adf",63836:"95e06d9c",63907:"57e245ce",63951:"8bbbdfbb",63966:"2278c359",63975:"b81d6cb2",64045:"c47fd21a",64099:"e34c241a",64184:"f4f77456",64209:"88ed6425",64264:"192402a6",64275:"48a72e90",64351:"7c0dabe4",64391:"b65b26fc",64442:"dfa86ee4",64447:"e4307155",64465:"e9f47cd4",64473:"9c8cec5f",64502:"5d817499",64527:"dae07133",64535:"3756d209",64577:"a2a8ce40",64597:"9c273b44",64623:"e52d80fa",64659:"6ae279e9",64670:"46693c0b",64737:"1d00ec5b",64749:"2da5f59f",64838:"9a8a6b3c",64899:"e80f330c",65013:"a3fde46e",65017:"f08c8322",65091:"75914136",65094:"f04b050d",65133:"c9bcea67",65193:"d0cf31b7",65221:"67922501",65223:"2a11e6a7",65233:"0530d392",65246:"ba47c7e8",65350:"517960c0",65356:"8db42357",65359:"070df7e5",65473:"a21d3492",65497:"26115f23",65569:"8008d958",65611:"5d7f3e2f",65625:"e1ba57a0",65629:"26cbee69",65647:"75ba6401",65648:"c7850e27",65658:"329c66a5",65727:"e29a1e9b",65745:"d1be0dfb",65756:"02ff5d42",65759:"7d5633f0",65835:"fbd3de42",65897:"38147e44",65918:"587a0b3e",65926:"567cfed1",65957:"21719437",66061:"1f391b9e",66062:"fada9729",66097:"ceca85fb",66135:"331ad65a",66140:"136587f9",66162:"cf5645b0",66169:"0b7cbed9",66210:"2b53c3fa",66237:"cc9fb6c4",66327:"eee168db",66338:"dcbc8483",66372:"8fb804ed",66381:"9019bc0f",66394:"c0d2ab5d",66415:"6f47e633",66461:"cb8920e1",66499:"85e49a93",66504:"b01f49d5",66510:"79d5f27a",66518:"c6ec7d6a",66524:"83037ff5",66538:"d0550e1e",66559:"ecb74126",66575:"b0e34666",66602:"9fce9e91",66715:"48455675",66753:"5389895e",66759:"e381e7b7",66822:"62e9fea7",66832:"e1fde1ef",66867:"58b9aab4",66903:"47f8f95a",66912:"6e955522",66947:"6faab85c",66995:"ba8ea34b",67044:"1b143eab",67061:"c1050649",67098:"a7bd4aaa",67100:"27c39d7f",67104:"06073810",67114:"0d86fe8d",67149:"11235f31",67189:"9736a6b2",67229:"2d4f05ca",67252:"60cef16d",67376:"01ed4ae3",67438:"b6d7433d",67472:"814f3328",67489:"3477cc36",67507:"77139df7",67545:"d89aa357",67581:"a1a96ebc",67587:"8775fa1c",67649:"eedfeff5",67668:"47766edd",67683:"cace8592",67704:"8fb89e11",67720:"4d59e059",67729:"2c18bef5",67755:"b57b3f07",67775:"39b26ead",67914:"0a0dd03a",67954:"8aec9a60",67989:"59171146",68038:"863f8530",68043:"c1c9577e",68045:"0924bdf1",68094:"e981d05e",68210:"a148d810",68246:"3b8a31a8",68249:"e48e4dc9",68253:"89c88bb6",68304:"8ea5f648",68315:"98826249",68334:"9d05a91f",68434:"975aa772",68467:"bb6faf52",68470:"60753781",68515:"f553a3a8",68551:"9a8c2230",68569:"d81421b8",68665:"f645a9e6",68685:"0b87289b",68710:"dbf13a93",68765:"719a7d9b",68770:"14033dfb",68839:"3ac10f9a",68861:"bb1a3e9c",68865:"08299697",68910:"62cdbb35",68930:"5aa406a5",68955:"f9d38c6e",68991:"b60b94c8",69017:"ef13094f",69030:"8a752d6d",69038:"8df5dc78",69129:"4a55387a",69169:"fc384521",69199:"f0589cfc",69347:"de9d53d2",69365:"4d2a5110",69384:"5a360e6e",69457:"7ffd9a5f",69501:"fc15228a",69517:"8a4df687",69570:"4d573718",69588:"a3713279",69633:"05c96a0d",69638:"9c10cdcf",69671:"36e398ec",69770:"b1e8d27b",69787:"787caf51",69844:"615fd764",69875:"b58f381c",69982:"8e981336",70008:"0860d759",70022:"c8d1e051",70058:"73a7a1a2",70100:"b318f3b6",70117:"9f3d620b",70126:"f68988f2",70127:"dae6ce88",70133:"2f2d1edf",70174:"e04f6385",70179:"cb543535",70209:"529aff45",70210:"8367b76d",70234:"e76e0aa8",70253:"5e1c183f",70268:"aefdabf4",70399:"d2786aa3",70418:"0bbbad22",70452:"b71c2d32",70502:"8c596cb7",70551:"0bd456dc",70560:"ac5d28bd",70606:"39583919",70612:"6d5145e1",70645:"a9f446ca",70649:"0920b03b",70669:"dcc774d2",70687:"70ca6744",70689:"52375367",70696:"52be8247",70712:"cb488bcc",70728:"07338b13",70749:"ecba2f16",70755:"f3e1c72c",70784:"ac00e857",70791:"fda5ba18",70794:"c1dff2d3",70814:"c3c05d38",70825:"1d89db06",70826:"23421dc8",70828:"6763bf32",70853:"e34cf2a2",70870:"4f594038",70947:"d1fa94a6",70953:"87d75f25",70963:"16029e49",71021:"05f6486e",71026:"5e0cf2ca",71051:"9b8a5dc6",71111:"48c07d39",71117:"92f96583",71162:"af7ce5e3",71227:"131dae92",71235:"18891827",71374:"40d0141e",71407:"7380ddcc",71473:"ff761994",71531:"f89e6cd1",71565:"83c3bea7",71585:"d7e2cd1f",71590:"ed47441b",71612:"a29b4047",71656:"7ec29cb2",71715:"4a74d13d",71784:"e3aab494",71792:"d2a882d8",71858:"09138901",71863:"acec9ba2",71877:"466f5a64",71910:"f783f7a9",71964:"835d6f03",72011:"31b399ad",72043:"f0dc2560",72053:"95f36a1a",72122:"3d095b6b",72135:"8cec3a3f",72246:"cf8b2bc1",72322:"4941c7ed",72353:"f23d42f7",72358:"4419dbb7",72446:"27d3d4e0",72527:"ac84c584",72573:"bd203bf3",72599:"3336b76f",72634:"c8768069",72668:"049fd4ea",72675:"f81aef8e",72707:"eb3eed60",72759:"69f425f4",72815:"56040849",72835:"2d34e835",72880:"43b52908",72907:"491d56e0",72914:"37eca2aa",72925:"b724edf8",73019:"fca979f9",73084:"b8385eea",73193:"da144c90",73264:"33e8889e",73276:"7bd25ebf",73290:"c7f860be",73316:"8b1b48fb",73331:"97d2cbab",73376:"21838a86",73397:"681664ad",73472:"092425ad",73493:"4c26df07",73539:"5ebbf59b",73540:"73d617e8",73569:"ac6289fa",73607:"01689e9b",73609:"622b7911",73624:"edefc60b",73684:"35be90fa",73693:"c6ea0115",73741:"eb08ed44",73756:"d8415e6f",73800:"5cf43a2c",73841:"569e0588",73846:"4d015a5e",73888:"06dd3057",73891:"d721a748",73905:"d502d9c9",73940:"f4731b9a",74019:"14841d7a",74027:"9d891c91",74039:"2e6d9cc0",74046:"b7aeb8c2",74069:"ea5e46ff",74121:"da7bf323",74134:"393be207",74282:"2fd7ee6b",74312:"acf7953e",74388:"57eb6545",74403:"bc42f283",74418:"103c2fe7",74490:"3d4e54c7",74524:"111d9467",74588:"84939cad",74657:"033d3629",74684:"e6ce3bd9",74792:"a5ac74f6",74880:"ec7a5af3",74902:"43c329f5",74945:"ad848ffa",74961:"9b3f2ab9",74994:"eb5c136f",75036:"0c90e710",75043:"1c4d0413",75051:"13d28288",75062:"8df3f976",75078:"dd620de6",75105:"e2585025",75106:"9f49c871",75173:"3126d1b1",75267:"6dab9497",75309:"07deb48b",75359:"84aa8d64",75378:"847c1b9e",75405:"ceaa6e69",75450:"97c492a6",75459:"605c3775",75666:"2aa42d18",75695:"68f2aae3",75696:"1ad07866",75718:"d2250181",75735:"691ef278",75759:"217c03e5",75775:"724a4edc",75805:"ad132b09",75866:"a3541f3b",75911:"f18854fb",75919:"e01c8f09",76041:"03240ae1",76123:"52166518",76163:"20a6876f",76166:"1778a2f7",76215:"d9605822",76235:"6a3d5afb",76257:"cc7760fb",76262:"075f4b70",76264:"cb46984a",76274:"5d06256e",76285:"0d71ae17",76289:"1ed4b1ad",76368:"9267add8",76530:"594c10ba",76641:"eb0d63b4",76683:"b6f16885",76719:"a50107bb",76748:"9a68cfff",76850:"d5a221f8",76854:"2b00a752",76871:"e364f7ff",76873:"a91e6a0a",77077:"a01a15fc",77088:"bf048e24",77096:"61ee4158",77123:"62825cc3",77145:"30ae1693",77174:"ce019e24",77183:"eaf93d9c",77200:"a95aede3",77222:"fe2389d2",77261:"1aef3c3b",77298:"109d395b",77369:"c4acdd50",77391:"d0c84d34",77596:"0235a442",77679:"5ef28561",77698:"60381fc6",77749:"3a31d53f",77764:"8c3793bd",77787:"d9e41302",77794:"62e7b5b0",77920:"a52788ff",78001:"33ab05f6",78037:"c9599a77",78055:"30175d3c",78087:"90caa6a1",78159:"8df10e0f",78199:"5455ca0e",78312:"1682c6e0",78354:"c1b680b7",78421:"b1a2ea9a",78497:"28ebe10c",78512:"b7e5c092",78522:"ff4a022d",78544:"4adc4126",78583:"95365ba3",78632:"4f1d1584",78704:"b80df7ca",78758:"667a9f57",78764:"6000d05b",78787:"a7f76e11",78820:"04c3dd09",78858:"4f0afd2f",78876:"499efff2",78880:"335badf9",78912:"c8939218",78919:"f73fc2b7",78926:"73bc484b",78985:"ccd7c88f",78988:"0ee8b775",79039:"803df44c",79048:"a94703ab",79150:"cdc43a7d",79157:"c884c38e",79180:"cc426672",79302:"74555694",79416:"a2030166",79474:"120bc0be",79485:"8b7dab17",79506:"368f9c22",79520:"34900881",79521:"09b9b020",79656:"1bfbaba8",79739:"8be24c4d",79841:"fc26a5d5",79866:"619263e9",79987:"dd6ea91b",80018:"5e4ec6cd",80046:"93c339e4",80091:"587df15f",80102:"7349f9da",80198:"d38a6f54",80205:"0cc20099",80259:"45cd5638",80260:"8aabb1ce",80299:"8657657d",80310:"3c89ed52",80362:"2fd10e89",80375:"6e954188",80408:"81c96f91",80424:"abf14b6e",80442:"e1a35040",80457:"992bda30",80458:"318980ec",80491:"7080aeb9",80531:"27321f21",80557:"5cb06d06",80598:"1f97a7ff",80632:"9abd9d66",80649:"80f24550",80677:"6d70fa33",80744:"641786b9",80750:"84ac5929",80772:"fd440a88",80781:"cfdba83a",80794:"442da8c1",80841:"413bf93f",80863:"b6aee096",80947:"a7f213e3",80957:"c141421f",81012:"820b9813",81019:"f74d3bfd",81050:"c88d5fcd",81085:"104fa664",81125:"aee88ae1",81175:"f98df6cf",81188:"ddf9f187",81212:"347b93da",81285:"54e1e905",81300:"2195a7cc",81319:"12363bc4",81346:"a20f65a6",81390:"e6c5d4a7",81410:"7af33a93",81482:"ae0ec9f4",81530:"654827c2",81544:"13685ceb",81583:"15a3ce9a",81639:"ffe2a86d",81666:"a74d38f6",81692:"0d9e9c41",81737:"c75c3ca5",81761:"932d66ca",81770:"fddd2104",81791:"d3571192",81795:"b90928c1",81802:"6066240e",81891:"b3dc9cb4",81902:"29470dda",81903:"acecf23e",81943:"4b93078e",81982:"2a59c009",81990:"f9ecf61e",82012:"4a7139ae",82019:"755e7c7e",82242:"cb7dc7d1",82261:"d2c9f0b8",82297:"226eff01",82354:"b87f99cc",82362:"7345c721",82431:"5d210bb4",82456:"24f4e7d7",82486:"4393156b",82553:"0add85e5",82581:"907fc94b",82588:"4cd82028",82663:"dd0ce7a8",82722:"72e3e702",82754:"4ccba4b8",82847:"dd09cc7b",82905:"12cecbf6",82939:"2a7d5452",82987:"732a7c43",83027:"dac8839d",83078:"42174d87",83089:"686a74aa",83103:"21cd7edb",83109:"79ca5236",83111:"6060cc06",83121:"deacbd9b",83127:"10757cc8",83182:"e5cd0e7f",83203:"95126e44",83213:"828aa184",83249:"ccc49370",83260:"5c26bf07",83273:"2b6eabf2",83316:"0ba08ea0",83378:"d0f169c8",83470:"e6ccb422",83574:"c85c7bc6",83608:"19560f91",83670:"9e37e644",83700:"a8c902bd",83779:"e6fc8a9b",83797:"b6cd0ba6",83824:"f5dd5915",83841:"cc517726",83849:"c48426e8",83928:"345eb925",83932:"757c6784",83954:"e67a83cf",83958:"2bf0d1a9",83970:"e200187b",83976:"0e384e19",84013:"c17d6939",84070:"511d3e84",84107:"ed83b9b9",84152:"d2ed2b82",84227:"17d9fbbc",84232:"ddd99a22",84255:"901160ed",84259:"2ffd8514",84319:"1132d5f6",84332:"0b82d45d",84527:"8180940b",84530:"2e72ea50",84563:"2760fb69",84572:"d9a7203a",84760:"21f4f354",84773:"b738b15f",84813:"6875c492",84869:"6bd2e919",84963:"d9d878d8",85014:"ae42df76",85095:"ec2ed6e2",85097:"7c86d680",85105:"ab380486",85205:"0cf3b0b8",85209:"8beefa16",85243:"78c968cb",85257:"ffec2b37",85279:"8f4a15da",85322:"c9ff545e",85358:"813f53ba",85442:"2f6614a5",85453:"de5aeda1",85537:"2c88985c",85581:"8f9bc413",85588:"24e645af",85736:"0e8c3a89",85796:"a4a3eadf",85806:"3e226f70",85817:"d40aab8b",85896:"ddb0149a",85899:"41639dad",85917:"716f14d9",85960:"5268c144",85973:"5733876a",85997:"83b6afd8",86053:"ae271c01",86074:"199e6876",86121:"677325f0",86129:"fb03d32d",86167:"1f8198a4",86215:"95ee2976",86216:"abe1b280",86246:"7a2c4c43",86354:"d630316d",86417:"669193a7",86434:"d5f51703",86478:"2a1b9f9a",86480:"3a311bb0",86501:"f4e1ab69",86586:"b76e61dd",86597:"0e0dfb6a",86623:"49bbb99a",86659:"7ff6577b",86669:"927f97a3",86685:"58421c87",86807:"b0445ba0",86829:"5626901c",86846:"57ec2762",86856:"c942990c",86888:"5687c7cb",86890:"319b6f13",86914:"869411c7",86939:"7a06eb83",86992:"d2a66e94",87004:"b62a4e5f",87087:"40df2769",87159:"4eec7d8d",87184:"4462c0cc",87186:"767cce31",87205:"bad31f72",87217:"9c71777e",87269:"33ac18ed",87282:"c884ad6a",87377:"c4f2c374",87407:"cd08112a",87432:"0b425eb3",87456:"33bf4676",87459:"b8f2cc13",87473:"baf4021b",87483:"d5762a9f",87485:"643da015",87547:"e29c32f4",87589:"d2b2e5aa",87614:"7c29a585",87688:"137db567",87747:"3f1a4665",87781:"6f831635",87863:"4194f325",87875:"01f93a4b",87880:"f553a7ec",87905:"fc17862c",87945:"5b17955f",87971:"dc4ee1bb",87995:"85f9f5a6",87996:"72a68b3b",88047:"a5f4f54c",88111:"ca3d464c",88163:"2e05ee79",88222:"b8fd1a8c",88248:"ed990644",88298:"85e50ec2",88310:"0169a509",88327:"fbfb0bb0",88393:"3cf947bf",88394:"5ae50f21",88399:"078339bb",88436:"a13618bc",88441:"40e4b7d4",88468:"bf79ed6f",88478:"23a811a2",88522:"d3f32089",88547:"2283659c",88622:"5871fbee",88626:"6e8a38ea",88654:"6dc022b8",88696:"6be5dbf9",88775:"b330ac5c",88815:"71cde98c",88817:"d3065e4e",88821:"b0b71a2a",88847:"014c2cb9",88852:"508d741f",88855:"5edce1ad",88865:"d15e98db",88903:"568a0660",88986:"7b658d8e",89018:"2c9ce68e",89093:"fc93a2ea",89097:"24f450eb",89172:"adb96665",89175:"9f15c6e7",89210:"9b191135",89289:"d8e6b3db",89338:"324cb577",89353:"b919dab0",89362:"cb2f16be",89402:"39ac0d5b",89415:"9ed26de9",89437:"ce1709a8",89455:"978ca64f",89489:"145c1cc4",89534:"66405406",89554:"7ff4234e",89610:"2709984f",89641:"cde9af88",89696:"837dc7e4",89801:"084a80b5",89810:"9bfa1541",89858:"36994c47",89873:"65f8c8fd",89986:"e582f766",90066:"bfb3b697",90071:"7e60f0fb",90110:"dbe2cfea",90115:"1026ad00",90135:"8abd7194",90214:"9db841c0",90257:"7b773c92",90280:"a80ed580",90317:"b3abe364",90378:"c633f93f",90414:"ed2667fa",90416:"9947fe09",90438:"fe49dcfb",90513:"6157713d",90562:"32eb34e5",90593:"99144b30",90598:"2563c97b",90625:"73554d19",90627:"bfa9eb5d",90717:"fff1effb",90749:"48dc1693",90831:"e96ddf11",90842:"837010ac",90849:"0058b4c6",90910:"e03c6723",90950:"271ea2f9",90972:"70e8710e",91025:"466375b7",91058:"42003ca2",91109:"014625f4",91187:"3629c812",91203:"ece3ee5e",91257:"f657ed9f",91305:"af1525aa",91319:"0f84f33f",91333:"d1980a1b",91359:"88f66091",91405:"ae3304ee",91428:"284c57b4",91530:"7118d0f0",91540:"980d0b86",91581:"8614d32b",91587:"1fecd571",91597:"e33cf622",91621:"7907a033",91695:"6033c607",91724:"9964550a",91787:"df99aa82",91844:"41ca93cc",91895:"b7201a27",91916:"72fdaa88",91993:"8ce0215f",92018:"0ce689e1",92031:"e2c3948d",92105:"6342cf5e",92125:"4cce6e5a",92196:"56a0d755",92223:"e772c536",92241:"bef29d9e",92273:"85fe7d70",92329:"4a4c6752",92410:"4c434b32",92412:"d888d821",92475:"1be20fdf",92535:"b6eb7220",92544:"e53ee22f",92653:"efa442c3",92664:"7c1d6f5a",92673:"9dfaf941",92955:"3e082aad",93057:"adfd3bc1",93085:"b5ea0533",93166:"fed5be48",93206:"2ed2a721",93209:"47b26a6d",93210:"e55ca189",93269:"b12147a6",93273:"cf32db66",93341:"aba187b8",93361:"2357dc71",93362:"7a234181",93438:"f5e9d1c4",93442:"63786366",93534:"a9c52207",93546:"d9e43198",93602:"44c6f0c9",93636:"e1847ef7",93795:"f179621d",93849:"09bacf3b",93854:"0a07ac32",93884:"c3c082a1",94e3:"45a5cd1f",94065:"ee71d619",94082:"8c0fb9c6",94091:"beceb38b",94106:"acddd5ca",94153:"6ee6a410",94168:"c6b4dc09",94227:"f0cc57e7",94251:"824e6f8c",94314:"00f9d3c8",94344:"8b44df1d",94361:"657bf45c",94417:"85c8b6c7",94538:"13feb7a8",94555:"8e7aaae8",94561:"a1e44e64",94568:"745d4b8c",94625:"0a02a2d0",94646:"30549b42",94648:"50774ec6",94685:"8520f5db",94711:"0c2c2d88",94713:"6dfe2e3e",94788:"c26e67a5",94796:"256701a3",94841:"a1282170",94843:"3e1f4a39",94863:"fc66001f",94864:"95ec5604",94910:"bb63d03e",94925:"e8b41ff0",94926:"98700a51",94980:"5fc8caff",94997:"8d65e26f",95020:"63c4b13e",95058:"4b69979c",95078:"10ac9a3e",95093:"818bb2cd",95136:"c0df5757",95159:"cdb727d9",95183:"f58814fb",95187:"59700627",95231:"1cb0fe52",95240:"b4030b00",95245:"ed211a79",95330:"9b4185c1",95427:"52910a8f",95443:"e1afbf8c",95470:"8a40ff6b",95542:"5917c028",95605:"9a8fdb53",95641:"188015be",95649:"4885c521",95659:"30d4e84f",95768:"89ce4ba3",95785:"8c79a60d",95818:"dbea5ca6",95909:"9e488927",95911:"accd2b0e",95912:"cbc2448c",96022:"5180e398",96086:"eb84cef2",96155:"a267572b",96225:"838df61f",96239:"380e17bf",96268:"638db053",96276:"82a7c68f",96325:"65d1ec04",96463:"ddd9290b",96470:"1b260ed9",96555:"d4083900",96652:"5b0025cd",96693:"8c42b153",96745:"4f748135",96771:"a98ffe6a",96778:"181eb1b5",96793:"c3c125b1",96881:"d527c17c",96963:"98749210",97006:"7ab01152",97033:"e9032a0d",97047:"763a8986",97107:"9331da7d",97114:"32bb5bcb",97123:"9d2e74da",97167:"20a30ea0",97189:"1b3061f3",97223:"7fa05810",97246:"365a7fd7",97253:"1a8a162f",97296:"205cdcf8",97342:"c2a473ad",97358:"76276d52",97379:"3bfbd79c",97420:"224c7e3e",97424:"ba47c136",97476:"b93a682d",97536:"3a6c6e5b",97554:"6ccd70b6",97562:"3d08e0be",97585:"8e81e342",97664:"facb0528",97670:"1dba1ecf",97732:"b58e88de",97737:"8fa9d00b",97753:"41b1becf",97763:"7d2fd416",97821:"97a057b3",97829:"e5562b89",97863:"47edbc34",97887:"26ec8ae2",97895:"ab8b4014",97901:"bf7dfc7c",97905:"ff509459",97913:"1e228808",98012:"350cc719",98020:"16243474",98021:"139d61ea",98030:"95f25ea6",98058:"fbb50b04",98059:"75dbe45b",98087:"b18455bc",98094:"48f67822",98139:"a2219ebb",98147:"fc46979d",98165:"c8e5bf38",98237:"3539b92c",98271:"f62bafa2",98308:"756094a5",98311:"9e297776",98392:"1c3a958e",98425:"b943b6ea",98473:"94dcd3de",98477:"ea6f04d4",98534:"6065ad54",98700:"99e64d04",98750:"19bfb940",98758:"af6e9729",98761:"3314c9d3",98779:"a48978f5",98804:"bf667688",98867:"3e8495a1",98908:"b4e94af8",98925:"ab37b32f",98935:"5de60d34",98941:"c9ba7c72",98947:"ef00b8a0",98982:"e8178b53",99001:"82a5d7f7",99095:"1c27379d",99146:"210305d4",99151:"8e563661",99252:"3047f3e7",99254:"c14fcab2",99299:"8944e56a",99306:"bdecca0c",99330:"b3e98411",99335:"df09200e",99437:"5ffb8f23",99451:"c0684476",99502:"2263a65b",99504:"76af5d51",99528:"16e939a3",99531:"8c2cbe8e",99534:"c49eb59b",99543:"25c655c3",99611:"9cc8ffa2",99644:"bfbfac54",99678:"25e15e7c",99701:"5a4dd75d",99705:"5fb14ca8",99709:"dc330c71",99710:"6e34d00c",99746:"f4a839f6",99747:"a32c0324",99872:"8e34a11f",99905:"2f7d15c4",99913:"38836367",99987:"4b2e4980"}[e]||e)+"."+{16:"8a09bc3c",25:"6875f362",28:"8eabdc91",39:"068bc723",48:"849a2f6b",60:"38634d1d",67:"7eac7b6d",106:"5d0f86c7",129:"ad091a76",159:"d17cfd99",217:"0bc1c4b6",278:"7a133af3",292:"0bd61be4",323:"2ca86a04",335:"10e8b68a",354:"40cf0362",387:"cd12a486",437:"dfb6628b",569:"02866324",614:"44e4a263",805:"07c12e99",822:"ef8339d8",851:"ee36c626",878:"8fad3634",910:"efe21154",917:"6161fd8b",924:"356e5ddd",940:"cac0b0df",1103:"3f89d301",1107:"7e8866c9",1119:"4bbc5be2",1136:"21f72e0a",1190:"8e56610a",1251:"49f694ba",1284:"f5a3911b",1337:"ed03be8e",1360:"ecf0e7ca",1422:"0e1d3b9a",1442:"47fac533",1506:"554d3065",1517:"1a694762",1550:"39a1cecd",1635:"35a7771d",1656:"ca0647f1",1658:"9711c713",1784:"0fef97f2",1785:"bc4d1952",1791:"5fd94693",1841:"b4a6b9c8",1846:"9ac595fa",1857:"4c849a8f",1882:"e0a7b21d",1928:"877dfaf1",1946:"61767f79",1955:"8a0bfb14",1983:"1982ff74",2016:"bd8bb947",2053:"a44f5cd9",2121:"e27f349b",2172:"33f2ad32",2214:"6a299945",2225:"7d7028c3",2239:"0f532176",2256:"779aea49",2262:"0bf08dd9",2298:"571d78d4",2385:"a6abe910",2450:"57f1681a",2472:"1b1cbadb",2540:"ccf5521a",2577:"9d61156a",2638:"0d848ef2",2640:"f6e9fddd",2652:"7c607567",2663:"4e8fefa1",2679:"dd0dc46a",2700:"2fafe158",2739:"0ca6b901",2748:"6afd10a6",2759:"38d24c2d",2768:"13d74cf3",2772:"5a7711f9",2791:"35fa4f4e",2839:"27e3f7c1",2854:"45badd67",2867:"2d1bdc8f",2884:"bc593027",2912:"4a994ce6",2932:"da39912d",2954:"f91b5b76",3075:"2f46f339",3125:"d85555ca",3131:"feeade58",3137:"a059264d",3144:"86fccc09",3179:"d88e0655",3303:"e1d40ac6",3313:"c4669891",3323:"a9b14b11",3471:"5be54bd3",3558:"4acdbe49",3693:"18efdc0f",3694:"cc0b4859",3722:"35a202b7",3740:"f2909c31",3764:"98e31955",3847:"6ae5e13d",3940:"06f163c7",3965:"c5d1f980",3968:"27784d55",4014:"fa271ba2",4028:"fdcb2d50",4040:"f4d4f02f",4060:"9b076617",4074:"33fd4cf3",4117:"9e3d6091",4133:"255f8728",4137:"75ef07ad",4205:"600be0a6",4269:"04fc581f",4355:"ef747ac7",4368:"2a18d792",4382:"5dd5bfe7",4405:"5da3e763",4467:"a9d685ff",4514:"b9b659a7",4530:"ae1c75a3",4574:"fc708067",4591:"93231c73",4602:"f0ac3d8f",4777:"5ba3f668",4790:"904f169a",4863:"2586fc4c",4890:"d278e426",4919:"c4d3cd3f",4949:"ffee880a",4980:"df50cb4e",5008:"6b1d60d5",5028:"7c782fc8",5029:"92619e1a",5065:"f9e7905c",5193:"1b398db6",5195:"774314fd",5206:"705fa214",5269:"3ef4f7b7",5270:"babf4f83",5289:"7a6d5275",5295:"55c2bba2",5304:"bb7c5b01",5322:"746c214e",5334:"5dddcc55",5338:"29eb08ba",5594:"341820cd",5622:"b7d8d7b8",5631:"e2b9b99a",5633:"a518d9fe",5647:"5d077e51",5679:"e005a1d1",5721:"d9f96b25",5806:"3440443d",5834:"8a7b0abb",5887:"68e33d36",5888:"5fa93c63",5936:"b9ce6fac",6022:"68004ad0",6054:"9b9387b6",6090:"f09b8f13",6158:"077cf067",6170:"31c32ba9",6203:"9f3b730b",6225:"4e2d8d24",6289:"a054c626",6310:"2da72e95",6350:"3246d056",6368:"505a4db3",6371:"091b2713",6372:"368025e5",6407:"c92fd14a",6467:"fecfe081",6475:"6ba96ded",6552:"b2b49c3f",6559:"c4675766",6636:"d669b70d",6677:"144addec",6692:"1df14aa2",6697:"5901c946",6709:"5ec644bd",6739:"cbfa6ce5",6761:"5d49f1ec",6778:"d15a57d6",6791:"2a8b7bcc",6823:"ff319028",6897:"55b30925",6930:"ef13ba85",6944:"cadf7087",7035:"04f7671e",7071:"9a7aef67",7100:"078c6553",7103:"6b60b9e8",7125:"b49e987b",7167:"960b2669",7317:"56224cc2",7355:"04e349af",7444:"42b1316f",7458:"24779c3b",7517:"0120af7d",7566:"dff45037",7622:"77095e00",7629:"1c27322a",7715:"5d269ad1",7734:"99a612f4",7747:"6f5217dc",7773:"a5129a8e",7802:"e5622638",7850:"a753f976",7907:"df25cd3e",7953:"ccc96052",8003:"308e1ef3",8070:"264340d9",8139:"61f67727",8174:"34dbd8a5",8177:"b6e0383c",8183:"147ec233",8192:"db4decd8",8209:"c1611e12",8289:"ffb3c473",8348:"3268c830",8354:"fb86f6fe",8363:"7fb1b5be",8367:"b027c2f9",8407:"99cf73d6",8413:"f12343b6",8417:"1b4b7f2b",8585:"223cec3e",8619:"df2fa06f",8632:"d4174f8d",8645:"e8362a79",8651:"e168587e",8699:"91e6e14c",8762:"4a09a33e",8766:"249ad394",8840:"8b4ab2c8",8888:"c84e49fc",8889:"86b477eb",8900:"fd8d3c4a",8902:"0f380f56",8965:"79e095ac",8978:"dc9f01fa",9003:"2d33db3b",9045:"4ab2e7e1",9066:"e4d3738b",9076:"916fbe5f",9127:"714c4851",9152:"c66c2521",9189:"0dc2870e",9195:"f73b50f1",9228:"46600037",9257:"c66aef02",9262:"793c41ca",9351:"61c33e87",9379:"6a5733b2",9380:"f90d89c5",9385:"dcf32ec4",9428:"6546d84d",9432:"48aca857",9446:"15fade35",9458:"7babc575",9460:"ca2fd5da",9471:"4a10d39c",9499:"ce206b69",9516:"c1f50d60",9525:"6395de95",9570:"e3b8ddf6",9573:"d316bf10",9631:"10562627",9647:"4bc8eb58",9650:"7174236d",9676:"cfd0f282",9702:"b965bad8",9722:"664860a4",9724:"52a83b67",9737:"79dbbe2c",9790:"e789ce2c",9836:"8fa50ab8",9860:"74431a06",9870:"438c6e6b",9891:"6a85c793",9907:"8492cade",9914:"d91c7b9a",9944:"f78b15cc",9986:"d714f752",10030:"59d03524",10118:"ab9fc9b2",10181:"3af67d97",10198:"86d665d9",10238:"50e474ba",10285:"bb7ecfb9",10354:"47cf3433",10472:"bdde685c",10488:"7d9ce470",10492:"e72abd0f",10535:"0d2ae118",10545:"f8206ac1",10610:"87868eba",10637:"0495e0c8",10642:"91b10c43",10653:"14a3241d",10688:"01c09458",10727:"2cdd7370",10767:"5998e785",10789:"3ced43a2",10884:"53e13b06",10887:"c5d8d489",10907:"a8d1a1dc",10920:"193c16db",10924:"95f4527b",10956:"b3581086",10978:"7f66cc97",11052:"c175ef33",11058:"b579a417",11111:"92a9adda",11156:"9f4df04c",11275:"7652321b",11289:"f07258e3",11299:"70c0805d",11326:"6ff62e03",11355:"670963d6",11508:"80fc6cd5",11565:"c60cdfb0",11631:"cc98017f",11684:"35726479",11716:"42a92ee7",11725:"127bbd01",11745:"4bbb96df",11768:"20e072bc",11782:"ed1984f6",11795:"ad2f4f53",11804:"b5477c1c",11806:"42b6e782",11842:"0baed23a",11855:"bfd20606",11869:"ced68d00",11905:"3420ebde",11906:"68908acd",11977:"c7bbc133",12047:"dce108f4",12134:"ae61bf97",12144:"1c1bbfbc",12186:"6a19cda2",12316:"4f2ec008",12318:"6eee13e3",12378:"27f2d38e",12393:"c1d28ac0",12395:"d9ff1cb1",12431:"a60ff215",12452:"97321323",12477:"e0de1e38",12485:"926a358a",12513:"80c6f08a",12535:"0c941d5a",12542:"9713e3ea",12608:"5c3e8335",12613:"ed4c9e9b",12654:"2320fb02",12679:"1987a2a6",12686:"8a06023e",12763:"91b8b830",12869:"442fc6d5",12945:"fc3acd38",12995:"ab02afd2",13021:"48555bb8",13060:"aacf116d",13063:"f4ce2b61",13152:"c62fcc56",13203:"c8815658",13210:"c492ed74",13236:"b09128dd",13298:"a8dfd245",13321:"4eb4493a",13366:"3c71c883",13374:"cfd37507",13375:"8ee7aefa",13406:"13cd1c37",13419:"b7a37f75",13465:"dd6217fa",13511:"d5b3caaa",13514:"b12fdebd",13522:"288b7d48",13525:"b906d866",13534:"17c9040e",13535:"5f863db7",13575:"0d1292de",13580:"8d40e9a7",13626:"720ad675",13698:"4cea1995",13733:"38f0a082",13762:"bca2bb57",13803:"bad250a1",13849:"0d423bae",13918:"bed720cf",13943:"453b7813",13982:"17e1f232",13996:"06631037",14003:"b85e041a",14027:"f1e7d467",14042:"de2f466d",14043:"69af2a51",14125:"527729ad",14134:"adda3dca",14136:"cc61dbbb",14152:"578ea5b7",14163:"af95fa0b",14179:"4aa5ccd8",14257:"da0d1ce3",14259:"9642e3bc",14261:"3d684545",14264:"61ebcfca",14317:"dcc8d0e6",14340:"8dd3b9be",14363:"9fdbf1ab",14388:"ae4bc2d3",14399:"ffd8a59d",14410:"cfbbb6cf",14433:"2c05622e",14435:"1ee34b91",14496:"17b3929b",14536:"10fb693d",14599:"a88c7ee0",14627:"cdb12c57",14726:"e51ab934",14750:"e9c21c9f",14802:"de3cd2b2",14822:"a06e5d99",14835:"0c69e520",14868:"b144ac54",14871:"c80525bf",14917:"28bfdf38",14919:"011e3248",14943:"fd40645a",15038:"8fb987eb",15078:"3707a6b6",15079:"97fca775",15144:"71664a42",15165:"3c006089",15186:"9c7f1b63",15220:"109d2e90",15235:"71ebac3a",15272:"13610938",15292:"328425c6",15316:"9cf140bc",15403:"8aefcfa4",15407:"ea6e5a65",15427:"6dbd0648",15473:"14fe1707",15501:"c8870324",15552:"7f6375ba",15607:"487239af",15764:"bd93ba8d",15805:"a819ffcc",15857:"4a7c5609",15917:"28cf866b",15947:"648ebe12",15955:"514b69ce",16017:"9d0f4c08",16032:"47c54457",16100:"0611b6f9",16209:"fbdc06f9",16231:"4dce858a",16319:"4732e626",16377:"367c7435",16499:"3b79297d",16500:"b4e686b5",16622:"b9a182ac",16641:"63243294",16747:"0771fd0b",16885:"9c0ac3a2",16942:"92d06824",16950:"48871f0d",17009:"10072815",17033:"eb12c1ef",17115:"0a45bdc0",17119:"b280a982",17232:"e57d15e6",17294:"0542258d",17332:"8b8baae3",17368:"115e5928",17402:"bdf033c0",17411:"03677a17",17441:"96e8dc5a",17474:"d3cc75ea",17502:"279389ac",17508:"1fb03d3f",17530:"88f83b35",17581:"2b842d8c",17626:"36956c06",17681:"5dd2ea64",17709:"7802d0a5",17739:"82146eff",17805:"8a44ead4",17818:"d75d0b8d",17860:"cc12893d",17867:"cac87718",17944:"53bb5673",17987:"53f18aee",17988:"6cc28a3d",18006:"c111595f",18033:"10fe31fe",18103:"a388784d",18121:"617b2ffd",18146:"95d7ba5d",18350:"f98fdabe",18351:"002d2df4",18401:"e94b36b4",18438:"7e8561ff",18445:"4e7d45c5",18520:"9fcfbaad",18620:"14588da2",18645:"66f4e3ae",18738:"88fed3ac",18757:"af5d2223",18798:"059f696d",18940:"1ee0f351",18950:"69aec1f5",18998:"db0ba0ec",19042:"f2d5ba0f",19052:"99eea1e9",19070:"f854e03e",19108:"ecbc268a",19109:"9741eeee",19136:"497d63f1",19237:"f651ae7f",19252:"120b0fba",19267:"0b9c27a3",19271:"81fe6522",19369:"43355738",19387:"99f2a106",19396:"cdf3109c",19425:"1d1db863",19428:"05a9e187",19474:"09d53c64",19513:"36062fee",19521:"c3aa46ab",19542:"95ba9a85",19664:"46dfd0b9",19687:"300fafa6",19711:"75e966bf",19736:"0a635609",19774:"ce35f8cf",19798:"b9d535eb",19801:"51ca69aa",19865:"cfdb0b35",19891:"3fbe9ab9",19999:"12a3ab26",20038:"db96bb57",20103:"ad5960b8",20160:"270b34ef",20194:"f1baf75d",20229:"40f68aa0",20286:"b249e590",20325:"960a510b",20358:"43110a16",20361:"ad2d7e59",20405:"c035349a",20415:"83d5d036",20424:"e5504750",20434:"bd31090e",20491:"907553b6",20535:"9f958eb8",20541:"dcfadc81",20619:"23928919",20676:"3388d156",20713:"3d6d71ad",20812:"0f0199c5",20857:"bb3132aa",20892:"31d8f230",20924:"e4fef5c4",20931:"b8036f1b",20951:"a20fcb32",21070:"a8900866",21077:"a68cdbf0",21121:"4c137c15",21177:"093ad022",21191:"5de11232",21273:"908689e4",21275:"eb589f3d",21290:"0c7c0365",21303:"bdf31e12",21316:"fe15db87",21341:"475db505",21346:"f490b9f9",21373:"9fcc5ee7",21401:"06aacf16",21417:"38f28434",21448:"9908b8dd",21482:"9ecbf669",21524:"d7e79944",21560:"a0edd7dc",21574:"4deddbeb",21585:"7b244de2",21630:"15f37768",21635:"95acc70e",21662:"95e27da1",21729:"ed2a8c6b",21764:"a96fe837",21805:"b555879a",21816:"ced1acf1",21837:"c401bc3e",21874:"d29acb33",21909:"4300cff4",21921:"193750e7",22141:"3ae5c6dc",22313:"cffab16f",22369:"8efee04e",22385:"b551598c",22428:"4573e534",22447:"d7010a6e",22472:"ecd49f93",22483:"9d782227",22499:"2575e3c3",22501:"4f13697b",22504:"c055c741",22534:"85034a52",22595:"dc8a546e",22628:"c7870ff3",22640:"1a855b7e",22647:"1318d50e",22673:"9b329e63",22709:"d92e340b",22750:"bf71f62d",22816:"3d4baa1b",22844:"c586deb2",22911:"202546a1",22916:"8d456179",22932:"dcb09864",22973:"1b43a4e6",23002:"4fad494a",23041:"f35f208e",23081:"15bf2d5c",23083:"bc82584b",23119:"b3b53229",23141:"70ca6181",23172:"c3a54128",23332:"d979405b",23388:"1678b3a5",23438:"adce0003",23475:"475adf50",23547:"5809152d",23563:"2ff8adf5",23591:"0056b029",23600:"a5202301",23625:"4b0929bb",23634:"5b74042a",23637:"039d8f53",23641:"3627f9fd",23666:"b60b4f41",23676:"c467ba2b",23700:"49359600",23765:"cd26f4f3",23773:"520cf3dc",23880:"f23d8063",23938:"d312764a",23998:"79a7d02e",24022:"870880c2",24065:"66cb1805",24084:"0194a56d",24098:"a0602b5b",24126:"37fede99",24140:"eb4a5735",24172:"1e513f10",24234:"8cab740f",24265:"9a476a2b",24340:"10e55999",24366:"8a444fd8",24373:"ab56a496",24377:"1cd6ec42",24381:"8cc6de74",24437:"d4b0c4cb",24475:"24d8adfa",24476:"2f952c5e",24704:"cf0a496d",24728:"0b98b763",24729:"055e6d18",24741:"90d7367f",24776:"4c19f189",24815:"6a3bcf36",24826:"f46941c8",24853:"91dc934e",24866:"d50b77ce",24867:"a8a3b8e9",24868:"64d89cc1",24891:"acea9caf",24912:"c8b91976",24919:"4ba924a0",24932:"ba390371",24970:"00f2ab70",25013:"ad26ed51",25044:"c495349e",25067:"a053ee4d",25101:"e136e901",25130:"4fb7b9f4",25186:"b0b2572a",25273:"cf02c70c",25285:"45a1ec19",25335:"b58a27d3",25396:"89ddaa62",25397:"f44c23ce",25426:"5994dd2a",25452:"e356ced0",25531:"70bc702b",25532:"973de6d2",25539:"4073dbe6",25578:"42db5727",25582:"2844b50a",25601:"aacfb2eb",25778:"6e2353ca",25781:"72653a4c",25845:"3668f0da",25869:"7ff9c8b3",25871:"6d4f8d18",25874:"a999abad",25959:"03b0d503",25987:"b6951fae",25993:"5a60b486",26003:"0dbe838e",26009:"6f80e4a4",26060:"49615372",26064:"2c34cf8a",26081:"4fd365ee",26183:"f197b658",26216:"0d38f496",26260:"d8f05c88",26305:"deccd138",26333:"ff2d3221",26337:"a16833a6",26414:"34f7cb93",26548:"9e235a1b",26577:"23f41c03",26627:"369f52dc",26630:"e90418e0",26632:"0dd9e951",26722:"14ac1711",26724:"7b16081c",26744:"58600e3c",26797:"0e6e3c17",26833:"ce805620",26859:"aff1f02a",26882:"d62acd2d",26900:"81479d5b",27012:"c29dba83",27020:"a2996298",27088:"47e88723",27115:"294cd6b8",27123:"6eed6c70",27179:"55360a14",27212:"4e430747",27369:"56253630",27388:"b0340f4a",27398:"50cba2a0",27444:"ea243234",27473:"5eca5e0b",27663:"e88ba1c1",27737:"6ce228fd",27823:"8a88b285",27887:"a3b9de04",27921:"45a287dc",27955:"be1e4c9d",27995:"09203cc8",28020:"ae6c303b",28086:"964d7344",28127:"52851c3a",28147:"1f3d479d",28166:"24865c80",28169:"5ab36c27",28178:"f62bdc9d",28204:"d075bb24",28235:"5ae34328",28252:"c129dd81",28319:"5d3fafd5",28410:"0b21d6e0",28428:"b0be705a",28435:"fceffd0e",28498:"1948bf76",28499:"f005668d",28627:"ae271a52",28679:"1d8f6a45",28687:"aacd5569",28701:"a28f8b91",28782:"40f66cca",28793:"e76cb139",28803:"49549dd3",28834:"3ee8e8b4",28853:"aa92d9dc",28889:"c2c9914f",28920:"9629e812",28941:"8c7af014",28945:"65b1a092",28993:"f632a39c",29060:"32b1dc91",29123:"dbd291d1",29171:"71258614",29204:"a6a43695",29243:"1356f2ff",29337:"01639ff2",29408:"bd89f274",29414:"8660a85b",29421:"fbe24116",29452:"3e9a03a7",29457:"7d14d05f",29498:"7aab7361",29523:"7c6695ac",29526:"2f8989c1",29653:"8f3e3c80",29679:"04a3f313",29690:"942394eb",29698:"0a4d3c74",29708:"8f01ba9e",29733:"13ded0ec",29795:"3b9ed063",29820:"86a6b03d",29829:"d1f3ee13",29845:"a1cbc44a",29891:"850e5356",29922:"a7552bd2",29950:"c6942778",29969:"67f28cd0",30035:"175fda04",30066:"bb46b5af",30154:"ff90619a",30167:"47c6ce34",30262:"e0931477",30265:"a50c2e7c",30317:"35aae90c",30355:"ef8a4417",30378:"6d8d95c9",30396:"87304d95",30410:"4e970141",30564:"97976d72",30569:"e40e0024",30579:"d0ae98f0",30583:"0349d956",30628:"c26c6054",30654:"73dd606b",30790:"a0648778",30827:"49ffcf71",30831:"64198a51",30842:"b50c451d",30848:"6480bf83",30934:"79541782",31004:"1d4923d1",31016:"035c650a",31039:"8dc64df6",31047:"cc5471ac",31125:"0d364546",31141:"652afae1",31144:"28477358",31220:"28b772dc",31255:"f94efe8c",31257:"771d4335",31272:"c7607cd9",31393:"a1e486ff",31465:"fff27cf7",31541:"4b8b9953",31544:"f4a84c54",31567:"a41f71c0",31680:"72a131db",31705:"4c5db215",31720:"77528df6",31726:"d1ef5da1",31749:"45dfc10c",31791:"392e408a",31797:"0a23a6de",31861:"93b5dcbf",31862:"9fb160ee",31869:"691241cf",31957:"09fc98dc",31972:"a2543041",31996:"7a9af77d",32005:"3ab64a28",32012:"287d33ab",32039:"4917de23",32183:"e4c13218",32225:"07d3987f",32270:"640b03e4",32278:"b3715c88",32335:"2a952a69",32365:"fe0329c9",32391:"a83eb43b",32472:"a7687f19",32527:"9df1b2f1",32558:"e82fa20b",32691:"c97ef1a0",32728:"a5d1f40a",32824:"d88681ec",32825:"20536cef",32847:"b0005fda",32875:"0800d3ee",32949:"7a6c4ac1",32960:"b5798ca2",32993:"6c882733",33049:"8364c70d",33065:"7304db46",33192:"9a5863ec",33214:"bc8b27d8",33234:"a84fa661",33290:"219552f4",33303:"43ad6989",33350:"75f07d8c",33458:"a7557404",33517:"3db9b64d",33524:"2e7a87c2",33533:"73ed698a",33540:"148c3af3",33559:"8fc1b433",33567:"350cc4b3",33572:"09b03628",33577:"d40cdbfd",33612:"c687cf88",33621:"bda44dbf",33638:"5fca92de",33643:"569e4e05",33659:"21d9ac33",33724:"88ca3752",33804:"d3822e25",33806:"7a355084",33816:"d55e6964",33825:"5a0253e7",33854:"94a59e40",33881:"5b41d6d8",33889:"b061c751",33999:"668ec8b1",34127:"fde6c8af",34156:"bbf91755",34179:"1bd8bf5b",34182:"21781a90",34196:"5c27b80b",34255:"3851abf4",34331:"9ed21c85",34335:"449a955c",34388:"63680cd5",34520:"7924cd1f",34528:"a8871870",34543:"0b1c0a0d",34549:"c9123c09",34597:"2d91134c",34621:"ae85fbee",34638:"16fa9b5f",34662:"091b3d87",34691:"a7c8fe63",34706:"89e46f0f",34750:"224f43cb",34803:"b8ca48f9",34808:"909e26ac",34810:"99716174",34837:"61c61de3",34847:"c1ad7786",34877:"5060c2b1",34904:"25d15372",34918:"cbd6f71e",34970:"cfbb2faa",35044:"cf8fed2a",35067:"d17b3136",35088:"cea1a9b9",35139:"1eb36ee6",35144:"bffb4b7d",35179:"10e7c1bf",35248:"74d1ce2f",35288:"991ef11e",35325:"e01489cc",35329:"9e684943",35379:"8ab1496f",35402:"375ee7c7",35532:"1beb2628",35618:"e4f3db36",35730:"f9137192",35733:"28dd7656",35742:"de61f764",35766:"9a1fa293",35767:"dfa52db3",35784:"fc8fde7f",35837:"1e5ce2e7",35881:"c8a2bef3",35922:"4cfe6391",35932:"506269e4",35960:"059789bb",35983:"a3ffd343",36009:"cbae7f1b",36055:"4770b5a2",36067:"fbcdaf70",36113:"2d48d701",36144:"725b0d44",36192:"26c2e85f",36264:"3ee1d673",36302:"72b0bcd9",36379:"b702241b",36387:"251632c5",36549:"e2747c6f",36569:"64ebfa77",36594:"bd886889",36619:"617e0e66",36640:"52eef1a4",36672:"1702d861",36689:"8a2dde69",36747:"e29bc85b",36790:"9de7a9c1",36851:"203b548f",36885:"1ac872db",37109:"0784f49d",37115:"b776b259",37147:"088d2c22",37191:"2d0ea897",37204:"1605f8c9",37206:"af6fb7c2",37209:"9b43eef4",37257:"dceda108",37329:"9a57546b",37366:"160a0072",37403:"c5e2093a",37486:"e7d57a3f",37489:"9a67fcbe",37577:"a1199bf2",37578:"0177c4c6",37617:"8e607010",37643:"4865d39b",37673:"7883a91c",37740:"fab7fc26",37799:"403baa89",37809:"9b6986e9",37827:"5fde1cd7",37839:"05558af7",37841:"670e19e8",37842:"1d81fddc",37872:"e9177a7e",37954:"076d3157",38001:"cb066494",38003:"70d204fd",38056:"20662e51",38107:"43c0bc0c",38158:"51aebc6d",38238:"5390d965",38348:"adc210d2",38370:"fdc0ca5e",38385:"97d9f92d",38416:"20851e98",38439:"461b0f80",38449:"39bb4f65",38451:"6bf97463",38468:"716e8fd3",38600:"5d6ce15c",38605:"f3f72edd",38607:"fbd8a71b",38700:"f325af51",38767:"cd1ba66e",38786:"31e8cea7",38816:"7447358c",38835:"40a51f33",38839:"dae4efea",38888:"6753bae4",38974:"4d5d8235",38999:"35806b80",39058:"effdc4b7",39088:"f567553b",39110:"50177589",39138:"10007fdc",39142:"6a331906",39153:"e21126fe",39192:"ef51e43e",39197:"ea14ffa4",39202:"40e4ffae",39208:"a97db242",39230:"5527100c",39240:"5079f647",39260:"ba5c24ff",39265:"5d0ed436",39283:"79d7db8a",39405:"6242229c",39407:"a7203d6b",39417:"d02a7f26",39423:"17d4a9fd",39444:"a22313d8",39487:"5b0731f4",39494:"f30b5ac5",39496:"ebba264e",39505:"5bda256e",39518:"c3c9d6ae",39520:"27de82f7",39555:"cccbb5c8",39569:"3bef5f39",39587:"af0d65e6",39608:"1a5e64fb",39712:"0a24b98c",39715:"bbc66cf5",39758:"a3fc9df7",39773:"b393be97",39777:"3432c599",39781:"47d6c4a6",39840:"53dbfad3",39887:"e46ee796",39901:"eb9119ba",39908:"6d740ec1",39921:"0e79eaf9",39973:"e8d22580",40065:"b7fdc9d7",40124:"40c231c1",40178:"b61f965a",40205:"c94f090b",40245:"b96db1fd",40369:"92864a0d",40463:"427c10a1",40464:"acd8440a",40555:"f52300e8",40606:"12108f89",40616:"a9f41f0f",40651:"2d49ff29",40690:"a60e3624",40697:"b4dd2319",40755:"3e432e6b",40758:"4d3e9419",40797:"3db6824f",40827:"1a80c281",40847:"cd35205d",40907:"d1b3edb9",40919:"6f0cfe05",40927:"73302b80",40959:"dfce7692",40999:"a9ca37f2",41030:"1804232c",41125:"2c45e5dd",41135:"e8aa7f97",41159:"3ac9a111",41260:"8ca7fc77",41361:"784c9a03",41455:"eddcbb4b",41461:"3443ba16",41491:"56b3d829",41504:"6389ddd7",41518:"d9e6c91e",41594:"af01423f",41640:"9e866231",41651:"616b00c8",41677:"8c940416",41739:"e8f5e70c",41750:"ee28ae45",41852:"3dd9673c",41867:"0e7edf3f",41872:"b0386e86",41892:"9f8c189d",41907:"f8e6d852",41944:"9bd9e858",41954:"e362215b",42068:"5e07d0e9",42187:"d9057212",42196:"71100c70",42268:"6c4ef9de",42298:"a50f4480",42299:"09b7f6e8",42350:"1142cfde",42368:"f8b7383f",42465:"3ff7fed9",42487:"56c461db",42518:"198e0c40",42566:"b345a309",42576:"9a6fd0e8",42596:"7100815e",42690:"f8d6e169",42702:"5f8c0f9d",42723:"4fe95284",42794:"7bc80366",42799:"439a565c",42801:"98f49e04",42879:"7c6ac9f2",42881:"45103050",42900:"a68c15fc",42905:"6bb55442",42911:"5bb7f601",42920:"c438d5bb",42937:"58f6b800",42942:"5cf3f15a",42995:"695ebee8",43029:"0d532445",43035:"dfc57d02",43040:"1e4a40d7",43042:"f5fd14f0",43078:"a02f005d",43088:"49e4451b",43113:"0310d8f0",43195:"92666359",43286:"f606a9ec",43352:"24cd4507",43421:"0a5a165b",43432:"fd8b8122",43445:"93a5e7b9",43461:"b6d04d9b",43512:"df29c6ae",43642:"d6ff100f",43677:"d746dcb5",43739:"fc70054f",43868:"a3261008",43922:"32416b57",43980:"4a4883a1",44008:"643184c5",44022:"6fcc094e",44040:"2dd42aaf",44045:"c0d86ede",44118:"8cf0d0b2",44123:"11ceffaf",44125:"e33c2ae7",44129:"86e86df2",44143:"044af036",44290:"0d453a69",44326:"2ee9dd8c",44339:"b32856ef",44380:"cf224181",44405:"51b730f8",44419:"a05b3029",44489:"66da1d5e",44492:"ec2797ef",44551:"b4882b70",44567:"3c62c94b",44640:"d04281b8",44662:"d947bd90",44682:"087d97b5",44719:"9f19c736",44840:"a727ef1c",44876:"e3d3c1eb",44891:"06e055c0",44933:"1455309f",44939:"e64332af",44948:"07d7525c",44960:"843bad26",44964:"b983a3df",44994:"41cd7968",44997:"3ffd623b",45113:"523985ee",45117:"8c2c2ced",45146:"e85c17b6",45210:"52123a36",45231:"a109a3fa",45238:"f9e62925",45290:"d75a994a",45323:"faea1743",45339:"3bf0bfb0",45460:"550b46d2",45500:"065ed853",45541:"6fd99ec3",45561:"f24f5701",45640:"6c7f6a66",45670:"50c1ba0a",45682:"888518b8",45919:"55919297",45983:"13ee651a",45988:"17b1ea4d",46001:"f432c2ec",46008:"f69f5c00",46025:"7650608b",46039:"f72f1171",46063:"3541e9ea",46074:"6f20e0a9",46156:"7d11bc0b",46197:"df2b2640",46244:"cad26181",46296:"f08eab96",46375:"df5f4d23",46425:"cdb28597",46451:"a6a11df4",46472:"fadf22a1",46513:"56c87a15",46515:"254b9405",46547:"8eec7030",46578:"59c56ca8",46598:"c169c2b6",46652:"63a78692",46662:"e6c9e2f6",46754:"5fcdc1a8",46802:"d659f98a",46840:"2eaf8b00",46850:"b6f5cc68",46888:"4961c159",46891:"3a3be22d",46905:"e3c70d3d",46911:"2300f302",46917:"762b651e",46921:"be8d197f",46937:"5c1b7fd5",46979:"008db483",47041:"255878c7",47060:"9049c3c7",47087:"13a87ecf",47106:"bccff21d",47110:"56dd8cf9",47127:"5f600751",47206:"4cdf2089",47236:"3d4f63b4",47247:"4d3adf16",47248:"816064a8",47297:"eeb342f5",47433:"ac211919",47517:"4328267e",47537:"f3a77ab2",47555:"b251b72a",47622:"09db78d4",47668:"680ae267",47693:"c10aa10f",47699:"492e4477",47730:"4bc36570",47917:"2dde4c6a",48003:"a29cbb41",48023:"0f86cac1",48050:"1fdb78aa",48063:"3a568226",48071:"d796c291",48091:"df7b8959",48130:"e6c193ae",48132:"953d34cb",48158:"d48e413c",48162:"b4c8061b",48167:"4dcfa6c4",48188:"04a8ab5c",48214:"46956ca7",48285:"d6809f4f",48295:"0b50cf9f",48304:"5f48cd0c",48310:"64b1d5ea",48355:"8e22de30",48360:"cfbd7304",48361:"c850e674",48532:"a494db1c",48544:"86f74040",48592:"349dffae",48613:"3f89da26",48661:"3216d8bf",48673:"9d684a78",48733:"c32bd61c",48768:"079b97bc",48793:"5daa512b",48803:"88f5f6df",48842:"f3bf8406",48883:"a235bfe0",49013:"07816850",49046:"9e2e99da",49055:"eb422c4d",49071:"ba636502",49112:"d7a24cb5",49174:"03c624f6",49199:"efe75a67",49226:"b6e04903",49280:"4dfb5a77",49305:"acecb563",49350:"b3dbf751",49388:"9bd7db7c",49461:"5ae94b5a",49521:"60482983",49569:"b4567b28",49579:"e16a6c78",49583:"393a5e78",49587:"4cf9d174",49655:"68089f3d",49676:"2bc5bc8b",49783:"5250a56f",49786:"7b91213f",49880:"54c0806d",49991:"403e6dc1",50012:"68d419a1",50095:"6ed54966",50176:"417fb999",50178:"18db7c21",50183:"1be5d08f",50259:"f7eb5e14",50261:"7008ff7c",50293:"42163459",50361:"bb9593e9",50414:"9d3834fc",50428:"f90b3bfe",50446:"5d436275",50472:"f3bb4c29",50486:"0f8a2868",50507:"8f74cee6",50570:"c66081cd",50584:"8ff63e94",50642:"da2e25f5",50700:"11590bcc",50703:"bd356fa0",50747:"b4c4b315",50769:"bbaddabb",50800:"1ada2615",50819:"551c9c7c",50827:"394e36ad",50839:"7fc947a3",50934:"7c4a4744",51047:"e05467fa",51049:"e5ceb9fb",51060:"7c84a54d",51090:"d3055e1c",51102:"25d73939",51116:"5d147865",51170:"b528a493",51175:"3e23e8e6",51184:"3fbba03e",51229:"24c83fdf",51237:"83d2b73c",51378:"27ce1cb5",51412:"e5d9f101",51420:"51bfdb8b",51438:"153db1c7",51494:"da27cda2",51548:"c84495e4",51578:"260cb010",51584:"24cf19af",51589:"990a5fd7",51636:"b532bd62",51649:"c406b631",51665:"4e1dca5f",51694:"d22c56f1",51711:"6ed1019a",51755:"e7683163",51763:"614f3b14",51769:"4709b0b0",51814:"e54b3ca6",51828:"cce51e7a",51837:"7e552679",51878:"72c458a7",51890:"33cbdd96",51937:"7b8f3921",52036:"4ce3fd22",52042:"f430fae8",52093:"0bc9c087",52102:"8459ac52",52132:"18679067",52135:"24738c10",52176:"9d3aca59",52207:"3fd074b7",52218:"715862a4",52235:"b3475660",52250:"d133f712",52266:"7f7e57bb",52267:"999b9003",52282:"63565f78",52286:"4a595239",52342:"e54f9f4c",52408:"d3db7d01",52450:"99135bd3",52462:"3529eb94",52466:"0060b180",52478:"f31e943b",52488:"2e672211",52563:"63e5002f",52569:"0978d751",52572:"66b2d328",52612:"fe83ac5d",52634:"f1140671",52680:"c4c41065",52711:"07c974a8",52731:"c514a24b",52770:"479aefff",52785:"db98dd4c",52826:"83f0710d",52840:"3fb65c88",52882:"f217557d",52895:"6f6f0499",52911:"2bf6f0df",52912:"9734e003",52913:"3bc2819d",53074:"21f6a3c7",53132:"26e4c97f",53170:"46c4c760",53178:"12177593",53201:"0d2201ce",53237:"9d662527",53256:"de10895e",53266:"ec0e5b63",53288:"58febaaa",53314:"aeb9ccb6",53449:"8c293a1d",53458:"d06e4d8a",53492:"c61db897",53540:"b615600c",53619:"07903309",53629:"0e9da356",53633:"91cfa4ca",53646:"252e93c1",53670:"5d9992cf",53742:"e3fb6085",53771:"503e9c26",53786:"295cce71",53832:"ea177f3b",53860:"045fe6c1",53861:"89eb5be8",53871:"06715a06",53881:"baaec368",53901:"176898ea",53925:"c90ab7e0",53982:"ac914839",53990:"ada2a249",54097:"7fe429a5",54106:"0ae8620d",54200:"d2bacd5b",54240:"9d3e04a4",54246:"60f12cee",54270:"091906f6",54272:"15bec62e",54274:"de7a41fb",54277:"203dd83d",54399:"52bbe215",54415:"f56879e1",54462:"7f730c3e",54487:"c5676dbb",54500:"8b8b005b",54540:"a02f600f",54568:"b4a9dfa4",54730:"18e0d97a",54749:"506a8f87",54759:"03807f37",54772:"1026b8f9",54851:"d6dbe9b4",54879:"50ccf7ac",54881:"d0c3c6aa",54927:"40575db9",54950:"c2be47fc",54951:"d7f390d4",54992:"712f1c60",55004:"a9d23daf",55100:"538b7d12",55144:"9e0ba362",55167:"b4d49358",55206:"0d70c765",55276:"e3d30ccb",55302:"1f220875",55342:"baf46857",55401:"b92ec369",55483:"9bee99fb",55526:"93ae64fa",55534:"25d024d7",55559:"b83a883d",55566:"82130cb7",55579:"4e0724fd",55616:"6658ee4a",55664:"42d281a8",55668:"ee74c105",55754:"89d9928b",55780:"a19a9af8",55836:"96bafd1b",55929:"77f3f6c1",55942:"d374a5b6",55955:"da177b3e",56028:"af871c03",56037:"ab8234ba",56062:"60867da1",56087:"13c91d14",56097:"f053aab6",56122:"74d598a4",56123:"c789ce2d",56133:"762b0fec",56190:"ef64dc1f",56256:"e36aeffd",56259:"68ac1963",56297:"26633708",56384:"fb72bb40",56408:"5bae03da",56493:"7ee5fd86",56510:"0a75fbb3",56560:"3239f135",56607:"48d20332",56648:"0e90e357",56662:"5e826942",56688:"2cca9c43",56723:"1293a3d4",56725:"7b698c07",56767:"e8136b8f",56779:"d6ff4e16",56826:"50015c12",56916:"550c02b1",56948:"46365228",56974:"4839cefe",57054:"6fb01f30",57119:"987dbf1a",57141:"7d7eec25",57155:"e94ea6ca",57159:"052fafd0",57165:"d5789e63",57178:"8bfa9b62",57182:"69123d57",57222:"c8de5978",57245:"8d409ef9",57251:"58acefa5",57283:"21045dc1",57302:"aa7169af",57326:"8b59f195",57359:"8ee1fa90",57361:"4a3a3cb3",57426:"57e1c591",57529:"7bab9ba3",57547:"3a1cad18",57581:"fb0331ac",57594:"1e47fc98",57614:"8825c03e",57647:"56c9542f",57730:"2bca685b",57749:"4bc6c338",57760:"1092dcab",57762:"c39c754a",57885:"5d86fd4d",58e3:"734d343b",58002:"748d2de6",58079:"573a9aa6",58111:"73a02d4a",58144:"36de5476",58255:"5ee8f8b1",58259:"cd10818c",58301:"4a5e22ca",58351:"cdf49691",58359:"dcd461e7",58393:"399932cd",58428:"3ffcb7ac",58479:"44dcc97b",58484:"f70a46c3",58492:"d2645f8d",58581:"83d42deb",58594:"cf6bd0a0",58666:"f2504d8d",58667:"97e291ec",58704:"25df67a5",58707:"15c67f3b",58721:"f353fceb",58780:"d0db0c75",58857:"865eb012",58874:"1a0a7c4a",58880:"1513efa4",58913:"e13e4900",59004:"328f4441",59005:"cfef6f48",59009:"dc31e278",59020:"8f1ae138",59053:"6eeb65e2",59070:"00082207",59104:"12590816",59114:"7e53ee45",59123:"564f8b9a",59182:"15f5ff3a",59262:"aaf827b1",59326:"73c25d16",59349:"c28d69e6",59475:"006f4c07",59493:"631868f6",59521:"c957bd71",59614:"b7e2e03b",59680:"3c671a5e",59702:"69bdff54",59732:"2a459114",59740:"04c67354",59779:"2a5eec41",59781:"f87d58a7",59866:"aa994063",59873:"1880b633",59874:"a8378fd7",59923:"b38aed5f",59940:"f75ceebc",59981:"76c5bdaf",60020:"0f554f37",60119:"5b9bc0b4",60167:"b7af2062",60185:"eb38933e",60187:"7636ec06",60193:"2bef0842",60198:"a93ff4e2",60203:"8d7339c1",60257:"5032408d",60267:"d687473f",60294:"5b4136af",60315:"55f56501",60445:"36846bdf",60510:"ffb2f2a0",60565:"d1460e67",60613:"cb3cc081",60625:"ad1309c8",60676:"81736ab9",60696:"a608baca",60725:"dbd85841",60762:"a573b13f",60765:"2dc71301",60819:"5e525d6e",60830:"00f6a771",60837:"fc61e2ff",60861:"b786146f",60884:"14ad0b43",60901:"0a407239",60908:"4b488b61",60953:"0a991989",60965:"60e4101b",61027:"449d7005",61114:"6ed53af4",61228:"71606809",61235:"b3fca070",61265:"f983c95e",61300:"834cd9b6",61345:"a53b2883",61378:"f5f5ceb0",61416:"179f2544",61481:"27590116",61559:"e81e8852",61584:"8bd4f52e",61627:"269c1cb3",61652:"48519009",61702:"3ab98eb7",61721:"1bede07a",61734:"76fade34",61740:"ae48d0eb",61786:"11544c15",61844:"b500c1f5",61886:"838ac25b",61939:"63ea2061",61950:"d36ac652",61959:"81d6b244",61991:"5a92f8a3",62015:"8ec5ca20",62037:"306ee7ed",62061:"1efccb35",62111:"cd0efc31",62120:"99d514b9",62138:"567ed5e1",62147:"8390c8e4",62149:"f514162d",62162:"5acdb074",62182:"503834a5",62188:"3e551357",62254:"245e993a",62264:"e4528e15",62400:"e7e7907a",62431:"b246e6b9",62456:"43e0651c",62469:"3c915cc0",62501:"de4df872",62549:"301ee9e3",62568:"68871186",62635:"22cbfad7",62639:"210bc494",62641:"75bfbfe9",62672:"6358ea51",62691:"753414c4",62703:"564b723c",62712:"d5ff0699",62714:"fe7a976f",62827:"296a4530",62845:"90aeb106",62867:"672ac76f",62875:"c31ca64d",62927:"d4d0ae7b",62953:"0792ec6e",62957:"6f1cdfbd",62975:"995912fe",63005:"6d110437",63064:"c0bf5dca",63075:"3ee1025d",63124:"34aa01f3",63151:"b9cb4815",63181:"ea8d8fd9",63228:"10bd0d98",63262:"bf4c2c34",63291:"9ffac09b",63298:"acbff002",63317:"811a54d4",63330:"bc478490",63332:"500cc836",63367:"7a6eb5b0",63382:"e45271df",63387:"489643ee",63420:"489eff90",63430:"26ae4f11",63455:"62b0c186",63515:"a47e4d5b",63521:"66036a01",63553:"ea92fe5c",63576:"0cc97e30",63627:"6f0e73b2",63646:"24a98389",63682:"e81f5366",63700:"7e8b055f",63739:"a9c24938",63760:"d6e4a1fe",63779:"8304ce91",63795:"90a8fc97",63836:"c50ffd2f",63907:"7efe218f",63951:"b86c7710",63966:"d1164b7f",63975:"7ff61510",64045:"8cd8acd5",64099:"a0ff270b",64184:"80d3a705",64209:"a2d5ce2d",64264:"304d81a8",64275:"336cbfa6",64351:"38e048a0",64391:"a3d0ce9f",64442:"e6149577",64447:"ab60d1bd",64465:"3c9d3adc",64473:"cf1bf285",64502:"3c32636b",64527:"8724400a",64535:"2a06b13a",64577:"d5c345aa",64597:"16a203d3",64623:"c358ac06",64659:"61c9f121",64670:"9186d3ad",64737:"9eaa058a",64749:"867c6f08",64838:"b35964d0",64899:"bf652241",65013:"9e1d762b",65017:"566badfd",65091:"33aa1367",65094:"9c1741e7",65133:"6b78163f",65193:"c9d51b91",65221:"f8300e9e",65223:"fc7b2fde",65233:"4caa50f4",65246:"b0e81973",65350:"360c493c",65356:"044c021b",65359:"367c24bc",65473:"5e0a91a2",65497:"8399cb89",65569:"b0072d01",65611:"5f89a380",65625:"08954e37",65629:"16ecda58",65647:"24aaa41f",65648:"e4fb5853",65658:"3b939be5",65727:"ded58870",65745:"316ea106",65756:"15cd943a",65759:"819fc53e",65835:"a3453b0a",65897:"a381fb32",65918:"146740b5",65926:"e5ecb06c",65957:"68a2cc05",66061:"45757a8b",66062:"eff2a5a4",66097:"25d8d490",66135:"e4b972b4",66140:"8dff9019",66162:"2b8e16e9",66169:"02c3df8a",66210:"d091b4c8",66237:"c01df0e8",66327:"5e088192",66338:"66797f3d",66372:"b09b0736",66381:"ad989954",66394:"8bded28d",66415:"cb19b5da",66461:"d833c819",66499:"38de97de",66504:"ca582e0b",66510:"24f0668b",66518:"c9394214",66524:"8bc94417",66538:"78f9375b",66559:"e664ab4b",66575:"a2383db1",66602:"5af37c6b",66715:"b7e0c845",66753:"69af30ac",66759:"dfe5642b",66822:"a5b3c79d",66832:"6af1a81d",66867:"9bff7442",66903:"1a459fba",66912:"54eafe75",66947:"c4ee53cb",66995:"784da00a",67044:"f4de3f48",67061:"d9896998",67098:"75ab85df",67100:"9ce14990",67104:"02c01d9f",67114:"8c651089",67149:"eda30c95",67189:"7efc5baf",67229:"600b8e16",67252:"9e177ed7",67376:"e257bb32",67438:"594c8d9d",67472:"522d8645",67489:"982012ab",67507:"f3e3cc96",67545:"326fab84",67581:"3158be0d",67587:"79c7e4c3",67649:"3e6502fb",67668:"b61eb3fe",67683:"d02d9bcf",67704:"90a45e0a",67720:"a584e6c3",67729:"20a9cf6e",67755:"1e073c84",67775:"b9dde3dc",67914:"332bff15",67954:"48148f16",67989:"b94c1f76",68038:"353d6f0d",68043:"e18e6796",68045:"7e4deb9f",68094:"1571ddda",68210:"58aca33b",68246:"f542fe1c",68249:"b11b303c",68253:"c30a59ca",68304:"6409542f",68315:"91b0f913",68334:"e92472f4",68434:"c764b111",68467:"a09054a8",68470:"20998ed9",68515:"48625b37",68551:"847ce269",68569:"8cddf474",68665:"36bf7051",68685:"4b2f7100",68710:"62339bd0",68765:"a74cb64a",68770:"cab2adc5",68839:"6cd72ebd",68861:"7c45d32f",68865:"a511cfb9",68910:"93d8f036",68930:"67d62767",68955:"d59c084b",68991:"fee3dba7",69017:"51df1e44",69030:"105e1b32",69038:"aade7e7f",69129:"af425d1b",69169:"310f0e66",69199:"aea55b19",69347:"146443c3",69365:"e3667910",69384:"efa43be3",69457:"0159f9de",69501:"99f0cd42",69517:"0c43557c",69570:"915a5618",69588:"919af855",69633:"322b0d34",69638:"05f50075",69671:"c73392f3",69770:"c4d5f69c",69787:"13159f81",69844:"3b524105",69875:"32cda754",69982:"3d6dd370",70008:"58761afb",70022:"066cbf14",70058:"943c427c",70100:"572c029c",70117:"b3c36679",70126:"d4115426",70127:"b084e3ed",70133:"9100024b",70174:"54e46a31",70179:"6bd088b0",70209:"f942e04a",70210:"baea7766",70234:"a553ab8f",70253:"f5ded396",70268:"4452b64a",70399:"e6d32efa",70418:"4c6611da",70452:"e2a2f968",70502:"c692a6f9",70551:"5ef3d7c7",70560:"215f7919",70606:"38763243",70612:"b8278f3e",70645:"4df69f83",70649:"416d6f2a",70669:"d4aecb8b",70687:"2dc36e49",70689:"2651b58f",70696:"c124e296",70712:"4c96e1d5",70728:"bf4215fc",70749:"42771251",70755:"3a5a31a3",70784:"0e4575ad",70791:"ad524d20",70794:"0f408514",70814:"2d650128",70825:"6ae11483",70826:"27591451",70828:"8b297c90",70853:"88bc6dfe",70870:"84bc6d3e",70947:"dc575b6a",70953:"0932b99a",70963:"6b5459a5",71021:"48d0f323",71026:"65a1b93e",71051:"bd187ac1",71111:"fc43fabc",71117:"209f2f28",71162:"54a7c705",71227:"c59bf196",71235:"ffab41df",71374:"ce3ef14b",71407:"7a2306d2",71473:"664a5e47",71531:"e7ecea81",71565:"e5390c69",71585:"72b12311",71590:"c9f8c1ee",71612:"5e412c85",71656:"e6be372d",71715:"5b33b525",71784:"fa873b97",71792:"a6dc84f6",71858:"4bc2929e",71863:"92de78c1",71877:"2ec93eca",71910:"16d68f54",71964:"70da7109",72011:"7070f410",72043:"dc1ef129",72053:"6c6b59b0",72122:"23fdf7ea",72135:"5d936594",72246:"0f8806ae",72322:"8245e8b1",72353:"8d7cc6bd",72358:"7f58ed97",72446:"4a887c84",72527:"bc4dd3e0",72573:"dda0531c",72599:"9a7c5694",72634:"2c57b1ce",72668:"a2ebdf55",72675:"062aa684",72707:"7fcd7169",72759:"16cfab86",72815:"370f154f",72835:"07d5ab1d",72880:"82729a0d",72907:"8897656e",72914:"a6b31fb7",72925:"04c27620",73019:"5eb6a457",73084:"48ab7c0f",73193:"fc9f3e78",73264:"6ccefbd7",73276:"4d9bb901",73290:"d109a779",73316:"d595c06c",73331:"3e2d12a8",73376:"df8d786f",73397:"5d1ec000",73472:"2c6afda4",73493:"6c824c9e",73539:"a9e1fae9",73540:"d2683542",73569:"3aebe524",73607:"5d5dfef0",73609:"c5233a93",73624:"7161ba9d",73684:"8c31bfad",73693:"7d0c657f",73741:"c203b285",73756:"e59be998",73800:"bd55d6a9",73841:"d5ca1fa3",73846:"669a7fc7",73888:"46723720",73891:"335d1a8a",73905:"74bdfa12",73940:"e1201e1b",74019:"da089245",74027:"f505499b",74039:"a154b9e1",74046:"80b43a0e",74069:"bf9b50b1",74121:"976ee526",74134:"78e4ce3b",74282:"530f2a45",74312:"1776f3bf",74388:"455542b1",74403:"f4e7381a",74418:"3ab1a45d",74490:"800980fe",74524:"411bcbe4",74588:"4657c99b",74657:"071be4bc",74684:"08c31347",74792:"7bee6d52",74880:"8302c738",74902:"404c15e2",74945:"ae83d79c",74961:"1ab74c24",74994:"50e20aa9",75036:"6923b486",75043:"9bf3467b",75051:"483179e4",75062:"c02cd0c9",75078:"a71daff9",75105:"a4f9c66a",75106:"b618c014",75173:"0650181c",75267:"1905bdc5",75309:"d03c9c6d",75359:"1199c533",75378:"7152a165",75405:"98e83edf",75450:"0e82a67d",75459:"0049814e",75666:"0d1db8f4",75695:"09919d70",75696:"49a8d4e6",75718:"3efe0489",75735:"0b7281c8",75759:"29828f86",75775:"034e9612",75805:"cdf0f609",75866:"b5edd83e",75911:"e2c5cb8b",75919:"5d46debe",76041:"73d8152b",76123:"ee2b987d",76163:"eec9e7aa",76166:"d34b4fe7",76215:"f8abf276",76235:"02562da7",76257:"3ae01493",76262:"5ea97bda",76264:"e627ec8b",76274:"ab0d6c32",76285:"0742bfe0",76289:"208c9bec",76368:"500ae3ed",76530:"595f3583",76641:"2e5945d0",76683:"45b5ebf0",76719:"3fa3d99b",76748:"3d43af04",76850:"b0aa9be8",76854:"54e40af3",76871:"3cc12515",76873:"6ade5fbb",77077:"9dd7d520",77088:"19b3c698",77096:"d15e01a9",77123:"f06a5da1",77145:"470371c5",77174:"7f4d362f",77183:"3e975e2c",77200:"2f3da969",77222:"32602dc6",77261:"34bc5584",77298:"0e6e1364",77369:"789a51cd",77391:"a5d2357e",77596:"f28bf1fc",77679:"3ef8cf50",77698:"897d517b",77749:"57524945",77764:"96ce0e27",77787:"ebaf27f5",77794:"a1d36ea7",77920:"32922c56",78001:"494fb1aa",78037:"380f20f2",78055:"64043d5a",78087:"792a9bde",78159:"4f38a4f8",78199:"7352f166",78312:"4227ef96",78354:"e1a80164",78421:"90fce091",78497:"f3762333",78512:"53548a02",78522:"3a8bb73f",78544:"6a741e05",78583:"16c7d56d",78632:"2e8da915",78704:"8074d8fc",78758:"bd259526",78764:"21ea3c33",78787:"79135f9e",78820:"8b478ac3",78858:"34bc29ba",78876:"59d1e0a4",78880:"57c23eae",78912:"b1c543a5",78919:"dfaaf169",78926:"01258cb1",78985:"b28dc1c0",78988:"76e47634",79039:"56222027",79048:"a78c49a8",79150:"7f9b8fbc",79157:"5c77fbba",79180:"001af72a",79302:"4003d6b5",79416:"d8803eb4",79474:"ce9bb68e",79485:"abc97de2",79506:"c5a051c0",79520:"7ee03445",79521:"34c92224",79656:"aaf407c1",79739:"28dafb21",79841:"bb848e98",79866:"72d729c2",79987:"ec16d719",80018:"8fe0883c",80046:"6b819d70",80091:"cf6c8fc7",80102:"9c77890f",80198:"ae37fddf",80205:"8b305082",80259:"2a9fd9bf",80260:"cb06e1c4",80299:"67a193ac",80310:"825dba4c",80362:"777aaaa9",80375:"c431c14a",80408:"65e68152",80424:"2b9e2beb",80442:"138beb03",80457:"a89f5ad8",80458:"05820e47",80491:"31389da7",80531:"16becf9e",80557:"c57e7621",80598:"f70e808c",80632:"3aff62bf",80649:"47ad624c",80677:"4c372437",80744:"595e9d0b",80750:"73b5b786",80772:"832cafa9",80781:"304645e6",80794:"1edbd938",80841:"aefb64da",80863:"9f9ec4bc",80947:"39b525ca",80957:"97a18955",81012:"7a14d07b",81019:"0757a63a",81050:"4ca32e8c",81085:"e860a612",81125:"4f6a5985",81175:"cd0071f2",81188:"9d386ddb",81212:"6b747f92",81285:"b35bde59",81300:"8a132e02",81319:"5eca8594",81346:"96f50bd6",81390:"35ed9e45",81410:"1b63b983",81482:"bf8e657a",81530:"c7de49da",81544:"9f053080",81583:"5d60f872",81639:"2f3e574a",81666:"310e4ba3",81692:"cd67aa57",81737:"bc202717",81761:"c2e8f014",81770:"f9f0bc0a",81791:"514758f4",81795:"fe21b98a",81802:"33f4867c",81891:"85e8a888",81902:"972577b8",81903:"6ea756ad",81943:"bda6f1f4",81982:"45b0a42d",81990:"7dbdd021",82012:"b83ac31c",82019:"09be3a79",82237:"96c3210a",82242:"a8849529",82261:"65a3a905",82297:"4ce54554",82354:"e6079d9a",82362:"4bfee9a1",82431:"c58b9a06",82456:"a76fa164",82486:"6ee2f109",82553:"b2f3c04c",82581:"4ac5a8bc",82588:"22192259",82663:"e2968488",82722:"a43dfc55",82754:"1f4c6ca0",82847:"f1160e5f",82905:"facce5bc",82939:"3ffe9e05",82987:"27acfdea",83027:"eff4badb",83078:"e1a47202",83089:"c7ada7b9",83103:"3840f51a",83109:"6414a4ba",83111:"56e10872",83121:"006dc26d",83127:"f69c75fe",83182:"cef47da6",83203:"6e8afdc3",83213:"569e617a",83249:"32d3403c",83260:"c6525628",83273:"38f30c97",83316:"b0b27b89",83378:"aac56db8",83470:"71c00062",83574:"a3dc9dce",83608:"ee9f57b5",83670:"8fa9b576",83700:"060932b3",83779:"50eea83c",83797:"ca389040",83824:"09a61fd1",83841:"e29e4267",83849:"a66ff3dc",83928:"6c3fffad",83932:"7e88d302",83954:"0e32797d",83958:"bbdcafaa",83970:"2b543913",83976:"968c9e99",84013:"6a35ce6e",84070:"f86c329b",84107:"cdeafde2",84152:"00a2579f",84227:"dd167da5",84232:"fb6c362a",84255:"5f7c3639",84259:"a83b5248",84319:"add20ab0",84332:"ed66d686",84527:"a2acefd3",84530:"7b34a9d4",84563:"692274ef",84572:"790e2aae",84760:"175a5b9f",84773:"2493d544",84813:"39aa10b6",84869:"b3c6188c",84963:"75f5a097",85014:"3b5a5dea",85095:"6e925ba8",85097:"1682c825",85105:"b97600ae",85205:"b80556e6",85209:"5ee954e1",85243:"01e4ab56",85257:"b95de7f5",85279:"9d896a2f",85322:"18b44b2b",85358:"30512b2a",85442:"7970ece3",85453:"fb1e1d1e",85537:"c0626242",85581:"7500bb20",85588:"53c18d68",85736:"48cb5964",85796:"e203b806",85806:"fbd285ee",85817:"f02bbb8e",85896:"e9b359a5",85899:"ef3290b1",85917:"e7d86b26",85960:"f16cb757",85973:"b1b5df36",85997:"db6152cf",86053:"4dc95938",86074:"741302c3",86121:"594f4aa4",86129:"7cb605a7",86167:"3ce460dc",86215:"77a5d5a6",86216:"bef3c0fc",86246:"e0349554",86354:"bc7c5dc8",86417:"a1124a7e",86434:"dc8e21d6",86478:"84cf68a0",86480:"06949ebe",86501:"9d7d8716",86586:"8c579e0a",86597:"9bd3dc22",86623:"a74edf8c",86659:"0e19cb86",86669:"3e544ff8",86685:"45e653c2",86807:"3d614f42",86829:"00572ffd",86846:"9070c2f2",86856:"28df0c06",86888:"0615aab2",86890:"fce759e0",86914:"0fb4f3b2",86939:"8b586edf",86992:"8a207a53",87004:"c486e5ea",87087:"05aae5c6",87159:"443cc7aa",87184:"a039315d",87186:"57a6b3ca",87205:"b2690949",87217:"1f696713",87269:"35be2680",87282:"c1f408d5",87377:"bbb33c7e",87407:"5da9e9b5",87432:"2a58f20b",87456:"30fa1b01",87459:"ca9b021b",87473:"7572fe2b",87483:"e76cd711",87485:"074d2936",87547:"8a3e31a9",87589:"a5c128db",87614:"29322f97",87688:"abe56e79",87747:"99cd7148",87781:"89e64bd0",87863:"5fd2edbf",87875:"a9ccf0bd",87880:"bd9c4f07",87905:"5b6b1941",87945:"05f869bf",87971:"1b696ed9",87995:"341abd86",87996:"23c299a7",88047:"a9a1b642",88111:"6e8bb269",88163:"2c3c6b0d",88222:"9b39dc9b",88248:"f8851e5d",88298:"10ee8dff",88310:"36767ebb",88327:"774b80c7",88393:"5ffb00fa",88394:"903eae4c",88399:"c0edc4c4",88436:"2424bc8e",88441:"98d13908",88468:"ee29a68a",88478:"46e46ed6",88522:"8af566d6",88547:"7ac33681",88622:"441ab315",88626:"5b4795b9",88654:"eadecd16",88696:"3e68290f",88775:"06f05fbd",88815:"c077e43d",88817:"9e1a7664",88821:"cee6cb12",88847:"988a32ab",88852:"414f6661",88855:"0a2c96aa",88865:"e8542a46",88903:"e122d6fd",88986:"0bc77a5d",89018:"8a55ca24",89093:"b18e7a97",89097:"aa5fb21a",89172:"f79b59df",89175:"d0983721",89210:"395fbb55",89289:"d1f4bf4f",89338:"efe97fec",89353:"e7814320",89362:"780293e3",89402:"4de66462",89415:"6fb43246",89437:"90d090a3",89455:"f8dbee27",89489:"c8cc5158",89534:"da6951a5",89554:"a784419c",89610:"cc130b82",89641:"9d5b345b",89696:"abcb22e4",89801:"cb16760f",89810:"9e2ddc23",89858:"8b837402",89873:"0d66abd8",89986:"49804246",90066:"ecb816b0",90071:"f51c662f",90110:"cf0b8805",90115:"f497c160",90135:"d3605a71",90214:"23fc1725",90257:"6105e05b",90280:"9e2d7785",90317:"0dbcd032",90378:"47701034",90414:"4258fc22",90416:"410119bd",90438:"817c940d",90513:"91bc9a3b",90562:"b9ee764a",90593:"eac7c7dc",90598:"bd31cfd9",90625:"183eaa34",90627:"69608c40",90717:"f97b214a",90749:"f70bc7ca",90831:"3b92aace",90842:"ea14dbf2",90849:"39cca96c",90910:"0098609d",90950:"e0bca964",90972:"4dfbb0c4",91025:"baab9bea",91058:"6d57ca17",91109:"22f0de7c",91187:"ff56e27b",91203:"7e4cc839",91257:"5b4a526c",91305:"c2578a79",91319:"302da679",91333:"bb1412e1",91359:"67ea2072",91405:"c7144869",91428:"d8c3b11a",91530:"9b1454b8",91540:"18f2c91b",91581:"2c4a9a6c",91587:"5ac8f274",91597:"1fa2692d",91621:"67795e38",91695:"55515b28",91724:"20a3ceb2",91787:"bf5ab72e",91844:"46dfa8ad",91895:"5d4b082b",91916:"9ebdbb47",91993:"8c22cc82",92018:"8b6adee1",92031:"fe024597",92105:"5cefee77",92125:"683c7611",92196:"4b555b09",92223:"38a9da1c",92241:"28e0f6e0",92273:"f9a086ad",92329:"021f8e8b",92410:"4f523206",92412:"0fea7b37",92475:"3181ced3",92535:"63822ab0",92544:"15f57ee3",92653:"c1972b89",92664:"728dae56",92673:"39debf2e",92955:"89495ccb",93057:"a805415e",93085:"6b1e261e",93166:"5f6aac76",93206:"a2bf9194",93209:"13044cf3",93210:"85ec11dd",93269:"1029ee46",93273:"a527fd2c",93341:"7dde88bf",93361:"3d0e28ec",93362:"b3248d4e",93438:"eda79199",93442:"8344dacb",93534:"ede04b1c",93546:"d382b725",93602:"56ae7f49",93636:"259ed984",93795:"395d0490",93849:"00e1630b",93854:"c47af2be",93884:"e3d67ca0",94e3:"789c1bc4",94065:"6e8f3ab7",94082:"b6dbf702",94091:"578df47f",94106:"7ccfcc6b",94153:"b039c689",94168:"d6078fe7",94227:"970cd00b",94251:"9e3cf38b",94314:"d3489683",94344:"cd30ed80",94361:"9a28b459",94417:"394f998e",94538:"f82ebe9d",94555:"70e87010",94561:"a0b349d3",94568:"5a296dcb",94625:"98b3088f",94646:"e5304cee",94648:"f6228d59",94685:"438776dc",94711:"611b1dc8",94713:"58810f4d",94788:"6fc1dd4f",94796:"4010c4d3",94841:"89ae3e4e",94843:"828d0dc0",94863:"55a59675",94864:"7c75a8b5",94910:"d465e058",94925:"8f642dfc",94926:"aae8d19c",94980:"93fccaed",94997:"d853cfde",95020:"5e8185a7",95058:"75d979d0",95078:"41cdca60",95093:"423cea85",95136:"797d7fa0",95159:"1a0d6a0e",95183:"dfc87c30",95187:"85f160a6",95231:"b9c65497",95240:"7c6cb8db",95245:"dbd1b852",95330:"8e4b3ab8",95427:"9aa9c150",95443:"f7ec7ba1",95470:"87fd0551",95542:"91ccb5d3",95605:"6c76232e",95641:"0addadba",95649:"6a65e3ce",95659:"0a1f4efb",95768:"80c58542",95785:"48f89240",95818:"16d5276d",95909:"aa84deac",95911:"529b3233",95912:"423bd8a9",96022:"afa72882",96086:"085de370",96155:"e89275d6",96225:"fbcf69e0",96239:"843c315c",96268:"8b0816d9",96276:"82ebc6ff",96325:"75b0ccfb",96425:"769273ca",96463:"f832da4d",96470:"be797944",96555:"d21049e9",96652:"2f23cc7f",96693:"1d428461",96745:"6967bd1e",96771:"dc2aa01a",96778:"f5680f39",96793:"e0a23a9d",96881:"d4be0935",96963:"8985d561",97006:"598d7afd",97033:"fe730233",97047:"87cbec5b",97107:"18c4752e",97114:"9cc27416",97123:"040c8599",97167:"1238b646",97189:"23708f52",97223:"c8e4b82c",97246:"17ce0e36",97253:"b41c2a94",97296:"a57c26cb",97342:"52841075",97358:"c769b875",97379:"57d3846e",97420:"81fe43ee",97424:"0223e146",97476:"187bd928",97536:"b4e3b3c7",97554:"005cc38e",97562:"bb8d8d2e",97585:"59a181fb",97664:"4e7bff16",97670:"a4274e75",97732:"0b31f7d8",97737:"2f8afde7",97753:"53aae94a",97763:"f430820c",97821:"f80efef2",97829:"5d36ccb6",97863:"f8cc590a",97887:"8c7c5b59",97895:"05685ac0",97901:"67fd6666",97905:"fd91bb9d",97913:"43f18326",98012:"6c29418a",98020:"96dae171",98021:"460aa0c1",98030:"a757e267",98058:"1c320273",98059:"6e7c26fa",98087:"0b0abad7",98094:"4dd0cc8b",98139:"7eab7866",98147:"7e92b0f4",98165:"2a383035",98237:"4eecdd21",98271:"a3781957",98308:"82707bbb",98311:"2e182187",98392:"2f73849c",98425:"bdd345cf",98473:"34829e79",98477:"ec2092dc",98534:"15f60b76",98700:"1fd78c55",98750:"89bad922",98758:"6d0f7cf8",98761:"e46ef735",98779:"84885d5b",98804:"6f3a5963",98867:"b7ac1bd7",98908:"e0e363a6",98925:"57095a25",98935:"86078de4",98941:"dcb08cf0",98947:"5f9ce70b",98982:"bdfdb56f",99001:"bc35de59",99095:"ee8e46b4",99146:"7b22214e",99151:"42d6760d",99252:"8f9df3bd",99254:"0688518f",99299:"a59c607a",99306:"56c8bfe8",99330:"3327c94a",99335:"b5287441",99437:"2c8966f9",99451:"b554daf7",99502:"8b774927",99504:"2e6b3e55",99528:"7e312bbc",99531:"e8f43f56",99534:"428df931",99543:"b88fd084",99611:"1da1e411",99644:"d506a7c9",99678:"1b66abd1",99701:"10059403",99705:"1a05760e",99709:"3640fd16",99710:"ffd9ccf4",99746:"4b4c2ec6",99747:"9fa0b3a5",99872:"d78a1b1f",99905:"0b59e46f",99913:"7b5a7e5b",99987:"e88defa7"}[e]+".js",r.miniCssF=e=>{},r.g=function(){if("object"==typeof globalThis)return globalThis;try{return this||new Function("return this")()}catch(e){if("object"==typeof window)return window}}(),r.o=(e,a)=>Object.prototype.hasOwnProperty.call(e,a),c={},b="hudi:",r.l=(e,a,f,d)=>{if(c[e])c[e].push(a);else{var t,o;if(void 0!==f)for(var n=document.getElementsByTagName("script"),i=0;i{t.onerror=t.onload=null,clearTimeout(s);var b=c[e];if(delete c[e],t.parentNode&&t.parentNode.removeChild(t),b&&b.forEach((e=>e(f))),a)return a(f)},s=setTimeout(l.bind(null,void 0,{type:"timeout",target:t}),12e4);t.onerror=l.bind(null,t.onerror),t.onload=l.bind(null,t.onload),o&&document.head.appendChild(t)}},r.r=e=>{"undefined"!=typeof Symbol&&Symbol.toStringTag&&Object.defineProperty(e,Symbol.toStringTag,{value:"Module"}),Object.defineProperty(e,"__esModule",{value:!0})},r.p="/",r.gca=function(e){return e={12164231:"43113",12935692:"36790",13171783:"29452",13883961:"54772",14828316:"41944",16243474:"98020",16640689:"53619",17896441:"18401",18891827:"71235",21719437:"65957",23348138:"57155",27077527:"44840",27211971:"57547",30769210:"44891",31863783:"2256",33241236:"33621",33732613:"14363",34689118:"51937",34900881:"79520",35313843:"23641",37844666:"63075",38836367:"99913",39583919:"70606",42545929:"37827",43764587:"32949",47074836:"39260",48455675:"66715",50536743:"43078",50737734:"21373",52166518:"76123",52375367:"70689",52676037:"54487",53317765:"39058",53798754:"21191",55946737:"45113",56040849:"72815",59171146:"67989",59700627:"95187",60165173:"44719",60753781:"68470",61773850:"22504",63786366:"93442",65511420:"32875",66405406:"89534",67474760:"63332",67922501:"65221",70570476:"12513",74555694:"79302",75914136:"65091",77787075:"36689",88937603:"61939",89514438:"17709",91415840:"9003",92328698:"20924",93340492:"41159",95789563:"59923",98749210:"96963",98826249:"68315","0583dad8":"16",a33c645e:"25","0da1f28a":"28",a978e5ab:"39",a674d2f7:"48",e9bd6ef2:"60","56169fa6":"67","6c6aeb2d":"106","275b5c51":"129",af8190e8:"159",b1f77b90:"217",a1a7cea0:"278",fcc9c1b0:"292",be9655ff:"323","43d16b11":"335","0b4da2f0":"354","19ac452e":"387",a168e2e0:"437",b4a99b76:"569","0a87a91b":"614","5c109a5d":"805","79cc09db":"822","8b10f6da":"851",e9038c17:"878",ce3da75c:"910","02b5547e":"917","9699c87d":"924","0c459987":"940","4b935b7f":"1103","060b2af9":"1107",e3b4090c:"1119",ae09e9a1:"1136",b0127f30:"1190",ea1c8f1c:"1251","62a07427":"1284","9cab4fb3":"1337",eb8e7dea:"1360","562c897e":"1422","50f3bb83":"1442","4b25608f":"1506",da868c82:"1517","8425e24c":"1550","73c8d421":"1635",cc5200ca:"1656","7a51e13d":"1658","89dd9b54":"1784","269ce614":"1785","4b2ed2d0":"1791","92bb01f4":"1841","9f349e0c":"1846",d032d8fc:"1857",c28fec2b:"1882","6e47a3c2":"1928","45b94beb":"1946","7fbadf7c":"1955",b93dc3c9:"1983","3c15683d":"2016","7a3be72c":"2053",fdb4abad:"2121",ef5805c5:"2172","32aa7863":"2214","9781112c":"2225","2b154460":"2239","7d9ec3a3":"2262",d2a01f74:"2298",fb4f9065:"2385","3b3f1ad3":"2450","1b7e3feb":"2472",e8b490f7:"2540",b9e56da2:"2577","372d2263":"2638",dba6e014:"2640","9ed88412":"2652","9d0312da":"2663","7e9327af":"2679","5e58268b":"2700","4c5bb0e3":"2739","7c87f31d":"2748",d28b7dcd:"2759",ad32f2e8:"2768","48f24bab":"2772",ba2305d7:"2791","00cd9d46":"2839","093ec6b1":"2854",c03096de:"2867","8539bc43":"2884","404a423b":"2912","580eda40":"2932","6e7eafb1":"2954",a80470ee:"3075","32dcc326":"3125",d29911ee:"3131","4f7cd6bb":"3137",aed1d196:"3144","5c3bdea9":"3179","6be904db":"3303",a3e90044:"3313",b9073401:"3323","1c4da86a":"3471","7fb3fac1":"3558",b783cafb:"3693","5e85aa31":"3694","50cd3db4":"3722",dc1f5b39:"3740","394bb252":"3764","43d31177":"3847","260d7fd0":"3940",d18b6106:"3965","50494a26":"3968",c943b655:"4014","04ae9608":"4028",b4ced939:"4040","0bf991f2":"4060","817ed3cf":"4074",d772531a:"4117",b18f8924:"4133","1a47b5e7":"4137","0c12eeea":"4205","18ffe98c":"4269","1497bce1":"4355","25fe5fdb":"4368","56471e5f":"4382","1dcf0194":"4405",ff5a6f64:"4467","3e8d408e":"4514","1cb4ff83":"4530",d9f219f0:"4574","405284b3":"4591","9920fffc":"4602","43d0cc21":"4777","3b6474f3":"4790",a0c4b68b:"4863","7c52b6f4":"4890","4929a6fa":"4919","066b815f":"4949","006c64a3":"4980","6880fe95":"5008",f5c507c6:"5028",ca8c3af7:"5029","25ed107d":"5065",d0ff2798:"5193","8c7f4a45":"5195","4b9df2bb":"5206","627b1f39":"5269","6ff0c558":"5270",b3a451d2:"5289",bbc3d059:"5295","44fefb3e":"5304","22b4f68c":"5322","86642e8b":"5334","0e11b6db":"5338","0c74166d":"5594",fb863cab:"5622","32ba6a4c":"5631",a804bb8e:"5633",d03e0408:"5647","58c4a37c":"5679","42bda306":"5721","1adb4ab6":"5806","6a34bd67":"5834",b5960565:"5887","478a7123":"5888","4959f63e":"5936","959a592e":"6022","87c914ef":"6054","384f1d9f":"6090","0d706529":"6158","14ed99c2":"6170","0ba05648":"6203",baed6a8a:"6225","59ba4d58":"6289","14a73a68":"6310",ce3c8465:"6350","207e0bd3":"6368","4658250f":"6371","5c865244":"6372","47ddb104":"6407","3faff22d":"6467","316bb886":"6475",ae59a273:"6552","21b16c5b":"6559",f6a1f09d:"6636",ea2cf37a:"6677","0cca5757":"6692","17ace3c3":"6697",debf2b54:"6709","01ceb854":"6739","507d925b":"6761","9cb5abfe":"6778",ae0384e2:"6791","4f684eef":"6823",b52407fb:"6897",ac385e9e:"6930","52f660a4":"6944",e841f655:"7035","6e78b29b":"7071",e7c12d1f:"7100","9d67d055":"7103",b744ccc8:"7125",bfd461a8:"7167",b1b9fe16:"7317","96f48bd9":"7355","502758cc":"7444","8fdc7f76":"7458","4abd5d51":"7517","12a626af":"7566","9df106ad":"7622","2a74f6a7":"7629","06a1e3b2":"7715",d6b46894:"7734","79a2ad2d":"7747","869d1599":"7773",deab0229:"7802","68c81474":"7850","7ba20019":"7907",df4b657d:"7953",f85e4dd8:"8003","0480b142":"8070",addb1e87:"8139","464278e7":"8174","38122bdb":"8177",c7a3e7d6:"8183","934057b1":"8192","01a85c17":"8209",c286af15:"8289","09ff3d76":"8348",db9fbbab:"8354","24179e00":"8363",abd23114:"8367","7a049829":"8407",cd17745a:"8413",bbced460:"8417","497095d8":"8585","15c2f51a":"8619","22351c69":"8632","212a125d":"8645","80bd773e":"8651","8f08e203":"8699","1cd42d14":"8762","7a11cfa3":"8766",e161c803:"8840","3a6a48c9":"8888",effd3c61:"8889",eebb3a9f:"8900",dc8afb47:"8902",b6e28713:"8965","6d5644d6":"8978","7c9ee56f":"9045","4c492fb9":"9066","52b6ceb8":"9076","29c12ff9":"9127","17666b14":"9152",d4eb85a9:"9189",b2d2727d:"9195",d336ca2e:"9228","0687df23":"9257","29a0dcae":"9262","4c14260d":"9351","7fd1d1a0":"9379","409e52b6":"9380","8ea09047":"9385",cfb1206c:"9428","8cd4ff44":"9432","49b92117":"9446","762ed4b3":"9458","01964c8d":"9460",ebf12a38:"9471","953153ea":"9499","2fec12c7":"9516",bf1307fc:"9525","8eea179e":"9570","13ab27aa":"9573","79e67a34":"9631","5e95c892":"9647","747973bc":"9650",e4d28161:"9676",a0d28f55:"9702",c35662b8:"9722","46e7f252":"9724","0d37dd31":"9737","6a0b8fcc":"9790","2fd2285d":"9836",d6d00b14:"9860","36e3724e":"9870","8a5cd0c4":"9891","9c0f572d":"9907","06b7dbb6":"9914","9ee88b57":"9944",c782ecef:"9986",c2277200:"10030",f89af623:"10118",f7519df0:"10181",ce5d6b55:"10198","0eb44ec6":"10238",b69fdc1c:"10285","8c29db87":"10354","0ca8e4c5":"10472",dc8e8e39:"10488","40011a75":"10492","79c374cf":"10535","1ca50c03":"10545",b1051fbd:"10610","6a326c1d":"10637","936cf648":"10642","5bf02344":"10653",f71afd42:"10688",f53cf472:"10727","931768b8":"10767","10fb8f29":"10789","9009d9a7":"10884",ac65ed35:"10887",b964b0d3:"10907","42746bb6":"10920",f0c78ca3:"10924","726b1d2e":"10956","90db14b9":"10978","9f1f18ec":"11052","7c44e50a":"11058","2750cc84":"11111",f97de939:"11156","2e961a80":"11275","82b971d3":"11289",ac338f84:"11299",f9cb0cea:"11326","6ed20719":"11355","039c7621":"11508","3d026c66":"11565",d2a270f8:"11631","9aee6bed":"11684","8fa68fae":"11716","51e360fd":"11725","446810fe":"11745","4cc89b71":"11768","3f65fb56":"11782","43ff550e":"11795",b9168984:"11804","7cb0ba19":"11806",f630dd79:"11842","51e592eb":"11855","43c57b15":"11869","41b6dd58":"11905","757b41cf":"11906","608a85b6":"11977",d25544dd:"12047","088c57bd":"12134","5617941b":"12144","3deb4153":"12186","77a5f3f8":"12316","9b28f794":"12318","2592f25d":"12378","7d05d2dd":"12393",a0fb693a:"12395","61c30610":"12431",f4a568da:"12452",ac9a039d:"12477","3e8ec4bd":"12485","2ca30cc7":"12535","0bce5bfd":"12542",ae632d28:"12608",bb6acec0:"12613","98af85a1":"12654","755aa830":"12679",f81e2128:"12686","0166538a":"12763","236efcfc":"12869",bc561320:"12945","72ed5e19":"12995","9b078b3d":"13021","075d0b63":"13060","6695bce8":"13063","4d01f822":"13152",a5fd3751:"13203","326f3c2b":"13210","814487fc":"13236",be65306b:"13298",bb230ab4:"13321",c7690f1a:"13366","0c9b8a46":"13374","15be7f5e":"13375",cdd81d53:"13406","05f125a2":"13419",f7887fd4:"13465","3f9b0969":"13511","35f2b245":"13514","559e18a5":"13522",aeb9cb2b:"13525",a221d960:"13534","27aec4ea":"13535","1b3749bd":"13575",b7ffc82e:"13580","16444e78":"13626",e32c93f6:"13698",e4126631:"13733","35ace877":"13762","5005f1cd":"13803",fdf3b0ba:"13849",a586ac12:"13918","79c522c5":"13943",f4dd7c05:"13982",bb5bef2a:"13996",fe0a5e13:"14003","4fe2812d":"14027","6c6eb236":"14042","2da40935":"14043",c58d5881:"14125","88677a17":"14134","5ba7f3a0":"14136","0c792305":"14152","79f5ed7e":"14163","0466f8a1":"14179","973a6c2e":"14257","4d62164b":"14259","881e0310":"14261","15b03510":"14264","2d2f3863":"14317","02704d8d":"14340","4330e2c5":"14388","95a29cbe":"14399","262f048a":"14410","5a1ac6cd":"14433",a2218d2d:"14435","0dc66419":"14496",b812e3a7:"14536","04287605":"14599","6b0d4b59":"14627",cb290368:"14726","8049dc2a":"14750","4193b389":"14802",da9a6aef:"14822","3145e86d":"14835","9c9e6d14":"14868","281df6d1":"14871",b2f79aaf:"14917","68aea991":"14919",b634826e:"14943",e8bf575a:"15038","6773b5f7":"15078","461201e5":"15079",e6691bac:"15144","29fcaeb8":"15165","809a1b46":"15186",e0719818:"15220","246d116d":"15235",ee5dc595:"15272","7bd6096e":"15292",a0583d0f:"15316",bbbd3731:"15403","413b2e2c":"15407","63067f40":"15427",ad07d5ed:"15473","259868d4":"15501","1ff337bc":"15552",a309ba1a:"15607","0d11ee2a":"15764","65df1ffd":"15805","520bf372":"15857","04d2875d":"15917",f05cb9af:"15947","1ea67680":"15955","777c5b61":"16017","7f5a2db2":"16032","5685cb00":"16100",f0db470e:"16209",d05ef008:"16231","1839a246":"16319","8eca1d9c":"16377","58c0e7a1":"16499",b5eb2b1f:"16500","3d8e248c":"16622","4ccf094c":"16641",ab84da8d:"16747","7c3644b1":"16885","426a3c0f":"16942",bd7a3684:"16950",a3848d24:"17009","30e67995":"17033","3687929f":"17115","87557f51":"17119",bc36781c:"17232","82fc01c6":"17294",b4762a45:"17332",de821e02:"17368",b629aa71:"17402","496a5e3a":"17411",c5c8072f:"17441",f7750a15:"17474",dddd9631:"17502","80e60ab1":"17508","8c212f85":"17530","46e89fee":"17581","47f3a505":"17626",a2af438a:"17681",b1e22259:"17739",bc4a985b:"17805","2d9024ae":"17818","7cfc22c1":"17860","9130943e":"17867","36f96221":"17944",f67021ea:"17987",a3255c75:"17988","059d2658":"18006",c1fa975f:"18033","7707ed5e":"18103","3a2db09e":"18121",c15d9823:"18146","889bcd45":"18350","670426f7":"18351","7eea383c":"18438","762cb2d4":"18445","905fdac9":"18520","1bdcf2af":"18620",e798c07e:"18645","3b375bfc":"18738","10b6d210":"18757","6c76cc4e":"18798","6cc2587c":"18940",d75d009b:"18950","7487c019":"18998","378587af":"19042","6e547e58":"19052","6f878c05":"19070","6f3d983d":"19108","868e6cc1":"19109","9c3fccaa":"19136",e77e0240:"19237",d9884433:"19252","1354c4fd":"19267","345a8726":"19271",a2cfe627:"19369","9228ff94":"19387",c61ed569:"19396",efd9e5ca:"19425","4b6ad6c1":"19428",a3c850e8:"19474","3faa4dcb":"19513","7f5a4e86":"19521","3548485f":"19542","76b52ed8":"19664","547af68c":"19687","0e4a3360":"19711","9e46f997":"19736","3c6c92ad":"19774","83c71362":"19798",a5052667:"19801","7e53000f":"19865","389b06c3":"19891","99bce14a":"19999","0ef00d46":"20038","430ee71a":"20103",ecc82a52:"20160","69fce7d3":"20194","5aae4c30":"20229",f60a8e85:"20286","6b7b009f":"20325",a55cf427:"20358","784d2d5b":"20361",a170c4b2:"20405","839320c8":"20415",c97a4f9f:"20424","39c0c867":"20434","4c056f0f":"20491",ca16b0fe:"20535","2925f13d":"20541",eae39fd7:"20619","874c4928":"20676",ff781369:"20713",a827f20e:"20812",bd440596:"20857","1d99c39c":"20892",de69fdc1:"20931","5ef30e9e":"20951",e3b4fd8a:"21070","7ff678ac":"21077",c1b7702d:"21121","574aa417":"21177","58f98693":"21273",e737da10:"21275",fab95ca2:"21290",e4bea80c:"21303","266d37b9":"21316",b23dd436:"21341",b2c62a3d:"21346","5105c354":"21401","9db9c0e0":"21417","995d9cd2":"21448","5f404bc7":"21482","69f9ca18":"21524","6c888f35":"21560","263bc6da":"21574",d17a9646:"21585",aad02470:"21630","6224d640":"21635",b1722c22:"21662",ea9d899a:"21729",df2a910d:"21764","1926d3d7":"21805",fe2d6fd6:"21816","7ce47929":"21837",a54a9a31:"21874",d3de5519:"21909",a8a7f86a:"21921","625beb64":"22141","334d7764":"22313","83a16324":"22369","9a5e9795":"22385","47425d5b":"22428",f93d9ca9:"22447","494588b9":"22472","95d688dd":"22483","2923e8f3":"22499",ff15e2eb:"22501","6f37f43d":"22534",ca5d6ba8:"22595",a36f1f19:"22628","69375b61":"22640",c760fa6a:"22647",c699b29d:"22673","74c5eee6":"22709",f701b058:"22750","70db0e3d":"22816",d6ededeb:"22844","9a29e675":"22911",d49fdba3:"22916","0d5f3865":"22932","70a32eac":"22973","1fd06df8":"23002","07da43c6":"23041",ef1c4348:"23081","7050405a":"23083","39307ee6":"23119",a1c39b3f:"23141",b994507e:"23172",fe86f3f9:"23332",fa5f4abf:"23388","95d97af4":"23438",fdeab78a:"23475",a67ef186:"23547",f24f712d:"23563","53746e73":"23591",af9b00f1:"23600","996abb7f":"23625","59e9dc06":"23634","9c150758":"23637",cb16b650:"23666",d235b83f:"23676",ea8366f8:"23700",fac8c34b:"23765","2153fb85":"23773","3affb2e4":"23880",be354741:"23938",afe4349e:"23998","9c7497bb":"24022","5ef1cef9":"24065","4ed60a8a":"24084","89ad6a53":"24098","897856d4":"24126","5ce7213e":"24140","1d2c1565":"24172","3af3061a":"24234","7ebf6f1e":"24265",a4e075be:"24340","99d3870c":"24366","5cbd92da":"24373",b5b376a0:"24377","4066930f":"24381","77b23935":"24437","045bd6f5":"24475","6cf93e89":"24476","3befffe2":"24704","91c64349":"24728","931beacc":"24729",e2c09daf:"24741",a852ca5b:"24776",a68f650d:"24815",ed6ec893:"24826","7cfd7c3a":"24853",e0c51b98:"24866",b4972e92:"24867",c6301cab:"24868",f0709bee:"24891","8e01a36e":"24912",cf6129a8:"24919","51fff4d3":"24932","231cac3a":"24970","1e3d98df":"25013","9cd11b72":"25044","9531441e":"25067","8bd2257d":"25101",b277530c:"25130","96046d19":"25186","19531bab":"25273","1da16fcf":"25285","537936fe":"25335",d78b8cbc:"25396","4083b748":"25397",b973fe72:"25426","5ef8e026":"25452",af5c68f9:"25531","40ebd41e":"25532","7fb09d55":"25539",c7382291:"25578","65603bfa":"25582","7205cbcf":"25601",d56bdeaf:"25778",ba115cad:"25781",d2c3f17d:"25845","6d1cdd35":"25869",c9953bfc:"25871","17268e53":"25874",f7f19049:"25959","1645b95d":"25987","92db1d5a":"25993","90c5833e":"26003","20f9256f":"26009","7662a79c":"26060","2316ae30":"26064",de97f7b0:"26081",fd6c914a:"26183","4e3e1e17":"26216","477cc2f7":"26260","878dce9b":"26305","9579dcb9":"26333","65009e94":"26337","14fd0d1c":"26414","9a5789a7":"26548","8eca70a5":"26577","8353278b":"26627",ba839983:"26630","960d3eb8":"26632","1802ae9e":"26722","78bc6f25":"26724","21e535c3":"26744","4681e75f":"26797",e2fe7a08:"26833","04fb6378":"26859","958a296c":"26882",b3749361:"26900",aca5eee4:"27012","58714c61":"27020","1547c10f":"27088",e37692b4:"27115",c4d1e829:"27123",cbec0238:"27179",ce710151:"27212",f52b34fb:"27369",f7e205e5:"27388",a3983124:"27398","7a3be7e3":"27444","31ed8f07":"27473","9f8f3c52":"27663","6d701be3":"27737",f95764fc:"27823",b0d5c266:"27887","10ccea9f":"27921","2e7e1134":"27955",ee4b8f19:"27995",ecd026d3:"28020","2cc67ca6":"28086","859500dc":"28127",abde445c:"28147","8120957a":"28166","513b54fb":"28169","9adef974":"28178","758b9bc9":"28204","20a81622":"28235",b8d3e47d:"28252",ec410c33:"28319","89cbd1b8":"28410",b212a261:"28428","5b374304":"28435","261fe657":"28499","54ad050e":"28627","0871002b":"28679",ab48a39f:"28687",c3856c66:"28701",fd5c5192:"28782","379ac6de":"28793","8b6fbbb4":"28803","7ed2bc06":"28834","4c00d6b9":"28853","2a0b5233":"28889",e7ef6e16:"28920","8c5b1685":"28941",f382fc88:"28945","8e7128cd":"28993","9c7a2f87":"29060","763fa02a":"29123","6f3c73f2":"29171",d4836e14:"29204",d330b699:"29243","8a472aba":"29337",fbce6636:"29408",b92135a6:"29414","163a1bae":"29421",e4d4ec4e:"29457",da05f515:"29498",b2f58160:"29523",b45a54e9:"29526","13f72d6c":"29653","6e0a413f":"29679",b1633424:"29690",a642dcef:"29698","80b93830":"29708",da10e58f:"29733",e6557239:"29795","16381f20":"29820",d973177d:"29829","5d94dea0":"29845","26f5f607":"29891","5397b577":"29922","5de1c39a":"29950","332b14e0":"29969","87546ad3":"30035","6b836ac4":"30066","339988c0":"30154","22bf7f69":"30167","5320ed26":"30262","132be4b3":"30265",af953e30:"30317","4b678e99":"30355","5c5d3a2d":"30378",e4a43002:"30396","90ffbe81":"30410","407230da":"30564",c7165ecb:"30569","1ee0a129":"30579","775c5b43":"30583","6ee9522e":"30628","5d70257a":"30654",a7c3ebee:"30790","5814834c":"30827",ced554c7:"30831",d8412727:"30842","677757ca":"30848","44c28e7f":"30934","7dea1595":"31004","416fb909":"31016","71d5b1f8":"31039","3f9a993f":"31047",c3c8ddc4:"31125",be9442ac:"31141",fbf25af7:"31144",b8ab477f:"31220","5c81ce12":"31255","67f1ea1e":"31257",fa366c46:"31272","07f6d4d1":"31393",cc564cf4:"31465","1668cad9":"31541",c602d51d:"31544","8c4bf82a":"31567",b38306ed:"31680","05957343":"31705","0a70c05a":"31720","624fba40":"31726","974bc2fa":"31749","04eab06f":"31791","14a26e2a":"31797",a14af959:"31861",c791a767:"31862","2dcd9099":"31869","2e31e40f":"31957","4ef864ee":"31972","8dbbcff6":"31996",f0dc0726:"32005","9ba9284f":"32012","116b7e55":"32039","5110a7b7":"32183",eb05a324:"32225","53ba3577":"32270","149a2d9e":"32278",f9a4941d:"32335",fe345ee5:"32365",e1407f5b:"32391",c08ba4cb:"32472",e6c2d44b:"32527","40c3731f":"32558","798484e3":"32691","1b520177":"32728","943ce597":"32824",fdddbf66:"32825","8590fd3e":"32847","225cc45a":"32960","604cd7c5":"32993","97e51b1f":"33049",a172f6b1:"33065",ee003a92:"33192","2a422024":"33214","465994bd":"33234","7da68ccc":"33290",f5ff1a23:"33303","2d0c5b52":"33350",c16e0158:"33458","3ed70074":"33517",f81c13fc:"33524","65ea2434":"33533",fe7ac0b7:"33540","8f00bf20":"33559","6075be34":"33567","269d704a":"33572",df68b7b1:"33577","3f933ca5":"33612",d294bdb0:"33638",da7149d8:"33643","81a6b262":"33659","755872dc":"33724","1b31aebb":"33804","07399e71":"33806","45f2496e":"33816",f8a443f2:"33825","06758c2b":"33854",a7690409:"33881",e9c7c2b7:"33889","49aa1a16":"33999","18b5779a":"34127",f5c63bac:"34156","3f4396e6":"34179","80f4f5b6":"34182","13e6af21":"34196",e8a8dea2:"34255","164e61b9":"34331","6153f283":"34335","79fbf53e":"34388","538371ec":"34520",b038a3ec:"34528","782eae4d":"34543","0735e5be":"34549","40cd51eb":"34597",f30fab6f:"34621","852f3aa7":"34638",ef071c32:"34662","02e54e09":"34691","47a1bde9":"34706",a5fea1bd:"34750",e076fc92:"34803","2e6ef254":"34808","924dcf98":"34810","10fcd509":"34837",dc7f8f8b:"34847",f7702246:"34877",f44e1bcd:"34904",c9cac9d9:"34918","40dc6b06":"34970","5e2f1fff":"35044","47cb0ede":"35067","35788c78":"35088","6fe5390f":"35139","83829ea7":"35144","372475fb":"35179","2947aa63":"35248","0f3b9f0c":"35288",d42065ad:"35325",a7827f50:"35329",d9454955:"35379","976ab233":"35402","388d5845":"35532","97bb73fe":"35618","2b4cfa56":"35730","8fc868c2":"35733",aba21aa0:"35742",ff6f6072:"35766","3019fa66":"35767",d240c26d:"35784",dd61deac:"35837","8852ad6f":"35881","92928c02":"35922",a3142532:"35932",dfa4e294:"35960",b5a599e0:"35983","04afe417":"36009","59be0e8d":"36055",ceff0062:"36067","28f38ec5":"36113","307d712e":"36144",b60e28b6:"36192","8a759f03":"36264",eafd7a74:"36302","467ff4a0":"36379",e23d1069:"36387","5ea64d6c":"36549","1472dfa3":"36569","94a5efbc":"36594","2573355f":"36619","1084a32f":"36640",b6fb003b:"36672",a3015275:"36747","649faa0e":"36851","1007513a":"36885",d73e6b67:"37109","0b4fcfba":"37115","97da4333":"37147",d786420c:"37191","8f3b27f1":"37204","43dcd746":"37206",e34cac0e:"37209","37524cd6":"37257","554882a9":"37329","6013ac2e":"37366","32b44a8d":"37403",ce46a18e:"37486","6a293d29":"37489",e9705ef5:"37577",e17d733a:"37578",be2ede3f:"37617",a6aa9e1f:"37643",e2657c7c:"37673","2cfb265a":"37740",feab7dcb:"37799","5f87fd99":"37809",d5e29b01:"37839","9ad03801":"37841",cd060f40:"37842","88d0aec4":"37872","3f272b07":"37954","690cbdd8":"38001","8d086d51":"38003",baad6bf9:"38056","1efbb938":"38107","66372f10":"38158","2e9f383c":"38238",bbc1ba35:"38348",cf858c26:"38370","7c15a7a1":"38385",dcd7017c:"38416","9474fc0c":"38439","91335d19":"38449",f0fb7623:"38451",b4a84943:"38468","0f497157":"38600","6dc06ee1":"38605","7eaa135a":"38607","1bf4b05b":"38700",c4de5678:"38767","872a41d4":"38786","5ea125b2":"38816","6495713a":"38835","110e78c3":"38839","6f5eeb66":"38888","2c50796a":"38974",daa4f13d:"38999","5560f355":"39088",cb9e1f3d:"39110","755ebe0d":"39138",b5f01515:"39142","0b8a4e62":"39153","09b62bf8":"39192","484cbf74":"39197","8b795044":"39202","93f7a5ff":"39208",b992801d:"39230","90f157dc":"39240",b40642d4:"39265",b1043e58:"39283",cc6505da:"39405","995840bc":"39407","085b8128":"39417",a615dd65:"39423","0f65ccfb":"39444","9d833fad":"39487",b326207c:"39494","7b2a260d":"39496","782d926e":"39505","630bda65":"39518","441dded5":"39520","4a80e27a":"39555","06a8eab3":"39569","7d0e1375":"39587","97ae1fff":"39608","8f64cb7a":"39712",a68f0e64:"39715","07bef82a":"39758",fb3759c3:"39773","320da59d":"39777","3c2d0811":"39781","8fa753de":"39840","9b0c8626":"39887",d76c455f:"39901","09a52b17":"39908","6471fe03":"39921",d4a2e930:"39973","5f469a3b":"40065",bc817d28:"40124","0508dae2":"40178",edf7dc61:"40205",ecaa52af:"40245","3c36ce76":"40369","0cd1a20c":"40463","4d1df301":"40464","3415fffa":"40555","3e032765":"40606","12a5de9b":"40616",bf5126e1:"40651",dfc3c224:"40690",cc01a9d7:"40697","8393a516":"40755","1a7b33c3":"40758",d2110cc0:"40797",ddebc8bf:"40827","2662e92d":"40847","321dafc4":"40907","2884dc3d":"40919","881bebdd":"40927",ca0149b2:"40959","68c5632a":"40999","33d4a0d4":"41030","0fa5c29b":"41125","85ffab62":"41135","6c9200cf":"41260","247909f7":"41361","2bf5edd4":"41455",fd517388:"41461",c3f88c36:"41491","8cd118b0":"41504",bf9d9be8:"41518",da172daa:"41594",d407c2c6:"41640",e8e3bd30:"41651","9017a110":"41677",b9edf71b:"41739",bc38a1dd:"41750","29db9f25":"41852","78f0e259":"41867","58b3152e":"41872",dc5506b6:"41892",b7a56be4:"41907","7c49023e":"41954",c413ce11:"42068","295cb0c0":"42187","671726ea":"42196","89c8b6fe":"42268","3fa17fc2":"42298","1f979aba":"42299","6df57399":"42350",ffac4af0:"42368",b8212f90:"42465","203cc654":"42487","34825c6d":"42518","2b505997":"42566",b184a577:"42576",eafac744:"42596","165a93e5":"42690","28826a4b":"42702",d8ed3ccd:"42723","89f1622a":"42794","5314580d":"42799",bf3d61ec:"42801",aeca0a21:"42879","432e670c":"42881",c9d95cbd:"42900","145364e0":"42905","39c7a203":"42911",ebbd9274:"42920",a8c92c97:"42937","19028add":"42942","2812e3b9":"42995",e3477e52:"43029","2907d5eb":"43035","676c7c8e":"43040",bec2896b:"43042","341293cf":"43088",d831e3b2:"43195",fe9b6676:"43286","4c07ba53":"43352",f048922c:"43421",fbf82623:"43432","83fec35c":"43445","7b89671b":"43461",d319e9f2:"43512","4cd0ba1b":"43642","7117a8cd":"43677","752351f9":"43739",c09831a6:"43868",a8b66660:"43922","509aac2d":"43980","5a8b22a0":"44008","454b7ded":"44022","43d1bad2":"44040","894f5c0e":"44045","6503f568":"44118",e2fc22cf:"44123",ca2b0f7e:"44125",d653d7ed:"44129","19acc4ed":"44143","4f087d20":"44290",cfca4f26:"44326","0a91021f":"44339","8fd374dc":"44380",d363a80c:"44405","216bf65d":"44419","6fb01322":"44489","9882af90":"44492","9b2ae5ff":"44551","502906a9":"44567","20bfa334":"44640",d65b2c25:"44662",bdc69003:"44682","260982a7":"44876",f3d02b29:"44933","43f3c18b":"44939",c7a67184:"44948","12b957b7":"44960","8914fdfe":"44964","41af2803":"44994","6e34e766":"44997","86fe1f77":"45117",b528d4d0:"45146","49a37e79":"45210","7737fe61":"45231",c3f790da:"45238",db90ec2b:"45290","815e47ce":"45323","24d4b044":"45339","459dbf85":"45460",e85bde03:"45500",a722b43a:"45541","7dce0476":"45561",ef7891f5:"45640",c36346e3:"45670","3533dbd1":"45682",a2a2954f:"45919","9926d60d":"45983","95aba771":"45988","8cf74eb7":"46001","2e048b0f":"46008","1fb7523b":"46025",f724037a:"46039","67893f6a":"46063",f1087137:"46074","53cdeba4":"46156","8e4df3f0":"46197",ea511817:"46244","04b49851":"46296",e49bd305:"46375","00239e8e":"46425","4bb02a47":"46451",bcb734aa:"46472","9dfe5a97":"46513",c13f246c:"46515",c5427124:"46547","8012465a":"46578","8393d26f":"46598","5f0630da":"46652","4f48cd24":"46662","2b6a7ab0":"46754","8f07070c":"46802",b2e2853b:"46840","917a523c":"46850",db9d1243:"46888","85f8bce5":"46891","57a58e78":"46905",ff13b684:"46911","7b0a3d41":"46917",a28b89d0:"46921","37ffd066":"46937",a9afee85:"46979",bbb92300:"47041","26996ad6":"47060",a5c8a2aa:"47087","696d889c":"47106","1366dd9b":"47110",c648308f:"47127","4d2e69a6":"47206","131c6f6e":"47236","5048202c":"47247",c7e2144d:"47248","6111afa9":"47297",b9a767d8:"47433","09e5dcaa":"47517",df0967b2:"47537","0029660c":"47555","0c1ff092":"47622",b1899f7e:"47668","39a07549":"47693","17093d00":"47699","4edf53e5":"47730","6bb76d2c":"47917","7ad7688e":"48003","47f96c4a":"48023",ca713325:"48050",c17f9567:"48063","852ff8c6":"48071","0cc98403":"48091",f81c1134:"48130",c467a11b:"48132","519b5759":"48162","00021e3e":"48167","9d8965aa":"48188",bef59fc9:"48214",e672d61a:"48285",e2b4c006:"48295","189e3ac3":"48304",f6884a75:"48310",d8c836b4:"48355","959b45a2":"48360","712c16a5":"48361","2896ce7a":"48532","0a5f31be":"48544",c1506482:"48592","7a556257":"48613",ef4f3413:"48661","5db19d16":"48673",d009d47f:"48733",e054d919:"48768",b71be52f:"48793","79392e36":"48803",d0b9fc21:"48842",ce18dbde:"48883","9d9f8394":"49013",c68848f4:"49046",be94b749:"49055","97d17d75":"49071","3455c0d5":"49112","0ac5df82":"49174","7795e379":"49199",f2a06fea:"49226","6111bc4c":"49280",ad0a2b75:"49305","2417a912":"49350","8f1510f6":"49388","2019135a":"49461","49d63cc6":"49521","0d819826":"49569",e01bb6c7:"49579","8ec8c5c5":"49583","2cbb324b":"49587",b0ba51ed:"49655","2391d372":"49676",acf72eb8:"49783","5bd6f6db":"49786","3fc6d2b7":"49880","1d5cbb7b":"49991","40c88f7a":"50012","837c6843":"50095","756c7559":"50176","18dddf22":"50178",f2790563:"50183",ea981684:"50259",ec6a0daf:"50261",fc6a53b6:"50293","09a0b8d6":"50361",c59a4667:"50414",a33de290:"50428",eaebafad:"50446",a4e24d6c:"50472","313c653e":"50486","129c4e7a":"50507",c112d1b7:"50570","3fd21eb6":"50584",e964b083:"50642","75be343d":"50700","0914ee26":"50703","0d98f85b":"50747",a339c33e:"50769",b365f6bc:"50800","4251704e":"50819",aa04bdb6:"50827","8aa0dce2":"50839",e7b82dc0:"50934","4dd6f8d8":"51047","6bd6fe03":"51049",e1758f93:"51060","12aaf145":"51090","3304ac90":"51102",d366555a:"51116","5e85b509":"51170",c39359c5:"51175","90d97cfa":"51184",a06af3ed:"51229","85a6e3f4":"51237","0a8d92af":"51378",fe2deb8c:"51412","7a04b795":"51420","14515c80":"51438","65eb0274":"51494","45b02367":"51548","8447ad38":"51578","4c2a4e19":"51584",f762fff5:"51589",ba5671ab:"51636","6537c712":"51649","65d842b9":"51665","612cc46d":"51694","89bdbd96":"51711",b50c8022:"51755","9199f8bd":"51763","1bd7f091":"51769","91a39dd0":"51814","44e51e65":"51828","5c238f81":"51837","48b6c251":"51878",b5fb4df8:"51890","1d79e793":"52036","7f8ebea7":"52042",edea4b8a:"52093","95cc61bd":"52102","3498be82":"52132",d0ed28b7:"52135","81d19844":"52176","508f03f4":"52207","1562cf35":"52218",fadc68df:"52235","8ec5e058":"52250","3e088827":"52266","6818c2e9":"52267",be224150:"52282","8e1e17e5":"52286","0dd8c0ac":"52342",ab586159:"52408","2dc793da":"52450","3016c86b":"52462","8c67a0ff":"52466",cda1618e:"52478",ca57223f:"52488",f2da277e:"52563","7199ad43":"52569",a9417ee3:"52572","27f2e2a4":"52612",c4f5d8e4:"52634","2124517a":"52680","9e4087bc":"52711",c8f57d29:"52731","85afc7f5":"52770","25eae11f":"52785",ce319efa:"52826","9e1bed9d":"52840",fa2ae802:"52882","69fcecaa":"52895","4d4179b3":"52911","82c60778":"52912","4adafdbf":"52913","2f32a261":"53074","5de85215":"53132",aaf3ff15:"53170","71514a42":"53178",d670063b:"53201","79ab3378":"53237",fa713049:"53256","76a0389c":"53266",f8fe23f1:"53288","5440b68b":"53314",e2bce80a:"53449","63436e22":"53458",a45dd8f5:"53492","92f1a745":"53540","5a2d67ad":"53629",f6ef9721:"53633","6c9978fa":"53646","52ed38a1":"53670","77612fce":"53742","3a447539":"53771",e352cccc:"53786","7c8deb08":"53832",e623e42b:"53860","2a5e97be":"53861","94dd9534":"53871",e9a95c5e:"53881",a4da17e3:"53901",a8933372:"53925",d5f056f5:"53982",f72b3fe4:"53990","8c317219":"54097","93b4d825":"54106",f97d7d88:"54200","3a55b748":"54240","5ba9c4b5":"54246",e5c0ea66:"54270",f020ef51:"54272",b7bc328b:"54274","71e2c4b4":"54277","25aa47d2":"54399",c87c5e1b:"54415","0c3d0366":"54462",fb9e14c7:"54500","66193a96":"54540","633af835":"54568",d4522125:"54730",ff4fb41e:"54749","0f7d8652":"54759",a2de6851:"54851",a872c629:"54879",c46bba44:"54881","55f8cc28":"54927","0a893fdf":"54950","98bd463b":"54951","5bcffa9a":"54992","37d125cb":"55004","70d55848":"55100","152d0582":"55144",c1cb0a0b:"55167",aaa8a12d:"55206",c962ae4a:"55276",facc4cc2:"55302",ee44120f:"55342","2027fd18":"55401","342d2e6b":"55483",f7fa3381:"55526",ab685cdb:"55534","29cf75d4":"55559",c8861997:"55566","3a9c66ce":"55579",d5caed5f:"55616",a7511015:"55664",bbaa8144:"55668","069b26b8":"55754",d425e9d6:"55780","94b063ba":"55836","2cbaf53f":"55929","41adf405":"55942","194c8540":"55955",e1a6f6ca:"56028","69ef23f7":"56037",cde3e47b:"56062",c96d4604:"56087","096bca72":"56097",a7b4d0d7:"56122","602e2465":"56123","887c7539":"56133","5160d3b0":"56190",e028a908:"56256","75a12619":"56259","575869e5":"56297",fec58251:"56384",cbb5064a:"56408","858f1266":"56493","86a45dc2":"56510",c9c31477:"56560","609f9177":"56607","06eed450":"56648",beb32294:"56662","8dd3eb38":"56688","69e6c429":"56723","9a502a1c":"56725","2d57b909":"56767","6664c24a":"56779",eec5649b:"56826",ebc13825:"56916","1dbfdc18":"56948","39ed0ae4":"56974","97b1d23e":"57054","4d68fc5d":"57119","037241c6":"57141",e15f47bb:"57159","034f0308":"57165","2575da36":"57178",a2f498c0:"57182","8ee4a7d7":"57222",e3b05f38:"57245","779bba71":"57251","5e727800":"57283","55b89dea":"57302","3ed58e4a":"57326","841b525c":"57359","79cc2eba":"57361","533bed85":"57426","0b7a8a63":"57529",b20f9cb2:"57581","2b2d1be1":"57594",dd48b375:"57614","0ed7fb46":"57647","658997e4":"57730",c07b5739:"57749","73d3ccec":"57760","27b8ef72":"57762",f1691dde:"57885",e2070fcf:"58000","31fce735":"58002","2655f170":"58079",fa4b529c:"58111",a2171a4d:"58144","400deb23":"58255","078daaf7":"58259",b296abb0:"58301",cbf83637:"58351",a5e62092:"58359","15ea2a5f":"58393",ec854811:"58428","2a882da6":"58479","36d29ed8":"58484","0ba9210d":"58492",f0e5f3ed:"58581","6bd22ece":"58594",a1e59af5:"58666",a36e07dd:"58667","7fde9a4c":"58704","6b6aadc5":"58707","9ffdcfdf":"58721",b7d1b016:"58780",bec552c1:"58857","030dfd2b":"58874","52e97b50":"58880","7dd8c933":"59004","39fca8ac":"59005",a169aa70:"59009",e84457bb:"59020","11b61292":"59053",c43c6f4f:"59070",c762272b:"59104","1a20bc57":"59114","17ffd4ff":"59123","97db775f":"59262",e2b886c9:"59326","156bd387":"59349","55d26943":"59475","150f4998":"59493",b2555332:"59521",c5c8b091:"59614","70cca634":"59680",c9b0e9c8:"59702","9873f053":"59732",ab90a7b7:"59740","28553a2f":"59779","0e7c02dc":"59781","03f08ad1":"59866",b37c8625:"59873",b62e8859:"59874","888f35af":"59940","03137a3d":"59981",cc3a93a6:"60020","66eb519d":"60119","152819f0":"60167",e04f784f:"60185",a52bfe45:"60187","5c430979":"60193","5c417b7f":"60198","5c5dd48c":"60203","3fa5d64e":"60257","3d3aadb0":"60267","93d54c10":"60294","42d452f1":"60315",f83acba0:"60445","821320eb":"60510","492bd0ed":"60565","3523854b":"60613","7a82ef89":"60625","244c7b0a":"60676",d163928b:"60696","73e80d5d":"60725",e9d18558:"60762","0614dce3":"60765",c316a3d7:"60819","2df3fdca":"60830",d2436a2b:"60837",c9d25191:"60861","709e1a02":"60884","00b58b18":"60901",fe53d354:"60908","3e85731b":"60953","4d18bf1b":"60965",b1506a50:"61027","127f988d":"61114","4521c19b":"61228",a7456010:"61235","84536aab":"61265","31eb78c6":"61300",f12e5474:"61345",c90911b0:"61378",a1e3441b:"61416",b0f00280:"61481","6b206066":"61559","55b57e06":"61584","35190cab":"61627","495a3878":"61652",faa24029:"61702","4f3239be":"61721",ad2a3b86:"61734",bc80aebf:"61740",aa1329f2:"61786","14da6f38":"61844","124bf435":"61886","820ef6f8":"61950","9df3c51e":"61959",a6972a3c:"61991","5cc68a8f":"62015",f1b1ae9c:"62037","2638eb20":"62061",d2289dcb:"62111",cf1cd5da:"62120","1a4e3797":"62138","8d364c04":"62147","5955b5ee":"62149","22c027ab":"62162","0e7e04d8":"62182","1e38d1de":"62188","9b695413":"62254",cb69dde4:"62264","8041f857":"62400","856bc38c":"62431","3e9695b6":"62456",f448ea15:"62469","564f4b07":"62501",cca423c7:"62549",abfb0eb9:"62568","61b91652":"62635","0f378f0f":"62639","58728d91":"62641",fcaa2a90:"62672",d9175e87:"62691","345b88a3":"62703",f9a8c23e:"62712","1769198d":"62714","4e78ea4f":"62827",ae5c2262:"62845","07853d90":"62867",f1096413:"62875",c03ef713:"62927","7a0d5539":"62953",e5c6728d:"62957",baa95a18:"62975",b0635fce:"63005","754efc6e":"63064",d4bcf69a:"63124","93f4c2fc":"63151","29fd9202":"63181","34c77011":"63228",df8bdaca:"63262","2f1bf685":"63291","259b7e50":"63298","7dd3b2a7":"63317","60b75cbd":"63330","44bdb747":"63367","5f87f962":"63382","9780a9d6":"63387","04ea9cef":"63420","465af2b7":"63430",f30739cc:"63455",a3041c66:"63515","58f053e0":"63521","1fd2ca4a":"63553",b9795b3d:"63576","43a56d47":"63627","47298c37":"63646",f82e3d74:"63682",eadd254f:"63700","81a5dcd4":"63739",c2750739:"63779",a4974adf:"63795","95e06d9c":"63836","57e245ce":"63907","8bbbdfbb":"63951","2278c359":"63966",b81d6cb2:"63975",c47fd21a:"64045",e34c241a:"64099",f4f77456:"64184","88ed6425":"64209","192402a6":"64264","48a72e90":"64275","7c0dabe4":"64351",b65b26fc:"64391",dfa86ee4:"64442",e4307155:"64447",e9f47cd4:"64465","9c8cec5f":"64473","5d817499":"64502",dae07133:"64527","3756d209":"64535",a2a8ce40:"64577","9c273b44":"64597",e52d80fa:"64623","6ae279e9":"64659","46693c0b":"64670","1d00ec5b":"64737","2da5f59f":"64749","9a8a6b3c":"64838",e80f330c:"64899",a3fde46e:"65013",f08c8322:"65017",f04b050d:"65094",c9bcea67:"65133",d0cf31b7:"65193","2a11e6a7":"65223","0530d392":"65233",ba47c7e8:"65246","517960c0":"65350","8db42357":"65356","070df7e5":"65359",a21d3492:"65473","26115f23":"65497","8008d958":"65569","5d7f3e2f":"65611",e1ba57a0:"65625","26cbee69":"65629","75ba6401":"65647",c7850e27:"65648","329c66a5":"65658",e29a1e9b:"65727",d1be0dfb:"65745","02ff5d42":"65756","7d5633f0":"65759",fbd3de42:"65835","38147e44":"65897","587a0b3e":"65918","567cfed1":"65926","1f391b9e":"66061",fada9729:"66062",ceca85fb:"66097","331ad65a":"66135","136587f9":"66140",cf5645b0:"66162","0b7cbed9":"66169","2b53c3fa":"66210",cc9fb6c4:"66237",eee168db:"66327",dcbc8483:"66338","8fb804ed":"66372","9019bc0f":"66381",c0d2ab5d:"66394","6f47e633":"66415",cb8920e1:"66461","85e49a93":"66499",b01f49d5:"66504","79d5f27a":"66510",c6ec7d6a:"66518","83037ff5":"66524",d0550e1e:"66538",ecb74126:"66559",b0e34666:"66575","9fce9e91":"66602","5389895e":"66753",e381e7b7:"66759","62e9fea7":"66822",e1fde1ef:"66832","58b9aab4":"66867","47f8f95a":"66903","6e955522":"66912","6faab85c":"66947",ba8ea34b:"66995","1b143eab":"67044",c1050649:"67061",a7bd4aaa:"67098","27c39d7f":"67100","06073810":"67104","0d86fe8d":"67114","11235f31":"67149","9736a6b2":"67189","2d4f05ca":"67229","60cef16d":"67252","01ed4ae3":"67376",b6d7433d:"67438","814f3328":"67472","3477cc36":"67489","77139df7":"67507",d89aa357:"67545",a1a96ebc:"67581","8775fa1c":"67587",eedfeff5:"67649","47766edd":"67668",cace8592:"67683","8fb89e11":"67704","4d59e059":"67720","2c18bef5":"67729",b57b3f07:"67755","39b26ead":"67775","0a0dd03a":"67914","8aec9a60":"67954","863f8530":"68038",c1c9577e:"68043","0924bdf1":"68045",e981d05e:"68094",a148d810:"68210","3b8a31a8":"68246",e48e4dc9:"68249","89c88bb6":"68253","8ea5f648":"68304","9d05a91f":"68334","975aa772":"68434",bb6faf52:"68467",f553a3a8:"68515","9a8c2230":"68551",d81421b8:"68569",f645a9e6:"68665","0b87289b":"68685",dbf13a93:"68710","719a7d9b":"68765","14033dfb":"68770","3ac10f9a":"68839",bb1a3e9c:"68861","08299697":"68865","62cdbb35":"68910","5aa406a5":"68930",f9d38c6e:"68955",b60b94c8:"68991",ef13094f:"69017","8a752d6d":"69030","8df5dc78":"69038","4a55387a":"69129",fc384521:"69169",f0589cfc:"69199",de9d53d2:"69347","4d2a5110":"69365","5a360e6e":"69384","7ffd9a5f":"69457",fc15228a:"69501","8a4df687":"69517","4d573718":"69570",a3713279:"69588","05c96a0d":"69633","9c10cdcf":"69638","36e398ec":"69671",b1e8d27b:"69770","787caf51":"69787","615fd764":"69844",b58f381c:"69875","8e981336":"69982","0860d759":"70008",c8d1e051:"70022","73a7a1a2":"70058",b318f3b6:"70100","9f3d620b":"70117",f68988f2:"70126",dae6ce88:"70127","2f2d1edf":"70133",e04f6385:"70174",cb543535:"70179","529aff45":"70209","8367b76d":"70210",e76e0aa8:"70234","5e1c183f":"70253",aefdabf4:"70268",d2786aa3:"70399","0bbbad22":"70418",b71c2d32:"70452","8c596cb7":"70502","0bd456dc":"70551",ac5d28bd:"70560","6d5145e1":"70612",a9f446ca:"70645","0920b03b":"70649",dcc774d2:"70669","70ca6744":"70687","52be8247":"70696",cb488bcc:"70712","07338b13":"70728",ecba2f16:"70749",f3e1c72c:"70755",ac00e857:"70784",fda5ba18:"70791",c1dff2d3:"70794",c3c05d38:"70814","1d89db06":"70825","23421dc8":"70826","6763bf32":"70828",e34cf2a2:"70853","4f594038":"70870",d1fa94a6:"70947","87d75f25":"70953","16029e49":"70963","05f6486e":"71021","5e0cf2ca":"71026","9b8a5dc6":"71051","48c07d39":"71111","92f96583":"71117",af7ce5e3:"71162","131dae92":"71227","40d0141e":"71374","7380ddcc":"71407",ff761994:"71473",f89e6cd1:"71531","83c3bea7":"71565",d7e2cd1f:"71585",ed47441b:"71590",a29b4047:"71612","7ec29cb2":"71656","4a74d13d":"71715",e3aab494:"71784",d2a882d8:"71792","09138901":"71858",acec9ba2:"71863","466f5a64":"71877",f783f7a9:"71910","835d6f03":"71964","31b399ad":"72011",f0dc2560:"72043","95f36a1a":"72053","3d095b6b":"72122","8cec3a3f":"72135",cf8b2bc1:"72246","4941c7ed":"72322",f23d42f7:"72353","4419dbb7":"72358","27d3d4e0":"72446",ac84c584:"72527",bd203bf3:"72573","3336b76f":"72599",c8768069:"72634","049fd4ea":"72668",f81aef8e:"72675",eb3eed60:"72707","69f425f4":"72759","2d34e835":"72835","43b52908":"72880","491d56e0":"72907","37eca2aa":"72914",b724edf8:"72925",fca979f9:"73019",b8385eea:"73084",da144c90:"73193","33e8889e":"73264","7bd25ebf":"73276",c7f860be:"73290","8b1b48fb":"73316","97d2cbab":"73331","21838a86":"73376","681664ad":"73397","092425ad":"73472","4c26df07":"73493","5ebbf59b":"73539","73d617e8":"73540",ac6289fa:"73569","01689e9b":"73607","622b7911":"73609",edefc60b:"73624","35be90fa":"73684",c6ea0115:"73693",eb08ed44:"73741",d8415e6f:"73756","5cf43a2c":"73800","569e0588":"73841","4d015a5e":"73846","06dd3057":"73888",d721a748:"73891",d502d9c9:"73905",f4731b9a:"73940","14841d7a":"74019","9d891c91":"74027","2e6d9cc0":"74039",b7aeb8c2:"74046",ea5e46ff:"74069",da7bf323:"74121","393be207":"74134","2fd7ee6b":"74282",acf7953e:"74312","57eb6545":"74388",bc42f283:"74403","103c2fe7":"74418","3d4e54c7":"74490","111d9467":"74524","84939cad":"74588","033d3629":"74657",e6ce3bd9:"74684",a5ac74f6:"74792",ec7a5af3:"74880","43c329f5":"74902",ad848ffa:"74945","9b3f2ab9":"74961",eb5c136f:"74994","0c90e710":"75036","1c4d0413":"75043","13d28288":"75051","8df3f976":"75062",dd620de6:"75078",e2585025:"75105","9f49c871":"75106","3126d1b1":"75173","6dab9497":"75267","07deb48b":"75309","84aa8d64":"75359","847c1b9e":"75378",ceaa6e69:"75405","97c492a6":"75450","605c3775":"75459","2aa42d18":"75666","68f2aae3":"75695","1ad07866":"75696",d2250181:"75718","691ef278":"75735","217c03e5":"75759","724a4edc":"75775",ad132b09:"75805",a3541f3b:"75866",f18854fb:"75911",e01c8f09:"75919","03240ae1":"76041","20a6876f":"76163","1778a2f7":"76166",d9605822:"76215","6a3d5afb":"76235",cc7760fb:"76257","075f4b70":"76262",cb46984a:"76264","5d06256e":"76274","0d71ae17":"76285","1ed4b1ad":"76289","9267add8":"76368","594c10ba":"76530",eb0d63b4:"76641",b6f16885:"76683",a50107bb:"76719","9a68cfff":"76748",d5a221f8:"76850","2b00a752":"76854",e364f7ff:"76871",a91e6a0a:"76873",a01a15fc:"77077",bf048e24:"77088","61ee4158":"77096","62825cc3":"77123","30ae1693":"77145",ce019e24:"77174",eaf93d9c:"77183",a95aede3:"77200",fe2389d2:"77222","1aef3c3b":"77261","109d395b":"77298",c4acdd50:"77369",d0c84d34:"77391","0235a442":"77596","5ef28561":"77679","60381fc6":"77698","3a31d53f":"77749","8c3793bd":"77764",d9e41302:"77787","62e7b5b0":"77794",a52788ff:"77920","33ab05f6":"78001",c9599a77:"78037","30175d3c":"78055","90caa6a1":"78087","8df10e0f":"78159","5455ca0e":"78199","1682c6e0":"78312",c1b680b7:"78354",b1a2ea9a:"78421","28ebe10c":"78497",b7e5c092:"78512",ff4a022d:"78522","4adc4126":"78544","95365ba3":"78583","4f1d1584":"78632",b80df7ca:"78704","667a9f57":"78758","6000d05b":"78764",a7f76e11:"78787","04c3dd09":"78820","4f0afd2f":"78858","499efff2":"78876","335badf9":"78880",c8939218:"78912",f73fc2b7:"78919","73bc484b":"78926",ccd7c88f:"78985","0ee8b775":"78988","803df44c":"79039",a94703ab:"79048",cdc43a7d:"79150",c884c38e:"79157",cc426672:"79180",a2030166:"79416","120bc0be":"79474","8b7dab17":"79485","368f9c22":"79506","09b9b020":"79521","1bfbaba8":"79656","8be24c4d":"79739",fc26a5d5:"79841","619263e9":"79866",dd6ea91b:"79987","5e4ec6cd":"80018","93c339e4":"80046","587df15f":"80091","7349f9da":"80102",d38a6f54:"80198","0cc20099":"80205","45cd5638":"80259","8aabb1ce":"80260","8657657d":"80299","3c89ed52":"80310","2fd10e89":"80362","6e954188":"80375","81c96f91":"80408",abf14b6e:"80424",e1a35040:"80442","992bda30":"80457","318980ec":"80458","7080aeb9":"80491","27321f21":"80531","5cb06d06":"80557","1f97a7ff":"80598","9abd9d66":"80632","80f24550":"80649","6d70fa33":"80677","641786b9":"80744","84ac5929":"80750",fd440a88:"80772",cfdba83a:"80781","442da8c1":"80794","413bf93f":"80841",b6aee096:"80863",a7f213e3:"80947",c141421f:"80957","820b9813":"81012",f74d3bfd:"81019",c88d5fcd:"81050","104fa664":"81085",aee88ae1:"81125",f98df6cf:"81175",ddf9f187:"81188","347b93da":"81212","54e1e905":"81285","2195a7cc":"81300","12363bc4":"81319",a20f65a6:"81346",e6c5d4a7:"81390","7af33a93":"81410",ae0ec9f4:"81482","654827c2":"81530","13685ceb":"81544","15a3ce9a":"81583",ffe2a86d:"81639",a74d38f6:"81666","0d9e9c41":"81692",c75c3ca5:"81737","932d66ca":"81761",fddd2104:"81770",d3571192:"81791",b90928c1:"81795","6066240e":"81802",b3dc9cb4:"81891","29470dda":"81902",acecf23e:"81903","4b93078e":"81943","2a59c009":"81982",f9ecf61e:"81990","4a7139ae":"82012","755e7c7e":"82019",cb7dc7d1:"82242",d2c9f0b8:"82261","226eff01":"82297",b87f99cc:"82354","7345c721":"82362","5d210bb4":"82431","24f4e7d7":"82456","4393156b":"82486","0add85e5":"82553","907fc94b":"82581","4cd82028":"82588",dd0ce7a8:"82663","72e3e702":"82722","4ccba4b8":"82754",dd09cc7b:"82847","12cecbf6":"82905","2a7d5452":"82939","732a7c43":"82987",dac8839d:"83027","42174d87":"83078","686a74aa":"83089","21cd7edb":"83103","79ca5236":"83109","6060cc06":"83111",deacbd9b:"83121","10757cc8":"83127",e5cd0e7f:"83182","95126e44":"83203","828aa184":"83213",ccc49370:"83249","5c26bf07":"83260","2b6eabf2":"83273","0ba08ea0":"83316",d0f169c8:"83378",e6ccb422:"83470",c85c7bc6:"83574","19560f91":"83608","9e37e644":"83670",a8c902bd:"83700",e6fc8a9b:"83779",b6cd0ba6:"83797",f5dd5915:"83824",cc517726:"83841",c48426e8:"83849","345eb925":"83928","757c6784":"83932",e67a83cf:"83954","2bf0d1a9":"83958",e200187b:"83970","0e384e19":"83976",c17d6939:"84013","511d3e84":"84070",ed83b9b9:"84107",d2ed2b82:"84152","17d9fbbc":"84227",ddd99a22:"84232","901160ed":"84255","2ffd8514":"84259","1132d5f6":"84319","0b82d45d":"84332","8180940b":"84527","2e72ea50":"84530","2760fb69":"84563",d9a7203a:"84572","21f4f354":"84760",b738b15f:"84773","6875c492":"84813","6bd2e919":"84869",d9d878d8:"84963",ae42df76:"85014",ec2ed6e2:"85095","7c86d680":"85097",ab380486:"85105","0cf3b0b8":"85205","8beefa16":"85209","78c968cb":"85243",ffec2b37:"85257","8f4a15da":"85279",c9ff545e:"85322","813f53ba":"85358","2f6614a5":"85442",de5aeda1:"85453","2c88985c":"85537","8f9bc413":"85581","24e645af":"85588","0e8c3a89":"85736",a4a3eadf:"85796","3e226f70":"85806",d40aab8b:"85817",ddb0149a:"85896","41639dad":"85899","716f14d9":"85917","5268c144":"85960","5733876a":"85973","83b6afd8":"85997",ae271c01:"86053","199e6876":"86074","677325f0":"86121",fb03d32d:"86129","1f8198a4":"86167","95ee2976":"86215",abe1b280:"86216","7a2c4c43":"86246",d630316d:"86354","669193a7":"86417",d5f51703:"86434","2a1b9f9a":"86478","3a311bb0":"86480",f4e1ab69:"86501",b76e61dd:"86586","0e0dfb6a":"86597","49bbb99a":"86623","7ff6577b":"86659","927f97a3":"86669","58421c87":"86685",b0445ba0:"86807","5626901c":"86829","57ec2762":"86846",c942990c:"86856","5687c7cb":"86888","319b6f13":"86890","869411c7":"86914","7a06eb83":"86939",d2a66e94:"86992",b62a4e5f:"87004","40df2769":"87087","4eec7d8d":"87159","4462c0cc":"87184","767cce31":"87186",bad31f72:"87205","9c71777e":"87217","33ac18ed":"87269",c884ad6a:"87282",c4f2c374:"87377",cd08112a:"87407","0b425eb3":"87432","33bf4676":"87456",b8f2cc13:"87459",baf4021b:"87473",d5762a9f:"87483","643da015":"87485",e29c32f4:"87547",d2b2e5aa:"87589","7c29a585":"87614","137db567":"87688","3f1a4665":"87747","6f831635":"87781","4194f325":"87863","01f93a4b":"87875",f553a7ec:"87880",fc17862c:"87905","5b17955f":"87945",dc4ee1bb:"87971","85f9f5a6":"87995","72a68b3b":"87996",a5f4f54c:"88047",ca3d464c:"88111","2e05ee79":"88163",b8fd1a8c:"88222",ed990644:"88248","85e50ec2":"88298","0169a509":"88310",fbfb0bb0:"88327","3cf947bf":"88393","5ae50f21":"88394","078339bb":"88399",a13618bc:"88436","40e4b7d4":"88441",bf79ed6f:"88468","23a811a2":"88478",d3f32089:"88522","2283659c":"88547","5871fbee":"88622","6e8a38ea":"88626","6dc022b8":"88654","6be5dbf9":"88696",b330ac5c:"88775","71cde98c":"88815",d3065e4e:"88817",b0b71a2a:"88821","014c2cb9":"88847","508d741f":"88852","5edce1ad":"88855",d15e98db:"88865","568a0660":"88903","7b658d8e":"88986","2c9ce68e":"89018",fc93a2ea:"89093","24f450eb":"89097",adb96665:"89172","9f15c6e7":"89175","9b191135":"89210",d8e6b3db:"89289","324cb577":"89338",b919dab0:"89353",cb2f16be:"89362","39ac0d5b":"89402","9ed26de9":"89415",ce1709a8:"89437","978ca64f":"89455","145c1cc4":"89489","7ff4234e":"89554","2709984f":"89610",cde9af88:"89641","837dc7e4":"89696","084a80b5":"89801","9bfa1541":"89810","36994c47":"89858","65f8c8fd":"89873",e582f766:"89986",bfb3b697:"90066","7e60f0fb":"90071",dbe2cfea:"90110","1026ad00":"90115","8abd7194":"90135","9db841c0":"90214","7b773c92":"90257",a80ed580:"90280",b3abe364:"90317",c633f93f:"90378",ed2667fa:"90414","9947fe09":"90416",fe49dcfb:"90438","6157713d":"90513","32eb34e5":"90562","99144b30":"90593","2563c97b":"90598","73554d19":"90625",bfa9eb5d:"90627",fff1effb:"90717","48dc1693":"90749",e96ddf11:"90831","837010ac":"90842","0058b4c6":"90849",e03c6723:"90910","271ea2f9":"90950","70e8710e":"90972","466375b7":"91025","42003ca2":"91058","014625f4":"91109","3629c812":"91187",ece3ee5e:"91203",f657ed9f:"91257",af1525aa:"91305","0f84f33f":"91319",d1980a1b:"91333","88f66091":"91359",ae3304ee:"91405","284c57b4":"91428","7118d0f0":"91530","980d0b86":"91540","8614d32b":"91581","1fecd571":"91587",e33cf622:"91597","7907a033":"91621","6033c607":"91695","9964550a":"91724",df99aa82:"91787","41ca93cc":"91844",b7201a27:"91895","72fdaa88":"91916","8ce0215f":"91993","0ce689e1":"92018",e2c3948d:"92031","6342cf5e":"92105","4cce6e5a":"92125","56a0d755":"92196",e772c536:"92223",bef29d9e:"92241","85fe7d70":"92273","4a4c6752":"92329","4c434b32":"92410",d888d821:"92412","1be20fdf":"92475",b6eb7220:"92535",e53ee22f:"92544",efa442c3:"92653","7c1d6f5a":"92664","9dfaf941":"92673","3e082aad":"92955",adfd3bc1:"93057",b5ea0533:"93085",fed5be48:"93166","2ed2a721":"93206","47b26a6d":"93209",e55ca189:"93210",b12147a6:"93269",cf32db66:"93273",aba187b8:"93341","2357dc71":"93361","7a234181":"93362",f5e9d1c4:"93438",a9c52207:"93534",d9e43198:"93546","44c6f0c9":"93602",e1847ef7:"93636",f179621d:"93795","09bacf3b":"93849","0a07ac32":"93854",c3c082a1:"93884","45a5cd1f":"94000",ee71d619:"94065","8c0fb9c6":"94082",beceb38b:"94091",acddd5ca:"94106","6ee6a410":"94153",c6b4dc09:"94168",f0cc57e7:"94227","824e6f8c":"94251","00f9d3c8":"94314","8b44df1d":"94344","657bf45c":"94361","85c8b6c7":"94417","13feb7a8":"94538","8e7aaae8":"94555",a1e44e64:"94561","745d4b8c":"94568","0a02a2d0":"94625","30549b42":"94646","50774ec6":"94648","8520f5db":"94685","0c2c2d88":"94711","6dfe2e3e":"94713",c26e67a5:"94788","256701a3":"94796",a1282170:"94841","3e1f4a39":"94843",fc66001f:"94863","95ec5604":"94864",bb63d03e:"94910",e8b41ff0:"94925","98700a51":"94926","5fc8caff":"94980","8d65e26f":"94997","63c4b13e":"95020","4b69979c":"95058","10ac9a3e":"95078","818bb2cd":"95093",c0df5757:"95136",cdb727d9:"95159",f58814fb:"95183","1cb0fe52":"95231",b4030b00:"95240",ed211a79:"95245","9b4185c1":"95330","52910a8f":"95427",e1afbf8c:"95443","8a40ff6b":"95470","5917c028":"95542","9a8fdb53":"95605","188015be":"95641","4885c521":"95649","30d4e84f":"95659","89ce4ba3":"95768","8c79a60d":"95785",dbea5ca6:"95818","9e488927":"95909",accd2b0e:"95911",cbc2448c:"95912","5180e398":"96022",eb84cef2:"96086",a267572b:"96155","838df61f":"96225","380e17bf":"96239","638db053":"96268","82a7c68f":"96276","65d1ec04":"96325",ddd9290b:"96463","1b260ed9":"96470",d4083900:"96555","5b0025cd":"96652","8c42b153":"96693","4f748135":"96745",a98ffe6a:"96771","181eb1b5":"96778",c3c125b1:"96793",d527c17c:"96881","7ab01152":"97006",e9032a0d:"97033","763a8986":"97047","9331da7d":"97107","32bb5bcb":"97114","9d2e74da":"97123","20a30ea0":"97167","1b3061f3":"97189","7fa05810":"97223","365a7fd7":"97246","1a8a162f":"97253","205cdcf8":"97296",c2a473ad:"97342","76276d52":"97358","3bfbd79c":"97379","224c7e3e":"97420",ba47c136:"97424",b93a682d:"97476","3a6c6e5b":"97536","6ccd70b6":"97554","3d08e0be":"97562","8e81e342":"97585",facb0528:"97664","1dba1ecf":"97670",b58e88de:"97732","8fa9d00b":"97737","41b1becf":"97753","7d2fd416":"97763","97a057b3":"97821",e5562b89:"97829","47edbc34":"97863","26ec8ae2":"97887",ab8b4014:"97895",bf7dfc7c:"97901",ff509459:"97905","1e228808":"97913","350cc719":"98012","139d61ea":"98021","95f25ea6":"98030",fbb50b04:"98058","75dbe45b":"98059",b18455bc:"98087","48f67822":"98094",a2219ebb:"98139",fc46979d:"98147",c8e5bf38:"98165","3539b92c":"98237",f62bafa2:"98271","756094a5":"98308","9e297776":"98311","1c3a958e":"98392",b943b6ea:"98425","94dcd3de":"98473",ea6f04d4:"98477","6065ad54":"98534","99e64d04":"98700","19bfb940":"98750",af6e9729:"98758","3314c9d3":"98761",a48978f5:"98779",bf667688:"98804","3e8495a1":"98867",b4e94af8:"98908",ab37b32f:"98925","5de60d34":"98935",c9ba7c72:"98941",ef00b8a0:"98947",e8178b53:"98982","82a5d7f7":"99001","1c27379d":"99095","210305d4":"99146","8e563661":"99151","3047f3e7":"99252",c14fcab2:"99254","8944e56a":"99299",bdecca0c:"99306",b3e98411:"99330",df09200e:"99335","5ffb8f23":"99437",c0684476:"99451","2263a65b":"99502","76af5d51":"99504","16e939a3":"99528","8c2cbe8e":"99531",c49eb59b:"99534","25c655c3":"99543","9cc8ffa2":"99611",bfbfac54:"99644","25e15e7c":"99678","5a4dd75d":"99701","5fb14ca8":"99705",dc330c71:"99709","6e34d00c":"99710",f4a839f6:"99746",a32c0324:"99747","8e34a11f":"99872","2f7d15c4":"99905","4b2e4980":"99987"}[e]||e,r.p+r.u(e)},(()=>{var e={45354:0,71869:0};r.f.j=(a,f)=>{var c=r.o(e,a)?e[a]:void 0;if(0!==c)if(c)f.push(c[2]);else if(/^(45354|71869)$/.test(a))e[a]=0;else{var b=new Promise(((f,b)=>c=e[a]=[f,b]));f.push(c[2]=b);var d=r.p+r.u(a),t=new Error;r.l(d,(f=>{if(r.o(e,a)&&(0!==(c=e[a])&&(e[a]=void 0),c)){var b=f&&("load"===f.type?"missing":f.type),d=f&&f.target&&f.target.src;t.message="Loading chunk "+a+" failed.\n("+b+": "+d+")",t.name="ChunkLoadError",t.type=b,t.request=d,c[1](t)}}),"chunk-"+a,a)}},r.O.j=a=>0===e[a];var a=(a,f)=>{var c,b,d=f[0],t=f[1],o=f[2],n=0;if(d.some((a=>0!==e[a]))){for(c in t)r.o(t,c)&&(r.m[c]=t[c]);if(o)var i=o(r)}for(a&&a(f);n{"use strict";var e,a,f,c,b,d={},t={};function r(e){var a=t[e];if(void 0!==a)return a.exports;var f=t[e]={exports:{}};return d[e].call(f.exports,f,f.exports,r),f.exports}r.m=d,e=[],r.O=(a,f,c,b)=>{if(!f){var d=1/0;for(i=0;i=b)&&Object.keys(r.O).every((e=>r.O[e](f[o])))?f.splice(o--,1):(t=!1,b0&&e[i-1][2]>b;i--)e[i]=e[i-1];e[i]=[f,c,b]},r.n=e=>{var a=e&&e.__esModule?()=>e.default:()=>e;return r.d(a,{a:a}),a},f=Object.getPrototypeOf?e=>Object.getPrototypeOf(e):e=>e.__proto__,r.t=function(e,c){if(1&c&&(e=this(e)),8&c)return e;if("object"==typeof e&&e){if(4&c&&e.__esModule)return e;if(16&c&&"function"==typeof e.then)return e}var b=Object.create(null);r.r(b);var d={};a=a||[null,f({}),f([]),f(f)];for(var t=2&c&&e;"object"==typeof t&&!~a.indexOf(t);t=f(t))Object.getOwnPropertyNames(t).forEach((a=>d[a]=()=>e[a]));return d.default=()=>e,r.d(b,d),b},r.d=(e,a)=>{for(var f in a)r.o(a,f)&&!r.o(e,f)&&Object.defineProperty(e,f,{enumerable:!0,get:a[f]})},r.f={},r.e=e=>Promise.all(Object.keys(r.f).reduce(((a,f)=>(r.f[f](e,a),a)),[])),r.u=e=>"assets/js/"+({16:"0583dad8",25:"a33c645e",28:"0da1f28a",39:"a978e5ab",48:"a674d2f7",60:"e9bd6ef2",67:"56169fa6",106:"6c6aeb2d",129:"275b5c51",159:"af8190e8",217:"b1f77b90",278:"a1a7cea0",292:"fcc9c1b0",323:"be9655ff",335:"43d16b11",354:"0b4da2f0",387:"19ac452e",437:"a168e2e0",569:"b4a99b76",614:"0a87a91b",805:"5c109a5d",822:"79cc09db",851:"8b10f6da",878:"e9038c17",910:"ce3da75c",917:"02b5547e",924:"9699c87d",940:"0c459987",1103:"4b935b7f",1107:"060b2af9",1119:"e3b4090c",1136:"ae09e9a1",1190:"b0127f30",1251:"ea1c8f1c",1284:"62a07427",1337:"9cab4fb3",1360:"eb8e7dea",1422:"562c897e",1442:"50f3bb83",1506:"4b25608f",1517:"da868c82",1550:"8425e24c",1635:"73c8d421",1656:"cc5200ca",1658:"7a51e13d",1784:"89dd9b54",1785:"269ce614",1791:"4b2ed2d0",1841:"92bb01f4",1846:"9f349e0c",1857:"d032d8fc",1882:"c28fec2b",1928:"6e47a3c2",1946:"45b94beb",1955:"7fbadf7c",1983:"b93dc3c9",2016:"3c15683d",2053:"7a3be72c",2121:"fdb4abad",2172:"ef5805c5",2214:"32aa7863",2225:"9781112c",2239:"2b154460",2256:"31863783",2262:"7d9ec3a3",2298:"d2a01f74",2385:"fb4f9065",2450:"3b3f1ad3",2472:"1b7e3feb",2540:"e8b490f7",2577:"b9e56da2",2638:"372d2263",2640:"dba6e014",2652:"9ed88412",2663:"9d0312da",2679:"7e9327af",2700:"5e58268b",2739:"4c5bb0e3",2748:"7c87f31d",2759:"d28b7dcd",2768:"ad32f2e8",2772:"48f24bab",2791:"ba2305d7",2839:"00cd9d46",2854:"093ec6b1",2867:"c03096de",2884:"8539bc43",2912:"404a423b",2932:"580eda40",2954:"6e7eafb1",3075:"a80470ee",3125:"32dcc326",3131:"d29911ee",3137:"4f7cd6bb",3144:"aed1d196",3179:"5c3bdea9",3303:"6be904db",3313:"a3e90044",3323:"b9073401",3471:"1c4da86a",3558:"7fb3fac1",3693:"b783cafb",3694:"5e85aa31",3722:"50cd3db4",3740:"dc1f5b39",3764:"394bb252",3847:"43d31177",3940:"260d7fd0",3965:"d18b6106",3968:"50494a26",4014:"c943b655",4028:"04ae9608",4040:"b4ced939",4060:"0bf991f2",4074:"817ed3cf",4117:"d772531a",4133:"b18f8924",4137:"1a47b5e7",4205:"0c12eeea",4269:"18ffe98c",4355:"1497bce1",4368:"25fe5fdb",4382:"56471e5f",4405:"1dcf0194",4467:"ff5a6f64",4514:"3e8d408e",4530:"1cb4ff83",4574:"d9f219f0",4591:"405284b3",4602:"9920fffc",4777:"43d0cc21",4790:"3b6474f3",4863:"a0c4b68b",4890:"7c52b6f4",4919:"4929a6fa",4949:"066b815f",4980:"006c64a3",5008:"6880fe95",5028:"f5c507c6",5029:"ca8c3af7",5065:"25ed107d",5193:"d0ff2798",5195:"8c7f4a45",5206:"4b9df2bb",5269:"627b1f39",5270:"6ff0c558",5289:"b3a451d2",5295:"bbc3d059",5304:"44fefb3e",5322:"22b4f68c",5334:"86642e8b",5338:"0e11b6db",5594:"0c74166d",5622:"fb863cab",5631:"32ba6a4c",5633:"a804bb8e",5647:"d03e0408",5679:"58c4a37c",5721:"42bda306",5806:"1adb4ab6",5834:"6a34bd67",5887:"b5960565",5888:"478a7123",5936:"4959f63e",6022:"959a592e",6054:"87c914ef",6090:"384f1d9f",6158:"0d706529",6170:"14ed99c2",6203:"0ba05648",6225:"baed6a8a",6289:"59ba4d58",6310:"14a73a68",6350:"ce3c8465",6368:"207e0bd3",6371:"4658250f",6372:"5c865244",6407:"47ddb104",6467:"3faff22d",6475:"316bb886",6552:"ae59a273",6559:"21b16c5b",6636:"f6a1f09d",6677:"ea2cf37a",6692:"0cca5757",6697:"17ace3c3",6709:"debf2b54",6739:"01ceb854",6761:"507d925b",6778:"9cb5abfe",6791:"ae0384e2",6823:"4f684eef",6897:"b52407fb",6930:"ac385e9e",6944:"52f660a4",7035:"e841f655",7071:"6e78b29b",7100:"e7c12d1f",7103:"9d67d055",7125:"b744ccc8",7167:"bfd461a8",7317:"b1b9fe16",7355:"96f48bd9",7444:"502758cc",7458:"8fdc7f76",7517:"4abd5d51",7566:"12a626af",7622:"9df106ad",7629:"2a74f6a7",7715:"06a1e3b2",7734:"d6b46894",7747:"79a2ad2d",7773:"869d1599",7802:"deab0229",7850:"68c81474",7907:"7ba20019",7953:"df4b657d",8003:"f85e4dd8",8070:"0480b142",8139:"addb1e87",8174:"464278e7",8177:"38122bdb",8183:"c7a3e7d6",8192:"934057b1",8209:"01a85c17",8289:"c286af15",8348:"09ff3d76",8354:"db9fbbab",8363:"24179e00",8367:"abd23114",8407:"7a049829",8413:"cd17745a",8417:"bbced460",8585:"497095d8",8619:"15c2f51a",8632:"22351c69",8645:"212a125d",8651:"80bd773e",8699:"8f08e203",8762:"1cd42d14",8766:"7a11cfa3",8840:"e161c803",8888:"3a6a48c9",8889:"effd3c61",8900:"eebb3a9f",8902:"dc8afb47",8965:"b6e28713",8978:"6d5644d6",9003:"91415840",9045:"7c9ee56f",9066:"4c492fb9",9076:"52b6ceb8",9127:"29c12ff9",9152:"17666b14",9189:"d4eb85a9",9195:"b2d2727d",9228:"d336ca2e",9257:"0687df23",9262:"29a0dcae",9351:"4c14260d",9379:"7fd1d1a0",9380:"409e52b6",9385:"8ea09047",9428:"cfb1206c",9432:"8cd4ff44",9446:"49b92117",9458:"762ed4b3",9460:"01964c8d",9471:"ebf12a38",9499:"953153ea",9516:"2fec12c7",9525:"bf1307fc",9570:"8eea179e",9573:"13ab27aa",9631:"79e67a34",9647:"5e95c892",9650:"747973bc",9676:"e4d28161",9702:"a0d28f55",9722:"c35662b8",9724:"46e7f252",9737:"0d37dd31",9790:"6a0b8fcc",9836:"2fd2285d",9860:"d6d00b14",9870:"36e3724e",9891:"8a5cd0c4",9907:"9c0f572d",9914:"06b7dbb6",9944:"9ee88b57",9986:"c782ecef",10030:"c2277200",10118:"f89af623",10181:"f7519df0",10198:"ce5d6b55",10238:"0eb44ec6",10285:"b69fdc1c",10354:"8c29db87",10472:"0ca8e4c5",10488:"dc8e8e39",10492:"40011a75",10535:"79c374cf",10545:"1ca50c03",10610:"b1051fbd",10637:"6a326c1d",10642:"936cf648",10653:"5bf02344",10688:"f71afd42",10727:"f53cf472",10767:"931768b8",10789:"10fb8f29",10884:"9009d9a7",10887:"ac65ed35",10907:"b964b0d3",10920:"42746bb6",10924:"f0c78ca3",10956:"726b1d2e",10978:"90db14b9",11052:"9f1f18ec",11058:"7c44e50a",11111:"2750cc84",11156:"f97de939",11275:"2e961a80",11289:"82b971d3",11299:"ac338f84",11326:"f9cb0cea",11355:"6ed20719",11508:"039c7621",11565:"3d026c66",11631:"d2a270f8",11684:"9aee6bed",11716:"8fa68fae",11725:"51e360fd",11745:"446810fe",11768:"4cc89b71",11782:"3f65fb56",11795:"43ff550e",11804:"b9168984",11806:"7cb0ba19",11842:"f630dd79",11855:"51e592eb",11869:"43c57b15",11905:"41b6dd58",11906:"757b41cf",11977:"608a85b6",12047:"d25544dd",12134:"088c57bd",12144:"5617941b",12186:"3deb4153",12316:"77a5f3f8",12318:"9b28f794",12378:"2592f25d",12393:"7d05d2dd",12395:"a0fb693a",12431:"61c30610",12452:"f4a568da",12477:"ac9a039d",12485:"3e8ec4bd",12513:"70570476",12535:"2ca30cc7",12542:"0bce5bfd",12608:"ae632d28",12613:"bb6acec0",12654:"98af85a1",12679:"755aa830",12686:"f81e2128",12763:"0166538a",12869:"236efcfc",12945:"bc561320",12995:"72ed5e19",13021:"9b078b3d",13060:"075d0b63",13063:"6695bce8",13152:"4d01f822",13203:"a5fd3751",13210:"326f3c2b",13236:"814487fc",13298:"be65306b",13321:"bb230ab4",13366:"c7690f1a",13374:"0c9b8a46",13375:"15be7f5e",13406:"cdd81d53",13419:"05f125a2",13465:"f7887fd4",13511:"3f9b0969",13514:"35f2b245",13522:"559e18a5",13525:"aeb9cb2b",13534:"a221d960",13535:"27aec4ea",13575:"1b3749bd",13580:"b7ffc82e",13626:"16444e78",13698:"e32c93f6",13733:"e4126631",13762:"35ace877",13803:"5005f1cd",13849:"fdf3b0ba",13918:"a586ac12",13943:"79c522c5",13982:"f4dd7c05",13996:"bb5bef2a",14003:"fe0a5e13",14027:"4fe2812d",14042:"6c6eb236",14043:"2da40935",14125:"c58d5881",14134:"88677a17",14136:"5ba7f3a0",14152:"0c792305",14163:"79f5ed7e",14179:"0466f8a1",14257:"973a6c2e",14259:"4d62164b",14261:"881e0310",14264:"15b03510",14317:"2d2f3863",14340:"02704d8d",14363:"33732613",14388:"4330e2c5",14399:"95a29cbe",14410:"262f048a",14433:"5a1ac6cd",14435:"a2218d2d",14496:"0dc66419",14536:"b812e3a7",14599:"04287605",14627:"6b0d4b59",14726:"cb290368",14750:"8049dc2a",14802:"4193b389",14822:"da9a6aef",14835:"3145e86d",14868:"9c9e6d14",14871:"281df6d1",14917:"b2f79aaf",14919:"68aea991",14943:"b634826e",15038:"e8bf575a",15078:"6773b5f7",15079:"461201e5",15144:"e6691bac",15165:"29fcaeb8",15186:"809a1b46",15220:"e0719818",15235:"246d116d",15272:"ee5dc595",15292:"7bd6096e",15316:"a0583d0f",15403:"bbbd3731",15407:"413b2e2c",15427:"63067f40",15473:"ad07d5ed",15501:"259868d4",15552:"1ff337bc",15607:"a309ba1a",15764:"0d11ee2a",15805:"65df1ffd",15857:"520bf372",15917:"04d2875d",15947:"f05cb9af",15955:"1ea67680",16017:"777c5b61",16032:"7f5a2db2",16100:"5685cb00",16209:"f0db470e",16231:"d05ef008",16319:"1839a246",16377:"8eca1d9c",16499:"58c0e7a1",16500:"b5eb2b1f",16622:"3d8e248c",16641:"4ccf094c",16747:"ab84da8d",16885:"7c3644b1",16942:"426a3c0f",16950:"bd7a3684",17009:"a3848d24",17033:"30e67995",17115:"3687929f",17119:"87557f51",17232:"bc36781c",17294:"82fc01c6",17332:"b4762a45",17368:"de821e02",17402:"b629aa71",17411:"496a5e3a",17441:"c5c8072f",17474:"f7750a15",17502:"dddd9631",17508:"80e60ab1",17530:"8c212f85",17581:"46e89fee",17626:"47f3a505",17681:"a2af438a",17709:"89514438",17739:"b1e22259",17805:"bc4a985b",17818:"2d9024ae",17860:"7cfc22c1",17867:"9130943e",17944:"36f96221",17987:"f67021ea",17988:"a3255c75",18006:"059d2658",18033:"c1fa975f",18103:"7707ed5e",18121:"3a2db09e",18146:"c15d9823",18350:"889bcd45",18351:"670426f7",18401:"17896441",18438:"7eea383c",18445:"762cb2d4",18520:"905fdac9",18620:"1bdcf2af",18645:"e798c07e",18738:"3b375bfc",18757:"10b6d210",18798:"6c76cc4e",18940:"6cc2587c",18950:"d75d009b",18998:"7487c019",19042:"378587af",19052:"6e547e58",19070:"6f878c05",19108:"6f3d983d",19109:"868e6cc1",19136:"9c3fccaa",19237:"e77e0240",19252:"d9884433",19267:"1354c4fd",19271:"345a8726",19369:"a2cfe627",19387:"9228ff94",19396:"c61ed569",19425:"efd9e5ca",19428:"4b6ad6c1",19474:"a3c850e8",19513:"3faa4dcb",19521:"7f5a4e86",19542:"3548485f",19664:"76b52ed8",19687:"547af68c",19711:"0e4a3360",19736:"9e46f997",19774:"3c6c92ad",19798:"83c71362",19801:"a5052667",19865:"7e53000f",19891:"389b06c3",19999:"99bce14a",20038:"0ef00d46",20103:"430ee71a",20160:"ecc82a52",20194:"69fce7d3",20229:"5aae4c30",20286:"f60a8e85",20325:"6b7b009f",20358:"a55cf427",20361:"784d2d5b",20405:"a170c4b2",20415:"839320c8",20424:"c97a4f9f",20434:"39c0c867",20491:"4c056f0f",20535:"ca16b0fe",20541:"2925f13d",20619:"eae39fd7",20676:"874c4928",20713:"ff781369",20812:"a827f20e",20857:"bd440596",20892:"1d99c39c",20924:"92328698",20931:"de69fdc1",20951:"5ef30e9e",21070:"e3b4fd8a",21077:"7ff678ac",21121:"c1b7702d",21177:"574aa417",21191:"53798754",21273:"58f98693",21275:"e737da10",21290:"fab95ca2",21303:"e4bea80c",21316:"266d37b9",21341:"b23dd436",21346:"b2c62a3d",21373:"50737734",21401:"5105c354",21417:"9db9c0e0",21448:"995d9cd2",21482:"5f404bc7",21524:"69f9ca18",21560:"6c888f35",21574:"263bc6da",21585:"d17a9646",21630:"aad02470",21635:"6224d640",21662:"b1722c22",21729:"ea9d899a",21764:"df2a910d",21805:"1926d3d7",21816:"fe2d6fd6",21837:"7ce47929",21874:"a54a9a31",21909:"d3de5519",21921:"a8a7f86a",22141:"625beb64",22313:"334d7764",22369:"83a16324",22385:"9a5e9795",22428:"47425d5b",22447:"f93d9ca9",22472:"494588b9",22483:"95d688dd",22499:"2923e8f3",22501:"ff15e2eb",22504:"61773850",22534:"6f37f43d",22595:"ca5d6ba8",22628:"a36f1f19",22640:"69375b61",22647:"c760fa6a",22673:"c699b29d",22709:"74c5eee6",22750:"f701b058",22816:"70db0e3d",22844:"d6ededeb",22911:"9a29e675",22916:"d49fdba3",22932:"0d5f3865",22973:"70a32eac",23002:"1fd06df8",23041:"07da43c6",23081:"ef1c4348",23083:"7050405a",23119:"39307ee6",23141:"a1c39b3f",23172:"b994507e",23332:"fe86f3f9",23388:"fa5f4abf",23438:"95d97af4",23475:"fdeab78a",23547:"a67ef186",23563:"f24f712d",23591:"53746e73",23600:"af9b00f1",23625:"996abb7f",23634:"59e9dc06",23637:"9c150758",23641:"35313843",23666:"cb16b650",23676:"d235b83f",23700:"ea8366f8",23765:"fac8c34b",23773:"2153fb85",23880:"3affb2e4",23938:"be354741",23998:"afe4349e",24022:"9c7497bb",24065:"5ef1cef9",24084:"4ed60a8a",24098:"89ad6a53",24126:"897856d4",24140:"5ce7213e",24172:"1d2c1565",24234:"3af3061a",24265:"7ebf6f1e",24340:"a4e075be",24366:"99d3870c",24373:"5cbd92da",24377:"b5b376a0",24381:"4066930f",24437:"77b23935",24475:"045bd6f5",24476:"6cf93e89",24704:"3befffe2",24728:"91c64349",24729:"931beacc",24741:"e2c09daf",24776:"a852ca5b",24815:"a68f650d",24826:"ed6ec893",24853:"7cfd7c3a",24866:"e0c51b98",24867:"b4972e92",24868:"c6301cab",24891:"f0709bee",24912:"8e01a36e",24919:"cf6129a8",24932:"51fff4d3",24970:"231cac3a",25013:"1e3d98df",25044:"9cd11b72",25067:"9531441e",25101:"8bd2257d",25130:"b277530c",25186:"96046d19",25273:"19531bab",25285:"1da16fcf",25335:"537936fe",25396:"d78b8cbc",25397:"4083b748",25426:"b973fe72",25452:"5ef8e026",25531:"af5c68f9",25532:"40ebd41e",25539:"7fb09d55",25578:"c7382291",25582:"65603bfa",25601:"7205cbcf",25778:"d56bdeaf",25781:"ba115cad",25845:"d2c3f17d",25869:"6d1cdd35",25871:"c9953bfc",25874:"17268e53",25959:"f7f19049",25987:"1645b95d",25993:"92db1d5a",26003:"90c5833e",26009:"20f9256f",26060:"7662a79c",26064:"2316ae30",26081:"de97f7b0",26183:"fd6c914a",26216:"4e3e1e17",26260:"477cc2f7",26305:"878dce9b",26333:"9579dcb9",26337:"65009e94",26414:"14fd0d1c",26548:"9a5789a7",26577:"8eca70a5",26627:"8353278b",26630:"ba839983",26632:"960d3eb8",26722:"1802ae9e",26724:"78bc6f25",26744:"21e535c3",26797:"4681e75f",26833:"e2fe7a08",26859:"04fb6378",26882:"958a296c",26900:"b3749361",27012:"aca5eee4",27020:"58714c61",27088:"1547c10f",27115:"e37692b4",27123:"c4d1e829",27179:"cbec0238",27212:"ce710151",27369:"f52b34fb",27388:"f7e205e5",27398:"a3983124",27444:"7a3be7e3",27473:"31ed8f07",27663:"9f8f3c52",27737:"6d701be3",27823:"f95764fc",27887:"b0d5c266",27921:"10ccea9f",27955:"2e7e1134",27995:"ee4b8f19",28020:"ecd026d3",28086:"2cc67ca6",28127:"859500dc",28147:"abde445c",28166:"8120957a",28169:"513b54fb",28178:"9adef974",28204:"758b9bc9",28235:"20a81622",28252:"b8d3e47d",28319:"ec410c33",28410:"89cbd1b8",28428:"b212a261",28435:"5b374304",28499:"261fe657",28627:"54ad050e",28679:"0871002b",28687:"ab48a39f",28701:"c3856c66",28782:"fd5c5192",28793:"379ac6de",28803:"8b6fbbb4",28834:"7ed2bc06",28853:"4c00d6b9",28889:"2a0b5233",28920:"e7ef6e16",28941:"8c5b1685",28945:"f382fc88",28993:"8e7128cd",29060:"9c7a2f87",29123:"763fa02a",29171:"6f3c73f2",29204:"d4836e14",29243:"d330b699",29337:"8a472aba",29408:"fbce6636",29414:"b92135a6",29421:"163a1bae",29452:"13171783",29457:"e4d4ec4e",29498:"da05f515",29523:"b2f58160",29526:"b45a54e9",29653:"13f72d6c",29679:"6e0a413f",29690:"b1633424",29698:"a642dcef",29708:"80b93830",29733:"da10e58f",29795:"e6557239",29820:"16381f20",29829:"d973177d",29845:"5d94dea0",29891:"26f5f607",29922:"5397b577",29950:"5de1c39a",29969:"332b14e0",30035:"87546ad3",30066:"6b836ac4",30154:"339988c0",30167:"22bf7f69",30262:"5320ed26",30265:"132be4b3",30317:"af953e30",30355:"4b678e99",30378:"5c5d3a2d",30396:"e4a43002",30410:"90ffbe81",30564:"407230da",30569:"c7165ecb",30579:"1ee0a129",30583:"775c5b43",30628:"6ee9522e",30654:"5d70257a",30790:"a7c3ebee",30827:"5814834c",30831:"ced554c7",30842:"d8412727",30848:"677757ca",30934:"44c28e7f",31004:"7dea1595",31016:"416fb909",31039:"71d5b1f8",31047:"3f9a993f",31125:"c3c8ddc4",31141:"be9442ac",31144:"fbf25af7",31220:"b8ab477f",31255:"5c81ce12",31257:"67f1ea1e",31272:"fa366c46",31393:"07f6d4d1",31465:"cc564cf4",31541:"1668cad9",31544:"c602d51d",31567:"8c4bf82a",31680:"b38306ed",31705:"05957343",31720:"0a70c05a",31726:"624fba40",31749:"974bc2fa",31791:"04eab06f",31797:"14a26e2a",31861:"a14af959",31862:"c791a767",31869:"2dcd9099",31957:"2e31e40f",31972:"4ef864ee",31996:"8dbbcff6",32005:"f0dc0726",32012:"9ba9284f",32039:"116b7e55",32183:"5110a7b7",32225:"eb05a324",32270:"53ba3577",32278:"149a2d9e",32335:"f9a4941d",32365:"fe345ee5",32391:"e1407f5b",32472:"c08ba4cb",32527:"e6c2d44b",32558:"40c3731f",32691:"798484e3",32728:"1b520177",32824:"943ce597",32825:"fdddbf66",32847:"8590fd3e",32875:"65511420",32949:"43764587",32960:"225cc45a",32993:"604cd7c5",33049:"97e51b1f",33065:"a172f6b1",33192:"ee003a92",33214:"2a422024",33234:"465994bd",33290:"7da68ccc",33303:"f5ff1a23",33350:"2d0c5b52",33458:"c16e0158",33517:"3ed70074",33524:"f81c13fc",33533:"65ea2434",33540:"fe7ac0b7",33559:"8f00bf20",33567:"6075be34",33572:"269d704a",33577:"df68b7b1",33612:"3f933ca5",33621:"33241236",33638:"d294bdb0",33643:"da7149d8",33659:"81a6b262",33724:"755872dc",33804:"1b31aebb",33806:"07399e71",33816:"45f2496e",33825:"f8a443f2",33854:"06758c2b",33881:"a7690409",33889:"e9c7c2b7",33999:"49aa1a16",34127:"18b5779a",34156:"f5c63bac",34179:"3f4396e6",34182:"80f4f5b6",34196:"13e6af21",34255:"e8a8dea2",34331:"164e61b9",34335:"6153f283",34388:"79fbf53e",34520:"538371ec",34528:"b038a3ec",34543:"782eae4d",34549:"0735e5be",34597:"40cd51eb",34621:"f30fab6f",34638:"852f3aa7",34662:"ef071c32",34691:"02e54e09",34706:"47a1bde9",34750:"a5fea1bd",34803:"e076fc92",34808:"2e6ef254",34810:"924dcf98",34837:"10fcd509",34847:"dc7f8f8b",34877:"f7702246",34904:"f44e1bcd",34918:"c9cac9d9",34970:"40dc6b06",35044:"5e2f1fff",35067:"47cb0ede",35088:"35788c78",35139:"6fe5390f",35144:"83829ea7",35179:"372475fb",35248:"2947aa63",35288:"0f3b9f0c",35325:"d42065ad",35329:"a7827f50",35379:"d9454955",35402:"976ab233",35532:"388d5845",35618:"97bb73fe",35730:"2b4cfa56",35733:"8fc868c2",35742:"aba21aa0",35766:"ff6f6072",35767:"3019fa66",35784:"d240c26d",35837:"dd61deac",35881:"8852ad6f",35922:"92928c02",35932:"a3142532",35960:"dfa4e294",35983:"b5a599e0",36009:"04afe417",36055:"59be0e8d",36067:"ceff0062",36113:"28f38ec5",36144:"307d712e",36192:"b60e28b6",36264:"8a759f03",36302:"eafd7a74",36379:"467ff4a0",36387:"e23d1069",36549:"5ea64d6c",36569:"1472dfa3",36594:"94a5efbc",36619:"2573355f",36640:"1084a32f",36672:"b6fb003b",36689:"77787075",36747:"a3015275",36790:"12935692",36851:"649faa0e",36885:"1007513a",37109:"d73e6b67",37115:"0b4fcfba",37147:"97da4333",37191:"d786420c",37204:"8f3b27f1",37206:"43dcd746",37209:"e34cac0e",37257:"37524cd6",37329:"554882a9",37366:"6013ac2e",37403:"32b44a8d",37486:"ce46a18e",37489:"6a293d29",37577:"e9705ef5",37578:"e17d733a",37617:"be2ede3f",37643:"a6aa9e1f",37673:"e2657c7c",37740:"2cfb265a",37799:"feab7dcb",37809:"5f87fd99",37827:"42545929",37839:"d5e29b01",37841:"9ad03801",37842:"cd060f40",37872:"88d0aec4",37954:"3f272b07",38001:"690cbdd8",38003:"8d086d51",38056:"baad6bf9",38107:"1efbb938",38158:"66372f10",38238:"2e9f383c",38348:"bbc1ba35",38370:"cf858c26",38385:"7c15a7a1",38416:"dcd7017c",38439:"9474fc0c",38449:"91335d19",38451:"f0fb7623",38468:"b4a84943",38600:"0f497157",38605:"6dc06ee1",38607:"7eaa135a",38700:"1bf4b05b",38767:"c4de5678",38786:"872a41d4",38816:"5ea125b2",38835:"6495713a",38839:"110e78c3",38888:"6f5eeb66",38974:"2c50796a",38999:"daa4f13d",39058:"53317765",39088:"5560f355",39110:"cb9e1f3d",39138:"755ebe0d",39142:"b5f01515",39153:"0b8a4e62",39192:"09b62bf8",39197:"484cbf74",39202:"8b795044",39208:"93f7a5ff",39230:"b992801d",39240:"90f157dc",39260:"47074836",39265:"b40642d4",39283:"b1043e58",39405:"cc6505da",39407:"995840bc",39417:"085b8128",39423:"a615dd65",39444:"0f65ccfb",39487:"9d833fad",39494:"b326207c",39496:"7b2a260d",39505:"782d926e",39518:"630bda65",39520:"441dded5",39555:"4a80e27a",39569:"06a8eab3",39587:"7d0e1375",39608:"97ae1fff",39712:"8f64cb7a",39715:"a68f0e64",39758:"07bef82a",39773:"fb3759c3",39777:"320da59d",39781:"3c2d0811",39840:"8fa753de",39887:"9b0c8626",39901:"d76c455f",39908:"09a52b17",39921:"6471fe03",39973:"d4a2e930",40065:"5f469a3b",40124:"bc817d28",40178:"0508dae2",40205:"edf7dc61",40245:"ecaa52af",40369:"3c36ce76",40463:"0cd1a20c",40464:"4d1df301",40555:"3415fffa",40606:"3e032765",40616:"12a5de9b",40651:"bf5126e1",40690:"dfc3c224",40697:"cc01a9d7",40755:"8393a516",40758:"1a7b33c3",40797:"d2110cc0",40827:"ddebc8bf",40847:"2662e92d",40907:"321dafc4",40919:"2884dc3d",40927:"881bebdd",40959:"ca0149b2",40999:"68c5632a",41030:"33d4a0d4",41125:"0fa5c29b",41135:"85ffab62",41159:"93340492",41260:"6c9200cf",41361:"247909f7",41455:"2bf5edd4",41461:"fd517388",41491:"c3f88c36",41504:"8cd118b0",41518:"bf9d9be8",41594:"da172daa",41640:"d407c2c6",41651:"e8e3bd30",41677:"9017a110",41739:"b9edf71b",41750:"bc38a1dd",41852:"29db9f25",41867:"78f0e259",41872:"58b3152e",41892:"dc5506b6",41907:"b7a56be4",41944:"14828316",41954:"7c49023e",42068:"c413ce11",42187:"295cb0c0",42196:"671726ea",42268:"89c8b6fe",42298:"3fa17fc2",42299:"1f979aba",42350:"6df57399",42368:"ffac4af0",42465:"b8212f90",42487:"203cc654",42518:"34825c6d",42566:"2b505997",42576:"b184a577",42596:"eafac744",42690:"165a93e5",42702:"28826a4b",42723:"d8ed3ccd",42794:"89f1622a",42799:"5314580d",42801:"bf3d61ec",42879:"aeca0a21",42881:"432e670c",42900:"c9d95cbd",42905:"145364e0",42911:"39c7a203",42920:"ebbd9274",42937:"a8c92c97",42942:"19028add",42995:"2812e3b9",43029:"e3477e52",43035:"2907d5eb",43040:"676c7c8e",43042:"bec2896b",43078:"50536743",43088:"341293cf",43113:"12164231",43195:"d831e3b2",43286:"fe9b6676",43352:"4c07ba53",43421:"f048922c",43432:"fbf82623",43445:"83fec35c",43461:"7b89671b",43512:"d319e9f2",43642:"4cd0ba1b",43677:"7117a8cd",43739:"752351f9",43868:"c09831a6",43922:"a8b66660",43980:"509aac2d",44008:"5a8b22a0",44022:"454b7ded",44040:"43d1bad2",44045:"894f5c0e",44118:"6503f568",44123:"e2fc22cf",44125:"ca2b0f7e",44129:"d653d7ed",44143:"19acc4ed",44290:"4f087d20",44326:"cfca4f26",44339:"0a91021f",44380:"8fd374dc",44405:"d363a80c",44419:"216bf65d",44489:"6fb01322",44492:"9882af90",44551:"9b2ae5ff",44567:"502906a9",44640:"20bfa334",44662:"d65b2c25",44682:"bdc69003",44719:"60165173",44840:"27077527",44876:"260982a7",44891:"30769210",44933:"f3d02b29",44939:"43f3c18b",44948:"c7a67184",44960:"12b957b7",44964:"8914fdfe",44994:"41af2803",44997:"6e34e766",45113:"55946737",45117:"86fe1f77",45146:"b528d4d0",45210:"49a37e79",45231:"7737fe61",45238:"c3f790da",45290:"db90ec2b",45323:"815e47ce",45339:"24d4b044",45460:"459dbf85",45500:"e85bde03",45541:"a722b43a",45561:"7dce0476",45640:"ef7891f5",45670:"c36346e3",45682:"3533dbd1",45919:"a2a2954f",45983:"9926d60d",45988:"95aba771",46001:"8cf74eb7",46008:"2e048b0f",46025:"1fb7523b",46039:"f724037a",46063:"67893f6a",46074:"f1087137",46156:"53cdeba4",46197:"8e4df3f0",46244:"ea511817",46296:"04b49851",46375:"e49bd305",46425:"00239e8e",46451:"4bb02a47",46472:"bcb734aa",46513:"9dfe5a97",46515:"c13f246c",46547:"c5427124",46578:"8012465a",46598:"8393d26f",46652:"5f0630da",46662:"4f48cd24",46754:"2b6a7ab0",46802:"8f07070c",46840:"b2e2853b",46850:"917a523c",46888:"db9d1243",46891:"85f8bce5",46905:"57a58e78",46911:"ff13b684",46917:"7b0a3d41",46921:"a28b89d0",46937:"37ffd066",46979:"a9afee85",47041:"bbb92300",47060:"26996ad6",47087:"a5c8a2aa",47106:"696d889c",47110:"1366dd9b",47127:"c648308f",47206:"4d2e69a6",47236:"131c6f6e",47247:"5048202c",47248:"c7e2144d",47297:"6111afa9",47433:"b9a767d8",47517:"09e5dcaa",47537:"df0967b2",47555:"0029660c",47622:"0c1ff092",47668:"b1899f7e",47693:"39a07549",47699:"17093d00",47730:"4edf53e5",47917:"6bb76d2c",48003:"7ad7688e",48023:"47f96c4a",48050:"ca713325",48063:"c17f9567",48071:"852ff8c6",48091:"0cc98403",48130:"f81c1134",48132:"c467a11b",48162:"519b5759",48167:"00021e3e",48188:"9d8965aa",48214:"bef59fc9",48285:"e672d61a",48295:"e2b4c006",48304:"189e3ac3",48310:"f6884a75",48355:"d8c836b4",48360:"959b45a2",48361:"712c16a5",48532:"2896ce7a",48544:"0a5f31be",48592:"c1506482",48613:"7a556257",48661:"ef4f3413",48673:"5db19d16",48733:"d009d47f",48768:"e054d919",48793:"b71be52f",48803:"79392e36",48842:"d0b9fc21",48883:"ce18dbde",49013:"9d9f8394",49046:"c68848f4",49055:"be94b749",49071:"97d17d75",49112:"3455c0d5",49174:"0ac5df82",49199:"7795e379",49226:"f2a06fea",49280:"6111bc4c",49305:"ad0a2b75",49350:"2417a912",49388:"8f1510f6",49461:"2019135a",49521:"49d63cc6",49569:"0d819826",49579:"e01bb6c7",49583:"8ec8c5c5",49587:"2cbb324b",49655:"b0ba51ed",49676:"2391d372",49783:"acf72eb8",49786:"5bd6f6db",49880:"3fc6d2b7",49991:"1d5cbb7b",50012:"40c88f7a",50095:"837c6843",50176:"756c7559",50178:"18dddf22",50183:"f2790563",50259:"ea981684",50261:"ec6a0daf",50293:"fc6a53b6",50361:"09a0b8d6",50414:"c59a4667",50428:"a33de290",50446:"eaebafad",50472:"a4e24d6c",50486:"313c653e",50507:"129c4e7a",50570:"c112d1b7",50584:"3fd21eb6",50642:"e964b083",50700:"75be343d",50703:"0914ee26",50747:"0d98f85b",50769:"a339c33e",50800:"b365f6bc",50819:"4251704e",50827:"aa04bdb6",50839:"8aa0dce2",50934:"e7b82dc0",51047:"4dd6f8d8",51049:"6bd6fe03",51060:"e1758f93",51090:"12aaf145",51102:"3304ac90",51116:"d366555a",51170:"5e85b509",51175:"c39359c5",51184:"90d97cfa",51229:"a06af3ed",51237:"85a6e3f4",51378:"0a8d92af",51412:"fe2deb8c",51420:"7a04b795",51438:"14515c80",51494:"65eb0274",51548:"45b02367",51578:"8447ad38",51584:"4c2a4e19",51589:"f762fff5",51636:"ba5671ab",51649:"6537c712",51665:"65d842b9",51694:"612cc46d",51711:"89bdbd96",51755:"b50c8022",51763:"9199f8bd",51769:"1bd7f091",51814:"91a39dd0",51828:"44e51e65",51837:"5c238f81",51878:"48b6c251",51890:"b5fb4df8",51937:"34689118",52036:"1d79e793",52042:"7f8ebea7",52093:"edea4b8a",52102:"95cc61bd",52132:"3498be82",52135:"d0ed28b7",52176:"81d19844",52207:"508f03f4",52218:"1562cf35",52235:"fadc68df",52250:"8ec5e058",52266:"3e088827",52267:"6818c2e9",52282:"be224150",52286:"8e1e17e5",52342:"0dd8c0ac",52408:"ab586159",52450:"2dc793da",52462:"3016c86b",52466:"8c67a0ff",52478:"cda1618e",52488:"ca57223f",52563:"f2da277e",52569:"7199ad43",52572:"a9417ee3",52612:"27f2e2a4",52634:"c4f5d8e4",52680:"2124517a",52711:"9e4087bc",52731:"c8f57d29",52770:"85afc7f5",52785:"25eae11f",52826:"ce319efa",52840:"9e1bed9d",52882:"fa2ae802",52895:"69fcecaa",52911:"4d4179b3",52912:"82c60778",52913:"4adafdbf",53074:"2f32a261",53132:"5de85215",53170:"aaf3ff15",53178:"71514a42",53201:"d670063b",53237:"79ab3378",53256:"fa713049",53266:"76a0389c",53288:"f8fe23f1",53314:"5440b68b",53449:"e2bce80a",53458:"63436e22",53492:"a45dd8f5",53540:"92f1a745",53619:"16640689",53629:"5a2d67ad",53633:"f6ef9721",53646:"6c9978fa",53670:"52ed38a1",53742:"77612fce",53771:"3a447539",53786:"e352cccc",53832:"7c8deb08",53860:"e623e42b",53861:"2a5e97be",53871:"94dd9534",53881:"e9a95c5e",53901:"a4da17e3",53925:"a8933372",53982:"d5f056f5",53990:"f72b3fe4",54097:"8c317219",54106:"93b4d825",54200:"f97d7d88",54240:"3a55b748",54246:"5ba9c4b5",54270:"e5c0ea66",54272:"f020ef51",54274:"b7bc328b",54277:"71e2c4b4",54399:"25aa47d2",54415:"c87c5e1b",54462:"0c3d0366",54487:"52676037",54500:"fb9e14c7",54540:"66193a96",54568:"633af835",54730:"d4522125",54749:"ff4fb41e",54759:"0f7d8652",54772:"13883961",54851:"a2de6851",54879:"a872c629",54881:"c46bba44",54927:"55f8cc28",54950:"0a893fdf",54951:"98bd463b",54992:"5bcffa9a",55004:"37d125cb",55100:"70d55848",55144:"152d0582",55167:"c1cb0a0b",55206:"aaa8a12d",55276:"c962ae4a",55302:"facc4cc2",55342:"ee44120f",55401:"2027fd18",55483:"342d2e6b",55526:"f7fa3381",55534:"ab685cdb",55559:"29cf75d4",55566:"c8861997",55579:"3a9c66ce",55616:"d5caed5f",55664:"a7511015",55668:"bbaa8144",55754:"069b26b8",55780:"d425e9d6",55836:"94b063ba",55929:"2cbaf53f",55942:"41adf405",55955:"194c8540",56028:"e1a6f6ca",56037:"69ef23f7",56062:"cde3e47b",56087:"c96d4604",56097:"096bca72",56122:"a7b4d0d7",56123:"602e2465",56133:"887c7539",56190:"5160d3b0",56256:"e028a908",56259:"75a12619",56297:"575869e5",56384:"fec58251",56408:"cbb5064a",56493:"858f1266",56510:"86a45dc2",56560:"c9c31477",56607:"609f9177",56648:"06eed450",56662:"beb32294",56688:"8dd3eb38",56723:"69e6c429",56725:"9a502a1c",56767:"2d57b909",56779:"6664c24a",56826:"eec5649b",56916:"ebc13825",56948:"1dbfdc18",56974:"39ed0ae4",57054:"97b1d23e",57119:"4d68fc5d",57141:"037241c6",57155:"23348138",57159:"e15f47bb",57165:"034f0308",57178:"2575da36",57182:"a2f498c0",57222:"8ee4a7d7",57245:"e3b05f38",57251:"779bba71",57283:"5e727800",57302:"55b89dea",57326:"3ed58e4a",57359:"841b525c",57361:"79cc2eba",57426:"533bed85",57529:"0b7a8a63",57547:"27211971",57581:"b20f9cb2",57594:"2b2d1be1",57614:"dd48b375",57647:"0ed7fb46",57730:"658997e4",57749:"c07b5739",57760:"73d3ccec",57762:"27b8ef72",57885:"f1691dde",58e3:"e2070fcf",58002:"31fce735",58079:"2655f170",58111:"fa4b529c",58144:"a2171a4d",58255:"400deb23",58259:"078daaf7",58301:"b296abb0",58351:"cbf83637",58359:"a5e62092",58393:"15ea2a5f",58428:"ec854811",58479:"2a882da6",58484:"36d29ed8",58492:"0ba9210d",58581:"f0e5f3ed",58594:"6bd22ece",58666:"a1e59af5",58667:"a36e07dd",58704:"7fde9a4c",58707:"6b6aadc5",58721:"9ffdcfdf",58780:"b7d1b016",58857:"bec552c1",58874:"030dfd2b",58880:"52e97b50",59004:"7dd8c933",59005:"39fca8ac",59009:"a169aa70",59020:"e84457bb",59053:"11b61292",59070:"c43c6f4f",59104:"c762272b",59114:"1a20bc57",59123:"17ffd4ff",59262:"97db775f",59326:"e2b886c9",59349:"156bd387",59475:"55d26943",59493:"150f4998",59521:"b2555332",59614:"c5c8b091",59680:"70cca634",59702:"c9b0e9c8",59732:"9873f053",59740:"ab90a7b7",59779:"28553a2f",59781:"0e7c02dc",59866:"03f08ad1",59873:"b37c8625",59874:"b62e8859",59923:"95789563",59940:"888f35af",59981:"03137a3d",60020:"cc3a93a6",60119:"66eb519d",60167:"152819f0",60185:"e04f784f",60187:"a52bfe45",60193:"5c430979",60198:"5c417b7f",60203:"5c5dd48c",60257:"3fa5d64e",60267:"3d3aadb0",60294:"93d54c10",60315:"42d452f1",60445:"f83acba0",60510:"821320eb",60565:"492bd0ed",60613:"3523854b",60625:"7a82ef89",60676:"244c7b0a",60696:"d163928b",60725:"73e80d5d",60762:"e9d18558",60765:"0614dce3",60819:"c316a3d7",60830:"2df3fdca",60837:"d2436a2b",60861:"c9d25191",60884:"709e1a02",60901:"00b58b18",60908:"fe53d354",60953:"3e85731b",60965:"4d18bf1b",61027:"b1506a50",61114:"127f988d",61228:"4521c19b",61235:"a7456010",61265:"84536aab",61300:"31eb78c6",61345:"f12e5474",61378:"c90911b0",61416:"a1e3441b",61481:"b0f00280",61559:"6b206066",61584:"55b57e06",61627:"35190cab",61652:"495a3878",61702:"faa24029",61721:"4f3239be",61734:"ad2a3b86",61740:"bc80aebf",61786:"aa1329f2",61844:"14da6f38",61886:"124bf435",61939:"88937603",61950:"820ef6f8",61959:"9df3c51e",61991:"a6972a3c",62015:"5cc68a8f",62037:"f1b1ae9c",62061:"2638eb20",62111:"d2289dcb",62120:"cf1cd5da",62138:"1a4e3797",62147:"8d364c04",62149:"5955b5ee",62162:"22c027ab",62182:"0e7e04d8",62188:"1e38d1de",62254:"9b695413",62264:"cb69dde4",62400:"8041f857",62431:"856bc38c",62456:"3e9695b6",62469:"f448ea15",62501:"564f4b07",62549:"cca423c7",62568:"abfb0eb9",62635:"61b91652",62639:"0f378f0f",62641:"58728d91",62672:"fcaa2a90",62691:"d9175e87",62703:"345b88a3",62712:"f9a8c23e",62714:"1769198d",62827:"4e78ea4f",62845:"ae5c2262",62867:"07853d90",62875:"f1096413",62927:"c03ef713",62953:"7a0d5539",62957:"e5c6728d",62975:"baa95a18",63005:"b0635fce",63064:"754efc6e",63075:"37844666",63124:"d4bcf69a",63151:"93f4c2fc",63181:"29fd9202",63228:"34c77011",63262:"df8bdaca",63291:"2f1bf685",63298:"259b7e50",63317:"7dd3b2a7",63330:"60b75cbd",63332:"67474760",63367:"44bdb747",63382:"5f87f962",63387:"9780a9d6",63420:"04ea9cef",63430:"465af2b7",63455:"f30739cc",63515:"a3041c66",63521:"58f053e0",63553:"1fd2ca4a",63576:"b9795b3d",63627:"43a56d47",63646:"47298c37",63682:"f82e3d74",63700:"eadd254f",63739:"81a5dcd4",63779:"c2750739",63795:"a4974adf",63836:"95e06d9c",63907:"57e245ce",63951:"8bbbdfbb",63966:"2278c359",63975:"b81d6cb2",64045:"c47fd21a",64099:"e34c241a",64184:"f4f77456",64209:"88ed6425",64264:"192402a6",64275:"48a72e90",64351:"7c0dabe4",64391:"b65b26fc",64442:"dfa86ee4",64447:"e4307155",64465:"e9f47cd4",64473:"9c8cec5f",64502:"5d817499",64527:"dae07133",64535:"3756d209",64577:"a2a8ce40",64597:"9c273b44",64623:"e52d80fa",64659:"6ae279e9",64670:"46693c0b",64737:"1d00ec5b",64749:"2da5f59f",64838:"9a8a6b3c",64899:"e80f330c",65013:"a3fde46e",65017:"f08c8322",65091:"75914136",65094:"f04b050d",65133:"c9bcea67",65193:"d0cf31b7",65221:"67922501",65223:"2a11e6a7",65233:"0530d392",65246:"ba47c7e8",65350:"517960c0",65356:"8db42357",65359:"070df7e5",65473:"a21d3492",65497:"26115f23",65569:"8008d958",65611:"5d7f3e2f",65625:"e1ba57a0",65629:"26cbee69",65647:"75ba6401",65648:"c7850e27",65658:"329c66a5",65727:"e29a1e9b",65745:"d1be0dfb",65756:"02ff5d42",65759:"7d5633f0",65835:"fbd3de42",65897:"38147e44",65918:"587a0b3e",65926:"567cfed1",65957:"21719437",66061:"1f391b9e",66062:"fada9729",66097:"ceca85fb",66135:"331ad65a",66140:"136587f9",66162:"cf5645b0",66169:"0b7cbed9",66210:"2b53c3fa",66237:"cc9fb6c4",66327:"eee168db",66338:"dcbc8483",66372:"8fb804ed",66381:"9019bc0f",66394:"c0d2ab5d",66415:"6f47e633",66461:"cb8920e1",66499:"85e49a93",66504:"b01f49d5",66510:"79d5f27a",66518:"c6ec7d6a",66524:"83037ff5",66538:"d0550e1e",66559:"ecb74126",66575:"b0e34666",66602:"9fce9e91",66715:"48455675",66753:"5389895e",66759:"e381e7b7",66822:"62e9fea7",66832:"e1fde1ef",66867:"58b9aab4",66903:"47f8f95a",66912:"6e955522",66947:"6faab85c",66995:"ba8ea34b",67044:"1b143eab",67061:"c1050649",67098:"a7bd4aaa",67100:"27c39d7f",67104:"06073810",67114:"0d86fe8d",67149:"11235f31",67189:"9736a6b2",67229:"2d4f05ca",67252:"60cef16d",67376:"01ed4ae3",67438:"b6d7433d",67472:"814f3328",67489:"3477cc36",67507:"77139df7",67545:"d89aa357",67581:"a1a96ebc",67587:"8775fa1c",67649:"eedfeff5",67668:"47766edd",67683:"cace8592",67704:"8fb89e11",67720:"4d59e059",67729:"2c18bef5",67755:"b57b3f07",67775:"39b26ead",67914:"0a0dd03a",67954:"8aec9a60",67989:"59171146",68038:"863f8530",68043:"c1c9577e",68045:"0924bdf1",68094:"e981d05e",68210:"a148d810",68246:"3b8a31a8",68249:"e48e4dc9",68253:"89c88bb6",68304:"8ea5f648",68315:"98826249",68334:"9d05a91f",68434:"975aa772",68467:"bb6faf52",68470:"60753781",68515:"f553a3a8",68551:"9a8c2230",68569:"d81421b8",68665:"f645a9e6",68685:"0b87289b",68710:"dbf13a93",68765:"719a7d9b",68770:"14033dfb",68839:"3ac10f9a",68861:"bb1a3e9c",68865:"08299697",68910:"62cdbb35",68930:"5aa406a5",68955:"f9d38c6e",68991:"b60b94c8",69017:"ef13094f",69030:"8a752d6d",69038:"8df5dc78",69129:"4a55387a",69169:"fc384521",69199:"f0589cfc",69347:"de9d53d2",69365:"4d2a5110",69384:"5a360e6e",69457:"7ffd9a5f",69501:"fc15228a",69517:"8a4df687",69570:"4d573718",69588:"a3713279",69633:"05c96a0d",69638:"9c10cdcf",69671:"36e398ec",69770:"b1e8d27b",69787:"787caf51",69844:"615fd764",69875:"b58f381c",69982:"8e981336",70008:"0860d759",70022:"c8d1e051",70058:"73a7a1a2",70100:"b318f3b6",70117:"9f3d620b",70126:"f68988f2",70127:"dae6ce88",70133:"2f2d1edf",70174:"e04f6385",70179:"cb543535",70209:"529aff45",70210:"8367b76d",70234:"e76e0aa8",70253:"5e1c183f",70268:"aefdabf4",70399:"d2786aa3",70418:"0bbbad22",70452:"b71c2d32",70502:"8c596cb7",70551:"0bd456dc",70560:"ac5d28bd",70606:"39583919",70612:"6d5145e1",70645:"a9f446ca",70649:"0920b03b",70669:"dcc774d2",70687:"70ca6744",70689:"52375367",70696:"52be8247",70712:"cb488bcc",70728:"07338b13",70749:"ecba2f16",70755:"f3e1c72c",70784:"ac00e857",70791:"fda5ba18",70794:"c1dff2d3",70814:"c3c05d38",70825:"1d89db06",70826:"23421dc8",70828:"6763bf32",70853:"e34cf2a2",70870:"4f594038",70947:"d1fa94a6",70953:"87d75f25",70963:"16029e49",71021:"05f6486e",71026:"5e0cf2ca",71051:"9b8a5dc6",71111:"48c07d39",71117:"92f96583",71162:"af7ce5e3",71227:"131dae92",71235:"18891827",71374:"40d0141e",71407:"7380ddcc",71473:"ff761994",71531:"f89e6cd1",71565:"83c3bea7",71585:"d7e2cd1f",71590:"ed47441b",71612:"a29b4047",71656:"7ec29cb2",71715:"4a74d13d",71784:"e3aab494",71792:"d2a882d8",71858:"09138901",71863:"acec9ba2",71877:"466f5a64",71910:"f783f7a9",71964:"835d6f03",72011:"31b399ad",72043:"f0dc2560",72053:"95f36a1a",72122:"3d095b6b",72135:"8cec3a3f",72246:"cf8b2bc1",72322:"4941c7ed",72353:"f23d42f7",72358:"4419dbb7",72446:"27d3d4e0",72527:"ac84c584",72573:"bd203bf3",72599:"3336b76f",72634:"c8768069",72668:"049fd4ea",72675:"f81aef8e",72707:"eb3eed60",72759:"69f425f4",72815:"56040849",72835:"2d34e835",72880:"43b52908",72907:"491d56e0",72914:"37eca2aa",72925:"b724edf8",73019:"fca979f9",73084:"b8385eea",73193:"da144c90",73264:"33e8889e",73276:"7bd25ebf",73290:"c7f860be",73316:"8b1b48fb",73331:"97d2cbab",73376:"21838a86",73397:"681664ad",73472:"092425ad",73493:"4c26df07",73539:"5ebbf59b",73540:"73d617e8",73569:"ac6289fa",73607:"01689e9b",73609:"622b7911",73624:"edefc60b",73684:"35be90fa",73693:"c6ea0115",73741:"eb08ed44",73756:"d8415e6f",73800:"5cf43a2c",73841:"569e0588",73846:"4d015a5e",73888:"06dd3057",73891:"d721a748",73905:"d502d9c9",73940:"f4731b9a",74019:"14841d7a",74027:"9d891c91",74039:"2e6d9cc0",74046:"b7aeb8c2",74069:"ea5e46ff",74121:"da7bf323",74134:"393be207",74282:"2fd7ee6b",74312:"acf7953e",74388:"57eb6545",74403:"bc42f283",74418:"103c2fe7",74490:"3d4e54c7",74524:"111d9467",74588:"84939cad",74657:"033d3629",74684:"e6ce3bd9",74792:"a5ac74f6",74880:"ec7a5af3",74902:"43c329f5",74945:"ad848ffa",74961:"9b3f2ab9",74994:"eb5c136f",75036:"0c90e710",75043:"1c4d0413",75051:"13d28288",75062:"8df3f976",75078:"dd620de6",75105:"e2585025",75106:"9f49c871",75173:"3126d1b1",75267:"6dab9497",75309:"07deb48b",75359:"84aa8d64",75378:"847c1b9e",75405:"ceaa6e69",75450:"97c492a6",75459:"605c3775",75666:"2aa42d18",75695:"68f2aae3",75696:"1ad07866",75718:"d2250181",75735:"691ef278",75759:"217c03e5",75775:"724a4edc",75805:"ad132b09",75866:"a3541f3b",75911:"f18854fb",75919:"e01c8f09",76041:"03240ae1",76123:"52166518",76163:"20a6876f",76166:"1778a2f7",76215:"d9605822",76235:"6a3d5afb",76257:"cc7760fb",76262:"075f4b70",76264:"cb46984a",76274:"5d06256e",76285:"0d71ae17",76289:"1ed4b1ad",76368:"9267add8",76530:"594c10ba",76641:"eb0d63b4",76683:"b6f16885",76719:"a50107bb",76748:"9a68cfff",76850:"d5a221f8",76854:"2b00a752",76871:"e364f7ff",76873:"a91e6a0a",77077:"a01a15fc",77088:"bf048e24",77096:"61ee4158",77123:"62825cc3",77145:"30ae1693",77174:"ce019e24",77183:"eaf93d9c",77200:"a95aede3",77222:"fe2389d2",77261:"1aef3c3b",77298:"109d395b",77369:"c4acdd50",77391:"d0c84d34",77596:"0235a442",77679:"5ef28561",77698:"60381fc6",77749:"3a31d53f",77764:"8c3793bd",77787:"d9e41302",77794:"62e7b5b0",77920:"a52788ff",78001:"33ab05f6",78037:"c9599a77",78055:"30175d3c",78087:"90caa6a1",78159:"8df10e0f",78199:"5455ca0e",78312:"1682c6e0",78354:"c1b680b7",78421:"b1a2ea9a",78497:"28ebe10c",78512:"b7e5c092",78522:"ff4a022d",78544:"4adc4126",78583:"95365ba3",78632:"4f1d1584",78704:"b80df7ca",78758:"667a9f57",78764:"6000d05b",78787:"a7f76e11",78820:"04c3dd09",78858:"4f0afd2f",78876:"499efff2",78880:"335badf9",78912:"c8939218",78919:"f73fc2b7",78926:"73bc484b",78985:"ccd7c88f",78988:"0ee8b775",79039:"803df44c",79048:"a94703ab",79150:"cdc43a7d",79157:"c884c38e",79180:"cc426672",79302:"74555694",79416:"a2030166",79474:"120bc0be",79485:"8b7dab17",79506:"368f9c22",79520:"34900881",79521:"09b9b020",79656:"1bfbaba8",79739:"8be24c4d",79841:"fc26a5d5",79866:"619263e9",79987:"dd6ea91b",80018:"5e4ec6cd",80046:"93c339e4",80091:"587df15f",80102:"7349f9da",80198:"d38a6f54",80205:"0cc20099",80259:"45cd5638",80260:"8aabb1ce",80299:"8657657d",80310:"3c89ed52",80362:"2fd10e89",80375:"6e954188",80408:"81c96f91",80424:"abf14b6e",80442:"e1a35040",80457:"992bda30",80458:"318980ec",80491:"7080aeb9",80531:"27321f21",80557:"5cb06d06",80598:"1f97a7ff",80632:"9abd9d66",80649:"80f24550",80677:"6d70fa33",80744:"641786b9",80750:"84ac5929",80772:"fd440a88",80781:"cfdba83a",80794:"442da8c1",80841:"413bf93f",80863:"b6aee096",80947:"a7f213e3",80957:"c141421f",81012:"820b9813",81019:"f74d3bfd",81050:"c88d5fcd",81085:"104fa664",81125:"aee88ae1",81175:"f98df6cf",81188:"ddf9f187",81212:"347b93da",81285:"54e1e905",81300:"2195a7cc",81319:"12363bc4",81346:"a20f65a6",81390:"e6c5d4a7",81410:"7af33a93",81482:"ae0ec9f4",81530:"654827c2",81544:"13685ceb",81583:"15a3ce9a",81639:"ffe2a86d",81666:"a74d38f6",81692:"0d9e9c41",81737:"c75c3ca5",81761:"932d66ca",81770:"fddd2104",81791:"d3571192",81795:"b90928c1",81802:"6066240e",81891:"b3dc9cb4",81902:"29470dda",81903:"acecf23e",81943:"4b93078e",81982:"2a59c009",81990:"f9ecf61e",82012:"4a7139ae",82019:"755e7c7e",82242:"cb7dc7d1",82261:"d2c9f0b8",82297:"226eff01",82354:"b87f99cc",82362:"7345c721",82431:"5d210bb4",82456:"24f4e7d7",82486:"4393156b",82553:"0add85e5",82581:"907fc94b",82588:"4cd82028",82663:"dd0ce7a8",82722:"72e3e702",82754:"4ccba4b8",82847:"dd09cc7b",82905:"12cecbf6",82939:"2a7d5452",82987:"732a7c43",83027:"dac8839d",83078:"42174d87",83089:"686a74aa",83103:"21cd7edb",83109:"79ca5236",83111:"6060cc06",83121:"deacbd9b",83127:"10757cc8",83182:"e5cd0e7f",83203:"95126e44",83213:"828aa184",83249:"ccc49370",83260:"5c26bf07",83273:"2b6eabf2",83316:"0ba08ea0",83378:"d0f169c8",83470:"e6ccb422",83574:"c85c7bc6",83608:"19560f91",83670:"9e37e644",83700:"a8c902bd",83779:"e6fc8a9b",83797:"b6cd0ba6",83824:"f5dd5915",83841:"cc517726",83849:"c48426e8",83928:"345eb925",83932:"757c6784",83954:"e67a83cf",83958:"2bf0d1a9",83970:"e200187b",83976:"0e384e19",84013:"c17d6939",84070:"511d3e84",84107:"ed83b9b9",84152:"d2ed2b82",84227:"17d9fbbc",84232:"ddd99a22",84255:"901160ed",84259:"2ffd8514",84319:"1132d5f6",84332:"0b82d45d",84527:"8180940b",84530:"2e72ea50",84563:"2760fb69",84572:"d9a7203a",84760:"21f4f354",84773:"b738b15f",84813:"6875c492",84869:"6bd2e919",84963:"d9d878d8",85014:"ae42df76",85095:"ec2ed6e2",85097:"7c86d680",85105:"ab380486",85205:"0cf3b0b8",85209:"8beefa16",85243:"78c968cb",85257:"ffec2b37",85279:"8f4a15da",85322:"c9ff545e",85358:"813f53ba",85442:"2f6614a5",85453:"de5aeda1",85537:"2c88985c",85581:"8f9bc413",85588:"24e645af",85736:"0e8c3a89",85796:"a4a3eadf",85806:"3e226f70",85817:"d40aab8b",85896:"ddb0149a",85899:"41639dad",85917:"716f14d9",85960:"5268c144",85973:"5733876a",85997:"83b6afd8",86053:"ae271c01",86074:"199e6876",86121:"677325f0",86129:"fb03d32d",86167:"1f8198a4",86215:"95ee2976",86216:"abe1b280",86246:"7a2c4c43",86354:"d630316d",86417:"669193a7",86434:"d5f51703",86478:"2a1b9f9a",86480:"3a311bb0",86501:"f4e1ab69",86586:"b76e61dd",86597:"0e0dfb6a",86623:"49bbb99a",86659:"7ff6577b",86669:"927f97a3",86685:"58421c87",86807:"b0445ba0",86829:"5626901c",86846:"57ec2762",86856:"c942990c",86888:"5687c7cb",86890:"319b6f13",86914:"869411c7",86939:"7a06eb83",86992:"d2a66e94",87004:"b62a4e5f",87087:"40df2769",87159:"4eec7d8d",87184:"4462c0cc",87186:"767cce31",87205:"bad31f72",87217:"9c71777e",87269:"33ac18ed",87282:"c884ad6a",87377:"c4f2c374",87407:"cd08112a",87432:"0b425eb3",87456:"33bf4676",87459:"b8f2cc13",87473:"baf4021b",87483:"d5762a9f",87485:"643da015",87547:"e29c32f4",87589:"d2b2e5aa",87614:"7c29a585",87688:"137db567",87747:"3f1a4665",87781:"6f831635",87863:"4194f325",87875:"01f93a4b",87880:"f553a7ec",87905:"fc17862c",87945:"5b17955f",87971:"dc4ee1bb",87995:"85f9f5a6",87996:"72a68b3b",88047:"a5f4f54c",88111:"ca3d464c",88163:"2e05ee79",88222:"b8fd1a8c",88248:"ed990644",88298:"85e50ec2",88310:"0169a509",88327:"fbfb0bb0",88393:"3cf947bf",88394:"5ae50f21",88399:"078339bb",88436:"a13618bc",88441:"40e4b7d4",88468:"bf79ed6f",88478:"23a811a2",88522:"d3f32089",88547:"2283659c",88622:"5871fbee",88626:"6e8a38ea",88654:"6dc022b8",88696:"6be5dbf9",88775:"b330ac5c",88815:"71cde98c",88817:"d3065e4e",88821:"b0b71a2a",88847:"014c2cb9",88852:"508d741f",88855:"5edce1ad",88865:"d15e98db",88903:"568a0660",88986:"7b658d8e",89018:"2c9ce68e",89093:"fc93a2ea",89097:"24f450eb",89172:"adb96665",89175:"9f15c6e7",89210:"9b191135",89289:"d8e6b3db",89338:"324cb577",89353:"b919dab0",89362:"cb2f16be",89402:"39ac0d5b",89415:"9ed26de9",89437:"ce1709a8",89455:"978ca64f",89489:"145c1cc4",89534:"66405406",89554:"7ff4234e",89610:"2709984f",89641:"cde9af88",89696:"837dc7e4",89801:"084a80b5",89810:"9bfa1541",89858:"36994c47",89873:"65f8c8fd",89986:"e582f766",90066:"bfb3b697",90071:"7e60f0fb",90110:"dbe2cfea",90115:"1026ad00",90135:"8abd7194",90214:"9db841c0",90257:"7b773c92",90280:"a80ed580",90317:"b3abe364",90378:"c633f93f",90414:"ed2667fa",90416:"9947fe09",90438:"fe49dcfb",90513:"6157713d",90562:"32eb34e5",90593:"99144b30",90598:"2563c97b",90625:"73554d19",90627:"bfa9eb5d",90717:"fff1effb",90749:"48dc1693",90831:"e96ddf11",90842:"837010ac",90849:"0058b4c6",90910:"e03c6723",90950:"271ea2f9",90972:"70e8710e",91025:"466375b7",91058:"42003ca2",91109:"014625f4",91187:"3629c812",91203:"ece3ee5e",91257:"f657ed9f",91305:"af1525aa",91319:"0f84f33f",91333:"d1980a1b",91359:"88f66091",91405:"ae3304ee",91428:"284c57b4",91530:"7118d0f0",91540:"980d0b86",91581:"8614d32b",91587:"1fecd571",91597:"e33cf622",91621:"7907a033",91695:"6033c607",91724:"9964550a",91787:"df99aa82",91844:"41ca93cc",91895:"b7201a27",91916:"72fdaa88",91993:"8ce0215f",92018:"0ce689e1",92031:"e2c3948d",92105:"6342cf5e",92125:"4cce6e5a",92196:"56a0d755",92223:"e772c536",92241:"bef29d9e",92273:"85fe7d70",92329:"4a4c6752",92410:"4c434b32",92412:"d888d821",92475:"1be20fdf",92535:"b6eb7220",92544:"e53ee22f",92653:"efa442c3",92664:"7c1d6f5a",92673:"9dfaf941",92955:"3e082aad",93057:"adfd3bc1",93085:"b5ea0533",93166:"fed5be48",93206:"2ed2a721",93209:"47b26a6d",93210:"e55ca189",93269:"b12147a6",93273:"cf32db66",93341:"aba187b8",93361:"2357dc71",93362:"7a234181",93438:"f5e9d1c4",93442:"63786366",93534:"a9c52207",93546:"d9e43198",93602:"44c6f0c9",93636:"e1847ef7",93795:"f179621d",93849:"09bacf3b",93854:"0a07ac32",93884:"c3c082a1",94e3:"45a5cd1f",94065:"ee71d619",94082:"8c0fb9c6",94091:"beceb38b",94106:"acddd5ca",94153:"6ee6a410",94168:"c6b4dc09",94227:"f0cc57e7",94251:"824e6f8c",94314:"00f9d3c8",94344:"8b44df1d",94361:"657bf45c",94417:"85c8b6c7",94538:"13feb7a8",94555:"8e7aaae8",94561:"a1e44e64",94568:"745d4b8c",94625:"0a02a2d0",94646:"30549b42",94648:"50774ec6",94685:"8520f5db",94711:"0c2c2d88",94713:"6dfe2e3e",94788:"c26e67a5",94796:"256701a3",94841:"a1282170",94843:"3e1f4a39",94863:"fc66001f",94864:"95ec5604",94910:"bb63d03e",94925:"e8b41ff0",94926:"98700a51",94980:"5fc8caff",94997:"8d65e26f",95020:"63c4b13e",95058:"4b69979c",95078:"10ac9a3e",95093:"818bb2cd",95136:"c0df5757",95159:"cdb727d9",95183:"f58814fb",95187:"59700627",95231:"1cb0fe52",95240:"b4030b00",95245:"ed211a79",95330:"9b4185c1",95427:"52910a8f",95443:"e1afbf8c",95470:"8a40ff6b",95542:"5917c028",95605:"9a8fdb53",95641:"188015be",95649:"4885c521",95659:"30d4e84f",95768:"89ce4ba3",95785:"8c79a60d",95818:"dbea5ca6",95909:"9e488927",95911:"accd2b0e",95912:"cbc2448c",96022:"5180e398",96086:"eb84cef2",96155:"a267572b",96225:"838df61f",96239:"380e17bf",96268:"638db053",96276:"82a7c68f",96325:"65d1ec04",96463:"ddd9290b",96470:"1b260ed9",96555:"d4083900",96652:"5b0025cd",96693:"8c42b153",96745:"4f748135",96771:"a98ffe6a",96778:"181eb1b5",96793:"c3c125b1",96881:"d527c17c",96963:"98749210",97006:"7ab01152",97033:"e9032a0d",97047:"763a8986",97107:"9331da7d",97114:"32bb5bcb",97123:"9d2e74da",97167:"20a30ea0",97189:"1b3061f3",97223:"7fa05810",97246:"365a7fd7",97253:"1a8a162f",97296:"205cdcf8",97342:"c2a473ad",97358:"76276d52",97379:"3bfbd79c",97420:"224c7e3e",97424:"ba47c136",97476:"b93a682d",97536:"3a6c6e5b",97554:"6ccd70b6",97562:"3d08e0be",97585:"8e81e342",97664:"facb0528",97670:"1dba1ecf",97732:"b58e88de",97737:"8fa9d00b",97753:"41b1becf",97763:"7d2fd416",97821:"97a057b3",97829:"e5562b89",97863:"47edbc34",97887:"26ec8ae2",97895:"ab8b4014",97901:"bf7dfc7c",97905:"ff509459",97913:"1e228808",98012:"350cc719",98020:"16243474",98021:"139d61ea",98030:"95f25ea6",98058:"fbb50b04",98059:"75dbe45b",98087:"b18455bc",98094:"48f67822",98139:"a2219ebb",98147:"fc46979d",98165:"c8e5bf38",98237:"3539b92c",98271:"f62bafa2",98308:"756094a5",98311:"9e297776",98392:"1c3a958e",98425:"b943b6ea",98473:"94dcd3de",98477:"ea6f04d4",98534:"6065ad54",98700:"99e64d04",98750:"19bfb940",98758:"af6e9729",98761:"3314c9d3",98779:"a48978f5",98804:"bf667688",98867:"3e8495a1",98908:"b4e94af8",98925:"ab37b32f",98935:"5de60d34",98941:"c9ba7c72",98947:"ef00b8a0",98982:"e8178b53",99001:"82a5d7f7",99095:"1c27379d",99146:"210305d4",99151:"8e563661",99252:"3047f3e7",99254:"c14fcab2",99299:"8944e56a",99306:"bdecca0c",99330:"b3e98411",99335:"df09200e",99437:"5ffb8f23",99451:"c0684476",99502:"2263a65b",99504:"76af5d51",99528:"16e939a3",99531:"8c2cbe8e",99534:"c49eb59b",99543:"25c655c3",99611:"9cc8ffa2",99644:"bfbfac54",99678:"25e15e7c",99701:"5a4dd75d",99705:"5fb14ca8",99709:"dc330c71",99710:"6e34d00c",99746:"f4a839f6",99747:"a32c0324",99872:"8e34a11f",99905:"2f7d15c4",99913:"38836367",99987:"4b2e4980"}[e]||e)+"."+{16:"8a09bc3c",25:"6875f362",28:"8eabdc91",39:"068bc723",48:"849a2f6b",60:"38634d1d",67:"7eac7b6d",106:"5d0f86c7",129:"ad091a76",159:"d17cfd99",217:"0bc1c4b6",278:"7a133af3",292:"0bd61be4",323:"2ca86a04",335:"10e8b68a",354:"40cf0362",387:"cd12a486",437:"dfb6628b",569:"02866324",614:"44e4a263",805:"07c12e99",822:"ef8339d8",851:"ee36c626",878:"8fad3634",910:"efe21154",917:"6161fd8b",924:"356e5ddd",940:"cac0b0df",1103:"3f89d301",1107:"7e8866c9",1119:"4bbc5be2",1136:"21f72e0a",1190:"8e56610a",1251:"49f694ba",1284:"f5a3911b",1337:"ed03be8e",1360:"ecf0e7ca",1422:"0e1d3b9a",1442:"47fac533",1506:"554d3065",1517:"1a694762",1550:"39a1cecd",1635:"35a7771d",1656:"ca0647f1",1658:"9711c713",1784:"0fef97f2",1785:"bc4d1952",1791:"5fd94693",1841:"b4a6b9c8",1846:"9ac595fa",1857:"4c849a8f",1882:"e0a7b21d",1928:"877dfaf1",1946:"61767f79",1955:"8a0bfb14",1983:"1982ff74",2016:"bd8bb947",2053:"a44f5cd9",2121:"e27f349b",2172:"33f2ad32",2214:"6a299945",2225:"7d7028c3",2239:"0f532176",2256:"779aea49",2262:"0bf08dd9",2298:"571d78d4",2385:"a6abe910",2450:"57f1681a",2472:"1b1cbadb",2540:"ccf5521a",2577:"9d61156a",2638:"0d848ef2",2640:"f6e9fddd",2652:"7c607567",2663:"4e8fefa1",2679:"dd0dc46a",2700:"2fafe158",2739:"0ca6b901",2748:"6afd10a6",2759:"38d24c2d",2768:"13d74cf3",2772:"5a7711f9",2791:"35fa4f4e",2839:"27e3f7c1",2854:"45badd67",2867:"2d1bdc8f",2884:"bc593027",2912:"4a994ce6",2932:"da39912d",2954:"f91b5b76",3075:"2f46f339",3125:"d85555ca",3131:"feeade58",3137:"a059264d",3144:"86fccc09",3179:"d88e0655",3303:"e1d40ac6",3313:"c4669891",3323:"a9b14b11",3471:"5be54bd3",3558:"4acdbe49",3693:"18efdc0f",3694:"cc0b4859",3722:"35a202b7",3740:"f2909c31",3764:"98e31955",3847:"6ae5e13d",3940:"06f163c7",3965:"c5d1f980",3968:"27784d55",4014:"fa271ba2",4028:"fdcb2d50",4040:"f4d4f02f",4060:"9b076617",4074:"33fd4cf3",4117:"9e3d6091",4133:"255f8728",4137:"75ef07ad",4205:"600be0a6",4269:"04fc581f",4355:"ef747ac7",4368:"2a18d792",4382:"5dd5bfe7",4405:"5da3e763",4467:"a9d685ff",4514:"b9b659a7",4530:"ae1c75a3",4574:"fc708067",4591:"93231c73",4602:"f0ac3d8f",4777:"5ba3f668",4790:"904f169a",4863:"2586fc4c",4890:"d278e426",4919:"c4d3cd3f",4949:"ffee880a",4980:"df50cb4e",5008:"6b1d60d5",5028:"7c782fc8",5029:"92619e1a",5065:"f9e7905c",5193:"1b398db6",5195:"774314fd",5206:"705fa214",5269:"3ef4f7b7",5270:"babf4f83",5289:"7a6d5275",5295:"55c2bba2",5304:"bb7c5b01",5322:"746c214e",5334:"5dddcc55",5338:"29eb08ba",5594:"341820cd",5622:"b7d8d7b8",5631:"e2b9b99a",5633:"a518d9fe",5647:"5d077e51",5679:"e005a1d1",5721:"d9f96b25",5806:"3440443d",5834:"8a7b0abb",5887:"68e33d36",5888:"5fa93c63",5936:"b9ce6fac",6022:"68004ad0",6054:"9b9387b6",6090:"f09b8f13",6158:"077cf067",6170:"31c32ba9",6203:"9f3b730b",6225:"4e2d8d24",6289:"a054c626",6310:"2da72e95",6350:"3246d056",6368:"505a4db3",6371:"091b2713",6372:"368025e5",6407:"c92fd14a",6467:"fecfe081",6475:"6ba96ded",6552:"b2b49c3f",6559:"c4675766",6636:"d669b70d",6677:"144addec",6692:"1df14aa2",6697:"5901c946",6709:"5ec644bd",6739:"cbfa6ce5",6761:"5d49f1ec",6778:"d15a57d6",6791:"2a8b7bcc",6823:"ff319028",6897:"55b30925",6930:"ef13ba85",6944:"cadf7087",7035:"04f7671e",7071:"9a7aef67",7100:"078c6553",7103:"6b60b9e8",7125:"b49e987b",7167:"960b2669",7317:"56224cc2",7355:"04e349af",7444:"42b1316f",7458:"24779c3b",7517:"0120af7d",7566:"dff45037",7622:"77095e00",7629:"1c27322a",7715:"5d269ad1",7734:"99a612f4",7747:"6f5217dc",7773:"a5129a8e",7802:"e5622638",7850:"a753f976",7907:"df25cd3e",7953:"ccc96052",8003:"308e1ef3",8070:"264340d9",8139:"61f67727",8174:"34dbd8a5",8177:"b6e0383c",8183:"147ec233",8192:"db4decd8",8209:"c1611e12",8289:"ffb3c473",8348:"3268c830",8354:"fb86f6fe",8363:"7fb1b5be",8367:"b027c2f9",8407:"99cf73d6",8413:"f12343b6",8417:"1b4b7f2b",8585:"223cec3e",8619:"df2fa06f",8632:"d4174f8d",8645:"e8362a79",8651:"e168587e",8699:"91e6e14c",8762:"4a09a33e",8766:"249ad394",8840:"8b4ab2c8",8888:"c84e49fc",8889:"65cd727e",8900:"fd8d3c4a",8902:"0f380f56",8965:"79e095ac",8978:"dc9f01fa",9003:"2d33db3b",9045:"4ab2e7e1",9066:"e4d3738b",9076:"916fbe5f",9127:"714c4851",9152:"c66c2521",9189:"0dc2870e",9195:"f73b50f1",9228:"46600037",9257:"c66aef02",9262:"793c41ca",9351:"61c33e87",9379:"6a5733b2",9380:"f90d89c5",9385:"dcf32ec4",9428:"6546d84d",9432:"48aca857",9446:"15fade35",9458:"7babc575",9460:"ca2fd5da",9471:"4a10d39c",9499:"ce206b69",9516:"c1f50d60",9525:"6395de95",9570:"e3b8ddf6",9573:"d316bf10",9631:"10562627",9647:"4bc8eb58",9650:"7174236d",9676:"cfd0f282",9702:"b965bad8",9722:"664860a4",9724:"52a83b67",9737:"79dbbe2c",9790:"e789ce2c",9836:"8fa50ab8",9860:"74431a06",9870:"438c6e6b",9891:"6a85c793",9907:"8492cade",9914:"d91c7b9a",9944:"f78b15cc",9986:"d714f752",10030:"59d03524",10118:"ab9fc9b2",10181:"3af67d97",10198:"86d665d9",10238:"50e474ba",10285:"bb7ecfb9",10354:"47cf3433",10472:"bdde685c",10488:"7d9ce470",10492:"e72abd0f",10535:"0d2ae118",10545:"f8206ac1",10610:"87868eba",10637:"0495e0c8",10642:"91b10c43",10653:"14a3241d",10688:"01c09458",10727:"2cdd7370",10767:"5998e785",10789:"3ced43a2",10884:"53e13b06",10887:"c5d8d489",10907:"a8d1a1dc",10920:"193c16db",10924:"95f4527b",10956:"b3581086",10978:"7f66cc97",11052:"c175ef33",11058:"b579a417",11111:"92a9adda",11156:"9f4df04c",11275:"7652321b",11289:"f07258e3",11299:"70c0805d",11326:"6ff62e03",11355:"670963d6",11508:"80fc6cd5",11565:"c60cdfb0",11631:"cc98017f",11684:"35726479",11716:"42a92ee7",11725:"127bbd01",11745:"4bbb96df",11768:"20e072bc",11782:"ed1984f6",11795:"ad2f4f53",11804:"b5477c1c",11806:"42b6e782",11842:"0baed23a",11855:"bfd20606",11869:"ced68d00",11905:"3420ebde",11906:"68908acd",11977:"c7bbc133",12047:"dce108f4",12134:"ae61bf97",12144:"1c1bbfbc",12186:"6a19cda2",12316:"4f2ec008",12318:"6eee13e3",12378:"27f2d38e",12393:"c1d28ac0",12395:"d9ff1cb1",12431:"a60ff215",12452:"97321323",12477:"e0de1e38",12485:"926a358a",12513:"80c6f08a",12535:"0c941d5a",12542:"9713e3ea",12608:"5c3e8335",12613:"ed4c9e9b",12654:"2320fb02",12679:"1987a2a6",12686:"8a06023e",12763:"91b8b830",12869:"442fc6d5",12945:"fc3acd38",12995:"ab02afd2",13021:"48555bb8",13060:"aacf116d",13063:"f4ce2b61",13152:"c62fcc56",13203:"c8815658",13210:"c492ed74",13236:"b09128dd",13298:"a8dfd245",13321:"4eb4493a",13366:"3c71c883",13374:"cfd37507",13375:"8ee7aefa",13406:"13cd1c37",13419:"b7a37f75",13465:"dd6217fa",13511:"d5b3caaa",13514:"b12fdebd",13522:"288b7d48",13525:"b906d866",13534:"17c9040e",13535:"5f863db7",13575:"0d1292de",13580:"8d40e9a7",13626:"720ad675",13698:"4cea1995",13733:"38f0a082",13762:"bca2bb57",13803:"bad250a1",13849:"0d423bae",13918:"bed720cf",13943:"453b7813",13982:"17e1f232",13996:"06631037",14003:"b85e041a",14027:"f1e7d467",14042:"de2f466d",14043:"69af2a51",14125:"527729ad",14134:"adda3dca",14136:"cc61dbbb",14152:"578ea5b7",14163:"af95fa0b",14179:"4aa5ccd8",14257:"da0d1ce3",14259:"9642e3bc",14261:"3d684545",14264:"61ebcfca",14317:"dcc8d0e6",14340:"8dd3b9be",14363:"9fdbf1ab",14388:"ae4bc2d3",14399:"ffd8a59d",14410:"cfbbb6cf",14433:"2c05622e",14435:"1ee34b91",14496:"17b3929b",14536:"10fb693d",14599:"a88c7ee0",14627:"cdb12c57",14726:"e51ab934",14750:"e9c21c9f",14802:"de3cd2b2",14822:"a06e5d99",14835:"0c69e520",14868:"b144ac54",14871:"c80525bf",14917:"28bfdf38",14919:"011e3248",14943:"fd40645a",15038:"8fb987eb",15078:"3707a6b6",15079:"97fca775",15144:"71664a42",15165:"3c006089",15186:"9c7f1b63",15220:"109d2e90",15235:"71ebac3a",15272:"13610938",15292:"328425c6",15316:"9cf140bc",15403:"8aefcfa4",15407:"ea6e5a65",15427:"6dbd0648",15473:"14fe1707",15501:"c8870324",15552:"7f6375ba",15607:"487239af",15764:"bd93ba8d",15805:"a819ffcc",15857:"4a7c5609",15917:"28cf866b",15947:"648ebe12",15955:"514b69ce",16017:"9d0f4c08",16032:"47c54457",16100:"0611b6f9",16209:"fbdc06f9",16231:"4dce858a",16319:"4732e626",16377:"367c7435",16499:"3b79297d",16500:"b4e686b5",16622:"b9a182ac",16641:"63243294",16747:"0771fd0b",16885:"9c0ac3a2",16942:"92d06824",16950:"48871f0d",17009:"10072815",17033:"eb12c1ef",17115:"0a45bdc0",17119:"b280a982",17232:"e57d15e6",17294:"0542258d",17332:"8b8baae3",17368:"115e5928",17402:"bdf033c0",17411:"03677a17",17441:"96e8dc5a",17474:"d3cc75ea",17502:"279389ac",17508:"1fb03d3f",17530:"88f83b35",17581:"2b842d8c",17626:"36956c06",17681:"5dd2ea64",17709:"7802d0a5",17739:"82146eff",17805:"8a44ead4",17818:"d75d0b8d",17860:"cc12893d",17867:"cac87718",17944:"53bb5673",17987:"79ab4ac0",17988:"6cc28a3d",18006:"c111595f",18033:"10fe31fe",18103:"a388784d",18121:"617b2ffd",18146:"95d7ba5d",18350:"f98fdabe",18351:"002d2df4",18401:"e94b36b4",18438:"7e8561ff",18445:"4e7d45c5",18520:"9fcfbaad",18620:"14588da2",18645:"66f4e3ae",18738:"88fed3ac",18757:"bc03ff70",18798:"059f696d",18940:"1ee0f351",18950:"69aec1f5",18998:"db0ba0ec",19042:"f2d5ba0f",19052:"99eea1e9",19070:"f854e03e",19108:"ecbc268a",19109:"9741eeee",19136:"497d63f1",19237:"f651ae7f",19252:"120b0fba",19267:"0b9c27a3",19271:"81fe6522",19369:"43355738",19387:"99f2a106",19396:"cdf3109c",19425:"1d1db863",19428:"05a9e187",19474:"09d53c64",19513:"36062fee",19521:"c3aa46ab",19542:"95ba9a85",19664:"46dfd0b9",19687:"300fafa6",19711:"75e966bf",19736:"0a635609",19774:"ce35f8cf",19798:"b9d535eb",19801:"51ca69aa",19865:"cfdb0b35",19891:"3fbe9ab9",19999:"12a3ab26",20038:"db96bb57",20103:"ad5960b8",20160:"270b34ef",20194:"f1baf75d",20229:"40f68aa0",20286:"b249e590",20325:"960a510b",20358:"43110a16",20361:"ad2d7e59",20405:"c035349a",20415:"83d5d036",20424:"e5504750",20434:"bd31090e",20491:"907553b6",20535:"9f958eb8",20541:"dcfadc81",20619:"23928919",20676:"3388d156",20713:"3d6d71ad",20812:"0f0199c5",20857:"bb3132aa",20892:"31d8f230",20924:"e4fef5c4",20931:"b8036f1b",20951:"a20fcb32",21070:"a8900866",21077:"a68cdbf0",21121:"4c137c15",21177:"093ad022",21191:"5de11232",21273:"908689e4",21275:"eb589f3d",21290:"0c7c0365",21303:"bdf31e12",21316:"fe15db87",21341:"475db505",21346:"f490b9f9",21373:"9fcc5ee7",21401:"06aacf16",21417:"38f28434",21448:"9908b8dd",21482:"9ecbf669",21524:"d7e79944",21560:"a0edd7dc",21574:"4deddbeb",21585:"7b244de2",21630:"15f37768",21635:"95acc70e",21662:"95e27da1",21729:"ed2a8c6b",21764:"a96fe837",21805:"b555879a",21816:"ced1acf1",21837:"c401bc3e",21874:"d29acb33",21909:"4300cff4",21921:"193750e7",22141:"3ae5c6dc",22313:"cffab16f",22369:"8efee04e",22385:"b551598c",22428:"4573e534",22447:"d7010a6e",22472:"ecd49f93",22483:"9d782227",22499:"2575e3c3",22501:"4f13697b",22504:"c055c741",22534:"85034a52",22595:"dc8a546e",22628:"c7870ff3",22640:"1a855b7e",22647:"1318d50e",22673:"9b329e63",22709:"d92e340b",22750:"bf71f62d",22816:"3d4baa1b",22844:"c586deb2",22911:"202546a1",22916:"8d456179",22932:"dcb09864",22973:"1b43a4e6",23002:"4fad494a",23041:"f35f208e",23081:"15bf2d5c",23083:"bc82584b",23119:"b3b53229",23141:"70ca6181",23172:"c3a54128",23332:"d979405b",23388:"1678b3a5",23438:"adce0003",23475:"475adf50",23547:"5809152d",23563:"2ff8adf5",23591:"0056b029",23600:"a5202301",23625:"4b0929bb",23634:"5b74042a",23637:"039d8f53",23641:"3627f9fd",23666:"b60b4f41",23676:"c467ba2b",23700:"60a4a0cc",23765:"cd26f4f3",23773:"520cf3dc",23880:"f23d8063",23938:"d312764a",23998:"79a7d02e",24022:"870880c2",24065:"66cb1805",24084:"0194a56d",24098:"a0602b5b",24126:"37fede99",24140:"eb4a5735",24172:"1e513f10",24234:"8cab740f",24265:"9a476a2b",24340:"10e55999",24366:"8a444fd8",24373:"ab56a496",24377:"1cd6ec42",24381:"8cc6de74",24437:"d4b0c4cb",24475:"24d8adfa",24476:"2f952c5e",24704:"cf0a496d",24728:"0b98b763",24729:"055e6d18",24741:"90d7367f",24776:"4c19f189",24815:"6a3bcf36",24826:"f46941c8",24853:"91dc934e",24866:"d50b77ce",24867:"a8a3b8e9",24868:"64d89cc1",24891:"acea9caf",24912:"c8b91976",24919:"4ba924a0",24932:"ba390371",24970:"00f2ab70",25013:"ad26ed51",25044:"c495349e",25067:"a053ee4d",25101:"e136e901",25130:"4fb7b9f4",25186:"b0b2572a",25273:"cf02c70c",25285:"45a1ec19",25335:"b58a27d3",25396:"89ddaa62",25397:"f44c23ce",25426:"5994dd2a",25452:"e356ced0",25531:"70bc702b",25532:"973de6d2",25539:"4073dbe6",25578:"42db5727",25582:"2844b50a",25601:"aacfb2eb",25778:"6e2353ca",25781:"72653a4c",25845:"3668f0da",25869:"7ff9c8b3",25871:"6d4f8d18",25874:"a999abad",25959:"03b0d503",25987:"b6951fae",25993:"5a60b486",26003:"0dbe838e",26009:"6f80e4a4",26060:"49615372",26064:"2c34cf8a",26081:"4fd365ee",26183:"f197b658",26216:"0d38f496",26260:"d8f05c88",26305:"deccd138",26333:"ff2d3221",26337:"a16833a6",26414:"34f7cb93",26548:"9e235a1b",26577:"23f41c03",26627:"369f52dc",26630:"e90418e0",26632:"0dd9e951",26722:"14ac1711",26724:"7b16081c",26744:"58600e3c",26797:"0e6e3c17",26833:"ce805620",26859:"aff1f02a",26882:"d62acd2d",26900:"81479d5b",27012:"c29dba83",27020:"a2996298",27088:"47e88723",27115:"294cd6b8",27123:"6eed6c70",27179:"55360a14",27212:"4e430747",27369:"56253630",27388:"b0340f4a",27398:"50cba2a0",27444:"ea243234",27473:"5eca5e0b",27663:"e88ba1c1",27737:"6ce228fd",27823:"8a88b285",27887:"a3b9de04",27921:"45a287dc",27955:"be1e4c9d",27995:"09203cc8",28020:"ae6c303b",28086:"964d7344",28127:"52851c3a",28147:"1f3d479d",28166:"24865c80",28169:"5ab36c27",28178:"f62bdc9d",28204:"d075bb24",28235:"5ae34328",28252:"c129dd81",28319:"5d3fafd5",28410:"0b21d6e0",28428:"b0be705a",28435:"fceffd0e",28498:"1948bf76",28499:"f005668d",28627:"ae271a52",28679:"1d8f6a45",28687:"aacd5569",28701:"a28f8b91",28782:"40f66cca",28793:"e76cb139",28803:"49549dd3",28834:"3ee8e8b4",28853:"aa92d9dc",28889:"c2c9914f",28920:"9629e812",28941:"8c7af014",28945:"65b1a092",28993:"f632a39c",29060:"32b1dc91",29123:"dbd291d1",29171:"71258614",29204:"a6a43695",29243:"1356f2ff",29337:"01639ff2",29408:"bd89f274",29414:"8660a85b",29421:"fbe24116",29452:"3e9a03a7",29457:"7d14d05f",29498:"7aab7361",29523:"7c6695ac",29526:"2f8989c1",29653:"8f3e3c80",29679:"04a3f313",29690:"942394eb",29698:"0a4d3c74",29708:"8f01ba9e",29733:"13ded0ec",29795:"3b9ed063",29820:"86a6b03d",29829:"d1f3ee13",29845:"a1cbc44a",29891:"850e5356",29922:"a7552bd2",29950:"c6942778",29969:"67f28cd0",30035:"175fda04",30066:"bb46b5af",30154:"ff90619a",30167:"47c6ce34",30262:"e0931477",30265:"a50c2e7c",30317:"35aae90c",30355:"ef8a4417",30378:"6d8d95c9",30396:"87304d95",30410:"4e970141",30564:"97976d72",30569:"e40e0024",30579:"d0ae98f0",30583:"0349d956",30628:"c26c6054",30654:"73dd606b",30790:"a0648778",30827:"49ffcf71",30831:"64198a51",30842:"b50c451d",30848:"6480bf83",30934:"79541782",31004:"1d4923d1",31016:"035c650a",31039:"8dc64df6",31047:"cc5471ac",31125:"0d364546",31141:"652afae1",31144:"28477358",31220:"28b772dc",31255:"f94efe8c",31257:"771d4335",31272:"c7607cd9",31393:"a1e486ff",31465:"fff27cf7",31541:"4b8b9953",31544:"f4a84c54",31567:"a41f71c0",31680:"72a131db",31705:"4c5db215",31720:"77528df6",31726:"d1ef5da1",31749:"45dfc10c",31791:"392e408a",31797:"0a23a6de",31861:"93b5dcbf",31862:"9fb160ee",31869:"691241cf",31957:"09fc98dc",31972:"a2543041",31996:"7a9af77d",32005:"3ab64a28",32012:"287d33ab",32039:"4917de23",32183:"e4c13218",32225:"07d3987f",32270:"640b03e4",32278:"b3715c88",32335:"2a952a69",32365:"fe0329c9",32391:"a83eb43b",32472:"a7687f19",32527:"9df1b2f1",32558:"e82fa20b",32691:"c97ef1a0",32728:"a5d1f40a",32824:"d88681ec",32825:"20536cef",32847:"b0005fda",32875:"0800d3ee",32949:"7a6c4ac1",32960:"b5798ca2",32993:"6c882733",33049:"8364c70d",33065:"7304db46",33192:"9a5863ec",33214:"bc8b27d8",33234:"a84fa661",33290:"219552f4",33303:"43ad6989",33350:"75f07d8c",33458:"a7557404",33517:"3db9b64d",33524:"2e7a87c2",33533:"73ed698a",33540:"148c3af3",33559:"8fc1b433",33567:"350cc4b3",33572:"09b03628",33577:"d40cdbfd",33612:"c687cf88",33621:"bda44dbf",33638:"5fca92de",33643:"569e4e05",33659:"21d9ac33",33724:"88ca3752",33804:"d3822e25",33806:"7a355084",33816:"d55e6964",33825:"5a0253e7",33854:"94a59e40",33881:"5b41d6d8",33889:"b061c751",33999:"668ec8b1",34127:"fde6c8af",34156:"bbf91755",34179:"1bd8bf5b",34182:"21781a90",34196:"5c27b80b",34255:"3851abf4",34331:"9ed21c85",34335:"449a955c",34388:"63680cd5",34520:"7924cd1f",34528:"a8871870",34543:"bd5caa0e",34549:"c9123c09",34597:"2d91134c",34621:"ae85fbee",34638:"16fa9b5f",34662:"091b3d87",34691:"dc70936f",34706:"89e46f0f",34750:"224f43cb",34803:"b8ca48f9",34808:"909e26ac",34810:"99716174",34837:"61c61de3",34847:"c1ad7786",34877:"5060c2b1",34904:"25d15372",34918:"cbd6f71e",34970:"cfbb2faa",35044:"cf8fed2a",35067:"d17b3136",35088:"cea1a9b9",35139:"1eb36ee6",35144:"bffb4b7d",35179:"10e7c1bf",35248:"74d1ce2f",35288:"991ef11e",35325:"e01489cc",35329:"9e684943",35379:"8ab1496f",35402:"375ee7c7",35532:"1beb2628",35618:"e4f3db36",35730:"f9137192",35733:"28dd7656",35742:"de61f764",35766:"9a1fa293",35767:"dfa52db3",35784:"fc8fde7f",35837:"1e5ce2e7",35881:"c8a2bef3",35922:"4cfe6391",35932:"506269e4",35960:"059789bb",35983:"a3ffd343",36009:"cbae7f1b",36055:"4770b5a2",36067:"fbcdaf70",36113:"2d48d701",36144:"725b0d44",36192:"26c2e85f",36264:"3ee1d673",36302:"72b0bcd9",36379:"b702241b",36387:"251632c5",36549:"e2747c6f",36569:"64ebfa77",36594:"bd886889",36619:"617e0e66",36640:"52eef1a4",36672:"1702d861",36689:"8a2dde69",36747:"e29bc85b",36790:"9de7a9c1",36851:"203b548f",36885:"1ac872db",37109:"0784f49d",37115:"b776b259",37147:"088d2c22",37191:"2d0ea897",37204:"1605f8c9",37206:"af6fb7c2",37209:"9b43eef4",37257:"dceda108",37329:"9a57546b",37366:"160a0072",37403:"c5e2093a",37486:"e7d57a3f",37489:"9a67fcbe",37577:"a1199bf2",37578:"0177c4c6",37617:"8e607010",37643:"4865d39b",37673:"7883a91c",37740:"fab7fc26",37799:"403baa89",37809:"9b6986e9",37827:"5fde1cd7",37839:"05558af7",37841:"670e19e8",37842:"1d81fddc",37872:"e9177a7e",37954:"076d3157",38001:"cb066494",38003:"70d204fd",38056:"20662e51",38107:"43c0bc0c",38158:"51aebc6d",38238:"5390d965",38348:"adc210d2",38370:"fdc0ca5e",38385:"97d9f92d",38416:"20851e98",38439:"461b0f80",38449:"39bb4f65",38451:"6bf97463",38468:"716e8fd3",38600:"5d6ce15c",38605:"f3f72edd",38607:"fbd8a71b",38700:"f325af51",38767:"cd1ba66e",38786:"31e8cea7",38816:"7447358c",38835:"40a51f33",38839:"dae4efea",38888:"6753bae4",38974:"4d5d8235",38999:"35806b80",39058:"effdc4b7",39088:"f567553b",39110:"50177589",39138:"10007fdc",39142:"6a331906",39153:"e21126fe",39192:"ef51e43e",39197:"ea14ffa4",39202:"40e4ffae",39208:"a97db242",39230:"5527100c",39240:"5079f647",39260:"ba5c24ff",39265:"5d0ed436",39283:"79d7db8a",39405:"6242229c",39407:"a7203d6b",39417:"d02a7f26",39423:"17d4a9fd",39444:"a22313d8",39487:"5b0731f4",39494:"f30b5ac5",39496:"ebba264e",39505:"5bda256e",39518:"c3c9d6ae",39520:"27de82f7",39555:"cccbb5c8",39569:"3bef5f39",39587:"af0d65e6",39608:"1a5e64fb",39712:"0a24b98c",39715:"bbc66cf5",39758:"a3fc9df7",39773:"b393be97",39777:"3432c599",39781:"47d6c4a6",39840:"53dbfad3",39887:"e46ee796",39901:"eb9119ba",39908:"6d740ec1",39921:"0e79eaf9",39973:"e8d22580",40065:"b7fdc9d7",40124:"40c231c1",40178:"b61f965a",40205:"c94f090b",40245:"b96db1fd",40369:"92864a0d",40463:"427c10a1",40464:"acd8440a",40555:"f52300e8",40606:"12108f89",40616:"a9f41f0f",40651:"2d49ff29",40690:"a60e3624",40697:"b4dd2319",40755:"3e432e6b",40758:"4d3e9419",40797:"3db6824f",40827:"1a80c281",40847:"cd35205d",40907:"d1b3edb9",40919:"6f0cfe05",40927:"73302b80",40959:"dfce7692",40999:"a9ca37f2",41030:"1804232c",41125:"2c45e5dd",41135:"e8aa7f97",41159:"3ac9a111",41260:"8ca7fc77",41361:"784c9a03",41455:"eddcbb4b",41461:"3443ba16",41491:"56b3d829",41504:"6389ddd7",41518:"d9e6c91e",41594:"af01423f",41640:"9e866231",41651:"616b00c8",41677:"8c940416",41739:"e8f5e70c",41750:"ee28ae45",41852:"3dd9673c",41867:"0e7edf3f",41872:"b0386e86",41892:"9f8c189d",41907:"f8e6d852",41944:"9bd9e858",41954:"e362215b",42068:"5e07d0e9",42187:"d9057212",42196:"71100c70",42268:"6c4ef9de",42298:"a50f4480",42299:"09b7f6e8",42350:"1142cfde",42368:"f8b7383f",42465:"3ff7fed9",42487:"56c461db",42518:"198e0c40",42566:"b345a309",42576:"9a6fd0e8",42596:"7100815e",42690:"f8d6e169",42702:"5f8c0f9d",42723:"4fe95284",42794:"7bc80366",42799:"439a565c",42801:"98f49e04",42879:"7c6ac9f2",42881:"45103050",42900:"a68c15fc",42905:"6bb55442",42911:"5bb7f601",42920:"c438d5bb",42937:"58f6b800",42942:"5cf3f15a",42995:"695ebee8",43029:"0d532445",43035:"dfc57d02",43040:"1e4a40d7",43042:"f5fd14f0",43078:"a02f005d",43088:"49e4451b",43113:"0310d8f0",43195:"92666359",43286:"f606a9ec",43352:"24cd4507",43421:"0a5a165b",43432:"fd8b8122",43445:"93a5e7b9",43461:"b6d04d9b",43512:"df29c6ae",43642:"d6ff100f",43677:"d746dcb5",43739:"fc70054f",43868:"a3261008",43922:"32416b57",43980:"4a4883a1",44008:"643184c5",44022:"6fcc094e",44040:"2dd42aaf",44045:"c0d86ede",44118:"8cf0d0b2",44123:"11ceffaf",44125:"e33c2ae7",44129:"86e86df2",44143:"044af036",44290:"0d453a69",44326:"2ee9dd8c",44339:"b32856ef",44380:"cf224181",44405:"51b730f8",44419:"a05b3029",44489:"66da1d5e",44492:"ec2797ef",44551:"b4882b70",44567:"3c62c94b",44640:"d04281b8",44662:"d947bd90",44682:"087d97b5",44719:"9f19c736",44840:"a727ef1c",44876:"e3d3c1eb",44891:"06e055c0",44933:"1455309f",44939:"e64332af",44948:"07d7525c",44960:"843bad26",44964:"b983a3df",44994:"41cd7968",44997:"3ffd623b",45113:"523985ee",45117:"8c2c2ced",45146:"e85c17b6",45210:"52123a36",45231:"a109a3fa",45238:"f9e62925",45290:"d75a994a",45323:"faea1743",45339:"3bf0bfb0",45460:"550b46d2",45500:"065ed853",45541:"6fd99ec3",45561:"f24f5701",45640:"6c7f6a66",45670:"50c1ba0a",45682:"888518b8",45919:"55919297",45983:"13ee651a",45988:"17b1ea4d",46001:"f432c2ec",46008:"f69f5c00",46025:"7650608b",46039:"f72f1171",46063:"3541e9ea",46074:"6f20e0a9",46156:"7d11bc0b",46197:"df2b2640",46244:"cad26181",46296:"f08eab96",46375:"df5f4d23",46425:"cdb28597",46451:"a6a11df4",46472:"fadf22a1",46513:"56c87a15",46515:"254b9405",46547:"8eec7030",46578:"59c56ca8",46598:"c169c2b6",46652:"63a78692",46662:"e6c9e2f6",46754:"5fcdc1a8",46802:"d659f98a",46840:"2eaf8b00",46850:"b6f5cc68",46888:"4961c159",46891:"3a3be22d",46905:"e3c70d3d",46911:"2300f302",46917:"762b651e",46921:"be8d197f",46937:"5c1b7fd5",46979:"008db483",47041:"255878c7",47060:"9049c3c7",47087:"13a87ecf",47106:"bccff21d",47110:"56dd8cf9",47127:"5f600751",47206:"4cdf2089",47236:"3d4f63b4",47247:"4d3adf16",47248:"816064a8",47297:"eeb342f5",47433:"ac211919",47517:"4328267e",47537:"f3a77ab2",47555:"b251b72a",47622:"09db78d4",47668:"680ae267",47693:"c10aa10f",47699:"492e4477",47730:"4bc36570",47917:"2dde4c6a",48003:"a29cbb41",48023:"0f86cac1",48050:"1fdb78aa",48063:"3a568226",48071:"d796c291",48091:"df7b8959",48130:"e6c193ae",48132:"953d34cb",48158:"d48e413c",48162:"b4c8061b",48167:"4dcfa6c4",48188:"04a8ab5c",48214:"46956ca7",48285:"d6809f4f",48295:"0b50cf9f",48304:"5f48cd0c",48310:"64b1d5ea",48355:"8e22de30",48360:"cfbd7304",48361:"c850e674",48532:"a494db1c",48544:"86f74040",48592:"349dffae",48613:"3f89da26",48661:"3216d8bf",48673:"9d684a78",48733:"c32bd61c",48768:"079b97bc",48793:"5daa512b",48803:"88f5f6df",48842:"f3bf8406",48883:"a235bfe0",49013:"07816850",49046:"9e2e99da",49055:"eb422c4d",49071:"ba636502",49112:"d7a24cb5",49174:"03c624f6",49199:"efe75a67",49226:"b6e04903",49280:"4dfb5a77",49305:"acecb563",49350:"b3dbf751",49388:"9bd7db7c",49461:"5ae94b5a",49521:"60482983",49569:"b4567b28",49579:"e16a6c78",49583:"393a5e78",49587:"4cf9d174",49655:"68089f3d",49676:"2bc5bc8b",49783:"5250a56f",49786:"7b91213f",49880:"54c0806d",49991:"403e6dc1",50012:"68d419a1",50095:"6ed54966",50176:"417fb999",50178:"18db7c21",50183:"1be5d08f",50259:"f7eb5e14",50261:"7008ff7c",50293:"42163459",50361:"bb9593e9",50414:"9d3834fc",50428:"f90b3bfe",50446:"5d436275",50472:"f3bb4c29",50486:"0f8a2868",50507:"8f74cee6",50570:"c66081cd",50584:"8ff63e94",50642:"da2e25f5",50700:"11590bcc",50703:"bd356fa0",50747:"b4c4b315",50769:"bbaddabb",50800:"1ada2615",50819:"551c9c7c",50827:"394e36ad",50839:"7fc947a3",50934:"7c4a4744",51047:"e05467fa",51049:"e5ceb9fb",51060:"7c84a54d",51090:"d3055e1c",51102:"25d73939",51116:"5d147865",51170:"b528a493",51175:"3e23e8e6",51184:"3fbba03e",51229:"24c83fdf",51237:"83d2b73c",51378:"27ce1cb5",51412:"e5d9f101",51420:"51bfdb8b",51438:"153db1c7",51494:"da27cda2",51548:"c84495e4",51578:"260cb010",51584:"24cf19af",51589:"990a5fd7",51636:"b532bd62",51649:"c406b631",51665:"4e1dca5f",51694:"d22c56f1",51711:"6ed1019a",51755:"e7683163",51763:"614f3b14",51769:"4709b0b0",51814:"e54b3ca6",51828:"cce51e7a",51837:"7e552679",51878:"72c458a7",51890:"33cbdd96",51937:"7b8f3921",52036:"4ce3fd22",52042:"f430fae8",52093:"0bc9c087",52102:"8459ac52",52132:"18679067",52135:"24738c10",52176:"9d3aca59",52207:"3fd074b7",52218:"715862a4",52235:"b3475660",52250:"d133f712",52266:"7f7e57bb",52267:"999b9003",52282:"63565f78",52286:"4a595239",52342:"e54f9f4c",52408:"d3db7d01",52450:"99135bd3",52462:"3529eb94",52466:"0060b180",52478:"f31e943b",52488:"2e672211",52563:"63e5002f",52569:"0978d751",52572:"66b2d328",52612:"fe83ac5d",52634:"f1140671",52680:"c4c41065",52711:"07c974a8",52731:"c514a24b",52770:"479aefff",52785:"db98dd4c",52826:"83f0710d",52840:"3fb65c88",52882:"f217557d",52895:"6f6f0499",52911:"2bf6f0df",52912:"9734e003",52913:"3bc2819d",53074:"21f6a3c7",53132:"26e4c97f",53170:"46c4c760",53178:"12177593",53201:"0d2201ce",53237:"9d662527",53256:"de10895e",53266:"ec0e5b63",53288:"58febaaa",53314:"aeb9ccb6",53449:"8c293a1d",53458:"d06e4d8a",53492:"c61db897",53540:"b615600c",53619:"07903309",53629:"0e9da356",53633:"91cfa4ca",53646:"252e93c1",53670:"5d9992cf",53742:"e3fb6085",53771:"503e9c26",53786:"295cce71",53832:"ea177f3b",53860:"045fe6c1",53861:"89eb5be8",53871:"06715a06",53881:"baaec368",53901:"176898ea",53925:"c90ab7e0",53982:"ac914839",53990:"ada2a249",54097:"7fe429a5",54106:"0ae8620d",54200:"d2bacd5b",54240:"9d3e04a4",54246:"60f12cee",54270:"091906f6",54272:"15bec62e",54274:"de7a41fb",54277:"203dd83d",54399:"52bbe215",54415:"f56879e1",54462:"7f730c3e",54487:"c5676dbb",54500:"8b8b005b",54540:"a02f600f",54568:"b4a9dfa4",54730:"18e0d97a",54749:"506a8f87",54759:"03807f37",54772:"1026b8f9",54851:"d6dbe9b4",54879:"50ccf7ac",54881:"d0c3c6aa",54927:"40575db9",54950:"c2be47fc",54951:"d7f390d4",54992:"712f1c60",55004:"a9d23daf",55100:"538b7d12",55144:"9e0ba362",55167:"b4d49358",55206:"0d70c765",55276:"e3d30ccb",55302:"1f220875",55342:"baf46857",55401:"b92ec369",55483:"9bee99fb",55526:"93ae64fa",55534:"25d024d7",55559:"b83a883d",55566:"82130cb7",55579:"4e0724fd",55616:"6658ee4a",55664:"42d281a8",55668:"ee74c105",55754:"89d9928b",55780:"a19a9af8",55836:"96bafd1b",55929:"77f3f6c1",55942:"d374a5b6",55955:"da177b3e",56028:"af871c03",56037:"ab8234ba",56062:"60867da1",56087:"13c91d14",56097:"f053aab6",56122:"74d598a4",56123:"c789ce2d",56133:"762b0fec",56190:"ef64dc1f",56256:"e36aeffd",56259:"68ac1963",56297:"26633708",56384:"fb72bb40",56408:"5bae03da",56493:"7ee5fd86",56510:"0a75fbb3",56560:"3239f135",56607:"48d20332",56648:"0e90e357",56662:"5e826942",56688:"2cca9c43",56723:"1293a3d4",56725:"7b698c07",56767:"e8136b8f",56779:"d6ff4e16",56826:"50015c12",56916:"550c02b1",56948:"46365228",56974:"4839cefe",57054:"6fb01f30",57119:"987dbf1a",57141:"7d7eec25",57155:"e94ea6ca",57159:"052fafd0",57165:"d5789e63",57178:"8bfa9b62",57182:"69123d57",57222:"c8de5978",57245:"8d409ef9",57251:"58acefa5",57283:"21045dc1",57302:"aa7169af",57326:"8b59f195",57359:"8ee1fa90",57361:"4a3a3cb3",57426:"57e1c591",57529:"7bab9ba3",57547:"3a1cad18",57581:"fb0331ac",57594:"1e47fc98",57614:"8825c03e",57647:"56c9542f",57730:"2bca685b",57749:"4bc6c338",57760:"1092dcab",57762:"c39c754a",57885:"5d86fd4d",58e3:"734d343b",58002:"748d2de6",58079:"573a9aa6",58111:"73a02d4a",58144:"36de5476",58255:"5ee8f8b1",58259:"cd10818c",58301:"4a5e22ca",58351:"cdf49691",58359:"dcd461e7",58393:"399932cd",58428:"3ffcb7ac",58479:"44dcc97b",58484:"f70a46c3",58492:"d2645f8d",58581:"83d42deb",58594:"cf6bd0a0",58666:"f2504d8d",58667:"97e291ec",58704:"25df67a5",58707:"15c67f3b",58721:"f353fceb",58780:"d0db0c75",58857:"865eb012",58874:"1a0a7c4a",58880:"1513efa4",58913:"e13e4900",59004:"328f4441",59005:"cfef6f48",59009:"dc31e278",59020:"8f1ae138",59053:"6eeb65e2",59070:"00082207",59104:"12590816",59114:"83de7b83",59123:"564f8b9a",59182:"15f5ff3a",59262:"aaf827b1",59326:"73c25d16",59349:"c28d69e6",59475:"006f4c07",59493:"631868f6",59521:"c957bd71",59614:"b7e2e03b",59680:"3c671a5e",59702:"69bdff54",59732:"2a459114",59740:"04c67354",59779:"2a5eec41",59781:"f87d58a7",59866:"aa994063",59873:"1880b633",59874:"a8378fd7",59923:"b38aed5f",59940:"f75ceebc",59981:"76c5bdaf",60020:"0f554f37",60119:"5b9bc0b4",60167:"b7af2062",60185:"eb38933e",60187:"7636ec06",60193:"2bef0842",60198:"a93ff4e2",60203:"8d7339c1",60257:"5032408d",60267:"d687473f",60294:"5b4136af",60315:"55f56501",60445:"36846bdf",60510:"ffb2f2a0",60565:"d1460e67",60613:"cb3cc081",60625:"ad1309c8",60676:"81736ab9",60696:"a608baca",60725:"dbd85841",60762:"a573b13f",60765:"2dc71301",60819:"5e525d6e",60830:"00f6a771",60837:"fc61e2ff",60861:"b786146f",60884:"14ad0b43",60901:"0a407239",60908:"4b488b61",60953:"0a991989",60965:"60e4101b",61027:"449d7005",61114:"6ed53af4",61228:"71606809",61235:"b3fca070",61265:"f983c95e",61300:"834cd9b6",61345:"a53b2883",61378:"f5f5ceb0",61416:"179f2544",61481:"27590116",61559:"e81e8852",61584:"8bd4f52e",61627:"269c1cb3",61652:"48519009",61702:"3ab98eb7",61721:"1bede07a",61734:"76fade34",61740:"ae48d0eb",61786:"11544c15",61844:"b500c1f5",61886:"838ac25b",61939:"63ea2061",61950:"d36ac652",61959:"81d6b244",61991:"5a92f8a3",62015:"8ec5ca20",62037:"306ee7ed",62061:"1efccb35",62111:"cd0efc31",62120:"99d514b9",62138:"567ed5e1",62147:"8390c8e4",62149:"f514162d",62162:"5acdb074",62182:"503834a5",62188:"3e551357",62254:"245e993a",62264:"e4528e15",62400:"e7e7907a",62431:"b246e6b9",62456:"43e0651c",62469:"3c915cc0",62501:"de4df872",62549:"301ee9e3",62568:"68871186",62635:"22cbfad7",62639:"210bc494",62641:"75bfbfe9",62672:"6358ea51",62691:"753414c4",62703:"564b723c",62712:"d5ff0699",62714:"fe7a976f",62827:"296a4530",62845:"90aeb106",62867:"672ac76f",62875:"c31ca64d",62927:"d4d0ae7b",62953:"0792ec6e",62957:"6f1cdfbd",62975:"995912fe",63005:"6d110437",63064:"c0bf5dca",63075:"3ee1025d",63124:"34aa01f3",63151:"b9cb4815",63181:"ea8d8fd9",63228:"10bd0d98",63262:"bf4c2c34",63291:"9ffac09b",63298:"acbff002",63317:"811a54d4",63330:"bc478490",63332:"500cc836",63367:"7a6eb5b0",63382:"e45271df",63387:"489643ee",63420:"489eff90",63430:"26ae4f11",63455:"62b0c186",63515:"a47e4d5b",63521:"66036a01",63553:"ea92fe5c",63576:"0cc97e30",63627:"6f0e73b2",63646:"24a98389",63682:"e81f5366",63700:"7e8b055f",63739:"a9c24938",63760:"d6e4a1fe",63779:"8304ce91",63795:"90a8fc97",63836:"c50ffd2f",63907:"7efe218f",63951:"b86c7710",63966:"d1164b7f",63975:"7ff61510",64045:"8cd8acd5",64099:"a0ff270b",64184:"80d3a705",64209:"a2d5ce2d",64264:"304d81a8",64275:"336cbfa6",64351:"38e048a0",64391:"a3d0ce9f",64442:"e6149577",64447:"ab60d1bd",64465:"3c9d3adc",64473:"cf1bf285",64502:"3c32636b",64527:"8724400a",64535:"2a06b13a",64577:"d5c345aa",64597:"16a203d3",64623:"c358ac06",64659:"61c9f121",64670:"9186d3ad",64737:"9eaa058a",64749:"867c6f08",64838:"b35964d0",64899:"bf652241",65013:"9e1d762b",65017:"566badfd",65091:"33aa1367",65094:"9c1741e7",65133:"6b78163f",65193:"c9d51b91",65221:"f8300e9e",65223:"fc7b2fde",65233:"4caa50f4",65246:"b0e81973",65350:"360c493c",65356:"044c021b",65359:"367c24bc",65473:"5e0a91a2",65497:"8399cb89",65569:"b0072d01",65611:"5f89a380",65625:"08954e37",65629:"16ecda58",65647:"24aaa41f",65648:"e4fb5853",65658:"3b939be5",65727:"ded58870",65745:"316ea106",65756:"15cd943a",65759:"819fc53e",65835:"a3453b0a",65897:"a381fb32",65918:"146740b5",65926:"e5ecb06c",65957:"68a2cc05",66061:"45757a8b",66062:"eff2a5a4",66097:"25d8d490",66135:"e4b972b4",66140:"8dff9019",66162:"2b8e16e9",66169:"02c3df8a",66210:"d091b4c8",66237:"c01df0e8",66327:"5e088192",66338:"66797f3d",66372:"b09b0736",66381:"ad989954",66394:"8bded28d",66415:"cb19b5da",66461:"d833c819",66499:"38de97de",66504:"ca582e0b",66510:"24f0668b",66518:"c9394214",66524:"8bc94417",66538:"78f9375b",66559:"e664ab4b",66575:"a2383db1",66602:"5af37c6b",66715:"b7e0c845",66753:"69af30ac",66759:"dfe5642b",66822:"a5b3c79d",66832:"6af1a81d",66867:"9bff7442",66903:"1a459fba",66912:"54eafe75",66947:"c4ee53cb",66995:"784da00a",67044:"f4de3f48",67061:"d9896998",67098:"75ab85df",67100:"9ce14990",67104:"02c01d9f",67114:"8c651089",67149:"eda30c95",67189:"7efc5baf",67229:"600b8e16",67252:"9e177ed7",67376:"e257bb32",67438:"594c8d9d",67472:"522d8645",67489:"982012ab",67507:"f3e3cc96",67545:"326fab84",67581:"3158be0d",67587:"79c7e4c3",67649:"3e6502fb",67668:"b61eb3fe",67683:"d02d9bcf",67704:"90a45e0a",67720:"a584e6c3",67729:"20a9cf6e",67755:"1e073c84",67775:"b9dde3dc",67914:"332bff15",67954:"48148f16",67989:"b94c1f76",68038:"353d6f0d",68043:"e18e6796",68045:"7e4deb9f",68094:"1571ddda",68210:"58aca33b",68246:"f542fe1c",68249:"b11b303c",68253:"c30a59ca",68304:"6409542f",68315:"91b0f913",68334:"e92472f4",68434:"c764b111",68467:"a09054a8",68470:"20998ed9",68515:"48625b37",68551:"847ce269",68569:"8cddf474",68665:"36bf7051",68685:"4b2f7100",68710:"62339bd0",68765:"a74cb64a",68770:"cab2adc5",68839:"6cd72ebd",68861:"7c45d32f",68865:"a511cfb9",68910:"93d8f036",68930:"67d62767",68955:"d59c084b",68991:"fee3dba7",69017:"51df1e44",69030:"105e1b32",69038:"aade7e7f",69129:"af425d1b",69169:"310f0e66",69199:"aea55b19",69347:"146443c3",69365:"e3667910",69384:"efa43be3",69457:"0159f9de",69501:"99f0cd42",69517:"0c43557c",69570:"915a5618",69588:"919af855",69633:"322b0d34",69638:"05f50075",69671:"c73392f3",69770:"c4d5f69c",69787:"13159f81",69844:"3b524105",69875:"32cda754",69982:"3d6dd370",70008:"58761afb",70022:"066cbf14",70058:"943c427c",70100:"572c029c",70117:"b3c36679",70126:"d4115426",70127:"b084e3ed",70133:"9100024b",70174:"54e46a31",70179:"6bd088b0",70209:"f942e04a",70210:"baea7766",70234:"a553ab8f",70253:"30c1c2e5",70268:"4452b64a",70399:"e6d32efa",70418:"4c6611da",70452:"e2a2f968",70502:"c692a6f9",70551:"5ef3d7c7",70560:"215f7919",70606:"38763243",70612:"b8278f3e",70645:"4df69f83",70649:"416d6f2a",70669:"d4aecb8b",70687:"2dc36e49",70689:"2651b58f",70696:"c124e296",70712:"4c96e1d5",70728:"bf4215fc",70749:"42771251",70755:"3a5a31a3",70784:"0e4575ad",70791:"ad524d20",70794:"0f408514",70814:"2d650128",70825:"6ae11483",70826:"27591451",70828:"8b297c90",70853:"88bc6dfe",70870:"84bc6d3e",70947:"dc575b6a",70953:"0932b99a",70963:"6b5459a5",71021:"48d0f323",71026:"65a1b93e",71051:"bd187ac1",71111:"fc43fabc",71117:"209f2f28",71162:"54a7c705",71227:"c59bf196",71235:"ffab41df",71374:"ce3ef14b",71407:"7a2306d2",71473:"664a5e47",71531:"e7ecea81",71565:"e5390c69",71585:"72b12311",71590:"65b131bc",71612:"5e412c85",71656:"e6be372d",71715:"5b33b525",71784:"fa873b97",71792:"a6dc84f6",71858:"4bc2929e",71863:"92de78c1",71877:"2ec93eca",71910:"16d68f54",71964:"70da7109",72011:"7070f410",72043:"dc1ef129",72053:"6c6b59b0",72122:"23fdf7ea",72135:"5d936594",72246:"0f8806ae",72322:"8245e8b1",72353:"8d7cc6bd",72358:"7f58ed97",72446:"4a887c84",72527:"bc4dd3e0",72573:"dda0531c",72599:"9a7c5694",72634:"2c57b1ce",72668:"a2ebdf55",72675:"062aa684",72707:"7fcd7169",72759:"16cfab86",72815:"370f154f",72835:"07d5ab1d",72880:"82729a0d",72907:"6e8fb4cd",72914:"a6b31fb7",72925:"04c27620",73019:"5eb6a457",73084:"48ab7c0f",73193:"fc9f3e78",73264:"6ccefbd7",73276:"4d9bb901",73290:"d109a779",73316:"d595c06c",73331:"3e2d12a8",73376:"df8d786f",73397:"5d1ec000",73472:"2c6afda4",73493:"6c824c9e",73539:"a9e1fae9",73540:"d2683542",73569:"3aebe524",73607:"5d5dfef0",73609:"c5233a93",73624:"660e3035",73684:"8c31bfad",73693:"7d0c657f",73741:"c203b285",73756:"e59be998",73800:"bd55d6a9",73841:"d5ca1fa3",73846:"669a7fc7",73888:"46723720",73891:"335d1a8a",73905:"74bdfa12",73940:"e1201e1b",74019:"da089245",74027:"f505499b",74039:"a154b9e1",74046:"80b43a0e",74069:"bf9b50b1",74121:"976ee526",74134:"78e4ce3b",74282:"530f2a45",74312:"1776f3bf",74388:"455542b1",74403:"f4e7381a",74418:"3ab1a45d",74490:"800980fe",74524:"411bcbe4",74588:"4657c99b",74657:"071be4bc",74684:"08c31347",74792:"7bee6d52",74880:"8302c738",74902:"404c15e2",74945:"ae83d79c",74961:"1ab74c24",74994:"50e20aa9",75036:"6923b486",75043:"9bf3467b",75051:"483179e4",75062:"c02cd0c9",75078:"a71daff9",75105:"a4f9c66a",75106:"b618c014",75173:"0650181c",75267:"1905bdc5",75309:"d03c9c6d",75359:"1199c533",75378:"7152a165",75405:"98e83edf",75450:"0e82a67d",75459:"0049814e",75666:"0d1db8f4",75695:"09919d70",75696:"49a8d4e6",75718:"3efe0489",75735:"0b7281c8",75759:"29828f86",75775:"034e9612",75805:"cdf0f609",75866:"b5edd83e",75911:"e2c5cb8b",75919:"5d46debe",76041:"73d8152b",76123:"ee2b987d",76163:"eec9e7aa",76166:"d34b4fe7",76215:"f8abf276",76235:"02562da7",76257:"3ae01493",76262:"5ea97bda",76264:"e627ec8b",76274:"ab0d6c32",76285:"0742bfe0",76289:"208c9bec",76368:"500ae3ed",76530:"595f3583",76641:"2e5945d0",76683:"45b5ebf0",76719:"3fa3d99b",76748:"3d43af04",76850:"b0aa9be8",76854:"54e40af3",76871:"3cc12515",76873:"6ade5fbb",77077:"9dd7d520",77088:"19b3c698",77096:"d15e01a9",77123:"f06a5da1",77145:"470371c5",77174:"7f4d362f",77183:"3e975e2c",77200:"2f3da969",77222:"32602dc6",77261:"34bc5584",77298:"0e6e1364",77369:"789a51cd",77391:"a5d2357e",77596:"f28bf1fc",77679:"3ef8cf50",77698:"897d517b",77749:"57524945",77764:"96ce0e27",77787:"ebaf27f5",77794:"a1d36ea7",77920:"32922c56",78001:"494fb1aa",78037:"380f20f2",78055:"64043d5a",78087:"792a9bde",78159:"4f38a4f8",78199:"7352f166",78312:"4227ef96",78354:"e1a80164",78421:"90fce091",78497:"f3762333",78512:"53548a02",78522:"3a8bb73f",78544:"6a741e05",78583:"16c7d56d",78632:"2e8da915",78704:"8074d8fc",78758:"bd259526",78764:"21ea3c33",78787:"79135f9e",78820:"8b478ac3",78858:"34bc29ba",78876:"59d1e0a4",78880:"57c23eae",78912:"b1c543a5",78919:"dfaaf169",78926:"01258cb1",78985:"b28dc1c0",78988:"76e47634",79039:"56222027",79048:"a78c49a8",79150:"7f9b8fbc",79157:"5c77fbba",79180:"001af72a",79302:"4003d6b5",79416:"d8803eb4",79474:"ce9bb68e",79485:"abc97de2",79506:"c5a051c0",79520:"7ee03445",79521:"34c92224",79656:"aaf407c1",79739:"28dafb21",79841:"bb848e98",79866:"72d729c2",79987:"ec16d719",80018:"8fe0883c",80046:"6b819d70",80091:"cf6c8fc7",80102:"9c77890f",80198:"ae37fddf",80205:"8b305082",80259:"2a9fd9bf",80260:"cb06e1c4",80299:"67a193ac",80310:"825dba4c",80362:"777aaaa9",80375:"c431c14a",80408:"65e68152",80424:"2b9e2beb",80442:"138beb03",80457:"a89f5ad8",80458:"05820e47",80491:"31389da7",80531:"16becf9e",80557:"c57e7621",80598:"f70e808c",80632:"3aff62bf",80649:"47ad624c",80677:"4c372437",80744:"595e9d0b",80750:"73b5b786",80772:"832cafa9",80781:"304645e6",80794:"1edbd938",80841:"aefb64da",80863:"9f9ec4bc",80947:"39b525ca",80957:"97a18955",81012:"7a14d07b",81019:"0757a63a",81050:"4ca32e8c",81085:"e860a612",81125:"4f6a5985",81175:"cd0071f2",81188:"9d386ddb",81212:"6b747f92",81285:"b35bde59",81300:"8a132e02",81319:"5eca8594",81346:"96f50bd6",81390:"35ed9e45",81410:"1b63b983",81482:"bf8e657a",81530:"c7de49da",81544:"9f053080",81583:"5d60f872",81639:"2f3e574a",81666:"310e4ba3",81692:"cd67aa57",81737:"bc202717",81761:"c2e8f014",81770:"f9f0bc0a",81791:"514758f4",81795:"fe21b98a",81802:"33f4867c",81891:"85e8a888",81902:"972577b8",81903:"6ea756ad",81943:"bda6f1f4",81982:"45b0a42d",81990:"7dbdd021",82012:"b83ac31c",82019:"09be3a79",82237:"96c3210a",82242:"a8849529",82261:"65a3a905",82297:"4ce54554",82354:"e6079d9a",82362:"4bfee9a1",82431:"c58b9a06",82456:"a76fa164",82486:"6ee2f109",82553:"b2f3c04c",82581:"4ac5a8bc",82588:"22192259",82663:"e2968488",82722:"a43dfc55",82754:"1f4c6ca0",82847:"f1160e5f",82905:"facce5bc",82939:"3ffe9e05",82987:"27acfdea",83027:"eff4badb",83078:"e1a47202",83089:"c7ada7b9",83103:"3840f51a",83109:"6414a4ba",83111:"56e10872",83121:"006dc26d",83127:"f69c75fe",83182:"cef47da6",83203:"6e8afdc3",83213:"569e617a",83249:"32d3403c",83260:"c6525628",83273:"38f30c97",83316:"b0b27b89",83378:"aac56db8",83470:"71c00062",83574:"a3dc9dce",83608:"ee9f57b5",83670:"8fa9b576",83700:"060932b3",83779:"50eea83c",83797:"ca389040",83824:"09a61fd1",83841:"e29e4267",83849:"a66ff3dc",83928:"6c3fffad",83932:"7e88d302",83954:"0e32797d",83958:"bbdcafaa",83970:"2b543913",83976:"968c9e99",84013:"6a35ce6e",84070:"f86c329b",84107:"cdeafde2",84152:"00a2579f",84227:"dd167da5",84232:"fb6c362a",84255:"5f7c3639",84259:"a83b5248",84319:"add20ab0",84332:"ed66d686",84527:"a2acefd3",84530:"7b34a9d4",84563:"692274ef",84572:"790e2aae",84760:"175a5b9f",84773:"2493d544",84813:"39aa10b6",84869:"b3c6188c",84963:"75f5a097",85014:"3b5a5dea",85095:"6e925ba8",85097:"1682c825",85105:"b97600ae",85205:"b80556e6",85209:"5ee954e1",85243:"01e4ab56",85257:"b95de7f5",85279:"9d896a2f",85322:"18b44b2b",85358:"30512b2a",85442:"7970ece3",85453:"fb1e1d1e",85537:"c0626242",85581:"7500bb20",85588:"53c18d68",85736:"48cb5964",85796:"e203b806",85806:"fbd285ee",85817:"f02bbb8e",85896:"e9b359a5",85899:"ef3290b1",85917:"e7d86b26",85960:"f16cb757",85973:"1a5c8931",85997:"db6152cf",86053:"4dc95938",86074:"741302c3",86121:"594f4aa4",86129:"7cb605a7",86167:"3ce460dc",86215:"77a5d5a6",86216:"bef3c0fc",86246:"e0349554",86354:"bc7c5dc8",86417:"a1124a7e",86434:"dc8e21d6",86478:"84cf68a0",86480:"06949ebe",86501:"9d7d8716",86586:"8c579e0a",86597:"9bd3dc22",86623:"a74edf8c",86659:"0e19cb86",86669:"3e544ff8",86685:"45e653c2",86807:"3d614f42",86829:"00572ffd",86846:"9070c2f2",86856:"28df0c06",86888:"0615aab2",86890:"fce759e0",86914:"0fb4f3b2",86939:"8b586edf",86992:"8a207a53",87004:"c486e5ea",87087:"05aae5c6",87159:"443cc7aa",87184:"a039315d",87186:"57a6b3ca",87205:"b2690949",87217:"1f696713",87269:"35be2680",87282:"c1f408d5",87377:"bbb33c7e",87407:"5da9e9b5",87432:"2a58f20b",87456:"30fa1b01",87459:"ca9b021b",87473:"7572fe2b",87483:"e76cd711",87485:"074d2936",87547:"8a3e31a9",87589:"a5c128db",87614:"29322f97",87688:"abe56e79",87747:"99cd7148",87781:"89e64bd0",87863:"5fd2edbf",87875:"a9ccf0bd",87880:"bd9c4f07",87905:"5b6b1941",87945:"05f869bf",87971:"1b696ed9",87995:"341abd86",87996:"23c299a7",88047:"a9a1b642",88111:"6e8bb269",88163:"2c3c6b0d",88222:"9b39dc9b",88248:"f8851e5d",88298:"10ee8dff",88310:"36767ebb",88327:"774b80c7",88393:"5ffb00fa",88394:"903eae4c",88399:"c0edc4c4",88436:"2424bc8e",88441:"98d13908",88468:"ee29a68a",88478:"46e46ed6",88522:"8af566d6",88547:"7ac33681",88622:"441ab315",88626:"5b4795b9",88654:"eadecd16",88696:"3e68290f",88775:"06f05fbd",88815:"c077e43d",88817:"9e1a7664",88821:"cee6cb12",88847:"988a32ab",88852:"414f6661",88855:"0a2c96aa",88865:"e8542a46",88903:"e122d6fd",88986:"0bc77a5d",89018:"8a55ca24",89093:"b18e7a97",89097:"aa5fb21a",89172:"f79b59df",89175:"d0983721",89210:"395fbb55",89289:"d1f4bf4f",89338:"efe97fec",89353:"e7814320",89362:"780293e3",89402:"4de66462",89415:"6fb43246",89437:"90d090a3",89455:"f8dbee27",89489:"c8cc5158",89534:"da6951a5",89554:"a784419c",89610:"cc130b82",89641:"9d5b345b",89696:"abcb22e4",89801:"cb16760f",89810:"9e2ddc23",89858:"8b837402",89873:"0d66abd8",89986:"49804246",90066:"ecb816b0",90071:"f51c662f",90110:"cf0b8805",90115:"f497c160",90135:"d3605a71",90214:"23fc1725",90257:"6105e05b",90280:"9e2d7785",90317:"0dbcd032",90378:"47701034",90414:"4258fc22",90416:"410119bd",90438:"817c940d",90513:"91bc9a3b",90562:"b9ee764a",90593:"eac7c7dc",90598:"bd31cfd9",90625:"183eaa34",90627:"69608c40",90717:"f97b214a",90749:"f70bc7ca",90831:"3b92aace",90842:"ea14dbf2",90849:"39cca96c",90910:"0098609d",90950:"e0bca964",90972:"4dfbb0c4",91025:"baab9bea",91058:"6d57ca17",91109:"22f0de7c",91187:"ff56e27b",91203:"7e4cc839",91257:"5b4a526c",91305:"c2578a79",91319:"302da679",91333:"bb1412e1",91359:"67ea2072",91405:"c7144869",91428:"d8c3b11a",91530:"9b1454b8",91540:"18f2c91b",91581:"2c4a9a6c",91587:"5ac8f274",91597:"1fa2692d",91621:"67795e38",91695:"55515b28",91724:"20a3ceb2",91787:"bf5ab72e",91844:"46dfa8ad",91895:"5d4b082b",91916:"9ebdbb47",91993:"8c22cc82",92018:"8b6adee1",92031:"fe024597",92105:"5cefee77",92125:"683c7611",92196:"4b555b09",92223:"38a9da1c",92241:"28e0f6e0",92273:"f9a086ad",92329:"021f8e8b",92410:"4f523206",92412:"0fea7b37",92475:"3181ced3",92535:"63822ab0",92544:"15f57ee3",92653:"c1972b89",92664:"728dae56",92673:"39debf2e",92955:"89495ccb",93057:"a805415e",93085:"6b1e261e",93166:"5f6aac76",93206:"a2bf9194",93209:"13044cf3",93210:"85ec11dd",93269:"1029ee46",93273:"a527fd2c",93341:"7dde88bf",93361:"3d0e28ec",93362:"b3248d4e",93438:"eda79199",93442:"8344dacb",93534:"ede04b1c",93546:"d382b725",93602:"56ae7f49",93636:"259ed984",93795:"395d0490",93849:"00e1630b",93854:"c47af2be",93884:"e3d67ca0",94e3:"85a8b76e",94065:"6e8f3ab7",94082:"b6dbf702",94091:"578df47f",94106:"7ccfcc6b",94153:"b039c689",94168:"d6078fe7",94227:"970cd00b",94251:"9e3cf38b",94314:"d3489683",94344:"cd30ed80",94361:"9a28b459",94417:"394f998e",94538:"f82ebe9d",94555:"70e87010",94561:"a0b349d3",94568:"5a296dcb",94625:"98b3088f",94646:"e5304cee",94648:"f6228d59",94685:"438776dc",94711:"611b1dc8",94713:"58810f4d",94788:"6fc1dd4f",94796:"4010c4d3",94841:"89ae3e4e",94843:"828d0dc0",94863:"55a59675",94864:"7c75a8b5",94910:"d465e058",94925:"8f642dfc",94926:"aae8d19c",94980:"93fccaed",94997:"d853cfde",95020:"5e8185a7",95058:"75d979d0",95078:"41cdca60",95093:"423cea85",95136:"797d7fa0",95159:"1a0d6a0e",95183:"dfc87c30",95187:"85f160a6",95231:"b9c65497",95240:"7c6cb8db",95245:"dbd1b852",95330:"8e4b3ab8",95427:"9aa9c150",95443:"f7ec7ba1",95470:"87fd0551",95542:"91ccb5d3",95605:"6c76232e",95641:"0addadba",95649:"6a65e3ce",95659:"0a1f4efb",95768:"80c58542",95785:"48f89240",95818:"16d5276d",95909:"aa84deac",95911:"529b3233",95912:"423bd8a9",96022:"afa72882",96086:"085de370",96155:"e89275d6",96225:"fbcf69e0",96239:"843c315c",96268:"8b0816d9",96276:"82ebc6ff",96325:"75b0ccfb",96425:"769273ca",96463:"f832da4d",96470:"be797944",96555:"d21049e9",96652:"2f23cc7f",96693:"1d428461",96745:"6967bd1e",96771:"dc2aa01a",96778:"f5680f39",96793:"e0a23a9d",96881:"d4be0935",96963:"8985d561",97006:"598d7afd",97033:"fe730233",97047:"87cbec5b",97107:"18c4752e",97114:"9cc27416",97123:"040c8599",97167:"1238b646",97189:"23708f52",97223:"c8e4b82c",97246:"17ce0e36",97253:"b41c2a94",97296:"a57c26cb",97342:"52841075",97358:"c769b875",97379:"57d3846e",97420:"81fe43ee",97424:"a55b1584",97476:"56cd85f4",97536:"b4e3b3c7",97554:"005cc38e",97562:"bb8d8d2e",97585:"59a181fb",97664:"4e7bff16",97670:"a4274e75",97732:"0b31f7d8",97737:"2f8afde7",97753:"53aae94a",97763:"f430820c",97821:"f80efef2",97829:"5d36ccb6",97863:"f8cc590a",97887:"8c7c5b59",97895:"05685ac0",97901:"67fd6666",97905:"fd91bb9d",97913:"43f18326",98012:"6c29418a",98020:"96dae171",98021:"460aa0c1",98030:"a757e267",98058:"1c320273",98059:"6e7c26fa",98087:"0b0abad7",98094:"4dd0cc8b",98139:"7eab7866",98147:"7e92b0f4",98165:"2a383035",98237:"4eecdd21",98271:"a3781957",98308:"82707bbb",98311:"2e182187",98392:"2f73849c",98425:"bdd345cf",98473:"34829e79",98477:"ec2092dc",98534:"15f60b76",98700:"1fd78c55",98750:"89bad922",98758:"6d0f7cf8",98761:"e46ef735",98779:"84885d5b",98804:"6f3a5963",98867:"b7ac1bd7",98908:"e0e363a6",98925:"57095a25",98935:"86078de4",98941:"dcb08cf0",98947:"5f9ce70b",98982:"bdfdb56f",99001:"bc35de59",99095:"ee8e46b4",99146:"7b22214e",99151:"42d6760d",99252:"8f9df3bd",99254:"0688518f",99299:"a59c607a",99306:"56c8bfe8",99330:"3327c94a",99335:"b5287441",99437:"2c8966f9",99451:"b554daf7",99502:"00a479c5",99504:"2e6b3e55",99528:"7e312bbc",99531:"e8f43f56",99534:"428df931",99543:"b88fd084",99611:"1da1e411",99644:"d506a7c9",99678:"1b66abd1",99701:"10059403",99705:"1a05760e",99709:"3640fd16",99710:"ffd9ccf4",99746:"4b4c2ec6",99747:"9fa0b3a5",99872:"d78a1b1f",99905:"0b59e46f",99913:"7b5a7e5b",99987:"e88defa7"}[e]+".js",r.miniCssF=e=>{},r.g=function(){if("object"==typeof globalThis)return globalThis;try{return this||new Function("return this")()}catch(e){if("object"==typeof window)return window}}(),r.o=(e,a)=>Object.prototype.hasOwnProperty.call(e,a),c={},b="hudi:",r.l=(e,a,f,d)=>{if(c[e])c[e].push(a);else{var t,o;if(void 0!==f)for(var n=document.getElementsByTagName("script"),i=0;i{t.onerror=t.onload=null,clearTimeout(s);var b=c[e];if(delete c[e],t.parentNode&&t.parentNode.removeChild(t),b&&b.forEach((e=>e(f))),a)return a(f)},s=setTimeout(l.bind(null,void 0,{type:"timeout",target:t}),12e4);t.onerror=l.bind(null,t.onerror),t.onload=l.bind(null,t.onload),o&&document.head.appendChild(t)}},r.r=e=>{"undefined"!=typeof Symbol&&Symbol.toStringTag&&Object.defineProperty(e,Symbol.toStringTag,{value:"Module"}),Object.defineProperty(e,"__esModule",{value:!0})},r.p="/",r.gca=function(e){return e={12164231:"43113",12935692:"36790",13171783:"29452",13883961:"54772",14828316:"41944",16243474:"98020",16640689:"53619",17896441:"18401",18891827:"71235",21719437:"65957",23348138:"57155",27077527:"44840",27211971:"57547",30769210:"44891",31863783:"2256",33241236:"33621",33732613:"14363",34689118:"51937",34900881:"79520",35313843:"23641",37844666:"63075",38836367:"99913",39583919:"70606",42545929:"37827",43764587:"32949",47074836:"39260",48455675:"66715",50536743:"43078",50737734:"21373",52166518:"76123",52375367:"70689",52676037:"54487",53317765:"39058",53798754:"21191",55946737:"45113",56040849:"72815",59171146:"67989",59700627:"95187",60165173:"44719",60753781:"68470",61773850:"22504",63786366:"93442",65511420:"32875",66405406:"89534",67474760:"63332",67922501:"65221",70570476:"12513",74555694:"79302",75914136:"65091",77787075:"36689",88937603:"61939",89514438:"17709",91415840:"9003",92328698:"20924",93340492:"41159",95789563:"59923",98749210:"96963",98826249:"68315","0583dad8":"16",a33c645e:"25","0da1f28a":"28",a978e5ab:"39",a674d2f7:"48",e9bd6ef2:"60","56169fa6":"67","6c6aeb2d":"106","275b5c51":"129",af8190e8:"159",b1f77b90:"217",a1a7cea0:"278",fcc9c1b0:"292",be9655ff:"323","43d16b11":"335","0b4da2f0":"354","19ac452e":"387",a168e2e0:"437",b4a99b76:"569","0a87a91b":"614","5c109a5d":"805","79cc09db":"822","8b10f6da":"851",e9038c17:"878",ce3da75c:"910","02b5547e":"917","9699c87d":"924","0c459987":"940","4b935b7f":"1103","060b2af9":"1107",e3b4090c:"1119",ae09e9a1:"1136",b0127f30:"1190",ea1c8f1c:"1251","62a07427":"1284","9cab4fb3":"1337",eb8e7dea:"1360","562c897e":"1422","50f3bb83":"1442","4b25608f":"1506",da868c82:"1517","8425e24c":"1550","73c8d421":"1635",cc5200ca:"1656","7a51e13d":"1658","89dd9b54":"1784","269ce614":"1785","4b2ed2d0":"1791","92bb01f4":"1841","9f349e0c":"1846",d032d8fc:"1857",c28fec2b:"1882","6e47a3c2":"1928","45b94beb":"1946","7fbadf7c":"1955",b93dc3c9:"1983","3c15683d":"2016","7a3be72c":"2053",fdb4abad:"2121",ef5805c5:"2172","32aa7863":"2214","9781112c":"2225","2b154460":"2239","7d9ec3a3":"2262",d2a01f74:"2298",fb4f9065:"2385","3b3f1ad3":"2450","1b7e3feb":"2472",e8b490f7:"2540",b9e56da2:"2577","372d2263":"2638",dba6e014:"2640","9ed88412":"2652","9d0312da":"2663","7e9327af":"2679","5e58268b":"2700","4c5bb0e3":"2739","7c87f31d":"2748",d28b7dcd:"2759",ad32f2e8:"2768","48f24bab":"2772",ba2305d7:"2791","00cd9d46":"2839","093ec6b1":"2854",c03096de:"2867","8539bc43":"2884","404a423b":"2912","580eda40":"2932","6e7eafb1":"2954",a80470ee:"3075","32dcc326":"3125",d29911ee:"3131","4f7cd6bb":"3137",aed1d196:"3144","5c3bdea9":"3179","6be904db":"3303",a3e90044:"3313",b9073401:"3323","1c4da86a":"3471","7fb3fac1":"3558",b783cafb:"3693","5e85aa31":"3694","50cd3db4":"3722",dc1f5b39:"3740","394bb252":"3764","43d31177":"3847","260d7fd0":"3940",d18b6106:"3965","50494a26":"3968",c943b655:"4014","04ae9608":"4028",b4ced939:"4040","0bf991f2":"4060","817ed3cf":"4074",d772531a:"4117",b18f8924:"4133","1a47b5e7":"4137","0c12eeea":"4205","18ffe98c":"4269","1497bce1":"4355","25fe5fdb":"4368","56471e5f":"4382","1dcf0194":"4405",ff5a6f64:"4467","3e8d408e":"4514","1cb4ff83":"4530",d9f219f0:"4574","405284b3":"4591","9920fffc":"4602","43d0cc21":"4777","3b6474f3":"4790",a0c4b68b:"4863","7c52b6f4":"4890","4929a6fa":"4919","066b815f":"4949","006c64a3":"4980","6880fe95":"5008",f5c507c6:"5028",ca8c3af7:"5029","25ed107d":"5065",d0ff2798:"5193","8c7f4a45":"5195","4b9df2bb":"5206","627b1f39":"5269","6ff0c558":"5270",b3a451d2:"5289",bbc3d059:"5295","44fefb3e":"5304","22b4f68c":"5322","86642e8b":"5334","0e11b6db":"5338","0c74166d":"5594",fb863cab:"5622","32ba6a4c":"5631",a804bb8e:"5633",d03e0408:"5647","58c4a37c":"5679","42bda306":"5721","1adb4ab6":"5806","6a34bd67":"5834",b5960565:"5887","478a7123":"5888","4959f63e":"5936","959a592e":"6022","87c914ef":"6054","384f1d9f":"6090","0d706529":"6158","14ed99c2":"6170","0ba05648":"6203",baed6a8a:"6225","59ba4d58":"6289","14a73a68":"6310",ce3c8465:"6350","207e0bd3":"6368","4658250f":"6371","5c865244":"6372","47ddb104":"6407","3faff22d":"6467","316bb886":"6475",ae59a273:"6552","21b16c5b":"6559",f6a1f09d:"6636",ea2cf37a:"6677","0cca5757":"6692","17ace3c3":"6697",debf2b54:"6709","01ceb854":"6739","507d925b":"6761","9cb5abfe":"6778",ae0384e2:"6791","4f684eef":"6823",b52407fb:"6897",ac385e9e:"6930","52f660a4":"6944",e841f655:"7035","6e78b29b":"7071",e7c12d1f:"7100","9d67d055":"7103",b744ccc8:"7125",bfd461a8:"7167",b1b9fe16:"7317","96f48bd9":"7355","502758cc":"7444","8fdc7f76":"7458","4abd5d51":"7517","12a626af":"7566","9df106ad":"7622","2a74f6a7":"7629","06a1e3b2":"7715",d6b46894:"7734","79a2ad2d":"7747","869d1599":"7773",deab0229:"7802","68c81474":"7850","7ba20019":"7907",df4b657d:"7953",f85e4dd8:"8003","0480b142":"8070",addb1e87:"8139","464278e7":"8174","38122bdb":"8177",c7a3e7d6:"8183","934057b1":"8192","01a85c17":"8209",c286af15:"8289","09ff3d76":"8348",db9fbbab:"8354","24179e00":"8363",abd23114:"8367","7a049829":"8407",cd17745a:"8413",bbced460:"8417","497095d8":"8585","15c2f51a":"8619","22351c69":"8632","212a125d":"8645","80bd773e":"8651","8f08e203":"8699","1cd42d14":"8762","7a11cfa3":"8766",e161c803:"8840","3a6a48c9":"8888",effd3c61:"8889",eebb3a9f:"8900",dc8afb47:"8902",b6e28713:"8965","6d5644d6":"8978","7c9ee56f":"9045","4c492fb9":"9066","52b6ceb8":"9076","29c12ff9":"9127","17666b14":"9152",d4eb85a9:"9189",b2d2727d:"9195",d336ca2e:"9228","0687df23":"9257","29a0dcae":"9262","4c14260d":"9351","7fd1d1a0":"9379","409e52b6":"9380","8ea09047":"9385",cfb1206c:"9428","8cd4ff44":"9432","49b92117":"9446","762ed4b3":"9458","01964c8d":"9460",ebf12a38:"9471","953153ea":"9499","2fec12c7":"9516",bf1307fc:"9525","8eea179e":"9570","13ab27aa":"9573","79e67a34":"9631","5e95c892":"9647","747973bc":"9650",e4d28161:"9676",a0d28f55:"9702",c35662b8:"9722","46e7f252":"9724","0d37dd31":"9737","6a0b8fcc":"9790","2fd2285d":"9836",d6d00b14:"9860","36e3724e":"9870","8a5cd0c4":"9891","9c0f572d":"9907","06b7dbb6":"9914","9ee88b57":"9944",c782ecef:"9986",c2277200:"10030",f89af623:"10118",f7519df0:"10181",ce5d6b55:"10198","0eb44ec6":"10238",b69fdc1c:"10285","8c29db87":"10354","0ca8e4c5":"10472",dc8e8e39:"10488","40011a75":"10492","79c374cf":"10535","1ca50c03":"10545",b1051fbd:"10610","6a326c1d":"10637","936cf648":"10642","5bf02344":"10653",f71afd42:"10688",f53cf472:"10727","931768b8":"10767","10fb8f29":"10789","9009d9a7":"10884",ac65ed35:"10887",b964b0d3:"10907","42746bb6":"10920",f0c78ca3:"10924","726b1d2e":"10956","90db14b9":"10978","9f1f18ec":"11052","7c44e50a":"11058","2750cc84":"11111",f97de939:"11156","2e961a80":"11275","82b971d3":"11289",ac338f84:"11299",f9cb0cea:"11326","6ed20719":"11355","039c7621":"11508","3d026c66":"11565",d2a270f8:"11631","9aee6bed":"11684","8fa68fae":"11716","51e360fd":"11725","446810fe":"11745","4cc89b71":"11768","3f65fb56":"11782","43ff550e":"11795",b9168984:"11804","7cb0ba19":"11806",f630dd79:"11842","51e592eb":"11855","43c57b15":"11869","41b6dd58":"11905","757b41cf":"11906","608a85b6":"11977",d25544dd:"12047","088c57bd":"12134","5617941b":"12144","3deb4153":"12186","77a5f3f8":"12316","9b28f794":"12318","2592f25d":"12378","7d05d2dd":"12393",a0fb693a:"12395","61c30610":"12431",f4a568da:"12452",ac9a039d:"12477","3e8ec4bd":"12485","2ca30cc7":"12535","0bce5bfd":"12542",ae632d28:"12608",bb6acec0:"12613","98af85a1":"12654","755aa830":"12679",f81e2128:"12686","0166538a":"12763","236efcfc":"12869",bc561320:"12945","72ed5e19":"12995","9b078b3d":"13021","075d0b63":"13060","6695bce8":"13063","4d01f822":"13152",a5fd3751:"13203","326f3c2b":"13210","814487fc":"13236",be65306b:"13298",bb230ab4:"13321",c7690f1a:"13366","0c9b8a46":"13374","15be7f5e":"13375",cdd81d53:"13406","05f125a2":"13419",f7887fd4:"13465","3f9b0969":"13511","35f2b245":"13514","559e18a5":"13522",aeb9cb2b:"13525",a221d960:"13534","27aec4ea":"13535","1b3749bd":"13575",b7ffc82e:"13580","16444e78":"13626",e32c93f6:"13698",e4126631:"13733","35ace877":"13762","5005f1cd":"13803",fdf3b0ba:"13849",a586ac12:"13918","79c522c5":"13943",f4dd7c05:"13982",bb5bef2a:"13996",fe0a5e13:"14003","4fe2812d":"14027","6c6eb236":"14042","2da40935":"14043",c58d5881:"14125","88677a17":"14134","5ba7f3a0":"14136","0c792305":"14152","79f5ed7e":"14163","0466f8a1":"14179","973a6c2e":"14257","4d62164b":"14259","881e0310":"14261","15b03510":"14264","2d2f3863":"14317","02704d8d":"14340","4330e2c5":"14388","95a29cbe":"14399","262f048a":"14410","5a1ac6cd":"14433",a2218d2d:"14435","0dc66419":"14496",b812e3a7:"14536","04287605":"14599","6b0d4b59":"14627",cb290368:"14726","8049dc2a":"14750","4193b389":"14802",da9a6aef:"14822","3145e86d":"14835","9c9e6d14":"14868","281df6d1":"14871",b2f79aaf:"14917","68aea991":"14919",b634826e:"14943",e8bf575a:"15038","6773b5f7":"15078","461201e5":"15079",e6691bac:"15144","29fcaeb8":"15165","809a1b46":"15186",e0719818:"15220","246d116d":"15235",ee5dc595:"15272","7bd6096e":"15292",a0583d0f:"15316",bbbd3731:"15403","413b2e2c":"15407","63067f40":"15427",ad07d5ed:"15473","259868d4":"15501","1ff337bc":"15552",a309ba1a:"15607","0d11ee2a":"15764","65df1ffd":"15805","520bf372":"15857","04d2875d":"15917",f05cb9af:"15947","1ea67680":"15955","777c5b61":"16017","7f5a2db2":"16032","5685cb00":"16100",f0db470e:"16209",d05ef008:"16231","1839a246":"16319","8eca1d9c":"16377","58c0e7a1":"16499",b5eb2b1f:"16500","3d8e248c":"16622","4ccf094c":"16641",ab84da8d:"16747","7c3644b1":"16885","426a3c0f":"16942",bd7a3684:"16950",a3848d24:"17009","30e67995":"17033","3687929f":"17115","87557f51":"17119",bc36781c:"17232","82fc01c6":"17294",b4762a45:"17332",de821e02:"17368",b629aa71:"17402","496a5e3a":"17411",c5c8072f:"17441",f7750a15:"17474",dddd9631:"17502","80e60ab1":"17508","8c212f85":"17530","46e89fee":"17581","47f3a505":"17626",a2af438a:"17681",b1e22259:"17739",bc4a985b:"17805","2d9024ae":"17818","7cfc22c1":"17860","9130943e":"17867","36f96221":"17944",f67021ea:"17987",a3255c75:"17988","059d2658":"18006",c1fa975f:"18033","7707ed5e":"18103","3a2db09e":"18121",c15d9823:"18146","889bcd45":"18350","670426f7":"18351","7eea383c":"18438","762cb2d4":"18445","905fdac9":"18520","1bdcf2af":"18620",e798c07e:"18645","3b375bfc":"18738","10b6d210":"18757","6c76cc4e":"18798","6cc2587c":"18940",d75d009b:"18950","7487c019":"18998","378587af":"19042","6e547e58":"19052","6f878c05":"19070","6f3d983d":"19108","868e6cc1":"19109","9c3fccaa":"19136",e77e0240:"19237",d9884433:"19252","1354c4fd":"19267","345a8726":"19271",a2cfe627:"19369","9228ff94":"19387",c61ed569:"19396",efd9e5ca:"19425","4b6ad6c1":"19428",a3c850e8:"19474","3faa4dcb":"19513","7f5a4e86":"19521","3548485f":"19542","76b52ed8":"19664","547af68c":"19687","0e4a3360":"19711","9e46f997":"19736","3c6c92ad":"19774","83c71362":"19798",a5052667:"19801","7e53000f":"19865","389b06c3":"19891","99bce14a":"19999","0ef00d46":"20038","430ee71a":"20103",ecc82a52:"20160","69fce7d3":"20194","5aae4c30":"20229",f60a8e85:"20286","6b7b009f":"20325",a55cf427:"20358","784d2d5b":"20361",a170c4b2:"20405","839320c8":"20415",c97a4f9f:"20424","39c0c867":"20434","4c056f0f":"20491",ca16b0fe:"20535","2925f13d":"20541",eae39fd7:"20619","874c4928":"20676",ff781369:"20713",a827f20e:"20812",bd440596:"20857","1d99c39c":"20892",de69fdc1:"20931","5ef30e9e":"20951",e3b4fd8a:"21070","7ff678ac":"21077",c1b7702d:"21121","574aa417":"21177","58f98693":"21273",e737da10:"21275",fab95ca2:"21290",e4bea80c:"21303","266d37b9":"21316",b23dd436:"21341",b2c62a3d:"21346","5105c354":"21401","9db9c0e0":"21417","995d9cd2":"21448","5f404bc7":"21482","69f9ca18":"21524","6c888f35":"21560","263bc6da":"21574",d17a9646:"21585",aad02470:"21630","6224d640":"21635",b1722c22:"21662",ea9d899a:"21729",df2a910d:"21764","1926d3d7":"21805",fe2d6fd6:"21816","7ce47929":"21837",a54a9a31:"21874",d3de5519:"21909",a8a7f86a:"21921","625beb64":"22141","334d7764":"22313","83a16324":"22369","9a5e9795":"22385","47425d5b":"22428",f93d9ca9:"22447","494588b9":"22472","95d688dd":"22483","2923e8f3":"22499",ff15e2eb:"22501","6f37f43d":"22534",ca5d6ba8:"22595",a36f1f19:"22628","69375b61":"22640",c760fa6a:"22647",c699b29d:"22673","74c5eee6":"22709",f701b058:"22750","70db0e3d":"22816",d6ededeb:"22844","9a29e675":"22911",d49fdba3:"22916","0d5f3865":"22932","70a32eac":"22973","1fd06df8":"23002","07da43c6":"23041",ef1c4348:"23081","7050405a":"23083","39307ee6":"23119",a1c39b3f:"23141",b994507e:"23172",fe86f3f9:"23332",fa5f4abf:"23388","95d97af4":"23438",fdeab78a:"23475",a67ef186:"23547",f24f712d:"23563","53746e73":"23591",af9b00f1:"23600","996abb7f":"23625","59e9dc06":"23634","9c150758":"23637",cb16b650:"23666",d235b83f:"23676",ea8366f8:"23700",fac8c34b:"23765","2153fb85":"23773","3affb2e4":"23880",be354741:"23938",afe4349e:"23998","9c7497bb":"24022","5ef1cef9":"24065","4ed60a8a":"24084","89ad6a53":"24098","897856d4":"24126","5ce7213e":"24140","1d2c1565":"24172","3af3061a":"24234","7ebf6f1e":"24265",a4e075be:"24340","99d3870c":"24366","5cbd92da":"24373",b5b376a0:"24377","4066930f":"24381","77b23935":"24437","045bd6f5":"24475","6cf93e89":"24476","3befffe2":"24704","91c64349":"24728","931beacc":"24729",e2c09daf:"24741",a852ca5b:"24776",a68f650d:"24815",ed6ec893:"24826","7cfd7c3a":"24853",e0c51b98:"24866",b4972e92:"24867",c6301cab:"24868",f0709bee:"24891","8e01a36e":"24912",cf6129a8:"24919","51fff4d3":"24932","231cac3a":"24970","1e3d98df":"25013","9cd11b72":"25044","9531441e":"25067","8bd2257d":"25101",b277530c:"25130","96046d19":"25186","19531bab":"25273","1da16fcf":"25285","537936fe":"25335",d78b8cbc:"25396","4083b748":"25397",b973fe72:"25426","5ef8e026":"25452",af5c68f9:"25531","40ebd41e":"25532","7fb09d55":"25539",c7382291:"25578","65603bfa":"25582","7205cbcf":"25601",d56bdeaf:"25778",ba115cad:"25781",d2c3f17d:"25845","6d1cdd35":"25869",c9953bfc:"25871","17268e53":"25874",f7f19049:"25959","1645b95d":"25987","92db1d5a":"25993","90c5833e":"26003","20f9256f":"26009","7662a79c":"26060","2316ae30":"26064",de97f7b0:"26081",fd6c914a:"26183","4e3e1e17":"26216","477cc2f7":"26260","878dce9b":"26305","9579dcb9":"26333","65009e94":"26337","14fd0d1c":"26414","9a5789a7":"26548","8eca70a5":"26577","8353278b":"26627",ba839983:"26630","960d3eb8":"26632","1802ae9e":"26722","78bc6f25":"26724","21e535c3":"26744","4681e75f":"26797",e2fe7a08:"26833","04fb6378":"26859","958a296c":"26882",b3749361:"26900",aca5eee4:"27012","58714c61":"27020","1547c10f":"27088",e37692b4:"27115",c4d1e829:"27123",cbec0238:"27179",ce710151:"27212",f52b34fb:"27369",f7e205e5:"27388",a3983124:"27398","7a3be7e3":"27444","31ed8f07":"27473","9f8f3c52":"27663","6d701be3":"27737",f95764fc:"27823",b0d5c266:"27887","10ccea9f":"27921","2e7e1134":"27955",ee4b8f19:"27995",ecd026d3:"28020","2cc67ca6":"28086","859500dc":"28127",abde445c:"28147","8120957a":"28166","513b54fb":"28169","9adef974":"28178","758b9bc9":"28204","20a81622":"28235",b8d3e47d:"28252",ec410c33:"28319","89cbd1b8":"28410",b212a261:"28428","5b374304":"28435","261fe657":"28499","54ad050e":"28627","0871002b":"28679",ab48a39f:"28687",c3856c66:"28701",fd5c5192:"28782","379ac6de":"28793","8b6fbbb4":"28803","7ed2bc06":"28834","4c00d6b9":"28853","2a0b5233":"28889",e7ef6e16:"28920","8c5b1685":"28941",f382fc88:"28945","8e7128cd":"28993","9c7a2f87":"29060","763fa02a":"29123","6f3c73f2":"29171",d4836e14:"29204",d330b699:"29243","8a472aba":"29337",fbce6636:"29408",b92135a6:"29414","163a1bae":"29421",e4d4ec4e:"29457",da05f515:"29498",b2f58160:"29523",b45a54e9:"29526","13f72d6c":"29653","6e0a413f":"29679",b1633424:"29690",a642dcef:"29698","80b93830":"29708",da10e58f:"29733",e6557239:"29795","16381f20":"29820",d973177d:"29829","5d94dea0":"29845","26f5f607":"29891","5397b577":"29922","5de1c39a":"29950","332b14e0":"29969","87546ad3":"30035","6b836ac4":"30066","339988c0":"30154","22bf7f69":"30167","5320ed26":"30262","132be4b3":"30265",af953e30:"30317","4b678e99":"30355","5c5d3a2d":"30378",e4a43002:"30396","90ffbe81":"30410","407230da":"30564",c7165ecb:"30569","1ee0a129":"30579","775c5b43":"30583","6ee9522e":"30628","5d70257a":"30654",a7c3ebee:"30790","5814834c":"30827",ced554c7:"30831",d8412727:"30842","677757ca":"30848","44c28e7f":"30934","7dea1595":"31004","416fb909":"31016","71d5b1f8":"31039","3f9a993f":"31047",c3c8ddc4:"31125",be9442ac:"31141",fbf25af7:"31144",b8ab477f:"31220","5c81ce12":"31255","67f1ea1e":"31257",fa366c46:"31272","07f6d4d1":"31393",cc564cf4:"31465","1668cad9":"31541",c602d51d:"31544","8c4bf82a":"31567",b38306ed:"31680","05957343":"31705","0a70c05a":"31720","624fba40":"31726","974bc2fa":"31749","04eab06f":"31791","14a26e2a":"31797",a14af959:"31861",c791a767:"31862","2dcd9099":"31869","2e31e40f":"31957","4ef864ee":"31972","8dbbcff6":"31996",f0dc0726:"32005","9ba9284f":"32012","116b7e55":"32039","5110a7b7":"32183",eb05a324:"32225","53ba3577":"32270","149a2d9e":"32278",f9a4941d:"32335",fe345ee5:"32365",e1407f5b:"32391",c08ba4cb:"32472",e6c2d44b:"32527","40c3731f":"32558","798484e3":"32691","1b520177":"32728","943ce597":"32824",fdddbf66:"32825","8590fd3e":"32847","225cc45a":"32960","604cd7c5":"32993","97e51b1f":"33049",a172f6b1:"33065",ee003a92:"33192","2a422024":"33214","465994bd":"33234","7da68ccc":"33290",f5ff1a23:"33303","2d0c5b52":"33350",c16e0158:"33458","3ed70074":"33517",f81c13fc:"33524","65ea2434":"33533",fe7ac0b7:"33540","8f00bf20":"33559","6075be34":"33567","269d704a":"33572",df68b7b1:"33577","3f933ca5":"33612",d294bdb0:"33638",da7149d8:"33643","81a6b262":"33659","755872dc":"33724","1b31aebb":"33804","07399e71":"33806","45f2496e":"33816",f8a443f2:"33825","06758c2b":"33854",a7690409:"33881",e9c7c2b7:"33889","49aa1a16":"33999","18b5779a":"34127",f5c63bac:"34156","3f4396e6":"34179","80f4f5b6":"34182","13e6af21":"34196",e8a8dea2:"34255","164e61b9":"34331","6153f283":"34335","79fbf53e":"34388","538371ec":"34520",b038a3ec:"34528","782eae4d":"34543","0735e5be":"34549","40cd51eb":"34597",f30fab6f:"34621","852f3aa7":"34638",ef071c32:"34662","02e54e09":"34691","47a1bde9":"34706",a5fea1bd:"34750",e076fc92:"34803","2e6ef254":"34808","924dcf98":"34810","10fcd509":"34837",dc7f8f8b:"34847",f7702246:"34877",f44e1bcd:"34904",c9cac9d9:"34918","40dc6b06":"34970","5e2f1fff":"35044","47cb0ede":"35067","35788c78":"35088","6fe5390f":"35139","83829ea7":"35144","372475fb":"35179","2947aa63":"35248","0f3b9f0c":"35288",d42065ad:"35325",a7827f50:"35329",d9454955:"35379","976ab233":"35402","388d5845":"35532","97bb73fe":"35618","2b4cfa56":"35730","8fc868c2":"35733",aba21aa0:"35742",ff6f6072:"35766","3019fa66":"35767",d240c26d:"35784",dd61deac:"35837","8852ad6f":"35881","92928c02":"35922",a3142532:"35932",dfa4e294:"35960",b5a599e0:"35983","04afe417":"36009","59be0e8d":"36055",ceff0062:"36067","28f38ec5":"36113","307d712e":"36144",b60e28b6:"36192","8a759f03":"36264",eafd7a74:"36302","467ff4a0":"36379",e23d1069:"36387","5ea64d6c":"36549","1472dfa3":"36569","94a5efbc":"36594","2573355f":"36619","1084a32f":"36640",b6fb003b:"36672",a3015275:"36747","649faa0e":"36851","1007513a":"36885",d73e6b67:"37109","0b4fcfba":"37115","97da4333":"37147",d786420c:"37191","8f3b27f1":"37204","43dcd746":"37206",e34cac0e:"37209","37524cd6":"37257","554882a9":"37329","6013ac2e":"37366","32b44a8d":"37403",ce46a18e:"37486","6a293d29":"37489",e9705ef5:"37577",e17d733a:"37578",be2ede3f:"37617",a6aa9e1f:"37643",e2657c7c:"37673","2cfb265a":"37740",feab7dcb:"37799","5f87fd99":"37809",d5e29b01:"37839","9ad03801":"37841",cd060f40:"37842","88d0aec4":"37872","3f272b07":"37954","690cbdd8":"38001","8d086d51":"38003",baad6bf9:"38056","1efbb938":"38107","66372f10":"38158","2e9f383c":"38238",bbc1ba35:"38348",cf858c26:"38370","7c15a7a1":"38385",dcd7017c:"38416","9474fc0c":"38439","91335d19":"38449",f0fb7623:"38451",b4a84943:"38468","0f497157":"38600","6dc06ee1":"38605","7eaa135a":"38607","1bf4b05b":"38700",c4de5678:"38767","872a41d4":"38786","5ea125b2":"38816","6495713a":"38835","110e78c3":"38839","6f5eeb66":"38888","2c50796a":"38974",daa4f13d:"38999","5560f355":"39088",cb9e1f3d:"39110","755ebe0d":"39138",b5f01515:"39142","0b8a4e62":"39153","09b62bf8":"39192","484cbf74":"39197","8b795044":"39202","93f7a5ff":"39208",b992801d:"39230","90f157dc":"39240",b40642d4:"39265",b1043e58:"39283",cc6505da:"39405","995840bc":"39407","085b8128":"39417",a615dd65:"39423","0f65ccfb":"39444","9d833fad":"39487",b326207c:"39494","7b2a260d":"39496","782d926e":"39505","630bda65":"39518","441dded5":"39520","4a80e27a":"39555","06a8eab3":"39569","7d0e1375":"39587","97ae1fff":"39608","8f64cb7a":"39712",a68f0e64:"39715","07bef82a":"39758",fb3759c3:"39773","320da59d":"39777","3c2d0811":"39781","8fa753de":"39840","9b0c8626":"39887",d76c455f:"39901","09a52b17":"39908","6471fe03":"39921",d4a2e930:"39973","5f469a3b":"40065",bc817d28:"40124","0508dae2":"40178",edf7dc61:"40205",ecaa52af:"40245","3c36ce76":"40369","0cd1a20c":"40463","4d1df301":"40464","3415fffa":"40555","3e032765":"40606","12a5de9b":"40616",bf5126e1:"40651",dfc3c224:"40690",cc01a9d7:"40697","8393a516":"40755","1a7b33c3":"40758",d2110cc0:"40797",ddebc8bf:"40827","2662e92d":"40847","321dafc4":"40907","2884dc3d":"40919","881bebdd":"40927",ca0149b2:"40959","68c5632a":"40999","33d4a0d4":"41030","0fa5c29b":"41125","85ffab62":"41135","6c9200cf":"41260","247909f7":"41361","2bf5edd4":"41455",fd517388:"41461",c3f88c36:"41491","8cd118b0":"41504",bf9d9be8:"41518",da172daa:"41594",d407c2c6:"41640",e8e3bd30:"41651","9017a110":"41677",b9edf71b:"41739",bc38a1dd:"41750","29db9f25":"41852","78f0e259":"41867","58b3152e":"41872",dc5506b6:"41892",b7a56be4:"41907","7c49023e":"41954",c413ce11:"42068","295cb0c0":"42187","671726ea":"42196","89c8b6fe":"42268","3fa17fc2":"42298","1f979aba":"42299","6df57399":"42350",ffac4af0:"42368",b8212f90:"42465","203cc654":"42487","34825c6d":"42518","2b505997":"42566",b184a577:"42576",eafac744:"42596","165a93e5":"42690","28826a4b":"42702",d8ed3ccd:"42723","89f1622a":"42794","5314580d":"42799",bf3d61ec:"42801",aeca0a21:"42879","432e670c":"42881",c9d95cbd:"42900","145364e0":"42905","39c7a203":"42911",ebbd9274:"42920",a8c92c97:"42937","19028add":"42942","2812e3b9":"42995",e3477e52:"43029","2907d5eb":"43035","676c7c8e":"43040",bec2896b:"43042","341293cf":"43088",d831e3b2:"43195",fe9b6676:"43286","4c07ba53":"43352",f048922c:"43421",fbf82623:"43432","83fec35c":"43445","7b89671b":"43461",d319e9f2:"43512","4cd0ba1b":"43642","7117a8cd":"43677","752351f9":"43739",c09831a6:"43868",a8b66660:"43922","509aac2d":"43980","5a8b22a0":"44008","454b7ded":"44022","43d1bad2":"44040","894f5c0e":"44045","6503f568":"44118",e2fc22cf:"44123",ca2b0f7e:"44125",d653d7ed:"44129","19acc4ed":"44143","4f087d20":"44290",cfca4f26:"44326","0a91021f":"44339","8fd374dc":"44380",d363a80c:"44405","216bf65d":"44419","6fb01322":"44489","9882af90":"44492","9b2ae5ff":"44551","502906a9":"44567","20bfa334":"44640",d65b2c25:"44662",bdc69003:"44682","260982a7":"44876",f3d02b29:"44933","43f3c18b":"44939",c7a67184:"44948","12b957b7":"44960","8914fdfe":"44964","41af2803":"44994","6e34e766":"44997","86fe1f77":"45117",b528d4d0:"45146","49a37e79":"45210","7737fe61":"45231",c3f790da:"45238",db90ec2b:"45290","815e47ce":"45323","24d4b044":"45339","459dbf85":"45460",e85bde03:"45500",a722b43a:"45541","7dce0476":"45561",ef7891f5:"45640",c36346e3:"45670","3533dbd1":"45682",a2a2954f:"45919","9926d60d":"45983","95aba771":"45988","8cf74eb7":"46001","2e048b0f":"46008","1fb7523b":"46025",f724037a:"46039","67893f6a":"46063",f1087137:"46074","53cdeba4":"46156","8e4df3f0":"46197",ea511817:"46244","04b49851":"46296",e49bd305:"46375","00239e8e":"46425","4bb02a47":"46451",bcb734aa:"46472","9dfe5a97":"46513",c13f246c:"46515",c5427124:"46547","8012465a":"46578","8393d26f":"46598","5f0630da":"46652","4f48cd24":"46662","2b6a7ab0":"46754","8f07070c":"46802",b2e2853b:"46840","917a523c":"46850",db9d1243:"46888","85f8bce5":"46891","57a58e78":"46905",ff13b684:"46911","7b0a3d41":"46917",a28b89d0:"46921","37ffd066":"46937",a9afee85:"46979",bbb92300:"47041","26996ad6":"47060",a5c8a2aa:"47087","696d889c":"47106","1366dd9b":"47110",c648308f:"47127","4d2e69a6":"47206","131c6f6e":"47236","5048202c":"47247",c7e2144d:"47248","6111afa9":"47297",b9a767d8:"47433","09e5dcaa":"47517",df0967b2:"47537","0029660c":"47555","0c1ff092":"47622",b1899f7e:"47668","39a07549":"47693","17093d00":"47699","4edf53e5":"47730","6bb76d2c":"47917","7ad7688e":"48003","47f96c4a":"48023",ca713325:"48050",c17f9567:"48063","852ff8c6":"48071","0cc98403":"48091",f81c1134:"48130",c467a11b:"48132","519b5759":"48162","00021e3e":"48167","9d8965aa":"48188",bef59fc9:"48214",e672d61a:"48285",e2b4c006:"48295","189e3ac3":"48304",f6884a75:"48310",d8c836b4:"48355","959b45a2":"48360","712c16a5":"48361","2896ce7a":"48532","0a5f31be":"48544",c1506482:"48592","7a556257":"48613",ef4f3413:"48661","5db19d16":"48673",d009d47f:"48733",e054d919:"48768",b71be52f:"48793","79392e36":"48803",d0b9fc21:"48842",ce18dbde:"48883","9d9f8394":"49013",c68848f4:"49046",be94b749:"49055","97d17d75":"49071","3455c0d5":"49112","0ac5df82":"49174","7795e379":"49199",f2a06fea:"49226","6111bc4c":"49280",ad0a2b75:"49305","2417a912":"49350","8f1510f6":"49388","2019135a":"49461","49d63cc6":"49521","0d819826":"49569",e01bb6c7:"49579","8ec8c5c5":"49583","2cbb324b":"49587",b0ba51ed:"49655","2391d372":"49676",acf72eb8:"49783","5bd6f6db":"49786","3fc6d2b7":"49880","1d5cbb7b":"49991","40c88f7a":"50012","837c6843":"50095","756c7559":"50176","18dddf22":"50178",f2790563:"50183",ea981684:"50259",ec6a0daf:"50261",fc6a53b6:"50293","09a0b8d6":"50361",c59a4667:"50414",a33de290:"50428",eaebafad:"50446",a4e24d6c:"50472","313c653e":"50486","129c4e7a":"50507",c112d1b7:"50570","3fd21eb6":"50584",e964b083:"50642","75be343d":"50700","0914ee26":"50703","0d98f85b":"50747",a339c33e:"50769",b365f6bc:"50800","4251704e":"50819",aa04bdb6:"50827","8aa0dce2":"50839",e7b82dc0:"50934","4dd6f8d8":"51047","6bd6fe03":"51049",e1758f93:"51060","12aaf145":"51090","3304ac90":"51102",d366555a:"51116","5e85b509":"51170",c39359c5:"51175","90d97cfa":"51184",a06af3ed:"51229","85a6e3f4":"51237","0a8d92af":"51378",fe2deb8c:"51412","7a04b795":"51420","14515c80":"51438","65eb0274":"51494","45b02367":"51548","8447ad38":"51578","4c2a4e19":"51584",f762fff5:"51589",ba5671ab:"51636","6537c712":"51649","65d842b9":"51665","612cc46d":"51694","89bdbd96":"51711",b50c8022:"51755","9199f8bd":"51763","1bd7f091":"51769","91a39dd0":"51814","44e51e65":"51828","5c238f81":"51837","48b6c251":"51878",b5fb4df8:"51890","1d79e793":"52036","7f8ebea7":"52042",edea4b8a:"52093","95cc61bd":"52102","3498be82":"52132",d0ed28b7:"52135","81d19844":"52176","508f03f4":"52207","1562cf35":"52218",fadc68df:"52235","8ec5e058":"52250","3e088827":"52266","6818c2e9":"52267",be224150:"52282","8e1e17e5":"52286","0dd8c0ac":"52342",ab586159:"52408","2dc793da":"52450","3016c86b":"52462","8c67a0ff":"52466",cda1618e:"52478",ca57223f:"52488",f2da277e:"52563","7199ad43":"52569",a9417ee3:"52572","27f2e2a4":"52612",c4f5d8e4:"52634","2124517a":"52680","9e4087bc":"52711",c8f57d29:"52731","85afc7f5":"52770","25eae11f":"52785",ce319efa:"52826","9e1bed9d":"52840",fa2ae802:"52882","69fcecaa":"52895","4d4179b3":"52911","82c60778":"52912","4adafdbf":"52913","2f32a261":"53074","5de85215":"53132",aaf3ff15:"53170","71514a42":"53178",d670063b:"53201","79ab3378":"53237",fa713049:"53256","76a0389c":"53266",f8fe23f1:"53288","5440b68b":"53314",e2bce80a:"53449","63436e22":"53458",a45dd8f5:"53492","92f1a745":"53540","5a2d67ad":"53629",f6ef9721:"53633","6c9978fa":"53646","52ed38a1":"53670","77612fce":"53742","3a447539":"53771",e352cccc:"53786","7c8deb08":"53832",e623e42b:"53860","2a5e97be":"53861","94dd9534":"53871",e9a95c5e:"53881",a4da17e3:"53901",a8933372:"53925",d5f056f5:"53982",f72b3fe4:"53990","8c317219":"54097","93b4d825":"54106",f97d7d88:"54200","3a55b748":"54240","5ba9c4b5":"54246",e5c0ea66:"54270",f020ef51:"54272",b7bc328b:"54274","71e2c4b4":"54277","25aa47d2":"54399",c87c5e1b:"54415","0c3d0366":"54462",fb9e14c7:"54500","66193a96":"54540","633af835":"54568",d4522125:"54730",ff4fb41e:"54749","0f7d8652":"54759",a2de6851:"54851",a872c629:"54879",c46bba44:"54881","55f8cc28":"54927","0a893fdf":"54950","98bd463b":"54951","5bcffa9a":"54992","37d125cb":"55004","70d55848":"55100","152d0582":"55144",c1cb0a0b:"55167",aaa8a12d:"55206",c962ae4a:"55276",facc4cc2:"55302",ee44120f:"55342","2027fd18":"55401","342d2e6b":"55483",f7fa3381:"55526",ab685cdb:"55534","29cf75d4":"55559",c8861997:"55566","3a9c66ce":"55579",d5caed5f:"55616",a7511015:"55664",bbaa8144:"55668","069b26b8":"55754",d425e9d6:"55780","94b063ba":"55836","2cbaf53f":"55929","41adf405":"55942","194c8540":"55955",e1a6f6ca:"56028","69ef23f7":"56037",cde3e47b:"56062",c96d4604:"56087","096bca72":"56097",a7b4d0d7:"56122","602e2465":"56123","887c7539":"56133","5160d3b0":"56190",e028a908:"56256","75a12619":"56259","575869e5":"56297",fec58251:"56384",cbb5064a:"56408","858f1266":"56493","86a45dc2":"56510",c9c31477:"56560","609f9177":"56607","06eed450":"56648",beb32294:"56662","8dd3eb38":"56688","69e6c429":"56723","9a502a1c":"56725","2d57b909":"56767","6664c24a":"56779",eec5649b:"56826",ebc13825:"56916","1dbfdc18":"56948","39ed0ae4":"56974","97b1d23e":"57054","4d68fc5d":"57119","037241c6":"57141",e15f47bb:"57159","034f0308":"57165","2575da36":"57178",a2f498c0:"57182","8ee4a7d7":"57222",e3b05f38:"57245","779bba71":"57251","5e727800":"57283","55b89dea":"57302","3ed58e4a":"57326","841b525c":"57359","79cc2eba":"57361","533bed85":"57426","0b7a8a63":"57529",b20f9cb2:"57581","2b2d1be1":"57594",dd48b375:"57614","0ed7fb46":"57647","658997e4":"57730",c07b5739:"57749","73d3ccec":"57760","27b8ef72":"57762",f1691dde:"57885",e2070fcf:"58000","31fce735":"58002","2655f170":"58079",fa4b529c:"58111",a2171a4d:"58144","400deb23":"58255","078daaf7":"58259",b296abb0:"58301",cbf83637:"58351",a5e62092:"58359","15ea2a5f":"58393",ec854811:"58428","2a882da6":"58479","36d29ed8":"58484","0ba9210d":"58492",f0e5f3ed:"58581","6bd22ece":"58594",a1e59af5:"58666",a36e07dd:"58667","7fde9a4c":"58704","6b6aadc5":"58707","9ffdcfdf":"58721",b7d1b016:"58780",bec552c1:"58857","030dfd2b":"58874","52e97b50":"58880","7dd8c933":"59004","39fca8ac":"59005",a169aa70:"59009",e84457bb:"59020","11b61292":"59053",c43c6f4f:"59070",c762272b:"59104","1a20bc57":"59114","17ffd4ff":"59123","97db775f":"59262",e2b886c9:"59326","156bd387":"59349","55d26943":"59475","150f4998":"59493",b2555332:"59521",c5c8b091:"59614","70cca634":"59680",c9b0e9c8:"59702","9873f053":"59732",ab90a7b7:"59740","28553a2f":"59779","0e7c02dc":"59781","03f08ad1":"59866",b37c8625:"59873",b62e8859:"59874","888f35af":"59940","03137a3d":"59981",cc3a93a6:"60020","66eb519d":"60119","152819f0":"60167",e04f784f:"60185",a52bfe45:"60187","5c430979":"60193","5c417b7f":"60198","5c5dd48c":"60203","3fa5d64e":"60257","3d3aadb0":"60267","93d54c10":"60294","42d452f1":"60315",f83acba0:"60445","821320eb":"60510","492bd0ed":"60565","3523854b":"60613","7a82ef89":"60625","244c7b0a":"60676",d163928b:"60696","73e80d5d":"60725",e9d18558:"60762","0614dce3":"60765",c316a3d7:"60819","2df3fdca":"60830",d2436a2b:"60837",c9d25191:"60861","709e1a02":"60884","00b58b18":"60901",fe53d354:"60908","3e85731b":"60953","4d18bf1b":"60965",b1506a50:"61027","127f988d":"61114","4521c19b":"61228",a7456010:"61235","84536aab":"61265","31eb78c6":"61300",f12e5474:"61345",c90911b0:"61378",a1e3441b:"61416",b0f00280:"61481","6b206066":"61559","55b57e06":"61584","35190cab":"61627","495a3878":"61652",faa24029:"61702","4f3239be":"61721",ad2a3b86:"61734",bc80aebf:"61740",aa1329f2:"61786","14da6f38":"61844","124bf435":"61886","820ef6f8":"61950","9df3c51e":"61959",a6972a3c:"61991","5cc68a8f":"62015",f1b1ae9c:"62037","2638eb20":"62061",d2289dcb:"62111",cf1cd5da:"62120","1a4e3797":"62138","8d364c04":"62147","5955b5ee":"62149","22c027ab":"62162","0e7e04d8":"62182","1e38d1de":"62188","9b695413":"62254",cb69dde4:"62264","8041f857":"62400","856bc38c":"62431","3e9695b6":"62456",f448ea15:"62469","564f4b07":"62501",cca423c7:"62549",abfb0eb9:"62568","61b91652":"62635","0f378f0f":"62639","58728d91":"62641",fcaa2a90:"62672",d9175e87:"62691","345b88a3":"62703",f9a8c23e:"62712","1769198d":"62714","4e78ea4f":"62827",ae5c2262:"62845","07853d90":"62867",f1096413:"62875",c03ef713:"62927","7a0d5539":"62953",e5c6728d:"62957",baa95a18:"62975",b0635fce:"63005","754efc6e":"63064",d4bcf69a:"63124","93f4c2fc":"63151","29fd9202":"63181","34c77011":"63228",df8bdaca:"63262","2f1bf685":"63291","259b7e50":"63298","7dd3b2a7":"63317","60b75cbd":"63330","44bdb747":"63367","5f87f962":"63382","9780a9d6":"63387","04ea9cef":"63420","465af2b7":"63430",f30739cc:"63455",a3041c66:"63515","58f053e0":"63521","1fd2ca4a":"63553",b9795b3d:"63576","43a56d47":"63627","47298c37":"63646",f82e3d74:"63682",eadd254f:"63700","81a5dcd4":"63739",c2750739:"63779",a4974adf:"63795","95e06d9c":"63836","57e245ce":"63907","8bbbdfbb":"63951","2278c359":"63966",b81d6cb2:"63975",c47fd21a:"64045",e34c241a:"64099",f4f77456:"64184","88ed6425":"64209","192402a6":"64264","48a72e90":"64275","7c0dabe4":"64351",b65b26fc:"64391",dfa86ee4:"64442",e4307155:"64447",e9f47cd4:"64465","9c8cec5f":"64473","5d817499":"64502",dae07133:"64527","3756d209":"64535",a2a8ce40:"64577","9c273b44":"64597",e52d80fa:"64623","6ae279e9":"64659","46693c0b":"64670","1d00ec5b":"64737","2da5f59f":"64749","9a8a6b3c":"64838",e80f330c:"64899",a3fde46e:"65013",f08c8322:"65017",f04b050d:"65094",c9bcea67:"65133",d0cf31b7:"65193","2a11e6a7":"65223","0530d392":"65233",ba47c7e8:"65246","517960c0":"65350","8db42357":"65356","070df7e5":"65359",a21d3492:"65473","26115f23":"65497","8008d958":"65569","5d7f3e2f":"65611",e1ba57a0:"65625","26cbee69":"65629","75ba6401":"65647",c7850e27:"65648","329c66a5":"65658",e29a1e9b:"65727",d1be0dfb:"65745","02ff5d42":"65756","7d5633f0":"65759",fbd3de42:"65835","38147e44":"65897","587a0b3e":"65918","567cfed1":"65926","1f391b9e":"66061",fada9729:"66062",ceca85fb:"66097","331ad65a":"66135","136587f9":"66140",cf5645b0:"66162","0b7cbed9":"66169","2b53c3fa":"66210",cc9fb6c4:"66237",eee168db:"66327",dcbc8483:"66338","8fb804ed":"66372","9019bc0f":"66381",c0d2ab5d:"66394","6f47e633":"66415",cb8920e1:"66461","85e49a93":"66499",b01f49d5:"66504","79d5f27a":"66510",c6ec7d6a:"66518","83037ff5":"66524",d0550e1e:"66538",ecb74126:"66559",b0e34666:"66575","9fce9e91":"66602","5389895e":"66753",e381e7b7:"66759","62e9fea7":"66822",e1fde1ef:"66832","58b9aab4":"66867","47f8f95a":"66903","6e955522":"66912","6faab85c":"66947",ba8ea34b:"66995","1b143eab":"67044",c1050649:"67061",a7bd4aaa:"67098","27c39d7f":"67100","06073810":"67104","0d86fe8d":"67114","11235f31":"67149","9736a6b2":"67189","2d4f05ca":"67229","60cef16d":"67252","01ed4ae3":"67376",b6d7433d:"67438","814f3328":"67472","3477cc36":"67489","77139df7":"67507",d89aa357:"67545",a1a96ebc:"67581","8775fa1c":"67587",eedfeff5:"67649","47766edd":"67668",cace8592:"67683","8fb89e11":"67704","4d59e059":"67720","2c18bef5":"67729",b57b3f07:"67755","39b26ead":"67775","0a0dd03a":"67914","8aec9a60":"67954","863f8530":"68038",c1c9577e:"68043","0924bdf1":"68045",e981d05e:"68094",a148d810:"68210","3b8a31a8":"68246",e48e4dc9:"68249","89c88bb6":"68253","8ea5f648":"68304","9d05a91f":"68334","975aa772":"68434",bb6faf52:"68467",f553a3a8:"68515","9a8c2230":"68551",d81421b8:"68569",f645a9e6:"68665","0b87289b":"68685",dbf13a93:"68710","719a7d9b":"68765","14033dfb":"68770","3ac10f9a":"68839",bb1a3e9c:"68861","08299697":"68865","62cdbb35":"68910","5aa406a5":"68930",f9d38c6e:"68955",b60b94c8:"68991",ef13094f:"69017","8a752d6d":"69030","8df5dc78":"69038","4a55387a":"69129",fc384521:"69169",f0589cfc:"69199",de9d53d2:"69347","4d2a5110":"69365","5a360e6e":"69384","7ffd9a5f":"69457",fc15228a:"69501","8a4df687":"69517","4d573718":"69570",a3713279:"69588","05c96a0d":"69633","9c10cdcf":"69638","36e398ec":"69671",b1e8d27b:"69770","787caf51":"69787","615fd764":"69844",b58f381c:"69875","8e981336":"69982","0860d759":"70008",c8d1e051:"70022","73a7a1a2":"70058",b318f3b6:"70100","9f3d620b":"70117",f68988f2:"70126",dae6ce88:"70127","2f2d1edf":"70133",e04f6385:"70174",cb543535:"70179","529aff45":"70209","8367b76d":"70210",e76e0aa8:"70234","5e1c183f":"70253",aefdabf4:"70268",d2786aa3:"70399","0bbbad22":"70418",b71c2d32:"70452","8c596cb7":"70502","0bd456dc":"70551",ac5d28bd:"70560","6d5145e1":"70612",a9f446ca:"70645","0920b03b":"70649",dcc774d2:"70669","70ca6744":"70687","52be8247":"70696",cb488bcc:"70712","07338b13":"70728",ecba2f16:"70749",f3e1c72c:"70755",ac00e857:"70784",fda5ba18:"70791",c1dff2d3:"70794",c3c05d38:"70814","1d89db06":"70825","23421dc8":"70826","6763bf32":"70828",e34cf2a2:"70853","4f594038":"70870",d1fa94a6:"70947","87d75f25":"70953","16029e49":"70963","05f6486e":"71021","5e0cf2ca":"71026","9b8a5dc6":"71051","48c07d39":"71111","92f96583":"71117",af7ce5e3:"71162","131dae92":"71227","40d0141e":"71374","7380ddcc":"71407",ff761994:"71473",f89e6cd1:"71531","83c3bea7":"71565",d7e2cd1f:"71585",ed47441b:"71590",a29b4047:"71612","7ec29cb2":"71656","4a74d13d":"71715",e3aab494:"71784",d2a882d8:"71792","09138901":"71858",acec9ba2:"71863","466f5a64":"71877",f783f7a9:"71910","835d6f03":"71964","31b399ad":"72011",f0dc2560:"72043","95f36a1a":"72053","3d095b6b":"72122","8cec3a3f":"72135",cf8b2bc1:"72246","4941c7ed":"72322",f23d42f7:"72353","4419dbb7":"72358","27d3d4e0":"72446",ac84c584:"72527",bd203bf3:"72573","3336b76f":"72599",c8768069:"72634","049fd4ea":"72668",f81aef8e:"72675",eb3eed60:"72707","69f425f4":"72759","2d34e835":"72835","43b52908":"72880","491d56e0":"72907","37eca2aa":"72914",b724edf8:"72925",fca979f9:"73019",b8385eea:"73084",da144c90:"73193","33e8889e":"73264","7bd25ebf":"73276",c7f860be:"73290","8b1b48fb":"73316","97d2cbab":"73331","21838a86":"73376","681664ad":"73397","092425ad":"73472","4c26df07":"73493","5ebbf59b":"73539","73d617e8":"73540",ac6289fa:"73569","01689e9b":"73607","622b7911":"73609",edefc60b:"73624","35be90fa":"73684",c6ea0115:"73693",eb08ed44:"73741",d8415e6f:"73756","5cf43a2c":"73800","569e0588":"73841","4d015a5e":"73846","06dd3057":"73888",d721a748:"73891",d502d9c9:"73905",f4731b9a:"73940","14841d7a":"74019","9d891c91":"74027","2e6d9cc0":"74039",b7aeb8c2:"74046",ea5e46ff:"74069",da7bf323:"74121","393be207":"74134","2fd7ee6b":"74282",acf7953e:"74312","57eb6545":"74388",bc42f283:"74403","103c2fe7":"74418","3d4e54c7":"74490","111d9467":"74524","84939cad":"74588","033d3629":"74657",e6ce3bd9:"74684",a5ac74f6:"74792",ec7a5af3:"74880","43c329f5":"74902",ad848ffa:"74945","9b3f2ab9":"74961",eb5c136f:"74994","0c90e710":"75036","1c4d0413":"75043","13d28288":"75051","8df3f976":"75062",dd620de6:"75078",e2585025:"75105","9f49c871":"75106","3126d1b1":"75173","6dab9497":"75267","07deb48b":"75309","84aa8d64":"75359","847c1b9e":"75378",ceaa6e69:"75405","97c492a6":"75450","605c3775":"75459","2aa42d18":"75666","68f2aae3":"75695","1ad07866":"75696",d2250181:"75718","691ef278":"75735","217c03e5":"75759","724a4edc":"75775",ad132b09:"75805",a3541f3b:"75866",f18854fb:"75911",e01c8f09:"75919","03240ae1":"76041","20a6876f":"76163","1778a2f7":"76166",d9605822:"76215","6a3d5afb":"76235",cc7760fb:"76257","075f4b70":"76262",cb46984a:"76264","5d06256e":"76274","0d71ae17":"76285","1ed4b1ad":"76289","9267add8":"76368","594c10ba":"76530",eb0d63b4:"76641",b6f16885:"76683",a50107bb:"76719","9a68cfff":"76748",d5a221f8:"76850","2b00a752":"76854",e364f7ff:"76871",a91e6a0a:"76873",a01a15fc:"77077",bf048e24:"77088","61ee4158":"77096","62825cc3":"77123","30ae1693":"77145",ce019e24:"77174",eaf93d9c:"77183",a95aede3:"77200",fe2389d2:"77222","1aef3c3b":"77261","109d395b":"77298",c4acdd50:"77369",d0c84d34:"77391","0235a442":"77596","5ef28561":"77679","60381fc6":"77698","3a31d53f":"77749","8c3793bd":"77764",d9e41302:"77787","62e7b5b0":"77794",a52788ff:"77920","33ab05f6":"78001",c9599a77:"78037","30175d3c":"78055","90caa6a1":"78087","8df10e0f":"78159","5455ca0e":"78199","1682c6e0":"78312",c1b680b7:"78354",b1a2ea9a:"78421","28ebe10c":"78497",b7e5c092:"78512",ff4a022d:"78522","4adc4126":"78544","95365ba3":"78583","4f1d1584":"78632",b80df7ca:"78704","667a9f57":"78758","6000d05b":"78764",a7f76e11:"78787","04c3dd09":"78820","4f0afd2f":"78858","499efff2":"78876","335badf9":"78880",c8939218:"78912",f73fc2b7:"78919","73bc484b":"78926",ccd7c88f:"78985","0ee8b775":"78988","803df44c":"79039",a94703ab:"79048",cdc43a7d:"79150",c884c38e:"79157",cc426672:"79180",a2030166:"79416","120bc0be":"79474","8b7dab17":"79485","368f9c22":"79506","09b9b020":"79521","1bfbaba8":"79656","8be24c4d":"79739",fc26a5d5:"79841","619263e9":"79866",dd6ea91b:"79987","5e4ec6cd":"80018","93c339e4":"80046","587df15f":"80091","7349f9da":"80102",d38a6f54:"80198","0cc20099":"80205","45cd5638":"80259","8aabb1ce":"80260","8657657d":"80299","3c89ed52":"80310","2fd10e89":"80362","6e954188":"80375","81c96f91":"80408",abf14b6e:"80424",e1a35040:"80442","992bda30":"80457","318980ec":"80458","7080aeb9":"80491","27321f21":"80531","5cb06d06":"80557","1f97a7ff":"80598","9abd9d66":"80632","80f24550":"80649","6d70fa33":"80677","641786b9":"80744","84ac5929":"80750",fd440a88:"80772",cfdba83a:"80781","442da8c1":"80794","413bf93f":"80841",b6aee096:"80863",a7f213e3:"80947",c141421f:"80957","820b9813":"81012",f74d3bfd:"81019",c88d5fcd:"81050","104fa664":"81085",aee88ae1:"81125",f98df6cf:"81175",ddf9f187:"81188","347b93da":"81212","54e1e905":"81285","2195a7cc":"81300","12363bc4":"81319",a20f65a6:"81346",e6c5d4a7:"81390","7af33a93":"81410",ae0ec9f4:"81482","654827c2":"81530","13685ceb":"81544","15a3ce9a":"81583",ffe2a86d:"81639",a74d38f6:"81666","0d9e9c41":"81692",c75c3ca5:"81737","932d66ca":"81761",fddd2104:"81770",d3571192:"81791",b90928c1:"81795","6066240e":"81802",b3dc9cb4:"81891","29470dda":"81902",acecf23e:"81903","4b93078e":"81943","2a59c009":"81982",f9ecf61e:"81990","4a7139ae":"82012","755e7c7e":"82019",cb7dc7d1:"82242",d2c9f0b8:"82261","226eff01":"82297",b87f99cc:"82354","7345c721":"82362","5d210bb4":"82431","24f4e7d7":"82456","4393156b":"82486","0add85e5":"82553","907fc94b":"82581","4cd82028":"82588",dd0ce7a8:"82663","72e3e702":"82722","4ccba4b8":"82754",dd09cc7b:"82847","12cecbf6":"82905","2a7d5452":"82939","732a7c43":"82987",dac8839d:"83027","42174d87":"83078","686a74aa":"83089","21cd7edb":"83103","79ca5236":"83109","6060cc06":"83111",deacbd9b:"83121","10757cc8":"83127",e5cd0e7f:"83182","95126e44":"83203","828aa184":"83213",ccc49370:"83249","5c26bf07":"83260","2b6eabf2":"83273","0ba08ea0":"83316",d0f169c8:"83378",e6ccb422:"83470",c85c7bc6:"83574","19560f91":"83608","9e37e644":"83670",a8c902bd:"83700",e6fc8a9b:"83779",b6cd0ba6:"83797",f5dd5915:"83824",cc517726:"83841",c48426e8:"83849","345eb925":"83928","757c6784":"83932",e67a83cf:"83954","2bf0d1a9":"83958",e200187b:"83970","0e384e19":"83976",c17d6939:"84013","511d3e84":"84070",ed83b9b9:"84107",d2ed2b82:"84152","17d9fbbc":"84227",ddd99a22:"84232","901160ed":"84255","2ffd8514":"84259","1132d5f6":"84319","0b82d45d":"84332","8180940b":"84527","2e72ea50":"84530","2760fb69":"84563",d9a7203a:"84572","21f4f354":"84760",b738b15f:"84773","6875c492":"84813","6bd2e919":"84869",d9d878d8:"84963",ae42df76:"85014",ec2ed6e2:"85095","7c86d680":"85097",ab380486:"85105","0cf3b0b8":"85205","8beefa16":"85209","78c968cb":"85243",ffec2b37:"85257","8f4a15da":"85279",c9ff545e:"85322","813f53ba":"85358","2f6614a5":"85442",de5aeda1:"85453","2c88985c":"85537","8f9bc413":"85581","24e645af":"85588","0e8c3a89":"85736",a4a3eadf:"85796","3e226f70":"85806",d40aab8b:"85817",ddb0149a:"85896","41639dad":"85899","716f14d9":"85917","5268c144":"85960","5733876a":"85973","83b6afd8":"85997",ae271c01:"86053","199e6876":"86074","677325f0":"86121",fb03d32d:"86129","1f8198a4":"86167","95ee2976":"86215",abe1b280:"86216","7a2c4c43":"86246",d630316d:"86354","669193a7":"86417",d5f51703:"86434","2a1b9f9a":"86478","3a311bb0":"86480",f4e1ab69:"86501",b76e61dd:"86586","0e0dfb6a":"86597","49bbb99a":"86623","7ff6577b":"86659","927f97a3":"86669","58421c87":"86685",b0445ba0:"86807","5626901c":"86829","57ec2762":"86846",c942990c:"86856","5687c7cb":"86888","319b6f13":"86890","869411c7":"86914","7a06eb83":"86939",d2a66e94:"86992",b62a4e5f:"87004","40df2769":"87087","4eec7d8d":"87159","4462c0cc":"87184","767cce31":"87186",bad31f72:"87205","9c71777e":"87217","33ac18ed":"87269",c884ad6a:"87282",c4f2c374:"87377",cd08112a:"87407","0b425eb3":"87432","33bf4676":"87456",b8f2cc13:"87459",baf4021b:"87473",d5762a9f:"87483","643da015":"87485",e29c32f4:"87547",d2b2e5aa:"87589","7c29a585":"87614","137db567":"87688","3f1a4665":"87747","6f831635":"87781","4194f325":"87863","01f93a4b":"87875",f553a7ec:"87880",fc17862c:"87905","5b17955f":"87945",dc4ee1bb:"87971","85f9f5a6":"87995","72a68b3b":"87996",a5f4f54c:"88047",ca3d464c:"88111","2e05ee79":"88163",b8fd1a8c:"88222",ed990644:"88248","85e50ec2":"88298","0169a509":"88310",fbfb0bb0:"88327","3cf947bf":"88393","5ae50f21":"88394","078339bb":"88399",a13618bc:"88436","40e4b7d4":"88441",bf79ed6f:"88468","23a811a2":"88478",d3f32089:"88522","2283659c":"88547","5871fbee":"88622","6e8a38ea":"88626","6dc022b8":"88654","6be5dbf9":"88696",b330ac5c:"88775","71cde98c":"88815",d3065e4e:"88817",b0b71a2a:"88821","014c2cb9":"88847","508d741f":"88852","5edce1ad":"88855",d15e98db:"88865","568a0660":"88903","7b658d8e":"88986","2c9ce68e":"89018",fc93a2ea:"89093","24f450eb":"89097",adb96665:"89172","9f15c6e7":"89175","9b191135":"89210",d8e6b3db:"89289","324cb577":"89338",b919dab0:"89353",cb2f16be:"89362","39ac0d5b":"89402","9ed26de9":"89415",ce1709a8:"89437","978ca64f":"89455","145c1cc4":"89489","7ff4234e":"89554","2709984f":"89610",cde9af88:"89641","837dc7e4":"89696","084a80b5":"89801","9bfa1541":"89810","36994c47":"89858","65f8c8fd":"89873",e582f766:"89986",bfb3b697:"90066","7e60f0fb":"90071",dbe2cfea:"90110","1026ad00":"90115","8abd7194":"90135","9db841c0":"90214","7b773c92":"90257",a80ed580:"90280",b3abe364:"90317",c633f93f:"90378",ed2667fa:"90414","9947fe09":"90416",fe49dcfb:"90438","6157713d":"90513","32eb34e5":"90562","99144b30":"90593","2563c97b":"90598","73554d19":"90625",bfa9eb5d:"90627",fff1effb:"90717","48dc1693":"90749",e96ddf11:"90831","837010ac":"90842","0058b4c6":"90849",e03c6723:"90910","271ea2f9":"90950","70e8710e":"90972","466375b7":"91025","42003ca2":"91058","014625f4":"91109","3629c812":"91187",ece3ee5e:"91203",f657ed9f:"91257",af1525aa:"91305","0f84f33f":"91319",d1980a1b:"91333","88f66091":"91359",ae3304ee:"91405","284c57b4":"91428","7118d0f0":"91530","980d0b86":"91540","8614d32b":"91581","1fecd571":"91587",e33cf622:"91597","7907a033":"91621","6033c607":"91695","9964550a":"91724",df99aa82:"91787","41ca93cc":"91844",b7201a27:"91895","72fdaa88":"91916","8ce0215f":"91993","0ce689e1":"92018",e2c3948d:"92031","6342cf5e":"92105","4cce6e5a":"92125","56a0d755":"92196",e772c536:"92223",bef29d9e:"92241","85fe7d70":"92273","4a4c6752":"92329","4c434b32":"92410",d888d821:"92412","1be20fdf":"92475",b6eb7220:"92535",e53ee22f:"92544",efa442c3:"92653","7c1d6f5a":"92664","9dfaf941":"92673","3e082aad":"92955",adfd3bc1:"93057",b5ea0533:"93085",fed5be48:"93166","2ed2a721":"93206","47b26a6d":"93209",e55ca189:"93210",b12147a6:"93269",cf32db66:"93273",aba187b8:"93341","2357dc71":"93361","7a234181":"93362",f5e9d1c4:"93438",a9c52207:"93534",d9e43198:"93546","44c6f0c9":"93602",e1847ef7:"93636",f179621d:"93795","09bacf3b":"93849","0a07ac32":"93854",c3c082a1:"93884","45a5cd1f":"94000",ee71d619:"94065","8c0fb9c6":"94082",beceb38b:"94091",acddd5ca:"94106","6ee6a410":"94153",c6b4dc09:"94168",f0cc57e7:"94227","824e6f8c":"94251","00f9d3c8":"94314","8b44df1d":"94344","657bf45c":"94361","85c8b6c7":"94417","13feb7a8":"94538","8e7aaae8":"94555",a1e44e64:"94561","745d4b8c":"94568","0a02a2d0":"94625","30549b42":"94646","50774ec6":"94648","8520f5db":"94685","0c2c2d88":"94711","6dfe2e3e":"94713",c26e67a5:"94788","256701a3":"94796",a1282170:"94841","3e1f4a39":"94843",fc66001f:"94863","95ec5604":"94864",bb63d03e:"94910",e8b41ff0:"94925","98700a51":"94926","5fc8caff":"94980","8d65e26f":"94997","63c4b13e":"95020","4b69979c":"95058","10ac9a3e":"95078","818bb2cd":"95093",c0df5757:"95136",cdb727d9:"95159",f58814fb:"95183","1cb0fe52":"95231",b4030b00:"95240",ed211a79:"95245","9b4185c1":"95330","52910a8f":"95427",e1afbf8c:"95443","8a40ff6b":"95470","5917c028":"95542","9a8fdb53":"95605","188015be":"95641","4885c521":"95649","30d4e84f":"95659","89ce4ba3":"95768","8c79a60d":"95785",dbea5ca6:"95818","9e488927":"95909",accd2b0e:"95911",cbc2448c:"95912","5180e398":"96022",eb84cef2:"96086",a267572b:"96155","838df61f":"96225","380e17bf":"96239","638db053":"96268","82a7c68f":"96276","65d1ec04":"96325",ddd9290b:"96463","1b260ed9":"96470",d4083900:"96555","5b0025cd":"96652","8c42b153":"96693","4f748135":"96745",a98ffe6a:"96771","181eb1b5":"96778",c3c125b1:"96793",d527c17c:"96881","7ab01152":"97006",e9032a0d:"97033","763a8986":"97047","9331da7d":"97107","32bb5bcb":"97114","9d2e74da":"97123","20a30ea0":"97167","1b3061f3":"97189","7fa05810":"97223","365a7fd7":"97246","1a8a162f":"97253","205cdcf8":"97296",c2a473ad:"97342","76276d52":"97358","3bfbd79c":"97379","224c7e3e":"97420",ba47c136:"97424",b93a682d:"97476","3a6c6e5b":"97536","6ccd70b6":"97554","3d08e0be":"97562","8e81e342":"97585",facb0528:"97664","1dba1ecf":"97670",b58e88de:"97732","8fa9d00b":"97737","41b1becf":"97753","7d2fd416":"97763","97a057b3":"97821",e5562b89:"97829","47edbc34":"97863","26ec8ae2":"97887",ab8b4014:"97895",bf7dfc7c:"97901",ff509459:"97905","1e228808":"97913","350cc719":"98012","139d61ea":"98021","95f25ea6":"98030",fbb50b04:"98058","75dbe45b":"98059",b18455bc:"98087","48f67822":"98094",a2219ebb:"98139",fc46979d:"98147",c8e5bf38:"98165","3539b92c":"98237",f62bafa2:"98271","756094a5":"98308","9e297776":"98311","1c3a958e":"98392",b943b6ea:"98425","94dcd3de":"98473",ea6f04d4:"98477","6065ad54":"98534","99e64d04":"98700","19bfb940":"98750",af6e9729:"98758","3314c9d3":"98761",a48978f5:"98779",bf667688:"98804","3e8495a1":"98867",b4e94af8:"98908",ab37b32f:"98925","5de60d34":"98935",c9ba7c72:"98941",ef00b8a0:"98947",e8178b53:"98982","82a5d7f7":"99001","1c27379d":"99095","210305d4":"99146","8e563661":"99151","3047f3e7":"99252",c14fcab2:"99254","8944e56a":"99299",bdecca0c:"99306",b3e98411:"99330",df09200e:"99335","5ffb8f23":"99437",c0684476:"99451","2263a65b":"99502","76af5d51":"99504","16e939a3":"99528","8c2cbe8e":"99531",c49eb59b:"99534","25c655c3":"99543","9cc8ffa2":"99611",bfbfac54:"99644","25e15e7c":"99678","5a4dd75d":"99701","5fb14ca8":"99705",dc330c71:"99709","6e34d00c":"99710",f4a839f6:"99746",a32c0324:"99747","8e34a11f":"99872","2f7d15c4":"99905","4b2e4980":"99987"}[e]||e,r.p+r.u(e)},(()=>{var e={45354:0,71869:0};r.f.j=(a,f)=>{var c=r.o(e,a)?e[a]:void 0;if(0!==c)if(c)f.push(c[2]);else if(/^(45354|71869)$/.test(a))e[a]=0;else{var b=new Promise(((f,b)=>c=e[a]=[f,b]));f.push(c[2]=b);var d=r.p+r.u(a),t=new Error;r.l(d,(f=>{if(r.o(e,a)&&(0!==(c=e[a])&&(e[a]=void 0),c)){var b=f&&("load"===f.type?"missing":f.type),d=f&&f.target&&f.target.src;t.message="Loading chunk "+a+" failed.\n("+b+": "+d+")",t.name="ChunkLoadError",t.type=b,t.request=d,c[1](t)}}),"chunk-"+a,a)}},r.O.j=a=>0===e[a];var a=(a,f)=>{var c,b,d=f[0],t=f[1],o=f[2],n=0;if(d.some((a=>0!==e[a]))){for(c in t)r.o(t,c)&&(r.m[c]=t[c]);if(o)var i=o(r)}for(a&&a(f);n
-
+
diff --git a/content/blog/2016/12/30/strata-talk-2017/index.html b/content/blog/2016/12/30/strata-talk-2017/index.html
index 4e728cef62205..87e08e42e4180 100644
--- a/content/blog/2016/12/30/strata-talk-2017/index.html
+++ b/content/blog/2016/12/30/strata-talk-2017/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2017/03/12/Hoodie-Uber-Engineerings-Incremental-Processing-Framework-on-Hadoop/index.html b/content/blog/2017/03/12/Hoodie-Uber-Engineerings-Incremental-Processing-Framework-on-Hadoop/index.html
index 0d6e16bfc704f..0f777f58b7910 100644
--- a/content/blog/2017/03/12/Hoodie-Uber-Engineerings-Incremental-Processing-Framework-on-Hadoop/index.html
+++ b/content/blog/2017/03/12/Hoodie-Uber-Engineerings-Incremental-Processing-Framework-on-Hadoop/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2019/01/18/asf-incubation/index.html b/content/blog/2019/01/18/asf-incubation/index.html
index 8d491b1545c9e..f795499f2f15e 100644
--- a/content/blog/2019/01/18/asf-incubation/index.html
+++ b/content/blog/2019/01/18/asf-incubation/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2019/03/07/batch-vs-incremental/index.html b/content/blog/2019/03/07/batch-vs-incremental/index.html
index a6fb3435a0c56..4f7919622672e 100644
--- a/content/blog/2019/03/07/batch-vs-incremental/index.html
+++ b/content/blog/2019/03/07/batch-vs-incremental/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2019/05/14/registering-dataset-to-hive/index.html b/content/blog/2019/05/14/registering-dataset-to-hive/index.html
index 3b75130835770..0696462291581 100644
--- a/content/blog/2019/05/14/registering-dataset-to-hive/index.html
+++ b/content/blog/2019/05/14/registering-dataset-to-hive/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2019/09/09/ingesting-database-changes/index.html b/content/blog/2019/09/09/ingesting-database-changes/index.html
index cdc9b37b67530..8e5f00452bb61 100644
--- a/content/blog/2019/09/09/ingesting-database-changes/index.html
+++ b/content/blog/2019/09/09/ingesting-database-changes/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2019/10/22/Hudi-On-Hops/index.html b/content/blog/2019/10/22/Hudi-On-Hops/index.html
index ea5bf129bf60c..e66dc2585299a 100644
--- a/content/blog/2019/10/22/Hudi-On-Hops/index.html
+++ b/content/blog/2019/10/22/Hudi-On-Hops/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2019/11/15/New-Insert-Update-Delete-Data-on-S3-with-Amazon-EMR-and-Apache-Hudi/index.html b/content/blog/2019/11/15/New-Insert-Update-Delete-Data-on-S3-with-Amazon-EMR-and-Apache-Hudi/index.html
index 61c4d8af532b3..9bea10a66b92d 100644
--- a/content/blog/2019/11/15/New-Insert-Update-Delete-Data-on-S3-with-Amazon-EMR-and-Apache-Hudi/index.html
+++ b/content/blog/2019/11/15/New-Insert-Update-Delete-Data-on-S3-with-Amazon-EMR-and-Apache-Hudi/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2020/01/15/delete-support-in-hudi/index.html b/content/blog/2020/01/15/delete-support-in-hudi/index.html
index 6de204458748e..990e0a795af49 100644
--- a/content/blog/2020/01/15/delete-support-in-hudi/index.html
+++ b/content/blog/2020/01/15/delete-support-in-hudi/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2020/01/20/change-capture-using-aws/index.html b/content/blog/2020/01/20/change-capture-using-aws/index.html
index 89cd0da6a7d95..81999be8fb31f 100644
--- a/content/blog/2020/01/20/change-capture-using-aws/index.html
+++ b/content/blog/2020/01/20/change-capture-using-aws/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2020/03/22/exporting-hudi-datasets/index.html b/content/blog/2020/03/22/exporting-hudi-datasets/index.html
index 055c001af8fd7..080688f141bfc 100644
--- a/content/blog/2020/03/22/exporting-hudi-datasets/index.html
+++ b/content/blog/2020/03/22/exporting-hudi-datasets/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2020/04/27/apache-hudi-apache-zepplin/index.html b/content/blog/2020/04/27/apache-hudi-apache-zepplin/index.html
index 5bf402873dc42..c2842e1f2d733 100644
--- a/content/blog/2020/04/27/apache-hudi-apache-zepplin/index.html
+++ b/content/blog/2020/04/27/apache-hudi-apache-zepplin/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2020/05/28/monitoring-hudi-metrics-with-datadog/index.html b/content/blog/2020/05/28/monitoring-hudi-metrics-with-datadog/index.html
index 930bedc8a2be7..c7d8f50b145c0 100644
--- a/content/blog/2020/05/28/monitoring-hudi-metrics-with-datadog/index.html
+++ b/content/blog/2020/05/28/monitoring-hudi-metrics-with-datadog/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2020/06/04/The-Apache-Software-Foundation-Announces-Apache-Hudi-as-a-Top-Level-Project/index.html b/content/blog/2020/06/04/The-Apache-Software-Foundation-Announces-Apache-Hudi-as-a-Top-Level-Project/index.html
index f10a77098021e..b371c2bab32d1 100644
--- a/content/blog/2020/06/04/The-Apache-Software-Foundation-Announces-Apache-Hudi-as-a-Top-Level-Project/index.html
+++ b/content/blog/2020/06/04/The-Apache-Software-Foundation-Announces-Apache-Hudi-as-a-Top-Level-Project/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2020/06/09/Building-a-Large-scale-Transactional-Data-Lake-at-Uber-Using-Apache-Hudi/index.html b/content/blog/2020/06/09/Building-a-Large-scale-Transactional-Data-Lake-at-Uber-Using-Apache-Hudi/index.html
index 7248b9bfe0cb4..1da7f170fc3bf 100644
--- a/content/blog/2020/06/09/Building-a-Large-scale-Transactional-Data-Lake-at-Uber-Using-Apache-Hudi/index.html
+++ b/content/blog/2020/06/09/Building-a-Large-scale-Transactional-Data-Lake-at-Uber-Using-Apache-Hudi/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2020/06/16/Apache-Hudi-grows-cloud-data-lake-maturity/index.html b/content/blog/2020/06/16/Apache-Hudi-grows-cloud-data-lake-maturity/index.html
index f2977e4ece4cb..d318bb97d3e60 100644
--- a/content/blog/2020/06/16/Apache-Hudi-grows-cloud-data-lake-maturity/index.html
+++ b/content/blog/2020/06/16/Apache-Hudi-grows-cloud-data-lake-maturity/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2020/08/04/PrestoDB-and-Apache-Hudi/index.html b/content/blog/2020/08/04/PrestoDB-and-Apache-Hudi/index.html
index c58f2118e5322..66479ccc0bd65 100644
--- a/content/blog/2020/08/04/PrestoDB-and-Apache-Hudi/index.html
+++ b/content/blog/2020/08/04/PrestoDB-and-Apache-Hudi/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2020/08/18/hudi-incremental-processing-on-data-lakes/index.html b/content/blog/2020/08/18/hudi-incremental-processing-on-data-lakes/index.html
index 5d1cf2fa79a4a..615189c8eaa27 100644
--- a/content/blog/2020/08/18/hudi-incremental-processing-on-data-lakes/index.html
+++ b/content/blog/2020/08/18/hudi-incremental-processing-on-data-lakes/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2020/08/20/efficient-migration-of-large-parquet-tables/index.html b/content/blog/2020/08/20/efficient-migration-of-large-parquet-tables/index.html
index 597979bcbafac..e6c1a2fff2090 100644
--- a/content/blog/2020/08/20/efficient-migration-of-large-parquet-tables/index.html
+++ b/content/blog/2020/08/20/efficient-migration-of-large-parquet-tables/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2020/08/21/async-compaction-deployment-model/index.html b/content/blog/2020/08/21/async-compaction-deployment-model/index.html
index bd467339b944a..747a054220f70 100644
--- a/content/blog/2020/08/21/async-compaction-deployment-model/index.html
+++ b/content/blog/2020/08/21/async-compaction-deployment-model/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2020/08/22/ingest-multiple-tables-using-hudi/index.html b/content/blog/2020/08/22/ingest-multiple-tables-using-hudi/index.html
index c030271807256..be7c9da009717 100644
--- a/content/blog/2020/08/22/ingest-multiple-tables-using-hudi/index.html
+++ b/content/blog/2020/08/22/ingest-multiple-tables-using-hudi/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2020/10/06/cdc-solution-using-hudi-by-nclouds/index.html b/content/blog/2020/10/06/cdc-solution-using-hudi-by-nclouds/index.html
index c98920ca5763a..a0f1b9254137a 100644
--- a/content/blog/2020/10/06/cdc-solution-using-hudi-by-nclouds/index.html
+++ b/content/blog/2020/10/06/cdc-solution-using-hudi-by-nclouds/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2020/10/15/apache-hudi-meets-apache-flink/index.html b/content/blog/2020/10/15/apache-hudi-meets-apache-flink/index.html
index d4d32750ec9eb..d30af5afd635e 100644
--- a/content/blog/2020/10/15/apache-hudi-meets-apache-flink/index.html
+++ b/content/blog/2020/10/15/apache-hudi-meets-apache-flink/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2020/10/19/Origins-of-Data-Lake-at-Grofers/index.html b/content/blog/2020/10/19/Origins-of-Data-Lake-at-Grofers/index.html
index abe1b19acedc9..5890372f563a6 100644
--- a/content/blog/2020/10/19/Origins-of-Data-Lake-at-Grofers/index.html
+++ b/content/blog/2020/10/19/Origins-of-Data-Lake-at-Grofers/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2020/10/19/hudi-meets-aws-emr-and-aws-dms/index.html b/content/blog/2020/10/19/hudi-meets-aws-emr-and-aws-dms/index.html
index c2cf210054412..92baf34afdc8c 100644
--- a/content/blog/2020/10/19/hudi-meets-aws-emr-and-aws-dms/index.html
+++ b/content/blog/2020/10/19/hudi-meets-aws-emr-and-aws-dms/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2020/10/21/Architecting-Data-Lakes-for-the-Modern-Enterprise-at-Data-Summit-Connect-Fall-2020/index.html b/content/blog/2020/10/21/Architecting-Data-Lakes-for-the-Modern-Enterprise-at-Data-Summit-Connect-Fall-2020/index.html
index 32f827b0ec1a5..7f8675917a714 100644
--- a/content/blog/2020/10/21/Architecting-Data-Lakes-for-the-Modern-Enterprise-at-Data-Summit-Connect-Fall-2020/index.html
+++ b/content/blog/2020/10/21/Architecting-Data-Lakes-for-the-Modern-Enterprise-at-Data-Summit-Connect-Fall-2020/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2020/10/21/Data-Lake-Change-Capture-using-Apache-Hudi-and-Amazon-AMS-EMR/index.html b/content/blog/2020/10/21/Data-Lake-Change-Capture-using-Apache-Hudi-and-Amazon-AMS-EMR/index.html
index 417847d8cb193..1781b4383214a 100644
--- a/content/blog/2020/10/21/Data-Lake-Change-Capture-using-Apache-Hudi-and-Amazon-AMS-EMR/index.html
+++ b/content/blog/2020/10/21/Data-Lake-Change-Capture-using-Apache-Hudi-and-Amazon-AMS-EMR/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2020/11/11/hudi-indexing-mechanisms/index.html b/content/blog/2020/11/11/hudi-indexing-mechanisms/index.html
index 962feef28dd7a..535ff4e3358bd 100644
--- a/content/blog/2020/11/11/hudi-indexing-mechanisms/index.html
+++ b/content/blog/2020/11/11/hudi-indexing-mechanisms/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2020/11/29/Can-Big-Data-Solutions-Be-Affordable/index.html b/content/blog/2020/11/29/Can-Big-Data-Solutions-Be-Affordable/index.html
index 5dfd3a2f1fae5..2e95ae4c626b0 100644
--- a/content/blog/2020/11/29/Can-Big-Data-Solutions-Be-Affordable/index.html
+++ b/content/blog/2020/11/29/Can-Big-Data-Solutions-Be-Affordable/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2020/12/01/high-perf-data-lake-with-hudi-and-alluxio-t3go/index.html b/content/blog/2020/12/01/high-perf-data-lake-with-hudi-and-alluxio-t3go/index.html
index 0d2af074f7d16..b4460f68e82bf 100644
--- a/content/blog/2020/12/01/high-perf-data-lake-with-hudi-and-alluxio-t3go/index.html
+++ b/content/blog/2020/12/01/high-perf-data-lake-with-hudi-and-alluxio-t3go/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2021/01/27/hudi-clustering-intro/index.html b/content/blog/2021/01/27/hudi-clustering-intro/index.html
index 085e85d59fbb9..ea24940a52a4f 100644
--- a/content/blog/2021/01/27/hudi-clustering-intro/index.html
+++ b/content/blog/2021/01/27/hudi-clustering-intro/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2021/02/13/hudi-key-generators/index.html b/content/blog/2021/02/13/hudi-key-generators/index.html
index da7b98e2cd6bb..bc99d2e026def 100644
--- a/content/blog/2021/02/13/hudi-key-generators/index.html
+++ b/content/blog/2021/02/13/hudi-key-generators/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2021/02/24/Time-travel-operations-in-Hopsworks-Feature-Store/index.html b/content/blog/2021/02/24/Time-travel-operations-in-Hopsworks-Feature-Store/index.html
index 0e51e9ff528ef..62952e38cd92d 100644
--- a/content/blog/2021/02/24/Time-travel-operations-in-Hopsworks-Feature-Store/index.html
+++ b/content/blog/2021/02/24/Time-travel-operations-in-Hopsworks-Feature-Store/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2021/03/01/Data-Lakehouse-Building-the-Next-Generation-of-Data-Lakes-using-Apache-Hudi/index.html b/content/blog/2021/03/01/Data-Lakehouse-Building-the-Next-Generation-of-Data-Lakes-using-Apache-Hudi/index.html
index f38cacc98afbe..775381754afe7 100644
--- a/content/blog/2021/03/01/Data-Lakehouse-Building-the-Next-Generation-of-Data-Lakes-using-Apache-Hudi/index.html
+++ b/content/blog/2021/03/01/Data-Lakehouse-Building-the-Next-Generation-of-Data-Lakes-using-Apache-Hudi/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2021/03/01/hudi-file-sizing/index.html b/content/blog/2021/03/01/hudi-file-sizing/index.html
index f32ccbf399721..1b44b45ba71f5 100644
--- a/content/blog/2021/03/01/hudi-file-sizing/index.html
+++ b/content/blog/2021/03/01/hudi-file-sizing/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2021/03/04/Build-a-data-lake-using-amazon-kinesis-data-stream-for-amazon-dynamodb-and-apache-hudi/index.html b/content/blog/2021/03/04/Build-a-data-lake-using-amazon-kinesis-data-stream-for-amazon-dynamodb-and-apache-hudi/index.html
index 9a1f93738c36d..ad33eb31be3aa 100644
--- a/content/blog/2021/03/04/Build-a-data-lake-using-amazon-kinesis-data-stream-for-amazon-dynamodb-and-apache-hudi/index.html
+++ b/content/blog/2021/03/04/Build-a-data-lake-using-amazon-kinesis-data-stream-for-amazon-dynamodb-and-apache-hudi/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2021/03/11/New-features-from-Apache-hudi-in-Amazon-EMR/index.html b/content/blog/2021/03/11/New-features-from-Apache-hudi-in-Amazon-EMR/index.html
index d45de21a05667..430906ba21ae3 100644
--- a/content/blog/2021/03/11/New-features-from-Apache-hudi-in-Amazon-EMR/index.html
+++ b/content/blog/2021/03/11/New-features-from-Apache-hudi-in-Amazon-EMR/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2021/04/12/Build-Slowly-Changing-Dimensions-Type-2-SCD2-with-Apache-Spark-and-Apache-Hudi-on-Amazon-EMR/index.html b/content/blog/2021/04/12/Build-Slowly-Changing-Dimensions-Type-2-SCD2-with-Apache-Spark-and-Apache-Hudi-on-Amazon-EMR/index.html
index da79ce4a0b0d3..691f422f86a4f 100644
--- a/content/blog/2021/04/12/Build-Slowly-Changing-Dimensions-Type-2-SCD2-with-Apache-Spark-and-Apache-Hudi-on-Amazon-EMR/index.html
+++ b/content/blog/2021/04/12/Build-Slowly-Changing-Dimensions-Type-2-SCD2-with-Apache-Spark-and-Apache-Hudi-on-Amazon-EMR/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2021/05/12/Experts-primer-on-Apache-Hudi/index.html b/content/blog/2021/05/12/Experts-primer-on-Apache-Hudi/index.html
index 5889bc4129c3a..e5fffd993907e 100644
--- a/content/blog/2021/05/12/Experts-primer-on-Apache-Hudi/index.html
+++ b/content/blog/2021/05/12/Experts-primer-on-Apache-Hudi/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2021/06/04/Apache-Hudi-How-Uber-gets-data-a-ride-to-its-destination/index.html b/content/blog/2021/06/04/Apache-Hudi-How-Uber-gets-data-a-ride-to-its-destination/index.html
index bbfc0fca678d4..941a47405dd83 100644
--- a/content/blog/2021/06/04/Apache-Hudi-How-Uber-gets-data-a-ride-to-its-destination/index.html
+++ b/content/blog/2021/06/04/Apache-Hudi-How-Uber-gets-data-a-ride-to-its-destination/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2021/06/10/employing-right-configurations-for-hudi-cleaner/index.html b/content/blog/2021/06/10/employing-right-configurations-for-hudi-cleaner/index.html
index 1db9b63fc330a..72c13f627aa1e 100644
--- a/content/blog/2021/06/10/employing-right-configurations-for-hudi-cleaner/index.html
+++ b/content/blog/2021/06/10/employing-right-configurations-for-hudi-cleaner/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2021/07/16/Amazon-Athena-expands-Apache-Hudi-support/index.html b/content/blog/2021/07/16/Amazon-Athena-expands-Apache-Hudi-support/index.html
index e8e490ded021c..c210db7a0e3fb 100644
--- a/content/blog/2021/07/16/Amazon-Athena-expands-Apache-Hudi-support/index.html
+++ b/content/blog/2021/07/16/Amazon-Athena-expands-Apache-Hudi-support/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2021/07/16/Query-apache-hudi-dataset-in-an-amazon-S3-data-lake-with-amazon-athena-Read-optimized-queries/index.html b/content/blog/2021/07/16/Query-apache-hudi-dataset-in-an-amazon-S3-data-lake-with-amazon-athena-Read-optimized-queries/index.html
index 0f3abec81e3a3..36f0500cd3315 100644
--- a/content/blog/2021/07/16/Query-apache-hudi-dataset-in-an-amazon-S3-data-lake-with-amazon-athena-Read-optimized-queries/index.html
+++ b/content/blog/2021/07/16/Query-apache-hudi-dataset-in-an-amazon-S3-data-lake-with-amazon-athena-Read-optimized-queries/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2021/07/21/streaming-data-lake-platform/index.html b/content/blog/2021/07/21/streaming-data-lake-platform/index.html
index 8ad7ef01711e0..86c75457becb2 100644
--- a/content/blog/2021/07/21/streaming-data-lake-platform/index.html
+++ b/content/blog/2021/07/21/streaming-data-lake-platform/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2021/07/26/Baixin-banksreal-time-data-lake-evolution-scheme-based-on-Apache-Hudi/index.html b/content/blog/2021/07/26/Baixin-banksreal-time-data-lake-evolution-scheme-based-on-Apache-Hudi/index.html
index e27b15214ac5e..449a88d2cee93 100644
--- a/content/blog/2021/07/26/Baixin-banksreal-time-data-lake-evolution-scheme-based-on-Apache-Hudi/index.html
+++ b/content/blog/2021/07/26/Baixin-banksreal-time-data-lake-evolution-scheme-based-on-Apache-Hudi/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2021/08/03/MLOps-Wars-Versioned-Feature-Data-with-a-Lakehouse/index.html b/content/blog/2021/08/03/MLOps-Wars-Versioned-Feature-Data-with-a-Lakehouse/index.html
index f28887c2ee16f..6ca8551365546 100644
--- a/content/blog/2021/08/03/MLOps-Wars-Versioned-Feature-Data-with-a-Lakehouse/index.html
+++ b/content/blog/2021/08/03/MLOps-Wars-Versioned-Feature-Data-with-a-Lakehouse/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2021/08/11/Cost-Efficient-Open-Source-Big-Data-Platform-at-Uber/index.html b/content/blog/2021/08/11/Cost-Efficient-Open-Source-Big-Data-Platform-at-Uber/index.html
index a4ebff8bd419e..8ec6869fd9dd7 100644
--- a/content/blog/2021/08/11/Cost-Efficient-Open-Source-Big-Data-Platform-at-Uber/index.html
+++ b/content/blog/2021/08/11/Cost-Efficient-Open-Source-Big-Data-Platform-at-Uber/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2021/08/16/kafka-custom-deserializer/index.html b/content/blog/2021/08/16/kafka-custom-deserializer/index.html
index b3f1e08366a5b..70f45f6fdafdb 100644
--- a/content/blog/2021/08/16/kafka-custom-deserializer/index.html
+++ b/content/blog/2021/08/16/kafka-custom-deserializer/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2021/08/18/improving-marker-mechanism/index.html b/content/blog/2021/08/18/improving-marker-mechanism/index.html
index 027bc197dc840..9620a16f772a4 100644
--- a/content/blog/2021/08/18/improving-marker-mechanism/index.html
+++ b/content/blog/2021/08/18/improving-marker-mechanism/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2021/08/18/virtual-keys/index.html b/content/blog/2021/08/18/virtual-keys/index.html
index 96f8c33ddecb2..b5740912561fd 100644
--- a/content/blog/2021/08/18/virtual-keys/index.html
+++ b/content/blog/2021/08/18/virtual-keys/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2021/08/23/async-clustering/index.html b/content/blog/2021/08/23/async-clustering/index.html
index 38bdab5ea10a0..fdeeb17f80535 100644
--- a/content/blog/2021/08/23/async-clustering/index.html
+++ b/content/blog/2021/08/23/async-clustering/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2021/08/23/s3-events-source/index.html b/content/blog/2021/08/23/s3-events-source/index.html
index 5500ac0f5820b..8ac5bf26abe68 100644
--- a/content/blog/2021/08/23/s3-events-source/index.html
+++ b/content/blog/2021/08/23/s3-events-source/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2021/09/01/building-eb-level-data-lake-using-hudi-at-bytedance/index.html b/content/blog/2021/09/01/building-eb-level-data-lake-using-hudi-at-bytedance/index.html
index 5029545cd0722..69d03f42ea6ba 100644
--- a/content/blog/2021/09/01/building-eb-level-data-lake-using-hudi-at-bytedance/index.html
+++ b/content/blog/2021/09/01/building-eb-level-data-lake-using-hudi-at-bytedance/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2021/10/05/Data-Platform-2.0-Part-I/index.html b/content/blog/2021/10/05/Data-Platform-2.0-Part-I/index.html
index 5af4344f8d79f..ba16ae12d69c6 100644
--- a/content/blog/2021/10/05/Data-Platform-2.0-Part-I/index.html
+++ b/content/blog/2021/10/05/Data-Platform-2.0-Part-I/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2021/10/14/How-Amazon-Transportation-Service-enabled-near-real-time-event-analytics-at-petabyte-scale-using-AWS-Glue-with-Apache-Hudi/index.html b/content/blog/2021/10/14/How-Amazon-Transportation-Service-enabled-near-real-time-event-analytics-at-petabyte-scale-using-AWS-Glue-with-Apache-Hudi/index.html
index ec81844831413..b2d6dbbe74382 100644
--- a/content/blog/2021/10/14/How-Amazon-Transportation-Service-enabled-near-real-time-event-analytics-at-petabyte-scale-using-AWS-Glue-with-Apache-Hudi/index.html
+++ b/content/blog/2021/10/14/How-Amazon-Transportation-Service-enabled-near-real-time-event-analytics-at-petabyte-scale-using-AWS-Glue-with-Apache-Hudi/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2021/10/21/Practice-of-Apache-Hudi-in-building-real-time-data-lake-at-station-B/index.html b/content/blog/2021/10/21/Practice-of-Apache-Hudi-in-building-real-time-data-lake-at-station-B/index.html
index e409c957d928a..f81a2b9ed98db 100644
--- a/content/blog/2021/10/21/Practice-of-Apache-Hudi-in-building-real-time-data-lake-at-station-B/index.html
+++ b/content/blog/2021/10/21/Practice-of-Apache-Hudi-in-building-real-time-data-lake-at-station-B/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2021/11/16/How-GE-Aviation-built-cloud-native-data-pipelines-at-enterprise-scale-using-the-AWS-platform/index.html b/content/blog/2021/11/16/How-GE-Aviation-built-cloud-native-data-pipelines-at-enterprise-scale-using-the-AWS-platform/index.html
index 0325c1b3289a7..0b3b2197e3fe2 100644
--- a/content/blog/2021/11/16/How-GE-Aviation-built-cloud-native-data-pipelines-at-enterprise-scale-using-the-AWS-platform/index.html
+++ b/content/blog/2021/11/16/How-GE-Aviation-built-cloud-native-data-pipelines-at-enterprise-scale-using-the-AWS-platform/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2021/11/22/Apache-Hudi-Architecture-Tools-and-Best-Practices/index.html b/content/blog/2021/11/22/Apache-Hudi-Architecture-Tools-and-Best-Practices/index.html
index 752977c2866a4..9b57c1a90519a 100644
--- a/content/blog/2021/11/22/Apache-Hudi-Architecture-Tools-and-Best-Practices/index.html
+++ b/content/blog/2021/11/22/Apache-Hudi-Architecture-Tools-and-Best-Practices/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2021/12/16/lakehouse-concurrency-control-are-we-too-optimistic/index.html b/content/blog/2021/12/16/lakehouse-concurrency-control-are-we-too-optimistic/index.html
index 9f498a13f1101..fd8b37775ef89 100644
--- a/content/blog/2021/12/16/lakehouse-concurrency-control-are-we-too-optimistic/index.html
+++ b/content/blog/2021/12/16/lakehouse-concurrency-control-are-we-too-optimistic/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2021/12/20/New-features-from-Apache-Hudi-0.7.0-and-0.8.0-available-on-Amazon-EMR/index.html b/content/blog/2021/12/20/New-features-from-Apache-Hudi-0.7.0-and-0.8.0-available-on-Amazon-EMR/index.html
index bfcf560a18e4d..48c01f5da99b2 100644
--- a/content/blog/2021/12/20/New-features-from-Apache-Hudi-0.7.0-and-0.8.0-available-on-Amazon-EMR/index.html
+++ b/content/blog/2021/12/20/New-features-from-Apache-Hudi-0.7.0-and-0.8.0-available-on-Amazon-EMR/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2021/12/29/hudi-zorder-and-hilbert-space-filling-curves/index.html b/content/blog/2021/12/29/hudi-zorder-and-hilbert-space-filling-curves/index.html
index ccdf529bb1277..d34ac7ee7655e 100644
--- a/content/blog/2021/12/29/hudi-zorder-and-hilbert-space-filling-curves/index.html
+++ b/content/blog/2021/12/29/hudi-zorder-and-hilbert-space-filling-curves/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2021/12/31/The-Art-of-Building-Open-Data-Lakes-with-Apache-Hudi-Kafka-Hive-and-Debezium/index.html b/content/blog/2021/12/31/The-Art-of-Building-Open-Data-Lakes-with-Apache-Hudi-Kafka-Hive-and-Debezium/index.html
index 2f5e3d60ea9b0..1a5b5083f9d3a 100644
--- a/content/blog/2021/12/31/The-Art-of-Building-Open-Data-Lakes-with-Apache-Hudi-Kafka-Hive-and-Debezium/index.html
+++ b/content/blog/2021/12/31/The-Art-of-Building-Open-Data-Lakes-with-Apache-Hudi-Kafka-Hive-and-Debezium/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2022/01/06/apache-hudi-2021-a-year-in-review/index.html b/content/blog/2022/01/06/apache-hudi-2021-a-year-in-review/index.html
index a457fdd67f43f..a4372eeac2596 100644
--- a/content/blog/2022/01/06/apache-hudi-2021-a-year-in-review/index.html
+++ b/content/blog/2022/01/06/apache-hudi-2021-a-year-in-review/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2022/01/14/change-data-capture-with-debezium-and-apache-hudi/index.html b/content/blog/2022/01/14/change-data-capture-with-debezium-and-apache-hudi/index.html
index 20f9b771aecbf..0e2fc598adf21 100644
--- a/content/blog/2022/01/14/change-data-capture-with-debezium-and-apache-hudi/index.html
+++ b/content/blog/2022/01/14/change-data-capture-with-debezium-and-apache-hudi/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2022/01/18/Why-and-How-I-Integrated-Airbyte-and-Apache-Hudi/index.html b/content/blog/2022/01/18/Why-and-How-I-Integrated-Airbyte-and-Apache-Hudi/index.html
index a1bf93ff997ae..ca2195540317a 100644
--- a/content/blog/2022/01/18/Why-and-How-I-Integrated-Airbyte-and-Apache-Hudi/index.html
+++ b/content/blog/2022/01/18/Why-and-How-I-Integrated-Airbyte-and-Apache-Hudi/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2022/01/20/Hudi-powering-data-lake-efforts-at-Walmart-and-Disney-Hotstar/index.html b/content/blog/2022/01/20/Hudi-powering-data-lake-efforts-at-Walmart-and-Disney-Hotstar/index.html
index 4bde4d898db62..e241457985382 100644
--- a/content/blog/2022/01/20/Hudi-powering-data-lake-efforts-at-Walmart-and-Disney-Hotstar/index.html
+++ b/content/blog/2022/01/20/Hudi-powering-data-lake-efforts-at-Walmart-and-Disney-Hotstar/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2022/01/25/Cost-Efficiency-Scale-in-Big-Data-File-Format/index.html b/content/blog/2022/01/25/Cost-Efficiency-Scale-in-Big-Data-File-Format/index.html
index 9fd34f81eec22..574a3ef836201 100644
--- a/content/blog/2022/01/25/Cost-Efficiency-Scale-in-Big-Data-File-Format/index.html
+++ b/content/blog/2022/01/25/Cost-Efficiency-Scale-in-Big-Data-File-Format/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2022/02/02/Onehouse-Commitment-to-Openness/index.html b/content/blog/2022/02/02/Onehouse-Commitment-to-Openness/index.html
index 7e28255a2abcc..41aa60ae24052 100644
--- a/content/blog/2022/02/02/Onehouse-Commitment-to-Openness/index.html
+++ b/content/blog/2022/02/02/Onehouse-Commitment-to-Openness/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2022/02/03/Onehouse-brings-a-fully-managed-lakehouse-to-Apache-Hudi/index.html b/content/blog/2022/02/03/Onehouse-brings-a-fully-managed-lakehouse-to-Apache-Hudi/index.html
index df9fb4f5214b5..4ee9936ceb382 100644
--- a/content/blog/2022/02/03/Onehouse-brings-a-fully-managed-lakehouse-to-Apache-Hudi/index.html
+++ b/content/blog/2022/02/03/Onehouse-brings-a-fully-managed-lakehouse-to-Apache-Hudi/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2022/02/09/ACID-transformations-on-Distributed-file-system/index.html b/content/blog/2022/02/09/ACID-transformations-on-Distributed-file-system/index.html
index 9a52e91c14dd1..f420fba02d0c9 100644
--- a/content/blog/2022/02/09/ACID-transformations-on-Distributed-file-system/index.html
+++ b/content/blog/2022/02/09/ACID-transformations-on-Distributed-file-system/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2022/02/12/Open-Source-Data-Lake-Table-Formats-Evaluating-Current-Interest-and-Rate-of-Adoption/index.html b/content/blog/2022/02/12/Open-Source-Data-Lake-Table-Formats-Evaluating-Current-Interest-and-Rate-of-Adoption/index.html
index d6dda91e7d0bc..7b7d4a3756902 100644
--- a/content/blog/2022/02/12/Open-Source-Data-Lake-Table-Formats-Evaluating-Current-Interest-and-Rate-of-Adoption/index.html
+++ b/content/blog/2022/02/12/Open-Source-Data-Lake-Table-Formats-Evaluating-Current-Interest-and-Rate-of-Adoption/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2022/02/17/Fresher-Data-Lake-on-AWS-S3/index.html b/content/blog/2022/02/17/Fresher-Data-Lake-on-AWS-S3/index.html
index 0bf9860b6d568..757c5cc3b752e 100644
--- a/content/blog/2022/02/17/Fresher-Data-Lake-on-AWS-S3/index.html
+++ b/content/blog/2022/02/17/Fresher-Data-Lake-on-AWS-S3/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2022/02/20/Understanding-its-core-concepts-from-hudi-persistence-files/index.html b/content/blog/2022/02/20/Understanding-its-core-concepts-from-hudi-persistence-files/index.html
index d647de27da23c..1e3aaf3f8febe 100644
--- a/content/blog/2022/02/20/Understanding-its-core-concepts-from-hudi-persistence-files/index.html
+++ b/content/blog/2022/02/20/Understanding-its-core-concepts-from-hudi-persistence-files/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2022/03/01/Create-a-low-latency-source-to-data-lake-pipeline-using-Amazon-MSK-Connect-Apache-Flink-and-Apache-Hudi/index.html b/content/blog/2022/03/01/Create-a-low-latency-source-to-data-lake-pipeline-using-Amazon-MSK-Connect-Apache-Flink-and-Apache-Hudi/index.html
index 24ca0dd789171..326e4def20f01 100644
--- a/content/blog/2022/03/01/Create-a-low-latency-source-to-data-lake-pipeline-using-Amazon-MSK-Connect-Apache-Flink-and-Apache-Hudi/index.html
+++ b/content/blog/2022/03/01/Create-a-low-latency-source-to-data-lake-pipeline-using-Amazon-MSK-Connect-Apache-Flink-and-Apache-Hudi/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2022/03/09/Build-a-serverless-pipeline-to-analyze-streaming-data-using-AWS-Glue-Apache-Hudi-and-Amazon-S3/index.html b/content/blog/2022/03/09/Build-a-serverless-pipeline-to-analyze-streaming-data-using-AWS-Glue-Apache-Hudi-and-Amazon-S3/index.html
index db92068f0b8e0..526be0037135a 100644
--- a/content/blog/2022/03/09/Build-a-serverless-pipeline-to-analyze-streaming-data-using-AWS-Glue-Apache-Hudi-and-Amazon-S3/index.html
+++ b/content/blog/2022/03/09/Build-a-serverless-pipeline-to-analyze-streaming-data-using-AWS-Glue-Apache-Hudi-and-Amazon-S3/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2022/03/24/Zendesk-Insights-for-CTOs-Part-3-Growing-your-business-with-modern-data-capabilities/index.html b/content/blog/2022/03/24/Zendesk-Insights-for-CTOs-Part-3-Growing-your-business-with-modern-data-capabilities/index.html
index a878ed4daf13d..6abeb85aa2d31 100644
--- a/content/blog/2022/03/24/Zendesk-Insights-for-CTOs-Part-3-Growing-your-business-with-modern-data-capabilities/index.html
+++ b/content/blog/2022/03/24/Zendesk-Insights-for-CTOs-Part-3-Growing-your-business-with-modern-data-capabilities/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2022/04/04/Key-Learnings-on-Using-Apache-HUDI-in-building-Lakehouse-Architecture-at-Halodoc/index.html b/content/blog/2022/04/04/Key-Learnings-on-Using-Apache-HUDI-in-building-Lakehouse-Architecture-at-Halodoc/index.html
index a646d43539df7..5b4b3ae71b622 100644
--- a/content/blog/2022/04/04/Key-Learnings-on-Using-Apache-HUDI-in-building-Lakehouse-Architecture-at-Halodoc/index.html
+++ b/content/blog/2022/04/04/Key-Learnings-on-Using-Apache-HUDI-in-building-Lakehouse-Architecture-at-Halodoc/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2022/04/04/New-features-from-Apache-Hudi-0.9.0-on-Amazon-EMR/index.html b/content/blog/2022/04/04/New-features-from-Apache-Hudi-0.9.0-on-Amazon-EMR/index.html
index a82a48170fed6..0291b5e0fa5d5 100644
--- a/content/blog/2022/04/04/New-features-from-Apache-Hudi-0.9.0-on-Amazon-EMR/index.html
+++ b/content/blog/2022/04/04/New-features-from-Apache-Hudi-0.9.0-on-Amazon-EMR/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2022/04/19/Corrections-in-data-lakehouse-table-format-comparisons/index.html b/content/blog/2022/04/19/Corrections-in-data-lakehouse-table-format-comparisons/index.html
index 739432e48a9cb..b025d997d727c 100644
--- a/content/blog/2022/04/19/Corrections-in-data-lakehouse-table-format-comparisons/index.html
+++ b/content/blog/2022/04/19/Corrections-in-data-lakehouse-table-format-comparisons/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2022/05/17/Introducing-Multi-Modal-Index-for-the-Lakehouse-in-Apache-Hudi/index.html b/content/blog/2022/05/17/Introducing-Multi-Modal-Index-for-the-Lakehouse-in-Apache-Hudi/index.html
index 6eb3740cd54f7..d7e5220c30347 100644
--- a/content/blog/2022/05/17/Introducing-Multi-Modal-Index-for-the-Lakehouse-in-Apache-Hudi/index.html
+++ b/content/blog/2022/05/17/Introducing-Multi-Modal-Index-for-the-Lakehouse-in-Apache-Hudi/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2022/05/25/Record-by-record-deletable-data-lake-using-Apache-Hudi/index.html b/content/blog/2022/05/25/Record-by-record-deletable-data-lake-using-Apache-Hudi/index.html
index 30a27f81a490d..b748c938c2987 100644
--- a/content/blog/2022/05/25/Record-by-record-deletable-data-lake-using-Apache-Hudi/index.html
+++ b/content/blog/2022/05/25/Record-by-record-deletable-data-lake-using-Apache-Hudi/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2022/06/04/Asynchronous-Indexing-Using-Hudi/index.html b/content/blog/2022/06/04/Asynchronous-Indexing-Using-Hudi/index.html
index 4e993c9aca53c..93c2a7c337e60 100644
--- a/content/blog/2022/06/04/Asynchronous-Indexing-Using-Hudi/index.html
+++ b/content/blog/2022/06/04/Asynchronous-Indexing-Using-Hudi/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2022/06/09/Singificant-queries-speedup-from-Hudi-Column-Stats-Index-and-Data-Skipping-features/index.html b/content/blog/2022/06/09/Singificant-queries-speedup-from-Hudi-Column-Stats-Index-and-Data-Skipping-features/index.html
index 6af1e40bc80c6..350b304551967 100644
--- a/content/blog/2022/06/09/Singificant-queries-speedup-from-Hudi-Column-Stats-Index-and-Data-Skipping-features/index.html
+++ b/content/blog/2022/06/09/Singificant-queries-speedup-from-Hudi-Column-Stats-Index-and-Data-Skipping-features/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2022/06/29/Apache-Hudi-vs-Delta-Lake-transparent-tpc-ds-lakehouse-performance-benchmarks/index.html b/content/blog/2022/06/29/Apache-Hudi-vs-Delta-Lake-transparent-tpc-ds-lakehouse-performance-benchmarks/index.html
index d57df77c4ed04..0bf7515240ed8 100644
--- a/content/blog/2022/06/29/Apache-Hudi-vs-Delta-Lake-transparent-tpc-ds-lakehouse-performance-benchmarks/index.html
+++ b/content/blog/2022/06/29/Apache-Hudi-vs-Delta-Lake-transparent-tpc-ds-lakehouse-performance-benchmarks/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2022/07/11/build-open-lakehouse-using-apache-hudi-and-dbt/index.html b/content/blog/2022/07/11/build-open-lakehouse-using-apache-hudi-and-dbt/index.html
index e008bafd84dc6..bee0af3cf6542 100644
--- a/content/blog/2022/07/11/build-open-lakehouse-using-apache-hudi-and-dbt/index.html
+++ b/content/blog/2022/07/11/build-open-lakehouse-using-apache-hudi-and-dbt/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2022/08/09/How-NerdWallet-uses-AWS-and-Apache-Hudi-to-build-a-serverless-real-time-analytics-platform/index.html b/content/blog/2022/08/09/How-NerdWallet-uses-AWS-and-Apache-Hudi-to-build-a-serverless-real-time-analytics-platform/index.html
index 4246efdb74426..d2a7b0ad62683 100644
--- a/content/blog/2022/08/09/How-NerdWallet-uses-AWS-and-Apache-Hudi-to-build-a-serverless-real-time-analytics-platform/index.html
+++ b/content/blog/2022/08/09/How-NerdWallet-uses-AWS-and-Apache-Hudi-to-build-a-serverless-real-time-analytics-platform/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2022/08/12/Use-Flink-Hudi-to-Build-a-Streaming-Data-Lake-Platform/index.html b/content/blog/2022/08/12/Use-Flink-Hudi-to-Build-a-Streaming-Data-Lake-Platform/index.html
index ea880d7c3c752..112ba8969ebc8 100644
--- a/content/blog/2022/08/12/Use-Flink-Hudi-to-Build-a-Streaming-Data-Lake-Platform/index.html
+++ b/content/blog/2022/08/12/Use-Flink-Hudi-to-Build-a-Streaming-Data-Lake-Platform/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2022/08/24/Implementation-of-SCD-2-with-Apache-Hudi-and-Spark/index.html b/content/blog/2022/08/24/Implementation-of-SCD-2-with-Apache-Hudi-and-Spark/index.html
index cc52555281cb0..5001ffafd9f0a 100644
--- a/content/blog/2022/08/24/Implementation-of-SCD-2-with-Apache-Hudi-and-Spark/index.html
+++ b/content/blog/2022/08/24/Implementation-of-SCD-2-with-Apache-Hudi-and-Spark/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2022/08/25/Data-Lake-Lakehouse-Guide-Powered-by-Data-Lake-Table-Formats-Delta-Lake-Iceberg-Hudi/index.html b/content/blog/2022/08/25/Data-Lake-Lakehouse-Guide-Powered-by-Data-Lake-Table-Formats-Delta-Lake-Iceberg-Hudi/index.html
index 1a54ebb567402..b0c26c9b7c78c 100644
--- a/content/blog/2022/08/25/Data-Lake-Lakehouse-Guide-Powered-by-Data-Lake-Table-Formats-Delta-Lake-Iceberg-Hudi/index.html
+++ b/content/blog/2022/08/25/Data-Lake-Lakehouse-Guide-Powered-by-Data-Lake-Table-Formats-Delta-Lake-Iceberg-Hudi/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2022/09/20/Building-Streaming-Data-Lakes-with-Hudi-and-MinIO/index.html b/content/blog/2022/09/20/Building-Streaming-Data-Lakes-with-Hudi-and-MinIO/index.html
index 66ed406294636..2be210bf554d5 100644
--- a/content/blog/2022/09/20/Building-Streaming-Data-Lakes-with-Hudi-and-MinIO/index.html
+++ b/content/blog/2022/09/20/Building-Streaming-Data-Lakes-with-Hudi-and-MinIO/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2022/09/28/Data-processing-with-Spark-time-traveling/index.html b/content/blog/2022/09/28/Data-processing-with-Spark-time-traveling/index.html
index ed1d007b14048..88a8a49f60fa8 100644
--- a/content/blog/2022/09/28/Data-processing-with-Spark-time-traveling/index.html
+++ b/content/blog/2022/09/28/Data-processing-with-Spark-time-traveling/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2022/10/06/Ingest-streaming-data-to-Apache-Hudi-using-AWS-Glue-and-DeltaStreamer/index.html b/content/blog/2022/10/06/Ingest-streaming-data-to-Apache-Hudi-using-AWS-Glue-and-DeltaStreamer/index.html
index 1af5bd540f649..dc59ac6b3b60f 100644
--- a/content/blog/2022/10/06/Ingest-streaming-data-to-Apache-Hudi-using-AWS-Glue-and-DeltaStreamer/index.html
+++ b/content/blog/2022/10/06/Ingest-streaming-data-to-Apache-Hudi-using-AWS-Glue-and-DeltaStreamer/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2022/10/08/what-why-and-how-apache-hudis-bloom-index/index.html b/content/blog/2022/10/08/what-why-and-how-apache-hudis-bloom-index/index.html
index dc5a0366e7793..ea13e7a499c17 100644
--- a/content/blog/2022/10/08/what-why-and-how-apache-hudis-bloom-index/index.html
+++ b/content/blog/2022/10/08/what-why-and-how-apache-hudis-bloom-index/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2022/10/17/Get-started-with-Apache-Hudi-using-AWS/index.html b/content/blog/2022/10/17/Get-started-with-Apache-Hudi-using-AWS/index.html
index 174dee12597b5..e658c7ee6e724 100644
--- a/content/blog/2022/10/17/Get-started-with-Apache-Hudi-using-AWS/index.html
+++ b/content/blog/2022/10/17/Get-started-with-Apache-Hudi-using-AWS/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2022/11/10/How-Hudl-built-a-cost-optimized-AWS-Glue-pipeline-with-Apache-Hudi-datasets/index.html b/content/blog/2022/11/10/How-Hudl-built-a-cost-optimized-AWS-Glue-pipeline-with-Apache-Hudi-datasets/index.html
index 3abeedb19a230..21b3fa136a977 100644
--- a/content/blog/2022/11/10/How-Hudl-built-a-cost-optimized-AWS-Glue-pipeline-with-Apache-Hudi-datasets/index.html
+++ b/content/blog/2022/11/10/How-Hudl-built-a-cost-optimized-AWS-Glue-pipeline-with-Apache-Hudi-datasets/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2022/11/22/Build-your-Apache-Hudi-data-lake-on-AWS-using-Amazon-EMR-Part-1/index.html b/content/blog/2022/11/22/Build-your-Apache-Hudi-data-lake-on-AWS-using-Amazon-EMR-Part-1/index.html
index a0d9e970574a3..13594b43df529 100644
--- a/content/blog/2022/11/22/Build-your-Apache-Hudi-data-lake-on-AWS-using-Amazon-EMR-Part-1/index.html
+++ b/content/blog/2022/11/22/Build-your-Apache-Hudi-data-lake-on-AWS-using-Amazon-EMR-Part-1/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2022/12/01/Run-apache-hudi-at-scale-on-aws/index.html b/content/blog/2022/12/01/Run-apache-hudi-at-scale-on-aws/index.html
index a1beacc1185f5..64fdf25ea2780 100644
--- a/content/blog/2022/12/01/Run-apache-hudi-at-scale-on-aws/index.html
+++ b/content/blog/2022/12/01/Run-apache-hudi-at-scale-on-aws/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2022/12/19/Build-Your-First-Hudi-Lakehouse-with-AWS-Glue-and-AWS-S3/index.html b/content/blog/2022/12/19/Build-Your-First-Hudi-Lakehouse-with-AWS-Glue-and-AWS-S3/index.html
index 9b1c32f4e5de9..c8d1734fdd588 100644
--- a/content/blog/2022/12/19/Build-Your-First-Hudi-Lakehouse-with-AWS-Glue-and-AWS-S3/index.html
+++ b/content/blog/2022/12/19/Build-Your-First-Hudi-Lakehouse-with-AWS-Glue-and-AWS-S3/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2022/12/29/Apache-Hudi-2022-A-Year-In-Review/index.html b/content/blog/2022/12/29/Apache-Hudi-2022-A-Year-In-Review/index.html
index 7c3bfe10812b4..0bf10a4d58ac3 100644
--- a/content/blog/2022/12/29/Apache-Hudi-2022-A-Year-In-Review/index.html
+++ b/content/blog/2022/12/29/Apache-Hudi-2022-A-Year-In-Review/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2023/01/11/Apache-Hudi-vs-Delta-Lake-vs-Apache-Iceberg-Lakehouse-Feature-Comparison/index.html b/content/blog/2023/01/11/Apache-Hudi-vs-Delta-Lake-vs-Apache-Iceberg-Lakehouse-Feature-Comparison/index.html
index a09945ab2052d..1027eef632830 100644
--- a/content/blog/2023/01/11/Apache-Hudi-vs-Delta-Lake-vs-Apache-Iceberg-Lakehouse-Feature-Comparison/index.html
+++ b/content/blog/2023/01/11/Apache-Hudi-vs-Delta-Lake-vs-Apache-Iceberg-Lakehouse-Feature-Comparison/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2023/01/27/Introducing-native-support-for-Apache-Hudi-Delta-Lake-Apache-Iceberg-on-AWS-Glue-for-Apache-Spark/index.html b/content/blog/2023/01/27/Introducing-native-support-for-Apache-Hudi-Delta-Lake-Apache-Iceberg-on-AWS-Glue-for-Apache-Spark/index.html
index cda73ee04a284..296569893d60a 100644
--- a/content/blog/2023/01/27/Introducing-native-support-for-Apache-Hudi-Delta-Lake-Apache-Iceberg-on-AWS-Glue-for-Apache-Spark/index.html
+++ b/content/blog/2023/01/27/Introducing-native-support-for-Apache-Hudi-Delta-Lake-Apache-Iceberg-on-AWS-Glue-for-Apache-Spark/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2023/02/07/automate-schema-evolution-at-scale-with-apache-hudi-in-aws-glue/index.html b/content/blog/2023/02/07/automate-schema-evolution-at-scale-with-apache-hudi-in-aws-glue/index.html
index e3d0ee9539a78..e0d5dbcee7ba4 100644
--- a/content/blog/2023/02/07/automate-schema-evolution-at-scale-with-apache-hudi-in-aws-glue/index.html
+++ b/content/blog/2023/02/07/automate-schema-evolution-at-scale-with-apache-hudi-in-aws-glue/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2023/02/12/table-service-deployment-models-in-apache-hudi/index.html b/content/blog/2023/02/12/table-service-deployment-models-in-apache-hudi/index.html
index b1436e8ec54bd..4ef6c357f312a 100644
--- a/content/blog/2023/02/12/table-service-deployment-models-in-apache-hudi/index.html
+++ b/content/blog/2023/02/12/table-service-deployment-models-in-apache-hudi/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2023/02/19/bulk-insert-sort-modes-with-apache-hudi/index.html b/content/blog/2023/02/19/bulk-insert-sort-modes-with-apache-hudi/index.html
index a93a97bc6d767..6b0c2dfa6b955 100644
--- a/content/blog/2023/02/19/bulk-insert-sort-modes-with-apache-hudi/index.html
+++ b/content/blog/2023/02/19/bulk-insert-sort-modes-with-apache-hudi/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2023/02/22/Getting-Started-Manage-your-Hudi-tables-with-the-admin-Hudi-CLI-tool/index.html b/content/blog/2023/02/22/Getting-Started-Manage-your-Hudi-tables-with-the-admin-Hudi-CLI-tool/index.html
index 3c47149435e61..7be56269a9590 100644
--- a/content/blog/2023/02/22/Getting-Started-Manage-your-Hudi-tables-with-the-admin-Hudi-CLI-tool/index.html
+++ b/content/blog/2023/02/22/Getting-Started-Manage-your-Hudi-tables-with-the-admin-Hudi-CLI-tool/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2023/03/16/Setting-Uber-Transactional-Data-Lake-in-Motion-with-Incremental-ETL-Using-Apache-Hudi/index.html b/content/blog/2023/03/16/Setting-Uber-Transactional-Data-Lake-in-Motion-with-Incremental-ETL-Using-Apache-Hudi/index.html
index b097d50a8d174..7e6547008c097 100644
--- a/content/blog/2023/03/16/Setting-Uber-Transactional-Data-Lake-in-Motion-with-Incremental-ETL-Using-Apache-Hudi/index.html
+++ b/content/blog/2023/03/16/Setting-Uber-Transactional-Data-Lake-in-Motion-with-Incremental-ETL-Using-Apache-Hudi/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2023/03/17/introduction-to-apache-hudi/index.html b/content/blog/2023/03/17/introduction-to-apache-hudi/index.html
index e1eef610201fc..819549bb91d20 100644
--- a/content/blog/2023/03/17/introduction-to-apache-hudi/index.html
+++ b/content/blog/2023/03/17/introduction-to-apache-hudi/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2023/03/20/Introducing-native-support-for-Apache Hudi-Delta-Lake-and-Apache-Iceberg-on-AWS-Glue-for-Apache-Spark-Part-2-AWS-Glue-Studio-Visual-Editor/index.html b/content/blog/2023/03/20/Introducing-native-support-for-Apache Hudi-Delta-Lake-and-Apache-Iceberg-on-AWS-Glue-for-Apache-Spark-Part-2-AWS-Glue-Studio-Visual-Editor/index.html
index ec448f0489651..a4e3866954f33 100644
--- a/content/blog/2023/03/20/Introducing-native-support-for-Apache Hudi-Delta-Lake-and-Apache-Iceberg-on-AWS-Glue-for-Apache-Spark-Part-2-AWS-Glue-Studio-Visual-Editor/index.html
+++ b/content/blog/2023/03/20/Introducing-native-support-for-Apache Hudi-Delta-Lake-and-Apache-Iceberg-on-AWS-Glue-for-Apache-Spark-Part-2-AWS-Glue-Studio-Visual-Editor/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2023/03/23/Spark-ETL-Chapter-8-with-Lakehouse-Apache-HUDI/index.html b/content/blog/2023/03/23/Spark-ETL-Chapter-8-with-Lakehouse-Apache-HUDI/index.html
index c5fc45fbe7d0f..835e3e3b52129 100644
--- a/content/blog/2023/03/23/Spark-ETL-Chapter-8-with-Lakehouse-Apache-HUDI/index.html
+++ b/content/blog/2023/03/23/Spark-ETL-Chapter-8-with-Lakehouse-Apache-HUDI/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2023/04/02/global-vs-non-global-index-in-apache-hudi/index.html b/content/blog/2023/04/02/global-vs-non-global-index-in-apache-hudi/index.html
index d87d63322d21d..cd8baee54e964 100644
--- a/content/blog/2023/04/02/global-vs-non-global-index-in-apache-hudi/index.html
+++ b/content/blog/2023/04/02/global-vs-non-global-index-in-apache-hudi/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2023/04/07/Speed-up-your-write-latencies-using-Bucket-Index-in-Apache-Hudi/index.html b/content/blog/2023/04/07/Speed-up-your-write-latencies-using-Bucket-Index-in-Apache-Hudi/index.html
index 8c9ce75a1af99..2f4b9c1a09098 100644
--- a/content/blog/2023/04/07/Speed-up-your-write-latencies-using-Bucket-Index-in-Apache-Hudi/index.html
+++ b/content/blog/2023/04/07/Speed-up-your-write-latencies-using-Bucket-Index-in-Apache-Hudi/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2023/04/18/getting-started-incrementally-process-data-with-apache-hudi/index.html b/content/blog/2023/04/18/getting-started-incrementally-process-data-with-apache-hudi/index.html
index 58d220f3f4434..92e7bb672d901 100644
--- a/content/blog/2023/04/18/getting-started-incrementally-process-data-with-apache-hudi/index.html
+++ b/content/blog/2023/04/18/getting-started-incrementally-process-data-with-apache-hudi/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2023/04/26/the-lakehouse-trifecta/index.html b/content/blog/2023/04/26/the-lakehouse-trifecta/index.html
index 90534fd37cc6f..5a8269ddb7eff 100644
--- a/content/blog/2023/04/26/the-lakehouse-trifecta/index.html
+++ b/content/blog/2023/04/26/the-lakehouse-trifecta/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2023/04/29/can-you-concurrently-write-data-to-apache-hudi-w-o-any-lock-provider/index.html b/content/blog/2023/04/29/can-you-concurrently-write-data-to-apache-hudi-w-o-any-lock-provider/index.html
index d9d5af13ae8f3..72438252102d5 100644
--- a/content/blog/2023/04/29/can-you-concurrently-write-data-to-apache-hudi-w-o-any-lock-provider/index.html
+++ b/content/blog/2023/04/29/can-you-concurrently-write-data-to-apache-hudi-w-o-any-lock-provider/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2023/05/02/intro-to-hudi-and-flink/index.html b/content/blog/2023/05/02/intro-to-hudi-and-flink/index.html
index d8b1650af1a19..00241c6731321 100644
--- a/content/blog/2023/05/02/intro-to-hudi-and-flink/index.html
+++ b/content/blog/2023/05/02/intro-to-hudi-and-flink/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2023/05/03/lakehouse-at-fortune-1-scale/index.html b/content/blog/2023/05/03/lakehouse-at-fortune-1-scale/index.html
index 1d9bfefb95bed..9af6a06d0c88f 100644
--- a/content/blog/2023/05/03/lakehouse-at-fortune-1-scale/index.html
+++ b/content/blog/2023/05/03/lakehouse-at-fortune-1-scale/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2023/05/09/amazon-athena-apache-hudi/index.html b/content/blog/2023/05/09/amazon-athena-apache-hudi/index.html
index 222f38eed3101..641efb3d91296 100644
--- a/content/blog/2023/05/09/amazon-athena-apache-hudi/index.html
+++ b/content/blog/2023/05/09/amazon-athena-apache-hudi/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2023/05/10/top-3-things-you-can-do-to-get-fast-upsert-performance-in-apache-hudi/index.html b/content/blog/2023/05/10/top-3-things-you-can-do-to-get-fast-upsert-performance-in-apache-hudi/index.html
index 07da68a3535c8..9a4b7d1962fdd 100644
--- a/content/blog/2023/05/10/top-3-things-you-can-do-to-get-fast-upsert-performance-in-apache-hudi/index.html
+++ b/content/blog/2023/05/10/top-3-things-you-can-do-to-get-fast-upsert-performance-in-apache-hudi/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2023/05/12/ingesting-data-to-apache-hudi-using-spark-sql/index.html b/content/blog/2023/05/12/ingesting-data-to-apache-hudi-using-spark-sql/index.html
index caf96fe5b78b2..241a231f2e824 100644
--- a/content/blog/2023/05/12/ingesting-data-to-apache-hudi-using-spark-sql/index.html
+++ b/content/blog/2023/05/12/ingesting-data-to-apache-hudi-using-spark-sql/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2023/05/16/how-zoom-implemented-streaming-log-ingestion-and-efficient-gdpr-deletes-using-apache-hudi-on-amazon-emr/index.html b/content/blog/2023/05/16/how-zoom-implemented-streaming-log-ingestion-and-efficient-gdpr-deletes-using-apache-hudi-on-amazon-emr/index.html
index b155361696ce1..a291ee2784171 100644
--- a/content/blog/2023/05/16/how-zoom-implemented-streaming-log-ingestion-and-efficient-gdpr-deletes-using-apache-hudi-on-amazon-emr/index.html
+++ b/content/blog/2023/05/16/how-zoom-implemented-streaming-log-ingestion-and-efficient-gdpr-deletes-using-apache-hudi-on-amazon-emr/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2023/05/19/hudi-metafields-demystified/index.html b/content/blog/2023/05/19/hudi-metafields-demystified/index.html
index 699a57ddc5d0f..9c541f9bb1ed8 100644
--- a/content/blog/2023/05/19/hudi-metafields-demystified/index.html
+++ b/content/blog/2023/05/19/hudi-metafields-demystified/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2023/05/29/different-query-types-with-apache-hudi/index.html b/content/blog/2023/05/29/different-query-types-with-apache-hudi/index.html
index ad3fadc69420e..aed7a6a0ded45 100644
--- a/content/blog/2023/05/29/different-query-types-with-apache-hudi/index.html
+++ b/content/blog/2023/05/29/different-query-types-with-apache-hudi/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2023/06/03/text-based-search-from-elastic-search-to-vector-search/index.html b/content/blog/2023/06/03/text-based-search-from-elastic-search-to-vector-search/index.html
index 21ecebd467b27..84d6ec79ce467 100644
--- a/content/blog/2023/06/03/text-based-search-from-elastic-search-to-vector-search/index.html
+++ b/content/blog/2023/06/03/text-based-search-from-elastic-search-to-vector-search/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2023/06/11/cleaner-and-archival-in-apache-hudi/index.html b/content/blog/2023/06/11/cleaner-and-archival-in-apache-hudi/index.html
index a63d741813ea4..ef7c8292a839b 100644
--- a/content/blog/2023/06/11/cleaner-and-archival-in-apache-hudi/index.html
+++ b/content/blog/2023/06/11/cleaner-and-archival-in-apache-hudi/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2023/06/16/Exploring-New-Frontiers-How-Apache-Flink-Apache-Hudi-and-Presto-Power-New-Insights-at-Scale/index.html b/content/blog/2023/06/16/Exploring-New-Frontiers-How-Apache-Flink-Apache-Hudi-and-Presto-Power-New-Insights-at-Scale/index.html
index 05c7185e33a49..df55b0c27950a 100644
--- a/content/blog/2023/06/16/Exploring-New-Frontiers-How-Apache-Flink-Apache-Hudi-and-Presto-Power-New-Insights-at-Scale/index.html
+++ b/content/blog/2023/06/16/Exploring-New-Frontiers-How-Apache-Flink-Apache-Hudi-and-Presto-Power-New-Insights-at-Scale/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2023/06/20/How-to-query-data-in-Apache-Hudi-using-StarRocks/index.html b/content/blog/2023/06/20/How-to-query-data-in-Apache-Hudi-using-StarRocks/index.html
index f1de37425699a..155f1371bcc3b 100644
--- a/content/blog/2023/06/20/How-to-query-data-in-Apache-Hudi-using-StarRocks/index.html
+++ b/content/blog/2023/06/20/How-to-query-data-in-Apache-Hudi-using-StarRocks/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2023/06/20/timeline-server-in-apache-hudi/index.html b/content/blog/2023/06/20/timeline-server-in-apache-hudi/index.html
index 3fa8e2084b1c8..cd54dba4da29d 100644
--- a/content/blog/2023/06/20/timeline-server-in-apache-hudi/index.html
+++ b/content/blog/2023/06/20/timeline-server-in-apache-hudi/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2023/06/24/multi-writer-support-in-apache-hudi/index.html b/content/blog/2023/06/24/multi-writer-support-in-apache-hudi/index.html
index a1efab9aba7a4..e5f335a575c4c 100644
--- a/content/blog/2023/06/24/multi-writer-support-in-apache-hudi/index.html
+++ b/content/blog/2023/06/24/multi-writer-support-in-apache-hudi/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2023/06/26/Unlimited-Big-Data-Exchange-A-Wonderful-Review-of-Apache-DolphinScheduler-and-Hudi-Hangzhou-Meetup/index.html b/content/blog/2023/06/26/Unlimited-Big-Data-Exchange-A-Wonderful-Review-of-Apache-DolphinScheduler-and-Hudi-Hangzhou-Meetup/index.html
index b5427d55df02d..9652ebd6d7478 100644
--- a/content/blog/2023/06/26/Unlimited-Big-Data-Exchange-A-Wonderful-Review-of-Apache-DolphinScheduler-and-Hudi-Hangzhou-Meetup/index.html
+++ b/content/blog/2023/06/26/Unlimited-Big-Data-Exchange-A-Wonderful-Review-of-Apache-DolphinScheduler-and-Hudi-Hangzhou-Meetup/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2023/06/30/What-about-Apache-Hudi-Apache-Iceberg-and-Delta-Lake/index.html b/content/blog/2023/06/30/What-about-Apache-Hudi-Apache-Iceberg-and-Delta-Lake/index.html
index e85b7675a47fe..04ddd8fde6a4a 100644
--- a/content/blog/2023/06/30/What-about-Apache-Hudi-Apache-Iceberg-and-Delta-Lake/index.html
+++ b/content/blog/2023/06/30/What-about-Apache-Hudi-Apache-Iceberg-and-Delta-Lake/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2023/07/01/monitoring-table-size-stats/index.html b/content/blog/2023/07/01/monitoring-table-size-stats/index.html
index 52058fd836e81..39dacb01f2957 100644
--- a/content/blog/2023/07/01/monitoring-table-size-stats/index.html
+++ b/content/blog/2023/07/01/monitoring-table-size-stats/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2023/07/02/Hudi-Best-Practices-Handling-Failed-Inserts-Upserts-with-Error-Tables/index.html b/content/blog/2023/07/02/Hudi-Best-Practices-Handling-Failed-Inserts-Upserts-with-Error-Tables/index.html
index 74f665326e89e..1e9ed32d5ed91 100644
--- a/content/blog/2023/07/02/Hudi-Best-Practices-Handling-Failed-Inserts-Upserts-with-Error-Tables/index.html
+++ b/content/blog/2023/07/02/Hudi-Best-Practices-Handling-Failed-Inserts-Upserts-with-Error-Tables/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2023/07/07/Skip-rocks-and-files-Turbocharge-Trino-queries-with-Hudi-multi-modal-indexing-subsystem/index.html b/content/blog/2023/07/07/Skip-rocks-and-files-Turbocharge-Trino-queries-with-Hudi-multi-modal-indexing-subsystem/index.html
index 8d62b749bcdb1..d5e751ccf3999 100644
--- a/content/blog/2023/07/07/Skip-rocks-and-files-Turbocharge-Trino-queries-with-Hudi-multi-modal-indexing-subsystem/index.html
+++ b/content/blog/2023/07/07/Skip-rocks-and-files-Turbocharge-Trino-queries-with-Hudi-multi-modal-indexing-subsystem/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2023/07/08/Quickly-start-using-Apache-Hudi-on-AWS-EMR/index.html b/content/blog/2023/07/08/Quickly-start-using-Apache-Hudi-on-AWS-EMR/index.html
index 2494200ce22ba..360156ea337ee 100644
--- a/content/blog/2023/07/08/Quickly-start-using-Apache-Hudi-on-AWS-EMR/index.html
+++ b/content/blog/2023/07/08/Quickly-start-using-Apache-Hudi-on-AWS-EMR/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2023/07/09/Hoodie-Timeline-Foundational-pillar-for-ACID-transactions/index.html b/content/blog/2023/07/09/Hoodie-Timeline-Foundational-pillar-for-ACID-transactions/index.html
index 0b8921c216f44..4e04400bf942f 100644
--- a/content/blog/2023/07/09/Hoodie-Timeline-Foundational-pillar-for-ACID-transactions/index.html
+++ b/content/blog/2023/07/09/Hoodie-Timeline-Foundational-pillar-for-ACID-transactions/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2023/07/20/Backfilling-Apache-Hudi-Tables-in-Production-Techniques-and-Approaches-Using-AWS-Glue-by-Job-Target-LLC/index.html b/content/blog/2023/07/20/Backfilling-Apache-Hudi-Tables-in-Production-Techniques-and-Approaches-Using-AWS-Glue-by-Job-Target-LLC/index.html
index fc9d88e826138..c11442a8c52f3 100644
--- a/content/blog/2023/07/20/Backfilling-Apache-Hudi-Tables-in-Production-Techniques-and-Approaches-Using-AWS-Glue-by-Job-Target-LLC/index.html
+++ b/content/blog/2023/07/20/Backfilling-Apache-Hudi-Tables-in-Production-Techniques-and-Approaches-Using-AWS-Glue-by-Job-Target-LLC/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2023/07/21/AWS-Glue-Crawlers-now-supports-Apache-Hudi-Tables/index.html b/content/blog/2023/07/21/AWS-Glue-Crawlers-now-supports-Apache-Hudi-Tables/index.html
index d95e5382b3f73..1d0871f7eb604 100644
--- a/content/blog/2023/07/21/AWS-Glue-Crawlers-now-supports-Apache-Hudi-Tables/index.html
+++ b/content/blog/2023/07/21/AWS-Glue-Crawlers-now-supports-Apache-Hudi-Tables/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2023/07/27/Apache-Hudi-Revolutionizing-Big-Data-Management-for-Real-Time-Analytics/index.html b/content/blog/2023/07/27/Apache-Hudi-Revolutionizing-Big-Data-Management-for-Real-Time-Analytics/index.html
index 8fb6cb8566109..14d7a50bfd60d 100644
--- a/content/blog/2023/07/27/Apache-Hudi-Revolutionizing-Big-Data-Management-for-Real-Time-Analytics/index.html
+++ b/content/blog/2023/07/27/Apache-Hudi-Revolutionizing-Big-Data-Management-for-Real-Time-Analytics/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2023/08/03/Apache-Hudi-on-AWS-Glue-A-Step-by-Step-Guide/index.html b/content/blog/2023/08/03/Apache-Hudi-on-AWS-Glue-A-Step-by-Step-Guide/index.html
index db5ace5d1183a..93bd5e7810995 100644
--- a/content/blog/2023/08/03/Apache-Hudi-on-AWS-Glue-A-Step-by-Step-Guide/index.html
+++ b/content/blog/2023/08/03/Apache-Hudi-on-AWS-Glue-A-Step-by-Step-Guide/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2023/08/03/Create-an-Apache-Hudi-based-near-real-time-transactional-data lake-using-AWS-DMS-Amazon-Kinesis-AWS-Glue-streaming-ETL-and-data-visualization-using-Amazon-QuickSight/index.html b/content/blog/2023/08/03/Create-an-Apache-Hudi-based-near-real-time-transactional-data lake-using-AWS-DMS-Amazon-Kinesis-AWS-Glue-streaming-ETL-and-data-visualization-using-Amazon-QuickSight/index.html
index 0e16b1c358882..115e413f60f1f 100644
--- a/content/blog/2023/08/03/Create-an-Apache-Hudi-based-near-real-time-transactional-data lake-using-AWS-DMS-Amazon-Kinesis-AWS-Glue-streaming-ETL-and-data-visualization-using-Amazon-QuickSight/index.html
+++ b/content/blog/2023/08/03/Create-an-Apache-Hudi-based-near-real-time-transactional-data lake-using-AWS-DMS-Amazon-Kinesis-AWS-Glue-streaming-ETL-and-data-visualization-using-Amazon-QuickSight/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2023/08/03/Data-lake-Table-formats-Apache-Iceberg-vs-Apache-Hudi-vs-Delta-lake/index.html b/content/blog/2023/08/03/Data-lake-Table-formats-Apache-Iceberg-vs-Apache-Hudi-vs-Delta-lake/index.html
index e9fd05b17abe9..d283254e07fff 100644
--- a/content/blog/2023/08/03/Data-lake-Table-formats-Apache-Iceberg-vs-Apache-Hudi-vs-Delta-lake/index.html
+++ b/content/blog/2023/08/03/Data-lake-Table-formats-Apache-Iceberg-vs-Apache-Hudi-vs-Delta-lake/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2023/08/05/Data-Lakehouse-Architecture-for-Big-Data-with-Apache-Hudi/index.html b/content/blog/2023/08/05/Data-Lakehouse-Architecture-for-Big-Data-with-Apache-Hudi/index.html
index 82f7370501310..93526310e9c7b 100644
--- a/content/blog/2023/08/05/Data-Lakehouse-Architecture-for-Big-Data-with-Apache-Hudi/index.html
+++ b/content/blog/2023/08/05/Data-Lakehouse-Architecture-for-Big-Data-with-Apache-Hudi/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2023/08/09/Lakehouse-Trifecta-Delta-Lake-Apache-Iceberg-and-Apache-Hudi/index.html b/content/blog/2023/08/09/Lakehouse-Trifecta-Delta-Lake-Apache-Iceberg-and-Apache-Hudi/index.html
index d7e5ed6b2f8b5..c67dae9d14580 100644
--- a/content/blog/2023/08/09/Lakehouse-Trifecta-Delta-Lake-Apache-Iceberg-and-Apache-Hudi/index.html
+++ b/content/blog/2023/08/09/Lakehouse-Trifecta-Delta-Lake-Apache-Iceberg-and-Apache-Hudi/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2023/08/22/Exploring-various-storage-types-in-Apache-Hudi/index.html b/content/blog/2023/08/22/Exploring-various-storage-types-in-Apache-Hudi/index.html
index 09e91e12191fd..515a0a1f58e0a 100644
--- a/content/blog/2023/08/22/Exploring-various-storage-types-in-Apache-Hudi/index.html
+++ b/content/blog/2023/08/22/Exploring-various-storage-types-in-Apache-Hudi/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2023/08/25/Delta-Hudi-Iceberg-Which-is-most-popular/index.html b/content/blog/2023/08/25/Delta-Hudi-Iceberg-Which-is-most-popular/index.html
index 52f8a05257a77..b39ab30008a95 100644
--- a/content/blog/2023/08/25/Delta-Hudi-Iceberg-Which-is-most-popular/index.html
+++ b/content/blog/2023/08/25/Delta-Hudi-Iceberg-Which-is-most-popular/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2023/08/28/Apache-Hudi-From-Zero-To-One/index.html b/content/blog/2023/08/28/Apache-Hudi-From-Zero-To-One/index.html
index 30efe91eb53f6..48e624821225c 100644
--- a/content/blog/2023/08/28/Apache-Hudi-From-Zero-To-One/index.html
+++ b/content/blog/2023/08/28/Apache-Hudi-From-Zero-To-One/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2023/08/28/Delta-Hudi-Iceberg-A-Benchmark-Compilation/index.html b/content/blog/2023/08/28/Delta-Hudi-Iceberg-A-Benchmark-Compilation/index.html
index d90ff4cc072cf..17961ed45e616 100644
--- a/content/blog/2023/08/28/Delta-Hudi-Iceberg-A-Benchmark-Compilation/index.html
+++ b/content/blog/2023/08/28/Delta-Hudi-Iceberg-A-Benchmark-Compilation/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2023/08/31/Incremental-Queries-with-Apache-Hudi-and-Apache-Flink/index.html b/content/blog/2023/08/31/Incremental-Queries-with-Apache-Hudi-and-Apache-Flink/index.html
index de10533351cbd..f27ac8c277b75 100644
--- a/content/blog/2023/08/31/Incremental-Queries-with-Apache-Hudi-and-Apache-Flink/index.html
+++ b/content/blog/2023/08/31/Incremental-Queries-with-Apache-Hudi-and-Apache-Flink/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2023/09/06/Apache-Hudi-From-Zero-To-One-blog-2/index.html b/content/blog/2023/09/06/Apache-Hudi-From-Zero-To-One-blog-2/index.html
index 7a7d17d0de91a..915bd24942d0d 100644
--- a/content/blog/2023/09/06/Apache-Hudi-From-Zero-To-One-blog-2/index.html
+++ b/content/blog/2023/09/06/Apache-Hudi-From-Zero-To-One-blog-2/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2023/09/06/Lakehouse-or-Warehouse-Part-1-of-2/index.html b/content/blog/2023/09/06/Lakehouse-or-Warehouse-Part-1-of-2/index.html
index d79b4ff939093..4f16279088428 100644
--- a/content/blog/2023/09/06/Lakehouse-or-Warehouse-Part-1-of-2/index.html
+++ b/content/blog/2023/09/06/Lakehouse-or-Warehouse-Part-1-of-2/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2023/09/10/Demystifying-Copy-on-Write-in-Apache-Hudi-Understanding-Read-and-Write-Operations/index.html b/content/blog/2023/09/10/Demystifying-Copy-on-Write-in-Apache-Hudi-Understanding-Read-and-Write-Operations/index.html
index c549f78b5de17..268b3b463990f 100644
--- a/content/blog/2023/09/10/Demystifying-Copy-on-Write-in-Apache-Hudi-Understanding-Read-and-Write-Operations/index.html
+++ b/content/blog/2023/09/10/Demystifying-Copy-on-Write-in-Apache-Hudi-Understanding-Read-and-Write-Operations/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2023/09/12/Lakehouse-or-Warehouse-Part-2-of-2/index.html b/content/blog/2023/09/12/Lakehouse-or-Warehouse-Part-2-of-2/index.html
index f8f0a468421a4..94ef93027e6e1 100644
--- a/content/blog/2023/09/12/Lakehouse-or-Warehouse-Part-2-of-2/index.html
+++ b/content/blog/2023/09/12/Lakehouse-or-Warehouse-Part-2-of-2/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2023/09/13/Simplify-operational-data-processing-in-data-lakes-using-AWS-Glue-and-Apache-Hudi/index.html b/content/blog/2023/09/13/Simplify-operational-data-processing-in-data-lakes-using-AWS-Glue-and-Apache-Hudi/index.html
index cb138c177eae4..fa5a7192641af 100644
--- a/content/blog/2023/09/13/Simplify-operational-data-processing-in-data-lakes-using-AWS-Glue-and-Apache-Hudi/index.html
+++ b/content/blog/2023/09/13/Simplify-operational-data-processing-in-data-lakes-using-AWS-Glue-and-Apache-Hudi/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2023/09/15/Apache-Hudi-From-Zero-To-One-blog-3/index.html b/content/blog/2023/09/15/Apache-Hudi-From-Zero-To-One-blog-3/index.html
index 9315d17a88fc2..4e98a356b43d0 100644
--- a/content/blog/2023/09/15/Apache-Hudi-From-Zero-To-One-blog-3/index.html
+++ b/content/blog/2023/09/15/Apache-Hudi-From-Zero-To-One-blog-3/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2023/09/19/A-Beginners-Guide-to-Apache-Hudi-with-PySpark-Part-1-of-2/index.html b/content/blog/2023/09/19/A-Beginners-Guide-to-Apache-Hudi-with-PySpark-Part-1-of-2/index.html
index 7bfcb854c435b..30c565a8f4e60 100644
--- a/content/blog/2023/09/19/A-Beginners-Guide-to-Apache-Hudi-with-PySpark-Part-1-of-2/index.html
+++ b/content/blog/2023/09/19/A-Beginners-Guide-to-Apache-Hudi-with-PySpark-Part-1-of-2/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2023/09/22/Exploring-the-Architecture-of-Apache-Iceberg-Delta-Lake-and-Apache-Hudi/index.html b/content/blog/2023/09/22/Exploring-the-Architecture-of-Apache-Iceberg-Delta-Lake-and-Apache-Hudi/index.html
index e1f6885c527b2..a7ccfce0162dd 100644
--- a/content/blog/2023/09/22/Exploring-the-Architecture-of-Apache-Iceberg-Delta-Lake-and-Apache-Hudi/index.html
+++ b/content/blog/2023/09/22/Exploring-the-Architecture-of-Apache-Iceberg-Delta-Lake-and-Apache-Hudi/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2023/09/27/Apache-Hudi-From-Zero-To-One-blog-4/index.html b/content/blog/2023/09/27/Apache-Hudi-From-Zero-To-One-blog-4/index.html
index c555516ebacd3..3e18a581ee96e 100644
--- a/content/blog/2023/09/27/Apache-Hudi-From-Zero-To-One-blog-4/index.html
+++ b/content/blog/2023/09/27/Apache-Hudi-From-Zero-To-One-blog-4/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2023/10/06/Apache-Hudi-Copy-on-Write-CoW-Table/index.html b/content/blog/2023/10/06/Apache-Hudi-Copy-on-Write-CoW-Table/index.html
index 3b5bf20b136d0..c8be1085be4ce 100644
--- a/content/blog/2023/10/06/Apache-Hudi-Copy-on-Write-CoW-Table/index.html
+++ b/content/blog/2023/10/06/Apache-Hudi-Copy-on-Write-CoW-Table/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2023/10/11/starrocks-query-performance-with-apache-hudi-and-onehouse/index.html b/content/blog/2023/10/11/starrocks-query-performance-with-apache-hudi-and-onehouse/index.html
index 428373333ae7f..ddb5edcce828d 100644
--- a/content/blog/2023/10/11/starrocks-query-performance-with-apache-hudi-and-onehouse/index.html
+++ b/content/blog/2023/10/11/starrocks-query-performance-with-apache-hudi-and-onehouse/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2023/10/17/Get-started-with-Apache-Hudi-using-AWS-Glue-by-implementing-key-design-concepts-Part-1/index.html b/content/blog/2023/10/17/Get-started-with-Apache-Hudi-using-AWS-Glue-by-implementing-key-design-concepts-Part-1/index.html
index c11a04fb2ad56..eaf7fefd36f46 100644
--- a/content/blog/2023/10/17/Get-started-with-Apache-Hudi-using-AWS-Glue-by-implementing-key-design-concepts-Part-1/index.html
+++ b/content/blog/2023/10/17/Get-started-with-Apache-Hudi-using-AWS-Glue-by-implementing-key-design-concepts-Part-1/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2023/10/18/Apache-Hudi-From-Zero-To-One-blog-5/index.html b/content/blog/2023/10/18/Apache-Hudi-From-Zero-To-One-blog-5/index.html
index 6a5ee33c72043..aeeda70086962 100644
--- a/content/blog/2023/10/18/Apache-Hudi-From-Zero-To-One-blog-5/index.html
+++ b/content/blog/2023/10/18/Apache-Hudi-From-Zero-To-One-blog-5/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2023/10/19/load-data-incrementally-from-transactional-data-lakes-to-data-warehouses/index.html b/content/blog/2023/10/19/load-data-incrementally-from-transactional-data-lakes-to-data-warehouses/index.html
index a294e8f36e5c4..e35dcde66434c 100644
--- a/content/blog/2023/10/19/load-data-incrementally-from-transactional-data-lakes-to-data-warehouses/index.html
+++ b/content/blog/2023/10/19/load-data-incrementally-from-transactional-data-lakes-to-data-warehouses/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2023/10/20/Its-Time-for-the-Universal-Data-Lakehouse/index.html b/content/blog/2023/10/20/Its-Time-for-the-Universal-Data-Lakehouse/index.html
index 8fb1f203f4330..ab7879dfa7e5a 100644
--- a/content/blog/2023/10/20/Its-Time-for-the-Universal-Data-Lakehouse/index.html
+++ b/content/blog/2023/10/20/Its-Time-for-the-Universal-Data-Lakehouse/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2023/10/22/Tipico-Facilitates-Faster-Data-Access-with-a-Modern-Data-Strategy-on-AWS/index.html b/content/blog/2023/10/22/Tipico-Facilitates-Faster-Data-Access-with-a-Modern-Data-Strategy-on-AWS/index.html
index 9005a6b99da68..d0b210cb23669 100644
--- a/content/blog/2023/10/22/Tipico-Facilitates-Faster-Data-Access-with-a-Modern-Data-Strategy-on-AWS/index.html
+++ b/content/blog/2023/10/22/Tipico-Facilitates-Faster-Data-Access-with-a-Modern-Data-Strategy-on-AWS/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2023/10/29/UPSERT-Performance-Evaluation-of-Hudi-0-14-and-Spark-3-4-1-Record-Level-Index-Global-Bloom-Global-Simple-Indexes/index.html b/content/blog/2023/10/29/UPSERT-Performance-Evaluation-of-Hudi-0-14-and-Spark-3-4-1-Record-Level-Index-Global-Bloom-Global-Simple-Indexes/index.html
index 4ef906247f2f6..ac5d37bfbe01c 100644
--- a/content/blog/2023/10/29/UPSERT-Performance-Evaluation-of-Hudi-0-14-and-Spark-3-4-1-Record-Level-Index-Global-Bloom-Global-Simple-Indexes/index.html
+++ b/content/blog/2023/10/29/UPSERT-Performance-Evaluation-of-Hudi-0-14-and-Spark-3-4-1-Record-Level-Index-Global-Bloom-Global-Simple-Indexes/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2023/11/01/record-level-index/index.html b/content/blog/2023/11/01/record-level-index/index.html
index 212e6f1ba3036..71f01d9a089e2 100644
--- a/content/blog/2023/11/01/record-level-index/index.html
+++ b/content/blog/2023/11/01/record-level-index/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2023/11/13/Apache-Hudi-From-Zero-To-One-blog-6/index.html b/content/blog/2023/11/13/Apache-Hudi-From-Zero-To-One-blog-6/index.html
index 9542c777af526..d3c01acd409f1 100644
--- a/content/blog/2023/11/13/Apache-Hudi-From-Zero-To-One-blog-6/index.html
+++ b/content/blog/2023/11/13/Apache-Hudi-From-Zero-To-One-blog-6/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2023/11/19/Hudi-Streamer-DeltaStreamer-Hands-On-Guide-Local-Ingestion-from-Parquet-Source/index.html b/content/blog/2023/11/19/Hudi-Streamer-DeltaStreamer-Hands-On-Guide-Local-Ingestion-from-Parquet-Source/index.html
index 618cd39508cf4..2e422fa11d6d6 100644
--- a/content/blog/2023/11/19/Hudi-Streamer-DeltaStreamer-Hands-On-Guide-Local-Ingestion-from-Parquet-Source/index.html
+++ b/content/blog/2023/11/19/Hudi-Streamer-DeltaStreamer-Hands-On-Guide-Local-Ingestion-from-Parquet-Source/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2023/11/22/Introducing-Apache-Hudi-support-with-AWS-Glue-crawlers/index.html b/content/blog/2023/11/22/Introducing-Apache-Hudi-support-with-AWS-Glue-crawlers/index.html
index 134d8fc1b43ec..148a85c5f9358 100644
--- a/content/blog/2023/11/22/Introducing-Apache-Hudi-support-with-AWS-Glue-crawlers/index.html
+++ b/content/blog/2023/11/22/Introducing-Apache-Hudi-support-with-AWS-Glue-crawlers/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2023/11/26/Real-Time-Data-Processing-with-Postgres-Debezium-Kafka-Schema-Registry-and-DeltaStreamer-Guide-for-Begineers/index.html b/content/blog/2023/11/26/Real-Time-Data-Processing-with-Postgres-Debezium-Kafka-Schema-Registry-and-DeltaStreamer-Guide-for-Begineers/index.html
index 7ea003220d742..a1b65fa6bfa00 100644
--- a/content/blog/2023/11/26/Real-Time-Data-Processing-with-Postgres-Debezium-Kafka-Schema-Registry-and-DeltaStreamer-Guide-for-Begineers/index.html
+++ b/content/blog/2023/11/26/Real-Time-Data-Processing-with-Postgres-Debezium-Kafka-Schema-Registry-and-DeltaStreamer-Guide-for-Begineers/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2023/11/28/Apache-Hudi-Part-1-History-Getting-Started/index.html b/content/blog/2023/11/28/Apache-Hudi-Part-1-History-Getting-Started/index.html
index dfbb1b8f38321..efa7bde52a074 100644
--- a/content/blog/2023/11/28/Apache-Hudi-Part-1-History-Getting-Started/index.html
+++ b/content/blog/2023/11/28/Apache-Hudi-Part-1-History-Getting-Started/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2023/11/30/Mastering-Data-Lakes-A-Deep-Dive-into-MINIO-Hudi-and-Delta-Streamer/index.html b/content/blog/2023/11/30/Mastering-Data-Lakes-A-Deep-Dive-into-MINIO-Hudi-and-Delta-Streamer/index.html
index 1e0c9d7cc3619..67326a61ae614 100644
--- a/content/blog/2023/11/30/Mastering-Data-Lakes-A-Deep-Dive-into-MINIO-Hudi-and-Delta-Streamer/index.html
+++ b/content/blog/2023/11/30/Mastering-Data-Lakes-A-Deep-Dive-into-MINIO-Hudi-and-Delta-Streamer/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2023/12/01/Getting-started-with-Apache-Hudi/index.html b/content/blog/2023/12/01/Getting-started-with-Apache-Hudi/index.html
index 97e65407e60f2..6f8cbf83e32e6 100644
--- a/content/blog/2023/12/01/Getting-started-with-Apache-Hudi/index.html
+++ b/content/blog/2023/12/01/Getting-started-with-Apache-Hudi/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2023/12/06/Apache-Hudi-From-Zero-To-One-blog-7/index.html b/content/blog/2023/12/06/Apache-Hudi-From-Zero-To-One-blog-7/index.html
index 9859ec1281a46..765aced325b51 100644
--- a/content/blog/2023/12/06/Apache-Hudi-From-Zero-To-One-blog-7/index.html
+++ b/content/blog/2023/12/06/Apache-Hudi-From-Zero-To-One-blog-7/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2023/12/09/Getting-started-with-Apache-Hudi/index.html b/content/blog/2023/12/09/Getting-started-with-Apache-Hudi/index.html
index c7104f61766d2..73f9b8d0c5f0a 100644
--- a/content/blog/2023/12/09/Getting-started-with-Apache-Hudi/index.html
+++ b/content/blog/2023/12/09/Getting-started-with-Apache-Hudi/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2023/12/13/what-is-apache-hudi/index.html b/content/blog/2023/12/13/what-is-apache-hudi/index.html
index 2e1db31ea825d..9e127cf157d0f 100644
--- a/content/blog/2023/12/13/what-is-apache-hudi/index.html
+++ b/content/blog/2023/12/13/what-is-apache-hudi/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2023/12/28/apache-hudi-2023-a-year-in-review/index.html b/content/blog/2023/12/28/apache-hudi-2023-a-year-in-review/index.html
index 2067a221d49fb..695ac756b5061 100644
--- a/content/blog/2023/12/28/apache-hudi-2023-a-year-in-review/index.html
+++ b/content/blog/2023/12/28/apache-hudi-2023-a-year-in-review/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2024/01/01/From-Data-lake-to-Microservices-Unleashing-the-Power-of-Apache-Hudi-Record-Level-Index-with-FastAPI-and-Spark-Connect/index.html b/content/blog/2024/01/01/From-Data-lake-to-Microservices-Unleashing-the-Power-of-Apache-Hudi-Record-Level-Index-with-FastAPI-and-Spark-Connect/index.html
index fa99fc19d9da8..1c5f0308ce8a9 100644
--- a/content/blog/2024/01/01/From-Data-lake-to-Microservices-Unleashing-the-Power-of-Apache-Hudi-Record-Level-Index-with-FastAPI-and-Spark-Connect/index.html
+++ b/content/blog/2024/01/01/From-Data-lake-to-Microservices-Unleashing-the-Power-of-Apache-Hudi-Record-Level-Index-with-FastAPI-and-Spark-Connect/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2024/01/02/Build-a-federated-query-solution-with-Apache-Doris-Apache-Flink-and-Apache-Hudi/index.html b/content/blog/2024/01/02/Build-a-federated-query-solution-with-Apache-Doris-Apache-Flink-and-Apache-Hudi/index.html
index bac75396db713..e3bdc95546d8e 100644
--- a/content/blog/2024/01/02/Build-a-federated-query-solution-with-Apache-Doris-Apache-Flink-and-Apache-Hudi/index.html
+++ b/content/blog/2024/01/02/Build-a-federated-query-solution-with-Apache-Doris-Apache-Flink-and-Apache-Hudi/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2024/01/05/Small-Talk-about-Apache-Hudi/index.html b/content/blog/2024/01/05/Small-Talk-about-Apache-Hudi/index.html
index ebda285cac18c..24b99cfc5c95a 100644
--- a/content/blog/2024/01/05/Small-Talk-about-Apache-Hudi/index.html
+++ b/content/blog/2024/01/05/Small-Talk-about-Apache-Hudi/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2024/01/09/introduction-to-apache-hudi/index.html b/content/blog/2024/01/09/introduction-to-apache-hudi/index.html
index 929566bcff286..7206f4711039a 100644
--- a/content/blog/2024/01/09/introduction-to-apache-hudi/index.html
+++ b/content/blog/2024/01/09/introduction-to-apache-hudi/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2024/01/11/In-House-Data-Lake-with-CDC-Processing-Hudi-Docker/index.html b/content/blog/2024/01/11/In-House-Data-Lake-with-CDC-Processing-Hudi-Docker/index.html
index 3ca52baae8f35..2ff3e7def2ee1 100644
--- a/content/blog/2024/01/11/In-House-Data-Lake-with-CDC-Processing-Hudi-Docker/index.html
+++ b/content/blog/2024/01/11/In-House-Data-Lake-with-CDC-Processing-Hudi-Docker/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2024/01/17/Enforce-fine-grained-access-control-on-Open-Table-Formats-via-Amazon-EMR-integrated-with-AWS-Lake-Formation/index.html b/content/blog/2024/01/17/Enforce-fine-grained-access-control-on-Open-Table-Formats-via-Amazon-EMR-integrated-with-AWS-Lake-Formation/index.html
index b6fd14e259efd..0284f9e243b2d 100644
--- a/content/blog/2024/01/17/Enforce-fine-grained-access-control-on-Open-Table-Formats-via-Amazon-EMR-integrated-with-AWS-Lake-Formation/index.html
+++ b/content/blog/2024/01/17/Enforce-fine-grained-access-control-on-Open-Table-Formats-via-Amazon-EMR-integrated-with-AWS-Lake-Formation/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2024/01/18/Deleting-Items-from-Apache-Hudi-using-Delta-Streamer-in-UPSERT-Mode-with-Kafka-Avro-Messages/index.html b/content/blog/2024/01/18/Deleting-Items-from-Apache-Hudi-using-Delta-Streamer-in-UPSERT-Mode-with-Kafka-Avro-Messages/index.html
index 8d69ab8caf38c..26719e8cc49b1 100644
--- a/content/blog/2024/01/18/Deleting-Items-from-Apache-Hudi-using-Delta-Streamer-in-UPSERT-Mode-with-Kafka-Avro-Messages/index.html
+++ b/content/blog/2024/01/18/Deleting-Items-from-Apache-Hudi-using-Delta-Streamer-in-UPSERT-Mode-with-Kafka-Avro-Messages/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2024/01/20/Data-Engineering-Bootstrapping-Data-lake-with-Apache-Hudi/index.html b/content/blog/2024/01/20/Data-Engineering-Bootstrapping-Data-lake-with-Apache-Hudi/index.html
index a9f8b85abe859..da4568d019b25 100644
--- a/content/blog/2024/01/20/Data-Engineering-Bootstrapping-Data-lake-with-Apache-Hudi/index.html
+++ b/content/blog/2024/01/20/Data-Engineering-Bootstrapping-Data-lake-with-Apache-Hudi/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2024/01/20/Learn-How-to-Move-Data-From-MongoDB-to-Apache-Hudi-Using-PySpark/index.html b/content/blog/2024/01/20/Learn-How-to-Move-Data-From-MongoDB-to-Apache-Hudi-Using-PySpark/index.html
index eca230ed3882f..80297b9d9c365 100644
--- a/content/blog/2024/01/20/Learn-How-to-Move-Data-From-MongoDB-to-Apache-Hudi-Using-PySpark/index.html
+++ b/content/blog/2024/01/20/Learn-How-to-Move-Data-From-MongoDB-to-Apache-Hudi-Using-PySpark/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2024/01/24/Use-Amazon-Athena-with-Spark-SQL-for-your-open-source-transactional-table-formats/index.html b/content/blog/2024/01/24/Use-Amazon-Athena-with-Spark-SQL-for-your-open-source-transactional-table-formats/index.html
index 28cbceb261a73..1f6929290a987 100644
--- a/content/blog/2024/01/24/Use-Amazon-Athena-with-Spark-SQL-for-your-open-source-transactional-table-formats/index.html
+++ b/content/blog/2024/01/24/Use-Amazon-Athena-with-Spark-SQL-for-your-open-source-transactional-table-formats/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2024/01/30/Leverage-Partition-Paths-of-your-data-lake-tables-to-Optimize-Data-Retrieval-Costs-on-the-cloud/index.html b/content/blog/2024/01/30/Leverage-Partition-Paths-of-your-data-lake-tables-to-Optimize-Data-Retrieval-Costs-on-the-cloud/index.html
index 54f30b1c21589..f75b1bd13410a 100644
--- a/content/blog/2024/01/30/Leverage-Partition-Paths-of-your-data-lake-tables-to-Optimize-Data-Retrieval-Costs-on-the-cloud/index.html
+++ b/content/blog/2024/01/30/Leverage-Partition-Paths-of-your-data-lake-tables-to-Optimize-Data-Retrieval-Costs-on-the-cloud/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2024/02/04/Apache-Hudi-Managing-Partition-on-a-petabyte-scale-table/index.html b/content/blog/2024/02/04/Apache-Hudi-Managing-Partition-on-a-petabyte-scale-table/index.html
index 62d2de18b9684..ce0444cc36dc3 100644
--- a/content/blog/2024/02/04/Apache-Hudi-Managing-Partition-on-a-petabyte-scale-table/index.html
+++ b/content/blog/2024/02/04/Apache-Hudi-Managing-Partition-on-a-petabyte-scale-table/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2024/02/06/Building-an-Open-Source-Data-Lake-House-with-Hudi-Postgres-Hive-Metastore-Minio-and-StarRocks/index.html b/content/blog/2024/02/06/Building-an-Open-Source-Data-Lake-House-with-Hudi-Postgres-Hive-Metastore-Minio-and-StarRocks/index.html
index b152d16be11d1..1039479155933 100644
--- a/content/blog/2024/02/06/Building-an-Open-Source-Data-Lake-House-with-Hudi-Postgres-Hive-Metastore-Minio-and-StarRocks/index.html
+++ b/content/blog/2024/02/06/Building-an-Open-Source-Data-Lake-House-with-Hudi-Postgres-Hive-Metastore-Minio-and-StarRocks/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2024/02/06/Combine-Transactional-Integrity-and-Data-Lake-Operations-with-YugabyteDB-and-Apache-Hudi/index.html b/content/blog/2024/02/06/Combine-Transactional-Integrity-and-Data-Lake-Operations-with-YugabyteDB-and-Apache-Hudi/index.html
index e52aeba29fbab..2cca89be3b16e 100644
--- a/content/blog/2024/02/06/Combine-Transactional-Integrity-and-Data-Lake-Operations-with-YugabyteDB-and-Apache-Hudi/index.html
+++ b/content/blog/2024/02/06/Combine-Transactional-Integrity-and-Data-Lake-Operations-with-YugabyteDB-and-Apache-Hudi/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2024/02/12/How-a-POC-became-a-production-ready-Hudi-data-lakehouse-through-close-team-collaboration/index.html b/content/blog/2024/02/12/How-a-POC-became-a-production-ready-Hudi-data-lakehouse-through-close-team-collaboration/index.html
index 7afbc7009662c..a18d184e07a6c 100644
--- a/content/blog/2024/02/12/How-a-POC-became-a-production-ready-Hudi-data-lakehouse-through-close-team-collaboration/index.html
+++ b/content/blog/2024/02/12/How-a-POC-became-a-production-ready-Hudi-data-lakehouse-through-close-team-collaboration/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2024/02/23/Enabling-near-real-time-data-analytics-on-the-data-lake/index.html b/content/blog/2024/02/23/Enabling-near-real-time-data-analytics-on-the-data-lake/index.html
index 3a587bff4db4c..5b0c81aafecaa 100644
--- a/content/blog/2024/02/23/Enabling-near-real-time-data-analytics-on-the-data-lake/index.html
+++ b/content/blog/2024/02/23/Enabling-near-real-time-data-analytics-on-the-data-lake/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2024/02/27/Building-Data-Lakes-on-AWS-with-Kafka-Connect-Debezium-Apicurio-Registry-and-Apache-Hudi/index.html b/content/blog/2024/02/27/Building-Data-Lakes-on-AWS-with-Kafka-Connect-Debezium-Apicurio-Registry-and-Apache-Hudi/index.html
index 53cd364c108c1..41892f0c4d82b 100644
--- a/content/blog/2024/02/27/Building-Data-Lakes-on-AWS-with-Kafka-Connect-Debezium-Apicurio-Registry-and-Apache-Hudi/index.html
+++ b/content/blog/2024/02/27/Building-Data-Lakes-on-AWS-with-Kafka-Connect-Debezium-Apicurio-Registry-and-Apache-Hudi/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2024/02/27/empowering-data-driven-excellence-how-the-bluestone-data-platform-embraced-data-mesh-for-success/index.html b/content/blog/2024/02/27/empowering-data-driven-excellence-how-the-bluestone-data-platform-embraced-data-mesh-for-success/index.html
index 3907ecbd05b0b..20fb67cfbd41d 100644
--- a/content/blog/2024/02/27/empowering-data-driven-excellence-how-the-bluestone-data-platform-embraced-data-mesh-for-success/index.html
+++ b/content/blog/2024/02/27/empowering-data-driven-excellence-how-the-bluestone-data-platform-embraced-data-mesh-for-success/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2024/03/05/Apache-Hudi-From-Zero-To-One-blog-9/index.html b/content/blog/2024/03/05/Apache-Hudi-From-Zero-To-One-blog-9/index.html
index 46b271ea99099..800ab8f8b82f4 100644
--- a/content/blog/2024/03/05/Apache-Hudi-From-Zero-To-One-blog-9/index.html
+++ b/content/blog/2024/03/05/Apache-Hudi-From-Zero-To-One-blog-9/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2024/03/10/navigating-the-future-the-evolutionary-journey-of-upstoxs-data-platform/index.html b/content/blog/2024/03/10/navigating-the-future-the-evolutionary-journey-of-upstoxs-data-platform/index.html
index c9f0fabd34af7..42a11b1a28001 100644
--- a/content/blog/2024/03/10/navigating-the-future-the-evolutionary-journey-of-upstoxs-data-platform/index.html
+++ b/content/blog/2024/03/10/navigating-the-future-the-evolutionary-journey-of-upstoxs-data-platform/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2024/03/14/Modern-Datalakes-with-Hudi--MinIO--and-HMS/index.html b/content/blog/2024/03/14/Modern-Datalakes-with-Hudi--MinIO--and-HMS/index.html
index d742eb618d60b..016c0e8080354 100644
--- a/content/blog/2024/03/14/Modern-Datalakes-with-Hudi--MinIO--and-HMS/index.html
+++ b/content/blog/2024/03/14/Modern-Datalakes-with-Hudi--MinIO--and-HMS/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2024/03/16/Open-Table-Formats-part-1-Apache-Hudi-Hadoop-Upserts-Deletes-and-Incrementals/index.html b/content/blog/2024/03/16/Open-Table-Formats-part-1-Apache-Hudi-Hadoop-Upserts-Deletes-and-Incrementals/index.html
index ababc73184894..7f2781eb6c18d 100644
--- a/content/blog/2024/03/16/Open-Table-Formats-part-1-Apache-Hudi-Hadoop-Upserts-Deletes-and-Incrementals/index.html
+++ b/content/blog/2024/03/16/Open-Table-Formats-part-1-Apache-Hudi-Hadoop-Upserts-Deletes-and-Incrementals/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2024/03/22/data-lake-cost-optimisation-strategies/index.html b/content/blog/2024/03/22/data-lake-cost-optimisation-strategies/index.html
index dd1b2a4f65c5d..cb4b98cf229b2 100644
--- a/content/blog/2024/03/22/data-lake-cost-optimisation-strategies/index.html
+++ b/content/blog/2024/03/22/data-lake-cost-optimisation-strategies/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2024/03/23/options-on-kafka-sink-to-open-table-formats-apache-iceberg-and-apache-hudi/index.html b/content/blog/2024/03/23/options-on-kafka-sink-to-open-table-formats-apache-iceberg-and-apache-hudi/index.html
index 04855f369a11a..f049b364bd023 100644
--- a/content/blog/2024/03/23/options-on-kafka-sink-to-open-table-formats-apache-iceberg-and-apache-hudi/index.html
+++ b/content/blog/2024/03/23/options-on-kafka-sink-to-open-table-formats-apache-iceberg-and-apache-hudi/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2024/03/30/record-level-indexing-apache-hudi-delivers-70-faster-point/index.html b/content/blog/2024/03/30/record-level-indexing-apache-hudi-delivers-70-faster-point/index.html
index ce49aa5f200dc..3c7af4e81b3d8 100644
--- a/content/blog/2024/03/30/record-level-indexing-apache-hudi-delivers-70-faster-point/index.html
+++ b/content/blog/2024/03/30/record-level-indexing-apache-hudi-delivers-70-faster-point/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2024/04/03/hands-on-guide-reading-data-from-hudi-tables-joining-delta/index.html b/content/blog/2024/04/03/hands-on-guide-reading-data-from-hudi-tables-joining-delta/index.html
index 91ee677da8a0b..1adccc7319880 100644
--- a/content/blog/2024/04/03/hands-on-guide-reading-data-from-hudi-tables-joining-delta/index.html
+++ b/content/blog/2024/04/03/hands-on-guide-reading-data-from-hudi-tables-joining-delta/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2024/04/21/build-real-time-streaming-pipeline-with-kinesis-apache-flink-and-apache-hudi/index.html b/content/blog/2024/04/21/build-real-time-streaming-pipeline-with-kinesis-apache-flink-and-apache-hudi/index.html
index 734ac5e24beed..44b01349dc970 100644
--- a/content/blog/2024/04/21/build-real-time-streaming-pipeline-with-kinesis-apache-flink-and-apache-hudi/index.html
+++ b/content/blog/2024/04/21/build-real-time-streaming-pipeline-with-kinesis-apache-flink-and-apache-hudi/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2024/04/24/understanding-apache-hudi-consistency-model-part-1/index.html b/content/blog/2024/04/24/understanding-apache-hudi-consistency-model-part-1/index.html
index 891bca3093c5c..d46ad69ebfd7b 100644
--- a/content/blog/2024/04/24/understanding-apache-hudi-consistency-model-part-1/index.html
+++ b/content/blog/2024/04/24/understanding-apache-hudi-consistency-model-part-1/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2024/04/24/understanding-apache-hudi-consistency-model-part-2/index.html b/content/blog/2024/04/24/understanding-apache-hudi-consistency-model-part-2/index.html
index c9eaa34ee34c0..487164108a14b 100644
--- a/content/blog/2024/04/24/understanding-apache-hudi-consistency-model-part-2/index.html
+++ b/content/blog/2024/04/24/understanding-apache-hudi-consistency-model-part-2/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2024/04/24/understanding-apache-hudi-consistency-model-part-3/index.html b/content/blog/2024/04/24/understanding-apache-hudi-consistency-model-part-3/index.html
index 23b59d95d6f4b..6c9f01be27ac2 100644
--- a/content/blog/2024/04/24/understanding-apache-hudi-consistency-model-part-3/index.html
+++ b/content/blog/2024/04/24/understanding-apache-hudi-consistency-model-part-3/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2024/04/25/apache-hudi-vs-apache-iceberg-a-comprehensive-comparison/index.html b/content/blog/2024/04/25/apache-hudi-vs-apache-iceberg-a-comprehensive-comparison/index.html
index 6fcc8b2900157..f012dcc16894c 100644
--- a/content/blog/2024/04/25/apache-hudi-vs-apache-iceberg-a-comprehensive-comparison/index.html
+++ b/content/blog/2024/04/25/apache-hudi-vs-apache-iceberg-a-comprehensive-comparison/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2024/05/02/how-query-apache-hudi-tables-python-using-daft-spark-free/index.html b/content/blog/2024/05/02/how-query-apache-hudi-tables-python-using-daft-spark-free/index.html
index d05acf31359f0..6311b9558b222 100644
--- a/content/blog/2024/05/02/how-query-apache-hudi-tables-python-using-daft-spark-free/index.html
+++ b/content/blog/2024/05/02/how-query-apache-hudi-tables-python-using-daft-spark-free/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2024/05/07/learn-how-read-hudi-data-aws-glue-ray-using-daft-spark/index.html b/content/blog/2024/05/07/learn-how-read-hudi-data-aws-glue-ray-using-daft-spark/index.html
index ba720f89e3e01..5a74a5a47f083 100644
--- a/content/blog/2024/05/07/learn-how-read-hudi-data-aws-glue-ray-using-daft-spark/index.html
+++ b/content/blog/2024/05/07/learn-how-read-hudi-data-aws-glue-ray-using-daft-spark/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2024/05/10/building-analytical-apps-on-the-lakehouse-using-apache-hudi-daft-streamlit/index.html b/content/blog/2024/05/10/building-analytical-apps-on-the-lakehouse-using-apache-hudi-daft-streamlit/index.html
index 79a16f5f35786..ab18843f094d4 100644
--- a/content/blog/2024/05/10/building-analytical-apps-on-the-lakehouse-using-apache-hudi-daft-streamlit/index.html
+++ b/content/blog/2024/05/10/building-analytical-apps-on-the-lakehouse-using-apache-hudi-daft-streamlit/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2024/05/19/apache-hudi-on-aws-glue/index.html b/content/blog/2024/05/19/apache-hudi-on-aws-glue/index.html
index 155a897257266..40f5b8dc2ce2c 100644
--- a/content/blog/2024/05/19/apache-hudi-on-aws-glue/index.html
+++ b/content/blog/2024/05/19/apache-hudi-on-aws-glue/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2024/05/22/use-aws-data-exchange-to-seamlessly-share-apache-hudi-datasets/index.html b/content/blog/2024/05/22/use-aws-data-exchange-to-seamlessly-share-apache-hudi-datasets/index.html
index c3add62568fbd..a48336a9d9d40 100644
--- a/content/blog/2024/05/22/use-aws-data-exchange-to-seamlessly-share-apache-hudi-datasets/index.html
+++ b/content/blog/2024/05/22/use-aws-data-exchange-to-seamlessly-share-apache-hudi-datasets/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2024/05/27/apache-hudi-vs-delta-lake-choosing-the-right-tool-for-your-data-lake-on-aws/index.html b/content/blog/2024/05/27/apache-hudi-vs-delta-lake-choosing-the-right-tool-for-your-data-lake-on-aws/index.html
index 2af27b5846851..b80f776f6c1d6 100644
--- a/content/blog/2024/05/27/apache-hudi-vs-delta-lake-choosing-the-right-tool-for-your-data-lake-on-aws/index.html
+++ b/content/blog/2024/05/27/apache-hudi-vs-delta-lake-choosing-the-right-tool-for-your-data-lake-on-aws/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2024/06/07/apache-hudi-a-deep-dive-with-python-code-examples/index.html b/content/blog/2024/06/07/apache-hudi-a-deep-dive-with-python-code-examples/index.html
index 3810d07ec1e62..a250c7420f2a8 100644
--- a/content/blog/2024/06/07/apache-hudi-a-deep-dive-with-python-code-examples/index.html
+++ b/content/blog/2024/06/07/apache-hudi-a-deep-dive-with-python-code-examples/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2024/06/18/how-to-use-apache-hudi-with-databricks/index.html b/content/blog/2024/06/18/how-to-use-apache-hudi-with-databricks/index.html
index 313ffc42339ee..eaddc8da1addb 100644
--- a/content/blog/2024/06/18/how-to-use-apache-hudi-with-databricks/index.html
+++ b/content/blog/2024/06/18/how-to-use-apache-hudi-with-databricks/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2024/07/11/what-is-a-data-lakehouse/index.html b/content/blog/2024/07/11/what-is-a-data-lakehouse/index.html
index d4f6ff5743971..fbd11ab84bddb 100644
--- a/content/blog/2024/07/11/what-is-a-data-lakehouse/index.html
+++ b/content/blog/2024/07/11/what-is-a-data-lakehouse/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2024/07/30/data-lake-cdc/index.html b/content/blog/2024/07/30/data-lake-cdc/index.html
index c9a41e540b6c4..e76d4e76cdc6b 100644
--- a/content/blog/2024/07/30/data-lake-cdc/index.html
+++ b/content/blog/2024/07/30/data-lake-cdc/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2024/07/31/hudi-file-formats/index.html b/content/blog/2024/07/31/hudi-file-formats/index.html
index b4d473dbd6fc9..f9dc7bf58741a 100644
--- a/content/blog/2024/07/31/hudi-file-formats/index.html
+++ b/content/blog/2024/07/31/hudi-file-formats/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2024/09/04/developer-guide-how-to-submit-hudi-pyspark-python-jobs-to-emr-serverless/index.html b/content/blog/2024/09/04/developer-guide-how-to-submit-hudi-pyspark-python-jobs-to-emr-serverless/index.html
index 9697b7051d85c..946c40a3ddd13 100644
--- a/content/blog/2024/09/04/developer-guide-how-to-submit-hudi-pyspark-python-jobs-to-emr-serverless/index.html
+++ b/content/blog/2024/09/04/developer-guide-how-to-submit-hudi-pyspark-python-jobs-to-emr-serverless/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2024/09/09/use-apache-hudi-tables-in-athena-for-spark/index.html b/content/blog/2024/09/09/use-apache-hudi-tables-in-athena-for-spark/index.html
index 8776c8929f2fc..c70be15483ee5 100644
--- a/content/blog/2024/09/09/use-apache-hudi-tables-in-athena-for-spark/index.html
+++ b/content/blog/2024/09/09/use-apache-hudi-tables-in-athena-for-spark/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2024/09/11/comparing-apache-hudi-apache-iceberg-and-delta-lake/index.html b/content/blog/2024/09/11/comparing-apache-hudi-apache-iceberg-and-delta-lake/index.html
index 3e2adf426c10f..8c0579f53b457 100644
--- a/content/blog/2024/09/11/comparing-apache-hudi-apache-iceberg-and-delta-lake/index.html
+++ b/content/blog/2024/09/11/comparing-apache-hudi-apache-iceberg-and-delta-lake/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2024/09/14/Ubers-Big-Data-Revolution-From-MySQL-to-Hadoop-and-Beyond/index.html b/content/blog/2024/09/14/Ubers-Big-Data-Revolution-From-MySQL-to-Hadoop-and-Beyond/index.html
index b1762bbcdff98..fe043291f93e4 100644
--- a/content/blog/2024/09/14/Ubers-Big-Data-Revolution-From-MySQL-to-Hadoop-and-Beyond/index.html
+++ b/content/blog/2024/09/14/Ubers-Big-Data-Revolution-From-MySQL-to-Hadoop-and-Beyond/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2024/09/17/how-apache-hudi-transformed-yuno-s-data-lake/index.html b/content/blog/2024/09/17/how-apache-hudi-transformed-yuno-s-data-lake/index.html
index c71bd71c31247..0a8e6e1ce1260 100644
--- a/content/blog/2024/09/17/how-apache-hudi-transformed-yuno-s-data-lake/index.html
+++ b/content/blog/2024/09/17/how-apache-hudi-transformed-yuno-s-data-lake/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2024/09/22/hands-on-with-apache-hudi-and-spark/index.html b/content/blog/2024/09/22/hands-on-with-apache-hudi-and-spark/index.html
index ea880a3eeace8..900b07174b444 100644
--- a/content/blog/2024/09/22/hands-on-with-apache-hudi-and-spark/index.html
+++ b/content/blog/2024/09/22/hands-on-with-apache-hudi-and-spark/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2024/09/24/hudi-iceberg-and-delta-lake-data-lake-table-formats-compared/index.html b/content/blog/2024/09/24/hudi-iceberg-and-delta-lake-data-lake-table-formats-compared/index.html
index e5ae3f88c0be4..798df06842c60 100644
--- a/content/blog/2024/09/24/hudi-iceberg-and-delta-lake-data-lake-table-formats-compared/index.html
+++ b/content/blog/2024/09/24/hudi-iceberg-and-delta-lake-data-lake-table-formats-compared/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2024/09/30/change-query-support-in-apache-hudi-0-15/index.html b/content/blog/2024/09/30/change-query-support-in-apache-hudi-0-15/index.html
index 63a019d9a59d5..15e9039a47d91 100644
--- a/content/blog/2024/09/30/change-query-support-in-apache-hudi-0-15/index.html
+++ b/content/blog/2024/09/30/change-query-support-in-apache-hudi-0-15/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2024/10/02/apache-hudi-spark-and-minio-hands-on-lab-in-docker/index.html b/content/blog/2024/10/02/apache-hudi-spark-and-minio-hands-on-lab-in-docker/index.html
index 7858b5bb9ae91..dd15134d15a4e 100644
--- a/content/blog/2024/10/02/apache-hudi-spark-and-minio-hands-on-lab-in-docker/index.html
+++ b/content/blog/2024/10/02/apache-hudi-spark-and-minio-hands-on-lab-in-docker/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2024/10/07/iceberg-vs-delta-lake-vs-hudi-a-comparative-look-at-lakehouse-architectures/index.html b/content/blog/2024/10/07/iceberg-vs-delta-lake-vs-hudi-a-comparative-look-at-lakehouse-architectures/index.html
index 9bb59f7fffd01..d756e2f1e72e9 100644
--- a/content/blog/2024/10/07/iceberg-vs-delta-lake-vs-hudi-a-comparative-look-at-lakehouse-architectures/index.html
+++ b/content/blog/2024/10/07/iceberg-vs-delta-lake-vs-hudi-a-comparative-look-at-lakehouse-architectures/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2024/10/07/mastering-slowly-changing-dimensions-with-apache-hudi-and-spark-sql/index.html b/content/blog/2024/10/07/mastering-slowly-changing-dimensions-with-apache-hudi-and-spark-sql/index.html
index a930d0d6cd51e..c072648e4477d 100644
--- a/content/blog/2024/10/07/mastering-slowly-changing-dimensions-with-apache-hudi-and-spark-sql/index.html
+++ b/content/blog/2024/10/07/mastering-slowly-changing-dimensions-with-apache-hudi-and-spark-sql/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2024/10/14/streaming-dynamodb-data-into-a-hudi-table-aws-glue-in-action/index.html b/content/blog/2024/10/14/streaming-dynamodb-data-into-a-hudi-table-aws-glue-in-action/index.html
index 1df77a5ffff47..d7285173558d4 100644
--- a/content/blog/2024/10/14/streaming-dynamodb-data-into-a-hudi-table-aws-glue-in-action/index.html
+++ b/content/blog/2024/10/14/streaming-dynamodb-data-into-a-hudi-table-aws-glue-in-action/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2024/10/22/exploring-time-travel-queries-in-apache-hudi/index.html b/content/blog/2024/10/22/exploring-time-travel-queries-in-apache-hudi/index.html
index 87ea7998c277c..855e9c54b056b 100644
--- a/content/blog/2024/10/22/exploring-time-travel-queries-in-apache-hudi/index.html
+++ b/content/blog/2024/10/22/exploring-time-travel-queries-in-apache-hudi/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2024/10/23/Using-Apache-Hudi-with-Apache-Flink/index.html b/content/blog/2024/10/23/Using-Apache-Hudi-with-Apache-Flink/index.html
index 4412db9957369..fd9e2095dc91a 100644
--- a/content/blog/2024/10/23/Using-Apache-Hudi-with-Apache-Flink/index.html
+++ b/content/blog/2024/10/23/Using-Apache-Hudi-with-Apache-Flink/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2024/10/23/mastering-open-table-formats-a-guide-to-apache-iceberg-hudi-and-delta-lake/index.html b/content/blog/2024/10/23/mastering-open-table-formats-a-guide-to-apache-iceberg-hudi-and-delta-lake/index.html
index 1d17cd8ecce3e..9df9ee05b1395 100644
--- a/content/blog/2024/10/23/mastering-open-table-formats-a-guide-to-apache-iceberg-hudi-and-delta-lake/index.html
+++ b/content/blog/2024/10/23/mastering-open-table-formats-a-guide-to-apache-iceberg-hudi-and-delta-lake/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2024/10/26/moving-large-tables-from-snowflake-to-s3-using-the-copy-into-command-and-hudi/index.html b/content/blog/2024/10/26/moving-large-tables-from-snowflake-to-s3-using-the-copy-into-command-and-hudi/index.html
index a1bbfedb25dc5..9cf42efa490a3 100644
--- a/content/blog/2024/10/26/moving-large-tables-from-snowflake-to-s3-using-the-copy-into-command-and-hudi/index.html
+++ b/content/blog/2024/10/26/moving-large-tables-from-snowflake-to-s3-using-the-copy-into-command-and-hudi/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2024/10/27/I-spent-5-hours-exploring-the-story-behind-Apache-Hudi/index.html b/content/blog/2024/10/27/I-spent-5-hours-exploring-the-story-behind-Apache-Hudi/index.html
index 5a952c0e468bd..dac82cdd83d21 100644
--- a/content/blog/2024/10/27/I-spent-5-hours-exploring-the-story-behind-Apache-Hudi/index.html
+++ b/content/blog/2024/10/27/I-spent-5-hours-exploring-the-story-behind-Apache-Hudi/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2024/11/12/record-level-indexing-in-apache-hudi/index.html b/content/blog/2024/11/12/record-level-indexing-in-apache-hudi/index.html
index 162b0f94c7e4e..a175234c66ee3 100644
--- a/content/blog/2024/11/12/record-level-indexing-in-apache-hudi/index.html
+++ b/content/blog/2024/11/12/record-level-indexing-in-apache-hudi/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2024/11/12/storing-200-billion-entities-notions/index.html b/content/blog/2024/11/12/storing-200-billion-entities-notions/index.html
index f165080235fa9..322d47132dcd4 100644
--- a/content/blog/2024/11/12/storing-200-billion-entities-notions/index.html
+++ b/content/blog/2024/11/12/storing-200-billion-entities-notions/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2024/11/12/understanding-cow-and-mor-in-apache-hudi/index.html b/content/blog/2024/11/12/understanding-cow-and-mor-in-apache-hudi/index.html
index 7dcddf0f71f84..349f47d85a90a 100644
--- a/content/blog/2024/11/12/understanding-cow-and-mor-in-apache-hudi/index.html
+++ b/content/blog/2024/11/12/understanding-cow-and-mor-in-apache-hudi/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2024/11/19/automated-small-file-handling/index.html b/content/blog/2024/11/19/automated-small-file-handling/index.html
index f2bb33dd98aaf..390997ab51a3c 100644
--- a/content/blog/2024/11/19/automated-small-file-handling/index.html
+++ b/content/blog/2024/11/19/automated-small-file-handling/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2024/12/06/non-blocking-concurrency-control/index.html b/content/blog/2024/12/06/non-blocking-concurrency-control/index.html
index 51b6e517fc84e..be8ba62b5013b 100644
--- a/content/blog/2024/12/06/non-blocking-concurrency-control/index.html
+++ b/content/blog/2024/12/06/non-blocking-concurrency-control/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2024/12/16/announcing-hudi-1-0-0/index.html b/content/blog/2024/12/16/announcing-hudi-1-0-0/index.html
index 3970c697082e2..c1ad724cc3f72 100644
--- a/content/blog/2024/12/16/announcing-hudi-1-0-0/index.html
+++ b/content/blog/2024/12/16/announcing-hudi-1-0-0/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/2024/12/29/apache-hudi-2024-a-year-in-review/index.html b/content/blog/2024/12/29/apache-hudi-2024-a-year-in-review/index.html
index 6265fabe327c7..1433ab5f59980 100644
--- a/content/blog/2024/12/29/apache-hudi-2024-a-year-in-review/index.html
+++ b/content/blog/2024/12/29/apache-hudi-2024-a-year-in-review/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/archive/index.html b/content/blog/archive/index.html
index eaaa804b64130..911c4bde11176 100644
--- a/content/blog/archive/index.html
+++ b/content/blog/archive/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/index.html b/content/blog/index.html
index cf40424d8b094..18e16c831f597 100644
--- a/content/blog/index.html
+++ b/content/blog/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/page/10/index.html b/content/blog/page/10/index.html
index e9483a00eb728..87a37fa4d05e4 100644
--- a/content/blog/page/10/index.html
+++ b/content/blog/page/10/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/page/11/index.html b/content/blog/page/11/index.html
index 835a869aa1889..6519b43a3c52c 100644
--- a/content/blog/page/11/index.html
+++ b/content/blog/page/11/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/page/12/index.html b/content/blog/page/12/index.html
index c765cde482eab..30e9fea595d43 100644
--- a/content/blog/page/12/index.html
+++ b/content/blog/page/12/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/page/13/index.html b/content/blog/page/13/index.html
index c7e2ffab0b72a..679c96b357fba 100644
--- a/content/blog/page/13/index.html
+++ b/content/blog/page/13/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/page/14/index.html b/content/blog/page/14/index.html
index 8ae4e6a908015..e9346598a44e0 100644
--- a/content/blog/page/14/index.html
+++ b/content/blog/page/14/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/page/15/index.html b/content/blog/page/15/index.html
index b8f8360dc08a7..58a1e48fa8263 100644
--- a/content/blog/page/15/index.html
+++ b/content/blog/page/15/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/page/16/index.html b/content/blog/page/16/index.html
index a5df1b9f09a72..87ecfda4bce84 100644
--- a/content/blog/page/16/index.html
+++ b/content/blog/page/16/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/page/17/index.html b/content/blog/page/17/index.html
index 1ee5f4a3e26dd..0b9222b7e9bd7 100644
--- a/content/blog/page/17/index.html
+++ b/content/blog/page/17/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/page/18/index.html b/content/blog/page/18/index.html
index 2b7175855ed3b..ea2c5a0dc7f3a 100644
--- a/content/blog/page/18/index.html
+++ b/content/blog/page/18/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/page/19/index.html b/content/blog/page/19/index.html
index 7e5f676ba5c3f..8936113d1b4b5 100644
--- a/content/blog/page/19/index.html
+++ b/content/blog/page/19/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/page/2/index.html b/content/blog/page/2/index.html
index 3cac668167129..2160807a2bd25 100644
--- a/content/blog/page/2/index.html
+++ b/content/blog/page/2/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/page/20/index.html b/content/blog/page/20/index.html
index 2fb459d46af2d..ba9ce6fa13ab4 100644
--- a/content/blog/page/20/index.html
+++ b/content/blog/page/20/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/page/21/index.html b/content/blog/page/21/index.html
index 7cdc4fc3701b8..08c075970c7df 100644
--- a/content/blog/page/21/index.html
+++ b/content/blog/page/21/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/page/3/index.html b/content/blog/page/3/index.html
index 730121924ef27..e273cb0c04a28 100644
--- a/content/blog/page/3/index.html
+++ b/content/blog/page/3/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/page/4/index.html b/content/blog/page/4/index.html
index dc39a1d125505..e5c355baa8230 100644
--- a/content/blog/page/4/index.html
+++ b/content/blog/page/4/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/page/5/index.html b/content/blog/page/5/index.html
index 91fbfb5ca2769..86a3584e43c20 100644
--- a/content/blog/page/5/index.html
+++ b/content/blog/page/5/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/page/6/index.html b/content/blog/page/6/index.html
index 2a0026ea6f5cb..ace5d77da92c6 100644
--- a/content/blog/page/6/index.html
+++ b/content/blog/page/6/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/page/7/index.html b/content/blog/page/7/index.html
index b8102be265495..26259f621e268 100644
--- a/content/blog/page/7/index.html
+++ b/content/blog/page/7/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/page/8/index.html b/content/blog/page/8/index.html
index 34d021bae7d05..18a84972ba353 100644
--- a/content/blog/page/8/index.html
+++ b/content/blog/page/8/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/page/9/index.html b/content/blog/page/9/index.html
index fe1be2257c52f..5ed6dc2074a8e 100644
--- a/content/blog/page/9/index.html
+++ b/content/blog/page/9/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/streaming-data-lake-platform/index.html b/content/blog/streaming-data-lake-platform/index.html
index b207a69b5461e..bb4710fc92d28 100644
--- a/content/blog/streaming-data-lake-platform/index.html
+++ b/content/blog/streaming-data-lake-platform/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/access-control/index.html b/content/blog/tags/access-control/index.html
index 68fa0e1dd32bd..9fd0cf01aeefb 100644
--- a/content/blog/tags/access-control/index.html
+++ b/content/blog/tags/access-control/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/acid/index.html b/content/blog/tags/acid/index.html
index 0ab78818f2145..c8742b0d2fa74 100644
--- a/content/blog/tags/acid/index.html
+++ b/content/blog/tags/acid/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/active-timeline/index.html b/content/blog/tags/active-timeline/index.html
index 73e5623468553..84291f147a572 100644
--- a/content/blog/tags/active-timeline/index.html
+++ b/content/blog/tags/active-timeline/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/airbyte/index.html b/content/blog/tags/airbyte/index.html
index 9676de920e685..5e6537c8d7015 100644
--- a/content/blog/tags/airbyte/index.html
+++ b/content/blog/tags/airbyte/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/alibabacloud/index.html b/content/blog/tags/alibabacloud/index.html
index 1a3220a58fadd..d6a5baaa092a8 100644
--- a/content/blog/tags/alibabacloud/index.html
+++ b/content/blog/tags/alibabacloud/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/amazon-athena/index.html b/content/blog/tags/amazon-athena/index.html
index 6b2c5c70d1217..a40224683aed5 100644
--- a/content/blog/tags/amazon-athena/index.html
+++ b/content/blog/tags/amazon-athena/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/amazon-dynamodb/index.html b/content/blog/tags/amazon-dynamodb/index.html
index 86b8574ecb325..3c3a92151f59a 100644
--- a/content/blog/tags/amazon-dynamodb/index.html
+++ b/content/blog/tags/amazon-dynamodb/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/amazon-eks/index.html b/content/blog/tags/amazon-eks/index.html
index 4fe06c7c9a3d9..9075b0f212e9f 100644
--- a/content/blog/tags/amazon-eks/index.html
+++ b/content/blog/tags/amazon-eks/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/amazon-emr/index.html b/content/blog/tags/amazon-emr/index.html
index 843f7ed11eb3e..4adfc75ddb088 100644
--- a/content/blog/tags/amazon-emr/index.html
+++ b/content/blog/tags/amazon-emr/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/amazon-kinesis/index.html b/content/blog/tags/amazon-kinesis/index.html
index bf17fa5cb1a30..6a2680cff6abf 100644
--- a/content/blog/tags/amazon-kinesis/index.html
+++ b/content/blog/tags/amazon-kinesis/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/amazon-mks/index.html b/content/blog/tags/amazon-mks/index.html
index 0722ee7deb6b6..1dfca8c4521a5 100644
--- a/content/blog/tags/amazon-mks/index.html
+++ b/content/blog/tags/amazon-mks/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/amazon-rds/index.html b/content/blog/tags/amazon-rds/index.html
index d462e5d3b6642..7ca69a046ff57 100644
--- a/content/blog/tags/amazon-rds/index.html
+++ b/content/blog/tags/amazon-rds/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/amazon-redshift/index.html b/content/blog/tags/amazon-redshift/index.html
index e7c4ec61b541d..13ff15ed5fc40 100644
--- a/content/blog/tags/amazon-redshift/index.html
+++ b/content/blog/tags/amazon-redshift/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/amazon-s-3/index.html b/content/blog/tags/amazon-s-3/index.html
index 38a22ecaa06c9..0f5878d1da0a8 100644
--- a/content/blog/tags/amazon-s-3/index.html
+++ b/content/blog/tags/amazon-s-3/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/amazon-sagemaker/index.html b/content/blog/tags/amazon-sagemaker/index.html
index 01b1753bd1b08..4ce86f7e45d6b 100644
--- a/content/blog/tags/amazon-sagemaker/index.html
+++ b/content/blog/tags/amazon-sagemaker/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/amazon-spark/index.html b/content/blog/tags/amazon-spark/index.html
index 0d6335b1cec47..cb26e88e77ee1 100644
--- a/content/blog/tags/amazon-spark/index.html
+++ b/content/blog/tags/amazon-spark/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/amazon/index.html b/content/blog/tags/amazon/index.html
index bd3449f06757b..8cd8a4966b80f 100644
--- a/content/blog/tags/amazon/index.html
+++ b/content/blog/tags/amazon/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/amazon/page/2/index.html b/content/blog/tags/amazon/page/2/index.html
index cf7a97386d0a8..7361c1515adc4 100644
--- a/content/blog/tags/amazon/page/2/index.html
+++ b/content/blog/tags/amazon/page/2/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/amazon/page/3/index.html b/content/blog/tags/amazon/page/3/index.html
index 954ae1df194f9..2ff65eed3d38d 100644
--- a/content/blog/tags/amazon/page/3/index.html
+++ b/content/blog/tags/amazon/page/3/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/analytics-at-scale/index.html b/content/blog/tags/analytics-at-scale/index.html
index 69ae53d811a5c..3279b3adf1336 100644
--- a/content/blog/tags/analytics-at-scale/index.html
+++ b/content/blog/tags/analytics-at-scale/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/analyticsinsight/index.html b/content/blog/tags/analyticsinsight/index.html
index ea43bbcaf6f74..a53aa80f4a070 100644
--- a/content/blog/tags/analyticsinsight/index.html
+++ b/content/blog/tags/analyticsinsight/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/antstack/index.html b/content/blog/tags/antstack/index.html
index 1ccc6ae97f936..0b006a0841f78 100644
--- a/content/blog/tags/antstack/index.html
+++ b/content/blog/tags/antstack/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/apache-avro/index.html b/content/blog/tags/apache-avro/index.html
index 60305c266adcb..d25f6805d5b9e 100644
--- a/content/blog/tags/apache-avro/index.html
+++ b/content/blog/tags/apache-avro/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/apache-dolphin-scheduler/index.html b/content/blog/tags/apache-dolphin-scheduler/index.html
index cfa89d195c6c3..1219a4df8bbb1 100644
--- a/content/blog/tags/apache-dolphin-scheduler/index.html
+++ b/content/blog/tags/apache-dolphin-scheduler/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/apache-doris/index.html b/content/blog/tags/apache-doris/index.html
index 02ed2634c87db..5a8b8fee651d4 100644
--- a/content/blog/tags/apache-doris/index.html
+++ b/content/blog/tags/apache-doris/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/apache-flink/index.html b/content/blog/tags/apache-flink/index.html
index 57cec0763cda8..1a2a9bb91c0a5 100644
--- a/content/blog/tags/apache-flink/index.html
+++ b/content/blog/tags/apache-flink/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/apache-hive/index.html b/content/blog/tags/apache-hive/index.html
index b8bcf63cacc72..99de7328ecaa3 100644
--- a/content/blog/tags/apache-hive/index.html
+++ b/content/blog/tags/apache-hive/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/apache-hudi-blogs/index.html b/content/blog/tags/apache-hudi-blogs/index.html
index 11c5b395054fb..eb0f8b9161920 100644
--- a/content/blog/tags/apache-hudi-blogs/index.html
+++ b/content/blog/tags/apache-hudi-blogs/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/apache-hudi/index.html b/content/blog/tags/apache-hudi/index.html
index 8444199e4c3f9..9c786c3c84de7 100644
--- a/content/blog/tags/apache-hudi/index.html
+++ b/content/blog/tags/apache-hudi/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/apache-hudi/page/10/index.html b/content/blog/tags/apache-hudi/page/10/index.html
index 79d39fb29185f..94a212abc5e60 100644
--- a/content/blog/tags/apache-hudi/page/10/index.html
+++ b/content/blog/tags/apache-hudi/page/10/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/apache-hudi/page/11/index.html b/content/blog/tags/apache-hudi/page/11/index.html
index 9e45c8cab6d90..f089ad9b787d1 100644
--- a/content/blog/tags/apache-hudi/page/11/index.html
+++ b/content/blog/tags/apache-hudi/page/11/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/apache-hudi/page/12/index.html b/content/blog/tags/apache-hudi/page/12/index.html
index b8e47dccdcee2..56efcb092bb51 100644
--- a/content/blog/tags/apache-hudi/page/12/index.html
+++ b/content/blog/tags/apache-hudi/page/12/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/apache-hudi/page/2/index.html b/content/blog/tags/apache-hudi/page/2/index.html
index b888462c62055..6f3a8fa1868e0 100644
--- a/content/blog/tags/apache-hudi/page/2/index.html
+++ b/content/blog/tags/apache-hudi/page/2/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/apache-hudi/page/3/index.html b/content/blog/tags/apache-hudi/page/3/index.html
index bc58eae5c2110..c1aa61da1400a 100644
--- a/content/blog/tags/apache-hudi/page/3/index.html
+++ b/content/blog/tags/apache-hudi/page/3/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/apache-hudi/page/4/index.html b/content/blog/tags/apache-hudi/page/4/index.html
index 2fa9a3c4f050a..9bd6fa0a17e0a 100644
--- a/content/blog/tags/apache-hudi/page/4/index.html
+++ b/content/blog/tags/apache-hudi/page/4/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/apache-hudi/page/5/index.html b/content/blog/tags/apache-hudi/page/5/index.html
index 8233960f88bb5..b72acaab93fb8 100644
--- a/content/blog/tags/apache-hudi/page/5/index.html
+++ b/content/blog/tags/apache-hudi/page/5/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/apache-hudi/page/6/index.html b/content/blog/tags/apache-hudi/page/6/index.html
index 43995d14d52cd..b053d3549aef3 100644
--- a/content/blog/tags/apache-hudi/page/6/index.html
+++ b/content/blog/tags/apache-hudi/page/6/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/apache-hudi/page/7/index.html b/content/blog/tags/apache-hudi/page/7/index.html
index 7d24bc7dd0af6..7e4180150263c 100644
--- a/content/blog/tags/apache-hudi/page/7/index.html
+++ b/content/blog/tags/apache-hudi/page/7/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/apache-hudi/page/8/index.html b/content/blog/tags/apache-hudi/page/8/index.html
index 163ee40d97697..d87bb1c3cd6cf 100644
--- a/content/blog/tags/apache-hudi/page/8/index.html
+++ b/content/blog/tags/apache-hudi/page/8/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/apache-hudi/page/9/index.html b/content/blog/tags/apache-hudi/page/9/index.html
index 9d589383b816b..6ad8570694fb6 100644
--- a/content/blog/tags/apache-hudi/page/9/index.html
+++ b/content/blog/tags/apache-hudi/page/9/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/apache-iceberg/index.html b/content/blog/tags/apache-iceberg/index.html
index 049d29a16f33e..cd8aff000c542 100644
--- a/content/blog/tags/apache-iceberg/index.html
+++ b/content/blog/tags/apache-iceberg/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/apache-kafka/index.html b/content/blog/tags/apache-kafka/index.html
index 1eea074e0a85a..336824743ebf5 100644
--- a/content/blog/tags/apache-kafka/index.html
+++ b/content/blog/tags/apache-kafka/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/apache-orc/index.html b/content/blog/tags/apache-orc/index.html
index d71b0303c6b7e..3cfb356d83314 100644
--- a/content/blog/tags/apache-orc/index.html
+++ b/content/blog/tags/apache-orc/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/apache-parquet/index.html b/content/blog/tags/apache-parquet/index.html
index 1506d67b05b58..2a2f99ef24006 100644
--- a/content/blog/tags/apache-parquet/index.html
+++ b/content/blog/tags/apache-parquet/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/apache-spark/index.html b/content/blog/tags/apache-spark/index.html
index 798c4db55ea97..5846e46d38cf9 100644
--- a/content/blog/tags/apache-spark/index.html
+++ b/content/blog/tags/apache-spark/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/apache-spark/page/2/index.html b/content/blog/tags/apache-spark/page/2/index.html
index eb8de19fa5501..5e8b7a939d1f3 100644
--- a/content/blog/tags/apache-spark/page/2/index.html
+++ b/content/blog/tags/apache-spark/page/2/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/apache-zeppelin/index.html b/content/blog/tags/apache-zeppelin/index.html
index 5b89c46ad68ec..e62a6b1f647e1 100644
--- a/content/blog/tags/apache-zeppelin/index.html
+++ b/content/blog/tags/apache-zeppelin/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/apache/index.html b/content/blog/tags/apache/index.html
index 291b7465b4634..a07b364c7f33c 100644
--- a/content/blog/tags/apache/index.html
+++ b/content/blog/tags/apache/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/apcache-spark/index.html b/content/blog/tags/apcache-spark/index.html
index 98c9ef4c095c5..17dddc07cf6f3 100644
--- a/content/blog/tags/apcache-spark/index.html
+++ b/content/blog/tags/apcache-spark/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/apicurio-registry/index.html b/content/blog/tags/apicurio-registry/index.html
index 9bff113e1b697..a07d4ad9b1758 100644
--- a/content/blog/tags/apicurio-registry/index.html
+++ b/content/blog/tags/apicurio-registry/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/architecture/index.html b/content/blog/tags/architecture/index.html
index 8e94361d715db..66912ae0223e9 100644
--- a/content/blog/tags/architecture/index.html
+++ b/content/blog/tags/architecture/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/archival-timeline/index.html b/content/blog/tags/archival-timeline/index.html
index 65da37e374bf2..91227dd739b07 100644
--- a/content/blog/tags/archival-timeline/index.html
+++ b/content/blog/tags/archival-timeline/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/async-indexing/index.html b/content/blog/tags/async-indexing/index.html
index df317e73bad62..67d2c31b64245 100644
--- a/content/blog/tags/async-indexing/index.html
+++ b/content/blog/tags/async-indexing/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/athena/index.html b/content/blog/tags/athena/index.html
index 2c0d8d361bc2f..c2304d6a3b247 100644
--- a/content/blog/tags/athena/index.html
+++ b/content/blog/tags/athena/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/aws-athena/index.html b/content/blog/tags/aws-athena/index.html
index 813f454f64324..0cdbb458c45c6 100644
--- a/content/blog/tags/aws-athena/index.html
+++ b/content/blog/tags/aws-athena/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/aws-cloud-9/index.html b/content/blog/tags/aws-cloud-9/index.html
index 1b6ff6dfb3eb1..db79a6268a539 100644
--- a/content/blog/tags/aws-cloud-9/index.html
+++ b/content/blog/tags/aws-cloud-9/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/aws-data-exchange/index.html b/content/blog/tags/aws-data-exchange/index.html
index 4f6ca2606afce..c938f84e0e545 100644
--- a/content/blog/tags/aws-data-exchange/index.html
+++ b/content/blog/tags/aws-data-exchange/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/aws-emr/index.html b/content/blog/tags/aws-emr/index.html
index 4e504c1ba436e..11865058adaaf 100644
--- a/content/blog/tags/aws-emr/index.html
+++ b/content/blog/tags/aws-emr/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/aws-glue-crawlers/index.html b/content/blog/tags/aws-glue-crawlers/index.html
index 3f0496b8928ab..e59e8957563f4 100644
--- a/content/blog/tags/aws-glue-crawlers/index.html
+++ b/content/blog/tags/aws-glue-crawlers/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/aws-glue/index.html b/content/blog/tags/aws-glue/index.html
index 0e039e4fb7e76..e4f083986fe0c 100644
--- a/content/blog/tags/aws-glue/index.html
+++ b/content/blog/tags/aws-glue/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/aws-glue/page/2/index.html b/content/blog/tags/aws-glue/page/2/index.html
index cab3292e74b53..1b25c427d8d74 100644
--- a/content/blog/tags/aws-glue/page/2/index.html
+++ b/content/blog/tags/aws-glue/page/2/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/aws-lake-formation/index.html b/content/blog/tags/aws-lake-formation/index.html
index dac825a1d1f3e..ff1aa5271c557 100644
--- a/content/blog/tags/aws-lake-formation/index.html
+++ b/content/blog/tags/aws-lake-formation/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/aws-s-3/index.html b/content/blog/tags/aws-s-3/index.html
index 8f1821d14d0ee..c8b5682057050 100644
--- a/content/blog/tags/aws-s-3/index.html
+++ b/content/blog/tags/aws-s-3/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/aws/index.html b/content/blog/tags/aws/index.html
index 08d4aee72d284..13c0c15982ccc 100644
--- a/content/blog/tags/aws/index.html
+++ b/content/blog/tags/aws/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/backfilling/index.html b/content/blog/tags/backfilling/index.html
index 17e1e2ecfc8da..a31b1032b0f30 100644
--- a/content/blog/tags/backfilling/index.html
+++ b/content/blog/tags/backfilling/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/beginner/index.html b/content/blog/tags/beginner/index.html
index f445be55a7033..36190280c930a 100644
--- a/content/blog/tags/beginner/index.html
+++ b/content/blog/tags/beginner/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/beginner/page/2/index.html b/content/blog/tags/beginner/page/2/index.html
index d9eda7c53babf..a4ed850ec6beb 100644
--- a/content/blog/tags/beginner/page/2/index.html
+++ b/content/blog/tags/beginner/page/2/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/best-practices/index.html b/content/blog/tags/best-practices/index.html
index 0cb5e74746d53..82b2ca51eedad 100644
--- a/content/blog/tags/best-practices/index.html
+++ b/content/blog/tags/best-practices/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/big-data/index.html b/content/blog/tags/big-data/index.html
index b471df5f35772..503b5473bdc35 100644
--- a/content/blog/tags/big-data/index.html
+++ b/content/blog/tags/big-data/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/bigdata/index.html b/content/blog/tags/bigdata/index.html
index 3967fb2cf8e98..5b5879628e30e 100644
--- a/content/blog/tags/bigdata/index.html
+++ b/content/blog/tags/bigdata/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/blog/index.html b/content/blog/tags/blog/index.html
index 92d49f5e9ae74..e399ecc36f0f3 100644
--- a/content/blog/tags/blog/index.html
+++ b/content/blog/tags/blog/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/blog/page/10/index.html b/content/blog/tags/blog/page/10/index.html
index c616354bebdbc..23ca94b4b13fd 100644
--- a/content/blog/tags/blog/page/10/index.html
+++ b/content/blog/tags/blog/page/10/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/blog/page/11/index.html b/content/blog/tags/blog/page/11/index.html
index 5a8881a6bb2aa..9f3ed79480642 100644
--- a/content/blog/tags/blog/page/11/index.html
+++ b/content/blog/tags/blog/page/11/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/blog/page/12/index.html b/content/blog/tags/blog/page/12/index.html
index ca51600a1409a..93cbc633fef21 100644
--- a/content/blog/tags/blog/page/12/index.html
+++ b/content/blog/tags/blog/page/12/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/blog/page/2/index.html b/content/blog/tags/blog/page/2/index.html
index 3763b78c4c0f5..506dad54edc63 100644
--- a/content/blog/tags/blog/page/2/index.html
+++ b/content/blog/tags/blog/page/2/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/blog/page/3/index.html b/content/blog/tags/blog/page/3/index.html
index 3ccda63ab5798..389d43f921192 100644
--- a/content/blog/tags/blog/page/3/index.html
+++ b/content/blog/tags/blog/page/3/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/blog/page/4/index.html b/content/blog/tags/blog/page/4/index.html
index 3e6d07de0edef..4fc1c596b3317 100644
--- a/content/blog/tags/blog/page/4/index.html
+++ b/content/blog/tags/blog/page/4/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/blog/page/5/index.html b/content/blog/tags/blog/page/5/index.html
index 1384f6e25adcf..d7d025ce7070a 100644
--- a/content/blog/tags/blog/page/5/index.html
+++ b/content/blog/tags/blog/page/5/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/blog/page/6/index.html b/content/blog/tags/blog/page/6/index.html
index efbe06c95faf6..8d52342c695cd 100644
--- a/content/blog/tags/blog/page/6/index.html
+++ b/content/blog/tags/blog/page/6/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/blog/page/7/index.html b/content/blog/tags/blog/page/7/index.html
index 57f7560a2f299..2ab50a8583c9f 100644
--- a/content/blog/tags/blog/page/7/index.html
+++ b/content/blog/tags/blog/page/7/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/blog/page/8/index.html b/content/blog/tags/blog/page/8/index.html
index 0f15049e296c2..6cac71d4ed184 100644
--- a/content/blog/tags/blog/page/8/index.html
+++ b/content/blog/tags/blog/page/8/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/blog/page/9/index.html b/content/blog/tags/blog/page/9/index.html
index a482cbfdd672c..1d4b9f73e034e 100644
--- a/content/blog/tags/blog/page/9/index.html
+++ b/content/blog/tags/blog/page/9/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/bloom-index/index.html b/content/blog/tags/bloom-index/index.html
index 6f116e52bbf21..3e7c1555013b6 100644
--- a/content/blog/tags/bloom-index/index.html
+++ b/content/blog/tags/bloom-index/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/bloom/index.html b/content/blog/tags/bloom/index.html
index 045cd5e8c81ae..4b03261dfc66f 100644
--- a/content/blog/tags/bloom/index.html
+++ b/content/blog/tags/bloom/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/bootstrap/index.html b/content/blog/tags/bootstrap/index.html
index f1490f926be65..d52a60d3e8c1b 100644
--- a/content/blog/tags/bootstrap/index.html
+++ b/content/blog/tags/bootstrap/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/bucket-index/index.html b/content/blog/tags/bucket-index/index.html
index 7a02daddcad60..efb6d9e66e102 100644
--- a/content/blog/tags/bucket-index/index.html
+++ b/content/blog/tags/bucket-index/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/bulk-insert/index.html b/content/blog/tags/bulk-insert/index.html
index 39f0ecf459480..d0215604ef542 100644
--- a/content/blog/tags/bulk-insert/index.html
+++ b/content/blog/tags/bulk-insert/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/bytearray/index.html b/content/blog/tags/bytearray/index.html
index 84c7d7b986a97..33cee728e25f9 100644
--- a/content/blog/tags/bytearray/index.html
+++ b/content/blog/tags/bytearray/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/bytebytego/index.html b/content/blog/tags/bytebytego/index.html
index 0d742d49de415..7b329f3297eb3 100644
--- a/content/blog/tags/bytebytego/index.html
+++ b/content/blog/tags/bytebytego/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/caching/index.html b/content/blog/tags/caching/index.html
index feea70fa8c075..70bf23b139310 100644
--- a/content/blog/tags/caching/index.html
+++ b/content/blog/tags/caching/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/case-study/index.html b/content/blog/tags/case-study/index.html
index d3ca18c119a71..98a3075f3aa53 100644
--- a/content/blog/tags/case-study/index.html
+++ b/content/blog/tags/case-study/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/cdc/index.html b/content/blog/tags/cdc/index.html
index dc3cbae63b49b..b6ee37bfcae01 100644
--- a/content/blog/tags/cdc/index.html
+++ b/content/blog/tags/cdc/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/change-data-capture/index.html b/content/blog/tags/change-data-capture/index.html
index 056c7c7bbf1d7..63cbc068c2cda 100644
--- a/content/blog/tags/change-data-capture/index.html
+++ b/content/blog/tags/change-data-capture/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/cleaner/index.html b/content/blog/tags/cleaner/index.html
index dbf2be146101c..7873703d9ca2e 100644
--- a/content/blog/tags/cleaner/index.html
+++ b/content/blog/tags/cleaner/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/cleaning/index.html b/content/blog/tags/cleaning/index.html
index 5a0714d8b8831..b04ca9cf11deb 100644
--- a/content/blog/tags/cleaning/index.html
+++ b/content/blog/tags/cleaning/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/cloudthat/index.html b/content/blog/tags/cloudthat/index.html
index 6781048ebe52a..7599746baa4d9 100644
--- a/content/blog/tags/cloudthat/index.html
+++ b/content/blog/tags/cloudthat/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/clustering/index.html b/content/blog/tags/clustering/index.html
index ed47c144da5cb..995195c4424c4 100644
--- a/content/blog/tags/clustering/index.html
+++ b/content/blog/tags/clustering/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/code-sample/index.html b/content/blog/tags/code-sample/index.html
index 6174407011c31..7793161220ef7 100644
--- a/content/blog/tags/code-sample/index.html
+++ b/content/blog/tags/code-sample/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/commits/index.html b/content/blog/tags/commits/index.html
index 6f8f6be7776c2..e1946ce73b7d7 100644
--- a/content/blog/tags/commits/index.html
+++ b/content/blog/tags/commits/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/community/index.html b/content/blog/tags/community/index.html
index f1351cdf9772e..3fe4e41ffc66d 100644
--- a/content/blog/tags/community/index.html
+++ b/content/blog/tags/community/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/compaction/index.html b/content/blog/tags/compaction/index.html
index eb786c965da03..60d80a1ecb362 100644
--- a/content/blog/tags/compaction/index.html
+++ b/content/blog/tags/compaction/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/comparison/index.html b/content/blog/tags/comparison/index.html
index 79c893b241933..e6d60a0198b93 100644
--- a/content/blog/tags/comparison/index.html
+++ b/content/blog/tags/comparison/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/comparison/page/2/index.html b/content/blog/tags/comparison/page/2/index.html
index eee12f6d3cfe1..aaaf011f0fc6a 100644
--- a/content/blog/tags/comparison/page/2/index.html
+++ b/content/blog/tags/comparison/page/2/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/compression/index.html b/content/blog/tags/compression/index.html
index 963491af81423..a5db7f7206f4a 100644
--- a/content/blog/tags/compression/index.html
+++ b/content/blog/tags/compression/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/concurrency-control/index.html b/content/blog/tags/concurrency-control/index.html
index f19f37fcf9f85..8e66c95bd77f1 100644
--- a/content/blog/tags/concurrency-control/index.html
+++ b/content/blog/tags/concurrency-control/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/concurrency/index.html b/content/blog/tags/concurrency/index.html
index 8f02922a5dedb..004a754399d2b 100644
--- a/content/blog/tags/concurrency/index.html
+++ b/content/blog/tags/concurrency/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/conference/index.html b/content/blog/tags/conference/index.html
index 480b40c129189..5eacec9e1a807 100644
--- a/content/blog/tags/conference/index.html
+++ b/content/blog/tags/conference/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/consistency/index.html b/content/blog/tags/consistency/index.html
index eaf04b442ce54..161bb86e462c4 100644
--- a/content/blog/tags/consistency/index.html
+++ b/content/blog/tags/consistency/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/cost-efficiency/index.html b/content/blog/tags/cost-efficiency/index.html
index 6953d56c9f056..3833252ed37ae 100644
--- a/content/blog/tags/cost-efficiency/index.html
+++ b/content/blog/tags/cost-efficiency/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/cost-optimization/index.html b/content/blog/tags/cost-optimization/index.html
index bfa366ae24058..eed5d2da72bf2 100644
--- a/content/blog/tags/cost-optimization/index.html
+++ b/content/blog/tags/cost-optimization/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/cost/index.html b/content/blog/tags/cost/index.html
index 1eca6866feb74..a491dce399211 100644
--- a/content/blog/tags/cost/index.html
+++ b/content/blog/tags/cost/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/cow/index.html b/content/blog/tags/cow/index.html
index 9ee7645982e63..279d9d4d03b05 100644
--- a/content/blog/tags/cow/index.html
+++ b/content/blog/tags/cow/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/daft/index.html b/content/blog/tags/daft/index.html
index 3fb3edb0604b4..e04a6a228a44c 100644
--- a/content/blog/tags/daft/index.html
+++ b/content/blog/tags/daft/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/data-lake/index.html b/content/blog/tags/data-lake/index.html
index c0d3a8b06cd99..f39cbbd41ab11 100644
--- a/content/blog/tags/data-lake/index.html
+++ b/content/blog/tags/data-lake/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/data-lakehouse/index.html b/content/blog/tags/data-lakehouse/index.html
index c764178721d86..3939ba8165bfc 100644
--- a/content/blog/tags/data-lakehouse/index.html
+++ b/content/blog/tags/data-lakehouse/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/data-mesh/index.html b/content/blog/tags/data-mesh/index.html
index 076a3865bc549..bcc9d506692aa 100644
--- a/content/blog/tags/data-mesh/index.html
+++ b/content/blog/tags/data-mesh/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/data-platform/index.html b/content/blog/tags/data-platform/index.html
index e677637499000..893300e8bf082 100644
--- a/content/blog/tags/data-platform/index.html
+++ b/content/blog/tags/data-platform/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/data-processing/index.html b/content/blog/tags/data-processing/index.html
index 966f6584558a8..eb22aea144f35 100644
--- a/content/blog/tags/data-processing/index.html
+++ b/content/blog/tags/data-processing/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/data-sahring/index.html b/content/blog/tags/data-sahring/index.html
index 0309f7118783d..885e383bc1e76 100644
--- a/content/blog/tags/data-sahring/index.html
+++ b/content/blog/tags/data-sahring/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/data-skipping/index.html b/content/blog/tags/data-skipping/index.html
index 6bf1a53e81c83..b868fa51ea10f 100644
--- a/content/blog/tags/data-skipping/index.html
+++ b/content/blog/tags/data-skipping/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/data-warehouse/index.html b/content/blog/tags/data-warehouse/index.html
index 977f2de234690..72e0d2d936a4f 100644
--- a/content/blog/tags/data-warehouse/index.html
+++ b/content/blog/tags/data-warehouse/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/databricks/index.html b/content/blog/tags/databricks/index.html
index 7c23a89cf893c..960188aac5141 100644
--- a/content/blog/tags/databricks/index.html
+++ b/content/blog/tags/databricks/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/datalake-platform/index.html b/content/blog/tags/datalake-platform/index.html
index eebda7264abb3..bc01d59066615 100644
--- a/content/blog/tags/datalake-platform/index.html
+++ b/content/blog/tags/datalake-platform/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/datalake/index.html b/content/blog/tags/datalake/index.html
index 656916fd6486a..3f73d0b2ccf22 100644
--- a/content/blog/tags/datalake/index.html
+++ b/content/blog/tags/datalake/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/datumagic/index.html b/content/blog/tags/datumagic/index.html
index 11b8aa0fdc671..2990a931e7e48 100644
--- a/content/blog/tags/datumagic/index.html
+++ b/content/blog/tags/datumagic/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/dbta/index.html b/content/blog/tags/dbta/index.html
index cb8195e2633e7..66bd3025e9c2a 100644
--- a/content/blog/tags/dbta/index.html
+++ b/content/blog/tags/dbta/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/debezium/index.html b/content/blog/tags/debezium/index.html
index 994a7d4332a96..71f2d1cf5b29d 100644
--- a/content/blog/tags/debezium/index.html
+++ b/content/blog/tags/debezium/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/deep-dive/index.html b/content/blog/tags/deep-dive/index.html
index d6ebb9b76afdf..a1fbc2b3964b2 100644
--- a/content/blog/tags/deep-dive/index.html
+++ b/content/blog/tags/deep-dive/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/defogdata/index.html b/content/blog/tags/defogdata/index.html
index e7be4967b8ab4..fabbcbe0a2ac1 100644
--- a/content/blog/tags/defogdata/index.html
+++ b/content/blog/tags/defogdata/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/delete-partition/index.html b/content/blog/tags/delete-partition/index.html
index 72f46681fbe21..699c13b290524 100644
--- a/content/blog/tags/delete-partition/index.html
+++ b/content/blog/tags/delete-partition/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/delete/index.html b/content/blog/tags/delete/index.html
index 8f8d2a8b15014..dad0d17945619 100644
--- a/content/blog/tags/delete/index.html
+++ b/content/blog/tags/delete/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/deletes/index.html b/content/blog/tags/deletes/index.html
index 35f74134081b5..d5a377f44fd22 100644
--- a/content/blog/tags/deletes/index.html
+++ b/content/blog/tags/deletes/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/delta-lake/index.html b/content/blog/tags/delta-lake/index.html
index 2f576b05f8449..b712f8434e77e 100644
--- a/content/blog/tags/delta-lake/index.html
+++ b/content/blog/tags/delta-lake/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/delta-lake/page/2/index.html b/content/blog/tags/delta-lake/page/2/index.html
index 36024caa32f78..b7f3d4cf05ddb 100644
--- a/content/blog/tags/delta-lake/page/2/index.html
+++ b/content/blog/tags/delta-lake/page/2/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/delta/index.html b/content/blog/tags/delta/index.html
index 89b1cab455fac..2552fd5e07e42 100644
--- a/content/blog/tags/delta/index.html
+++ b/content/blog/tags/delta/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/deltastreamer/index.html b/content/blog/tags/deltastreamer/index.html
index a2f900882de45..71e63416ee985 100644
--- a/content/blog/tags/deltastreamer/index.html
+++ b/content/blog/tags/deltastreamer/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/deployment/index.html b/content/blog/tags/deployment/index.html
index bc7760127c553..32a255cf6f900 100644
--- a/content/blog/tags/deployment/index.html
+++ b/content/blog/tags/deployment/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/design/index.html b/content/blog/tags/design/index.html
index 7be9dc3571f33..e763b5f66fedf 100644
--- a/content/blog/tags/design/index.html
+++ b/content/blog/tags/design/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/design/page/2/index.html b/content/blog/tags/design/page/2/index.html
index 285426eae2619..543418c8d39af 100644
--- a/content/blog/tags/design/page/2/index.html
+++ b/content/blog/tags/design/page/2/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/det/index.html b/content/blog/tags/det/index.html
index 945471eaba67d..e3a58bb0c8984 100644
--- a/content/blog/tags/det/index.html
+++ b/content/blog/tags/det/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/dev-to/index.html b/content/blog/tags/dev-to/index.html
index 246b7ad5cc351..32c141b669e4c 100644
--- a/content/blog/tags/dev-to/index.html
+++ b/content/blog/tags/dev-to/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/developpaper/index.html b/content/blog/tags/developpaper/index.html
index 43b20efbfcd22..32be8890ca9dd 100644
--- a/content/blog/tags/developpaper/index.html
+++ b/content/blog/tags/developpaper/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/devgenius/index.html b/content/blog/tags/devgenius/index.html
index 961f826307d9f..b5771d986ba2d 100644
--- a/content/blog/tags/devgenius/index.html
+++ b/content/blog/tags/devgenius/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/diva-portal/index.html b/content/blog/tags/diva-portal/index.html
index c581807ed8973..e7fd8a839e330 100644
--- a/content/blog/tags/diva-portal/index.html
+++ b/content/blog/tags/diva-portal/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/docker/index.html b/content/blog/tags/docker/index.html
index 99a4a682cd870..8056073da5218 100644
--- a/content/blog/tags/docker/index.html
+++ b/content/blog/tags/docker/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/dremio/index.html b/content/blog/tags/dremio/index.html
index fcc02a115c488..3a78b0301c02c 100644
--- a/content/blog/tags/dremio/index.html
+++ b/content/blog/tags/dremio/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/dzone/index.html b/content/blog/tags/dzone/index.html
index 57e9e6a59b232..dbddcfd4709a6 100644
--- a/content/blog/tags/dzone/index.html
+++ b/content/blog/tags/dzone/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/etl/index.html b/content/blog/tags/etl/index.html
index 09b09c4a0101e..d99427f350020 100644
--- a/content/blog/tags/etl/index.html
+++ b/content/blog/tags/etl/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/fast-api/index.html b/content/blog/tags/fast-api/index.html
index cdc6df503f0be..c230bdf86d92b 100644
--- a/content/blog/tags/fast-api/index.html
+++ b/content/blog/tags/fast-api/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/feature-store/index.html b/content/blog/tags/feature-store/index.html
index ddb1a8bfb9c7b..3fcb0b9b8a893 100644
--- a/content/blog/tags/feature-store/index.html
+++ b/content/blog/tags/feature-store/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/fiel-sizing/index.html b/content/blog/tags/fiel-sizing/index.html
index 82999d68aa6c7..f770e788b72f6 100644
--- a/content/blog/tags/fiel-sizing/index.html
+++ b/content/blog/tags/fiel-sizing/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/file-sizing/index.html b/content/blog/tags/file-sizing/index.html
index bb44aba8e5dca..aecd41dd3fa34 100644
--- a/content/blog/tags/file-sizing/index.html
+++ b/content/blog/tags/file-sizing/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/file-system-view/index.html b/content/blog/tags/file-system-view/index.html
index 9d83d9dda9d88..63006a01d4061 100644
--- a/content/blog/tags/file-system-view/index.html
+++ b/content/blog/tags/file-system-view/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/flink/index.html b/content/blog/tags/flink/index.html
index b45cd800be09c..813b49d688fac 100644
--- a/content/blog/tags/flink/index.html
+++ b/content/blog/tags/flink/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/forefathers/index.html b/content/blog/tags/forefathers/index.html
index a809436ddb4b0..934744d3b2ae4 100644
--- a/content/blog/tags/forefathers/index.html
+++ b/content/blog/tags/forefathers/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/gdpr-deletion/index.html b/content/blog/tags/gdpr-deletion/index.html
index 22d6a6bb87cf3..264db0a924a29 100644
--- a/content/blog/tags/gdpr-deletion/index.html
+++ b/content/blog/tags/gdpr-deletion/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/getting-started/index.html b/content/blog/tags/getting-started/index.html
index 1061ffab942b4..d5018756ea912 100644
--- a/content/blog/tags/getting-started/index.html
+++ b/content/blog/tags/getting-started/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/glue-crawler/index.html b/content/blog/tags/glue-crawler/index.html
index 5e0da5ebb547f..225e73c63c5c2 100644
--- a/content/blog/tags/glue-crawler/index.html
+++ b/content/blog/tags/glue-crawler/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/glue-studio/index.html b/content/blog/tags/glue-studio/index.html
index d278d5e915c20..73267c5b8f9c1 100644
--- a/content/blog/tags/glue-studio/index.html
+++ b/content/blog/tags/glue-studio/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/google-scholar/index.html b/content/blog/tags/google-scholar/index.html
index acff59307169f..594afd86d134b 100644
--- a/content/blog/tags/google-scholar/index.html
+++ b/content/blog/tags/google-scholar/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/grab/index.html b/content/blog/tags/grab/index.html
index b223f561e02cb..53e59fdad22f8 100644
--- a/content/blog/tags/grab/index.html
+++ b/content/blog/tags/grab/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/grofers/index.html b/content/blog/tags/grofers/index.html
index 79cc965880392..5e42d061c9d43 100644
--- a/content/blog/tags/grofers/index.html
+++ b/content/blog/tags/grofers/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/guide/index.html b/content/blog/tags/guide/index.html
index 7b8e049497364..fa2f8c9af086b 100644
--- a/content/blog/tags/guide/index.html
+++ b/content/blog/tags/guide/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/halodoc/index.html b/content/blog/tags/halodoc/index.html
index c09d41445274e..3be5430d48075 100644
--- a/content/blog/tags/halodoc/index.html
+++ b/content/blog/tags/halodoc/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/harshdaiya/index.html b/content/blog/tags/harshdaiya/index.html
index 9c225581dd821..22aad0cb47fb2 100644
--- a/content/blog/tags/harshdaiya/index.html
+++ b/content/blog/tags/harshdaiya/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/hbase-index/index.html b/content/blog/tags/hbase-index/index.html
index 8b112a6ec4dd8..38acde7700016 100644
--- a/content/blog/tags/hbase-index/index.html
+++ b/content/blog/tags/hbase-index/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/hive-metastore/index.html b/content/blog/tags/hive-metastore/index.html
index 820fcfc1b42d2..9246febf608f3 100644
--- a/content/blog/tags/hive-metastore/index.html
+++ b/content/blog/tags/hive-metastore/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/hms/index.html b/content/blog/tags/hms/index.html
index f70e416457376..054b7be718e7e 100644
--- a/content/blog/tags/hms/index.html
+++ b/content/blog/tags/hms/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/hopsworks/index.html b/content/blog/tags/hopsworks/index.html
index d042ee3e7dcf9..3f64c226a6383 100644
--- a/content/blog/tags/hopsworks/index.html
+++ b/content/blog/tags/hopsworks/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/how-to/index.html b/content/blog/tags/how-to/index.html
index f13b6ed8b9a75..254826b10c158 100644
--- a/content/blog/tags/how-to/index.html
+++ b/content/blog/tags/how-to/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/how-to/page/2/index.html b/content/blog/tags/how-to/page/2/index.html
index 38cf41ae5fac9..5a5928bf345ce 100644
--- a/content/blog/tags/how-to/page/2/index.html
+++ b/content/blog/tags/how-to/page/2/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/how-to/page/3/index.html b/content/blog/tags/how-to/page/3/index.html
index 8bbb13bd2161d..6ec3034f94c7f 100644
--- a/content/blog/tags/how-to/page/3/index.html
+++ b/content/blog/tags/how-to/page/3/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/how-to/page/4/index.html b/content/blog/tags/how-to/page/4/index.html
index c03ff3bd4e31e..75dec482f652d 100644
--- a/content/blog/tags/how-to/page/4/index.html
+++ b/content/blog/tags/how-to/page/4/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/how-to/page/5/index.html b/content/blog/tags/how-to/page/5/index.html
index 3aa02ff5a9e61..26a8ab1805164 100644
--- a/content/blog/tags/how-to/page/5/index.html
+++ b/content/blog/tags/how-to/page/5/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/hudi-cli/index.html b/content/blog/tags/hudi-cli/index.html
index d6e46982db4a4..bdee8774bfb03 100644
--- a/content/blog/tags/hudi-cli/index.html
+++ b/content/blog/tags/hudi-cli/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/hudi-streamer/index.html b/content/blog/tags/hudi-streamer/index.html
index 489cd818ce278..a7335009c373c 100644
--- a/content/blog/tags/hudi-streamer/index.html
+++ b/content/blog/tags/hudi-streamer/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/hudi/index.html b/content/blog/tags/hudi/index.html
index 06a04f696a944..6a1e1d33cd667 100644
--- a/content/blog/tags/hudi/index.html
+++ b/content/blog/tags/hudi/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/iceberg/index.html b/content/blog/tags/iceberg/index.html
index 5b69c3c87574c..d051c2c05accf 100644
--- a/content/blog/tags/iceberg/index.html
+++ b/content/blog/tags/iceberg/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/incremental-etl/index.html b/content/blog/tags/incremental-etl/index.html
index 1e70b9e434aa9..efe2a38267111 100644
--- a/content/blog/tags/incremental-etl/index.html
+++ b/content/blog/tags/incremental-etl/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/incremental-processing/index.html b/content/blog/tags/incremental-processing/index.html
index fc43d56f82d86..a67cd1ac0e8fc 100644
--- a/content/blog/tags/incremental-processing/index.html
+++ b/content/blog/tags/incremental-processing/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/incremental-processing/page/2/index.html b/content/blog/tags/incremental-processing/page/2/index.html
index db778b910cc1d..7cdd8da41dbf9 100644
--- a/content/blog/tags/incremental-processing/page/2/index.html
+++ b/content/blog/tags/incremental-processing/page/2/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/incremental-query/index.html b/content/blog/tags/incremental-query/index.html
index b5242e17bb423..3c1ac169456b6 100644
--- a/content/blog/tags/incremental-query/index.html
+++ b/content/blog/tags/incremental-query/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/incremental-updates/index.html b/content/blog/tags/incremental-updates/index.html
index 73eeee0f62801..95b8e4d0f70fc 100644
--- a/content/blog/tags/incremental-updates/index.html
+++ b/content/blog/tags/incremental-updates/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/index.html b/content/blog/tags/index.html
index 8ad1427b15084..ac1ec4949fc60 100644
--- a/content/blog/tags/index.html
+++ b/content/blog/tags/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/indexing/index.html b/content/blog/tags/indexing/index.html
index 77971cf21666d..610abeff54d10 100644
--- a/content/blog/tags/indexing/index.html
+++ b/content/blog/tags/indexing/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/inserts/index.html b/content/blog/tags/inserts/index.html
index f103726a46c0c..5bb27870cc3ea 100644
--- a/content/blog/tags/inserts/index.html
+++ b/content/blog/tags/inserts/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/intermediate/index.html b/content/blog/tags/intermediate/index.html
index 1de876502da3f..a662186742237 100644
--- a/content/blog/tags/intermediate/index.html
+++ b/content/blog/tags/intermediate/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/interoperability/index.html b/content/blog/tags/interoperability/index.html
index 70e1cdac5b781..ecd537ca1c2f7 100644
--- a/content/blog/tags/interoperability/index.html
+++ b/content/blog/tags/interoperability/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/introduction/index.html b/content/blog/tags/introduction/index.html
index 93281946d542a..9ebeb8b3d4859 100644
--- a/content/blog/tags/introduction/index.html
+++ b/content/blog/tags/introduction/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/itnext/index.html b/content/blog/tags/itnext/index.html
index e863f89568a0d..968eee046da51 100644
--- a/content/blog/tags/itnext/index.html
+++ b/content/blog/tags/itnext/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/jack-vanlightly/index.html b/content/blog/tags/jack-vanlightly/index.html
index abb184174ccba..24c3c40fb4f5e 100644
--- a/content/blog/tags/jack-vanlightly/index.html
+++ b/content/blog/tags/jack-vanlightly/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/kafka-connect/index.html b/content/blog/tags/kafka-connect/index.html
index 164a0973c314e..967d117feb8b9 100644
--- a/content/blog/tags/kafka-connect/index.html
+++ b/content/blog/tags/kafka-connect/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/key-generators/index.html b/content/blog/tags/key-generators/index.html
index 0036484894ad6..8f3184e735bf8 100644
--- a/content/blog/tags/key-generators/index.html
+++ b/content/blog/tags/key-generators/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/lakefs/index.html b/content/blog/tags/lakefs/index.html
index da6777e9febf9..6d16b3fe2a9d1 100644
--- a/content/blog/tags/lakefs/index.html
+++ b/content/blog/tags/lakefs/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/lakehouse/index.html b/content/blog/tags/lakehouse/index.html
index 36f7ea84b22d8..f18613402470f 100644
--- a/content/blog/tags/lakehouse/index.html
+++ b/content/blog/tags/lakehouse/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/leboncoin-tech-blog/index.html b/content/blog/tags/leboncoin-tech-blog/index.html
index e3ca512765efb..518d26fee5388 100644
--- a/content/blog/tags/leboncoin-tech-blog/index.html
+++ b/content/blog/tags/leboncoin-tech-blog/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/linkedin/index.html b/content/blog/tags/linkedin/index.html
index d401b2a6a7e89..bbad035bf06cb 100644
--- a/content/blog/tags/linkedin/index.html
+++ b/content/blog/tags/linkedin/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/linkedin/page/2/index.html b/content/blog/tags/linkedin/page/2/index.html
index 8b32c7f02cdf1..615a2b57fcd97 100644
--- a/content/blog/tags/linkedin/page/2/index.html
+++ b/content/blog/tags/linkedin/page/2/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/lock-provider/index.html b/content/blog/tags/lock-provider/index.html
index 2e87f442104b9..3966f662f14b1 100644
--- a/content/blog/tags/lock-provider/index.html
+++ b/content/blog/tags/lock-provider/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/logicalclocks/index.html b/content/blog/tags/logicalclocks/index.html
index 2e0477679daa5..04c7308172906 100644
--- a/content/blog/tags/logicalclocks/index.html
+++ b/content/blog/tags/logicalclocks/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/markers/index.html b/content/blog/tags/markers/index.html
index d3170db6ae852..4bfb8ecd0b1e8 100644
--- a/content/blog/tags/markers/index.html
+++ b/content/blog/tags/markers/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/medallion-architecture/index.html b/content/blog/tags/medallion-architecture/index.html
index 67e99cfa6f1f7..b25a982d5c0ed 100644
--- a/content/blog/tags/medallion-architecture/index.html
+++ b/content/blog/tags/medallion-architecture/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/medium/index.html b/content/blog/tags/medium/index.html
index bf2f86b2f64a5..c901016b1ffab 100644
--- a/content/blog/tags/medium/index.html
+++ b/content/blog/tags/medium/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/medium/page/2/index.html b/content/blog/tags/medium/page/2/index.html
index 0456859930fbc..de30b80e43687 100644
--- a/content/blog/tags/medium/page/2/index.html
+++ b/content/blog/tags/medium/page/2/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/medium/page/3/index.html b/content/blog/tags/medium/page/3/index.html
index 3edb5ea59e782..e027860999010 100644
--- a/content/blog/tags/medium/page/3/index.html
+++ b/content/blog/tags/medium/page/3/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/medium/page/4/index.html b/content/blog/tags/medium/page/4/index.html
index 579c23080359a..95e6c4682fece 100644
--- a/content/blog/tags/medium/page/4/index.html
+++ b/content/blog/tags/medium/page/4/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/meetup/index.html b/content/blog/tags/meetup/index.html
index 00863d14e4923..d270dca3259eb 100644
--- a/content/blog/tags/meetup/index.html
+++ b/content/blog/tags/meetup/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/metadata/index.html b/content/blog/tags/metadata/index.html
index b764ea560e9ca..d26362406b699 100644
--- a/content/blog/tags/metadata/index.html
+++ b/content/blog/tags/metadata/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/metafields/index.html b/content/blog/tags/metafields/index.html
index 921d0f6887147..581efa82bd66e 100644
--- a/content/blog/tags/metafields/index.html
+++ b/content/blog/tags/metafields/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/metrics/index.html b/content/blog/tags/metrics/index.html
index 3376d2fbc27f0..56f33cdecb1ff 100644
--- a/content/blog/tags/metrics/index.html
+++ b/content/blog/tags/metrics/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/migration/index.html b/content/blog/tags/migration/index.html
index 49de034f4c6bd..64d4a35d40936 100644
--- a/content/blog/tags/migration/index.html
+++ b/content/blog/tags/migration/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/min/index.html b/content/blog/tags/min/index.html
index eeb16eabb673b..ec75b05a3e6f0 100644
--- a/content/blog/tags/min/index.html
+++ b/content/blog/tags/min/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/minio/index.html b/content/blog/tags/minio/index.html
index 88a076d43fbe8..0d51397d06fd6 100644
--- a/content/blog/tags/minio/index.html
+++ b/content/blog/tags/minio/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/mino/index.html b/content/blog/tags/mino/index.html
index 4f15805296619..2c65ed48dd4a6 100644
--- a/content/blog/tags/mino/index.html
+++ b/content/blog/tags/mino/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/mlops/index.html b/content/blog/tags/mlops/index.html
index b65876d093014..44224433aeeaa 100644
--- a/content/blog/tags/mlops/index.html
+++ b/content/blog/tags/mlops/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/modern-data-architecture/index.html b/content/blog/tags/modern-data-architecture/index.html
index ca49db8b407ac..67f3d8abbec3b 100644
--- a/content/blog/tags/modern-data-architecture/index.html
+++ b/content/blog/tags/modern-data-architecture/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/mongodb/index.html b/content/blog/tags/mongodb/index.html
index af81535bb6e00..47cf23ff9db45 100644
--- a/content/blog/tags/mongodb/index.html
+++ b/content/blog/tags/mongodb/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/monotonic-timestamp/index.html b/content/blog/tags/monotonic-timestamp/index.html
index 34b3b97d9c1d3..692dcc487d892 100644
--- a/content/blog/tags/monotonic-timestamp/index.html
+++ b/content/blog/tags/monotonic-timestamp/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/mor/index.html b/content/blog/tags/mor/index.html
index b2d3e8a283c36..c30deb7ffa86e 100644
--- a/content/blog/tags/mor/index.html
+++ b/content/blog/tags/mor/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/multi-deltastreamer/index.html b/content/blog/tags/multi-deltastreamer/index.html
index e031b6f71703c..d96cbd23a1efc 100644
--- a/content/blog/tags/multi-deltastreamer/index.html
+++ b/content/blog/tags/multi-deltastreamer/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/multi-modal-indexing/index.html b/content/blog/tags/multi-modal-indexing/index.html
index caa76e034cd08..46fb1d15697f3 100644
--- a/content/blog/tags/multi-modal-indexing/index.html
+++ b/content/blog/tags/multi-modal-indexing/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/multi-writer/index.html b/content/blog/tags/multi-writer/index.html
index 561d1e7b0d130..f3e8750e52a6d 100644
--- a/content/blog/tags/multi-writer/index.html
+++ b/content/blog/tags/multi-writer/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/near-real-time-analytics/index.html b/content/blog/tags/near-real-time-analytics/index.html
index 6d517780442e3..e0bb496d7d69f 100644
--- a/content/blog/tags/near-real-time-analytics/index.html
+++ b/content/blog/tags/near-real-time-analytics/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/onehouse/index.html b/content/blog/tags/onehouse/index.html
index 74a7a4aa38462..36a3b0ba55183 100644
--- a/content/blog/tags/onehouse/index.html
+++ b/content/blog/tags/onehouse/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/onehouse/page/2/index.html b/content/blog/tags/onehouse/page/2/index.html
index ef4a8d77c7968..a41ab47229942 100644
--- a/content/blog/tags/onehouse/page/2/index.html
+++ b/content/blog/tags/onehouse/page/2/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/open-architecture/index.html b/content/blog/tags/open-architecture/index.html
index f1e3baa52a5e1..3957c44542afb 100644
--- a/content/blog/tags/open-architecture/index.html
+++ b/content/blog/tags/open-architecture/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/opstree/index.html b/content/blog/tags/opstree/index.html
index 7c532f721edb0..57a91c7be833e 100644
--- a/content/blog/tags/opstree/index.html
+++ b/content/blog/tags/opstree/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/optimization/index.html b/content/blog/tags/optimization/index.html
index 576df6c5e297f..31b9c0a96c504 100644
--- a/content/blog/tags/optimization/index.html
+++ b/content/blog/tags/optimization/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/oreilly/index.html b/content/blog/tags/oreilly/index.html
index 80bbeef2580d7..69ff4163e9f83 100644
--- a/content/blog/tags/oreilly/index.html
+++ b/content/blog/tags/oreilly/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/partition/index.html b/content/blog/tags/partition/index.html
index e8c5222e9d026..5739fbabaa0a6 100644
--- a/content/blog/tags/partition/index.html
+++ b/content/blog/tags/partition/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/performance/index.html b/content/blog/tags/performance/index.html
index 244bb29116023..14d7743414262 100644
--- a/content/blog/tags/performance/index.html
+++ b/content/blog/tags/performance/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/postgres/index.html b/content/blog/tags/postgres/index.html
index 8e8daa43df44f..1044514fbe7e9 100644
--- a/content/blog/tags/postgres/index.html
+++ b/content/blog/tags/postgres/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/postgresql/index.html b/content/blog/tags/postgresql/index.html
index 0a975a5765b78..de0fae3d1bcb2 100644
--- a/content/blog/tags/postgresql/index.html
+++ b/content/blog/tags/postgresql/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/presto/index.html b/content/blog/tags/presto/index.html
index c972c149e6a89..7d28bbf6a29d6 100644
--- a/content/blog/tags/presto/index.html
+++ b/content/blog/tags/presto/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/prestocon/index.html b/content/blog/tags/prestocon/index.html
index edf99da1cb243..eaf883961af48 100644
--- a/content/blog/tags/prestocon/index.html
+++ b/content/blog/tags/prestocon/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/prestodb/index.html b/content/blog/tags/prestodb/index.html
index d20e9bd54d0e5..809a793e51ba8 100644
--- a/content/blog/tags/prestodb/index.html
+++ b/content/blog/tags/prestodb/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/programmer/index.html b/content/blog/tags/programmer/index.html
index d3df0b90b0fe8..39153428fbbee 100644
--- a/content/blog/tags/programmer/index.html
+++ b/content/blog/tags/programmer/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/pyspark/index.html b/content/blog/tags/pyspark/index.html
index 1c978898337ab..5310412531bfc 100644
--- a/content/blog/tags/pyspark/index.html
+++ b/content/blog/tags/pyspark/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/python/index.html b/content/blog/tags/python/index.html
index a111c410a3c56..32db3e4940257 100644
--- a/content/blog/tags/python/index.html
+++ b/content/blog/tags/python/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/queries/index.html b/content/blog/tags/queries/index.html
index 997c2e6d4b52d..ad5cf755560c8 100644
--- a/content/blog/tags/queries/index.html
+++ b/content/blog/tags/queries/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/query-performance/index.html b/content/blog/tags/query-performance/index.html
index d9393d8ccca70..132e383289136 100644
--- a/content/blog/tags/query-performance/index.html
+++ b/content/blog/tags/query-performance/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/querying/index.html b/content/blog/tags/querying/index.html
index 61de9e942b543..3924e18d73f40 100644
--- a/content/blog/tags/querying/index.html
+++ b/content/blog/tags/querying/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/ray/index.html b/content/blog/tags/ray/index.html
index 63f8c349fedc3..b29783fbc6847 100644
--- a/content/blog/tags/ray/index.html
+++ b/content/blog/tags/ray/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/read-optimized-query/index.html b/content/blog/tags/read-optimized-query/index.html
index 26d707932e931..a977e53850d31 100644
--- a/content/blog/tags/read-optimized-query/index.html
+++ b/content/blog/tags/read-optimized-query/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/reads/index.html b/content/blog/tags/reads/index.html
index 9f38388dfe8f6..1e618184b260e 100644
--- a/content/blog/tags/reads/index.html
+++ b/content/blog/tags/reads/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/real-time-datalake/index.html b/content/blog/tags/real-time-datalake/index.html
index 8e54e1d1df81d..6271ac9f81e9a 100644
--- a/content/blog/tags/real-time-datalake/index.html
+++ b/content/blog/tags/real-time-datalake/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/real-time-query/index.html b/content/blog/tags/real-time-query/index.html
index aa8fbbcde35b3..a00835351a28d 100644
--- a/content/blog/tags/real-time-query/index.html
+++ b/content/blog/tags/real-time-query/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/record-index/index.html b/content/blog/tags/record-index/index.html
index bfbadcc457e00..0ad3b0451fa90 100644
--- a/content/blog/tags/record-index/index.html
+++ b/content/blog/tags/record-index/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/record-level-index/index.html b/content/blog/tags/record-level-index/index.html
index ad38de151945c..383f2c8caa799 100644
--- a/content/blog/tags/record-level-index/index.html
+++ b/content/blog/tags/record-level-index/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/release/index.html b/content/blog/tags/release/index.html
index bacffc0a8c56d..adce9e4e14a57 100644
--- a/content/blog/tags/release/index.html
+++ b/content/blog/tags/release/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/risingwave/index.html b/content/blog/tags/risingwave/index.html
index 952b838f61d1b..07c68683983dd 100644
--- a/content/blog/tags/risingwave/index.html
+++ b/content/blog/tags/risingwave/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/robinhood/index.html b/content/blog/tags/robinhood/index.html
index 0b7e35e4928a4..511f021ebb743 100644
--- a/content/blog/tags/robinhood/index.html
+++ b/content/blog/tags/robinhood/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/rtinsights/index.html b/content/blog/tags/rtinsights/index.html
index 9b0957bcab2d4..541d5ffeeb3b0 100644
--- a/content/blog/tags/rtinsights/index.html
+++ b/content/blog/tags/rtinsights/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/scd-1/index.html b/content/blog/tags/scd-1/index.html
index 06d3d6969e721..df1d413ac922a 100644
--- a/content/blog/tags/scd-1/index.html
+++ b/content/blog/tags/scd-1/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/scd-2/index.html b/content/blog/tags/scd-2/index.html
index ddf295493966e..e33782501f1ac 100644
--- a/content/blog/tags/scd-2/index.html
+++ b/content/blog/tags/scd-2/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/scd-3/index.html b/content/blog/tags/scd-3/index.html
index 649d22ea43b6c..b57f92c7424dd 100644
--- a/content/blog/tags/scd-3/index.html
+++ b/content/blog/tags/scd-3/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/schema-evolution/index.html b/content/blog/tags/schema-evolution/index.html
index 3ce6fb4764bb6..9ad5c69b6b67c 100644
--- a/content/blog/tags/schema-evolution/index.html
+++ b/content/blog/tags/schema-evolution/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/schema/index.html b/content/blog/tags/schema/index.html
index 3d803310183de..674f0548e9aae 100644
--- a/content/blog/tags/schema/index.html
+++ b/content/blog/tags/schema/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/selectfrom/index.html b/content/blog/tags/selectfrom/index.html
index 2ac6bb51f351f..4325dfced663f 100644
--- a/content/blog/tags/selectfrom/index.html
+++ b/content/blog/tags/selectfrom/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/snapshot-exporter/index.html b/content/blog/tags/snapshot-exporter/index.html
index 6a75a21cbef72..f4b2d64bc4a04 100644
--- a/content/blog/tags/snapshot-exporter/index.html
+++ b/content/blog/tags/snapshot-exporter/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/snapshot-query/index.html b/content/blog/tags/snapshot-query/index.html
index d7fbc2fad0650..0ba6756db3dea 100644
--- a/content/blog/tags/snapshot-query/index.html
+++ b/content/blog/tags/snapshot-query/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/space-filling-curves/index.html b/content/blog/tags/space-filling-curves/index.html
index 4d9576fe2daba..13dfc3caf1d7e 100644
--- a/content/blog/tags/space-filling-curves/index.html
+++ b/content/blog/tags/space-filling-curves/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/spark-sql/index.html b/content/blog/tags/spark-sql/index.html
index f9b8a06f29d59..ba07f11979aac 100644
--- a/content/blog/tags/spark-sql/index.html
+++ b/content/blog/tags/spark-sql/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/sql-transformer/index.html b/content/blog/tags/sql-transformer/index.html
index f27477c21748d..1913776b78dc2 100644
--- a/content/blog/tags/sql-transformer/index.html
+++ b/content/blog/tags/sql-transformer/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/starrocks/index.html b/content/blog/tags/starrocks/index.html
index 113650a5d785b..dba9a648d28d7 100644
--- a/content/blog/tags/starrocks/index.html
+++ b/content/blog/tags/starrocks/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/storage-spec/index.html b/content/blog/tags/storage-spec/index.html
index 57ecc90429341..8c50694d25508 100644
--- a/content/blog/tags/storage-spec/index.html
+++ b/content/blog/tags/storage-spec/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/storage-types/index.html b/content/blog/tags/storage-types/index.html
index 91ae3d25b9421..60c978554183a 100644
--- a/content/blog/tags/storage-types/index.html
+++ b/content/blog/tags/storage-types/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/storage/index.html b/content/blog/tags/storage/index.html
index 8625ea28549e4..815c0a1e972c0 100644
--- a/content/blog/tags/storage/index.html
+++ b/content/blog/tags/storage/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/streaming-ingestion/index.html b/content/blog/tags/streaming-ingestion/index.html
index 0233ceb48ee1c..322d9e6611953 100644
--- a/content/blog/tags/streaming-ingestion/index.html
+++ b/content/blog/tags/streaming-ingestion/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/streaming/index.html b/content/blog/tags/streaming/index.html
index 4b51258ce8252..d03a5a7e6f197 100644
--- a/content/blog/tags/streaming/index.html
+++ b/content/blog/tags/streaming/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/streamlit/index.html b/content/blog/tags/streamlit/index.html
index 915221fc37ff3..78e7e86286d0d 100644
--- a/content/blog/tags/streamlit/index.html
+++ b/content/blog/tags/streamlit/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/substack/index.html b/content/blog/tags/substack/index.html
index 71a16b057a24e..99b416739c3db 100644
--- a/content/blog/tags/substack/index.html
+++ b/content/blog/tags/substack/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/table-formats/index.html b/content/blog/tags/table-formats/index.html
index d111a81363e58..6df7f8ba34409 100644
--- a/content/blog/tags/table-formats/index.html
+++ b/content/blog/tags/table-formats/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/table-service/index.html b/content/blog/tags/table-service/index.html
index c2fb3b414ded4..349046ef8fcae 100644
--- a/content/blog/tags/table-service/index.html
+++ b/content/blog/tags/table-service/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/table-services/index.html b/content/blog/tags/table-services/index.html
index 369ceb83537fc..72a53f02abd28 100644
--- a/content/blog/tags/table-services/index.html
+++ b/content/blog/tags/table-services/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/table-size-stats/index.html b/content/blog/tags/table-size-stats/index.html
index e54bc529ede9d..66de2cb196a2b 100644
--- a/content/blog/tags/table-size-stats/index.html
+++ b/content/blog/tags/table-size-stats/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/techtarget/index.html b/content/blog/tags/techtarget/index.html
index 6b9e9c4963109..a6384147df6a7 100644
--- a/content/blog/tags/techtarget/index.html
+++ b/content/blog/tags/techtarget/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/time-travel-query/index.html b/content/blog/tags/time-travel-query/index.html
index 5fa6f41a3168c..027a973b7c6d2 100644
--- a/content/blog/tags/time-travel-query/index.html
+++ b/content/blog/tags/time-travel-query/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/timeline-server/index.html b/content/blog/tags/timeline-server/index.html
index f23a9c116ca83..6d3ef25cea92e 100644
--- a/content/blog/tags/timeline-server/index.html
+++ b/content/blog/tags/timeline-server/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/timeline/index.html b/content/blog/tags/timeline/index.html
index 970fbfe95ebb3..e3ca8216fdd3b 100644
--- a/content/blog/tags/timeline/index.html
+++ b/content/blog/tags/timeline/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/timestamp-as-of-query/index.html b/content/blog/tags/timestamp-as-of-query/index.html
index 6d20cf82c04f6..bf144f1544f89 100644
--- a/content/blog/tags/timestamp-as-of-query/index.html
+++ b/content/blog/tags/timestamp-as-of-query/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/timestamp-collision/index.html b/content/blog/tags/timestamp-collision/index.html
index 26a6e790cfe75..94e579c57e8f8 100644
--- a/content/blog/tags/timestamp-collision/index.html
+++ b/content/blog/tags/timestamp-collision/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/tla-specification/index.html b/content/blog/tags/tla-specification/index.html
index 4a456f95d7279..f5a99bfa29e6f 100644
--- a/content/blog/tags/tla-specification/index.html
+++ b/content/blog/tags/tla-specification/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/towardsdatascience/index.html b/content/blog/tags/towardsdatascience/index.html
index 8c34cd6a3a695..c7ff9e69a0f49 100644
--- a/content/blog/tags/towardsdatascience/index.html
+++ b/content/blog/tags/towardsdatascience/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/transactions/index.html b/content/blog/tags/transactions/index.html
index 5f40dc7437e76..105c300f173eb 100644
--- a/content/blog/tags/transactions/index.html
+++ b/content/blog/tags/transactions/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/trino/index.html b/content/blog/tags/trino/index.html
index 06e15f88978ad..2e72106c7c3bd 100644
--- a/content/blog/tags/trino/index.html
+++ b/content/blog/tags/trino/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/uber/index.html b/content/blog/tags/uber/index.html
index e7e0deeeff715..823fe2990deb1 100644
--- a/content/blog/tags/uber/index.html
+++ b/content/blog/tags/uber/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/upsert/index.html b/content/blog/tags/upsert/index.html
index 62e758aee70ad..e5e399cf8d2fa 100644
--- a/content/blog/tags/upsert/index.html
+++ b/content/blog/tags/upsert/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/upserts/index.html b/content/blog/tags/upserts/index.html
index 5ef05b21de331..d811ede7e0365 100644
--- a/content/blog/tags/upserts/index.html
+++ b/content/blog/tags/upserts/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/upstox-engineering/index.html b/content/blog/tags/upstox-engineering/index.html
index 405af00753ec4..17b4751886b2f 100644
--- a/content/blog/tags/upstox-engineering/index.html
+++ b/content/blog/tags/upstox-engineering/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/use-case/index.html b/content/blog/tags/use-case/index.html
index fd666dd6f0d13..8f0bc2db8808c 100644
--- a/content/blog/tags/use-case/index.html
+++ b/content/blog/tags/use-case/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/use-case/page/2/index.html b/content/blog/tags/use-case/page/2/index.html
index 1ae8cc3c4a052..c23326282222e 100644
--- a/content/blog/tags/use-case/page/2/index.html
+++ b/content/blog/tags/use-case/page/2/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/use-case/page/3/index.html b/content/blog/tags/use-case/page/3/index.html
index d5fb16ede1c67..9a19045aed0f4 100644
--- a/content/blog/tags/use-case/page/3/index.html
+++ b/content/blog/tags/use-case/page/3/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/vector-search/index.html b/content/blog/tags/vector-search/index.html
index c5c032337d7ae..007bceca96536 100644
--- a/content/blog/tags/vector-search/index.html
+++ b/content/blog/tags/vector-search/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/venturebeat/index.html b/content/blog/tags/venturebeat/index.html
index 9f32d74bb8945..912cff4b3d262 100644
--- a/content/blog/tags/venturebeat/index.html
+++ b/content/blog/tags/venturebeat/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/walmartglobaltech/index.html b/content/blog/tags/walmartglobaltech/index.html
index 78c45f7f2025a..1df4defbc081e 100644
--- a/content/blog/tags/walmartglobaltech/index.html
+++ b/content/blog/tags/walmartglobaltech/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/writes/index.html b/content/blog/tags/writes/index.html
index f706a65b001c0..992ddf30ab76c 100644
--- a/content/blog/tags/writes/index.html
+++ b/content/blog/tags/writes/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/xenonstack/index.html b/content/blog/tags/xenonstack/index.html
index 1badd4d9b00cc..9eba56ee12c49 100644
--- a/content/blog/tags/xenonstack/index.html
+++ b/content/blog/tags/xenonstack/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/y-uno/index.html b/content/blog/tags/y-uno/index.html
index e9e5c2b976d9d..76e3c175db8f1 100644
--- a/content/blog/tags/y-uno/index.html
+++ b/content/blog/tags/y-uno/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/yahoo/index.html b/content/blog/tags/yahoo/index.html
index 3350eecb5253e..d5f0b7e3415ce 100644
--- a/content/blog/tags/yahoo/index.html
+++ b/content/blog/tags/yahoo/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/blog/tags/yugabyte/index.html b/content/blog/tags/yugabyte/index.html
index 49e77b9ba6e88..4947b80d5cbe7 100644
--- a/content/blog/tags/yugabyte/index.html
+++ b/content/blog/tags/yugabyte/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/cn/404.html b/content/cn/404.html
index c8dfea712b5a5..9761b4e19e188 100644
--- a/content/cn/404.html
+++ b/content/cn/404.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/cn/404/index.html b/content/cn/404/index.html
index 40c3b8d9ea2cb..93404ec31ba77 100644
--- a/content/cn/404/index.html
+++ b/content/cn/404/index.html
@@ -19,7 +19,7 @@
-
+
diff --git a/content/cn/assets/js/02e54e09.429268a3.js b/content/cn/assets/js/02e54e09.641379dc.js
similarity index 95%
rename from content/cn/assets/js/02e54e09.429268a3.js
rename to content/cn/assets/js/02e54e09.641379dc.js
index e988186363bfc..0c6bfc614f663 100644
--- a/content/cn/assets/js/02e54e09.429268a3.js
+++ b/content/cn/assets/js/02e54e09.641379dc.js
@@ -1 +1 @@
-"use strict";(self.webpackChunkhudi=self.webpackChunkhudi||[]).push([[34691],{28372:(e,t,i)=>{i.r(t),i.d(t,{assets:()=>o,contentTitle:()=>l,default:()=>u,frontMatter:()=>a,metadata:()=>n,toc:()=>c});const n=JSON.parse('{"id":"clustering","title":"Clustering","description":"Background","source":"@site/docs/clustering.md","sourceDirName":".","slug":"/clustering","permalink":"/cn/docs/next/clustering","draft":false,"unlisted":false,"editUrl":"https://github.com/apache/hudi/tree/asf-site/website/docs/clustering.md","tags":[],"version":"current","frontMatter":{"title":"Clustering","summary":"In this page, we describe async compaction in Hudi.","toc":true,"last_modified_at":null},"sidebar":"docs","previous":{"title":"Compaction","permalink":"/cn/docs/next/compaction"},"next":{"title":"Indexing","permalink":"/cn/docs/next/metadata_indexing"}}');var s=i(74848),r=i(28453);const a={title:"Clustering",summary:"In this page, we describe async compaction in Hudi.",toc:!0,last_modified_at:null},l=void 0,o={},c=[{value:"Background",id:"background",level:2},{value:"How is compaction different from clustering?",id:"how-is-compaction-different-from-clustering",level:2},{value:"Clustering Architecture",id:"clustering-architecture",level:2},{value:"Overall, there are 2 steps to clustering",id:"overall-there-are-2-steps-to-clustering",level:3},{value:"Schedule clustering",id:"schedule-clustering",level:3},{value:"Execute clustering",id:"execute-clustering",level:3},{value:"Clustering Usecases",id:"clustering-usecases",level:2},{value:"Batching small files",id:"batching-small-files",level:3},{value:"Cluster by sort key",id:"cluster-by-sort-key",level:3},{value:"Clustering Strategies",id:"clustering-strategies",level:2},{value:"Plan Strategy",id:"plan-strategy",level:3},{value:"Size-based clustering strategies",id:"size-based-clustering-strategies",level:4},{value:"SparkSingleFileSortPlanStrategy",id:"sparksinglefilesortplanstrategy",level:4},{value:"SparkConsistentBucketClusteringPlanStrategy",id:"sparkconsistentbucketclusteringplanstrategy",level:4},{value:"Execution Strategy",id:"execution-strategy",level:3},{value:"Update Strategy",id:"update-strategy",level:3},{value:"Inline clustering",id:"inline-clustering",level:2},{value:"Async Clustering",id:"async-clustering",level:2},{value:"Setup Asynchronous Clustering",id:"setup-asynchronous-clustering",level:2},{value:"HoodieClusteringJob",id:"hoodieclusteringjob",level:3},{value:"HoodieStreamer",id:"hoodiestreamer",level:3},{value:"Spark Structured Streaming",id:"spark-structured-streaming",level:3},{value:"Java Client",id:"java-client",level:2},{value:"Related Resources",id:"related-resources",level:2}];function d(e){const t={a:"a",admonition:"admonition",code:"code",em:"em",h2:"h2",h3:"h3",h4:"h4",img:"img",li:"li",ol:"ol",p:"p",pre:"pre",strong:"strong",table:"table",tbody:"tbody",td:"td",th:"th",thead:"thead",tr:"tr",ul:"ul",...(0,r.R)(),...e.components};return(0,s.jsxs)(s.Fragment,{children:[(0,s.jsx)(t.h2,{id:"background",children:"Background"}),"\n",(0,s.jsxs)(t.p,{children:["Apache Hudi brings stream processing to big data, providing fresh data while being an order of magnitude efficient over traditional batch processing. In a data lake/warehouse, one of the key trade-offs is between ingestion speed and query performance. Data ingestion typically prefers small files to improve parallelism and make data available to queries as soon as possible. However, query performance degrades poorly with a lot of small files. Also, during ingestion, data is typically co-located based on arrival time. However, the query engines perform better when the data frequently queried is co-located together. In most architectures each of these systems tend to add optimizations independently to improve performance which hits limitations due to un-optimized data layouts. This doc introduces a new kind of table service called clustering ",(0,s.jsx)(t.a,{href:"https://cwiki.apache.org/confluence/display/HUDI/RFC+-+19+Clustering+data+for+freshness+and+query+performance",children:"[RFC-19]"})," to reorganize data for improved query performance without compromising on ingestion speed."]}),"\n",(0,s.jsx)(t.h2,{id:"how-is-compaction-different-from-clustering",children:"How is compaction different from clustering?"}),"\n",(0,s.jsxs)(t.p,{children:["Hudi is modeled like a log-structured storage engine with multiple versions of the data.\nParticularly, ",(0,s.jsx)(t.a,{href:"/docs/table_types#merge-on-read-table",children:"Merge-On-Read"}),"\ntables in Hudi store data using a combination of base file in columnar format and row-based delta logs that contain\nupdates. Compaction is a way to merge the delta logs with base files to produce the latest file slices with the most\nrecent snapshot of data. Compaction helps to keep the query performance in check (larger delta log files would incur\nlonger merge times on query side). On the other hand, clustering is a data layout optimization technique. One can stitch\ntogether small files into larger files using clustering. Additionally, data can be clustered by sort key so that queries\ncan take advantage of data locality."]}),"\n",(0,s.jsx)(t.h2,{id:"clustering-architecture",children:"Clustering Architecture"}),"\n",(0,s.jsxs)(t.p,{children:["At a high level, Hudi provides different operations such as insert/upsert/bulk_insert through it\u2019s write client API to be able to write data to a Hudi table. To be able to choose a trade-off between file size and ingestion speed, Hudi provides a knob ",(0,s.jsx)(t.code,{children:"hoodie.parquet.small.file.limit"})," to be able to configure the smallest allowable file size. Users are able to configure the small file ",(0,s.jsx)(t.a,{href:"https://hudi.apache.org/docs/configurations/#hoodieparquetsmallfilelimit",children:"soft limit"})," to ",(0,s.jsx)(t.code,{children:"0"})," to force new data to go into a new set of filegroups or set it to a higher value to ensure new data gets \u201cpadded\u201d to existing files until it meets that limit that adds to ingestion latencies."]}),"\n",(0,s.jsx)(t.p,{children:"To be able to support an architecture that allows for fast ingestion without compromising query performance, we have introduced a \u2018clustering\u2019 service to rewrite the data to optimize Hudi data lake file layout."}),"\n",(0,s.jsx)(t.p,{children:"Clustering table service can run asynchronously or synchronously adding a new action type called \u201cREPLACE\u201d, that will mark the clustering action in the Hudi metadata timeline."}),"\n",(0,s.jsx)(t.h3,{id:"overall-there-are-2-steps-to-clustering",children:"Overall, there are 2 steps to clustering"}),"\n",(0,s.jsxs)(t.ol,{children:["\n",(0,s.jsx)(t.li,{children:"Scheduling clustering: Create a clustering plan using a pluggable clustering strategy."}),"\n",(0,s.jsx)(t.li,{children:"Execute clustering: Process the plan using an execution strategy to create new files and replace old files."}),"\n"]}),"\n",(0,s.jsx)(t.h3,{id:"schedule-clustering",children:"Schedule clustering"}),"\n",(0,s.jsx)(t.p,{children:"Following steps are followed to schedule clustering."}),"\n",(0,s.jsxs)(t.ol,{children:["\n",(0,s.jsx)(t.li,{children:"Identify files that are eligible for clustering: Depending on the clustering strategy chosen, the scheduling logic will identify the files eligible for clustering."}),"\n",(0,s.jsx)(t.li,{children:"Group files that are eligible for clustering based on specific criteria. Each group is expected to have data size in multiples of \u2018targetFileSize\u2019. Grouping is done as part of \u2018strategy\u2019 defined in the plan. Additionally, there is an option to put a cap on group size to improve parallelism and avoid shuffling large amounts of data."}),"\n",(0,s.jsxs)(t.li,{children:["Finally, the clustering plan is saved to the timeline in an avro ",(0,s.jsx)(t.a,{href:"https://github.com/apache/hudi/blob/master/hudi-common/src/main/avro/HoodieClusteringPlan.avsc",children:"metadata format"}),"."]}),"\n"]}),"\n",(0,s.jsx)(t.h3,{id:"execute-clustering",children:"Execute clustering"}),"\n",(0,s.jsxs)(t.ol,{children:["\n",(0,s.jsx)(t.li,{children:"Read the clustering plan and get the \u2018clusteringGroups\u2019 that mark the file groups that need to be clustered."}),"\n",(0,s.jsx)(t.li,{children:"For each group, we instantiate appropriate strategy class with strategyParams (example: sortColumns) and apply that strategy to rewrite the data."}),"\n",(0,s.jsxs)(t.li,{children:["Create a \u201cREPLACE\u201d commit and update the metadata in ",(0,s.jsx)(t.a,{href:"https://github.com/apache/hudi/blob/master/hudi-common/src/main/java/org/apache/hudi/common/model/HoodieReplaceCommitMetadata.java",children:"HoodieReplaceCommitMetadata"}),"."]}),"\n"]}),"\n",(0,s.jsx)(t.p,{children:"Clustering Service builds on Hudi\u2019s MVCC based design to allow for writers to continue to insert new data while clustering action runs in the background to reformat data layout, ensuring snapshot isolation between concurrent readers and writers."}),"\n",(0,s.jsx)(t.p,{children:"NOTE: Clustering can only be scheduled for tables / partitions not receiving any concurrent updates. In the future, concurrent updates use-case will be supported as well."}),"\n",(0,s.jsxs)(t.p,{children:[(0,s.jsx)(t.img,{alt:"Clustering example",src:i(7004).A+"",width:"6827",height:"3334"}),"\n",(0,s.jsx)(t.em,{children:"Figure: Illustrating query performance improvements by clustering"})]}),"\n",(0,s.jsx)(t.h2,{id:"clustering-usecases",children:"Clustering Usecases"}),"\n",(0,s.jsx)(t.h3,{id:"batching-small-files",children:"Batching small files"}),"\n",(0,s.jsx)(t.p,{children:"As mentioned in the intro, streaming ingestion generally results in smaller files in your data lake. But having a lot of\nsuch small files could lead to higher query latency. From our experience supporting community users, there are quite a\nfew users who are using Hudi just for small file handling capabilities. So, you could employ clustering to batch a lot\nof such small files into larger ones."}),"\n",(0,s.jsx)(t.p,{children:(0,s.jsx)(t.img,{alt:"Batching small files",src:i(67491).A+"",width:"3100",height:"1620"})}),"\n",(0,s.jsx)(t.h3,{id:"cluster-by-sort-key",children:"Cluster by sort key"}),"\n",(0,s.jsx)(t.p,{children:"Another classic problem in data lake is the arrival time vs event time problem. Generally you write data based on\narrival time, while query predicates do not sit well with it. With clustering, you can re-write your data by sorting\nbased on query predicates and so, your data skipping will be very efficient and your query can ignore scanning a lot of\nunnecessary data."}),"\n",(0,s.jsx)(t.p,{children:(0,s.jsx)(t.img,{alt:"Batching small files",src:i(14438).A+"",width:"5003",height:"1464"})}),"\n",(0,s.jsx)(t.h2,{id:"clustering-strategies",children:"Clustering Strategies"}),"\n",(0,s.jsx)(t.p,{children:"On a high level, clustering creates a plan based on a configurable strategy, groups eligible files based on specific\ncriteria and then executes the plan. As mentioned before, clustering plan as well as execution depends on configurable\nstrategy. These strategies can be broadly classified into three types: clustering plan strategy, execution strategy and\nupdate strategy."}),"\n",(0,s.jsx)(t.h3,{id:"plan-strategy",children:"Plan Strategy"}),"\n",(0,s.jsxs)(t.p,{children:["This strategy comes into play while creating clustering plan. It helps to decide what file groups should be clustered\nand how many output file groups should the clustering produce. Note that these strategies are easily pluggable using the\nconfig ",(0,s.jsx)(t.a,{href:"/docs/configurations#hoodieclusteringplanstrategyclass",children:"hoodie.clustering.plan.strategy.class"}),"."]}),"\n",(0,s.jsx)(t.p,{children:"Different plan strategies are as follows:"}),"\n",(0,s.jsx)(t.h4,{id:"size-based-clustering-strategies",children:"Size-based clustering strategies"}),"\n",(0,s.jsxs)(t.p,{children:["This strategy creates clustering groups based on max size allowed per group. Also, it excludes files that are greater\nthan the small file limit from the clustering plan. Available strategies depending on write client\nare: ",(0,s.jsx)(t.code,{children:"SparkSizeBasedClusteringPlanStrategy"}),", ",(0,s.jsx)(t.code,{children:"FlinkSizeBasedClusteringPlanStrategy"}),"\nand ",(0,s.jsx)(t.code,{children:"JavaSizeBasedClusteringPlanStrategy"}),". Furthermore, Hudi provides flexibility to include or exclude partitions for\nclustering, tune the file size limits, maximum number of output groups. Please refer to ",(0,s.jsx)(t.a,{href:"https://hudi.apache.org/docs/next/configurations/#hoodieclusteringplanstrategysmallfilelimit",children:"hoodie.clustering.plan.strategy.small.file.limit"}),"\n, ",(0,s.jsx)(t.a,{href:"https://hudi.apache.org/docs/next/configurations/#hoodieclusteringplanstrategymaxnumgroups",children:"hoodie.clustering.plan.strategy.max.num.groups"}),", ",(0,s.jsx)(t.a,{href:"https://hudi.apache.org/docs/next/configurations/#hoodieclusteringplanstrategymaxbytespergroup",children:"hoodie.clustering.plan.strategy.max.bytes.per.group"}),"\n, ",(0,s.jsx)(t.a,{href:"https://hudi.apache.org/docs/next/configurations/#hoodieclusteringplanstrategytargetfilemaxbytes",children:"hoodie.clustering.plan.strategy.target.file.max.bytes"})," for more details."]}),"\n",(0,s.jsxs)(t.table,{children:[(0,s.jsx)(t.thead,{children:(0,s.jsxs)(t.tr,{children:[(0,s.jsx)(t.th,{children:"Config Name"}),(0,s.jsx)(t.th,{children:"Default"}),(0,s.jsx)(t.th,{children:"Description"})]})}),(0,s.jsxs)(t.tbody,{children:[(0,s.jsxs)(t.tr,{children:[(0,s.jsx)(t.td,{children:"hoodie.clustering.plan.strategy.partition.selected"}),(0,s.jsxs)(t.td,{children:["N/A ",(0,s.jsx)(t.strong,{children:"(Required)"})]}),(0,s.jsxs)(t.td,{children:["Comma separated list of partitions to run clustering",(0,s.jsx)("br",{}),(0,s.jsx)("br",{}),(0,s.jsx)(t.code,{children:"Config Param: PARTITION_SELECTED"}),(0,s.jsx)("br",{}),(0,s.jsx)(t.code,{children:"Since Version: 0.11.0"})]})]}),(0,s.jsxs)(t.tr,{children:[(0,s.jsx)(t.td,{children:"hoodie.clustering.plan.strategy.partition.regex.pattern"}),(0,s.jsxs)(t.td,{children:["N/A ",(0,s.jsx)(t.strong,{children:"(Required)"})]}),(0,s.jsxs)(t.td,{children:["Filter clustering partitions that matched regex pattern",(0,s.jsx)("br",{}),(0,s.jsx)("br",{}),(0,s.jsx)(t.code,{children:"Config Param: PARTITION_REGEX_PATTERN"}),(0,s.jsx)("br",{}),(0,s.jsx)(t.code,{children:"Since Version: 0.11.0"})]})]}),(0,s.jsxs)(t.tr,{children:[(0,s.jsx)(t.td,{children:"hoodie.clustering.plan.partition.filter.mode"}),(0,s.jsx)(t.td,{children:"NONE (Optional)"}),(0,s.jsxs)(t.td,{children:["Partition filter mode used in the creation of clustering plan. Possible values:",(0,s.jsx)("br",{}),(0,s.jsxs)("ul",{children:[(0,s.jsxs)("li",{children:[(0,s.jsx)(t.code,{children:"NONE"}),": Do not filter partitions. The clustering plan will include all partitions that have clustering candidates."]}),(0,s.jsxs)("li",{children:[(0,s.jsx)(t.code,{children:"RECENT_DAYS"}),": This filter assumes that your data is partitioned by date. The clustering plan will only include partitions from K days ago to N days ago, where K >= N. K is determined by ",(0,s.jsx)(t.code,{children:"hoodie.clustering.plan.strategy.daybased.lookback.partitions"})," and N is determined by ",(0,s.jsx)(t.code,{children:"hoodie.clustering.plan.strategy.daybased.skipfromlatest.partitions"}),"."]}),(0,s.jsxs)("li",{children:[(0,s.jsx)(t.code,{children:"SELECTED_PARTITIONS"}),": The clustering plan will include only partition paths with names that sort within the inclusive range [",(0,s.jsx)(t.code,{children:"hoodie.clustering.plan.strategy.cluster.begin.partition"}),", ",(0,s.jsx)(t.code,{children:"hoodie.clustering.plan.strategy.cluster.end.partition"}),"]."]}),(0,s.jsxs)("li",{children:[(0,s.jsx)(t.code,{children:"DAY_ROLLING"}),": To determine the partitions in the clustering plan, the eligible partitions will be sorted in ascending order. Each partition will have an index i in that list. The clustering plan will only contain partitions such that i mod 24 = H, where H is the current hour of the day (from 0 to 23)."]})]}),(0,s.jsx)("br",{}),(0,s.jsx)(t.code,{children:"Config Param: PLAN_PARTITION_FILTER_MODE_NAME"}),(0,s.jsx)("br",{}),(0,s.jsx)(t.code,{children:"Since Version: 0.11.0"})]})]})]})]}),"\n",(0,s.jsx)(t.h4,{id:"sparksinglefilesortplanstrategy",children:"SparkSingleFileSortPlanStrategy"}),"\n",(0,s.jsxs)(t.p,{children:["In this strategy, clustering group for each partition is built in the same way as ",(0,s.jsx)(t.code,{children:"SparkSizeBasedClusteringPlanStrategy"}),"\n. The difference is that the output group is 1 and file group id remains the same,\nwhile ",(0,s.jsx)(t.code,{children:"SparkSizeBasedClusteringPlanStrategy"})," can create multiple file groups with newer fileIds."]}),"\n",(0,s.jsx)(t.h4,{id:"sparkconsistentbucketclusteringplanstrategy",children:"SparkConsistentBucketClusteringPlanStrategy"}),"\n",(0,s.jsx)(t.p,{children:"This strategy is specifically used for consistent bucket index. This will be leveraged to expand your bucket index (from\nstatic partitioning to dynamic). Typically, users don\u2019t need to use this strategy. Hudi internally uses this for\ndynamically expanding the buckets for bucket index datasets."}),"\n",(0,s.jsx)(t.admonition,{title:"The latter two strategies are applicable only for the Spark engine.",type:"note"}),"\n",(0,s.jsx)(t.h3,{id:"execution-strategy",children:"Execution Strategy"}),"\n",(0,s.jsxs)(t.p,{children:["After building the clustering groups in the planning phase, Hudi applies execution strategy, for each group, primarily\nbased on sort columns and size. The strategy can be specified using the\nconfig ",(0,s.jsx)(t.a,{href:"/docs/configurations/#hoodieclusteringexecutionstrategyclass",children:"hoodie.clustering.execution.strategy.class"}),". By\ndefault, Hudi sorts the file groups in the plan by the specified columns, while meeting the configured target file\nsizes."]}),"\n",(0,s.jsxs)(t.table,{children:[(0,s.jsx)(t.thead,{children:(0,s.jsxs)(t.tr,{children:[(0,s.jsx)(t.th,{children:"Config Name"}),(0,s.jsx)(t.th,{children:"Default"}),(0,s.jsx)(t.th,{children:"Description"})]})}),(0,s.jsx)(t.tbody,{children:(0,s.jsxs)(t.tr,{children:[(0,s.jsx)(t.td,{children:"hoodie.clustering.execution.strategy.class"}),(0,s.jsx)(t.td,{children:"org.apache.hudi.client.clustering.run.strategy.SparkSortAndSizeExecutionStrategy (Optional)"}),(0,s.jsxs)(t.td,{children:["Config to provide a strategy class (subclass of RunClusteringStrategy) to define how the clustering plan is executed. By default, we sort the file groups in th plan by the specified columns, while meeting the configured target file sizes.",(0,s.jsx)("br",{}),(0,s.jsx)("br",{}),(0,s.jsx)(t.code,{children:"Config Param: EXECUTION_STRATEGY_CLASS_NAME"}),(0,s.jsx)("br",{}),(0,s.jsx)(t.code,{children:"Since Version: 0.7.0"})]})]})})]}),"\n",(0,s.jsx)(t.p,{children:"The available strategies are as follows:"}),"\n",(0,s.jsxs)(t.ol,{children:["\n",(0,s.jsxs)(t.li,{children:[(0,s.jsx)(t.code,{children:"SPARK_SORT_AND_SIZE_EXECUTION_STRATEGY"}),": Uses bulk_insert to re-write data from input file groups.","\n",(0,s.jsxs)(t.ol,{children:["\n",(0,s.jsxs)(t.li,{children:["Set ",(0,s.jsx)(t.code,{children:"hoodie.clustering.execution.strategy.class"}),"\nto ",(0,s.jsx)(t.code,{children:"org.apache.hudi.client.clustering.run.strategy.SparkSortAndSizeExecutionStrategy"}),"."]}),"\n",(0,s.jsxs)(t.li,{children:[(0,s.jsx)(t.code,{children:"hoodie.clustering.plan.strategy.sort.columns"}),": Columns to sort the data while clustering. This goes in\nconjunction with layout optimization strategies depending on your query predicates. One can set comma separated\nlist of columns that needs to be sorted in this config."]}),"\n"]}),"\n"]}),"\n",(0,s.jsxs)(t.li,{children:[(0,s.jsx)(t.code,{children:"JAVA_SORT_AND_SIZE_EXECUTION_STRATEGY"}),": Similar to ",(0,s.jsx)(t.code,{children:"SPARK_SORT_AND_SIZE_EXECUTION_STRATEGY"}),", for the Java and Flink\nengines. Set ",(0,s.jsx)(t.code,{children:"hoodie.clustering.execution.strategy.class"}),"\nto ",(0,s.jsx)(t.code,{children:"org.apache.hudi.client.clustering.run.strategy.JavaSortAndSizeExecutionStrategy"}),"."]}),"\n",(0,s.jsxs)(t.li,{children:[(0,s.jsx)(t.code,{children:"SPARK_CONSISTENT_BUCKET_EXECUTION_STRATEGY"}),": As the name implies, this is applicable to dynamically expand\nconsistent bucket index and only applicable to the Spark engine. Set ",(0,s.jsx)(t.code,{children:"hoodie.clustering.execution.strategy.class"}),"\nto ",(0,s.jsx)(t.code,{children:"org.apache.hudi.client.clustering.run.strategy.SparkConsistentBucketClusteringExecutionStrategy"}),"."]}),"\n"]}),"\n",(0,s.jsx)(t.h3,{id:"update-strategy",children:"Update Strategy"}),"\n",(0,s.jsxs)(t.p,{children:["Currently, clustering can only be scheduled for tables/partitions not receiving any concurrent updates. By default,\nthe config for update strategy - ",(0,s.jsx)(t.a,{href:"/docs/configurations/#hoodieclusteringupdatesstrategy",children:(0,s.jsx)(t.code,{children:"hoodie.clustering.updates.strategy"})})," is set to ",(0,s.jsx)(t.em,{children:(0,s.jsx)(t.strong,{children:"SparkRejectUpdateStrategy"})}),". If some file group has updates during clustering then it will reject updates and throw an\nexception. However, in some use-cases updates are very sparse and do not touch most file groups. The default strategy to\nsimply reject updates does not seem fair. In such use-cases, users can set the config to ",(0,s.jsx)(t.em,{children:(0,s.jsx)(t.strong,{children:"SparkAllowUpdateStrategy"})}),"."]}),"\n",(0,s.jsxs)(t.p,{children:["We discussed the critical strategy configurations. All other configurations related to clustering are\nlisted ",(0,s.jsx)(t.a,{href:"/docs/configurations/#Clustering-Configs",children:"here"}),". Out of this list, a few configurations that will be very useful\nfor inline or async clustering are shown below with code samples."]}),"\n",(0,s.jsx)(t.h2,{id:"inline-clustering",children:"Inline clustering"}),"\n",(0,s.jsx)(t.p,{children:"Inline clustering happens synchronously with the regular ingestion writer or as part of the data ingestion pipeline. This means the next round of ingestion cannot proceed until the clustering is complete With inline clustering, Hudi will schedule, plan clustering operations after each commit is completed and execute the clustering plans after it\u2019s created. This is the simplest deployment model to run because it\u2019s easier to manage than running different asynchronous Spark jobs. This mode is supported on Spark Datasource, Flink, Spark-SQL and DeltaStreamer in a sync-once mode."}),"\n",(0,s.jsxs)(t.p,{children:["For this deployment mode, please enable and set: ",(0,s.jsx)(t.code,{children:"hoodie.clustering.inline"})]}),"\n",(0,s.jsxs)(t.p,{children:["To choose how often clustering is triggered, also set: ",(0,s.jsx)(t.code,{children:"hoodie.clustering.inline.max.commits"}),"."]}),"\n",(0,s.jsx)(t.p,{children:"Inline clustering can be setup easily using spark dataframe options.\nSee sample below:"}),"\n",(0,s.jsx)(t.pre,{children:(0,s.jsx)(t.code,{className:"language-scala",children:'import org.apache.hudi.QuickstartUtils._\nimport scala.collection.JavaConversions._\nimport org.apache.spark.sql.SaveMode._\nimport org.apache.hudi.DataSourceReadOptions._\nimport org.apache.hudi.DataSourceWriteOptions._\nimport org.apache.hudi.config.HoodieWriteConfig._\n\n\nval df = //generate data frame\ndf.write.format("org.apache.hudi").\n options(getQuickstartWriteConfigs).\n option("hoodie.datasource.write.precombine.field", "ts").\n option("hoodie.datasource.write.recordkey.field", "uuid").\n option("hoodie.datasource.write.partitionpath.field", "partitionpath").\n option("hoodie.table.name", "tableName").\n option("hoodie.parquet.small.file.limit", "0").\n option("hoodie.clustering.inline", "true").\n option("hoodie.clustering.inline.max.commits", "4").\n option("hoodie.clustering.plan.strategy.target.file.max.bytes", "1073741824").\n option("hoodie.clustering.plan.strategy.small.file.limit", "629145600").\n option("hoodie.clustering.plan.strategy.sort.columns", "column1,column2"). //optional, if sorting is needed as part of rewriting data\n mode(Append).\n save("dfs://location");\n'})}),"\n",(0,s.jsx)(t.h2,{id:"async-clustering",children:"Async Clustering"}),"\n",(0,s.jsx)(t.p,{children:"Async clustering runs the clustering table service in the background without blocking the regular ingestions writers. There are three different ways to deploy an asynchronous clustering process:"}),"\n",(0,s.jsxs)(t.ul,{children:["\n",(0,s.jsxs)(t.li,{children:[(0,s.jsx)(t.strong,{children:"Asynchronous execution within the same process"}),": In this deployment mode, Hudi will schedule and plan the clustering operations after each commit is completed as part of the ingestion pipeline. Separately, Hudi spins up another thread within the same job and executes the clustering table service. This is supported by Spark Streaming, Flink and DeltaStreamer in continuous mode. For this deployment mode, please enable ",(0,s.jsx)(t.code,{children:"hoodie.clustering.async.enabled"})," and ",(0,s.jsx)(t.code,{children:"hoodie.clustering.async.max.commits\u200b"}),"."]}),"\n",(0,s.jsxs)(t.li,{children:[(0,s.jsx)(t.strong,{children:"Asynchronous scheduling and execution by a separate process"}),": In this deployment mode, the application will write data to a Hudi table as part of the ingestion pipeline. A separate clustering job will schedule, plan and execute the clustering operation. By running a different job for the clustering operation, it rebalances how Hudi uses compute resources: fewer compute resources are needed for the ingestion, which makes ingestion latency stable, and an independent set of compute resources are reserved for the clustering process. Please configure the lock providers for the concurrency control among all jobs (both writer and table service jobs). In general, configure lock providers when there are two different jobs or two different processes occurring. All writers support this deployment model. For this deployment mode, no clustering configs should be set for the ingestion writer."]}),"\n",(0,s.jsxs)(t.li,{children:[(0,s.jsx)(t.strong,{children:"Scheduling inline and executing async"}),": In this deployment mode, the application ingests data and schedules the clustering in one job; in another, the application executes the clustering plan. The supported writers (see below) won\u2019t be blocked from ingesting data. If the metadata table is enabled, a lock provider is not needed. However, if the metadata table is enabled, please ensure all jobs have the lock providers configured for concurrency control. All writers support this deployment option. For this deployment mode, please enable, ",(0,s.jsx)(t.code,{children:"hoodie.clustering.schedule.inline"})," and ",(0,s.jsx)(t.code,{children:"hoodie.clustering.async.enabled"}),"."]}),"\n"]}),"\n",(0,s.jsxs)(t.p,{children:["Hudi supports ",(0,s.jsx)(t.a,{href:"https://hudi.apache.org/docs/concurrency_control#enabling-multi-writing",children:"multi-writers"})," which provides\nsnapshot isolation between multiple table services, thus allowing writers to continue with ingestion while clustering\nruns in the background."]}),"\n",(0,s.jsxs)(t.table,{children:[(0,s.jsx)(t.thead,{children:(0,s.jsxs)(t.tr,{children:[(0,s.jsx)(t.th,{children:"Config Name"}),(0,s.jsx)(t.th,{children:"Default"}),(0,s.jsx)(t.th,{children:"Description"})]})}),(0,s.jsxs)(t.tbody,{children:[(0,s.jsxs)(t.tr,{children:[(0,s.jsx)(t.td,{children:"hoodie.clustering.async.enabled"}),(0,s.jsx)(t.td,{children:"false (Optional)"}),(0,s.jsxs)(t.td,{children:["Enable running of clustering service, asynchronously as inserts happen on the table.",(0,s.jsx)("br",{}),(0,s.jsx)("br",{}),(0,s.jsx)(t.code,{children:"Config Param: ASYNC_CLUSTERING_ENABLE"}),(0,s.jsx)("br",{}),(0,s.jsx)(t.code,{children:"Since Version: 0.7.0"})]})]}),(0,s.jsxs)(t.tr,{children:[(0,s.jsx)(t.td,{children:"hoodie.clustering.async.max.commits"}),(0,s.jsx)(t.td,{children:"4 (Optional)"}),(0,s.jsxs)(t.td,{children:["Config to control frequency of async clustering",(0,s.jsx)("br",{}),(0,s.jsx)("br",{}),(0,s.jsx)(t.code,{children:"Config Param: ASYNC_CLUSTERING_MAX_COMMITS"}),(0,s.jsx)("br",{}),(0,s.jsx)(t.code,{children:"Since Version: 0.9.0"})]})]})]})]}),"\n",(0,s.jsx)(t.h2,{id:"setup-asynchronous-clustering",children:"Setup Asynchronous Clustering"}),"\n",(0,s.jsxs)(t.p,{children:["Users can leverage ",(0,s.jsx)(t.a,{href:"https://cwiki.apache.org/confluence/display/HUDI/RFC+-+19+Clustering+data+for+freshness+and+query+performance#RFC19Clusteringdataforfreshnessandqueryperformance-SetupforAsyncclusteringJob",children:"HoodieClusteringJob"}),"\nto setup 2-step asynchronous clustering."]}),"\n",(0,s.jsx)(t.h3,{id:"hoodieclusteringjob",children:"HoodieClusteringJob"}),"\n",(0,s.jsxs)(t.p,{children:["By specifying the ",(0,s.jsx)(t.code,{children:"scheduleAndExecute"})," mode both schedule as well as clustering can be achieved in the same step.\nThe appropriate mode can be specified using ",(0,s.jsx)(t.code,{children:"-mode"})," or ",(0,s.jsx)(t.code,{children:"-m"})," option. There are three modes:"]}),"\n",(0,s.jsxs)(t.ol,{children:["\n",(0,s.jsxs)(t.li,{children:[(0,s.jsx)(t.code,{children:"schedule"}),": Make a clustering plan. This gives an instant which can be passed in execute mode."]}),"\n",(0,s.jsxs)(t.li,{children:[(0,s.jsx)(t.code,{children:"execute"}),": Execute a clustering plan at a particular instant. If no instant-time is specified, HoodieClusteringJob will execute for the earliest instant on the Hudi timeline."]}),"\n",(0,s.jsxs)(t.li,{children:[(0,s.jsx)(t.code,{children:"scheduleAndExecute"}),": Make a clustering plan first and execute that plan immediately."]}),"\n"]}),"\n",(0,s.jsx)(t.p,{children:"Note that to run this job while the original writer is still running, please enable multi-writing:"}),"\n",(0,s.jsx)(t.pre,{children:(0,s.jsx)(t.code,{children:"hoodie.write.concurrency.mode=optimistic_concurrency_control\nhoodie.write.lock.provider=org.apache.hudi.client.transaction.lock.ZookeeperBasedLockProvider\n"})}),"\n",(0,s.jsx)(t.p,{children:"A sample spark-submit command to setup HoodieClusteringJob is as below:"}),"\n",(0,s.jsx)(t.pre,{children:(0,s.jsx)(t.code,{className:"language-bash",children:'spark-submit \\\n--jars "packaging/hudi-utilities-slim-bundle/target/hudi-utilities-slim-bundle_2.12-1.0.0.jar,packaging/hudi-spark-bundle/target/hudi-spark3.5-bundle_2.12-1.0.0.jar" \\\n--class org.apache.hudi.utilities.HoodieClusteringJob \\\n/path/to/hudi-utilities-slim-bundle/target/hudi-utilities-slim-bundle_2.12-1.0.0.jar \\\n--props /path/to/config/clusteringjob.properties \\\n--mode scheduleAndExecute \\\n--base-path /path/to/hudi_table/basePath \\\n--table-name hudi_table_schedule_clustering \\\n--spark-memory 1g\n'})}),"\n",(0,s.jsxs)(t.p,{children:["A sample ",(0,s.jsx)(t.code,{children:"clusteringjob.properties"})," file:"]}),"\n",(0,s.jsx)(t.pre,{children:(0,s.jsx)(t.code,{children:"hoodie.clustering.async.enabled=true\nhoodie.clustering.async.max.commits=4\nhoodie.clustering.plan.strategy.target.file.max.bytes=1073741824\nhoodie.clustering.plan.strategy.small.file.limit=629145600\nhoodie.clustering.execution.strategy.class=org.apache.hudi.client.clustering.run.strategy.SparkSortAndSizeExecutionStrategy\nhoodie.clustering.plan.strategy.sort.columns=column1,column2\n"})}),"\n",(0,s.jsx)(t.h3,{id:"hoodiestreamer",children:"HoodieStreamer"}),"\n",(0,s.jsxs)(t.p,{children:["This brings us to our users' favorite utility in Hudi. Now, we can trigger asynchronous clustering with Hudi Streamer.\nJust set the ",(0,s.jsx)(t.code,{children:"hoodie.clustering.async.enabled"})," config to true and specify other clustering config in properties file\nwhose location can be pased as ",(0,s.jsx)(t.code,{children:"\u2014props"})," when starting the Hudi Streamer (just like in the case of HoodieClusteringJob)."]}),"\n",(0,s.jsx)(t.p,{children:"A sample spark-submit command to setup HoodieStreamer is as below:"}),"\n",(0,s.jsx)(t.pre,{children:(0,s.jsx)(t.code,{className:"language-bash",children:'spark-submit \\\n--jars "packaging/hudi-utilities-slim-bundle/target/hudi-utilities-slim-bundle_2.12-1.0.0.jar,packaging/hudi-spark-bundle/target/hudi-spark3.5-bundle_2.12-1.0.0.jar" \\\n--class org.apache.hudi.utilities.streamer.HoodieStreamer \\\n/path/to/hudi-utilities-slim-bundle/target/hudi-utilities-slim-bundle_2.12-1.0.0.jar \\\n--props /path/to/config/clustering_kafka.properties \\\n--schemaprovider-class org.apache.hudi.utilities.schema.SchemaRegistryProvider \\\n--source-class org.apache.hudi.utilities.sources.AvroKafkaSource \\\n--source-ordering-field impresssiontime \\\n--table-type COPY_ON_WRITE \\\n--target-base-path /path/to/hudi_table/basePath \\\n--target-table impressions_cow_cluster \\\n--op INSERT \\\n--hoodie-conf hoodie.clustering.async.enabled=true \\\n--continuous\n'})}),"\n",(0,s.jsx)(t.h3,{id:"spark-structured-streaming",children:"Spark Structured Streaming"}),"\n",(0,s.jsx)(t.p,{children:"We can also enable asynchronous clustering with Spark structured streaming sink as shown below."}),"\n",(0,s.jsx)(t.pre,{children:(0,s.jsx)(t.code,{className:"language-scala",children:'val commonOpts = Map(\n "hoodie.insert.shuffle.parallelism" -> "4",\n "hoodie.upsert.shuffle.parallelism" -> "4",\n "hoodie.datasource.write.recordkey.field" -> "_row_key",\n "hoodie.datasource.write.partitionpath.field" -> "partition",\n "hoodie.datasource.write.precombine.field" -> "timestamp",\n "hoodie.table.name" -> "hoodie_test"\n)\n\ndef getAsyncClusteringOpts(isAsyncClustering: String, \n clusteringNumCommit: String, \n executionStrategy: String):Map[String, String] = {\n commonOpts + (DataSourceWriteOptions.ASYNC_CLUSTERING_ENABLE.key -> isAsyncClustering,\n HoodieClusteringConfig.ASYNC_CLUSTERING_MAX_COMMITS.key -> clusteringNumCommit,\n HoodieClusteringConfig.EXECUTION_STRATEGY_CLASS_NAME.key -> executionStrategy\n )\n}\n\ndef initStreamingWriteFuture(hudiOptions: Map[String, String]): Future[Unit] = {\n val streamingInput = // define the source of streaming\n Future {\n println("streaming starting")\n streamingInput\n .writeStream\n .format("org.apache.hudi")\n .options(hudiOptions)\n .option("checkpointLocation", basePath + "/checkpoint")\n .mode(Append)\n .start()\n .awaitTermination(10000)\n println("streaming ends")\n }\n}\n\ndef structuredStreamingWithClustering(): Unit = {\n val df = //generate data frame\n val hudiOptions = getClusteringOpts("true", "1", "org.apache.hudi.client.clustering.run.strategy.SparkSortAndSizeExecutionStrategy")\n val f1 = initStreamingWriteFuture(hudiOptions)\n Await.result(f1, Duration.Inf)\n}\n'})}),"\n",(0,s.jsx)(t.h2,{id:"java-client",children:"Java Client"}),"\n",(0,s.jsxs)(t.p,{children:["Clustering is also supported via Java client. Plan strategy ",(0,s.jsx)(t.code,{children:"org.apache.hudi.client.clustering.plan.strategy.JavaSizeBasedClusteringPlanStrategy"}),"\nand execution strategy ",(0,s.jsx)(t.code,{children:"org.apache.hudi.client.clustering.run.strategy.JavaSortAndSizeExecutionStrategy"})," are supported\nout-of-the-box. Note that as of now only linear sort is supported in Java execution strategy."]}),"\n",(0,s.jsx)(t.h2,{id:"related-resources",children:"Related Resources"}),"\n",(0,s.jsx)("h3",{children:"Videos"}),"\n",(0,s.jsxs)(t.ul,{children:["\n",(0,s.jsx)(t.li,{children:(0,s.jsx)(t.a,{href:"https://www.youtube.com/watch?v=R_sm4wlGXuE",children:"Understanding Clustering in Apache Hudi and the Benefits of Asynchronous Clustering"})}),"\n"]})]})}function u(e={}){const{wrapper:t}={...(0,r.R)(),...e.components};return t?(0,s.jsx)(t,{...e,children:(0,s.jsx)(d,{...e})}):d(e)}},7004:(e,t,i)=>{i.d(t,{A:()=>n});const n=i.p+"assets/images/clustering1_new-d67c9e691d235b140f7c80d68400f425.png"},67491:(e,t,i)=>{i.d(t,{A:()=>n});const n=i.p+"assets/images/clustering2_new-0837c07b6db44ab75873633f0eab2e2c.png"},14438:(e,t,i)=>{i.d(t,{A:()=>n});const n=i.p+"assets/images/clustering_3-8bc286ab44c48137f8409b5c342a7207.png"},28453:(e,t,i)=>{i.d(t,{R:()=>a,x:()=>l});var n=i(96540);const s={},r=n.createContext(s);function a(e){const t=n.useContext(r);return n.useMemo((function(){return"function"==typeof e?e(t):{...t,...e}}),[t,e])}function l(e){let t;return t=e.disableParentContext?"function"==typeof e.components?e.components(s):e.components||s:a(e.components),n.createElement(r.Provider,{value:t},e.children)}}}]);
\ No newline at end of file
+"use strict";(self.webpackChunkhudi=self.webpackChunkhudi||[]).push([[34691],{28372:(e,t,i)=>{i.r(t),i.d(t,{assets:()=>o,contentTitle:()=>l,default:()=>u,frontMatter:()=>a,metadata:()=>n,toc:()=>c});const n=JSON.parse('{"id":"clustering","title":"Clustering","description":"Background","source":"@site/docs/clustering.md","sourceDirName":".","slug":"/clustering","permalink":"/cn/docs/next/clustering","draft":false,"unlisted":false,"editUrl":"https://github.com/apache/hudi/tree/asf-site/website/docs/clustering.md","tags":[],"version":"current","frontMatter":{"title":"Clustering","summary":"In this page, we describe async compaction in Hudi.","toc":true,"last_modified_at":null},"sidebar":"docs","previous":{"title":"Compaction","permalink":"/cn/docs/next/compaction"},"next":{"title":"Indexing","permalink":"/cn/docs/next/metadata_indexing"}}');var s=i(74848),r=i(28453);const a={title:"Clustering",summary:"In this page, we describe async compaction in Hudi.",toc:!0,last_modified_at:null},l=void 0,o={},c=[{value:"Background",id:"background",level:2},{value:"How is compaction different from clustering?",id:"how-is-compaction-different-from-clustering",level:2},{value:"Clustering Architecture",id:"clustering-architecture",level:2},{value:"Overall, there are 2 steps to clustering",id:"overall-there-are-2-steps-to-clustering",level:3},{value:"Schedule clustering",id:"schedule-clustering",level:3},{value:"Execute clustering",id:"execute-clustering",level:3},{value:"Clustering Usecases",id:"clustering-usecases",level:2},{value:"Batching small files",id:"batching-small-files",level:3},{value:"Cluster by sort key",id:"cluster-by-sort-key",level:3},{value:"Clustering Strategies",id:"clustering-strategies",level:2},{value:"Plan Strategy",id:"plan-strategy",level:3},{value:"Size-based clustering strategies",id:"size-based-clustering-strategies",level:4},{value:"SparkSingleFileSortPlanStrategy",id:"sparksinglefilesortplanstrategy",level:4},{value:"SparkConsistentBucketClusteringPlanStrategy",id:"sparkconsistentbucketclusteringplanstrategy",level:4},{value:"Execution Strategy",id:"execution-strategy",level:3},{value:"Update Strategy",id:"update-strategy",level:3},{value:"Inline clustering",id:"inline-clustering",level:2},{value:"Async Clustering",id:"async-clustering",level:2},{value:"Setup Asynchronous Clustering",id:"setup-asynchronous-clustering",level:2},{value:"HoodieClusteringJob",id:"hoodieclusteringjob",level:3},{value:"HoodieStreamer",id:"hoodiestreamer",level:3},{value:"Spark Structured Streaming",id:"spark-structured-streaming",level:3},{value:"Java Client",id:"java-client",level:2},{value:"Related Resources",id:"related-resources",level:2}];function d(e){const t={a:"a",admonition:"admonition",code:"code",em:"em",h2:"h2",h3:"h3",h4:"h4",img:"img",li:"li",ol:"ol",p:"p",pre:"pre",strong:"strong",table:"table",tbody:"tbody",td:"td",th:"th",thead:"thead",tr:"tr",ul:"ul",...(0,r.R)(),...e.components};return(0,s.jsxs)(s.Fragment,{children:[(0,s.jsx)(t.h2,{id:"background",children:"Background"}),"\n",(0,s.jsxs)(t.p,{children:["Apache Hudi brings stream processing to big data, providing fresh data while being an order of magnitude efficient over traditional batch processing. In a data lake/warehouse, one of the key trade-offs is between ingestion speed and query performance. Data ingestion typically prefers small files to improve parallelism and make data available to queries as soon as possible. However, query performance degrades poorly with a lot of small files. Also, during ingestion, data is typically co-located based on arrival time. However, the query engines perform better when the data frequently queried is co-located together. In most architectures each of these systems tend to add optimizations independently to improve performance which hits limitations due to un-optimized data layouts. This doc introduces a new kind of table service called clustering ",(0,s.jsx)(t.a,{href:"https://cwiki.apache.org/confluence/display/HUDI/RFC+-+19+Clustering+data+for+freshness+and+query+performance",children:"[RFC-19]"})," to reorganize data for improved query performance without compromising on ingestion speed."]}),"\n",(0,s.jsx)(t.h2,{id:"how-is-compaction-different-from-clustering",children:"How is compaction different from clustering?"}),"\n",(0,s.jsxs)(t.p,{children:["Hudi is modeled like a log-structured storage engine with multiple versions of the data.\nParticularly, ",(0,s.jsx)(t.a,{href:"/docs/table_types#merge-on-read-table",children:"Merge-On-Read"}),"\ntables in Hudi store data using a combination of base file in columnar format and row-based delta logs that contain\nupdates. Compaction is a way to merge the delta logs with base files to produce the latest file slices with the most\nrecent snapshot of data. Compaction helps to keep the query performance in check (larger delta log files would incur\nlonger merge times on query side). On the other hand, clustering is a data layout optimization technique. One can stitch\ntogether small files into larger files using clustering. Additionally, data can be clustered by sort key so that queries\ncan take advantage of data locality."]}),"\n",(0,s.jsx)(t.h2,{id:"clustering-architecture",children:"Clustering Architecture"}),"\n",(0,s.jsxs)(t.p,{children:["At a high level, Hudi provides different operations such as insert/upsert/bulk_insert through it\u2019s write client API to be able to write data to a Hudi table. To be able to choose a trade-off between file size and ingestion speed, Hudi provides a knob ",(0,s.jsx)(t.code,{children:"hoodie.parquet.small.file.limit"})," to be able to configure the smallest allowable file size. Users are able to configure the small file ",(0,s.jsx)(t.a,{href:"https://hudi.apache.org/docs/configurations/#hoodieparquetsmallfilelimit",children:"soft limit"})," to ",(0,s.jsx)(t.code,{children:"0"})," to force new data to go into a new set of filegroups or set it to a higher value to ensure new data gets \u201cpadded\u201d to existing files until it meets that limit that adds to ingestion latencies."]}),"\n",(0,s.jsx)(t.p,{children:"To be able to support an architecture that allows for fast ingestion without compromising query performance, we have introduced a \u2018clustering\u2019 service to rewrite the data to optimize Hudi data lake file layout."}),"\n",(0,s.jsx)(t.p,{children:"Clustering table service can run asynchronously or synchronously adding a new action type called \u201cREPLACE\u201d, that will mark the clustering action in the Hudi metadata timeline."}),"\n",(0,s.jsx)(t.h3,{id:"overall-there-are-2-steps-to-clustering",children:"Overall, there are 2 steps to clustering"}),"\n",(0,s.jsxs)(t.ol,{children:["\n",(0,s.jsx)(t.li,{children:"Scheduling clustering: Create a clustering plan using a pluggable clustering strategy."}),"\n",(0,s.jsx)(t.li,{children:"Execute clustering: Process the plan using an execution strategy to create new files and replace old files."}),"\n"]}),"\n",(0,s.jsx)(t.h3,{id:"schedule-clustering",children:"Schedule clustering"}),"\n",(0,s.jsx)(t.p,{children:"Following steps are followed to schedule clustering."}),"\n",(0,s.jsxs)(t.ol,{children:["\n",(0,s.jsx)(t.li,{children:"Identify files that are eligible for clustering: Depending on the clustering strategy chosen, the scheduling logic will identify the files eligible for clustering."}),"\n",(0,s.jsx)(t.li,{children:"Group files that are eligible for clustering based on specific criteria. Each group is expected to have data size in multiples of \u2018targetFileSize\u2019. Grouping is done as part of \u2018strategy\u2019 defined in the plan. Additionally, there is an option to put a cap on group size to improve parallelism and avoid shuffling large amounts of data."}),"\n",(0,s.jsxs)(t.li,{children:["Finally, the clustering plan is saved to the timeline in an avro ",(0,s.jsx)(t.a,{href:"https://github.com/apache/hudi/blob/master/hudi-common/src/main/avro/HoodieClusteringPlan.avsc",children:"metadata format"}),"."]}),"\n"]}),"\n",(0,s.jsx)(t.h3,{id:"execute-clustering",children:"Execute clustering"}),"\n",(0,s.jsxs)(t.ol,{children:["\n",(0,s.jsx)(t.li,{children:"Read the clustering plan and get the \u2018clusteringGroups\u2019 that mark the file groups that need to be clustered."}),"\n",(0,s.jsx)(t.li,{children:"For each group, we instantiate appropriate strategy class with strategyParams (example: sortColumns) and apply that strategy to rewrite the data."}),"\n",(0,s.jsxs)(t.li,{children:["Create a \u201cREPLACE\u201d commit and update the metadata in ",(0,s.jsx)(t.a,{href:"https://github.com/apache/hudi/blob/master/hudi-common/src/main/java/org/apache/hudi/common/model/HoodieReplaceCommitMetadata.java",children:"HoodieReplaceCommitMetadata"}),"."]}),"\n"]}),"\n",(0,s.jsx)(t.p,{children:"Clustering Service builds on Hudi\u2019s MVCC based design to allow for writers to continue to insert new data while clustering action runs in the background to reformat data layout, ensuring snapshot isolation between concurrent readers and writers."}),"\n",(0,s.jsx)(t.p,{children:"NOTE: Clustering can only be scheduled for tables / partitions not receiving any concurrent updates. In the future, concurrent updates use-case will be supported as well."}),"\n",(0,s.jsxs)(t.p,{children:[(0,s.jsx)(t.img,{alt:"Clustering example",src:i(7004).A+"",width:"6827",height:"3334"}),"\n",(0,s.jsx)(t.em,{children:"Figure: Illustrating query performance improvements by clustering"})]}),"\n",(0,s.jsx)(t.h2,{id:"clustering-usecases",children:"Clustering Usecases"}),"\n",(0,s.jsx)(t.h3,{id:"batching-small-files",children:"Batching small files"}),"\n",(0,s.jsx)(t.p,{children:"As mentioned in the intro, streaming ingestion generally results in smaller files in your data lake. But having a lot of\nsuch small files could lead to higher query latency. From our experience supporting community users, there are quite a\nfew users who are using Hudi just for small file handling capabilities. So, you could employ clustering to batch a lot\nof such small files into larger ones."}),"\n",(0,s.jsx)(t.p,{children:(0,s.jsx)(t.img,{alt:"Batching small files",src:i(67491).A+"",width:"3100",height:"1620"})}),"\n",(0,s.jsx)(t.h3,{id:"cluster-by-sort-key",children:"Cluster by sort key"}),"\n",(0,s.jsx)(t.p,{children:"Another classic problem in data lake is the arrival time vs event time problem. Generally you write data based on\narrival time, while query predicates do not sit well with it. With clustering, you can re-write your data by sorting\nbased on query predicates and so, your data skipping will be very efficient and your query can ignore scanning a lot of\nunnecessary data."}),"\n",(0,s.jsx)(t.p,{children:(0,s.jsx)(t.img,{alt:"Batching small files",src:i(14438).A+"",width:"5003",height:"1464"})}),"\n",(0,s.jsx)(t.h2,{id:"clustering-strategies",children:"Clustering Strategies"}),"\n",(0,s.jsx)(t.p,{children:"On a high level, clustering creates a plan based on a configurable strategy, groups eligible files based on specific\ncriteria and then executes the plan. As mentioned before, clustering plan as well as execution depends on configurable\nstrategy. These strategies can be broadly classified into three types: clustering plan strategy, execution strategy and\nupdate strategy."}),"\n",(0,s.jsx)(t.h3,{id:"plan-strategy",children:"Plan Strategy"}),"\n",(0,s.jsxs)(t.p,{children:["This strategy comes into play while creating clustering plan. It helps to decide what file groups should be clustered\nand how many output file groups should the clustering produce. Note that these strategies are easily pluggable using the\nconfig ",(0,s.jsx)(t.a,{href:"/docs/configurations#hoodieclusteringplanstrategyclass",children:"hoodie.clustering.plan.strategy.class"}),"."]}),"\n",(0,s.jsx)(t.p,{children:"Different plan strategies are as follows:"}),"\n",(0,s.jsx)(t.h4,{id:"size-based-clustering-strategies",children:"Size-based clustering strategies"}),"\n",(0,s.jsxs)(t.p,{children:["This strategy creates clustering groups based on max size allowed per group. Also, it excludes files that are greater\nthan the small file limit from the clustering plan. Available strategies depending on write client\nare: ",(0,s.jsx)(t.code,{children:"SparkSizeBasedClusteringPlanStrategy"}),", ",(0,s.jsx)(t.code,{children:"FlinkSizeBasedClusteringPlanStrategy"}),"\nand ",(0,s.jsx)(t.code,{children:"JavaSizeBasedClusteringPlanStrategy"}),". Furthermore, Hudi provides flexibility to include or exclude partitions for\nclustering, tune the file size limits, maximum number of output groups. Please refer to ",(0,s.jsx)(t.a,{href:"https://hudi.apache.org/docs/next/configurations/#hoodieclusteringplanstrategysmallfilelimit",children:"hoodie.clustering.plan.strategy.small.file.limit"}),"\n, ",(0,s.jsx)(t.a,{href:"https://hudi.apache.org/docs/next/configurations/#hoodieclusteringplanstrategymaxnumgroups",children:"hoodie.clustering.plan.strategy.max.num.groups"}),", ",(0,s.jsx)(t.a,{href:"https://hudi.apache.org/docs/next/configurations/#hoodieclusteringplanstrategymaxbytespergroup",children:"hoodie.clustering.plan.strategy.max.bytes.per.group"}),"\n, ",(0,s.jsx)(t.a,{href:"https://hudi.apache.org/docs/next/configurations/#hoodieclusteringplanstrategytargetfilemaxbytes",children:"hoodie.clustering.plan.strategy.target.file.max.bytes"})," for more details."]}),"\n",(0,s.jsxs)(t.table,{children:[(0,s.jsx)(t.thead,{children:(0,s.jsxs)(t.tr,{children:[(0,s.jsx)(t.th,{children:"Config Name"}),(0,s.jsx)(t.th,{children:"Default"}),(0,s.jsx)(t.th,{children:"Description"})]})}),(0,s.jsxs)(t.tbody,{children:[(0,s.jsxs)(t.tr,{children:[(0,s.jsx)(t.td,{children:"hoodie.clustering.plan.strategy.partition.selected"}),(0,s.jsxs)(t.td,{children:["N/A ",(0,s.jsx)(t.strong,{children:"(Required)"})]}),(0,s.jsxs)(t.td,{children:["Comma separated list of partitions to run clustering",(0,s.jsx)("br",{}),(0,s.jsx)("br",{}),(0,s.jsx)(t.code,{children:"Config Param: PARTITION_SELECTED"}),(0,s.jsx)("br",{}),(0,s.jsx)(t.code,{children:"Since Version: 0.11.0"})]})]}),(0,s.jsxs)(t.tr,{children:[(0,s.jsx)(t.td,{children:"hoodie.clustering.plan.strategy.partition.regex.pattern"}),(0,s.jsxs)(t.td,{children:["N/A ",(0,s.jsx)(t.strong,{children:"(Required)"})]}),(0,s.jsxs)(t.td,{children:["Filter clustering partitions that matched regex pattern",(0,s.jsx)("br",{}),(0,s.jsx)("br",{}),(0,s.jsx)(t.code,{children:"Config Param: PARTITION_REGEX_PATTERN"}),(0,s.jsx)("br",{}),(0,s.jsx)(t.code,{children:"Since Version: 0.11.0"})]})]}),(0,s.jsxs)(t.tr,{children:[(0,s.jsx)(t.td,{children:"hoodie.clustering.plan.partition.filter.mode"}),(0,s.jsx)(t.td,{children:"NONE (Optional)"}),(0,s.jsxs)(t.td,{children:["Partition filter mode used in the creation of clustering plan. Possible values:",(0,s.jsx)("br",{}),(0,s.jsxs)("ul",{children:[(0,s.jsxs)("li",{children:[(0,s.jsx)(t.code,{children:"NONE"}),": Do not filter partitions. The clustering plan will include all partitions that have clustering candidates."]}),(0,s.jsxs)("li",{children:[(0,s.jsx)(t.code,{children:"RECENT_DAYS"}),": This filter assumes that your data is partitioned by date. The clustering plan will only include partitions from K days ago to N days ago, where K >= N. K is determined by ",(0,s.jsx)(t.code,{children:"hoodie.clustering.plan.strategy.daybased.lookback.partitions"})," and N is determined by ",(0,s.jsx)(t.code,{children:"hoodie.clustering.plan.strategy.daybased.skipfromlatest.partitions"}),"."]}),(0,s.jsxs)("li",{children:[(0,s.jsx)(t.code,{children:"SELECTED_PARTITIONS"}),": The clustering plan will include only partition paths with names that sort within the inclusive range [",(0,s.jsx)(t.code,{children:"hoodie.clustering.plan.strategy.cluster.begin.partition"}),", ",(0,s.jsx)(t.code,{children:"hoodie.clustering.plan.strategy.cluster.end.partition"}),"]."]}),(0,s.jsxs)("li",{children:[(0,s.jsx)(t.code,{children:"DAY_ROLLING"}),": To determine the partitions in the clustering plan, the eligible partitions will be sorted in ascending order. Each partition will have an index i in that list. The clustering plan will only contain partitions such that i mod 24 = H, where H is the current hour of the day (from 0 to 23)."]})]}),(0,s.jsx)("br",{}),(0,s.jsx)(t.code,{children:"Config Param: PLAN_PARTITION_FILTER_MODE_NAME"}),(0,s.jsx)("br",{}),(0,s.jsx)(t.code,{children:"Since Version: 0.11.0"})]})]})]})]}),"\n",(0,s.jsx)(t.h4,{id:"sparksinglefilesortplanstrategy",children:"SparkSingleFileSortPlanStrategy"}),"\n",(0,s.jsxs)(t.p,{children:["In this strategy, clustering group for each partition is built in the same way as ",(0,s.jsx)(t.code,{children:"SparkSizeBasedClusteringPlanStrategy"}),"\n. The difference is that the output group is 1 and file group id remains the same,\nwhile ",(0,s.jsx)(t.code,{children:"SparkSizeBasedClusteringPlanStrategy"})," can create multiple file groups with newer fileIds."]}),"\n",(0,s.jsx)(t.h4,{id:"sparkconsistentbucketclusteringplanstrategy",children:"SparkConsistentBucketClusteringPlanStrategy"}),"\n",(0,s.jsx)(t.p,{children:"This strategy is specifically used for consistent bucket index. This will be leveraged to expand your bucket index (from\nstatic partitioning to dynamic). Typically, users don\u2019t need to use this strategy. Hudi internally uses this for\ndynamically expanding the buckets for bucket index datasets."}),"\n",(0,s.jsx)(t.admonition,{title:"The latter two strategies are applicable only for the Spark engine.",type:"note"}),"\n",(0,s.jsx)(t.h3,{id:"execution-strategy",children:"Execution Strategy"}),"\n",(0,s.jsxs)(t.p,{children:["After building the clustering groups in the planning phase, Hudi applies execution strategy, for each group, primarily\nbased on sort columns and size. The strategy can be specified using the\nconfig ",(0,s.jsx)(t.a,{href:"/docs/configurations/#hoodieclusteringexecutionstrategyclass",children:"hoodie.clustering.execution.strategy.class"}),". By\ndefault, Hudi sorts the file groups in the plan by the specified columns, while meeting the configured target file\nsizes."]}),"\n",(0,s.jsxs)(t.table,{children:[(0,s.jsx)(t.thead,{children:(0,s.jsxs)(t.tr,{children:[(0,s.jsx)(t.th,{children:"Config Name"}),(0,s.jsx)(t.th,{children:"Default"}),(0,s.jsx)(t.th,{children:"Description"})]})}),(0,s.jsx)(t.tbody,{children:(0,s.jsxs)(t.tr,{children:[(0,s.jsx)(t.td,{children:"hoodie.clustering.execution.strategy.class"}),(0,s.jsx)(t.td,{children:"org.apache.hudi.client.clustering.run.strategy.SparkSortAndSizeExecutionStrategy (Optional)"}),(0,s.jsxs)(t.td,{children:["Config to provide a strategy class (subclass of RunClusteringStrategy) to define how the clustering plan is executed. By default, we sort the file groups in th plan by the specified columns, while meeting the configured target file sizes.",(0,s.jsx)("br",{}),(0,s.jsx)("br",{}),(0,s.jsx)(t.code,{children:"Config Param: EXECUTION_STRATEGY_CLASS_NAME"}),(0,s.jsx)("br",{}),(0,s.jsx)(t.code,{children:"Since Version: 0.7.0"})]})]})})]}),"\n",(0,s.jsx)(t.p,{children:"The available strategies are as follows:"}),"\n",(0,s.jsxs)(t.ol,{children:["\n",(0,s.jsxs)(t.li,{children:[(0,s.jsx)(t.code,{children:"SPARK_SORT_AND_SIZE_EXECUTION_STRATEGY"}),": Uses bulk_insert to re-write data from input file groups.","\n",(0,s.jsxs)(t.ol,{children:["\n",(0,s.jsxs)(t.li,{children:["Set ",(0,s.jsx)(t.code,{children:"hoodie.clustering.execution.strategy.class"}),"\nto ",(0,s.jsx)(t.code,{children:"org.apache.hudi.client.clustering.run.strategy.SparkSortAndSizeExecutionStrategy"}),"."]}),"\n",(0,s.jsxs)(t.li,{children:[(0,s.jsx)(t.code,{children:"hoodie.clustering.plan.strategy.sort.columns"}),": Columns to sort the data while clustering. This goes in\nconjunction with layout optimization strategies depending on your query predicates. One can set comma separated\nlist of columns that needs to be sorted in this config."]}),"\n"]}),"\n"]}),"\n",(0,s.jsxs)(t.li,{children:[(0,s.jsx)(t.code,{children:"JAVA_SORT_AND_SIZE_EXECUTION_STRATEGY"}),": Similar to ",(0,s.jsx)(t.code,{children:"SPARK_SORT_AND_SIZE_EXECUTION_STRATEGY"}),", for the Java and Flink\nengines. Set ",(0,s.jsx)(t.code,{children:"hoodie.clustering.execution.strategy.class"}),"\nto ",(0,s.jsx)(t.code,{children:"org.apache.hudi.client.clustering.run.strategy.JavaSortAndSizeExecutionStrategy"}),"."]}),"\n",(0,s.jsxs)(t.li,{children:[(0,s.jsx)(t.code,{children:"SPARK_CONSISTENT_BUCKET_EXECUTION_STRATEGY"}),": As the name implies, this is applicable to dynamically expand\nconsistent bucket index and only applicable to the Spark engine. Set ",(0,s.jsx)(t.code,{children:"hoodie.clustering.execution.strategy.class"}),"\nto ",(0,s.jsx)(t.code,{children:"org.apache.hudi.client.clustering.run.strategy.SparkConsistentBucketClusteringExecutionStrategy"}),"."]}),"\n"]}),"\n",(0,s.jsx)(t.h3,{id:"update-strategy",children:"Update Strategy"}),"\n",(0,s.jsxs)(t.p,{children:["Currently, clustering can only be scheduled for tables/partitions not receiving any concurrent updates. By default,\nthe config for update strategy - ",(0,s.jsx)(t.a,{href:"/docs/configurations/#hoodieclusteringupdatesstrategy",children:(0,s.jsx)(t.code,{children:"hoodie.clustering.updates.strategy"})})," is set to ",(0,s.jsx)(t.em,{children:(0,s.jsx)(t.strong,{children:"SparkRejectUpdateStrategy"})}),". If some file group has updates during clustering then it will reject updates and throw an\nexception. However, in some use-cases updates are very sparse and do not touch most file groups. The default strategy to\nsimply reject updates does not seem fair. In such use-cases, users can set the config to ",(0,s.jsx)(t.em,{children:(0,s.jsx)(t.strong,{children:"SparkAllowUpdateStrategy"})}),"."]}),"\n",(0,s.jsxs)(t.p,{children:["We discussed the critical strategy configurations. All other configurations related to clustering are\nlisted ",(0,s.jsx)(t.a,{href:"/docs/configurations/#Clustering-Configs",children:"here"}),". Out of this list, a few configurations that will be very useful\nfor inline or async clustering are shown below with code samples."]}),"\n",(0,s.jsx)(t.h2,{id:"inline-clustering",children:"Inline clustering"}),"\n",(0,s.jsx)(t.p,{children:"Inline clustering happens synchronously with the regular ingestion writer or as part of the data ingestion pipeline. This means the next round of ingestion cannot proceed until the clustering is complete With inline clustering, Hudi will schedule, plan clustering operations after each commit is completed and execute the clustering plans after it\u2019s created. This is the simplest deployment model to run because it\u2019s easier to manage than running different asynchronous Spark jobs. This mode is supported on Spark Datasource, Flink, Spark-SQL and DeltaStreamer in a sync-once mode."}),"\n",(0,s.jsxs)(t.p,{children:["For this deployment mode, please enable and set: ",(0,s.jsx)(t.code,{children:"hoodie.clustering.inline"})]}),"\n",(0,s.jsxs)(t.p,{children:["To choose how often clustering is triggered, also set: ",(0,s.jsx)(t.code,{children:"hoodie.clustering.inline.max.commits"}),"."]}),"\n",(0,s.jsx)(t.p,{children:"Inline clustering can be setup easily using spark dataframe options.\nSee sample below:"}),"\n",(0,s.jsx)(t.pre,{children:(0,s.jsx)(t.code,{className:"language-scala",children:'import org.apache.hudi.QuickstartUtils._\nimport scala.collection.JavaConversions._\nimport org.apache.spark.sql.SaveMode._\nimport org.apache.hudi.DataSourceReadOptions._\nimport org.apache.hudi.DataSourceWriteOptions._\nimport org.apache.hudi.config.HoodieWriteConfig._\n\n\nval df = //generate data frame\ndf.write.format("org.apache.hudi").\n options(getQuickstartWriteConfigs).\n option("hoodie.datasource.write.precombine.field", "ts").\n option("hoodie.datasource.write.recordkey.field", "uuid").\n option("hoodie.datasource.write.partitionpath.field", "partitionpath").\n option("hoodie.table.name", "tableName").\n option("hoodie.parquet.small.file.limit", "0").\n option("hoodie.clustering.inline", "true").\n option("hoodie.clustering.inline.max.commits", "4").\n option("hoodie.clustering.plan.strategy.target.file.max.bytes", "1073741824").\n option("hoodie.clustering.plan.strategy.small.file.limit", "629145600").\n option("hoodie.clustering.plan.strategy.sort.columns", "column1,column2"). //optional, if sorting is needed as part of rewriting data\n mode(Append).\n save("dfs://location");\n'})}),"\n",(0,s.jsx)(t.h2,{id:"async-clustering",children:"Async Clustering"}),"\n",(0,s.jsx)(t.p,{children:"Async clustering runs the clustering table service in the background without blocking the regular ingestions writers. There are three different ways to deploy an asynchronous clustering process:"}),"\n",(0,s.jsxs)(t.ul,{children:["\n",(0,s.jsxs)(t.li,{children:[(0,s.jsx)(t.strong,{children:"Asynchronous execution within the same process"}),": In this deployment mode, Hudi will schedule and plan the clustering operations after each commit is completed as part of the ingestion pipeline. Separately, Hudi spins up another thread within the same job and executes the clustering table service. This is supported by Spark Streaming, Flink and DeltaStreamer in continuous mode. For this deployment mode, please enable ",(0,s.jsx)(t.code,{children:"hoodie.clustering.async.enabled"})," and ",(0,s.jsx)(t.code,{children:"hoodie.clustering.async.max.commits\u200b"}),"."]}),"\n",(0,s.jsxs)(t.li,{children:[(0,s.jsx)(t.strong,{children:"Asynchronous scheduling and execution by a separate process"}),": In this deployment mode, the application will write data to a Hudi table as part of the ingestion pipeline. A separate clustering job will schedule, plan and execute the clustering operation. By running a different job for the clustering operation, it rebalances how Hudi uses compute resources: fewer compute resources are needed for the ingestion, which makes ingestion latency stable, and an independent set of compute resources are reserved for the clustering process. Please configure the lock providers for the concurrency control among all jobs (both writer and table service jobs). In general, configure lock providers when there are two different jobs or two different processes occurring. All writers support this deployment model. For this deployment mode, no clustering configs should be set for the ingestion writer."]}),"\n",(0,s.jsxs)(t.li,{children:[(0,s.jsx)(t.strong,{children:"Scheduling inline and executing async"}),": In this deployment mode, the application ingests data and schedules the clustering in one job; in another, the application executes the clustering plan. The supported writers (see below) won\u2019t be blocked from ingesting data. If the metadata table is enabled, a lock provider is not needed. However, if the metadata table is enabled, please ensure all jobs have the lock providers configured for concurrency control. All writers support this deployment option. For this deployment mode, please enable, ",(0,s.jsx)(t.code,{children:"hoodie.clustering.schedule.inline"})," and ",(0,s.jsx)(t.code,{children:"hoodie.clustering.async.enabled"}),"."]}),"\n"]}),"\n",(0,s.jsxs)(t.p,{children:["Hudi supports ",(0,s.jsx)(t.a,{href:"https://hudi.apache.org/docs/concurrency_control#enabling-multi-writing",children:"multi-writers"})," which provides\nsnapshot isolation between multiple table services, thus allowing writers to continue with ingestion while clustering\nruns in the background."]}),"\n",(0,s.jsxs)(t.table,{children:[(0,s.jsx)(t.thead,{children:(0,s.jsxs)(t.tr,{children:[(0,s.jsx)(t.th,{children:"Config Name"}),(0,s.jsx)(t.th,{children:"Default"}),(0,s.jsx)(t.th,{children:"Description"})]})}),(0,s.jsxs)(t.tbody,{children:[(0,s.jsxs)(t.tr,{children:[(0,s.jsx)(t.td,{children:"hoodie.clustering.async.enabled"}),(0,s.jsx)(t.td,{children:"false (Optional)"}),(0,s.jsxs)(t.td,{children:["Enable running of clustering service, asynchronously as inserts happen on the table.",(0,s.jsx)("br",{}),(0,s.jsx)("br",{}),(0,s.jsx)(t.code,{children:"Config Param: ASYNC_CLUSTERING_ENABLE"}),(0,s.jsx)("br",{}),(0,s.jsx)(t.code,{children:"Since Version: 0.7.0"})]})]}),(0,s.jsxs)(t.tr,{children:[(0,s.jsx)(t.td,{children:"hoodie.clustering.async.max.commits"}),(0,s.jsx)(t.td,{children:"4 (Optional)"}),(0,s.jsxs)(t.td,{children:["Config to control frequency of async clustering",(0,s.jsx)("br",{}),(0,s.jsx)("br",{}),(0,s.jsx)(t.code,{children:"Config Param: ASYNC_CLUSTERING_MAX_COMMITS"}),(0,s.jsx)("br",{}),(0,s.jsx)(t.code,{children:"Since Version: 0.9.0"})]})]})]})]}),"\n",(0,s.jsx)(t.h2,{id:"setup-asynchronous-clustering",children:"Setup Asynchronous Clustering"}),"\n",(0,s.jsxs)(t.p,{children:["Users can leverage ",(0,s.jsx)(t.a,{href:"https://cwiki.apache.org/confluence/display/HUDI/RFC+-+19+Clustering+data+for+freshness+and+query+performance#RFC19Clusteringdataforfreshnessandqueryperformance-SetupforAsyncclusteringJob",children:"HoodieClusteringJob"}),"\nto setup 2-step asynchronous clustering."]}),"\n",(0,s.jsx)(t.h3,{id:"hoodieclusteringjob",children:"HoodieClusteringJob"}),"\n",(0,s.jsxs)(t.p,{children:["By specifying the ",(0,s.jsx)(t.code,{children:"scheduleAndExecute"})," mode both schedule as well as clustering can be achieved in the same step.\nThe appropriate mode can be specified using ",(0,s.jsx)(t.code,{children:"-mode"})," or ",(0,s.jsx)(t.code,{children:"-m"})," option. There are three modes:"]}),"\n",(0,s.jsxs)(t.ol,{children:["\n",(0,s.jsxs)(t.li,{children:[(0,s.jsx)(t.code,{children:"schedule"}),": Make a clustering plan. This gives an instant which can be passed in execute mode."]}),"\n",(0,s.jsxs)(t.li,{children:[(0,s.jsx)(t.code,{children:"execute"}),": Execute a clustering plan at a particular instant. If no instant-time is specified, HoodieClusteringJob will execute for the earliest instant on the Hudi timeline."]}),"\n",(0,s.jsxs)(t.li,{children:[(0,s.jsx)(t.code,{children:"scheduleAndExecute"}),": Make a clustering plan first and execute that plan immediately."]}),"\n"]}),"\n",(0,s.jsx)(t.p,{children:"Note that to run this job while the original writer is still running, please enable multi-writing:"}),"\n",(0,s.jsx)(t.pre,{children:(0,s.jsx)(t.code,{children:"hoodie.write.concurrency.mode=optimistic_concurrency_control\nhoodie.write.lock.provider=org.apache.hudi.client.transaction.lock.ZookeeperBasedLockProvider\n"})}),"\n",(0,s.jsx)(t.p,{children:"A sample spark-submit command to setup HoodieClusteringJob is as below:"}),"\n",(0,s.jsx)(t.pre,{children:(0,s.jsx)(t.code,{className:"language-bash",children:'spark-submit \\\n--jars "packaging/hudi-utilities-slim-bundle/target/hudi-utilities-slim-bundle_2.12-1.0.0.jar,packaging/hudi-spark-bundle/target/hudi-spark3.5-bundle_2.12-1.0.0.jar" \\\n--class org.apache.hudi.utilities.HoodieClusteringJob \\\n/path/to/hudi-utilities-slim-bundle/target/hudi-utilities-slim-bundle_2.12-1.0.0.jar \\\n--props /path/to/config/clusteringjob.properties \\\n--mode scheduleAndExecute \\\n--base-path /path/to/hudi_table/basePath \\\n--table-name hudi_table_schedule_clustering \\\n--spark-memory 1g\n'})}),"\n",(0,s.jsxs)(t.p,{children:["A sample ",(0,s.jsx)(t.code,{children:"clusteringjob.properties"})," file:"]}),"\n",(0,s.jsx)(t.pre,{children:(0,s.jsx)(t.code,{children:"hoodie.clustering.async.enabled=true\nhoodie.clustering.async.max.commits=4\nhoodie.clustering.plan.strategy.target.file.max.bytes=1073741824\nhoodie.clustering.plan.strategy.small.file.limit=629145600\nhoodie.clustering.execution.strategy.class=org.apache.hudi.client.clustering.run.strategy.SparkSortAndSizeExecutionStrategy\nhoodie.clustering.plan.strategy.sort.columns=column1,column2\n"})}),"\n",(0,s.jsx)(t.h3,{id:"hoodiestreamer",children:"HoodieStreamer"}),"\n",(0,s.jsxs)(t.p,{children:["This brings us to our users' favorite utility in Hudi. Now, we can trigger asynchronous clustering with Hudi Streamer.\nJust set the ",(0,s.jsx)(t.code,{children:"hoodie.clustering.async.enabled"})," config to true and specify other clustering config in properties file\nwhose location can be pased as ",(0,s.jsx)(t.code,{children:"\u2014props"})," when starting the Hudi Streamer (just like in the case of HoodieClusteringJob)."]}),"\n",(0,s.jsx)(t.p,{children:"A sample spark-submit command to setup HoodieStreamer is as below:"}),"\n",(0,s.jsx)(t.pre,{children:(0,s.jsx)(t.code,{className:"language-bash",children:'spark-submit \\\n--jars "packaging/hudi-utilities-slim-bundle/target/hudi-utilities-slim-bundle_2.12-1.0.0.jar,packaging/hudi-spark-bundle/target/hudi-spark3.5-bundle_2.12-1.0.0.jar" \\\n--class org.apache.hudi.utilities.streamer.HoodieStreamer \\\n/path/to/hudi-utilities-slim-bundle/target/hudi-utilities-slim-bundle_2.12-1.0.0.jar \\\n--props /path/to/config/clustering_kafka.properties \\\n--schemaprovider-class org.apache.hudi.utilities.schema.SchemaRegistryProvider \\\n--source-class org.apache.hudi.utilities.sources.AvroKafkaSource \\\n--source-ordering-field impresssiontime \\\n--table-type COPY_ON_WRITE \\\n--target-base-path /path/to/hudi_table/basePath \\\n--target-table impressions_cow_cluster \\\n--op INSERT \\\n--hoodie-conf hoodie.clustering.async.enabled=true \\\n--continuous\n'})}),"\n",(0,s.jsx)(t.h3,{id:"spark-structured-streaming",children:"Spark Structured Streaming"}),"\n",(0,s.jsx)(t.p,{children:"We can also enable asynchronous clustering with Spark structured streaming sink as shown below."}),"\n",(0,s.jsx)(t.pre,{children:(0,s.jsx)(t.code,{className:"language-scala",children:'val commonOpts = Map(\n "hoodie.insert.shuffle.parallelism" -> "4",\n "hoodie.upsert.shuffle.parallelism" -> "4",\n "hoodie.datasource.write.recordkey.field" -> "_row_key",\n "hoodie.datasource.write.partitionpath.field" -> "partition",\n "hoodie.datasource.write.precombine.field" -> "timestamp",\n "hoodie.table.name" -> "hoodie_test"\n)\n\ndef getAsyncClusteringOpts(isAsyncClustering: String, \n clusteringNumCommit: String, \n executionStrategy: String):Map[String, String] = {\n commonOpts + (DataSourceWriteOptions.ASYNC_CLUSTERING_ENABLE.key -> isAsyncClustering,\n HoodieClusteringConfig.ASYNC_CLUSTERING_MAX_COMMITS.key -> clusteringNumCommit,\n HoodieClusteringConfig.EXECUTION_STRATEGY_CLASS_NAME.key -> executionStrategy\n )\n}\n\ndef initStreamingWriteFuture(hudiOptions: Map[String, String]): Future[Unit] = {\n val streamingInput = // define the source of streaming\n Future {\n println("streaming starting")\n streamingInput\n .writeStream\n .format("org.apache.hudi")\n .options(hudiOptions)\n .option("checkpointLocation", basePath + "/checkpoint")\n .mode(Append)\n .start()\n .awaitTermination(10000)\n println("streaming ends")\n }\n}\n\ndef structuredStreamingWithClustering(): Unit = {\n val df = //generate data frame\n val hudiOptions = getClusteringOpts("true", "1", "org.apache.hudi.client.clustering.run.strategy.SparkSortAndSizeExecutionStrategy")\n val f1 = initStreamingWriteFuture(hudiOptions)\n Await.result(f1, Duration.Inf)\n}\n'})}),"\n",(0,s.jsx)(t.h2,{id:"java-client",children:"Java Client"}),"\n",(0,s.jsxs)(t.p,{children:["Clustering is also supported via Java client. Plan strategy ",(0,s.jsx)(t.code,{children:"org.apache.hudi.client.clustering.plan.strategy.JavaSizeBasedClusteringPlanStrategy"}),"\nand execution strategy ",(0,s.jsx)(t.code,{children:"org.apache.hudi.client.clustering.run.strategy.JavaSortAndSizeExecutionStrategy"})," are supported\nout-of-the-box. Note that as of now only linear sort is supported in Java execution strategy."]}),"\n",(0,s.jsx)(t.h2,{id:"related-resources",children:"Related Resources"}),"\n",(0,s.jsxs)(t.p,{children:[(0,s.jsx)("h3",{children:"Blogs"}),"\n",(0,s.jsx)(t.a,{href:"https://www.onehouse.ai/blog/apachehudi-z-order-and-hilbert-space-filling-curves",children:"Apache Hudi Z-Order and Hilbert Space Filling Curves"}),"\n",(0,s.jsx)(t.a,{href:"https://medium.com/apache-hudi-blogs/hudi-z-order-and-hilbert-space-filling-curves-68fa28bffaf0",children:"Hudi Z-Order and Hilbert Space-filling Curves"})]}),"\n",(0,s.jsx)("h3",{children:"Videos"}),"\n",(0,s.jsxs)(t.ul,{children:["\n",(0,s.jsx)(t.li,{children:(0,s.jsx)(t.a,{href:"https://www.youtube.com/watch?v=R_sm4wlGXuE",children:"Understanding Clustering in Apache Hudi and the Benefits of Asynchronous Clustering"})}),"\n"]})]})}function u(e={}){const{wrapper:t}={...(0,r.R)(),...e.components};return t?(0,s.jsx)(t,{...e,children:(0,s.jsx)(d,{...e})}):d(e)}},7004:(e,t,i)=>{i.d(t,{A:()=>n});const n=i.p+"assets/images/clustering1_new-d67c9e691d235b140f7c80d68400f425.png"},67491:(e,t,i)=>{i.d(t,{A:()=>n});const n=i.p+"assets/images/clustering2_new-0837c07b6db44ab75873633f0eab2e2c.png"},14438:(e,t,i)=>{i.d(t,{A:()=>n});const n=i.p+"assets/images/clustering_3-8bc286ab44c48137f8409b5c342a7207.png"},28453:(e,t,i)=>{i.d(t,{R:()=>a,x:()=>l});var n=i(96540);const s={},r=n.createContext(s);function a(e){const t=n.useContext(r);return n.useMemo((function(){return"function"==typeof e?e(t):{...t,...e}}),[t,e])}function l(e){let t;return t=e.disableParentContext?"function"==typeof e.components?e.components(s):e.components||s:a(e.components),n.createElement(r.Provider,{value:t},e.children)}}}]);
\ No newline at end of file
diff --git a/content/cn/assets/js/10b6d210.3f6cd12f.js b/content/cn/assets/js/10b6d210.3f6cd12f.js
deleted file mode 100644
index 18f63107f1f72..0000000000000
--- a/content/cn/assets/js/10b6d210.3f6cd12f.js
+++ /dev/null
@@ -1 +0,0 @@
-"use strict";(self.webpackChunkhudi=self.webpackChunkhudi||[]).push([[18757],{78367:(e,i,t)=>{t.r(i),t.d(i,{assets:()=>c,contentTitle:()=>r,default:()=>h,frontMatter:()=>s,metadata:()=>n,toc:()=>l});const n=JSON.parse('{"id":"compaction","title":"Compaction","description":"Background","source":"@site/docs/compaction.md","sourceDirName":".","slug":"/compaction","permalink":"/cn/docs/next/compaction","draft":false,"unlisted":false,"editUrl":"https://github.com/apache/hudi/tree/asf-site/website/docs/compaction.md","tags":[],"version":"current","frontMatter":{"title":"Compaction","summary":"In this page, we describe async compaction in Hudi.","toc":true,"toc_min_heading_level":2,"toc_max_heading_level":4,"last_modified_at":null},"sidebar":"docs","previous":{"title":"Cleaning","permalink":"/cn/docs/next/cleaning"},"next":{"title":"Clustering","permalink":"/cn/docs/next/clustering"}}');var o=t(74848),a=t(28453);const s={title:"Compaction",summary:"In this page, we describe async compaction in Hudi.",toc:!0,toc_min_heading_level:2,toc_max_heading_level:4,last_modified_at:null},r=void 0,c={},l=[{value:"Background",id:"background",level:2},{value:"Why MOR tables need compaction?",id:"why-mor-tables-need-compaction",level:3},{value:"Compaction Architecture",id:"compaction-architecture",level:2},{value:"Strategies in Compaction Scheduling",id:"strategies-in-compaction-scheduling",level:3},{value:"Trigger Strategies",id:"trigger-strategies",level:4},{value:"Compaction Strategies",id:"compaction-strategies",level:4},{value:"Ways to trigger Compaction",id:"ways-to-trigger-compaction",level:2},{value:"Inline",id:"inline",level:3},{value:"Async & Offline Compaction models",id:"async--offline-compaction-models",level:3},{value:"Async execution within the same process",id:"async-execution-within-the-same-process",level:4},{value:"Spark Structured Streaming",id:"spark-structured-streaming",level:5},{value:"Hudi Streamer Continuous Mode",id:"hudi-streamer-continuous-mode",level:5},{value:"Scheduling and Execution by a separate process",id:"scheduling-and-execution-by-a-separate-process",level:4},{value:"Scheduling inline and executing async",id:"scheduling-inline-and-executing-async",level:4},{value:"Hudi Compactor Utility",id:"hudi-compactor-utility",level:4},{value:"Hudi CLI",id:"hudi-cli",level:4},{value:"Flink Offline Compaction",id:"flink-offline-compaction",level:4},{value:"Options",id:"options",level:4}];function d(e){const i={a:"a",admonition:"admonition",br:"br",code:"code",em:"em",h2:"h2",h3:"h3",h4:"h4",h5:"h5",img:"img",li:"li",p:"p",pre:"pre",strong:"strong",table:"table",tbody:"tbody",td:"td",th:"th",thead:"thead",tr:"tr",ul:"ul",...(0,a.R)(),...e.components};return(0,o.jsxs)(o.Fragment,{children:[(0,o.jsx)(i.h2,{id:"background",children:"Background"}),"\n",(0,o.jsx)(i.p,{children:"Compaction is a table service employed by Hudi specifically in Merge On Read(MOR) tables to merge updates from row-based log\nfiles to the corresponding columnar-based base file periodically to produce a new version of the base file. Compaction is\nnot applicable to Copy On Write(COW) tables and only applies to MOR tables."}),"\n",(0,o.jsx)(i.h3,{id:"why-mor-tables-need-compaction",children:"Why MOR tables need compaction?"}),"\n",(0,o.jsxs)(i.p,{children:["To understand the significance of compaction in MOR tables, it is helpful to understand the MOR table layout first. In Hudi,\ndata is organized in terms of ",(0,o.jsx)(i.a,{href:"https://hudi.apache.org/docs/file_layouts/",children:"file groups"}),". Each file group in a MOR table\nconsists of a base file and one or more log files. Typically, during writes, inserts are stored in the base file, and updates\nare appended to log files."]}),"\n",(0,o.jsxs)(i.p,{children:[(0,o.jsx)(i.img,{alt:"mor_table_file_layout",src:t(67587).A+"",width:"6528",height:"2450"}),"\n",(0,o.jsx)(i.em,{children:"Figure: MOR table file layout showing different file groups with base data file and log files"})]}),"\n",(0,o.jsx)(i.p,{children:"During the compaction process, updates from the log files are merged with the base file to form a new version of the\nbase file as shown below. Since MOR is designed to be write-optimized, on new writes, after index tagging is complete,\nHudi appends the records pertaining to each file groups as log blocks in log files. There is no synchronous merge\nhappening during write, resulting in a lower write amplification and better write latency. In contrast, on new writes to a\nCOW table, Hudi combines the new writes with the older base file to produce a new version of the base file resulting in\na higher write amplification and higher write latencies."}),"\n",(0,o.jsxs)(i.p,{children:[(0,o.jsx)(i.img,{alt:"mor_table_file_layout",src:t(56040).A+"",width:"5081",height:"3148"}),"\n",(0,o.jsx)(i.em,{children:"Figure: Compaction on a given file group"})]}),"\n",(0,o.jsx)(i.p,{children:"While serving the read query(snapshot read), for each file group, records in base file and all its corresponding log\nfiles are merged together and served. And hence the read latency for MOR snapshot query might be higher compared to\nCOW table since there is no merge involved in case of COW at read time. Compaction takes care of merging the updates from\nlog files with the base file at regular intervals to bound the growth of log files and to ensure the read latencies do not\nspike up."}),"\n",(0,o.jsx)(i.h2,{id:"compaction-architecture",children:"Compaction Architecture"}),"\n",(0,o.jsx)(i.p,{children:"There are two steps to compaction."}),"\n",(0,o.jsxs)(i.ul,{children:["\n",(0,o.jsxs)(i.li,{children:[(0,o.jsx)(i.em,{children:(0,o.jsx)(i.strong,{children:"Compaction Scheduling"})}),": In this step, Hudi scans the partitions and selects file slices to be compacted. A compaction\nplan is finally written to Hudi timeline."]}),"\n",(0,o.jsxs)(i.li,{children:[(0,o.jsx)(i.em,{children:(0,o.jsx)(i.strong,{children:"Compaction Execution"})}),": In this step the compaction plan is read and file slices are compacted."]}),"\n"]}),"\n",(0,o.jsx)(i.h3,{id:"strategies-in-compaction-scheduling",children:"Strategies in Compaction Scheduling"}),"\n",(0,o.jsx)(i.p,{children:"There are two strategies involved in scheduling the compaction:"}),"\n",(0,o.jsxs)(i.ul,{children:["\n",(0,o.jsx)(i.li,{children:"Trigger Strategy: Determines how often to trigger scheduling of the compaction."}),"\n",(0,o.jsx)(i.li,{children:"Compaction Strategy: Determines which file groups to compact."}),"\n"]}),"\n",(0,o.jsx)(i.p,{children:"Hudi provides various options for both these strategies as discussed below."}),"\n",(0,o.jsx)(i.h4,{id:"trigger-strategies",children:"Trigger Strategies"}),"\n",(0,o.jsxs)(i.table,{children:[(0,o.jsx)(i.thead,{children:(0,o.jsxs)(i.tr,{children:[(0,o.jsx)(i.th,{children:"Config Name"}),(0,o.jsx)(i.th,{children:"Default"}),(0,o.jsx)(i.th,{children:"Description"})]})}),(0,o.jsxs)(i.tbody,{children:[(0,o.jsxs)(i.tr,{children:[(0,o.jsx)(i.td,{children:"hoodie.compact.inline.trigger.strategy"}),(0,o.jsx)(i.td,{children:"NUM_COMMITS (Optional)"}),(0,o.jsxs)(i.td,{children:["org.apache.hudi.table.action.compact.CompactionTriggerStrategy: Controls when compaction is scheduled.",(0,o.jsx)("br",{}),(0,o.jsx)(i.code,{children:"Config Param: INLINE_COMPACT_TRIGGER_STRATEGY"})," ",(0,o.jsx)("br",{})]})]}),(0,o.jsxs)(i.tr,{children:[(0,o.jsx)(i.td,{children:(0,o.jsxs)("ul",{children:[(0,o.jsxs)("li",{children:[(0,o.jsx)(i.code,{children:"NUM_COMMITS"}),": triggers compaction when there are at least N delta commits after last completed compaction."]}),(0,o.jsxs)("li",{children:[(0,o.jsx)(i.code,{children:"NUM_COMMITS_AFTER_LAST_REQUEST"}),": triggers compaction when there are at least N delta commits after last completed or requested compaction."]}),(0,o.jsxs)("li",{children:[(0,o.jsx)(i.code,{children:"TIME_ELAPSED"}),": triggers compaction after N seconds since last compaction."]}),(0,o.jsxs)("li",{children:[(0,o.jsx)(i.code,{children:"NUM_AND_TIME"}),": triggers compaction when both there are at least N delta commits and N seconds elapsed (both must be satisfied) after last completed compaction."]}),(0,o.jsxs)("li",{children:[(0,o.jsx)(i.code,{children:"NUM_OR_TIME"}),": triggers compaction when both there are at least N delta commits or N seconds elapsed (either condition is satisfied) after last completed compaction."]})]})}),(0,o.jsx)(i.td,{}),(0,o.jsx)(i.td,{})]})]})]}),"\n",(0,o.jsx)(i.h4,{id:"compaction-strategies",children:"Compaction Strategies"}),"\n",(0,o.jsxs)(i.table,{children:[(0,o.jsx)(i.thead,{children:(0,o.jsxs)(i.tr,{children:[(0,o.jsx)(i.th,{children:"Config Name"}),(0,o.jsx)(i.th,{children:"Default"}),(0,o.jsx)(i.th,{children:"Description"})]})}),(0,o.jsx)(i.tbody,{children:(0,o.jsxs)(i.tr,{children:[(0,o.jsx)(i.td,{children:"hoodie.compaction.strategy"}),(0,o.jsx)(i.td,{children:"org.apache.hudi.table.action.compact.strategy.LogFileSizeBasedCompactionStrategy (Optional)"}),(0,o.jsxs)(i.td,{children:["Compaction strategy decides which file groups are picked up for compaction during each compaction run. By default. Hudi picks the log file with most accumulated unmerged data. ",(0,o.jsx)("br",{}),(0,o.jsx)("br",{}),(0,o.jsx)(i.code,{children:"Config Param: COMPACTION_STRATEGY"})]})]})})]}),"\n",(0,o.jsxs)(i.p,{children:["Available Strategies (Provide the full package name when using the strategy): ",(0,o.jsxs)("ul",{children:[(0,o.jsxs)("li",{children:[(0,o.jsx)(i.code,{children:"LogFileNumBasedCompactionStrategy"}),":\norders the compactions based on the total log files count, filters the file group with log files count greater than the\nthreshold and limits the compactions within a configured IO bound."]}),(0,o.jsxs)("li",{children:[(0,o.jsx)(i.code,{children:"LogFileSizeBasedCompactionStrategy"}),": orders\nthe compactions based on the total log files size, filters the file group which log files size is greater than the\nthreshold and limits the compactions within a configured IO bound."]}),(0,o.jsxs)("li",{children:[(0,o.jsx)(i.code,{children:"BoundedIOCompactionStrategy"}),": CompactionStrategy\nwhich looks at total IO to be done for the compaction (read + write) and limits the list of compactions to be under a\nconfigured limit on the IO."]}),(0,o.jsxs)("li",{children:[(0,o.jsx)(i.code,{children:"BoundedPartitionAwareCompactionStrategy"}),":This"," strategy ensures that the last N partitions\nare picked up even if there are later partitions created for the table. lastNPartitions is defined as the N partitions before\nthe currentDate. currentDay = 2018/01/01 The table has partitions for 2018/02/02 and 2018/03/03 beyond the currentDay This\nstrategy will pick up the following partitions for compaction : (2018/01/01, allPartitionsInRange[(2018/01/01 - lastNPartitions)\nto 2018/01/01), 2018/02/02, 2018/03/03)"]}),(0,o.jsxs)("li",{children:[(0,o.jsx)(i.code,{children:"DayBasedCompactionStrategy"}),":This"," strategy orders compactions in reverse\norder of creation of Hive Partitions. It helps to compact data in latest partitions first and then older capped at the\nTotal_IO allowed."]}),(0,o.jsxs)("li",{children:[(0,o.jsx)(i.code,{children:"UnBoundedCompactionStrategy"}),": UnBoundedCompactionStrategy will not change ordering or filter\nany compaction. It is a pass-through and will compact all the base files which has a log file. This usually means\nno-intelligence on compaction."]}),(0,o.jsxs)("li",{children:[(0,o.jsx)(i.code,{children:"UnBoundedPartitionAwareCompactionStrategy"}),":UnBoundedPartitionAwareCompactionStrategy"," is a custom UnBounded Strategy. This will filter all the partitions that\nare eligible to be compacted by a {@link BoundedPartitionAwareCompactionStrategy} and return the result. This is done\nso that a long running UnBoundedPartitionAwareCompactionStrategy does not step over partitions in a shorter running\nBoundedPartitionAwareCompactionStrategy. Essentially, this is an inverse of the partitions chosen in\nBoundedPartitionAwareCompactionStrategy"]})]})]}),"\n",(0,o.jsx)(i.admonition,{type:"note",children:(0,o.jsxs)(i.p,{children:["Please refer to ",(0,o.jsx)(i.a,{href:"https://hudi.apache.org/docs/next/configurations#Compaction-Configs",children:"advanced configs"})," for more details."]})}),"\n",(0,o.jsx)(i.h2,{id:"ways-to-trigger-compaction",children:"Ways to trigger Compaction"}),"\n",(0,o.jsx)(i.h3,{id:"inline",children:"Inline"}),"\n",(0,o.jsx)(i.p,{children:"By default, compaction is run asynchronously."}),"\n",(0,o.jsx)(i.p,{children:"If latency of ingesting records is important for you, you are most likely using Merge-On-Read tables.\nMerge-On-Read tables store data using a combination of columnar (e.g parquet) + row based (e.g avro) file formats.\nUpdates are logged to delta files & later compacted to produce new versions of columnar files.\nTo improve ingestion latency, Async Compaction is the default configuration."}),"\n",(0,o.jsx)(i.p,{children:"If immediate read performance of a new commit is important for you, or you want simplicity of not managing separate compaction jobs,\nyou may want synchronous inline compaction, which means that as a commit is written it is also compacted by the same job."}),"\n",(0,o.jsxs)(i.p,{children:["For this deployment mode, please use ",(0,o.jsx)(i.code,{children:"hoodie.compact.inline = true"})," for Spark Datasource and Spark SQL writers. For\nHoodieStreamer sync once mode inline compaction can be achieved by passing the flag ",(0,o.jsx)(i.code,{children:"--disable-compaction"})," (Meaning to\ndisable async compaction). Further in HoodieStreamer when both\ningestion and compaction is running in the same spark context, you can use resource allocation configuration\nin Hudi Streamer CLI such as (",(0,o.jsx)(i.code,{children:"--delta-sync-scheduling-weight"}),",\n",(0,o.jsx)(i.code,{children:"--compact-scheduling-weight"}),", ",(0,o.jsx)(i.code,{children:"--delta-sync-scheduling-minshare"}),", and ",(0,o.jsx)(i.code,{children:"--compact-scheduling-minshare"}),")\nto control executor allocation between ingestion and compaction."]}),"\n",(0,o.jsx)(i.h3,{id:"async--offline-compaction-models",children:"Async & Offline Compaction models"}),"\n",(0,o.jsx)(i.p,{children:"There are a couple of ways here to trigger compaction ."}),"\n",(0,o.jsx)(i.h4,{id:"async-execution-within-the-same-process",children:"Async execution within the same process"}),"\n",(0,o.jsx)(i.p,{children:"In streaming ingestion write models like HoodieStreamer\ncontinuous mode, Flink and Spark Streaming, async compaction is enabled by default and runs alongside without blocking\nregular ingestion."}),"\n",(0,o.jsx)(i.h5,{id:"spark-structured-streaming",children:"Spark Structured Streaming"}),"\n",(0,o.jsx)(i.p,{children:"Compactions are scheduled and executed asynchronously inside the\nstreaming job.Here is an example snippet in java"}),"\n",(0,o.jsx)(i.pre,{children:(0,o.jsx)(i.code,{className:"language-properties",children:'import org.apache.hudi.DataSourceWriteOptions;\nimport org.apache.hudi.HoodieDataSourceHelpers;\nimport org.apache.hudi.config.HoodieCompactionConfig;\nimport org.apache.hudi.config.HoodieWriteConfig;\n\nimport org.apache.spark.sql.streaming.OutputMode;\nimport org.apache.spark.sql.streaming.ProcessingTime;\n\n\n DataStreamWriter writer = streamingInput.writeStream().format("org.apache.hudi")\n .option("hoodie.datasource.write.operation", operationType)\n .option("hoodie.datasource.write.table.type", tableType)\n .option("hoodie.datasource.write.recordkey.field", "_row_key")\n .option("hoodie.datasource.write.partitionpath.field", "partition")\n .option("hoodie.datasource.write.precombine.field"(), "timestamp")\n .option("hoodie.compact.inline.max.delta.commits", "10")\n .option("hoodie.datasource.compaction.async.enable", "true")\n .option("hoodie.table.name", tableName).option("checkpointLocation", checkpointLocation)\n .outputMode(OutputMode.Append());\n writer.trigger(new ProcessingTime(30000)).start(tablePath);\n'})}),"\n",(0,o.jsx)(i.h5,{id:"hudi-streamer-continuous-mode",children:"Hudi Streamer Continuous Mode"}),"\n",(0,o.jsxs)(i.p,{children:["Hudi Streamer provides continuous ingestion mode where a single long running spark application",(0,o.jsx)(i.br,{}),"\n","ingests data to Hudi table continuously from upstream sources. In this mode, Hudi supports managing asynchronous\ncompactions. Here is an example snippet for running in continuous mode with async compactions"]}),"\n",(0,o.jsx)(i.pre,{children:(0,o.jsx)(i.code,{className:"language-properties",children:"spark-submit --packages org.apache.hudi:hudi-utilities-slim-bundle_2.12:1.0.0,org.apache.hudi:hudi-spark3.5-bundle_2.12:1.0.0 \\\n--class org.apache.hudi.utilities.streamer.HoodieStreamer \\\n--table-type MERGE_ON_READ \\\n--target-base-path \\\n--target-table \\\n--source-class org.apache.hudi.utilities.sources.JsonDFSSource \\\n--source-ordering-field ts \\\n--props /path/to/source.properties \\\n--continous\n"})}),"\n",(0,o.jsx)(i.h4,{id:"scheduling-and-execution-by-a-separate-process",children:"Scheduling and Execution by a separate process"}),"\n",(0,o.jsxs)(i.p,{children:["For some use cases with long running table services, instead of having the regular writes block, users have the option to run\nboth steps of the compaction (",(0,o.jsx)(i.a,{href:"#compaction-architecture",children:"scheduling and execution"}),") offline in a separate process altogether.\nThis allows for regular writers to not bother about these compaction steps and allows users to provide more resources for\nthe compaction job as needed."]}),"\n",(0,o.jsx)(i.admonition,{type:"note",children:(0,o.jsx)(i.p,{children:"This model needs a lock provider configured for all jobs - the regular writer as well as the offline compaction job."})}),"\n",(0,o.jsx)(i.h4,{id:"scheduling-inline-and-executing-async",children:"Scheduling inline and executing async"}),"\n",(0,o.jsx)(i.p,{children:"In this model, it is possible for a Spark Datasource writer or a Flink job to just schedule the compaction inline ( that\nwill serialize the compaction plan in the timeline but will not execute it). And then a separate utility like\nHudiCompactor or HoodieFlinkCompactor can take care of periodically executing the compaction plan."}),"\n",(0,o.jsx)(i.admonition,{type:"note",children:(0,o.jsxs)(i.p,{children:["This model may need a lock provider ",(0,o.jsx)(i.strong,{children:"if"})," metadata table is enabled."]})}),"\n",(0,o.jsx)(i.h4,{id:"hudi-compactor-utility",children:"Hudi Compactor Utility"}),"\n",(0,o.jsxs)(i.p,{children:["Hudi provides a standalone tool to execute specific compactions asynchronously. Below is an example and you can read more in the ",(0,o.jsx)(i.a,{href:"/docs/cli#compactions",children:"deployment guide"}),"\nThe compactor utility allows to do scheduling and execution of compaction."]}),"\n",(0,o.jsx)(i.p,{children:"Example:"}),"\n",(0,o.jsx)(i.pre,{children:(0,o.jsx)(i.code,{className:"language-properties",children:"spark-submit --packages org.apache.hudi:hudi-utilities-slim-bundle_2.12:1.0.0,org.apache.hudi:hudi-spark3.5-bundle_2.12:1.0.0 \\\n--class org.apache.hudi.utilities.HoodieCompactor \\\n--base-path \\\n--table-name \\\n--schema-file \\\n--instant-time \n"})}),"\n",(0,o.jsxs)(i.p,{children:["Note, the ",(0,o.jsx)(i.code,{children:"instant-time"})," parameter is now optional for the Hudi Compactor Utility. If using the utility without ",(0,o.jsx)(i.code,{children:"--instant time"}),",\nthe spark-submit will execute the earliest scheduled compaction on the Hudi timeline."]}),"\n",(0,o.jsx)(i.h4,{id:"hudi-cli",children:"Hudi CLI"}),"\n",(0,o.jsxs)(i.p,{children:["Hudi CLI is yet another way to execute specific compactions asynchronously. Here is an example and you can read more in the ",(0,o.jsx)(i.a,{href:"/docs/cli#compactions",children:"deployment guide"})]}),"\n",(0,o.jsx)(i.p,{children:"Example:"}),"\n",(0,o.jsx)(i.pre,{children:(0,o.jsx)(i.code,{className:"language-properties",children:"hudi:trips->compaction run --tableName --parallelism --compactionInstant \n...\n"})}),"\n",(0,o.jsx)(i.h4,{id:"flink-offline-compaction",children:"Flink Offline Compaction"}),"\n",(0,o.jsxs)(i.p,{children:["Offline compaction needs to submit the Flink task on the command line. The program entry is as follows: ",(0,o.jsx)(i.code,{children:"hudi-flink-bundle_2.11-0.9.0-SNAPSHOT.jar"})," :\n",(0,o.jsx)(i.code,{children:"org.apache.hudi.sink.compact.HoodieFlinkCompactor"})]}),"\n",(0,o.jsx)(i.pre,{children:(0,o.jsx)(i.code,{className:"language-bash",children:"# Command line\n./bin/flink run -c org.apache.hudi.sink.compact.HoodieFlinkCompactor lib/hudi-flink-bundle_2.11-0.9.0.jar --path hdfs://xxx:9000/table\n"})}),"\n",(0,o.jsx)(i.h4,{id:"options",children:"Options"}),"\n",(0,o.jsxs)(i.table,{children:[(0,o.jsx)(i.thead,{children:(0,o.jsxs)(i.tr,{children:[(0,o.jsx)(i.th,{children:"Option Name"}),(0,o.jsx)(i.th,{children:"Default"}),(0,o.jsx)(i.th,{children:"Description"})]})}),(0,o.jsxs)(i.tbody,{children:[(0,o.jsxs)(i.tr,{children:[(0,o.jsx)(i.td,{children:(0,o.jsx)(i.code,{children:"--path"})}),(0,o.jsx)(i.td,{children:(0,o.jsx)(i.code,{children:"n/a **(Required)**"})}),(0,o.jsx)(i.td,{children:"The path where the target table is stored on Hudi"})]}),(0,o.jsxs)(i.tr,{children:[(0,o.jsx)(i.td,{children:(0,o.jsx)(i.code,{children:"--compaction-max-memory"})}),(0,o.jsxs)(i.td,{children:[(0,o.jsx)(i.code,{children:"100"})," (Optional)"]}),(0,o.jsx)(i.td,{children:"The index map size of log data during compaction, 100 MB by default. If you have enough memory, you can turn up this parameter"})]}),(0,o.jsxs)(i.tr,{children:[(0,o.jsx)(i.td,{children:(0,o.jsx)(i.code,{children:"--schedule"})}),(0,o.jsxs)(i.td,{children:[(0,o.jsx)(i.code,{children:"false"})," (Optional)"]}),(0,o.jsx)(i.td,{children:"whether to execute the operation of scheduling compaction plan. When the write process is still writing\uff0c turning on this parameter have a risk of losing data. Therefore, it must be ensured that there are no write tasks currently writing data to this table when this parameter is turned on"})]}),(0,o.jsxs)(i.tr,{children:[(0,o.jsx)(i.td,{children:(0,o.jsx)(i.code,{children:"--seq"})}),(0,o.jsxs)(i.td,{children:[(0,o.jsx)(i.code,{children:"LIFO"})," (Optional)"]}),(0,o.jsxs)(i.td,{children:["The order in which compaction tasks are executed. Executing from the latest compaction plan by default. ",(0,o.jsx)(i.code,{children:"LIFO"}),": executing from the latest plan. ",(0,o.jsx)(i.code,{children:"FIFO"}),": executing from the oldest plan."]})]}),(0,o.jsxs)(i.tr,{children:[(0,o.jsx)(i.td,{children:(0,o.jsx)(i.code,{children:"--service"})}),(0,o.jsxs)(i.td,{children:[(0,o.jsx)(i.code,{children:"false"})," (Optional)"]}),(0,o.jsx)(i.td,{children:"Whether to start a monitoring service that checks and schedules new compaction task in configured interval."})]}),(0,o.jsxs)(i.tr,{children:[(0,o.jsx)(i.td,{children:(0,o.jsx)(i.code,{children:"--min-compaction-interval-seconds"})}),(0,o.jsxs)(i.td,{children:[(0,o.jsx)(i.code,{children:"600(s)"})," (optional)"]}),(0,o.jsx)(i.td,{children:"The checking interval for service mode, by default 10 minutes."})]})]})]})]})}function h(e={}){const{wrapper:i}={...(0,a.R)(),...e.components};return i?(0,o.jsx)(i,{...e,children:(0,o.jsx)(d,{...e})}):d(e)}},67587:(e,i,t)=>{t.d(i,{A:()=>n});const n=t.p+"assets/images/hudi_mor_file_layout-643f9f7fda5aa0d532682af27fe3e42c.jpg"},56040:(e,i,t)=>{t.d(i,{A:()=>n});const n=t.p+"assets/images/hudi_mor_file_layout_post_compaction-9f10af785d4927dc3d66303dac5bc7ba.jpg"},28453:(e,i,t)=>{t.d(i,{R:()=>s,x:()=>r});var n=t(96540);const o={},a=n.createContext(o);function s(e){const i=n.useContext(a);return n.useMemo((function(){return"function"==typeof e?e(i):{...i,...e}}),[i,e])}function r(e){let i;return i=e.disableParentContext?"function"==typeof e.components?e.components(o):e.components||o:s(e.components),n.createElement(a.Provider,{value:i},e.children)}}}]);
\ No newline at end of file
diff --git a/content/cn/assets/js/10b6d210.c61f07fc.js b/content/cn/assets/js/10b6d210.c61f07fc.js
new file mode 100644
index 0000000000000..214be08b076ef
--- /dev/null
+++ b/content/cn/assets/js/10b6d210.c61f07fc.js
@@ -0,0 +1 @@
+"use strict";(self.webpackChunkhudi=self.webpackChunkhudi||[]).push([[18757],{78367:(e,i,t)=>{t.r(i),t.d(i,{assets:()=>c,contentTitle:()=>r,default:()=>h,frontMatter:()=>s,metadata:()=>n,toc:()=>l});const n=JSON.parse('{"id":"compaction","title":"Compaction","description":"Background","source":"@site/docs/compaction.md","sourceDirName":".","slug":"/compaction","permalink":"/cn/docs/next/compaction","draft":false,"unlisted":false,"editUrl":"https://github.com/apache/hudi/tree/asf-site/website/docs/compaction.md","tags":[],"version":"current","frontMatter":{"title":"Compaction","summary":"In this page, we describe async compaction in Hudi.","toc":true,"toc_min_heading_level":2,"toc_max_heading_level":4,"last_modified_at":null},"sidebar":"docs","previous":{"title":"Cleaning","permalink":"/cn/docs/next/cleaning"},"next":{"title":"Clustering","permalink":"/cn/docs/next/clustering"}}');var o=t(74848),a=t(28453);const s={title:"Compaction",summary:"In this page, we describe async compaction in Hudi.",toc:!0,toc_min_heading_level:2,toc_max_heading_level:4,last_modified_at:null},r=void 0,c={},l=[{value:"Background",id:"background",level:2},{value:"Why MOR tables need compaction?",id:"why-mor-tables-need-compaction",level:3},{value:"Compaction Architecture",id:"compaction-architecture",level:2},{value:"Strategies in Compaction Scheduling",id:"strategies-in-compaction-scheduling",level:3},{value:"Trigger Strategies",id:"trigger-strategies",level:4},{value:"Compaction Strategies",id:"compaction-strategies",level:4},{value:"Ways to trigger Compaction",id:"ways-to-trigger-compaction",level:2},{value:"Inline",id:"inline",level:3},{value:"Async & Offline Compaction models",id:"async--offline-compaction-models",level:3},{value:"Async execution within the same process",id:"async-execution-within-the-same-process",level:4},{value:"Spark Structured Streaming",id:"spark-structured-streaming",level:5},{value:"Hudi Streamer Continuous Mode",id:"hudi-streamer-continuous-mode",level:5},{value:"Scheduling and Execution by a separate process",id:"scheduling-and-execution-by-a-separate-process",level:4},{value:"Scheduling inline and executing async",id:"scheduling-inline-and-executing-async",level:4},{value:"Hudi Compactor Utility",id:"hudi-compactor-utility",level:4},{value:"Hudi CLI",id:"hudi-cli",level:4},{value:"Flink Offline Compaction",id:"flink-offline-compaction",level:4},{value:"Options",id:"options",level:4},{value:"Related Resources",id:"related-resources",level:2}];function d(e){const i={a:"a",admonition:"admonition",br:"br",code:"code",em:"em",h2:"h2",h3:"h3",h4:"h4",h5:"h5",img:"img",li:"li",p:"p",pre:"pre",strong:"strong",table:"table",tbody:"tbody",td:"td",th:"th",thead:"thead",tr:"tr",ul:"ul",...(0,a.R)(),...e.components};return(0,o.jsxs)(o.Fragment,{children:[(0,o.jsx)(i.h2,{id:"background",children:"Background"}),"\n",(0,o.jsx)(i.p,{children:"Compaction is a table service employed by Hudi specifically in Merge On Read(MOR) tables to merge updates from row-based log\nfiles to the corresponding columnar-based base file periodically to produce a new version of the base file. Compaction is\nnot applicable to Copy On Write(COW) tables and only applies to MOR tables."}),"\n",(0,o.jsx)(i.h3,{id:"why-mor-tables-need-compaction",children:"Why MOR tables need compaction?"}),"\n",(0,o.jsxs)(i.p,{children:["To understand the significance of compaction in MOR tables, it is helpful to understand the MOR table layout first. In Hudi,\ndata is organized in terms of ",(0,o.jsx)(i.a,{href:"https://hudi.apache.org/docs/file_layouts/",children:"file groups"}),". Each file group in a MOR table\nconsists of a base file and one or more log files. Typically, during writes, inserts are stored in the base file, and updates\nare appended to log files."]}),"\n",(0,o.jsxs)(i.p,{children:[(0,o.jsx)(i.img,{alt:"mor_table_file_layout",src:t(67587).A+"",width:"6528",height:"2450"}),"\n",(0,o.jsx)(i.em,{children:"Figure: MOR table file layout showing different file groups with base data file and log files"})]}),"\n",(0,o.jsx)(i.p,{children:"During the compaction process, updates from the log files are merged with the base file to form a new version of the\nbase file as shown below. Since MOR is designed to be write-optimized, on new writes, after index tagging is complete,\nHudi appends the records pertaining to each file groups as log blocks in log files. There is no synchronous merge\nhappening during write, resulting in a lower write amplification and better write latency. In contrast, on new writes to a\nCOW table, Hudi combines the new writes with the older base file to produce a new version of the base file resulting in\na higher write amplification and higher write latencies."}),"\n",(0,o.jsxs)(i.p,{children:[(0,o.jsx)(i.img,{alt:"mor_table_file_layout",src:t(56040).A+"",width:"5081",height:"3148"}),"\n",(0,o.jsx)(i.em,{children:"Figure: Compaction on a given file group"})]}),"\n",(0,o.jsx)(i.p,{children:"While serving the read query(snapshot read), for each file group, records in base file and all its corresponding log\nfiles are merged together and served. And hence the read latency for MOR snapshot query might be higher compared to\nCOW table since there is no merge involved in case of COW at read time. Compaction takes care of merging the updates from\nlog files with the base file at regular intervals to bound the growth of log files and to ensure the read latencies do not\nspike up."}),"\n",(0,o.jsx)(i.h2,{id:"compaction-architecture",children:"Compaction Architecture"}),"\n",(0,o.jsx)(i.p,{children:"There are two steps to compaction."}),"\n",(0,o.jsxs)(i.ul,{children:["\n",(0,o.jsxs)(i.li,{children:[(0,o.jsx)(i.em,{children:(0,o.jsx)(i.strong,{children:"Compaction Scheduling"})}),": In this step, Hudi scans the partitions and selects file slices to be compacted. A compaction\nplan is finally written to Hudi timeline."]}),"\n",(0,o.jsxs)(i.li,{children:[(0,o.jsx)(i.em,{children:(0,o.jsx)(i.strong,{children:"Compaction Execution"})}),": In this step the compaction plan is read and file slices are compacted."]}),"\n"]}),"\n",(0,o.jsx)(i.h3,{id:"strategies-in-compaction-scheduling",children:"Strategies in Compaction Scheduling"}),"\n",(0,o.jsx)(i.p,{children:"There are two strategies involved in scheduling the compaction:"}),"\n",(0,o.jsxs)(i.ul,{children:["\n",(0,o.jsx)(i.li,{children:"Trigger Strategy: Determines how often to trigger scheduling of the compaction."}),"\n",(0,o.jsx)(i.li,{children:"Compaction Strategy: Determines which file groups to compact."}),"\n"]}),"\n",(0,o.jsx)(i.p,{children:"Hudi provides various options for both these strategies as discussed below."}),"\n",(0,o.jsx)(i.h4,{id:"trigger-strategies",children:"Trigger Strategies"}),"\n",(0,o.jsxs)(i.table,{children:[(0,o.jsx)(i.thead,{children:(0,o.jsxs)(i.tr,{children:[(0,o.jsx)(i.th,{children:"Config Name"}),(0,o.jsx)(i.th,{children:"Default"}),(0,o.jsx)(i.th,{children:"Description"})]})}),(0,o.jsxs)(i.tbody,{children:[(0,o.jsxs)(i.tr,{children:[(0,o.jsx)(i.td,{children:"hoodie.compact.inline.trigger.strategy"}),(0,o.jsx)(i.td,{children:"NUM_COMMITS (Optional)"}),(0,o.jsxs)(i.td,{children:["org.apache.hudi.table.action.compact.CompactionTriggerStrategy: Controls when compaction is scheduled.",(0,o.jsx)("br",{}),(0,o.jsx)(i.code,{children:"Config Param: INLINE_COMPACT_TRIGGER_STRATEGY"})," ",(0,o.jsx)("br",{})]})]}),(0,o.jsxs)(i.tr,{children:[(0,o.jsx)(i.td,{children:(0,o.jsxs)("ul",{children:[(0,o.jsxs)("li",{children:[(0,o.jsx)(i.code,{children:"NUM_COMMITS"}),": triggers compaction when there are at least N delta commits after last completed compaction."]}),(0,o.jsxs)("li",{children:[(0,o.jsx)(i.code,{children:"NUM_COMMITS_AFTER_LAST_REQUEST"}),": triggers compaction when there are at least N delta commits after last completed or requested compaction."]}),(0,o.jsxs)("li",{children:[(0,o.jsx)(i.code,{children:"TIME_ELAPSED"}),": triggers compaction after N seconds since last compaction."]}),(0,o.jsxs)("li",{children:[(0,o.jsx)(i.code,{children:"NUM_AND_TIME"}),": triggers compaction when both there are at least N delta commits and N seconds elapsed (both must be satisfied) after last completed compaction."]}),(0,o.jsxs)("li",{children:[(0,o.jsx)(i.code,{children:"NUM_OR_TIME"}),": triggers compaction when both there are at least N delta commits or N seconds elapsed (either condition is satisfied) after last completed compaction."]})]})}),(0,o.jsx)(i.td,{}),(0,o.jsx)(i.td,{})]})]})]}),"\n",(0,o.jsx)(i.h4,{id:"compaction-strategies",children:"Compaction Strategies"}),"\n",(0,o.jsxs)(i.table,{children:[(0,o.jsx)(i.thead,{children:(0,o.jsxs)(i.tr,{children:[(0,o.jsx)(i.th,{children:"Config Name"}),(0,o.jsx)(i.th,{children:"Default"}),(0,o.jsx)(i.th,{children:"Description"})]})}),(0,o.jsx)(i.tbody,{children:(0,o.jsxs)(i.tr,{children:[(0,o.jsx)(i.td,{children:"hoodie.compaction.strategy"}),(0,o.jsx)(i.td,{children:"org.apache.hudi.table.action.compact.strategy.LogFileSizeBasedCompactionStrategy (Optional)"}),(0,o.jsxs)(i.td,{children:["Compaction strategy decides which file groups are picked up for compaction during each compaction run. By default. Hudi picks the log file with most accumulated unmerged data. ",(0,o.jsx)("br",{}),(0,o.jsx)("br",{}),(0,o.jsx)(i.code,{children:"Config Param: COMPACTION_STRATEGY"})]})]})})]}),"\n",(0,o.jsxs)(i.p,{children:["Available Strategies (Provide the full package name when using the strategy): ",(0,o.jsxs)("ul",{children:[(0,o.jsxs)("li",{children:[(0,o.jsx)(i.code,{children:"LogFileNumBasedCompactionStrategy"}),":\norders the compactions based on the total log files count, filters the file group with log files count greater than the\nthreshold and limits the compactions within a configured IO bound."]}),(0,o.jsxs)("li",{children:[(0,o.jsx)(i.code,{children:"LogFileSizeBasedCompactionStrategy"}),": orders\nthe compactions based on the total log files size, filters the file group which log files size is greater than the\nthreshold and limits the compactions within a configured IO bound."]}),(0,o.jsxs)("li",{children:[(0,o.jsx)(i.code,{children:"BoundedIOCompactionStrategy"}),": CompactionStrategy\nwhich looks at total IO to be done for the compaction (read + write) and limits the list of compactions to be under a\nconfigured limit on the IO."]}),(0,o.jsxs)("li",{children:[(0,o.jsx)(i.code,{children:"BoundedPartitionAwareCompactionStrategy"}),":This"," strategy ensures that the last N partitions\nare picked up even if there are later partitions created for the table. lastNPartitions is defined as the N partitions before\nthe currentDate. currentDay = 2018/01/01 The table has partitions for 2018/02/02 and 2018/03/03 beyond the currentDay This\nstrategy will pick up the following partitions for compaction : (2018/01/01, allPartitionsInRange[(2018/01/01 - lastNPartitions)\nto 2018/01/01), 2018/02/02, 2018/03/03)"]}),(0,o.jsxs)("li",{children:[(0,o.jsx)(i.code,{children:"DayBasedCompactionStrategy"}),":This"," strategy orders compactions in reverse\norder of creation of Hive Partitions. It helps to compact data in latest partitions first and then older capped at the\nTotal_IO allowed."]}),(0,o.jsxs)("li",{children:[(0,o.jsx)(i.code,{children:"UnBoundedCompactionStrategy"}),": UnBoundedCompactionStrategy will not change ordering or filter\nany compaction. It is a pass-through and will compact all the base files which has a log file. This usually means\nno-intelligence on compaction."]}),(0,o.jsxs)("li",{children:[(0,o.jsx)(i.code,{children:"UnBoundedPartitionAwareCompactionStrategy"}),":UnBoundedPartitionAwareCompactionStrategy"," is a custom UnBounded Strategy. This will filter all the partitions that\nare eligible to be compacted by a {@link BoundedPartitionAwareCompactionStrategy} and return the result. This is done\nso that a long running UnBoundedPartitionAwareCompactionStrategy does not step over partitions in a shorter running\nBoundedPartitionAwareCompactionStrategy. Essentially, this is an inverse of the partitions chosen in\nBoundedPartitionAwareCompactionStrategy"]})]})]}),"\n",(0,o.jsx)(i.admonition,{type:"note",children:(0,o.jsxs)(i.p,{children:["Please refer to ",(0,o.jsx)(i.a,{href:"https://hudi.apache.org/docs/next/configurations#Compaction-Configs",children:"advanced configs"})," for more details."]})}),"\n",(0,o.jsx)(i.h2,{id:"ways-to-trigger-compaction",children:"Ways to trigger Compaction"}),"\n",(0,o.jsx)(i.h3,{id:"inline",children:"Inline"}),"\n",(0,o.jsx)(i.p,{children:"By default, compaction is run asynchronously."}),"\n",(0,o.jsx)(i.p,{children:"If latency of ingesting records is important for you, you are most likely using Merge-On-Read tables.\nMerge-On-Read tables store data using a combination of columnar (e.g parquet) + row based (e.g avro) file formats.\nUpdates are logged to delta files & later compacted to produce new versions of columnar files.\nTo improve ingestion latency, Async Compaction is the default configuration."}),"\n",(0,o.jsx)(i.p,{children:"If immediate read performance of a new commit is important for you, or you want simplicity of not managing separate compaction jobs,\nyou may want synchronous inline compaction, which means that as a commit is written it is also compacted by the same job."}),"\n",(0,o.jsxs)(i.p,{children:["For this deployment mode, please use ",(0,o.jsx)(i.code,{children:"hoodie.compact.inline = true"})," for Spark Datasource and Spark SQL writers. For\nHoodieStreamer sync once mode inline compaction can be achieved by passing the flag ",(0,o.jsx)(i.code,{children:"--disable-compaction"})," (Meaning to\ndisable async compaction). Further in HoodieStreamer when both\ningestion and compaction is running in the same spark context, you can use resource allocation configuration\nin Hudi Streamer CLI such as (",(0,o.jsx)(i.code,{children:"--delta-sync-scheduling-weight"}),",\n",(0,o.jsx)(i.code,{children:"--compact-scheduling-weight"}),", ",(0,o.jsx)(i.code,{children:"--delta-sync-scheduling-minshare"}),", and ",(0,o.jsx)(i.code,{children:"--compact-scheduling-minshare"}),")\nto control executor allocation between ingestion and compaction."]}),"\n",(0,o.jsx)(i.h3,{id:"async--offline-compaction-models",children:"Async & Offline Compaction models"}),"\n",(0,o.jsx)(i.p,{children:"There are a couple of ways here to trigger compaction ."}),"\n",(0,o.jsx)(i.h4,{id:"async-execution-within-the-same-process",children:"Async execution within the same process"}),"\n",(0,o.jsx)(i.p,{children:"In streaming ingestion write models like HoodieStreamer\ncontinuous mode, Flink and Spark Streaming, async compaction is enabled by default and runs alongside without blocking\nregular ingestion."}),"\n",(0,o.jsx)(i.h5,{id:"spark-structured-streaming",children:"Spark Structured Streaming"}),"\n",(0,o.jsx)(i.p,{children:"Compactions are scheduled and executed asynchronously inside the\nstreaming job.Here is an example snippet in java"}),"\n",(0,o.jsx)(i.pre,{children:(0,o.jsx)(i.code,{className:"language-properties",children:'import org.apache.hudi.DataSourceWriteOptions;\nimport org.apache.hudi.HoodieDataSourceHelpers;\nimport org.apache.hudi.config.HoodieCompactionConfig;\nimport org.apache.hudi.config.HoodieWriteConfig;\n\nimport org.apache.spark.sql.streaming.OutputMode;\nimport org.apache.spark.sql.streaming.ProcessingTime;\n\n\n DataStreamWriter writer = streamingInput.writeStream().format("org.apache.hudi")\n .option("hoodie.datasource.write.operation", operationType)\n .option("hoodie.datasource.write.table.type", tableType)\n .option("hoodie.datasource.write.recordkey.field", "_row_key")\n .option("hoodie.datasource.write.partitionpath.field", "partition")\n .option("hoodie.datasource.write.precombine.field"(), "timestamp")\n .option("hoodie.compact.inline.max.delta.commits", "10")\n .option("hoodie.datasource.compaction.async.enable", "true")\n .option("hoodie.table.name", tableName).option("checkpointLocation", checkpointLocation)\n .outputMode(OutputMode.Append());\n writer.trigger(new ProcessingTime(30000)).start(tablePath);\n'})}),"\n",(0,o.jsx)(i.h5,{id:"hudi-streamer-continuous-mode",children:"Hudi Streamer Continuous Mode"}),"\n",(0,o.jsxs)(i.p,{children:["Hudi Streamer provides continuous ingestion mode where a single long running spark application",(0,o.jsx)(i.br,{}),"\n","ingests data to Hudi table continuously from upstream sources. In this mode, Hudi supports managing asynchronous\ncompactions. Here is an example snippet for running in continuous mode with async compactions"]}),"\n",(0,o.jsx)(i.pre,{children:(0,o.jsx)(i.code,{className:"language-properties",children:"spark-submit --packages org.apache.hudi:hudi-utilities-slim-bundle_2.12:1.0.0,org.apache.hudi:hudi-spark3.5-bundle_2.12:1.0.0 \\\n--class org.apache.hudi.utilities.streamer.HoodieStreamer \\\n--table-type MERGE_ON_READ \\\n--target-base-path \\\n--target-table \\\n--source-class org.apache.hudi.utilities.sources.JsonDFSSource \\\n--source-ordering-field ts \\\n--props /path/to/source.properties \\\n--continous\n"})}),"\n",(0,o.jsx)(i.h4,{id:"scheduling-and-execution-by-a-separate-process",children:"Scheduling and Execution by a separate process"}),"\n",(0,o.jsxs)(i.p,{children:["For some use cases with long running table services, instead of having the regular writes block, users have the option to run\nboth steps of the compaction (",(0,o.jsx)(i.a,{href:"#compaction-architecture",children:"scheduling and execution"}),") offline in a separate process altogether.\nThis allows for regular writers to not bother about these compaction steps and allows users to provide more resources for\nthe compaction job as needed."]}),"\n",(0,o.jsx)(i.admonition,{type:"note",children:(0,o.jsx)(i.p,{children:"This model needs a lock provider configured for all jobs - the regular writer as well as the offline compaction job."})}),"\n",(0,o.jsx)(i.h4,{id:"scheduling-inline-and-executing-async",children:"Scheduling inline and executing async"}),"\n",(0,o.jsx)(i.p,{children:"In this model, it is possible for a Spark Datasource writer or a Flink job to just schedule the compaction inline ( that\nwill serialize the compaction plan in the timeline but will not execute it). And then a separate utility like\nHudiCompactor or HoodieFlinkCompactor can take care of periodically executing the compaction plan."}),"\n",(0,o.jsx)(i.admonition,{type:"note",children:(0,o.jsxs)(i.p,{children:["This model may need a lock provider ",(0,o.jsx)(i.strong,{children:"if"})," metadata table is enabled."]})}),"\n",(0,o.jsx)(i.h4,{id:"hudi-compactor-utility",children:"Hudi Compactor Utility"}),"\n",(0,o.jsxs)(i.p,{children:["Hudi provides a standalone tool to execute specific compactions asynchronously. Below is an example and you can read more in the ",(0,o.jsx)(i.a,{href:"/docs/cli#compactions",children:"deployment guide"}),"\nThe compactor utility allows to do scheduling and execution of compaction."]}),"\n",(0,o.jsx)(i.p,{children:"Example:"}),"\n",(0,o.jsx)(i.pre,{children:(0,o.jsx)(i.code,{className:"language-properties",children:"spark-submit --packages org.apache.hudi:hudi-utilities-slim-bundle_2.12:1.0.0,org.apache.hudi:hudi-spark3.5-bundle_2.12:1.0.0 \\\n--class org.apache.hudi.utilities.HoodieCompactor \\\n--base-path \\\n--table-name \\\n--schema-file \\\n--instant-time \n"})}),"\n",(0,o.jsxs)(i.p,{children:["Note, the ",(0,o.jsx)(i.code,{children:"instant-time"})," parameter is now optional for the Hudi Compactor Utility. If using the utility without ",(0,o.jsx)(i.code,{children:"--instant time"}),",\nthe spark-submit will execute the earliest scheduled compaction on the Hudi timeline."]}),"\n",(0,o.jsx)(i.h4,{id:"hudi-cli",children:"Hudi CLI"}),"\n",(0,o.jsxs)(i.p,{children:["Hudi CLI is yet another way to execute specific compactions asynchronously. Here is an example and you can read more in the ",(0,o.jsx)(i.a,{href:"/docs/cli#compactions",children:"deployment guide"})]}),"\n",(0,o.jsx)(i.p,{children:"Example:"}),"\n",(0,o.jsx)(i.pre,{children:(0,o.jsx)(i.code,{className:"language-properties",children:"hudi:trips->compaction run --tableName --parallelism --compactionInstant \n...\n"})}),"\n",(0,o.jsx)(i.h4,{id:"flink-offline-compaction",children:"Flink Offline Compaction"}),"\n",(0,o.jsxs)(i.p,{children:["Offline compaction needs to submit the Flink task on the command line. The program entry is as follows: ",(0,o.jsx)(i.code,{children:"hudi-flink-bundle_2.11-0.9.0-SNAPSHOT.jar"})," :\n",(0,o.jsx)(i.code,{children:"org.apache.hudi.sink.compact.HoodieFlinkCompactor"})]}),"\n",(0,o.jsx)(i.pre,{children:(0,o.jsx)(i.code,{className:"language-bash",children:"# Command line\n./bin/flink run -c org.apache.hudi.sink.compact.HoodieFlinkCompactor lib/hudi-flink-bundle_2.11-0.9.0.jar --path hdfs://xxx:9000/table\n"})}),"\n",(0,o.jsx)(i.h4,{id:"options",children:"Options"}),"\n",(0,o.jsxs)(i.table,{children:[(0,o.jsx)(i.thead,{children:(0,o.jsxs)(i.tr,{children:[(0,o.jsx)(i.th,{children:"Option Name"}),(0,o.jsx)(i.th,{children:"Default"}),(0,o.jsx)(i.th,{children:"Description"})]})}),(0,o.jsxs)(i.tbody,{children:[(0,o.jsxs)(i.tr,{children:[(0,o.jsx)(i.td,{children:(0,o.jsx)(i.code,{children:"--path"})}),(0,o.jsx)(i.td,{children:(0,o.jsx)(i.code,{children:"n/a **(Required)**"})}),(0,o.jsx)(i.td,{children:"The path where the target table is stored on Hudi"})]}),(0,o.jsxs)(i.tr,{children:[(0,o.jsx)(i.td,{children:(0,o.jsx)(i.code,{children:"--compaction-max-memory"})}),(0,o.jsxs)(i.td,{children:[(0,o.jsx)(i.code,{children:"100"})," (Optional)"]}),(0,o.jsx)(i.td,{children:"The index map size of log data during compaction, 100 MB by default. If you have enough memory, you can turn up this parameter"})]}),(0,o.jsxs)(i.tr,{children:[(0,o.jsx)(i.td,{children:(0,o.jsx)(i.code,{children:"--schedule"})}),(0,o.jsxs)(i.td,{children:[(0,o.jsx)(i.code,{children:"false"})," (Optional)"]}),(0,o.jsx)(i.td,{children:"whether to execute the operation of scheduling compaction plan. When the write process is still writing\uff0c turning on this parameter have a risk of losing data. Therefore, it must be ensured that there are no write tasks currently writing data to this table when this parameter is turned on"})]}),(0,o.jsxs)(i.tr,{children:[(0,o.jsx)(i.td,{children:(0,o.jsx)(i.code,{children:"--seq"})}),(0,o.jsxs)(i.td,{children:[(0,o.jsx)(i.code,{children:"LIFO"})," (Optional)"]}),(0,o.jsxs)(i.td,{children:["The order in which compaction tasks are executed. Executing from the latest compaction plan by default. ",(0,o.jsx)(i.code,{children:"LIFO"}),": executing from the latest plan. ",(0,o.jsx)(i.code,{children:"FIFO"}),": executing from the oldest plan."]})]}),(0,o.jsxs)(i.tr,{children:[(0,o.jsx)(i.td,{children:(0,o.jsx)(i.code,{children:"--service"})}),(0,o.jsxs)(i.td,{children:[(0,o.jsx)(i.code,{children:"false"})," (Optional)"]}),(0,o.jsx)(i.td,{children:"Whether to start a monitoring service that checks and schedules new compaction task in configured interval."})]}),(0,o.jsxs)(i.tr,{children:[(0,o.jsx)(i.td,{children:(0,o.jsx)(i.code,{children:"--min-compaction-interval-seconds"})}),(0,o.jsxs)(i.td,{children:[(0,o.jsx)(i.code,{children:"600(s)"})," (optional)"]}),(0,o.jsx)(i.td,{children:"The checking interval for service mode, by default 10 minutes."})]})]})]}),"\n",(0,o.jsx)(i.h2,{id:"related-resources",children:"Related Resources"}),"\n",(0,o.jsxs)(i.p,{children:[(0,o.jsx)("h3",{children:"Blogs"}),"\n",(0,o.jsx)(i.a,{href:"https://medium.com/@simpsons/apache-hudi-compaction-6e6383790234",children:"Apache Hudi Compaction"}),"\n",(0,o.jsx)(i.a,{href:"https://medium.com/@simpsons/standalone-hoodiecompactor-utility-890198e4c539",children:"Standalone HoodieCompactor Utility"})]})]})}function h(e={}){const{wrapper:i}={...(0,a.R)(),...e.components};return i?(0,o.jsx)(i,{...e,children:(0,o.jsx)(d,{...e})}):d(e)}},67587:(e,i,t)=>{t.d(i,{A:()=>n});const n=t.p+"assets/images/hudi_mor_file_layout-643f9f7fda5aa0d532682af27fe3e42c.jpg"},56040:(e,i,t)=>{t.d(i,{A:()=>n});const n=t.p+"assets/images/hudi_mor_file_layout_post_compaction-9f10af785d4927dc3d66303dac5bc7ba.jpg"},28453:(e,i,t)=>{t.d(i,{R:()=>s,x:()=>r});var n=t(96540);const o={},a=n.createContext(o);function s(e){const i=n.useContext(a);return n.useMemo((function(){return"function"==typeof e?e(i):{...i,...e}}),[i,e])}function r(e){let i;return i=e.disableParentContext?"function"==typeof e.components?e.components(o):e.components||o:s(e.components),n.createElement(a.Provider,{value:i},e.children)}}}]);
\ No newline at end of file
diff --git a/content/cn/assets/js/1a20bc57.01c0b799.js b/content/cn/assets/js/1a20bc57.01c0b799.js
deleted file mode 100644
index c9787f67f8e3e..0000000000000
--- a/content/cn/assets/js/1a20bc57.01c0b799.js
+++ /dev/null
@@ -1 +0,0 @@
-"use strict";(self.webpackChunkhudi=self.webpackChunkhudi||[]).push([[59114],{46768:(e,_,n)=>{n.r(_),n.d(_,{assets:()=>l,contentTitle:()=>s,default:()=>d,frontMatter:()=>o,metadata:()=>t,toc:()=>r});const t=JSON.parse('{"id":"cli","title":"CLI","description":"Local set up","source":"@site/docs/cli.md","sourceDirName":".","slug":"/cli","permalink":"/cn/docs/next/cli","draft":false,"unlisted":false,"editUrl":"https://github.com/apache/hudi/tree/asf-site/website/docs/cli.md","tags":[],"version":"current","frontMatter":{"title":"CLI","keywords":["hudi","cli"],"last_modified_at":"2021-08-18T19:59:57.000Z"},"sidebar":"docs","previous":{"title":"SQL Procedures","permalink":"/cn/docs/next/procedures"},"next":{"title":"Metrics","permalink":"/cn/docs/next/metrics"}}');var i=n(74848),a=n(28453);const o={title:"CLI",keywords:["hudi","cli"],last_modified_at:new Date("2021-08-18T19:59:57.000Z")},s=void 0,l={},r=[{value:"Local set up",id:"local-set-up",level:3},{value:"Hudi CLI Bundle setup",id:"hudi-cli-bundle-setup",level:3},{value:"Base path",id:"base-path",level:3},{value:"Using Hudi-cli in S3",id:"using-hudi-cli-in-s3",level:3},{value:"Note: These AWS jar versions below are specific to Spark 3.2.0",id:"note-these-aws-jar-versions-below-are-specific-to-spark-320",level:4},{value:"Using hudi-cli on Google Dataproc",id:"using-hudi-cli-on-google-dataproc",level:3},{value:"Connect to a Kerberized cluster",id:"connect-to-a-kerberized-cluster",level:2},{value:"Using hudi-cli",id:"using-hudi-cli",level:2},{value:"Inspecting Commits",id:"inspecting-commits",level:3},{value:"Drilling Down to a specific Commit",id:"drilling-down-to-a-specific-commit",level:3},{value:"FileSystem View",id:"filesystem-view",level:3},{value:"Statistics",id:"statistics",level:3},{value:"Archived Commits",id:"archived-commits",level:3},{value:"Compactions",id:"compactions",level:3},{value:"Validate Compaction",id:"validate-compaction",level:3},{value:"Unscheduling Compaction",id:"unscheduling-compaction",level:3},{value:"Repair Compaction",id:"repair-compaction",level:3},{value:"Savepoint and Restore",id:"savepoint-and-restore",level:3},{value:"Upgrade and Downgrade Table",id:"upgrade-and-downgrade-table",level:3},{value:"Change Hudi Table Type",id:"change-hudi-table-type",level:3}];function c(e){const _={a:"a",admonition:"admonition",br:"br",code:"code",h2:"h2",h3:"h3",h4:"h4",li:"li",ol:"ol",p:"p",pre:"pre",strong:"strong",table:"table",tbody:"tbody",td:"td",th:"th",thead:"thead",tr:"tr",...(0,a.R)(),...e.components};return(0,i.jsxs)(i.Fragment,{children:[(0,i.jsx)(_.h3,{id:"local-set-up",children:"Local set up"}),"\n",(0,i.jsxs)(_.p,{children:["Once hudi has been built, the shell can be fired by via ",(0,i.jsx)(_.code,{children:"cd hudi-cli && ./hudi-cli.sh"}),"."]}),"\n",(0,i.jsx)(_.h3,{id:"hudi-cli-bundle-setup",children:"Hudi CLI Bundle setup"}),"\n",(0,i.jsxs)(_.p,{children:["In release ",(0,i.jsx)(_.code,{children:"0.13.0"})," we have now added another way of launching the ",(0,i.jsx)(_.code,{children:"hudi cli"}),", which is using the ",(0,i.jsx)(_.code,{children:"hudi-cli-bundle"}),"."]}),"\n",(0,i.jsxs)(_.p,{children:["There are a couple of requirements when using this approach such as having ",(0,i.jsx)(_.code,{children:"spark"})," installed locally on your machine.\nIt is required to use a spark distribution with hadoop dependencies packaged such as ",(0,i.jsx)(_.code,{children:"spark-3.3.1-bin-hadoop2.tgz"})," from ",(0,i.jsx)(_.a,{href:"https://archive.apache.org/dist/spark/",children:"https://archive.apache.org/dist/spark/"}),".\nWe also recommend you set an env variable ",(0,i.jsx)(_.code,{children:"$SPARK_HOME"})," to the path of where spark is installed on your machine.\nOne important thing to note is that the ",(0,i.jsx)(_.code,{children:"hudi-spark-bundle"})," should also be present when using the ",(0,i.jsx)(_.code,{children:"hudi-cli-bundle"}),".",(0,i.jsx)(_.br,{}),"\n","To provide the locations of these bundle jars you can set them in your shell like so:\n",(0,i.jsx)(_.code,{children:"export CLI_BUNDLE_JAR="})," , ",(0,i.jsx)(_.code,{children:"export SPARK_BUNDLE_JAR="}),"."]}),"\n",(0,i.jsx)(_.p,{children:"For steps see below if you are not compiling the project and downloading the jars:"}),"\n",(0,i.jsxs)(_.ol,{children:["\n",(0,i.jsx)(_.li,{children:"Create an empty folder as a new directory"}),"\n",(0,i.jsx)(_.li,{children:"Copy the hudi-cli-bundle jars and hudi-spark*-bundle jars to this directory"}),"\n",(0,i.jsx)(_.li,{children:"Copy the following script and folder to this directory"}),"\n"]}),"\n",(0,i.jsx)(_.pre,{children:(0,i.jsx)(_.code,{children:"packaging/hudi-cli-bundle/hudi-cli-with-bundle.sh\npackaging/hudi-cli-bundle/conf . the `conf` folder should be in this directory.\n"})}),"\n",(0,i.jsxs)(_.ol,{start:"4",children:["\n",(0,i.jsx)(_.li,{children:"Start Hudi CLI shell with environment variables set"}),"\n"]}),"\n",(0,i.jsx)(_.pre,{children:(0,i.jsx)(_.code,{children:"export SPARK_HOME=\nexport CLI_BUNDLE_JAR=\nexport SPARK_BUNDLE_JAR=\n\n./hudi-cli-with-bundle.sh\n\n"})}),"\n",(0,i.jsx)(_.h3,{id:"base-path",children:"Base path"}),"\n",(0,i.jsxs)(_.p,{children:["A hudi table resides on DFS, in a location referred to as the ",(0,i.jsx)(_.code,{children:"basePath"})," and\nwe would need this location in order to connect to a Hudi table. Hudi library effectively manages this table internally, using ",(0,i.jsx)(_.code,{children:".hoodie"})," subfolder to track all metadata."]}),"\n",(0,i.jsx)(_.h3,{id:"using-hudi-cli-in-s3",children:"Using Hudi-cli in S3"}),"\n",(0,i.jsxs)(_.p,{children:["If you are using hudi that comes packaged with AWS EMR, you can find instructions to use hudi-cli ",(0,i.jsx)(_.a,{href:"https://docs.aws.amazon.com/emr/latest/ReleaseGuide/emr-hudi-cli.html",children:"here"}),".\nIf you are not using EMR, or would like to use latest hudi-cli from master, you can follow the below steps to access S3 dataset in your local environment (laptop)."]}),"\n",(0,i.jsx)(_.p,{children:"Build Hudi with corresponding Spark version, for eg, -Dspark3.1.x"}),"\n",(0,i.jsx)(_.p,{children:"Set the following environment variables."}),"\n",(0,i.jsx)(_.pre,{children:(0,i.jsx)(_.code,{children:"export AWS_REGION=us-east-2\nexport AWS_ACCESS_KEY_ID=\nexport AWS_SECRET_ACCESS_KEY=\nexport SPARK_HOME=\n"})}),"\n",(0,i.jsx)(_.p,{children:"Ensure you set the SPARK_HOME to your local spark home compatible to compiled hudi spark version above."}),"\n",(0,i.jsx)(_.p,{children:"Apart from these, we might need to add aws jars to class path so that accessing S3 is feasible from local.\nWe need two jars, namely, aws-java-sdk-bundle jar and hadoop-aws jar which you can find online.\nFor eg:"}),"\n",(0,i.jsx)(_.pre,{children:(0,i.jsx)(_.code,{children:"wget https://repo1.maven.org/maven2/org/apache/hadoop/hadoop-aws/3.2.0/hadoop-aws-3.2.0.jar -o /lib/spark-3.2.0-bin-hadoop3.2/jars/hadoop-aws-3.2.0.jar\nwget https://repo1.maven.org/maven2/com/amazonaws/aws-java-sdk-bundle/1.11.375/aws-java-sdk-bundle-1.11.375.jar -o /lib/spark-3.2.0-bin-hadoop3.2/jars/aws-java-sdk-bundle-1.11.375.jar\n"})}),"\n",(0,i.jsx)(_.h4,{id:"note-these-aws-jar-versions-below-are-specific-to-spark-320",children:"Note: These AWS jar versions below are specific to Spark 3.2.0"}),"\n",(0,i.jsx)(_.pre,{children:(0,i.jsx)(_.code,{children:"export CLIENT_JAR=/lib/spark-3.2.0-bin-hadoop3.2/jars/aws-java-sdk-bundle-1.12.48.jar:/lib/spark-3.2.0-bin-hadoop3.2/jars/hadoop-aws-3.3.1.jar\n"})}),"\n",(0,i.jsx)(_.p,{children:"Once these are set, you are good to launch hudi-cli and access S3 dataset."}),"\n",(0,i.jsx)(_.pre,{children:(0,i.jsx)(_.code,{children:"./hudi-cli/hudi-cli.sh\n"})}),"\n",(0,i.jsx)(_.h3,{id:"using-hudi-cli-on-google-dataproc",children:"Using hudi-cli on Google Dataproc"}),"\n",(0,i.jsxs)(_.p,{children:[(0,i.jsx)(_.a,{href:"https://cloud.google.com/dataproc",children:"Dataproc"})," is Google's managed service for running Apache Hadoop, Apache Spark,\nApache Flink, Presto and many other frameworks, including Hudi. If you want to run the Hudi CLI on a Dataproc node\nwhich has not been launched with Hudi support enabled, you can use the steps below:"]}),"\n",(0,i.jsx)(_.p,{children:"These steps use Hudi version 0.13.0. If you want to use a different version you will have to edit the below commands\nappropriately:"}),"\n",(0,i.jsxs)(_.ol,{children:["\n",(0,i.jsx)(_.li,{children:"Once you've started the Dataproc cluster, you can ssh into it as follows:"}),"\n"]}),"\n",(0,i.jsx)(_.pre,{children:(0,i.jsx)(_.code,{children:'$ gcloud compute ssh --zone "YOUR_ZONE" "HOSTNAME_OF_MASTER_NODE" --project "YOUR_PROJECT"\n'})}),"\n",(0,i.jsxs)(_.ol,{start:"2",children:["\n",(0,i.jsx)(_.li,{children:"Download the Hudi CLI bundle"}),"\n"]}),"\n",(0,i.jsx)(_.pre,{children:(0,i.jsx)(_.code,{children:"wget https://repo1.maven.org/maven2/org/apache/hudi/hudi-cli-bundle_2.12/0.13.0/hudi-cli-bundle_2.12-0.13.0.jar \n"})}),"\n",(0,i.jsxs)(_.ol,{start:"3",children:["\n",(0,i.jsx)(_.li,{children:"Download the Hudi Spark bundle"}),"\n"]}),"\n",(0,i.jsx)(_.pre,{children:(0,i.jsx)(_.code,{children:"wget https://repo1.maven.org/maven2/org/apache/hudi/hudi-spark-bundle_2.12/0.13.0/hudi-spark-bundle_2.12-0.13.0.jar\n"})}),"\n",(0,i.jsxs)(_.ol,{start:"4",children:["\n",(0,i.jsx)(_.li,{children:"Download the shell script that launches Hudi CLI bundle"}),"\n"]}),"\n",(0,i.jsx)(_.pre,{children:(0,i.jsx)(_.code,{children:"wget https://raw.githubusercontent.com/apache/hudi/release-0.13.0/packaging/hudi-cli-bundle/hudi-cli-with-bundle.sh\n"})}),"\n",(0,i.jsxs)(_.ol,{start:"5",children:["\n",(0,i.jsx)(_.li,{children:"Launch Hudi CLI bundle with appropriate environment variables as follows:"}),"\n"]}),"\n",(0,i.jsx)(_.pre,{children:(0,i.jsx)(_.code,{children:"CLIENT_JAR=$DATAPROC_DIR/lib/gcs-connector.jar CLI_BUNDLE_JAR=hudi-cli-bundle_2.12-0.13.0.jar SPARK_BUNDLE_JAR=hudi-spark-bundle_2.12-0.13.0.jar ./hudi-cli-with-bundle.sh \n"})}),"\n",(0,i.jsxs)(_.ol,{start:"6",children:["\n",(0,i.jsxs)(_.li,{children:["\n",(0,i.jsxs)(_.p,{children:["hudi->connect --path gs://path_to_some_table",(0,i.jsx)(_.br,{}),"\n","Metadata for table some_table loaded"]}),"\n"]}),"\n",(0,i.jsxs)(_.li,{children:["\n",(0,i.jsxs)(_.p,{children:["hudi:some_table->commits show --limit 5",(0,i.jsx)(_.br,{}),"\n","This command should show the recent commits, if the above steps work correctly."]}),"\n"]}),"\n"]}),"\n",(0,i.jsx)(_.h2,{id:"connect-to-a-kerberized-cluster",children:"Connect to a Kerberized cluster"}),"\n",(0,i.jsxs)(_.p,{children:["Before connecting to a Kerberized cluster, you can use ",(0,i.jsx)(_.strong,{children:"kerberos kinit"})," command. Following is the usage of this command."]}),"\n",(0,i.jsx)(_.pre,{children:(0,i.jsx)(_.code,{className:"language-shell",children:"hudi->help kerberos kinit\nNAME\n kerberos kinit - Perform Kerberos authentication\n\nSYNOPSIS\n kerberos kinit --krb5conf String [--principal String] [--keytab String]\n\nOPTIONS\n --krb5conf String\n Path to krb5.conf\n [Optional, default = /etc/krb5.conf]\n\n --principal String\n Kerberos principal\n [Mandatory]\n\n --keytab String\n Path to keytab\n [Mandatory]\n"})}),"\n",(0,i.jsx)(_.p,{children:"For example:"}),"\n",(0,i.jsx)(_.pre,{children:(0,i.jsx)(_.code,{className:"language-shell",children:"hudi->kerberos kinit --principal user/host@DOMAIN --keytab /etc/security/keytabs/user.keytab\nPerform Kerberos authentication\nParameters:\n--krb5conf: /etc/krb5.conf\n--principal: user/host@DOMAIN\n--keytab: /etc/security/keytabs/user.keytab\nKerberos current user: user/host@DOMAIN (auth:KERBEROS)\nKerberos login user: user/host@DOMAIN (auth:KERBEROS)\nKerberos authentication success\n"})}),"\n",(0,i.jsx)(_.p,{children:'If you see "Kerberos authentication success" in the command output, it means Kerberos authentication has been successful.'}),"\n",(0,i.jsx)(_.h2,{id:"using-hudi-cli",children:"Using hudi-cli"}),"\n",(0,i.jsx)(_.p,{children:"To initialize a hudi table, use the following command."}),"\n",(0,i.jsx)(_.pre,{children:(0,i.jsx)(_.code,{className:"language-java",children:"===================================================================\n* ___ ___ *\n* /\\__\\ ___ /\\ \\ ___ *\n* / / / /\\__\\ / \\ \\ /\\ \\ *\n* / /__/ / / / / /\\ \\ \\ \\ \\ \\ *\n* / \\ \\ ___ / / / / / \\ \\__\\ / \\__\\ *\n* / /\\ \\ /\\__\\ / /__/ ___ / /__/ \\ |__| / /\\/__/ *\n* \\/ \\ \\/ / / \\ \\ \\ /\\__\\ \\ \\ \\ / / / /\\/ / / *\n* \\ / / \\ \\ / / / \\ \\ / / / \\ /__/ *\n* / / / \\ \\/ / / \\ \\/ / / \\ \\__\\ *\n* / / / \\ / / \\ / / \\/__/ *\n* \\/__/ \\/__/ \\/__/ Apache Hudi CLI *\n* *\n===================================================================\n\nhudi->create --path /user/hive/warehouse/table1 --tableName hoodie_table_1 --tableType COPY_ON_WRITE\n.....\n"})}),"\n",(0,i.jsx)(_.p,{children:"To see the description of hudi table, use the command:"}),"\n",(0,i.jsx)(_.pre,{children:(0,i.jsx)(_.code,{className:"language-java",children:"hudi:hoodie_table_1->desc\n18/09/06 15:57:19 INFO timeline.HoodieActiveTimeline: Loaded instants []\n _________________________________________________________\n | Property | Value |\n |========================================================|\n | basePath | ... |\n | metaPath | ... |\n | fileSystem | hdfs |\n | hoodie.table.name | hoodie_table_1 |\n | hoodie.table.type | COPY_ON_WRITE |\n | hoodie.archivelog.folder| |\n"})}),"\n",(0,i.jsx)(_.p,{children:"Following is a sample command to connect to a Hudi table contains uber trips."}),"\n",(0,i.jsx)(_.pre,{children:(0,i.jsx)(_.code,{className:"language-java",children:"hudi:trips->connect --path /app/uber/trips\n\n16/10/05 23:20:37 INFO model.HoodieTableMetadata: All commits :HoodieCommits{commitList=[20161002045850, 20161002052915, 20161002055918, 20161002065317, 20161002075932, 20161002082904, 20161002085949, 20161002092936, 20161002105903, 20161002112938, 20161002123005, 20161002133002, 20161002155940, 20161002165924, 20161002172907, 20161002175905, 20161002190016, 20161002192954, 20161002195925, 20161002205935, 20161002215928, 20161002222938, 20161002225915, 20161002232906, 20161003003028, 20161003005958, 20161003012936, 20161003022924, 20161003025859, 20161003032854, 20161003042930, 20161003052911, 20161003055907, 20161003062946, 20161003065927, 20161003075924, 20161003082926, 20161003085925, 20161003092909, 20161003100010, 20161003102913, 20161003105850, 20161003112910, 20161003115851, 20161003122929, 20161003132931, 20161003142952, 20161003145856, 20161003152953, 20161003155912, 20161003162922, 20161003165852, 20161003172923, 20161003175923, 20161003195931, 20161003210118, 20161003212919, 20161003215928, 20161003223000, 20161003225858, 20161004003042, 20161004011345, 20161004015235, 20161004022234, 20161004063001, 20161004072402, 20161004074436, 20161004080224, 20161004082928, 20161004085857, 20161004105922, 20161004122927, 20161004142929, 20161004163026, 20161004175925, 20161004194411, 20161004203202, 20161004211210, 20161004214115, 20161004220437, 20161004223020, 20161004225321, 20161004231431, 20161004233643, 20161005010227, 20161005015927, 20161005022911, 20161005032958, 20161005035939, 20161005052904, 20161005070028, 20161005074429, 20161005081318, 20161005083455, 20161005085921, 20161005092901, 20161005095936, 20161005120158, 20161005123418, 20161005125911, 20161005133107, 20161005155908, 20161005163517, 20161005165855, 20161005180127, 20161005184226, 20161005191051, 20161005193234, 20161005203112, 20161005205920, 20161005212949, 20161005223034, 20161005225920]}\nMetadata for table trips loaded\n"})}),"\n",(0,i.jsx)(_.p,{children:"Once connected to the table, a lot of other commands become available. The shell has contextual autocomplete help (press TAB) and below is a list of all commands, few of which are reviewed in this section"}),"\n",(0,i.jsx)(_.pre,{children:(0,i.jsx)(_.code,{className:"language-shell",children:"hudi:trips->help\n* ! - Allows execution of operating system (OS) commands\n* // - Inline comment markers (start of line only)\n* ; - Inline comment markers (start of line only)\n* bootstrap index showmapping - Show bootstrap index mapping\n* bootstrap index showpartitions - Show bootstrap indexed partitions\n* bootstrap run - Run a bootstrap action for current Hudi table\n* clean showpartitions - Show partition level details of a clean\n* cleans refresh - Refresh table metadata\n* cleans run - run clean\n* cleans show - Show the cleans\n* clear - Clears the console\n* cls - Clears the console\n* clustering run - Run Clustering\n* clustering schedule - Schedule Clustering\n* clustering scheduleAndExecute - Run Clustering. Make a cluster plan first and execute that plan immediately\n* commit rollback - Rollback a commit\n* commits compare - Compare commits with another Hoodie table\n* commit show_write_stats - Show write stats of a commit\n* commit showfiles - Show file level details of a commit\n* commit showpartitions - Show partition level details of a commit\n* commits refresh - Refresh table metadata\n* commits show - Show the commits\n* commits showarchived - Show the archived commits\n* commits sync - Sync commits with another Hoodie table\n* compaction repair - Renames the files to make them consistent with the timeline as dictated by Hoodie metadata. Use when compaction unschedule fails partially.\n* compaction run - Run Compaction for given instant time\n* compaction schedule - Schedule Compaction\n* compaction scheduleAndExecute - Schedule compaction plan and execute this plan\n* compaction show - Shows compaction details for a specific compaction instant\n* compaction showarchived - Shows compaction details for a specific compaction instant\n* compactions show all - Shows all compactions that are in active timeline\n* compactions showarchived - Shows compaction details for specified time window\n* compaction unschedule - Unschedule Compaction\n* compaction unscheduleFileId - UnSchedule Compaction for a fileId\n* compaction validate - Validate Compaction\n* connect - Connect to a hoodie table\n* create - Create a hoodie table if not present\n* date - Displays the local date and time\n* desc - Describe Hoodie Table properties\n* downgrade table - Downgrades a table\n* exit - Exits the shell\n* export instants - Export Instants and their metadata from the Timeline\n* fetch table schema - Fetches latest table schema\n* hdfsparquetimport - Imports Parquet table to a hoodie table\n* help - List all commands usage\n* marker delete - Delete the marker\n* metadata create - Create the Metadata Table if it does not exist\n* metadata delete - Remove the Metadata Table\n* metadata init - Update the metadata table from commits since the creation\n* metadata list-files - Print a list of all files in a partition from the metadata\n* metadata list-partitions - List all partitions from metadata\n* metadata refresh - Refresh table metadata\n* metadata set - Set options for Metadata Table\n* metadata stats - Print stats about the metadata\n* metadata validate-files - Validate all files in all partitions from the metadata\n* quit - Exits the shell\n* refresh - Refresh table metadata\n* repair addpartitionmeta - Add partition metadata to a table, if not present\n* repair corrupted clean files - repair corrupted clean files\n* repair deduplicate - De-duplicate a partition path contains duplicates & produce repaired files to replace with\n* repair migrate-partition-meta - Migrate all partition meta file currently stored in text format to be stored in base file format. See HoodieTableConfig#PARTITION_METAFILE_USE_DATA_FORMAT.\n* repair overwrite-hoodie-props - Overwrite hoodie.properties with provided file. Risky operation. Proceed with caution!\n* savepoint create - Savepoint a commit\n* savepoint delete - Delete the savepoint\n* savepoint rollback - Savepoint a commit\n* savepoints refresh - Refresh table metadata\n* savepoints show - Show the savepoints\n* script - Parses the specified resource file and executes its commands\n* set - Set spark launcher env to cli\n* show archived commits - Read commits from archived files and show details\n* show archived commit stats - Read commits from archived files and show details\n* show env - Show spark launcher env by key\n* show envs all - Show spark launcher envs\n* show fsview all - Show entire file-system view\n* show fsview latest - Show latest file-system view\n* show logfile metadata - Read commit metadata from log files\n* show logfile records - Read records from log files\n* show rollback - Show details of a rollback instant\n* show rollbacks - List all rollback instants\n* stats filesizes - File Sizes. Display summary stats on sizes of files\n* stats wa - Write Amplification. Ratio of how many records were upserted to how many records were actually written\n* sync validate - Validate the sync by counting the number of records\n* system properties - Shows the shell's properties\n* table delete-configs - Delete the supplied table configs from the table.\n* table recover-configs - Recover table configs, from update/delete that failed midway.\n* table update-configs - Update the table configs with configs with provided file.\n* temp_delete - Delete view name\n* temp_query - query against created temp view\n* temp delete - Delete view name\n* temp query - query against created temp view\n* temps_show - Show all views name\n* temps show - Show all views name\n* upgrade table - Upgrades a table\n* utils loadClass - Load a class\n* version - Displays shell version\n\nhudi:trips->\n"})}),"\n",(0,i.jsx)(_.h3,{id:"inspecting-commits",children:"Inspecting Commits"}),"\n",(0,i.jsxs)(_.p,{children:["The task of upserting or inserting a batch of incoming records is known as a ",(0,i.jsx)(_.strong,{children:"commit"})," in Hudi. A commit provides basic atomicity guarantees such that only committed data is available for querying.\nEach commit has a monotonically increasing string/number called the ",(0,i.jsx)(_.strong,{children:"commit number"}),". Typically, this is the time at which we started the commit."]}),"\n",(0,i.jsx)(_.p,{children:"To view some basic information about the last 10 commits,"}),"\n",(0,i.jsx)(_.pre,{children:(0,i.jsx)(_.code,{className:"language-java",children:'hudi:trips->commits show --sortBy "Total Bytes Written" --desc true --limit 10\n ________________________________________________________________________________________________________________________________________________________________________\n | CommitTime | Total Bytes Written| Total Files Added| Total Files Updated| Total Partitions Written| Total Records Written| Total Update Records Written| Total Errors|\n |=======================================================================================================================================================================|\n ....\n ....\n ....\n'})}),"\n",(0,i.jsx)(_.p,{children:"At the start of each write, Hudi also writes a .inflight commit to the .hoodie folder. You can use the timestamp there to estimate how long the commit has been inflight"}),"\n",(0,i.jsx)(_.pre,{children:(0,i.jsx)(_.code,{className:"language-java",children:"$ hdfs dfs -ls /app/uber/trips/.hoodie/*.inflight\n-rw-r--r-- 3 vinoth supergroup 321984 2016-10-05 23:18 /app/uber/trips/.hoodie/20161005225920.inflight\n"})}),"\n",(0,i.jsx)(_.h3,{id:"drilling-down-to-a-specific-commit",children:"Drilling Down to a specific Commit"}),"\n",(0,i.jsx)(_.p,{children:"To understand how the writes spread across specific partiions,"}),"\n",(0,i.jsx)(_.pre,{children:(0,i.jsx)(_.code,{className:"language-java",children:'hudi:trips->commit showpartitions --commit 20161005165855 --sortBy "Total Bytes Written" --desc true --limit 10\n __________________________________________________________________________________________________________________________________________\n | Partition Path| Total Files Added| Total Files Updated| Total Records Inserted| Total Records Updated| Total Bytes Written| Total Errors|\n |=========================================================================================================================================|\n ....\n ....\n'})}),"\n",(0,i.jsx)(_.p,{children:"If you need file level granularity , we can do the following"}),"\n",(0,i.jsx)(_.pre,{children:(0,i.jsx)(_.code,{className:"language-java",children:'hudi:trips->commit showfiles --commit 20161005165855 --sortBy "Partition Path"\n ________________________________________________________________________________________________________________________________________________________\n | Partition Path| File ID | Previous Commit| Total Records Updated| Total Records Written| Total Bytes Written| Total Errors|\n |=======================================================================================================================================================|\n ....\n ....\n'})}),"\n",(0,i.jsx)(_.h3,{id:"filesystem-view",children:"FileSystem View"}),"\n",(0,i.jsx)(_.p,{children:"Hudi views each partition as a collection of file-groups with each file-group containing a list of file-slices in commit order (See concepts).\nThe below commands allow users to view the file-slices for a data-set."}),"\n",(0,i.jsx)(_.pre,{children:(0,i.jsx)(_.code,{className:"language-java",children:'hudi:stock_ticks_mor->show fsview all\n ....\n _______________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________\n | Partition | FileId | Base-Instant | Data-File | Data-File Size| Num Delta Files| Total Delta File Size| Delta Files |\n |==============================================================================================================================================================================================================================================================================================================================================================================================================|\n | 2018/08/31| 111415c3-f26d-4639-86c8-f9956f245ac3| 20181002180759| hdfs://namenode:8020/user/hive/warehouse/stock_ticks_mor/2018/08/31/111415c3-f26d-4639-86c8-f9956f245ac3_0_20181002180759.parquet| 432.5 KB | 1 | 20.8 KB | [HoodieLogFile {hdfs://namenode:8020/user/hive/warehouse/stock_ticks_mor/2018/08/31/.111415c3-f26d-4639-86c8-f9956f245ac3_20181002180759.log.1}]|\n\n\n\nhudi:stock_ticks_mor->show fsview latest --partitionPath "2018/08/31"\n ......\n __________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________\n | Partition | FileId | Base-Instant | Data-File | Data-File Size| Num Delta Files| Total Delta Size| Delta Size - compaction scheduled| Delta Size - compaction unscheduled| Delta To Base Ratio - compaction scheduled| Delta To Base Ratio - compaction unscheduled| Delta Files - compaction scheduled | Delta Files - compaction unscheduled|\n |=================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================|\n | 2018/08/31| 111415c3-f26d-4639-86c8-f9956f245ac3| 20181002180759| hdfs://namenode:8020/user/hive/warehouse/stock_ticks_mor/2018/08/31/111415c3-f26d-4639-86c8-f9956f245ac3_0_20181002180759.parquet| 432.5 KB | 1 | 20.8 KB | 20.8 KB | 0.0 B | 0.0 B | 0.0 B | [HoodieLogFile {hdfs://namenode:8020/user/hive/warehouse/stock_ticks_mor/2018/08/31/.111415c3-f26d-4639-86c8-f9956f245ac3_20181002180759.log.1}]| [] |\n\n'})}),"\n",(0,i.jsx)(_.h3,{id:"statistics",children:"Statistics"}),"\n",(0,i.jsx)(_.p,{children:"Since Hudi directly manages file sizes for DFS table, it might be good to get an overall picture"}),"\n",(0,i.jsx)(_.pre,{children:(0,i.jsx)(_.code,{className:"language-java",children:'hudi:trips->stats filesizes --partitionPath 2016/09/01 --sortBy "95th" --desc true --limit 10\n ________________________________________________________________________________________________\n | CommitTime | Min | 10th | 50th | avg | 95th | Max | NumFiles| StdDev |\n |===============================================================================================|\n | | 93.9 MB | 93.9 MB | 93.9 MB | 93.9 MB | 93.9 MB | 93.9 MB | 2 | 2.3 KB |\n ....\n ....\n'})}),"\n",(0,i.jsx)(_.p,{children:"In case of Hudi write taking much longer, it might be good to see the write amplification for any sudden increases"}),"\n",(0,i.jsx)(_.pre,{children:(0,i.jsx)(_.code,{className:"language-java",children:"hudi:trips->stats wa\n __________________________________________________________________________\n | CommitTime | Total Upserted| Total Written| Write Amplifiation Factor|\n |=========================================================================|\n ....\n ....\n"})}),"\n",(0,i.jsx)(_.h3,{id:"archived-commits",children:"Archived Commits"}),"\n",(0,i.jsx)(_.p,{children:"In order to limit the amount of growth of .commit files on DFS, Hudi archives older .commit files (with due respect to the cleaner policy) into a commits.archived file.\nThis is a sequence file that contains a mapping from commitNumber => json with raw information about the commit (same that is nicely rolled up above)."}),"\n",(0,i.jsx)(_.h3,{id:"compactions",children:"Compactions"}),"\n",(0,i.jsx)(_.p,{children:"To get an idea of the lag between compaction and writer applications, use the below command to list down all\npending compactions."}),"\n",(0,i.jsx)(_.pre,{children:(0,i.jsx)(_.code,{className:"language-java",children:"hudi:trips->compactions show all\n ___________________________________________________________________\n | Compaction Instant Time| State | Total FileIds to be Compacted|\n |==================================================================|\n | | REQUESTED| 35 |\n | | INFLIGHT | 27 |\n"})}),"\n",(0,i.jsx)(_.p,{children:"To inspect a specific compaction plan, use"}),"\n",(0,i.jsx)(_.pre,{children:(0,i.jsx)(_.code,{className:"language-java",children:"hudi:trips->compaction show --instant \n _________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________\n | Partition Path| File Id | Base Instant | Data File Path | Total Delta Files| getMetrics |\n |================================================================================================================================================================================================================================================\n | 2018/07/17 | | | viewfs://ns-default/.../../UUID_.parquet | 1 | {TOTAL_LOG_FILES=1.0, TOTAL_IO_READ_MB=1230.0, TOTAL_LOG_FILES_SIZE=2.51255751E8, TOTAL_IO_WRITE_MB=991.0, TOTAL_IO_MB=2221.0}|\n\n"})}),"\n",(0,i.jsx)(_.p,{children:"To manually schedule or run a compaction, use the below command. This command uses spark launcher to perform compaction\noperations."}),"\n",(0,i.jsxs)(_.p,{children:[(0,i.jsx)(_.strong,{children:"NOTE:"})," Make sure no other application is scheduling compaction for this table concurrently\n{: .notice--info}"]}),"\n",(0,i.jsx)(_.pre,{children:(0,i.jsx)(_.code,{className:"language-java",children:"hudi:trips->help compaction schedule\nKeyword: compaction schedule\nDescription: Schedule Compaction\n Keyword: sparkMemory\n Help: Spark executor memory\n Mandatory: false\n Default if specified: '__NULL__'\n Default if unspecified: '1G'\n\n* compaction schedule - Schedule Compaction\n"})}),"\n",(0,i.jsx)(_.pre,{children:(0,i.jsx)(_.code,{className:"language-java",children:"hudi:trips->help compaction run\nKeyword: compaction run\nDescription: Run Compaction for given instant time\n Keyword: tableName\n Help: Table name\n Mandatory: true\n Default if specified: '__NULL__'\n Default if unspecified: '__NULL__'\n\n Keyword: parallelism\n Help: Parallelism for hoodie compaction\n Mandatory: true\n Default if specified: '__NULL__'\n Default if unspecified: '__NULL__'\n\n Keyword: schemaFilePath\n Help: Path for Avro schema file\n Mandatory: true\n Default if specified: '__NULL__'\n Default if unspecified: '__NULL__'\n\n Keyword: sparkMemory\n Help: Spark executor memory\n Mandatory: true\n Default if specified: '__NULL__'\n Default if unspecified: '__NULL__'\n\n Keyword: retry\n Help: Number of retries\n Mandatory: true\n Default if specified: '__NULL__'\n Default if unspecified: '__NULL__'\n\n Keyword: compactionInstant\n Help: Base path for the target hoodie table\n Mandatory: true\n Default if specified: '__NULL__'\n Default if unspecified: '__NULL__'\n\n* compaction run - Run Compaction for given instant time\n"})}),"\n",(0,i.jsx)(_.h3,{id:"validate-compaction",children:"Validate Compaction"}),"\n",(0,i.jsx)(_.p,{children:"Validating a compaction plan : Check if all the files necessary for compactions are present and are valid"}),"\n",(0,i.jsx)(_.pre,{children:(0,i.jsx)(_.code,{className:"language-java",children:"hudi:stock_ticks_mor->compaction validate --instant 20181005222611\n...\n\n COMPACTION PLAN VALID\n\n ___________________________________________________________________________________________________________________________________________________________________________________________________________________________\n | File Id | Base Instant Time| Base Data File | Num Delta Files| Valid| Error|\n |==========================================================================================================================================================================================================================|\n | 05320e98-9a57-4c38-b809-a6beaaeb36bd| 20181005222445 | hdfs://namenode:8020/user/hive/warehouse/stock_ticks_mor/2018/08/31/05320e98-9a57-4c38-b809-a6beaaeb36bd_0_20181005222445.parquet| 1 | true | |\n\n\n\nhudi:stock_ticks_mor->compaction validate --instant 20181005222601\n\n COMPACTION PLAN INVALID\n\n _______________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________\n | File Id | Base Instant Time| Base Data File | Num Delta Files| Valid| Error |\n |=====================================================================================================================================================================================================================================================================================================|\n | 05320e98-9a57-4c38-b809-a6beaaeb36bd| 20181005222445 | hdfs://namenode:8020/user/hive/warehouse/stock_ticks_mor/2018/08/31/05320e98-9a57-4c38-b809-a6beaaeb36bd_0_20181005222445.parquet| 1 | false| All log files specified in compaction operation is not present. Missing .... |\n"})}),"\n",(0,i.jsxs)(_.p,{children:[(0,i.jsx)(_.strong,{children:"NOTE:"})," The following commands must be executed without any other writer/ingestion application running.\n{: .notice--warning}"]}),"\n",(0,i.jsx)(_.p,{children:"Sometimes, it becomes necessary to remove a fileId from a compaction-plan inorder to speed-up or unblock compaction\noperation. Any new log-files that happened on this file after the compaction got scheduled will be safely renamed\nso that are preserved. Hudi provides the following CLI to support it"}),"\n",(0,i.jsx)(_.h3,{id:"unscheduling-compaction",children:"Unscheduling Compaction"}),"\n",(0,i.jsx)(_.pre,{children:(0,i.jsx)(_.code,{className:"language-java",children:"hudi:trips->compaction unscheduleFileId --fileId \n....\nNo File renames needed to unschedule file from pending compaction. Operation successful.\n"})}),"\n",(0,i.jsx)(_.p,{children:"In other cases, an entire compaction plan needs to be reverted. This is supported by the following CLI"}),"\n",(0,i.jsx)(_.pre,{children:(0,i.jsx)(_.code,{className:"language-java",children:"hudi:trips->compaction unschedule --instant \n.....\nNo File renames needed to unschedule pending compaction. Operation successful.\n"})}),"\n",(0,i.jsx)(_.h3,{id:"repair-compaction",children:"Repair Compaction"}),"\n",(0,i.jsxs)(_.p,{children:["The above compaction unscheduling operations could sometimes fail partially (e",":g"," -> DFS temporarily unavailable). With\npartial failures, the compaction operation could become inconsistent with the state of file-slices. When you run\n",(0,i.jsx)(_.code,{children:"compaction validate"}),", you can notice invalid compaction operations if there is one. In these cases, the repair\ncommand comes to the rescue, it will rearrange the file-slices so that there is no loss and the file-slices are\nconsistent with the compaction plan"]}),"\n",(0,i.jsx)(_.pre,{children:(0,i.jsx)(_.code,{className:"language-java",children:"hudi:stock_ticks_mor->compaction repair --instant 20181005222611\n......\nCompaction successfully repaired\n.....\n"})}),"\n",(0,i.jsx)(_.h3,{id:"savepoint-and-restore",children:"Savepoint and Restore"}),"\n",(0,i.jsxs)(_.p,{children:['As the name suggest, "savepoint" saves the table as of the commit time, so that it lets you restore the table to this\nsavepoint at a later point in time if need be. You can read more about savepoints and restore ',(0,i.jsx)(_.a,{href:"disaster_recovery",children:"here"})]}),"\n",(0,i.jsx)(_.p,{children:"To trigger savepoint for a hudi table"}),"\n",(0,i.jsx)(_.pre,{children:(0,i.jsx)(_.code,{className:"language-java",children:"connect --path /tmp/hudi_trips_cow/\ncommits show\nset --conf SPARK_HOME=\nsavepoint create --commit 20220128160245447 --sparkMaster local[2]\n"})}),"\n",(0,i.jsx)(_.p,{children:"To restore the table to one of the savepointed commit:"}),"\n",(0,i.jsx)(_.pre,{children:(0,i.jsx)(_.code,{className:"language-java",children:"connect --path /tmp/hudi_trips_cow/\ncommits show\nset --conf SPARK_HOME=\nsavepoints show\n\u2554\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2557\n\u2551 SavepointTime \u2551\n\u2560\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2563\n\u2551 20220128160245447 \u2551\n\u255a\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u255d\nsavepoint rollback --savepoint 20220128160245447 --sparkMaster local[2]\n"})}),"\n",(0,i.jsx)(_.h3,{id:"upgrade-and-downgrade-table",children:"Upgrade and Downgrade Table"}),"\n",(0,i.jsx)(_.p,{children:"In case the user needs to downgrade the version of Hudi library used, the Hudi table needs to be manually downgraded\non the newer version of Hudi CLI before library downgrade. To downgrade a Hudi table through CLI, user needs to specify\nthe target Hudi table version as follows:"}),"\n",(0,i.jsx)(_.pre,{children:(0,i.jsx)(_.code,{className:"language-shell",children:"connect --path \ndowngrade table --toVersion \n"})}),"\n",(0,i.jsx)(_.p,{children:"The following table shows the Hudi table versions corresponding to the Hudi release versions:"}),"\n",(0,i.jsxs)(_.table,{children:[(0,i.jsx)(_.thead,{children:(0,i.jsxs)(_.tr,{children:[(0,i.jsx)(_.th,{style:{textAlign:"left"},children:"Hudi Table Version"}),(0,i.jsx)(_.th,{style:{textAlign:"left"},children:"Hudi Release Version(s)"})]})}),(0,i.jsxs)(_.tbody,{children:[(0,i.jsxs)(_.tr,{children:[(0,i.jsxs)(_.td,{style:{textAlign:"left"},children:[(0,i.jsx)(_.code,{children:"FIVE"})," or ",(0,i.jsx)(_.code,{children:"5"})]}),(0,i.jsx)(_.td,{style:{textAlign:"left"},children:"0.12.x"})]}),(0,i.jsxs)(_.tr,{children:[(0,i.jsxs)(_.td,{style:{textAlign:"left"},children:[(0,i.jsx)(_.code,{children:"FOUR"})," or ",(0,i.jsx)(_.code,{children:"4"})]}),(0,i.jsx)(_.td,{style:{textAlign:"left"},children:"0.11.x"})]}),(0,i.jsxs)(_.tr,{children:[(0,i.jsxs)(_.td,{style:{textAlign:"left"},children:[(0,i.jsx)(_.code,{children:"THREE"})," or ",(0,i.jsx)(_.code,{children:"3"})]}),(0,i.jsx)(_.td,{style:{textAlign:"left"},children:"0.10.x"})]}),(0,i.jsxs)(_.tr,{children:[(0,i.jsxs)(_.td,{style:{textAlign:"left"},children:[(0,i.jsx)(_.code,{children:"TWO"})," or ",(0,i.jsx)(_.code,{children:"2"})]}),(0,i.jsx)(_.td,{style:{textAlign:"left"},children:"0.9.x"})]}),(0,i.jsxs)(_.tr,{children:[(0,i.jsxs)(_.td,{style:{textAlign:"left"},children:[(0,i.jsx)(_.code,{children:"ONE"})," or ",(0,i.jsx)(_.code,{children:"1"})]}),(0,i.jsx)(_.td,{style:{textAlign:"left"},children:"0.6.x - 0.8.x"})]}),(0,i.jsxs)(_.tr,{children:[(0,i.jsxs)(_.td,{style:{textAlign:"left"},children:[(0,i.jsx)(_.code,{children:"ZERO"})," or ",(0,i.jsx)(_.code,{children:"0"})]}),(0,i.jsx)(_.td,{style:{textAlign:"left"},children:"0.5.x and below"})]})]})]}),"\n",(0,i.jsxs)(_.p,{children:["For example, to downgrade a table from version ",(0,i.jsx)(_.code,{children:"FIVE"}),"(",(0,i.jsx)(_.code,{children:"5"}),") (current version) to ",(0,i.jsx)(_.code,{children:"TWO"}),"(",(0,i.jsx)(_.code,{children:"2"}),"), you should run (use proper Spark master based\non your environment)"]}),"\n",(0,i.jsx)(_.pre,{children:(0,i.jsx)(_.code,{className:"language-shell",children:"downgrade table --toVersion TWO --sparkMaster local[2]\n"})}),"\n",(0,i.jsx)(_.p,{children:"or"}),"\n",(0,i.jsx)(_.pre,{children:(0,i.jsx)(_.code,{className:"language-shell",children:"downgrade table --toVersion 2 --sparkMaster local[2]\n"})}),"\n",(0,i.jsxs)(_.p,{children:["You can verify the table version by looking at the ",(0,i.jsx)(_.code,{children:"hoodie.table.version"})," property in ",(0,i.jsx)(_.code,{children:".hoodie/hoodie.properties"})," under\nthe table path:"]}),"\n",(0,i.jsx)(_.pre,{children:(0,i.jsx)(_.code,{className:"language-properties",children:"hoodie.table.version=2\n"})}),"\n",(0,i.jsx)(_.p,{children:"Hudi CLI also provides the ability to manually upgrade a Hudi table. To upgrade a Hudi table through CLI:"}),"\n",(0,i.jsx)(_.pre,{children:(0,i.jsx)(_.code,{className:"language-shell",children:"upgrade table --toVersion \n"})}),"\n",(0,i.jsxs)(_.admonition,{type:"note",children:[(0,i.jsxs)(_.p,{children:["Table upgrade is automatically handled by the Hudi write client in different deployment modes such as DeltaStreamer\nafter upgrading the Hudi library so that the user does not have to do manual upgrade. Such automatic table upgrade\nis the ",(0,i.jsx)(_.strong,{children:"recommended"})," way in general, instead of using ",(0,i.jsx)(_.code,{children:"upgrade"})," CLI command."]}),(0,i.jsx)(_.p,{children:'Table upgrade from table version ONE to TWO requires key generator related configs such as\n"hoodie.datasource.write.recordkey.field", which is only available when user configures the write job. So the table\nupgrade from version ONE to TWO through CLI is not supported, and user should rely on the automatic upgrade in the write\nclient instead.'})]}),"\n",(0,i.jsx)(_.p,{children:"You may also run the upgrade command without specifying the target version. In such a case, the latest table version\ncorresponding to the library release version is used:"}),"\n",(0,i.jsx)(_.pre,{children:(0,i.jsx)(_.code,{className:"language-shell",children:"upgrade table\n"})}),"\n",(0,i.jsx)(_.h3,{id:"change-hudi-table-type",children:"Change Hudi Table Type"}),"\n",(0,i.jsx)(_.p,{children:"There are cases we want to change the hudi table type. For example, change COW table to MOR for more efficient and\nlower latency ingestion; change MOR to COW for better read performance and compatibility with downstream engines.\nSo we offer the table command to perform this modification conveniently."}),"\n",(0,i.jsxs)(_.p,{children:["Changing ",(0,i.jsx)(_.strong,{children:"COW to MOR"}),", we can simply modify the ",(0,i.jsx)(_.code,{children:"hoodie.table.type"})," in ",(0,i.jsx)(_.code,{children:"hoodie.properties"})," to MERGE_ON_READ."]}),"\n",(0,i.jsxs)(_.p,{children:["While changing ",(0,i.jsx)(_.strong,{children:"MOR to COW"}),", we must make sure all the log files are compacted before modifying the table type,\nor it will cause data loss."]}),"\n",(0,i.jsx)(_.pre,{children:(0,i.jsx)(_.code,{className:"language-shell",children:"connect --path \ntable change-table-type \n"})}),"\n",(0,i.jsxs)(_.p,{children:["The parameter ",(0,i.jsx)(_.code,{children:"target_table_type"})," candidates are below:"]}),"\n",(0,i.jsxs)(_.table,{children:[(0,i.jsx)(_.thead,{children:(0,i.jsxs)(_.tr,{children:[(0,i.jsx)(_.th,{style:{textAlign:"left"},children:"target table type"}),(0,i.jsx)(_.th,{style:{textAlign:"left"},children:"comment"})]})}),(0,i.jsxs)(_.tbody,{children:[(0,i.jsxs)(_.tr,{children:[(0,i.jsx)(_.td,{style:{textAlign:"left"},children:"MOR"}),(0,i.jsx)(_.td,{style:{textAlign:"left"},children:"Change COW table to MERGE_ON_READ."})]}),(0,i.jsxs)(_.tr,{children:[(0,i.jsx)(_.td,{style:{textAlign:"left"},children:"COW"}),(0,i.jsxs)(_.td,{style:{textAlign:"left"},children:["Change MOR table to COPY_ON_WRITE. ",(0,i.jsx)("br",{}),"By default, changing to COW will ",(0,i.jsx)(_.strong,{children:"execute all pending compactions"})," and ",(0,i.jsx)(_.strong,{children:"perform a full compaction"})," if any log file left. Set ",(0,i.jsx)(_.code,{children:"--enable-compaction=false"})," will disable the default compaction. ",(0,i.jsx)("br",{}),"There are params can be set for the compaction operation:",(0,i.jsx)("br",{}),(0,i.jsx)(_.code,{children:"--parallelism"}),": Default ",(0,i.jsx)(_.code,{children:"3"}),". Parallelism for hoodie compaction",(0,i.jsx)("br",{}),(0,i.jsx)(_.code,{children:"--sparkMaster"}),": Default ",(0,i.jsx)(_.code,{children:"local"}),". Spark Master",(0,i.jsx)("br",{}),(0,i.jsx)(_.code,{children:"--sparkMemory"}),": Default ",(0,i.jsx)(_.code,{children:"4G"}),". Spark executor memory",(0,i.jsx)("br",{}),(0,i.jsx)(_.code,{children:"--retry"}),": Default ",(0,i.jsx)(_.code,{children:"1"}),". Number of retries",(0,i.jsx)("br",{}),(0,i.jsx)(_.code,{children:"--propsFilePath"}),": Default ",(0,i.jsx)(_.code,{children:" "}),". path to properties file on localfs or dfs with configurations for hoodie client for compacting",(0,i.jsx)("br",{}),(0,i.jsx)(_.code,{children:"--hoodieConfigs"}),": Default ",(0,i.jsx)(_.code,{children:" "}),". Any configuration that can be set in the properties file can be passed here in the form of an array"]})]})]})]}),"\n",(0,i.jsx)(_.p,{children:"Example below is changing MOR table to COW:"}),"\n",(0,i.jsx)(_.pre,{children:(0,i.jsx)(_.code,{className:"language-shell",children:"connect --path /var/dataset/test_table_mor2cow\ndesc\n\u2554\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2564\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2557\n\u2551 Property \u2502 Value \u2551\n\u2560\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u256a\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2563\n\u2551 basePath \u2502 /var/dataset/test_table_mor2cow \u2551\n\u255f\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2562\n\u2551 metaPath \u2502 /var/dataset/test_table_mor2cow/.hoodie \u2551\n\u255f\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2562\n\u2551 fileSystem \u2502 file \u2551\n\u255f\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2562\n\u2551 hoodie.table.name \u2502 test_table \u2551\n\u255f\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2562\n\u2551 hoodie.compaction.record.merger.strategy \u2502 eeb8d96f-b1e4-49fd-bbf8-28ac514178e5 \u2551\n\u255f\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2562\n\u2551 hoodie.table.metadata.partitions \u2502 files \u2551\n\u255f\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2562\n\u2551 hoodie.table.type \u2502 MERGE_ON_READ \u2551\n\u255f\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2562\n\u2551 hoodie.table.metadata.partitions.inflight \u2502 \u2551\n\u255f\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2562\n\u2551 hoodie.archivelog.folder \u2502 archived \u2551\n\u255f\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2562\n\u2551 hoodie.timeline.layout.version \u2502 1 \u2551\n\u255f\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2562\n\u2551 hoodie.table.checksum \u2502 2702201862 \u2551\n\u255f\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2562\n\u2551 hoodie.compaction.payload.type \u2502 HOODIE_AVRO \u2551\n\u255f\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2562\n\u2551 hoodie.table.version \u2502 6 \u2551\n\u255f\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2562\n\u2551 hoodie.datasource.write.drop.partition.columns \u2502 false \u2551\n\u255a\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2567\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u255d\n\ntable change-table-type COW\n\u2554\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2564\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2564\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2557\n\u2551 Property \u2502 Old Value \u2502 New Value \u2551\n\u2560\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u256a\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u256a\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2563\n\u2551 hoodie.archivelog.folder \u2502 archived \u2502 archived \u2551\n\u255f\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2562\n\u2551 hoodie.compaction.payload.type \u2502 HOODIE_AVRO \u2502 HOODIE_AVRO \u2551\n\u255f\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2562\n\u2551 hoodie.compaction.record.merger.strategy \u2502 eeb8d96f-b1e4-49fd-bbf8-28ac514178e5 \u2502 eeb8d96f-b1e4-49fd-bbf8-28ac514178e5 \u2551\n\u255f\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2562\n\u2551 hoodie.datasource.write.drop.partition.columns \u2502 false \u2502 false \u2551\n\u255f\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2562\n\u2551 hoodie.table.checksum \u2502 2702201862 \u2502 2702201862 \u2551\n\u255f\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2562\n\u2551 hoodie.table.metadata.partitions \u2502 files \u2502 files \u2551\n\u255f\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2562\n\u2551 hoodie.table.metadata.partitions.inflight \u2502 \u2502 \u2551\n\u255f\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2562\n\u2551 hoodie.table.name \u2502 test_table \u2502 test_table \u2551\n\u255f\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2562\n\u2551 hoodie.table.type \u2502 MERGE_ON_READ \u2502 COPY_ON_WRITE \u2551\n\u255f\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2562\n\u2551 hoodie.table.version \u2502 6 \u2502 6 \u2551\n\u255f\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2562\n\u2551 hoodie.timeline.layout.version \u2502 1 \u2502 1 \u2551\n\u255a\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2567\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2567\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u255d\n"})})]})}function d(e={}){const{wrapper:_}={...(0,a.R)(),...e.components};return _?(0,i.jsx)(_,{...e,children:(0,i.jsx)(c,{...e})}):c(e)}},28453:(e,_,n)=>{n.d(_,{R:()=>o,x:()=>s});var t=n(96540);const i={},a=t.createContext(i);function o(e){const _=t.useContext(a);return t.useMemo((function(){return"function"==typeof e?e(_):{..._,...e}}),[_,e])}function s(e){let _;return _=e.disableParentContext?"function"==typeof e.components?e.components(i):e.components||i:o(e.components),t.createElement(a.Provider,{value:_},e.children)}}}]);
\ No newline at end of file
diff --git a/content/cn/assets/js/1a20bc57.4eb3f9ea.js b/content/cn/assets/js/1a20bc57.4eb3f9ea.js
new file mode 100644
index 0000000000000..b2a35e111a2c0
--- /dev/null
+++ b/content/cn/assets/js/1a20bc57.4eb3f9ea.js
@@ -0,0 +1 @@
+"use strict";(self.webpackChunkhudi=self.webpackChunkhudi||[]).push([[59114],{46768:(e,_,n)=>{n.r(_),n.d(_,{assets:()=>l,contentTitle:()=>s,default:()=>d,frontMatter:()=>o,metadata:()=>t,toc:()=>r});const t=JSON.parse('{"id":"cli","title":"CLI","description":"Local set up","source":"@site/docs/cli.md","sourceDirName":".","slug":"/cli","permalink":"/cn/docs/next/cli","draft":false,"unlisted":false,"editUrl":"https://github.com/apache/hudi/tree/asf-site/website/docs/cli.md","tags":[],"version":"current","frontMatter":{"title":"CLI","keywords":["hudi","cli"],"last_modified_at":"2021-08-18T19:59:57.000Z"},"sidebar":"docs","previous":{"title":"SQL Procedures","permalink":"/cn/docs/next/procedures"},"next":{"title":"Metrics","permalink":"/cn/docs/next/metrics"}}');var i=n(74848),a=n(28453);const o={title:"CLI",keywords:["hudi","cli"],last_modified_at:new Date("2021-08-18T19:59:57.000Z")},s=void 0,l={},r=[{value:"Local set up",id:"local-set-up",level:3},{value:"Hudi CLI Bundle setup",id:"hudi-cli-bundle-setup",level:3},{value:"Base path",id:"base-path",level:3},{value:"Using Hudi-cli in S3",id:"using-hudi-cli-in-s3",level:3},{value:"Note: These AWS jar versions below are specific to Spark 3.2.0",id:"note-these-aws-jar-versions-below-are-specific-to-spark-320",level:4},{value:"Using hudi-cli on Google Dataproc",id:"using-hudi-cli-on-google-dataproc",level:3},{value:"Connect to a Kerberized cluster",id:"connect-to-a-kerberized-cluster",level:2},{value:"Using hudi-cli",id:"using-hudi-cli",level:2},{value:"Inspecting Commits",id:"inspecting-commits",level:3},{value:"Drilling Down to a specific Commit",id:"drilling-down-to-a-specific-commit",level:3},{value:"FileSystem View",id:"filesystem-view",level:3},{value:"Statistics",id:"statistics",level:3},{value:"Archived Commits",id:"archived-commits",level:3},{value:"Compactions",id:"compactions",level:3},{value:"Validate Compaction",id:"validate-compaction",level:3},{value:"Unscheduling Compaction",id:"unscheduling-compaction",level:3},{value:"Repair Compaction",id:"repair-compaction",level:3},{value:"Savepoint and Restore",id:"savepoint-and-restore",level:3},{value:"Upgrade and Downgrade Table",id:"upgrade-and-downgrade-table",level:3},{value:"Change Hudi Table Type",id:"change-hudi-table-type",level:3},{value:"Related Resources",id:"related-resources",level:2}];function c(e){const _={a:"a",admonition:"admonition",br:"br",code:"code",h2:"h2",h3:"h3",h4:"h4",li:"li",ol:"ol",p:"p",pre:"pre",strong:"strong",table:"table",tbody:"tbody",td:"td",th:"th",thead:"thead",tr:"tr",ul:"ul",...(0,a.R)(),...e.components};return(0,i.jsxs)(i.Fragment,{children:[(0,i.jsx)(_.h3,{id:"local-set-up",children:"Local set up"}),"\n",(0,i.jsxs)(_.p,{children:["Once hudi has been built, the shell can be fired by via ",(0,i.jsx)(_.code,{children:"cd hudi-cli && ./hudi-cli.sh"}),"."]}),"\n",(0,i.jsx)(_.h3,{id:"hudi-cli-bundle-setup",children:"Hudi CLI Bundle setup"}),"\n",(0,i.jsxs)(_.p,{children:["In release ",(0,i.jsx)(_.code,{children:"0.13.0"})," we have now added another way of launching the ",(0,i.jsx)(_.code,{children:"hudi cli"}),", which is using the ",(0,i.jsx)(_.code,{children:"hudi-cli-bundle"}),"."]}),"\n",(0,i.jsxs)(_.p,{children:["There are a couple of requirements when using this approach such as having ",(0,i.jsx)(_.code,{children:"spark"})," installed locally on your machine.\nIt is required to use a spark distribution with hadoop dependencies packaged such as ",(0,i.jsx)(_.code,{children:"spark-3.3.1-bin-hadoop2.tgz"})," from ",(0,i.jsx)(_.a,{href:"https://archive.apache.org/dist/spark/",children:"https://archive.apache.org/dist/spark/"}),".\nWe also recommend you set an env variable ",(0,i.jsx)(_.code,{children:"$SPARK_HOME"})," to the path of where spark is installed on your machine.\nOne important thing to note is that the ",(0,i.jsx)(_.code,{children:"hudi-spark-bundle"})," should also be present when using the ",(0,i.jsx)(_.code,{children:"hudi-cli-bundle"}),".",(0,i.jsx)(_.br,{}),"\n","To provide the locations of these bundle jars you can set them in your shell like so:\n",(0,i.jsx)(_.code,{children:"export CLI_BUNDLE_JAR="})," , ",(0,i.jsx)(_.code,{children:"export SPARK_BUNDLE_JAR="}),"."]}),"\n",(0,i.jsx)(_.p,{children:"For steps see below if you are not compiling the project and downloading the jars:"}),"\n",(0,i.jsxs)(_.ol,{children:["\n",(0,i.jsx)(_.li,{children:"Create an empty folder as a new directory"}),"\n",(0,i.jsx)(_.li,{children:"Copy the hudi-cli-bundle jars and hudi-spark*-bundle jars to this directory"}),"\n",(0,i.jsx)(_.li,{children:"Copy the following script and folder to this directory"}),"\n"]}),"\n",(0,i.jsx)(_.pre,{children:(0,i.jsx)(_.code,{children:"packaging/hudi-cli-bundle/hudi-cli-with-bundle.sh\npackaging/hudi-cli-bundle/conf . the `conf` folder should be in this directory.\n"})}),"\n",(0,i.jsxs)(_.ol,{start:"4",children:["\n",(0,i.jsx)(_.li,{children:"Start Hudi CLI shell with environment variables set"}),"\n"]}),"\n",(0,i.jsx)(_.pre,{children:(0,i.jsx)(_.code,{children:"export SPARK_HOME=\nexport CLI_BUNDLE_JAR=\nexport SPARK_BUNDLE_JAR=\n\n./hudi-cli-with-bundle.sh\n\n"})}),"\n",(0,i.jsx)(_.h3,{id:"base-path",children:"Base path"}),"\n",(0,i.jsxs)(_.p,{children:["A hudi table resides on DFS, in a location referred to as the ",(0,i.jsx)(_.code,{children:"basePath"})," and\nwe would need this location in order to connect to a Hudi table. Hudi library effectively manages this table internally, using ",(0,i.jsx)(_.code,{children:".hoodie"})," subfolder to track all metadata."]}),"\n",(0,i.jsx)(_.h3,{id:"using-hudi-cli-in-s3",children:"Using Hudi-cli in S3"}),"\n",(0,i.jsxs)(_.p,{children:["If you are using hudi that comes packaged with AWS EMR, you can find instructions to use hudi-cli ",(0,i.jsx)(_.a,{href:"https://docs.aws.amazon.com/emr/latest/ReleaseGuide/emr-hudi-cli.html",children:"here"}),".\nIf you are not using EMR, or would like to use latest hudi-cli from master, you can follow the below steps to access S3 dataset in your local environment (laptop)."]}),"\n",(0,i.jsx)(_.p,{children:"Build Hudi with corresponding Spark version, for eg, -Dspark3.1.x"}),"\n",(0,i.jsx)(_.p,{children:"Set the following environment variables."}),"\n",(0,i.jsx)(_.pre,{children:(0,i.jsx)(_.code,{children:"export AWS_REGION=us-east-2\nexport AWS_ACCESS_KEY_ID=\nexport AWS_SECRET_ACCESS_KEY=\nexport SPARK_HOME=\n"})}),"\n",(0,i.jsx)(_.p,{children:"Ensure you set the SPARK_HOME to your local spark home compatible to compiled hudi spark version above."}),"\n",(0,i.jsx)(_.p,{children:"Apart from these, we might need to add aws jars to class path so that accessing S3 is feasible from local.\nWe need two jars, namely, aws-java-sdk-bundle jar and hadoop-aws jar which you can find online.\nFor eg:"}),"\n",(0,i.jsx)(_.pre,{children:(0,i.jsx)(_.code,{children:"wget https://repo1.maven.org/maven2/org/apache/hadoop/hadoop-aws/3.2.0/hadoop-aws-3.2.0.jar -o /lib/spark-3.2.0-bin-hadoop3.2/jars/hadoop-aws-3.2.0.jar\nwget https://repo1.maven.org/maven2/com/amazonaws/aws-java-sdk-bundle/1.11.375/aws-java-sdk-bundle-1.11.375.jar -o /lib/spark-3.2.0-bin-hadoop3.2/jars/aws-java-sdk-bundle-1.11.375.jar\n"})}),"\n",(0,i.jsx)(_.h4,{id:"note-these-aws-jar-versions-below-are-specific-to-spark-320",children:"Note: These AWS jar versions below are specific to Spark 3.2.0"}),"\n",(0,i.jsx)(_.pre,{children:(0,i.jsx)(_.code,{children:"export CLIENT_JAR=/lib/spark-3.2.0-bin-hadoop3.2/jars/aws-java-sdk-bundle-1.12.48.jar:/lib/spark-3.2.0-bin-hadoop3.2/jars/hadoop-aws-3.3.1.jar\n"})}),"\n",(0,i.jsx)(_.p,{children:"Once these are set, you are good to launch hudi-cli and access S3 dataset."}),"\n",(0,i.jsx)(_.pre,{children:(0,i.jsx)(_.code,{children:"./hudi-cli/hudi-cli.sh\n"})}),"\n",(0,i.jsx)(_.h3,{id:"using-hudi-cli-on-google-dataproc",children:"Using hudi-cli on Google Dataproc"}),"\n",(0,i.jsxs)(_.p,{children:[(0,i.jsx)(_.a,{href:"https://cloud.google.com/dataproc",children:"Dataproc"})," is Google's managed service for running Apache Hadoop, Apache Spark,\nApache Flink, Presto and many other frameworks, including Hudi. If you want to run the Hudi CLI on a Dataproc node\nwhich has not been launched with Hudi support enabled, you can use the steps below:"]}),"\n",(0,i.jsx)(_.p,{children:"These steps use Hudi version 0.13.0. If you want to use a different version you will have to edit the below commands\nappropriately:"}),"\n",(0,i.jsxs)(_.ol,{children:["\n",(0,i.jsx)(_.li,{children:"Once you've started the Dataproc cluster, you can ssh into it as follows:"}),"\n"]}),"\n",(0,i.jsx)(_.pre,{children:(0,i.jsx)(_.code,{children:'$ gcloud compute ssh --zone "YOUR_ZONE" "HOSTNAME_OF_MASTER_NODE" --project "YOUR_PROJECT"\n'})}),"\n",(0,i.jsxs)(_.ol,{start:"2",children:["\n",(0,i.jsx)(_.li,{children:"Download the Hudi CLI bundle"}),"\n"]}),"\n",(0,i.jsx)(_.pre,{children:(0,i.jsx)(_.code,{children:"wget https://repo1.maven.org/maven2/org/apache/hudi/hudi-cli-bundle_2.12/0.13.0/hudi-cli-bundle_2.12-0.13.0.jar \n"})}),"\n",(0,i.jsxs)(_.ol,{start:"3",children:["\n",(0,i.jsx)(_.li,{children:"Download the Hudi Spark bundle"}),"\n"]}),"\n",(0,i.jsx)(_.pre,{children:(0,i.jsx)(_.code,{children:"wget https://repo1.maven.org/maven2/org/apache/hudi/hudi-spark-bundle_2.12/0.13.0/hudi-spark-bundle_2.12-0.13.0.jar\n"})}),"\n",(0,i.jsxs)(_.ol,{start:"4",children:["\n",(0,i.jsx)(_.li,{children:"Download the shell script that launches Hudi CLI bundle"}),"\n"]}),"\n",(0,i.jsx)(_.pre,{children:(0,i.jsx)(_.code,{children:"wget https://raw.githubusercontent.com/apache/hudi/release-0.13.0/packaging/hudi-cli-bundle/hudi-cli-with-bundle.sh\n"})}),"\n",(0,i.jsxs)(_.ol,{start:"5",children:["\n",(0,i.jsx)(_.li,{children:"Launch Hudi CLI bundle with appropriate environment variables as follows:"}),"\n"]}),"\n",(0,i.jsx)(_.pre,{children:(0,i.jsx)(_.code,{children:"CLIENT_JAR=$DATAPROC_DIR/lib/gcs-connector.jar CLI_BUNDLE_JAR=hudi-cli-bundle_2.12-0.13.0.jar SPARK_BUNDLE_JAR=hudi-spark-bundle_2.12-0.13.0.jar ./hudi-cli-with-bundle.sh \n"})}),"\n",(0,i.jsxs)(_.ol,{start:"6",children:["\n",(0,i.jsxs)(_.li,{children:["\n",(0,i.jsxs)(_.p,{children:["hudi->connect --path gs://path_to_some_table",(0,i.jsx)(_.br,{}),"\n","Metadata for table some_table loaded"]}),"\n"]}),"\n",(0,i.jsxs)(_.li,{children:["\n",(0,i.jsxs)(_.p,{children:["hudi:some_table->commits show --limit 5",(0,i.jsx)(_.br,{}),"\n","This command should show the recent commits, if the above steps work correctly."]}),"\n"]}),"\n"]}),"\n",(0,i.jsx)(_.h2,{id:"connect-to-a-kerberized-cluster",children:"Connect to a Kerberized cluster"}),"\n",(0,i.jsxs)(_.p,{children:["Before connecting to a Kerberized cluster, you can use ",(0,i.jsx)(_.strong,{children:"kerberos kinit"})," command. Following is the usage of this command."]}),"\n",(0,i.jsx)(_.pre,{children:(0,i.jsx)(_.code,{className:"language-shell",children:"hudi->help kerberos kinit\nNAME\n kerberos kinit - Perform Kerberos authentication\n\nSYNOPSIS\n kerberos kinit --krb5conf String [--principal String] [--keytab String]\n\nOPTIONS\n --krb5conf String\n Path to krb5.conf\n [Optional, default = /etc/krb5.conf]\n\n --principal String\n Kerberos principal\n [Mandatory]\n\n --keytab String\n Path to keytab\n [Mandatory]\n"})}),"\n",(0,i.jsx)(_.p,{children:"For example:"}),"\n",(0,i.jsx)(_.pre,{children:(0,i.jsx)(_.code,{className:"language-shell",children:"hudi->kerberos kinit --principal user/host@DOMAIN --keytab /etc/security/keytabs/user.keytab\nPerform Kerberos authentication\nParameters:\n--krb5conf: /etc/krb5.conf\n--principal: user/host@DOMAIN\n--keytab: /etc/security/keytabs/user.keytab\nKerberos current user: user/host@DOMAIN (auth:KERBEROS)\nKerberos login user: user/host@DOMAIN (auth:KERBEROS)\nKerberos authentication success\n"})}),"\n",(0,i.jsx)(_.p,{children:'If you see "Kerberos authentication success" in the command output, it means Kerberos authentication has been successful.'}),"\n",(0,i.jsx)(_.h2,{id:"using-hudi-cli",children:"Using hudi-cli"}),"\n",(0,i.jsx)(_.p,{children:"To initialize a hudi table, use the following command."}),"\n",(0,i.jsx)(_.pre,{children:(0,i.jsx)(_.code,{className:"language-java",children:"===================================================================\n* ___ ___ *\n* /\\__\\ ___ /\\ \\ ___ *\n* / / / /\\__\\ / \\ \\ /\\ \\ *\n* / /__/ / / / / /\\ \\ \\ \\ \\ \\ *\n* / \\ \\ ___ / / / / / \\ \\__\\ / \\__\\ *\n* / /\\ \\ /\\__\\ / /__/ ___ / /__/ \\ |__| / /\\/__/ *\n* \\/ \\ \\/ / / \\ \\ \\ /\\__\\ \\ \\ \\ / / / /\\/ / / *\n* \\ / / \\ \\ / / / \\ \\ / / / \\ /__/ *\n* / / / \\ \\/ / / \\ \\/ / / \\ \\__\\ *\n* / / / \\ / / \\ / / \\/__/ *\n* \\/__/ \\/__/ \\/__/ Apache Hudi CLI *\n* *\n===================================================================\n\nhudi->create --path /user/hive/warehouse/table1 --tableName hoodie_table_1 --tableType COPY_ON_WRITE\n.....\n"})}),"\n",(0,i.jsx)(_.p,{children:"To see the description of hudi table, use the command:"}),"\n",(0,i.jsx)(_.pre,{children:(0,i.jsx)(_.code,{className:"language-java",children:"hudi:hoodie_table_1->desc\n18/09/06 15:57:19 INFO timeline.HoodieActiveTimeline: Loaded instants []\n _________________________________________________________\n | Property | Value |\n |========================================================|\n | basePath | ... |\n | metaPath | ... |\n | fileSystem | hdfs |\n | hoodie.table.name | hoodie_table_1 |\n | hoodie.table.type | COPY_ON_WRITE |\n | hoodie.archivelog.folder| |\n"})}),"\n",(0,i.jsx)(_.p,{children:"Following is a sample command to connect to a Hudi table contains uber trips."}),"\n",(0,i.jsx)(_.pre,{children:(0,i.jsx)(_.code,{className:"language-java",children:"hudi:trips->connect --path /app/uber/trips\n\n16/10/05 23:20:37 INFO model.HoodieTableMetadata: All commits :HoodieCommits{commitList=[20161002045850, 20161002052915, 20161002055918, 20161002065317, 20161002075932, 20161002082904, 20161002085949, 20161002092936, 20161002105903, 20161002112938, 20161002123005, 20161002133002, 20161002155940, 20161002165924, 20161002172907, 20161002175905, 20161002190016, 20161002192954, 20161002195925, 20161002205935, 20161002215928, 20161002222938, 20161002225915, 20161002232906, 20161003003028, 20161003005958, 20161003012936, 20161003022924, 20161003025859, 20161003032854, 20161003042930, 20161003052911, 20161003055907, 20161003062946, 20161003065927, 20161003075924, 20161003082926, 20161003085925, 20161003092909, 20161003100010, 20161003102913, 20161003105850, 20161003112910, 20161003115851, 20161003122929, 20161003132931, 20161003142952, 20161003145856, 20161003152953, 20161003155912, 20161003162922, 20161003165852, 20161003172923, 20161003175923, 20161003195931, 20161003210118, 20161003212919, 20161003215928, 20161003223000, 20161003225858, 20161004003042, 20161004011345, 20161004015235, 20161004022234, 20161004063001, 20161004072402, 20161004074436, 20161004080224, 20161004082928, 20161004085857, 20161004105922, 20161004122927, 20161004142929, 20161004163026, 20161004175925, 20161004194411, 20161004203202, 20161004211210, 20161004214115, 20161004220437, 20161004223020, 20161004225321, 20161004231431, 20161004233643, 20161005010227, 20161005015927, 20161005022911, 20161005032958, 20161005035939, 20161005052904, 20161005070028, 20161005074429, 20161005081318, 20161005083455, 20161005085921, 20161005092901, 20161005095936, 20161005120158, 20161005123418, 20161005125911, 20161005133107, 20161005155908, 20161005163517, 20161005165855, 20161005180127, 20161005184226, 20161005191051, 20161005193234, 20161005203112, 20161005205920, 20161005212949, 20161005223034, 20161005225920]}\nMetadata for table trips loaded\n"})}),"\n",(0,i.jsx)(_.p,{children:"Once connected to the table, a lot of other commands become available. The shell has contextual autocomplete help (press TAB) and below is a list of all commands, few of which are reviewed in this section"}),"\n",(0,i.jsx)(_.pre,{children:(0,i.jsx)(_.code,{className:"language-shell",children:"hudi:trips->help\n* ! - Allows execution of operating system (OS) commands\n* // - Inline comment markers (start of line only)\n* ; - Inline comment markers (start of line only)\n* bootstrap index showmapping - Show bootstrap index mapping\n* bootstrap index showpartitions - Show bootstrap indexed partitions\n* bootstrap run - Run a bootstrap action for current Hudi table\n* clean showpartitions - Show partition level details of a clean\n* cleans refresh - Refresh table metadata\n* cleans run - run clean\n* cleans show - Show the cleans\n* clear - Clears the console\n* cls - Clears the console\n* clustering run - Run Clustering\n* clustering schedule - Schedule Clustering\n* clustering scheduleAndExecute - Run Clustering. Make a cluster plan first and execute that plan immediately\n* commit rollback - Rollback a commit\n* commits compare - Compare commits with another Hoodie table\n* commit show_write_stats - Show write stats of a commit\n* commit showfiles - Show file level details of a commit\n* commit showpartitions - Show partition level details of a commit\n* commits refresh - Refresh table metadata\n* commits show - Show the commits\n* commits showarchived - Show the archived commits\n* commits sync - Sync commits with another Hoodie table\n* compaction repair - Renames the files to make them consistent with the timeline as dictated by Hoodie metadata. Use when compaction unschedule fails partially.\n* compaction run - Run Compaction for given instant time\n* compaction schedule - Schedule Compaction\n* compaction scheduleAndExecute - Schedule compaction plan and execute this plan\n* compaction show - Shows compaction details for a specific compaction instant\n* compaction showarchived - Shows compaction details for a specific compaction instant\n* compactions show all - Shows all compactions that are in active timeline\n* compactions showarchived - Shows compaction details for specified time window\n* compaction unschedule - Unschedule Compaction\n* compaction unscheduleFileId - UnSchedule Compaction for a fileId\n* compaction validate - Validate Compaction\n* connect - Connect to a hoodie table\n* create - Create a hoodie table if not present\n* date - Displays the local date and time\n* desc - Describe Hoodie Table properties\n* downgrade table - Downgrades a table\n* exit - Exits the shell\n* export instants - Export Instants and their metadata from the Timeline\n* fetch table schema - Fetches latest table schema\n* hdfsparquetimport - Imports Parquet table to a hoodie table\n* help - List all commands usage\n* marker delete - Delete the marker\n* metadata create - Create the Metadata Table if it does not exist\n* metadata delete - Remove the Metadata Table\n* metadata init - Update the metadata table from commits since the creation\n* metadata list-files - Print a list of all files in a partition from the metadata\n* metadata list-partitions - List all partitions from metadata\n* metadata refresh - Refresh table metadata\n* metadata set - Set options for Metadata Table\n* metadata stats - Print stats about the metadata\n* metadata validate-files - Validate all files in all partitions from the metadata\n* quit - Exits the shell\n* refresh - Refresh table metadata\n* repair addpartitionmeta - Add partition metadata to a table, if not present\n* repair corrupted clean files - repair corrupted clean files\n* repair deduplicate - De-duplicate a partition path contains duplicates & produce repaired files to replace with\n* repair migrate-partition-meta - Migrate all partition meta file currently stored in text format to be stored in base file format. See HoodieTableConfig#PARTITION_METAFILE_USE_DATA_FORMAT.\n* repair overwrite-hoodie-props - Overwrite hoodie.properties with provided file. Risky operation. Proceed with caution!\n* savepoint create - Savepoint a commit\n* savepoint delete - Delete the savepoint\n* savepoint rollback - Savepoint a commit\n* savepoints refresh - Refresh table metadata\n* savepoints show - Show the savepoints\n* script - Parses the specified resource file and executes its commands\n* set - Set spark launcher env to cli\n* show archived commits - Read commits from archived files and show details\n* show archived commit stats - Read commits from archived files and show details\n* show env - Show spark launcher env by key\n* show envs all - Show spark launcher envs\n* show fsview all - Show entire file-system view\n* show fsview latest - Show latest file-system view\n* show logfile metadata - Read commit metadata from log files\n* show logfile records - Read records from log files\n* show rollback - Show details of a rollback instant\n* show rollbacks - List all rollback instants\n* stats filesizes - File Sizes. Display summary stats on sizes of files\n* stats wa - Write Amplification. Ratio of how many records were upserted to how many records were actually written\n* sync validate - Validate the sync by counting the number of records\n* system properties - Shows the shell's properties\n* table delete-configs - Delete the supplied table configs from the table.\n* table recover-configs - Recover table configs, from update/delete that failed midway.\n* table update-configs - Update the table configs with configs with provided file.\n* temp_delete - Delete view name\n* temp_query - query against created temp view\n* temp delete - Delete view name\n* temp query - query against created temp view\n* temps_show - Show all views name\n* temps show - Show all views name\n* upgrade table - Upgrades a table\n* utils loadClass - Load a class\n* version - Displays shell version\n\nhudi:trips->\n"})}),"\n",(0,i.jsx)(_.h3,{id:"inspecting-commits",children:"Inspecting Commits"}),"\n",(0,i.jsxs)(_.p,{children:["The task of upserting or inserting a batch of incoming records is known as a ",(0,i.jsx)(_.strong,{children:"commit"})," in Hudi. A commit provides basic atomicity guarantees such that only committed data is available for querying.\nEach commit has a monotonically increasing string/number called the ",(0,i.jsx)(_.strong,{children:"commit number"}),". Typically, this is the time at which we started the commit."]}),"\n",(0,i.jsx)(_.p,{children:"To view some basic information about the last 10 commits,"}),"\n",(0,i.jsx)(_.pre,{children:(0,i.jsx)(_.code,{className:"language-java",children:'hudi:trips->commits show --sortBy "Total Bytes Written" --desc true --limit 10\n ________________________________________________________________________________________________________________________________________________________________________\n | CommitTime | Total Bytes Written| Total Files Added| Total Files Updated| Total Partitions Written| Total Records Written| Total Update Records Written| Total Errors|\n |=======================================================================================================================================================================|\n ....\n ....\n ....\n'})}),"\n",(0,i.jsx)(_.p,{children:"At the start of each write, Hudi also writes a .inflight commit to the .hoodie folder. You can use the timestamp there to estimate how long the commit has been inflight"}),"\n",(0,i.jsx)(_.pre,{children:(0,i.jsx)(_.code,{className:"language-java",children:"$ hdfs dfs -ls /app/uber/trips/.hoodie/*.inflight\n-rw-r--r-- 3 vinoth supergroup 321984 2016-10-05 23:18 /app/uber/trips/.hoodie/20161005225920.inflight\n"})}),"\n",(0,i.jsx)(_.h3,{id:"drilling-down-to-a-specific-commit",children:"Drilling Down to a specific Commit"}),"\n",(0,i.jsx)(_.p,{children:"To understand how the writes spread across specific partiions,"}),"\n",(0,i.jsx)(_.pre,{children:(0,i.jsx)(_.code,{className:"language-java",children:'hudi:trips->commit showpartitions --commit 20161005165855 --sortBy "Total Bytes Written" --desc true --limit 10\n __________________________________________________________________________________________________________________________________________\n | Partition Path| Total Files Added| Total Files Updated| Total Records Inserted| Total Records Updated| Total Bytes Written| Total Errors|\n |=========================================================================================================================================|\n ....\n ....\n'})}),"\n",(0,i.jsx)(_.p,{children:"If you need file level granularity , we can do the following"}),"\n",(0,i.jsx)(_.pre,{children:(0,i.jsx)(_.code,{className:"language-java",children:'hudi:trips->commit showfiles --commit 20161005165855 --sortBy "Partition Path"\n ________________________________________________________________________________________________________________________________________________________\n | Partition Path| File ID | Previous Commit| Total Records Updated| Total Records Written| Total Bytes Written| Total Errors|\n |=======================================================================================================================================================|\n ....\n ....\n'})}),"\n",(0,i.jsx)(_.h3,{id:"filesystem-view",children:"FileSystem View"}),"\n",(0,i.jsx)(_.p,{children:"Hudi views each partition as a collection of file-groups with each file-group containing a list of file-slices in commit order (See concepts).\nThe below commands allow users to view the file-slices for a data-set."}),"\n",(0,i.jsx)(_.pre,{children:(0,i.jsx)(_.code,{className:"language-java",children:'hudi:stock_ticks_mor->show fsview all\n ....\n _______________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________\n | Partition | FileId | Base-Instant | Data-File | Data-File Size| Num Delta Files| Total Delta File Size| Delta Files |\n |==============================================================================================================================================================================================================================================================================================================================================================================================================|\n | 2018/08/31| 111415c3-f26d-4639-86c8-f9956f245ac3| 20181002180759| hdfs://namenode:8020/user/hive/warehouse/stock_ticks_mor/2018/08/31/111415c3-f26d-4639-86c8-f9956f245ac3_0_20181002180759.parquet| 432.5 KB | 1 | 20.8 KB | [HoodieLogFile {hdfs://namenode:8020/user/hive/warehouse/stock_ticks_mor/2018/08/31/.111415c3-f26d-4639-86c8-f9956f245ac3_20181002180759.log.1}]|\n\n\n\nhudi:stock_ticks_mor->show fsview latest --partitionPath "2018/08/31"\n ......\n __________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________\n | Partition | FileId | Base-Instant | Data-File | Data-File Size| Num Delta Files| Total Delta Size| Delta Size - compaction scheduled| Delta Size - compaction unscheduled| Delta To Base Ratio - compaction scheduled| Delta To Base Ratio - compaction unscheduled| Delta Files - compaction scheduled | Delta Files - compaction unscheduled|\n |=================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================|\n | 2018/08/31| 111415c3-f26d-4639-86c8-f9956f245ac3| 20181002180759| hdfs://namenode:8020/user/hive/warehouse/stock_ticks_mor/2018/08/31/111415c3-f26d-4639-86c8-f9956f245ac3_0_20181002180759.parquet| 432.5 KB | 1 | 20.8 KB | 20.8 KB | 0.0 B | 0.0 B | 0.0 B | [HoodieLogFile {hdfs://namenode:8020/user/hive/warehouse/stock_ticks_mor/2018/08/31/.111415c3-f26d-4639-86c8-f9956f245ac3_20181002180759.log.1}]| [] |\n\n'})}),"\n",(0,i.jsx)(_.h3,{id:"statistics",children:"Statistics"}),"\n",(0,i.jsx)(_.p,{children:"Since Hudi directly manages file sizes for DFS table, it might be good to get an overall picture"}),"\n",(0,i.jsx)(_.pre,{children:(0,i.jsx)(_.code,{className:"language-java",children:'hudi:trips->stats filesizes --partitionPath 2016/09/01 --sortBy "95th" --desc true --limit 10\n ________________________________________________________________________________________________\n | CommitTime | Min | 10th | 50th | avg | 95th | Max | NumFiles| StdDev |\n |===============================================================================================|\n | | 93.9 MB | 93.9 MB | 93.9 MB | 93.9 MB | 93.9 MB | 93.9 MB | 2 | 2.3 KB |\n ....\n ....\n'})}),"\n",(0,i.jsx)(_.p,{children:"In case of Hudi write taking much longer, it might be good to see the write amplification for any sudden increases"}),"\n",(0,i.jsx)(_.pre,{children:(0,i.jsx)(_.code,{className:"language-java",children:"hudi:trips->stats wa\n __________________________________________________________________________\n | CommitTime | Total Upserted| Total Written| Write Amplifiation Factor|\n |=========================================================================|\n ....\n ....\n"})}),"\n",(0,i.jsx)(_.h3,{id:"archived-commits",children:"Archived Commits"}),"\n",(0,i.jsx)(_.p,{children:"In order to limit the amount of growth of .commit files on DFS, Hudi archives older .commit files (with due respect to the cleaner policy) into a commits.archived file.\nThis is a sequence file that contains a mapping from commitNumber => json with raw information about the commit (same that is nicely rolled up above)."}),"\n",(0,i.jsx)(_.h3,{id:"compactions",children:"Compactions"}),"\n",(0,i.jsx)(_.p,{children:"To get an idea of the lag between compaction and writer applications, use the below command to list down all\npending compactions."}),"\n",(0,i.jsx)(_.pre,{children:(0,i.jsx)(_.code,{className:"language-java",children:"hudi:trips->compactions show all\n ___________________________________________________________________\n | Compaction Instant Time| State | Total FileIds to be Compacted|\n |==================================================================|\n | | REQUESTED| 35 |\n | | INFLIGHT | 27 |\n"})}),"\n",(0,i.jsx)(_.p,{children:"To inspect a specific compaction plan, use"}),"\n",(0,i.jsx)(_.pre,{children:(0,i.jsx)(_.code,{className:"language-java",children:"hudi:trips->compaction show --instant \n _________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________\n | Partition Path| File Id | Base Instant | Data File Path | Total Delta Files| getMetrics |\n |================================================================================================================================================================================================================================================\n | 2018/07/17 | | | viewfs://ns-default/.../../UUID_.parquet | 1 | {TOTAL_LOG_FILES=1.0, TOTAL_IO_READ_MB=1230.0, TOTAL_LOG_FILES_SIZE=2.51255751E8, TOTAL_IO_WRITE_MB=991.0, TOTAL_IO_MB=2221.0}|\n\n"})}),"\n",(0,i.jsx)(_.p,{children:"To manually schedule or run a compaction, use the below command. This command uses spark launcher to perform compaction\noperations."}),"\n",(0,i.jsxs)(_.p,{children:[(0,i.jsx)(_.strong,{children:"NOTE:"})," Make sure no other application is scheduling compaction for this table concurrently\n{: .notice--info}"]}),"\n",(0,i.jsx)(_.pre,{children:(0,i.jsx)(_.code,{className:"language-java",children:"hudi:trips->help compaction schedule\nKeyword: compaction schedule\nDescription: Schedule Compaction\n Keyword: sparkMemory\n Help: Spark executor memory\n Mandatory: false\n Default if specified: '__NULL__'\n Default if unspecified: '1G'\n\n* compaction schedule - Schedule Compaction\n"})}),"\n",(0,i.jsx)(_.pre,{children:(0,i.jsx)(_.code,{className:"language-java",children:"hudi:trips->help compaction run\nKeyword: compaction run\nDescription: Run Compaction for given instant time\n Keyword: tableName\n Help: Table name\n Mandatory: true\n Default if specified: '__NULL__'\n Default if unspecified: '__NULL__'\n\n Keyword: parallelism\n Help: Parallelism for hoodie compaction\n Mandatory: true\n Default if specified: '__NULL__'\n Default if unspecified: '__NULL__'\n\n Keyword: schemaFilePath\n Help: Path for Avro schema file\n Mandatory: true\n Default if specified: '__NULL__'\n Default if unspecified: '__NULL__'\n\n Keyword: sparkMemory\n Help: Spark executor memory\n Mandatory: true\n Default if specified: '__NULL__'\n Default if unspecified: '__NULL__'\n\n Keyword: retry\n Help: Number of retries\n Mandatory: true\n Default if specified: '__NULL__'\n Default if unspecified: '__NULL__'\n\n Keyword: compactionInstant\n Help: Base path for the target hoodie table\n Mandatory: true\n Default if specified: '__NULL__'\n Default if unspecified: '__NULL__'\n\n* compaction run - Run Compaction for given instant time\n"})}),"\n",(0,i.jsx)(_.h3,{id:"validate-compaction",children:"Validate Compaction"}),"\n",(0,i.jsx)(_.p,{children:"Validating a compaction plan : Check if all the files necessary for compactions are present and are valid"}),"\n",(0,i.jsx)(_.pre,{children:(0,i.jsx)(_.code,{className:"language-java",children:"hudi:stock_ticks_mor->compaction validate --instant 20181005222611\n...\n\n COMPACTION PLAN VALID\n\n ___________________________________________________________________________________________________________________________________________________________________________________________________________________________\n | File Id | Base Instant Time| Base Data File | Num Delta Files| Valid| Error|\n |==========================================================================================================================================================================================================================|\n | 05320e98-9a57-4c38-b809-a6beaaeb36bd| 20181005222445 | hdfs://namenode:8020/user/hive/warehouse/stock_ticks_mor/2018/08/31/05320e98-9a57-4c38-b809-a6beaaeb36bd_0_20181005222445.parquet| 1 | true | |\n\n\n\nhudi:stock_ticks_mor->compaction validate --instant 20181005222601\n\n COMPACTION PLAN INVALID\n\n _______________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________\n | File Id | Base Instant Time| Base Data File | Num Delta Files| Valid| Error |\n |=====================================================================================================================================================================================================================================================================================================|\n | 05320e98-9a57-4c38-b809-a6beaaeb36bd| 20181005222445 | hdfs://namenode:8020/user/hive/warehouse/stock_ticks_mor/2018/08/31/05320e98-9a57-4c38-b809-a6beaaeb36bd_0_20181005222445.parquet| 1 | false| All log files specified in compaction operation is not present. Missing .... |\n"})}),"\n",(0,i.jsxs)(_.p,{children:[(0,i.jsx)(_.strong,{children:"NOTE:"})," The following commands must be executed without any other writer/ingestion application running.\n{: .notice--warning}"]}),"\n",(0,i.jsx)(_.p,{children:"Sometimes, it becomes necessary to remove a fileId from a compaction-plan inorder to speed-up or unblock compaction\noperation. Any new log-files that happened on this file after the compaction got scheduled will be safely renamed\nso that are preserved. Hudi provides the following CLI to support it"}),"\n",(0,i.jsx)(_.h3,{id:"unscheduling-compaction",children:"Unscheduling Compaction"}),"\n",(0,i.jsx)(_.pre,{children:(0,i.jsx)(_.code,{className:"language-java",children:"hudi:trips->compaction unscheduleFileId --fileId \n....\nNo File renames needed to unschedule file from pending compaction. Operation successful.\n"})}),"\n",(0,i.jsx)(_.p,{children:"In other cases, an entire compaction plan needs to be reverted. This is supported by the following CLI"}),"\n",(0,i.jsx)(_.pre,{children:(0,i.jsx)(_.code,{className:"language-java",children:"hudi:trips->compaction unschedule --instant \n.....\nNo File renames needed to unschedule pending compaction. Operation successful.\n"})}),"\n",(0,i.jsx)(_.h3,{id:"repair-compaction",children:"Repair Compaction"}),"\n",(0,i.jsxs)(_.p,{children:["The above compaction unscheduling operations could sometimes fail partially (e",":g"," -> DFS temporarily unavailable). With\npartial failures, the compaction operation could become inconsistent with the state of file-slices. When you run\n",(0,i.jsx)(_.code,{children:"compaction validate"}),", you can notice invalid compaction operations if there is one. In these cases, the repair\ncommand comes to the rescue, it will rearrange the file-slices so that there is no loss and the file-slices are\nconsistent with the compaction plan"]}),"\n",(0,i.jsx)(_.pre,{children:(0,i.jsx)(_.code,{className:"language-java",children:"hudi:stock_ticks_mor->compaction repair --instant 20181005222611\n......\nCompaction successfully repaired\n.....\n"})}),"\n",(0,i.jsx)(_.h3,{id:"savepoint-and-restore",children:"Savepoint and Restore"}),"\n",(0,i.jsxs)(_.p,{children:['As the name suggest, "savepoint" saves the table as of the commit time, so that it lets you restore the table to this\nsavepoint at a later point in time if need be. You can read more about savepoints and restore ',(0,i.jsx)(_.a,{href:"disaster_recovery",children:"here"})]}),"\n",(0,i.jsx)(_.p,{children:"To trigger savepoint for a hudi table"}),"\n",(0,i.jsx)(_.pre,{children:(0,i.jsx)(_.code,{className:"language-java",children:"connect --path /tmp/hudi_trips_cow/\ncommits show\nset --conf SPARK_HOME=\nsavepoint create --commit 20220128160245447 --sparkMaster local[2]\n"})}),"\n",(0,i.jsx)(_.p,{children:"To restore the table to one of the savepointed commit:"}),"\n",(0,i.jsx)(_.pre,{children:(0,i.jsx)(_.code,{className:"language-java",children:"connect --path /tmp/hudi_trips_cow/\ncommits show\nset --conf SPARK_HOME=\nsavepoints show\n\u2554\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2557\n\u2551 SavepointTime \u2551\n\u2560\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2563\n\u2551 20220128160245447 \u2551\n\u255a\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u255d\nsavepoint rollback --savepoint 20220128160245447 --sparkMaster local[2]\n"})}),"\n",(0,i.jsx)(_.h3,{id:"upgrade-and-downgrade-table",children:"Upgrade and Downgrade Table"}),"\n",(0,i.jsx)(_.p,{children:"In case the user needs to downgrade the version of Hudi library used, the Hudi table needs to be manually downgraded\non the newer version of Hudi CLI before library downgrade. To downgrade a Hudi table through CLI, user needs to specify\nthe target Hudi table version as follows:"}),"\n",(0,i.jsx)(_.pre,{children:(0,i.jsx)(_.code,{className:"language-shell",children:"connect --path \ndowngrade table --toVersion \n"})}),"\n",(0,i.jsx)(_.p,{children:"The following table shows the Hudi table versions corresponding to the Hudi release versions:"}),"\n",(0,i.jsxs)(_.table,{children:[(0,i.jsx)(_.thead,{children:(0,i.jsxs)(_.tr,{children:[(0,i.jsx)(_.th,{style:{textAlign:"left"},children:"Hudi Table Version"}),(0,i.jsx)(_.th,{style:{textAlign:"left"},children:"Hudi Release Version(s)"})]})}),(0,i.jsxs)(_.tbody,{children:[(0,i.jsxs)(_.tr,{children:[(0,i.jsxs)(_.td,{style:{textAlign:"left"},children:[(0,i.jsx)(_.code,{children:"FIVE"})," or ",(0,i.jsx)(_.code,{children:"5"})]}),(0,i.jsx)(_.td,{style:{textAlign:"left"},children:"0.12.x"})]}),(0,i.jsxs)(_.tr,{children:[(0,i.jsxs)(_.td,{style:{textAlign:"left"},children:[(0,i.jsx)(_.code,{children:"FOUR"})," or ",(0,i.jsx)(_.code,{children:"4"})]}),(0,i.jsx)(_.td,{style:{textAlign:"left"},children:"0.11.x"})]}),(0,i.jsxs)(_.tr,{children:[(0,i.jsxs)(_.td,{style:{textAlign:"left"},children:[(0,i.jsx)(_.code,{children:"THREE"})," or ",(0,i.jsx)(_.code,{children:"3"})]}),(0,i.jsx)(_.td,{style:{textAlign:"left"},children:"0.10.x"})]}),(0,i.jsxs)(_.tr,{children:[(0,i.jsxs)(_.td,{style:{textAlign:"left"},children:[(0,i.jsx)(_.code,{children:"TWO"})," or ",(0,i.jsx)(_.code,{children:"2"})]}),(0,i.jsx)(_.td,{style:{textAlign:"left"},children:"0.9.x"})]}),(0,i.jsxs)(_.tr,{children:[(0,i.jsxs)(_.td,{style:{textAlign:"left"},children:[(0,i.jsx)(_.code,{children:"ONE"})," or ",(0,i.jsx)(_.code,{children:"1"})]}),(0,i.jsx)(_.td,{style:{textAlign:"left"},children:"0.6.x - 0.8.x"})]}),(0,i.jsxs)(_.tr,{children:[(0,i.jsxs)(_.td,{style:{textAlign:"left"},children:[(0,i.jsx)(_.code,{children:"ZERO"})," or ",(0,i.jsx)(_.code,{children:"0"})]}),(0,i.jsx)(_.td,{style:{textAlign:"left"},children:"0.5.x and below"})]})]})]}),"\n",(0,i.jsxs)(_.p,{children:["For example, to downgrade a table from version ",(0,i.jsx)(_.code,{children:"FIVE"}),"(",(0,i.jsx)(_.code,{children:"5"}),") (current version) to ",(0,i.jsx)(_.code,{children:"TWO"}),"(",(0,i.jsx)(_.code,{children:"2"}),"), you should run (use proper Spark master based\non your environment)"]}),"\n",(0,i.jsx)(_.pre,{children:(0,i.jsx)(_.code,{className:"language-shell",children:"downgrade table --toVersion TWO --sparkMaster local[2]\n"})}),"\n",(0,i.jsx)(_.p,{children:"or"}),"\n",(0,i.jsx)(_.pre,{children:(0,i.jsx)(_.code,{className:"language-shell",children:"downgrade table --toVersion 2 --sparkMaster local[2]\n"})}),"\n",(0,i.jsxs)(_.p,{children:["You can verify the table version by looking at the ",(0,i.jsx)(_.code,{children:"hoodie.table.version"})," property in ",(0,i.jsx)(_.code,{children:".hoodie/hoodie.properties"})," under\nthe table path:"]}),"\n",(0,i.jsx)(_.pre,{children:(0,i.jsx)(_.code,{className:"language-properties",children:"hoodie.table.version=2\n"})}),"\n",(0,i.jsx)(_.p,{children:"Hudi CLI also provides the ability to manually upgrade a Hudi table. To upgrade a Hudi table through CLI:"}),"\n",(0,i.jsx)(_.pre,{children:(0,i.jsx)(_.code,{className:"language-shell",children:"upgrade table --toVersion \n"})}),"\n",(0,i.jsxs)(_.admonition,{type:"note",children:[(0,i.jsxs)(_.p,{children:["Table upgrade is automatically handled by the Hudi write client in different deployment modes such as DeltaStreamer\nafter upgrading the Hudi library so that the user does not have to do manual upgrade. Such automatic table upgrade\nis the ",(0,i.jsx)(_.strong,{children:"recommended"})," way in general, instead of using ",(0,i.jsx)(_.code,{children:"upgrade"})," CLI command."]}),(0,i.jsx)(_.p,{children:'Table upgrade from table version ONE to TWO requires key generator related configs such as\n"hoodie.datasource.write.recordkey.field", which is only available when user configures the write job. So the table\nupgrade from version ONE to TWO through CLI is not supported, and user should rely on the automatic upgrade in the write\nclient instead.'})]}),"\n",(0,i.jsx)(_.p,{children:"You may also run the upgrade command without specifying the target version. In such a case, the latest table version\ncorresponding to the library release version is used:"}),"\n",(0,i.jsx)(_.pre,{children:(0,i.jsx)(_.code,{className:"language-shell",children:"upgrade table\n"})}),"\n",(0,i.jsx)(_.h3,{id:"change-hudi-table-type",children:"Change Hudi Table Type"}),"\n",(0,i.jsx)(_.p,{children:"There are cases we want to change the hudi table type. For example, change COW table to MOR for more efficient and\nlower latency ingestion; change MOR to COW for better read performance and compatibility with downstream engines.\nSo we offer the table command to perform this modification conveniently."}),"\n",(0,i.jsxs)(_.p,{children:["Changing ",(0,i.jsx)(_.strong,{children:"COW to MOR"}),", we can simply modify the ",(0,i.jsx)(_.code,{children:"hoodie.table.type"})," in ",(0,i.jsx)(_.code,{children:"hoodie.properties"})," to MERGE_ON_READ."]}),"\n",(0,i.jsxs)(_.p,{children:["While changing ",(0,i.jsx)(_.strong,{children:"MOR to COW"}),", we must make sure all the log files are compacted before modifying the table type,\nor it will cause data loss."]}),"\n",(0,i.jsx)(_.pre,{children:(0,i.jsx)(_.code,{className:"language-shell",children:"connect --path \ntable change-table-type \n"})}),"\n",(0,i.jsxs)(_.p,{children:["The parameter ",(0,i.jsx)(_.code,{children:"target_table_type"})," candidates are below:"]}),"\n",(0,i.jsxs)(_.table,{children:[(0,i.jsx)(_.thead,{children:(0,i.jsxs)(_.tr,{children:[(0,i.jsx)(_.th,{style:{textAlign:"left"},children:"target table type"}),(0,i.jsx)(_.th,{style:{textAlign:"left"},children:"comment"})]})}),(0,i.jsxs)(_.tbody,{children:[(0,i.jsxs)(_.tr,{children:[(0,i.jsx)(_.td,{style:{textAlign:"left"},children:"MOR"}),(0,i.jsx)(_.td,{style:{textAlign:"left"},children:"Change COW table to MERGE_ON_READ."})]}),(0,i.jsxs)(_.tr,{children:[(0,i.jsx)(_.td,{style:{textAlign:"left"},children:"COW"}),(0,i.jsxs)(_.td,{style:{textAlign:"left"},children:["Change MOR table to COPY_ON_WRITE. ",(0,i.jsx)("br",{}),"By default, changing to COW will ",(0,i.jsx)(_.strong,{children:"execute all pending compactions"})," and ",(0,i.jsx)(_.strong,{children:"perform a full compaction"})," if any log file left. Set ",(0,i.jsx)(_.code,{children:"--enable-compaction=false"})," will disable the default compaction. ",(0,i.jsx)("br",{}),"There are params can be set for the compaction operation:",(0,i.jsx)("br",{}),(0,i.jsx)(_.code,{children:"--parallelism"}),": Default ",(0,i.jsx)(_.code,{children:"3"}),". Parallelism for hoodie compaction",(0,i.jsx)("br",{}),(0,i.jsx)(_.code,{children:"--sparkMaster"}),": Default ",(0,i.jsx)(_.code,{children:"local"}),". Spark Master",(0,i.jsx)("br",{}),(0,i.jsx)(_.code,{children:"--sparkMemory"}),": Default ",(0,i.jsx)(_.code,{children:"4G"}),". Spark executor memory",(0,i.jsx)("br",{}),(0,i.jsx)(_.code,{children:"--retry"}),": Default ",(0,i.jsx)(_.code,{children:"1"}),". Number of retries",(0,i.jsx)("br",{}),(0,i.jsx)(_.code,{children:"--propsFilePath"}),": Default ",(0,i.jsx)(_.code,{children:" "}),". path to properties file on localfs or dfs with configurations for hoodie client for compacting",(0,i.jsx)("br",{}),(0,i.jsx)(_.code,{children:"--hoodieConfigs"}),": Default ",(0,i.jsx)(_.code,{children:" "}),". Any configuration that can be set in the properties file can be passed here in the form of an array"]})]})]})]}),"\n",(0,i.jsx)(_.p,{children:"Example below is changing MOR table to COW:"}),"\n",(0,i.jsx)(_.pre,{children:(0,i.jsx)(_.code,{className:"language-shell",children:"connect --path /var/dataset/test_table_mor2cow\ndesc\n\u2554\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2564\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2557\n\u2551 Property \u2502 Value \u2551\n\u2560\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u256a\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2563\n\u2551 basePath \u2502 /var/dataset/test_table_mor2cow \u2551\n\u255f\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2562\n\u2551 metaPath \u2502 /var/dataset/test_table_mor2cow/.hoodie \u2551\n\u255f\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2562\n\u2551 fileSystem \u2502 file \u2551\n\u255f\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2562\n\u2551 hoodie.table.name \u2502 test_table \u2551\n\u255f\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2562\n\u2551 hoodie.compaction.record.merger.strategy \u2502 eeb8d96f-b1e4-49fd-bbf8-28ac514178e5 \u2551\n\u255f\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2562\n\u2551 hoodie.table.metadata.partitions \u2502 files \u2551\n\u255f\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2562\n\u2551 hoodie.table.type \u2502 MERGE_ON_READ \u2551\n\u255f\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2562\n\u2551 hoodie.table.metadata.partitions.inflight \u2502 \u2551\n\u255f\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2562\n\u2551 hoodie.archivelog.folder \u2502 archived \u2551\n\u255f\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2562\n\u2551 hoodie.timeline.layout.version \u2502 1 \u2551\n\u255f\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2562\n\u2551 hoodie.table.checksum \u2502 2702201862 \u2551\n\u255f\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2562\n\u2551 hoodie.compaction.payload.type \u2502 HOODIE_AVRO \u2551\n\u255f\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2562\n\u2551 hoodie.table.version \u2502 6 \u2551\n\u255f\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2562\n\u2551 hoodie.datasource.write.drop.partition.columns \u2502 false \u2551\n\u255a\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2567\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u255d\n\ntable change-table-type COW\n\u2554\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2564\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2564\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2557\n\u2551 Property \u2502 Old Value \u2502 New Value \u2551\n\u2560\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u256a\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u256a\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2563\n\u2551 hoodie.archivelog.folder \u2502 archived \u2502 archived \u2551\n\u255f\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2562\n\u2551 hoodie.compaction.payload.type \u2502 HOODIE_AVRO \u2502 HOODIE_AVRO \u2551\n\u255f\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2562\n\u2551 hoodie.compaction.record.merger.strategy \u2502 eeb8d96f-b1e4-49fd-bbf8-28ac514178e5 \u2502 eeb8d96f-b1e4-49fd-bbf8-28ac514178e5 \u2551\n\u255f\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2562\n\u2551 hoodie.datasource.write.drop.partition.columns \u2502 false \u2502 false \u2551\n\u255f\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2562\n\u2551 hoodie.table.checksum \u2502 2702201862 \u2502 2702201862 \u2551\n\u255f\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2562\n\u2551 hoodie.table.metadata.partitions \u2502 files \u2502 files \u2551\n\u255f\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2562\n\u2551 hoodie.table.metadata.partitions.inflight \u2502 \u2502 \u2551\n\u255f\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2562\n\u2551 hoodie.table.name \u2502 test_table \u2502 test_table \u2551\n\u255f\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2562\n\u2551 hoodie.table.type \u2502 MERGE_ON_READ \u2502 COPY_ON_WRITE \u2551\n\u255f\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2562\n\u2551 hoodie.table.version \u2502 6 \u2502 6 \u2551\n\u255f\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2562\n\u2551 hoodie.timeline.layout.version \u2502 1 \u2502 1 \u2551\n\u255a\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2567\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2567\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u255d\n"})}),"\n",(0,i.jsx)(_.h2,{id:"related-resources",children:"Related Resources"}),"\n",(0,i.jsx)("h3",{children:"Blogs"}),"\n",(0,i.jsxs)(_.ul,{children:["\n",(0,i.jsx)(_.li,{children:(0,i.jsx)(_.a,{href:"https://www.onehouse.ai/blog/getting-started-manage-your-hudi-tables-with-the-admin-hudi-cli-tool",children:"Getting Started: Manage your Hudi tables with the admin Hudi-CLI tool"})}),"\n"]})]})}function d(e={}){const{wrapper:_}={...(0,a.R)(),...e.components};return _?(0,i.jsx)(_,{...e,children:(0,i.jsx)(c,{...e})}):c(e)}},28453:(e,_,n)=>{n.d(_,{R:()=>o,x:()=>s});var t=n(96540);const i={},a=t.createContext(i);function o(e){const _=t.useContext(a);return t.useMemo((function(){return"function"==typeof e?e(_):{..._,...e}}),[_,e])}function s(e){let _;return _=e.disableParentContext?"function"==typeof e.components?e.components(i):e.components||i:o(e.components),t.createElement(a.Provider,{value:_},e.children)}}}]);
\ No newline at end of file
diff --git a/content/cn/assets/js/2263a65b.8d8f2ab4.js b/content/cn/assets/js/2263a65b.48601d47.js
similarity index 89%
rename from content/cn/assets/js/2263a65b.8d8f2ab4.js
rename to content/cn/assets/js/2263a65b.48601d47.js
index 215b9ac0e0d5c..6e397d1d40f8a 100644
--- a/content/cn/assets/js/2263a65b.8d8f2ab4.js
+++ b/content/cn/assets/js/2263a65b.48601d47.js
@@ -1 +1 @@
-"use strict";(self.webpackChunkhudi=self.webpackChunkhudi||[]).push([[99502],{8946:(e,a,t)=>{t.r(a),t.d(a,{assets:()=>s,contentTitle:()=>l,default:()=>u,frontMatter:()=>n,metadata:()=>i,toc:()=>d});const i=JSON.parse('{"id":"precommit_validator","title":"Data Quality","description":"Data quality refers to the overall accuracy, completeness, consistency, and validity of data. Ensuring data quality is vital for accurate analysis and reporting, as well as for compliance with regulations and maintaining trust in your organization\'s data infrastructure.","source":"@site/docs/precommit_validator.md","sourceDirName":".","slug":"/precommit_validator","permalink":"/cn/docs/next/precommit_validator","draft":false,"unlisted":false,"editUrl":"https://github.com/apache/hudi/tree/asf-site/website/docs/precommit_validator.md","tags":[],"version":"current","frontMatter":{"title":"Data Quality","keywords":["hudi","quality","expectations","pre-commit validator"]},"sidebar":"docs","previous":{"title":"Exporter","permalink":"/cn/docs/next/snapshot_exporter"},"next":{"title":"Post-commit Callback","permalink":"/cn/docs/next/platform_services_post_commit_callback"}}');var o=t(74848),r=t(28453);const n={title:"Data Quality",keywords:["hudi","quality","expectations","pre-commit validator"]},l=void 0,s={},d=[{value:"SQL Query Single Result",id:"sql-query-single-result",level:2},{value:"SQL Query Equality",id:"sql-query-equality",level:2},{value:"SQL Query Inequality",id:"sql-query-inequality",level:2},{value:"Extend Custom Validator",id:"extend-custom-validator",level:2},{value:"Additional Monitoring with Notifications",id:"additional-monitoring-with-notifications",level:2},{value:"Related Resources",id:"related-resources",level:2}];function c(e){const a={a:"a",code:"code",h2:"h2",li:"li",p:"p",pre:"pre",strong:"strong",ul:"ul",...(0,r.R)(),...e.components};return(0,o.jsxs)(o.Fragment,{children:[(0,o.jsx)(a.p,{children:"Data quality refers to the overall accuracy, completeness, consistency, and validity of data. Ensuring data quality is vital for accurate analysis and reporting, as well as for compliance with regulations and maintaining trust in your organization's data infrastructure."}),"\n",(0,o.jsxs)(a.p,{children:["Hudi offers ",(0,o.jsx)(a.strong,{children:"Pre-Commit Validators"})," that allow you to ensure that your data meets certain data quality expectations as you are writing with Hudi Streamer or Spark Datasource writers."]}),"\n",(0,o.jsxs)(a.p,{children:["To configure pre-commit validators, use this setting ",(0,o.jsx)(a.code,{children:"hoodie.precommit.validators="}),"."]}),"\n",(0,o.jsx)(a.p,{children:"Example:"}),"\n",(0,o.jsx)(a.pre,{children:(0,o.jsx)(a.code,{className:"language-scala",children:'spark.write.format("hudi")\n .option("hoodie.precommit.validators", "org.apache.hudi.client.validator.SqlQueryEqualityPreCommitValidator")\n'})}),"\n",(0,o.jsx)(a.p,{children:"Today you can use any of these validators and even have the flexibility to extend your own:"}),"\n",(0,o.jsx)(a.h2,{id:"sql-query-single-result",children:"SQL Query Single Result"}),"\n",(0,o.jsx)(a.p,{children:(0,o.jsx)(a.a,{href:"https://github.com/apache/hudi/blob/bf5a52e51bbeaa089995335a0a4c55884792e505/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/validator/SqlQuerySingleResultPreCommitValidator.java",children:"org.apache.hudi.client.validator.SqlQuerySingleResultPreCommitValidator"})}),"\n",(0,o.jsx)(a.p,{children:"The SQL Query Single Result validator can be used to validate that a query on the table results in a specific value. This validator allows you to run a SQL query and abort the commit if it does not match the expected output."}),"\n",(0,o.jsxs)(a.p,{children:["Multiple queries can be separated by ",(0,o.jsx)(a.code,{children:";"})," delimiter. Include the expected result as part of the query separated by ",(0,o.jsx)(a.code,{children:"#"}),"."]}),"\n",(0,o.jsxs)(a.p,{children:["Syntax: ",(0,o.jsx)(a.code,{children:"query1#result1;query2#result2"})]}),"\n",(0,o.jsx)(a.p,{children:"Example:"}),"\n",(0,o.jsx)(a.pre,{children:(0,o.jsx)(a.code,{className:"language-scala",children:'// In this example, we set up a validator that expects there is no row with `col` column as `null`\n\nimport org.apache.hudi.config.HoodiePreCommitValidatorConfig._\n\ndf.write.format("hudi").mode(Overwrite).\n option("hoodie.table.name", tableName).\n option("hoodie.precommit.validators", "org.apache.hudi.client.validator.SqlQuerySingleResultPreCommitValidator").\n option("hoodie.precommit.validators.single.value.sql.queries", "select count(*) from where col is null#0").\n save(basePath)\n'})}),"\n",(0,o.jsx)(a.h2,{id:"sql-query-equality",children:"SQL Query Equality"}),"\n",(0,o.jsx)(a.p,{children:(0,o.jsx)(a.a,{href:"https://github.com/apache/hudi/blob/bf5a52e51bbeaa089995335a0a4c55884792e505/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/validator/SqlQueryEqualityPreCommitValidator.java",children:"org.apache.hudi.client.validator.SqlQueryEqualityPreCommitValidator"})}),"\n",(0,o.jsx)(a.p,{children:"The SQL Query Equality validator runs a query before ingesting the data, then runs the same query after ingesting the data and confirms that both outputs match. This allows you to validate for equality of rows before and after the commit."}),"\n",(0,o.jsx)(a.p,{children:"This validator is useful when you want to verify that your query does not change a specific subset of the data. Some examples:"}),"\n",(0,o.jsxs)(a.ul,{children:["\n",(0,o.jsx)(a.li,{children:"Validate that the number of null fields is the same before and after your query"}),"\n",(0,o.jsx)(a.li,{children:"Validate that there are no duplicate records after your query runs"}),"\n",(0,o.jsx)(a.li,{children:"Validate that you are only updating the data, and no inserts slip through"}),"\n"]}),"\n",(0,o.jsx)(a.p,{children:"Example:"}),"\n",(0,o.jsx)(a.pre,{children:(0,o.jsx)(a.code,{className:"language-scala",children:'// In this example, we set up a validator that expects no change of null rows with the new commit\n\nimport org.apache.hudi.config.HoodiePreCommitValidatorConfig._\n\ndf.write.format("hudi").mode(Overwrite).\n option("hoodie.table.name", tableName).\n option("hoodie.precommit.validators", "org.apache.hudi.client.validator.SqlQueryEqualityPreCommitValidator").\n option("hoodie.precommit.validators.equality.sql.queries", "select count(*) from where col is null").\n save(basePath)\n'})}),"\n",(0,o.jsx)(a.h2,{id:"sql-query-inequality",children:"SQL Query Inequality"}),"\n",(0,o.jsx)(a.p,{children:(0,o.jsx)(a.a,{href:"https://github.com/apache/hudi/blob/bf5a52e51bbeaa089995335a0a4c55884792e505/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/validator/SqlQueryInequalityPreCommitValidator.java",children:"org.apache.hudi.client.validator.SqlQueryInequalityPreCommitValidator"})}),"\n",(0,o.jsx)(a.p,{children:"The SQL Query Inquality validator runs a query before ingesting the data, then runs the same query after ingesting the data and confirms that both outputs DO NOT match. This allows you to confirm changes in the rows before and after the commit."}),"\n",(0,o.jsx)(a.p,{children:"Example:"}),"\n",(0,o.jsx)(a.pre,{children:(0,o.jsx)(a.code,{className:"language-scala",children:'// In this example, we set up a validator that expects a change of null rows with the new commit\n\nimport org.apache.hudi.config.HoodiePreCommitValidatorConfig._\n\ndf.write.format("hudi").mode(Overwrite).\n option("hoodie.table.name", tableName).\n option("hoodie.precommit.validators", "org.apache.hudi.client.validator.SqlQueryInequalityPreCommitValidator").\n option("hoodie.precommit.validators.inequality.sql.queries", "select count(*) from where col is null").\n save(basePath)\n'})}),"\n",(0,o.jsx)(a.h2,{id:"extend-custom-validator",children:"Extend Custom Validator"}),"\n",(0,o.jsxs)(a.p,{children:["Users can also provide their own implementations by extending the abstract class ",(0,o.jsx)(a.a,{href:"https://github.com/apache/hudi/blob/bf5a52e51bbeaa089995335a0a4c55884792e505/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/validator/SparkPreCommitValidator.java",children:"SparkPreCommitValidator"}),"\nand overriding this method"]}),"\n",(0,o.jsx)(a.pre,{children:(0,o.jsx)(a.code,{className:"language-java",children:"void validateRecordsBeforeAndAfter(Dataset before, \n Dataset after, \n Set partitionsAffected)\n"})}),"\n",(0,o.jsx)(a.h2,{id:"additional-monitoring-with-notifications",children:"Additional Monitoring with Notifications"}),"\n",(0,o.jsxs)(a.p,{children:["Hudi offers a ",(0,o.jsx)(a.a,{href:"platform_services_post_commit_callback",children:"commit notification service"})," that can be configured to trigger notifications about write commits."]}),"\n",(0,o.jsx)(a.p,{children:"The commit notification service can be combined with pre-commit validators to send a notification when a commit fails a validation. This is possible by passing details about the validation as a custom value to the HTTP endpoint."}),"\n",(0,o.jsx)(a.h2,{id:"related-resources",children:"Related Resources"}),"\n",(0,o.jsx)("h3",{children:"Videos"}),"\n",(0,o.jsxs)(a.ul,{children:["\n",(0,o.jsx)(a.li,{children:(0,o.jsx)(a.a,{href:"https://www.youtube.com/watch?v=KNzs9dj_Btc",children:"Learn About Apache Hudi Pre Commit Validator with Hands on Lab"})}),"\n"]})]})}function u(e={}){const{wrapper:a}={...(0,r.R)(),...e.components};return a?(0,o.jsx)(a,{...e,children:(0,o.jsx)(c,{...e})}):c(e)}},28453:(e,a,t)=>{t.d(a,{R:()=>n,x:()=>l});var i=t(96540);const o={},r=i.createContext(o);function n(e){const a=i.useContext(r);return i.useMemo((function(){return"function"==typeof e?e(a):{...a,...e}}),[a,e])}function l(e){let a;return a=e.disableParentContext?"function"==typeof e.components?e.components(o):e.components||o:n(e.components),i.createElement(r.Provider,{value:a},e.children)}}}]);
\ No newline at end of file
+"use strict";(self.webpackChunkhudi=self.webpackChunkhudi||[]).push([[99502],{8946:(e,a,t)=>{t.r(a),t.d(a,{assets:()=>s,contentTitle:()=>l,default:()=>u,frontMatter:()=>n,metadata:()=>i,toc:()=>d});const i=JSON.parse('{"id":"precommit_validator","title":"Data Quality","description":"Data quality refers to the overall accuracy, completeness, consistency, and validity of data. Ensuring data quality is vital for accurate analysis and reporting, as well as for compliance with regulations and maintaining trust in your organization\'s data infrastructure.","source":"@site/docs/precommit_validator.md","sourceDirName":".","slug":"/precommit_validator","permalink":"/cn/docs/next/precommit_validator","draft":false,"unlisted":false,"editUrl":"https://github.com/apache/hudi/tree/asf-site/website/docs/precommit_validator.md","tags":[],"version":"current","frontMatter":{"title":"Data Quality","keywords":["hudi","quality","expectations","pre-commit validator"]},"sidebar":"docs","previous":{"title":"Exporter","permalink":"/cn/docs/next/snapshot_exporter"},"next":{"title":"Post-commit Callback","permalink":"/cn/docs/next/platform_services_post_commit_callback"}}');var o=t(74848),r=t(28453);const n={title:"Data Quality",keywords:["hudi","quality","expectations","pre-commit validator"]},l=void 0,s={},d=[{value:"SQL Query Single Result",id:"sql-query-single-result",level:2},{value:"SQL Query Equality",id:"sql-query-equality",level:2},{value:"SQL Query Inequality",id:"sql-query-inequality",level:2},{value:"Extend Custom Validator",id:"extend-custom-validator",level:2},{value:"Additional Monitoring with Notifications",id:"additional-monitoring-with-notifications",level:2},{value:"Related Resources",id:"related-resources",level:2}];function c(e){const a={a:"a",code:"code",h2:"h2",li:"li",p:"p",pre:"pre",strong:"strong",ul:"ul",...(0,r.R)(),...e.components};return(0,o.jsxs)(o.Fragment,{children:[(0,o.jsx)(a.p,{children:"Data quality refers to the overall accuracy, completeness, consistency, and validity of data. Ensuring data quality is vital for accurate analysis and reporting, as well as for compliance with regulations and maintaining trust in your organization's data infrastructure."}),"\n",(0,o.jsxs)(a.p,{children:["Hudi offers ",(0,o.jsx)(a.strong,{children:"Pre-Commit Validators"})," that allow you to ensure that your data meets certain data quality expectations as you are writing with Hudi Streamer or Spark Datasource writers."]}),"\n",(0,o.jsxs)(a.p,{children:["To configure pre-commit validators, use this setting ",(0,o.jsx)(a.code,{children:"hoodie.precommit.validators="}),"."]}),"\n",(0,o.jsx)(a.p,{children:"Example:"}),"\n",(0,o.jsx)(a.pre,{children:(0,o.jsx)(a.code,{className:"language-scala",children:'spark.write.format("hudi")\n .option("hoodie.precommit.validators", "org.apache.hudi.client.validator.SqlQueryEqualityPreCommitValidator")\n'})}),"\n",(0,o.jsx)(a.p,{children:"Today you can use any of these validators and even have the flexibility to extend your own:"}),"\n",(0,o.jsx)(a.h2,{id:"sql-query-single-result",children:"SQL Query Single Result"}),"\n",(0,o.jsx)(a.p,{children:(0,o.jsx)(a.a,{href:"https://github.com/apache/hudi/blob/bf5a52e51bbeaa089995335a0a4c55884792e505/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/validator/SqlQuerySingleResultPreCommitValidator.java",children:"org.apache.hudi.client.validator.SqlQuerySingleResultPreCommitValidator"})}),"\n",(0,o.jsx)(a.p,{children:"The SQL Query Single Result validator can be used to validate that a query on the table results in a specific value. This validator allows you to run a SQL query and abort the commit if it does not match the expected output."}),"\n",(0,o.jsxs)(a.p,{children:["Multiple queries can be separated by ",(0,o.jsx)(a.code,{children:";"})," delimiter. Include the expected result as part of the query separated by ",(0,o.jsx)(a.code,{children:"#"}),"."]}),"\n",(0,o.jsxs)(a.p,{children:["Syntax: ",(0,o.jsx)(a.code,{children:"query1#result1;query2#result2"})]}),"\n",(0,o.jsx)(a.p,{children:"Example:"}),"\n",(0,o.jsx)(a.pre,{children:(0,o.jsx)(a.code,{className:"language-scala",children:'// In this example, we set up a validator that expects there is no row with `col` column as `null`\n\nimport org.apache.hudi.config.HoodiePreCommitValidatorConfig._\n\ndf.write.format("hudi").mode(Overwrite).\n option("hoodie.table.name", tableName).\n option("hoodie.precommit.validators", "org.apache.hudi.client.validator.SqlQuerySingleResultPreCommitValidator").\n option("hoodie.precommit.validators.single.value.sql.queries", "select count(*) from where col is null#0").\n save(basePath)\n'})}),"\n",(0,o.jsx)(a.h2,{id:"sql-query-equality",children:"SQL Query Equality"}),"\n",(0,o.jsx)(a.p,{children:(0,o.jsx)(a.a,{href:"https://github.com/apache/hudi/blob/bf5a52e51bbeaa089995335a0a4c55884792e505/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/validator/SqlQueryEqualityPreCommitValidator.java",children:"org.apache.hudi.client.validator.SqlQueryEqualityPreCommitValidator"})}),"\n",(0,o.jsx)(a.p,{children:"The SQL Query Equality validator runs a query before ingesting the data, then runs the same query after ingesting the data and confirms that both outputs match. This allows you to validate for equality of rows before and after the commit."}),"\n",(0,o.jsx)(a.p,{children:"This validator is useful when you want to verify that your query does not change a specific subset of the data. Some examples:"}),"\n",(0,o.jsxs)(a.ul,{children:["\n",(0,o.jsx)(a.li,{children:"Validate that the number of null fields is the same before and after your query"}),"\n",(0,o.jsx)(a.li,{children:"Validate that there are no duplicate records after your query runs"}),"\n",(0,o.jsx)(a.li,{children:"Validate that you are only updating the data, and no inserts slip through"}),"\n"]}),"\n",(0,o.jsx)(a.p,{children:"Example:"}),"\n",(0,o.jsx)(a.pre,{children:(0,o.jsx)(a.code,{className:"language-scala",children:'// In this example, we set up a validator that expects no change of null rows with the new commit\n\nimport org.apache.hudi.config.HoodiePreCommitValidatorConfig._\n\ndf.write.format("hudi").mode(Overwrite).\n option("hoodie.table.name", tableName).\n option("hoodie.precommit.validators", "org.apache.hudi.client.validator.SqlQueryEqualityPreCommitValidator").\n option("hoodie.precommit.validators.equality.sql.queries", "select count(*) from where col is null").\n save(basePath)\n'})}),"\n",(0,o.jsx)(a.h2,{id:"sql-query-inequality",children:"SQL Query Inequality"}),"\n",(0,o.jsx)(a.p,{children:(0,o.jsx)(a.a,{href:"https://github.com/apache/hudi/blob/bf5a52e51bbeaa089995335a0a4c55884792e505/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/validator/SqlQueryInequalityPreCommitValidator.java",children:"org.apache.hudi.client.validator.SqlQueryInequalityPreCommitValidator"})}),"\n",(0,o.jsx)(a.p,{children:"The SQL Query Inquality validator runs a query before ingesting the data, then runs the same query after ingesting the data and confirms that both outputs DO NOT match. This allows you to confirm changes in the rows before and after the commit."}),"\n",(0,o.jsx)(a.p,{children:"Example:"}),"\n",(0,o.jsx)(a.pre,{children:(0,o.jsx)(a.code,{className:"language-scala",children:'// In this example, we set up a validator that expects a change of null rows with the new commit\n\nimport org.apache.hudi.config.HoodiePreCommitValidatorConfig._\n\ndf.write.format("hudi").mode(Overwrite).\n option("hoodie.table.name", tableName).\n option("hoodie.precommit.validators", "org.apache.hudi.client.validator.SqlQueryInequalityPreCommitValidator").\n option("hoodie.precommit.validators.inequality.sql.queries", "select count(*) from where col is null").\n save(basePath)\n'})}),"\n",(0,o.jsx)(a.h2,{id:"extend-custom-validator",children:"Extend Custom Validator"}),"\n",(0,o.jsxs)(a.p,{children:["Users can also provide their own implementations by extending the abstract class ",(0,o.jsx)(a.a,{href:"https://github.com/apache/hudi/blob/bf5a52e51bbeaa089995335a0a4c55884792e505/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/validator/SparkPreCommitValidator.java",children:"SparkPreCommitValidator"}),"\nand overriding this method"]}),"\n",(0,o.jsx)(a.pre,{children:(0,o.jsx)(a.code,{className:"language-java",children:"void validateRecordsBeforeAndAfter(Dataset before, \n Dataset after, \n Set partitionsAffected)\n"})}),"\n",(0,o.jsx)(a.h2,{id:"additional-monitoring-with-notifications",children:"Additional Monitoring with Notifications"}),"\n",(0,o.jsxs)(a.p,{children:["Hudi offers a ",(0,o.jsx)(a.a,{href:"platform_services_post_commit_callback",children:"commit notification service"})," that can be configured to trigger notifications about write commits."]}),"\n",(0,o.jsx)(a.p,{children:"The commit notification service can be combined with pre-commit validators to send a notification when a commit fails a validation. This is possible by passing details about the validation as a custom value to the HTTP endpoint."}),"\n",(0,o.jsx)(a.h2,{id:"related-resources",children:"Related Resources"}),"\n",(0,o.jsx)("h3",{children:"Blogs"}),"\n",(0,o.jsxs)(a.ul,{children:["\n",(0,o.jsx)(a.li,{children:(0,o.jsx)(a.a,{href:"https://www.onehouse.ai/blog/apply-pre-commit-validation-for-data-quality-in-apache-hudi",children:"Apply Pre-Commit Validation for Data Quality in Apache Hudi"})}),"\n"]}),"\n",(0,o.jsx)("h3",{children:"Videos"}),"\n",(0,o.jsxs)(a.ul,{children:["\n",(0,o.jsx)(a.li,{children:(0,o.jsx)(a.a,{href:"https://www.youtube.com/watch?v=KNzs9dj_Btc",children:"Learn About Apache Hudi Pre Commit Validator with Hands on Lab"})}),"\n"]})]})}function u(e={}){const{wrapper:a}={...(0,r.R)(),...e.components};return a?(0,o.jsx)(a,{...e,children:(0,o.jsx)(c,{...e})}):c(e)}},28453:(e,a,t)=>{t.d(a,{R:()=>n,x:()=>l});var i=t(96540);const o={},r=i.createContext(o);function n(e){const a=i.useContext(r);return i.useMemo((function(){return"function"==typeof e?e(a):{...a,...e}}),[a,e])}function l(e){let a;return a=e.disableParentContext?"function"==typeof e.components?e.components(o):e.components||o:n(e.components),i.createElement(r.Provider,{value:a},e.children)}}}]);
\ No newline at end of file
diff --git a/content/cn/assets/js/491d56e0.0fac960c.js b/content/cn/assets/js/491d56e0.0fac960c.js
deleted file mode 100644
index 7449f0d683d73..0000000000000
--- a/content/cn/assets/js/491d56e0.0fac960c.js
+++ /dev/null
@@ -1 +0,0 @@
-"use strict";(self.webpackChunkhudi=self.webpackChunkhudi||[]).push([[72907],{48105:(e,r,i)=>{i.r(r),i.d(r,{assets:()=>d,contentTitle:()=>s,default:()=>h,frontMatter:()=>a,metadata:()=>n,toc:()=>c});const n=JSON.parse('{"id":"record_merger","title":"Record Mergers","description":"Hudi handles mutations to records and streaming data, as we briefly touched upon in timeline ordering section.","source":"@site/docs/record_merger.md","sourceDirName":".","slug":"/record_merger","permalink":"/cn/docs/next/record_merger","draft":false,"unlisted":false,"editUrl":"https://github.com/apache/hudi/tree/asf-site/website/docs/record_merger.md","tags":[],"version":"current","frontMatter":{"title":"Record Mergers","keywords":["hudi","merge","upsert","precombine"],"toc":true,"toc_min_heading_level":2,"toc_max_heading_level":4},"sidebar":"docs","previous":{"title":"Key Generation","permalink":"/cn/docs/next/key_generation"},"next":{"title":"Table Metadata","permalink":"/cn/docs/next/metadata"}}');var o=i(74848),t=i(28453);const a={title:"Record Mergers",keywords:["hudi","merge","upsert","precombine"],toc:!0,toc_min_heading_level:2,toc_max_heading_level:4},s=void 0,d={},c=[{value:"COMMIT_TIME_ORDERING",id:"commit_time_ordering",level:3},{value:"EVENT_TIME_ORDERING",id:"event_time_ordering",level:3},{value:"CUSTOM",id:"custom",level:3},{value:"Record Merge Configs",id:"record-merge-configs",level:3},{value:"Record Payloads",id:"record-payloads",level:3},{value:"OverwriteWithLatestAvroPayload",id:"overwritewithlatestavropayload",level:4},{value:"DefaultHoodieRecordPayload",id:"defaulthoodierecordpayload",level:4},{value:"EventTimeAvroPayload",id:"eventtimeavropayload",level:4},{value:"OverwriteNonDefaultsWithLatestAvroPayload",id:"overwritenondefaultswithlatestavropayload",level:4},{value:"PartialUpdateAvroPayload",id:"partialupdateavropayload",level:4},{value:"Configs",id:"configs",level:4}];function l(e){const r={a:"a",admonition:"admonition",code:"code",em:"em",h3:"h3",h4:"h4",li:"li",p:"p",pre:"pre",strong:"strong",table:"table",tbody:"tbody",td:"td",th:"th",thead:"thead",tr:"tr",ul:"ul",...(0,t.R)(),...e.components};return(0,o.jsxs)(o.Fragment,{children:[(0,o.jsxs)(r.p,{children:["Hudi handles mutations to records and streaming data, as we briefly touched upon in ",(0,o.jsx)(r.a,{href:"timeline#ordering-of-actions",children:"timeline ordering"})," section.\nTo provide users full-fledged support for stream processing, Hudi goes all the way making the storage engine and the underlying storage format\nunderstand how to merge changes to the same record key, that may arrive even in different order at different times. With the rise of mobile applications\nand IoT, these scenarios have become the normal than an exception. For e.g. a social networking application uploading user events several hours after they happened,\nwhen the user connects to WiFi networks."]}),"\n",(0,o.jsx)(r.p,{children:"To achieve this, Hudi supports merge modes, which define how the base and log files are ordered in a file slice and further how different records with\nthe same record key within that file slice are merged consistently to produce the same deterministic results for snapshot queries, writers and table services. Specifically,\nthere are three merge modes supported as a table-level configuration, invoked in the following places."}),"\n",(0,o.jsxs)(r.ul,{children:["\n",(0,o.jsxs)(r.li,{children:["\n",(0,o.jsxs)(r.p,{children:[(0,o.jsx)(r.strong,{children:"(writing)"})," Combining multiple change records for the same record key while reading input data during writes. This is an optional optimization that\nreduces the number of records written to log files to improve query and write performance subsequently."]}),"\n"]}),"\n",(0,o.jsxs)(r.li,{children:["\n",(0,o.jsxs)(r.p,{children:[(0,o.jsx)(r.strong,{children:"(writing)"})," Merging final change record (partial/full update/delete) against existing record in storage for CoW tables."]}),"\n"]}),"\n",(0,o.jsxs)(r.li,{children:["\n",(0,o.jsxs)(r.p,{children:[(0,o.jsx)(r.strong,{children:"(compaction)"})," Compaction service merges all change records in log files against base files, respecting the merge mode."]}),"\n"]}),"\n",(0,o.jsxs)(r.li,{children:["\n",(0,o.jsxs)(r.p,{children:[(0,o.jsx)(r.strong,{children:"(query)"})," Merging change records in log files, after filtering/projections against base file for MoR table queries."]}),"\n"]}),"\n"]}),"\n",(0,o.jsx)(r.p,{children:"Note that the merge mode should not be altered once the table is created to avoid inconsistent behavior due to compaction producing\ndifferent merge results when switching between the modes."}),"\n",(0,o.jsx)(r.h3,{id:"commit_time_ordering",children:"COMMIT_TIME_ORDERING"}),"\n",(0,o.jsx)(r.p,{children:"Here, we expect the input records to arrive in strict order such that arrival order is same as their\ndelta commit order on the table. Merging simply picks the record belonging to the latest write as the merged result. In relational data mode speak,\nthis provides overwrite semantics aligned with serializable writes on the timeline."}),"\n",(0,o.jsx)("figure",{children:(0,o.jsx)("img",{className:"docimage",src:i(38526).A,alt:"upsert_path.png"})}),"\n",(0,o.jsx)(r.p,{children:"In the example above, the writer process consumes a database change log, expected to be in strict order of a logical sequence number (lsn)\nthat denotes the ordering of the writes in the upstream database."}),"\n",(0,o.jsx)(r.h3,{id:"event_time_ordering",children:"EVENT_TIME_ORDERING"}),"\n",(0,o.jsxs)(r.p,{children:["This is the default merge mode. While commit time ordering provides a well-understood standard behavior, it's hardly sufficient. The commit time is unrelated to the actual\nordering of data that a user may care about and strict ordering of input in complex distributed systems is difficult to achieve.\nWith event time ordering, the merging picks the record with the highest value on a user specified ",(0,o.jsx)(r.em,{children:(0,o.jsx)(r.strong,{children:"ordering or precombine field"})})," as the merged result."]}),"\n",(0,o.jsx)("figure",{children:(0,o.jsx)("img",{className:"docimage",src:i(57159).A,alt:"upsert_path.png"})}),"\n",(0,o.jsxs)(r.p,{children:['In the example above, two microservices product change records about orders at different times, that can arrive out-of-order. As color coded,\nthis can lead to application-level inconsistent states in the table if simply merged in commit time order like a cancelled order being re-created or\na paid order moved back to just created state expecting payment again. Event time ordering helps by ignoring older state changes that arrive late and\navoiding order status from "jumping back" in time. Combined with ',(0,o.jsx)(r.a,{href:"concurrency_control#non-blocking-concurrency-control-mode",children:"non-blocking concurrency control"}),",\nthis provides a very powerful way for processing such data streams efficiently and correctly."]}),"\n",(0,o.jsx)(r.h3,{id:"custom",children:"CUSTOM"}),"\n",(0,o.jsx)(r.p,{children:'In some cases, even more control and customization may be needed. Extending the same example above, the two microservices could be updating two different\nset of columns "order_info" and "payment_info", along with order state. The merge logic is then expected to not only resolve the correct status, but merge\norder_info from the record in created state, into the record in cancelled state that already has payment_info fields populated with reasons payment failed.\nSuch reconciliation provide a simple denormalized data model for downstream consumption where queries (for e.g. fraud detection) can simply filter fields\nacross order_info and payment_info without costly self-join on each access.'}),"\n",(0,o.jsxs)(r.p,{children:["Hudi allows authoring of cross-language custom record mergers on top of a standard record merger API, that supports full and partial merges. The java APIs\nare sketched below at a high-level. It simply takes older/newer records in engine native formats and produces a merged record or returns empty to skip them entirely (e.g. soft deletes).\nRecord merger is configured using a ",(0,o.jsx)(r.code,{children:"hoodie.write.record.merge.strategy.id"})," write config whose value is an uuid, that is taken by the writer to persist in the table config, and is expected to be returned by ",(0,o.jsx)(r.code,{children:"getMergingStrategy()"}),"\nmethod below. Using this mechanism, Hudi can automatically deduce the record merger to use for the table across different language/engine runtimes."]}),"\n",(0,o.jsx)(r.pre,{children:(0,o.jsx)(r.code,{className:"language-Java",children:"interface HoodieRecordMerger {\n\n Option> merge(HoodieRecord older, Schema oldSchema, \n HoodieRecord newer, Schema newSchema, \n TypedProperties props) {\n ...\n }\n\n Option> partialMerge(HoodieRecord older, Schema oldSchema, \n HoodieRecord newer, Schema newSchema, \n Schema readerSchema, TypedProperties props) {\n ...\n }\n \n HoodieRecordType getRecordType() {...}\n \n String getMergingStrategy(); {...}\n}\n"})}),"\n",(0,o.jsx)(r.h3,{id:"record-merge-configs",children:"Record Merge Configs"}),"\n",(0,o.jsx)(r.p,{children:"The record merge mode and optional record merge strategy ID and custom merge implementation classes can be specified using the below configs."}),"\n",(0,o.jsxs)(r.table,{children:[(0,o.jsx)(r.thead,{children:(0,o.jsxs)(r.tr,{children:[(0,o.jsx)(r.th,{children:"Config Name"}),(0,o.jsx)(r.th,{children:"Default"}),(0,o.jsx)(r.th,{children:"Description"})]})}),(0,o.jsxs)(r.tbody,{children:[(0,o.jsxs)(r.tr,{children:[(0,o.jsx)(r.td,{children:"hoodie.write.record.merge.mode"}),(0,o.jsx)(r.td,{children:"EVENT_TIME_ORDERING"}),(0,o.jsxs)(r.td,{children:["Determines the logic of merging different records with the same record key. Valid values: (1) ",(0,o.jsx)(r.code,{children:"COMMIT_TIME_ORDERING"}),": use commit time to merge records, i.e., the record from later commit overwrites the earlier record with the same key. (2) ",(0,o.jsx)(r.code,{children:"EVENT_TIME_ORDERING"})," (default): use event time as the ordering to merge records, i.e., the record with the larger event time overwrites the record with the smaller event time on the same key, regardless of commit time. The event time or preCombine field needs to be specified by the user. (3) ",(0,o.jsx)(r.code,{children:"CUSTOM"}),": use custom merging logic specified by the user.",(0,o.jsx)("br",{}),(0,o.jsx)(r.code,{children:"Config Param: RECORD_MERGE_MODE"}),(0,o.jsx)("br",{}),(0,o.jsx)(r.code,{children:"Since Version: 1.0.0"})]})]}),(0,o.jsxs)(r.tr,{children:[(0,o.jsx)(r.td,{children:"hoodie.write.record.merge.strategy.id"}),(0,o.jsx)(r.td,{children:"N/A (Optional)"}),(0,o.jsxs)(r.td,{children:["ID of record merge strategy. When you specify this config, you also need to specify ",(0,o.jsx)(r.code,{children:"hoodie.write.record.merge.custom.implementation.classes"}),". Hudi picks the ",(0,o.jsx)(r.code,{children:"HoodieRecordMerger"})," implementation class from the list of classes in ",(0,o.jsx)(r.code,{children:"hoodie.write.record.merge.custom.implementation.classes"})," that has the specified merge strategy ID.",(0,o.jsx)("br",{}),(0,o.jsx)(r.code,{children:"Config Param: RECORD_MERGE_STRATEGY_ID"}),(0,o.jsx)("br",{}),(0,o.jsx)(r.code,{children:"Since Version: 0.13.0"})]})]}),(0,o.jsxs)(r.tr,{children:[(0,o.jsx)(r.td,{children:"hoodie.write.record.merge.custom.implementation.classes"}),(0,o.jsx)(r.td,{children:"N/A (Optional)"}),(0,o.jsxs)(r.td,{children:["List of ",(0,o.jsx)(r.code,{children:"HoodieRecordMerger"})," implementations constituting Hudi's merging strategy based on the engine used. Hudi picks the ",(0,o.jsx)(r.code,{children:"HoodieRecordMerger"})," implementation class from this list based on the specified ",(0,o.jsx)(r.code,{children:"hoodie.write.record.merge.strategy.id"}),".",(0,o.jsx)("br",{}),(0,o.jsx)(r.code,{children:"Config Param: RECORD_MERGE_IMPL_CLASSES"}),(0,o.jsx)("br",{}),(0,o.jsx)(r.code,{children:"Since Version: 0.13.0"})]})]})]})]}),"\n",(0,o.jsx)(r.h3,{id:"record-payloads",children:"Record Payloads"}),"\n",(0,o.jsx)(r.admonition,{type:"caution",children:(0,o.jsx)(r.p,{children:"Going forward, we recommend users to migrate and use the record merger APIs and not write new payload implementations."})}),"\n",(0,o.jsx)(r.p,{children:"Record payload is an older abstraction/API for achieving similar record-level merge capabilities. While record payloads were very useful and popular,\nit had drawbacks like lower performance due to conversion of engine native record formats to Apache Avro for merging and lack of cross-language support.\nAs we shall see below, Hudi provides out-of-box support for different payloads for different use cases. Hudi implements fallback from\nrecord merger APIs to payload APIs internally, to provide backwards compatibility for existing payload implementations."}),"\n",(0,o.jsx)(r.h4,{id:"overwritewithlatestavropayload",children:"OverwriteWithLatestAvroPayload"}),"\n",(0,o.jsx)(r.pre,{children:(0,o.jsx)(r.code,{className:"language-scala",children:"hoodie.datasource.write.payload.class=org.apache.hudi.common.model.OverwriteWithLatestAvroPayload\n"})}),"\n",(0,o.jsxs)(r.p,{children:["This is the default record payload implementation. It picks the record with the greatest value (determined by calling\n",(0,o.jsx)(r.code,{children:".compareTo()"})," on the value of precombine key) to break ties and simply picks the latest record while merging. This gives\nlatest-write-wins style semantics."]}),"\n",(0,o.jsx)(r.h4,{id:"defaulthoodierecordpayload",children:"DefaultHoodieRecordPayload"}),"\n",(0,o.jsx)(r.pre,{children:(0,o.jsx)(r.code,{className:"language-scala",children:"hoodie.datasource.write.payload.class=org.apache.hudi.common.model.DefaultHoodieRecordPayload\n"})}),"\n",(0,o.jsxs)(r.p,{children:["While ",(0,o.jsx)(r.code,{children:"OverwriteWithLatestAvroPayload"})," precombines based on an ordering field and picks the latest record while merging,\n",(0,o.jsx)(r.code,{children:"DefaultHoodieRecordPayload"})," honors the ordering field for both precombinig and merging. Let's understand the difference with an example:"]}),"\n",(0,o.jsxs)(r.p,{children:["Let's say the ordering field is ",(0,o.jsx)(r.code,{children:"ts"})," and record key is ",(0,o.jsx)(r.code,{children:"id"})," and schema is:"]}),"\n",(0,o.jsx)(r.pre,{children:(0,o.jsx)(r.code,{children:'{\n [\n {"name":"id","type":"string"},\n {"name":"ts","type":"long"},\n {"name":"name","type":"string"},\n {"name":"price","type":"string"}\n ]\n}\n'})}),"\n",(0,o.jsx)(r.p,{children:"Current record in storage:"}),"\n",(0,o.jsx)(r.pre,{children:(0,o.jsx)(r.code,{children:" id ts name price\n 1 2 name_2 price_2\n"})}),"\n",(0,o.jsx)(r.p,{children:"Incoming record:"}),"\n",(0,o.jsx)(r.pre,{children:(0,o.jsx)(r.code,{children:" id ts name price\n 1 1 name_1 price_1\n"})}),"\n",(0,o.jsxs)(r.p,{children:["Result data after merging using ",(0,o.jsx)(r.code,{children:"OverwriteWithLatestAvroPayload"})," (latest-write-wins):"]}),"\n",(0,o.jsx)(r.pre,{children:(0,o.jsx)(r.code,{children:" id ts name price\n 1 1 name_1 price_1\n"})}),"\n",(0,o.jsxs)(r.p,{children:["Result data after merging using ",(0,o.jsx)(r.code,{children:"DefaultHoodieRecordPayload"})," (always honors ordering field):"]}),"\n",(0,o.jsx)(r.pre,{children:(0,o.jsx)(r.code,{children:" id ts name price\n 1 2 name_2 price_2\n"})}),"\n",(0,o.jsx)(r.h4,{id:"eventtimeavropayload",children:"EventTimeAvroPayload"}),"\n",(0,o.jsx)(r.pre,{children:(0,o.jsx)(r.code,{className:"language-scala",children:"hoodie.datasource.write.payload.class=org.apache.hudi.common.model.EventTimeAvroPayload\n"})}),"\n",(0,o.jsxs)(r.p,{children:["This is the default record payload for Flink based writing. Some use cases require merging records by event time and\nthus event time plays the role of an ordering field. This payload is particularly useful in the case of late-arriving data.\nFor such use cases, users need to set the ",(0,o.jsx)(r.a,{href:"/docs/configurations#RECORD_PAYLOAD",children:"payload event time field"})," configuration."]}),"\n",(0,o.jsx)(r.h4,{id:"overwritenondefaultswithlatestavropayload",children:"OverwriteNonDefaultsWithLatestAvroPayload"}),"\n",(0,o.jsx)(r.pre,{children:(0,o.jsx)(r.code,{className:"language-scala",children:"hoodie.datasource.write.payload.class=org.apache.hudi.common.model.OverwriteNonDefaultsWithLatestAvroPayload\n"})}),"\n",(0,o.jsxs)(r.p,{children:["This payload is quite similar to ",(0,o.jsx)(r.code,{children:"OverwriteWithLatestAvroPayload"})," with slight difference while merging records. For\nprecombining, just like ",(0,o.jsx)(r.code,{children:"OverwriteWithLatestAvroPayload"}),", it picks the latest record for a key, based on an ordering\nfield. While merging, it overwrites the existing record on storage only for the specified ",(0,o.jsx)(r.strong,{children:"fields that don't equal\ndefault value"})," for that field."]}),"\n",(0,o.jsx)(r.h4,{id:"partialupdateavropayload",children:"PartialUpdateAvroPayload"}),"\n",(0,o.jsx)(r.pre,{children:(0,o.jsx)(r.code,{className:"language-scala",children:"hoodie.datasource.write.payload.class=org.apache.hudi.common.model.PartialUpdateAvroPayload\n"})}),"\n",(0,o.jsxs)(r.p,{children:["This payload supports partial update. Typically, once the merge step resolves which record to pick, then the record on\nstorage is fully replaced by the resolved record. But, in some cases, the requirement is to update only certain fields\nand not replace the whole record. This is called partial update. ",(0,o.jsx)(r.code,{children:"PartialUpdateAvroPayload"})," provides out-of-box support\nfor such use cases. To illustrate the point, let us look at a simple example:"]}),"\n",(0,o.jsxs)(r.p,{children:["Let's say the ordering field is ",(0,o.jsx)(r.code,{children:"ts"})," and record key is ",(0,o.jsx)(r.code,{children:"id"})," and schema is:"]}),"\n",(0,o.jsx)(r.pre,{children:(0,o.jsx)(r.code,{children:'{\n [\n {"name":"id","type":"string"},\n {"name":"ts","type":"long"},\n {"name":"name","type":"string"},\n {"name":"price","type":"string"}\n ]\n}\n'})}),"\n",(0,o.jsx)(r.p,{children:"Current record in storage:"}),"\n",(0,o.jsx)(r.pre,{children:(0,o.jsx)(r.code,{children:" id ts name price\n 1 2 name_1 null\n"})}),"\n",(0,o.jsx)(r.p,{children:"Incoming record:"}),"\n",(0,o.jsx)(r.pre,{children:(0,o.jsx)(r.code,{children:" id ts name price\n 1 1 null price_1\n"})}),"\n",(0,o.jsxs)(r.p,{children:["Result data after merging using ",(0,o.jsx)(r.code,{children:"PartialUpdateAvroPayload"}),":"]}),"\n",(0,o.jsx)(r.pre,{children:(0,o.jsx)(r.code,{children:" id ts name price\n 1 2 name_1 price_1\n"})}),"\n",(0,o.jsx)(r.h4,{id:"configs",children:"Configs"}),"\n",(0,o.jsxs)(r.p,{children:["Payload class can be specified using the below configs. For more advanced configs refer ",(0,o.jsx)(r.a,{href:"https://hudi.apache.org/docs/configurations#RECORD_PAYLOAD",children:"here"})]}),"\n",(0,o.jsx)(r.p,{children:(0,o.jsx)(r.strong,{children:"Spark based configs:"})}),"\n",(0,o.jsxs)(r.table,{children:[(0,o.jsx)(r.thead,{children:(0,o.jsxs)(r.tr,{children:[(0,o.jsx)(r.th,{children:"Config Name"}),(0,o.jsx)(r.th,{children:"Default"}),(0,o.jsx)(r.th,{children:"Description"})]})}),(0,o.jsx)(r.tbody,{children:(0,o.jsxs)(r.tr,{children:[(0,o.jsx)(r.td,{children:"hoodie.datasource.write.payload.class"}),(0,o.jsx)(r.td,{children:"org.apache.hudi.common.model.OverwriteWithLatestAvroPayload (Optional)"}),(0,o.jsxs)(r.td,{children:["Payload class used. Override this, if you like to roll your own merge logic, when upserting/inserting. This will render any value set for PRECOMBINE_FIELD_OPT_VAL in-effective",(0,o.jsx)("br",{}),(0,o.jsx)("br",{}),(0,o.jsx)(r.code,{children:"Config Param: WRITE_PAYLOAD_CLASS_NAME"})]})]})})]}),"\n",(0,o.jsx)(r.p,{children:(0,o.jsx)(r.strong,{children:"Flink based configs:"})}),"\n",(0,o.jsxs)(r.table,{children:[(0,o.jsx)(r.thead,{children:(0,o.jsxs)(r.tr,{children:[(0,o.jsx)(r.th,{children:"Config Name"}),(0,o.jsx)(r.th,{children:"Default"}),(0,o.jsx)(r.th,{children:"Description"})]})}),(0,o.jsx)(r.tbody,{children:(0,o.jsxs)(r.tr,{children:[(0,o.jsx)(r.td,{children:"payload.class"}),(0,o.jsx)(r.td,{children:"org.apache.hudi.common.model.EventTimeAvroPayload (Optional)"}),(0,o.jsxs)(r.td,{children:["Payload class used. Override this, if you like to roll your own merge logic, when upserting/inserting. This will render any value set for the option in-effective",(0,o.jsx)("br",{}),(0,o.jsx)("br",{})," ",(0,o.jsx)(r.code,{children:"Config Param: PAYLOAD_CLASS_NAME"})]})]})})]}),"\n",(0,o.jsxs)(r.p,{children:["There are also quite a few other implementations. Developers may be interested in looking at the hierarchy of ",(0,o.jsx)(r.code,{children:"HoodieRecordPayload"})," interface. For\nexample, ",(0,o.jsx)(r.a,{href:"https://github.com/apache/hudi/blob/e76dd102bcaf8aec5a932e7277ccdbfd73ce1a32/hudi-common/src/main/java/org/apache/hudi/common/model/debezium/MySqlDebeziumAvroPayload.java",children:(0,o.jsx)(r.code,{children:"MySqlDebeziumAvroPayload"})})," and ",(0,o.jsx)(r.a,{href:"https://github.com/apache/hudi/blob/e76dd102bcaf8aec5a932e7277ccdbfd73ce1a32/hudi-common/src/main/java/org/apache/hudi/common/model/debezium/PostgresDebeziumAvroPayload.java",children:(0,o.jsx)(r.code,{children:"PostgresDebeziumAvroPayload"})})," provides support for seamlessly applying changes\ncaptured via Debezium for MySQL and PostgresDB. ",(0,o.jsx)(r.a,{href:"https://github.com/apache/hudi/blob/e76dd102bcaf8aec5a932e7277ccdbfd73ce1a32/hudi-common/src/main/java/org/apache/hudi/common/model/AWSDmsAvroPayload.java",children:(0,o.jsx)(r.code,{children:"AWSDmsAvroPayload"})})," provides support for applying changes captured via Amazon Database Migration Service onto S3.\nFor full configurations, go ",(0,o.jsx)(r.a,{href:"/docs/configurations#RECORD_PAYLOAD",children:"here"})," and please check out ",(0,o.jsx)(r.a,{href:"faq_writing_tables/#can-i-implement-my-own-logic-for-how-input-records-are-merged-with-record-on-storage",children:"this FAQ"})," if you want to implement your own custom payloads."]})]})}function h(e={}){const{wrapper:r}={...(0,t.R)(),...e.components};return r?(0,o.jsx)(r,{...e,children:(0,o.jsx)(l,{...e})}):l(e)}},38526:(e,r,i)=>{i.d(r,{A:()=>n});const n=i.p+"assets/images/commit-time-ordering-merge-mode-e9b6af3dcdb508053202617218f3ffe6.png"},57159:(e,r,i)=>{i.d(r,{A:()=>n});const n=i.p+"assets/images/event-time-ordering-merge-mode-c8164e035840388bf4290fa81ac6262a.png"},28453:(e,r,i)=>{i.d(r,{R:()=>a,x:()=>s});var n=i(96540);const o={},t=n.createContext(o);function a(e){const r=n.useContext(t);return n.useMemo((function(){return"function"==typeof e?e(r):{...r,...e}}),[r,e])}function s(e){let r;return r=e.disableParentContext?"function"==typeof e.components?e.components(o):e.components||o:a(e.components),n.createElement(t.Provider,{value:r},e.children)}}}]);
\ No newline at end of file
diff --git a/content/cn/assets/js/491d56e0.76e97535.js b/content/cn/assets/js/491d56e0.76e97535.js
new file mode 100644
index 0000000000000..561ad457fa1c2
--- /dev/null
+++ b/content/cn/assets/js/491d56e0.76e97535.js
@@ -0,0 +1 @@
+"use strict";(self.webpackChunkhudi=self.webpackChunkhudi||[]).push([[72907],{48105:(e,r,i)=>{i.r(r),i.d(r,{assets:()=>d,contentTitle:()=>s,default:()=>h,frontMatter:()=>a,metadata:()=>n,toc:()=>c});const n=JSON.parse('{"id":"record_merger","title":"Record Mergers","description":"Hudi handles mutations to records and streaming data, as we briefly touched upon in timeline ordering section.","source":"@site/docs/record_merger.md","sourceDirName":".","slug":"/record_merger","permalink":"/cn/docs/next/record_merger","draft":false,"unlisted":false,"editUrl":"https://github.com/apache/hudi/tree/asf-site/website/docs/record_merger.md","tags":[],"version":"current","frontMatter":{"title":"Record Mergers","keywords":["hudi","merge","upsert","precombine"],"toc":true,"toc_min_heading_level":2,"toc_max_heading_level":4},"sidebar":"docs","previous":{"title":"Key Generation","permalink":"/cn/docs/next/key_generation"},"next":{"title":"Table Metadata","permalink":"/cn/docs/next/metadata"}}');var o=i(74848),t=i(28453);const a={title:"Record Mergers",keywords:["hudi","merge","upsert","precombine"],toc:!0,toc_min_heading_level:2,toc_max_heading_level:4},s=void 0,d={},c=[{value:"COMMIT_TIME_ORDERING",id:"commit_time_ordering",level:3},{value:"EVENT_TIME_ORDERING",id:"event_time_ordering",level:3},{value:"CUSTOM",id:"custom",level:3},{value:"Record Merge Configs",id:"record-merge-configs",level:3},{value:"Record Payloads",id:"record-payloads",level:3},{value:"OverwriteWithLatestAvroPayload",id:"overwritewithlatestavropayload",level:4},{value:"DefaultHoodieRecordPayload",id:"defaulthoodierecordpayload",level:4},{value:"EventTimeAvroPayload",id:"eventtimeavropayload",level:4},{value:"OverwriteNonDefaultsWithLatestAvroPayload",id:"overwritenondefaultswithlatestavropayload",level:4},{value:"PartialUpdateAvroPayload",id:"partialupdateavropayload",level:4},{value:"Configs",id:"configs",level:4},{value:"Related Resources",id:"related-resources",level:2}];function l(e){const r={a:"a",admonition:"admonition",code:"code",em:"em",h2:"h2",h3:"h3",h4:"h4",li:"li",p:"p",pre:"pre",strong:"strong",table:"table",tbody:"tbody",td:"td",th:"th",thead:"thead",tr:"tr",ul:"ul",...(0,t.R)(),...e.components};return(0,o.jsxs)(o.Fragment,{children:[(0,o.jsxs)(r.p,{children:["Hudi handles mutations to records and streaming data, as we briefly touched upon in ",(0,o.jsx)(r.a,{href:"timeline#ordering-of-actions",children:"timeline ordering"})," section.\nTo provide users full-fledged support for stream processing, Hudi goes all the way making the storage engine and the underlying storage format\nunderstand how to merge changes to the same record key, that may arrive even in different order at different times. With the rise of mobile applications\nand IoT, these scenarios have become the normal than an exception. For e.g. a social networking application uploading user events several hours after they happened,\nwhen the user connects to WiFi networks."]}),"\n",(0,o.jsx)(r.p,{children:"To achieve this, Hudi supports merge modes, which define how the base and log files are ordered in a file slice and further how different records with\nthe same record key within that file slice are merged consistently to produce the same deterministic results for snapshot queries, writers and table services. Specifically,\nthere are three merge modes supported as a table-level configuration, invoked in the following places."}),"\n",(0,o.jsxs)(r.ul,{children:["\n",(0,o.jsxs)(r.li,{children:["\n",(0,o.jsxs)(r.p,{children:[(0,o.jsx)(r.strong,{children:"(writing)"})," Combining multiple change records for the same record key while reading input data during writes. This is an optional optimization that\nreduces the number of records written to log files to improve query and write performance subsequently."]}),"\n"]}),"\n",(0,o.jsxs)(r.li,{children:["\n",(0,o.jsxs)(r.p,{children:[(0,o.jsx)(r.strong,{children:"(writing)"})," Merging final change record (partial/full update/delete) against existing record in storage for CoW tables."]}),"\n"]}),"\n",(0,o.jsxs)(r.li,{children:["\n",(0,o.jsxs)(r.p,{children:[(0,o.jsx)(r.strong,{children:"(compaction)"})," Compaction service merges all change records in log files against base files, respecting the merge mode."]}),"\n"]}),"\n",(0,o.jsxs)(r.li,{children:["\n",(0,o.jsxs)(r.p,{children:[(0,o.jsx)(r.strong,{children:"(query)"})," Merging change records in log files, after filtering/projections against base file for MoR table queries."]}),"\n"]}),"\n"]}),"\n",(0,o.jsx)(r.p,{children:"Note that the merge mode should not be altered once the table is created to avoid inconsistent behavior due to compaction producing\ndifferent merge results when switching between the modes."}),"\n",(0,o.jsx)(r.h3,{id:"commit_time_ordering",children:"COMMIT_TIME_ORDERING"}),"\n",(0,o.jsx)(r.p,{children:"Here, we expect the input records to arrive in strict order such that arrival order is same as their\ndelta commit order on the table. Merging simply picks the record belonging to the latest write as the merged result. In relational data mode speak,\nthis provides overwrite semantics aligned with serializable writes on the timeline."}),"\n",(0,o.jsx)("figure",{children:(0,o.jsx)("img",{className:"docimage",src:i(38526).A,alt:"upsert_path.png"})}),"\n",(0,o.jsx)(r.p,{children:"In the example above, the writer process consumes a database change log, expected to be in strict order of a logical sequence number (lsn)\nthat denotes the ordering of the writes in the upstream database."}),"\n",(0,o.jsx)(r.h3,{id:"event_time_ordering",children:"EVENT_TIME_ORDERING"}),"\n",(0,o.jsxs)(r.p,{children:["This is the default merge mode. While commit time ordering provides a well-understood standard behavior, it's hardly sufficient. The commit time is unrelated to the actual\nordering of data that a user may care about and strict ordering of input in complex distributed systems is difficult to achieve.\nWith event time ordering, the merging picks the record with the highest value on a user specified ",(0,o.jsx)(r.em,{children:(0,o.jsx)(r.strong,{children:"ordering or precombine field"})})," as the merged result."]}),"\n",(0,o.jsx)("figure",{children:(0,o.jsx)("img",{className:"docimage",src:i(57159).A,alt:"upsert_path.png"})}),"\n",(0,o.jsxs)(r.p,{children:['In the example above, two microservices product change records about orders at different times, that can arrive out-of-order. As color coded,\nthis can lead to application-level inconsistent states in the table if simply merged in commit time order like a cancelled order being re-created or\na paid order moved back to just created state expecting payment again. Event time ordering helps by ignoring older state changes that arrive late and\navoiding order status from "jumping back" in time. Combined with ',(0,o.jsx)(r.a,{href:"concurrency_control#non-blocking-concurrency-control-mode",children:"non-blocking concurrency control"}),",\nthis provides a very powerful way for processing such data streams efficiently and correctly."]}),"\n",(0,o.jsx)(r.h3,{id:"custom",children:"CUSTOM"}),"\n",(0,o.jsx)(r.p,{children:'In some cases, even more control and customization may be needed. Extending the same example above, the two microservices could be updating two different\nset of columns "order_info" and "payment_info", along with order state. The merge logic is then expected to not only resolve the correct status, but merge\norder_info from the record in created state, into the record in cancelled state that already has payment_info fields populated with reasons payment failed.\nSuch reconciliation provide a simple denormalized data model for downstream consumption where queries (for e.g. fraud detection) can simply filter fields\nacross order_info and payment_info without costly self-join on each access.'}),"\n",(0,o.jsxs)(r.p,{children:["Hudi allows authoring of cross-language custom record mergers on top of a standard record merger API, that supports full and partial merges. The java APIs\nare sketched below at a high-level. It simply takes older/newer records in engine native formats and produces a merged record or returns empty to skip them entirely (e.g. soft deletes).\nRecord merger is configured using a ",(0,o.jsx)(r.code,{children:"hoodie.write.record.merge.strategy.id"})," write config whose value is an uuid, that is taken by the writer to persist in the table config, and is expected to be returned by ",(0,o.jsx)(r.code,{children:"getMergingStrategy()"}),"\nmethod below. Using this mechanism, Hudi can automatically deduce the record merger to use for the table across different language/engine runtimes."]}),"\n",(0,o.jsx)(r.pre,{children:(0,o.jsx)(r.code,{className:"language-Java",children:"interface HoodieRecordMerger {\n\n Option> merge(HoodieRecord older, Schema oldSchema, \n HoodieRecord newer, Schema newSchema, \n TypedProperties props) {\n ...\n }\n\n Option> partialMerge(HoodieRecord older, Schema oldSchema, \n HoodieRecord newer, Schema newSchema, \n Schema readerSchema, TypedProperties props) {\n ...\n }\n \n HoodieRecordType getRecordType() {...}\n \n String getMergingStrategy(); {...}\n}\n"})}),"\n",(0,o.jsx)(r.h3,{id:"record-merge-configs",children:"Record Merge Configs"}),"\n",(0,o.jsx)(r.p,{children:"The record merge mode and optional record merge strategy ID and custom merge implementation classes can be specified using the below configs."}),"\n",(0,o.jsxs)(r.table,{children:[(0,o.jsx)(r.thead,{children:(0,o.jsxs)(r.tr,{children:[(0,o.jsx)(r.th,{children:"Config Name"}),(0,o.jsx)(r.th,{children:"Default"}),(0,o.jsx)(r.th,{children:"Description"})]})}),(0,o.jsxs)(r.tbody,{children:[(0,o.jsxs)(r.tr,{children:[(0,o.jsx)(r.td,{children:"hoodie.write.record.merge.mode"}),(0,o.jsx)(r.td,{children:"EVENT_TIME_ORDERING"}),(0,o.jsxs)(r.td,{children:["Determines the logic of merging different records with the same record key. Valid values: (1) ",(0,o.jsx)(r.code,{children:"COMMIT_TIME_ORDERING"}),": use commit time to merge records, i.e., the record from later commit overwrites the earlier record with the same key. (2) ",(0,o.jsx)(r.code,{children:"EVENT_TIME_ORDERING"})," (default): use event time as the ordering to merge records, i.e., the record with the larger event time overwrites the record with the smaller event time on the same key, regardless of commit time. The event time or preCombine field needs to be specified by the user. (3) ",(0,o.jsx)(r.code,{children:"CUSTOM"}),": use custom merging logic specified by the user.",(0,o.jsx)("br",{}),(0,o.jsx)(r.code,{children:"Config Param: RECORD_MERGE_MODE"}),(0,o.jsx)("br",{}),(0,o.jsx)(r.code,{children:"Since Version: 1.0.0"})]})]}),(0,o.jsxs)(r.tr,{children:[(0,o.jsx)(r.td,{children:"hoodie.write.record.merge.strategy.id"}),(0,o.jsx)(r.td,{children:"N/A (Optional)"}),(0,o.jsxs)(r.td,{children:["ID of record merge strategy. When you specify this config, you also need to specify ",(0,o.jsx)(r.code,{children:"hoodie.write.record.merge.custom.implementation.classes"}),". Hudi picks the ",(0,o.jsx)(r.code,{children:"HoodieRecordMerger"})," implementation class from the list of classes in ",(0,o.jsx)(r.code,{children:"hoodie.write.record.merge.custom.implementation.classes"})," that has the specified merge strategy ID.",(0,o.jsx)("br",{}),(0,o.jsx)(r.code,{children:"Config Param: RECORD_MERGE_STRATEGY_ID"}),(0,o.jsx)("br",{}),(0,o.jsx)(r.code,{children:"Since Version: 0.13.0"})]})]}),(0,o.jsxs)(r.tr,{children:[(0,o.jsx)(r.td,{children:"hoodie.write.record.merge.custom.implementation.classes"}),(0,o.jsx)(r.td,{children:"N/A (Optional)"}),(0,o.jsxs)(r.td,{children:["List of ",(0,o.jsx)(r.code,{children:"HoodieRecordMerger"})," implementations constituting Hudi's merging strategy based on the engine used. Hudi picks the ",(0,o.jsx)(r.code,{children:"HoodieRecordMerger"})," implementation class from this list based on the specified ",(0,o.jsx)(r.code,{children:"hoodie.write.record.merge.strategy.id"}),".",(0,o.jsx)("br",{}),(0,o.jsx)(r.code,{children:"Config Param: RECORD_MERGE_IMPL_CLASSES"}),(0,o.jsx)("br",{}),(0,o.jsx)(r.code,{children:"Since Version: 0.13.0"})]})]})]})]}),"\n",(0,o.jsx)(r.h3,{id:"record-payloads",children:"Record Payloads"}),"\n",(0,o.jsx)(r.admonition,{type:"caution",children:(0,o.jsx)(r.p,{children:"Going forward, we recommend users to migrate and use the record merger APIs and not write new payload implementations."})}),"\n",(0,o.jsx)(r.p,{children:"Record payload is an older abstraction/API for achieving similar record-level merge capabilities. While record payloads were very useful and popular,\nit had drawbacks like lower performance due to conversion of engine native record formats to Apache Avro for merging and lack of cross-language support.\nAs we shall see below, Hudi provides out-of-box support for different payloads for different use cases. Hudi implements fallback from\nrecord merger APIs to payload APIs internally, to provide backwards compatibility for existing payload implementations."}),"\n",(0,o.jsx)(r.h4,{id:"overwritewithlatestavropayload",children:"OverwriteWithLatestAvroPayload"}),"\n",(0,o.jsx)(r.pre,{children:(0,o.jsx)(r.code,{className:"language-scala",children:"hoodie.datasource.write.payload.class=org.apache.hudi.common.model.OverwriteWithLatestAvroPayload\n"})}),"\n",(0,o.jsxs)(r.p,{children:["This is the default record payload implementation. It picks the record with the greatest value (determined by calling\n",(0,o.jsx)(r.code,{children:".compareTo()"})," on the value of precombine key) to break ties and simply picks the latest record while merging. This gives\nlatest-write-wins style semantics."]}),"\n",(0,o.jsx)(r.h4,{id:"defaulthoodierecordpayload",children:"DefaultHoodieRecordPayload"}),"\n",(0,o.jsx)(r.pre,{children:(0,o.jsx)(r.code,{className:"language-scala",children:"hoodie.datasource.write.payload.class=org.apache.hudi.common.model.DefaultHoodieRecordPayload\n"})}),"\n",(0,o.jsxs)(r.p,{children:["While ",(0,o.jsx)(r.code,{children:"OverwriteWithLatestAvroPayload"})," precombines based on an ordering field and picks the latest record while merging,\n",(0,o.jsx)(r.code,{children:"DefaultHoodieRecordPayload"})," honors the ordering field for both precombinig and merging. Let's understand the difference with an example:"]}),"\n",(0,o.jsxs)(r.p,{children:["Let's say the ordering field is ",(0,o.jsx)(r.code,{children:"ts"})," and record key is ",(0,o.jsx)(r.code,{children:"id"})," and schema is:"]}),"\n",(0,o.jsx)(r.pre,{children:(0,o.jsx)(r.code,{children:'{\n [\n {"name":"id","type":"string"},\n {"name":"ts","type":"long"},\n {"name":"name","type":"string"},\n {"name":"price","type":"string"}\n ]\n}\n'})}),"\n",(0,o.jsx)(r.p,{children:"Current record in storage:"}),"\n",(0,o.jsx)(r.pre,{children:(0,o.jsx)(r.code,{children:" id ts name price\n 1 2 name_2 price_2\n"})}),"\n",(0,o.jsx)(r.p,{children:"Incoming record:"}),"\n",(0,o.jsx)(r.pre,{children:(0,o.jsx)(r.code,{children:" id ts name price\n 1 1 name_1 price_1\n"})}),"\n",(0,o.jsxs)(r.p,{children:["Result data after merging using ",(0,o.jsx)(r.code,{children:"OverwriteWithLatestAvroPayload"})," (latest-write-wins):"]}),"\n",(0,o.jsx)(r.pre,{children:(0,o.jsx)(r.code,{children:" id ts name price\n 1 1 name_1 price_1\n"})}),"\n",(0,o.jsxs)(r.p,{children:["Result data after merging using ",(0,o.jsx)(r.code,{children:"DefaultHoodieRecordPayload"})," (always honors ordering field):"]}),"\n",(0,o.jsx)(r.pre,{children:(0,o.jsx)(r.code,{children:" id ts name price\n 1 2 name_2 price_2\n"})}),"\n",(0,o.jsx)(r.h4,{id:"eventtimeavropayload",children:"EventTimeAvroPayload"}),"\n",(0,o.jsx)(r.pre,{children:(0,o.jsx)(r.code,{className:"language-scala",children:"hoodie.datasource.write.payload.class=org.apache.hudi.common.model.EventTimeAvroPayload\n"})}),"\n",(0,o.jsxs)(r.p,{children:["This is the default record payload for Flink based writing. Some use cases require merging records by event time and\nthus event time plays the role of an ordering field. This payload is particularly useful in the case of late-arriving data.\nFor such use cases, users need to set the ",(0,o.jsx)(r.a,{href:"/docs/configurations#RECORD_PAYLOAD",children:"payload event time field"})," configuration."]}),"\n",(0,o.jsx)(r.h4,{id:"overwritenondefaultswithlatestavropayload",children:"OverwriteNonDefaultsWithLatestAvroPayload"}),"\n",(0,o.jsx)(r.pre,{children:(0,o.jsx)(r.code,{className:"language-scala",children:"hoodie.datasource.write.payload.class=org.apache.hudi.common.model.OverwriteNonDefaultsWithLatestAvroPayload\n"})}),"\n",(0,o.jsxs)(r.p,{children:["This payload is quite similar to ",(0,o.jsx)(r.code,{children:"OverwriteWithLatestAvroPayload"})," with slight difference while merging records. For\nprecombining, just like ",(0,o.jsx)(r.code,{children:"OverwriteWithLatestAvroPayload"}),", it picks the latest record for a key, based on an ordering\nfield. While merging, it overwrites the existing record on storage only for the specified ",(0,o.jsx)(r.strong,{children:"fields that don't equal\ndefault value"})," for that field."]}),"\n",(0,o.jsx)(r.h4,{id:"partialupdateavropayload",children:"PartialUpdateAvroPayload"}),"\n",(0,o.jsx)(r.pre,{children:(0,o.jsx)(r.code,{className:"language-scala",children:"hoodie.datasource.write.payload.class=org.apache.hudi.common.model.PartialUpdateAvroPayload\n"})}),"\n",(0,o.jsxs)(r.p,{children:["This payload supports partial update. Typically, once the merge step resolves which record to pick, then the record on\nstorage is fully replaced by the resolved record. But, in some cases, the requirement is to update only certain fields\nand not replace the whole record. This is called partial update. ",(0,o.jsx)(r.code,{children:"PartialUpdateAvroPayload"})," provides out-of-box support\nfor such use cases. To illustrate the point, let us look at a simple example:"]}),"\n",(0,o.jsxs)(r.p,{children:["Let's say the ordering field is ",(0,o.jsx)(r.code,{children:"ts"})," and record key is ",(0,o.jsx)(r.code,{children:"id"})," and schema is:"]}),"\n",(0,o.jsx)(r.pre,{children:(0,o.jsx)(r.code,{children:'{\n [\n {"name":"id","type":"string"},\n {"name":"ts","type":"long"},\n {"name":"name","type":"string"},\n {"name":"price","type":"string"}\n ]\n}\n'})}),"\n",(0,o.jsx)(r.p,{children:"Current record in storage:"}),"\n",(0,o.jsx)(r.pre,{children:(0,o.jsx)(r.code,{children:" id ts name price\n 1 2 name_1 null\n"})}),"\n",(0,o.jsx)(r.p,{children:"Incoming record:"}),"\n",(0,o.jsx)(r.pre,{children:(0,o.jsx)(r.code,{children:" id ts name price\n 1 1 null price_1\n"})}),"\n",(0,o.jsxs)(r.p,{children:["Result data after merging using ",(0,o.jsx)(r.code,{children:"PartialUpdateAvroPayload"}),":"]}),"\n",(0,o.jsx)(r.pre,{children:(0,o.jsx)(r.code,{children:" id ts name price\n 1 2 name_1 price_1\n"})}),"\n",(0,o.jsx)(r.h4,{id:"configs",children:"Configs"}),"\n",(0,o.jsxs)(r.p,{children:["Payload class can be specified using the below configs. For more advanced configs refer ",(0,o.jsx)(r.a,{href:"https://hudi.apache.org/docs/configurations#RECORD_PAYLOAD",children:"here"})]}),"\n",(0,o.jsx)(r.p,{children:(0,o.jsx)(r.strong,{children:"Spark based configs:"})}),"\n",(0,o.jsxs)(r.table,{children:[(0,o.jsx)(r.thead,{children:(0,o.jsxs)(r.tr,{children:[(0,o.jsx)(r.th,{children:"Config Name"}),(0,o.jsx)(r.th,{children:"Default"}),(0,o.jsx)(r.th,{children:"Description"})]})}),(0,o.jsx)(r.tbody,{children:(0,o.jsxs)(r.tr,{children:[(0,o.jsx)(r.td,{children:"hoodie.datasource.write.payload.class"}),(0,o.jsx)(r.td,{children:"org.apache.hudi.common.model.OverwriteWithLatestAvroPayload (Optional)"}),(0,o.jsxs)(r.td,{children:["Payload class used. Override this, if you like to roll your own merge logic, when upserting/inserting. This will render any value set for PRECOMBINE_FIELD_OPT_VAL in-effective",(0,o.jsx)("br",{}),(0,o.jsx)("br",{}),(0,o.jsx)(r.code,{children:"Config Param: WRITE_PAYLOAD_CLASS_NAME"})]})]})})]}),"\n",(0,o.jsx)(r.p,{children:(0,o.jsx)(r.strong,{children:"Flink based configs:"})}),"\n",(0,o.jsxs)(r.table,{children:[(0,o.jsx)(r.thead,{children:(0,o.jsxs)(r.tr,{children:[(0,o.jsx)(r.th,{children:"Config Name"}),(0,o.jsx)(r.th,{children:"Default"}),(0,o.jsx)(r.th,{children:"Description"})]})}),(0,o.jsx)(r.tbody,{children:(0,o.jsxs)(r.tr,{children:[(0,o.jsx)(r.td,{children:"payload.class"}),(0,o.jsx)(r.td,{children:"org.apache.hudi.common.model.EventTimeAvroPayload (Optional)"}),(0,o.jsxs)(r.td,{children:["Payload class used. Override this, if you like to roll your own merge logic, when upserting/inserting. This will render any value set for the option in-effective",(0,o.jsx)("br",{}),(0,o.jsx)("br",{})," ",(0,o.jsx)(r.code,{children:"Config Param: PAYLOAD_CLASS_NAME"})]})]})})]}),"\n",(0,o.jsxs)(r.p,{children:["There are also quite a few other implementations. Developers may be interested in looking at the hierarchy of ",(0,o.jsx)(r.code,{children:"HoodieRecordPayload"})," interface. For\nexample, ",(0,o.jsx)(r.a,{href:"https://github.com/apache/hudi/blob/e76dd102bcaf8aec5a932e7277ccdbfd73ce1a32/hudi-common/src/main/java/org/apache/hudi/common/model/debezium/MySqlDebeziumAvroPayload.java",children:(0,o.jsx)(r.code,{children:"MySqlDebeziumAvroPayload"})})," and ",(0,o.jsx)(r.a,{href:"https://github.com/apache/hudi/blob/e76dd102bcaf8aec5a932e7277ccdbfd73ce1a32/hudi-common/src/main/java/org/apache/hudi/common/model/debezium/PostgresDebeziumAvroPayload.java",children:(0,o.jsx)(r.code,{children:"PostgresDebeziumAvroPayload"})})," provides support for seamlessly applying changes\ncaptured via Debezium for MySQL and PostgresDB. ",(0,o.jsx)(r.a,{href:"https://github.com/apache/hudi/blob/e76dd102bcaf8aec5a932e7277ccdbfd73ce1a32/hudi-common/src/main/java/org/apache/hudi/common/model/AWSDmsAvroPayload.java",children:(0,o.jsx)(r.code,{children:"AWSDmsAvroPayload"})})," provides support for applying changes captured via Amazon Database Migration Service onto S3.\nFor full configurations, go ",(0,o.jsx)(r.a,{href:"/docs/configurations#RECORD_PAYLOAD",children:"here"})," and please check out ",(0,o.jsx)(r.a,{href:"faq_writing_tables/#can-i-implement-my-own-logic-for-how-input-records-are-merged-with-record-on-storage",children:"this FAQ"})," if you want to implement your own custom payloads."]}),"\n",(0,o.jsx)(r.h2,{id:"related-resources",children:"Related Resources"}),"\n",(0,o.jsx)("h3",{children:"Blogs"}),"\n",(0,o.jsxs)(r.ul,{children:["\n",(0,o.jsx)(r.li,{children:(0,o.jsx)(r.a,{href:"https://medium.com/@simpsons/how-to-define-your-own-merge-logic-with-apache-hudi-622ee5ccab1e",children:"How to define your own merge logic with Apache Hudi"})}),"\n"]})]})}function h(e={}){const{wrapper:r}={...(0,t.R)(),...e.components};return r?(0,o.jsx)(r,{...e,children:(0,o.jsx)(l,{...e})}):l(e)}},38526:(e,r,i)=>{i.d(r,{A:()=>n});const n=i.p+"assets/images/commit-time-ordering-merge-mode-e9b6af3dcdb508053202617218f3ffe6.png"},57159:(e,r,i)=>{i.d(r,{A:()=>n});const n=i.p+"assets/images/event-time-ordering-merge-mode-c8164e035840388bf4290fa81ac6262a.png"},28453:(e,r,i)=>{i.d(r,{R:()=>a,x:()=>s});var n=i(96540);const o={},t=n.createContext(o);function a(e){const r=n.useContext(t);return n.useMemo((function(){return"function"==typeof e?e(r):{...r,...e}}),[r,e])}function s(e){let r;return r=e.disableParentContext?"function"==typeof e.components?e.components(o):e.components||o:a(e.components),n.createElement(t.Provider,{value:r},e.children)}}}]);
\ No newline at end of file
diff --git a/content/cn/assets/js/5733876a.664154f3.js b/content/cn/assets/js/5733876a.664154f3.js
deleted file mode 100644
index 0c43ba6000e18..0000000000000
--- a/content/cn/assets/js/5733876a.664154f3.js
+++ /dev/null
@@ -1 +0,0 @@
-"use strict";(self.webpackChunkhudi=self.webpackChunkhudi||[]).push([[85973],{91855:(e,t,a)=>{a.r(t),a.d(t,{assets:()=>c,contentTitle:()=>u,default:()=>m,frontMatter:()=>l,metadata:()=>n,toc:()=>d});const n=JSON.parse('{"id":"writing_tables_streaming_writes","title":"Streaming Writes","description":"Spark Streaming","source":"@site/docs/writing_tables_streaming_writes.md","sourceDirName":".","slug":"/writing_tables_streaming_writes","permalink":"/cn/docs/next/writing_tables_streaming_writes","draft":false,"unlisted":false,"editUrl":"https://github.com/apache/hudi/tree/asf-site/website/docs/writing_tables_streaming_writes.md","tags":[],"version":"current","frontMatter":{"title":"Streaming Writes","keywords":["hudi","spark","flink","streaming","processing"],"last_modified_at":"2024-03-13T19:59:57.000Z"},"sidebar":"docs","previous":{"title":"\u5199\u5165 Hudi \u6570\u636e\u96c6","permalink":"/cn/docs/next/writing_data"},"next":{"title":"SQL Queries","permalink":"/cn/docs/next/sql_queries"}}');var r=a(74848),i=a(28453),s=a(11470),o=a(19365);const l={title:"Streaming Writes",keywords:["hudi","spark","flink","streaming","processing"],last_modified_at:new Date("2024-03-13T19:59:57.000Z")},u=void 0,c={},d=[{value:"Spark Streaming",id:"spark-streaming",level:2}];function p(e){const t={code:"code",h2:"h2",p:"p",pre:"pre",...(0,i.R)(),...e.components};return(0,r.jsxs)(r.Fragment,{children:[(0,r.jsx)(t.h2,{id:"spark-streaming",children:"Spark Streaming"}),"\n",(0,r.jsx)(t.p,{children:"You can write Hudi tables using spark's structured streaming."}),"\n",(0,r.jsxs)(s.A,{groupId:"programming-language",defaultValue:"python",values:[{label:"Scala",value:"scala"},{label:"Python",value:"python"}],children:[(0,r.jsx)(o.A,{value:"scala",children:(0,r.jsx)(t.pre,{children:(0,r.jsx)(t.code,{className:"language-scala",children:'// spark-shell\n// prepare to stream write to new table\nimport org.apache.spark.sql.streaming.Trigger\n\nval streamingTableName = "hudi_trips_cow_streaming"\nval baseStreamingPath = "file:///tmp/hudi_trips_cow_streaming"\nval checkpointLocation = "file:///tmp/checkpoints/hudi_trips_cow_streaming"\n\n// create streaming df\nval df = spark.readStream.\n format("hudi").\n load(basePath)\n\n// write stream to new hudi table\ndf.writeStream.format("hudi").\n options(getQuickstartWriteConfigs).\n option("hoodie.datasource.write.precombine.field", "ts").\n option("hoodie.datasource.write.recordkey.field", "uuid").\n option("hoodie.datasource.write.partitionpath.field", "partitionpath").\n option("hoodie.table.name", streamingTableName).\n outputMode("append").\n option("path", baseStreamingPath).\n option("checkpointLocation", checkpointLocation).\n trigger(Trigger.Once()).\n start()\n\n'})})}),(0,r.jsx)(o.A,{value:"python",children:(0,r.jsx)(t.pre,{children:(0,r.jsx)(t.code,{className:"language-python",children:"# pyspark\n# prepare to stream write to new table\nstreamingTableName = \"hudi_trips_cow_streaming\"\nbaseStreamingPath = \"file:///tmp/hudi_trips_cow_streaming\"\ncheckpointLocation = \"file:///tmp/checkpoints/hudi_trips_cow_streaming\"\n\nhudi_streaming_options = {\n 'hoodie.table.name': streamingTableName,\n 'hoodie.datasource.write.recordkey.field': 'uuid',\n 'hoodie.datasource.write.partitionpath.field': 'partitionpath',\n 'hoodie.datasource.write.table.name': streamingTableName,\n 'hoodie.datasource.write.operation': 'upsert',\n 'hoodie.datasource.write.precombine.field': 'ts',\n 'hoodie.upsert.shuffle.parallelism': 2,\n 'hoodie.insert.shuffle.parallelism': 2\n}\n\n# create streaming df\ndf = spark.readStream \n .format(\"hudi\") \n .load(basePath)\n\n# write stream to new hudi table\ndf.writeStream.format(\"hudi\") \n .options(**hudi_streaming_options) \n .outputMode(\"append\") \n .option(\"path\", baseStreamingPath) \n .option(\"checkpointLocation\", checkpointLocation) \n .trigger(once=True) \n .start()\n\n"})})})]})]})}function m(e={}){const{wrapper:t}={...(0,i.R)(),...e.components};return t?(0,r.jsx)(t,{...e,children:(0,r.jsx)(p,{...e})}):p(e)}},19365:(e,t,a)=>{a.d(t,{A:()=>s});a(96540);var n=a(34164);const r={tabItem:"tabItem_Ymn6"};var i=a(74848);function s(e){let{children:t,hidden:a,className:s}=e;return(0,i.jsx)("div",{role:"tabpanel",className:(0,n.A)(r.tabItem,s),hidden:a,children:t})}},11470:(e,t,a)=>{a.d(t,{A:()=>y});var n=a(96540),r=a(34164),i=a(23104),s=a(56347),o=a(205),l=a(57485),u=a(31682),c=a(70679);function d(e){return n.Children.toArray(e).filter((e=>"\n"!==e)).map((e=>{if(!e||(0,n.isValidElement)(e)&&function(e){const{props:t}=e;return!!t&&"object"==typeof t&&"value"in t}(e))return e;throw new Error(`Docusaurus error: Bad child <${"string"==typeof e.type?e.type:e.type.name}>: all children of the component should be , and every should have a unique "value" prop.`)}))?.filter(Boolean)??[]}function p(e){const{values:t,children:a}=e;return(0,n.useMemo)((()=>{const e=t??function(e){return d(e).map((e=>{let{props:{value:t,label:a,attributes:n,default:r}}=e;return{value:t,label:a,attributes:n,default:r}}))}(a);return function(e){const t=(0,u.XI)(e,((e,t)=>e.value===t.value));if(t.length>0)throw new Error(`Docusaurus error: Duplicate values "${t.map((e=>e.value)).join(", ")}" found in . Every value needs to be unique.`)}(e),e}),[t,a])}function m(e){let{value:t,tabValues:a}=e;return a.some((e=>e.value===t))}function h(e){let{queryString:t=!1,groupId:a}=e;const r=(0,s.W6)(),i=function(e){let{queryString:t=!1,groupId:a}=e;if("string"==typeof t)return t;if(!1===t)return null;if(!0===t&&!a)throw new Error('Docusaurus error: The component groupId prop is required if queryString=true, because this value is used as the search param name. You can also provide an explicit value such as queryString="my-search-param".');return a??null}({queryString:t,groupId:a});return[(0,l.aZ)(i),(0,n.useCallback)((e=>{if(!i)return;const t=new URLSearchParams(r.location.search);t.set(i,e),r.replace({...r.location,search:t.toString()})}),[i,r])]}function f(e){const{defaultValue:t,queryString:a=!1,groupId:r}=e,i=p(e),[s,l]=(0,n.useState)((()=>function(e){let{defaultValue:t,tabValues:a}=e;if(0===a.length)throw new Error("Docusaurus error: the component requires at least one children component");if(t){if(!m({value:t,tabValues:a}))throw new Error(`Docusaurus error: The has a defaultValue "${t}" but none of its children has the corresponding value. Available values are: ${a.map((e=>e.value)).join(", ")}. If you intend to show no default tab, use defaultValue={null} instead.`);return t}const n=a.find((e=>e.default))??a[0];if(!n)throw new Error("Unexpected error: 0 tabValues");return n.value}({defaultValue:t,tabValues:i}))),[u,d]=h({queryString:a,groupId:r}),[f,g]=function(e){let{groupId:t}=e;const a=function(e){return e?`docusaurus.tab.${e}`:null}(t),[r,i]=(0,c.Dv)(a);return[r,(0,n.useCallback)((e=>{a&&i.set(e)}),[a,i])]}({groupId:r}),b=(()=>{const e=u??f;return m({value:e,tabValues:i})?e:null})();(0,o.A)((()=>{b&&l(b)}),[b]);return{selectedValue:s,selectValue:(0,n.useCallback)((e=>{if(!m({value:e,tabValues:i}))throw new Error(`Can't select invalid tab value=${e}`);l(e),d(e),g(e)}),[d,g,i]),tabValues:i}}var g=a(92303);const b={tabList:"tabList__CuJ",tabItem:"tabItem_LNqP"};var w=a(74848);function v(e){let{className:t,block:a,selectedValue:n,selectValue:s,tabValues:o}=e;const l=[],{blockElementScrollPositionUntilNextRender:u}=(0,i.a_)(),c=e=>{const t=e.currentTarget,a=l.indexOf(t),r=o[a].value;r!==n&&(u(t),s(r))},d=e=>{let t=null;switch(e.key){case"Enter":c(e);break;case"ArrowRight":{const a=l.indexOf(e.currentTarget)+1;t=l[a]??l[0];break}case"ArrowLeft":{const a=l.indexOf(e.currentTarget)-1;t=l[a]??l[l.length-1];break}}t?.focus()};return(0,w.jsx)("ul",{role:"tablist","aria-orientation":"horizontal",className:(0,r.A)("tabs",{"tabs--block":a},t),children:o.map((e=>{let{value:t,label:a,attributes:i}=e;return(0,w.jsx)("li",{role:"tab",tabIndex:n===t?0:-1,"aria-selected":n===t,ref:e=>l.push(e),onKeyDown:d,onClick:c,...i,className:(0,r.A)("tabs__item",b.tabItem,i?.className,{"tabs__item--active":n===t}),children:a??t},t)}))})}function _(e){let{lazy:t,children:a,selectedValue:i}=e;const s=(Array.isArray(a)?a:[a]).filter(Boolean);if(t){const e=s.find((e=>e.props.value===i));return e?(0,n.cloneElement)(e,{className:(0,r.A)("margin-top--md",e.props.className)}):null}return(0,w.jsx)("div",{className:"margin-top--md",children:s.map(((e,t)=>(0,n.cloneElement)(e,{key:t,hidden:e.props.value!==i})))})}function k(e){const t=f(e);return(0,w.jsxs)("div",{className:(0,r.A)("tabs-container",b.tabList),children:[(0,w.jsx)(v,{...t,...e}),(0,w.jsx)(_,{...t,...e})]})}function y(e){const t=(0,g.A)();return(0,w.jsx)(k,{...e,children:d(e.children)},String(t))}},28453:(e,t,a)=>{a.d(t,{R:()=>s,x:()=>o});var n=a(96540);const r={},i=n.createContext(r);function s(e){const t=n.useContext(i);return n.useMemo((function(){return"function"==typeof e?e(t):{...t,...e}}),[t,e])}function o(e){let t;return t=e.disableParentContext?"function"==typeof e.components?e.components(r):e.components||r:s(e.components),n.createElement(i.Provider,{value:t},e.children)}}}]);
\ No newline at end of file
diff --git a/content/cn/assets/js/5733876a.a87a72dc.js b/content/cn/assets/js/5733876a.a87a72dc.js
new file mode 100644
index 0000000000000..a9fd6ff2e82d0
--- /dev/null
+++ b/content/cn/assets/js/5733876a.a87a72dc.js
@@ -0,0 +1 @@
+"use strict";(self.webpackChunkhudi=self.webpackChunkhudi||[]).push([[85973],{91855:(e,t,n)=>{n.r(t),n.d(t,{assets:()=>c,contentTitle:()=>u,default:()=>h,frontMatter:()=>l,metadata:()=>r,toc:()=>d});const r=JSON.parse('{"id":"writing_tables_streaming_writes","title":"Streaming Writes","description":"Spark Streaming","source":"@site/docs/writing_tables_streaming_writes.md","sourceDirName":".","slug":"/writing_tables_streaming_writes","permalink":"/cn/docs/next/writing_tables_streaming_writes","draft":false,"unlisted":false,"editUrl":"https://github.com/apache/hudi/tree/asf-site/website/docs/writing_tables_streaming_writes.md","tags":[],"version":"current","frontMatter":{"title":"Streaming Writes","keywords":["hudi","spark","flink","streaming","processing"],"last_modified_at":"2024-03-13T19:59:57.000Z"},"sidebar":"docs","previous":{"title":"\u5199\u5165 Hudi \u6570\u636e\u96c6","permalink":"/cn/docs/next/writing_data"},"next":{"title":"SQL Queries","permalink":"/cn/docs/next/sql_queries"}}');var a=n(74848),i=n(28453),s=n(11470),o=n(19365);const l={title:"Streaming Writes",keywords:["hudi","spark","flink","streaming","processing"],last_modified_at:new Date("2024-03-13T19:59:57.000Z")},u=void 0,c={},d=[{value:"Spark Streaming",id:"spark-streaming",level:2},{value:"Related Resources",id:"related-resources",level:2}];function p(e){const t={a:"a",code:"code",h2:"h2",li:"li",p:"p",pre:"pre",ul:"ul",...(0,i.R)(),...e.components};return(0,a.jsxs)(a.Fragment,{children:[(0,a.jsx)(t.h2,{id:"spark-streaming",children:"Spark Streaming"}),"\n",(0,a.jsx)(t.p,{children:"You can write Hudi tables using spark's structured streaming."}),"\n",(0,a.jsxs)(s.A,{groupId:"programming-language",defaultValue:"python",values:[{label:"Scala",value:"scala"},{label:"Python",value:"python"}],children:[(0,a.jsx)(o.A,{value:"scala",children:(0,a.jsx)(t.pre,{children:(0,a.jsx)(t.code,{className:"language-scala",children:'// spark-shell\n// prepare to stream write to new table\nimport org.apache.spark.sql.streaming.Trigger\n\nval streamingTableName = "hudi_trips_cow_streaming"\nval baseStreamingPath = "file:///tmp/hudi_trips_cow_streaming"\nval checkpointLocation = "file:///tmp/checkpoints/hudi_trips_cow_streaming"\n\n// create streaming df\nval df = spark.readStream.\n format("hudi").\n load(basePath)\n\n// write stream to new hudi table\ndf.writeStream.format("hudi").\n options(getQuickstartWriteConfigs).\n option("hoodie.datasource.write.precombine.field", "ts").\n option("hoodie.datasource.write.recordkey.field", "uuid").\n option("hoodie.datasource.write.partitionpath.field", "partitionpath").\n option("hoodie.table.name", streamingTableName).\n outputMode("append").\n option("path", baseStreamingPath).\n option("checkpointLocation", checkpointLocation).\n trigger(Trigger.Once()).\n start()\n\n'})})}),(0,a.jsx)(o.A,{value:"python",children:(0,a.jsx)(t.pre,{children:(0,a.jsx)(t.code,{className:"language-python",children:"# pyspark\n# prepare to stream write to new table\nstreamingTableName = \"hudi_trips_cow_streaming\"\nbaseStreamingPath = \"file:///tmp/hudi_trips_cow_streaming\"\ncheckpointLocation = \"file:///tmp/checkpoints/hudi_trips_cow_streaming\"\n\nhudi_streaming_options = {\n 'hoodie.table.name': streamingTableName,\n 'hoodie.datasource.write.recordkey.field': 'uuid',\n 'hoodie.datasource.write.partitionpath.field': 'partitionpath',\n 'hoodie.datasource.write.table.name': streamingTableName,\n 'hoodie.datasource.write.operation': 'upsert',\n 'hoodie.datasource.write.precombine.field': 'ts',\n 'hoodie.upsert.shuffle.parallelism': 2,\n 'hoodie.insert.shuffle.parallelism': 2\n}\n\n# create streaming df\ndf = spark.readStream \n .format(\"hudi\") \n .load(basePath)\n\n# write stream to new hudi table\ndf.writeStream.format(\"hudi\") \n .options(**hudi_streaming_options) \n .outputMode(\"append\") \n .option(\"path\", baseStreamingPath) \n .option(\"checkpointLocation\", checkpointLocation) \n .trigger(once=True) \n .start()\n\n"})})})]}),"\n",(0,a.jsx)(t.h2,{id:"related-resources",children:"Related Resources"}),"\n",(0,a.jsx)("h3",{children:"Blogs"}),"\n",(0,a.jsxs)(t.ul,{children:["\n",(0,a.jsx)(t.li,{children:(0,a.jsx)(t.a,{href:"https://www.onehouse.ai/blog/intro-to-hudi-and-flink",children:"An Introduction to the Hudi and Flink Integration"})}),"\n",(0,a.jsx)(t.li,{children:(0,a.jsx)(t.a,{href:"https://medium.com/@simpsons/bulk-insert-sort-modes-with-apache-hudi-c781e77841bc",children:"Bulk Insert Sort Modes with Apache Hudi"})}),"\n"]})]})}function h(e={}){const{wrapper:t}={...(0,i.R)(),...e.components};return t?(0,a.jsx)(t,{...e,children:(0,a.jsx)(p,{...e})}):p(e)}},19365:(e,t,n)=>{n.d(t,{A:()=>s});n(96540);var r=n(34164);const a={tabItem:"tabItem_Ymn6"};var i=n(74848);function s(e){let{children:t,hidden:n,className:s}=e;return(0,i.jsx)("div",{role:"tabpanel",className:(0,r.A)(a.tabItem,s),hidden:n,children:t})}},11470:(e,t,n)=>{n.d(t,{A:()=>x});var r=n(96540),a=n(34164),i=n(23104),s=n(56347),o=n(205),l=n(57485),u=n(31682),c=n(70679);function d(e){return r.Children.toArray(e).filter((e=>"\n"!==e)).map((e=>{if(!e||(0,r.isValidElement)(e)&&function(e){const{props:t}=e;return!!t&&"object"==typeof t&&"value"in t}(e))return e;throw new Error(`Docusaurus error: Bad child <${"string"==typeof e.type?e.type:e.type.name}>: all children of the component should be , and every