From 6ca05d88c452dcde2c63341ffcb25afee60c91c2 Mon Sep 17 00:00:00 2001 From: Rishabh Goyal Date: Sat, 21 Nov 2015 22:34:00 +0530 Subject: [PATCH 01/64] Hazelcast Upgrade --- foxtrot-core/pom.xml | 4 ++-- foxtrot-server/pom.xml | 2 +- .../com/flipkart/foxtrot/server/cluster/ClusterManager.java | 3 ++- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/foxtrot-core/pom.xml b/foxtrot-core/pom.xml index 18a1e6bba..ab0ed87f4 100644 --- a/foxtrot-core/pom.xml +++ b/foxtrot-core/pom.xml @@ -64,13 +64,13 @@ com.hazelcast hazelcast - 3.3.3 + 3.5.3 com.hazelcast hazelcast - 3.3.3 + 3.5.3 test-jar test diff --git a/foxtrot-server/pom.xml b/foxtrot-server/pom.xml index 79dd50ed9..3021dfd74 100644 --- a/foxtrot-server/pom.xml +++ b/foxtrot-server/pom.xml @@ -62,7 +62,7 @@ com.hazelcast hazelcast - 3.3.3 + 3.5.3 test-jar test diff --git a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/cluster/ClusterManager.java b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/cluster/ClusterManager.java index 05010b9aa..79cee2941 100644 --- a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/cluster/ClusterManager.java +++ b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/cluster/ClusterManager.java @@ -2,6 +2,7 @@ import com.flipkart.foxtrot.core.querystore.impl.HazelcastConnection; import com.google.common.collect.ImmutableList; +import com.hazelcast.config.EvictionPolicy; import com.hazelcast.config.MapConfig; import com.hazelcast.core.IMap; import com.yammer.dropwizard.lifecycle.Managed; @@ -35,7 +36,7 @@ public ClusterManager(HazelcastConnection connection, List healthCh mapConfig.setTimeToLiveSeconds(MAP_REFRESH_TIME+ 2); //Reduce jitter mapConfig.setBackupCount(1); mapConfig.setAsyncBackupCount(2); - mapConfig.setEvictionPolicy(MapConfig.EvictionPolicy.NONE); + mapConfig.setEvictionPolicy(EvictionPolicy.NONE); hazelcastConnection.getHazelcastConfig().getMapConfigs().put(MAP_NAME, mapConfig); String hostname = Inet4Address.getLocalHost().getCanonicalHostName(); From 24dfff2fb64f03cad3e08bdc8214fd6b2f5014bb Mon Sep 17 00:00:00 2001 From: Swapnil Marghade Date: Mon, 23 Nov 2015 22:24:35 +0530 Subject: [PATCH 02/64] Successful integration with docker using docker compose --- Dockerfile | 23 ++++++++++ config/docker.yml | 28 ++++++++++++ config/local.yml | 2 + docker-compose.yml | 34 ++++++++++++++ .../core/datastore/impl/hbase/HBaseUtil.java | 8 ++++ .../datastore/impl/hbase/HbaseConfig.java | 19 ++++++++ foxtrot-server/pom.xml | 44 +++++++++++++++++++ 7 files changed, 158 insertions(+) create mode 100644 Dockerfile create mode 100755 config/docker.yml create mode 100644 docker-compose.yml diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 000000000..3adae0e20 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,23 @@ +FROM ubuntu:14.04 +MAINTAINER Shashank + + +RUN \ + apt-get install -y --no-install-recommends software-properties-common \ + && add-apt-repository ppa:webupd8team/java \ + && gpg --keyserver hkp://keys.gnupg.net --recv-keys 409B6B1796C275462A1703113804BB82D39DC0E3 \ + && apt-get update \ + && echo debconf shared/accepted-oracle-license-v1-1 select true | debconf-set-selections \ + && echo debconf shared/accepted-oracle-license-v1-1 seen true | debconf-set-selections \ + && apt-get install -y --no-install-recommends oracle-java8-installer + +EXPOSE 17000 +EXPOSE 17001 + +VOLUME /var/log/foxtrot-server + +ADD config/docker.yml docker.yml +ADD foxtrot-server/target/foxtrot*.jar server.jar + +CMD sh -c "sleep 10; java -jar -Djavax.xml.parsers.DocumentBuilderFactory=com.sun.org.apache.xerces.internal.jaxp.DocumentBuilderFactoryImpl server.jar server docker.yml" + diff --git a/config/docker.yml b/config/docker.yml new file mode 100755 index 000000000..288fa0d24 --- /dev/null +++ b/config/docker.yml @@ -0,0 +1,28 @@ +http: + port: 17000 + adminPort: 17001 + connectorType: nonblocking + +elasticsearch: + hosts: + - 192.168.99.100 + cluster: elasticsearch + +hbase: + secure : false + tableName: foxtrot + hbaseZookeeperQuorum: 192.168.99.100 + hbaseZookeeperClientPort: 2181 + +cluster: + name: foxtrot + disableMulticast: true + members: ["localhost:5701", "localhost:5702"] + +logging: + level: INFO + +deletionconfig: + active: true + interval: 86400 + initialdelay: 60 diff --git a/config/local.yml b/config/local.yml index 841f66a82..131de29ec 100755 --- a/config/local.yml +++ b/config/local.yml @@ -11,6 +11,8 @@ elasticsearch: hbase: secure : false tableName: foxtrot + hbaseZookeeperQuorum: localhost + hbaseZookeeperClientPort: 2181 cluster: name: foxtrot diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 000000000..800a26738 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,34 @@ +elasticsearch: + image: library/elasticsearch:1.7.3 + hostname: elasticsearch + command: elasticsearch -Des.http.cors.enabled=true + ports: + - "9200:9200" + - "9300:9300" + +# hbase compose +hbase: + image: nerdammer/hbase:1.0.0 + hostname: hbase + ports: + - "2181:2181" + - "60000:60000" + - "60010:60010" + - "60020:60020" + - "60030:60030" + +#enitity store compose +foxtrot-server: + links: + - elasticsearch + - hbase + container_name: foxtrot_server + build: . + ports: + - "17000:17000" + - "17001:17001" + volumes: + - /var/log/foxtrot-server + # enviroment valiables +# environment: +# - ZK_CONNECTION_STRING=hbase:2181 diff --git a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/datastore/impl/hbase/HBaseUtil.java b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/datastore/impl/hbase/HBaseUtil.java index ad1ce757b..fe65c2588 100644 --- a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/datastore/impl/hbase/HBaseUtil.java +++ b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/datastore/impl/hbase/HBaseUtil.java @@ -60,6 +60,14 @@ public static Configuration create(final HbaseConfig hbaseConfig) throws IOExcep hbaseConfig.getAuthString(), hbaseConfig.getKeytabFileName()); logger.info("Logged into Hbase with User: " + UserGroupInformation.getLoginUser()); } + + if(null != hbaseConfig.getHbaseZookeeperQuorum()){ + configuration.set("hbase.zookeeper.quorum", hbaseConfig.getHbaseZookeeperQuorum()); + } + + if(null != hbaseConfig.getHbaseZookeeperClientPort()){ + configuration.setInt("hbase.zookeeper.property.clientPort", hbaseConfig.getHbaseZookeeperClientPort()); + } return configuration; } diff --git a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/datastore/impl/hbase/HbaseConfig.java b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/datastore/impl/hbase/HbaseConfig.java index 6b291f1e0..61c4c8387 100644 --- a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/datastore/impl/hbase/HbaseConfig.java +++ b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/datastore/impl/hbase/HbaseConfig.java @@ -36,6 +36,9 @@ public class HbaseConfig { private String kinitPath; private String authString; private String seggregatedTablePrefix; + private String hbaseZookeeperQuorum; + private Integer hbaseZookeeperClientPort; + @NotNull @NotEmpty @@ -131,4 +134,20 @@ public String getSeggregatedTablePrefix() { public void setSeggregatedTablePrefix(String seggregatedTablePrefix) { this.seggregatedTablePrefix = seggregatedTablePrefix; } + + public Integer getHbaseZookeeperClientPort() { + return hbaseZookeeperClientPort; + } + + public void setHbaseZookeeperClientPort(Integer hbaseZookeeperClientPort) { + this.hbaseZookeeperClientPort = hbaseZookeeperClientPort; + } + + public String getHbaseZookeeperQuorum() { + return hbaseZookeeperQuorum; + } + + public void setHbaseZookeeperQuorum(String hbaseZookeeperQuorum) { + this.hbaseZookeeperQuorum = hbaseZookeeperQuorum; + } } diff --git a/foxtrot-server/pom.xml b/foxtrot-server/pom.xml index 79dd50ed9..ab3433473 100644 --- a/foxtrot-server/pom.xml +++ b/foxtrot-server/pom.xml @@ -69,6 +69,50 @@ + + + docker + + + + + + + + + com.spotify + docker-maven-plugin + 0.1.1 + + foxtrot-server:${project.version} + ${project.parent.basedir} + + + / + ${project.build.directory} + ${project.build.finalName}.jar + + + / + ${project.parent.basedir}/config + *.yml + + + + + + package + + build + + + + + + + + + From 0df73b470f9444c6503a0d4a44d94decd101973b Mon Sep 17 00:00:00 2001 From: Swapnil Marghade Date: Tue, 24 Nov 2015 14:33:36 +0530 Subject: [PATCH 03/64] Fixed pom to create single image without docker-compose --- foxtrot-server/pom.xml | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/foxtrot-server/pom.xml b/foxtrot-server/pom.xml index ab3433473..97fc789ba 100644 --- a/foxtrot-server/pom.xml +++ b/foxtrot-server/pom.xml @@ -72,11 +72,6 @@ docker - - - - - @@ -85,7 +80,7 @@ 0.1.1 foxtrot-server:${project.version} - ${project.parent.basedir} + ${project.parent.basedir}/../foxtrot / From ff193d1f032c657d4cd8f8420e796bd29a0952ef Mon Sep 17 00:00:00 2001 From: Avanish Pandey Date: Tue, 24 Nov 2015 15:05:16 +0530 Subject: [PATCH 04/64] Support for is null and is not null --- .../flipkart/foxtrot/common/query/Filter.java | 1 + .../foxtrot/common/query/FilterOperator.java | 1 + .../foxtrot/common/query/FilterVisitor.java | 2 ++ .../common/query/general/MissingFilter.java | 28 +++++++++++++++++++ .../foxtrot/core/common/PeriodSelector.java | 5 ++++ .../query/ElasticSearchQueryGenerator.java | 4 +++ .../flipkart/foxtrot/sql/QueryTranslator.java | 16 ++++++++--- .../com/flipkart/foxtrot/sql/ParseTest.java | 17 ++++++++++- 8 files changed, 69 insertions(+), 5 deletions(-) create mode 100644 foxtrot-common/src/main/java/com/flipkart/foxtrot/common/query/general/MissingFilter.java diff --git a/foxtrot-common/src/main/java/com/flipkart/foxtrot/common/query/Filter.java b/foxtrot-common/src/main/java/com/flipkart/foxtrot/common/query/Filter.java index e7a261f75..7d2a1433a 100644 --- a/foxtrot-common/src/main/java/com/flipkart/foxtrot/common/query/Filter.java +++ b/foxtrot-common/src/main/java/com/flipkart/foxtrot/common/query/Filter.java @@ -46,6 +46,7 @@ @JsonSubTypes.Type(value = NotEqualsFilter.class, name = FilterOperator.not_equals), @JsonSubTypes.Type(value = AnyFilter.class, name = FilterOperator.any), @JsonSubTypes.Type(value = ExistsFilter.class, name = FilterOperator.exists), + @JsonSubTypes.Type(value = MissingFilter.class, name = FilterOperator.missing), //String @JsonSubTypes.Type(value = ContainsFilter.class, name = FilterOperator.contains), diff --git a/foxtrot-common/src/main/java/com/flipkart/foxtrot/common/query/FilterOperator.java b/foxtrot-common/src/main/java/com/flipkart/foxtrot/common/query/FilterOperator.java index cb35e53c6..4657275d1 100644 --- a/foxtrot-common/src/main/java/com/flipkart/foxtrot/common/query/FilterOperator.java +++ b/foxtrot-common/src/main/java/com/flipkart/foxtrot/common/query/FilterOperator.java @@ -27,6 +27,7 @@ public interface FilterOperator { public static final String any = "any"; public static final String in = "in"; public static final String exists = "exists"; + public static final String missing = "missing"; //Numeric public static final String less_than = "less_than"; diff --git a/foxtrot-common/src/main/java/com/flipkart/foxtrot/common/query/FilterVisitor.java b/foxtrot-common/src/main/java/com/flipkart/foxtrot/common/query/FilterVisitor.java index 0cdf4276a..37841f320 100644 --- a/foxtrot-common/src/main/java/com/flipkart/foxtrot/common/query/FilterVisitor.java +++ b/foxtrot-common/src/main/java/com/flipkart/foxtrot/common/query/FilterVisitor.java @@ -50,4 +50,6 @@ public abstract class FilterVisitor { public abstract void visit(ExistsFilter existsFilter) throws Exception; public abstract void visit(LastFilter lastFilter) throws Exception; + + public abstract void visit(MissingFilter missingFilter) throws Exception; } diff --git a/foxtrot-common/src/main/java/com/flipkart/foxtrot/common/query/general/MissingFilter.java b/foxtrot-common/src/main/java/com/flipkart/foxtrot/common/query/general/MissingFilter.java new file mode 100644 index 000000000..c6658a9fa --- /dev/null +++ b/foxtrot-common/src/main/java/com/flipkart/foxtrot/common/query/general/MissingFilter.java @@ -0,0 +1,28 @@ +package com.flipkart.foxtrot.common.query.general; + +import com.flipkart.foxtrot.common.query.Filter; +import com.flipkart.foxtrot.common.query.FilterOperator; +import com.flipkart.foxtrot.common.query.FilterVisitor; + +/** + * Created by avanish.pandey on 23/11/15. + */ +public class MissingFilter extends Filter{ + + + public MissingFilter() { + super(FilterOperator.missing); + } + + public MissingFilter(String field) { + super(FilterOperator.missing, field); + } + + @Override + public void accept(FilterVisitor visitor) throws Exception { + visitor.visit(this); + + } + + +} diff --git a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/common/PeriodSelector.java b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/common/PeriodSelector.java index 6acc873a6..ab5c8ea13 100644 --- a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/common/PeriodSelector.java +++ b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/common/PeriodSelector.java @@ -112,4 +112,9 @@ public void visit(LastFilter lastFilter) throws Exception { timeWindow.setStartTime(window.getStartTime()); timeWindow.setEndTime(window.getEndTime()); } + + @Override + public void visit(MissingFilter missingFilter) throws Exception{ + + } } diff --git a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/query/ElasticSearchQueryGenerator.java b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/query/ElasticSearchQueryGenerator.java index c7a482122..76e63fa25 100644 --- a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/query/ElasticSearchQueryGenerator.java +++ b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/query/ElasticSearchQueryGenerator.java @@ -129,6 +129,10 @@ public void visit(LastFilter lastFilter) throws Exception { .cache(false)); } + @Override public void visit(MissingFilter missingFilter) throws Exception { + addFilter(FilterBuilders.missingFilter(missingFilter.getField())); + } + private void addFilter(FilterBuilder elasticSearchFilter) throws Exception { if (combinerType == FilterCombinerType.and) { boolFilterBuilder.must(elasticSearchFilter); diff --git a/foxtrot-sql/src/main/java/com/flipkart/foxtrot/sql/QueryTranslator.java b/foxtrot-sql/src/main/java/com/flipkart/foxtrot/sql/QueryTranslator.java index 12ce23af2..1851fdd24 100644 --- a/foxtrot-sql/src/main/java/com/flipkart/foxtrot/sql/QueryTranslator.java +++ b/foxtrot-sql/src/main/java/com/flipkart/foxtrot/sql/QueryTranslator.java @@ -1,6 +1,5 @@ package com.flipkart.foxtrot.sql; - import com.flipkart.foxtrot.common.ActionRequest; import com.flipkart.foxtrot.common.Period; import com.flipkart.foxtrot.common.count.CountRequest; @@ -11,9 +10,7 @@ import com.flipkart.foxtrot.common.query.Query; import com.flipkart.foxtrot.common.query.ResultSort; import com.flipkart.foxtrot.common.query.datetime.LastFilter; -import com.flipkart.foxtrot.common.query.general.EqualsFilter; -import com.flipkart.foxtrot.common.query.general.InFilter; -import com.flipkart.foxtrot.common.query.general.NotEqualsFilter; +import com.flipkart.foxtrot.common.query.general.*; import com.flipkart.foxtrot.common.query.numeric.*; import com.flipkart.foxtrot.common.stats.StatsRequest; import com.flipkart.foxtrot.common.stats.StatsTrendRequest; @@ -510,6 +507,17 @@ public void visit(InExpression inExpression) { @Override public void visit(IsNullExpression isNullExpression) { super.visit(isNullExpression); + ColumnData columnData = setupColumn(isNullExpression.getLeftExpression()); + if (isNullExpression.isNot()) { + ExistsFilter existsFilter = new ExistsFilter(); + + existsFilter.setField(columnData.getColumnName().replaceAll(Constants.SQL_FIELD_REGEX, "")); + filters.add(existsFilter); + } else { + MissingFilter missingFilter = new MissingFilter(); + missingFilter.setField(columnData.getColumnName().replaceAll(Constants.SQL_FIELD_REGEX, "")); + filters.add(missingFilter); + } } @Override diff --git a/foxtrot-sql/src/test/java/com/flipkart/foxtrot/sql/ParseTest.java b/foxtrot-sql/src/test/java/com/flipkart/foxtrot/sql/ParseTest.java index b3395f32c..e806ee435 100644 --- a/foxtrot-sql/src/test/java/com/flipkart/foxtrot/sql/ParseTest.java +++ b/foxtrot-sql/src/test/java/com/flipkart/foxtrot/sql/ParseTest.java @@ -1,10 +1,13 @@ package com.flipkart.foxtrot.sql; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.ObjectWriter; import org.junit.Test; public class ParseTest { @Test - public void test() { + public void test() throws Exception { //TODO /*ObjectMapper objectMapper = new ObjectMapper(); objectMapper.setSerializationInclusion(JsonInclude.Include.NON_EMPTY); @@ -65,5 +68,17 @@ public void test() { ActionRequest request = queryTranslator.translate(sql); System.out.println(writer.writeValueAsString(request)); }*/ + + + ObjectMapper objectMapper = new ObjectMapper(); + objectMapper.setSerializationInclusion(JsonInclude.Include.NON_EMPTY); + objectMapper.setSerializationInclusion(JsonInclude.Include.NON_NULL); + ObjectWriter writer = objectMapper.writerWithDefaultPrettyPrinter(); + { + String sql = "select * from europa where a is null"; + QueryTranslator queryTranslator = new QueryTranslator(); + FqlQuery query = queryTranslator.translate(sql); + System.out.println(writer.writeValueAsString(query)); + } } } From 9916a1f55ad817ed6130c151d595838d7865cd32 Mon Sep 17 00:00:00 2001 From: Vivek Kothari Date: Tue, 24 Nov 2015 15:11:35 +0530 Subject: [PATCH 05/64] issue#53 IndexMissingException while running queries Set indexOptions to ignore unavailable indices. Ref: https://www.elastic.co/guide/en/elasticsearch/reference/current/multi-index.html --- .../core/querystore/actions/CountAction.java | 2 ++ .../querystore/actions/DistinctAction.java | 6 ++-- .../core/querystore/actions/FilterAction.java | 28 +++++++++++-------- .../core/querystore/actions/GroupAction.java | 6 ++-- .../querystore/actions/HistogramAction.java | 2 ++ .../core/querystore/actions/StatsAction.java | 2 ++ .../querystore/actions/StatsTrendAction.java | 2 ++ .../core/querystore/actions/TrendAction.java | 2 ++ .../core/querystore/actions/Utils.java | 5 ++++ 9 files changed, 40 insertions(+), 15 deletions(-) diff --git a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/CountAction.java b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/CountAction.java index d4b755940..d44e20de4 100644 --- a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/CountAction.java +++ b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/CountAction.java @@ -17,6 +17,7 @@ import com.flipkart.foxtrot.core.querystore.impl.ElasticsearchUtils; import com.flipkart.foxtrot.core.querystore.query.ElasticSearchQueryGenerator; import com.google.common.collect.Lists; + import org.elasticsearch.action.count.CountRequestBuilder; import org.elasticsearch.action.search.SearchRequestBuilder; import org.elasticsearch.action.search.SearchResponse; @@ -77,6 +78,7 @@ public ActionResponse execute(CountRequest parameter) throws QueryStoreException if (parameter.isDistinct()){ SearchRequestBuilder query = getConnection().getClient() .prepareSearch(ElasticsearchUtils.getIndices(parameter.getTable(), parameter)) + .setIndicesOptions(Utils.indicesOptions()) .setSearchType(SearchType.COUNT) .setQuery(new ElasticSearchQueryGenerator(FilterCombinerType.and) .genFilter(parameter.getFilters())) diff --git a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/DistinctAction.java b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/DistinctAction.java index ce34c8e49..55f29738c 100644 --- a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/DistinctAction.java +++ b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/DistinctAction.java @@ -17,6 +17,7 @@ import com.flipkart.foxtrot.core.querystore.impl.ElasticsearchUtils; import com.flipkart.foxtrot.core.querystore.query.ElasticSearchQueryGenerator; import com.google.common.collect.Lists; + import org.elasticsearch.action.search.SearchRequestBuilder; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.search.SearchType; @@ -73,8 +74,9 @@ public ActionResponse execute(DistinctRequest request) throws QueryStoreExceptio throw new QueryStoreException(QueryStoreException.ErrorCode.INVALID_REQUEST, "Invalid Table"); } try { - SearchRequestBuilder query = getConnection().getClient().prepareSearch(ElasticsearchUtils.getIndices( - request.getTable(), request)); + SearchRequestBuilder query = getConnection().getClient() + .prepareSearch(ElasticsearchUtils.getIndices(request.getTable(), request)) + .setIndicesOptions(Utils.indicesOptions()); TermsBuilder rootBuilder = null; TermsBuilder termsBuilder = null; diff --git a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/FilterAction.java b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/FilterAction.java index a236ec1eb..7af9f5fa4 100644 --- a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/FilterAction.java +++ b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/FilterAction.java @@ -15,8 +15,23 @@ */ package com.flipkart.foxtrot.core.querystore.actions; +import java.util.Collections; +import java.util.Vector; + +import org.elasticsearch.action.search.SearchRequestBuilder; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.action.search.SearchType; +import org.elasticsearch.search.SearchHit; +import org.elasticsearch.search.sort.SortOrder; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + import com.flipkart.foxtrot.common.Document; -import com.flipkart.foxtrot.common.query.*; +import com.flipkart.foxtrot.common.query.Filter; +import com.flipkart.foxtrot.common.query.FilterCombinerType; +import com.flipkart.foxtrot.common.query.Query; +import com.flipkart.foxtrot.common.query.QueryResponse; +import com.flipkart.foxtrot.common.query.ResultSort; import com.flipkart.foxtrot.common.query.general.AnyFilter; import com.flipkart.foxtrot.core.common.Action; import com.flipkart.foxtrot.core.datastore.DataStore; @@ -28,16 +43,6 @@ import com.flipkart.foxtrot.core.querystore.impl.ElasticsearchUtils; import com.flipkart.foxtrot.core.querystore.query.ElasticSearchQueryGenerator; import com.google.common.collect.Lists; -import org.elasticsearch.action.search.SearchRequestBuilder; -import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.action.search.SearchType; -import org.elasticsearch.search.SearchHit; -import org.elasticsearch.search.sort.SortOrder; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.Collections; -import java.util.Vector; /** * User: Santanu Sinha (santanu.sinha@flipkart.com) @@ -93,6 +98,7 @@ public QueryResponse execute(Query parameter) throws QueryStoreException { }*/ search = getConnection().getClient().prepareSearch(ElasticsearchUtils.getIndices(parameter.getTable(), parameter)) .setTypes(ElasticsearchUtils.TYPE_NAME) + .setIndicesOptions(Utils.indicesOptions()) .setQuery(new ElasticSearchQueryGenerator(FilterCombinerType.and).genFilter(parameter.getFilters())) .setSearchType(SearchType.QUERY_THEN_FETCH) .setFrom(parameter.getFrom()) diff --git a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/GroupAction.java b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/GroupAction.java index b904f94e9..4d2263adc 100644 --- a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/GroupAction.java +++ b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/GroupAction.java @@ -31,6 +31,7 @@ import com.flipkart.foxtrot.core.querystore.impl.ElasticsearchUtils; import com.flipkart.foxtrot.core.querystore.query.ElasticSearchQueryGenerator; import com.google.common.collect.Lists; + import org.elasticsearch.action.search.SearchRequestBuilder; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.search.SearchType; @@ -86,8 +87,9 @@ public ActionResponse execute(GroupRequest parameter) throws QueryStoreException throw new QueryStoreException(QueryStoreException.ErrorCode.INVALID_REQUEST, "Invalid Table"); } try { - SearchRequestBuilder query = getConnection().getClient().prepareSearch(ElasticsearchUtils.getIndices( - parameter.getTable(), parameter)); + SearchRequestBuilder query = getConnection().getClient() + .prepareSearch(ElasticsearchUtils.getIndices(parameter.getTable(), parameter)) + .setIndicesOptions(Utils.indicesOptions()); TermsBuilder rootBuilder = null; TermsBuilder termsBuilder = null; for (String field : parameter.getNesting()) { diff --git a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/HistogramAction.java b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/HistogramAction.java index 623a8e1c7..13057409d 100644 --- a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/HistogramAction.java +++ b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/HistogramAction.java @@ -33,6 +33,7 @@ import com.flipkart.foxtrot.core.querystore.query.ElasticSearchQueryGenerator; import com.google.common.collect.Lists; import com.yammer.dropwizard.util.Duration; + import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.search.SearchType; import org.elasticsearch.search.aggregations.AggregationBuilders; @@ -114,6 +115,7 @@ public ActionResponse execute(HistogramRequest parameter) throws QueryStoreExcep String dateHistogramKey = Utils.sanitizeFieldForAggregation(parameter.getField()); SearchResponse response = getConnection().getClient().prepareSearch( ElasticsearchUtils.getIndices(parameter.getTable(), parameter)) + .setIndicesOptions(Utils.indicesOptions()) .setTypes(ElasticsearchUtils.TYPE_NAME) .setQuery(new ElasticSearchQueryGenerator(FilterCombinerType.and) .genFilter(parameter.getFilters())) diff --git a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/StatsAction.java b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/StatsAction.java index bced0953b..8f2caaca4 100644 --- a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/StatsAction.java +++ b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/StatsAction.java @@ -16,6 +16,7 @@ import com.flipkart.foxtrot.core.querystore.impl.ElasticsearchUtils; import com.flipkart.foxtrot.core.querystore.query.ElasticSearchQueryGenerator; import com.google.common.collect.Lists; + import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.search.SearchType; import org.elasticsearch.search.aggregations.Aggregations; @@ -72,6 +73,7 @@ public ActionResponse execute(StatsRequest request) throws QueryStoreException { try { SearchResponse response = getConnection().getClient().prepareSearch( ElasticsearchUtils.getIndices(request.getTable(), request)) + .setIndicesOptions(Utils.indicesOptions()) .setTypes(ElasticsearchUtils.TYPE_NAME) .setQuery(new ElasticSearchQueryGenerator(request.getCombiner()).genFilter(request.getFilters())) .setSize(0) diff --git a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/StatsTrendAction.java b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/StatsTrendAction.java index 70aba22a8..6f787c583 100644 --- a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/StatsTrendAction.java +++ b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/StatsTrendAction.java @@ -18,6 +18,7 @@ import com.flipkart.foxtrot.core.querystore.query.ElasticSearchQueryGenerator; import com.google.common.collect.Lists; import com.yammer.dropwizard.util.Duration; + import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.search.SearchType; import org.elasticsearch.search.aggregations.AbstractAggregationBuilder; @@ -85,6 +86,7 @@ public ActionResponse execute(StatsTrendRequest parameter) throws QueryStoreExce AbstractAggregationBuilder aggregation = buildAggregation(parameter); SearchResponse response = getConnection().getClient().prepareSearch( ElasticsearchUtils.getIndices(parameter.getTable(), parameter)) + .setIndicesOptions(Utils.indicesOptions()) .setTypes(ElasticsearchUtils.TYPE_NAME) .setQuery(new ElasticSearchQueryGenerator(parameter.getCombiner()).genFilter(parameter.getFilters())) .setSize(0) diff --git a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/TrendAction.java b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/TrendAction.java index c634a08f4..caf24b9cb 100644 --- a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/TrendAction.java +++ b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/TrendAction.java @@ -34,6 +34,7 @@ import com.flipkart.foxtrot.core.querystore.query.ElasticSearchQueryGenerator; import com.google.common.collect.Lists; import com.yammer.dropwizard.util.Duration; + import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.search.SearchType; import org.elasticsearch.search.aggregations.AbstractAggregationBuilder; @@ -114,6 +115,7 @@ public ActionResponse execute(TrendRequest parameter) throws QueryStoreException AbstractAggregationBuilder aggregationBuilder = buildAggregation(parameter); SearchResponse searchResponse = getConnection().getClient() .prepareSearch(ElasticsearchUtils.getIndices(parameter.getTable(), parameter)) + .setIndicesOptions(Utils.indicesOptions()) .setQuery(new ElasticSearchQueryGenerator(FilterCombinerType.and).genFilter(parameter.getFilters())) .setSearchType(SearchType.COUNT) .addAggregation(aggregationBuilder) diff --git a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/Utils.java b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/Utils.java index 6896b1fd1..d816b14c2 100644 --- a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/Utils.java +++ b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/Utils.java @@ -1,5 +1,6 @@ package com.flipkart.foxtrot.core.querystore.actions; +import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.search.aggregations.AbstractAggregationBuilder; import org.elasticsearch.search.aggregations.AggregationBuilders; import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogram; @@ -41,4 +42,8 @@ public static String getPercentileAggregationKey(String field){ public static String getDateHistogramKey(String field){ return sanitizeFieldForAggregation(field) + "_date_histogram"; } + + public static IndicesOptions indicesOptions() { + return IndicesOptions.fromOptions(true, false, false, false); + } } From 9c4850be5f742cde02b0bff804bc5d94f5f04bc4 Mon Sep 17 00:00:00 2001 From: Vivek Kothari Date: Tue, 24 Nov 2015 15:43:22 +0530 Subject: [PATCH 06/64] allow expansion of wildcards. --- .../com/flipkart/foxtrot/core/querystore/actions/Utils.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/Utils.java b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/Utils.java index d816b14c2..52a2d491c 100644 --- a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/Utils.java +++ b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/Utils.java @@ -44,6 +44,6 @@ public static String getDateHistogramKey(String field){ } public static IndicesOptions indicesOptions() { - return IndicesOptions.fromOptions(true, false, false, false); + return IndicesOptions.fromOptions(true, true, true, false); } } From 5975d2665431fad0de4b5ff433c162f78d6e3167 Mon Sep 17 00:00:00 2001 From: Swapnil Marghade Date: Tue, 24 Nov 2015 18:27:53 +0530 Subject: [PATCH 07/64] changes for docker hbase version --- Dockerfile | 8 +++++--- docker-compose.yml | 2 +- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index 3adae0e20..01a0d0a2f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ FROM ubuntu:14.04 -MAINTAINER Shashank +MAINTAINER Swapnil Marghade RUN \ @@ -9,7 +9,8 @@ RUN \ && apt-get update \ && echo debconf shared/accepted-oracle-license-v1-1 select true | debconf-set-selections \ && echo debconf shared/accepted-oracle-license-v1-1 seen true | debconf-set-selections \ - && apt-get install -y --no-install-recommends oracle-java8-installer + && apt-get install -y --no-install-recommends oracle-java8-installer \ + && apt-get install -y --no-install-recommends curl EXPOSE 17000 EXPOSE 17001 @@ -18,6 +19,7 @@ VOLUME /var/log/foxtrot-server ADD config/docker.yml docker.yml ADD foxtrot-server/target/foxtrot*.jar server.jar +ADD scripts/local_es_setup.sh local_es_setup.sh -CMD sh -c "sleep 10; java -jar -Djavax.xml.parsers.DocumentBuilderFactory=com.sun.org.apache.xerces.internal.jaxp.DocumentBuilderFactoryImpl server.jar server docker.yml" +CMD sh -c "sleep 20 ; ./local_es_setup.sh 192.168.99.100 || true ; java -Dfile.encoding=utf-8 -jar server.jar server docker.yml" diff --git a/docker-compose.yml b/docker-compose.yml index 800a26738..64f53196c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -8,7 +8,7 @@ elasticsearch: # hbase compose hbase: - image: nerdammer/hbase:1.0.0 + image: banno/hbase-standalone hostname: hbase ports: - "2181:2181" From f69cc7de42f01085744d7652cf09242606c9c488 Mon Sep 17 00:00:00 2001 From: Vivek Kothari Date: Tue, 24 Nov 2015 19:17:52 +0530 Subject: [PATCH 08/64] Add jUnit to test missing index query. --- .../com/flipkart/foxtrot/core/TestUtils.java | 43 ++++++++++++++----- .../querystore/actions/FilterActionTest.java | 29 ++++++++++++- 2 files changed, 60 insertions(+), 12 deletions(-) diff --git a/foxtrot-core/src/test/java/com/flipkart/foxtrot/core/TestUtils.java b/foxtrot-core/src/test/java/com/flipkart/foxtrot/core/TestUtils.java index 3a0787d1d..cc6b3904c 100644 --- a/foxtrot-core/src/test/java/com/flipkart/foxtrot/core/TestUtils.java +++ b/foxtrot-core/src/test/java/com/flipkart/foxtrot/core/TestUtils.java @@ -15,6 +15,23 @@ */ package com.flipkart.foxtrot.core; +import static org.mockito.Mockito.when; + +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.Vector; + +import org.apache.hadoop.hbase.client.HTableInterface; +import org.mockito.Matchers; +import org.mockito.Mockito; +import org.reflections.Reflections; +import org.reflections.scanners.SubTypesScanner; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.jsontype.NamedType; import com.flipkart.foxtrot.common.Document; @@ -27,17 +44,6 @@ import com.flipkart.foxtrot.core.querystore.actions.spi.ActionMetadata; import com.flipkart.foxtrot.core.querystore.actions.spi.AnalyticsLoader; import com.flipkart.foxtrot.core.querystore.actions.spi.AnalyticsProvider; -import org.apache.hadoop.hbase.client.HTableInterface; -import org.mockito.Matchers; -import org.mockito.Mockito; -import org.reflections.Reflections; -import org.reflections.scanners.SubTypesScanner; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.*; - -import static org.mockito.Mockito.when; /** * Created by rishabh.goyal on 28/04/14. @@ -198,4 +204,19 @@ public static List getMappingDocuments(ObjectMapper mapper) { documents.add(new Document("Y", System.currentTimeMillis(), mapper.valueToTree(document))); return documents; } + + public static List getQueryDocumentsDifferentDate(ObjectMapper mapper, long startTimestamp) { + List documents = new Vector(); + documents.add(TestUtils.getDocument("Z", startTimestamp, new Object[]{"os", "android", "device", "nexus", "battery", 24}, mapper)); + documents.add(TestUtils.getDocument("Y", startTimestamp++, new Object[]{"os", "android", "device", "nexus", "battery", 48}, mapper)); + documents.add(TestUtils.getDocument("X", startTimestamp++, new Object[]{"os", "android", "device", "nexus", "battery", 74}, mapper)); + documents.add(TestUtils.getDocument("W", startTimestamp++, new Object[] { "os", "android", "device", "nexus", "battery", 99 }, mapper)); + documents.add(TestUtils.getDocument("A", startTimestamp++, new Object[]{"os", "android", "version", 1, "device", "nexus"}, mapper)); + documents.add(TestUtils.getDocument("B", startTimestamp++, new Object[]{"os", "android", "version", 1, "device", "galaxy"}, mapper)); + documents.add(TestUtils.getDocument("C", startTimestamp++, new Object[]{"os", "android", "version", 2, "device", "nexus"}, mapper)); + documents.add(TestUtils.getDocument("D", startTimestamp++, new Object[]{"os", "ios", "version", 1, "device", "iphone"}, mapper)); + documents.add(TestUtils.getDocument("E", startTimestamp++, new Object[]{"os", "ios", "version", 2, "device", "ipad"}, mapper)); + return documents; + } + } diff --git a/foxtrot-core/src/test/java/com/flipkart/foxtrot/core/querystore/actions/FilterActionTest.java b/foxtrot-core/src/test/java/com/flipkart/foxtrot/core/querystore/actions/FilterActionTest.java index 546dc3772..a3fae8457 100644 --- a/foxtrot-core/src/test/java/com/flipkart/foxtrot/core/querystore/actions/FilterActionTest.java +++ b/foxtrot-core/src/test/java/com/flipkart/foxtrot/core/querystore/actions/FilterActionTest.java @@ -41,6 +41,10 @@ import com.google.common.collect.Lists; import com.hazelcast.core.HazelcastInstance; import com.hazelcast.test.TestHazelcastInstanceFactory; + +import org.elasticsearch.action.admin.indices.get.GetIndexRequest; +import org.elasticsearch.action.admin.indices.get.GetIndexResponse; +import org.joda.time.LocalDateTime; import org.junit.After; import org.junit.Before; import org.junit.Ignore; @@ -66,6 +70,7 @@ public class FilterActionTest { private ObjectMapper mapper = new ObjectMapper(); private MockElasticsearchServer elasticsearchServer; private HazelcastInstance hazelcastInstance; + private QueryStore queryStore; @Before public void setUp() throws Exception { @@ -87,7 +92,7 @@ public void setUp() throws Exception { TableMetadataManager tableMetadataManager = Mockito.mock(TableMetadataManager.class); when(tableMetadataManager.exists(TestUtils.TEST_TABLE_NAME)).thenReturn(true); when(tableMetadataManager.get(anyString())).thenReturn(TestUtils.TEST_TABLE); - QueryStore queryStore = new ElasticsearchQueryStore(tableMetadataManager, elasticsearchConnection, dataStore); + queryStore = new ElasticsearchQueryStore(tableMetadataManager, elasticsearchConnection, dataStore); List documents = TestUtils.getQueryDocuments(mapper); //when(queryStore.get(anyString(), Matchers.anyListOf(String.class))).thenReturn(documents); AnalyticsLoader analyticsLoader = new AnalyticsLoader(tableMetadataManager, dataStore, queryStore, elasticsearchConnection); @@ -659,6 +664,28 @@ public void testQueryCaching() throws QueryStoreException, JsonProcessingExcepti compare(documents, actualResponse.getDocuments()); } + @SuppressWarnings("deprecation") + @Test + public void testMissingIndicesQuery() throws QueryStoreException { + Query query = new Query(); + query.setTable(TestUtils.TEST_TABLE_NAME); + + List documents = TestUtils.getQueryDocumentsDifferentDate(mapper, new Date(2014, 5, 1).getTime()); + documents.addAll( TestUtils.getQueryDocumentsDifferentDate(mapper, new Date(2014, 5, 5).getTime())); + queryStore.save(TestUtils.TEST_TABLE_NAME, documents); + for (Document document : documents) { + elasticsearchServer.getClient().admin().indices() + .prepareRefresh(ElasticsearchUtils.getCurrentIndex(TestUtils.TEST_TABLE_NAME, document.getTimestamp())) + .setForce(true).execute().actionGet(); + } + GetIndexResponse response = elasticsearchServer.getClient().admin().indices().getIndex(new GetIndexRequest()).actionGet(); + assertEquals(3, response.getIndices()); + + QueryResponse actualResponse = QueryResponse.class.cast(queryExecutor.execute(query)); + assertEquals(30, actualResponse.getDocuments().size()); + } + + public void compare(List expectedDocuments, List actualDocuments){ assertEquals(expectedDocuments.size(), actualDocuments.size()); for (int i = 0 ; i < expectedDocuments.size(); i++){ From d76a7c175dd3e11a232f093dd7a87a869975c56c Mon Sep 17 00:00:00 2001 From: Swapnil Marghade Date: Tue, 24 Nov 2015 19:59:48 +0530 Subject: [PATCH 09/64] added initialise in command --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 01a0d0a2f..b4e55c05f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -21,5 +21,5 @@ ADD config/docker.yml docker.yml ADD foxtrot-server/target/foxtrot*.jar server.jar ADD scripts/local_es_setup.sh local_es_setup.sh -CMD sh -c "sleep 20 ; ./local_es_setup.sh 192.168.99.100 || true ; java -Dfile.encoding=utf-8 -jar server.jar server docker.yml" +CMD sh -c "sleep 15 ; java -jar server.jar initialize docker.yml || true ; java -Dfile.encoding=utf-8 -jar server.jar server docker.yml" From 966838c3131066bc1f688ce7e13023bc20715bd2 Mon Sep 17 00:00:00 2001 From: Vivek Kothari Date: Tue, 24 Nov 2015 20:21:39 +0530 Subject: [PATCH 10/64] fix junit. --- .../querystore/actions/FilterActionTest.java | 76 ++++++++++++------- 1 file changed, 48 insertions(+), 28 deletions(-) diff --git a/foxtrot-core/src/test/java/com/flipkart/foxtrot/core/querystore/actions/FilterActionTest.java b/foxtrot-core/src/test/java/com/flipkart/foxtrot/core/querystore/actions/FilterActionTest.java index a3fae8457..87687a7da 100644 --- a/foxtrot-core/src/test/java/com/flipkart/foxtrot/core/querystore/actions/FilterActionTest.java +++ b/foxtrot-core/src/test/java/com/flipkart/foxtrot/core/querystore/actions/FilterActionTest.java @@ -15,6 +15,30 @@ */ package com.flipkart.foxtrot.core.querystore.actions; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.mockito.Matchers.anyString; +import static org.mockito.Mockito.when; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Date; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import java.util.Vector; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; + +import org.elasticsearch.action.admin.indices.get.GetIndexRequest; +import org.elasticsearch.action.admin.indices.get.GetIndexResponse; +import org.junit.After; +import org.junit.Before; +import org.junit.Ignore; +import org.junit.Test; +import org.mockito.Mockito; + import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; @@ -26,7 +50,11 @@ import com.flipkart.foxtrot.common.query.general.AnyFilter; import com.flipkart.foxtrot.common.query.general.EqualsFilter; import com.flipkart.foxtrot.common.query.general.NotEqualsFilter; -import com.flipkart.foxtrot.common.query.numeric.*; +import com.flipkart.foxtrot.common.query.numeric.BetweenFilter; +import com.flipkart.foxtrot.common.query.numeric.GreaterEqualFilter; +import com.flipkart.foxtrot.common.query.numeric.GreaterThanFilter; +import com.flipkart.foxtrot.common.query.numeric.LessEqualFilter; +import com.flipkart.foxtrot.common.query.numeric.LessThanFilter; import com.flipkart.foxtrot.common.query.string.ContainsFilter; import com.flipkart.foxtrot.core.MockElasticsearchServer; import com.flipkart.foxtrot.core.TestUtils; @@ -37,30 +65,15 @@ import com.flipkart.foxtrot.core.querystore.QueryStoreException; import com.flipkart.foxtrot.core.querystore.TableMetadataManager; import com.flipkart.foxtrot.core.querystore.actions.spi.AnalyticsLoader; -import com.flipkart.foxtrot.core.querystore.impl.*; +import com.flipkart.foxtrot.core.querystore.impl.DistributedCacheFactory; +import com.flipkart.foxtrot.core.querystore.impl.ElasticsearchConnection; +import com.flipkart.foxtrot.core.querystore.impl.ElasticsearchQueryStore; +import com.flipkart.foxtrot.core.querystore.impl.ElasticsearchUtils; +import com.flipkart.foxtrot.core.querystore.impl.HazelcastConnection; import com.google.common.collect.Lists; import com.hazelcast.core.HazelcastInstance; import com.hazelcast.test.TestHazelcastInstanceFactory; -import org.elasticsearch.action.admin.indices.get.GetIndexRequest; -import org.elasticsearch.action.admin.indices.get.GetIndexResponse; -import org.joda.time.LocalDateTime; -import org.junit.After; -import org.junit.Before; -import org.junit.Ignore; -import org.junit.Test; -import org.mockito.Mockito; - -import java.io.IOException; -import java.util.*; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; -import static org.mockito.Matchers.anyString; -import static org.mockito.Mockito.when; - /** * Created by rishabh.goyal on 28/04/14. */ @@ -667,11 +680,8 @@ public void testQueryCaching() throws QueryStoreException, JsonProcessingExcepti @SuppressWarnings("deprecation") @Test public void testMissingIndicesQuery() throws QueryStoreException { - Query query = new Query(); - query.setTable(TestUtils.TEST_TABLE_NAME); - - List documents = TestUtils.getQueryDocumentsDifferentDate(mapper, new Date(2014, 5, 1).getTime()); - documents.addAll( TestUtils.getQueryDocumentsDifferentDate(mapper, new Date(2014, 5, 5).getTime())); + List documents = TestUtils.getQueryDocumentsDifferentDate(mapper, new Date(2014 - 1900, 4, 1).getTime()); + documents.addAll(TestUtils.getQueryDocumentsDifferentDate(mapper, new Date(2014 - 1900, 4, 5).getTime())); queryStore.save(TestUtils.TEST_TABLE_NAME, documents); for (Document document : documents) { elasticsearchServer.getClient().admin().indices() @@ -679,10 +689,20 @@ public void testMissingIndicesQuery() throws QueryStoreException { .setForce(true).execute().actionGet(); } GetIndexResponse response = elasticsearchServer.getClient().admin().indices().getIndex(new GetIndexRequest()).actionGet(); - assertEquals(3, response.getIndices()); + assertEquals(3, response.getIndices().length); + + Query query = new Query(); + query.setLimit(20); + query.setTable(TestUtils.TEST_TABLE_NAME); + BetweenFilter betweenFilter = new BetweenFilter(); + betweenFilter.setField("_timestamp"); + betweenFilter.setFrom(documents.get(0).getTimestamp()); + betweenFilter.setTo(documents.get(documents.size() -1).getTimestamp()); + betweenFilter.setTemporal(true); + query.setFilters(Lists.newArrayList(betweenFilter)); QueryResponse actualResponse = QueryResponse.class.cast(queryExecutor.execute(query)); - assertEquals(30, actualResponse.getDocuments().size()); + assertEquals(18, actualResponse.getDocuments().size()); } From 7c1a151c49ca763687e4aaccdc05a660a7865ce1 Mon Sep 17 00:00:00 2001 From: Avanish Pandey Date: Tue, 24 Nov 2015 23:38:15 +0530 Subject: [PATCH 11/64] Test fixed as per comment --- .../com/flipkart/foxtrot/sql/ParseTest.java | 20 +++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/foxtrot-sql/src/test/java/com/flipkart/foxtrot/sql/ParseTest.java b/foxtrot-sql/src/test/java/com/flipkart/foxtrot/sql/ParseTest.java index e806ee435..fda09362c 100644 --- a/foxtrot-sql/src/test/java/com/flipkart/foxtrot/sql/ParseTest.java +++ b/foxtrot-sql/src/test/java/com/flipkart/foxtrot/sql/ParseTest.java @@ -3,8 +3,15 @@ import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectWriter; +import com.flipkart.foxtrot.common.query.Query; +import com.flipkart.foxtrot.common.query.general.MissingFilter; +import com.flipkart.foxtrot.sql.query.FqlActionQuery; +import com.google.common.collect.ImmutableList; +import org.junit.Assert; import org.junit.Test; +import java.util.ArrayList; + public class ParseTest { @Test public void test() throws Exception { @@ -77,8 +84,17 @@ public void test() throws Exception { { String sql = "select * from europa where a is null"; QueryTranslator queryTranslator = new QueryTranslator(); - FqlQuery query = queryTranslator.translate(sql); - System.out.println(writer.writeValueAsString(query)); + //FqlQuery query = queryTranslator.translate(sql); + Query query = new Query(); + query.setTable("europa"); + query.setFrom(0); + query.setLimit(10); + query.setSort(null); + ImmutableList filters = ImmutableList.of(new MissingFilter("a")); + query.setFilters(filters); + FqlActionQuery fqlActionQuery = new FqlActionQuery(FqlQueryType.select,query, new ArrayList()); + Assert.assertEquals(writer.writeValueAsString(fqlActionQuery), writer.writeValueAsString(queryTranslator.translate(sql))); + } } } From 5b99c73a0d1c42f18f7e5b57e16af90852429dfb Mon Sep 17 00:00:00 2001 From: Vivek Kothari Date: Wed, 25 Nov 2015 23:49:50 +0530 Subject: [PATCH 12/64] incorporate comments. --- .../foxtrot/core/querystore/actions/FilterActionTest.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/foxtrot-core/src/test/java/com/flipkart/foxtrot/core/querystore/actions/FilterActionTest.java b/foxtrot-core/src/test/java/com/flipkart/foxtrot/core/querystore/actions/FilterActionTest.java index 87687a7da..9c2ecacaf 100644 --- a/foxtrot-core/src/test/java/com/flipkart/foxtrot/core/querystore/actions/FilterActionTest.java +++ b/foxtrot-core/src/test/java/com/flipkart/foxtrot/core/querystore/actions/FilterActionTest.java @@ -692,7 +692,7 @@ public void testMissingIndicesQuery() throws QueryStoreException { assertEquals(3, response.getIndices().length); Query query = new Query(); - query.setLimit(20); + query.setLimit(documents.size()); query.setTable(TestUtils.TEST_TABLE_NAME); BetweenFilter betweenFilter = new BetweenFilter(); betweenFilter.setField("_timestamp"); @@ -702,7 +702,8 @@ public void testMissingIndicesQuery() throws QueryStoreException { query.setFilters(Lists.newArrayList(betweenFilter)); QueryResponse actualResponse = QueryResponse.class.cast(queryExecutor.execute(query)); - assertEquals(18, actualResponse.getDocuments().size()); + assertEquals(documents.size(), actualResponse.getDocuments().size()); + compare(documents, actualResponse.getDocuments()); } From 64aaaffa52f2778fdeacacca817f0a50cffb7c67 Mon Sep 17 00:00:00 2001 From: Vivek Kothari Date: Wed, 25 Nov 2015 23:59:38 +0530 Subject: [PATCH 13/64] fix jUnit --- .../foxtrot/core/querystore/actions/FilterActionTest.java | 1 - 1 file changed, 1 deletion(-) diff --git a/foxtrot-core/src/test/java/com/flipkart/foxtrot/core/querystore/actions/FilterActionTest.java b/foxtrot-core/src/test/java/com/flipkart/foxtrot/core/querystore/actions/FilterActionTest.java index 9c2ecacaf..39c214d3f 100644 --- a/foxtrot-core/src/test/java/com/flipkart/foxtrot/core/querystore/actions/FilterActionTest.java +++ b/foxtrot-core/src/test/java/com/flipkart/foxtrot/core/querystore/actions/FilterActionTest.java @@ -703,7 +703,6 @@ public void testMissingIndicesQuery() throws QueryStoreException { QueryResponse actualResponse = QueryResponse.class.cast(queryExecutor.execute(query)); assertEquals(documents.size(), actualResponse.getDocuments().size()); - compare(documents, actualResponse.getDocuments()); } From 246b6f27876c7c8a5c7b98d88ad2f0cc825dce51 Mon Sep 17 00:00:00 2001 From: Santanu Sinha Date: Thu, 26 Nov 2015 04:22:04 +0530 Subject: [PATCH 14/64] Sorting of hbase keys - Meta field added to document - DocumentTranslator created that translates between low level and high level doc - HbaseDataStore saves transparently and returns translated document - Translated doc is saved in es - Data part goes as "dcoument" type - Metadata part goes as "metadata" type - Get accepts flag to do search or not - If flag is set, it searched for provided id's in metadata documents - FilterAction sets flags false and directly goes through to pull data from HBase - No other action affected --- .../com/flipkart/foxtrot/common/Document.java | 27 ++++ .../foxtrot/common/DocumentMetadata.java | 41 ++++++ .../foxtrot/core/datastore/DataStore.java | 4 +- .../datastore/impl/hbase/HBaseDataStore.java | 61 ++++++--- .../core/querystore/DocumentTranslator.java | 75 +++++++++++ .../foxtrot/core/querystore/QueryStore.java | 2 + .../core/querystore/actions/FilterAction.java | 4 +- .../querystore/actions/HistogramAction.java | 2 +- .../core/querystore/actions/StatsAction.java | 2 +- .../querystore/actions/StatsTrendAction.java | 2 +- .../impl/ElasticsearchQueryStore.java | 124 +++++++++++++++--- .../querystore/impl/ElasticsearchUtils.java | 54 ++++++-- .../query/ElasticSearchQueryGenerator.java | 2 +- .../foxtrot/core/MockElasticsearchServer.java | 5 + .../impl/hbase/HBaseDataStoreTest.java | 30 +++-- .../impl/ElasticsearchQueryStoreTest.java | 48 +++---- .../resources/DocumentResourceTest.java | 6 +- 17 files changed, 403 insertions(+), 86 deletions(-) create mode 100644 foxtrot-common/src/main/java/com/flipkart/foxtrot/common/DocumentMetadata.java create mode 100644 foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/DocumentTranslator.java diff --git a/foxtrot-common/src/main/java/com/flipkart/foxtrot/common/Document.java b/foxtrot-common/src/main/java/com/flipkart/foxtrot/common/Document.java index db4170c97..245ad6851 100644 --- a/foxtrot-common/src/main/java/com/flipkart/foxtrot/common/Document.java +++ b/foxtrot-common/src/main/java/com/flipkart/foxtrot/common/Document.java @@ -36,6 +36,8 @@ public class Document implements Serializable { @JsonProperty private long timestamp; + private DocumentMetadata metadata; + @NotNull @JsonProperty private JsonNode data; @@ -50,6 +52,13 @@ public Document(String id, long timestamp, JsonNode data) { this.data = data; } + public Document(String id, long timestamp, DocumentMetadata metadata, JsonNode data) { + this.id = id; + this.timestamp = timestamp; + this.metadata = metadata; + this.data = data; + } + public String getId() { return id; } @@ -73,4 +82,22 @@ public long getTimestamp() { public void setTimestamp(long timestamp) { this.timestamp = timestamp; } + + public DocumentMetadata getMetadata() { + return metadata; + } + + public void setMetadata(DocumentMetadata metadata) { + this.metadata = metadata; + } + + @Override + public String toString() { + return "Document{" + + "id='" + id + '\'' + + ", timestamp=" + timestamp + + ", metadata=" + metadata + + ", data=" + data + + '}'; + } } diff --git a/foxtrot-common/src/main/java/com/flipkart/foxtrot/common/DocumentMetadata.java b/foxtrot-common/src/main/java/com/flipkart/foxtrot/common/DocumentMetadata.java new file mode 100644 index 000000000..1dce2f436 --- /dev/null +++ b/foxtrot-common/src/main/java/com/flipkart/foxtrot/common/DocumentMetadata.java @@ -0,0 +1,41 @@ +package com.flipkart.foxtrot.common; + +/** + * Metadata for a document + */ +public class DocumentMetadata { + private String id; + private String rowKey; + + public DocumentMetadata() { + } + + public DocumentMetadata(String id, String rowKey) { + this.id = id; + this.rowKey = rowKey; + } + + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + } + + public String getRowKey() { + return rowKey; + } + + public void setRowKey(String rowKey) { + this.rowKey = rowKey; + } + + @Override + public String toString() { + return "DocumentMetadata{" + + "id='" + id + '\'' + + ", rowKey='" + rowKey + '\'' + + '}'; + } +} diff --git a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/datastore/DataStore.java b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/datastore/DataStore.java index c758613da..a060a75be 100644 --- a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/datastore/DataStore.java +++ b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/datastore/DataStore.java @@ -26,8 +26,8 @@ * Time: 9:17 PM */ public interface DataStore { - public void save(final Table table, final Document document) throws DataStoreException; - public void save(final Table table, final List documents) throws DataStoreException; + public Document save(final Table table, final Document document) throws DataStoreException; + public List save(final Table table, final List documents) throws DataStoreException; public Document get(final Table table, final String id) throws DataStoreException; public List get(final Table table, final List ids) throws DataStoreException; } diff --git a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/datastore/impl/hbase/HBaseDataStore.java b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/datastore/impl/hbase/HBaseDataStore.java index bddfa45f0..48c44d2f6 100644 --- a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/datastore/impl/hbase/HBaseDataStore.java +++ b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/datastore/impl/hbase/HBaseDataStore.java @@ -18,10 +18,13 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import com.flipkart.foxtrot.common.Document; +import com.flipkart.foxtrot.common.DocumentMetadata; import com.flipkart.foxtrot.common.Table; import com.flipkart.foxtrot.core.datastore.DataStore; import com.flipkart.foxtrot.core.datastore.DataStoreException; +import com.flipkart.foxtrot.core.querystore.DocumentTranslator; import com.google.common.base.Stopwatch; +import com.google.common.collect.ImmutableList; import org.apache.hadoop.hbase.client.Get; import org.apache.hadoop.hbase.client.HTableInterface; import org.apache.hadoop.hbase.client.Put; @@ -44,11 +47,13 @@ public class HBaseDataStore implements DataStore { private static final Logger logger = LoggerFactory.getLogger(HBaseDataStore.class.getSimpleName()); private static final byte[] COLUMN_FAMILY = Bytes.toBytes("d"); - private static final byte[] DATA_FIELD_NAME = Bytes.toBytes("data"); + private static final byte[] DOCUMENT_FIELD_NAME = Bytes.toBytes("data"); + private static final byte[] DOCUMENT_META_FIELD_NAME = Bytes.toBytes("metadata"); private static final byte[] TIMESTAMP_FIELD_NAME = Bytes.toBytes("timestamp"); private final HbaseTableConnection tableWrapper; private final ObjectMapper mapper; + private final DocumentTranslator translator = new DocumentTranslator(); public HBaseDataStore(HbaseTableConnection tableWrapper, ObjectMapper mapper) { this.tableWrapper = tableWrapper; @@ -56,16 +61,18 @@ public HBaseDataStore(HbaseTableConnection tableWrapper, ObjectMapper mapper) { } @Override - public void save(final Table table, Document document) throws DataStoreException { + public Document save(final Table table, Document document) throws DataStoreException { if (document == null || document.getData() == null || document.getId() == null) { throw new DataStoreException(DataStoreException.ErrorCode.STORE_INVALID_REQUEST, "Invalid Document"); } HTableInterface hTable = null; + Document translatedDocument = null; try { + translatedDocument = translator.translate(table, document); hTable = tableWrapper.getTable(table); Stopwatch stopwatch = new Stopwatch(); stopwatch.start(); - hTable.put(getPutForDocument(table, document)); + hTable.put(getPutForDocument(table, translatedDocument)); logger.error(String.format("HBASE put took : %d table : %s", stopwatch.elapsedMillis(), table)); } catch (JsonProcessingException e) { throw new DataStoreException(DataStoreException.ErrorCode.STORE_INVALID_REQUEST, @@ -85,21 +92,25 @@ public void save(final Table table, Document document) throws DataStoreException } } } + return translatedDocument; } @Override - public void save(final Table table, List documents) throws DataStoreException { + public List save(final Table table, List documents) throws DataStoreException { if (documents == null || documents.isEmpty()) { throw new DataStoreException(DataStoreException.ErrorCode.STORE_INVALID_REQUEST, "Invalid Documents List"); } List puts = new Vector(); + ImmutableList.Builder translatedDocuments = ImmutableList.builder(); try { for (Document document : documents) { if (document == null || document.getData() == null || document.getId() == null) { throw new DataStoreException(DataStoreException.ErrorCode.STORE_INVALID_REQUEST, "Invalid Document"); } - puts.add(getPutForDocument(table, document)); + Document translatedDocument = translator.translate(table, document); + puts.add(getPutForDocument(table, translatedDocument)); + translatedDocuments.add(translatedDocument); } } catch (JsonProcessingException e) { throw new DataStoreException(DataStoreException.ErrorCode.STORE_INVALID_REQUEST, @@ -128,22 +139,27 @@ public void save(final Table table, List documents) throws DataStoreEx } } } + return translatedDocuments.build(); } @Override public Document get(final Table table, String id) throws DataStoreException { HTableInterface hTable = null; try { - Get get = new Get(Bytes.toBytes(id + ":" + table.getName())) - .addColumn(COLUMN_FAMILY, DATA_FIELD_NAME) - .addColumn(COLUMN_FAMILY, TIMESTAMP_FIELD_NAME); + Get get = new Get(Bytes.toBytes(id)) + .addColumn(COLUMN_FAMILY, DOCUMENT_FIELD_NAME) + .addColumn(COLUMN_FAMILY, DOCUMENT_META_FIELD_NAME) + .addColumn(COLUMN_FAMILY, TIMESTAMP_FIELD_NAME) + .setMaxVersions(1); hTable = tableWrapper.getTable(table); Result getResult = hTable.get(get); if (!getResult.isEmpty()) { - byte[] data = getResult.getValue(COLUMN_FAMILY, DATA_FIELD_NAME); + byte[] data = getResult.getValue(COLUMN_FAMILY, DOCUMENT_FIELD_NAME); + byte[] metadata = getResult.getValue(COLUMN_FAMILY, DOCUMENT_META_FIELD_NAME); byte[] timestamp = getResult.getValue(COLUMN_FAMILY, TIMESTAMP_FIELD_NAME); long time = Bytes.toLong(timestamp); - return new Document(id, time, mapper.readTree(data)); + DocumentMetadata documentMetadata = (null != metadata) ? mapper.readValue(metadata, DocumentMetadata.class) : null; + return translator.translateBack(new Document(id, time, documentMetadata, mapper.readTree(data))); } else { throw new DataStoreException(DataStoreException.ErrorCode.STORE_NO_DATA_FOUND_FOR_ID, String.format("No data found for ID: %s", id)); @@ -177,9 +193,11 @@ public List get(final Table table, List ids) throws DataStoreE try { List gets = new ArrayList(ids.size()); for (String id : ids) { - Get get = new Get(Bytes.toBytes(id + ":" + table.getName())) - .addColumn(COLUMN_FAMILY, DATA_FIELD_NAME) - .addColumn(COLUMN_FAMILY, TIMESTAMP_FIELD_NAME); + Get get = new Get(Bytes.toBytes(id)) + .addColumn(COLUMN_FAMILY, DOCUMENT_FIELD_NAME) + .addColumn(COLUMN_FAMILY, DOCUMENT_META_FIELD_NAME) + .addColumn(COLUMN_FAMILY, TIMESTAMP_FIELD_NAME) + .setMaxVersions(1); gets.add(get); } hTable = tableWrapper.getTable(table); @@ -188,11 +206,17 @@ public List get(final Table table, List ids) throws DataStoreE for (int index = 0; index < getResults.length; index++) { Result getResult = getResults[index]; if (!getResult.isEmpty()) { - byte[] data = getResult.getValue(COLUMN_FAMILY, DATA_FIELD_NAME); + byte[] data = getResult.getValue(COLUMN_FAMILY, DOCUMENT_FIELD_NAME); + byte[] metadata = getResult.getValue(COLUMN_FAMILY, DOCUMENT_META_FIELD_NAME); byte[] timestamp = getResult.getValue(COLUMN_FAMILY, TIMESTAMP_FIELD_NAME); long time = Bytes.toLong(timestamp); - results.add(new Document(Bytes.toString(getResult.getRow()).split(":")[0], - time, mapper.readTree(data))); + DocumentMetadata documentMetadata = (null != metadata) + ? mapper.readValue(metadata, DocumentMetadata.class) + : null; + final String docId = (null == metadata) + ? Bytes.toString(getResult.getRow()).split(":")[0] + : documentMetadata.getRowKey(); + results.add(translator.translateBack(new Document(docId, time, documentMetadata, mapper.readTree(data)))); } else { throw new DataStoreException(DataStoreException.ErrorCode.STORE_NO_DATA_FOUND_FOR_IDS, String.format("No data found for ID: %s", ids.get(index))); @@ -222,8 +246,9 @@ public List get(final Table table, List ids) throws DataStoreE } public Put getPutForDocument(final Table table, Document document) throws JsonProcessingException { - return new Put(Bytes.toBytes(document.getId() + ":" + table.getName())) - .add(COLUMN_FAMILY, DATA_FIELD_NAME, mapper.writeValueAsBytes(document.getData())) + return new Put(Bytes.toBytes(document.getId())) + .add(COLUMN_FAMILY, DOCUMENT_META_FIELD_NAME, mapper.writeValueAsBytes(document.getMetadata())) + .add(COLUMN_FAMILY, DOCUMENT_FIELD_NAME, mapper.writeValueAsBytes(document.getData())) .add(COLUMN_FAMILY, TIMESTAMP_FIELD_NAME, Bytes.toBytes(document.getTimestamp())); } } diff --git a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/DocumentTranslator.java b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/DocumentTranslator.java new file mode 100644 index 000000000..d6e360a0e --- /dev/null +++ b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/DocumentTranslator.java @@ -0,0 +1,75 @@ +package com.flipkart.foxtrot.core.querystore; + +import com.flipkart.foxtrot.common.Document; +import com.flipkart.foxtrot.common.DocumentMetadata; +import com.flipkart.foxtrot.common.Table; +import com.google.common.collect.ImmutableList; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.List; + +/** + * Created by santanu.s on 24/11/15. + */ +public class DocumentTranslator { + private static final Logger logger = LoggerFactory.getLogger(DocumentTranslator.class); + + public List translate(final Table table, final List inDocuments) { + ImmutableList.Builder docListBuilder = ImmutableList.builder(); + for(Document document : inDocuments) { + docListBuilder.add(translate(table, document)); + } + return docListBuilder.build(); + } + + public Document translate(final Table table, final Document inDocument) { + Document document = new Document(); + final String rowKey = rowKey(table, inDocument); + + DocumentMetadata metadata = new DocumentMetadata(); + metadata.setRowKey(rowKey); + metadata.setId(inDocument.getId()); + + document.setId(rowKey); + document.setTimestamp(inDocument.getTimestamp()); + document.setMetadata(metadata); + document.setData(inDocument.getData()); + + logger.info("Translated doc row key: {}, {}", rowKey, document); + return document; + } + + public List translateBack(final List inDocuments) { + ImmutableList.Builder listBuilder = ImmutableList.builder(); + if(null != inDocuments) { + for (Document document : inDocuments) { + listBuilder.add(translateBack(document)); + } + } + return listBuilder.build(); + } + + public Document translateBack(final Document inDocument) { + Document document = new Document(); + document.setId(inDocument.getMetadata() != null ? inDocument.getMetadata().getId() : inDocument.getId()); + document.setTimestamp(inDocument.getTimestamp()); + document.setData(inDocument.getData()); + return document; + } + + public DocumentMetadata metadata(final Table table, final Document inDocument) { + final String rowKey = rowKey(table, inDocument); + + DocumentMetadata metadata = new DocumentMetadata(); + metadata.setRowKey(rowKey); + metadata.setId(inDocument.getId()); + + logger.info("Doc row key: {}, {}", rowKey, inDocument); + return metadata; + } + + public String rowKey(final Table table, final Document document) { + return String.format("%s:%d:%s", table.getName(), document.getTimestamp(), document.getId()); + } +} diff --git a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/QueryStore.java b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/QueryStore.java index 3a4534d45..a390ba5a0 100644 --- a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/QueryStore.java +++ b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/QueryStore.java @@ -36,6 +36,8 @@ public interface QueryStore { public List get(final String table, final List ids) throws QueryStoreException; + public List get(final String table, final List ids, boolean bypassMetaLookup) throws QueryStoreException; + public TableFieldMapping getFieldMappings(final String table) throws QueryStoreException; public void cleanupAll() throws QueryStoreException; diff --git a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/FilterAction.java b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/FilterAction.java index a236ec1eb..b73948cbe 100644 --- a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/FilterAction.java +++ b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/FilterAction.java @@ -92,7 +92,7 @@ public QueryResponse execute(Query parameter) throws QueryStoreException { "There is no table called: " + query.getTable()); }*/ search = getConnection().getClient().prepareSearch(ElasticsearchUtils.getIndices(parameter.getTable(), parameter)) - .setTypes(ElasticsearchUtils.TYPE_NAME) + .setTypes(ElasticsearchUtils.DOCUMENT_TYPE_NAME) .setQuery(new ElasticSearchQueryGenerator(FilterCombinerType.and).genFilter(parameter.getFilters())) .setSearchType(SearchType.QUERY_THEN_FETCH) .setFrom(parameter.getFrom()) @@ -107,7 +107,7 @@ public QueryResponse execute(Query parameter) throws QueryStoreException { if (ids.isEmpty()) { return new QueryResponse(Collections.emptyList()); } - return new QueryResponse(getQueryStore().get(parameter.getTable(), ids)); + return new QueryResponse(getQueryStore().get(parameter.getTable(), ids, true)); } catch (Exception e) { if (null != search) { logger.error("Error running generated query: " + search, e); diff --git a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/HistogramAction.java b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/HistogramAction.java index 623a8e1c7..e6a655539 100644 --- a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/HistogramAction.java +++ b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/HistogramAction.java @@ -114,7 +114,7 @@ public ActionResponse execute(HistogramRequest parameter) throws QueryStoreExcep String dateHistogramKey = Utils.sanitizeFieldForAggregation(parameter.getField()); SearchResponse response = getConnection().getClient().prepareSearch( ElasticsearchUtils.getIndices(parameter.getTable(), parameter)) - .setTypes(ElasticsearchUtils.TYPE_NAME) + .setTypes(ElasticsearchUtils.DOCUMENT_TYPE_NAME) .setQuery(new ElasticSearchQueryGenerator(FilterCombinerType.and) .genFilter(parameter.getFilters())) .setSize(0) diff --git a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/StatsAction.java b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/StatsAction.java index bced0953b..dadb51f94 100644 --- a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/StatsAction.java +++ b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/StatsAction.java @@ -72,7 +72,7 @@ public ActionResponse execute(StatsRequest request) throws QueryStoreException { try { SearchResponse response = getConnection().getClient().prepareSearch( ElasticsearchUtils.getIndices(request.getTable(), request)) - .setTypes(ElasticsearchUtils.TYPE_NAME) + .setTypes(ElasticsearchUtils.DOCUMENT_TYPE_NAME) .setQuery(new ElasticSearchQueryGenerator(request.getCombiner()).genFilter(request.getFilters())) .setSize(0) .setSearchType(SearchType.COUNT) diff --git a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/StatsTrendAction.java b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/StatsTrendAction.java index 70aba22a8..96ec00726 100644 --- a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/StatsTrendAction.java +++ b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/StatsTrendAction.java @@ -85,7 +85,7 @@ public ActionResponse execute(StatsTrendRequest parameter) throws QueryStoreExce AbstractAggregationBuilder aggregation = buildAggregation(parameter); SearchResponse response = getConnection().getClient().prepareSearch( ElasticsearchUtils.getIndices(parameter.getTable(), parameter)) - .setTypes(ElasticsearchUtils.TYPE_NAME) + .setTypes(ElasticsearchUtils.DOCUMENT_TYPE_NAME) .setQuery(new ElasticSearchQueryGenerator(parameter.getCombiner()).genFilter(parameter.getFilters())) .setSize(0) .setSearchType(SearchType.COUNT) diff --git a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/impl/ElasticsearchQueryStore.java b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/impl/ElasticsearchQueryStore.java index 4f33a82b7..7c3c11a1d 100644 --- a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/impl/ElasticsearchQueryStore.java +++ b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/impl/ElasticsearchQueryStore.java @@ -17,29 +17,38 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; -import com.flipkart.foxtrot.common.Document; -import com.flipkart.foxtrot.common.FieldTypeMapping; -import com.flipkart.foxtrot.common.Table; -import com.flipkart.foxtrot.common.TableFieldMapping; +import com.flipkart.foxtrot.common.*; import com.flipkart.foxtrot.core.datastore.DataStore; import com.flipkart.foxtrot.core.datastore.DataStoreException; import com.flipkart.foxtrot.core.parsers.ElasticsearchMappingParser; +import com.flipkart.foxtrot.core.querystore.DocumentTranslator; import com.flipkart.foxtrot.core.querystore.QueryStore; import com.flipkart.foxtrot.core.querystore.QueryStoreException; import com.flipkart.foxtrot.core.querystore.TableMetadataManager; import com.google.common.base.Stopwatch; +import com.google.common.collect.ImmutableSet; import com.google.common.collect.Lists; +import com.google.common.collect.Sets; import org.elasticsearch.action.WriteConsistencyLevel; +import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsRequest; import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse; import org.elasticsearch.action.admin.indices.status.IndicesStatusResponse; +import org.elasticsearch.action.admin.indices.template.get.GetIndexTemplatesRequest; import org.elasticsearch.action.bulk.BulkItemResponse; import org.elasticsearch.action.bulk.BulkRequestBuilder; import org.elasticsearch.action.bulk.BulkResponse; +import org.elasticsearch.action.get.MultiGetRequest; import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.action.index.IndexRequestBuilder; +import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.cluster.metadata.MappingMetaData; +import org.elasticsearch.common.collect.ImmutableOpenMap; import org.elasticsearch.common.hppc.cursors.ObjectCursor; import org.elasticsearch.common.joda.time.DateTime; import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.index.query.FilterBuilders; +import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.search.SearchHit; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -80,17 +89,25 @@ public void save(String table, Document document) throws QueryStoreException { if (new DateTime().plusDays(1).minus(document.getTimestamp()).getMillis() < 0) { return; } - dataStore.save(tableMetadataManager.get(table), document); - long timestamp = document.getTimestamp(); + final Table tableMeta = tableMetadataManager.get(table); + final Document translatedDocuement = dataStore.save(tableMeta, document); + long timestamp = translatedDocuement.getTimestamp(); Stopwatch stopwatch = new Stopwatch(); stopwatch.start(); connection.getClient() - .prepareIndex() - .setIndex(ElasticsearchUtils.getCurrentIndex(table, timestamp)) - .setType(ElasticsearchUtils.TYPE_NAME) - .setId(document.getId()) - .setTimestamp(Long.toString(timestamp)) - .setSource(mapper.writeValueAsBytes(document.getData())) + .prepareBulk() + .add(new IndexRequest() + .index(ElasticsearchUtils.getCurrentIndex(table, timestamp)) + .type(ElasticsearchUtils.DOCUMENT_TYPE_NAME) + .id(translatedDocuement.getId()) + .timestamp(Long.toString(timestamp)) + .source(mapper.writeValueAsBytes(translatedDocuement.getData()))) + .add(new IndexRequest() + .index(ElasticsearchUtils.getCurrentIndex(table, timestamp)) + .type(ElasticsearchUtils.DOCUMENT_META_TYPE_NAME) + .id(translatedDocuement.getId()) + .timestamp(Long.toString(timestamp)) + .source(mapper.writeValueAsBytes(translatedDocuement.getMetadata()))) .setConsistencyLevel(WriteConsistencyLevel.QUORUM) .execute() .get(2, TimeUnit.SECONDS); @@ -128,12 +145,13 @@ public void save(String table, List documents) throws QueryStoreExcept throw new QueryStoreException(QueryStoreException.ErrorCode.INVALID_REQUEST, "Invalid Document List"); } - dataStore.save(tableMetadataManager.get(table), documents); + final Table tableMeta = tableMetadataManager.get(table); + final List translatedDocuments = dataStore.save(tableMeta, documents); BulkRequestBuilder bulkRequestBuilder = connection.getClient().prepareBulk(); DateTime dateTime = new DateTime().plusDays(1); - for (Document document : documents) { + for (Document document : translatedDocuments) { long timestamp = document.getTimestamp(); if (dateTime.minus(timestamp).getMillis() < 0) { continue; @@ -141,11 +159,17 @@ public void save(String table, List documents) throws QueryStoreExcept final String index = ElasticsearchUtils.getCurrentIndex(table, timestamp); IndexRequest indexRequest = new IndexRequest() .index(index) - .type(ElasticsearchUtils.TYPE_NAME) + .type(ElasticsearchUtils.DOCUMENT_TYPE_NAME) .id(document.getId()) .timestamp(Long.toString(timestamp)) .source(mapper.writeValueAsBytes(document.getData())); bulkRequestBuilder.add(indexRequest); + IndexRequest metaIndexRequest = new IndexRequest() + .index(index) + .type(ElasticsearchUtils.DOCUMENT_META_TYPE_NAME) + .id(document.getId()) + .source(mapper.writeValueAsBytes(document.getMetadata())); + bulkRequestBuilder.add(metaIndexRequest); } if (bulkRequestBuilder.numberOfActions() > 0){ Stopwatch stopwatch = new Stopwatch(); @@ -202,8 +226,28 @@ public Document get(String table, String id) throws QueryStoreException { throw new QueryStoreException(QueryStoreException.ErrorCode.DOCUMENT_GET_ERROR, ex.getMessage(), ex); } + String lookupKey = null; try { - return dataStore.get(fxTable, id); + SearchResponse searchResponse = connection.getClient() + .prepareSearch(ElasticsearchUtils.getIndices(table)) + .setTypes(ElasticsearchUtils.DOCUMENT_META_TYPE_NAME) + .setQuery( + QueryBuilders.constantScoreQuery( + FilterBuilders.boolFilter() + .must(FilterBuilders.termFilter("id", id)))) + .setNoFields() + .setSize(1) + .execute() + .actionGet(); + if(searchResponse.getHits().totalHits() == 0 ) { + logger.warn("Going into compatibility mode, looks using passed in ID as the data store id: {}", id); + lookupKey = id; + } + else { + lookupKey = searchResponse.getHits().getHits()[0].getId(); + logger.debug("Translated lookup key for {} is {}.", id, lookupKey); + } + return dataStore.get(fxTable, lookupKey); } catch (DataStoreException ex) { if (ex.getErrorCode().equals(DataStoreException.ErrorCode.STORE_NO_DATA_FOUND_FOR_ID)) { throw new QueryStoreException(QueryStoreException.ErrorCode.DOCUMENT_NOT_FOUND, @@ -217,13 +261,54 @@ public Document get(String table, String id) throws QueryStoreException { @Override public List get(String table, List ids) throws QueryStoreException { + return get(table, ids, false); + } + + @Override + public List get(String table, List ids, boolean bypassMetalookup) throws QueryStoreException { table = ElasticsearchUtils.getValidTableName(table); try { if (!tableMetadataManager.exists(table)) { throw new QueryStoreException(QueryStoreException.ErrorCode.NO_SUCH_TABLE, "No table exists with the name: " + table); } - return dataStore.get(tableMetadataManager.get(table), ids); + ImmutableOpenMap> mappings = connection.getClient() + .admin().indices().getMappings( + new GetMappingsRequest() + .indices(ElasticsearchUtils.getIndices(table)) + .types("metadata")) + .actionGet() + .mappings(); + + List rowKeys; + if(!bypassMetalookup) { + rowKeys = Lists.newArrayList(); + SearchResponse response = connection.getClient().prepareSearch(ElasticsearchUtils.getIndices(table)) + .setQuery( + QueryBuilders.constantScoreQuery( + FilterBuilders.inFilter("id", ids.toArray(new String[ids.size()])))) + .setFetchSource(false) + .addField("id") + .setSize(ids.size()) + .execute() + .actionGet(); + Set inputKeys = ImmutableSet.copyOf(ids); + Set foundKeys = Sets.newHashSet(); + for (SearchHit hit : response.getHits()) { + rowKeys.add(hit.getId()); + foundKeys.add(hit.getFields().get("id").getValue().toString()); + } + + //Now we've got new keys + //Add them all to fetchList + //Add old generation keys and invalid keys as is + rowKeys.addAll(Sets.difference(inputKeys, foundKeys)); + } + else { + rowKeys = ids; + } + logger.info("Get row keys: {}", rowKeys.size()); + return dataStore.get(tableMetadataManager.get(table), rowKeys); } catch (DataStoreException ex) { if (ex.getErrorCode().equals(DataStoreException.ErrorCode.STORE_NO_DATA_FOUND_FOR_IDS)) { throw new QueryStoreException(QueryStoreException.ErrorCode.DOCUMENT_NOT_FOUND, @@ -252,7 +337,7 @@ public TableFieldMapping getFieldMappings(String table) throws QueryStoreExcepti .indices().prepareGetMappings(ElasticsearchUtils.getIndices(table)).execute().actionGet(); for (ObjectCursor index : mappingsResponse.getMappings().keys()) { - MappingMetaData mappingData = mappingsResponse.mappings().get(index.value).get(ElasticsearchUtils.TYPE_NAME); + MappingMetaData mappingData = mappingsResponse.mappings().get(index.value).get(ElasticsearchUtils.DOCUMENT_TYPE_NAME); mappings.addAll(mappingParser.getFieldMappings(mappingData)); } return new TableFieldMapping(table, mappings); @@ -325,4 +410,7 @@ public void cleanup(Set tables) throws QueryStoreException { String.format("Unable to delete Indexes - %s", indicesToDelete), ex); } } + + + } diff --git a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/impl/ElasticsearchUtils.java b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/impl/ElasticsearchUtils.java index ea151f1c1..0e198cf1a 100644 --- a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/impl/ElasticsearchUtils.java +++ b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/impl/ElasticsearchUtils.java @@ -45,7 +45,8 @@ public class ElasticsearchUtils { private static final Logger logger = LoggerFactory.getLogger(ElasticsearchUtils.class.getSimpleName()); - public static final String TYPE_NAME = "document"; + public static final String DOCUMENT_TYPE_NAME = "document"; + public static final String DOCUMENT_META_TYPE_NAME = "metadata"; public static final String TABLENAME_PREFIX = "foxtrot"; public static final String TABLENAME_POSTFIX = "table"; private static final DateTimeFormatter FORMATTER = DateTimeFormat.forPattern("dd-M-yyyy"); @@ -64,15 +65,15 @@ public static String getIndexPrefix(final String table) { return String.format("%s-%s-%s-", ElasticsearchUtils.TABLENAME_PREFIX, table, ElasticsearchUtils.TABLENAME_POSTFIX); } - public static String[] getIndices(final String table) { + public static String getIndices(final String table) { /*long currentTime = new Date().getTime(); String names[] = new String[30]; //TODO::USE TABLE METADATA for(int i = 0 ; i < 30; i++) { String postfix = new SimpleDateFormat("dd-M-yyyy").format(new Date(currentTime)); names[i] = String.format("%s-%s-%s", TABLENAME_PREFIX, table, postfix); }*/ - return new String[]{String.format("%s-%s-%s-*", - ElasticsearchUtils.TABLENAME_PREFIX, table, ElasticsearchUtils.TABLENAME_POSTFIX)}; + return String.format("%s-%s-%s-*", + ElasticsearchUtils.TABLENAME_PREFIX, table, ElasticsearchUtils.TABLENAME_POSTFIX); } public static String[] getIndices(final String table, final ActionRequest request) throws Exception { @@ -84,7 +85,7 @@ public static String[] getIndices(final String table, final ActionRequest reques DateTime start = interval.getStart().toLocalDate().toDateTimeAtStartOfDay(); if(start.getYear() <= 1970) { logger.warn("Request of type {} running on all indices", request.getClass().getSimpleName()); - return getIndices(table); + return new String[] {getIndices(table)}; } List indices = Lists.newArrayList(); final DateTime end = interval.getEnd().plusDays(1).toLocalDate().toDateTimeAtStartOfDay(); @@ -106,10 +107,10 @@ public static String getCurrentIndex(final String table, long timestamp) { public static PutIndexTemplateRequest getClusterTemplateMapping(IndicesAdminClient indicesAdminClient){ try { - XContentBuilder contentBuilder = getDocumentMapping(); PutIndexTemplateRequestBuilder builder = new PutIndexTemplateRequestBuilder(indicesAdminClient, "generic_template"); builder.setTemplate("foxtrot-*"); - builder.addMapping(TYPE_NAME, contentBuilder); + builder.addMapping(DOCUMENT_TYPE_NAME, getDocumentMapping()); + builder.addMapping(DOCUMENT_META_TYPE_NAME, getDocumentMetadataMapping()); return builder.request(); } catch (IOException ex){ logger.error("TEMPLATE_CREATION_FAILED", ex); @@ -117,10 +118,47 @@ public static PutIndexTemplateRequest getClusterTemplateMapping(IndicesAdminClie } } + public static XContentBuilder getDocumentMetadataMapping() throws IOException { + return XContentFactory.jsonBuilder() + .startObject() + .field(DOCUMENT_META_TYPE_NAME) + .startObject() + .field("_source") + .startObject() + .field("enabled", false) + .endObject() + .field("_all") + .startObject() + .field("enabled", false) + .endObject() + .field("dynamic_templates") + .startArray() + .startObject() + .field("template_no_store") + .startObject() + .field("match_mapping_type", "string|date|boolean|double|long|integer") + .field("match_pattern", "regex") + .field("path_match", ".*") + .field("mapping") + .startObject() + .field("store", true) + .field("doc_values", true) + .field("index", "not_analyzed") + .field("fielddata") + .startObject() + .field("format", "doc_values") + .endObject() + .endObject() + .endObject() + .endObject() + .endArray() + .endObject() + .endObject(); + } public static XContentBuilder getDocumentMapping() throws IOException { return XContentFactory.jsonBuilder() .startObject() - .field(TYPE_NAME) + .field(DOCUMENT_TYPE_NAME) .startObject() .field("_source") .startObject() diff --git a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/query/ElasticSearchQueryGenerator.java b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/query/ElasticSearchQueryGenerator.java index 76e63fa25..2ef0d3ad7 100644 --- a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/query/ElasticSearchQueryGenerator.java +++ b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/query/ElasticSearchQueryGenerator.java @@ -146,7 +146,7 @@ public QueryBuilder genFilter(List filters) throws Exception { for (Filter filter : filters) { filter.accept(this); } - return QueryBuilders.filteredQuery(QueryBuilders.matchAllQuery(), boolFilterBuilder); + return QueryBuilders.constantScoreQuery(boolFilterBuilder); } } diff --git a/foxtrot-core/src/test/java/com/flipkart/foxtrot/core/MockElasticsearchServer.java b/foxtrot-core/src/test/java/com/flipkart/foxtrot/core/MockElasticsearchServer.java index 4dbcc232b..a8b8a8691 100644 --- a/foxtrot-core/src/test/java/com/flipkart/foxtrot/core/MockElasticsearchServer.java +++ b/foxtrot-core/src/test/java/com/flipkart/foxtrot/core/MockElasticsearchServer.java @@ -17,6 +17,7 @@ import org.apache.commons.io.FileUtils; import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest; +import org.elasticsearch.action.admin.indices.refresh.RefreshRequest; import org.elasticsearch.client.Client; import org.elasticsearch.common.settings.ImmutableSettings; import org.elasticsearch.node.Node; @@ -47,6 +48,10 @@ public MockElasticsearchServer(String directory) { .node(); } + public void refresh(final String index) { + node.client().admin().indices().refresh(new RefreshRequest().indices(index)).actionGet(); + } + public Client getClient() { return node.client(); } diff --git a/foxtrot-core/src/test/java/com/flipkart/foxtrot/core/datastore/impl/hbase/HBaseDataStoreTest.java b/foxtrot-core/src/test/java/com/flipkart/foxtrot/core/datastore/impl/hbase/HBaseDataStoreTest.java index cd3beab66..137df1438 100644 --- a/foxtrot-core/src/test/java/com/flipkart/foxtrot/core/datastore/impl/hbase/HBaseDataStoreTest.java +++ b/foxtrot-core/src/test/java/com/flipkart/foxtrot/core/datastore/impl/hbase/HBaseDataStoreTest.java @@ -18,9 +18,11 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.flipkart.foxtrot.common.Document; +import com.flipkart.foxtrot.common.DocumentMetadata; import com.flipkart.foxtrot.common.Table; import com.flipkart.foxtrot.core.MockHTable; import com.flipkart.foxtrot.core.datastore.DataStoreException; +import com.flipkart.foxtrot.core.querystore.DocumentTranslator; import org.apache.hadoop.hbase.client.Get; import org.apache.hadoop.hbase.client.HTableInterface; import org.apache.hadoop.hbase.client.Put; @@ -149,11 +151,14 @@ public void testSaveSingleNullHBaseTableConnection() throws Exception { } } + private DocumentTranslator translator = new DocumentTranslator(); @Test public void testSaveBulk() throws Exception { List documents = new Vector(); for (int i = 0; i < 10; i++) { - documents.add(new Document(UUID.randomUUID().toString(), + final String id = UUID.randomUUID().toString(); + documents.add(new Document( + id, System.currentTimeMillis(), mapper.valueToTree(Collections.singletonMap("TEST_NAME", "SINGLE_SAVE_TEST")))); } @@ -264,7 +269,7 @@ public void testSaveBulkNullHBaseTableConnection() throws Exception { } public void validateSave(Document savedDocument) throws Exception { - String rowkey = String.format("%s:%s", savedDocument.getId(), TEST_APP_NAME); + String rowkey = new DocumentTranslator().rowKey(TEST_APP, savedDocument); Get get = new Get(Bytes.toBytes(rowkey)); Result result = tableInterface.get(get); assertNotNull("Get for Id should not be null", result); @@ -278,7 +283,7 @@ public void validateSave(Document savedDocument) throws Exception { public void testGetSingle() throws Exception { String id = UUID.randomUUID().toString(); JsonNode data = mapper.valueToTree(Collections.singletonMap("TEST_NAME", "SINGLE_SAVE_TEST")); - Document expectedDocument = new Document(id, System.currentTimeMillis(), data); + Document expectedDocument = new Document(id, System.currentTimeMillis(), new DocumentMetadata(id, "row.1"), data); tableInterface.put(HBaseDataStore.getPutForDocument(TEST_APP, expectedDocument)); Document actualDocument = HBaseDataStore.get(TEST_APP, id); compare(expectedDocument, actualDocument); @@ -349,12 +354,13 @@ public void testGetBulk() throws Exception { for (int i = 0; i < 10; i++) { String id = UUID.randomUUID().toString(); ids.add(id); - idValues.put(id, - new Document(id, System.currentTimeMillis(), mapper.valueToTree(Collections.singletonMap("TEST_NAME", "BULK_GET_TEST")))); + Document document = new Document(id, System.currentTimeMillis(), + new DocumentMetadata(id, id), + mapper.valueToTree(Collections.singletonMap("TEST_NAME", "BULK_GET_TEST"))); + idValues.put(document.getMetadata().getId(), document); putList.add(HBaseDataStore.getPutForDocument(TEST_APP, idValues.get(id))); } tableInterface.put(putList); - List actualDocuments = HBaseDataStore.get(TEST_APP, ids); HashMap actualIdValues = new HashMap(); for (Document doc : actualDocuments) { @@ -398,7 +404,9 @@ public void testGetBulkHBaseReadException() throws Exception { List putList = new Vector(); for (int i = 0; i < 10; i++) { String id = UUID.randomUUID().toString(); - Document document = new Document(id, System.currentTimeMillis(), mapper.valueToTree(Collections.singletonMap("TEST_NAME", "BULK_GET_TEST"))); + Document document = new Document(id, System.currentTimeMillis(), + new DocumentMetadata(id, String.format("row:%d", i)), + mapper.valueToTree(Collections.singletonMap("TEST_NAME", "BULK_GET_TEST"))); putList.add(HBaseDataStore.getPutForDocument(TEST_APP, document)); } tableInterface.put(putList); @@ -419,7 +427,9 @@ public void testGetBulkHBaseCloseException() throws Exception { List putList = new Vector(); for (int i = 0; i < 10; i++) { String id = UUID.randomUUID().toString(); - Document document = new Document(id, System.currentTimeMillis(), mapper.valueToTree(Collections.singletonMap("TEST_NAME", "BULK_GET_TEST"))); + Document document = new Document(id, System.currentTimeMillis(), + new DocumentMetadata(id, String.format("row:%d", i)), + mapper.valueToTree(Collections.singletonMap("TEST_NAME", "BULK_GET_TEST"))); putList.add(HBaseDataStore.getPutForDocument(TEST_APP, document)); } tableInterface.put(putList); @@ -439,7 +449,9 @@ public void testGetBulkNullHBaseTableConnection() throws Exception { String id = UUID.randomUUID().toString(); ids.add(id); idValues.put(id, - new Document(id, System.currentTimeMillis(), mapper.valueToTree(Collections.singletonMap("TEST_NAME", "BULK_GET_TEST")))); + new Document(id, System.currentTimeMillis(), + new DocumentMetadata(id, String.format("row:%d", i)), + mapper.valueToTree(Collections.singletonMap("TEST_NAME", "BULK_GET_TEST")))); putList.add(HBaseDataStore.getPutForDocument(TEST_APP, idValues.get(id))); } tableInterface.put(putList); diff --git a/foxtrot-core/src/test/java/com/flipkart/foxtrot/core/querystore/impl/ElasticsearchQueryStoreTest.java b/foxtrot-core/src/test/java/com/flipkart/foxtrot/core/querystore/impl/ElasticsearchQueryStoreTest.java index 5b93f4b1a..75b64cda5 100644 --- a/foxtrot-core/src/test/java/com/flipkart/foxtrot/core/querystore/impl/ElasticsearchQueryStoreTest.java +++ b/foxtrot-core/src/test/java/com/flipkart/foxtrot/core/querystore/impl/ElasticsearchQueryStoreTest.java @@ -24,10 +24,12 @@ import com.flipkart.foxtrot.core.MockElasticsearchServer; import com.flipkart.foxtrot.core.TestUtils; import com.flipkart.foxtrot.core.datastore.DataStore; +import com.flipkart.foxtrot.core.querystore.DocumentTranslator; import com.flipkart.foxtrot.core.querystore.QueryExecutor; import com.flipkart.foxtrot.core.querystore.QueryStoreException; import com.flipkart.foxtrot.core.querystore.TableMetadataManager; import com.flipkart.foxtrot.core.querystore.actions.spi.AnalyticsLoader; +import com.google.common.collect.ImmutableList; import org.elasticsearch.action.get.GetResponse; import org.junit.After; import org.junit.Before; @@ -50,6 +52,7 @@ public class ElasticsearchQueryStoreTest { private DataStore dataStore; private ElasticsearchQueryStore queryStore; private ObjectMapper mapper; + private TableMetadataManager tableMetadataManager; @Before public void setUp() throws Exception { @@ -61,7 +64,7 @@ public void setUp() throws Exception { ElasticsearchConnection elasticsearchConnection = Mockito.mock(ElasticsearchConnection.class); when(elasticsearchConnection.getClient()).thenReturn(elasticsearchServer.getClient()); ElasticsearchUtils.initializeMappings(elasticsearchConnection.getClient()); - TableMetadataManager tableMetadataManager = Mockito.mock(TableMetadataManager.class); + tableMetadataManager = Mockito.mock(TableMetadataManager.class); when(tableMetadataManager.exists(TestUtils.TEST_TABLE_NAME)).thenReturn(true); when(tableMetadataManager.get(anyString())).thenReturn(TestUtils.TEST_TABLE); AnalyticsLoader analyticsLoader = new AnalyticsLoader(tableMetadataManager, dataStore, queryStore, elasticsearchConnection); @@ -77,22 +80,22 @@ public void tearDown() throws Exception { @Test public void testSaveSingle() throws Exception { - Document expectedDocument = new Document(); - expectedDocument.setId(UUID.randomUUID().toString()); - expectedDocument.setTimestamp(System.currentTimeMillis()); + Document originalDocument = new Document(); + originalDocument.setId(UUID.randomUUID().toString()); + originalDocument.setTimestamp(System.currentTimeMillis()); JsonNode data = mapper.valueToTree(Collections.singletonMap("TEST_NAME", "SINGLE_SAVE_TEST")); - expectedDocument.setData(data); - queryStore.save(TestUtils.TEST_TABLE_NAME, expectedDocument); - + originalDocument.setData(data); + queryStore.save(TestUtils.TEST_TABLE_NAME, originalDocument); + final Document translatedDocuemnt = new DocumentTranslator().translate(tableMetadataManager.get(TestUtils.TEST_TABLE_NAME), originalDocument); GetResponse getResponse = elasticsearchServer - .getClient() - .prepareGet(ElasticsearchUtils.getCurrentIndex(TestUtils.TEST_TABLE_NAME, expectedDocument.getTimestamp()), - ElasticsearchUtils.TYPE_NAME, - expectedDocument.getId()) - .setFields("_timestamp").execute().actionGet(); + .getClient() + .prepareGet(ElasticsearchUtils.getCurrentIndex(TestUtils.TEST_TABLE_NAME, originalDocument.getTimestamp()), + ElasticsearchUtils.DOCUMENT_TYPE_NAME, + translatedDocuemnt.getId()) + .setFields("_timestamp").execute().actionGet(); assertTrue("Id should exist in ES", getResponse.isExists()); - assertEquals("Id should match requestId", expectedDocument.getId(), getResponse.getId()); - assertEquals("Timestamp should match request timestamp", expectedDocument.getTimestamp(), getResponse.getField("_timestamp").getValue()); + assertEquals("Id should match requestId", translatedDocuemnt.getId(), getResponse.getId()); + assertEquals("Timestamp should match request timestamp", translatedDocuemnt.getTimestamp(), getResponse.getField("_timestamp").getValue()); } @Test @@ -119,12 +122,12 @@ public void testSaveBulk() throws Exception { mapper.valueToTree(Collections.singletonMap("TEST_NAME", "SINGLE_SAVE_TEST")))); } queryStore.save(TestUtils.TEST_TABLE_NAME, documents); - - for (Document document : documents) { + final List translatedDocuemtns = new DocumentTranslator().translate(tableMetadataManager.get(TestUtils.TEST_TABLE_NAME), documents); + for (Document document : translatedDocuemtns) { GetResponse getResponse = elasticsearchServer .getClient() .prepareGet(ElasticsearchUtils.getCurrentIndex(TestUtils.TEST_TABLE_NAME, document.getTimestamp()), - ElasticsearchUtils.TYPE_NAME, + ElasticsearchUtils.DOCUMENT_TYPE_NAME, document.getId()) .setFields("_timestamp").execute().actionGet(); assertTrue("Id should exist in ES", getResponse.isExists()); @@ -178,8 +181,8 @@ public void testGetSingle() throws Exception { JsonNode data = mapper.valueToTree(Collections.singletonMap("TEST_NAME", "SINGLE_SAVE_TEST")); Document document = new Document(id, System.currentTimeMillis(), data); document.setTimestamp(timestamp); - dataStore.save(TestUtils.TEST_TABLE, document); - + queryStore.save(TestUtils.TEST_TABLE_NAME, document); + elasticsearchServer.refresh(ElasticsearchUtils.getIndices(TestUtils.TEST_TABLE_NAME)); Document responseDocument = queryStore.get(TestUtils.TEST_TABLE_NAME, id); assertNotNull(responseDocument); assertEquals(id, responseDocument.getId()); @@ -208,10 +211,8 @@ public void testGetBulk() throws Exception { mapper.valueToTree(Collections.singletonMap("TEST_NAME", "SINGLE_SAVE_TEST")))); idValues.get(id).setTimestamp(System.currentTimeMillis()); } - for (Document document : idValues.values()) { - dataStore.save(TestUtils.TEST_TABLE, document); - } - + queryStore.save(TestUtils.TEST_TABLE_NAME, ImmutableList.copyOf(idValues.values())); + elasticsearchServer.refresh(ElasticsearchUtils.getIndices(TestUtils.TEST_TABLE_NAME)); List responseDocuments = queryStore.get(TestUtils.TEST_TABLE_NAME, ids); HashMap responseIdValues = new HashMap(); for (Document doc : responseDocuments) { @@ -223,6 +224,7 @@ public void testGetBulk() throws Exception { assertNotNull(responseIdValues.get(id)); assertEquals(id, responseIdValues.get(id).getId()); assertEquals("Timestamp should match request timestamp", idValues.get(id).getTimestamp(), responseIdValues.get(id).getTimestamp()); + System.out.println("OK: " + id); } } diff --git a/foxtrot-server/src/test/java/com/flipkart/foxtrot/server/resources/DocumentResourceTest.java b/foxtrot-server/src/test/java/com/flipkart/foxtrot/server/resources/DocumentResourceTest.java index 6bb5d8dfd..e511cf59b 100644 --- a/foxtrot-server/src/test/java/com/flipkart/foxtrot/server/resources/DocumentResourceTest.java +++ b/foxtrot-server/src/test/java/com/flipkart/foxtrot/server/resources/DocumentResourceTest.java @@ -122,6 +122,7 @@ public void testSaveDocument() throws Exception { System.currentTimeMillis(), factory.objectNode().put("hello", "world")); client().resource("/v1/document/" + TestUtils.TEST_TABLE_NAME).type(MediaType.APPLICATION_JSON_TYPE).post(document); + elasticsearchServer.refresh(ElasticsearchUtils.getIndices(TestUtils.TEST_TABLE_NAME)); Document response = queryStore.get(TestUtils.TEST_TABLE_NAME, id); compare(document, response); } @@ -181,7 +182,7 @@ public void testSaveDocuments() throws Exception { documents.add(document1); documents.add(document2); client().resource(String.format("/v1/document/%s/bulk", TestUtils.TEST_TABLE_NAME)).type(MediaType.APPLICATION_JSON_TYPE).post(documents); - + elasticsearchServer.refresh(ElasticsearchUtils.getIndices(TestUtils.TEST_TABLE_NAME)); compare(document1, queryStore.get(TestUtils.TEST_TABLE_NAME, id1)); compare(document2, queryStore.get(TestUtils.TEST_TABLE_NAME, id2)); } @@ -273,7 +274,7 @@ public void testGetDocument() throws Exception { String id = UUID.randomUUID().toString(); Document document = new Document(id, System.currentTimeMillis(), factory.objectNode().put("D", "data")); queryStore.save(TestUtils.TEST_TABLE_NAME, document); - + elasticsearchServer.refresh(ElasticsearchUtils.getIndices(TestUtils.TEST_TABLE_NAME)); Document response = client().resource(String.format("/v1/document/%s/%s", TestUtils.TEST_TABLE_NAME, id)) .get(Document.class); compare(document, response); @@ -314,6 +315,7 @@ public void testGetDocuments() throws Exception { documents.add(document1); documents.add(document2); queryStore.save(TestUtils.TEST_TABLE_NAME, documents); + elasticsearchServer.refresh(ElasticsearchUtils.getIndices(TestUtils.TEST_TABLE_NAME)); String response = client().resource(String.format("/v1/document/%s", TestUtils.TEST_TABLE_NAME)) .queryParam("id", id1) .queryParam("id", id2) From 8fa3724d477922fe63d7f6687da506735be73ba8 Mon Sep 17 00:00:00 2001 From: Santanu Sinha Date: Fri, 27 Nov 2015 00:50:57 +0530 Subject: [PATCH 15/64] Changed metadata placement * Metadata on Elasticsearch is now indexed embedded __FOXTROT_METADATA__ field * Metadata is bypassed in get mapping API --- .travis.yml | 1 + .../parsers/ElasticsearchMappingParser.java | 4 ++ .../impl/ElasticsearchQueryStore.java | 72 +++++++++---------- .../querystore/impl/ElasticsearchUtils.java | 22 +++++- 4 files changed, 58 insertions(+), 41 deletions(-) diff --git a/.travis.yml b/.travis.yml index 10904f3de..d0f8f79ea 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,3 +1,4 @@ +sudo: false language: java jdk: - oraclejdk7 diff --git a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/parsers/ElasticsearchMappingParser.java b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/parsers/ElasticsearchMappingParser.java index 4c4b69376..542869d3f 100644 --- a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/parsers/ElasticsearchMappingParser.java +++ b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/parsers/ElasticsearchMappingParser.java @@ -19,6 +19,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.flipkart.foxtrot.common.FieldType; import com.flipkart.foxtrot.common.FieldTypeMapping; +import com.flipkart.foxtrot.core.querystore.impl.ElasticsearchUtils; import org.elasticsearch.cluster.metadata.MappingMetaData; import java.io.IOException; @@ -48,6 +49,9 @@ private Set generateFieldMappings(String parentField, JsonNode Iterator> iterator = jsonNode.fields(); while (iterator.hasNext()) { Map.Entry entry = iterator.next(); + if(entry.getKey().equals(ElasticsearchUtils.DOCUMENT_META_FIELD_NAME)) { + continue; + } String currentField = (parentField == null) ? entry.getKey() : (String.format("%s.%s", parentField, entry.getKey())); if (entry.getValue().has("properties")) { fieldTypeMappings.addAll(generateFieldMappings(currentField, entry.getValue().get("properties"))); diff --git a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/impl/ElasticsearchQueryStore.java b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/impl/ElasticsearchQueryStore.java index 7c3c11a1d..96ed216a1 100644 --- a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/impl/ElasticsearchQueryStore.java +++ b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/impl/ElasticsearchQueryStore.java @@ -16,8 +16,11 @@ package com.flipkart.foxtrot.core.querystore.impl; import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; import com.flipkart.foxtrot.common.*; +import com.flipkart.foxtrot.common.Table; import com.flipkart.foxtrot.core.datastore.DataStore; import com.flipkart.foxtrot.core.datastore.DataStoreException; import com.flipkart.foxtrot.core.parsers.ElasticsearchMappingParser; @@ -26,9 +29,7 @@ import com.flipkart.foxtrot.core.querystore.QueryStoreException; import com.flipkart.foxtrot.core.querystore.TableMetadataManager; import com.google.common.base.Stopwatch; -import com.google.common.collect.ImmutableSet; -import com.google.common.collect.Lists; -import com.google.common.collect.Sets; +import com.google.common.collect.*; import org.elasticsearch.action.WriteConsistencyLevel; import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsRequest; import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse; @@ -92,22 +93,17 @@ public void save(String table, Document document) throws QueryStoreException { final Table tableMeta = tableMetadataManager.get(table); final Document translatedDocuement = dataStore.save(tableMeta, document); long timestamp = translatedDocuement.getTimestamp(); + + //translatedDocuement.getData(). Stopwatch stopwatch = new Stopwatch(); stopwatch.start(); connection.getClient() - .prepareBulk() - .add(new IndexRequest() - .index(ElasticsearchUtils.getCurrentIndex(table, timestamp)) - .type(ElasticsearchUtils.DOCUMENT_TYPE_NAME) - .id(translatedDocuement.getId()) - .timestamp(Long.toString(timestamp)) - .source(mapper.writeValueAsBytes(translatedDocuement.getData()))) - .add(new IndexRequest() - .index(ElasticsearchUtils.getCurrentIndex(table, timestamp)) - .type(ElasticsearchUtils.DOCUMENT_META_TYPE_NAME) - .id(translatedDocuement.getId()) - .timestamp(Long.toString(timestamp)) - .source(mapper.writeValueAsBytes(translatedDocuement.getMetadata()))) + .prepareIndex() + .setIndex(ElasticsearchUtils.getCurrentIndex(table, timestamp)) + .setType(ElasticsearchUtils.DOCUMENT_TYPE_NAME) + .setId(translatedDocuement.getId()) + .setTimestamp(Long.toString(timestamp)) + .setSource(convert(translatedDocuement)) .setConsistencyLevel(WriteConsistencyLevel.QUORUM) .execute() .get(2, TimeUnit.SECONDS); @@ -162,14 +158,8 @@ public void save(String table, List documents) throws QueryStoreExcept .type(ElasticsearchUtils.DOCUMENT_TYPE_NAME) .id(document.getId()) .timestamp(Long.toString(timestamp)) - .source(mapper.writeValueAsBytes(document.getData())); + .source(convert(document)); bulkRequestBuilder.add(indexRequest); - IndexRequest metaIndexRequest = new IndexRequest() - .index(index) - .type(ElasticsearchUtils.DOCUMENT_META_TYPE_NAME) - .id(document.getId()) - .source(mapper.writeValueAsBytes(document.getMetadata())); - bulkRequestBuilder.add(metaIndexRequest); } if (bulkRequestBuilder.numberOfActions() > 0){ Stopwatch stopwatch = new Stopwatch(); @@ -230,11 +220,11 @@ public Document get(String table, String id) throws QueryStoreException { try { SearchResponse searchResponse = connection.getClient() .prepareSearch(ElasticsearchUtils.getIndices(table)) - .setTypes(ElasticsearchUtils.DOCUMENT_META_TYPE_NAME) + .setTypes(ElasticsearchUtils.DOCUMENT_TYPE_NAME) .setQuery( QueryBuilders.constantScoreQuery( FilterBuilders.boolFilter() - .must(FilterBuilders.termFilter("id", id)))) + .must(FilterBuilders.termFilter(ElasticsearchUtils.DOCUMENT_META_ID_FIELD_NAME, id)))) .setNoFields() .setSize(1) .execute() @@ -280,35 +270,31 @@ public List get(String table, List ids, boolean bypassMetalook .actionGet() .mappings(); - List rowKeys; + Map rowKeys = Maps.newLinkedHashMap(); + for(String id: ids) { + rowKeys.put(id, id); + } if(!bypassMetalookup) { - rowKeys = Lists.newArrayList(); SearchResponse response = connection.getClient().prepareSearch(ElasticsearchUtils.getIndices(table)) + .setTypes(ElasticsearchUtils.DOCUMENT_TYPE_NAME) .setQuery( QueryBuilders.constantScoreQuery( - FilterBuilders.inFilter("id", ids.toArray(new String[ids.size()])))) + FilterBuilders.inFilter(ElasticsearchUtils.DOCUMENT_META_ID_FIELD_NAME, ids.toArray(new String[ids.size()])))) .setFetchSource(false) - .addField("id") + .addField(ElasticsearchUtils.DOCUMENT_META_ID_FIELD_NAME) //Used for compatibility .setSize(ids.size()) .execute() .actionGet(); Set inputKeys = ImmutableSet.copyOf(ids); Set foundKeys = Sets.newHashSet(); for (SearchHit hit : response.getHits()) { - rowKeys.add(hit.getId()); - foundKeys.add(hit.getFields().get("id").getValue().toString()); + final String id = hit.getFields().get(ElasticsearchUtils.DOCUMENT_META_ID_FIELD_NAME).getValue().toString(); + rowKeys.put(id,hit.getId()); + foundKeys.add(id); } - - //Now we've got new keys - //Add them all to fetchList - //Add old generation keys and invalid keys as is - rowKeys.addAll(Sets.difference(inputKeys, foundKeys)); - } - else { - rowKeys = ids; } logger.info("Get row keys: {}", rowKeys.size()); - return dataStore.get(tableMetadataManager.get(table), rowKeys); + return dataStore.get(tableMetadataManager.get(table), ImmutableList.copyOf(rowKeys.values())); } catch (DataStoreException ex) { if (ex.getErrorCode().equals(DataStoreException.ErrorCode.STORE_NO_DATA_FOUND_FOR_IDS)) { throw new QueryStoreException(QueryStoreException.ErrorCode.DOCUMENT_NOT_FOUND, @@ -411,6 +397,12 @@ public void cleanup(Set tables) throws QueryStoreException { } } + private String convert(Document translatedDocuement) { + JsonNode metaNode = mapper.valueToTree(translatedDocuement.getMetadata()); + ObjectNode dataNode = translatedDocuement.getData().deepCopy(); + dataNode.put(ElasticsearchUtils.DOCUMENT_META_FIELD_NAME, metaNode); + return dataNode.toString(); + } } diff --git a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/impl/ElasticsearchUtils.java b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/impl/ElasticsearchUtils.java index 0e198cf1a..540cff3ad 100644 --- a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/impl/ElasticsearchUtils.java +++ b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/impl/ElasticsearchUtils.java @@ -47,6 +47,9 @@ public class ElasticsearchUtils { public static final String DOCUMENT_TYPE_NAME = "document"; public static final String DOCUMENT_META_TYPE_NAME = "metadata"; + public static final String DOCUMENT_META_FIELD_NAME = "__FOXTROT_METADATA__"; + public static final String DOCUMENT_META_ID_FIELD_NAME = String.format("%s.id", DOCUMENT_META_FIELD_NAME); + public static final String TABLENAME_PREFIX = "foxtrot"; public static final String TABLENAME_POSTFIX = "table"; private static final DateTimeFormatter FORMATTER = DateTimeFormat.forPattern("dd-M-yyyy"); @@ -109,8 +112,9 @@ public static PutIndexTemplateRequest getClusterTemplateMapping(IndicesAdminClie try { PutIndexTemplateRequestBuilder builder = new PutIndexTemplateRequestBuilder(indicesAdminClient, "generic_template"); builder.setTemplate("foxtrot-*"); + System.out.println(getDocumentMapping().string()); builder.addMapping(DOCUMENT_TYPE_NAME, getDocumentMapping()); - builder.addMapping(DOCUMENT_META_TYPE_NAME, getDocumentMetadataMapping()); + //builder.addMapping(DOCUMENT_META_TYPE_NAME, getDocumentMetadataMapping()); return builder.request(); } catch (IOException ex){ logger.error("TEMPLATE_CREATION_FAILED", ex); @@ -175,6 +179,22 @@ public static XContentBuilder getDocumentMapping() throws IOException { .endObject() .field("dynamic_templates") .startArray() + .startObject() + .field("template_metadata_fields") + .startObject() + .field("path_match", ElasticsearchUtils.DOCUMENT_META_FIELD_NAME + ".*") + .field("mapping") + .startObject() + .field("store", true) + .field("doc_values", true) + .field("index", "not_analyzed") + .field("fielddata") + .startObject() + .field("format", "doc_values") + .endObject() + .endObject() + .endObject() + .endObject() .startObject() .field("template_timestamp") .startObject() From 0ea4ece2fa608ded19298b5bb8ed12cfc79950d8 Mon Sep 17 00:00:00 2001 From: Santanu Sinha Date: Fri, 27 Nov 2015 01:17:41 +0530 Subject: [PATCH 16/64] Removed useless metadata mapping --- .../querystore/impl/ElasticsearchUtils.java | 38 ------------------- 1 file changed, 38 deletions(-) diff --git a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/impl/ElasticsearchUtils.java b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/impl/ElasticsearchUtils.java index 540cff3ad..562bf5483 100644 --- a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/impl/ElasticsearchUtils.java +++ b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/impl/ElasticsearchUtils.java @@ -114,7 +114,6 @@ public static PutIndexTemplateRequest getClusterTemplateMapping(IndicesAdminClie builder.setTemplate("foxtrot-*"); System.out.println(getDocumentMapping().string()); builder.addMapping(DOCUMENT_TYPE_NAME, getDocumentMapping()); - //builder.addMapping(DOCUMENT_META_TYPE_NAME, getDocumentMetadataMapping()); return builder.request(); } catch (IOException ex){ logger.error("TEMPLATE_CREATION_FAILED", ex); @@ -122,43 +121,6 @@ public static PutIndexTemplateRequest getClusterTemplateMapping(IndicesAdminClie } } - public static XContentBuilder getDocumentMetadataMapping() throws IOException { - return XContentFactory.jsonBuilder() - .startObject() - .field(DOCUMENT_META_TYPE_NAME) - .startObject() - .field("_source") - .startObject() - .field("enabled", false) - .endObject() - .field("_all") - .startObject() - .field("enabled", false) - .endObject() - .field("dynamic_templates") - .startArray() - .startObject() - .field("template_no_store") - .startObject() - .field("match_mapping_type", "string|date|boolean|double|long|integer") - .field("match_pattern", "regex") - .field("path_match", ".*") - .field("mapping") - .startObject() - .field("store", true) - .field("doc_values", true) - .field("index", "not_analyzed") - .field("fielddata") - .startObject() - .field("format", "doc_values") - .endObject() - .endObject() - .endObject() - .endObject() - .endArray() - .endObject() - .endObject(); - } public static XContentBuilder getDocumentMapping() throws IOException { return XContentFactory.jsonBuilder() .startObject() From 3fd16975a4a928b9784f880b0ce87d2997d67979 Mon Sep 17 00:00:00 2001 From: Santanu Sinha Date: Wed, 2 Dec 2015 10:50:27 +0530 Subject: [PATCH 17/64] Rowkeys distributed * Added row key distributor * Changed testcases to return config from hbase --- .../core/datastore/impl/hbase/HbaseKeyTranslator.java | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 foxtrot-core/src/main/java/com/flipkart/foxtrot/core/datastore/impl/hbase/HbaseKeyTranslator.java diff --git a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/datastore/impl/hbase/HbaseKeyTranslator.java b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/datastore/impl/hbase/HbaseKeyTranslator.java new file mode 100644 index 000000000..fb89c3275 --- /dev/null +++ b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/datastore/impl/hbase/HbaseKeyTranslator.java @@ -0,0 +1,7 @@ +package com.flipkart.foxtrot.core.datastore.impl.hbase; + +/** + * Created by santanu.s on 02/12/15. + */ +public class HbaseKeyTranslator { +} From 1daf7758f6c09734fe39eeee42ff127bf9963e08 Mon Sep 17 00:00:00 2001 From: Santanu Sinha Date: Wed, 2 Dec 2015 11:23:44 +0530 Subject: [PATCH 18/64] Actual files added for bucketing --- .../core/datastore/impl/hbase/HBaseDataStore.java | 11 ++++++++--- .../core/datastore/impl/hbase/HbaseConfig.java | 13 +++++++++++++ .../datastore/impl/hbase/HbaseKeyTranslator.java | 12 ++++++++++++ .../datastore/impl/hbase/HbaseTableConnection.java | 4 ++++ .../java/com/flipkart/foxtrot/core/TestUtils.java | 2 ++ .../datastore/impl/hbase/HBaseDataStoreTest.java | 7 ++++++- pom.xml | 10 ++++++++++ 7 files changed, 55 insertions(+), 4 deletions(-) diff --git a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/datastore/impl/hbase/HBaseDataStore.java b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/datastore/impl/hbase/HBaseDataStore.java index 48c44d2f6..797fdf894 100644 --- a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/datastore/impl/hbase/HBaseDataStore.java +++ b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/datastore/impl/hbase/HBaseDataStore.java @@ -25,6 +25,7 @@ import com.flipkart.foxtrot.core.querystore.DocumentTranslator; import com.google.common.base.Stopwatch; import com.google.common.collect.ImmutableList; +import com.shash.hbase.ds.RowKeyDistributorByHashPrefix; import org.apache.hadoop.hbase.client.Get; import org.apache.hadoop.hbase.client.HTableInterface; import org.apache.hadoop.hbase.client.Put; @@ -54,10 +55,13 @@ public class HBaseDataStore implements DataStore { private final HbaseTableConnection tableWrapper; private final ObjectMapper mapper; private final DocumentTranslator translator = new DocumentTranslator(); + private final HbaseKeyTranslator keyTranslator; public HBaseDataStore(HbaseTableConnection tableWrapper, ObjectMapper mapper) { this.tableWrapper = tableWrapper; this.mapper = mapper; + keyTranslator = new HbaseKeyTranslator(new RowKeyDistributorByHashPrefix( + new RowKeyDistributorByHashPrefix.OneByteSimpleHash(tableWrapper.getHbaseConfig().getNumBuckets()))); } @Override @@ -146,7 +150,7 @@ public List save(final Table table, List documents) throws D public Document get(final Table table, String id) throws DataStoreException { HTableInterface hTable = null; try { - Get get = new Get(Bytes.toBytes(id)) + Get get = new Get(keyTranslator.idToRowKey(id)) .addColumn(COLUMN_FAMILY, DOCUMENT_FIELD_NAME) .addColumn(COLUMN_FAMILY, DOCUMENT_META_FIELD_NAME) .addColumn(COLUMN_FAMILY, TIMESTAMP_FIELD_NAME) @@ -193,7 +197,7 @@ public List get(final Table table, List ids) throws DataStoreE try { List gets = new ArrayList(ids.size()); for (String id : ids) { - Get get = new Get(Bytes.toBytes(id)) + Get get = new Get(keyTranslator.idToRowKey(id)) .addColumn(COLUMN_FAMILY, DOCUMENT_FIELD_NAME) .addColumn(COLUMN_FAMILY, DOCUMENT_META_FIELD_NAME) .addColumn(COLUMN_FAMILY, TIMESTAMP_FIELD_NAME) @@ -246,9 +250,10 @@ public List get(final Table table, List ids) throws DataStoreE } public Put getPutForDocument(final Table table, Document document) throws JsonProcessingException { - return new Put(Bytes.toBytes(document.getId())) + return new Put(keyTranslator.idToRowKey(document.getId())) .add(COLUMN_FAMILY, DOCUMENT_META_FIELD_NAME, mapper.writeValueAsBytes(document.getMetadata())) .add(COLUMN_FAMILY, DOCUMENT_FIELD_NAME, mapper.writeValueAsBytes(document.getData())) .add(COLUMN_FAMILY, TIMESTAMP_FIELD_NAME, Bytes.toBytes(document.getTimestamp())); } + } diff --git a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/datastore/impl/hbase/HbaseConfig.java b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/datastore/impl/hbase/HbaseConfig.java index 61c4c8387..41a73c20f 100644 --- a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/datastore/impl/hbase/HbaseConfig.java +++ b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/datastore/impl/hbase/HbaseConfig.java @@ -17,6 +17,8 @@ import org.hibernate.validator.constraints.NotEmpty; +import javax.validation.constraints.Max; +import javax.validation.constraints.Min; import javax.validation.constraints.NotNull; /** @@ -38,10 +40,13 @@ public class HbaseConfig { private String seggregatedTablePrefix; private String hbaseZookeeperQuorum; private Integer hbaseZookeeperClientPort; + private short numBuckets = 32; @NotNull @NotEmpty + @Min(1) + @Max(Byte.MAX_VALUE) private String tableName; public HbaseConfig() { @@ -150,4 +155,12 @@ public String getHbaseZookeeperQuorum() { public void setHbaseZookeeperQuorum(String hbaseZookeeperQuorum) { this.hbaseZookeeperQuorum = hbaseZookeeperQuorum; } + + public int getNumBuckets() { + return numBuckets; + } + + public void setNumBuckets(short numBuckets) { + this.numBuckets = numBuckets; + } } diff --git a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/datastore/impl/hbase/HbaseKeyTranslator.java b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/datastore/impl/hbase/HbaseKeyTranslator.java index fb89c3275..b8bb04793 100644 --- a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/datastore/impl/hbase/HbaseKeyTranslator.java +++ b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/datastore/impl/hbase/HbaseKeyTranslator.java @@ -1,7 +1,19 @@ package com.flipkart.foxtrot.core.datastore.impl.hbase; +import com.shash.hbase.ds.AbstractRowKeyDistributor; +import org.apache.hadoop.hbase.util.Bytes; + /** * Created by santanu.s on 02/12/15. */ public class HbaseKeyTranslator { + private final AbstractRowKeyDistributor keyDistributor; + + public HbaseKeyTranslator(AbstractRowKeyDistributor keyDistributor) { + this.keyDistributor = keyDistributor; + } + + public byte[] idToRowKey(String id) { + return keyDistributor.getDistributedKey(Bytes.toBytes(id)); + } } diff --git a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/datastore/impl/hbase/HbaseTableConnection.java b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/datastore/impl/hbase/HbaseTableConnection.java index 54658fd4a..3c3337abb 100644 --- a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/datastore/impl/hbase/HbaseTableConnection.java +++ b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/datastore/impl/hbase/HbaseTableConnection.java @@ -69,4 +69,8 @@ public void start() throws Exception { public void stop() throws Exception { tablePool.close(); } + + public HbaseConfig getHbaseConfig() { + return hbaseConfig; + } } diff --git a/foxtrot-core/src/test/java/com/flipkart/foxtrot/core/TestUtils.java b/foxtrot-core/src/test/java/com/flipkart/foxtrot/core/TestUtils.java index cc6b3904c..82585db0a 100644 --- a/foxtrot-core/src/test/java/com/flipkart/foxtrot/core/TestUtils.java +++ b/foxtrot-core/src/test/java/com/flipkart/foxtrot/core/TestUtils.java @@ -24,6 +24,7 @@ import java.util.Set; import java.util.Vector; +import com.flipkart.foxtrot.core.datastore.impl.hbase.HbaseConfig; import org.apache.hadoop.hbase.client.HTableInterface; import org.mockito.Matchers; import org.mockito.Mockito; @@ -58,6 +59,7 @@ public static DataStore getDataStore() throws DataStoreException { HTableInterface tableInterface = MockHTable.create(); HbaseTableConnection tableConnection = Mockito.mock(HbaseTableConnection.class); when(tableConnection.getTable(Matchers.any())).thenReturn(tableInterface); + when(tableConnection.getHbaseConfig()).thenReturn(new HbaseConfig()); return new HBaseDataStore(tableConnection, new ObjectMapper()); } diff --git a/foxtrot-core/src/test/java/com/flipkart/foxtrot/core/datastore/impl/hbase/HBaseDataStoreTest.java b/foxtrot-core/src/test/java/com/flipkart/foxtrot/core/datastore/impl/hbase/HBaseDataStoreTest.java index 137df1438..75a7b44af 100644 --- a/foxtrot-core/src/test/java/com/flipkart/foxtrot/core/datastore/impl/hbase/HBaseDataStoreTest.java +++ b/foxtrot-core/src/test/java/com/flipkart/foxtrot/core/datastore/impl/hbase/HBaseDataStoreTest.java @@ -23,6 +23,7 @@ import com.flipkart.foxtrot.core.MockHTable; import com.flipkart.foxtrot.core.datastore.DataStoreException; import com.flipkart.foxtrot.core.querystore.DocumentTranslator; +import com.shash.hbase.ds.RowKeyDistributorByHashPrefix; import org.apache.hadoop.hbase.client.Get; import org.apache.hadoop.hbase.client.HTableInterface; import org.apache.hadoop.hbase.client.Put; @@ -53,6 +54,9 @@ public class HBaseDataStoreTest { private static final byte[] DATA_FIELD_NAME = Bytes.toBytes("data"); private static final String TEST_APP_NAME = "test-app"; private static final Table TEST_APP = new Table(TEST_APP_NAME, 7); + private final HbaseKeyTranslator keyTranslator = new HbaseKeyTranslator(new RowKeyDistributorByHashPrefix( + new RowKeyDistributorByHashPrefix.OneByteSimpleHash(32))); + @Before public void setUp() throws Exception { @@ -60,6 +64,7 @@ public void setUp() throws Exception { tableInterface = spy(tableInterface); hBaseTableConnection = Mockito.mock(HbaseTableConnection.class); when(hBaseTableConnection.getTable(Matchers.
any())).thenReturn(tableInterface); + when(hBaseTableConnection.getHbaseConfig()).thenReturn(new HbaseConfig()); HBaseDataStore = new HBaseDataStore(hBaseTableConnection, mapper); } @@ -270,7 +275,7 @@ public void testSaveBulkNullHBaseTableConnection() throws Exception { public void validateSave(Document savedDocument) throws Exception { String rowkey = new DocumentTranslator().rowKey(TEST_APP, savedDocument); - Get get = new Get(Bytes.toBytes(rowkey)); + Get get = new Get(keyTranslator.idToRowKey(rowkey)); Result result = tableInterface.get(get); assertNotNull("Get for Id should not be null", result); Document actualDocument = new Document(savedDocument.getId(), diff --git a/pom.xml b/pom.xml index edb4040d7..7e4c94d76 100644 --- a/pom.xml +++ b/pom.xml @@ -63,6 +63,11 @@ clouderahttps://repository.cloudera.com/artifactory/cloudera-repos/ + + clojars + Clojars repository + https://clojars.org/repo + @@ -135,6 +140,11 @@ xalan 2.7.2 + + com.shash.hbase.ds + hbase-ds + 0.0.1 + From 90439589c71b9a399b2fedc36abc6f4bf3b835b7 Mon Sep 17 00:00:00 2001 From: Santanu Sinha Date: Wed, 2 Dec 2015 11:28:49 +0530 Subject: [PATCH 19/64] Moved validations to proper place --- .../foxtrot/core/datastore/impl/hbase/HbaseConfig.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/datastore/impl/hbase/HbaseConfig.java b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/datastore/impl/hbase/HbaseConfig.java index 41a73c20f..724b448db 100644 --- a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/datastore/impl/hbase/HbaseConfig.java +++ b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/datastore/impl/hbase/HbaseConfig.java @@ -40,13 +40,14 @@ public class HbaseConfig { private String seggregatedTablePrefix; private String hbaseZookeeperQuorum; private Integer hbaseZookeeperClientPort; + + @Min(1) + @Max(Byte.MAX_VALUE) private short numBuckets = 32; @NotNull @NotEmpty - @Min(1) - @Max(Byte.MAX_VALUE) private String tableName; public HbaseConfig() { From 9d4da544868a0dba5caee959999de328a1d716d8 Mon Sep 17 00:00:00 2001 From: Santanu Sinha Date: Thu, 3 Dec 2015 11:56:27 +0530 Subject: [PATCH 20/64] AOP based timers * Uses aspects to track and profile methods annotated with yammer @Timed * Minor code cleanup (immutable set, debugging of log) --- foxtrot-core/pom.xml | 25 ++++++++++++++ .../foxtrot/core/aspect/MetricsAspect.java | 33 +++++++++++++++++++ .../foxtrot/core/datastore/DataStore.java | 4 +-- .../datastore/impl/hbase/HBaseDataStore.java | 15 ++++----- .../core/querystore/DocumentTranslator.java | 2 +- .../foxtrot/core/querystore/QueryStore.java | 4 +-- .../core/querystore/actions/FilterAction.java | 2 +- .../impl/DistributedTableMetadataManager.java | 2 +- .../impl/ElasticsearchQueryStore.java | 30 ++++++++--------- .../impl/hbase/HBaseDataStoreTest.java | 28 ++++++++-------- .../impl/ElasticsearchQueryStoreTest.java | 4 +-- .../server/resources/DocumentResource.java | 10 ++++-- .../resources/DocumentResourceTest.java | 2 +- 13 files changed, 111 insertions(+), 50 deletions(-) create mode 100644 foxtrot-core/src/main/java/com/flipkart/foxtrot/core/aspect/MetricsAspect.java diff --git a/foxtrot-core/pom.xml b/foxtrot-core/pom.xml index 4702dbe39..361d10106 100644 --- a/foxtrot-core/pom.xml +++ b/foxtrot-core/pom.xml @@ -95,6 +95,13 @@ + + + org.aspectj + aspectjrt + 1.8.7 + + @@ -110,6 +117,24 @@ + + org.codehaus.mojo + aspectj-maven-plugin + 1.8 + + 1.6 + 1.6 + 1.6 + + + + + compile + test-compile + + + + diff --git a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/aspect/MetricsAspect.java b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/aspect/MetricsAspect.java new file mode 100644 index 000000000..d5efbfab2 --- /dev/null +++ b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/aspect/MetricsAspect.java @@ -0,0 +1,33 @@ +package com.flipkart.foxtrot.core.aspect; + +import com.yammer.metrics.Metrics; +import com.yammer.metrics.core.MetricsRegistry; +import com.yammer.metrics.core.Timer; +import com.yammer.metrics.core.TimerContext; +import org.aspectj.lang.ProceedingJoinPoint; +import org.aspectj.lang.annotation.Around; +import org.aspectj.lang.annotation.Aspect; +import org.aspectj.lang.annotation.Pointcut; + +/** + * Created by santanu.s on 03/12/15. + */ +@Aspect +public class MetricsAspect { + + @Pointcut(value = "execution(@com.yammer.metrics.annotation.Timed * *(..))") + public void timeChecked() {} + + @Around(value = "timeChecked()") + public Object around(ProceedingJoinPoint joinPoint) throws Throwable { + MetricsRegistry registry = Metrics.defaultRegistry(); + + Timer timer = registry.newTimer(joinPoint.getThis().getClass(), joinPoint.getSignature().getName()); + TimerContext time = timer.time(); + try { + return joinPoint.proceed(); + } finally { + time.stop(); + } + } +} diff --git a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/datastore/DataStore.java b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/datastore/DataStore.java index a060a75be..0aabd5f8e 100644 --- a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/datastore/DataStore.java +++ b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/datastore/DataStore.java @@ -27,7 +27,7 @@ */ public interface DataStore { public Document save(final Table table, final Document document) throws DataStoreException; - public List save(final Table table, final List documents) throws DataStoreException; + public List saveAll(final Table table, final List documents) throws DataStoreException; public Document get(final Table table, final String id) throws DataStoreException; - public List get(final Table table, final List ids) throws DataStoreException; + public List getAll(final Table table, final List ids) throws DataStoreException; } diff --git a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/datastore/impl/hbase/HBaseDataStore.java b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/datastore/impl/hbase/HBaseDataStore.java index 797fdf894..5cc440f6c 100644 --- a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/datastore/impl/hbase/HBaseDataStore.java +++ b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/datastore/impl/hbase/HBaseDataStore.java @@ -26,6 +26,7 @@ import com.google.common.base.Stopwatch; import com.google.common.collect.ImmutableList; import com.shash.hbase.ds.RowKeyDistributorByHashPrefix; +import com.yammer.metrics.annotation.Timed; import org.apache.hadoop.hbase.client.Get; import org.apache.hadoop.hbase.client.HTableInterface; import org.apache.hadoop.hbase.client.Put; @@ -65,6 +66,7 @@ public HBaseDataStore(HbaseTableConnection tableWrapper, ObjectMapper mapper) { } @Override + @Timed public Document save(final Table table, Document document) throws DataStoreException { if (document == null || document.getData() == null || document.getId() == null) { throw new DataStoreException(DataStoreException.ErrorCode.STORE_INVALID_REQUEST, "Invalid Document"); @@ -74,10 +76,7 @@ public Document save(final Table table, Document document) throws DataStoreExcep try { translatedDocument = translator.translate(table, document); hTable = tableWrapper.getTable(table); - Stopwatch stopwatch = new Stopwatch(); - stopwatch.start(); hTable.put(getPutForDocument(table, translatedDocument)); - logger.error(String.format("HBASE put took : %d table : %s", stopwatch.elapsedMillis(), table)); } catch (JsonProcessingException e) { throw new DataStoreException(DataStoreException.ErrorCode.STORE_INVALID_REQUEST, e.getMessage(), e); @@ -100,7 +99,8 @@ public Document save(final Table table, Document document) throws DataStoreExcep } @Override - public List save(final Table table, List documents) throws DataStoreException { + @Timed + public List saveAll(final Table table, List documents) throws DataStoreException { if (documents == null || documents.isEmpty()) { throw new DataStoreException(DataStoreException.ErrorCode.STORE_INVALID_REQUEST, "Invalid Documents List"); } @@ -124,10 +124,7 @@ public List save(final Table table, List documents) throws D HTableInterface hTable = null; try { hTable = tableWrapper.getTable(table); - Stopwatch stopwatch = new Stopwatch(); - stopwatch.start(); hTable.put(puts); - logger.error(String.format("HBASE put took : %d table : %s", stopwatch.elapsedMillis(), table)); } catch (IOException e) { throw new DataStoreException(DataStoreException.ErrorCode.STORE_MULTI_SAVE, e.getMessage(), e); @@ -147,6 +144,7 @@ public List save(final Table table, List documents) throws D } @Override + @Timed public Document get(final Table table, String id) throws DataStoreException { HTableInterface hTable = null; try { @@ -188,7 +186,8 @@ public Document get(final Table table, String id) throws DataStoreException { } @Override - public List get(final Table table, List ids) throws DataStoreException { + @Timed + public List getAll(final Table table, List ids) throws DataStoreException { if (ids == null) { throw new DataStoreException(DataStoreException.ErrorCode.STORE_INVALID_REQUEST, "Invalid Request IDs"); } diff --git a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/DocumentTranslator.java b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/DocumentTranslator.java index d6e360a0e..c46ad9b78 100644 --- a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/DocumentTranslator.java +++ b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/DocumentTranslator.java @@ -36,7 +36,7 @@ public Document translate(final Table table, final Document inDocument) { document.setMetadata(metadata); document.setData(inDocument.getData()); - logger.info("Translated doc row key: {}, {}", rowKey, document); + logger.debug("Translated doc row key: {}, {}", rowKey, document); return document; } diff --git a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/QueryStore.java b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/QueryStore.java index a390ba5a0..e168758af 100644 --- a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/QueryStore.java +++ b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/QueryStore.java @@ -34,9 +34,9 @@ public interface QueryStore { public Document get(final String table, final String id) throws QueryStoreException; - public List get(final String table, final List ids) throws QueryStoreException; + public List getAll(final String table, final List ids) throws QueryStoreException; - public List get(final String table, final List ids, boolean bypassMetaLookup) throws QueryStoreException; + public List getAll(final String table, final List ids, boolean bypassMetaLookup) throws QueryStoreException; public TableFieldMapping getFieldMappings(final String table) throws QueryStoreException; diff --git a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/FilterAction.java b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/FilterAction.java index 28b8f40c7..c90c3d39c 100644 --- a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/FilterAction.java +++ b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/FilterAction.java @@ -113,7 +113,7 @@ public QueryResponse execute(Query parameter) throws QueryStoreException { if (ids.isEmpty()) { return new QueryResponse(Collections.emptyList()); } - return new QueryResponse(getQueryStore().get(parameter.getTable(), ids, true)); + return new QueryResponse(getQueryStore().getAll(parameter.getTable(), ids, true)); } catch (Exception e) { if (null != search) { logger.error("Error running generated query: " + search, e); diff --git a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/impl/DistributedTableMetadataManager.java b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/impl/DistributedTableMetadataManager.java index 4e295e1d7..a156b50ce 100644 --- a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/impl/DistributedTableMetadataManager.java +++ b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/impl/DistributedTableMetadataManager.java @@ -64,7 +64,7 @@ public void save(Table table) throws Exception { @Override public Table get(String tableName) throws Exception { - logger.info(String.format("Getting Table : %s", tableName)); + logger.debug(String.format("Getting Table : %s", tableName)); if (tableDataStore.containsKey(tableName)) { return tableDataStore.get(tableName); } diff --git a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/impl/ElasticsearchQueryStore.java b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/impl/ElasticsearchQueryStore.java index 96ed216a1..b7ac704cf 100644 --- a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/impl/ElasticsearchQueryStore.java +++ b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/impl/ElasticsearchQueryStore.java @@ -24,23 +24,20 @@ import com.flipkart.foxtrot.core.datastore.DataStore; import com.flipkart.foxtrot.core.datastore.DataStoreException; import com.flipkart.foxtrot.core.parsers.ElasticsearchMappingParser; -import com.flipkart.foxtrot.core.querystore.DocumentTranslator; import com.flipkart.foxtrot.core.querystore.QueryStore; import com.flipkart.foxtrot.core.querystore.QueryStoreException; import com.flipkart.foxtrot.core.querystore.TableMetadataManager; import com.google.common.base.Stopwatch; import com.google.common.collect.*; +import com.yammer.metrics.annotation.Timed; import org.elasticsearch.action.WriteConsistencyLevel; import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsRequest; import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse; import org.elasticsearch.action.admin.indices.status.IndicesStatusResponse; -import org.elasticsearch.action.admin.indices.template.get.GetIndexTemplatesRequest; import org.elasticsearch.action.bulk.BulkItemResponse; import org.elasticsearch.action.bulk.BulkRequestBuilder; import org.elasticsearch.action.bulk.BulkResponse; -import org.elasticsearch.action.get.MultiGetRequest; import org.elasticsearch.action.index.IndexRequest; -import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.cluster.metadata.MappingMetaData; import org.elasticsearch.common.collect.ImmutableOpenMap; @@ -80,6 +77,7 @@ public ElasticsearchQueryStore(TableMetadataManager tableMetadataManager, } @Override + @Timed(name = "single-document-save") public void save(String table, Document document) throws QueryStoreException { table = ElasticsearchUtils.getValidTableName(table); try { @@ -95,8 +93,6 @@ public void save(String table, Document document) throws QueryStoreException { long timestamp = translatedDocuement.getTimestamp(); //translatedDocuement.getData(). - Stopwatch stopwatch = new Stopwatch(); - stopwatch.start(); connection.getClient() .prepareIndex() .setIndex(ElasticsearchUtils.getCurrentIndex(table, timestamp)) @@ -107,7 +103,6 @@ public void save(String table, Document document) throws QueryStoreException { .setConsistencyLevel(WriteConsistencyLevel.QUORUM) .execute() .get(2, TimeUnit.SECONDS); - logger.info(String.format("ES took : %d table : %s", stopwatch.elapsedMillis(), table)); } catch (QueryStoreException ex) { throw ex; } catch (DataStoreException ex) { @@ -130,6 +125,7 @@ public void save(String table, Document document) throws QueryStoreException { } @Override + @Timed(name = "multiple-document-saveAll") public void save(String table, List documents) throws QueryStoreException { table = ElasticsearchUtils.getValidTableName(table); try { @@ -142,7 +138,7 @@ public void save(String table, List documents) throws QueryStoreExcept "Invalid Document List"); } final Table tableMeta = tableMetadataManager.get(table); - final List translatedDocuments = dataStore.save(tableMeta, documents); + final List translatedDocuments = dataStore.saveAll(tableMeta, documents); BulkRequestBuilder bulkRequestBuilder = connection.getClient().prepareBulk(); DateTime dateTime = new DateTime().plusDays(1); @@ -162,13 +158,10 @@ public void save(String table, List documents) throws QueryStoreExcept bulkRequestBuilder.add(indexRequest); } if (bulkRequestBuilder.numberOfActions() > 0){ - Stopwatch stopwatch = new Stopwatch(); - stopwatch.start(); BulkResponse responses = bulkRequestBuilder .setConsistencyLevel(WriteConsistencyLevel.QUORUM) .execute() .get(10, TimeUnit.SECONDS); - logger.info(String.format("ES took : %d table : %s", stopwatch.elapsedMillis(), table)); int failedCount = 0; for (int i = 0; i < responses.getItems().length; i++) { BulkItemResponse itemResponse = responses.getItems()[i]; @@ -203,6 +196,7 @@ public void save(String table, List documents) throws QueryStoreExcept } @Override + @Timed(name = "single-document-getAll") public Document get(String table, String id) throws QueryStoreException { table = ElasticsearchUtils.getValidTableName(table); Table fxTable = null; @@ -250,12 +244,13 @@ public Document get(String table, String id) throws QueryStoreException { } @Override - public List get(String table, List ids) throws QueryStoreException { - return get(table, ids, false); + public List getAll(String table, List ids) throws QueryStoreException { + return getAll(table, ids, false); } @Override - public List get(String table, List ids, boolean bypassMetalookup) throws QueryStoreException { + @Timed + public List getAll(String table, List ids, boolean bypassMetalookup) throws QueryStoreException { table = ElasticsearchUtils.getValidTableName(table); try { if (!tableMetadataManager.exists(table)) { @@ -294,7 +289,7 @@ public List get(String table, List ids, boolean bypassMetalook } } logger.info("Get row keys: {}", rowKeys.size()); - return dataStore.get(tableMetadataManager.get(table), ImmutableList.copyOf(rowKeys.values())); + return dataStore.getAll(tableMetadataManager.get(table), ImmutableList.copyOf(rowKeys.values())); } catch (DataStoreException ex) { if (ex.getErrorCode().equals(DataStoreException.ErrorCode.STORE_NO_DATA_FOUND_FOR_IDS)) { throw new QueryStoreException(QueryStoreException.ErrorCode.DOCUMENT_NOT_FOUND, @@ -309,6 +304,7 @@ public List get(String table, List ids, boolean bypassMetalook } @Override + @Timed public TableFieldMapping getFieldMappings(String table) throws QueryStoreException { table = ElasticsearchUtils.getValidTableName(table); try { @@ -351,11 +347,13 @@ public void cleanupAll() throws QueryStoreException { } @Override + @Timed public void cleanup(final String table) throws QueryStoreException { - cleanup(new HashSet(Arrays.asList(table))); + cleanup(ImmutableSet.of(table)); } @Override + @Timed public void cleanup(Set tables) throws QueryStoreException { List indicesToDelete = new ArrayList(); try { diff --git a/foxtrot-core/src/test/java/com/flipkart/foxtrot/core/datastore/impl/hbase/HBaseDataStoreTest.java b/foxtrot-core/src/test/java/com/flipkart/foxtrot/core/datastore/impl/hbase/HBaseDataStoreTest.java index 75a7b44af..b56c7bd3b 100644 --- a/foxtrot-core/src/test/java/com/flipkart/foxtrot/core/datastore/impl/hbase/HBaseDataStoreTest.java +++ b/foxtrot-core/src/test/java/com/flipkart/foxtrot/core/datastore/impl/hbase/HBaseDataStoreTest.java @@ -167,7 +167,7 @@ public void testSaveBulk() throws Exception { System.currentTimeMillis(), mapper.valueToTree(Collections.singletonMap("TEST_NAME", "SINGLE_SAVE_TEST")))); } - HBaseDataStore.save(TEST_APP, documents); + HBaseDataStore.saveAll(TEST_APP, documents); for (Document document : documents) { validateSave(document); } @@ -180,7 +180,7 @@ public void testSaveBulkNullDocuments() throws Exception { documents.add(null); } try { - HBaseDataStore.save(TEST_APP, documents); + HBaseDataStore.saveAll(TEST_APP, documents); fail(); } catch (DataStoreException ex) { assertEquals(DataStoreException.ErrorCode.STORE_INVALID_REQUEST, ex.getErrorCode()); @@ -191,7 +191,7 @@ public void testSaveBulkNullDocuments() throws Exception { public void testSaveBulkNullIdList() throws Exception { List documents = null; try { - HBaseDataStore.save(TEST_APP, documents); + HBaseDataStore.saveAll(TEST_APP, documents); fail(); } catch (DataStoreException ex) { assertEquals(DataStoreException.ErrorCode.STORE_INVALID_REQUEST, ex.getErrorCode()); @@ -205,7 +205,7 @@ public void testSaveBulkNullId() throws Exception { documents.add(new Document(null, System.currentTimeMillis(), mapper.valueToTree(Collections.singletonMap("TEST_NAME", "SINGLE_SAVE_TEST")))); } try { - HBaseDataStore.save(TEST_APP, documents); + HBaseDataStore.saveAll(TEST_APP, documents); fail(); } catch (DataStoreException ex) { assertEquals(DataStoreException.ErrorCode.STORE_INVALID_REQUEST, ex.getErrorCode()); @@ -219,7 +219,7 @@ public void testSaveBulkNullData() throws Exception { documents.add(new Document(UUID.randomUUID().toString(), System.currentTimeMillis(), null)); } try { - HBaseDataStore.save(TEST_APP, documents); + HBaseDataStore.saveAll(TEST_APP, documents); fail(); } catch (DataStoreException ex) { assertEquals(DataStoreException.ErrorCode.STORE_INVALID_REQUEST, ex.getErrorCode()); @@ -236,7 +236,7 @@ public void testSaveBulkHBaseWriteException() throws Exception { .when(tableInterface) .put(Matchers.anyListOf(Put.class)); try { - HBaseDataStore.save(TEST_APP, documents); + HBaseDataStore.saveAll(TEST_APP, documents); fail(); } catch (DataStoreException ex) { assertEquals(DataStoreException.ErrorCode.STORE_MULTI_SAVE, ex.getErrorCode()); @@ -252,7 +252,7 @@ public void testSaveBulkHBaseCloseException() throws Exception { doThrow(new IOException()) .when(tableInterface) .close(); - HBaseDataStore.save(TEST_APP, documents); + HBaseDataStore.saveAll(TEST_APP, documents); verify(tableInterface, times(1)).close(); } @@ -266,7 +266,7 @@ public void testSaveBulkNullHBaseTableConnection() throws Exception { } when(hBaseTableConnection.getTable(Matchers.
any())).thenReturn(null); try { - HBaseDataStore.save(TEST_APP, documents); + HBaseDataStore.saveAll(TEST_APP, documents); fail(); } catch (DataStoreException ex) { assertEquals(DataStoreException.ErrorCode.STORE_MULTI_SAVE, ex.getErrorCode()); @@ -366,7 +366,7 @@ public void testGetBulk() throws Exception { putList.add(HBaseDataStore.getPutForDocument(TEST_APP, idValues.get(id))); } tableInterface.put(putList); - List actualDocuments = HBaseDataStore.get(TEST_APP, ids); + List actualDocuments = HBaseDataStore.getAll(TEST_APP, ids); HashMap actualIdValues = new HashMap(); for (Document doc : actualDocuments) { actualIdValues.put(doc.getId(), doc); @@ -382,7 +382,7 @@ public void testGetBulk() throws Exception { public void testGetBulkNullIdList() throws Exception { List ids = null; try { - HBaseDataStore.get(TEST_APP, ids); + HBaseDataStore.getAll(TEST_APP, ids); fail(); } catch (DataStoreException ex) { assertEquals(DataStoreException.ErrorCode.STORE_INVALID_REQUEST, ex.getErrorCode()); @@ -396,7 +396,7 @@ public void testGetBulkMissingDocument() throws Exception { ids.add(UUID.randomUUID().toString()); } try { - HBaseDataStore.get(TEST_APP, ids); + HBaseDataStore.getAll(TEST_APP, ids); fail(); } catch (DataStoreException ex) { assertEquals(DataStoreException.ErrorCode.STORE_NO_DATA_FOUND_FOR_IDS, ex.getErrorCode()); @@ -419,7 +419,7 @@ public void testGetBulkHBaseReadException() throws Exception { .when(tableInterface) .get(Matchers.anyListOf(Get.class)); try { - HBaseDataStore.get(TEST_APP, ids); + HBaseDataStore.getAll(TEST_APP, ids); fail(); } catch (DataStoreException ex) { assertEquals(DataStoreException.ErrorCode.STORE_MULTI_GET, ex.getErrorCode()); @@ -441,7 +441,7 @@ public void testGetBulkHBaseCloseException() throws Exception { doThrow(new IOException()) .when(tableInterface) .close(); - HBaseDataStore.get(TEST_APP, ids); + HBaseDataStore.getAll(TEST_APP, ids); verify(tableInterface, times(1)).close(); } @@ -462,7 +462,7 @@ public void testGetBulkNullHBaseTableConnection() throws Exception { tableInterface.put(putList); when(hBaseTableConnection.getTable(Matchers.
any())).thenReturn(null); try { - HBaseDataStore.get(TEST_APP, ids); + HBaseDataStore.getAll(TEST_APP, ids); fail(); } catch (DataStoreException ex) { assertEquals(DataStoreException.ErrorCode.STORE_MULTI_GET, ex.getErrorCode()); diff --git a/foxtrot-core/src/test/java/com/flipkart/foxtrot/core/querystore/impl/ElasticsearchQueryStoreTest.java b/foxtrot-core/src/test/java/com/flipkart/foxtrot/core/querystore/impl/ElasticsearchQueryStoreTest.java index 75b64cda5..c0bea304e 100644 --- a/foxtrot-core/src/test/java/com/flipkart/foxtrot/core/querystore/impl/ElasticsearchQueryStoreTest.java +++ b/foxtrot-core/src/test/java/com/flipkart/foxtrot/core/querystore/impl/ElasticsearchQueryStoreTest.java @@ -213,7 +213,7 @@ public void testGetBulk() throws Exception { } queryStore.save(TestUtils.TEST_TABLE_NAME, ImmutableList.copyOf(idValues.values())); elasticsearchServer.refresh(ElasticsearchUtils.getIndices(TestUtils.TEST_TABLE_NAME)); - List responseDocuments = queryStore.get(TestUtils.TEST_TABLE_NAME, ids); + List responseDocuments = queryStore.getAll(TestUtils.TEST_TABLE_NAME, ids); HashMap responseIdValues = new HashMap(); for (Document doc : responseDocuments) { responseIdValues.put(doc.getId(), doc); @@ -231,7 +231,7 @@ public void testGetBulk() throws Exception { @Test public void testGetBulkInvalidIds() throws Exception { try { - queryStore.get(TestUtils.TEST_TABLE_NAME, Arrays.asList(UUID.randomUUID().toString(), UUID.randomUUID().toString())); + queryStore.getAll(TestUtils.TEST_TABLE_NAME, Arrays.asList(UUID.randomUUID().toString(), UUID.randomUUID().toString())); fail(); } catch (QueryStoreException qse) { assertEquals(QueryStoreException.ErrorCode.DOCUMENT_NOT_FOUND, qse.getErrorCode()); diff --git a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/resources/DocumentResource.java b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/resources/DocumentResource.java index 3ef29580a..917587bcd 100644 --- a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/resources/DocumentResource.java +++ b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/resources/DocumentResource.java @@ -18,6 +18,7 @@ import com.flipkart.foxtrot.common.Document; import com.flipkart.foxtrot.core.querystore.QueryStore; import com.flipkart.foxtrot.core.querystore.QueryStoreException; +import com.yammer.metrics.annotation.Timed; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -36,7 +37,6 @@ * Time: 10:55 PM */ @Path("/v1/document/{table}") -@Consumes(MediaType.APPLICATION_JSON) @Produces(MediaType.APPLICATION_JSON) public class DocumentResource { private static final Logger logger = LoggerFactory.getLogger(DocumentResource.class.getSimpleName()); @@ -48,6 +48,8 @@ public DocumentResource(QueryStore queryStore) { } @POST + @Consumes(MediaType.APPLICATION_JSON) + @Timed public Response saveDocument(@PathParam("table") final String table, @Valid final Document document) { try { queryStore.save(table, document); @@ -72,6 +74,8 @@ public Response saveDocument(@PathParam("table") final String table, @Valid fina @POST @Path("/bulk") + @Consumes(MediaType.APPLICATION_JSON) + @Timed public Response saveDocuments(@PathParam("table") final String table, @Valid final List document) { try { queryStore.save(table, document); @@ -96,6 +100,7 @@ public Response saveDocuments(@PathParam("table") final String table, @Valid fin @GET @Path("/{id}") + @Timed public Response getDocument(@PathParam("table") final String table, @PathParam("id") @NotNull final String id) { try { return Response.ok(queryStore.get(table, id)).build(); @@ -117,9 +122,10 @@ public Response getDocument(@PathParam("table") final String table, @PathParam(" } @GET + @Timed public Response getDocuments(@PathParam("table") final String table, @QueryParam("id") @NotNull final List ids) { try { - return Response.ok(queryStore.get(table, ids)).build(); + return Response.ok(queryStore.getAll(table, ids)).build(); } catch (QueryStoreException ex) { logger.error("Get error : ", ex); switch (ex.getErrorCode()) { diff --git a/foxtrot-server/src/test/java/com/flipkart/foxtrot/server/resources/DocumentResourceTest.java b/foxtrot-server/src/test/java/com/flipkart/foxtrot/server/resources/DocumentResourceTest.java index e511cf59b..1910c5f33 100644 --- a/foxtrot-server/src/test/java/com/flipkart/foxtrot/server/resources/DocumentResourceTest.java +++ b/foxtrot-server/src/test/java/com/flipkart/foxtrot/server/resources/DocumentResourceTest.java @@ -348,7 +348,7 @@ public void testGetDocumentsMissingIds() throws Exception { public void testGetDocumentsInternalError() throws Exception { try { doThrow(new QueryStoreException(QueryStoreException.ErrorCode.DOCUMENT_GET_ERROR, "Error")) - .when(queryStore).get(anyString(), anyListOf(String.class)); + .when(queryStore).getAll(anyString(), anyListOf(String.class)); client().resource(String.format("/v1/document/%s", TestUtils.TEST_TABLE_NAME)) .queryParam("id", UUID.randomUUID().toString()) .get(String.class); From f49dcf3aa5c94e995f070d16f11bd838b08ede18 Mon Sep 17 00:00:00 2001 From: Santanu Sinha Date: Thu, 3 Dec 2015 13:01:17 +0530 Subject: [PATCH 21/64] Bumping up java compatibility to 1.8 --- foxtrot-core/pom.xml | 6 +++--- pom.xml | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/foxtrot-core/pom.xml b/foxtrot-core/pom.xml index 361d10106..f282e00f0 100644 --- a/foxtrot-core/pom.xml +++ b/foxtrot-core/pom.xml @@ -122,9 +122,9 @@ aspectj-maven-plugin 1.8 - 1.6 - 1.6 - 1.6 + 1.8 + 1.8 + 1.8 diff --git a/pom.xml b/pom.xml index 7e4c94d76..e1083d584 100644 --- a/pom.xml +++ b/pom.xml @@ -154,8 +154,8 @@ maven-compiler-plugin 3.1 - 1.6 - 1.6 + 1.8 + 1.8 From 61f52579dd4b9d2927cea184bb330667f4b8ea14 Mon Sep 17 00:00:00 2001 From: Santanu Sinha Date: Thu, 3 Dec 2015 13:03:49 +0530 Subject: [PATCH 22/64] Bumping up java compatibility to 1.8 --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index d0f8f79ea..3916d52e1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,5 +1,5 @@ sudo: false language: java jdk: - - oraclejdk7 - - openjdk7 + - oraclejdk8 + - openjdk8 From 5ff801412ed9fcf54e00e6cff1af8ceac37749e6 Mon Sep 17 00:00:00 2001 From: Santanu Sinha Date: Thu, 3 Dec 2015 13:27:54 +0530 Subject: [PATCH 23/64] Minor cleanup --- .travis.yml | 1 - .../foxtrot/common/DocumentMetadata.java | 16 ++++++++-------- .../datastore/impl/hbase/HBaseDataStore.java | 5 ++--- .../core/querystore/DocumentTranslator.java | 4 ++-- 4 files changed, 12 insertions(+), 14 deletions(-) diff --git a/.travis.yml b/.travis.yml index 3916d52e1..35cc84488 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,4 +2,3 @@ sudo: false language: java jdk: - oraclejdk8 - - openjdk8 diff --git a/foxtrot-common/src/main/java/com/flipkart/foxtrot/common/DocumentMetadata.java b/foxtrot-common/src/main/java/com/flipkart/foxtrot/common/DocumentMetadata.java index 1dce2f436..1869a01bc 100644 --- a/foxtrot-common/src/main/java/com/flipkart/foxtrot/common/DocumentMetadata.java +++ b/foxtrot-common/src/main/java/com/flipkart/foxtrot/common/DocumentMetadata.java @@ -5,14 +5,14 @@ */ public class DocumentMetadata { private String id; - private String rowKey; + private String rawStorageId; public DocumentMetadata() { } - public DocumentMetadata(String id, String rowKey) { + public DocumentMetadata(String id, String rawStorageId) { this.id = id; - this.rowKey = rowKey; + this.rawStorageId = rawStorageId; } public String getId() { @@ -23,19 +23,19 @@ public void setId(String id) { this.id = id; } - public String getRowKey() { - return rowKey; + public String getRawStorageId() { + return rawStorageId; } - public void setRowKey(String rowKey) { - this.rowKey = rowKey; + public void setRawStorageId(String rawStorageId) { + this.rawStorageId = rawStorageId; } @Override public String toString() { return "DocumentMetadata{" + "id='" + id + '\'' + - ", rowKey='" + rowKey + '\'' + + ", rawStorageId='" + rawStorageId + '\'' + '}'; } } diff --git a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/datastore/impl/hbase/HBaseDataStore.java b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/datastore/impl/hbase/HBaseDataStore.java index 5cc440f6c..98effc1e2 100644 --- a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/datastore/impl/hbase/HBaseDataStore.java +++ b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/datastore/impl/hbase/HBaseDataStore.java @@ -23,7 +23,6 @@ import com.flipkart.foxtrot.core.datastore.DataStore; import com.flipkart.foxtrot.core.datastore.DataStoreException; import com.flipkart.foxtrot.core.querystore.DocumentTranslator; -import com.google.common.base.Stopwatch; import com.google.common.collect.ImmutableList; import com.shash.hbase.ds.RowKeyDistributorByHashPrefix; import com.yammer.metrics.annotation.Timed; @@ -61,7 +60,7 @@ public class HBaseDataStore implements DataStore { public HBaseDataStore(HbaseTableConnection tableWrapper, ObjectMapper mapper) { this.tableWrapper = tableWrapper; this.mapper = mapper; - keyTranslator = new HbaseKeyTranslator(new RowKeyDistributorByHashPrefix( + this.keyTranslator = new HbaseKeyTranslator(new RowKeyDistributorByHashPrefix( new RowKeyDistributorByHashPrefix.OneByteSimpleHash(tableWrapper.getHbaseConfig().getNumBuckets()))); } @@ -218,7 +217,7 @@ public List getAll(final Table table, List ids) throws DataSto : null; final String docId = (null == metadata) ? Bytes.toString(getResult.getRow()).split(":")[0] - : documentMetadata.getRowKey(); + : documentMetadata.getRawStorageId(); results.add(translator.translateBack(new Document(docId, time, documentMetadata, mapper.readTree(data)))); } else { throw new DataStoreException(DataStoreException.ErrorCode.STORE_NO_DATA_FOUND_FOR_IDS, diff --git a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/DocumentTranslator.java b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/DocumentTranslator.java index c46ad9b78..0328fa135 100644 --- a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/DocumentTranslator.java +++ b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/DocumentTranslator.java @@ -28,7 +28,7 @@ public Document translate(final Table table, final Document inDocument) { final String rowKey = rowKey(table, inDocument); DocumentMetadata metadata = new DocumentMetadata(); - metadata.setRowKey(rowKey); + metadata.setRawStorageId(rowKey); metadata.setId(inDocument.getId()); document.setId(rowKey); @@ -62,7 +62,7 @@ public DocumentMetadata metadata(final Table table, final Document inDocument) { final String rowKey = rowKey(table, inDocument); DocumentMetadata metadata = new DocumentMetadata(); - metadata.setRowKey(rowKey); + metadata.setRawStorageId(rowKey); metadata.setId(inDocument.getId()); logger.info("Doc row key: {}, {}", rowKey, inDocument); From ce1757d86f22faf5e6ffa1877b3f2b90697a9134 Mon Sep 17 00:00:00 2001 From: Santanu Sinha Date: Sat, 5 Dec 2015 21:38:35 +0530 Subject: [PATCH 24/64] Fixed behavior with older data * Key versioning addded * Key generation behavior consolidated * Version specific and overall testcases added --- .../impl/hbase/HbaseKeyTranslator.java | 19 ------------------- 1 file changed, 19 deletions(-) delete mode 100644 foxtrot-core/src/main/java/com/flipkart/foxtrot/core/datastore/impl/hbase/HbaseKeyTranslator.java diff --git a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/datastore/impl/hbase/HbaseKeyTranslator.java b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/datastore/impl/hbase/HbaseKeyTranslator.java deleted file mode 100644 index b8bb04793..000000000 --- a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/datastore/impl/hbase/HbaseKeyTranslator.java +++ /dev/null @@ -1,19 +0,0 @@ -package com.flipkart.foxtrot.core.datastore.impl.hbase; - -import com.shash.hbase.ds.AbstractRowKeyDistributor; -import org.apache.hadoop.hbase.util.Bytes; - -/** - * Created by santanu.s on 02/12/15. - */ -public class HbaseKeyTranslator { - private final AbstractRowKeyDistributor keyDistributor; - - public HbaseKeyTranslator(AbstractRowKeyDistributor keyDistributor) { - this.keyDistributor = keyDistributor; - } - - public byte[] idToRowKey(String id) { - return keyDistributor.getDistributedKey(Bytes.toBytes(id)); - } -} From ae34f87ad98099b9fa93a28710fd2dcad6a93b52 Mon Sep 17 00:00:00 2001 From: Santanu Sinha Date: Sat, 5 Dec 2015 21:51:48 +0530 Subject: [PATCH 25/64] Adding the files for last commit --- .../datastore/impl/hbase/HBaseDataStore.java | 13 +- .../core/querystore/DocumentTranslator.java | 38 ++++-- .../impl/ElasticsearchQueryStore.java | 9 +- .../impl/hbase/HBaseDataStoreTest.java | 123 +++++++++++++++--- .../impl/ElasticsearchQueryStoreTest.java | 7 +- 5 files changed, 151 insertions(+), 39 deletions(-) diff --git a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/datastore/impl/hbase/HBaseDataStore.java b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/datastore/impl/hbase/HBaseDataStore.java index 98effc1e2..c49a6d558 100644 --- a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/datastore/impl/hbase/HBaseDataStore.java +++ b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/datastore/impl/hbase/HBaseDataStore.java @@ -23,6 +23,7 @@ import com.flipkart.foxtrot.core.datastore.DataStore; import com.flipkart.foxtrot.core.datastore.DataStoreException; import com.flipkart.foxtrot.core.querystore.DocumentTranslator; +import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.ImmutableList; import com.shash.hbase.ds.RowKeyDistributorByHashPrefix; import com.yammer.metrics.annotation.Timed; @@ -54,13 +55,12 @@ public class HBaseDataStore implements DataStore { private final HbaseTableConnection tableWrapper; private final ObjectMapper mapper; - private final DocumentTranslator translator = new DocumentTranslator(); - private final HbaseKeyTranslator keyTranslator; + private final DocumentTranslator translator; public HBaseDataStore(HbaseTableConnection tableWrapper, ObjectMapper mapper) { this.tableWrapper = tableWrapper; this.mapper = mapper; - this.keyTranslator = new HbaseKeyTranslator(new RowKeyDistributorByHashPrefix( + translator = new DocumentTranslator(new RowKeyDistributorByHashPrefix( new RowKeyDistributorByHashPrefix.OneByteSimpleHash(tableWrapper.getHbaseConfig().getNumBuckets()))); } @@ -147,7 +147,7 @@ public List saveAll(final Table table, List documents) throw public Document get(final Table table, String id) throws DataStoreException { HTableInterface hTable = null; try { - Get get = new Get(keyTranslator.idToRowKey(id)) + Get get = new Get(Bytes.toBytes(translator.rawStorageIdFromDocumentId(table, id))) .addColumn(COLUMN_FAMILY, DOCUMENT_FIELD_NAME) .addColumn(COLUMN_FAMILY, DOCUMENT_META_FIELD_NAME) .addColumn(COLUMN_FAMILY, TIMESTAMP_FIELD_NAME) @@ -195,7 +195,7 @@ public List getAll(final Table table, List ids) throws DataSto try { List gets = new ArrayList(ids.size()); for (String id : ids) { - Get get = new Get(keyTranslator.idToRowKey(id)) + Get get = new Get(Bytes.toBytes(translator.rawStorageIdFromDocumentId(table, id))) .addColumn(COLUMN_FAMILY, DOCUMENT_FIELD_NAME) .addColumn(COLUMN_FAMILY, DOCUMENT_META_FIELD_NAME) .addColumn(COLUMN_FAMILY, TIMESTAMP_FIELD_NAME) @@ -247,8 +247,9 @@ public List getAll(final Table table, List ids) throws DataSto } } + @VisibleForTesting public Put getPutForDocument(final Table table, Document document) throws JsonProcessingException { - return new Put(keyTranslator.idToRowKey(document.getId())) + return new Put(Bytes.toBytes(document.getId())) .add(COLUMN_FAMILY, DOCUMENT_META_FIELD_NAME, mapper.writeValueAsBytes(document.getMetadata())) .add(COLUMN_FAMILY, DOCUMENT_FIELD_NAME, mapper.writeValueAsBytes(document.getData())) .add(COLUMN_FAMILY, TIMESTAMP_FIELD_NAME, Bytes.toBytes(document.getTimestamp())); diff --git a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/DocumentTranslator.java b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/DocumentTranslator.java index 0328fa135..b2aa3845d 100644 --- a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/DocumentTranslator.java +++ b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/DocumentTranslator.java @@ -4,6 +4,8 @@ import com.flipkart.foxtrot.common.DocumentMetadata; import com.flipkart.foxtrot.common.Table; import com.google.common.collect.ImmutableList; +import com.shash.hbase.ds.AbstractRowKeyDistributor; +import org.apache.hadoop.hbase.util.Bytes; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -13,8 +15,16 @@ * Created by santanu.s on 24/11/15. */ public class DocumentTranslator { + private static final String CURRENT_RAW_KEY_VERSION = "__RAW_KEY_VERSION_2__"; + private static final Logger logger = LoggerFactory.getLogger(DocumentTranslator.class); + private final AbstractRowKeyDistributor keyDistributor; + + public DocumentTranslator(AbstractRowKeyDistributor keyDistributor) { + this.keyDistributor = keyDistributor; + } + public List translate(final Table table, final List inDocuments) { ImmutableList.Builder docListBuilder = ImmutableList.builder(); for(Document document : inDocuments) { @@ -25,18 +35,15 @@ public List translate(final Table table, final List inDocume public Document translate(final Table table, final Document inDocument) { Document document = new Document(); - final String rowKey = rowKey(table, inDocument); + final String rowKey = rawStorageIdFromDocument(table, inDocument); - DocumentMetadata metadata = new DocumentMetadata(); - metadata.setRawStorageId(rowKey); - metadata.setId(inDocument.getId()); + DocumentMetadata metadata = metadata(table, inDocument); document.setId(rowKey); document.setTimestamp(inDocument.getTimestamp()); document.setMetadata(metadata); document.setData(inDocument.getData()); - logger.debug("Translated doc row key: {}, {}", rowKey, document); return document; } @@ -59,17 +66,30 @@ public Document translateBack(final Document inDocument) { } public DocumentMetadata metadata(final Table table, final Document inDocument) { - final String rowKey = rowKey(table, inDocument); + final String rowKey = generateScalableKey(rawStorageIdFromDocument(table, inDocument)); DocumentMetadata metadata = new DocumentMetadata(); metadata.setRawStorageId(rowKey); metadata.setId(inDocument.getId()); - logger.info("Doc row key: {}, {}", rowKey, inDocument); + logger.debug("Doc row key: {}, {}", rowKey, inDocument); return metadata; } - public String rowKey(final Table table, final Document document) { - return String.format("%s:%d:%s", table.getName(), document.getTimestamp(), document.getId()); + public String rawStorageIdFromDocument(final Table table, final Document document) { + return String.format("%s:%d:%s:%s", + table.getName(), document.getTimestamp(), document.getId(), CURRENT_RAW_KEY_VERSION); + } + + private String generateScalableKey(String id) { + return new String(keyDistributor.getDistributedKey(Bytes.toBytes(id))); + } + + public String rawStorageIdFromDocumentId(Table table, String id) { + if(!id.endsWith(CURRENT_RAW_KEY_VERSION)) { + return String.format("%s:%s", id, table.getName()); + } + //IMPLEMENTOR NOTE:: Handle older versions here + return id; } } diff --git a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/impl/ElasticsearchQueryStore.java b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/impl/ElasticsearchQueryStore.java index b7ac704cf..6bb98a73c 100644 --- a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/impl/ElasticsearchQueryStore.java +++ b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/impl/ElasticsearchQueryStore.java @@ -77,7 +77,7 @@ public ElasticsearchQueryStore(TableMetadataManager tableMetadataManager, } @Override - @Timed(name = "single-document-save") + @Timed public void save(String table, Document document) throws QueryStoreException { table = ElasticsearchUtils.getValidTableName(table); try { @@ -125,7 +125,7 @@ public void save(String table, Document document) throws QueryStoreException { } @Override - @Timed(name = "multiple-document-saveAll") + @Timed public void save(String table, List documents) throws QueryStoreException { table = ElasticsearchUtils.getValidTableName(table); try { @@ -196,7 +196,7 @@ public void save(String table, List documents) throws QueryStoreExcept } @Override - @Timed(name = "single-document-getAll") + @Timed public Document get(String table, String id) throws QueryStoreException { table = ElasticsearchUtils.getValidTableName(table); Table fxTable = null; @@ -280,12 +280,9 @@ public List getAll(String table, List ids, boolean bypassMetal .setSize(ids.size()) .execute() .actionGet(); - Set inputKeys = ImmutableSet.copyOf(ids); - Set foundKeys = Sets.newHashSet(); for (SearchHit hit : response.getHits()) { final String id = hit.getFields().get(ElasticsearchUtils.DOCUMENT_META_ID_FIELD_NAME).getValue().toString(); rowKeys.put(id,hit.getId()); - foundKeys.add(id); } } logger.info("Get row keys: {}", rowKeys.size()); diff --git a/foxtrot-core/src/test/java/com/flipkart/foxtrot/core/datastore/impl/hbase/HBaseDataStoreTest.java b/foxtrot-core/src/test/java/com/flipkart/foxtrot/core/datastore/impl/hbase/HBaseDataStoreTest.java index b56c7bd3b..0b2f9c5a1 100644 --- a/foxtrot-core/src/test/java/com/flipkart/foxtrot/core/datastore/impl/hbase/HBaseDataStoreTest.java +++ b/foxtrot-core/src/test/java/com/flipkart/foxtrot/core/datastore/impl/hbase/HBaseDataStoreTest.java @@ -23,6 +23,7 @@ import com.flipkart.foxtrot.core.MockHTable; import com.flipkart.foxtrot.core.datastore.DataStoreException; import com.flipkart.foxtrot.core.querystore.DocumentTranslator; +import com.google.common.collect.Lists; import com.shash.hbase.ds.RowKeyDistributorByHashPrefix; import org.apache.hadoop.hbase.client.Get; import org.apache.hadoop.hbase.client.HTableInterface; @@ -36,6 +37,7 @@ import java.io.IOException; import java.util.*; +import java.util.stream.Collectors; import static org.junit.Assert.*; import static org.mockito.Mockito.*; @@ -54,7 +56,7 @@ public class HBaseDataStoreTest { private static final byte[] DATA_FIELD_NAME = Bytes.toBytes("data"); private static final String TEST_APP_NAME = "test-app"; private static final Table TEST_APP = new Table(TEST_APP_NAME, 7); - private final HbaseKeyTranslator keyTranslator = new HbaseKeyTranslator(new RowKeyDistributorByHashPrefix( + private final DocumentTranslator translator = new DocumentTranslator(new RowKeyDistributorByHashPrefix( new RowKeyDistributorByHashPrefix.OneByteSimpleHash(32))); @@ -156,7 +158,6 @@ public void testSaveSingleNullHBaseTableConnection() throws Exception { } } - private DocumentTranslator translator = new DocumentTranslator(); @Test public void testSaveBulk() throws Exception { List documents = new Vector(); @@ -274,8 +275,8 @@ public void testSaveBulkNullHBaseTableConnection() throws Exception { } public void validateSave(Document savedDocument) throws Exception { - String rowkey = new DocumentTranslator().rowKey(TEST_APP, savedDocument); - Get get = new Get(keyTranslator.idToRowKey(rowkey)); + String rowkey = translator.rawStorageIdFromDocument(TEST_APP, savedDocument); + Get get = new Get(Bytes.toBytes(translator.rawStorageIdFromDocumentId(TEST_APP, rowkey))); Result result = tableInterface.get(get); assertNotNull("Get for Id should not be null", result); Document actualDocument = new Document(savedDocument.getId(), @@ -284,16 +285,34 @@ public void validateSave(Document savedDocument) throws Exception { compare(savedDocument, actualDocument); } + private String v1FormatKey(String id) { + return String.format("%s:%s", id, TEST_APP_NAME); + } + @Test - public void testGetSingle() throws Exception { + public void testGetSingleV1Format() throws Exception { String id = UUID.randomUUID().toString(); JsonNode data = mapper.valueToTree(Collections.singletonMap("TEST_NAME", "SINGLE_SAVE_TEST")); - Document expectedDocument = new Document(id, System.currentTimeMillis(), new DocumentMetadata(id, "row.1"), data); + final String newId = v1FormatKey(id); + Document expectedDocument = new Document(newId, System.currentTimeMillis(), + new DocumentMetadata(newId, "row.1"), data); tableInterface.put(HBaseDataStore.getPutForDocument(TEST_APP, expectedDocument)); Document actualDocument = HBaseDataStore.get(TEST_APP, id); compare(expectedDocument, actualDocument); } + @Test + public void testGetSingleV2Format() throws Exception { + String id = UUID.randomUUID().toString(); + long timestamp = System.currentTimeMillis(); + JsonNode data = mapper.valueToTree(Collections.singletonMap("TEST_NAME", "SINGLE_SAVE_TEST")); + Document originalDocument = new Document(id, timestamp, data); + final String newId = translator.rawStorageIdFromDocument(TEST_APP, originalDocument); + tableInterface.put(HBaseDataStore.getPutForDocument(TEST_APP, translator.translate(TEST_APP, originalDocument))); + Document actualDocument = HBaseDataStore.get(TEST_APP, newId); + compare(originalDocument, actualDocument); + } + @Test public void testGetSingleMissingDocument() { try { @@ -328,11 +347,12 @@ public void testGetSingleHBaseCloseException() throws Exception { JsonNode data = mapper.valueToTree(Collections.singletonMap("TEST_NAME", "SINGLE_SAVE_TEST")); Document expectedDocument = new Document(id, System.currentTimeMillis(), data); - tableInterface.put(HBaseDataStore.getPutForDocument(TEST_APP, expectedDocument)); + Document translated = translator.translate(TEST_APP, expectedDocument); + tableInterface.put(HBaseDataStore.getPutForDocument(TEST_APP, translated)); doThrow(new IOException()) .when(tableInterface) .close(); - HBaseDataStore.get(TEST_APP, id); + HBaseDataStore.get(TEST_APP, translated.getId()); verify(tableInterface, times(1)).close(); } @@ -352,22 +372,54 @@ public void testGetSingleNullHBaseTableConnection() throws Exception { } @Test - public void testGetBulk() throws Exception { + public void testV1GetBulk() throws Exception { Map idValues = new HashMap(); List ids = new Vector(); List putList = new Vector(); + HashMap actualIdValues = new HashMap(); for (int i = 0; i < 10; i++) { String id = UUID.randomUUID().toString(); + long timestamp = System.currentTimeMillis(); + JsonNode data = mapper.valueToTree(Collections.singletonMap("TEST_NAME", "BULK_GET_TEST")); + String rawId = v1FormatKey(id); ids.add(id); - Document document = new Document(id, System.currentTimeMillis(), - new DocumentMetadata(id, id), - mapper.valueToTree(Collections.singletonMap("TEST_NAME", "BULK_GET_TEST"))); - idValues.put(document.getMetadata().getId(), document); - putList.add(HBaseDataStore.getPutForDocument(TEST_APP, idValues.get(id))); + Document document = new Document(rawId, timestamp, + new DocumentMetadata(id, rawId), data); + putList.add(HBaseDataStore.getPutForDocument(TEST_APP, document)); + idValues.put(id, new Document(id, timestamp, data)); } tableInterface.put(putList); List actualDocuments = HBaseDataStore.getAll(TEST_APP, ids); + for (Document doc : actualDocuments) { + actualIdValues.put(doc.getId(), doc); + } + assertNotNull("List of returned Documents should not be null", actualDocuments); + for (String id : ids) { + assertTrue("Requested Id should be present in response", actualIdValues.containsKey(id)); + compare(idValues.get(id), actualIdValues.get(id)); + } + } + + @Test + public void testV2GetBulk() throws Exception { + Map idValues = new HashMap(); + List ids = new Vector(); + List rawIds = new Vector(); + List putList = new Vector(); HashMap actualIdValues = new HashMap(); + for (int i = 0; i < 10; i++) { + String id = UUID.randomUUID().toString(); + long timestamp = System.currentTimeMillis(); + JsonNode data = mapper.valueToTree(Collections.singletonMap("TEST_NAME", "BULK_GET_TEST")); + ids.add(id); + Document document = new Document(id, timestamp, data); + Document translated = translator.translate(TEST_APP, document); + putList.add(HBaseDataStore.getPutForDocument(TEST_APP, translated)); + idValues.put(id, new Document(id, timestamp, data)); + rawIds.add(translated.getId()); + } + tableInterface.put(putList); + List actualDocuments = HBaseDataStore.getAll(TEST_APP, rawIds); for (Document doc : actualDocuments) { actualIdValues.put(doc.getId(), doc); } @@ -380,9 +432,8 @@ public void testGetBulk() throws Exception { @Test public void testGetBulkNullIdList() throws Exception { - List ids = null; try { - HBaseDataStore.getAll(TEST_APP, ids); + HBaseDataStore.getAll(TEST_APP, null); fail(); } catch (DataStoreException ex) { assertEquals(DataStoreException.ErrorCode.STORE_INVALID_REQUEST, ex.getErrorCode()); @@ -469,6 +520,46 @@ public void testGetBulkNullHBaseTableConnection() throws Exception { } } + @Test + public void testBasicSetGet() throws Exception { + String id = UUID.randomUUID().toString(); + long timestamp = System.currentTimeMillis(); + JsonNode data = mapper.valueToTree(Collections.singletonMap("TEST_NAME", "SINGLE_SAVE_TEST")); + Document originalDocument = new Document(id, timestamp, data); + final String newId = translator.rawStorageIdFromDocument(TEST_APP, originalDocument); + HBaseDataStore.save(TEST_APP, originalDocument); + Document actualDocument = HBaseDataStore.get(TEST_APP, newId); + compare(originalDocument, actualDocument); + } + + @Test + public void testBasicBulkGetSet() throws Exception { + Map idValues = new HashMap(); + List ids = new Vector(); + HashMap actualIdValues = new HashMap(); + List documents = Lists.newArrayList(); + for (int i = 0; i < 10; i++) { + String id = UUID.randomUUID().toString(); + long timestamp = System.currentTimeMillis(); + JsonNode data = mapper.valueToTree(Collections.singletonMap("TEST_NAME", "BULK_GET_TEST")); + ids.add(id); + Document document = new Document(id, timestamp, data); + idValues.put(id, document); + documents.add(document); + } + List translatedDocs = HBaseDataStore.saveAll(TEST_APP, documents); + List rawIds = translatedDocs.stream().map(Document::getId).collect(Collectors.toCollection(ArrayList::new)); + List actualDocuments = HBaseDataStore.getAll(TEST_APP, rawIds); + for (Document doc : actualDocuments) { + actualIdValues.put(doc.getId(), doc); + } + assertNotNull("List of returned Documents should not be null", actualDocuments); + for (String id : ids) { + assertTrue("Requested Id should be present in response", actualIdValues.containsKey(id)); + compare(idValues.get(id), actualIdValues.get(id)); + } + } + public void compare(Document expected, Document actual) throws Exception { assertNotNull(expected); assertNotNull(actual); diff --git a/foxtrot-core/src/test/java/com/flipkart/foxtrot/core/querystore/impl/ElasticsearchQueryStoreTest.java b/foxtrot-core/src/test/java/com/flipkart/foxtrot/core/querystore/impl/ElasticsearchQueryStoreTest.java index c0bea304e..451a22aeb 100644 --- a/foxtrot-core/src/test/java/com/flipkart/foxtrot/core/querystore/impl/ElasticsearchQueryStoreTest.java +++ b/foxtrot-core/src/test/java/com/flipkart/foxtrot/core/querystore/impl/ElasticsearchQueryStoreTest.java @@ -30,6 +30,7 @@ import com.flipkart.foxtrot.core.querystore.TableMetadataManager; import com.flipkart.foxtrot.core.querystore.actions.spi.AnalyticsLoader; import com.google.common.collect.ImmutableList; +import com.shash.hbase.ds.RowKeyDistributorByHashPrefix; import org.elasticsearch.action.get.GetResponse; import org.junit.After; import org.junit.Before; @@ -53,6 +54,8 @@ public class ElasticsearchQueryStoreTest { private ElasticsearchQueryStore queryStore; private ObjectMapper mapper; private TableMetadataManager tableMetadataManager; + private final DocumentTranslator translator = new DocumentTranslator(new RowKeyDistributorByHashPrefix( + new RowKeyDistributorByHashPrefix.OneByteSimpleHash(32))); @Before public void setUp() throws Exception { @@ -86,7 +89,7 @@ public void testSaveSingle() throws Exception { JsonNode data = mapper.valueToTree(Collections.singletonMap("TEST_NAME", "SINGLE_SAVE_TEST")); originalDocument.setData(data); queryStore.save(TestUtils.TEST_TABLE_NAME, originalDocument); - final Document translatedDocuemnt = new DocumentTranslator().translate(tableMetadataManager.get(TestUtils.TEST_TABLE_NAME), originalDocument); + final Document translatedDocuemnt = translator.translate(tableMetadataManager.get(TestUtils.TEST_TABLE_NAME), originalDocument); GetResponse getResponse = elasticsearchServer .getClient() .prepareGet(ElasticsearchUtils.getCurrentIndex(TestUtils.TEST_TABLE_NAME, originalDocument.getTimestamp()), @@ -122,7 +125,7 @@ public void testSaveBulk() throws Exception { mapper.valueToTree(Collections.singletonMap("TEST_NAME", "SINGLE_SAVE_TEST")))); } queryStore.save(TestUtils.TEST_TABLE_NAME, documents); - final List translatedDocuemtns = new DocumentTranslator().translate(tableMetadataManager.get(TestUtils.TEST_TABLE_NAME), documents); + final List translatedDocuemtns = translator.translate(tableMetadataManager.get(TestUtils.TEST_TABLE_NAME), documents); for (Document document : translatedDocuemtns) { GetResponse getResponse = elasticsearchServer .getClient() From 703408df26b30accd2138a64b564b755dd177a3a Mon Sep 17 00:00:00 2001 From: Santanu Sinha Date: Sat, 5 Dec 2015 22:13:54 +0530 Subject: [PATCH 26/64] Minor coding style fix --- .../foxtrot/core/datastore/impl/hbase/HBaseDataStore.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/datastore/impl/hbase/HBaseDataStore.java b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/datastore/impl/hbase/HBaseDataStore.java index c49a6d558..c13379c05 100644 --- a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/datastore/impl/hbase/HBaseDataStore.java +++ b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/datastore/impl/hbase/HBaseDataStore.java @@ -60,7 +60,7 @@ public class HBaseDataStore implements DataStore { public HBaseDataStore(HbaseTableConnection tableWrapper, ObjectMapper mapper) { this.tableWrapper = tableWrapper; this.mapper = mapper; - translator = new DocumentTranslator(new RowKeyDistributorByHashPrefix( + this.translator = new DocumentTranslator(new RowKeyDistributorByHashPrefix( new RowKeyDistributorByHashPrefix.OneByteSimpleHash(tableWrapper.getHbaseConfig().getNumBuckets()))); } From d69c2aa250bd889942b6f3cc78d82da24dd25f93 Mon Sep 17 00:00:00 2001 From: Rishabh Goyal Date: Sat, 5 Dec 2015 22:33:07 +0530 Subject: [PATCH 27/64] Added TableManager. Added check for HBase table existence during table creation --- foxtrot-common/pom.xml | 4 +- foxtrot-core/pom.xml | 5 +- .../flipkart/foxtrot/core/common/Action.java | 17 +- .../foxtrot/core/common/Identifiable.java | 11 + .../foxtrot/core/datastore/DataStore.java | 15 +- .../core/datastore/DataStoreException.java | 9 +- .../datastore/impl/hbase/HBaseDataStore.java | 21 +- .../impl/hbase/HbaseTableConnection.java | 1 - .../foxtrot/core/querystore/QueryStore.java | 27 +- .../core/querystore/QueryStoreException.java | 5 +- .../core/querystore/actions/CountAction.java | 2 +- .../querystore/actions/DistinctAction.java | 2 +- .../core/querystore/actions/FilterAction.java | 2 +- .../core/querystore/actions/GroupAction.java | 2 +- .../querystore/actions/HistogramAction.java | 2 +- .../core/querystore/actions/StatsAction.java | 2 +- .../querystore/actions/StatsTrendAction.java | 2 +- .../core/querystore/actions/TrendAction.java | 2 +- .../actions/spi/AnalyticsLoader.java | 2 +- .../impl/ElasticsearchQueryStore.java | 32 ++- .../foxtrot/core/table/TableManager.java | 22 ++ .../core/table/TableManagerException.java | 46 +++ .../TableMetadataManager.java | 23 +- .../impl/DistributedTableMetadataManager.java | 13 +- .../core/table/impl/FoxtrotTableManager.java | 104 +++++++ .../impl/TableMapStore.java | 3 +- .../core/common/NonCacheableAction.java | 2 +- .../core/querystore/QueryExecutorTest.java | 1 + .../querystore/actions/CountActionTest.java | 2 +- .../actions/DistinctActionTest.java | 2 +- .../querystore/actions/FilterActionTest.java | 2 +- .../querystore/actions/GroupActionTest.java | 2 +- .../actions/HistogramActionTest.java | 2 +- .../actions/NonCacheableActionTest.java | 2 +- .../querystore/actions/TrendActionTest.java | 2 +- .../querystore/impl/DistributedCacheTest.java | 2 +- .../DistributedTableMetadataManagerTest.java | 2 + .../impl/ElasticsearchQueryStoreTest.java | 2 +- .../querystore/impl/TableMapStoreTest.java | 1 + .../table/impl/FoxtrotTableManagerTest.java | 268 ++++++++++++++++++ foxtrot-server/pom.xml | 8 +- .../foxtrot/server/FoxtrotServer.java | 22 +- .../server/resources/DocumentResource.java | 6 +- .../resources/TableManagerResource.java | 106 +++++++ .../resources/TableMetadataResource.java | 78 ----- .../resources/AnalyticsResourceTest.java | 3 +- .../server/resources/AsyncResourceTest.java | 3 +- .../resources/DocumentResourceTest.java | 3 +- .../TableFieldMappingResourceTest.java | 3 +- ...est.java => TableManagerResourceTest.java} | 42 ++- foxtrot-sql/pom.xml | 9 +- .../com/flipkart/foxtrot/sql/FqlEngine.java | 2 +- pom.xml | 2 +- 53 files changed, 749 insertions(+), 206 deletions(-) create mode 100644 foxtrot-core/src/main/java/com/flipkart/foxtrot/core/common/Identifiable.java create mode 100644 foxtrot-core/src/main/java/com/flipkart/foxtrot/core/table/TableManager.java create mode 100644 foxtrot-core/src/main/java/com/flipkart/foxtrot/core/table/TableManagerException.java rename foxtrot-core/src/main/java/com/flipkart/foxtrot/core/{querystore => table}/TableMetadataManager.java (72%) rename foxtrot-core/src/main/java/com/flipkart/foxtrot/core/{querystore => table}/impl/DistributedTableMetadataManager.java (92%) create mode 100644 foxtrot-core/src/main/java/com/flipkart/foxtrot/core/table/impl/FoxtrotTableManager.java rename foxtrot-core/src/main/java/com/flipkart/foxtrot/core/{querystore => table}/impl/TableMapStore.java (98%) create mode 100644 foxtrot-core/src/test/java/com/flipkart/foxtrot/core/table/impl/FoxtrotTableManagerTest.java create mode 100644 foxtrot-server/src/main/java/com/flipkart/foxtrot/server/resources/TableManagerResource.java delete mode 100644 foxtrot-server/src/main/java/com/flipkart/foxtrot/server/resources/TableMetadataResource.java rename foxtrot-server/src/test/java/com/flipkart/foxtrot/server/resources/{TableMetadataResourceTest.java => TableManagerResourceTest.java} (77%) diff --git a/foxtrot-common/pom.xml b/foxtrot-common/pom.xml index 6d50d632a..2eb21d635 100644 --- a/foxtrot-common/pom.xml +++ b/foxtrot-common/pom.xml @@ -5,12 +5,10 @@ foxtrot com.flipkart.foxtrot - 0.1 + 0.3-SNAPSHOT 4.0.0 foxtrot-common - 0.3 - \ No newline at end of file diff --git a/foxtrot-core/pom.xml b/foxtrot-core/pom.xml index 4702dbe39..1d4a3a841 100644 --- a/foxtrot-core/pom.xml +++ b/foxtrot-core/pom.xml @@ -5,12 +5,11 @@ foxtrot com.flipkart.foxtrot - 0.1 + 0.3-SNAPSHOT 4.0.0 foxtrot-core - 0.2 @@ -80,7 +79,7 @@ com.flipkart.foxtrot foxtrot-common - 0.3 + ${project.parent.version} diff --git a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/common/Action.java b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/common/Action.java index 22a886d01..a2ace7667 100644 --- a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/common/Action.java +++ b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/common/Action.java @@ -1,12 +1,12 @@ /** * Copyright 2014 Flipkart Internet Pvt. Ltd. - * + *

* Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + *

* http://www.apache.org/licenses/LICENSE-2.0 - * + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -22,8 +22,8 @@ import com.flipkart.foxtrot.core.datastore.DataStore; import com.flipkart.foxtrot.core.querystore.QueryStore; import com.flipkart.foxtrot.core.querystore.QueryStoreException; -import com.flipkart.foxtrot.core.querystore.TableMetadataManager; import com.flipkart.foxtrot.core.querystore.impl.ElasticsearchConnection; +import com.flipkart.foxtrot.core.table.TableMetadataManager; import com.google.common.collect.Lists; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -146,17 +146,16 @@ protected Filter getDefaultTimeSpan() { } private List checkAndAddTemporalBoundary(List filters) { - if(null != filters) { + if (null != filters) { for (Filter filter : filters) { - if(filter.isTemporal()) { + if (filter.isTemporal()) { return filters; } } } - if(null == filters) { + if (null == filters) { filters = Lists.newArrayList(); - } - else { + } else { filters = Lists.newArrayList(filters); } filters.add(getDefaultTimeSpan()); diff --git a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/common/Identifiable.java b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/common/Identifiable.java new file mode 100644 index 000000000..9b5f7bcb9 --- /dev/null +++ b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/common/Identifiable.java @@ -0,0 +1,11 @@ +package com.flipkart.foxtrot.core.common; + +/** + * Created by rishabh.goyal on 05/12/15. + */ + +public interface Identifiable { + + String getId(); + +} diff --git a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/datastore/DataStore.java b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/datastore/DataStore.java index c758613da..9fad73c3c 100644 --- a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/datastore/DataStore.java +++ b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/datastore/DataStore.java @@ -1,12 +1,12 @@ /** * Copyright 2014 Flipkart Internet Pvt. Ltd. - * + *

* Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + *

* http://www.apache.org/licenses/LICENSE-2.0 - * + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -17,6 +17,7 @@ import com.flipkart.foxtrot.common.Document; import com.flipkart.foxtrot.common.Table; +import com.flipkart.foxtrot.core.common.Identifiable; import java.util.List; @@ -25,9 +26,15 @@ * Date: 12/03/14 * Time: 9:17 PM */ -public interface DataStore { +public interface DataStore extends Identifiable { + + public void init(final Table table) throws DataStoreException; + public void save(final Table table, final Document document) throws DataStoreException; + public void save(final Table table, final List documents) throws DataStoreException; + public Document get(final Table table, final String id) throws DataStoreException; + public List get(final Table table, final List ids) throws DataStoreException; } diff --git a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/datastore/DataStoreException.java b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/datastore/DataStoreException.java index 28bab0953..0ae46fe56 100644 --- a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/datastore/DataStoreException.java +++ b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/datastore/DataStoreException.java @@ -1,12 +1,12 @@ /** * Copyright 2014 Flipkart Internet Pvt. Ltd. - * + *

* Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + *

* http://www.apache.org/licenses/LICENSE-2.0 - * + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -23,7 +23,8 @@ * Time: 9:18 PM */ public class DataStoreException extends Exception { - public static enum ErrorCode { + public enum ErrorCode { + TABLE_NOT_FOUND, STORE_CONNECTION, STORE_SINGLE_SAVE, STORE_MULTI_SAVE, diff --git a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/datastore/impl/hbase/HBaseDataStore.java b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/datastore/impl/hbase/HBaseDataStore.java index bddfa45f0..215179cc9 100644 --- a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/datastore/impl/hbase/HBaseDataStore.java +++ b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/datastore/impl/hbase/HBaseDataStore.java @@ -1,12 +1,12 @@ /** * Copyright 2014 Flipkart Internet Pvt. Ltd. - * + *

* Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + *

* http://www.apache.org/licenses/LICENSE-2.0 - * + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -55,6 +55,16 @@ public HBaseDataStore(HbaseTableConnection tableWrapper, ObjectMapper mapper) { this.mapper = mapper; } + @Override + public void init(Table table) throws DataStoreException { + // Check for existence of HBase table during init to make sure HBase is ready for taking writes + HTableInterface tableInterface = tableWrapper.getTable(table); + if (tableInterface == null) { + throw new DataStoreException(DataStoreException.ErrorCode.TABLE_NOT_FOUND, + "HBase Table Missing. Create HBase table"); + } + } + @Override public void save(final Table table, Document document) throws DataStoreException { if (document == null || document.getData() == null || document.getId() == null) { @@ -226,4 +236,9 @@ public Put getPutForDocument(final Table table, Document document) throws JsonPr .add(COLUMN_FAMILY, DATA_FIELD_NAME, mapper.writeValueAsBytes(document.getData())) .add(COLUMN_FAMILY, TIMESTAMP_FIELD_NAME, Bytes.toBytes(document.getTimestamp())); } + + @Override + public String getId() { + return "hbase"; + } } diff --git a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/datastore/impl/hbase/HbaseTableConnection.java b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/datastore/impl/hbase/HbaseTableConnection.java index 54658fd4a..e97dc73ed 100644 --- a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/datastore/impl/hbase/HbaseTableConnection.java +++ b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/datastore/impl/hbase/HbaseTableConnection.java @@ -44,7 +44,6 @@ public HbaseTableConnection(HbaseConfig hbaseConfig) { this.hbaseConfig = hbaseConfig; } - public synchronized HTableInterface getTable(final Table table) throws DataStoreException { try { if (hbaseConfig.isSecure() && UserGroupInformation.isSecurityEnabled()) { diff --git a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/QueryStore.java b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/QueryStore.java index 3a4534d45..8c444c93a 100644 --- a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/QueryStore.java +++ b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/QueryStore.java @@ -1,12 +1,12 @@ /** * Copyright 2014 Flipkart Internet Pvt. Ltd. - * + *

* Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + *

* http://www.apache.org/licenses/LICENSE-2.0 - * + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -17,6 +17,7 @@ import com.flipkart.foxtrot.common.Document; import com.flipkart.foxtrot.common.TableFieldMapping; +import com.flipkart.foxtrot.core.common.Identifiable; import java.util.List; import java.util.Set; @@ -26,21 +27,23 @@ * Date: 12/03/14 * Time: 9:25 PM */ -public interface QueryStore { +public interface QueryStore extends Identifiable { - public void save(final String table, final Document document) throws QueryStoreException; + void init(final String table) throws QueryStoreException; - public void save(final String table, final List documents) throws QueryStoreException; + void save(final String table, final Document document) throws QueryStoreException; - public Document get(final String table, final String id) throws QueryStoreException; + void save(final String table, final List documents) throws QueryStoreException; - public List get(final String table, final List ids) throws QueryStoreException; + Document get(final String table, final String id) throws QueryStoreException; - public TableFieldMapping getFieldMappings(final String table) throws QueryStoreException; + List get(final String table, final List ids) throws QueryStoreException; - public void cleanupAll() throws QueryStoreException; + TableFieldMapping getFieldMappings(final String table) throws QueryStoreException; - public void cleanup(final String table) throws QueryStoreException; + void cleanupAll() throws QueryStoreException; - public void cleanup(final Set tables) throws QueryStoreException; + void cleanup(final String table) throws QueryStoreException; + + void cleanup(final Set tables) throws QueryStoreException; } diff --git a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/QueryStoreException.java b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/QueryStoreException.java index 0968d4cfd..09d1fc4b4 100644 --- a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/QueryStoreException.java +++ b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/QueryStoreException.java @@ -31,7 +31,7 @@ public QueryStoreException(ErrorCode errorCode, String message, Throwable cause) this.errorCode = errorCode; } - public static enum ErrorCode { + public enum ErrorCode { NO_SUCH_TABLE, DOCUMENT_SAVE_ERROR, DOCUMENT_GET_ERROR, @@ -45,7 +45,8 @@ public static enum ErrorCode { INVALID_REQUEST, NO_METADATA_FOUND, TABLE_LIST_FETCH_ERROR, - DATA_CLEANUP_ERROR + DATA_CLEANUP_ERROR, + EXECUTION_ERROR } diff --git a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/CountAction.java b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/CountAction.java index d44e20de4..dfc5faec3 100644 --- a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/CountAction.java +++ b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/CountAction.java @@ -11,7 +11,7 @@ import com.flipkart.foxtrot.core.datastore.DataStore; import com.flipkart.foxtrot.core.querystore.QueryStore; import com.flipkart.foxtrot.core.querystore.QueryStoreException; -import com.flipkart.foxtrot.core.querystore.TableMetadataManager; +import com.flipkart.foxtrot.core.table.TableMetadataManager; import com.flipkart.foxtrot.core.querystore.actions.spi.AnalyticsProvider; import com.flipkart.foxtrot.core.querystore.impl.ElasticsearchConnection; import com.flipkart.foxtrot.core.querystore.impl.ElasticsearchUtils; diff --git a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/DistinctAction.java b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/DistinctAction.java index 55f29738c..3ad7c1138 100644 --- a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/DistinctAction.java +++ b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/DistinctAction.java @@ -11,7 +11,7 @@ import com.flipkart.foxtrot.core.datastore.DataStore; import com.flipkart.foxtrot.core.querystore.QueryStore; import com.flipkart.foxtrot.core.querystore.QueryStoreException; -import com.flipkart.foxtrot.core.querystore.TableMetadataManager; +import com.flipkart.foxtrot.core.table.TableMetadataManager; import com.flipkart.foxtrot.core.querystore.actions.spi.AnalyticsProvider; import com.flipkart.foxtrot.core.querystore.impl.ElasticsearchConnection; import com.flipkart.foxtrot.core.querystore.impl.ElasticsearchUtils; diff --git a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/FilterAction.java b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/FilterAction.java index 7af9f5fa4..9ff12d6b2 100644 --- a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/FilterAction.java +++ b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/FilterAction.java @@ -37,7 +37,7 @@ import com.flipkart.foxtrot.core.datastore.DataStore; import com.flipkart.foxtrot.core.querystore.QueryStore; import com.flipkart.foxtrot.core.querystore.QueryStoreException; -import com.flipkart.foxtrot.core.querystore.TableMetadataManager; +import com.flipkart.foxtrot.core.table.TableMetadataManager; import com.flipkart.foxtrot.core.querystore.actions.spi.AnalyticsProvider; import com.flipkart.foxtrot.core.querystore.impl.ElasticsearchConnection; import com.flipkart.foxtrot.core.querystore.impl.ElasticsearchUtils; diff --git a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/GroupAction.java b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/GroupAction.java index 4d2263adc..e02b5c7ea 100644 --- a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/GroupAction.java +++ b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/GroupAction.java @@ -25,7 +25,7 @@ import com.flipkart.foxtrot.core.datastore.DataStore; import com.flipkart.foxtrot.core.querystore.QueryStore; import com.flipkart.foxtrot.core.querystore.QueryStoreException; -import com.flipkart.foxtrot.core.querystore.TableMetadataManager; +import com.flipkart.foxtrot.core.table.TableMetadataManager; import com.flipkart.foxtrot.core.querystore.actions.spi.AnalyticsProvider; import com.flipkart.foxtrot.core.querystore.impl.ElasticsearchConnection; import com.flipkart.foxtrot.core.querystore.impl.ElasticsearchUtils; diff --git a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/HistogramAction.java b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/HistogramAction.java index 13057409d..c8d889746 100644 --- a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/HistogramAction.java +++ b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/HistogramAction.java @@ -26,7 +26,7 @@ import com.flipkart.foxtrot.core.datastore.DataStore; import com.flipkart.foxtrot.core.querystore.QueryStore; import com.flipkart.foxtrot.core.querystore.QueryStoreException; -import com.flipkart.foxtrot.core.querystore.TableMetadataManager; +import com.flipkart.foxtrot.core.table.TableMetadataManager; import com.flipkart.foxtrot.core.querystore.actions.spi.AnalyticsProvider; import com.flipkart.foxtrot.core.querystore.impl.ElasticsearchConnection; import com.flipkart.foxtrot.core.querystore.impl.ElasticsearchUtils; diff --git a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/StatsAction.java b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/StatsAction.java index 8f2caaca4..c01851238 100644 --- a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/StatsAction.java +++ b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/StatsAction.java @@ -10,7 +10,7 @@ import com.flipkart.foxtrot.core.datastore.DataStore; import com.flipkart.foxtrot.core.querystore.QueryStore; import com.flipkart.foxtrot.core.querystore.QueryStoreException; -import com.flipkart.foxtrot.core.querystore.TableMetadataManager; +import com.flipkart.foxtrot.core.table.TableMetadataManager; import com.flipkart.foxtrot.core.querystore.actions.spi.AnalyticsProvider; import com.flipkart.foxtrot.core.querystore.impl.ElasticsearchConnection; import com.flipkart.foxtrot.core.querystore.impl.ElasticsearchUtils; diff --git a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/StatsTrendAction.java b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/StatsTrendAction.java index 6f787c583..e8c46d3d8 100644 --- a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/StatsTrendAction.java +++ b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/StatsTrendAction.java @@ -11,7 +11,7 @@ import com.flipkart.foxtrot.core.datastore.DataStore; import com.flipkart.foxtrot.core.querystore.QueryStore; import com.flipkart.foxtrot.core.querystore.QueryStoreException; -import com.flipkart.foxtrot.core.querystore.TableMetadataManager; +import com.flipkart.foxtrot.core.table.TableMetadataManager; import com.flipkart.foxtrot.core.querystore.actions.spi.AnalyticsProvider; import com.flipkart.foxtrot.core.querystore.impl.ElasticsearchConnection; import com.flipkart.foxtrot.core.querystore.impl.ElasticsearchUtils; diff --git a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/TrendAction.java b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/TrendAction.java index caf24b9cb..73b54b76a 100644 --- a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/TrendAction.java +++ b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/TrendAction.java @@ -27,7 +27,7 @@ import com.flipkart.foxtrot.core.datastore.DataStore; import com.flipkart.foxtrot.core.querystore.QueryStore; import com.flipkart.foxtrot.core.querystore.QueryStoreException; -import com.flipkart.foxtrot.core.querystore.TableMetadataManager; +import com.flipkart.foxtrot.core.table.TableMetadataManager; import com.flipkart.foxtrot.core.querystore.actions.spi.AnalyticsProvider; import com.flipkart.foxtrot.core.querystore.impl.ElasticsearchConnection; import com.flipkart.foxtrot.core.querystore.impl.ElasticsearchUtils; diff --git a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/spi/AnalyticsLoader.java b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/spi/AnalyticsLoader.java index 8a56558a1..287603f79 100644 --- a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/spi/AnalyticsLoader.java +++ b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/spi/AnalyticsLoader.java @@ -20,7 +20,7 @@ import com.flipkart.foxtrot.core.common.CacheUtils; import com.flipkart.foxtrot.core.datastore.DataStore; import com.flipkart.foxtrot.core.querystore.QueryStore; -import com.flipkart.foxtrot.core.querystore.TableMetadataManager; +import com.flipkart.foxtrot.core.table.TableMetadataManager; import com.flipkart.foxtrot.core.querystore.impl.ElasticsearchConnection; import com.google.common.collect.Maps; diff --git a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/impl/ElasticsearchQueryStore.java b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/impl/ElasticsearchQueryStore.java index 4f33a82b7..a69009ee9 100644 --- a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/impl/ElasticsearchQueryStore.java +++ b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/impl/ElasticsearchQueryStore.java @@ -1,12 +1,12 @@ /** * Copyright 2014 Flipkart Internet Pvt. Ltd. - * + *

* Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + *

* http://www.apache.org/licenses/LICENSE-2.0 - * + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -26,7 +26,7 @@ import com.flipkart.foxtrot.core.parsers.ElasticsearchMappingParser; import com.flipkart.foxtrot.core.querystore.QueryStore; import com.flipkart.foxtrot.core.querystore.QueryStoreException; -import com.flipkart.foxtrot.core.querystore.TableMetadataManager; +import com.flipkart.foxtrot.core.table.TableMetadataManager; import com.google.common.base.Stopwatch; import com.google.common.collect.Lists; import org.elasticsearch.action.WriteConsistencyLevel; @@ -54,21 +54,26 @@ public class ElasticsearchQueryStore implements QueryStore { private static final Logger logger = LoggerFactory.getLogger(ElasticsearchQueryStore.class.getSimpleName()); - private final TableMetadataManager tableMetadataManager; private final ElasticsearchConnection connection; private final DataStore dataStore; + private final TableMetadataManager tableMetadataManager; private final ObjectMapper mapper; public ElasticsearchQueryStore(TableMetadataManager tableMetadataManager, ElasticsearchConnection connection, DataStore dataStore) { - this.tableMetadataManager = tableMetadataManager; this.connection = connection; this.dataStore = dataStore; + this.tableMetadataManager = tableMetadataManager; this.mapper = ElasticsearchUtils.getMapper(); } + @Override + public void init(String table) throws QueryStoreException { + // Nothing needs to be done here since indexes are created at runtime in elasticsearch + } + @Override public void save(String table, Document document) throws QueryStoreException { table = ElasticsearchUtils.getValidTableName(table); @@ -77,7 +82,7 @@ public void save(String table, Document document) throws QueryStoreException { throw new QueryStoreException(QueryStoreException.ErrorCode.NO_SUCH_TABLE, "No table exists with the name: " + table); } - if (new DateTime().plusDays(1).minus(document.getTimestamp()).getMillis() < 0) { + if (new DateTime().plusDays(1).minus(document.getTimestamp()).getMillis() < 0) { return; } dataStore.save(tableMetadataManager.get(table), document); @@ -135,7 +140,7 @@ public void save(String table, List documents) throws QueryStoreExcept for (Document document : documents) { long timestamp = document.getTimestamp(); - if (dateTime.minus(timestamp).getMillis() < 0) { + if (dateTime.minus(timestamp).getMillis() < 0) { continue; } final String index = ElasticsearchUtils.getCurrentIndex(table, timestamp); @@ -147,7 +152,7 @@ public void save(String table, List documents) throws QueryStoreExcept .source(mapper.writeValueAsBytes(document.getData())); bulkRequestBuilder.add(indexRequest); } - if (bulkRequestBuilder.numberOfActions() > 0){ + if (bulkRequestBuilder.numberOfActions() > 0) { Stopwatch stopwatch = new Stopwatch(); stopwatch.start(); BulkResponse responses = bulkRequestBuilder @@ -309,12 +314,12 @@ public void cleanup(Set tables) throws QueryStoreException { logger.warn(String.format("Deleting Indexes - Indexes - %s", indicesToDelete)); if (indicesToDelete.size() > 0) { List> subLists = Lists.partition(indicesToDelete, 5); - for ( List subList : subLists ){ + for (List subList : subLists) { try { connection.getClient().admin().indices().prepareDelete(subList.toArray(new String[subList.size()])) .execute().actionGet(TimeValue.timeValueMinutes(5)); logger.warn(String.format("Deleted Indexes - Indexes - %s", subList)); - } catch (Exception e){ + } catch (Exception e) { logger.error(String.format("Index deletion failed - Indexes - %s", subList), e); } } @@ -325,4 +330,9 @@ public void cleanup(Set tables) throws QueryStoreException { String.format("Unable to delete Indexes - %s", indicesToDelete), ex); } } + + @Override + public String getId() { + return "elasticsearch"; + } } diff --git a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/table/TableManager.java b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/table/TableManager.java new file mode 100644 index 000000000..c511252ba --- /dev/null +++ b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/table/TableManager.java @@ -0,0 +1,22 @@ +package com.flipkart.foxtrot.core.table; + +import com.flipkart.foxtrot.common.Table; + +import java.util.List; + +/** + * Created by rishabh.goyal on 05/12/15. + */ +public interface TableManager { + + void save(Table table) throws TableManagerException; + + Table get(String name) throws TableManagerException; + + List

getAll() throws TableManagerException; + + void update(Table table) throws TableManagerException; + + void delete(String name) throws TableManagerException; + +} diff --git a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/table/TableManagerException.java b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/table/TableManagerException.java new file mode 100644 index 000000000..0692a1b8b --- /dev/null +++ b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/table/TableManagerException.java @@ -0,0 +1,46 @@ +package com.flipkart.foxtrot.core.table; + +import org.apache.commons.lang3.builder.ToStringBuilder; + +/** + * Created by rishabh.goyal on 05/12/15. + */ +public class TableManagerException extends Exception { + + public enum ErrorCode { + BAD_REQUEST, + TABLE_NOT_FOUND, + TABLE_ALREADY_EXISTS, + INTERNAL_ERROR + } + + private final ErrorCode errorCode; + + public TableManagerException(ErrorCode errorCode, String message) { + super(message); + this.errorCode = errorCode; + } + + public TableManagerException(ErrorCode errorCode, Throwable cause) { + super(cause); + this.errorCode = errorCode; + } + + public TableManagerException(ErrorCode errorCode, String message, Throwable cause) { + super(message, cause); + this.errorCode = errorCode; + } + + public ErrorCode getErrorCode() { + return errorCode; + } + + @Override + public String toString() { + return new ToStringBuilder(this) + .appendSuper(super.toString()) + .append("errorCode", errorCode) + .toString(); + } + +} diff --git a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/TableMetadataManager.java b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/table/TableMetadataManager.java similarity index 72% rename from foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/TableMetadataManager.java rename to foxtrot-core/src/main/java/com/flipkart/foxtrot/core/table/TableMetadataManager.java index 94e510e0e..07c1c55af 100644 --- a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/TableMetadataManager.java +++ b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/table/TableMetadataManager.java @@ -1,19 +1,19 @@ /** * Copyright 2014 Flipkart Internet Pvt. Ltd. - * + *

* Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + *

* http://www.apache.org/licenses/LICENSE-2.0 - * + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ -package com.flipkart.foxtrot.core.querystore; +package com.flipkart.foxtrot.core.table; import com.flipkart.foxtrot.common.Table; import com.yammer.dropwizard.lifecycle.Managed; @@ -26,9 +26,14 @@ * Time: 10:08 PM */ public interface TableMetadataManager extends Managed { - public void save(Table table) throws Exception; - public Table get(String tableName) throws Exception; - public List

get() throws Exception; - public boolean exists(String tableName) throws Exception; - public void delete(String tableName) throws Exception; + + void save(Table table) throws Exception; + + Table get(String tableName) throws Exception; + + List
get() throws Exception; + + boolean exists(String tableName) throws Exception; + + void delete(String tableName) throws Exception; } diff --git a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/impl/DistributedTableMetadataManager.java b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/table/impl/DistributedTableMetadataManager.java similarity index 92% rename from foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/impl/DistributedTableMetadataManager.java rename to foxtrot-core/src/main/java/com/flipkart/foxtrot/core/table/impl/DistributedTableMetadataManager.java index 4e295e1d7..77e4e9fbe 100644 --- a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/impl/DistributedTableMetadataManager.java +++ b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/table/impl/DistributedTableMetadataManager.java @@ -1,22 +1,25 @@ /** * Copyright 2014 Flipkart Internet Pvt. Ltd. - * + *

* Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + *

* http://www.apache.org/licenses/LICENSE-2.0 - * + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ -package com.flipkart.foxtrot.core.querystore.impl; +package com.flipkart.foxtrot.core.table.impl; import com.flipkart.foxtrot.common.Table; -import com.flipkart.foxtrot.core.querystore.TableMetadataManager; +import com.flipkart.foxtrot.core.querystore.impl.DistributedCache; +import com.flipkart.foxtrot.core.querystore.impl.ElasticsearchConnection; +import com.flipkart.foxtrot.core.querystore.impl.HazelcastConnection; +import com.flipkart.foxtrot.core.table.TableMetadataManager; import com.google.common.collect.Lists; import com.hazelcast.config.MapConfig; import com.hazelcast.config.MapStoreConfig; diff --git a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/table/impl/FoxtrotTableManager.java b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/table/impl/FoxtrotTableManager.java new file mode 100644 index 000000000..b1e8a8245 --- /dev/null +++ b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/table/impl/FoxtrotTableManager.java @@ -0,0 +1,104 @@ +package com.flipkart.foxtrot.core.table.impl; + +import com.flipkart.foxtrot.common.Table; +import com.flipkart.foxtrot.core.datastore.DataStore; +import com.flipkart.foxtrot.core.datastore.DataStoreException; +import com.flipkart.foxtrot.core.querystore.QueryStore; +import com.flipkart.foxtrot.core.table.TableManager; +import com.flipkart.foxtrot.core.table.TableManagerException; +import com.flipkart.foxtrot.core.table.TableMetadataManager; + +import java.util.List; + +/** + * Created by rishabh.goyal on 05/12/15. + */ +public class FoxtrotTableManager implements TableManager { + + private final TableMetadataManager metadataManager; + private final QueryStore queryStore; + private final DataStore dataStore; + + public FoxtrotTableManager(TableMetadataManager metadataManager, QueryStore queryStore, DataStore dataStore) { + this.metadataManager = metadataManager; + this.queryStore = queryStore; + this.dataStore = dataStore; + } + + + @Override + public void save(Table table) throws TableManagerException { + validateTableParams(table); + try { + if (metadataManager.exists(table.getName())) { + throw new TableManagerException(TableManagerException.ErrorCode.BAD_REQUEST, "table already exists"); + } + queryStore.init(table.getName()); + dataStore.init(table); + metadataManager.save(table); + } catch (DataStoreException e) { + switch (e.getErrorCode()) { + case TABLE_NOT_FOUND: + throw new TableManagerException(TableManagerException.ErrorCode.BAD_REQUEST, e); + case STORE_CONNECTION: + throw new TableManagerException(TableManagerException.ErrorCode.INTERNAL_ERROR, e); + default: + throw new TableManagerException(TableManagerException.ErrorCode.INTERNAL_ERROR, e); + } + } catch (TableManagerException e) { + throw e; + } catch (Exception e) { + throw new TableManagerException(TableManagerException.ErrorCode.INTERNAL_ERROR, e); + } + } + + @Override + public Table get(String name) throws TableManagerException { + try { + Table table = metadataManager.get(name); + if (table == null) { + throw new TableManagerException(TableManagerException.ErrorCode.TABLE_NOT_FOUND, "table_does_not_exists"); + } + return table; + } catch (TableManagerException e) { + throw e; + } catch (Exception e) { + throw new TableManagerException(TableManagerException.ErrorCode.INTERNAL_ERROR, e); + } + } + + @Override + public List

getAll() throws TableManagerException { + try { + return metadataManager.get(); + } catch (Exception e) { + throw new TableManagerException(TableManagerException.ErrorCode.INTERNAL_ERROR, e); + } + } + + @Override + public void update(Table table) throws TableManagerException { + validateTableParams(table); + try { + if (!metadataManager.exists(table.getName())) { + throw new TableManagerException(TableManagerException.ErrorCode.TABLE_NOT_FOUND, "table_does_not_exists"); + } + metadataManager.save(table); + } catch (TableManagerException e) { + throw e; + } catch (Exception e) { + throw new TableManagerException(TableManagerException.ErrorCode.INTERNAL_ERROR, e); + } + } + + @Override + public void delete(String tableName) throws TableManagerException { + // TODO Implement this once downstream implications are figured out + } + + private void validateTableParams(Table table) throws TableManagerException { + if (table == null || table.getName() == null || table.getName().trim().isEmpty() || table.getTtl() <= 0) { + throw new TableManagerException(TableManagerException.ErrorCode.BAD_REQUEST, "Invalid Table Params"); + } + } +} diff --git a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/impl/TableMapStore.java b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/table/impl/TableMapStore.java similarity index 98% rename from foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/impl/TableMapStore.java rename to foxtrot-core/src/main/java/com/flipkart/foxtrot/core/table/impl/TableMapStore.java index 7682abd7d..98a740b2f 100644 --- a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/impl/TableMapStore.java +++ b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/table/impl/TableMapStore.java @@ -13,12 +13,13 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.flipkart.foxtrot.core.querystore.impl; +package com.flipkart.foxtrot.core.table.impl; import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import com.flipkart.foxtrot.common.Table; +import com.flipkart.foxtrot.core.querystore.impl.ElasticsearchConnection; import com.google.common.collect.Maps; import com.google.common.collect.Sets; import com.hazelcast.core.MapStore; diff --git a/foxtrot-core/src/test/java/com/flipkart/foxtrot/core/common/NonCacheableAction.java b/foxtrot-core/src/test/java/com/flipkart/foxtrot/core/common/NonCacheableAction.java index d4bfd9404..933976f1f 100644 --- a/foxtrot-core/src/test/java/com/flipkart/foxtrot/core/common/NonCacheableAction.java +++ b/foxtrot-core/src/test/java/com/flipkart/foxtrot/core/common/NonCacheableAction.java @@ -19,7 +19,7 @@ import com.flipkart.foxtrot.core.datastore.DataStore; import com.flipkart.foxtrot.core.querystore.QueryStore; import com.flipkart.foxtrot.core.querystore.QueryStoreException; -import com.flipkart.foxtrot.core.querystore.TableMetadataManager; +import com.flipkart.foxtrot.core.table.TableMetadataManager; import com.flipkart.foxtrot.core.querystore.actions.spi.AnalyticsProvider; import com.flipkart.foxtrot.core.querystore.impl.ElasticsearchConnection; import com.google.common.annotations.VisibleForTesting; diff --git a/foxtrot-core/src/test/java/com/flipkart/foxtrot/core/querystore/QueryExecutorTest.java b/foxtrot-core/src/test/java/com/flipkart/foxtrot/core/querystore/QueryExecutorTest.java index 387f91f53..ba3f6a1b8 100644 --- a/foxtrot-core/src/test/java/com/flipkart/foxtrot/core/querystore/QueryExecutorTest.java +++ b/foxtrot-core/src/test/java/com/flipkart/foxtrot/core/querystore/QueryExecutorTest.java @@ -29,6 +29,7 @@ import com.flipkart.foxtrot.core.querystore.impl.ElasticsearchConnection; import com.flipkart.foxtrot.core.querystore.impl.ElasticsearchUtils; import com.flipkart.foxtrot.core.querystore.impl.HazelcastConnection; +import com.flipkart.foxtrot.core.table.TableMetadataManager; import com.hazelcast.core.HazelcastInstance; import com.hazelcast.test.TestHazelcastInstanceFactory; import org.junit.After; diff --git a/foxtrot-core/src/test/java/com/flipkart/foxtrot/core/querystore/actions/CountActionTest.java b/foxtrot-core/src/test/java/com/flipkart/foxtrot/core/querystore/actions/CountActionTest.java index 50ef77c8f..2be374356 100644 --- a/foxtrot-core/src/test/java/com/flipkart/foxtrot/core/querystore/actions/CountActionTest.java +++ b/foxtrot-core/src/test/java/com/flipkart/foxtrot/core/querystore/actions/CountActionTest.java @@ -13,7 +13,7 @@ import com.flipkart.foxtrot.core.querystore.QueryExecutor; import com.flipkart.foxtrot.core.querystore.QueryStore; import com.flipkart.foxtrot.core.querystore.QueryStoreException; -import com.flipkart.foxtrot.core.querystore.TableMetadataManager; +import com.flipkart.foxtrot.core.table.TableMetadataManager; import com.flipkart.foxtrot.core.querystore.actions.spi.AnalyticsLoader; import com.flipkart.foxtrot.core.querystore.impl.*; import com.hazelcast.core.HazelcastInstance; diff --git a/foxtrot-core/src/test/java/com/flipkart/foxtrot/core/querystore/actions/DistinctActionTest.java b/foxtrot-core/src/test/java/com/flipkart/foxtrot/core/querystore/actions/DistinctActionTest.java index 9246b3b8e..f662c4109 100644 --- a/foxtrot-core/src/test/java/com/flipkart/foxtrot/core/querystore/actions/DistinctActionTest.java +++ b/foxtrot-core/src/test/java/com/flipkart/foxtrot/core/querystore/actions/DistinctActionTest.java @@ -13,7 +13,7 @@ import com.flipkart.foxtrot.core.querystore.QueryExecutor; import com.flipkart.foxtrot.core.querystore.QueryStore; import com.flipkart.foxtrot.core.querystore.QueryStoreException; -import com.flipkart.foxtrot.core.querystore.TableMetadataManager; +import com.flipkart.foxtrot.core.table.TableMetadataManager; import com.flipkart.foxtrot.core.querystore.actions.spi.AnalyticsLoader; import com.flipkart.foxtrot.core.querystore.impl.*; import com.hazelcast.core.HazelcastInstance; diff --git a/foxtrot-core/src/test/java/com/flipkart/foxtrot/core/querystore/actions/FilterActionTest.java b/foxtrot-core/src/test/java/com/flipkart/foxtrot/core/querystore/actions/FilterActionTest.java index 39c214d3f..190d5df22 100644 --- a/foxtrot-core/src/test/java/com/flipkart/foxtrot/core/querystore/actions/FilterActionTest.java +++ b/foxtrot-core/src/test/java/com/flipkart/foxtrot/core/querystore/actions/FilterActionTest.java @@ -63,7 +63,7 @@ import com.flipkart.foxtrot.core.querystore.QueryExecutor; import com.flipkart.foxtrot.core.querystore.QueryStore; import com.flipkart.foxtrot.core.querystore.QueryStoreException; -import com.flipkart.foxtrot.core.querystore.TableMetadataManager; +import com.flipkart.foxtrot.core.table.TableMetadataManager; import com.flipkart.foxtrot.core.querystore.actions.spi.AnalyticsLoader; import com.flipkart.foxtrot.core.querystore.impl.DistributedCacheFactory; import com.flipkart.foxtrot.core.querystore.impl.ElasticsearchConnection; diff --git a/foxtrot-core/src/test/java/com/flipkart/foxtrot/core/querystore/actions/GroupActionTest.java b/foxtrot-core/src/test/java/com/flipkart/foxtrot/core/querystore/actions/GroupActionTest.java index 66ae45427..00e4ef66d 100644 --- a/foxtrot-core/src/test/java/com/flipkart/foxtrot/core/querystore/actions/GroupActionTest.java +++ b/foxtrot-core/src/test/java/com/flipkart/foxtrot/core/querystore/actions/GroupActionTest.java @@ -30,7 +30,7 @@ import com.flipkart.foxtrot.core.querystore.QueryExecutor; import com.flipkart.foxtrot.core.querystore.QueryStore; import com.flipkart.foxtrot.core.querystore.QueryStoreException; -import com.flipkart.foxtrot.core.querystore.TableMetadataManager; +import com.flipkart.foxtrot.core.table.TableMetadataManager; import com.flipkart.foxtrot.core.querystore.actions.spi.AnalyticsLoader; import com.flipkart.foxtrot.core.querystore.impl.*; import com.google.common.collect.Maps; diff --git a/foxtrot-core/src/test/java/com/flipkart/foxtrot/core/querystore/actions/HistogramActionTest.java b/foxtrot-core/src/test/java/com/flipkart/foxtrot/core/querystore/actions/HistogramActionTest.java index c2acb8a55..42a17350a 100644 --- a/foxtrot-core/src/test/java/com/flipkart/foxtrot/core/querystore/actions/HistogramActionTest.java +++ b/foxtrot-core/src/test/java/com/flipkart/foxtrot/core/querystore/actions/HistogramActionTest.java @@ -32,7 +32,7 @@ import com.flipkart.foxtrot.core.querystore.QueryExecutor; import com.flipkart.foxtrot.core.querystore.QueryStore; import com.flipkart.foxtrot.core.querystore.QueryStoreException; -import com.flipkart.foxtrot.core.querystore.TableMetadataManager; +import com.flipkart.foxtrot.core.table.TableMetadataManager; import com.flipkart.foxtrot.core.querystore.actions.spi.AnalyticsLoader; import com.flipkart.foxtrot.core.querystore.impl.*; import com.google.common.collect.Lists; diff --git a/foxtrot-core/src/test/java/com/flipkart/foxtrot/core/querystore/actions/NonCacheableActionTest.java b/foxtrot-core/src/test/java/com/flipkart/foxtrot/core/querystore/actions/NonCacheableActionTest.java index b5977996c..d62ac7963 100644 --- a/foxtrot-core/src/test/java/com/flipkart/foxtrot/core/querystore/actions/NonCacheableActionTest.java +++ b/foxtrot-core/src/test/java/com/flipkart/foxtrot/core/querystore/actions/NonCacheableActionTest.java @@ -24,7 +24,7 @@ import com.flipkart.foxtrot.core.querystore.QueryExecutor; import com.flipkart.foxtrot.core.querystore.QueryStore; import com.flipkart.foxtrot.core.querystore.QueryStoreException; -import com.flipkart.foxtrot.core.querystore.TableMetadataManager; +import com.flipkart.foxtrot.core.table.TableMetadataManager; import com.flipkart.foxtrot.core.querystore.actions.spi.AnalyticsLoader; import com.flipkart.foxtrot.core.querystore.impl.*; import com.hazelcast.core.HazelcastInstance; diff --git a/foxtrot-core/src/test/java/com/flipkart/foxtrot/core/querystore/actions/TrendActionTest.java b/foxtrot-core/src/test/java/com/flipkart/foxtrot/core/querystore/actions/TrendActionTest.java index e0a1c42ed..df2010a5a 100644 --- a/foxtrot-core/src/test/java/com/flipkart/foxtrot/core/querystore/actions/TrendActionTest.java +++ b/foxtrot-core/src/test/java/com/flipkart/foxtrot/core/querystore/actions/TrendActionTest.java @@ -31,7 +31,7 @@ import com.flipkart.foxtrot.core.datastore.DataStore; import com.flipkart.foxtrot.core.querystore.QueryExecutor; import com.flipkart.foxtrot.core.querystore.QueryStoreException; -import com.flipkart.foxtrot.core.querystore.TableMetadataManager; +import com.flipkart.foxtrot.core.table.TableMetadataManager; import com.flipkart.foxtrot.core.querystore.actions.spi.AnalyticsLoader; import com.flipkart.foxtrot.core.querystore.impl.*; import com.google.common.collect.Lists; diff --git a/foxtrot-core/src/test/java/com/flipkart/foxtrot/core/querystore/impl/DistributedCacheTest.java b/foxtrot-core/src/test/java/com/flipkart/foxtrot/core/querystore/impl/DistributedCacheTest.java index 0f84a1fd4..70eefa4e3 100644 --- a/foxtrot-core/src/test/java/com/flipkart/foxtrot/core/querystore/impl/DistributedCacheTest.java +++ b/foxtrot-core/src/test/java/com/flipkart/foxtrot/core/querystore/impl/DistributedCacheTest.java @@ -22,7 +22,7 @@ import com.flipkart.foxtrot.core.TestUtils; import com.flipkart.foxtrot.core.common.CacheUtils; import com.flipkart.foxtrot.core.querystore.QueryStore; -import com.flipkart.foxtrot.core.querystore.TableMetadataManager; +import com.flipkart.foxtrot.core.table.TableMetadataManager; import com.flipkart.foxtrot.core.querystore.actions.spi.AnalyticsLoader; import com.hazelcast.core.HazelcastInstance; import com.hazelcast.test.TestHazelcastInstanceFactory; diff --git a/foxtrot-core/src/test/java/com/flipkart/foxtrot/core/querystore/impl/DistributedTableMetadataManagerTest.java b/foxtrot-core/src/test/java/com/flipkart/foxtrot/core/querystore/impl/DistributedTableMetadataManagerTest.java index 1fb0fd09f..3027ecba0 100644 --- a/foxtrot-core/src/test/java/com/flipkart/foxtrot/core/querystore/impl/DistributedTableMetadataManagerTest.java +++ b/foxtrot-core/src/test/java/com/flipkart/foxtrot/core/querystore/impl/DistributedTableMetadataManagerTest.java @@ -20,6 +20,8 @@ import com.flipkart.foxtrot.common.group.GroupResponse; import com.flipkart.foxtrot.core.MockElasticsearchServer; import com.flipkart.foxtrot.core.TestUtils; +import com.flipkart.foxtrot.core.table.impl.DistributedTableMetadataManager; +import com.flipkart.foxtrot.core.table.impl.TableMapStore; import com.hazelcast.config.Config; import com.hazelcast.core.HazelcastInstance; import com.hazelcast.core.IMap; diff --git a/foxtrot-core/src/test/java/com/flipkart/foxtrot/core/querystore/impl/ElasticsearchQueryStoreTest.java b/foxtrot-core/src/test/java/com/flipkart/foxtrot/core/querystore/impl/ElasticsearchQueryStoreTest.java index 5b93f4b1a..8fe94a0a5 100644 --- a/foxtrot-core/src/test/java/com/flipkart/foxtrot/core/querystore/impl/ElasticsearchQueryStoreTest.java +++ b/foxtrot-core/src/test/java/com/flipkart/foxtrot/core/querystore/impl/ElasticsearchQueryStoreTest.java @@ -26,7 +26,7 @@ import com.flipkart.foxtrot.core.datastore.DataStore; import com.flipkart.foxtrot.core.querystore.QueryExecutor; import com.flipkart.foxtrot.core.querystore.QueryStoreException; -import com.flipkart.foxtrot.core.querystore.TableMetadataManager; +import com.flipkart.foxtrot.core.table.TableMetadataManager; import com.flipkart.foxtrot.core.querystore.actions.spi.AnalyticsLoader; import org.elasticsearch.action.get.GetResponse; import org.junit.After; diff --git a/foxtrot-core/src/test/java/com/flipkart/foxtrot/core/querystore/impl/TableMapStoreTest.java b/foxtrot-core/src/test/java/com/flipkart/foxtrot/core/querystore/impl/TableMapStoreTest.java index d22134792..76e6f1a93 100644 --- a/foxtrot-core/src/test/java/com/flipkart/foxtrot/core/querystore/impl/TableMapStoreTest.java +++ b/foxtrot-core/src/test/java/com/flipkart/foxtrot/core/querystore/impl/TableMapStoreTest.java @@ -18,6 +18,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.flipkart.foxtrot.common.Table; import com.flipkart.foxtrot.core.MockElasticsearchServer; +import com.flipkart.foxtrot.core.table.impl.TableMapStore; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Iterables; import com.google.common.collect.Maps; diff --git a/foxtrot-core/src/test/java/com/flipkart/foxtrot/core/table/impl/FoxtrotTableManagerTest.java b/foxtrot-core/src/test/java/com/flipkart/foxtrot/core/table/impl/FoxtrotTableManagerTest.java new file mode 100644 index 000000000..eca1b8c8b --- /dev/null +++ b/foxtrot-core/src/test/java/com/flipkart/foxtrot/core/table/impl/FoxtrotTableManagerTest.java @@ -0,0 +1,268 @@ +package com.flipkart.foxtrot.core.table.impl; + +import com.flipkart.foxtrot.common.Table; +import com.flipkart.foxtrot.core.datastore.DataStore; +import com.flipkart.foxtrot.core.datastore.DataStoreException; +import com.flipkart.foxtrot.core.querystore.QueryStore; +import com.flipkart.foxtrot.core.querystore.QueryStoreException; +import com.flipkart.foxtrot.core.table.TableManager; +import com.flipkart.foxtrot.core.table.TableManagerException; +import com.flipkart.foxtrot.core.table.TableMetadataManager; +import org.junit.Before; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; +import static org.mockito.Matchers.any; +import static org.mockito.Mockito.*; + +/** + * Created by rishabh.goyal on 05/12/15. + */ +public class FoxtrotTableManagerTest { + + private TableManager tableManager; + private QueryStore queryStore; + private DataStore dataStore; + private TableMetadataManager metadataManager; + + + @Before + public void setUp() throws Exception { + this.queryStore = mock(QueryStore.class); + this.dataStore = mock(DataStore.class); + this.metadataManager = mock(TableMetadataManager.class); + this.tableManager = new FoxtrotTableManager(metadataManager, queryStore, dataStore); + } + + @Test + public void saveTableNullName() throws Exception { + doNothing().when(metadataManager).save(any(Table.class)); + doNothing().when(dataStore).init(any(Table.class)); + doNothing().when(queryStore).init(any(String.class)); + try { + Table table = new Table(); + table.setName(null); + table.setTtl(60); + tableManager.save(table); + fail(); + } catch (TableManagerException e) { + assertEquals(TableManagerException.ErrorCode.BAD_REQUEST, e.getErrorCode()); + } + } + + @Test + public void saveTableEmptyName() throws Exception { + doNothing().when(metadataManager).save(any(Table.class)); + doNothing().when(dataStore).init(any(Table.class)); + doNothing().when(queryStore).init(any(String.class)); + try { + Table table = new Table(); + table.setName(" "); + table.setTtl(60); + tableManager.save(table); + fail(); + } catch (TableManagerException e) { + assertEquals(TableManagerException.ErrorCode.BAD_REQUEST, e.getErrorCode()); + } + } + + @Test + public void saveNullTable() throws Exception { + doNothing().when(metadataManager).save(any(Table.class)); + doNothing().when(dataStore).init(any(Table.class)); + doNothing().when(queryStore).init(any(String.class)); + try { + tableManager.save(null); + fail(); + } catch (TableManagerException e) { + assertEquals(TableManagerException.ErrorCode.BAD_REQUEST, e.getErrorCode()); + } + } + + @Test + public void saveTableInvalidTtl() throws Exception { + doNothing().when(metadataManager).save(any(Table.class)); + doNothing().when(dataStore).init(any(Table.class)); + doNothing().when(queryStore).init(any(String.class)); + try { + Table table = new Table(); + table.setName("abcd"); + table.setTtl(0); + tableManager.save(table); + fail(); + } catch (TableManagerException e) { + assertEquals(TableManagerException.ErrorCode.BAD_REQUEST, e.getErrorCode()); + } + } + + @Test + public void saveTable() throws Exception { + doNothing().when(metadataManager).save(any(Table.class)); + doNothing().when(dataStore).init(any(Table.class)); + doNothing().when(queryStore).init(any(String.class)); + Table table = new Table(); + table.setName("abcd"); + table.setTtl(10); + tableManager.save(table); + } + + @Test + public void saveExistingTable() throws Exception { + doNothing().when(metadataManager).save(any(Table.class)); + doNothing().when(dataStore).init(any(Table.class)); + doNothing().when(queryStore).init(any(String.class)); + doReturn(true).when(metadataManager).exists(any(String.class)); + try { + Table table = new Table(); + table.setName("abcd"); + table.setTtl(10); + tableManager.save(table); + fail(); + } catch (TableManagerException e) { + assertEquals(TableManagerException.ErrorCode.BAD_REQUEST, e.getErrorCode()); + } + } + + @Test + public void saveTableQueryStoreFailed() throws Exception { + doNothing().when(metadataManager).save(any(Table.class)); + doNothing().when(dataStore).init(any(Table.class)); + doThrow(new QueryStoreException(QueryStoreException.ErrorCode.EXECUTION_ERROR, "Dummy")).when(queryStore).init(any(String.class)); + try { + Table table = new Table(); + table.setName("abcd"); + table.setTtl(10); + tableManager.save(table); + fail(); + } catch (TableManagerException e) { + assertEquals(e.getErrorCode(), TableManagerException.ErrorCode.INTERNAL_ERROR); + } + } + + @Test + public void saveTableDataStoreFailed() throws Exception { + doNothing().when(metadataManager).save(any(Table.class)); + doThrow(new DataStoreException(DataStoreException.ErrorCode.STORE_CONNECTION, "Dummy")).when(dataStore).init(any(Table.class)); + doNothing().when(queryStore).init(any(String.class)); + try { + Table table = new Table(); + table.setName("abcd"); + table.setTtl(10); + tableManager.save(table); + fail(); + } catch (TableManagerException e) { + assertEquals(e.getErrorCode(), TableManagerException.ErrorCode.INTERNAL_ERROR); + } + } + + @Test + public void saveTableDataStoreNoTableFound() throws Exception { + doNothing().when(metadataManager).save(any(Table.class)); + doThrow(new DataStoreException(DataStoreException.ErrorCode.TABLE_NOT_FOUND, "Dummy")).when(dataStore).init(any(Table.class)); + doNothing().when(queryStore).init(any(String.class)); + try { + Table table = new Table(); + table.setName("abcd"); + table.setTtl(10); + tableManager.save(table); + fail(); + } catch (TableManagerException e) { + assertEquals(TableManagerException.ErrorCode.BAD_REQUEST, e.getErrorCode()); + } + } + + @Test + public void updateTable() throws Exception { + doNothing().when(metadataManager).save(any(Table.class)); + doNothing().when(dataStore).init(any(Table.class)); + doNothing().when(queryStore).init(any(String.class)); + doReturn(true).when(metadataManager).exists(anyString()); + Table table = new Table(); + table.setName("abcd"); + table.setTtl(10); + tableManager.update(table); + } + + @Test + public void updateNonExistingTable() throws Exception { + doNothing().when(metadataManager).save(any(Table.class)); + doNothing().when(dataStore).init(any(Table.class)); + doNothing().when(queryStore).init(any(String.class)); + doReturn(false).when(metadataManager).exists(anyString()); + try { + Table table = new Table(); + table.setName("abcd"); + table.setTtl(10); + tableManager.update(table); + fail(); + } catch (TableManagerException e) { + assertEquals(TableManagerException.ErrorCode.TABLE_NOT_FOUND, e.getErrorCode()); + } + } + + @Test + public void updateNullTable() throws Exception { + doNothing().when(metadataManager).save(any(Table.class)); + doNothing().when(dataStore).init(any(Table.class)); + doNothing().when(queryStore).init(any(String.class)); + doReturn(false).when(metadataManager).exists(anyString()); + try { + tableManager.update(null); + fail(); + } catch (TableManagerException e) { + assertEquals(TableManagerException.ErrorCode.BAD_REQUEST, e.getErrorCode()); + } + } + + @Test + public void updateTableNullName() throws Exception { + doNothing().when(metadataManager).save(any(Table.class)); + doNothing().when(dataStore).init(any(Table.class)); + doNothing().when(queryStore).init(any(String.class)); + doReturn(false).when(metadataManager).exists(anyString()); + try { + Table table = new Table(); + table.setName(null); + table.setTtl(10); + tableManager.update(table); + fail(); + } catch (TableManagerException e) { + assertEquals(TableManagerException.ErrorCode.BAD_REQUEST, e.getErrorCode()); + } + } + + @Test + public void updateTableEmptyName() throws Exception { + doNothing().when(metadataManager).save(any(Table.class)); + doNothing().when(dataStore).init(any(Table.class)); + doNothing().when(queryStore).init(any(String.class)); + doReturn(false).when(metadataManager).exists(anyString()); + try { + Table table = new Table(); + table.setName(" "); + table.setTtl(10); + tableManager.update(table); + fail(); + } catch (TableManagerException e) { + assertEquals(TableManagerException.ErrorCode.BAD_REQUEST, e.getErrorCode()); + } + } + + @Test + public void updateTableInvalidTtl() throws Exception { + doNothing().when(metadataManager).save(any(Table.class)); + doNothing().when(dataStore).init(any(Table.class)); + doNothing().when(queryStore).init(any(String.class)); + doReturn(false).when(metadataManager).exists(anyString()); + try { + Table table = new Table(); + table.setName("abcd"); + table.setTtl(0); + tableManager.update(table); + fail(); + } catch (TableManagerException e) { + assertEquals(TableManagerException.ErrorCode.BAD_REQUEST, e.getErrorCode()); + } + } +} \ No newline at end of file diff --git a/foxtrot-server/pom.xml b/foxtrot-server/pom.xml index 4da7d6501..67f22cfcc 100644 --- a/foxtrot-server/pom.xml +++ b/foxtrot-server/pom.xml @@ -5,7 +5,7 @@ foxtrot com.flipkart.foxtrot - 0.1 + 0.3-SNAPSHOT 4.0.0 @@ -30,17 +30,17 @@ com.flipkart.foxtrot foxtrot-core - 0.2 + ${project.parent.version} com.flipkart.foxtrot foxtrot-sql - 0.1 + ${project.parent.version} com.flipkart.foxtrot foxtrot-core - 0.2 + ${project.parent.version} test-jar test diff --git a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/FoxtrotServer.java b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/FoxtrotServer.java index e990b5b90..38ab1e23d 100644 --- a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/FoxtrotServer.java +++ b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/FoxtrotServer.java @@ -1,12 +1,12 @@ /** * Copyright 2014 Flipkart Internet Pvt. Ltd. - * + *

* Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + *

* http://www.apache.org/licenses/LICENSE-2.0 - * + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -27,9 +27,11 @@ import com.flipkart.foxtrot.core.datastore.impl.hbase.HbaseTableConnection; import com.flipkart.foxtrot.core.querystore.QueryExecutor; import com.flipkart.foxtrot.core.querystore.QueryStore; -import com.flipkart.foxtrot.core.querystore.TableMetadataManager; import com.flipkart.foxtrot.core.querystore.actions.spi.AnalyticsLoader; import com.flipkart.foxtrot.core.querystore.impl.*; +import com.flipkart.foxtrot.core.table.TableMetadataManager; +import com.flipkart.foxtrot.core.table.impl.DistributedTableMetadataManager; +import com.flipkart.foxtrot.core.table.impl.FoxtrotTableManager; import com.flipkart.foxtrot.server.cluster.ClusterManager; import com.flipkart.foxtrot.server.config.FoxtrotServerConfiguration; import com.flipkart.foxtrot.server.console.ElasticsearchConsolePersistence; @@ -89,11 +91,11 @@ public void run(FoxtrotServerConfiguration configuration, Environment environmen ElasticsearchUtils.setMapper(objectMapper); + TableMetadataManager tableMetadataManager = new DistributedTableMetadataManager(hazelcastConnection, + elasticsearchConnection); DataStore dataStore = new HBaseDataStore(HBaseTableConnection, objectMapper); - - - TableMetadataManager tableMetadataManager = new DistributedTableMetadataManager(hazelcastConnection, elasticsearchConnection); QueryStore queryStore = new ElasticsearchQueryStore(tableMetadataManager, elasticsearchConnection, dataStore); + FoxtrotTableManager tableManager = new FoxtrotTableManager(tableMetadataManager, queryStore, dataStore); AnalyticsLoader analyticsLoader = new AnalyticsLoader(tableMetadataManager, dataStore, queryStore, elasticsearchConnection); @@ -106,7 +108,7 @@ public void run(FoxtrotServerConfiguration configuration, Environment environmen healthChecks.add(new ElasticSearchHealthCheck("ES Health Check", elasticsearchConnection)); ClusterManager clusterManager = new ClusterManager( - hazelcastConnection, healthChecks, configuration.getHttpConfiguration().getPort()); + hazelcastConnection, healthChecks, configuration.getHttpConfiguration().getPort()); environment.manage(HBaseTableConnection); environment.manage(elasticsearchConnection); @@ -119,7 +121,7 @@ public void run(FoxtrotServerConfiguration configuration, Environment environmen environment.addResource(new DocumentResource(queryStore)); environment.addResource(new AsyncResource()); environment.addResource(new AnalyticsResource(executor)); - environment.addResource(new TableMetadataResource(tableMetadataManager)); + environment.addResource(new TableManagerResource(tableManager)); environment.addResource(new TableFieldMappingResource(queryStore)); environment.addResource(new ConsoleResource( new ElasticsearchConsolePersistence(elasticsearchConnection, objectMapper))); @@ -128,7 +130,7 @@ public void run(FoxtrotServerConfiguration configuration, Environment environmen environment.addResource(new ClusterInfoResource(clusterManager)); environment.addResource(new UtilResource(configuration)); - for(HealthCheck healthCheck : healthChecks) { + for (HealthCheck healthCheck : healthChecks) { environment.addHealthCheck(healthCheck); } diff --git a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/resources/DocumentResource.java b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/resources/DocumentResource.java index 3ef29580a..cdc42bc95 100644 --- a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/resources/DocumentResource.java +++ b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/resources/DocumentResource.java @@ -1,12 +1,12 @@ /** * Copyright 2014 Flipkart Internet Pvt. Ltd. - * + *

* Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + *

* http://www.apache.org/licenses/LICENSE-2.0 - * + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/resources/TableManagerResource.java b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/resources/TableManagerResource.java new file mode 100644 index 000000000..b698639b4 --- /dev/null +++ b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/resources/TableManagerResource.java @@ -0,0 +1,106 @@ +/** + * Copyright 2014 Flipkart Internet Pvt. Ltd. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.flipkart.foxtrot.server.resources; + +import com.flipkart.foxtrot.common.Table; +import com.flipkart.foxtrot.core.table.TableManager; +import com.flipkart.foxtrot.core.table.TableManagerException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.validation.Valid; +import javax.ws.rs.*; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; +import java.util.Collections; + +@Path("/v1/tables") +@Consumes(MediaType.APPLICATION_JSON) +@Produces(MediaType.APPLICATION_JSON) +public class TableManagerResource { + private static final Logger logger = LoggerFactory.getLogger(TableManagerResource.class); + private final TableManager tableManager; + + public TableManagerResource(TableManager tableManager) { + this.tableManager = tableManager; + } + + @POST + public Response save(@Valid final Table table) { + try { + tableManager.save(table); + return Response.ok(table).build(); + } catch (TableManagerException e) { + logger.error(String.format("Unable to save table %s", table), e); + switch (e.getErrorCode()) { + case TABLE_ALREADY_EXISTS: + return Response.status(422).entity(Collections.singletonMap("error", e.getMessage())).build(); + case INTERNAL_ERROR: + return Response.status(Response.Status.INTERNAL_SERVER_ERROR) + .entity(Collections.singletonMap("error", e.getMessage())).build(); + default: + return Response.status(Response.Status.INTERNAL_SERVER_ERROR) + .entity(Collections.singletonMap("error", e.getMessage())).build(); + } + } + } + + @GET + @Path("/{name}") + public Response get(@PathParam("name") final String name) { + try { + Table table = tableManager.get(name); + return Response.ok().entity(table).build(); + } catch (TableManagerException e) { + logger.error(String.format("Unable to fetch table %s", name), e); + switch (e.getErrorCode()) { + case TABLE_NOT_FOUND: + return Response.status(Response.Status.NOT_FOUND) + .entity(Collections.singletonMap("error", e.getMessage())).build(); + case INTERNAL_ERROR: + return Response.status(Response.Status.INTERNAL_SERVER_ERROR) + .entity(Collections.singletonMap("error", e.getMessage())).build(); + default: + return Response.status(Response.Status.INTERNAL_SERVER_ERROR) + .entity(Collections.singletonMap("error", e.getMessage())).build(); + } + } + } + + @DELETE + @Path("/{name}/delete") + public Response delete(@PathParam("name") final String name) { + try { + tableManager.delete(name); + return Response.status(Response.Status.NO_CONTENT).build(); + } catch (Exception e) { + logger.error(String.format("Unable to delete table %s", name), e); + return Response.status(Response.Status.INTERNAL_SERVER_ERROR) + .entity(Collections.singletonMap("error", e.getMessage())).build(); + } + } + + @GET + public Response getAll() { + try { + return Response.ok().entity(tableManager.getAll()).build(); + } catch (TableManagerException e) { + logger.error("Unable to fetch table list", e); + return Response.status(Response.Status.INTERNAL_SERVER_ERROR) + .entity(Collections.singletonMap("error", e.getMessage())).build(); + } + } +} diff --git a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/resources/TableMetadataResource.java b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/resources/TableMetadataResource.java deleted file mode 100644 index 6f05c2945..000000000 --- a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/resources/TableMetadataResource.java +++ /dev/null @@ -1,78 +0,0 @@ -/** - * Copyright 2014 Flipkart Internet Pvt. Ltd. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.flipkart.foxtrot.server.resources; - -import com.flipkart.foxtrot.common.Table; -import com.flipkart.foxtrot.core.querystore.TableMetadataManager; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import javax.validation.Valid; -import javax.ws.rs.*; -import javax.ws.rs.core.MediaType; -import javax.ws.rs.core.Response; -import java.util.Collections; -import java.util.List; - -@Path("/v1/tables") -@Consumes(MediaType.APPLICATION_JSON) -@Produces(MediaType.APPLICATION_JSON) -public class TableMetadataResource { - private static final Logger logger = LoggerFactory.getLogger(TableMetadataResource.class); - private final TableMetadataManager tableMetadataManager; - - public TableMetadataResource(TableMetadataManager tableMetadataManager) { - this.tableMetadataManager = tableMetadataManager; - } - - @POST - public Table save(@Valid final Table table) { - try { - tableMetadataManager.save(table); - } catch (Exception e) { - logger.error(String.format("Unable to save table %s", table), e); - throw new WebApplicationException(Response.serverError().entity(Collections.singletonMap("error", e.getMessage())).build()); - } - return table; - } - - @GET - @Path("/{name}") - public Table get(@PathParam("name") final String name) throws Exception { - Table table = tableMetadataManager.get(name); - if (table == null) { - throw new WebApplicationException(Response.status(Response.Status.NOT_FOUND).build()); - } - return table; - } - - @DELETE - @Path("/{name}/delete") - public void delete(@PathParam("name") final String name) throws Exception { - try { - tableMetadataManager.delete(name); - } catch (Exception ex){ - throw new WebApplicationException(Response.serverError() - .entity(Collections.singletonMap("error", ex.getMessage())) - .build()); - } - } - - @GET - public List

get() throws Exception { - return tableMetadataManager.get(); - } -} diff --git a/foxtrot-server/src/test/java/com/flipkart/foxtrot/server/resources/AnalyticsResourceTest.java b/foxtrot-server/src/test/java/com/flipkart/foxtrot/server/resources/AnalyticsResourceTest.java index 12098e24d..6da18f350 100644 --- a/foxtrot-server/src/test/java/com/flipkart/foxtrot/server/resources/AnalyticsResourceTest.java +++ b/foxtrot-server/src/test/java/com/flipkart/foxtrot/server/resources/AnalyticsResourceTest.java @@ -29,9 +29,10 @@ import com.flipkart.foxtrot.core.datastore.DataStore; import com.flipkart.foxtrot.core.querystore.QueryExecutor; import com.flipkart.foxtrot.core.querystore.QueryStore; -import com.flipkart.foxtrot.core.querystore.TableMetadataManager; +import com.flipkart.foxtrot.core.table.TableMetadataManager; import com.flipkart.foxtrot.core.querystore.actions.spi.AnalyticsLoader; import com.flipkart.foxtrot.core.querystore.impl.*; +import com.flipkart.foxtrot.core.table.impl.TableMapStore; import com.hazelcast.core.HazelcastInstance; import com.hazelcast.test.TestHazelcastInstanceFactory; import com.sun.jersey.api.client.UniformInterfaceException; diff --git a/foxtrot-server/src/test/java/com/flipkart/foxtrot/server/resources/AsyncResourceTest.java b/foxtrot-server/src/test/java/com/flipkart/foxtrot/server/resources/AsyncResourceTest.java index c98ac5539..4dfbea63b 100644 --- a/foxtrot-server/src/test/java/com/flipkart/foxtrot/server/resources/AsyncResourceTest.java +++ b/foxtrot-server/src/test/java/com/flipkart/foxtrot/server/resources/AsyncResourceTest.java @@ -29,9 +29,10 @@ import com.flipkart.foxtrot.core.datastore.DataStore; import com.flipkart.foxtrot.core.querystore.QueryExecutor; import com.flipkart.foxtrot.core.querystore.QueryStore; -import com.flipkart.foxtrot.core.querystore.TableMetadataManager; +import com.flipkart.foxtrot.core.table.TableMetadataManager; import com.flipkart.foxtrot.core.querystore.actions.spi.AnalyticsLoader; import com.flipkart.foxtrot.core.querystore.impl.*; +import com.flipkart.foxtrot.core.table.impl.TableMapStore; import com.hazelcast.core.HazelcastInstance; import com.hazelcast.test.TestHazelcastInstanceFactory; import com.sun.jersey.api.client.UniformInterfaceException; diff --git a/foxtrot-server/src/test/java/com/flipkart/foxtrot/server/resources/DocumentResourceTest.java b/foxtrot-server/src/test/java/com/flipkart/foxtrot/server/resources/DocumentResourceTest.java index 6bb5d8dfd..a076f2013 100644 --- a/foxtrot-server/src/test/java/com/flipkart/foxtrot/server/resources/DocumentResourceTest.java +++ b/foxtrot-server/src/test/java/com/flipkart/foxtrot/server/resources/DocumentResourceTest.java @@ -25,9 +25,10 @@ import com.flipkart.foxtrot.core.datastore.DataStore; import com.flipkart.foxtrot.core.querystore.QueryStore; import com.flipkart.foxtrot.core.querystore.QueryStoreException; -import com.flipkart.foxtrot.core.querystore.TableMetadataManager; +import com.flipkart.foxtrot.core.table.TableMetadataManager; import com.flipkart.foxtrot.core.querystore.actions.spi.AnalyticsLoader; import com.flipkart.foxtrot.core.querystore.impl.*; +import com.flipkart.foxtrot.core.table.impl.TableMapStore; import com.hazelcast.core.HazelcastInstance; import com.hazelcast.test.TestHazelcastInstanceFactory; import com.sun.jersey.api.client.UniformInterfaceException; diff --git a/foxtrot-server/src/test/java/com/flipkart/foxtrot/server/resources/TableFieldMappingResourceTest.java b/foxtrot-server/src/test/java/com/flipkart/foxtrot/server/resources/TableFieldMappingResourceTest.java index 37207056e..b89873d8f 100644 --- a/foxtrot-server/src/test/java/com/flipkart/foxtrot/server/resources/TableFieldMappingResourceTest.java +++ b/foxtrot-server/src/test/java/com/flipkart/foxtrot/server/resources/TableFieldMappingResourceTest.java @@ -25,9 +25,10 @@ import com.flipkart.foxtrot.core.datastore.DataStore; import com.flipkart.foxtrot.core.querystore.QueryExecutor; import com.flipkart.foxtrot.core.querystore.QueryStore; -import com.flipkart.foxtrot.core.querystore.TableMetadataManager; +import com.flipkart.foxtrot.core.table.TableMetadataManager; import com.flipkart.foxtrot.core.querystore.actions.spi.AnalyticsLoader; import com.flipkart.foxtrot.core.querystore.impl.*; +import com.flipkart.foxtrot.core.table.impl.TableMapStore; import com.hazelcast.core.HazelcastInstance; import com.hazelcast.test.TestHazelcastInstanceFactory; import com.sun.jersey.api.client.UniformInterfaceException; diff --git a/foxtrot-server/src/test/java/com/flipkart/foxtrot/server/resources/TableMetadataResourceTest.java b/foxtrot-server/src/test/java/com/flipkart/foxtrot/server/resources/TableManagerResourceTest.java similarity index 77% rename from foxtrot-server/src/test/java/com/flipkart/foxtrot/server/resources/TableMetadataResourceTest.java rename to foxtrot-server/src/test/java/com/flipkart/foxtrot/server/resources/TableManagerResourceTest.java index 6cf4f3ff3..3150c3c60 100644 --- a/foxtrot-server/src/test/java/com/flipkart/foxtrot/server/resources/TableMetadataResourceTest.java +++ b/foxtrot-server/src/test/java/com/flipkart/foxtrot/server/resources/TableManagerResourceTest.java @@ -1,12 +1,12 @@ /** * Copyright 2014 Flipkart Internet Pvt. Ltd. - * + *

* Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + *

* http://www.apache.org/licenses/LICENSE-2.0 - * + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -20,8 +20,18 @@ import com.flipkart.foxtrot.core.MockElasticsearchServer; import com.flipkart.foxtrot.core.TestUtils; import com.flipkart.foxtrot.core.common.CacheUtils; -import com.flipkart.foxtrot.core.querystore.TableMetadataManager; -import com.flipkart.foxtrot.core.querystore.impl.*; +import com.flipkart.foxtrot.core.datastore.DataStore; +import com.flipkart.foxtrot.core.querystore.QueryStore; +import com.flipkart.foxtrot.core.querystore.impl.DistributedCacheFactory; +import com.flipkart.foxtrot.core.querystore.impl.ElasticsearchConnection; +import com.flipkart.foxtrot.core.querystore.impl.ElasticsearchUtils; +import com.flipkart.foxtrot.core.querystore.impl.HazelcastConnection; +import com.flipkart.foxtrot.core.table.TableManager; +import com.flipkart.foxtrot.core.table.TableManagerException; +import com.flipkart.foxtrot.core.table.TableMetadataManager; +import com.flipkart.foxtrot.core.table.impl.DistributedTableMetadataManager; +import com.flipkart.foxtrot.core.table.impl.FoxtrotTableManager; +import com.flipkart.foxtrot.core.table.impl.TableMapStore; import com.hazelcast.config.Config; import com.hazelcast.core.HazelcastInstance; import com.hazelcast.test.TestHazelcastInstanceFactory; @@ -47,13 +57,13 @@ /** * Created by rishabh.goyal on 04/05/14. */ -public class TableMetadataResourceTest extends ResourceTest { +public class TableManagerResourceTest extends ResourceTest { - private TableMetadataManager tableMetadataManager; private MockElasticsearchServer elasticsearchServer; private HazelcastInstance hazelcastInstance; + private TableManager tableManager; - public TableMetadataResourceTest() throws Exception { + public TableManagerResourceTest() throws Exception { ObjectMapper mapper = new ObjectMapper(); ElasticsearchUtils.setMapper(mapper); @@ -74,14 +84,19 @@ public TableMetadataResourceTest() throws Exception { elasticsearchServer.getClient().admin().indices().create(createRequest).actionGet(); elasticsearchServer.getClient().admin().cluster().prepareHealth().setWaitForGreenStatus().execute().actionGet(); - tableMetadataManager = new DistributedTableMetadataManager(hazelcastConnection, elasticsearchConnection); + TableMetadataManager tableMetadataManager = new DistributedTableMetadataManager(hazelcastConnection, elasticsearchConnection); tableMetadataManager = spy(tableMetadataManager); tableMetadataManager.start(); + + QueryStore queryStore = Mockito.mock(QueryStore.class); + DataStore dataStore = Mockito.mock(DataStore.class); + this.tableManager = new FoxtrotTableManager(tableMetadataManager, queryStore, dataStore); + this.tableManager = spy(tableManager); } @Override protected void setUpResources() throws Exception { - addResource(new TableMetadataResource(tableMetadataManager)); + addResource(new TableManagerResource(tableManager)); } @@ -89,7 +104,6 @@ protected void setUpResources() throws Exception { public void tearDown() throws Exception { elasticsearchServer.shutdown(); hazelcastInstance.shutdown(); - tableMetadataManager.stop(); } @@ -98,7 +112,7 @@ public void testSave() throws Exception { Table table = new Table(TestUtils.TEST_TABLE_NAME, 30); client().resource("/v1/tables").type(MediaType.APPLICATION_JSON_TYPE).post(table); - Table response = tableMetadataManager.get(table.getName()); + Table response = tableManager.get(table.getName()); assertNotNull(response); assertEquals(table.getName(), response.getName()); assertEquals(table.getTtl(), response.getTtl()); @@ -119,7 +133,7 @@ public void testSaveNullTableName() throws Exception { @Test public void testSaveBackendError() throws Exception { Table table = new Table(UUID.randomUUID().toString(), 30); - doThrow(new Exception()).when(tableMetadataManager).save(Matchers.

any()); + doThrow(new TableManagerException(TableManagerException.ErrorCode.INTERNAL_ERROR, "Dummy Message")).when(tableManager).save(Matchers.
any()); try { client().resource("/v1/tables").type(MediaType.APPLICATION_JSON_TYPE).post(table); } catch (UniformInterfaceException ex) { @@ -137,7 +151,7 @@ public void testSaveIllegalTtl() throws Exception { @Test public void testGet() throws Exception { Table table = new Table(TestUtils.TEST_TABLE_NAME, 30); - tableMetadataManager.save(table); + tableManager.save(table); Table response = client().resource(String.format("/v1/tables/%s", table.getName())).get(Table.class); assertNotNull(response); diff --git a/foxtrot-sql/pom.xml b/foxtrot-sql/pom.xml index 4b1264ae2..f3413b0fd 100644 --- a/foxtrot-sql/pom.xml +++ b/foxtrot-sql/pom.xml @@ -5,12 +5,11 @@ foxtrot com.flipkart.foxtrot - 0.1 + 0.3-SNAPSHOT 4.0.0foxtrot-sql - 0.1