diff --git a/README.md b/README.md new file mode 100644 index 0000000000..0394f41a23 --- /dev/null +++ b/README.md @@ -0,0 +1,18 @@ +Kyligence Gateway +===== +### Quickly Manual +1. ${gateway.home}/conf/gateway.properties + a. `kylin.gateway.datasource.type=file` is default. + b. `kylin.gateway.datasource.route-table-file-path=${gateway.home}/conf/route_table` config route table. + +2. route_table file + a. A GLOBAL route: {"id":111, "backends":["www.baidu.com", "wwww.google.com"], "resourceGroup":"default", "type":"GLOBAL"} + b. A CUBE route: {"id":4, "backends":["10.1.2.56:7070"], "project":"p1", "resourceGroup":"common_query_1", "type":"CUBE"}, if http request header/url/body `p1` exist will route to backends + c. A line a route, You can config multi routes. + +3. route_table jdbc + a. config `kylin.gateway.datasource.type=jdbc` + +4. ${gateway.home}/bin/gateway.sh + a. `./bin/gateway.sh start` to start gateway. + diff --git a/build/conf/route_table b/build/conf/route_table index 819c911150..0ee2e864a4 100644 --- a/build/conf/route_table +++ b/build/conf/route_table @@ -1,2 +1,4 @@ -{"id":4, "backends":["10.1.2.56:7070"], "project":"p1", "resourceGroup":"common_query_1", "type":"CUBE"} -{"id":111, "backends":["www.baidu.com"], "resourceGroup":"default", "type":"GLOBAL"} +{"id":4, "backends":["10.1.2.56:7070"], "project":"p1", "resourceGroup":"common_query_1", "type":"ASYNC"} +{"id":5, "backends":["10.1.2.56:7070"], "project":"p1", "resourceGroup":"common_query_2", "type":"CUBE"} +{"id":6, "backends":["10.1.2.56:7070"], "project":"p2", "resourceGroup":"common_query_3", "order": 10 , "type":"CUBE"} +{"id":1, "backends":["10.1.2.56:7070"], "resourceGroup":"default", "type":"GLOBAL"} diff --git a/kylin-gateway/src/main/java/io/kyligence/kap/gateway/entity/KylinRouteRaw.java b/kylin-gateway/src/main/java/io/kyligence/kap/gateway/entity/KylinRouteRaw.java index 924ff1b4d6..27acbf2a62 100644 --- a/kylin-gateway/src/main/java/io/kyligence/kap/gateway/entity/KylinRouteRaw.java +++ b/kylin-gateway/src/main/java/io/kyligence/kap/gateway/entity/KylinRouteRaw.java @@ -30,6 +30,8 @@ public class KylinRouteRaw { private String type; + private int order = 0; + private String cluster; private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); @@ -67,6 +69,7 @@ public static KylinRouteRaw convert(RouteDO routeDO) { kylinRouteRaw.setId(routeDO.getId()); kylinRouteRaw.setType(routeDO.getType()); + kylinRouteRaw.setOrder(routeDO.getOrder()); kylinRouteRaw.setProject(routeDO.getProject()); kylinRouteRaw.setResourceGroup(routeDO.getResourceGroup()); kylinRouteRaw.setStringBackends(Arrays.toString(routeDO.getBackends().toArray())); diff --git a/kylin-gateway/src/main/java/io/kyligence/kap/gateway/health/ConcurrentPingStrategy.java b/kylin-gateway/src/main/java/io/kyligence/kap/gateway/health/ConcurrentPingStrategy.java index 6c043b4f1f..6729bbb07a 100644 --- a/kylin-gateway/src/main/java/io/kyligence/kap/gateway/health/ConcurrentPingStrategy.java +++ b/kylin-gateway/src/main/java/io/kyligence/kap/gateway/health/ConcurrentPingStrategy.java @@ -3,8 +3,10 @@ import com.netflix.loadbalancer.IPing; import com.netflix.loadbalancer.IPingStrategy; import com.netflix.loadbalancer.Server; +import io.kyligence.kap.gateway.event.Kylin3XRefreshRoutesEvent; import lombok.Data; import lombok.extern.slf4j.Slf4j; +import org.springframework.context.ApplicationListener; import java.util.Map; import java.util.Objects; @@ -21,7 +23,7 @@ @Slf4j @Data -public class ConcurrentPingStrategy implements IPingStrategy { +public class ConcurrentPingStrategy implements IPingStrategy, ApplicationListener { private Map serversStatus = new ConcurrentHashMap<>(); @@ -68,6 +70,11 @@ public boolean[] pingServers(IPing ping, Server[] servers) { return results; } + @Override + public void onApplicationEvent(Kylin3XRefreshRoutesEvent event) { + serversStatus.clear(); + } + private class CheckServerTask implements Callable { private IPing ping; @@ -82,12 +89,17 @@ private CheckServerTask(IPing ping, Server server) { @Override public Boolean call() { serversStatus.putIfAbsent(server, new AtomicInteger(0)); + AtomicInteger errorTimes = serversStatus.get(server); + if (null == errorTimes) { + // for clear servers + return false; + } try { boolean isAlive = ping.isAlive(server); if (isAlive) { - if (serversStatus.get(server).get() > 0) { - serversStatus.get(server).set(0); + if (errorTimes.get() > 0) { + errorTimes.set(0); } return true; } @@ -95,7 +107,7 @@ public Boolean call() { log.error("Failed to ping server: {}", server, e); } - return serversStatus.get(server).incrementAndGet() < retryTimes; + return errorTimes.incrementAndGet() < retryTimes; } } diff --git a/kylin-gateway/src/main/java/io/kyligence/kap/gateway/persistent/domain/RouteDO.java b/kylin-gateway/src/main/java/io/kyligence/kap/gateway/persistent/domain/RouteDO.java index 71fa610007..13491329a2 100644 --- a/kylin-gateway/src/main/java/io/kyligence/kap/gateway/persistent/domain/RouteDO.java +++ b/kylin-gateway/src/main/java/io/kyligence/kap/gateway/persistent/domain/RouteDO.java @@ -25,4 +25,6 @@ public class RouteDO { private String type; + private int order = 0; + } diff --git a/kylin-gateway/src/main/java/io/kyligence/kap/gateway/route/RefreshRouteTableScheduler.java b/kylin-gateway/src/main/java/io/kyligence/kap/gateway/route/RefreshRouteTableScheduler.java index e799d4f56c..0e2f05985e 100644 --- a/kylin-gateway/src/main/java/io/kyligence/kap/gateway/route/RefreshRouteTableScheduler.java +++ b/kylin-gateway/src/main/java/io/kyligence/kap/gateway/route/RefreshRouteTableScheduler.java @@ -31,7 +31,6 @@ import java.net.URISyntaxException; import java.util.Collection; import java.util.List; -import java.util.Objects; import java.util.stream.Collectors; import static io.kyligence.kap.gateway.constant.KylinRouteConstant.DEFAULT_RESOURCE_GROUP; @@ -119,7 +118,7 @@ private RouteDefinition convert2RouteDefinition(KylinRouteRaw routeRaw) case CUBE: predicateDefinition.setName(KYLIN_ROUTE_PREDICATE); predicateDefinition.getArgs().put(PREDICATE_ARG_KEY_0, routeRaw.getProject()); - routeDefinition.setOrder(0); + routeDefinition.setOrder(routeRaw.getOrder()); break; case GLOBAL: predicateDefinition.setName("Path"); @@ -146,14 +145,19 @@ private Kylin3XLoadBalancer convert2Kylin3XLoadBalancer(KylinRouteRaw routeRaw) private boolean isRawRouteTableIllegal(List routeRawList) { boolean checkResult = false; + if (routeRawList.size() > routeRawList.stream().map(KylinRouteRaw::getId).distinct().count()) { + logger.error("Route table contain same id!"); + return true; + } + List errorList = routeRawList.stream().filter(kylinRouteRaw -> { - if (kylinRouteRaw.getId() < 0) { + if (StringUtils.isBlank(kylinRouteRaw.getStringBackends())) { return true; } - if (StringUtils.isBlank(kylinRouteRaw.getResourceGroup()) - || StringUtils.isBlank(kylinRouteRaw.getStringBackends()) - || StringUtils.isBlank(kylinRouteRaw.getType())) { + try { + KylinResourceGroupTypeEnum.valueOf(kylinRouteRaw.getType()); + } catch (IllegalArgumentException e) { return true; } @@ -162,16 +166,6 @@ private boolean isRawRouteTableIllegal(List routeRawList) { return true; } - try { - URI uri = new URI(getStringURI(kylinRouteRaw.getResourceGroup())); - if (Objects.isNull(uri.getHost()) || Objects.isNull(uri.getAuthority()) - || Objects.isNull(uri.getScheme())) { - return true; - } - } catch (URISyntaxException e) { - return true; - } - if (CollectionUtils.isEmpty(kylinRouteRaw.getBackends())) { return true; } diff --git a/kylin-gateway/src/main/resources/application.yml b/kylin-gateway/src/main/resources/application.yml index 1cb64bf4bc..50c525f06a 100644 --- a/kylin-gateway/src/main/resources/application.yml +++ b/kylin-gateway/src/main/resources/application.yml @@ -6,7 +6,7 @@ spring: cloud: gateway: httpclient: - connectTimeout: 3000 + connectTimeout: 15000 responseTimeout: 480000 logging: