diff --git a/pom.xml b/pom.xml
index 4753d24..33403af 100644
--- a/pom.xml
+++ b/pom.xml
@@ -45,7 +45,7 @@
org.asynchttpclient
async-http-client
- 2.12.1
+ 3.0.1
com.google.guava
@@ -108,7 +108,7 @@
org.apache.kafka
kafka-clients
- 2.8.1
+ 3.9.0
@@ -159,22 +159,22 @@
io.netty
netty-handler
- 4.1.108.Final
+ 4.1.115.Final
io.netty
netty-common
- 4.1.108.Final
+ 4.1.115.Final
io.netty
netty-codec
- 4.1.108.Final
+ 4.1.115.Final
io.netty
netty-codec-http
- 4.1.108.Final
+ 4.1.115.Final
@@ -207,7 +207,7 @@
org.springframework
spring-web
- 6.1.6
+ 6.1.12
diff --git a/src/main/java/com/datastat/constant/Constant.java b/src/main/java/com/datastat/constant/Constant.java
index 42f2e4b..7c5c6ae 100644
--- a/src/main/java/com/datastat/constant/Constant.java
+++ b/src/main/java/com/datastat/constant/Constant.java
@@ -39,7 +39,7 @@ private Constant() {
/**
* VALID_OPENMIND_ENV_REG used to match input string.
*/
- public static final String VALID_OPENMIND_ENV_REG = "pro|yidong|sh";
+ public static final String VALID_OPENMIND_ENV_REG = "pro|tianyi|sh";
/**
* VALID_REPO_ID used to match input string.
@@ -51,6 +51,16 @@ private Constant() {
*/
public static final String OPENMIND_COMMUNITY = "openmind";
+ /**
+ * The name of github platform.
+ */
+ public static final String GITHUB_PLATFORM = "github";
+
+ /**
+ * The name of gutee platform.
+ */
+ public static final String GITEE_PLATFORM = "GITEE";
+
/**
* 支持性能数据上传的社区.
*/
diff --git a/src/main/java/com/datastat/controller/QueryController.java b/src/main/java/com/datastat/controller/QueryController.java
index f321e99..6b3a7f9 100644
--- a/src/main/java/com/datastat/controller/QueryController.java
+++ b/src/main/java/com/datastat/controller/QueryController.java
@@ -152,7 +152,13 @@ public String queryCveDetails(HttpServletRequest request,
@RequestMapping("/newYear/report")
public String queryNewYear(HttpServletRequest request,
@CookieValue(value = "_oauth2_proxy", required = true) String oauth2_proxy) {
- return queryService.queryNewYearPer(request, oauth2_proxy);
+ return queryService.queryNewYearPer(request, oauth2_proxy, "gitee");
+ }
+
+ @RequestMapping("/newYear/report/github")
+ public String queryNewYearGithub(HttpServletRequest request,
+ @CookieValue(value = "_oauth2_proxy", required = true) String oauth2_proxy) {
+ return queryService.queryNewYearPer(request, oauth2_proxy, "github");
}
@RequestMapping("/newYear/monthcount")
@@ -830,10 +836,7 @@ public String monthDownCount(HttpServletRequest request,
* @return A string containing the monthly download count information for the repository.
*/
@RequestMapping(value = "/view/count")
- public String viewCount(HttpServletRequest request,
- @RequestParam(value = "repoType") String repoType,
- @RequestParam(value = "owner") String owner,
- @RequestParam(value = "repo") String repo) {
- return queryService.getViewCount(request, repoType, owner, repo);
+ public String viewCount(HttpServletRequest request, @Valid final RequestParams condition) {
+ return queryService.getViewCount(request, condition);
}
}
\ No newline at end of file
diff --git a/src/main/java/com/datastat/dao/QueryDao.java b/src/main/java/com/datastat/dao/QueryDao.java
index e68e790..7302851 100644
--- a/src/main/java/com/datastat/dao/QueryDao.java
+++ b/src/main/java/com/datastat/dao/QueryDao.java
@@ -356,18 +356,18 @@ public String queryCveDetails(CustomPropertiesConfig queryConf, String lastCurso
return esQueryUtils.esFromId(restHighLevelClient, item, lastCursor, Integer.parseInt(pageSize), indexName);
}
- public String queryNewYearPer(CustomPropertiesConfig queryConf, String oauth2_proxy, String community) {
+ public String queryNewYearPer(CustomPropertiesConfig queryConf, String oauth2_proxy, String community,
+ String platform) {
HashMap resMap = new HashMap<>();
try {
- String user = getUserFromCookie(queryConf, oauth2_proxy);
- String localFile = env.getProperty("export_path") + community.toLowerCase() + "_" + env.getProperty("year") + ".csv";
+ String localFile = String.format("%s%s_%s_%s.csv", env.getProperty("export_path"), community.toLowerCase(),
+ platform, env.getProperty("year"));
+ String user = getUserFromCookie(queryConf, oauth2_proxy, platform);
List> report = CsvFileUtil.readFile(localFile);
resMap.put("code", 200);
resMap.put("msg", "OK");
- if (report == null)
+ if (report == null || user == null)
resMap.put("data", new ArrayList<>());
- else if (user == null)
- resMap.put("data", report);
else {
List> user_login = report.stream()
.filter(m -> m.getOrDefault("user_login", "").equals(user)).collect(Collectors.toList());
@@ -392,13 +392,13 @@ else if (user == null)
return objectMapper.valueToTree(resMap).toString();
} catch (Exception e) {
logger.error("report exception - {}", e.getMessage());
- return objectMapper.valueToTree(resMap).toString();
+ throw new RuntimeException(e.getMessage());
}
}
@SneakyThrows
public String queryNewYearMonthCount(CustomPropertiesConfig queryConf, String oauth2_proxy) {
- String user = getUserFromCookie(queryConf, oauth2_proxy);
+ String user = getUserFromCookie(queryConf, oauth2_proxy, "gitee");
String queryJson = String.format(queryConf.getMonthCountQueryStr(), user);
ListenableFuture future = esAsyncHttpUtil.executeSearch(esUrl, queryConf.getGiteeAllIndex(), queryJson);
String responseBody = future.get().getResponseBody(UTF_8);
@@ -424,13 +424,22 @@ public String queryNewYearMonthCount(CustomPropertiesConfig queryConf, String oa
}
@SneakyThrows
- private String getUserFromCookie(CustomPropertiesConfig queryConf, String oauth2_proxy) {
+ private String getUserFromCookie(CustomPropertiesConfig queryConf, String oauth2_proxy, String platform) {
String cookie_oauth2_proxy = "_oauth2_proxy=" + oauth2_proxy;
- HttpResponse response = Unirest.get(queryConf.getGiteeUserInfoUrl())
+ String userInfoUrl;
+ if (Constant.GITEE_PLATFORM.equalsIgnoreCase(platform)) {
+ userInfoUrl = queryConf.getGiteeUserInfoUrl();
+ } else if (Constant.GITHUB_PLATFORM.equalsIgnoreCase(platform)) {
+ userInfoUrl = queryConf.getGithubUserInfoUrl();
+ } else {
+ throw new RuntimeException("error platform");
+ }
+ HttpResponse response = Unirest.get(userInfoUrl)
.header("cookie", cookie_oauth2_proxy)
.asString();
if (response.getStatus() != 200) {
+ logger.error("user auth execption - {}", response.getBody());
throw new Exception("unauthorized");
}
JsonNode res = objectMapper.readTree(response.getBody());
@@ -1470,7 +1479,7 @@ public String getOneIdUserGiteeLoginName(HttpServletRequest request) {
}
}
} catch (Exception e) {
- throw new RuntimeException(e);
+ throw new RuntimeException(e.getMessage());
}
return null;
}
@@ -3602,7 +3611,7 @@ public String queryUserOwnerRepos(CustomPropertiesConfig queryConf, String user)
}
@SneakyThrows
- public String saveFrontendEvents(String community, String requestBody) {
+ public String saveFrontendEvents(String community, String requestBody, String clientIp) {
// 检测请求体是否含有header和body
boolean hasHeader = requestBody.contains("\"header\"");
boolean hasBody = requestBody.contains("\"body\"");
@@ -3650,6 +3659,7 @@ public String saveFrontendEvents(String community, String requestBody) {
eventObj.put("created_at", nowStr);
eventObj.put("community", community);
+ eventObj.put("clientIp", clientIp);
JsonNode mergedJson = objectMapper.updateValue(eventObj, headerObj);
@@ -3717,7 +3727,7 @@ public String putSearchNpsByCommunity(CustomPropertiesConfig queryConf, String t
resMap.put("community", community);
String userId = "";
if (token != null && !"mindspore".equals(community)) {
- userId = userIdDao.getUserIdByCommunity(token, community);
+ userId = userIdDao.getUserIdByCommunity(token, queryConf);
if (null == userId || userId.equals("")) {
logger.warn("UserId parse error for token:" + token + ",community:" + community);
throw new IllegalArgumentException("UserId parse error");
@@ -3924,26 +3934,32 @@ public String getCommunityMonthDowncount(CustomPropertiesConfig queryConf, Strin
* Retrieves the view count statistics for a specified community and repository.
*
* @param queryConf Custom configuration properties containing necessary query configurations.
- * @param repoType The type of the repository, passed as a request parameter.
- * @param owner The owner of the repository, passed as a request parameter.
- * @param repo The repo name of the repository, passed as a request parameter.
+ * @param condition The search condition of the repository
* @return A JSON string containing the monthly download count statistics.
* @throws Exception If an error occurs during the query process.
*/
@SneakyThrows
- public String getViewCount(CustomPropertiesConfig queryConf, String repoType, String owner, String repo) {
- String query = String.format(queryConf.getRepoViewCountQueryStr(), repoType, owner, repo);
- ListenableFuture future = esAsyncHttpUtil.executeCount(esUrl, queryConf.getExportWebsiteViewIndex(), query);
+ public String getViewCount(CustomPropertiesConfig queryConf, RequestParams condition) {
+ String query = String.format(queryConf.getRepoViewCountQueryStr(), condition.getStart(),
+ condition.getEnd(), condition.getRepoType(), condition.getRepoId());
+ String index = queryConf.getExportWebsiteViewIndex();
+ ListenableFuture future = esAsyncHttpUtil.executeSearch(esUrl, index, query);
Response response = future.get();
int statusCode = response.getStatusCode();
String statusText = response.getStatusText();
String responseBody = response.getResponseBody(UTF_8);
JsonNode dataNode = objectMapper.readTree(responseBody);
- long count = dataNode.get("count").asLong();
- Map resData = new HashMap<>();
- resData.put("owner", owner);
- resData.put("repo", repo);
- resData.put("count", count);
- return ResultUtil.resultJsonStr(statusCode, objectMapper.valueToTree(resData), statusText);
+ JsonNode testStr = dataNode.get("aggregations").get("group_field").get("buckets");
+ ArrayNode buckets = objectMapper.createArrayNode();
+ if (testStr.isArray()) {
+ for (int i = 0; i < testStr.size(); i++) {
+ JsonNode item = testStr.get(i);
+ ObjectNode bucket = objectMapper.createObjectNode();
+ bucket.put("repo_id", item.get("key").asText());
+ bucket.put("count", item.get("doc_count").asInt());
+ buckets.add(bucket);
+ }
+ }
+ return ResultUtil.resultJsonStr(statusCode, buckets, statusText);
}
}
diff --git a/src/main/java/com/datastat/dao/UserIdDao.java b/src/main/java/com/datastat/dao/UserIdDao.java
index 9a10648..ffd2ad2 100644
--- a/src/main/java/com/datastat/dao/UserIdDao.java
+++ b/src/main/java/com/datastat/dao/UserIdDao.java
@@ -17,6 +17,7 @@
import com.auth0.jwt.JWT;
import com.auth0.jwt.interfaces.DecodedJWT;
+import com.datastat.model.CustomPropertiesConfig;
import com.datastat.util.RSAUtil;
import org.springframework.beans.factory.annotation.Autowired;
@@ -42,14 +43,15 @@ public String getUserId(String token){
return userId;
}
- public String getUserIdByCommunity(String token, String community) {
+ public String getUserIdByCommunity(String token, CustomPropertiesConfig queryConf) {
String userId = null;
try {
- RSAPrivateKey privateKey = RSAUtil.getPrivateKey(env.getProperty("rsa.authing." + community + ".privateKey"));
+ String authPrivateKey = queryConf.getRsaAuthPrivateKey();
+ RSAPrivateKey privateKey = RSAUtil.getPrivateKey(authPrivateKey);
DecodedJWT decode = JWT.decode(RSAUtil.privateDecrypt(token, privateKey));
userId = decode.getAudience().get(0);
} catch (Exception e) {
- logger.error("parse token exception - {}", e.getMessage());
+ logger.error("parse user id from token exception - {}", e.getMessage());
}
return userId;
}
diff --git a/src/main/java/com/datastat/interceptor/oneid/OneidInterceptor.java b/src/main/java/com/datastat/interceptor/oneid/OneidInterceptor.java
index ed6bbb1..bf1985f 100644
--- a/src/main/java/com/datastat/interceptor/oneid/OneidInterceptor.java
+++ b/src/main/java/com/datastat/interceptor/oneid/OneidInterceptor.java
@@ -13,6 +13,7 @@
import java.io.IOException;
import java.lang.reflect.Method;
+import java.nio.charset.StandardCharsets;
import java.security.interfaces.RSAPrivateKey;
import java.util.*;
@@ -28,14 +29,11 @@
import com.datastat.util.ObjectMapperUtil;
import com.datastat.util.RSAUtil;
import com.fasterxml.jackson.databind.JsonNode;
-import com.mashape.unirest.http.HttpResponse;
-import com.mashape.unirest.http.Unirest;
import jakarta.annotation.PostConstruct;
import jakarta.servlet.http.Cookie;
import jakarta.servlet.http.HttpServletRequest;
import jakarta.servlet.http.HttpServletResponse;
import org.apache.commons.lang3.StringUtils;
-import org.json.JSONArray;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
@@ -65,9 +63,6 @@ public class OneidInterceptor implements HandlerInterceptor {
@Value("${cookie.token.secures:default}")
private String cookieSecures;
- @Value("${oneid.token.base.password:default}")
- private String oneidTokenBasePassword;
-
private static HashMap domain2secure;
private static final Logger logger = LoggerFactory.getLogger(OneidInterceptor.class);
@@ -100,11 +95,12 @@ public boolean preHandle(HttpServletRequest httpServletRequest, HttpServletRespo
&& (companyToken == null || !companyToken.required())) {
return true;
}
+ CustomPropertiesConfig queryConf = getQueryConf(httpServletRequest);
// 从请求头中取出 token
String headerToken = httpServletRequest.getHeader("token");
- String headJwtTokenMd5 = verifyHeaderToken(headerToken);
- if (StringUtils.isBlank(headJwtTokenMd5)) {
+ String verifyHeaderMsg = verifyHeaderToken(headerToken, queryConf);
+ if (!verifyHeaderMsg.equals("success")) {
tokenError(httpServletRequest, httpServletResponse, "unauthorized");
return false;
}
@@ -126,9 +122,11 @@ public boolean preHandle(HttpServletRequest httpServletRequest, HttpServletRespo
// 解密cookie中加密的token
String token = tokenCookie.getValue();
try {
- RSAPrivateKey privateKey = RSAUtil.getPrivateKey(env.getProperty("rsa.authing.privateKey"));
+ String authPrivateKey = queryConf.getRsaAuthPrivateKey();
+ RSAPrivateKey privateKey = RSAUtil.getPrivateKey(authPrivateKey);
token = RSAUtil.privateDecrypt(token, privateKey);
} catch (Exception e) {
+ logger.error("decode token in cookie exception - {}", e.getMessage());
tokenError(httpServletRequest, httpServletResponse, "unauthorized");
return false;
}
@@ -148,12 +146,14 @@ public boolean preHandle(HttpServletRequest httpServletRequest, HttpServletRespo
permission = new String(Base64.getDecoder().decode(permissionTemp.getBytes()));
verifyToken = decode.getClaim("verifyToken").asString();
} catch (JWTDecodeException j) {
+ logger.error("parse token exception - {}", j.getMessage());
tokenError(httpServletRequest, httpServletResponse, "unauthorized");
return false;
}
// 校验token
- String verifyTokenMsg = verifyToken(headJwtTokenMd5, token, verifyToken, userId, issuedAt, expiresAt, permission);
+ String verifyTokenMsg = verifyCookieToken(headerToken, token, verifyToken, userId, issuedAt, expiresAt,
+ permission, queryConf);
if (!verifyTokenMsg.equals("success")) {
tokenError(httpServletRequest, httpServletResponse, verifyTokenMsg);
return false;
@@ -193,11 +193,15 @@ public void afterCompletion(HttpServletRequest httpServletRequest,
Object o, Exception e) throws Exception {
}
- private String verifyToken(String headerToken, String token, String verifyToken,
- String userId, Date issuedAt, Date expiresAt, String permission) {
+ private String verifyCookieToken(String headerToken, String token, String verifyToken,
+ String userId, Date issuedAt, Date expiresAt, String permission, CustomPropertiesConfig queryConf) {
try {
+ // 服务端校验headerToken是否有效
+ String shaToken = RSAUtil.encryptSha256(headerToken, queryConf.getAuthSalt());
+ String md5Token = DigestUtils.md5DigestAsHex(headerToken.getBytes(StandardCharsets.UTF_8));
+
// header中的token和cookie中的token不一样
- if (!headerToken.equals(verifyToken)) {
+ if (!shaToken.equals(verifyToken) && !md5Token.equals(verifyToken)) {
return "unauthorized";
}
@@ -207,17 +211,12 @@ private String verifyToken(String headerToken, String token, String verifyToken,
}
// token 签名密码验证
- String password = permission + oneidTokenBasePassword;
+ String password = permission + queryConf.getAuthTokenSessionPassword();
JWTVerifier jwtVerifier = JWT.require(Algorithm.HMAC256(password)).build();
jwtVerifier.verify(token);
- // 退出登录后token失效
- // String redisKey = userId + issuedAt.toString();
- // String beforeToken = (String) redisDao.get(redisKey);
- // if (token.equalsIgnoreCase(beforeToken)) {
- // return "unauthorized";
- // }
} catch (Exception e) {
+ logger.error("verify token exception - {}", e.getMessage());
return "unauthorized";
}
return "success";
@@ -227,27 +226,21 @@ private String verifyToken(String headerToken, String token, String verifyToken,
* 校验header中的token
*
* @param headerToken header中的token
- * @return 校验正确返回token的MD5值
+ * @return 返回校验信息
*/
- private String verifyHeaderToken(String headerToken) {
+ private String verifyHeaderToken(String headerToken, CustomPropertiesConfig queryConf) {
try {
if (StringUtils.isBlank(headerToken)) {
return "unauthorized";
}
- // 服务端校验headerToken是否有效
- String md5Token = DigestUtils.md5DigestAsHex(headerToken.getBytes());
- // if (!redisDao.exists("idToken_" + md5Token)) {
- // return "token expires";
- // }
-
// token 签名密码验证
- String password = oneidTokenBasePassword;
+ String password = queryConf.getAuthTokenBasePassword();
JWTVerifier jwtVerifier = JWT.require(Algorithm.HMAC256(password)).build();
jwtVerifier.verify(headerToken);
- return md5Token;
+ return "success";
} catch (Exception e) {
- logger.error("exception", e);
+ logger.error("verify Header Token exception - {}", e.getMessage());
return "unauthorized";
}
}
@@ -267,7 +260,7 @@ private String verifyUser(SigToken sigToken, HttpServletRequest httpServletReque
}
}
} catch (Exception e) {
- logger.error("exception", e);
+ logger.error("verify user permission exception - {}", e.getMessage());
return "has no permission";
}
return "has no permission";
@@ -285,6 +278,7 @@ private String verifyCompanyPer(CompanyToken companyToken, HttpServletRequest ht
}
}
} catch (Exception e) {
+ logger.error("verify user company permission exception - {}", e.getMessage());
return "has no permission";
}
return "has no permission";
@@ -341,6 +335,7 @@ private String getManageToken() {
JsonNode resJson = ObjectMapperUtil.toJsonNode(response);
return resJson.get("token").asText();
} catch (Exception e) {
+ logger.error("get manage token exception - {}", e.getMessage());
}
return null;
}
diff --git a/src/main/java/com/datastat/model/CustomPropertiesConfig.java b/src/main/java/com/datastat/model/CustomPropertiesConfig.java
index a8d4f45..3eef44a 100644
--- a/src/main/java/com/datastat/model/CustomPropertiesConfig.java
+++ b/src/main/java/com/datastat/model/CustomPropertiesConfig.java
@@ -45,6 +45,7 @@ public class CustomPropertiesConfig {
private String sigAction;
private String isvYamlUrl;
private String giteeUserInfoUrl;
+ private String githubUserInfoUrl;
private String orgName;
private String isvCountToken;
private String checkField;
@@ -58,6 +59,10 @@ public class CustomPropertiesConfig {
private String globalNpsIssueTitle;
private String esBaseUrl;
private String esAuth;
+ private String authTokenBasePassword;
+ private String rsaAuthPrivateKey;
+ private String authTokenSessionPassword;
+ private String authSalt;
// -- index --
private String extOsIndex;
diff --git a/src/main/java/com/datastat/service/QueryService.java b/src/main/java/com/datastat/service/QueryService.java
index 9f763f4..7e517eb 100644
--- a/src/main/java/com/datastat/service/QueryService.java
+++ b/src/main/java/com/datastat/service/QueryService.java
@@ -24,6 +24,7 @@
import com.datastat.model.vo.*;
import com.datastat.result.ReturnCode;
import com.datastat.util.ArrayListUtil;
+import com.datastat.util.ClientUtil;
import com.datastat.util.PageUtils;
import com.datastat.util.RSAUtil;
import com.datastat.util.ResultUtil;
@@ -322,19 +323,19 @@ public String queryCveDetails(HttpServletRequest request, String community, Stri
return result;
}
- public String queryNewYearPer(HttpServletRequest request, String oauth2_proxy) {
+ public String queryNewYearPer(HttpServletRequest request, String oauth2_proxy, String platform) {
QueryDao queryDao = getQueryDao(request);
String referer = request.getHeader("Referer");
String community = null;
try {
community = referer.split("\\.")[1];
} catch (Exception e) {
- logger.error("exception", e);
+ logger.error("parse community exception - {}", e.getMessage());
return resultJsonStr(404, "error", "Referer error");
}
CustomPropertiesConfig queryConf = getQueryConf(community);
- return queryDao.queryNewYearPer(queryConf, oauth2_proxy, community);
+ return queryDao.queryNewYearPer(queryConf, oauth2_proxy, community, platform);
}
public String queryNewYearMonthCount(HttpServletRequest request, String oauth2_proxy) {
@@ -1562,8 +1563,9 @@ public String queryUserOwnerRepos(HttpServletRequest request, String user) {
public String saveFrontendEvents(HttpServletRequest request, String community, String requestBody) {
QueryDao queryDao = getQueryDao(request);
+ String clientIp = ClientUtil.getClientIpAddress(request);
if (!checkCommunity(community)) return ResultUtil.resultJsonStr(404, "error", "not found");
- return queryDao.saveFrontendEvents(community, requestBody);
+ return queryDao.saveFrontendEvents(community, requestBody, clientIp);
}
public String putGlobalNpsIssue(HttpServletRequest request, String token, String community, NpsBody body) {
@@ -1683,13 +1685,19 @@ public String getCommunityMonthDowncount(HttpServletRequest request, String comm
* @param repo The repo name of the repository, passed as a request parameter.
* @return A JSON string containing the monthly download count statistics.
*/
- public String getViewCount(HttpServletRequest request, String repoType, String owner, String repo) {
+ public String getViewCount(HttpServletRequest request, RequestParams condition) {
QueryDao queryDao = getQueryDao(request);
CustomPropertiesConfig queryConf = getQueryConf("foundry");
- String key = "get_viewcount_" + repoType + owner + repo;
+ StringBuilder sb = new StringBuilder("get_viewcount_");
+ sb.append(condition.getPath())
+ .append(condition.getRepoType())
+ .append(condition.getRepoId())
+ .append(condition.getStart())
+ .append(condition.getEnd());
+ String key = sb.toString();
String result = (String) redisDao.get(key);
if (result == null) {
- result = queryDao.getViewCount(queryConf, repoType, owner, repo);
+ result = queryDao.getViewCount(queryConf, condition);
redisDao.set(key, result, redisDefaultExpire);
}
return result;
diff --git a/src/main/java/com/datastat/util/ClientUtil.java b/src/main/java/com/datastat/util/ClientUtil.java
index 4813bc2..d5b4590 100644
--- a/src/main/java/com/datastat/util/ClientUtil.java
+++ b/src/main/java/com/datastat/util/ClientUtil.java
@@ -8,49 +8,82 @@
See the Mulan PSL v2 for more details.
Create: 2024/02
*/
-
package com.datastat.util;
-import java.util.Enumeration;
-
-import jakarta.servlet.http.HttpServletRequest;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-public class ClientUtil {
- private static final Logger logger = LoggerFactory.getLogger(ClientUtil.class);
- public static String getClientIpAddress(HttpServletRequest request) {
- String[] headerNames = {"x-forwarded-for", "Proxy-Client-IP", "WL-Proxy-Client-IP",
- "HTTP_CLIENT_IP", "HTTP_X_FORWARDED_FOR", "X-Real-IP"};
+import jakarta.servlet.http.HttpServletRequest;
- for (String headerName : headerNames) {
- String ip = request.getHeader(headerName);
- if (isValidIp(ip)) {
- return extractIp(ip);
- }
- }
+import java.net.InetAddress;
+import java.net.UnknownHostException;
- return request.getRemoteAddr();
- }
+public final class ClientUtil {
- private static boolean isValidIp(String ip) {
- return ip != null && ip.length() > 0 && !"unknown".equalsIgnoreCase(ip);
+
+ // Private constructor to prevent instantiation of the utility class
+ private ClientUtil() {
+ // private constructor to hide the implicit public one
+ throw new AssertionError("ClientUtil class cannot be instantiated.");
}
- private static String extractIp(String ip) {
- if (ip.contains(",")) {
- return ip.split(",")[0];
+ /**
+ * Logger instance for ClientUtil.
+ */
+ private static final Logger LOGGER = LoggerFactory.getLogger(ClientUtil.class);
+
+ /**
+ * Retrieve the client's IP address from the HttpServletRequest.
+ *
+ * @param request The HttpServletRequest object
+ * @return The client's IP address as a string
+ */
+ public static String getClientIpAddress(final HttpServletRequest request) {
+ String ip = request.getHeader("x-real-ip");
+ if (checkIp(ip)) {
+ ip = getForwardedIP(request);
+ }
+ if (checkIp(ip)) {
+ ip = request.getRemoteAddr();
+ if ("127.0.0.1".equals(ip) || "0:0:0:0:0:0:0:1".equals(ip)) {
+ InetAddress inet = null;
+ try {
+ inet = InetAddress.getLocalHost();
+ } catch (UnknownHostException e) {
+ LOGGER.error("get local host error: " + e.getMessage());
+ }
+ ip = inet.getHostAddress();
+ }
}
return ip;
}
- public static void getHeaderValue(HttpServletRequest request) {
- Enumeration headerNames = request.getHeaderNames();
- while (headerNames.hasMoreElements()){
- String name = headerNames.nextElement();
- String value = request.getHeader(name);
- logger.info("request header: name = {}, value = {}", name, value);
- }
+ /**
+ * Check if the provided string is a valid IP address.
+ *
+ * @param ip The IP address to check.
+ * @return true if the IP address is valid, false otherwise.
+ */
+ private static boolean checkIp(final String ip) {
+ return null == ip || ip.length() == 0 || "unknown".equalsIgnoreCase(ip);
}
+ /**
+ * Retrieve the client's x-forwarded-for IP address from the HttpServletRequest.
+ *
+ * @param request The HttpServletRequest object
+ * @return The client's IP address as a string
+ */
+ private static String getForwardedIP(final HttpServletRequest request) {
+ String headerName = "x-forwarded-for";
+ String ip = request.getHeader(headerName);
+ if (!checkIp(ip)) {
+ // There will be multiple IP values after multiple reverse proxies, pick the first IP.
+ if (ip.contains(",")) {
+ ip = ip.split(",")[0];
+ }
+ }
+ return ip;
+ }
}
+
diff --git a/src/main/java/com/datastat/util/CsvFileUtil.java b/src/main/java/com/datastat/util/CsvFileUtil.java
index 2ed4e6a..da802f4 100644
--- a/src/main/java/com/datastat/util/CsvFileUtil.java
+++ b/src/main/java/com/datastat/util/CsvFileUtil.java
@@ -59,7 +59,7 @@ public static List> readFile(String file) {
return res;
} catch (Exception e) {
- logger.error("exception", e);
+ logger.error("read file exception - {}", e.getMessage());
return null;
}
}
diff --git a/src/main/java/com/datastat/util/EsAsyncHttpUtil.java b/src/main/java/com/datastat/util/EsAsyncHttpUtil.java
index 578612c..e8cfd22 100644
--- a/src/main/java/com/datastat/util/EsAsyncHttpUtil.java
+++ b/src/main/java/com/datastat/util/EsAsyncHttpUtil.java
@@ -20,9 +20,8 @@
import java.security.SecureRandom;
import java.security.cert.CertificateException;
import java.security.cert.X509Certificate;
+import java.time.Duration;
import java.util.Base64;
-import java.util.Map;
-import java.util.concurrent.ExecutionException;
import javax.net.ssl.*;
import org.asynchttpclient.*;
@@ -44,8 +43,9 @@ public class EsAsyncHttpUtil {
public static synchronized AsyncHttpClient getClient() throws KeyManagementException, NoSuchAlgorithmException {
if (asyncHttpClient == null) {
asyncHttpClient = new DefaultAsyncHttpClient(new DefaultAsyncHttpClientConfig.Builder()
- .setConnectTimeout(100000)
- .setRequestTimeout(100000).setSslContext(new JdkSslContext(skipSsl(), true, ClientAuth.NONE))
+ .setConnectTimeout(Duration.ofSeconds(300))
+ .setReadTimeout(Duration.ofSeconds(300))
+ .setRequestTimeout(Duration.ofSeconds(300)).setSslContext(new JdkSslContext(skipSsl(), true, ClientAuth.NONE))
.build());
}
diff --git a/src/main/java/com/datastat/util/RSAUtil.java b/src/main/java/com/datastat/util/RSAUtil.java
index 15dacc0..515b26e 100644
--- a/src/main/java/com/datastat/util/RSAUtil.java
+++ b/src/main/java/com/datastat/util/RSAUtil.java
@@ -26,6 +26,8 @@
import javax.crypto.NoSuchPaddingException;
import org.apache.commons.codec.binary.Base64;
import org.apache.tomcat.util.http.fileupload.IOUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
@@ -35,6 +37,8 @@ public class RSAUtil implements Serializable {
public static String KEY_ALGORITHM;
public static String RSA_ALGORITHM;
+ private static final Logger LOGGER = LoggerFactory.getLogger(RSAUtil.class);
+
@Value("${rsa.key.algorithm:RSA}")
public void setKeyAlgorithm(String keyAlgorithm) { RSAUtil.KEY_ALGORITHM = keyAlgorithm; }
@@ -177,6 +181,36 @@ private static byte[] rsaSplitCodec(Cipher cipher, int opmode, byte[] datas, int
return dataResult;
}
+ /**
+ * sha256加密.
+ *
+ * @param data 数据
+ * @param salt 盐
+ * @return 加密后数据
+ * @throws NoSuchAlgorithmException 异常
+ */
+ public static String encryptSha256(String data, String salt) {
+ MessageDigest md = null;
+ try {
+ md = MessageDigest.getInstance("SHA-256");
+ } catch (NoSuchAlgorithmException e) {
+ LOGGER.error("encryptSha256 failed {}", e.getMessage());
+ return null;
+ }
+ // 将盐值和数据拼接后进行哈希计算
+ String combinedData = data + salt;
+ byte[] hashBytes = md.digest(combinedData.getBytes(StandardCharsets.UTF_8));
+ StringBuilder hexString = new StringBuilder();
+ for (byte b : hashBytes) {
+ String hex = String.format("%02X", b);
+ if (hex.length() == 1) {
+ hexString.append('0');
+ }
+ hexString.append(hex);
+ }
+ return hexString.toString();
+ }
+
public static void main(String[] args) throws Exception {
Map keyMap = RSAUtil.createKeys(3072);
String publicKey = keyMap.get("publicKey");
diff --git a/src/test/java/com/datastat/ds/unit/DaoUnitTests.java b/src/test/java/com/datastat/ds/unit/DaoUnitTests.java
index a56e57d..74f0b2a 100644
--- a/src/test/java/com/datastat/ds/unit/DaoUnitTests.java
+++ b/src/test/java/com/datastat/ds/unit/DaoUnitTests.java
@@ -38,6 +38,7 @@
import com.datastat.dao.RedisDao;
import com.datastat.dao.context.QueryDaoContext;
import com.datastat.ds.common.CommonUtil;
+import com.datastat.model.dto.RequestParams;
import com.datastat.util.EsAsyncHttpUtil;
import com.fasterxml.jackson.databind.ObjectMapper;
@@ -116,22 +117,28 @@ void testUserOwnerTypeDao() throws Exception {
@Test()
void testViewCountDao() throws Exception {
- String respBody = "{\"count\": 1234, \"_shards\":{\"total\":4,\"successful\":4}}";
- when(esAsyncHttpUtil.executeCount(anyString(), isNull(), anyString())).thenReturn(mockFuture);
+ String respBody = "{\"aggregations\":{\"group_field\":{\"buckets\":[{\"key\":\"3828\",\"doc_count\":1609}]}}}";
+ when(esAsyncHttpUtil.executeSearch(anyString(), isNull(), anyString())).thenReturn(mockFuture);
when(mockFuture.get()).thenReturn(mockResponse);
when(mockResponse.getStatusCode()).thenReturn(200);
when(mockResponse.getStatusText()).thenReturn("OK");
when(mockResponse.getResponseBody(StandardCharsets.UTF_8)).thenReturn(respBody);
String community = "foundry";
- String repoType = "dataset";
- String owner = "owner";
- String repo = "repo";
+ RequestParams params = new RequestParams();
+ params.setStart("2024-01-01");
+ params.setEnd("2024-12-01");
+ params.setRepoType("model");
+ params.setRepoId("3828");
+
when(queryDaoContext.getQueryDao(community)).thenReturn(queryDao);
when(queryConfContext.getQueryConfig(community)).thenReturn(queryConfig);
- String query = "{\"query\":{\"bool\":{\"filter\":[{\"query_string\":{\"analyze_wildcard\":true,"
- + "\"query\":\"event.keyword:$PageView AND properties.$path:\\\"/%ss/%s/%s\\\"\"}}]}}}";
+ String query = "{\"size\":0,\"query\":{\"bool\":{\"filter\":[{\"range\":{\"created_at\":{\"gte\":\"%s\",\"lte\":\"%s\"}}},"
+ + "{\"query_string\":{\"analyze_wildcard\":true,"
+ + "\"query\":\"event.keyword:RV AND properties.module.keyword:%s AND properties.id.keyword:%s\"}}]}},"
+ + "\"aggs\":{\"group_field\":{\"terms\":{\"field\":\"properties.id.keyword\",\"size\":50,"
+ + "\"order\":{\"_count\":\"desc\"},\"min_doc_count\":1},\"aggs\":{}}}}";
when(queryConfig.getRepoViewCountQueryStr()).thenReturn(query);
- String res = queryDao.getViewCount(queryConfig, repoType, owner, repo);
+ String res = queryDao.getViewCount(queryConfig, params);
CommonUtil.assertOk(res);
}
diff --git a/src/test/java/com/datastat/ds/unit/ServiceUnitTests.java b/src/test/java/com/datastat/ds/unit/ServiceUnitTests.java
index 2cf3e65..7012423 100644
--- a/src/test/java/com/datastat/ds/unit/ServiceUnitTests.java
+++ b/src/test/java/com/datastat/ds/unit/ServiceUnitTests.java
@@ -129,21 +129,27 @@ void testUserOwnerTypeService() throws Exception {
@Test()
void testViewCountService() throws Exception {
HttpServletRequest request = mock(HttpServletRequest.class);
- String repoType = "dataset";
- String owner = "owner";
- String repo = "repo";
- String key = "get_viewcount_" + repoType + owner + repo;
- String result = "{\"code\":200,\"msg\":\"ok\",\"data\":{\"owner\":\"owner\",\"repo\":\"repo\",\"count\":30}}";
+ RequestParams params = new RequestParams();
+ params.setRepoType("model");
+
+ StringBuilder sb = new StringBuilder("get_viewcount_");
+ sb.append(params.getPath())
+ .append(params.getRepoType())
+ .append(params.getRepoId())
+ .append(params.getStart())
+ .append(params.getEnd());
+ String key = sb.toString();
+ String result = "{\"code\":200,\"msg\":\"ok\",\"data\":{\"repo_id\":\"1234\", \"count\":30}}";
when(redisDao.get(key)).thenReturn(result);
- String serviceRes = queryService.getViewCount(request, repoType, owner, repo);
+ String serviceRes = queryService.getViewCount(request, params);
CommonUtil.assertOk(serviceRes);
when(redisDao.get(key)).thenReturn(null);
when(queryDaoContext.getQueryDao("queryDao")).thenReturn(foundryDao);
when(queryConfContext.getQueryConfig("foundryConf")).thenReturn(queryConfig);
- when(foundryDao.getViewCount(queryConfig, repoType, owner, repo)).thenReturn(result);
+ when(foundryDao.getViewCount(queryConfig, params)).thenReturn(result);
when(redisDao.set(key, result, 1l)).thenReturn(true);
- String res = queryService.getViewCount(request, repoType, owner, repo);
+ String res = queryService.getViewCount(request, params);
CommonUtil.assertOk(res);
}
}