Skip to content

Commit

Permalink
Merge pull request #305 from opensourceways/release/1230
Browse files Browse the repository at this point in the history 
merge Release/1230 into main
  • Loading branch information
@zhongjun2
zhongjun2 authored Dec 31, 2024
2 parents fde71d3 + f3b4e15 commit c7b6487
Show file tree
Hide file tree
Showing 14 changed files with 321 additions and 133 deletions.
18 changes: 9 additions & 9 deletions pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@
<dependency>
<groupId>org.asynchttpclient</groupId>
<artifactId>async-http-client</artifactId>
<version>2.12.1</version>
<version>3.0.1</version>
</dependency>
<dependency>
<groupId>com.google.guava</groupId>
Expand Down Expand Up @@ -108,7 +108,7 @@
<dependency>
<groupId>org.apache.kafka</groupId>
<artifactId>kafka-clients</artifactId>
<version>2.8.1</version>
<version>3.9.0</version>
</dependency>

<dependency>
Expand Down Expand Up @@ -159,22 +159,22 @@
<dependency>
<groupId>io.netty</groupId>
<artifactId>netty-handler</artifactId>
<version>4.1.108.Final</version>
<version>4.1.115.Final</version>
</dependency>
<dependency>
<groupId>io.netty</groupId>
<artifactId>netty-common</artifactId>
<version>4.1.108.Final</version>
<version>4.1.115.Final</version>
</dependency>
<dependency>
<groupId>io.netty</groupId>
<artifactId>netty-codec</artifactId>
<version>4.1.108.Final</version>
<version>4.1.115.Final</version>
</dependency>
<dependency>
<groupId>io.netty</groupId>
<artifactId>netty-codec-http</artifactId>
<version>4.1.108.Final</version>
<version>4.1.115.Final</version>
</dependency>

<dependency>
Expand Down Expand Up @@ -207,19 +207,19 @@
<dependency>
<groupId>org.springframework</groupId>
<artifactId>spring-web</artifactId>
<version>6.1.6</version>
<version>6.1.12</version>
</dependency>

<dependency>
<groupId>org.apache.tomcat.embed</groupId>
<artifactId>tomcat-embed-core</artifactId>
<version>10.1.25</version>
<version>10.1.34</version>
</dependency>

<dependency>
<groupId>org.apache.tomcat.embed</groupId>
<artifactId>tomcat-embed-websocket</artifactId>
<version>10.1.25</version>
<version>10.1.34</version>
</dependency>
</dependencies>

Expand Down
12 changes: 11 additions & 1 deletion src/main/java/com/datastat/constant/Constant.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@ private Constant() {
/**
* VALID_OPENMIND_ENV_REG used to match input string.
*/
public static final String VALID_OPENMIND_ENV_REG = "pro|yidong|sh";
public static final String VALID_OPENMIND_ENV_REG = "pro|tianyi|sh";

/**
* VALID_REPO_ID used to match input string.
Expand All @@ -51,6 +51,16 @@ private Constant() {
*/
public static final String OPENMIND_COMMUNITY = "openmind";

/**
* The name of github platform.
*/
public static final String GITHUB_PLATFORM = "github";

/**
* The name of gutee platform.
*/
public static final String GITEE_PLATFORM = "GITEE";

/**
* 支持性能数据上传的社区.
*/
Expand Down
27 changes: 21 additions & 6 deletions src/main/java/com/datastat/controller/QueryController.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -152,7 +152,13 @@ public String queryCveDetails(HttpServletRequest request,
@RequestMapping("/newYear/report")
public String queryNewYear(HttpServletRequest request,
@CookieValue(value = "_oauth2_proxy", required = true) String oauth2_proxy) {
return queryService.queryNewYearPer(request, oauth2_proxy);
return queryService.queryNewYearPer(request, oauth2_proxy, "gitee");
}

@RequestMapping("/newYear/report/github")
public String queryNewYearGithub(HttpServletRequest request,
@CookieValue(value = "_oauth2_proxy", required = true) String oauth2_proxy) {
return queryService.queryNewYearPer(request, oauth2_proxy, "github");
}

@RequestMapping("/newYear/monthcount")
Expand Down Expand Up @@ -830,10 +836,19 @@ public String monthDownCount(HttpServletRequest request,
* @return A string containing the monthly download count information for the repository.
*/
@RequestMapping(value = "/view/count")
public String viewCount(HttpServletRequest request,
@RequestParam(value = "repoType") String repoType,
@RequestParam(value = "owner") String owner,
@RequestParam(value = "repo") String repo) {
return queryService.getViewCount(request, repoType, owner, repo);
public String viewCount(HttpServletRequest request, @Valid final RequestParams condition) {
return queryService.getViewCount(request, condition);
}


/**
* Handles the request to view the modelers blog count.
*
* @param request the HttpServletRequest object containing client request data
* @return a String representing the view count of the modelers blog
*/
@RequestMapping(value = "/modelers/blog/view")
public String modelersBlogView(HttpServletRequest request) {
return queryService.getModelersBlogViewCount(request);
}
}
100 changes: 75 additions & 25 deletions src/main/java/com/datastat/dao/QueryDao.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -356,18 +356,18 @@ public String queryCveDetails(CustomPropertiesConfig queryConf, String lastCurso
return esQueryUtils.esFromId(restHighLevelClient, item, lastCursor, Integer.parseInt(pageSize), indexName);
}

public String queryNewYearPer(CustomPropertiesConfig queryConf, String oauth2_proxy, String community) {
public String queryNewYearPer(CustomPropertiesConfig queryConf, String oauth2_proxy, String community,
String platform) {
HashMap<String, Object> resMap = new HashMap<>();
try {
String user = getUserFromCookie(queryConf, oauth2_proxy);
String localFile = env.getProperty("export_path") + community.toLowerCase() + "_" + env.getProperty("year") + ".csv";
String localFile = String.format("%s%s_%s_%s.csv", env.getProperty("export_path"), community.toLowerCase(),
platform, env.getProperty("year"));
String user = getUserFromCookie(queryConf, oauth2_proxy, platform);
List<HashMap<String, Object>> report = CsvFileUtil.readFile(localFile);
resMap.put("code", 200);
resMap.put("msg", "OK");
if (report == null)
if (report == null || user == null)
resMap.put("data", new ArrayList<>());
else if (user == null)
resMap.put("data", report);
else {
List<HashMap<String, Object>> user_login = report.stream()
.filter(m -> m.getOrDefault("user_login", "").equals(user)).collect(Collectors.toList());
Expand All @@ -383,6 +383,7 @@ else if (user == null)
dataMap.put("user_login", user);
dataMap.put("community", community);
dataMap.put("created_at", nowStr);
dataMap.put("platform", platform);
request.add(new IndexRequest("new_year_report", "_doc", uuid + nowStr).source(dataMap));
if (request.requests().size() != 0)
restHighLevelClient.bulk(request, RequestOptions.DEFAULT);
Expand All @@ -392,13 +393,13 @@ else if (user == null)
return objectMapper.valueToTree(resMap).toString();
} catch (Exception e) {
logger.error("report exception - {}", e.getMessage());
return objectMapper.valueToTree(resMap).toString();
throw new RuntimeException(e.getMessage());
}
}

@SneakyThrows
public String queryNewYearMonthCount(CustomPropertiesConfig queryConf, String oauth2_proxy) {
String user = getUserFromCookie(queryConf, oauth2_proxy);
String user = getUserFromCookie(queryConf, oauth2_proxy, "gitee");
String queryJson = String.format(queryConf.getMonthCountQueryStr(), user);
ListenableFuture<Response> future = esAsyncHttpUtil.executeSearch(esUrl, queryConf.getGiteeAllIndex(), queryJson);
String responseBody = future.get().getResponseBody(UTF_8);
Expand All @@ -424,13 +425,22 @@ public String queryNewYearMonthCount(CustomPropertiesConfig queryConf, String oa
}

@SneakyThrows
private String getUserFromCookie(CustomPropertiesConfig queryConf, String oauth2_proxy) {
private String getUserFromCookie(CustomPropertiesConfig queryConf, String oauth2_proxy, String platform) {
String cookie_oauth2_proxy = "_oauth2_proxy=" + oauth2_proxy;
HttpResponse<String> response = Unirest.get(queryConf.getGiteeUserInfoUrl())
String userInfoUrl;
if (Constant.GITEE_PLATFORM.equalsIgnoreCase(platform)) {
userInfoUrl = queryConf.getGiteeUserInfoUrl();
} else if (Constant.GITHUB_PLATFORM.equalsIgnoreCase(platform)) {
userInfoUrl = queryConf.getGithubUserInfoUrl();
} else {
throw new RuntimeException("error platform");
}
HttpResponse<String> response = Unirest.get(userInfoUrl)
.header("cookie", cookie_oauth2_proxy)
.asString();

if (response.getStatus() != 200) {
logger.error("user auth execption - {}", response.getBody());
throw new Exception("unauthorized");
}
JsonNode res = objectMapper.readTree(response.getBody());
Expand Down Expand Up @@ -1470,7 +1480,7 @@ public String getOneIdUserGiteeLoginName(HttpServletRequest request) {
}
}
} catch (Exception e) {
throw new RuntimeException(e);
throw new RuntimeException(e.getMessage());
}
return null;
}
Expand Down Expand Up @@ -3602,7 +3612,7 @@ public String queryUserOwnerRepos(CustomPropertiesConfig queryConf, String user)
}

@SneakyThrows
public String saveFrontendEvents(String community, String requestBody) {
public String saveFrontendEvents(String community, String requestBody, String clientIp) {
// 检测请求体是否含有header和body
boolean hasHeader = requestBody.contains("\"header\"");
boolean hasBody = requestBody.contains("\"body\"");
Expand Down Expand Up @@ -3650,6 +3660,7 @@ public String saveFrontendEvents(String community, String requestBody) {

eventObj.put("created_at", nowStr);
eventObj.put("community", community);
eventObj.put("clientIp", clientIp);

JsonNode mergedJson = objectMapper.updateValue(eventObj, headerObj);

Expand Down Expand Up @@ -3717,7 +3728,7 @@ public String putSearchNpsByCommunity(CustomPropertiesConfig queryConf, String t
resMap.put("community", community);
String userId = "";
if (token != null && !"mindspore".equals(community)) {
userId = userIdDao.getUserIdByCommunity(token, community);
userId = userIdDao.getUserIdByCommunity(token, queryConf);
if (null == userId || userId.equals("")) {
logger.warn("UserId parse error for token:" + token + ",community:" + community);
throw new IllegalArgumentException("UserId parse error");
Expand Down Expand Up @@ -3924,26 +3935,65 @@ public String getCommunityMonthDowncount(CustomPropertiesConfig queryConf, Strin
* Retrieves the view count statistics for a specified community and repository.
*
* @param queryConf Custom configuration properties containing necessary query configurations.
* @param repoType The type of the repository, passed as a request parameter.
* @param owner The owner of the repository, passed as a request parameter.
* @param repo The repo name of the repository, passed as a request parameter.
* @param condition The search condition of the repository
* @return A JSON string containing the monthly download count statistics.
* @throws Exception If an error occurs during the query process.
*/
@SneakyThrows
public String getViewCount(CustomPropertiesConfig queryConf, String repoType, String owner, String repo) {
String query = String.format(queryConf.getRepoViewCountQueryStr(), repoType, owner, repo);
ListenableFuture<Response> future = esAsyncHttpUtil.executeCount(esUrl, queryConf.getExportWebsiteViewIndex(), query);
public String getViewCount(CustomPropertiesConfig queryConf, RequestParams condition) {
String query = String.format(queryConf.getRepoViewCountQueryStr(), condition.getStart(),
condition.getEnd(), condition.getRepoType(), condition.getRepoId());
String index = queryConf.getExportWebsiteViewIndex();
ListenableFuture<Response> future = esAsyncHttpUtil.executeSearch(esUrl, index, query);
Response response = future.get();
int statusCode = response.getStatusCode();
String statusText = response.getStatusText();
String responseBody = response.getResponseBody(UTF_8);
JsonNode dataNode = objectMapper.readTree(responseBody);
long count = dataNode.get("count").asLong();
Map<String, Object> resData = new HashMap<>();
resData.put("owner", owner);
resData.put("repo", repo);
resData.put("count", count);
return ResultUtil.resultJsonStr(statusCode, objectMapper.valueToTree(resData), statusText);
JsonNode testStr = dataNode.get("aggregations").get("group_field").get("buckets");
ArrayNode buckets = objectMapper.createArrayNode();
if (testStr.isArray()) {
for (int i = 0; i < testStr.size(); i++) {
JsonNode item = testStr.get(i);
ObjectNode bucket = objectMapper.createObjectNode();
bucket.put("repo_id", item.get("key").asText());
bucket.put("count", item.get("doc_count").asInt());
buckets.add(bucket);
}
}
return ResultUtil.resultJsonStr(statusCode, buckets, statusText);
}

/**
* Retrieves the view count for modeler's blog.
*
* @param queryConf Custom properties configuration object containing the necessary configuration information for the query.
* @return A JSON string containing the status code, result data, and status text.
* @throws Exception If an exception occurs during the execution of the query or processing of the response.
*/
@SneakyThrows
public String getModelersBlogViewCount(CustomPropertiesConfig queryConf) {
String query = String.format(queryConf.getModelersBlogViewCountQueryStr());
ListenableFuture<Response> future = esAsyncHttpUtil.executeSearch(esUrl, queryConf.getExportWebsiteViewIndex(), query);
Response response = future.get();
int statusCode = response.getStatusCode();
String statusText = response.getStatusText();
String responseBody = response.getResponseBody(UTF_8);
JsonNode dataNode = objectMapper.readTree(responseBody);
var buckets = dataNode.get("aggregations").get("group_by_id_and_title").get("buckets");
var resJsonArray = objectMapper.createArrayNode();
try {
for (var bucket : buckets) {
var jsonObject = objectMapper.createObjectNode();
jsonObject.put("id", bucket.get("key").get("id").asText());
jsonObject.put("title", bucket.get("key").get("title").asText());
jsonObject.put("count", bucket.get("doc_count").asInt());
resJsonArray.add(jsonObject);
}
} catch (Exception e) {
logger.error("query/modelers/blog/view get error - {}", e.getMessage());
return ResultUtil.resultJsonStr(statusCode, objectMapper.valueToTree(resJsonArray), "No data found or query error");
}
return ResultUtil.resultJsonStr(statusCode, objectMapper.valueToTree(resJsonArray), statusText);
}
}
8 changes: 5 additions & 3 deletions src/main/java/com/datastat/dao/UserIdDao.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@

import com.auth0.jwt.JWT;
import com.auth0.jwt.interfaces.DecodedJWT;
import com.datastat.model.CustomPropertiesConfig;
import com.datastat.util.RSAUtil;

import org.springframework.beans.factory.annotation.Autowired;
Expand All @@ -42,14 +43,15 @@ public String getUserId(String token){
return userId;
}

public String getUserIdByCommunity(String token, String community) {
public String getUserIdByCommunity(String token, CustomPropertiesConfig queryConf) {
String userId = null;
try {
RSAPrivateKey privateKey = RSAUtil.getPrivateKey(env.getProperty("rsa.authing." + community + ".privateKey"));
String authPrivateKey = queryConf.getRsaAuthPrivateKey();
RSAPrivateKey privateKey = RSAUtil.getPrivateKey(authPrivateKey);
DecodedJWT decode = JWT.decode(RSAUtil.privateDecrypt(token, privateKey));
userId = decode.getAudience().get(0);
} catch (Exception e) {
logger.error("parse token exception - {}", e.getMessage());
logger.error("parse user id from token exception - {}", e.getMessage());
}
return userId;
}
Expand Down
Loading

0 comments on commit c7b6487

Please sign in to comment.