Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

merge Release/1230 into main #305

Merged
merged 9 commits into from
Dec 31, 2024
Merged

merge Release/1230 into main #305

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
8d1f951
适配不同社区不同账号鉴权
Kaede10 Dec 23, 2024
f0a7665
Merge pull request #292 from opensourceways/wcp
zhongjun2 Dec 23, 2024
54c7f8d
add track clientip, add github report api.
Kaede10 Dec 24, 2024
dc46cfb
Merge pull request #293 from opensourceways/wcp
zhongjun2 Dec 24, 2024
0a55a26
修改仓库访问量处理逻辑
Kaede10 Dec 27, 2024
28dac8b
Merge pull request #302 from opensourceways/feature/1230
zhongjun2 Dec 27, 2024
4d9f563
Add modelers blog view count API
G-Sacker Dec 30, 2024
cc79d68
年报用户数据采集区分贡献平台
Kaede10 Dec 30, 2024
f3b4e15
Merge pull request #304 from opensourceways/feature/1230
zhongjun2 Dec 30, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 9 additions & 9 deletions pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@
<dependency>
<groupId>org.asynchttpclient</groupId>
<artifactId>async-http-client</artifactId>
<version>2.12.1</version>
<version>3.0.1</version>
</dependency>
<dependency>
<groupId>com.google.guava</groupId>
Expand Down Expand Up @@ -108,7 +108,7 @@
<dependency>
<groupId>org.apache.kafka</groupId>
<artifactId>kafka-clients</artifactId>
<version>2.8.1</version>
<version>3.9.0</version>
</dependency>

<dependency>
Expand Down Expand Up @@ -159,22 +159,22 @@
<dependency>
<groupId>io.netty</groupId>
<artifactId>netty-handler</artifactId>
<version>4.1.108.Final</version>
<version>4.1.115.Final</version>
</dependency>
<dependency>
<groupId>io.netty</groupId>
<artifactId>netty-common</artifactId>
<version>4.1.108.Final</version>
<version>4.1.115.Final</version>
</dependency>
<dependency>
<groupId>io.netty</groupId>
<artifactId>netty-codec</artifactId>
<version>4.1.108.Final</version>
<version>4.1.115.Final</version>
</dependency>
<dependency>
<groupId>io.netty</groupId>
<artifactId>netty-codec-http</artifactId>
<version>4.1.108.Final</version>
<version>4.1.115.Final</version>
</dependency>

<dependency>
Expand Down Expand Up @@ -207,19 +207,19 @@
<dependency>
<groupId>org.springframework</groupId>
<artifactId>spring-web</artifactId>
<version>6.1.6</version>
<version>6.1.12</version>
</dependency>

<dependency>
<groupId>org.apache.tomcat.embed</groupId>
<artifactId>tomcat-embed-core</artifactId>
<version>10.1.25</version>
<version>10.1.34</version>
</dependency>

<dependency>
<groupId>org.apache.tomcat.embed</groupId>
<artifactId>tomcat-embed-websocket</artifactId>
<version>10.1.25</version>
<version>10.1.34</version>
</dependency>
</dependencies>

Expand Down
12 changes: 11 additions & 1 deletion src/main/java/com/datastat/constant/Constant.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@ private Constant() {
/**
* VALID_OPENMIND_ENV_REG used to match input string.
*/
public static final String VALID_OPENMIND_ENV_REG = "pro|yidong|sh";
public static final String VALID_OPENMIND_ENV_REG = "pro|tianyi|sh";

/**
* VALID_REPO_ID used to match input string.
Expand All @@ -51,6 +51,16 @@ private Constant() {
*/
public static final String OPENMIND_COMMUNITY = "openmind";

/**
* The name of github platform.
*/
public static final String GITHUB_PLATFORM = "github";

/**
* The name of gutee platform.
*/
public static final String GITEE_PLATFORM = "GITEE";

/**
* 支持性能数据上传的社区.
*/
Expand Down
27 changes: 21 additions & 6 deletions src/main/java/com/datastat/controller/QueryController.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -152,7 +152,13 @@ public String queryCveDetails(HttpServletRequest request,
@RequestMapping("/newYear/report")
public String queryNewYear(HttpServletRequest request,
@CookieValue(value = "_oauth2_proxy", required = true) String oauth2_proxy) {
return queryService.queryNewYearPer(request, oauth2_proxy);
return queryService.queryNewYearPer(request, oauth2_proxy, "gitee");
}

@RequestMapping("/newYear/report/github")
public String queryNewYearGithub(HttpServletRequest request,
@CookieValue(value = "_oauth2_proxy", required = true) String oauth2_proxy) {
return queryService.queryNewYearPer(request, oauth2_proxy, "github");
}

@RequestMapping("/newYear/monthcount")
Expand Down Expand Up @@ -830,10 +836,19 @@ public String monthDownCount(HttpServletRequest request,
* @return A string containing the monthly download count information for the repository.
*/
@RequestMapping(value = "/view/count")
public String viewCount(HttpServletRequest request,
@RequestParam(value = "repoType") String repoType,
@RequestParam(value = "owner") String owner,
@RequestParam(value = "repo") String repo) {
return queryService.getViewCount(request, repoType, owner, repo);
public String viewCount(HttpServletRequest request, @Valid final RequestParams condition) {
return queryService.getViewCount(request, condition);
}


/**
* Handles the request to view the modelers blog count.
*
* @param request the HttpServletRequest object containing client request data
* @return a String representing the view count of the modelers blog
*/
@RequestMapping(value = "/modelers/blog/view")
public String modelersBlogView(HttpServletRequest request) {
return queryService.getModelersBlogViewCount(request);
}
}
100 changes: 75 additions & 25 deletions src/main/java/com/datastat/dao/QueryDao.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -356,18 +356,18 @@ public String queryCveDetails(CustomPropertiesConfig queryConf, String lastCurso
return esQueryUtils.esFromId(restHighLevelClient, item, lastCursor, Integer.parseInt(pageSize), indexName);
}

public String queryNewYearPer(CustomPropertiesConfig queryConf, String oauth2_proxy, String community) {
public String queryNewYearPer(CustomPropertiesConfig queryConf, String oauth2_proxy, String community,
String platform) {
HashMap<String, Object> resMap = new HashMap<>();
try {
String user = getUserFromCookie(queryConf, oauth2_proxy);
String localFile = env.getProperty("export_path") + community.toLowerCase() + "_" + env.getProperty("year") + ".csv";
String localFile = String.format("%s%s_%s_%s.csv", env.getProperty("export_path"), community.toLowerCase(),
platform, env.getProperty("year"));
String user = getUserFromCookie(queryConf, oauth2_proxy, platform);
List<HashMap<String, Object>> report = CsvFileUtil.readFile(localFile);
resMap.put("code", 200);
resMap.put("msg", "OK");
if (report == null)
if (report == null || user == null)
resMap.put("data", new ArrayList<>());
else if (user == null)
resMap.put("data", report);
else {
List<HashMap<String, Object>> user_login = report.stream()
.filter(m -> m.getOrDefault("user_login", "").equals(user)).collect(Collectors.toList());
Expand All @@ -383,6 +383,7 @@ else if (user == null)
dataMap.put("user_login", user);
dataMap.put("community", community);
dataMap.put("created_at", nowStr);
dataMap.put("platform", platform);
request.add(new IndexRequest("new_year_report", "_doc", uuid + nowStr).source(dataMap));
if (request.requests().size() != 0)
restHighLevelClient.bulk(request, RequestOptions.DEFAULT);
Expand All @@ -392,13 +393,13 @@ else if (user == null)
return objectMapper.valueToTree(resMap).toString();
} catch (Exception e) {
logger.error("report exception - {}", e.getMessage());
return objectMapper.valueToTree(resMap).toString();
throw new RuntimeException(e.getMessage());
}
}

@SneakyThrows
public String queryNewYearMonthCount(CustomPropertiesConfig queryConf, String oauth2_proxy) {
String user = getUserFromCookie(queryConf, oauth2_proxy);
String user = getUserFromCookie(queryConf, oauth2_proxy, "gitee");
String queryJson = String.format(queryConf.getMonthCountQueryStr(), user);
ListenableFuture<Response> future = esAsyncHttpUtil.executeSearch(esUrl, queryConf.getGiteeAllIndex(), queryJson);
String responseBody = future.get().getResponseBody(UTF_8);
Expand All @@ -424,13 +425,22 @@ public String queryNewYearMonthCount(CustomPropertiesConfig queryConf, String oa
}

@SneakyThrows
private String getUserFromCookie(CustomPropertiesConfig queryConf, String oauth2_proxy) {
private String getUserFromCookie(CustomPropertiesConfig queryConf, String oauth2_proxy, String platform) {
String cookie_oauth2_proxy = "_oauth2_proxy=" + oauth2_proxy;
HttpResponse<String> response = Unirest.get(queryConf.getGiteeUserInfoUrl())
String userInfoUrl;
if (Constant.GITEE_PLATFORM.equalsIgnoreCase(platform)) {
userInfoUrl = queryConf.getGiteeUserInfoUrl();
} else if (Constant.GITHUB_PLATFORM.equalsIgnoreCase(platform)) {
userInfoUrl = queryConf.getGithubUserInfoUrl();
} else {
throw new RuntimeException("error platform");
}
HttpResponse<String> response = Unirest.get(userInfoUrl)
.header("cookie", cookie_oauth2_proxy)
.asString();

if (response.getStatus() != 200) {
logger.error("user auth execption - {}", response.getBody());
throw new Exception("unauthorized");
}
JsonNode res = objectMapper.readTree(response.getBody());
Expand Down Expand Up @@ -1470,7 +1480,7 @@ public String getOneIdUserGiteeLoginName(HttpServletRequest request) {
}
}
} catch (Exception e) {
throw new RuntimeException(e);
throw new RuntimeException(e.getMessage());
}
return null;
}
Expand Down Expand Up @@ -3602,7 +3612,7 @@ public String queryUserOwnerRepos(CustomPropertiesConfig queryConf, String user)
}

@SneakyThrows
public String saveFrontendEvents(String community, String requestBody) {
public String saveFrontendEvents(String community, String requestBody, String clientIp) {
// 检测请求体是否含有header和body
boolean hasHeader = requestBody.contains("\"header\"");
boolean hasBody = requestBody.contains("\"body\"");
Expand Down Expand Up @@ -3650,6 +3660,7 @@ public String saveFrontendEvents(String community, String requestBody) {

eventObj.put("created_at", nowStr);
eventObj.put("community", community);
eventObj.put("clientIp", clientIp);

JsonNode mergedJson = objectMapper.updateValue(eventObj, headerObj);

Expand Down Expand Up @@ -3717,7 +3728,7 @@ public String putSearchNpsByCommunity(CustomPropertiesConfig queryConf, String t
resMap.put("community", community);
String userId = "";
if (token != null && !"mindspore".equals(community)) {
userId = userIdDao.getUserIdByCommunity(token, community);
userId = userIdDao.getUserIdByCommunity(token, queryConf);
if (null == userId || userId.equals("")) {
logger.warn("UserId parse error for token:" + token + ",community:" + community);
throw new IllegalArgumentException("UserId parse error");
Expand Down Expand Up @@ -3924,26 +3935,65 @@ public String getCommunityMonthDowncount(CustomPropertiesConfig queryConf, Strin
* Retrieves the view count statistics for a specified community and repository.
*
* @param queryConf Custom configuration properties containing necessary query configurations.
* @param repoType The type of the repository, passed as a request parameter.
* @param owner The owner of the repository, passed as a request parameter.
* @param repo The repo name of the repository, passed as a request parameter.
* @param condition The search condition of the repository
* @return A JSON string containing the monthly download count statistics.
* @throws Exception If an error occurs during the query process.
*/
@SneakyThrows
public String getViewCount(CustomPropertiesConfig queryConf, String repoType, String owner, String repo) {
String query = String.format(queryConf.getRepoViewCountQueryStr(), repoType, owner, repo);
ListenableFuture<Response> future = esAsyncHttpUtil.executeCount(esUrl, queryConf.getExportWebsiteViewIndex(), query);
public String getViewCount(CustomPropertiesConfig queryConf, RequestParams condition) {
String query = String.format(queryConf.getRepoViewCountQueryStr(), condition.getStart(),
condition.getEnd(), condition.getRepoType(), condition.getRepoId());
String index = queryConf.getExportWebsiteViewIndex();
ListenableFuture<Response> future = esAsyncHttpUtil.executeSearch(esUrl, index, query);
Response response = future.get();
int statusCode = response.getStatusCode();
String statusText = response.getStatusText();
String responseBody = response.getResponseBody(UTF_8);
JsonNode dataNode = objectMapper.readTree(responseBody);
long count = dataNode.get("count").asLong();
Map<String, Object> resData = new HashMap<>();
resData.put("owner", owner);
resData.put("repo", repo);
resData.put("count", count);
return ResultUtil.resultJsonStr(statusCode, objectMapper.valueToTree(resData), statusText);
JsonNode testStr = dataNode.get("aggregations").get("group_field").get("buckets");
ArrayNode buckets = objectMapper.createArrayNode();
if (testStr.isArray()) {
for (int i = 0; i < testStr.size(); i++) {
JsonNode item = testStr.get(i);
ObjectNode bucket = objectMapper.createObjectNode();
bucket.put("repo_id", item.get("key").asText());
bucket.put("count", item.get("doc_count").asInt());
buckets.add(bucket);
}
}
return ResultUtil.resultJsonStr(statusCode, buckets, statusText);
}

/**
* Retrieves the view count for modeler's blog.
*
* @param queryConf Custom properties configuration object containing the necessary configuration information for the query.
* @return A JSON string containing the status code, result data, and status text.
* @throws Exception If an exception occurs during the execution of the query or processing of the response.
*/
@SneakyThrows
public String getModelersBlogViewCount(CustomPropertiesConfig queryConf) {
String query = String.format(queryConf.getModelersBlogViewCountQueryStr());
ListenableFuture<Response> future = esAsyncHttpUtil.executeSearch(esUrl, queryConf.getExportWebsiteViewIndex(), query);
Response response = future.get();
int statusCode = response.getStatusCode();
String statusText = response.getStatusText();
String responseBody = response.getResponseBody(UTF_8);
JsonNode dataNode = objectMapper.readTree(responseBody);
var buckets = dataNode.get("aggregations").get("group_by_id_and_title").get("buckets");
var resJsonArray = objectMapper.createArrayNode();
try {
for (var bucket : buckets) {
var jsonObject = objectMapper.createObjectNode();
jsonObject.put("id", bucket.get("key").get("id").asText());
jsonObject.put("title", bucket.get("key").get("title").asText());
jsonObject.put("count", bucket.get("doc_count").asInt());
resJsonArray.add(jsonObject);
}
} catch (Exception e) {
logger.error("query/modelers/blog/view get error - {}", e.getMessage());
return ResultUtil.resultJsonStr(statusCode, objectMapper.valueToTree(resJsonArray), "No data found or query error");
}
return ResultUtil.resultJsonStr(statusCode, objectMapper.valueToTree(resJsonArray), statusText);
}
}
8 changes: 5 additions & 3 deletions src/main/java/com/datastat/dao/UserIdDao.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@

import com.auth0.jwt.JWT;
import com.auth0.jwt.interfaces.DecodedJWT;
import com.datastat.model.CustomPropertiesConfig;
import com.datastat.util.RSAUtil;

import org.springframework.beans.factory.annotation.Autowired;
Expand All @@ -42,14 +43,15 @@ public String getUserId(String token){
return userId;
}

public String getUserIdByCommunity(String token, String community) {
public String getUserIdByCommunity(String token, CustomPropertiesConfig queryConf) {
String userId = null;
try {
RSAPrivateKey privateKey = RSAUtil.getPrivateKey(env.getProperty("rsa.authing." + community + ".privateKey"));
String authPrivateKey = queryConf.getRsaAuthPrivateKey();
RSAPrivateKey privateKey = RSAUtil.getPrivateKey(authPrivateKey);
DecodedJWT decode = JWT.decode(RSAUtil.privateDecrypt(token, privateKey));
userId = decode.getAudience().get(0);
} catch (Exception e) {
logger.error("parse token exception - {}", e.getMessage());
logger.error("parse user id from token exception - {}", e.getMessage());
}
return userId;
}
Expand Down
Loading
Loading