Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Feature/1230 #299

Closed
wants to merge 6 commits into from
Closed

Feature/1230 #299

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
8d1f951
适配不同社区不同账号鉴权
Kaede10 Dec 23, 2024
f0a7665
Merge pull request #292 from opensourceways/wcp
zhongjun2 Dec 23, 2024
54c7f8d
add track clientip, add github report api.
Kaede10 Dec 24, 2024
dc46cfb
Merge pull request #293 from opensourceways/wcp
zhongjun2 Dec 24, 2024
4b880c3
修改仓库访问量处理逻辑
Kaede10 Dec 25, 2024
b817751
Merge pull request #294 from opensourceways/wcp
Kaede10 Dec 26, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 7 additions & 7 deletions pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@
<dependency>
<groupId>org.asynchttpclient</groupId>
<artifactId>async-http-client</artifactId>
<version>2.12.1</version>
<version>3.0.1</version>
</dependency>
<dependency>
<groupId>com.google.guava</groupId>
Expand Down Expand Up @@ -108,7 +108,7 @@
<dependency>
<groupId>org.apache.kafka</groupId>
<artifactId>kafka-clients</artifactId>
<version>2.8.1</version>
<version>3.9.0</version>
</dependency>

<dependency>
Expand Down Expand Up @@ -159,22 +159,22 @@
<dependency>
<groupId>io.netty</groupId>
<artifactId>netty-handler</artifactId>
<version>4.1.108.Final</version>
<version>4.1.115.Final</version>
</dependency>
<dependency>
<groupId>io.netty</groupId>
<artifactId>netty-common</artifactId>
<version>4.1.108.Final</version>
<version>4.1.115.Final</version>
</dependency>
<dependency>
<groupId>io.netty</groupId>
<artifactId>netty-codec</artifactId>
<version>4.1.108.Final</version>
<version>4.1.115.Final</version>
</dependency>
<dependency>
<groupId>io.netty</groupId>
<artifactId>netty-codec-http</artifactId>
<version>4.1.108.Final</version>
<version>4.1.115.Final</version>
</dependency>

<dependency>
Expand Down Expand Up @@ -207,7 +207,7 @@
<dependency>
<groupId>org.springframework</groupId>
<artifactId>spring-web</artifactId>
<version>6.1.6</version>
<version>6.1.12</version>
</dependency>

<dependency>
Expand Down
12 changes: 11 additions & 1 deletion src/main/java/com/datastat/constant/Constant.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@ private Constant() {
/**
* VALID_OPENMIND_ENV_REG used to match input string.
*/
public static final String VALID_OPENMIND_ENV_REG = "pro|yidong|sh";
public static final String VALID_OPENMIND_ENV_REG = "pro|tianyi|sh";

/**
* VALID_REPO_ID used to match input string.
Expand All @@ -51,6 +51,16 @@ private Constant() {
*/
public static final String OPENMIND_COMMUNITY = "openmind";

/**
* The name of github platform.
*/
public static final String GITHUB_PLATFORM = "github";

/**
* The name of gutee platform.
*/
public static final String GITEE_PLATFORM = "GITEE";

/**
* 支持性能数据上传的社区.
*/
Expand Down
15 changes: 9 additions & 6 deletions src/main/java/com/datastat/controller/QueryController.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -152,7 +152,13 @@ public String queryCveDetails(HttpServletRequest request,
@RequestMapping("/newYear/report")
public String queryNewYear(HttpServletRequest request,
@CookieValue(value = "_oauth2_proxy", required = true) String oauth2_proxy) {
return queryService.queryNewYearPer(request, oauth2_proxy);
return queryService.queryNewYearPer(request, oauth2_proxy, "gitee");
}

@RequestMapping("/newYear/report/github")
public String queryNewYearGithub(HttpServletRequest request,
@CookieValue(value = "_oauth2_proxy", required = true) String oauth2_proxy) {
return queryService.queryNewYearPer(request, oauth2_proxy, "github");
}

@RequestMapping("/newYear/monthcount")
Expand Down Expand Up @@ -830,10 +836,7 @@ public String monthDownCount(HttpServletRequest request,
* @return A string containing the monthly download count information for the repository.
*/
@RequestMapping(value = "/view/count")
public String viewCount(HttpServletRequest request,
@RequestParam(value = "repoType") String repoType,
@RequestParam(value = "owner") String owner,
@RequestParam(value = "repo") String repo) {
return queryService.getViewCount(request, repoType, owner, repo);
public String viewCount(HttpServletRequest request, @Valid final RequestParams condition) {
return queryService.getViewCount(request, condition);
}
}
66 changes: 41 additions & 25 deletions src/main/java/com/datastat/dao/QueryDao.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -356,18 +356,18 @@ public String queryCveDetails(CustomPropertiesConfig queryConf, String lastCurso
return esQueryUtils.esFromId(restHighLevelClient, item, lastCursor, Integer.parseInt(pageSize), indexName);
}

public String queryNewYearPer(CustomPropertiesConfig queryConf, String oauth2_proxy, String community) {
public String queryNewYearPer(CustomPropertiesConfig queryConf, String oauth2_proxy, String community,
String platform) {
HashMap<String, Object> resMap = new HashMap<>();
try {
String user = getUserFromCookie(queryConf, oauth2_proxy);
String localFile = env.getProperty("export_path") + community.toLowerCase() + "_" + env.getProperty("year") + ".csv";
String localFile = String.format("%s%s_%s_%s.csv", env.getProperty("export_path"), community.toLowerCase(),
platform, env.getProperty("year"));
String user = getUserFromCookie(queryConf, oauth2_proxy, platform);
List<HashMap<String, Object>> report = CsvFileUtil.readFile(localFile);
resMap.put("code", 200);
resMap.put("msg", "OK");
if (report == null)
if (report == null || user == null)
resMap.put("data", new ArrayList<>());
else if (user == null)
resMap.put("data", report);
else {
List<HashMap<String, Object>> user_login = report.stream()
.filter(m -> m.getOrDefault("user_login", "").equals(user)).collect(Collectors.toList());
Expand All @@ -392,13 +392,13 @@ else if (user == null)
return objectMapper.valueToTree(resMap).toString();
} catch (Exception e) {
logger.error("report exception - {}", e.getMessage());
return objectMapper.valueToTree(resMap).toString();
throw new RuntimeException(e.getMessage());
}
}

@SneakyThrows
public String queryNewYearMonthCount(CustomPropertiesConfig queryConf, String oauth2_proxy) {
String user = getUserFromCookie(queryConf, oauth2_proxy);
String user = getUserFromCookie(queryConf, oauth2_proxy, "gitee");
String queryJson = String.format(queryConf.getMonthCountQueryStr(), user);
ListenableFuture<Response> future = esAsyncHttpUtil.executeSearch(esUrl, queryConf.getGiteeAllIndex(), queryJson);
String responseBody = future.get().getResponseBody(UTF_8);
Expand All @@ -424,13 +424,22 @@ public String queryNewYearMonthCount(CustomPropertiesConfig queryConf, String oa
}

@SneakyThrows
private String getUserFromCookie(CustomPropertiesConfig queryConf, String oauth2_proxy) {
private String getUserFromCookie(CustomPropertiesConfig queryConf, String oauth2_proxy, String platform) {
String cookie_oauth2_proxy = "_oauth2_proxy=" + oauth2_proxy;
HttpResponse<String> response = Unirest.get(queryConf.getGiteeUserInfoUrl())
String userInfoUrl;
if (Constant.GITEE_PLATFORM.equalsIgnoreCase(platform)) {
userInfoUrl = queryConf.getGiteeUserInfoUrl();
} else if (Constant.GITHUB_PLATFORM.equalsIgnoreCase(platform)) {
userInfoUrl = queryConf.getGithubUserInfoUrl();
} else {
throw new RuntimeException("error platform");
}
HttpResponse<String> response = Unirest.get(userInfoUrl)
.header("cookie", cookie_oauth2_proxy)
.asString();

if (response.getStatus() != 200) {
logger.error("user auth execption - {}", response.getBody());
throw new Exception("unauthorized");
}
JsonNode res = objectMapper.readTree(response.getBody());
Expand Down Expand Up @@ -1470,7 +1479,7 @@ public String getOneIdUserGiteeLoginName(HttpServletRequest request) {
}
}
} catch (Exception e) {
throw new RuntimeException(e);
throw new RuntimeException(e.getMessage());
}
return null;
}
Expand Down Expand Up @@ -3602,7 +3611,7 @@ public String queryUserOwnerRepos(CustomPropertiesConfig queryConf, String user)
}

@SneakyThrows
public String saveFrontendEvents(String community, String requestBody) {
public String saveFrontendEvents(String community, String requestBody, String clientIp) {
// 检测请求体是否含有header和body
boolean hasHeader = requestBody.contains("\"header\"");
boolean hasBody = requestBody.contains("\"body\"");
Expand Down Expand Up @@ -3650,6 +3659,7 @@ public String saveFrontendEvents(String community, String requestBody) {

eventObj.put("created_at", nowStr);
eventObj.put("community", community);
eventObj.put("clientIp", clientIp);

JsonNode mergedJson = objectMapper.updateValue(eventObj, headerObj);

Expand Down Expand Up @@ -3717,7 +3727,7 @@ public String putSearchNpsByCommunity(CustomPropertiesConfig queryConf, String t
resMap.put("community", community);
String userId = "";
if (token != null && !"mindspore".equals(community)) {
userId = userIdDao.getUserIdByCommunity(token, community);
userId = userIdDao.getUserIdByCommunity(token, queryConf);
if (null == userId || userId.equals("")) {
logger.warn("UserId parse error for token:" + token + ",community:" + community);
throw new IllegalArgumentException("UserId parse error");
Expand Down Expand Up @@ -3924,26 +3934,32 @@ public String getCommunityMonthDowncount(CustomPropertiesConfig queryConf, Strin
* Retrieves the view count statistics for a specified community and repository.
*
* @param queryConf Custom configuration properties containing necessary query configurations.
* @param repoType The type of the repository, passed as a request parameter.
* @param owner The owner of the repository, passed as a request parameter.
* @param repo The repo name of the repository, passed as a request parameter.
* @param condition The search condition of the repository
* @return A JSON string containing the monthly download count statistics.
* @throws Exception If an error occurs during the query process.
*/
@SneakyThrows
public String getViewCount(CustomPropertiesConfig queryConf, String repoType, String owner, String repo) {
String query = String.format(queryConf.getRepoViewCountQueryStr(), repoType, owner, repo);
ListenableFuture<Response> future = esAsyncHttpUtil.executeCount(esUrl, queryConf.getExportWebsiteViewIndex(), query);
public String getViewCount(CustomPropertiesConfig queryConf, RequestParams condition) {
String query = String.format(queryConf.getRepoViewCountQueryStr(), condition.getStart(),
condition.getEnd(), condition.getRepoType(), condition.getRepoId());
String index = queryConf.getExportWebsiteViewIndex();
ListenableFuture<Response> future = esAsyncHttpUtil.executeSearch(esUrl, index, query);
Response response = future.get();
int statusCode = response.getStatusCode();
String statusText = response.getStatusText();
String responseBody = response.getResponseBody(UTF_8);
JsonNode dataNode = objectMapper.readTree(responseBody);
long count = dataNode.get("count").asLong();
Map<String, Object> resData = new HashMap<>();
resData.put("owner", owner);
resData.put("repo", repo);
resData.put("count", count);
return ResultUtil.resultJsonStr(statusCode, objectMapper.valueToTree(resData), statusText);
JsonNode testStr = dataNode.get("aggregations").get("group_field").get("buckets");
ArrayNode buckets = objectMapper.createArrayNode();
if (testStr.isArray()) {
for (int i = 0; i < testStr.size(); i++) {
JsonNode item = testStr.get(i);
ObjectNode bucket = objectMapper.createObjectNode();
bucket.put("repo_id", item.get("key").asText());
bucket.put("count", item.get("doc_count").asInt());
buckets.add(bucket);
}
}
return ResultUtil.resultJsonStr(statusCode, buckets, statusText);
}
}
8 changes: 5 additions & 3 deletions src/main/java/com/datastat/dao/UserIdDao.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@

import com.auth0.jwt.JWT;
import com.auth0.jwt.interfaces.DecodedJWT;
import com.datastat.model.CustomPropertiesConfig;
import com.datastat.util.RSAUtil;

import org.springframework.beans.factory.annotation.Autowired;
Expand All @@ -42,14 +43,15 @@ public String getUserId(String token){
return userId;
}

public String getUserIdByCommunity(String token, String community) {
public String getUserIdByCommunity(String token, CustomPropertiesConfig queryConf) {
String userId = null;
try {
RSAPrivateKey privateKey = RSAUtil.getPrivateKey(env.getProperty("rsa.authing." + community + ".privateKey"));
String authPrivateKey = queryConf.getRsaAuthPrivateKey();
RSAPrivateKey privateKey = RSAUtil.getPrivateKey(authPrivateKey);
DecodedJWT decode = JWT.decode(RSAUtil.privateDecrypt(token, privateKey));
userId = decode.getAudience().get(0);
} catch (Exception e) {
logger.error("parse token exception - {}", e.getMessage());
logger.error("parse user id from token exception - {}", e.getMessage());
}
return userId;
}
Expand Down
Loading
Loading