diff --git a/.github/linters/checkstyle.xml b/.github/linters/checkstyle.xml deleted file mode 100644 index fb7ba1b..0000000 --- a/.github/linters/checkstyle.xml +++ /dev/null @@ -1,209 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/.github/script/codearts_check.sh b/.github/script/codearts_check.sh new file mode 100644 index 0000000..b0b84e2 --- /dev/null +++ b/.github/script/codearts_check.sh @@ -0,0 +1,79 @@ + +IAM_DATA=$(cat < label.name); + if (!labels.includes(requiredLabel)) { + throw new Error(`PR 必须包含标签: ${requiredLabel}`); + } diff --git a/.github/workflows/linter.yml b/.github/workflows/linter.yml deleted file mode 100644 index 0c13a16..0000000 --- a/.github/workflows/linter.yml +++ /dev/null @@ -1,35 +0,0 @@ -name: Lint Code Base -on: - push: - branches: [master, main, test, develop_test] - pull_request: - branches: [master, main, test, develop_test] -jobs: - build: - name: Lint - runs-on: ubuntu-latest - - permissions: - contents: read - packages: read - # To report GitHub Actions status checks - statuses: write - - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - # super-linter needs the full git history to get the - # list of files that changed across commits - fetch-depth: 0 - - - name: Super-linter - uses: super-linter/super-linter@v7.1.0 # x-release-please-version - env: - # To report GitHub Actions status checks - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - VALIDATE_JAVA: true - FILTER_REGEX_INCLUDE: .*src/.*/ # Only lint files in src/ - LINTER_RULES_PATH: .github/linters - JAVA_FILE_NAME: checkstyle.xml - diff --git a/pom.xml b/pom.xml index a22ffa3..4753d24 100644 --- a/pom.xml +++ b/pom.xml @@ -229,6 +229,36 @@ org.springframework.boot spring-boot-maven-plugin + + org.jacoco + jacoco-maven-plugin + 0.8.7 + + + default-prepare-agent + + prepare-agent + + + + default-report + test + + report + + + + + + org.apache.maven.plugins + maven-surefire-plugin + 3.2.5 + + + com/datastat/ds/unit/*.java + + + diff --git a/src/main/java/com/datastat/config/FoundryConfig.java b/src/main/java/com/datastat/config/FoundryConfig.java index 6db2aec..b96ed04 100644 --- a/src/main/java/com/datastat/config/FoundryConfig.java +++ b/src/main/java/com/datastat/config/FoundryConfig.java @@ -18,7 +18,7 @@ import org.springframework.context.annotation.PropertySource; @ConfigurationProperties(prefix = "foundry") -@PropertySource(value = {"file:${config.path}/foundry.properties"}, encoding = "UTF-8") +@PropertySource(value = {"file:${config.path}/foundry.properties"}, ignoreResourceNotFound = true, encoding = "UTF-8") @Configuration("foundryConf") @Data public class FoundryConfig extends CustomPropertiesConfig { diff --git a/src/main/java/com/datastat/config/MindSporeConfig.java b/src/main/java/com/datastat/config/MindSporeConfig.java index 1e1982e..17ed852 100644 --- a/src/main/java/com/datastat/config/MindSporeConfig.java +++ b/src/main/java/com/datastat/config/MindSporeConfig.java @@ -18,7 +18,7 @@ import org.springframework.context.annotation.PropertySource; @ConfigurationProperties(prefix = "mindspore") -@PropertySource(value = {"file:${config.path}/mindspore.properties"}, encoding = "UTF-8") +@PropertySource(value = {"file:${config.path}/mindspore.properties"}, ignoreResourceNotFound = true, encoding = "UTF-8") @Configuration("mindsporeConf") @Data public class MindSporeConfig extends CustomPropertiesConfig { diff --git a/src/main/java/com/datastat/config/OpenEulerConfig.java b/src/main/java/com/datastat/config/OpenEulerConfig.java index bc07a55..1692a3a 100644 --- a/src/main/java/com/datastat/config/OpenEulerConfig.java +++ b/src/main/java/com/datastat/config/OpenEulerConfig.java @@ -18,7 +18,7 @@ import org.springframework.context.annotation.PropertySource; @ConfigurationProperties(prefix = "openeuler") -@PropertySource(value = {"file:${config.path}/openeuler.properties"}, encoding = "UTF-8") +@PropertySource(value = {"file:${config.path}/openeuler.properties"}, ignoreResourceNotFound = true, encoding = "UTF-8") @Configuration("openeulerConf") @Data public class OpenEulerConfig extends CustomPropertiesConfig { diff --git a/src/main/java/com/datastat/config/OpenGaussConfig.java b/src/main/java/com/datastat/config/OpenGaussConfig.java index 6c903e8..9c55fbc 100644 --- a/src/main/java/com/datastat/config/OpenGaussConfig.java +++ b/src/main/java/com/datastat/config/OpenGaussConfig.java @@ -26,7 +26,7 @@ import org.springframework.core.env.Environment; @ConfigurationProperties(prefix = "opengauss") -@PropertySource(value = {"file:${config.path}/opengauss.properties"}, encoding = "UTF-8") +@PropertySource(value = {"file:${config.path}/opengauss.properties"}, ignoreResourceNotFound = true, encoding = "UTF-8") @Configuration("opengaussConf") @Data public class OpenGaussConfig extends CustomPropertiesConfig { diff --git a/src/main/java/com/datastat/config/OpenLookengConfig.java b/src/main/java/com/datastat/config/OpenLookengConfig.java index 51b7108..c4f1195 100644 --- a/src/main/java/com/datastat/config/OpenLookengConfig.java +++ b/src/main/java/com/datastat/config/OpenLookengConfig.java @@ -18,7 +18,7 @@ import org.springframework.context.annotation.PropertySource; @ConfigurationProperties(prefix = "openlookeng") -@PropertySource(value = {"file:${config.path}/openlookeng.properties"}, encoding = "UTF-8") +@PropertySource(value = {"file:${config.path}/openlookeng.properties"}, ignoreResourceNotFound = true, encoding = "UTF-8") @Configuration("openlookengConf") @Data public class OpenLookengConfig extends CustomPropertiesConfig { diff --git a/src/main/java/com/datastat/config/QueryConfig.java b/src/main/java/com/datastat/config/QueryConfig.java index d29fb10..a83ca7a 100644 --- a/src/main/java/com/datastat/config/QueryConfig.java +++ b/src/main/java/com/datastat/config/QueryConfig.java @@ -18,7 +18,7 @@ import org.springframework.context.annotation.PropertySource; @ConfigurationProperties(prefix = "custom") -@PropertySource(value = {"file:${config.path}/custom.properties"}) +@PropertySource(value = {"file:${config.path}/custom.properties"}, ignoreResourceNotFound = true, encoding = "UTF-8") @Configuration("queryConf") @Data public class QueryConfig extends CustomPropertiesConfig { diff --git a/src/main/java/com/datastat/config/SoftwareConfig.java b/src/main/java/com/datastat/config/SoftwareConfig.java index 675ce7d..9f1a09c 100644 --- a/src/main/java/com/datastat/config/SoftwareConfig.java +++ b/src/main/java/com/datastat/config/SoftwareConfig.java @@ -9,7 +9,7 @@ import lombok.Data; @ConfigurationProperties(prefix = "software") -@PropertySource(value = {"file:${config.path}/software.properties"}, encoding = "UTF-8") +@PropertySource(value = {"file:${config.path}/software.properties"}, ignoreResourceNotFound = true, encoding = "UTF-8") @Configuration("softwareConf") @Data public class SoftwareConfig extends CustomPropertiesConfig { } diff --git a/src/main/java/com/datastat/constant/Constant.java b/src/main/java/com/datastat/constant/Constant.java index d051413..42f2e4b 100644 --- a/src/main/java/com/datastat/constant/Constant.java +++ b/src/main/java/com/datastat/constant/Constant.java @@ -16,6 +16,36 @@ private Constant() { public static final String FEEDBACK_REPO = "easy-software"; + /** + * VALID_APPROVAL_REG used to match input string. + */ + public static final String VALID_APPROVAL_REG = "model|dataset|space|^\\*$"; + + /** + * VALID_DATE_REG used to match input string. + */ + public static final String VALID_DATE_REG = "^(\\d{4})-(\\d{2})-(\\d{2})$"; + + /** + * START_DATE used to match input string. + */ + public static final String START_DATE = "1970-01-01"; + + /** + * END_DATE used to match input string. + */ + public static final String END_DATE = "2050-01-01"; + + /** + * VALID_OPENMIND_ENV_REG used to match input string. + */ + public static final String VALID_OPENMIND_ENV_REG = "pro|yidong|sh"; + + /** + * VALID_REPO_ID used to match input string. + */ + public static final String VALID_REPO_ID = "^\\d+$|^\\*$"; + /** * openmind社区. */ diff --git a/src/main/java/com/datastat/controller/QueryController.java b/src/main/java/com/datastat/controller/QueryController.java index 9c7310a..aa9a61a 100644 --- a/src/main/java/com/datastat/controller/QueryController.java +++ b/src/main/java/com/datastat/controller/QueryController.java @@ -26,6 +26,7 @@ import com.datastat.model.TeamupApplyForm; import com.datastat.model.dto.ContributeRequestParams; import com.datastat.model.dto.NpsIssueBody; +import com.datastat.model.dto.RequestParams; import com.datastat.model.meetup.MeetupApplyForm; import com.datastat.model.vo.*; import com.datastat.service.QueryService; @@ -676,10 +677,16 @@ public String queryModelFoundrySH(HttpServletRequest request, return queryService.queryModelFoundrySH(request, repo); } + /** + * Compute repo download based on the specified search conditions. + * + * @param request HttpServletRequest request. + * @param condition search condition. + * @return Response string. + */ @RequestMapping(value = "/modelfoundry/download/count") - public String queryModelFoundryCountPath(HttpServletRequest request, - @RequestParam(value = "path", required = false) String path) { - return queryService.queryModelFoundryCountPath(request, path); + public String queryModelFoundryCountPath(HttpServletRequest request, @Valid final RequestParams condition) { + return queryService.queryModelFoundryCountPath(request, condition); } @RequestMapping(value = "/repo/developer") @@ -689,10 +696,16 @@ public String queryRepoDeveloper(HttpServletRequest request, return queryService.queryRepoDeveloper(request, timeRange); } + /** + * Compute repo view count based on the specified search conditions. + * + * @param request HttpServletRequest request. + * @param condition search condition. + * @return Response string. + */ @RequestMapping(value = "/modelfoundry/view/count") - public String queryViewCount(HttpServletRequest request, - @RequestParam(value = "path", required = false) String path) { - return queryService.queryViewCount(request, path); + public String queryViewCount(HttpServletRequest request, @Valid final RequestParams condition) { + return queryService.queryViewCount(request, condition); } @RequestMapping("/community/coreRepos") @@ -766,4 +779,29 @@ public String saveFrontendEvents(HttpServletRequest request, @RequestBody String requestBody) { return queryService.saveFrontendEvents(request, community, requestBody); } + + /** + * Compute repo star count based on the specified search conditions. + * + * @param request HttpServletRequest request. + * @param condition search condition. + * @return Response string. + */ + @RequestMapping(value = "/modelfoundry/star/count") + public String queryEventCount(HttpServletRequest request, @Valid final RequestParams condition) { + return queryService.queryEventCount(request, condition); + } + + /** + * Handles HTTP requests to the "/monthdowncount/openmind" endpoint to retrieve the monthly download count for a specified repository. + * + * @param request The HTTP request object containing details of the request. + * @param repoID The unique identifier of the repository, passed as a request parameter. + * @return A string containing the monthly download count information for the repository. + */ + @RequestMapping(value = "/monthdowncount/openmind") + public String monthDownCount(HttpServletRequest request, + @RequestParam(value = "repo_id") String repoID ) { + return queryService.getCommunityMonthDowncount(request, "foundry", repoID); + } } \ No newline at end of file diff --git a/src/main/java/com/datastat/dao/OpenUbmcQueryDao.java b/src/main/java/com/datastat/dao/OpenUbmcQueryDao.java new file mode 100644 index 0000000..4d3f99a --- /dev/null +++ b/src/main/java/com/datastat/dao/OpenUbmcQueryDao.java @@ -0,0 +1,51 @@ +/* This project is licensed under the Mulan PSL v2. + You can use this software according to the terms and conditions of the Mulan PSL v2. + You may obtain a copy of Mulan PSL v2 at: + http://license.coscl.org.cn/MulanPSL2 + THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + PURPOSE. + See the Mulan PSL v2 for more details. + Create: 2024 +*/ +package com.datastat.dao; + +import java.util.ArrayList; +import java.util.HashMap; + +import org.asynchttpclient.ListenableFuture; +import org.asynchttpclient.Response; +import org.springframework.stereotype.Repository; + +import com.datastat.model.CustomPropertiesConfig; +import com.datastat.util.ResultUtil; + +import lombok.SneakyThrows; + +import static java.nio.charset.StandardCharsets.UTF_8; + +@Repository("openubmcDao") +public class OpenUbmcQueryDao extends QueryDao { + + /** + * Search ownertype based on username. + * + * @param queryConf query config. + * @param userName user name. + * @return Response string. + */ + @Override + @SneakyThrows + public String queryUserOwnerType(CustomPropertiesConfig queryConf, String userName) { + String index = queryConf.getSigIndex(); + String queryStr = queryConf.getAllUserOwnerTypeQueryStr(); + ListenableFuture future = this.esAsyncHttpUtil.executeElasticSearch(queryConf.getEsBaseUrl(), + queryConf.getEsAuth(), index, queryStr); + + String responseBody = future.get().getResponseBody(UTF_8); + HashMap> userData = parseOwnerInfo(responseBody, userName); + + ArrayList ownerInfo = userData.get(userName.toLowerCase()); + return ResultUtil.resultJsonStr(200, objectMapper.valueToTree(ownerInfo), "success"); + } +} diff --git a/src/main/java/com/datastat/dao/QueryDao.java b/src/main/java/com/datastat/dao/QueryDao.java index 4982621..baebf3d 100644 --- a/src/main/java/com/datastat/dao/QueryDao.java +++ b/src/main/java/com/datastat/dao/QueryDao.java @@ -26,6 +26,7 @@ import com.datastat.model.UserTagInfo; import com.datastat.model.dto.ContributeRequestParams; import com.datastat.model.dto.NpsIssueBody; +import com.datastat.model.dto.RequestParams; import com.datastat.model.meetup.MeetupApplyForm; import com.datastat.model.vo.*; import com.datastat.model.yaml.*; @@ -75,6 +76,7 @@ import java.text.DecimalFormat; import java.text.ParseException; import java.text.SimpleDateFormat; +import java.time.LocalDate; import java.time.LocalDateTime; import java.time.OffsetDateTime; import java.time.ZoneOffset; @@ -128,8 +130,8 @@ public class QueryDao { public void init() { esUrl = String.format("%s://%s:%s/", env.getProperty("es.scheme"), env.getProperty("es.host"), env.getProperty("es.port")); esQueryUtils = new EsQueryUtils(); - robotUsers = Arrays.asList(Objects.requireNonNull(env.getProperty("skip.robot.user")).split(",")); - domain_ids = Arrays.asList(Objects.requireNonNull(env.getProperty("qa.domain.ids")).split(",")); + robotUsers = Arrays.asList(Objects.requireNonNull(env.getProperty("skip.robot.user", "robot")).split(",")); + domain_ids = Arrays.asList(Objects.requireNonNull(env.getProperty("qa.domain.ids", "qa.domain.ids")).split(",")); } @SneakyThrows @@ -1199,16 +1201,35 @@ public String querySigsOfTCOwners(CustomPropertiesConfig queryConf) { return objectMapper.valueToTree(resMap).toString(); } + /** + * Search ownertype based on username. + * + * @param queryConf query config. + * @param userName user name. + * @return Response string. + */ @SneakyThrows public String queryUserOwnerType(CustomPropertiesConfig queryConf, String userName) { String index = queryConf.getSigIndex(); String queryStr = queryConf.getAllUserOwnerTypeQueryStr(); - ListenableFuture future = esAsyncHttpUtil.executeSearch(esUrl, index, queryStr); String responseBody = future.get().getResponseBody(UTF_8); + HashMap> userData = parseOwnerInfo(responseBody, userName); + ArrayList ownerInfo = userData.get(userName.toLowerCase()); + return ResultUtil.resultJsonStr(200, objectMapper.valueToTree(ownerInfo), "success"); + } + + /** + * parse owner info based on username. + * + * @param responseBody response string based on username. + * @param userName user name. + * @return Response string. + */ + @SneakyThrows + public HashMap> parseOwnerInfo(String responseBody, String userName) { JsonNode dataNode = objectMapper.readTree(responseBody); Iterator buckets = dataNode.get("aggregations").get("group_field").get("buckets").elements(); - HashMap> userData = new HashMap<>(); while (buckets.hasNext()) { JsonNode bucket = buckets.next(); @@ -1217,7 +1238,8 @@ public String queryUserOwnerType(CustomPropertiesConfig queryConf, String userNa while (users.hasNext()) { JsonNode userBucket = users.next(); String user = userBucket.get("key").asText(); - if (!user.equalsIgnoreCase(userName)) continue; + if (!user.equalsIgnoreCase(userName)) + continue; Iterator types = userBucket.get("type").get("buckets").elements(); ArrayList typeList = new ArrayList<>(); @@ -1238,13 +1260,7 @@ public String queryUserOwnerType(CustomPropertiesConfig queryConf, String userNa } } } - - HashMap resMap = new HashMap<>(); - resMap.put("code", 200); - resMap.put("data", userData.get(userName.toLowerCase())); - resMap.put("msg", "success"); - resMap.put("update_at", (new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ssXXX")).format(new Date())); - return objectMapper.valueToTree(resMap).toString(); + return userData; } @SneakyThrows @@ -1645,7 +1661,6 @@ protected String getCountResult(ListenableFuture future, String dataFl protected String getGiteeResNum(String access_token, String community) throws Exception { AsyncHttpClient client = EsAsyncHttpUtil.getClient(); - RequestBuilder builder = esAsyncHttpUtil.getBuilder(); Param access_tokenParam = new Param("access_token", access_token); Param visibility = new Param("visibility", "public"); Param affiliation = new Param("affiliation", "admin"); @@ -1663,6 +1678,8 @@ protected String getGiteeResNum(String access_token, String community) throws Ex params.add(q); params.add(page); params.add(per_page); + + RequestBuilder builder = new RequestBuilder(); Request request = builder.setUrl(env.getProperty("gitee.user.repos")).setQueryParams(params) .addHeader("Content-Type", "application/json;charset=UTF-8").setMethod("GET").build(); ListenableFuture responseListenableFuture = client.executeRequest(request); @@ -3287,11 +3304,18 @@ public String queryModelFoundrySH(CustomPropertiesConfig queryConf, String repo) return ResultUtil.resultJsonStr(statusCode, count, statusText); } + /** + * Compute repo download based on the specified search conditions. + * + * @param queryConf query config. + * @param condition search condition + * @return Response string. + */ @SneakyThrows - public String queryModelFoundryCountPath(CustomPropertiesConfig queryConf, String path) { - long currentTimeMillis = System.currentTimeMillis(); - String query = String.format(queryConf.getModelFoundryDownloadCountQueryStr(), 0, currentTimeMillis); - String index = queryConf.getModelFoundryPathIndex(path); + public String queryModelFoundryCountPath(CustomPropertiesConfig queryConf, RequestParams condition) { + String query = String.format(queryConf.getModelFoundryDownloadCountQueryStr(), condition.getStart(), + condition.getEnd(), condition.getRepoType(), condition.getRepoId()); + String index = queryConf.getModelFoundryPathIndex(condition.getPath()); ListenableFuture future = esAsyncHttpUtil.executeSearch(esUrl, index, query); Response response = future.get(); int statusCode = response.getStatusCode(); @@ -3354,10 +3378,17 @@ public String queryUserEmail(CustomPropertiesConfig queryConf, String user) { return email; } + /** + * Compute repo view count based on the specified search conditions. + * + * @param queryConf query config. + * @param condition search condition + * @return Response string. + */ @SneakyThrows - public String queryViewCount(CustomPropertiesConfig queryConf, String path) { - long currentTimeMillis = System.currentTimeMillis(); - String query = String.format(queryConf.getViewCountQueryStr(), 0, currentTimeMillis, path); + public String queryViewCount(CustomPropertiesConfig queryConf, RequestParams condition) { + String query = String.format(queryConf.getViewCountQueryStr(), condition.getStart(), + condition.getEnd(), condition.getRepoType(), condition.getRepoId()); String index = queryConf.getExportWebsiteViewIndex(); ListenableFuture future = esAsyncHttpUtil.executeSearch(esUrl, index, query); Response response = future.get(); @@ -3717,4 +3748,134 @@ public String queryGlobalIssues(CustomPropertiesConfig queryConf, String userId, } return resultInfo; } + + /** + * Compute repo star count based on the specified search conditions. + * + * @param queryConf query config. + * @param condition search condition + * @return Response string. + */ + @SneakyThrows + public String queryEventCount(CustomPropertiesConfig queryConf, RequestParams condition) { + String query = String.format(queryConf.getStarCountQueryStr(), condition.getStart(), + condition.getEnd(), condition.getRepoId()); + String index = queryConf.getEventIndex(condition.getRepoType()); + ListenableFuture future = esAsyncHttpUtil.executeSearch(esUrl, index, query); + Response response = future.get(); + int statusCode = response.getStatusCode(); + String statusText = response.getStatusText(); + String responseBody = response.getResponseBody(UTF_8); + JsonNode dataNode = objectMapper.readTree(responseBody); + JsonNode repoData = dataNode.get("aggregations").get("group_field").get("buckets"); + ArrayNode buckets = objectMapper.createArrayNode(); + for (JsonNode item : repoData) { + ObjectNode bucket = objectMapper.createObjectNode(); + bucket.put("repo_id", item.get("key").asText()); + JsonNode events = item.get("event").get("buckets"); + bucket.put("count", getStarCount(events)); + buckets.add(bucket); + } + return ResultUtil.resultJsonStr(statusCode, buckets, statusText); + } + + /** + * Compute repo star count. + * + * @param event events of a repo + * @return like count. + */ + @SneakyThrows + public int getStarCount(JsonNode events) { + int like = 0; + int dislike = 0; + for (JsonNode event : events) { + String eventType = event.path("key").asText().toLowerCase(); + if ("like_create".equals(eventType)) { + like = event.get("doc_count").asInt(); + } else if ("like_delete".equals(eventType)) { + dislike = event.get("doc_count").asInt(); + } + } + return Math.max(like - dislike, 0); + } + + /** + * Retrieves the monthly download count statistics for a specified community and repository. + * + * @param queryConf Custom configuration properties containing necessary query configurations. + * @param community The name of the community for which to retrieve the statistics. + * @param repoID The unique identifier of the repository. + * @return A JSON string containing the monthly download count statistics. + * @throws Exception If an error occurs during the query process. + */ + @SneakyThrows + public String getCommunityMonthDowncount(CustomPropertiesConfig queryConf, String community, String repoID) { + String query = String.format(queryConf.getOpenmindRepoQueryStr(), repoID); + ListenableFuture future = esAsyncHttpUtil.executeSearch(esUrl, queryConf.getOpenmindRepoIndex(), query); + + Response response = future.get(); + + int statusCode = response.getStatusCode(); + String statusText = response.getStatusText(); + String responseBody = response.getResponseBody(UTF_8); + + var dataObject = objectMapper.createObjectNode(); + var jsonArray = objectMapper.createArrayNode(); + + LocalDate today = LocalDate.now(); + DateTimeFormatter df = DateTimeFormatter.ofPattern("yyyy-MM-dd"); + + try { + JsonNode dataNode = objectMapper.readTree(responseBody); + JsonNode hits = dataNode.get("hits").get("hits").get(0); + JsonNode source = hits.get("_source"); + JsonNode repoName = source.get("repo_name"); + JsonNode repoType = source.get("repo_type"); + JsonNode aggregations = dataNode.get("aggregations"); + JsonNode dataPerDay = aggregations.get("data_per_day"); + JsonNode buckets = dataPerDay.get("buckets"); + + for(JsonNode b : buckets) { + var jsonObject = objectMapper.createObjectNode(); + jsonObject.set("date",b.get("key_as_string")); + jsonObject.set("count",b.get("doc_count")); + jsonArray.add(jsonObject); + } + + int bLen = buckets.size(); + int j = 0; + var resJsonArray = objectMapper.createArrayNode(); + + for(int i = 29; i >= 0; i--) { + LocalDate date = today.minusDays(i); + String formattedDate = date.format(df); + + if(j < bLen && jsonArray.get(j).get("date").asText().equals(formattedDate)) { + var jsonObject = objectMapper.createObjectNode(); + jsonObject.put("date", formattedDate); + jsonObject.put("count", jsonArray.get(j).get("count").asInt()); + resJsonArray.add(jsonObject); + j++; + }else{ + var jsonObject = objectMapper.createObjectNode(); + jsonObject.put("date", formattedDate); + jsonObject.put("count", 0); + resJsonArray.add(jsonObject); + } + } + + dataObject.put("repo_id", repoID); + dataObject.set("repo_name", repoName); + dataObject.set("repo_type", repoType); + dataObject.set("daily_down_count", resJsonArray); + + return ResultUtil.resultJsonStr(statusCode, objectMapper.valueToTree(dataObject), statusText); + + } catch (Exception e) { + logger.error("query/user/owner/repos get error", e.getMessage()); + + return ResultUtil.resultJsonStr(statusCode, dataObject, "No data found for the given repo_id"); + } + } } diff --git a/src/main/java/com/datastat/dao/metric/MetricDao.java b/src/main/java/com/datastat/dao/metric/MetricDao.java index 3484063..26f4990 100644 --- a/src/main/java/com/datastat/dao/metric/MetricDao.java +++ b/src/main/java/com/datastat/dao/metric/MetricDao.java @@ -47,10 +47,10 @@ public abstract class MetricDao { @Autowired EsAsyncHttpUtil esAsyncHttpUtil; - @Value("${company.query}") + @Value("${company.query:companyQuery}") String companyQueryStr; - @Value("${user.query}") + @Value("${user.query:userQuery}") String userQueryStr; protected static String esUrl; diff --git a/src/main/java/com/datastat/interceptor/oneid/OneidInterceptor.java b/src/main/java/com/datastat/interceptor/oneid/OneidInterceptor.java index 718a263..ed6bbb1 100644 --- a/src/main/java/com/datastat/interceptor/oneid/OneidInterceptor.java +++ b/src/main/java/com/datastat/interceptor/oneid/OneidInterceptor.java @@ -56,16 +56,16 @@ public class OneidInterceptor implements HandlerInterceptor { @Autowired RedisDao redisDao; - @Value("${cookie.token.name}") + @Value("${cookie.token.name:default}") private String cookieTokenName; - @Value("${cookie.token.domains}") + @Value("${cookie.token.domains:default}") private String allowDomains; - @Value("${cookie.token.secures}") + @Value("${cookie.token.secures:default}") private String cookieSecures; - @Value("${oneid.token.base.password}") + @Value("${oneid.token.base.password:default}") private String oneidTokenBasePassword; private static HashMap domain2secure; diff --git a/src/main/java/com/datastat/model/CustomPropertiesConfig.java b/src/main/java/com/datastat/model/CustomPropertiesConfig.java index 8cd3a29..c51aeb4 100644 --- a/src/main/java/com/datastat/model/CustomPropertiesConfig.java +++ b/src/main/java/com/datastat/model/CustomPropertiesConfig.java @@ -56,6 +56,8 @@ public class CustomPropertiesConfig { private String npsIssueFilter; private String globalNpsIssueFormat; private String globalNpsIssueTitle; + private String esBaseUrl; + private String esAuth; // -- index -- private String extOsIndex; @@ -107,6 +109,10 @@ public class CustomPropertiesConfig { private String sigGatheringTemplate; private String softwareMaintainerIndex; private String giteeFeedbackIssueIndex; + private String spaceEventIndex; + private String modelEventIndex; + private String datasetEventIndex; + private String openmindRepoIndex; // -- query str -- private String extOsQueryStr; @@ -212,6 +218,8 @@ public class CustomPropertiesConfig { private String sigGatheringUserCount; private String repoIssueField; private String userOwnerReposQuery; + private String starCountQueryStr; + private String openmindRepoQueryStr; protected static final Map contributeTypeMap = new HashMap<>(); protected static final Map groupFieldMap = new HashMap<>(); @@ -638,4 +646,22 @@ public String getExportWebsiteViewPathIndex(String path) { return null; } } + + /** + * select which index to use. + * + * @param repoType the type of repo + * @return index name. + */ + public String getEventIndex(String repoType) { + if (repoType.equalsIgnoreCase("model")) { + return getModelEventIndex(); + } else if (repoType.equalsIgnoreCase("dataset")){ + return getDatasetEventIndex(); + } else if (repoType.equalsIgnoreCase("space")){ + return getSpaceEventIndex(); + } else { + throw new RuntimeException("Not support"); + } + } } diff --git a/src/main/java/com/datastat/model/dto/RequestParams.java b/src/main/java/com/datastat/model/dto/RequestParams.java new file mode 100644 index 0000000..78f1e98 --- /dev/null +++ b/src/main/java/com/datastat/model/dto/RequestParams.java @@ -0,0 +1,59 @@ +/* This project is licensed under the Mulan PSL v2. + You can use this software according to the terms and conditions of the Mulan PSL v2. + You may obtain a copy of Mulan PSL v2 at: + http://license.coscl.org.cn/MulanPSL2 + THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + PURPOSE. + See the Mulan PSL v2 for more details. + Create: 2024 +*/ + +package com.datastat.model.dto; + +import com.datastat.constant.Constant; + +import jakarta.validation.constraints.Pattern; +import jakarta.validation.constraints.Size; +import lombok.Getter; +import lombok.Setter; + +@Setter +@Getter +public class RequestParams { + /** + * repo type + */ + @Size(max = 20, message = "the length can not exceed 20") + @Pattern(regexp = Constant.VALID_APPROVAL_REG, message = "Input repo type error") + private String repoType = "*"; + + /** + * This parameter is used to indicate which environment the data comes from. + */ + @Size(max = 50, message = "the length can not exceed 50") + @Pattern(regexp = Constant.VALID_OPENMIND_ENV_REG, message = "Input path error") + private String path = "pro"; + + /** + * repo id + */ + @Size(max = 20, message = "the length can not exceed 20") + @Pattern(regexp = Constant.VALID_REPO_ID, message = "Input repo id error") + private String repoId = "*"; + + /** + * compute data from start time + */ + @Size(max = 20, message = "the length can not exceed 20") + @Pattern(regexp = Constant.VALID_DATE_REG, message = "Input start date error") + private String start = Constant.START_DATE; + + /** + * compute data to end time + */ + @Size(max = 20, message = "the length can not exceed 20") + @Pattern(regexp = Constant.VALID_DATE_REG, message = "Input end date error") + private String end = Constant.END_DATE; + +} diff --git a/src/main/java/com/datastat/service/QueryService.java b/src/main/java/com/datastat/service/QueryService.java index a374e4e..081f48d 100644 --- a/src/main/java/com/datastat/service/QueryService.java +++ b/src/main/java/com/datastat/service/QueryService.java @@ -40,6 +40,7 @@ import com.datastat.model.TeamupApplyForm; import com.datastat.model.dto.ContributeRequestParams; import com.datastat.model.dto.NpsIssueBody; +import com.datastat.model.dto.RequestParams; import com.datastat.model.meetup.MeetupApplyForm; import jakarta.annotation.PostConstruct; @@ -88,7 +89,7 @@ public class QueryService { @PostConstruct public void init() { redisDefaultExpire = Long.parseLong(env.getProperty("redis.keyExpire", "60")); - communityList = Arrays.asList(env.getProperty("communitys").split(",")); + communityList = Arrays.asList(env.getProperty("communitys", "").split(",")); } public Boolean checkCommunity(String community) { @@ -1407,14 +1408,26 @@ public String queryModelFoundrySH(HttpServletRequest request, String repo) { return result; } - public String queryModelFoundryCountPath(HttpServletRequest request, String path) { + /** + * Compute repo download based on the specified search conditions. + * + * @param request HttpServletRequest request. + * @param condition search condition. + * @return Response string. + */ + public String queryModelFoundryCountPath(HttpServletRequest request, RequestParams condition) { QueryDao queryDao = getQueryDao(request); CustomPropertiesConfig queryConf = getQueryConf("foundry"); - path = path == null ? "pro" : path; - String key = "modelfoundrycownload_repo_count_" + path; + StringBuilder sb = new StringBuilder("modelfoundrycownload_repo_count_"); + sb.append(condition.getPath()) + .append(condition.getRepoType()) + .append(condition.getRepoId()) + .append(condition.getStart()) + .append(condition.getEnd()); + String key = sb.toString(); String result = (String) redisDao.get(key); if (result == null) { - result = queryDao.queryModelFoundryCountPath(queryConf, path); + result = queryDao.queryModelFoundryCountPath(queryConf, condition); redisDao.set(key, result, 300l); } return result; @@ -1432,14 +1445,26 @@ public String queryRepoDeveloper(HttpServletRequest request, String timeRange) { return result; } - public String queryViewCount(HttpServletRequest request, String path) { + /** + * Compute repo view count based on the specified search conditions. + * + * @param request HttpServletRequest. + * @param condition search condition + * @return Response string. + */ + public String queryViewCount(HttpServletRequest request, RequestParams condition) { QueryDao queryDao = getQueryDao(request); CustomPropertiesConfig queryConf = getQueryConf("foundry"); - path = path == null ? "space" : path; - String key = "view_count_" + path; + StringBuilder sb = new StringBuilder("modelfoundryview_count_"); + sb.append(condition.getPath()) + .append(condition.getRepoType()) + .append(condition.getRepoId()) + .append(condition.getStart()) + .append(condition.getEnd()); + String key = sb.toString(); String result = (String) redisDao.get(key); if (result == null) { - result = queryDao.queryViewCount(queryConf, path); + result = queryDao.queryViewCount(queryConf, condition); redisDao.set(key, result, redisDefaultExpire); } return result; @@ -1589,4 +1614,53 @@ public String queryGolbalIssues(HttpServletRequest request,String token, Contrib result = objectMapper.valueToTree(resMap).toString(); return result; } + + /** + * Compute repo star count based on the specified search conditions. + * + * @param request HttpServletRequest. + * @param condition search condition + * @return Response string. + */ + public String queryEventCount(HttpServletRequest request, RequestParams condition) { + QueryDao queryDao = getQueryDao(request); + CustomPropertiesConfig queryConf = getQueryConf("foundry"); + StringBuilder sb = new StringBuilder("modelfoundrystar_count_"); + sb.append(condition.getRepoType()) + .append(condition.getRepoId()) + .append(condition.getStart()) + .append(condition.getEnd()); + String key = sb.toString(); + String result = (String) redisDao.get(key); + if (result == null) { + result = queryDao.queryEventCount(queryConf, condition); + redisDao.set(key, result, redisDefaultExpire); + } + return result; + } + + /** + * Retrieves the monthly download count statistics for a specified community and repository. + * This method first checks if the community exists, then queries the data either from a cache + * or the data source if not available in the cache. + * + * @param request The HTTP request object containing details of the request. + * @param community The name of the community for which to retrieve the statistics. + * @param repoID The unique identifier of the repository. + * @return A JSON string containing the monthly download count statistics. + */ + public String getCommunityMonthDowncount(HttpServletRequest request, String community, String repoID) { + QueryDao queryDao = getQueryDao(request); + if (!checkCommunity(community)) { + return ResultUtil.resultJsonStr(404, "error", "community not found"); + } + CustomPropertiesConfig queryConf = getQueryConf("foundry"); + String key = "get_community_month_downcount_" + community + repoID; + String result = (String) redisDao.get(key); + if (result == null) { + result = queryDao.getCommunityMonthDowncount(queryConf, community, repoID); + redisDao.set(key, result, redisDefaultExpire); + } + return result; + } } diff --git a/src/main/java/com/datastat/service/VersionService.java b/src/main/java/com/datastat/service/VersionService.java index c57f6d1..cef7070 100644 --- a/src/main/java/com/datastat/service/VersionService.java +++ b/src/main/java/com/datastat/service/VersionService.java @@ -63,7 +63,8 @@ public class VersionService { @PostConstruct public void init() { - esUrl = String.format("%s://%s:%s/", env.getProperty("es.scheme"), env.getProperty("es.host"), env.getProperty("es.port")); + esUrl = String.format("%s://%s:%s/", env.getProperty("es.scheme", "https"), + env.getProperty("es.host", "0.0.0.0"), env.getProperty("es.port", "9200")); } diff --git a/src/main/java/com/datastat/util/EsAsyncHttpUtil.java b/src/main/java/com/datastat/util/EsAsyncHttpUtil.java index b1f18b9..578612c 100644 --- a/src/main/java/com/datastat/util/EsAsyncHttpUtil.java +++ b/src/main/java/com/datastat/util/EsAsyncHttpUtil.java @@ -33,10 +33,10 @@ @Service public class EsAsyncHttpUtil { - @Value("${es.user}") + @Value("${es.user:user}") String esUser; - @Value("${es.password}") + @Value("${es.password:pwd}") String esPassword; static volatile AsyncHttpClient asyncHttpClient = null; @@ -127,4 +127,39 @@ public ListenableFuture executeCount(String esUrl, String index, Strin return client.executeRequest(builder.build()); } + + /** + * Get es builder based on auth. + * + * @param auth auth info. + * @return RequestBuilder. + */ + public RequestBuilder getEsBuilder(String auth) { + RequestBuilder builder = new RequestBuilder(); + builder.addHeader(HttpHeaders.CONTENT_TYPE, "application/json"); + builder.addHeader("Authorization", "Basic " + Base64.getEncoder().encodeToString((auth).getBytes())) + .setMethod("POST"); + return builder; + + } + + /** + * Execute search based on query. + * + * @param esUrl es url. + * @param auth auth info. + * @param index name of index. + * @param query string. + * @return ListenableFuture. + */ + public ListenableFuture executeElasticSearch(String esUrl, String auth, String index, + String query) throws NoSuchAlgorithmException, KeyManagementException { + AsyncHttpClient client = getClient(); + RequestBuilder builder = getEsBuilder(auth); + + builder.setUrl(esUrl + index + "/_search"); + builder.setBody(query); + + return client.executeRequest(builder.build()); + } } \ No newline at end of file diff --git a/src/main/java/com/datastat/util/RSAUtil.java b/src/main/java/com/datastat/util/RSAUtil.java index 2fe99c9..15dacc0 100644 --- a/src/main/java/com/datastat/util/RSAUtil.java +++ b/src/main/java/com/datastat/util/RSAUtil.java @@ -38,7 +38,7 @@ public class RSAUtil implements Serializable { @Value("${rsa.key.algorithm:RSA}") public void setKeyAlgorithm(String keyAlgorithm) { RSAUtil.KEY_ALGORITHM = keyAlgorithm; } - @Value("${rsa.authing.algorithm}") + @Value("${rsa.authing.algorithm:RSA}") public void setRsaAlgorithm(String rsaAlgorithm) { RSAUtil.RSA_ALGORITHM = rsaAlgorithm; } /** diff --git a/src/test/java/com/datastat/controller/MindSporeMonthDownCountTest.java b/src/test/java/com/datastat/controller/MindSporeMonthDownCountTest.java new file mode 100644 index 0000000..76ed312 --- /dev/null +++ b/src/test/java/com/datastat/controller/MindSporeMonthDownCountTest.java @@ -0,0 +1,44 @@ +package com.datastat.controller; + +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.nio.charset.StandardCharsets; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.autoconfigure.web.servlet.AutoConfigureMockMvc; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.http.MediaType; +import org.springframework.test.web.servlet.MockMvc; +import org.springframework.test.web.servlet.request.MockMvcRequestBuilders; +import org.springframework.test.web.servlet.setup.MockMvcBuilders; +import org.springframework.web.context.WebApplicationContext; + +import com.datastat.ds.DsApplication; + +@SpringBootTest(classes = DsApplication.class) +@AutoConfigureMockMvc +public class MindSporeMonthDownCountTest { + + @Autowired + private WebApplicationContext webApplicationContext; + + private MockMvc mockMvc; + + @BeforeEach + public void setUp() { + mockMvc = MockMvcBuilders.webAppContextSetup(webApplicationContext).build(); + } + + @Test + public void test() throws Exception { + String content = mockMvc.perform(MockMvcRequestBuilders.get("/query/monthdowncount/openmind/30387") + .accept(MediaType.APPLICATION_JSON_VALUE)) + .andReturn().getResponse().getContentAsString(StandardCharsets.UTF_8); + + assertNotNull(content); + assertTrue(content.contains("data")); + } +} \ No newline at end of file diff --git a/src/test/java/com/datastat/ds/DsApplicationTests.java b/src/test/java/com/datastat/ds/DsApplicationTests.java deleted file mode 100644 index aaa0ca2..0000000 --- a/src/test/java/com/datastat/ds/DsApplicationTests.java +++ /dev/null @@ -1,56 +0,0 @@ -package com.datastat.ds; - -import java.util.Iterator; -import java.util.Map.Entry; - -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.test.autoconfigure.web.servlet.AutoConfigureMockMvc; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.test.web.servlet.MockMvc; -import org.springframework.test.web.servlet.setup.MockMvcBuilders; -import org.springframework.util.LinkedMultiValueMap; -import org.springframework.util.MultiValueMap; -import org.springframework.web.context.WebApplicationContext; - -import com.datastat.ds.common.CommonUtil; -import com.datastat.ds.common.ReadCase; -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; - -@SpringBootTest -@AutoConfigureMockMvc -class DsApplicationTests { - - @Test - void contextLoads() { - } - - @Autowired - private WebApplicationContext webApplicationContext; - - private ObjectMapper mapper = new ObjectMapper(); - - private MockMvc mockMvc; - - @BeforeEach - public void setUp() throws Exception { - mockMvc = MockMvcBuilders.webAppContextSetup(webApplicationContext).build(); - } - - @Test - void testNps() throws Exception { - MultiValueMap paramMap = new LinkedMultiValueMap<>(); - paramMap.add("community", "openeuler"); - - JsonNode cases = ReadCase.readFile("src/test/java/com/datastat/ds/case/TestCase.json"); - JsonNode testCases = cases.get("nps"); - for (JsonNode testCase : testCases) { - String body = mapper.writeValueAsString(testCase); - String res = CommonUtil.executePost(mockMvc, "/query/nps", paramMap, body); - CommonUtil.assertOk(res); - } - - } -} diff --git a/src/test/java/com/datastat/ds/QueryDaoTests.java b/src/test/java/com/datastat/ds/QueryDaoTests.java deleted file mode 100644 index 82fe5d5..0000000 --- a/src/test/java/com/datastat/ds/QueryDaoTests.java +++ /dev/null @@ -1,85 +0,0 @@ -package com.datastat.ds; - -import static org.mockito.Mockito.when; - -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.test.autoconfigure.web.servlet.AutoConfigureMockMvc; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.boot.test.mock.mockito.MockBean; -import org.springframework.test.web.servlet.MockMvc; -import org.springframework.test.web.servlet.setup.MockMvcBuilders; -import org.springframework.util.LinkedMultiValueMap; -import org.springframework.util.MultiValueMap; -import org.springframework.web.context.WebApplicationContext; - -import com.datastat.dao.UserIdDao; -import com.datastat.ds.common.CommonUtil; -import com.datastat.ds.common.ReadCase; -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; - -import jakarta.servlet.http.Cookie; - -/** - * QueryDaoTests - */ -@SpringBootTest -@AutoConfigureMockMvc -public class QueryDaoTests { - @Test - void contextLoads() { - } - - @Autowired - private WebApplicationContext webApplicationContext; - - @MockBean - private UserIdDao userIdDao; - - private ObjectMapper mapper = new ObjectMapper(); - - private MockMvc mockMvc; - - @BeforeEach - public void setUp() throws Exception { - mockMvc = MockMvcBuilders.webAppContextSetup(webApplicationContext).build(); - } - - @Test - void test_get_nps() throws Exception{ - MultiValueMap paramMap = new LinkedMultiValueMap<>(); - paramMap.add("community", "openeuler"); - paramMap.add("repo", "easy-software"); - paramMap.add("page", "1"); - paramMap.add("pageSize", "1000"); - paramMap.add("sort", "desc"); - - String token = "abc123"; - Cookie cookie = new Cookie("_Y_G_", token); - - when(userIdDao.getUserId(token)).thenReturn("50216"); - String res = CommonUtil.executeGetWithCookie(mockMvc, "/query/get/nps", paramMap, cookie); - CommonUtil.assertOk(res); - } - - @Test - void test_globalnps_issue() throws Exception{ - MultiValueMap paramMap = new LinkedMultiValueMap<>(); - paramMap.add("community", "software"); - - String token = "abc123"; - Cookie cookie = new Cookie("_Y_G_", token); - - when(userIdDao.getUserId(token)).thenReturn("50216"); - JsonNode cases = ReadCase.readFile("src/test/java/com/datastat/ds/case/TestCase.json"); - JsonNode testCases = cases.get("nps"); - for (JsonNode testCase : testCases) { - String body = mapper.writeValueAsString(testCase); - String res = CommonUtil.executePostWithCookie(mockMvc, "/query/globalnps/issue", paramMap, body, cookie); - CommonUtil.assertOk(res); - } - } - -} \ No newline at end of file diff --git a/src/test/java/com/datastat/ds/case/TestCase.json b/src/test/java/com/datastat/ds/case/TestCase.json index 20c9d8a..a0fa3ad 100644 --- a/src/test/java/com/datastat/ds/case/TestCase.json +++ b/src/test/java/com/datastat/ds/case/TestCase.json @@ -1,19 +1,72 @@ { "nps": [ { - "feedbackPageUrl": "https://software-website.osinfra.cn", + "feedbackPageUrl": "https://software-website.osinfra.cn", "feedbackValue": 5, "feedbackText": "\"openeuler\"" }, { - "feedbackPageUrl": "https://software-website.osinfra.cn", + "feedbackPageUrl": "https://software-website.osinfra.cn", "feedbackValue": 5, "feedbackText": "openeuler: 23.09" }, { - "feedbackPageUrl": "https://software-website.osinfra.cn", + "feedbackPageUrl": "https://software-website.osinfra.cn", "feedbackValue": 5, "feedbackText": null } + ], + "modelfoundry_download_count": [ + { + "start": "2024-11-01", + "end": "2024-11-21", + "repoType": "model", + "repoId": "13046" + }, + { + "start": "2024-11-01", + "path": "pro", + "repoType": "dataset" + }, + { + "start": "2024-11-01", + "path": "yidong" + }, + { + "repoType": "space", + "path": "sh" + } + ], + "modelfoundry_view_count": [ + { + "start": "2024-11-01", + "end": "2024-11-21", + "repoType": "model", + "repoId": "13046" + }, + { + "start": "2024-11-01", + "path": "pro", + "repoType": "dataset" + }, + { + "start": "2024-11-01", + "path": "yidong" + }, + { + "repoType": "space", + "path": "sh" + } + ], + "modelfoundry_star_count": [ + { + "start": "2024-09-01", + "end": "2024-11-21", + "repoType": "model", + "repoId": "13046" + }, + { + "repoType": "space" + } ] } \ No newline at end of file diff --git a/src/test/java/com/datastat/ds/unit/DaoUnitTests.java b/src/test/java/com/datastat/ds/unit/DaoUnitTests.java new file mode 100644 index 0000000..3a58b27 --- /dev/null +++ b/src/test/java/com/datastat/ds/unit/DaoUnitTests.java @@ -0,0 +1,105 @@ +package com.datastat.ds.unit; + +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.isNull; +import static org.mockito.Mockito.when; + +import java.nio.charset.StandardCharsets; + +import org.asynchttpclient.ListenableFuture; +import org.asynchttpclient.Response; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.MockitoAnnotations; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.autoconfigure.web.servlet.AutoConfigureMockMvc; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.boot.test.mock.mockito.MockBean; +import org.springframework.test.util.ReflectionTestUtils; + +import com.datastat.config.QueryConfig; +import com.datastat.config.context.QueryConfContext; +import com.datastat.dao.KafkaDao; +import com.datastat.dao.ObsDao; +import com.datastat.dao.QueryDao; +import com.datastat.dao.RedisDao; +import com.datastat.dao.context.QueryDaoContext; +import com.datastat.ds.common.CommonUtil; +import com.datastat.util.EsAsyncHttpUtil; +import com.fasterxml.jackson.databind.ObjectMapper; + +@SpringBootTest +@AutoConfigureMockMvc +public class DaoUnitTests { + @Test + void contextLoads() { + } + + @Mock + QueryDaoContext queryDaoContext; + + @Mock + QueryConfContext queryConfContext; + + @Mock + ListenableFuture mockFuture; + + @Mock + Response mockResponse; + + @MockBean + KafkaDao kafkaDao; + + @MockBean + ObsDao obsDao; + + @Mock + private RedisDao redisDao; + + @Mock + private QueryConfig queryConfig; + + @InjectMocks + private QueryDao queryDao; + + @Mock + private EsAsyncHttpUtil esAsyncHttpUtil; + + @Autowired + private ObjectMapper objectMapper; + + @BeforeEach + public void setUp() throws Exception { + MockitoAnnotations.openMocks(this); + ReflectionTestUtils.setField(queryDao, "objectMapper", objectMapper); + } + + @Test() + void testUserOwnerTypeDao() throws Exception { + String respBody = "{\"aggregations\":{\"group_field\":{\"buckets\":[{\"key\":\"sig-python-modules\"," + + "\"user\":{\"buckets\":[{\"key\":\"myeuler\",\"doc_count\":1609," + + "\"type\":{\"buckets\":[{\"key\":\"maintainers\",\"doc_count\":1609}]}}," + + "{\"key\":\"shinwell_hu\",\"type\":{\"buckets\":[{\"key\":\"maintainers\",\"doc_count\":1609}]}}]}}]}}}"; + + when(esAsyncHttpUtil.executeSearch(anyString(), isNull(), isNull())).thenReturn(mockFuture); + when(mockFuture.get()).thenReturn(mockResponse); + when(mockResponse.getStatusCode()).thenReturn(200); + when(mockResponse.getStatusText()).thenReturn("OK"); + when(mockResponse.getResponseBody(StandardCharsets.UTF_8)).thenReturn(respBody); + String community = "openubmc"; + String user = "user"; + when(queryDaoContext.getQueryDao(community)).thenReturn(queryDao); + when(queryConfContext.getQueryConfig(community)).thenReturn(queryConfig); + String res = queryDao.queryUserOwnerType(queryConfig, user); + CommonUtil.assertOk(res); + + community = "openeuler"; + when(queryDaoContext.getQueryDao(community)).thenReturn(queryDao); + when(queryConfContext.getQueryConfig(community)).thenReturn(queryConfig); + res = queryDao.queryUserOwnerType(queryConfig, user); + CommonUtil.assertOk(res); + } + +} diff --git a/src/test/java/com/datastat/ds/unit/ServiceUnitTests.java b/src/test/java/com/datastat/ds/unit/ServiceUnitTests.java new file mode 100644 index 0000000..6bdd533 --- /dev/null +++ b/src/test/java/com/datastat/ds/unit/ServiceUnitTests.java @@ -0,0 +1,117 @@ +package com.datastat.ds.unit; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import java.util.Arrays; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.MockitoAnnotations; +import org.springframework.boot.test.autoconfigure.web.servlet.AutoConfigureMockMvc; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.boot.test.mock.mockito.MockBean; +import org.springframework.test.util.ReflectionTestUtils; + +import com.datastat.config.QueryConfig; +import com.datastat.config.context.QueryConfContext; +import com.datastat.dao.FoundryDao; +import com.datastat.dao.KafkaDao; +import com.datastat.dao.ObsDao; +import com.datastat.dao.QueryDao; +import com.datastat.dao.RedisDao; +import com.datastat.dao.context.QueryDaoContext; +import com.datastat.ds.common.CommonUtil; +import com.datastat.model.dto.RequestParams; +import com.datastat.service.QueryService; +import jakarta.servlet.http.HttpServletRequest; + +@SpringBootTest +@AutoConfigureMockMvc +public class ServiceUnitTests { + @Test + void contextLoads() { + } + + @Mock + QueryDaoContext queryDaoContext; + + @Mock + QueryConfContext queryConfContext; + + @MockBean + KafkaDao kafkaDao; + + @MockBean + ObsDao obsDao; + + @Mock + private RedisDao redisDao; + + @InjectMocks + private QueryService queryService; + + @Mock + private QueryConfig queryConfig; + + @Mock + private FoundryDao foundryDao; + + @Mock + private QueryDao queryDao; + + @BeforeEach + public void setUp() throws Exception { + MockitoAnnotations.openMocks(this); + ReflectionTestUtils.setField(queryService, "communityList", Arrays.asList("openeuler")); + } + + @Test() + void testDownloadCountService() throws Exception { + HttpServletRequest request = mock(HttpServletRequest.class); + RequestParams params = new RequestParams(); + params.setRepoType("model"); + + StringBuilder sb = new StringBuilder("modelfoundrycownload_repo_count_"); + sb.append(params.getPath()) + .append(params.getRepoType()) + .append(params.getRepoId()) + .append(params.getStart()) + .append(params.getEnd()); + String key = sb.toString(); + String result = "{\"code\":200,\"msg\":\"ok\",\"data\":[{\"repo_id\":\"1313\",\"repo\":\"Qwen\",\"download\":30}]}"; + when(redisDao.get(key)).thenReturn(result); + String serviceRes = queryService.queryModelFoundryCountPath(request, params); + CommonUtil.assertOk(serviceRes); + + when(redisDao.get(key)).thenReturn(null); + when(queryDaoContext.getQueryDao("queryDao")).thenReturn(foundryDao); + when(queryConfContext.getQueryConfig("foundryConf")).thenReturn(queryConfig); + when(foundryDao.queryModelFoundryCountPath(queryConfig, params)).thenReturn(result); + when(redisDao.set(key, result, 1l)).thenReturn(true); + String res = queryService.queryModelFoundryCountPath(request, params); + CommonUtil.assertOk(res); + } + + @Test() + void testUserOwnerTypeService() throws Exception { + HttpServletRequest request = mock(HttpServletRequest.class); + String user = "user"; + String community = "openeuler"; + String key = community.toLowerCase() + user + "ownertype"; + String result = "{\"code\":200,\"data\":[{\"sig\":\"infrastructrue\",\"type\":[\"committers\"]}],\"msg\":\"success\"}"; + when(redisDao.get(key)).thenReturn(result); + String serviceRes = queryService.queryUserOwnerType(request, community, user); + CommonUtil.assertOk(serviceRes); + + when(redisDao.get(key)).thenReturn(null); + when(queryDaoContext.getQueryDao("queryDao")).thenReturn(queryDao); + when(queryConfContext.getQueryConfig("queryConf")).thenReturn(queryConfig); + when(queryDao.queryUserOwnerType(queryConfig, user)).thenReturn(result); + when(redisDao.set(key, result, 1l)).thenReturn(true); + String res = queryService.queryUserOwnerType(request, community, user); + CommonUtil.assertOk(res); + } +}