From 0a55a26ecb518da1f0a85ab3c102685ef0d173fa Mon Sep 17 00:00:00 2001 From: kaede10 Date: Fri, 27 Dec 2024 11:51:07 +0800 Subject: [PATCH] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E4=BB=93=E5=BA=93=E8=AE=BF?= =?UTF-8?q?=E9=97=AE=E9=87=8F=E5=A4=84=E7=90=86=E9=80=BB=E8=BE=91?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pom.xml | 4 +-- .../datastat/controller/QueryController.java | 7 ++--- src/main/java/com/datastat/dao/QueryDao.java | 30 +++++++++++-------- .../com/datastat/service/QueryService.java | 12 ++++++-- .../com/datastat/ds/unit/DaoUnitTests.java | 23 +++++++++----- .../datastat/ds/unit/ServiceUnitTests.java | 22 +++++++++----- 6 files changed, 60 insertions(+), 38 deletions(-) diff --git a/pom.xml b/pom.xml index 33403af..1d29ff5 100644 --- a/pom.xml +++ b/pom.xml @@ -213,13 +213,13 @@ org.apache.tomcat.embed tomcat-embed-core - 10.1.25 + 10.1.34 org.apache.tomcat.embed tomcat-embed-websocket - 10.1.25 + 10.1.34 diff --git a/src/main/java/com/datastat/controller/QueryController.java b/src/main/java/com/datastat/controller/QueryController.java index 66956ba..6b3a7f9 100644 --- a/src/main/java/com/datastat/controller/QueryController.java +++ b/src/main/java/com/datastat/controller/QueryController.java @@ -836,10 +836,7 @@ public String monthDownCount(HttpServletRequest request, * @return A string containing the monthly download count information for the repository. */ @RequestMapping(value = "/view/count") - public String viewCount(HttpServletRequest request, - @RequestParam(value = "repoType") String repoType, - @RequestParam(value = "owner") String owner, - @RequestParam(value = "repo") String repo) { - return queryService.getViewCount(request, repoType, owner, repo); + public String viewCount(HttpServletRequest request, @Valid final RequestParams condition) { + return queryService.getViewCount(request, condition); } } \ No newline at end of file diff --git a/src/main/java/com/datastat/dao/QueryDao.java b/src/main/java/com/datastat/dao/QueryDao.java index 0699099..7302851 100644 --- a/src/main/java/com/datastat/dao/QueryDao.java +++ b/src/main/java/com/datastat/dao/QueryDao.java @@ -3934,26 +3934,32 @@ public String getCommunityMonthDowncount(CustomPropertiesConfig queryConf, Strin * Retrieves the view count statistics for a specified community and repository. * * @param queryConf Custom configuration properties containing necessary query configurations. - * @param repoType The type of the repository, passed as a request parameter. - * @param owner The owner of the repository, passed as a request parameter. - * @param repo The repo name of the repository, passed as a request parameter. + * @param condition The search condition of the repository * @return A JSON string containing the monthly download count statistics. * @throws Exception If an error occurs during the query process. */ @SneakyThrows - public String getViewCount(CustomPropertiesConfig queryConf, String repoType, String owner, String repo) { - String query = String.format(queryConf.getRepoViewCountQueryStr(), repoType, owner, repo); - ListenableFuture future = esAsyncHttpUtil.executeCount(esUrl, queryConf.getExportWebsiteViewIndex(), query); + public String getViewCount(CustomPropertiesConfig queryConf, RequestParams condition) { + String query = String.format(queryConf.getRepoViewCountQueryStr(), condition.getStart(), + condition.getEnd(), condition.getRepoType(), condition.getRepoId()); + String index = queryConf.getExportWebsiteViewIndex(); + ListenableFuture future = esAsyncHttpUtil.executeSearch(esUrl, index, query); Response response = future.get(); int statusCode = response.getStatusCode(); String statusText = response.getStatusText(); String responseBody = response.getResponseBody(UTF_8); JsonNode dataNode = objectMapper.readTree(responseBody); - long count = dataNode.get("count").asLong(); - Map resData = new HashMap<>(); - resData.put("owner", owner); - resData.put("repo", repo); - resData.put("count", count); - return ResultUtil.resultJsonStr(statusCode, objectMapper.valueToTree(resData), statusText); + JsonNode testStr = dataNode.get("aggregations").get("group_field").get("buckets"); + ArrayNode buckets = objectMapper.createArrayNode(); + if (testStr.isArray()) { + for (int i = 0; i < testStr.size(); i++) { + JsonNode item = testStr.get(i); + ObjectNode bucket = objectMapper.createObjectNode(); + bucket.put("repo_id", item.get("key").asText()); + bucket.put("count", item.get("doc_count").asInt()); + buckets.add(bucket); + } + } + return ResultUtil.resultJsonStr(statusCode, buckets, statusText); } } diff --git a/src/main/java/com/datastat/service/QueryService.java b/src/main/java/com/datastat/service/QueryService.java index 71b1cf3..7e517eb 100644 --- a/src/main/java/com/datastat/service/QueryService.java +++ b/src/main/java/com/datastat/service/QueryService.java @@ -1685,13 +1685,19 @@ public String getCommunityMonthDowncount(HttpServletRequest request, String comm * @param repo The repo name of the repository, passed as a request parameter. * @return A JSON string containing the monthly download count statistics. */ - public String getViewCount(HttpServletRequest request, String repoType, String owner, String repo) { + public String getViewCount(HttpServletRequest request, RequestParams condition) { QueryDao queryDao = getQueryDao(request); CustomPropertiesConfig queryConf = getQueryConf("foundry"); - String key = "get_viewcount_" + repoType + owner + repo; + StringBuilder sb = new StringBuilder("get_viewcount_"); + sb.append(condition.getPath()) + .append(condition.getRepoType()) + .append(condition.getRepoId()) + .append(condition.getStart()) + .append(condition.getEnd()); + String key = sb.toString(); String result = (String) redisDao.get(key); if (result == null) { - result = queryDao.getViewCount(queryConf, repoType, owner, repo); + result = queryDao.getViewCount(queryConf, condition); redisDao.set(key, result, redisDefaultExpire); } return result; diff --git a/src/test/java/com/datastat/ds/unit/DaoUnitTests.java b/src/test/java/com/datastat/ds/unit/DaoUnitTests.java index a56e57d..74f0b2a 100644 --- a/src/test/java/com/datastat/ds/unit/DaoUnitTests.java +++ b/src/test/java/com/datastat/ds/unit/DaoUnitTests.java @@ -38,6 +38,7 @@ import com.datastat.dao.RedisDao; import com.datastat.dao.context.QueryDaoContext; import com.datastat.ds.common.CommonUtil; +import com.datastat.model.dto.RequestParams; import com.datastat.util.EsAsyncHttpUtil; import com.fasterxml.jackson.databind.ObjectMapper; @@ -116,22 +117,28 @@ void testUserOwnerTypeDao() throws Exception { @Test() void testViewCountDao() throws Exception { - String respBody = "{\"count\": 1234, \"_shards\":{\"total\":4,\"successful\":4}}"; - when(esAsyncHttpUtil.executeCount(anyString(), isNull(), anyString())).thenReturn(mockFuture); + String respBody = "{\"aggregations\":{\"group_field\":{\"buckets\":[{\"key\":\"3828\",\"doc_count\":1609}]}}}"; + when(esAsyncHttpUtil.executeSearch(anyString(), isNull(), anyString())).thenReturn(mockFuture); when(mockFuture.get()).thenReturn(mockResponse); when(mockResponse.getStatusCode()).thenReturn(200); when(mockResponse.getStatusText()).thenReturn("OK"); when(mockResponse.getResponseBody(StandardCharsets.UTF_8)).thenReturn(respBody); String community = "foundry"; - String repoType = "dataset"; - String owner = "owner"; - String repo = "repo"; + RequestParams params = new RequestParams(); + params.setStart("2024-01-01"); + params.setEnd("2024-12-01"); + params.setRepoType("model"); + params.setRepoId("3828"); + when(queryDaoContext.getQueryDao(community)).thenReturn(queryDao); when(queryConfContext.getQueryConfig(community)).thenReturn(queryConfig); - String query = "{\"query\":{\"bool\":{\"filter\":[{\"query_string\":{\"analyze_wildcard\":true," - + "\"query\":\"event.keyword:$PageView AND properties.$path:\\\"/%ss/%s/%s\\\"\"}}]}}}"; + String query = "{\"size\":0,\"query\":{\"bool\":{\"filter\":[{\"range\":{\"created_at\":{\"gte\":\"%s\",\"lte\":\"%s\"}}}," + + "{\"query_string\":{\"analyze_wildcard\":true," + + "\"query\":\"event.keyword:RV AND properties.module.keyword:%s AND properties.id.keyword:%s\"}}]}}," + + "\"aggs\":{\"group_field\":{\"terms\":{\"field\":\"properties.id.keyword\",\"size\":50," + + "\"order\":{\"_count\":\"desc\"},\"min_doc_count\":1},\"aggs\":{}}}}"; when(queryConfig.getRepoViewCountQueryStr()).thenReturn(query); - String res = queryDao.getViewCount(queryConfig, repoType, owner, repo); + String res = queryDao.getViewCount(queryConfig, params); CommonUtil.assertOk(res); } diff --git a/src/test/java/com/datastat/ds/unit/ServiceUnitTests.java b/src/test/java/com/datastat/ds/unit/ServiceUnitTests.java index 2cf3e65..7012423 100644 --- a/src/test/java/com/datastat/ds/unit/ServiceUnitTests.java +++ b/src/test/java/com/datastat/ds/unit/ServiceUnitTests.java @@ -129,21 +129,27 @@ void testUserOwnerTypeService() throws Exception { @Test() void testViewCountService() throws Exception { HttpServletRequest request = mock(HttpServletRequest.class); - String repoType = "dataset"; - String owner = "owner"; - String repo = "repo"; - String key = "get_viewcount_" + repoType + owner + repo; - String result = "{\"code\":200,\"msg\":\"ok\",\"data\":{\"owner\":\"owner\",\"repo\":\"repo\",\"count\":30}}"; + RequestParams params = new RequestParams(); + params.setRepoType("model"); + + StringBuilder sb = new StringBuilder("get_viewcount_"); + sb.append(params.getPath()) + .append(params.getRepoType()) + .append(params.getRepoId()) + .append(params.getStart()) + .append(params.getEnd()); + String key = sb.toString(); + String result = "{\"code\":200,\"msg\":\"ok\",\"data\":{\"repo_id\":\"1234\", \"count\":30}}"; when(redisDao.get(key)).thenReturn(result); - String serviceRes = queryService.getViewCount(request, repoType, owner, repo); + String serviceRes = queryService.getViewCount(request, params); CommonUtil.assertOk(serviceRes); when(redisDao.get(key)).thenReturn(null); when(queryDaoContext.getQueryDao("queryDao")).thenReturn(foundryDao); when(queryConfContext.getQueryConfig("foundryConf")).thenReturn(queryConfig); - when(foundryDao.getViewCount(queryConfig, repoType, owner, repo)).thenReturn(result); + when(foundryDao.getViewCount(queryConfig, params)).thenReturn(result); when(redisDao.set(key, result, 1l)).thenReturn(true); - String res = queryService.getViewCount(request, repoType, owner, repo); + String res = queryService.getViewCount(request, params); CommonUtil.assertOk(res); } }