From 13995e5c0c538071b54b3c99296d4478b24dbddf Mon Sep 17 00:00:00 2001 From: kaede10 Date: Mon, 16 Dec 2024 14:25:30 +0800 Subject: [PATCH] add api /query/view/count for openmind --- .../datastat/controller/QueryController.java | 17 ++++++++++ .../com/datastat/dao/OpenUbmcQueryDao.java | 31 +++++++++++++++++ src/main/java/com/datastat/dao/QueryDao.java | 27 +++++++++++++++ .../model/CustomPropertiesConfig.java | 1 + .../com/datastat/service/QueryService.java | 25 +++++++++++++- .../com/datastat/ds/unit/DaoUnitTests.java | 33 +++++++++++++++++++ .../datastat/ds/unit/ServiceUnitTests.java | 32 ++++++++++++++++++ 7 files changed, 165 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/datastat/controller/QueryController.java b/src/main/java/com/datastat/controller/QueryController.java index aa9a61a..6841c00 100644 --- a/src/main/java/com/datastat/controller/QueryController.java +++ b/src/main/java/com/datastat/controller/QueryController.java @@ -804,4 +804,21 @@ public String monthDownCount(HttpServletRequest request, @RequestParam(value = "repo_id") String repoID ) { return queryService.getCommunityMonthDowncount(request, "foundry", repoID); } + + /** + * Handles HTTP requests to the "/view/count" endpoint to retrieve the view count for a specified repository. + * + * @param request The HTTP request object containing details of the request. + * @param repoType The type of the repository, passed as a request parameter. + * @param owner The owner of the repository, passed as a request parameter. + * @param repo The repo name of the repository, passed as a request parameter. + * @return A string containing the monthly download count information for the repository. + */ + @RequestMapping(value = "/view/count") + public String viewCount(HttpServletRequest request, + @RequestParam(value = "repoType") String repoType, + @RequestParam(value = "owner") String owner, + @RequestParam(value = "repo") String repo) { + return queryService.getViewCount(request, repoType, owner, repo); + } } \ No newline at end of file diff --git a/src/main/java/com/datastat/dao/OpenUbmcQueryDao.java b/src/main/java/com/datastat/dao/OpenUbmcQueryDao.java index 4d3f99a..848f4ec 100644 --- a/src/main/java/com/datastat/dao/OpenUbmcQueryDao.java +++ b/src/main/java/com/datastat/dao/OpenUbmcQueryDao.java @@ -12,6 +12,7 @@ import java.util.ArrayList; import java.util.HashMap; +import java.util.Iterator; import org.asynchttpclient.ListenableFuture; import org.asynchttpclient.Response; @@ -19,6 +20,8 @@ import com.datastat.model.CustomPropertiesConfig; import com.datastat.util.ResultUtil; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.JsonNode; import lombok.SneakyThrows; @@ -48,4 +51,32 @@ public String queryUserOwnerType(CustomPropertiesConfig queryConf, String userNa ArrayList ownerInfo = userData.get(userName.toLowerCase()); return ResultUtil.resultJsonStr(200, objectMapper.valueToTree(ownerInfo), "success"); } + + /** + * Search sig info based on sig name. + * + * @param queryConf query config. + * @param sig sig name. + * @return Response string. + */ + @Override + @SneakyThrows + public String querySigInfo(CustomPropertiesConfig queryConf, String sig) { + sig = sig == null ? "*" : sig; + String queryJson = String.format(queryConf.getSigInfoQueryStr(), sig); + ListenableFuture future = esAsyncHttpUtil.executeElasticSearch(queryConf.getEsBaseUrl(), + queryConf.getEsAuth(), queryConf.getSigIndex(), queryJson); + String responseBody = future.get().getResponseBody(UTF_8); + JsonNode dataNode = objectMapper.readTree(responseBody); + + Iterator buckets = dataNode.get("hits").get("hits").elements(); + ArrayList> sigList = new ArrayList<>(); + while (buckets.hasNext()) { + JsonNode bucket = buckets.next().get("_source"); + HashMap data = objectMapper.convertValue(bucket, new TypeReference<>() { + }); + sigList.add(data); + } + return ResultUtil.resultJsonStr(200, objectMapper.valueToTree(sigList), "ok"); + } } diff --git a/src/main/java/com/datastat/dao/QueryDao.java b/src/main/java/com/datastat/dao/QueryDao.java index baebf3d..bb18011 100644 --- a/src/main/java/com/datastat/dao/QueryDao.java +++ b/src/main/java/com/datastat/dao/QueryDao.java @@ -3878,4 +3878,31 @@ public String getCommunityMonthDowncount(CustomPropertiesConfig queryConf, Strin return ResultUtil.resultJsonStr(statusCode, dataObject, "No data found for the given repo_id"); } } + + /** + * Retrieves the view count statistics for a specified community and repository. + * + * @param queryConf Custom configuration properties containing necessary query configurations. + * @param repoType The type of the repository, passed as a request parameter. + * @param owner The owner of the repository, passed as a request parameter. + * @param repo The repo name of the repository, passed as a request parameter. + * @return A JSON string containing the monthly download count statistics. + * @throws Exception If an error occurs during the query process. + */ + @SneakyThrows + public String getViewCount(CustomPropertiesConfig queryConf, String repoType, String owner, String repo) { + String query = String.format(queryConf.getRepoViewCountQueryStr(), repoType, owner, repo); + ListenableFuture future = esAsyncHttpUtil.executeCount(esUrl, queryConf.getExportWebsiteViewIndex(), query); + Response response = future.get(); + int statusCode = response.getStatusCode(); + String statusText = response.getStatusText(); + String responseBody = response.getResponseBody(UTF_8); + JsonNode dataNode = objectMapper.readTree(responseBody); + long count = dataNode.get("count").asLong(); + Map resData = new HashMap<>(); + resData.put("owner", owner); + resData.put("repo", repo); + resData.put("count", count); + return ResultUtil.resultJsonStr(statusCode, objectMapper.valueToTree(resData), statusText); + } } diff --git a/src/main/java/com/datastat/model/CustomPropertiesConfig.java b/src/main/java/com/datastat/model/CustomPropertiesConfig.java index c51aeb4..6d7ba7b 100644 --- a/src/main/java/com/datastat/model/CustomPropertiesConfig.java +++ b/src/main/java/com/datastat/model/CustomPropertiesConfig.java @@ -220,6 +220,7 @@ public class CustomPropertiesConfig { private String userOwnerReposQuery; private String starCountQueryStr; private String openmindRepoQueryStr; + private String repoViewCountQueryStr; protected static final Map contributeTypeMap = new HashMap<>(); protected static final Map groupFieldMap = new HashMap<>(); diff --git a/src/main/java/com/datastat/service/QueryService.java b/src/main/java/com/datastat/service/QueryService.java index 081f48d..0bff5cb 100644 --- a/src/main/java/com/datastat/service/QueryService.java +++ b/src/main/java/com/datastat/service/QueryService.java @@ -1659,7 +1659,30 @@ public String getCommunityMonthDowncount(HttpServletRequest request, String comm String result = (String) redisDao.get(key); if (result == null) { result = queryDao.getCommunityMonthDowncount(queryConf, community, repoID); - redisDao.set(key, result, redisDefaultExpire); + redisDao.set(key, result, redisDefaultExpire); + } + return result; + } + + /** + * Retrieves the view count statistics for a specified community and repository. + * This method first checks if the community exists, then queries the data either from a cache + * or the data source if not available in the cache. + * + * @param request The HTTP request object containing details of the request. + * @param repoType The type of the repository, passed as a request parameter. + * @param owner The owner of the repository, passed as a request parameter. + * @param repo The repo name of the repository, passed as a request parameter. + * @return A JSON string containing the monthly download count statistics. + */ + public String getViewCount(HttpServletRequest request, String repoType, String owner, String repo) { + QueryDao queryDao = getQueryDao(request); + CustomPropertiesConfig queryConf = getQueryConf("foundry"); + String key = "get_viewcount_" + repoType + owner + repo; + String result = (String) redisDao.get(key); + if (result == null) { + result = queryDao.getViewCount(queryConf, repoType, owner, repo); + redisDao.set(key, result, redisDefaultExpire); } return result; } diff --git a/src/test/java/com/datastat/ds/unit/DaoUnitTests.java b/src/test/java/com/datastat/ds/unit/DaoUnitTests.java index 3a58b27..a56e57d 100644 --- a/src/test/java/com/datastat/ds/unit/DaoUnitTests.java +++ b/src/test/java/com/datastat/ds/unit/DaoUnitTests.java @@ -1,3 +1,14 @@ +/* This project is licensed under the Mulan PSL v2. + You can use this software according to the terms and conditions of the Mulan PSL v2. + You may obtain a copy of Mulan PSL v2 at: + http://license.coscl.org.cn/MulanPSL2 + THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + PURPOSE. + See the Mulan PSL v2 for more details. + Create: 2024 +*/ + package com.datastat.ds.unit; import static org.mockito.ArgumentMatchers.anyString; @@ -102,4 +113,26 @@ void testUserOwnerTypeDao() throws Exception { CommonUtil.assertOk(res); } + + @Test() + void testViewCountDao() throws Exception { + String respBody = "{\"count\": 1234, \"_shards\":{\"total\":4,\"successful\":4}}"; + when(esAsyncHttpUtil.executeCount(anyString(), isNull(), anyString())).thenReturn(mockFuture); + when(mockFuture.get()).thenReturn(mockResponse); + when(mockResponse.getStatusCode()).thenReturn(200); + when(mockResponse.getStatusText()).thenReturn("OK"); + when(mockResponse.getResponseBody(StandardCharsets.UTF_8)).thenReturn(respBody); + String community = "foundry"; + String repoType = "dataset"; + String owner = "owner"; + String repo = "repo"; + when(queryDaoContext.getQueryDao(community)).thenReturn(queryDao); + when(queryConfContext.getQueryConfig(community)).thenReturn(queryConfig); + String query = "{\"query\":{\"bool\":{\"filter\":[{\"query_string\":{\"analyze_wildcard\":true," + + "\"query\":\"event.keyword:$PageView AND properties.$path:\\\"/%ss/%s/%s\\\"\"}}]}}}"; + when(queryConfig.getRepoViewCountQueryStr()).thenReturn(query); + String res = queryDao.getViewCount(queryConfig, repoType, owner, repo); + CommonUtil.assertOk(res); + } + } diff --git a/src/test/java/com/datastat/ds/unit/ServiceUnitTests.java b/src/test/java/com/datastat/ds/unit/ServiceUnitTests.java index 6bdd533..2cf3e65 100644 --- a/src/test/java/com/datastat/ds/unit/ServiceUnitTests.java +++ b/src/test/java/com/datastat/ds/unit/ServiceUnitTests.java @@ -1,3 +1,14 @@ +/* This project is licensed under the Mulan PSL v2. + You can use this software according to the terms and conditions of the Mulan PSL v2. + You may obtain a copy of Mulan PSL v2 at: + http://license.coscl.org.cn/MulanPSL2 + THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + PURPOSE. + See the Mulan PSL v2 for more details. + Create: 2024 +*/ + package com.datastat.ds.unit; import static org.mockito.Mockito.mock; @@ -114,4 +125,25 @@ void testUserOwnerTypeService() throws Exception { String res = queryService.queryUserOwnerType(request, community, user); CommonUtil.assertOk(res); } + + @Test() + void testViewCountService() throws Exception { + HttpServletRequest request = mock(HttpServletRequest.class); + String repoType = "dataset"; + String owner = "owner"; + String repo = "repo"; + String key = "get_viewcount_" + repoType + owner + repo; + String result = "{\"code\":200,\"msg\":\"ok\",\"data\":{\"owner\":\"owner\",\"repo\":\"repo\",\"count\":30}}"; + when(redisDao.get(key)).thenReturn(result); + String serviceRes = queryService.getViewCount(request, repoType, owner, repo); + CommonUtil.assertOk(serviceRes); + + when(redisDao.get(key)).thenReturn(null); + when(queryDaoContext.getQueryDao("queryDao")).thenReturn(foundryDao); + when(queryConfContext.getQueryConfig("foundryConf")).thenReturn(queryConfig); + when(foundryDao.getViewCount(queryConfig, repoType, owner, repo)).thenReturn(result); + when(redisDao.set(key, result, 1l)).thenReturn(true); + String res = queryService.getViewCount(request, repoType, owner, repo); + CommonUtil.assertOk(res); + } }