add api /query/view/count for openmind. update newyear report

src/main/java/com/datastat/controller/QueryController.java

@@ -804,4 +804,21 @@ public String monthDownCount(HttpServletRequest request,
             @RequestParam(value = "repo_id") String repoID ) {
         return queryService.getCommunityMonthDowncount(request, "foundry", repoID);
     }
+
+    /**
+     * Handles HTTP requests to the "/view/count" endpoint to retrieve the view count for a specified repository.
+     *
+     * @param request The HTTP request object containing details of the request.
+     * @param repoType  The type of the repository, passed as a request parameter.
+     * @param owner  The owner of the repository, passed as a request parameter.
+     * @param repo  The repo name of the repository, passed as a request parameter.
+     * @return A string containing the monthly download count information for the repository.
+     */
+    @RequestMapping(value = "/view/count")
+    public String viewCount(HttpServletRequest request,
+            @RequestParam(value = "repoType") String repoType,
+            @RequestParam(value = "owner") String owner,
+            @RequestParam(value = "repo") String repo) {
+        return queryService.getViewCount(request, repoType, owner, repo);
+    }
 }

src/main/java/com/datastat/dao/OpenUbmcQueryDao.java

@@ -12,13 +12,16 @@
 
 import java.util.ArrayList;
 import java.util.HashMap;
+import java.util.Iterator;
 
 import org.asynchttpclient.ListenableFuture;
 import org.asynchttpclient.Response;
 import org.springframework.stereotype.Repository;
 
 import com.datastat.model.CustomPropertiesConfig;
 import com.datastat.util.ResultUtil;
+import com.fasterxml.jackson.core.type.TypeReference;
+import com.fasterxml.jackson.databind.JsonNode;
 
 import lombok.SneakyThrows;
 
@@ -48,4 +51,32 @@ public String queryUserOwnerType(CustomPropertiesConfig queryConf, String userNa
         ArrayList<Object> ownerInfo = userData.get(userName.toLowerCase());
         return ResultUtil.resultJsonStr(200, objectMapper.valueToTree(ownerInfo), "success");
     }
+
+    /**
+     * Search sig info based on sig name.
+     *
+     * @param queryConf query config.
+     * @param sig sig name.
+     * @return Response string.
+     */
+    @Override
+    @SneakyThrows
+    public String querySigInfo(CustomPropertiesConfig queryConf, String sig) {
+        sig = sig == null ? "*" : sig;
+        String queryJson = String.format(queryConf.getSigInfoQueryStr(), sig);
+        ListenableFuture<Response> future = esAsyncHttpUtil.executeElasticSearch(queryConf.getEsBaseUrl(),
+                queryConf.getEsAuth(), queryConf.getSigIndex(), queryJson);
+        String responseBody = future.get().getResponseBody(UTF_8);
+        JsonNode dataNode = objectMapper.readTree(responseBody);
+
+        Iterator<JsonNode> buckets = dataNode.get("hits").get("hits").elements();
+        ArrayList<HashMap<String, Object>> sigList = new ArrayList<>();
+        while (buckets.hasNext()) {
+            JsonNode bucket = buckets.next().get("_source");
+            HashMap<String, Object> data = objectMapper.convertValue(bucket, new TypeReference<>() {
+            });
+            sigList.add(data);
+        }
+        return ResultUtil.resultJsonStr(200, objectMapper.valueToTree(sigList), "ok");
+    }
 }

src/main/java/com/datastat/dao/QueryDao.java

@@ -356,41 +356,44 @@ public String queryCveDetails(CustomPropertiesConfig queryConf, String lastCurso
         return esQueryUtils.esFromId(restHighLevelClient, item, lastCursor, Integer.parseInt(pageSize), indexName);
     }
 
-    @SneakyThrows
     public String queryNewYearPer(CustomPropertiesConfig queryConf, String oauth2_proxy, String community) {
-        String user = getUserFromCookie(queryConf, oauth2_proxy);
-        String localFile = "om-data/obs/" + community.toLowerCase() + "_" + env.getProperty("year") + ".csv";
-        List<HashMap<String, Object>> report = CsvFileUtil.readFile(localFile);
         HashMap<String, Object> resMap = new HashMap<>();
-        resMap.put("code", 200);
-        resMap.put("msg", "OK");
-        if (report == null)
-            resMap.put("data", new ArrayList<>());
-        else if (user == null)
-            resMap.put("data", report);
-        else {
-            List<HashMap<String, Object>> user_login = report.stream()
-                    .filter(m -> m.getOrDefault("user_login", "").equals(user)).collect(Collectors.toList());
-            resMap.put("data", user_login);
-        }
-
-        BulkRequest request = new BulkRequest();
-        RestHighLevelClient restHighLevelClient = getRestHighLevelClient();
-        Date now = new Date();
-        SimpleDateFormat simpleDateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ssXXX");
-        String nowStr = simpleDateFormat.format(now);
-        String uuid = UUID.randomUUID().toString();
-        HashMap<String, Object> dataMap = new HashMap<>();
-        dataMap.put("user_login", user);
-        dataMap.put("community", community);
-        dataMap.put("created_at", nowStr);
-        request.add(new IndexRequest("new_year_report", "_doc", uuid + nowStr).source(dataMap));
-        if (request.requests().size() != 0)
-            restHighLevelClient.bulk(request, RequestOptions.DEFAULT);
-        restHighLevelClient.close();
-
-        resMap.put("update_at", (new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ssXXX")).format(new Date()));
-        return objectMapper.valueToTree(resMap).toString();
+        try {
+            String user = getUserFromCookie(queryConf, oauth2_proxy);
+            String localFile = env.getProperty("export_path") + community.toLowerCase() + "_" + env.getProperty("year") + ".csv";
+            List<HashMap<String, Object>> report = CsvFileUtil.readFile(localFile);
+            resMap.put("code", 200);
+            resMap.put("msg", "OK");
+            if (report == null)
+                resMap.put("data", new ArrayList<>());
+            else if (user == null)
+                resMap.put("data", report);
+            else {
+                List<HashMap<String, Object>> user_login = report.stream()
+                        .filter(m -> m.getOrDefault("user_login", "").equals(user)).collect(Collectors.toList());
+                resMap.put("data", user_login);
+            }
+            BulkRequest request = new BulkRequest();
+            RestHighLevelClient restHighLevelClient = getRestHighLevelClient();
+            Date now = new Date();
+            SimpleDateFormat simpleDateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ssXXX");
+            String nowStr = simpleDateFormat.format(now);
+            String uuid = UUID.randomUUID().toString();
+            HashMap<String, Object> dataMap = new HashMap<>();
+            dataMap.put("user_login", user);
+            dataMap.put("community", community);
+            dataMap.put("created_at", nowStr);
+            request.add(new IndexRequest("new_year_report", "_doc", uuid + nowStr).source(dataMap));
+            if (request.requests().size() != 0)
+                restHighLevelClient.bulk(request, RequestOptions.DEFAULT);
+            restHighLevelClient.close();
+
+            resMap.put("update_at", (new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ssXXX")).format(new Date()));
+            return objectMapper.valueToTree(resMap).toString();
+        } catch (Exception e) {
+            logger.error("report exception - {}", e.getMessage());
+            return objectMapper.valueToTree(resMap).toString();
+        }
     }
 
     @SneakyThrows
@@ -3878,4 +3881,31 @@ public String getCommunityMonthDowncount(CustomPropertiesConfig queryConf, Strin
             return ResultUtil.resultJsonStr(statusCode, dataObject, "No data found for the given repo_id");
         }
     }
+
+    /**
+     * Retrieves the view count statistics for a specified community and repository.
+     *
+     * @param queryConf Custom configuration properties containing necessary query configurations.
+     * @param repoType  The type of the repository, passed as a request parameter.
+     * @param owner  The owner of the repository, passed as a request parameter.
+     * @param repo  The repo name of the repository, passed as a request parameter.
+     * @return A JSON string containing the monthly download count statistics.
+     * @throws Exception If an error occurs during the query process.
+     */
+    @SneakyThrows
+    public String getViewCount(CustomPropertiesConfig queryConf, String repoType, String owner, String repo) {
+        String query = String.format(queryConf.getRepoViewCountQueryStr(), repoType, owner, repo);
+        ListenableFuture<Response> future =  esAsyncHttpUtil.executeCount(esUrl, queryConf.getExportWebsiteViewIndex(), query);
+        Response response = future.get();
+        int statusCode = response.getStatusCode();
+        String statusText = response.getStatusText();
+        String responseBody = response.getResponseBody(UTF_8);
+        JsonNode dataNode = objectMapper.readTree(responseBody);
+        long count = dataNode.get("count").asLong();
+        Map<String, Object> resData = new HashMap<>();
+        resData.put("owner", owner);
+        resData.put("repo", repo);
+        resData.put("count", count);
+        return ResultUtil.resultJsonStr(statusCode, objectMapper.valueToTree(resData), statusText);  
+    }
 }

src/main/java/com/datastat/model/CustomPropertiesConfig.java

@@ -220,6 +220,7 @@ public class CustomPropertiesConfig {
     private String userOwnerReposQuery;
     private String starCountQueryStr;
     private String openmindRepoQueryStr;
+    private String repoViewCountQueryStr;
 
     protected static final Map<String, String> contributeTypeMap = new HashMap<>();
     protected static final Map<String, String> groupFieldMap = new HashMap<>();

src/main/java/com/datastat/service/QueryService.java

@@ -1659,7 +1659,30 @@ public String getCommunityMonthDowncount(HttpServletRequest request, String comm
         String result = (String) redisDao.get(key);
         if (result == null) {
             result = queryDao.getCommunityMonthDowncount(queryConf, community, repoID);
-          redisDao.set(key, result, redisDefaultExpire);
+            redisDao.set(key, result, redisDefaultExpire);
+        }
+        return result;
+    }
+
+    /**
+     * Retrieves the view count statistics for a specified community and repository.
+     * This method first checks if the community exists, then queries the data either from a cache
+     * or the data source if not available in the cache.
+     *
+     * @param request  The HTTP request object containing details of the request.
+     * @param repoType  The type of the repository, passed as a request parameter.
+     * @param owner  The owner of the repository, passed as a request parameter.
+     * @param repo  The repo name of the repository, passed as a request parameter.
+     * @return A JSON string containing the monthly download count statistics.
+     */
+    public String getViewCount(HttpServletRequest request, String repoType, String owner, String repo) {
+        QueryDao queryDao = getQueryDao(request);
+        CustomPropertiesConfig queryConf = getQueryConf("foundry");
+        String key = "get_viewcount_" + repoType + owner + repo;
+        String result = (String) redisDao.get(key);
+        if (result == null) {
+            result = queryDao.getViewCount(queryConf, repoType, owner, repo);
+            redisDao.set(key, result, redisDefaultExpire);
         }
         return result;
     }

src/test/java/com/datastat/ds/unit/DaoUnitTests.java

@@ -1,3 +1,14 @@
+/* This project is licensed under the Mulan PSL v2.
+ You can use this software according to the terms and conditions of the Mulan PSL v2.
+ You may obtain a copy of Mulan PSL v2 at:
+     http://license.coscl.org.cn/MulanPSL2
+ THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
+ IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
+ PURPOSE.
+ See the Mulan PSL v2 for more details.
+ Create: 2024
+*/
+
 package com.datastat.ds.unit;
 
 import static org.mockito.ArgumentMatchers.anyString;
@@ -102,4 +113,26 @@ void testUserOwnerTypeDao() throws Exception {
         CommonUtil.assertOk(res);
     }
 
+
+    @Test()
+    void testViewCountDao() throws Exception {
+        String respBody = "{\"count\": 1234, \"_shards\":{\"total\":4,\"successful\":4}}";
+        when(esAsyncHttpUtil.executeCount(anyString(), isNull(), anyString())).thenReturn(mockFuture);
+        when(mockFuture.get()).thenReturn(mockResponse);
+        when(mockResponse.getStatusCode()).thenReturn(200);
+        when(mockResponse.getStatusText()).thenReturn("OK");
+        when(mockResponse.getResponseBody(StandardCharsets.UTF_8)).thenReturn(respBody);
+        String community = "foundry";
+        String repoType = "dataset";
+        String owner = "owner";
+        String repo = "repo";
+        when(queryDaoContext.getQueryDao(community)).thenReturn(queryDao);
+        when(queryConfContext.getQueryConfig(community)).thenReturn(queryConfig);
+        String query = "{\"query\":{\"bool\":{\"filter\":[{\"query_string\":{\"analyze_wildcard\":true,"
+                + "\"query\":\"event.keyword:$PageView AND properties.$path:\\\"/%ss/%s/%s\\\"\"}}]}}}";
+        when(queryConfig.getRepoViewCountQueryStr()).thenReturn(query);
+        String res = queryDao.getViewCount(queryConfig, repoType, owner, repo);
+        CommonUtil.assertOk(res);
+    }
+
 }

src/test/java/com/datastat/ds/unit/ServiceUnitTests.java

@@ -1,3 +1,14 @@
+/* This project is licensed under the Mulan PSL v2.
+ You can use this software according to the terms and conditions of the Mulan PSL v2.
+ You may obtain a copy of Mulan PSL v2 at:
+     http://license.coscl.org.cn/MulanPSL2
+ THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
+ IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
+ PURPOSE.
+ See the Mulan PSL v2 for more details.
+ Create: 2024
+*/
+
 package com.datastat.ds.unit;
 
 import static org.mockito.Mockito.mock;
@@ -114,4 +125,25 @@ void testUserOwnerTypeService() throws Exception {
         String res = queryService.queryUserOwnerType(request, community, user);
         CommonUtil.assertOk(res);
     }
+
+    @Test()
+    void testViewCountService() throws Exception {
+        HttpServletRequest request = mock(HttpServletRequest.class);
+        String repoType = "dataset";
+        String owner = "owner";
+        String repo = "repo";
+        String key = "get_viewcount_" + repoType + owner + repo;
+        String result = "{\"code\":200,\"msg\":\"ok\",\"data\":{\"owner\":\"owner\",\"repo\":\"repo\",\"count\":30}}";
+        when(redisDao.get(key)).thenReturn(result);
+        String serviceRes = queryService.getViewCount(request, repoType, owner, repo);
+        CommonUtil.assertOk(serviceRes);
+
+        when(redisDao.get(key)).thenReturn(null);
+        when(queryDaoContext.getQueryDao("queryDao")).thenReturn(foundryDao);
+        when(queryConfContext.getQueryConfig("foundryConf")).thenReturn(queryConfig);
+        when(foundryDao.getViewCount(queryConfig, repoType, owner, repo)).thenReturn(result);
+        when(redisDao.set(key, result, 1l)).thenReturn(true);
+        String res = queryService.getViewCount(request, repoType, owner, repo);
+        CommonUtil.assertOk(res);
+    }
 }