From 978f21d0f6f86e2f14c524843ff3ad4db1b27be6 Mon Sep 17 00:00:00 2001 From: kaede10 Date: Tue, 10 Dec 2024 10:55:22 +0800 Subject: [PATCH] =?UTF-8?q?=E9=80=82=E9=85=8Dopenubmc=E6=8E=A5=E5=8F=A3?= =?UTF-8?q?=EF=BC=8C=E5=A2=9E=E5=8A=A0DT=E6=B5=8B=E8=AF=95=E7=94=A8?= =?UTF-8?q?=E4=BE=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/datastat/dao/OpenUbmcQueryDao.java | 51 +++++++++ src/main/java/com/datastat/dao/QueryDao.java | 37 ++++-- .../model/CustomPropertiesConfig.java | 2 + .../com/datastat/util/EsAsyncHttpUtil.java | 35 ++++++ .../com/datastat/ds/unit/DaoUnitTests.java | 105 ++++++++++++++++++ .../datastat/ds/unit/ServiceUnitTests.java | 39 ++++++- 6 files changed, 252 insertions(+), 17 deletions(-) create mode 100644 src/main/java/com/datastat/dao/OpenUbmcQueryDao.java create mode 100644 src/test/java/com/datastat/ds/unit/DaoUnitTests.java diff --git a/src/main/java/com/datastat/dao/OpenUbmcQueryDao.java b/src/main/java/com/datastat/dao/OpenUbmcQueryDao.java new file mode 100644 index 0000000..4d3f99a --- /dev/null +++ b/src/main/java/com/datastat/dao/OpenUbmcQueryDao.java @@ -0,0 +1,51 @@ +/* This project is licensed under the Mulan PSL v2. + You can use this software according to the terms and conditions of the Mulan PSL v2. + You may obtain a copy of Mulan PSL v2 at: + http://license.coscl.org.cn/MulanPSL2 + THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + PURPOSE. + See the Mulan PSL v2 for more details. + Create: 2024 +*/ +package com.datastat.dao; + +import java.util.ArrayList; +import java.util.HashMap; + +import org.asynchttpclient.ListenableFuture; +import org.asynchttpclient.Response; +import org.springframework.stereotype.Repository; + +import com.datastat.model.CustomPropertiesConfig; +import com.datastat.util.ResultUtil; + +import lombok.SneakyThrows; + +import static java.nio.charset.StandardCharsets.UTF_8; + +@Repository("openubmcDao") +public class OpenUbmcQueryDao extends QueryDao { + + /** + * Search ownertype based on username. + * + * @param queryConf query config. + * @param userName user name. + * @return Response string. + */ + @Override + @SneakyThrows + public String queryUserOwnerType(CustomPropertiesConfig queryConf, String userName) { + String index = queryConf.getSigIndex(); + String queryStr = queryConf.getAllUserOwnerTypeQueryStr(); + ListenableFuture future = this.esAsyncHttpUtil.executeElasticSearch(queryConf.getEsBaseUrl(), + queryConf.getEsAuth(), index, queryStr); + + String responseBody = future.get().getResponseBody(UTF_8); + HashMap> userData = parseOwnerInfo(responseBody, userName); + + ArrayList ownerInfo = userData.get(userName.toLowerCase()); + return ResultUtil.resultJsonStr(200, objectMapper.valueToTree(ownerInfo), "success"); + } +} diff --git a/src/main/java/com/datastat/dao/QueryDao.java b/src/main/java/com/datastat/dao/QueryDao.java index d8560ea..baebf3d 100644 --- a/src/main/java/com/datastat/dao/QueryDao.java +++ b/src/main/java/com/datastat/dao/QueryDao.java @@ -1201,16 +1201,35 @@ public String querySigsOfTCOwners(CustomPropertiesConfig queryConf) { return objectMapper.valueToTree(resMap).toString(); } + /** + * Search ownertype based on username. + * + * @param queryConf query config. + * @param userName user name. + * @return Response string. + */ @SneakyThrows public String queryUserOwnerType(CustomPropertiesConfig queryConf, String userName) { String index = queryConf.getSigIndex(); String queryStr = queryConf.getAllUserOwnerTypeQueryStr(); - ListenableFuture future = esAsyncHttpUtil.executeSearch(esUrl, index, queryStr); String responseBody = future.get().getResponseBody(UTF_8); + HashMap> userData = parseOwnerInfo(responseBody, userName); + ArrayList ownerInfo = userData.get(userName.toLowerCase()); + return ResultUtil.resultJsonStr(200, objectMapper.valueToTree(ownerInfo), "success"); + } + + /** + * parse owner info based on username. + * + * @param responseBody response string based on username. + * @param userName user name. + * @return Response string. + */ + @SneakyThrows + public HashMap> parseOwnerInfo(String responseBody, String userName) { JsonNode dataNode = objectMapper.readTree(responseBody); Iterator buckets = dataNode.get("aggregations").get("group_field").get("buckets").elements(); - HashMap> userData = new HashMap<>(); while (buckets.hasNext()) { JsonNode bucket = buckets.next(); @@ -1219,7 +1238,8 @@ public String queryUserOwnerType(CustomPropertiesConfig queryConf, String userNa while (users.hasNext()) { JsonNode userBucket = users.next(); String user = userBucket.get("key").asText(); - if (!user.equalsIgnoreCase(userName)) continue; + if (!user.equalsIgnoreCase(userName)) + continue; Iterator types = userBucket.get("type").get("buckets").elements(); ArrayList typeList = new ArrayList<>(); @@ -1240,13 +1260,7 @@ public String queryUserOwnerType(CustomPropertiesConfig queryConf, String userNa } } } - - HashMap resMap = new HashMap<>(); - resMap.put("code", 200); - resMap.put("data", userData.get(userName.toLowerCase())); - resMap.put("msg", "success"); - resMap.put("update_at", (new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ssXXX")).format(new Date())); - return objectMapper.valueToTree(resMap).toString(); + return userData; } @SneakyThrows @@ -1647,7 +1661,6 @@ protected String getCountResult(ListenableFuture future, String dataFl protected String getGiteeResNum(String access_token, String community) throws Exception { AsyncHttpClient client = EsAsyncHttpUtil.getClient(); - RequestBuilder builder = esAsyncHttpUtil.getBuilder(); Param access_tokenParam = new Param("access_token", access_token); Param visibility = new Param("visibility", "public"); Param affiliation = new Param("affiliation", "admin"); @@ -1665,6 +1678,8 @@ protected String getGiteeResNum(String access_token, String community) throws Ex params.add(q); params.add(page); params.add(per_page); + + RequestBuilder builder = new RequestBuilder(); Request request = builder.setUrl(env.getProperty("gitee.user.repos")).setQueryParams(params) .addHeader("Content-Type", "application/json;charset=UTF-8").setMethod("GET").build(); ListenableFuture responseListenableFuture = client.executeRequest(request); diff --git a/src/main/java/com/datastat/model/CustomPropertiesConfig.java b/src/main/java/com/datastat/model/CustomPropertiesConfig.java index 6e20a27..c51aeb4 100644 --- a/src/main/java/com/datastat/model/CustomPropertiesConfig.java +++ b/src/main/java/com/datastat/model/CustomPropertiesConfig.java @@ -56,6 +56,8 @@ public class CustomPropertiesConfig { private String npsIssueFilter; private String globalNpsIssueFormat; private String globalNpsIssueTitle; + private String esBaseUrl; + private String esAuth; // -- index -- private String extOsIndex; diff --git a/src/main/java/com/datastat/util/EsAsyncHttpUtil.java b/src/main/java/com/datastat/util/EsAsyncHttpUtil.java index 3e7c743..578612c 100644 --- a/src/main/java/com/datastat/util/EsAsyncHttpUtil.java +++ b/src/main/java/com/datastat/util/EsAsyncHttpUtil.java @@ -127,4 +127,39 @@ public ListenableFuture executeCount(String esUrl, String index, Strin return client.executeRequest(builder.build()); } + + /** + * Get es builder based on auth. + * + * @param auth auth info. + * @return RequestBuilder. + */ + public RequestBuilder getEsBuilder(String auth) { + RequestBuilder builder = new RequestBuilder(); + builder.addHeader(HttpHeaders.CONTENT_TYPE, "application/json"); + builder.addHeader("Authorization", "Basic " + Base64.getEncoder().encodeToString((auth).getBytes())) + .setMethod("POST"); + return builder; + + } + + /** + * Execute search based on query. + * + * @param esUrl es url. + * @param auth auth info. + * @param index name of index. + * @param query string. + * @return ListenableFuture. + */ + public ListenableFuture executeElasticSearch(String esUrl, String auth, String index, + String query) throws NoSuchAlgorithmException, KeyManagementException { + AsyncHttpClient client = getClient(); + RequestBuilder builder = getEsBuilder(auth); + + builder.setUrl(esUrl + index + "/_search"); + builder.setBody(query); + + return client.executeRequest(builder.build()); + } } \ No newline at end of file diff --git a/src/test/java/com/datastat/ds/unit/DaoUnitTests.java b/src/test/java/com/datastat/ds/unit/DaoUnitTests.java new file mode 100644 index 0000000..3a58b27 --- /dev/null +++ b/src/test/java/com/datastat/ds/unit/DaoUnitTests.java @@ -0,0 +1,105 @@ +package com.datastat.ds.unit; + +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.isNull; +import static org.mockito.Mockito.when; + +import java.nio.charset.StandardCharsets; + +import org.asynchttpclient.ListenableFuture; +import org.asynchttpclient.Response; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.MockitoAnnotations; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.autoconfigure.web.servlet.AutoConfigureMockMvc; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.boot.test.mock.mockito.MockBean; +import org.springframework.test.util.ReflectionTestUtils; + +import com.datastat.config.QueryConfig; +import com.datastat.config.context.QueryConfContext; +import com.datastat.dao.KafkaDao; +import com.datastat.dao.ObsDao; +import com.datastat.dao.QueryDao; +import com.datastat.dao.RedisDao; +import com.datastat.dao.context.QueryDaoContext; +import com.datastat.ds.common.CommonUtil; +import com.datastat.util.EsAsyncHttpUtil; +import com.fasterxml.jackson.databind.ObjectMapper; + +@SpringBootTest +@AutoConfigureMockMvc +public class DaoUnitTests { + @Test + void contextLoads() { + } + + @Mock + QueryDaoContext queryDaoContext; + + @Mock + QueryConfContext queryConfContext; + + @Mock + ListenableFuture mockFuture; + + @Mock + Response mockResponse; + + @MockBean + KafkaDao kafkaDao; + + @MockBean + ObsDao obsDao; + + @Mock + private RedisDao redisDao; + + @Mock + private QueryConfig queryConfig; + + @InjectMocks + private QueryDao queryDao; + + @Mock + private EsAsyncHttpUtil esAsyncHttpUtil; + + @Autowired + private ObjectMapper objectMapper; + + @BeforeEach + public void setUp() throws Exception { + MockitoAnnotations.openMocks(this); + ReflectionTestUtils.setField(queryDao, "objectMapper", objectMapper); + } + + @Test() + void testUserOwnerTypeDao() throws Exception { + String respBody = "{\"aggregations\":{\"group_field\":{\"buckets\":[{\"key\":\"sig-python-modules\"," + + "\"user\":{\"buckets\":[{\"key\":\"myeuler\",\"doc_count\":1609," + + "\"type\":{\"buckets\":[{\"key\":\"maintainers\",\"doc_count\":1609}]}}," + + "{\"key\":\"shinwell_hu\",\"type\":{\"buckets\":[{\"key\":\"maintainers\",\"doc_count\":1609}]}}]}}]}}}"; + + when(esAsyncHttpUtil.executeSearch(anyString(), isNull(), isNull())).thenReturn(mockFuture); + when(mockFuture.get()).thenReturn(mockResponse); + when(mockResponse.getStatusCode()).thenReturn(200); + when(mockResponse.getStatusText()).thenReturn("OK"); + when(mockResponse.getResponseBody(StandardCharsets.UTF_8)).thenReturn(respBody); + String community = "openubmc"; + String user = "user"; + when(queryDaoContext.getQueryDao(community)).thenReturn(queryDao); + when(queryConfContext.getQueryConfig(community)).thenReturn(queryConfig); + String res = queryDao.queryUserOwnerType(queryConfig, user); + CommonUtil.assertOk(res); + + community = "openeuler"; + when(queryDaoContext.getQueryDao(community)).thenReturn(queryDao); + when(queryConfContext.getQueryConfig(community)).thenReturn(queryConfig); + res = queryDao.queryUserOwnerType(queryConfig, user); + CommonUtil.assertOk(res); + } + +} diff --git a/src/test/java/com/datastat/ds/unit/ServiceUnitTests.java b/src/test/java/com/datastat/ds/unit/ServiceUnitTests.java index 28a4c71..6bdd533 100644 --- a/src/test/java/com/datastat/ds/unit/ServiceUnitTests.java +++ b/src/test/java/com/datastat/ds/unit/ServiceUnitTests.java @@ -3,6 +3,8 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; +import java.util.Arrays; + import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.mockito.InjectMocks; @@ -11,12 +13,14 @@ import org.springframework.boot.test.autoconfigure.web.servlet.AutoConfigureMockMvc; import org.springframework.boot.test.context.SpringBootTest; import org.springframework.boot.test.mock.mockito.MockBean; +import org.springframework.test.util.ReflectionTestUtils; -import com.datastat.config.FoundryConfig; +import com.datastat.config.QueryConfig; import com.datastat.config.context.QueryConfContext; import com.datastat.dao.FoundryDao; import com.datastat.dao.KafkaDao; import com.datastat.dao.ObsDao; +import com.datastat.dao.QueryDao; import com.datastat.dao.RedisDao; import com.datastat.dao.context.QueryDaoContext; import com.datastat.ds.common.CommonUtil; @@ -50,14 +54,18 @@ void contextLoads() { private QueryService queryService; @Mock - private FoundryConfig queryConfig; + private QueryConfig queryConfig; + + @Mock + private FoundryDao foundryDao; @Mock - private FoundryDao queryDao; + private QueryDao queryDao; @BeforeEach public void setUp() throws Exception { MockitoAnnotations.openMocks(this); + ReflectionTestUtils.setField(queryService, "communityList", Arrays.asList("openeuler")); } @Test() @@ -66,7 +74,6 @@ void testDownloadCountService() throws Exception { RequestParams params = new RequestParams(); params.setRepoType("model"); - StringBuilder sb = new StringBuilder("modelfoundrycownload_repo_count_"); sb.append(params.getPath()) .append(params.getRepoType()) @@ -80,11 +87,31 @@ void testDownloadCountService() throws Exception { CommonUtil.assertOk(serviceRes); when(redisDao.get(key)).thenReturn(null); - when(queryDaoContext.getQueryDao("queryDao")).thenReturn(queryDao); + when(queryDaoContext.getQueryDao("queryDao")).thenReturn(foundryDao); when(queryConfContext.getQueryConfig("foundryConf")).thenReturn(queryConfig); - when(queryDao.queryModelFoundryCountPath(queryConfig, params)).thenReturn(result); + when(foundryDao.queryModelFoundryCountPath(queryConfig, params)).thenReturn(result); when(redisDao.set(key, result, 1l)).thenReturn(true); String res = queryService.queryModelFoundryCountPath(request, params); CommonUtil.assertOk(res); } + + @Test() + void testUserOwnerTypeService() throws Exception { + HttpServletRequest request = mock(HttpServletRequest.class); + String user = "user"; + String community = "openeuler"; + String key = community.toLowerCase() + user + "ownertype"; + String result = "{\"code\":200,\"data\":[{\"sig\":\"infrastructrue\",\"type\":[\"committers\"]}],\"msg\":\"success\"}"; + when(redisDao.get(key)).thenReturn(result); + String serviceRes = queryService.queryUserOwnerType(request, community, user); + CommonUtil.assertOk(serviceRes); + + when(redisDao.get(key)).thenReturn(null); + when(queryDaoContext.getQueryDao("queryDao")).thenReturn(queryDao); + when(queryConfContext.getQueryConfig("queryConf")).thenReturn(queryConfig); + when(queryDao.queryUserOwnerType(queryConfig, user)).thenReturn(result); + when(redisDao.set(key, result, 1l)).thenReturn(true); + String res = queryService.queryUserOwnerType(request, community, user); + CommonUtil.assertOk(res); + } }