Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

SCP with multiple retries of file from web to db #867

Merged
merged 3 commits into from
Nov 27, 2024
Merged

SCP with multiple retries of file from web to db #867

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
3b91037
SCP with multiple retries of file from web to db
Kantas2601 Nov 18, 2024
636b3a3
Updated Record already exist including completion
Kantas2601 Nov 27, 2024
628151c
IVR service removed
Kantas2601 Nov 27, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 3 additions & 4 deletions ...ari/src/main/java/org/motechproject/nms/kilkari/service/impl/SubscriptionServiceImpl.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1005,8 +1005,7 @@ public String getSqlQuery() {
"(firstMessageDayOfWeek = :dow OR " +
"(secondMessageDayOfWeek = :dow AND p.messagesPerWeek = 2)) AND " +
"status = 'ACTIVE' AND " +
"IVR_SERVICE IS TRUE "+
"AND (serviceStatus IN ('IVR', 'IVR_AND_WHATSAPP') OR serviceStatus IS NULL) "+
" (serviceStatus IN ('IVR', 'IVR_AND_WHATSAPP') OR serviceStatus IS NULL) "+
"ORDER BY s.id " +
"LIMIT :max";
LOGGER.debug(KilkariConstants.SQL_QUERY_LOG, query);
Expand Down Expand Up @@ -1194,9 +1193,9 @@ public List<String> findJhSubscriptionIds() {
public String getSqlQuery() {
String query = "select subscriptionId from (select subscriptionId from nms_mcts_mothers a LEFT JOIN " +
"nms_subscribers b on b.mother_id_oid=a.id LEFT JOIN nms_subscriptions c on c.subscriber_id_oid = b.id where rchId like 'JH%' " +
"and subscriptionPack_id_OID = 1 and IVR_SERVICE IS TRUE UNION ALL select subscriptionId from nms_mcts_children a LEFT JOIN " +
"and subscriptionPack_id_OID = 1 UNION ALL select subscriptionId from nms_mcts_children a LEFT JOIN " +
"nms_subscribers b on b.child_id_oid=a.id LEFT JOIN nms_subscriptions c on c.subscriber_id_oid = b.id " +
"where rchId like 'JH%' and subscriptionPack_id_OID = 2 and IVR_SERVICE IS TRUE AND (serviceStatus IN ('IVR', 'IVR_AND_WHATSAPP') OR serviceStatus IS NULL)) as a;";
"where rchId like 'JH%' and subscriptionPack_id_OID = 2 AND (serviceStatus IN ('IVR', 'IVR_AND_WHATSAPP') OR serviceStatus IS NULL)) as a;";
LOGGER.debug(KilkariConstants.SQL_QUERY_LOG, query);
return query;
}
Expand Down
6 changes: 3 additions & 3 deletions mobile-academy/src/main/resources/sms-content.properties
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,6 @@ sms.content.50 = Badhai ho didi, apne Mobile Academy course pura kar liya hai. C
# NATIONAL DEFAULT
sms.content.default = Badhai ho didi, apne Mobile Academy course pura kar liya hai. Certificate paane ke liye is SMS ko apne block meeting mein dikhayen.\\n\\nMoHFW

sms.templateId.default = 1007163065348946395
sms.entityId.default = 1301159100860122510
sms.telemarketerId.default = 1001096933494158
sms.templateId.default = localhost
sms.entityId.default = localhost
sms.telemarketerId.default = localhost
88 changes: 68 additions & 20 deletions rch/src/main/java/org/motechproject/nms/rch/service/impl/RchWsImportServiceImpl.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,13 +16,13 @@
import org.motechproject.nms.rch.domain.RchImportFacilitator;
import org.motechproject.nms.rch.domain.RchImportFailRecord;
import org.motechproject.nms.rch.domain.RchUserType;
import org.motechproject.nms.rch.exception.RchWebServiceException;
import org.motechproject.nms.rch.repository.RchImportAuditDataService;
import org.motechproject.nms.rch.repository.RchImportFacilitatorDataService;
import org.motechproject.nms.rch.repository.RchImportFailRecordDataService;
import org.motechproject.nms.rch.service.RchWebServiceFacade;
import org.motechproject.nms.rch.service.RchWsImportService;
import org.motechproject.nms.rch.utils.Constants;
import org.motechproject.nms.rch.utils.ExecutionHelper;
import org.motechproject.nms.region.domain.State;
import org.motechproject.nms.region.repository.StateDataService;
import org.motechproject.server.config.SettingsFacade;
Expand Down Expand Up @@ -219,6 +219,44 @@ public void importVillageHealthSubFacilityFromRch(List<Long> stateIds, LocalDate
LOGGER.info("Initiated import workflow from RCH for villagesubfacility");
}

private void executeScpCommand(String tempFilePath,boolean isRemoteLocal) {
String localDir = settingsFacade.getProperty("rch.folder.tmp");
String remoteDir = settingsFacade.getProperty("rch.folder.tmp.remote");

String command;

if(isRemoteLocal){
command = "scp " + settingsFacade.getProperty("rch.remote.url") + tempFilePath + " " + localDir;
}else {
command = String.format("scp %s %s", tempFilePath, remoteDir);
}
int maxRetries = 3;
long timeout = Long.parseLong(settingsFacade.getProperty("rch.scp_timeout"));

executeWithRetries(command, maxRetries, timeout);
}

private void executeWithRetries(String command, int maxRetries, long timeout) {
ExecutionHelper execHelper = new ExecutionHelper();
int attempt = 0;

while (attempt < maxRetries) {
attempt++;
try {
execHelper.exec(command, timeout);
LOGGER.info("SCP command executed successfully on attempt {}.", attempt);
return;
} catch (Exception e) {
if (attempt < maxRetries) {
LOGGER.warn("Attempt {} of {} failed. Retrying...", attempt, maxRetries, e);
} else {
LOGGER.error("SCP command failed after {} attempts. No more retries.", maxRetries, e);
return;
}
}
}
}

@MotechListener(subjects = { Constants.RCH_MOTHER_IMPORT_SUBJECT })
@Transactional
@Override
Expand Down Expand Up @@ -273,6 +311,7 @@ public void importRchMothersData(MotechEvent motechEvent) {

if (keelDeelApiResponse != null) {
String tempFilePath = rchWebServiceFacade.saveToFile(keelDeelApiResponse, "mother", stateId.toString());
executeScpCommand(tempFilePath,false);

Map<String, Object> params = new HashMap<>();
params.put("state", stateId.toString());
Expand Down Expand Up @@ -326,31 +365,33 @@ public void handleThirdApiEvent(MotechEvent event) {
RchUserType userType = RchUserType.valueOf(entityType.toUpperCase());

try {
String payload = rchWebServiceFacade.readPayloadFromTempFile(tempFilePath);

executeScpCommand(tempFilePath, true);
String payload = rchWebServiceFacade.readPayloadFromTempFile(tempFilePath);

if(payload!=null) {
thirdApiResponse = rchWebServiceFacade.callThirdApi(payload);
if (payload != null) {
thirdApiResponse = rchWebServiceFacade.callThirdApi(payload);
}


if(thirdApiResponse!=null) {
if (thirdApiResponse != null) {
File responseFile = rchWebServiceFacade.generateJsonResponseFile(thirdApiResponse, userType, Long.valueOf(stateId));

if (responseFile != null) {
LOGGER.info("RCH {} response successfully written to file. Copying to remote directory.", userType);
status = rchWebServiceFacade.retryScpAndAudit(responseFile.getName(), from, to, Long.valueOf(stateId), userType, 0);
} else {
LOGGER.error("Error writing {} response to file for state {}", userType, stateId);
}}
if (responseFile != null) {
LOGGER.info("RCH {} third api response successfully written to file. Copying to remote directory.", userType);
status = rchWebServiceFacade.retryScpAndAudit(responseFile.getName(), from, to, Long.valueOf(stateId), userType, 0);
} else {
LOGGER.error("Error writing {} third api response to file for state {}", userType, stateId);
}
}

if (status) {
LOGGER.info(FILE_RECORD_SUCCESS, stateId);
} else {
handleError(userType, Long.valueOf(stateId), stateName, Long.valueOf(stateCode), from, to, "Failed to process third API response.");
handleError(userType, Long.valueOf(stateId), stateName, Long.valueOf(stateCode), from, to, "Error during SCP file transfer.");
}

} catch (Exception e) {
}
catch (Exception e) {
handleError(userType, Long.valueOf(stateId), stateName, Long.valueOf(stateCode), from, to, e.getMessage());
}
}
Expand Down Expand Up @@ -407,6 +448,8 @@ public void importRchDistrictData(MotechEvent motechEvent) {

String tempFilePath = rchWebServiceFacade.saveToFile(keelDeelApiResponse, "district", stateId.toString());

executeScpCommand(tempFilePath,false);

Map<String, Object> params = new HashMap<>();
params.put("state", stateId.toString());
params.put("tempFilePath", tempFilePath);
Expand Down Expand Up @@ -466,6 +509,7 @@ public void handleApiEvent(MotechEvent event) {
String thirdApiResponse = null;

try {
executeScpCommand(tempFilePath,true);
String payload = rchWebServiceFacade.readPayloadFromTempFile(tempFilePath);

if(payload!=null) {
Expand All @@ -484,7 +528,7 @@ public void handleApiEvent(MotechEvent event) {
if (status) {
LOGGER.info("{} data processed successfully for state ID: {}", userType, stateId);
} else {
handleError(userType, Long.valueOf(stateId), stateName, Long.valueOf(stateCode), from, to, "Failed to process third API response.");
handleError(userType, Long.valueOf(stateId), stateName, Long.valueOf(stateCode), from, to, "Error during SCP file transfer.");
}

} catch (Exception e) {
Expand Down Expand Up @@ -522,6 +566,7 @@ public void importRchTalukaData(MotechEvent motechEvent) {
String keelDeelApiResponse = rchWebServiceFacade.callKeelDeelApiLocations(token, keel, deel);

String tempFilePath = rchWebServiceFacade.saveToFile(keelDeelApiResponse, "taluka", stateId.toString());
executeScpCommand(tempFilePath,false);

Map<String, Object> params = new HashMap<>();
params.put("state", stateId.toString());
Expand All @@ -546,7 +591,6 @@ public void importRchVillageData(MotechEvent motechEvent) {
Long stateId = (Long) motechEvent.getParameters().get(Constants.STATE_ID_PARAM);
LocalDate startDate = (LocalDate) motechEvent.getParameters().get(Constants.START_DATE_PARAM);
LocalDate endDate = (LocalDate) motechEvent.getParameters().get(Constants.END_DATE_PARAM);
URL endpoint = (URL) motechEvent.getParameters().get(Constants.ENDPOINT_PARAM);

State state = stateDataService.findByCode(stateId);
if (state == null) {
Expand All @@ -571,6 +615,7 @@ public void importRchVillageData(MotechEvent motechEvent) {
String keelDeelApiResponse = rchWebServiceFacade.callKeelDeelApiLocations(token, keel, deel);

String tempFilePath = rchWebServiceFacade.saveToFile(keelDeelApiResponse, "village", stateId.toString());
executeScpCommand(tempFilePath,false);

Map<String, Object> params = new HashMap<>();
params.put("state", stateId.toString());
Expand Down Expand Up @@ -620,6 +665,7 @@ public void importRchHealthBlockData(MotechEvent motechEvent) {
String keelDeelApiResponse = rchWebServiceFacade.callKeelDeelApiLocations(token, keel, deel);

String tempFilePath = rchWebServiceFacade.saveToFile(keelDeelApiResponse, "healthblock", stateId.toString());
executeScpCommand(tempFilePath,false);

Map<String, Object> params = new HashMap<>();
params.put("state", stateId.toString());
Expand Down Expand Up @@ -667,6 +713,7 @@ public void importRchTalukaHealthBlockData(MotechEvent motechEvent) {
String keelDeelApiResponse = rchWebServiceFacade.callKeelDeelApiLocations(token, keel, deel);

String tempFilePath = rchWebServiceFacade.saveToFile(keelDeelApiResponse, "talukahealthblock", stateId.toString());
executeScpCommand(tempFilePath,false);

Map<String, Object> params = new HashMap<>();
params.put("state", stateId.toString());
Expand Down Expand Up @@ -714,6 +761,7 @@ public void importRchHealthFacilityData(MotechEvent motechEvent) {
String keelDeelApiResponse = rchWebServiceFacade.callKeelDeelApiLocations(token, keel, deel);

String tempFilePath = rchWebServiceFacade.saveToFile(keelDeelApiResponse, "healthfacility", stateId.toString());
executeScpCommand(tempFilePath,false);

Map<String, Object> params = new HashMap<>();
params.put("state", stateId.toString());
Expand Down Expand Up @@ -762,6 +810,7 @@ public void importRchHealthSubFacilityData(MotechEvent motechEvent) {
String keelDeelApiResponse = rchWebServiceFacade.callKeelDeelApiLocations(token, keel, deel);

String tempFilePath = rchWebServiceFacade.saveToFile(keelDeelApiResponse, "healthsubfacility", stateId.toString());
executeScpCommand(tempFilePath,false);

Map<String, Object> params = new HashMap<>();
params.put("state", stateId.toString());
Expand Down Expand Up @@ -810,6 +859,7 @@ public void importRchVillageHealthSubFacilityData(MotechEvent motechEvent) {
String keelDeelApiResponse = rchWebServiceFacade.callKeelDeelApiLocations(token, keel, deel);

String tempFilePath = rchWebServiceFacade.saveToFile(keelDeelApiResponse, "villagehealthsubfacility", stateId.toString());
executeScpCommand(tempFilePath,false);

Map<String, Object> params = new HashMap<>();
params.put("state", stateId.toString());
Expand Down Expand Up @@ -883,7 +933,7 @@ public void importRchChildrenData(MotechEvent motechEvent) {

if (keelDeelApiResponse != null) {
String tempFilePath = rchWebServiceFacade.saveToFile(keelDeelApiResponse, "child", stateId.toString());

executeScpCommand(tempFilePath,false);

Map<String, Object> params = new HashMap<>();
params.put("state", stateId.toString());
Expand Down Expand Up @@ -941,9 +991,6 @@ public void importRchAshaData(MotechEvent motechEvent) {
String stateName = state.getName();
Long stateCode = state.getCode();




List<RchImportAudit> failedImports = rchImportAuditDataService.findByStateCodeAndDateRangeAndUserType(stateCode, startReferenceDate, endReferenceDate, RchUserType.ASHA);
boolean isDuplicate = !failedImports.isEmpty();

Expand Down Expand Up @@ -976,6 +1023,7 @@ public void importRchAshaData(MotechEvent motechEvent) {

if (keelDeelApiResponse != null) {
String tempFilePath = rchWebServiceFacade.saveToFile(keelDeelApiResponse, "asha", stateId.toString());
executeScpCommand(tempFilePath,false);

Map<String, Object> params = new HashMap<>();
params.put("state", stateId.toString());
Expand Down
28 changes: 16 additions & 12 deletions rch/src/main/resources/rch.properties
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -78,8 +78,8 @@ rch.loc_update_dir=/usr/local/RCH/loc_update_dir/
rch.scp_timeout=60000

#parameter for location API
rch.location.username=RCHLM_MOTECH
rch.location.api_token=vZgIoff2lQAdZx4HJ9A3yg==
rch.location.username=don'tsaythemagicword
rch.location.api_token=don'tsaythemagicword
rch.location.time_stamp=2011-01-01
rch.location.district_code=0
rch.location.taluka_code=0
Expand All @@ -100,21 +100,25 @@ rch.typeid.health_village=7
rch.typeid.taluka_health_block=11

#base url for location's API
rch.location.base_url=https://rchrpt.nhm.gov.in/Api_MWMIS/RCH_API/LocationMaster
rch.location.base_url=https://localhost:8080/Api_MWMIS/RCH_API/LocationMaster



rch.api.first=https://delhigw.napix.gov.in/nic/rch/encrypttext
rch.api.second=https://delhigw.napix.gov.in/nic/rch/beneficiarydata_rel_test
rch.api.third=https://delhigw.napix.gov.in/nic/rch/decrypttext
rch.api.token=https://delhigw.napix.gov.in/nic/rch//oauth2/token
rch.api.first=https://localhost:8080/nic/rch/encrypttext
rch.api.second=https://localhost:8080/nic/rch/beneficiarydata_rel_test
rch.api.third=https://localhost:8080/nic/rch/decrypttext
rch.api.token=https://localhost:8080/nic/rch//oauth2/token

rch.firstapi.securecode = 7j+hNOJeUxgv+UtX2fKi7A==
rch.firstapi.securecode=don'tsaythemagicword

rch.client.id=c2a64dbadb387bc8ba1702db79ad8463
rch.client.secret=42a30d3ced62b00cdbb37fbf17d63677
rch.client.id=don'tsaythemagicword
rch.client.secret=don'tsaythemagicword

rch.folder.tmp=/usr/local/tmp
rch.folder.records=/usr/local/RCH/rch-records

rch.location.second_api=https://delhigw.napix.gov.in/nic/rch/location_master_rel_test
rch.folder.tmp.remote=/usr/local/tmp/

rch.remote.url=


rch.location.second_api=https://localhost:8080/nic/rch/location_master_rel_test