Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

SCP with multiple retries of file from web to db #867

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
Open

SCP with multiple retries of file from web to db #867

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
88 changes: 68 additions & 20 deletions rch/src/main/java/org/motechproject/nms/rch/service/impl/RchWsImportServiceImpl.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,13 +16,13 @@
import org.motechproject.nms.rch.domain.RchImportFacilitator;
import org.motechproject.nms.rch.domain.RchImportFailRecord;
import org.motechproject.nms.rch.domain.RchUserType;
import org.motechproject.nms.rch.exception.RchWebServiceException;
import org.motechproject.nms.rch.repository.RchImportAuditDataService;
import org.motechproject.nms.rch.repository.RchImportFacilitatorDataService;
import org.motechproject.nms.rch.repository.RchImportFailRecordDataService;
import org.motechproject.nms.rch.service.RchWebServiceFacade;
import org.motechproject.nms.rch.service.RchWsImportService;
import org.motechproject.nms.rch.utils.Constants;
import org.motechproject.nms.rch.utils.ExecutionHelper;
import org.motechproject.nms.region.domain.State;
import org.motechproject.nms.region.repository.StateDataService;
import org.motechproject.server.config.SettingsFacade;
Expand Down Expand Up @@ -219,6 +219,44 @@ public void importVillageHealthSubFacilityFromRch(List<Long> stateIds, LocalDate
LOGGER.info("Initiated import workflow from RCH for villagesubfacility");
}

private void executeScpCommand(String tempFilePath,boolean isRemoteLocal) {
String localDir = settingsFacade.getProperty("rch.folder.tmp");
String remoteDir = settingsFacade.getProperty("rch.folder.tmp.remote");

String command;

if(isRemoteLocal){
command = "scp " + settingsFacade.getProperty("rch.remote.url") + tempFilePath + " " + localDir;
}else {
command = String.format("scp %s %s", tempFilePath, remoteDir);
}
int maxRetries = 3;
long timeout = Long.parseLong(settingsFacade.getProperty("rch.scp_timeout"));

executeWithRetries(command, maxRetries, timeout);
}

private void executeWithRetries(String command, int maxRetries, long timeout) {
ExecutionHelper execHelper = new ExecutionHelper();
int attempt = 0;

while (attempt < maxRetries) {
attempt++;
try {
execHelper.exec(command, timeout);
LOGGER.info("SCP command executed successfully on attempt {}.", attempt);
return;
} catch (Exception e) {
if (attempt < maxRetries) {
LOGGER.warn("Attempt {} of {} failed. Retrying...", attempt, maxRetries, e);
} else {
LOGGER.error("SCP command failed after {} attempts. No more retries.", maxRetries, e);
return;
}
}
}
}

@MotechListener(subjects = { Constants.RCH_MOTHER_IMPORT_SUBJECT })
@Transactional
@Override
Expand Down Expand Up @@ -273,6 +311,7 @@ public void importRchMothersData(MotechEvent motechEvent) {

if (keelDeelApiResponse != null) {
String tempFilePath = rchWebServiceFacade.saveToFile(keelDeelApiResponse, "mother", stateId.toString());
executeScpCommand(tempFilePath,false);

Map<String, Object> params = new HashMap<>();
params.put("state", stateId.toString());
Expand Down Expand Up @@ -326,31 +365,33 @@ public void handleThirdApiEvent(MotechEvent event) {
RchUserType userType = RchUserType.valueOf(entityType.toUpperCase());

try {
String payload = rchWebServiceFacade.readPayloadFromTempFile(tempFilePath);

executeScpCommand(tempFilePath, true);
String payload = rchWebServiceFacade.readPayloadFromTempFile(tempFilePath);

if(payload!=null) {
thirdApiResponse = rchWebServiceFacade.callThirdApi(payload);
if (payload != null) {
thirdApiResponse = rchWebServiceFacade.callThirdApi(payload);
}


if(thirdApiResponse!=null) {
if (thirdApiResponse != null) {
File responseFile = rchWebServiceFacade.generateJsonResponseFile(thirdApiResponse, userType, Long.valueOf(stateId));

if (responseFile != null) {
LOGGER.info("RCH {} response successfully written to file. Copying to remote directory.", userType);
status = rchWebServiceFacade.retryScpAndAudit(responseFile.getName(), from, to, Long.valueOf(stateId), userType, 0);
} else {
LOGGER.error("Error writing {} response to file for state {}", userType, stateId);
}}
if (responseFile != null) {
LOGGER.info("RCH {} third api response successfully written to file. Copying to remote directory.", userType);
status = rchWebServiceFacade.retryScpAndAudit(responseFile.getName(), from, to, Long.valueOf(stateId), userType, 0);
} else {
LOGGER.error("Error writing {} third api response to file for state {}", userType, stateId);
}
}

if (status) {
LOGGER.info(FILE_RECORD_SUCCESS, stateId);
} else {
handleError(userType, Long.valueOf(stateId), stateName, Long.valueOf(stateCode), from, to, "Failed to process third API response.");
handleError(userType, Long.valueOf(stateId), stateName, Long.valueOf(stateCode), from, to, "Error during SCP file transfer.");
}

} catch (Exception e) {
}
catch (Exception e) {
handleError(userType, Long.valueOf(stateId), stateName, Long.valueOf(stateCode), from, to, e.getMessage());
}
}
Expand Down Expand Up @@ -407,6 +448,8 @@ public void importRchDistrictData(MotechEvent motechEvent) {

String tempFilePath = rchWebServiceFacade.saveToFile(keelDeelApiResponse, "district", stateId.toString());

executeScpCommand(tempFilePath,false);

Map<String, Object> params = new HashMap<>();
params.put("state", stateId.toString());
params.put("tempFilePath", tempFilePath);
Expand Down Expand Up @@ -466,6 +509,7 @@ public void handleApiEvent(MotechEvent event) {
String thirdApiResponse = null;

try {
executeScpCommand(tempFilePath,true);
String payload = rchWebServiceFacade.readPayloadFromTempFile(tempFilePath);

if(payload!=null) {
Expand All @@ -484,7 +528,7 @@ public void handleApiEvent(MotechEvent event) {
if (status) {
LOGGER.info("{} data processed successfully for state ID: {}", userType, stateId);
} else {
handleError(userType, Long.valueOf(stateId), stateName, Long.valueOf(stateCode), from, to, "Failed to process third API response.");
handleError(userType, Long.valueOf(stateId), stateName, Long.valueOf(stateCode), from, to, "Error during SCP file transfer.");
}

} catch (Exception e) {
Expand Down Expand Up @@ -522,6 +566,7 @@ public void importRchTalukaData(MotechEvent motechEvent) {
String keelDeelApiResponse = rchWebServiceFacade.callKeelDeelApiLocations(token, keel, deel);

String tempFilePath = rchWebServiceFacade.saveToFile(keelDeelApiResponse, "taluka", stateId.toString());
executeScpCommand(tempFilePath,false);

Map<String, Object> params = new HashMap<>();
params.put("state", stateId.toString());
Expand All @@ -546,7 +591,6 @@ public void importRchVillageData(MotechEvent motechEvent) {
Long stateId = (Long) motechEvent.getParameters().get(Constants.STATE_ID_PARAM);
LocalDate startDate = (LocalDate) motechEvent.getParameters().get(Constants.START_DATE_PARAM);
LocalDate endDate = (LocalDate) motechEvent.getParameters().get(Constants.END_DATE_PARAM);
URL endpoint = (URL) motechEvent.getParameters().get(Constants.ENDPOINT_PARAM);

State state = stateDataService.findByCode(stateId);
if (state == null) {
Expand All @@ -571,6 +615,7 @@ public void importRchVillageData(MotechEvent motechEvent) {
String keelDeelApiResponse = rchWebServiceFacade.callKeelDeelApiLocations(token, keel, deel);

String tempFilePath = rchWebServiceFacade.saveToFile(keelDeelApiResponse, "village", stateId.toString());
executeScpCommand(tempFilePath,false);

Map<String, Object> params = new HashMap<>();
params.put("state", stateId.toString());
Expand Down Expand Up @@ -620,6 +665,7 @@ public void importRchHealthBlockData(MotechEvent motechEvent) {
String keelDeelApiResponse = rchWebServiceFacade.callKeelDeelApiLocations(token, keel, deel);

String tempFilePath = rchWebServiceFacade.saveToFile(keelDeelApiResponse, "healthblock", stateId.toString());
executeScpCommand(tempFilePath,false);

Map<String, Object> params = new HashMap<>();
params.put("state", stateId.toString());
Expand Down Expand Up @@ -667,6 +713,7 @@ public void importRchTalukaHealthBlockData(MotechEvent motechEvent) {
String keelDeelApiResponse = rchWebServiceFacade.callKeelDeelApiLocations(token, keel, deel);

String tempFilePath = rchWebServiceFacade.saveToFile(keelDeelApiResponse, "talukahealthblock", stateId.toString());
executeScpCommand(tempFilePath,false);

Map<String, Object> params = new HashMap<>();
params.put("state", stateId.toString());
Expand Down Expand Up @@ -714,6 +761,7 @@ public void importRchHealthFacilityData(MotechEvent motechEvent) {
String keelDeelApiResponse = rchWebServiceFacade.callKeelDeelApiLocations(token, keel, deel);

String tempFilePath = rchWebServiceFacade.saveToFile(keelDeelApiResponse, "healthfacility", stateId.toString());
executeScpCommand(tempFilePath,false);

Map<String, Object> params = new HashMap<>();
params.put("state", stateId.toString());
Expand Down Expand Up @@ -762,6 +810,7 @@ public void importRchHealthSubFacilityData(MotechEvent motechEvent) {
String keelDeelApiResponse = rchWebServiceFacade.callKeelDeelApiLocations(token, keel, deel);

String tempFilePath = rchWebServiceFacade.saveToFile(keelDeelApiResponse, "healthsubfacility", stateId.toString());
executeScpCommand(tempFilePath,false);

Map<String, Object> params = new HashMap<>();
params.put("state", stateId.toString());
Expand Down Expand Up @@ -810,6 +859,7 @@ public void importRchVillageHealthSubFacilityData(MotechEvent motechEvent) {
String keelDeelApiResponse = rchWebServiceFacade.callKeelDeelApiLocations(token, keel, deel);

String tempFilePath = rchWebServiceFacade.saveToFile(keelDeelApiResponse, "villagehealthsubfacility", stateId.toString());
executeScpCommand(tempFilePath,false);

Map<String, Object> params = new HashMap<>();
params.put("state", stateId.toString());
Expand Down Expand Up @@ -883,7 +933,7 @@ public void importRchChildrenData(MotechEvent motechEvent) {

if (keelDeelApiResponse != null) {
String tempFilePath = rchWebServiceFacade.saveToFile(keelDeelApiResponse, "child", stateId.toString());

executeScpCommand(tempFilePath,false);

Map<String, Object> params = new HashMap<>();
params.put("state", stateId.toString());
Expand Down Expand Up @@ -941,9 +991,6 @@ public void importRchAshaData(MotechEvent motechEvent) {
String stateName = state.getName();
Long stateCode = state.getCode();




List<RchImportAudit> failedImports = rchImportAuditDataService.findByStateCodeAndDateRangeAndUserType(stateCode, startReferenceDate, endReferenceDate, RchUserType.ASHA);
boolean isDuplicate = !failedImports.isEmpty();

Expand Down Expand Up @@ -976,6 +1023,7 @@ public void importRchAshaData(MotechEvent motechEvent) {

if (keelDeelApiResponse != null) {
String tempFilePath = rchWebServiceFacade.saveToFile(keelDeelApiResponse, "asha", stateId.toString());
executeScpCommand(tempFilePath,false);

Map<String, Object> params = new HashMap<>();
params.put("state", stateId.toString());
Expand Down
6 changes: 5 additions & 1 deletion rch/src/main/resources/rch.properties
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -115,6 +115,10 @@ rch.client.id=c2a64dbadb387bc8ba1702db79ad8463
rch.client.secret=42a30d3ced62b00cdbb37fbf17d63677

rch.folder.tmp=/usr/local/tmp
rch.folder.records=/usr/local/RCH/rch-records

rch.folder.tmp.remote=192.168.200.5:/usr/local/tmp/

rch.remote.url =192.168.200.5:


rch.location.second_api=https://delhigw.napix.gov.in/nic/rch/location_master_rel_test