From 100b0a811973327c552db99848c0aca31f20d627 Mon Sep 17 00:00:00 2001 From: Pranav Anbarasu Date: Tue, 9 Apr 2024 18:29:52 +0000 Subject: [PATCH] Delete existing dirs on each run in order to keep only newly indexed files and remove previously indexed files --- sts_synindex_internal.R | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/sts_synindex_internal.R b/sts_synindex_internal.R index 20c239a..083d0e9 100644 --- a/sts_synindex_internal.R +++ b/sts_synindex_internal.R @@ -62,6 +62,14 @@ junk <- invisible(lapply(list.dirs(AWS_PARQUET_DOWNLOAD_LOCATION), replace_equal # Generate manifest of existing files ------------------------------------- +existing_dirs <- synapser::synGetChildren(PARQUET_FOLDER) %>% synapser::as.list() + +if(length(existing_dirs)>0) { + for (i in seq_along(existing_dirs)) { + synapser::synDelete(existing_dirs[[i]]$id) + } +} + SYNAPSE_AUTH_TOKEN <- Sys.getenv('SYNAPSE_AUTH_TOKEN') manifest_cmd <- glue::glue('SYNAPSE_AUTH_TOKEN="{SYNAPSE_AUTH_TOKEN}" synapse manifest --parent-id {SYNAPSE_PARENT_ID} --manifest ./current_manifest.tsv {AWS_PARQUET_DOWNLOAD_LOCATION}') system(manifest_cmd)