Skip to content

Commit

Permalink
Merge pull request #28 from pranavanba/main
Browse files Browse the repository at this point in the history
Update S3 path in file provenance
  • Loading branch information
pranavanba authored Apr 22, 2024
2 parents c3dab47 + d1781a1 commit e06fb44
Showing 1 changed file with 6 additions and 1 deletion.
7 changes: 6 additions & 1 deletion scripts/main/sts_synindex_external.R
Original file line number Diff line number Diff line change
Expand Up @@ -91,6 +91,11 @@ replace_equal_with_underscore <- function(directory_path) {
synapser::synLogin(authToken = Sys.getenv('SYNAPSE_AUTH_TOKEN'))
config::get(config = "staging") %>% list2env(envir = .GlobalEnv)

unlink(x = c(AWS_PARQUET_DOWNLOAD_LOCATION,
AWS_ARCHIVE_DOWNLOAD_LOCATION,
PARQUET_FINAL_LOCATION),
recursive = TRUE,
force = TRUE)

# Get STS credentials for input data bucket -------------------------------
token <-
Expand Down Expand Up @@ -218,7 +223,7 @@ latest_commit_this_file <- paste0(latest_commit$html_url %>% stringr::str_replac

act <- synapser::Activity(name = "Indexing",
description = "Indexing external parquet datasets",
used = paste0("s3://", latest_archive),
used = paste0("https://s3.amazonaws.com/", latest_archive),
executed = latest_commit_this_file)

if(nrow(synapse_manifest_to_upload) > 0){
Expand Down

0 comments on commit e06fb44

Please sign in to comment.