From 5c648f5ca0babfcb29a37d704b7d790f156937ea Mon Sep 17 00:00:00 2001 From: David-YuWei Date: Wed, 28 Sep 2022 10:36:53 -0400 Subject: [PATCH] Ins 252 updated (#42) * Update INS-252 feature: delete partial scrape data files * Update INS-252 feature: add updated partial scrape data files * Update INS-252 feature: add sorting for data loading process to make sure data files will be read in order --- ...alScrape.tsv => patent_application_UpdatedPartialScrape.tsv} | 0 ..._PartialScrape.tsv => patent_grant_UpdatedPartialScrape.tsv} | 0 ...oject_PartialScrape.tsv => project_UpdatedPartialScrape.tsv} | 0 ...n_PartialScrape.tsv => publication_UpdatedPartialScrape.tsv} | 0 dataloader/loader.py | 2 +- 5 files changed, 1 insertion(+), 1 deletion(-) rename data/{patent_application_PartialScrape.tsv => patent_application_UpdatedPartialScrape.tsv} (100%) rename data/{patent_grant_PartialScrape.tsv => patent_grant_UpdatedPartialScrape.tsv} (100%) rename data/{project_PartialScrape.tsv => project_UpdatedPartialScrape.tsv} (100%) rename data/{publication_PartialScrape.tsv => publication_UpdatedPartialScrape.tsv} (100%) diff --git a/data/patent_application_PartialScrape.tsv b/data/patent_application_UpdatedPartialScrape.tsv similarity index 100% rename from data/patent_application_PartialScrape.tsv rename to data/patent_application_UpdatedPartialScrape.tsv diff --git a/data/patent_grant_PartialScrape.tsv b/data/patent_grant_UpdatedPartialScrape.tsv similarity index 100% rename from data/patent_grant_PartialScrape.tsv rename to data/patent_grant_UpdatedPartialScrape.tsv diff --git a/data/project_PartialScrape.tsv b/data/project_UpdatedPartialScrape.tsv similarity index 100% rename from data/project_PartialScrape.tsv rename to data/project_UpdatedPartialScrape.tsv diff --git a/data/publication_PartialScrape.tsv b/data/publication_UpdatedPartialScrape.tsv similarity index 100% rename from data/publication_PartialScrape.tsv rename to data/publication_UpdatedPartialScrape.tsv diff --git a/dataloader/loader.py b/dataloader/loader.py index 19c7250..bbe2f52 100755 --- a/dataloader/loader.py +++ b/dataloader/loader.py @@ -225,7 +225,7 @@ def main(): for plugin_config in config.plugins: plugins.append(prepare_plugin(plugin_config, schema)) loader = DataLoader(driver, schema, plugins) - + file_list.sort() loader.load(file_list, config.cheat_mode, config.dry_run, config.loading_mode, config.wipe_db, config.max_violations, split=config.split_transactions, no_backup=config.no_backup, neo4j_uri=config.neo4j_uri, backup_folder=config.backup_folder)