Skip to content

Commit

Permalink
Fix pool issue in FileUploadQueue (#78)
Browse files Browse the repository at this point in the history
FileUploadQueue had an incorrect call to the threadpool's submit that
caused the files to never be uploaded.
  • Loading branch information
mathialo authored Jan 7, 2021
1 parent a98636a commit accf9b7
Show file tree
Hide file tree
Showing 3 changed files with 5 additions and 8 deletions.
2 changes: 1 addition & 1 deletion cognite/extractorutils/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,4 +16,4 @@
Cognite extractor utils is a Python package that simplifies the development of new extractors.
"""

__version__ = "1.2.2"
__version__ = "1.2.3"
9 changes: 3 additions & 6 deletions cognite/extractorutils/uploader.py
Original file line number Diff line number Diff line change
Expand Up @@ -893,21 +893,18 @@ def upload(self) -> None:
backoff=RETRY_BACKOFF_FACTOR,
)
def _upload_single(self, index, file_name, file_meta):

# Upload file
file_meta = self.cdf_client.files.upload(file_name, **file_meta.dump(), overwrite=self.overwrite_existing)
file_meta = self.cdf_client.files.upload(file_name, overwrite=self.overwrite_existing, **file_meta.dump())

# Update meta-object in queue
with self.lock:
self.upload_queue[index] = (file_meta, file_name)
self.upload_queue[index] = (file_meta, file_name)

def _upload_batch(self):

# Concurrently execute file-uploads

with ThreadPoolExecutor(self.cdf_client.config.max_workers) as pool:
for i, (file_meta, file_name) in enumerate(self.upload_queue):
pool.submit(fn=self._upload_single, args=(i, file_name, file_meta))
pool.submit(self._upload_single, i, file_name, file_meta)

def __enter__(self) -> "FileUploadQueue":
"""
Expand Down
2 changes: 1 addition & 1 deletion pyproject.toml
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
[tool.poetry]
name = "cognite-extractor-utils"
version = "1.2.2"
version = "1.2.3"
description = "Utilities for easier development of extractors for CDF"
authors = ["Mathias Lohne <[email protected]>"]
license = "Apache-2.0"
Expand Down

0 comments on commit accf9b7

Please sign in to comment.