diff --git a/fusion/fusion_filesystem.py b/fusion/fusion_filesystem.py index 6d46bd7..56c5500 100644 --- a/fusion/fusion_filesystem.py +++ b/fusion/fusion_filesystem.py @@ -110,7 +110,9 @@ async def _ls_real(self, url, detail=True, **kwargs): is_file = False size = None try: - async with session.head(url + "/operationType/download", **self.kwargs) as r: + async with session.head( + url + "/operationType/download", **self.kwargs + ) as r: self._raise_not_found_for_status(r, url) out = [ url.split("/")[6] @@ -125,6 +127,7 @@ async def _ls_real(self, url, detail=True, **kwargs): size = int(r.headers["Content-Length"]) is_file = True except Exception as _: + logger.debug("Not a file - " + _) async with session.get(url, **self.kwargs) as r: self._raise_not_found_for_status(r, url) out = await r.json() @@ -353,7 +356,6 @@ async def put_data(): @staticmethod def _construct_headers(file_local, dt_iso, chunk_size=5 * 2**20, multipart=False): - headers = { "Content-Type": "application/octet-stream", "x-jpmc-distribution-created-date": dt_iso, diff --git a/fusion/utils.py b/fusion/utils.py index d749161..defab08 100644 --- a/fusion/utils.py +++ b/fusion/utils.py @@ -72,7 +72,8 @@ def tqdm_joblib(tqdm_object): """ class TqdmBatchCompletionCallback(joblib.parallel.BatchCompletionCallBack): - """ Tqdm execution wrapper.""" + """Tqdm execution wrapper.""" + def __call__(self, *args, **kwargs): n = 0 for i in args[0]._result: @@ -906,7 +907,9 @@ def _upload(row): delayed(_upload)(row) for index, row in loop.iterrows() ) else: - res = Parallel(n_jobs=n_par)(delayed(_upload)(row) for index, row in loop.iterrows()) + res = Parallel(n_jobs=n_par)( + delayed(_upload)(row) for index, row in loop.iterrows() + ) else: if show_progress: with tqdm_joblib(tqdm(total=len(loop.iterrows()))) as _: