Skip to content

Commit

Permalink
Noop: code reorganization (#24)
Browse files Browse the repository at this point in the history
  • Loading branch information
drscholly authored Nov 28, 2023
1 parent 1e0494c commit 4f4a04f
Show file tree
Hide file tree
Showing 19 changed files with 175 additions and 184 deletions.
1 change: 0 additions & 1 deletion tests/test_copy_dataprocessings.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
from toolbox.api.datagalaxy_api_dataprocessings import DataGalaxyApiDataprocessings
from toolbox.commands.copy_dataprocessings import copy_dataprocessings
from toolbox.api.datagalaxy_api_workspaces import DataGalaxyApiWorkspace

from toolbox.api.datagalaxy_api import DataGalaxyApiAuthentication, Token
import pytest as pytest

Expand Down
1 change: 0 additions & 1 deletion tests/test_copy_dictionary.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
from toolbox.api.datagalaxy_api_dictionary import DataGalaxyApiDictionary
from toolbox.commands.copy_dictionary import copy_dictionary
from toolbox.api.datagalaxy_api_workspaces import DataGalaxyApiWorkspace

from toolbox.api.datagalaxy_api import DataGalaxyApiAuthentication, Token
import pytest as pytest

Expand Down
20 changes: 10 additions & 10 deletions tests/test_copy_glossary.py
Original file line number Diff line number Diff line change
@@ -1,13 +1,13 @@
from toolbox.api.datagalaxy_api_workspaces import DataGalaxyApiWorkspace, DataGalaxyApiGlossary, \
DataGalaxyBulkResult
from toolbox.api.datagalaxy_api_glossary import DataGalaxyApiGlossary
from toolbox.api.datagalaxy_api_workspaces import DataGalaxyApiWorkspace
from toolbox.commands.copy_glossary import copy_glossary
from toolbox.api.datagalaxy_api import DataGalaxyApiAuthentication, Token
from toolbox.api.datagalaxy_api import DataGalaxyApiAuthentication, Token, DataGalaxyBulkResult
import pytest as pytest


# Mocks

def mock_list_glossary_properties_on_source_workspace(self, workspace_name):
def mock_list_properties_on_source_workspace(self, workspace_name):
if workspace_name == 'workspace_source':
return ['property1', 'property1', 'property1']
return []
Expand All @@ -27,10 +27,10 @@ def test_copy_glossary_when_no_property_on_target(mocker):
workspace_source_mock.return_value = 'workspace_source'
glossary_properties_on_source_workspace_mock = mocker.patch.object(
DataGalaxyApiGlossary,
'list_glossary_properties',
'list_properties',
autospec=True
)
glossary_properties_on_source_workspace_mock.side_effect = mock_list_glossary_properties_on_source_workspace
glossary_properties_on_source_workspace_mock.side_effect = mock_list_properties_on_source_workspace
bulk_upsert_properties_on_target_workspace_mock = mocker.patch.object(
DataGalaxyApiGlossary,
'bulk_upsert_property_tree',
Expand Down Expand Up @@ -73,9 +73,9 @@ def test_copy_glossary_same_client_space(mocker):
workspace_source_mock.return_value = 'workspace_source'
glossary_properties_on_source_workspace_mock = mocker.patch.object(
DataGalaxyApiGlossary,
'list_glossary_properties',
'list_properties',
autospec=True)
glossary_properties_on_source_workspace_mock.side_effect = mock_list_glossary_properties_on_source_workspace
glossary_properties_on_source_workspace_mock.side_effect = mock_list_properties_on_source_workspace
bulk_upsert_properties_on_target_workspace_mock = mocker.patch.object(
DataGalaxyApiGlossary,
'bulk_upsert_property_tree',
Expand Down Expand Up @@ -121,9 +121,9 @@ def test_copy_glossary_when_workspace_target_does_not_exist(mocker):
workspace_source_mock = mocker.patch.object(DataGalaxyApiWorkspace, 'get_workspace', autospec=True)
workspace_source_mock.return_value = None
glossary_properties_on_source_workspace_mock = mocker.patch.object(DataGalaxyApiGlossary,
'list_glossary_properties',
'list_properties',
autospec=True)
glossary_properties_on_source_workspace_mock.side_effect = mock_list_glossary_properties_on_source_workspace
glossary_properties_on_source_workspace_mock.side_effect = mock_list_properties_on_source_workspace

# ASSERT / VERIFY
with pytest.raises(Exception, match='workspace workspace_target does not exist'):
Expand Down
5 changes: 2 additions & 3 deletions tests/test_copy_usages.py
Original file line number Diff line number Diff line change
@@ -1,8 +1,7 @@
from toolbox.api.datagalaxy_api_usages import DataGalaxyApiUsages
from toolbox.commands.copy_usages import copy_usages
from toolbox.api.datagalaxy_api_workspaces import DataGalaxyApiWorkspace, DataGalaxyBulkResult

from toolbox.api.datagalaxy_api import DataGalaxyApiAuthentication, Token
from toolbox.api.datagalaxy_api_workspaces import DataGalaxyApiWorkspace
from toolbox.api.datagalaxy_api import DataGalaxyApiAuthentication, Token, DataGalaxyBulkResult
import pytest as pytest


Expand Down
2 changes: 1 addition & 1 deletion tests/test_del_useless_keys.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
from toolbox.api.datagalaxy_api_workspaces import del_useless_keys
from toolbox.api.datagalaxy_api import del_useless_keys


def test_del_useless_keys_with_too_many_keys():
Expand Down
1 change: 0 additions & 1 deletion tests/test_main.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@
import io
import re

from toolbox.__main__ import run


Expand Down
1 change: 0 additions & 1 deletion tests/test_to_bulk_item_att.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,4 @@
import pytest as pytest

from toolbox.api.datagalaxy_api_attributes import to_bulk_item_attribute


Expand Down
2 changes: 1 addition & 1 deletion tests/test_to_bulk_properties_tree.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
from toolbox.api.datagalaxy_api_workspaces import to_bulk_tree
from toolbox.api.datagalaxy_api import to_bulk_tree


def test_transform_ok():
Expand Down
77 changes: 77 additions & 0 deletions toolbox/api/datagalaxy_api.py
Original file line number Diff line number Diff line change
Expand Up @@ -60,3 +60,80 @@ def get_access_token(url: str, integration_token: Token):
client_space_id = integration_token.get_client_space_id()
logging.info(f'get_access_token - Authenticating [url: {url} , client_space: {client_space_id}]')
return access_token


@dataclass(frozen=True)
class DataGalaxyBulkResult:
total: int
created: int
deleted: int
unchanged: int
updated: int


BULK_PROPERTIES_FIELDS_TO_REMOVE = ['path', 'typePath', 'location', 'attributes', 'objectUrl', 'childrenCount',
'lastModificationTime', 'creationTime']


def del_useless_keys(members: dict):
for key in list(members.keys()):
if key in BULK_PROPERTIES_FIELDS_TO_REMOVE:
del members[key]
else:
pass

return members


PATH_SEPARATOR = "\\"


@dataclass(frozen=True)
class DataGalaxyPathWithType:
path: str
path_type: str


def handle_timeserie(property: dict) -> dict:
# Temporary solution: only copy the latest value of the TimeSerie
for key, value in property.items():
if isinstance(value, dict):
if 'lastEntry' in value:
# Expected format : "Date::Value"
property[key] = f"{value['lastEntry']['date']}::{value['lastEntry']['value']}"


def to_bulk_tree(properties: list) -> list:
nodes_map = {}
for property in properties:
nodes_map[DataGalaxyPathWithType(property['path'], property['typePath'])] = property

for property in properties:

if 'attributes' in property:
property.update(property['attributes'])

path = property['path']
path_type = property['typePath']
del_useless_keys(property)
handle_timeserie(property)

# TRANSFORM to bulk item
path_segments = path[1:].split(PATH_SEPARATOR)
if len(path_segments) > 1:
parent_path_segments = path_segments[:-1]
parent_path_type_segments = path_type[1:].split(PATH_SEPARATOR)[:-1]
parent_path = f"{PATH_SEPARATOR}{PATH_SEPARATOR.join(parent_path_segments)}"
parent_path_type = f"{PATH_SEPARATOR}{PATH_SEPARATOR.join(parent_path_type_segments)}"
parent = nodes_map[DataGalaxyPathWithType(parent_path, parent_path_type)]
if 'children' in parent:
parent['children'].append(property)
else:
parent['children'] = [property]

root_nodes = []
for key, value in nodes_map.items():
if len(key.path[1:].split(PATH_SEPARATOR)) == 1:
root_nodes.append(value)

return root_nodes
1 change: 0 additions & 1 deletion toolbox/api/datagalaxy_api_attributes.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@
import logging
from enum import Enum

import requests as requests


Expand Down
3 changes: 1 addition & 2 deletions toolbox/api/datagalaxy_api_dataprocessings.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
import logging

import requests as requests
from toolbox.api.datagalaxy_api_workspaces import DataGalaxyBulkResult, to_bulk_tree
from toolbox.api.datagalaxy_api import DataGalaxyBulkResult, to_bulk_tree


class DataGalaxyApiDataprocessings:
Expand Down
3 changes: 1 addition & 2 deletions toolbox/api/datagalaxy_api_dictionary.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
import logging

import requests as requests
from toolbox.api.datagalaxy_api_workspaces import DataGalaxyBulkResult, to_bulk_tree
from toolbox.api.datagalaxy_api import DataGalaxyBulkResult, to_bulk_tree


class DataGalaxyApiDictionary:
Expand Down
71 changes: 71 additions & 0 deletions toolbox/api/datagalaxy_api_glossary.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,71 @@
import logging
import requests as requests
from toolbox.api.datagalaxy_api import DataGalaxyBulkResult, to_bulk_tree


class DataGalaxyApiGlossary:
def __init__(self, url: str, access_token: str, workspace: dict):
self.url = url
self.access_token = access_token
self.workspace = workspace

def list_properties(self, workspace_name: str) -> list:
if not self.workspace["isVersioningEnabled"]:
version_id = self.workspace['defaultVersionId']
params = {'versionId': version_id, 'includeAttributes': 'true'}
headers = {'Authorization': f"Bearer {self.access_token}"}
response = requests.get(f"{self.url}/properties", params=params, headers=headers)
code = response.status_code
body_json = response.json()
if code == 200:
logging.info(
f'list_properties - {len(body_json["results"])} properties found on '
f'workspace {workspace_name} :')
result = []
result = result + body_json['results']
next_page = body_json["next_page"]
while next_page is not None:
headers = {'Authorization': f"Bearer {self.access_token}"}
response = requests.get(next_page, headers=headers)
body_json = response.json()
next_page = body_json["next_page"]
result = result + body_json['results']
return result

if 400 <= code < 500:
raise Exception(body_json['error'])

raise Exception(f'Unexpected error, code: {code}')

elif self.workspace["isVersioningEnabled"]:
raise Exception('pour l instant on ne gere pas le versioning')

def bulk_upsert_property_tree(self, workspace_name: str, properties: list) -> DataGalaxyBulkResult:
# Existing entities are updated and non-existing ones are created.
properties_ok_to_bulk = to_bulk_tree(properties)
logging.info(f'properties_ok_to_bulk: {properties_ok_to_bulk}')

if not self.workspace["isVersioningEnabled"]:
version_id = self.workspace['defaultVersionId']
headers = {'Authorization': f"Bearer {self.access_token}"}
response = requests.post(f"{self.url}/properties/bulktree/{version_id}", json=properties_ok_to_bulk,
headers=headers)
code = response.status_code
body_json = response.json()
if 200 <= code < 300:
result = DataGalaxyBulkResult(total=body_json["total"], created=body_json["created"],
deleted=body_json["deleted"], unchanged=body_json["unchanged"],
updated=body_json["updated"])
logging.info(
f'bulk_upsert_property_tree - {result.total} properties copied on workspace {workspace_name}:'
f'{result.created} were created, {result.updated} were updated, '
f'{result.deleted} were deleted and {result.unchanged} were unchanged')
return result

if 400 <= code < 500:
raise Exception(body_json['error'])

raise Exception(f'Unexpected error, code: {code}')

elif self.workspace["isVersioningEnabled"]:
raise Exception('pour l instant on ne sait pas si on accepte le versioning')
3 changes: 1 addition & 2 deletions toolbox/api/datagalaxy_api_usages.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
import logging

import requests as requests
from toolbox.api.datagalaxy_api_workspaces import DataGalaxyBulkResult, to_bulk_tree
from toolbox.api.datagalaxy_api import DataGalaxyBulkResult, to_bulk_tree


class DataGalaxyApiUsages:
Expand Down
Loading

0 comments on commit 4f4a04f

Please sign in to comment.