diff --git a/requirements.txt b/requirements.txt index 8c555c4..3a56341 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,3 @@ requests==2.20.* -# boto3==1.15.* # Only needed if using AwsOsduClient. \ No newline at end of file +python-dotenv +boto3==1.15.* # Only needed if using AwsOsduClient. \ No newline at end of file diff --git a/tests/integration.py b/tests/integration.py index e3ec992..12fffc1 100644 --- a/tests/integration.py +++ b/tests/integration.py @@ -5,6 +5,7 @@ """ import json import uuid +import unittest from functools import reduce from unittest import TestCase from dotenv import load_dotenv @@ -16,17 +17,18 @@ load_dotenv(verbose=True) +data_partition = 'osdu' class TestSimpleOsduClient(TestCase): - + def test_endpoint_access(self): # token = os.environ.get('OSDU_ACCESS_TOKEN') - token = AwsOsduClient('opendes').access_token + token = AwsOsduClient(data_partition).access_token query = { - "kind": f"opendes:osdu:*:*", + "kind": f"*:*:*:*", "limit": 1 } - client = SimpleOsduClient('opendes', token) + client = SimpleOsduClient(data_partition, token) result = client.search.query(query)['results'] @@ -36,7 +38,7 @@ def test_endpoint_access(self): class TestAwsOsduClient(TestCase): def test_get_access_token(self): - client = AwsOsduClient('opendes') + client = AwsOsduClient(data_partition) self.assertIsNotNone(client.access_token) @@ -45,63 +47,65 @@ class TestOsduServiceBase(TestCase): @classmethod def setUpClass(cls): # Authenticate once for the test fixture. - cls.osdu = AwsOsduClient('opendes') - + cls.osdu = AwsOsduClient(data_partition) + class TestSearchService_Query(TestOsduServiceBase): - def test_simple_search_for_10_items(self): + def test_simple_search_for_10_items(self): query = { - "kind": f"opendes:osdu:*:*", + "kind": f"*:*:*:*", "limit": 10 } result = self.osdu.search.query(query)['results'] self.assertEqual(10, len(result)) + def test_get_all_wells(self): query_get_all_wells = { - "kind": "opendes:osdu:*:0.2.0", - "query": "data.ResourceTypeID:\"srn:type:master-data/Well:\"" + "kind": "*:*:master-data--Well:*", } result = self.osdu.search.query(query_get_all_wells)['results'] # All results should be wells. - wells = filter(lambda x: x['type'] == 'well-master', result) + wells = filter(lambda x : x['type'] == 'master-data--Well', result) self.assertCountEqual(wells, result) + def test_get_all_wellbores(self): query = { - "kind": "opendes:osdu:*:0.2.0", - "query": "data.ResourceTypeID:\"srn:type:master-data/Wellbore:\"" + "kind": "*:*:master-data--Wellbore:*", } result = self.osdu.search.query(query)['results'] # All results should be well bores. - wellbores = filter(lambda x: x['type'] == 'wellbore-master', result) + wellbores = filter(lambda x : x['type'] == 'master-data--Wellbore', result) self.assertCountEqual(wellbores, result) + def test_full_text_search(self): - # Wildcard (*) search + #Wildcard (*) search query = { - "kind": "opendes:osdu:*:0.2.0", + "kind": "*:*:*:*", "query": "BIR*" } - expected_count = 37 + expected_count = 20 result = self.osdu.search.query(query) self.assertEqual(expected_count, result['totalCount']) + def test_find_well_by_id(self): - # Search By WellID - query = { - "kind": "opendes:osdu:*:0.2.0", - "query": "data.Data.IndividualTypeProperties.WellID:\"srn:master-data/Well:8690:\"" + #Search By WellID + query = { + "kind": "*:*:*:*", + "query": "data.WellID:\"osdu:master-data--Well:8690\"" } expected_count = 2 @@ -109,52 +113,41 @@ def test_find_well_by_id(self): self.assertEqual(expected_count, len(result)) + def test_query_find_matching_wells_by_id(self): # Boolean search with OR - well_ids = ['srn:master-data/Well:8690:', 'srn:master-data/Well:1000:'] + well_ids = [ 'osdu:master-data--Well:8690', 'osdu:master-data--Well:1000' ] query = { - "kind": "opendes:osdu:*:0.2.0", - "query": f"data.ResourceID:(\"{well_ids[0]}\" OR \"{well_ids[1]}\")" + "kind": "*:*:*:*", + "query": f"id:(\"{well_ids[0]}\" OR \"{well_ids[1]}\")" } expected_count = len(well_ids) result = self.osdu.search.query(query)['results'] self.assertEqual(expected_count, len(result)) - self.assertIn(result[0]['data']['ResourceID'], well_ids) - self.assertIn(result[1]['data']['ResourceID'], well_ids) - - def test_find_well_in_country(self): - # More Boolean search - query = { - "kind": "opendes:osdu:*:0.2.0", - "query": "(data.ResourceID:(\"srn:master-data/Well:8690:\" \ - OR \"srn:master-data/Well:1000:\")) \ - AND (data.Data.IndividualTypeProperties.CountryID: \"srn:master-data/GeopoliticalEntity:Netherlands:\")" - } - expected_count = 2 - - result = self.osdu.search.query(query)['results'] - - self.assertEqual(expected_count, len(result)) + self.assertIn(result[0]['id'], well_ids) + self.assertIn(result[1]['id'], well_ids) + @unittest.skip("not currently indexed") def test_find_welllogs_with_gr_curve(self): # WellLog with GR curve query = { - "kind": "opendes:osdu:*:0.2.0", - "query": "(data.ResourceTypeID: \"srn:type:work-product-component/WellLog:\") AND (data.Data.IndividualTypeProperties.Curves.Mnemonic: GR)" + "kind": "*:*:*:*", + "query": "(type: \"work-product-component--WellLog\") AND (data.Curves.Mnemonic: GR)" } - expected_greater_than = 900 + expected_count = 928 result = self.osdu.search.query(query) - self.assertGreater(result['totalCount'], expected_greater_than) + self.assertEqual(expected_count, result['totalCount']) + def test_find_markers_trajectories_for_wellbore(self): # Markers and Trajectories for a Wellbore query = { - "kind": "opendes:osdu:*:0.2.0", - "query": "(data.Data.IndividualTypeProperties.WellboreID: \"srn:master-data/Wellbore:3687:\") AND (data.ResourceTypeID: (\"srn:type:work-product-component/WellboreTrajectory:\" OR \"srn:type:work-product-component/WellboreMarker:\"))" + "kind": "*:*:*:*", + "query": "(data.WellboreID: \"osdu:master-data--Wellbore:3687:\") AND (type: (\"work-product-component--WellboreTrajectory\" OR \"work-product-component--WellboreMarkerSet\"))" } expected_count = 2 @@ -162,26 +155,26 @@ def test_find_markers_trajectories_for_wellbore(self): self.assertEqual(expected_count, len(result)) + def test_returned_fields(self): query = { - "kind": "opendes:osdu:*:0.2.0", - "query": "data.ResourceTypeID:\"srn:type:master-data/Well:\"", + "kind": "*:*:*:*", + "query": "type:\"master-data--Well\"", "limit": 10, - "returnedFields": ["data.Data.IndividualTypeProperties.CountryID"] + "returnedFields": ["data.FacilityID"] } result = self.osdu.search.query(query)['results'] + + returned_fields = lambda d : list(d.keys()) + self.assertListEqual(['FacilityID'], returned_fields(result[0]['data'])) - def returned_fields(d): return list(d.keys()) - # self.assertListEqual(['data'], returned_fields(result[0])) - self.assertListEqual( - ['Data.IndividualTypeProperties.CountryID'], returned_fields(result[0]['data'])) def test_find_number_of_wellbores_for_a_well(self): - # Get the number of wellbores for a well + # Get the number of wellbores for a well query = { - "kind": "opendes:osdu:*:0.2.0", - "query": "data.Data.IndividualTypeProperties.WellID:\"srn:master-data/Well:3687:\"", + "kind": "*:*:*:*", + "query": "data.WellID:\"osdu:master-data--Well:3687\"", "returnedFields": [""] } expected_count = 3 @@ -190,10 +183,11 @@ def test_find_number_of_wellbores_for_a_well(self): self.assertEqual(expected_count, count) + def test_malformed_query_raises_exception(self): # Query with 3-part kind. Must be full 4-part kind. query = { - "kind": "osdu:*:0.2.0" + "kind": "*:*:*" } with self.assertRaises(requests.HTTPError): should_fail = self.osdu.search.query(query) @@ -205,7 +199,7 @@ def test_basic_paging(self): page_size = 100 max_pages = 10 query = { - "kind": f"opendes:osdu:*:*", + "kind": f"*:*:*:*", "limit": page_size } result = self.osdu.search.query_with_paging(query) @@ -213,19 +207,19 @@ def test_basic_paging(self): # Iterate over first 'max_pages' pages and check that each page contains 'page_size' results. page_count = 1 for page, total_count in result: - with self.subTest(i=page_count): - self.assertEqual(page_size, len( - page), f'Failed on page #{page_count}') + with (self.subTest(i=page_count)): + self.assertEqual(page_size, len(page), f'Failed on page #{page_count}') page_count += 1 if page_count >= max_pages: break - + self.assertGreater(page_count, 1) + def test_paging_gets_all_results(self): page_size = 1000 query = { - 'kind': 'opendes:osdu:well-master:*', + 'kind': '*:*:master-data--Well:*', 'limit': page_size } result = self.osdu.search.query_with_paging(query) @@ -235,20 +229,13 @@ def test_paging_gets_all_results(self): for page, total in result: total_count = total record_count += len(page) - + self.assertGreater(record_count, 0) self.assertGreater(total_count, 0) self.assertEqual(total_count, record_count) - class TestStorageService(TestOsduServiceBase): - def test_query_all_kinds(self): - expected_min = 100 - result = self.osdu.storage.query_all_kinds()['results'] - - self.assertGreater(len(result), expected_min) - def test_get_record(self): record_id_query = { "kind": "*:*:*:*", @@ -261,14 +248,16 @@ def test_get_record(self): self.assertEqual(record_id, result['id']) + def test_get_nonexistant_record_raises_excpetion(self): - fake_record_id = 'opendes:doc:7be4fc7918e348c2bbc4d6f25b2ff334' # 'ABC123' + fake_record_id = 'opendes:doc:7be4fc7918e348c2bbc4d6f25b2ff334' #'ABC123' with self.assertRaises(requests.HTTPError): should_fail = self.osdu.storage.get_record(fake_record_id) + def test_get_all_record_versions(self): record_id_query = { - "kind": "*:*:well-master:*", + "kind": "*:*:master-data--Well:*", "limit": 1, "returnedFields": ["id"] } @@ -279,20 +268,19 @@ def test_get_all_record_versions(self): self.assertEqual(record_id, result['recordId']) self.assertGreaterEqual(len(result['versions']), 1) + def test_get_record_version(self): record_id_query = { - "kind": "*:*:well-master:*", + "kind": "*:*:master-data--Well:*", "limit": 1 } expected_record = self.osdu.search.query(record_id_query)['results'][0] - result = self.osdu.storage.get_record_version( - expected_record['id'], expected_record['version']) + result = self.osdu.storage.get_record_version(expected_record['id'], expected_record['version']) self.assertEqual(expected_record['id'], result['id']) self.assertEqual(expected_record['version'], result['version']) - class TestStorageService_WithSideEffects(TestOsduServiceBase): @classmethod @@ -313,76 +301,3 @@ def tearDownClass(cls): super().setUpClass() for record_id in cls.test_records: cls.osdu.storage.delete_record(record_id) - - -class TestDeliveryService(TestOsduServiceBase): - - def test_get_signed_urls_wells(self): - # Arrange - expected_count = { - 'unprocessed': 0 - } - srns_query = { - "kind": "opendes:osdu:*:0.2.0", - "query": "(data.Data.IndividualTypeProperties.WellboreID: \"srn:master-data/Wellbore:*:\") AND (data.ResourceTypeID: (\"srn:type:work-product-component/WellboreTrajectory:\" OR \"srn:type:work-product-component/WellboreMarker:\"))", - "returnedFields": ["data.Data.GroupTypeProperties.Files"] - } - srns_results = self.osdu.search.query(srns_query)['results'] - srns = reduce( - (lambda a, b: a + b), - map( - lambda result: result['data']['Data.GroupTypeProperties.Files'], - srns_results - ) - ) - expected_count['processed'] = len(srns) - - # Act - result = self.osdu.delivery.get_signed_urls(srns) - - # Assert - self.assertEqual(expected_count['processed'], len(result['processed'])) - self.assertEqual( - expected_count['unprocessed'], len(result['unprocessed'])) - - -class TestDatasetService_WithSideEffects(TestOsduServiceBase): - - @classmethod - def setUpClass(cls): - super().setUpClass() - cls.test_dataset_id = f'opendes:dataset--File.Generic:osdupy-{uuid.uuid4()}' - - def test_000_get_storage_instructions(self): - result = self.osdu.dataset.get_storage_instructions( - 'dataset--File.Generic') - - self.assertEqual('AWS_S3', result['providerKey']) - self.assertIsNotNone(result['storageLocation']) - - def test_001_register_dataset_file(self): - datasetRegistry = json.load( - open('tests/test_data/test_register_dataset.json')) - datasetRegistry['datasetRegistries'][0]['id'] = self.test_dataset_id - result = self.osdu.dataset.register_dataset(datasetRegistry) - self.assertEqual(self.test_dataset_id, - result['datasetRegistries'][0]['id']) - - def test_002_get_dataset_registry(self): - result = self.osdu.dataset.get_dataset_registry(self.test_dataset_id) - self.assertEqual(1, len(result['datasetRegistries'])) - self.assertEqual(self.test_dataset_id, - result['datasetRegistries'][0]['id']) - - def test_002_get_dataset_registries(self): - ids = [self.test_dataset_id] - result = self.osdu.dataset.get_dataset_registries(ids) - self.assertEqual(1, len(result['datasetRegistries'])) - self.assertEqual(self.test_dataset_id, - result['datasetRegistries'][0]['id']) - - def test_002_get_retrieval_instructions(self): - ids = [self.test_dataset_id] - result = self.osdu.dataset.get_retrieval_instructions(ids) - self.assertEqual(self.test_dataset_id, - result['delivery'][0]['datasetRegistryId']) diff --git a/tests/test_data/test_create_single_record.json b/tests/test_data/test_create_single_record.json index aec6dd5..6bf98e1 100644 --- a/tests/test_data/test_create_single_record.json +++ b/tests/test_data/test_create_single_record.json @@ -1,35 +1,29 @@ { "data": { - "ResourceTypeID": "srn:type:file/csv:", - "ResourceSecurityClassification": "srn:reference-data/ResourceSecurityClassification:RESTRICTED:", - "Data": { - "GroupTypeProperties": { - "FileSource": "", - "PreLoadFilePath": "s3://any_path/r1/data/provided/markers/137999999.csv" - }, - "IndividualTypeProperties": {}, - "ExtensionProperties": {} - }, - "AssociativeID": "f-1", - "ResourceID": "srn:file/csv:806407212248301982083####:" - }, - "meta": [], - "kind": "opendes:osdu:file:0.2.1", - "acl": { - "viewers": [ - "data.default.viewers@opendes.testing.com" - ], - "owners": [ - "data.default.owners@opendes.testing.com" - ] + "DatasetProperties.FileSourceInfo.FileSource": "", + "ResourceSecurityClassification": "osdu:reference-data--ResourceSecurityClassification:RESTRICTED:", + "SchemaFormatTypeID": "osdu:reference-data--SchemaFormatType:TabSeparatedColumnarText:" }, + "kind": "osdu:wks:dataset--File.Generic:1.0.0", + "namespace": "osdu:wks", "legal": { "legaltags": [ - "opendes-public-usa-dataset-1" + "osdu-public-usa-dataset-1" ], "otherRelevantDataCountries": [ "US" ], "status": "compliant" - } + }, + "id": "osdu:dataset--File.Generic:d180e2f8c63b041910c62d9defd5064639cb588d2a7ece7604dafa6a5ca9d", + "acl": { + "viewers": [ + "data.default.viewers@osdu.testing.com" + ], + "owners": [ + "data.default.owners@osdu.testing.com" + ] + }, + "type": "dataset--File.Generic", + "version": 1617223984663669 } \ No newline at end of file