From 80a62f104b43928828f214b8e57636677d19d89d Mon Sep 17 00:00:00 2001 From: Daniel Smith Date: Mon, 1 Oct 2018 13:58:17 -0400 Subject: [PATCH 1/8] OpenFF: Workflow doc and organization --- .../interface/collections/openffworkflow.py | 101 +++++++++++++----- 1 file changed, 75 insertions(+), 26 deletions(-) diff --git a/qcfractal/interface/collections/openffworkflow.py b/qcfractal/interface/collections/openffworkflow.py index 6687a6e27..18ae6a634 100644 --- a/qcfractal/interface/collections/openffworkflow.py +++ b/qcfractal/interface/collections/openffworkflow.py @@ -39,16 +39,15 @@ def __init__(self, name, options=None, client=None, **kwargs): """ super().__init__(name, client=client, options=options, **kwargs) + self._fragment_cache = {} + def _init_collection_data(self, additional_args): options = additional_args.get("options", None) if options is None: raise KeyError("No record of OpenFFWorkflow {} found and no initial options passed in.".format(name)) ret = copy.deepcopy(options) - ret["fragments"] = {} # No known fragments - ret["molecules"] = [] - - ret["fragment_cache"] = {} # Caches pulled fragment data + ret["fragments"] = {} return ret @@ -56,19 +55,63 @@ def _pre_save_prep(self, client): pass def get_options(self, key): + """ + Obtains "base" workflow options that do not change. + + Parameters + ---------- + key : str + The original workflow options. + + Returns + ------- + dict + The requested options dictionary. + """ if key not in self.__required_fields: raise KeyError("Key `{}` not understood.".format(key)) return copy.deepcopy(self.data[key]) def list_fragments(self): - return copy.deepcopy(list(self.data["fragments"])) + """ + List all fragments associated with this workflow. - def list_initial_molecules(self): - return copy.deepcopy(self.data["molecules"]) + Returns + ------- + list of str + A list of fragment id's. + """ + return list(self.data["fragments"]) def add_fragment(self, fragment_id, data, provenance={}): + """ + Adds a new fragment to the workflow along with the associated torsiondrives required. + Parameters + ---------- + fragment_id : str + The tag associated with fragment. In general this should be the canonical isomeric + explicit hydrogen mapped SMILES tag for this fragment. + data : dict + A dictionary of label : {intial_molecule, grid_spacing, dihedrals} keys. + + provenance : dict, optional + The provenance of the fragments creation + + Example + ------- + + data = { + "label1": { + "initial_molecule": ptl.data.get_molecule("butane.json"), + "grid_spacing": [60], + "dihedrals": [[0, 2, 3, 1]], + }, + ... + } + wf.add_fragment("CCCC", data=) + """ if fragment_id not in self.data["fragments"]: self.data["fragments"][fragment_id] = {} @@ -78,6 +121,7 @@ def add_fragment(self, fragment_id, data, provenance={}): print("Already found label {} for fragment_ID {}, skipping.".format(name, fragment_id)) continue + # Build out a new service torsion_meta = copy.deepcopy( {k: self.data[k] for k in ("torsiondrive_meta", "optimization_meta", "qc_meta")}) @@ -85,6 +129,7 @@ def add_fragment(self, fragment_id, data, provenance={}): for k in ["grid_spacing", "dihedrals"]: torsion_meta["torsiondrive_meta"][k] = packet[k] + # Get hash of torsion ret = self.client.add_service("torsiondrive", [packet["initial_molecule"]], torsion_meta) hash_lists = [] @@ -93,30 +138,34 @@ def add_fragment(self, fragment_id, data, provenance={}): if len(hash_lists) != 1: raise KeyError("Something went very wrong.") - hash_index = hash_lists[0] - frag_data[name] = hash_index + # add back to fragment data + packet["hash_index"] = hash_list[0] + frag_data[name] = packet - self.data["fragments"][fragment_id] = frag_data + def get_fragment_data(self, fragments=None, refresh_cache=False): + """Obtains fragment torsiondrives from server to local data. - def get_data(self): + Parameters + ---------- + fragments : None, optional + A list of fragment ID's to query upon + refresh_cache : bool, optional + If True requery everything, otherwise use the cache to prevent extra lookups. + """ + # If no fragments explicitly shown, grab all + if fragments is None: + fragments = self.data["fragments"].keys() + # Figure out the lookup lookup = [] - for k, v in self.data["fragments"].items(): - lookup.extend(list(v.values())) - - data = self.client.get_procedures({"hash_index": lookup}) - data = {x._hash_index: x for x in data} - - ret = {} - for frag, reqs in self.data["fragments"].items(): - ret[frag] = {} - for label, hash_index in reqs.items(): - try: - ret[frag][label] = {json.dumps(k):v for k, v in data[hash_index].final_energies().items()} - except KeyError: - ret[frag][label] = None + for frag in fragments: + lookup.extend([v["hash_index"] for v in self.data["fragments"][frag]]) + if refresh_cache is False: + lookup = list(set(lookup) - self._fragment_cache) - return ret + # Grab the data and update cache + data = self.client.get_procedures({"hash_index": lookup}) + self._fragment_cache.update({x._hash_index: x for x in data}) collection_utils.register_collection(OpenFFWorkflow) \ No newline at end of file From 2dfb94d820facf488ec95cda6a4636b90f0277ba Mon Sep 17 00:00:00 2001 From: Daniel Smith Date: Mon, 1 Oct 2018 14:34:09 -0400 Subject: [PATCH 2/8] OpenFF: reworks get operations to use a cache --- .../interface/collections/openffworkflow.py | 70 +++++++++++++++++-- 1 file changed, 66 insertions(+), 4 deletions(-) diff --git a/qcfractal/interface/collections/openffworkflow.py b/qcfractal/interface/collections/openffworkflow.py index 18ae6a634..24c530bec 100644 --- a/qcfractal/interface/collections/openffworkflow.py +++ b/qcfractal/interface/collections/openffworkflow.py @@ -37,10 +37,21 @@ def __init__(self, name, options=None, client=None, **kwargs): A Portal client to connect to a server """ + + if client is None: + raise KeyError("OpenFFWorkflow must have a client.") + super().__init__(name, client=client, options=options, **kwargs) self._fragment_cache = {} + # First workflow is saved + if "id" not in self.data: + ret = self.save() + if len(ret) == 0: + raise ValueError("Attempted to insert duplicate Workflow with name '{}'".format(name)) + self.data["id"] = ret[0][1] + def _init_collection_data(self, additional_args): options = additional_args.get("options", None) if options is None: @@ -139,9 +150,12 @@ def add_fragment(self, fragment_id, data, provenance={}): raise KeyError("Something went very wrong.") # add back to fragment data - packet["hash_index"] = hash_list[0] + packet["hash_index"] = hash_lists[0] frag_data[name] = packet + # Push collection data back to server + self.save(overwrite=True) + def get_fragment_data(self, fragments=None, refresh_cache=False): """Obtains fragment torsiondrives from server to local data. @@ -159,13 +173,61 @@ def get_fragment_data(self, fragments=None, refresh_cache=False): # Figure out the lookup lookup = [] for frag in fragments: - lookup.extend([v["hash_index"] for v in self.data["fragments"][frag]]) + lookup.extend([v["hash_index"] for v in self.data["fragments"][frag].values()]) if refresh_cache is False: - lookup = list(set(lookup) - self._fragment_cache) + lookup = list(set(lookup) - self._fragment_cache.keys()) # Grab the data and update cache data = self.client.get_procedures({"hash_index": lookup}) self._fragment_cache.update({x._hash_index: x for x in data}) -collection_utils.register_collection(OpenFFWorkflow) \ No newline at end of file + + def list_final_energies(self, fragments=None, refresh_cache=False): + + # If no fragments explicitly shown, grab all + if fragments is None: + fragments = self.data["fragments"].keys() + + # Get the data if available + self.get_fragment_data(fragments=fragments, refresh_cache=refresh_cache) + + ret = {} + for frag in fragments: + tmp = {} + for k, v in self.data["fragments"][frag].items(): + if v["hash_index"] in self._fragment_cache: + tmp[k] = self._fragment_cache[v["hash_index"]].final_energies() + else: + tmp[k] = None + + ret[frag] = tmp + + return ret + + + def list_final_molecules(self, fragments=None, refresh_cache=False): + + # If no fragments explicitly shown, grab all + if fragments is None: + fragments = self.data["fragments"].keys() + + # Get the data if available + self.get_fragment_data(fragments=fragments, refresh_cache=refresh_cache) + + ret = {} + for frag in fragments: + tmp = {} + for k, v in self.data["fragments"][frag].items(): + if v["hash_index"] in self._fragment_cache: + tmp[frag][k] = self._fragment_cache[v["hash_index"]].final_molecule() + else: + tmp[k] = None + + ret[frag] = tmp + + return ret + + +collection_utils.register_collection(OpenFFWorkflow) + From 059fed09bb40e9a91a09e68bcba4df401285adee Mon Sep 17 00:00:00 2001 From: Daniel Smith Date: Mon, 1 Oct 2018 14:45:18 -0400 Subject: [PATCH 3/8] Procedures: ORMs now own client objects --- qcfractal/interface/client.py | 2 +- qcfractal/interface/orm/build_orm.py | 8 ++-- qcfractal/interface/orm/optimization_orm.py | 10 +++-- qcfractal/interface/orm/torsiondrive_orm.py | 42 +++++++++++++++++++-- qcfractal/tests/test_procedures.py | 2 +- 5 files changed, 53 insertions(+), 11 deletions(-) diff --git a/qcfractal/interface/client.py b/qcfractal/interface/client.py index b33892d57..f428c5ad0 100644 --- a/qcfractal/interface/client.py +++ b/qcfractal/interface/client.py @@ -322,7 +322,7 @@ def get_procedures(self, procedure_id, return_objects=True): if return_objects: ret = [] for packet in r.json()["data"]: - tmp = orm.build_orm(packet) + tmp = orm.build_orm(packet, client=self) ret.append(tmp) return ret else: diff --git a/qcfractal/interface/orm/build_orm.py b/qcfractal/interface/orm/build_orm.py index 80917ba75..51e42b158 100644 --- a/qcfractal/interface/orm/build_orm.py +++ b/qcfractal/interface/orm/build_orm.py @@ -6,7 +6,7 @@ from .optimization_orm import OptimizationORM -def build_orm(data, procedure=None): +def build_orm(data, procedure=None, client=None): """ Constructs a Service ORM from incoming JSON data. @@ -16,6 +16,8 @@ def build_orm(data, procedure=None): A JSON representation of the procedure. procedure : None, optional The name of the procedure. If blank the procedure name is pulled from the `data["procedure"]` field. + client : FractalClient, optional + A activate server connection. Returns ------- @@ -44,8 +46,8 @@ def build_orm(data, procedure=None): # import json # print(json.dumps(data, indent=2)) if data["procedure"].lower() == "torsiondrive": - return TorsionDriveORM.from_json(data) + return TorsionDriveORM.from_json(data, client=client) elif data["procedure"].lower() == "optimization": - return OptimizationORM.from_json(data) + return OptimizationORM.from_json(data, client=client) else: raise KeyError("Service names {} not recognized.".format(data["procedure"])) diff --git a/qcfractal/interface/orm/optimization_orm.py b/qcfractal/interface/orm/optimization_orm.py index 399717f08..32b040fac 100644 --- a/qcfractal/interface/orm/optimization_orm.py +++ b/qcfractal/interface/orm/optimization_orm.py @@ -43,13 +43,14 @@ def __init__(self, initial_molecule, **kwargs): """ self._initial_molecule = initial_molecule + self._client = kwargs.pop("client", None) # Set kwargs for k in self.__json_mapper.keys(): setattr(self, k, kwargs.get(k[1:], None)) @classmethod - def from_json(cls, data): + def from_json(cls, data, client=None): """ Creates a OptimizationORM object from FractalServer data. @@ -65,6 +66,8 @@ def from_json(cls, data): - "final_molecule_id": The id of the optimizated molecule. - "trajectory": QC results for each step in the geometry optimization. - "energies": The final energies for each step in the geometry optimization. + client : FractalClient, optional + A activate server connection. Returns ------- @@ -80,6 +83,7 @@ def from_json(cls, data): if ("final_energies" in kwargs) and (kwargs["final_energies"] is not None): kwargs["final_energies"] = {tuple(json.loads(k)): v for k, v in kwargs["final_energies"].items()} + kwargs["client"] = client return cls(None, **kwargs) def __str__(self): @@ -131,7 +135,7 @@ def final_energy(self): """ return self._energies[-1] - def get_trajectory(self, client, projection=None): + def get_trajectory(self, projection=None): """Returns the raw documents for each gradient evaluation in the trajectory. Parameters @@ -148,6 +152,6 @@ def get_trajectory(self, client, projection=None): """ payload = copy.deepcopy(self._trajectory) payload["projection"] = projection - return client.locator(payload) + return self._client.locator(payload) diff --git a/qcfractal/interface/orm/torsiondrive_orm.py b/qcfractal/interface/orm/torsiondrive_orm.py index e9f719372..804f7bfba 100644 --- a/qcfractal/interface/orm/torsiondrive_orm.py +++ b/qcfractal/interface/orm/torsiondrive_orm.py @@ -2,8 +2,10 @@ A ORM for TorsionDrive """ +import copy import json +__all__ = ["TorsionDriveORM"] class TorsionDriveORM: """ @@ -19,6 +21,7 @@ class TorsionDriveORM: # Options "_optimization_history": "optimization_history", "_initial_molecule_id": "initial_molecule", + "_final_molecule_id": "final_molecule", "_torsiondrive_options": "torsiondrive_meta", "_geometric_options": "geometric_meta", "_qc_options": "qc_meta", @@ -43,13 +46,16 @@ def __init__(self, initial_molecule, **kwargs): """ self._initial_molecule = initial_molecule + self._client = kwargs.pop("client", None) # Set kwargs for k in self.__json_mapper.keys(): setattr(self, k, kwargs.get(k[1:], None)) + self._cache = {} + @classmethod - def from_json(cls, data): + def from_json(cls, data, client=None): """ Creates a TorsionDriveORM object from FractalServer data. @@ -64,6 +70,8 @@ def from_json(cls, data): - "geometric_meta": The options submitted to the Geometric method called by TorsionDrive - "qc_meta": The program, options, method, and basis to be run by Geometric. - "final_energies": A dictionary of final energies if the TorsionDrive service is finished + client : FractalClient + A server connection to Returns ------- @@ -75,12 +83,24 @@ def from_json(cls, data): for k, v in TorsionDriveORM.__json_mapper.items(): if v in data: kwargs[k[1:]] = data[v] + else: + kwargs[k[1:]] = None if ("final_energies" in kwargs) and (kwargs["final_energies"] is not None): kwargs["final_energies"] = {tuple(json.loads(k)): v for k, v in kwargs["final_energies"].items()} + self._client = client + return cls(None, **kwargs) + def _check_success(self): + if not self._success: + raise KeyError("{} has not completed or failed. Unable to process request.".format(self)) + + def _check_client(self): + if self._client is None: + raise KeyError("{} requires a FractalClient to aquire the requested information.".format(self)) + def __str__(self): """ Simplified torsiondrive string representation. @@ -133,8 +153,7 @@ def final_energies(self, key=None): {(-90,): -148.7641654446243, (180,): -148.76501336993732, (0,): -148.75056290106735, (90,): -148.7641654446148} """ - if not self._success: - raise KeyError("{} has not completed or failed. Unable to show final energies.".format(self)) + self._check_success() if key is None: return self._final_energies.copy() @@ -143,3 +162,20 @@ def final_energies(self, key=None): key = (int(key), ) return self._final_energies[key] + + def final_molecule(self): + """Returns the optimized molecule + + Returns + ------- + Molecule + The optimized molecule + """ + self._check_success() + self._check_client() + + if "final_molecule" not in self._cache: + self._cache["final_molecule"] = self._client.get_molecules({"mol": self._final_molecule_id}, index="id")["mol"] + + return copy.deepcopy(self._cache["final_molecule"]) + diff --git a/qcfractal/tests/test_procedures.py b/qcfractal/tests/test_procedures.py index a8a2b86ba..3f57d3e30 100644 --- a/qcfractal/tests/test_procedures.py +++ b/qcfractal/tests/test_procedures.py @@ -118,7 +118,7 @@ def test_procedure_optimization(fractal_compute_server): assert pytest.approx(-1.117530188962681, 1e-5) == results[0].final_energy() # Check pulls - traj = results[0].get_trajectory(client, projection={"properties": True}) + traj = results[0].get_trajectory(projection={"properties": True}) energies = results[0].energies() assert len(traj) == len(energies) From 0eb63b6f1ad87c8934da12cc9d4c6f4074878c14 Mon Sep 17 00:00:00 2001 From: Daniel Smith Date: Mon, 1 Oct 2018 15:44:08 -0400 Subject: [PATCH 4/8] Procedures: Reworks procedures for additional returns --- qcfractal/interface/orm/optimization_orm.py | 12 ++- qcfractal/interface/orm/torsiondrive_orm.py | 104 ++++++++++++++------ qcfractal/services/torsiondrive_service.py | 10 ++ qcfractal/tests/test_authentication.py | 1 + qcfractal/tests/test_procedures.py | 1 + qcfractal/tests/test_services.py | 12 ++- 6 files changed, 105 insertions(+), 35 deletions(-) diff --git a/qcfractal/interface/orm/optimization_orm.py b/qcfractal/interface/orm/optimization_orm.py index 32b040fac..175f38c2e 100644 --- a/qcfractal/interface/orm/optimization_orm.py +++ b/qcfractal/interface/orm/optimization_orm.py @@ -20,7 +20,6 @@ class OptimizationORM: # Options "_program": "program", "_qc_options": "qc_meta", - "_initial_molecule_id": "initial_molecule", "_final_molecule_id": "final_molecule", "_trajectory": "trajectory", @@ -150,8 +149,19 @@ def get_trajectory(self, projection=None): list of dict A list of results documents """ + payload = copy.deepcopy(self._trajectory) payload["projection"] = projection return self._client.locator(payload) + def final_molecule(self): + """Returns the optimized molecule + + Returns + ------- + Molecule + The optimized molecule + """ + ret = self._client.get_molecules([self._final_molecule_id], index="id") + return ret[0] diff --git a/qcfractal/interface/orm/torsiondrive_orm.py b/qcfractal/interface/orm/torsiondrive_orm.py index 804f7bfba..4fdc14ea1 100644 --- a/qcfractal/interface/orm/torsiondrive_orm.py +++ b/qcfractal/interface/orm/torsiondrive_orm.py @@ -7,6 +7,7 @@ __all__ = ["TorsionDriveORM"] + class TorsionDriveORM: """ A interface to the raw JSON data of a TorsionDrive torsion scan run. @@ -28,6 +29,7 @@ class TorsionDriveORM: # Energies "_final_energies": "final_energies", + "_minimum_positions": "minimum_positions", } def __init__(self, initial_molecule, **kwargs): @@ -86,21 +88,9 @@ def from_json(cls, data, client=None): else: kwargs[k[1:]] = None - if ("final_energies" in kwargs) and (kwargs["final_energies"] is not None): - kwargs["final_energies"] = {tuple(json.loads(k)): v for k, v in kwargs["final_energies"].items()} - - self._client = client - + kwargs["client"] = client return cls(None, **kwargs) - def _check_success(self): - if not self._success: - raise KeyError("{} has not completed or failed. Unable to process request.".format(self)) - - def _check_client(self): - if self._client is None: - raise KeyError("{} requires a FractalClient to aquire the requested information.".format(self)) - def __str__(self): """ Simplified torsiondrive string representation. @@ -130,6 +120,43 @@ def __str__(self): return ret + def _serialize_key(self, key): + if isinstance(key, (int, float)): + key = (int(key), ) + + return json.dumps(key) + + def _unserialize_key(self, key): + return tuple(json.loads(key)) + + def get_history(self): + """Pulls all optimization trajectories to local data. + + Returns + ------- + dict + The optimization history + """ + + if "history" not in self._cache: + + # Grab procedures + needed_hashes = [x for v in self._optimization_history.values() for x in v] + objects = self._client.get_procedures({"hash_index": needed_hashes}) + procedures = {v._hash_index: v for v in objects} + + # Move procedures into the correct order + ret = {} + for key, hashes in self._optimization_history.items(): + tmp = [] + for h in hashes: + tmp.append(procedures[h]) + ret[key] = tmp + + self._cache["history"] = ret + + return self._cache["history"] + def final_energies(self, key=None): """ Provides the final optimized energies at each grid point. @@ -137,7 +164,7 @@ def final_energies(self, key=None): Parameters ---------- key : None, optional - Returns the final energy at a single grid point. + Specifies a single entry to pull from. Returns @@ -153,29 +180,48 @@ def final_energies(self, key=None): {(-90,): -148.7641654446243, (180,): -148.76501336993732, (0,): -148.75056290106735, (90,): -148.7641654446148} """ - self._check_success() - if key is None: - return self._final_energies.copy() + return {self._unserialize_key(k): v for k, v in self._final_energies} else: - if isinstance(key, (int, float)): - key = (int(key), ) - return self._final_energies[key] + return self._final_energies[self._serialize_key(key)] + + def final_molecules(self, key=None): + """Returns the optimized molecules at each grid point + + Parameters + ---------- + key : None, optional + Specifies a single entry to pull from. - def final_molecule(self): - """Returns the optimized molecule Returns ------- - Molecule - The optimized molecule + energy : dict + Returns molecule at each grid point in a dictionary or at a + single point if a key is specified. + + Examples + -------- + + >>> torsiondrive_obj.final_energies() + {(-90,):{'symbols': ['H', 'O', 'O', 'H'], 'geometry': [1.72669422, 1.28135788, ... } """ - self._check_success() - self._check_client() - if "final_molecule" not in self._cache: - self._cache["final_molecule"] = self._client.get_molecules({"mol": self._final_molecule_id}, index="id")["mol"] + if "final_molecules" not in self._cache: + + ret = {} + for k, tasks in self.get_history().items(): + minpos = self._minimum_positions[k] + + ret[k] = tasks[minpos].final_molecule() - return copy.deepcopy(self._cache["final_molecule"]) + self._cache["final_molecules"] = ret + + data = self._cache["final_molecules"] + + if key is None: + return {self._unserialize_key(k): v for k, v in data.items()} + else: + return data[self._serialize_key(key)] diff --git a/qcfractal/services/torsiondrive_service.py b/qcfractal/services/torsiondrive_service.py index 824ef1db7..3c75f3f84 100644 --- a/qcfractal/services/torsiondrive_service.py +++ b/qcfractal/services/torsiondrive_service.py @@ -267,6 +267,15 @@ def finalize(self): self.data["minimum_positions"][key] = min_pos self.data["final_energies"][key] = v[min_pos][2] + self.data["optimization_history"] = { + json.dumps(td_api.grid_id_from_string(k)): v + for k, v in self.data["optimization_history"].items() + } + + # print(self.data["optimization_history"]) + # print(self.data["minimum_positions"]) + # print(self.data["final_energies"]) + # Pop temporaries del self.data["job_map"] del self.data["remaining_jobs"] @@ -274,6 +283,7 @@ def finalize(self): del self.data["queue_keys"] del self.data["torsiondrive_state"] del self.data["status"] + del self.data["required_jobs"] return self.data diff --git a/qcfractal/tests/test_authentication.py b/qcfractal/tests/test_authentication.py index 1c95a0147..238197ee2 100644 --- a/qcfractal/tests/test_authentication.py +++ b/qcfractal/tests/test_authentication.py @@ -62,6 +62,7 @@ def sec_server(request): ### Tests the compute queue stack def test_security_auth_decline_none(sec_server): client = portal.FractalClient(sec_server.get_address(), verify=False) + assert "FractalClient" in str(client) with pytest.raises(requests.exceptions.HTTPError): r = client.get_molecules([]) diff --git a/qcfractal/tests/test_procedures.py b/qcfractal/tests/test_procedures.py index 3f57d3e30..8b7592bb3 100644 --- a/qcfractal/tests/test_procedures.py +++ b/qcfractal/tests/test_procedures.py @@ -121,6 +121,7 @@ def test_procedure_optimization(fractal_compute_server): traj = results[0].get_trajectory(projection={"properties": True}) energies = results[0].energies() assert len(traj) == len(energies) + assert results[0].final_molecule()["symbols"] == ["H", "H"] # Check individual elements for ind in range(len(results[0]._trajectory)): diff --git a/qcfractal/tests/test_services.py b/qcfractal/tests/test_services.py index f142cb16f..217c2d3dc 100644 --- a/qcfractal/tests/test_services.py +++ b/qcfractal/tests/test_services.py @@ -61,9 +61,8 @@ def spin_up_test(grid_spacing=default_grid_spacing, **keyword_augments): def test_service_torsiondrive(torsiondrive_fixture): - """"Ensure torsiondrive works as intended gives the correct result""" - # This test does not ensure de-duplication of work - # Test will be skipped if missing RDKit, Geomertic, and TorsionDrive from fixture + """"Tests torsiondrive pathway and checks the result result""" + spin_up_test, client = torsiondrive_fixture ret = spin_up_test() @@ -78,14 +77,17 @@ def test_service_torsiondrive(torsiondrive_fixture): assert pytest.approx(0.000156553761859271, 1e-5) == result.final_energies(-90) assert pytest.approx(0.000753492556057886, 1e-5) == result.final_energies(180) + assert "symbols" in result.final_molecules()[(-90, )] + def test_service_torsiondrive_duplicates(torsiondrive_fixture): """Ensure that duplicates are properly caught and yield the same results without calculation""" - # This test does not ensure accuracy, there is another test for that - # Test will be skipped if missing RDKit, Geomertic, and TorsionDrive from fixture + spin_up_test, client = torsiondrive_fixture + # Run the test without modifications _ = spin_up_test() + # Augment the input for torsion drive to yield a new hash procedure hash, # but not a new task set _ = spin_up_test(torsiondrive_meta={"meaningless_entry_to_change_hash": "Waffles!"}) From 6b9137780d86e8fd9e0ce0dd962f9b084567135b Mon Sep 17 00:00:00 2001 From: Daniel Smith Date: Mon, 1 Oct 2018 15:53:20 -0400 Subject: [PATCH 5/8] OpenFF: Adds a workflow test --- .../interface/collections/openffworkflow.py | 14 ++-- qcfractal/interface/orm/torsiondrive_orm.py | 2 +- qcfractal/tests/test_collections.py | 79 ++++++++++++++++++- 3 files changed, 85 insertions(+), 10 deletions(-) diff --git a/qcfractal/interface/collections/openffworkflow.py b/qcfractal/interface/collections/openffworkflow.py index 24c530bec..cb8521a12 100644 --- a/qcfractal/interface/collections/openffworkflow.py +++ b/qcfractal/interface/collections/openffworkflow.py @@ -43,7 +43,7 @@ def __init__(self, name, options=None, client=None, **kwargs): super().__init__(name, client=client, options=options, **kwargs) - self._fragment_cache = {} + self._torsiondrive_cache = {} # First workflow is saved if "id" not in self.data: @@ -176,11 +176,11 @@ def get_fragment_data(self, fragments=None, refresh_cache=False): lookup.extend([v["hash_index"] for v in self.data["fragments"][frag].values()]) if refresh_cache is False: - lookup = list(set(lookup) - self._fragment_cache.keys()) + lookup = list(set(lookup) - self._torsiondrive_cache.keys()) # Grab the data and update cache data = self.client.get_procedures({"hash_index": lookup}) - self._fragment_cache.update({x._hash_index: x for x in data}) + self._torsiondrive_cache.update({x._hash_index: x for x in data}) def list_final_energies(self, fragments=None, refresh_cache=False): @@ -196,8 +196,8 @@ def list_final_energies(self, fragments=None, refresh_cache=False): for frag in fragments: tmp = {} for k, v in self.data["fragments"][frag].items(): - if v["hash_index"] in self._fragment_cache: - tmp[k] = self._fragment_cache[v["hash_index"]].final_energies() + if v["hash_index"] in self._torsiondrive_cache: + tmp[k] = self._torsiondrive_cache[v["hash_index"]].final_energies() else: tmp[k] = None @@ -219,8 +219,8 @@ def list_final_molecules(self, fragments=None, refresh_cache=False): for frag in fragments: tmp = {} for k, v in self.data["fragments"][frag].items(): - if v["hash_index"] in self._fragment_cache: - tmp[frag][k] = self._fragment_cache[v["hash_index"]].final_molecule() + if v["hash_index"] in self._torsiondrive_cache: + tmp[k] = self._torsiondrive_cache[v["hash_index"]].final_molecules() else: tmp[k] = None diff --git a/qcfractal/interface/orm/torsiondrive_orm.py b/qcfractal/interface/orm/torsiondrive_orm.py index 4fdc14ea1..5d1969b92 100644 --- a/qcfractal/interface/orm/torsiondrive_orm.py +++ b/qcfractal/interface/orm/torsiondrive_orm.py @@ -181,7 +181,7 @@ def final_energies(self, key=None): """ if key is None: - return {self._unserialize_key(k): v for k, v in self._final_energies} + return {self._unserialize_key(k): v for k, v in self._final_energies.items()} else: return self._final_energies[self._serialize_key(key)] diff --git a/qcfractal/tests/test_collections.py b/qcfractal/tests/test_collections.py index 562d41528..457357666 100644 --- a/qcfractal/tests/test_collections.py +++ b/qcfractal/tests/test_collections.py @@ -4,9 +4,11 @@ import qcfractal.interface as portal from qcfractal import testing -from qcfractal.testing import fractal_compute_server import pytest +# Only use dask +from qcfractal.testing import dask_server_fixture as fractal_compute_server + ### Tests an entire server and interaction energy database run @testing.using_psi4 @@ -60,7 +62,7 @@ def test_compute_database(fractal_compute_server): assert pytest.approx(0.00024477933196125805, 1.e-5) == ds.statistics("MUE", "SCF/STO-3G") -### Tests an entire server and interaction energy database run +### Tests the biofragment collection @testing.using_torsiondrive @testing.using_geometric @testing.using_rdkit @@ -110,3 +112,76 @@ def test_compute_biofragment(fractal_compute_server): # nanny.await_services(max_iter=5) # assert len(nanny.list_current_tasks()) == 0 +### Tests the openffworkflow collection +@testing.using_torsiondrive +@testing.using_geometric +@testing.using_rdkit +def test_compute_openffworkflow(fractal_compute_server): + + # Obtain a client and build a BioFragment + client = portal.FractalClient(fractal_compute_server.get_address("")) + nanny = fractal_compute_server.objects["queue_nanny"] + + openff_workflow_options = { + # Blank Fragmenter options + "enumerate_states": {}, + "enumerate_fragments": {}, + "torsiondrive_input": {}, + + # TorsionDrive, Geometric, and QC options + "torsiondrive_meta": {}, + "optimization_meta": { + "program": "geometric", + "coordsys": "tric", + }, + "qc_meta": { + "driver": "gradient", + "method": "UFF", + "basis": "", + "options": "none", + "program": "rdkit", + } + } + wf = portal.collections.OpenFFWorkflow("Workflow1", client=client, options=openff_workflow_options) + + # Add a fragment and wait for the compute + hooh = portal.data.get_molecule("hooh.json") + fragment_input = { + "label1": { + "initial_molecule": hooh.to_json(), + "grid_spacing": [120], + "dihedrals": [[0, 1, 2, 3]], + }, + } + wf.add_fragment("HOOH", fragment_input, provenance={}) + assert set(wf.list_fragments()) == {"HOOH"} + nanny.await_services(max_iter=5) + + final_energies = wf.list_final_energies() + assert final_energies.keys() == {"HOOH"} + assert final_energies["HOOH"].keys() == {"label1"} + + final_molecules = wf.list_final_molecules() + assert final_molecules.keys() == {"HOOH"} + assert final_molecules["HOOH"].keys() == {"label1"} + + # Add a second fragment + butane = portal.data.get_molecule("butane.json") + butane_id = butane.identifiers["canonical_isomeric_explicit_hydrogen_mapped_smiles"] + + fragment_input = { + "label1": { + "initial_molecule": butane.to_json(), + "grid_spacing": [90], + "dihedrals": [[0, 2, 3, 1]], + }, + } + wf.add_fragment(butane_id, fragment_input, provenance={}) + assert set(wf.list_fragments()) == {butane_id, "HOOH"} + + final_energies = wf.list_final_energies() + assert final_energies.keys() == {butane_id, "HOOH"} + assert final_energies[butane_id].keys() == {"label1"} + assert final_energies[butane_id]["label1"] is None + + From 9f51687abff6dc9062a2661a6d3587eea523e839 Mon Sep 17 00:00:00 2001 From: Daniel Smith Date: Mon, 1 Oct 2018 16:16:22 -0400 Subject: [PATCH 6/8] Test: Testing cleanup pass --- .../interface/collections/openffworkflow.py | 31 +++++++++++++++++++ qcfractal/interface/orm/torsiondrive_orm.py | 8 ++--- qcfractal/services/torsiondrive_service.py | 2 +- qcfractal/tests/test_collections.py | 6 ++-- qcfractal/tests/test_services.py | 2 +- 5 files changed, 40 insertions(+), 9 deletions(-) diff --git a/qcfractal/interface/collections/openffworkflow.py b/qcfractal/interface/collections/openffworkflow.py index cb8521a12..0426f42c1 100644 --- a/qcfractal/interface/collections/openffworkflow.py +++ b/qcfractal/interface/collections/openffworkflow.py @@ -166,6 +166,7 @@ def get_fragment_data(self, fragments=None, refresh_cache=False): refresh_cache : bool, optional If True requery everything, otherwise use the cache to prevent extra lookups. """ + # If no fragments explicitly shown, grab all if fragments is None: fragments = self.data["fragments"].keys() @@ -184,6 +185,21 @@ def get_fragment_data(self, fragments=None, refresh_cache=False): def list_final_energies(self, fragments=None, refresh_cache=False): + """ + Returns the final energies for the requested fragments. + + Parameters + ---------- + fragments : None, optional + A list of fragment ID's to query upon + refresh_cache : bool, optional + If True requery everything, otherwise use the cache to prevent extra lookups. + + Returns + ------- + dict + A dictionary structure with fragment and label fields available for access. + """ # If no fragments explicitly shown, grab all if fragments is None: @@ -207,6 +223,21 @@ def list_final_energies(self, fragments=None, refresh_cache=False): def list_final_molecules(self, fragments=None, refresh_cache=False): + """ + Returns the final molecules for the requested fragments. + + Parameters + ---------- + fragments : None, optional + A list of fragment ID's to query upon + refresh_cache : bool, optional + If True requery everything, otherwise use the cache to prevent extra lookups. + + Returns + ------- + dict + A dictionary structure with fragment and label fields available for access. + """ # If no fragments explicitly shown, grab all if fragments is None: diff --git a/qcfractal/interface/orm/torsiondrive_orm.py b/qcfractal/interface/orm/torsiondrive_orm.py index 5d1969b92..55ea67b09 100644 --- a/qcfractal/interface/orm/torsiondrive_orm.py +++ b/qcfractal/interface/orm/torsiondrive_orm.py @@ -141,9 +141,9 @@ def get_history(self): if "history" not in self._cache: # Grab procedures - needed_hashes = [x for v in self._optimization_history.values() for x in v] - objects = self._client.get_procedures({"hash_index": needed_hashes}) - procedures = {v._hash_index: v for v in objects} + needed_ids = [x for v in self._optimization_history.values() for x in v] + objects = self._client.get_procedures({"id": needed_ids}) + procedures = {v._id: v for v in objects} # Move procedures into the correct order ret = {} @@ -221,7 +221,7 @@ def final_molecules(self, key=None): data = self._cache["final_molecules"] if key is None: - return {self._unserialize_key(k): v for k, v in data.items()} + return {self._unserialize_key(k): copy.deepcopy(v) for k, v in data.items()} else: return data[self._serialize_key(key)] diff --git a/qcfractal/services/torsiondrive_service.py b/qcfractal/services/torsiondrive_service.py index 3c75f3f84..d3367fdf4 100644 --- a/qcfractal/services/torsiondrive_service.py +++ b/qcfractal/services/torsiondrive_service.py @@ -142,7 +142,7 @@ def iterate(self): job_results[key].append((mol_keys[0]["geometry"], mol_keys[1]["geometry"], ret["energies"][-1])) # Update history - self.data["optimization_history"][key].append(job_id) + self.data["optimization_history"][key].append(ret["id"]) td_api.update_state(self.data["torsiondrive_state"], job_results) diff --git a/qcfractal/tests/test_collections.py b/qcfractal/tests/test_collections.py index 457357666..635a5cc14 100644 --- a/qcfractal/tests/test_collections.py +++ b/qcfractal/tests/test_collections.py @@ -10,9 +10,9 @@ from qcfractal.testing import dask_server_fixture as fractal_compute_server -### Tests an entire server and interaction energy database run +### Tests an entire server and interaction energy dataset run @testing.using_psi4 -def test_compute_database(fractal_compute_server): +def test_compute_dataset(fractal_compute_server): client = portal.FractalClient(fractal_compute_server.get_address("")) ds_name = "He_PES" @@ -105,7 +105,7 @@ def test_compute_biofragment(fractal_compute_server): ] } # yapf: disable - frag.submit_torsion_drives("v1", needed_torsions) + # frag.submit_torsion_drives("v1", needed_torsions) # Compute! # nanny = fractal_compute_server.objects["queue_nanny"] diff --git a/qcfractal/tests/test_services.py b/qcfractal/tests/test_services.py index 217c2d3dc..aa2c015a9 100644 --- a/qcfractal/tests/test_services.py +++ b/qcfractal/tests/test_services.py @@ -60,7 +60,7 @@ def spin_up_test(grid_spacing=default_grid_spacing, **keyword_augments): yield spin_up_test, client -def test_service_torsiondrive(torsiondrive_fixture): +def test_service_torsiondrive_single(torsiondrive_fixture): """"Tests torsiondrive pathway and checks the result result""" spin_up_test, client = torsiondrive_fixture From f78eb21c6475d13fbf86d3fc57264fc1e801ccc7 Mon Sep 17 00:00:00 2001 From: Daniel Smith Date: Mon, 1 Oct 2018 18:25:11 -0400 Subject: [PATCH 7/8] Server: Adds tests for start/stop functions --- .../interface/collections/openffworkflow.py | 1 - qcfractal/server.py | 6 +-- qcfractal/tests/test_server.py | 39 +++++++++++++++---- 3 files changed, 34 insertions(+), 12 deletions(-) diff --git a/qcfractal/interface/collections/openffworkflow.py b/qcfractal/interface/collections/openffworkflow.py index 0426f42c1..ab5706ca3 100644 --- a/qcfractal/interface/collections/openffworkflow.py +++ b/qcfractal/interface/collections/openffworkflow.py @@ -1,7 +1,6 @@ """Mongo QCDB Fragment object and helpers """ -import json import copy from .collection import Collection diff --git a/qcfractal/server.py b/qcfractal/server.py index d97d3789a..47e0a6ff7 100644 --- a/qcfractal/server.py +++ b/qcfractal/server.py @@ -223,7 +223,7 @@ def __init__( # Add in periodic callbacks - self.logger.info("DQM Server successfully initialized at {}\n".format(self._address)) + self.logger.info("FractalServer successfully initialized at {}\n".format(self._address)) self.periodic = {} @@ -232,7 +232,7 @@ def start(self): Starts up all IOLoops and processes """ - self.logger.info("DQM Server successfully started. Starting IOLoop.\n") + self.logger.info("FractalServer successfully started. Starting IOLoop.\n") # If we have a queue socket start up the nanny if "queue_socket" in self.objects: @@ -260,7 +260,7 @@ def stop(self): for cb in self.periodic.values(): cb.stop() - self.logger.info("DQM Server stopping gracefully. Stopped IOLoop.\n") + self.logger.info("FractalServer stopping gracefully. Stopped IOLoop.\n") def get_address(self, function=""): return self._address + function diff --git a/qcfractal/tests/test_server.py b/qcfractal/tests/test_server.py index c95f6fdf9..3876ef24d 100644 --- a/qcfractal/tests/test_server.py +++ b/qcfractal/tests/test_server.py @@ -4,12 +4,37 @@ import qcfractal.interface as portal # Pytest Fixture -from qcfractal.testing import test_server +from qcfractal import FractalServer +from qcfractal.testing import test_server, pristine_loop, find_open_port import requests +import threading meta_set = {'errors', 'n_inserted', 'success', 'duplicates', 'error_description', 'validation_errors'} +def test_start_stop(): + + with pristine_loop() as loop: + + # Build server, manually handle IOLoop (no start/stop needed) + server = FractalServer( + port=find_open_port(), storage_project_name="something", io_loop=loop, ssl_options=False) + + thread = threading.Thread(target=server.start, name="test IOLoop") + thread.daemon = True + thread.start() + + loop_started = threading.Event() + loop.add_callback(loop_started.set) + loop_started.wait() + + try: + loop.add_callback(server.stop) + thread.join(timeout=5) + except: + pass + + def test_molecule_socket(test_server): mol_api_addr = test_server.get_address("molecule") @@ -86,13 +111,11 @@ def test_result_socket(test_server): water2 = portal.data.get_molecule("water_dimer_stretch.psimol") r = requests.post( test_server.get_address("molecule"), - json={ - "meta": {}, - "data": { - "water1": water.to_json(), - "water2": water2.to_json() - } - }) + json={"meta": {}, + "data": { + "water1": water.to_json(), + "water2": water2.to_json() + }}) assert r.status_code == 200 mol_insert = r.json() From 8822b34d74b1803074fd77a0de2cb7a494395519 Mon Sep 17 00:00:00 2001 From: Daniel Smith Date: Tue, 2 Oct 2018 11:53:56 -0400 Subject: [PATCH 8/8] Client: Additional collections docs --- qcfractal/interface/client.py | 32 +++++++++++++++++-- .../interface/collections/biofragment.py | 3 +- qcfractal/interface/collections/dataset.py | 3 +- 3 files changed, 34 insertions(+), 4 deletions(-) diff --git a/qcfractal/interface/client.py b/qcfractal/interface/client.py index f428c5ad0..434e595dd 100644 --- a/qcfractal/interface/client.py +++ b/qcfractal/interface/client.py @@ -246,12 +246,25 @@ def add_options(self, opt_list, full_return=False): ### Collections section def list_collections(self, collection_type=None): + """Lists the available collections currently on the server. + + Parameters + ---------- + collection_type : None, optional + If `None` all collection types will be returned, otherwise only the + specified collection type will be returned + + Returns + ------- + dict + A dictionary containing the available collection types. + """ query = {} if collection_type is not None: query = {"collection": collection_type.lower()} - payload = {"meta": {"projection": {"name": True, "collection":True}}, "data": query} + payload = {"meta": {"projection": {"name": True, "collection": True}}, "data": query} r = self._request("get", "collection", payload) if collection_type is None: @@ -262,8 +275,23 @@ def list_collections(self, collection_type=None): else: return [x["name"] for x in r.json()["data"]] - def get_collection(self, collection_type, collection_name, full_return=False): + """Aquires a given collection from the server + + Parameters + ---------- + collection_type : str + The collection type to be accessed + collection_name : str + The name of the collection to be accssed + full_return : bool, optional + If False, returns a Collection object otherwise returns raw JSON + + Returns + ------- + Collection + A Collection object if the given collection was found otherwise returns `None`. + """ payload = {"meta": {}, "data": [(collection_type.lower(), collection_name)]} r = self._request("get", "collection", payload) diff --git a/qcfractal/interface/collections/biofragment.py b/qcfractal/interface/collections/biofragment.py index 3b0932ec0..7d11968e8 100644 --- a/qcfractal/interface/collections/biofragment.py +++ b/qcfractal/interface/collections/biofragment.py @@ -115,4 +115,5 @@ def submit_torsion_drives(self, options_set, torsions): submissions.append(ret) return submissions -collection_utils.register_collection(BioFragment) \ No newline at end of file +collection_utils.register_collection(BioFragment) + diff --git a/qcfractal/interface/collections/dataset.py b/qcfractal/interface/collections/dataset.py index 503dba568..b3fa03ef2 100644 --- a/qcfractal/interface/collections/dataset.py +++ b/qcfractal/interface/collections/dataset.py @@ -703,4 +703,5 @@ def __getitem__(self, args): return self.df[args] -register_collection(Dataset) \ No newline at end of file +register_collection(Dataset) +