Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Implement New Weaviate Repository #61

Merged
merged 6 commits into from
Aug 28, 2024
Merged

Implement New Weaviate Repository #61

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
17ec7f7
update: datastew version
mehmetcanay Aug 22, 2024
9787834
refactor: implement new weaviate repository
mehmetcanay Aug 22, 2024
333b296
refactor: fix endpoint ids
mehmetcanay Aug 22, 2024
3ffff7c
Merge branch 'main' of https://github.com/SCAI-BIO/index into refacto…
mehmetcanay Aug 22, 2024
4731bc9
refactor: update datastew version
mehmetcanay Aug 26, 2024
a84e345
Fix port
tiadams Aug 28, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion api/requirements.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ httpx~=0.27.0
uvicorn~=0.30.1
fastapi~=0.111.0
starlette~=0.37.2
datastew~=0.3.3
datastew~=0.3.4
numpy~=1.25.2
pandas~=2.1.0
requests~=2.31.0
Expand Down
90 changes: 65 additions & 25 deletions api/routes.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -107,13 +107,19 @@ async def get_all_terminologies():


@app.put("/terminologies/{id}", tags=["terminologies"])
async def create_or_update_terminology(id: str, name: str):
async def create_terminology(id: str, name: str):
try:
terminology = Terminology(name=name, id=id)
repository.store(terminology)
return {"message": f"Terminology {id} created or updated successfully"}
return {"message": f"Terminology {id} created successfully"}
except Exception as e:
raise HTTPException(status_code=400, detail=f"Failed to create or update terminology: {str(e)}")
raise HTTPException(status_code=400, detail=f"Failed to create terminology: {str(e)}")


@app.get("/models", tags=["models"])
async def get_all_models():
sentence_embedders = repository.get_all_sentence_embedders()
return sentence_embedders


@app.get("/concepts", tags=["concepts"])
Expand All @@ -123,17 +129,26 @@ async def get_all_concepts():


@app.put("/concepts/{id}", tags=["concepts"])
async def create_or_update_concept(id: str, terminology_id: str, name: str):
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Unless the request param name matches the param name, this should not work afaik

Test for e.g./concepts/some_id

Copy link
Member Author

@mehmetcanay mehmetcanay Aug 22, 2024 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

You are right, I missed it. Fixed in the next push.

async def create_concept(concept_id: str, concept_name: str, terminology_name: str):
try:
terminology = repository.session.query(Terminology).filter(Terminology.id == terminology_id).first()
if not terminology:
raise HTTPException(status_code=404, detail=f"Terminology with id {terminology_id} not found")

concept = Concept(terminology=terminology, name=name, id=id)
if not repository._terminology_exists(terminology_name):
raise HTTPException(status_code=404, detail=f"Terminology {terminology_name} not found")
result = repository.client.query.get(
"Terminology",
["name", "_additional { id }"]
).with_where({
"path": "name",
"operator": "Equal",
"valueText": terminology_name
}).do()
terminology_data = result["data"]["Get"]["Terminology"][0]
terminology_id = terminology_data["_additional"]["id"]
terminology = Terminology(name=terminology_name, id=terminology_id)
concept = Concept(terminology=terminology, pref_label=concept_name, concept_identifier=concept_id)
repository.store(concept)
return {"message": f"Concept {id} created or updated successfully"}
return {"message": f"Concept {concept_id} created successfully"}
except Exception as e:
raise HTTPException(status_code=400, detail=f"Failed to create or update concept: {str(e)}")
raise HTTPException(status_code=400, detail=f"Failed to create concept: {str(e)}")


@app.get("/mappings", tags=["mappings"])
Expand All @@ -143,33 +158,58 @@ async def get_all_mappings():


@app.put("/concepts/{id}/mappings", tags=["concepts", "mappings"])
async def create_concept_and_attach_mapping(id: str, terminology_id: str, concept_name: str, text: str):
async def create_concept_and_attach_mapping(concept_id: str, concept_name: str, terminology_name, text: str):
try:
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

same here

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

And fixed here too.

terminology = repository.session.query(Terminology).filter(Terminology.id == terminology_id).first()
if not terminology:
raise HTTPException(status_code=404, detail=f"Terminology with id {terminology_id} not found")
concept = Concept(terminology=terminology, name=concept_name, id=id)
if not repository._terminology_exists(terminology_name):
raise HTTPException(status_code=404, detail=f"Terminology {terminology_name} not found")
result = repository.client.query.get(
"Terminology",
["name", "_additional { id }"]
).with_where({
"path": "name",
"operator": "Equal",
"valueText": terminology_name
}).do()
terminology_data = result["data"]["Get"]["Terminology"][0]
terminology_id = terminology_data["_additional"]["id"]
terminology = Terminology(name=terminology_name, id=terminology_id)
concept = Concept(terminology=terminology, pref_label=concept_name, concept_identifier=concept_id)
repository.store(concept)
embedding = embedding_model.get_embedding(text)
mapping = Mapping(concept=concept, text=text, embedding=embedding)
model_name = embedding_model.get_model_name()
mapping = Mapping(concept=concept, text=text, embedding=embedding, sentence_embedder=model_name)
repository.store(mapping)
return {"message": f"Concept {id} created or updated successfully"}
return {"message": f"Concept {concept_id} created successfully"}
except Exception as e:
raise HTTPException(status_code=400, detail=f"Failed to create or update concept: {str(e)}")
raise HTTPException(status_code=400, detail=f"Failed to create concept: {str(e)}")


@app.put("/mappings/", tags=["mappings"])
async def create_or_update_mapping(concept_id: str, text: str):
async def create_mapping(concept_id: str, text: str):
try:
concept = repository.session.query(Concept).filter(Concept.id == concept_id).first()
if not concept:
if not repository._concept_exists(concept_id=concept_id):
raise HTTPException(status_code=404, detail=f"Concept with id {concept_id} not found")
result = repository.client.query.get(
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

this logic should be part of datastew imo

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

You're right, we can have functions like get_terminology(terminology_name: str) and get_concept(concept_id: str). Should I start implementing it?

"Concept",
["conceptID", "prefLabel", "hasTerminology { ... on Terminology { _additional { id } name } }"]
).with_where({
"path": "conceptID",
"operator": "Equal",
"valueText": concept_id
}).do()
terminology_data = result["data"]["get"]["Concept"][0]["hasTerminology"]
terminology_name = terminology_data["name"]
terminology_id = terminology_data["_additional"]["id"]
terminology = Terminology(terminology_name, terminology_id)
concept_name = result["data"]["Get"]["Concept"][0]["prefLabel"]
concept = Concept(terminology=terminology, pref_label=concept_name, concept_identifier=concept_id)
embedding = embedding_model.get_embedding(text)
mapping = Mapping(concept=concept, text=text, embedding=embedding)
model_name = embedding_model.get_model_name()
mapping = Mapping(concept=concept, text=text, embedding=embedding, sentence_embedder=model_name)
repository.store(mapping)
return {"message": f"Mapping created or updated successfully"}
return {"message": "Mapping created successfully"}
except Exception as e:
raise HTTPException(status_code=400, detail=f"Failed to create or update mapping: {str(e)}")
raise HTTPException(status_code=400, detail=f"Failed to create mapping: {str(e)}")


@app.post("/mappings", tags=["mappings"])
Expand Down
Loading