diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..ea7f2ce --- /dev/null +++ b/.dockerignore @@ -0,0 +1,3 @@ +.git* +**/*.pyc +.venv/ \ No newline at end of file diff --git a/.github/workflows/development_dev-research-index.yml b/.github/workflows/development_dev-research-index.yml new file mode 100644 index 0000000..0bd1b93 --- /dev/null +++ b/.github/workflows/development_dev-research-index.yml @@ -0,0 +1,52 @@ +# Docs for the Azure Web Apps Deploy action: https://github.com/Azure/webapps-deploy +# More GitHub Actions for Azure: https://github.com/Azure/actions + +name: Build and deploy container app to Azure Web App - dev-research-index + +on: + push: + branches: + - development + workflow_dispatch: + +jobs: + build: + runs-on: 'ubuntu-latest' + + steps: + - name: Checkout the development branch + uses: actions/checkout@v2 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + + - name: Log in to registry + uses: docker/login-action@v2 + with: + registry: https://index.docker.io/v1/ + username: ${{ secrets.AzureAppService_ContainerUsername_9310ce663981441ab232b6b875b913e2 }} + password: ${{ secrets.AzureAppService_ContainerPassword_a6a4cadc786e40478e1d15d558c66044 }} + + - name: Build and push container image to registry + uses: docker/build-push-action@v3 + with: + push: true + tags: index.docker.io/${{ secrets.AzureAppService_ContainerUsername_9310ce663981441ab232b6b875b913e2 }}/ccg-research-index:${{ github.sha }} + file: ./Dockerfile + + deploy: + runs-on: ubuntu-latest + needs: build + environment: + name: 'production' + url: ${{ steps.deploy-to-webapp.outputs.webapp-url }} + + steps: + - name: Deploy to Azure Web App + id: deploy-to-webapp + uses: azure/webapps-deploy@v2 + with: + app-name: 'dev-research-index' + slot-name: 'production' + publish-profile: ${{ secrets.AzureAppService_PublishProfile_a74f0a22a9c3465a91fa629137af0f81 }} + images: 'index.docker.io/${{ secrets.AzureAppService_ContainerUsername_9310ce663981441ab232b6b875b913e2 }}/ccg-research-index:${{ github.sha }}' \ No newline at end of file diff --git a/.github/workflows/main_research-index.yml b/.github/workflows/main_research-index.yml index 68c0210..0d37ab4 100644 --- a/.github/workflows/main_research-index.yml +++ b/.github/workflows/main_research-index.yml @@ -14,7 +14,8 @@ jobs: runs-on: 'ubuntu-latest' steps: - - uses: actions/checkout@v2 + - name: Checkout to the branch + uses: actions/checkout@v2 - name: Set up Docker Buildx uses: docker/setup-buildx-action@v2 @@ -31,7 +32,7 @@ jobs: with: push: true tags: index.docker.io/${{ secrets.AzureAppService_ContainerUsername_dce678b65c414723a5c407473fe6c430 }}/ccg-research-index:${{ github.sha }} - file: ./Dockerfile + file: Dockerfile deploy: runs-on: ubuntu-latest diff --git a/Dockerfile b/Dockerfile index a9fbb5c..485bb2a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,21 +1,16 @@ -# Use an official Python runtime as a parent image FROM python:3.11.7-bookworm -# Set the working directory in the container to /app -WORKDIR /app +WORKDIR /research-index -# Add the current directory contents into the container at /app -ADD ./app /app -ADD requirements.txt /app/requirements.txt +ADD requirements.txt requirements.txt -# Install packages for the memgraph client -RUN apt update -y -RUN apt install -y python3-dev cmake make gcc g++ libssl-dev - -# Install any needed packages specified in requirements.txt RUN pip install --no-cache-dir -r requirements.txt -# Make port 80 available to the world outside this container -EXPOSE 80 +ADD . . + +EXPOSE 8000 -CMD ["gunicorn", "--bind", "0.0.0.0:80", "app:app"] +CMD ["fastapi", "run", "app/main.py", \ + "--port", "8000", \ + "--workers", "4", \ + "--proxy-headers"] \ No newline at end of file diff --git a/README.md b/README.md index 8c04d53..46c4ff1 100644 --- a/README.md +++ b/README.md @@ -2,9 +2,16 @@ ## Development +Create a `.env` file in the project root with the following environment variables: +```sh +MG_HOST= # Memgraph host address +MG_PORT= # Default Memgraph port +MG_PORT_ALT= # Alternative port +``` + To enter development mode of the website, with the memgraph database running in the background, run - python app/app.py + fastapi dev app/main.py ## Deployment @@ -17,10 +24,12 @@ Once the VM is up and running, SSH into the VM, download and install memgraph ### 2. Build Docker container - docker build + docker build -t research_index_web_app:development . Run the docker container in development mode to test + docker run -dp 8000:8000 research_index_web_app:development + docker run -dp 5001:80 -w /app -v "$(pwd):/app" research-index-gunicorn-311 sh -c "python app/app.py" ### 3. Deploy front-end to Azure and provision database diff --git a/app/api/__init__.py b/app/api/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/app/api/author.py b/app/api/author.py new file mode 100644 index 0000000..e54bb3e --- /dev/null +++ b/app/api/author.py @@ -0,0 +1,43 @@ +from fastapi import APIRouter, HTTPException, Query, Path +from typing import Annotated +from uuid import UUID + +from app.crud.author import Author +from app.schemas.author import AuthorListModel, AuthorOutputModel +from app.schemas.query import FilterWorkstream, FilterParams + +router = APIRouter(prefix="/api/authors", tags=["authors"]) + + +@router.get("") +def api_author_list(query: Annotated[FilterWorkstream, Query()] + ) -> AuthorListModel: + try: + authors = Author() + if result := authors.get_authors(skip=query.skip, + limit=query.limit, + workstream=query.workstream): + return result + except Exception as e: + raise HTTPException(status_code=500, detail=f"Database error: {str(e)}") from e + +@router.get("/{id}") +def api_author(id: Annotated[UUID, Path(title="Unique author identifier")], + query: Annotated[FilterParams, Query()] + ) -> AuthorOutputModel: + author = Author() + try: + result = author.get_author(id=id, + result_type=query.result_type, + skip=query.skip, + limit=query.limit) + + except KeyError: + raise HTTPException(status_code=404, + detail=f"Author '{id}' not found") + except ValueError as e: + raise HTTPException(status_code=500, + detail=f"Database error: {str(e)}") from e + else: + return result + diff --git a/app/api/country.py b/app/api/country.py new file mode 100644 index 0000000..1390d5e --- /dev/null +++ b/app/api/country.py @@ -0,0 +1,38 @@ +from fastapi import APIRouter, HTTPException, Query, Path +from typing import Annotated +from app.crud.country import Country +from app.schemas.country import CountryList, CountryOutputListModel +from app.schemas.query import FilterBase, FilterParams + +router = APIRouter(prefix="/api/countries", tags=["countries"]) + + +@router.get("") +def api_country_list(query: Annotated[FilterBase, Query()] + ) -> CountryList: + country_model = Country() + try: + return country_model.get_countries(query.skip, query.limit) + except Exception as e: + raise HTTPException(status_code=500, + detail=f"Server error: {str(e)}") from e + +@router.get("/{id}") +def api_country(id: Annotated[str, Path(examples=['KEN'], title="Country identifier", pattern="^([A-Z]{3})$")], + query: Annotated[FilterParams, Query()] + ) -> CountryOutputListModel: + country_model = Country() + try: + result = country_model.get_country(id, + query.skip, + query.limit, + query.result_type) + except KeyError: + raise HTTPException(status_code=404, + detail=f"Country with id {id} not found") + except ValueError as e: + raise HTTPException(status_code=500, + detail=f"Database error: {str(e)}") from e + + else: + return result diff --git a/app/api/output.py b/app/api/output.py new file mode 100644 index 0000000..447961e --- /dev/null +++ b/app/api/output.py @@ -0,0 +1,41 @@ +from fastapi import APIRouter, HTTPException, Query, Path +from fastapi.logger import logger +from typing import Annotated +from app.schemas.query import FilterOutputList +from uuid import UUID + +from app.crud.output import Output +from app.schemas.output import OutputListModel, OutputModel + +router = APIRouter(prefix="/api/outputs", tags=["outputs"]) + + +@router.get("") +def api_output_list( + query: Annotated[FilterOutputList, Query()] +) -> OutputListModel: + """Return a list of outputs""" + outputs = Output() + try: + return outputs.get_outputs(query.skip, + query.limit, + query.result_type, + query.country) + except ValueError as e: + raise HTTPException(status_code=500, detail=str(e)) from e + + +@router.get("/{id}") +def api_output(id: Annotated[UUID, Path(title="Unique output identifier")]) -> OutputModel: + output = Output() + try: + result = output.get_output(id) + except KeyError as e: + raise HTTPException( + status_code=404, detail=f"Output with id {id} not found" + ) from e + except Exception as e: + logger.error(f"Error in api_output: {str(e)}") + raise HTTPException(status_code=400, detail=str(e)) from e + else: + return result diff --git a/app/api/topics.py b/app/api/topics.py new file mode 100644 index 0000000..93c159b --- /dev/null +++ b/app/api/topics.py @@ -0,0 +1,17 @@ +from typing import List + +from fastapi import APIRouter + +from app.schemas.topic import TopicBaseModel + +router = APIRouter(prefix="/api/topics", tags=["countries"]) + + +@router.get("") +def api_topics_list() -> List[TopicBaseModel]: + raise NotImplementedError("Have not yet implemented topics in the database") + + +@router.get("/{id}") +def api_topics_list(id: str) -> TopicBaseModel: + raise NotImplementedError("Have not yet implemented topics in the database") diff --git a/app/api/workstream.py b/app/api/workstream.py new file mode 100644 index 0000000..4db8b09 --- /dev/null +++ b/app/api/workstream.py @@ -0,0 +1,47 @@ +from typing import List, Annotated + +from fastapi import APIRouter, HTTPException, Query, Path + +from app.crud.workstream import Workstream +from app.schemas.workstream import WorkstreamDetailModel, WorkstreamListModel +from app.schemas.query import FilterBase + +router = APIRouter(prefix="/api/workstreams", tags=["workstreams"]) + + +@router.get("") +def list_workstreams( + query: Annotated[FilterBase, Query()]) -> WorkstreamListModel: + """Return a list of workstreams + + Returns + ------- + app.schemas.workstream.WorkstreamListModel + + """ + model = Workstream() + try: + results = model.get_all(skip=query.skip, limit=query.limit) + except KeyError as e: + raise HTTPException(status_code=500, + detail=f"Database error: {str(e)}") from e + else: + return results + + + +@router.get("/{id}") +def get_workstream( + id: Annotated[str, Path(title="Unique workstream identifier")], + query: Annotated[FilterBase, Query()] + ) -> WorkstreamDetailModel: + """Return a single workstream + """ + model = Workstream() + try: + results = model.get(id, skip=query.skip, limit=query.limit) + except KeyError: + raise HTTPException(status_code=404, + detail=f"Workstream '{id}' not found") + else: + return results \ No newline at end of file diff --git a/app/app.py b/app/app.py deleted file mode 100644 index 04a0c61..0000000 --- a/app/app.py +++ /dev/null @@ -1,114 +0,0 @@ -from flask import Flask, render_template, request - -from model import (Author, Output, AuthorList, OutputList, Nodes, Edges, - CountryList, Country) - -from logging import getLogger, basicConfig, DEBUG - -logger = getLogger(__name__) -basicConfig(filename='example.log', filemode='w', encoding='utf-8', level=DEBUG) - - -app = Flask(__name__) - - - -@app.route('/countries/') -def country(id: str): - country_model = Country() - outputs, country = country_model.get(id) - - result_type = request.args.get('type') - logger.debug(f"Obtained filter {result_type}") - if result_type: - logger.info(f"Filtering outputs on {result_type}") - outputs, country = country_model.get(id, result_type=result_type) - else: - outputs, country = country_model.get(id) - - count = country_model.count(id) - - return render_template('country.html', title='Country', outputs=outputs, - country=country, count=count) - - -@app.route('/countries') -def country_list(): - country_model = CountryList() - entity = country_model.get() - return render_template('country_list.html', title='Countries', countries=entity) - - -@app.route('/authors/') -def author(id: str): - author_model = Author() - result_type = request.args.get('type') - logger.debug(f"Obtained filter {result_type}") - if result_type: - logger.info(f"Filtering outputs on {result_type}") - entity = author_model.get(id, type=result_type) - else: - entity = author_model.get(id) - logger.debug(entity) - - count = author_model.count(id) - - return render_template('author.html', - title='Author', - author=entity, - count=count) - - -@app.route('/authors') -def author_list(): - model = AuthorList() - entity = model.get() - return render_template('authors.html', title='Author List', authors=entity) - - -@app.route('/outputs') -def output_list(): - model = OutputList() - result_type = request.args.get('type') - logger.debug(f"Obtained filter {result_type}") - if result_type: - logger.info(f"Filtering outputs on {result_type}") - entity = model.filter_type(result_type=result_type) - else: - entity = model.get() - - count = model.count() - - return render_template('outputs.html', - title='Output List', - outputs=entity, - count=count) - - -@app.route('/outputs/') -def output(id: str): - output_model = Output() - - entity = output_model.get(id) - return render_template('output.html', title='Output', output=entity) - - -@app.route('/outputs//popup') -def output_popup(id: str): - output_model = Output() - entity = output_model.get(id) - return render_template('output_popup.html', title='Output', output=entity) - - -@app.route('/') -@app.route('/index') -def index(): - nodes = Nodes().get() - edges = Edges().get() - countries = CountryList().get() - return render_template('index.html', title='Home', - nodes=nodes, links=edges, countries=countries) - - -if __name__ == '__main__': - app.run(debug=True, port=5001) diff --git a/app/core/__init__.py b/app/core/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/app/core/config.py b/app/core/config.py new file mode 100644 index 0000000..0db9088 --- /dev/null +++ b/app/core/config.py @@ -0,0 +1,20 @@ +import os +from dotenv import load_dotenv + + +class Settings: + load_dotenv() + _instance = None + + def __new__(cls, *args, **kwargs): + if cls._instance is None: + cls._instance = super(Settings, cls).__new__(cls) + cls._instance._init_instance() + return cls._instance + + def _init_instance(self): + self.MG_HOST = os.getenv("MG_HOST") + self.MG_PORT = os.getenv("MG_PORT") + + +settings = Settings() diff --git a/app/crud/__init__.py b/app/crud/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/app/crud/author.py b/app/crud/author.py new file mode 100644 index 0000000..d309e17 --- /dev/null +++ b/app/crud/author.py @@ -0,0 +1,253 @@ +from typing import Any, Dict, List, Optional, Tuple +from uuid import UUID + +from fastapi.logger import logger + +from neo4j import Driver + +from app.db.session import connect_to_db +from app.schemas.author import (AuthorListModel, + AuthorOutputModel, + AuthorColabModel) +from app.schemas.meta import CountPublication + + +class Author: + + def get_authors(self, + skip: int, + limit: int, + workstream: list[str] = []) -> AuthorListModel: + """Get list of authors + + Arguments + --------- + skip: int + Number or records to skip + limit: int + Number of records to return + + Returns + ------- + AuthorListModel + + """ + authors = self.fetch_author_nodes(skip=skip, limit=limit, workstream=workstream) + if workstream: + count = len(authors) + else: + count = self.count_authors() + return {"meta": { + "count": {"total": count}, + "skip": skip, + "limit": limit}, + "results": authors} + + def get_author(self, id: UUID, result_type: str = 'publication', skip: int = 0, limit: int = 20) -> AuthorOutputModel: + """Get an author, collaborators and outputs + + Arguments + --------- + id: UUID + Unique author identifier + result_type: str, default = 'publication', + skip: int, default = 0 + limit: int, default = 20 + + Returns + ------- + AuthorOutputModel + """ + if author := self.fetch_author_node(str(id)): + collaborators = self.fetch_collaborator_nodes(str(id), result_type)[0] + collaborators = [collaborator.data() for collaborator in collaborators] + count = self.count_author_outputs(str(id)) + publications = self.fetch_publications(str(id), + result_type=result_type, + skip=skip, + limit=limit) + author['collaborators'] = collaborators + author['outputs'] = {'results': publications} + author['outputs']['meta'] = {"count": count, + "skip": skip, + "limit": limit, + "result_type": result_type} + return author + else: + msg = f"Could not find author with id: {id}" + logger.error(msg) + raise KeyError(msg) + + @connect_to_db + def fetch_author_nodes( + self, db: Driver, skip: int, limit: int, workstream: List[str] = [] + ) -> List[AuthorColabModel]: + if workstream: + query = """ + MATCH (a:Author)-[:member_of]->(u:Workstream) + WHERE u.id IN $workstream + OPTIONAL MATCH (a)-[:member_of]->(p:Partner) + RETURN a.first_name as first_name, + a.last_name as last_name, + a.uuid as uuid, + a.orcid as orcid, + collect(DISTINCT p) as affiliations, collect(DISTINCT u) as workstreams + ORDER BY last_name + SKIP $skip + LIMIT $limit;""" + else: + query = """ + MATCH (a:Author) + OPTIONAL MATCH (a)-[:member_of]->(u:Workstream) + OPTIONAL MATCH (a)-[:member_of]->(p:Partner) + RETURN a.first_name as first_name, + a.last_name as last_name, + a.uuid as uuid, + a.orcid as orcid, + collect(DISTINCT p) as affiliations, collect(DISTINCT u) as workstreams + ORDER BY last_name + SKIP $skip + LIMIT $limit;""" + records, _, _ = db.execute_query(query, skip=skip, limit=limit, workstream=workstream) + return [record.data() for record in records] + + @connect_to_db + def count_authors(self, db: Driver) -> int: + """Count the number of authors""" + query = """MATCH (a:Author) + RETURN COUNT(a) as count + """ + records, _, _ = db.execute_query(query) + + return [record.data() for record in records][0]["count"] + + + + @connect_to_db + def fetch_author_node(self, id: str, db: Driver) -> Dict[str, Any]: + author_query = """ + MATCH (a:Author) + WHERE a.uuid = $uuid + OPTIONAL MATCH (a)-[:member_of]->(p:Partner) + OPTIONAL MATCH (a)-[:member_of]->(u:Workstream) + RETURN a.uuid as uuid, a.orcid as orcid, + a.first_name as first_name, a.last_name as last_name, + collect(DISTINCT p) as affiliations, + collect(DISTINCT u) as workstreams;""" + records, _, _ = db.execute_query(author_query, uuid=id) + if len(records) == 0: + return None + else: + return records[0].data() + + @connect_to_db + def count_author_outputs(self, id: str, db: Driver) -> CountPublication: + query = """ + MATCH (a:Author)-[b:author_of]->(o:Output) + WHERE (a.uuid) = $uuid + RETURN o.result_type as result_type, count(DISTINCT o) as count + """ + records, _, _ = db.execute_query(query, uuid=id) + if len(records) < 1: + return { + "total": 0, + "publication": 0, + "dataset": 0, + "other": 0, + "software": 0, + } + counts = {x.data()["result_type"]: x.data()["count"] for x in records} + counts["total"] = sum(counts.values()) + return CountPublication(**counts) + + @connect_to_db + def fetch_collaborator_nodes( + self, id: str, result_type: str, db: Driver + ) -> List[Dict[str, Any]]: + collab_query = """ + MATCH (a:Author)-[:author_of]->(z:Output)<-[:author_of]-(b:Author) + WHERE a.uuid = $uuid AND b.uuid <> $uuid AND z.result_type = $result_type + RETURN DISTINCT b.uuid as uuid, + b.first_name as first_name, + b.last_name as last_name, + b.orcid as orcid, + count(z) as num_colabs + ORDER BY num_colabs DESCENDING + LIMIT 5""" + return db.execute_query(collab_query, uuid=id, result_type=result_type) + + @connect_to_db + def fetch_publications( + self, + id: str, + db: Driver, + result_type: Optional[str] = None, + limit: int = 20, + skip: int = 0, + ) -> Tuple: + if result_type in [ + "publication", + "dataset", + "software", + "other", + ]: + publications_query = """ + MATCH (a:Author)-[:author_of]->(p:Output) + WHERE (a.uuid) = $uuid AND (p.result_type = $result_type) + CALL { + WITH p + MATCH (b:Author)-[r:author_of]->(p) + RETURN b + ORDER BY r.rank + } + OPTIONAL MATCH (p)-[:refers_to]->(c:Country) + RETURN p as results, + collect(DISTINCT c) as countries, + collect(DISTINCT b) as authors + ORDER BY results.publication_year DESCENDING + SKIP $skip + LIMIT $limit + ;""" + + records, summary, keys = db.execute_query( + publications_query, + uuid=id, + result_type=result_type, + skip=skip, + limit=limit, + ) + + else: + publications_query = """ + MATCH (a:Author)-[:author_of]->(p:Output) + WHERE a.uuid = $uuid + CALL { + WITH p + MATCH (b:Author)-[r:author_of]->(p) + RETURN b + ORDER BY r.rank + } + OPTIONAL MATCH (p)-[:refers_to]->(c:Country) + RETURN p as results, + collect(DISTINCT c) as countries, + collect(DISTINCT b) as authors + ORDER BY results.publication_year DESCENDING + SKIP $skip + LIMIT $limit + ;""" + + records, summary, keys = db.execute_query( + publications_query, uuid=id, limit=limit, skip=skip + ) + publications = [] + + for record in records: + data = record.data() + package = data["results"] + package["authors"] = data["authors"] + package["countries"] = data["countries"] + publications.append(package) + + return publications + + diff --git a/app/crud/country.py b/app/crud/country.py new file mode 100644 index 0000000..e57b3bd --- /dev/null +++ b/app/crud/country.py @@ -0,0 +1,163 @@ +from typing import Any, Dict +from fastapi.logger import logger + +from neo4j import Driver + +from app.db.session import connect_to_db +from .output import Output + +from app.schemas.country import (CountryList, + CountryNodeModel, + CountryOutputListModel) +from app.schemas.meta import CountPublication + + +class Country: + + def get_country(self, + id: str, + skip: int = 0, + limit: int = 20, + result_type: str = 'publication' + ) -> CountryOutputListModel: + """Return a country + + Arguments + --------- + id: str + Three letter country code + skip: int = 0 + limit: int = 20 + result_type: str = 'publication' + + Returns + ------- + schemas.output.CountryOutputListModel + """ + try: + entity = self.fetch_country_node(id) + except KeyError as ex: + logger.error( + f"Country outputs not found {id}:{skip}:{limit}:{result_type}") + ex.add_note(f"Could not find {id} in the db") + raise KeyError(ex) + else: + outputs = Output() + package = outputs.get_outputs(skip=skip, + limit=limit, + result_type=result_type, + country=id) + counts = self.count_country_outputs(id) + package["meta"]["count"] = counts + return package | entity + + def get_countries(self, skip: int = 0, limit: int = 20) -> CountryList: + """Get a list of countries + + Arguments + --------- + skip: int, default=0 + limit: int, default=20 + + Returns + ------- + schemas.country.CountryList + """ + results = self.get_country_list(skip=skip, limit=limit) + count = self.count_countries() + return {"meta": {"count": {'total': count}, "skip": skip, "limit": limit}, + "results": results} + + @connect_to_db + def fetch_country_node(self, id: str, db: Driver) -> Dict[str, Any]: + """Retrieve country information + + Parameters + ---------- + id : str + The unique identifier of the country + db : Driver + Neo4j database driver instance + + + Returns + ------- + Dict[str, Any] + Country information dictionary + """ + query = """MATCH (c:Country) WHERE c.id = $id RETURN c as country;""" + results, summary, keys = db.execute_query(query, id=id) + logger.debug(f"Received {results}") + if results: + return results[0].data()["country"] + else: + msg = f"No results returned from query for country '{id}'" + logger.error(msg) + raise KeyError(msg) + + @connect_to_db + def count_country_outputs(self, id: str, db: Driver) -> CountPublication: + """Count articles by result type for a specific country. + + Parameters + ---------- + id : str + The unique identifier of the country + db : Driver + Neo4j database driver instance + + Returns + ------- + Dict[str, int] + Dictionary where: + - keys: result type strings + - values: count of articles for each result type + """ + query = """ + MATCH (o:Output)-[:refers_to]->(c:Country) + WHERE c.id = $id + RETURN o.result_type as result_type, count(DISTINCT o) as count + """ + records, _, _ = db.execute_query(query, id=id) + if len(records) <= 0: + return {'total': 0, + 'publication': 0, + 'dataset': 0, + 'other': 0, + 'software': 0} + counts = {x.data()["result_type"]: x.data()["count"] for x in records} + counts['total'] = sum(counts.values()) + + return CountPublication(**counts) + + @connect_to_db + def count_countries(self, db: Driver) -> int: + """Count the countries""" + query = """MATCH (c:Country)<-[:refers_to]-(p:Output) + RETURN count(DISTINCT c.id) as count""" + results, _, _ = db.execute_query(query) + return results[0].data()['count'] + + @connect_to_db + def get_country_list(self, + db: Driver, + skip: int = 0, + limit: int = 20) -> list[CountryNodeModel]: + """Retrieve all countries that have associated articles. + + Parameters + ---------- + db : Driver + Neo4j database driver instances + + Returns + ------- + list[schemas.country.CountryNodeModel] + """ + query = """MATCH (c:Country)<-[:refers_to]-(p:Output) + RETURN DISTINCT c as country + SKIP $skip + LIMIT $limit + """ + records, _, _ = db.execute_query(query, skip=skip, limit=limit) + return [result.data()['country'] for result in records] diff --git a/app/crud/graph.py b/app/crud/graph.py new file mode 100644 index 0000000..4adc9f0 --- /dev/null +++ b/app/crud/graph.py @@ -0,0 +1,78 @@ +from typing import Any, Dict, List + +from neo4j import Driver + +from app.db.session import connect_to_db + + +class Nodes: + """Class for retrieving graph nodes representing authors and articles.""" + + @connect_to_db + def get(self, db: Driver) -> List[Dict[str, Any]]: + """Retrieve all author and article nodes from the database. + + Parameters + ---------- + db : Driver + Neo4j database driver + + Returns + ------- + List[Dict[str, Any]] + List of node dictionaries containing: + - id : str + Node UUID + - group : int + Node type (0=Author, 1=Article) + - name : str + Display name (full name for authors, title for articles) + - url : str + Associated URL (ORCID for authors, DOI for articles) + + Raises + ------ + Neo4jError + If database query fails + """ + query = """MATCH (a:Author) + RETURN a.uuid as id, 0 as group, a.first_name + " " + a.last_name as name, a.orcid as url + UNION ALL + MATCH (b:Output) + RETURN b.uuid as id, 1 as group, b.title as name, "https://doi.org/" + b.doi as url + """ + results, summary, keys = db.execute_query(query) + return [x.data() for x in results] + + +class Edges: + """Class for retrieving graph edges representing author-article relationships.""" + + @connect_to_db + def get(self, db: Driver) -> List[Dict[str, str]]: + """Retrieve all author-article relationships from the database. + + Parameters + ---------- + db : Driver + Neo4j database driver + + Returns + ------- + List[Dict[str, str]] + List of edge dictionaries containing: + - source : str + Author node UUID + - target : str + Article node UUID + + Raises + ------ + Neo4jError + If database query fails + """ + query = """MATCH (p:Output)<-[author_of]-(a:Author) + RETURN p.uuid as target, a.uuid as source + """ + results, summary, keys = db.execute_query(query) + return [x.data() for x in results] diff --git a/app/crud/output.py b/app/crud/output.py new file mode 100644 index 0000000..777cae2 --- /dev/null +++ b/app/crud/output.py @@ -0,0 +1,253 @@ +from typing import Any, Dict, List +from uuid import UUID +from neo4j import Driver +from fastapi.logger import logger + +from app.db.session import connect_to_db +from app.schemas.output import OutputListModel, OutputModel + + +class Output: + @connect_to_db + def get_output(self, id: UUID, db: Driver) -> OutputModel: + """Retrieve article output information from the database. + + Parameters + ---------- + id : str + UUID of the article + db : Driver + Neo4j database driver + + Returns + ------- + Dict[str, Any] + Article output information containing: + - Article properties from Neo4j + - countries : List[Dict] + List of countries referenced in the article + - authors : List[Dict] + List of author dictionaries containing: + - uuid : str + Author's unique identifier + - first_name : str + Author's first name + - last_name : str + Author's last name + - orcid : str + Author's ORCID identifier + """ + + query = """ + MATCH (o:Output) + WHERE o.uuid = $uuid + OPTIONAL MATCH (o)-[:refers_to]->(c:Country) + CALL + { + WITH o + MATCH (a:Author)-[b:author_of]->(o) + RETURN a + ORDER BY b.rank + } + RETURN o as outputs, collect(DISTINCT c) as countries, collect(DISTINCT a) as authors + """ + records, _, _ = db.execute_query(query, uuid=str(id)) + if records: + data = [x.data() for x in records][0] + package = data['outputs'] + package['authors'] = data['authors'] + package['countries'] = data['countries'] + + return package + else: + logger.error(f"Output {str(id)} does not exist in database") + raise KeyError(f"Output {str(id)} does not exist") + + @connect_to_db + def count(self, db: Driver) -> Dict[str, int]: + """Count articles by result type. + + Parameters + ---------- + db : Driver + Neo4j database driver + + Returns + ------- + Dict[str, int] + Dictionary mapping result types to their counts + Example: {'journal_article': 5, 'conference_paper': 3} + """ + query = """ + MATCH (a:Author)-[b:author_of]->(o:Output) + RETURN o.result_type as result_type, count(DISTINCT o) as count + """ + records, _, _ = db.execute_query(query) + if len(records) <= 0: + return {'total': 0, + 'publication': 0, + 'dataset': 0, + 'other': 0, + 'software': 0} + counts = {x.data()["result_type"]: x.data()["count"] for x in records} + counts['total'] = sum(counts.values()) + return counts + + @connect_to_db + def filter_type(self, db: Driver, result_type: str, skip: int, limit: int) -> List[Dict[str, Any]]: + """Filter articles by result type and return with ordered authors. + + Parameters + ---------- + db : Driver + Neo4j database driver + result_type : str + Type of result to filter by (e.g. 'journal_article') + + Returns + ------- + List[Dict[str, Any]] + Filtered list of articles containing: + - outputs : Dict + Article properties + - countries : List[Dict] + List of referenced countries + - authors : List[Dict] + List of authors ordered by rank + + Raises + ------ + ValueError + If result_type is invalid + """ + query = """ + MATCH (o:Output) + WHERE o.result_type = $result_type + OPTIONAL MATCH (o)-[:refers_to]->(c:Country) + CALL + { + WITH o + MATCH (a:Author)-[b:author_of]->(o) + RETURN a + ORDER BY b.rank + } + + RETURN o as outputs, + collect(DISTINCT c) as countries, + collect(DISTINCT a) as authors + SKIP $skip + LIMIT $limit; + """ + records, _, _ = db.execute_query(query, + result_type=result_type, + skip=skip, + limit=limit) + outputs = [] + for x in records: + data = x.data() + package = data['outputs'] + package['authors'] = data['authors'] + package['countries'] = data['countries'] + outputs.append(package) + + return outputs + + @connect_to_db + def filter_country(self, + db: Driver, + result_type: str, + skip: int, + limit: int, + country: str) -> List[Dict[str, Any]]: + """Filter articles by country and result type and return with ordered authors. + + Parameters + ---------- + db : Driver + Neo4j database driver + result_type : str + Type of result to filter by (e.g. 'journal_article') + skip: int + Number of rows in the output to skip + limit: int + Number of rows to return + country: str + Three letter ISO country code + + Returns + ------- + List[Dict[str, Any]] + Filtered list of articles containing: + - outputs : Dict + Article properties + - countries : List[Dict] + List of referenced countries + - authors : List[Dict] + List of authors ordered by rank + + Raises + ------ + ValueError + If result_type is invalid + """ + query = """ + MATCH (o:Output)-[:refers_to]->(c:Country) + WHERE o.result_type = $result_type + AND c.id = $country_id + CALL + { + WITH o + MATCH (a:Author)-[b:author_of]->(o) + RETURN a + ORDER BY b.rank + } + RETURN o as outputs, + collect(DISTINCT c) as countries, + collect(DISTINCT a) as authors + SKIP $skip + LIMIT $limit; + """ + records, summary, keys = db.execute_query(query, + result_type=result_type, + country_id=country, + skip=skip, + limit=limit) + outputs = [] + for x in records: + data = x.data() + package = data['outputs'] + package['authors'] = data['authors'] + package['countries'] = data['countries'] + outputs.append(package) + + return outputs + + def get_outputs(self, + skip: int = 0, + limit: int = 20, + result_type: str = 'publication', + country: str = None) -> OutputListModel: + """Return a list of outputs""" + try: + if country: + results = self.filter_country( + result_type=result_type, skip=skip, limit=limit, country=country + ) + else: + results = self.filter_type(result_type=result_type, + skip=skip, + limit=limit) + + count = self.count() + + return { + "meta": { + "count": count, + "skip": skip, + "limit": limit, + "result_type": result_type + }, + "results": results, + } + except ValueError as e: + raise ValueError(str(e)) from e \ No newline at end of file diff --git a/app/crud/workstream.py b/app/crud/workstream.py new file mode 100644 index 0000000..1ebde77 --- /dev/null +++ b/app/crud/workstream.py @@ -0,0 +1,109 @@ +from typing import Any, Dict, List + +from neo4j import Driver + +from app.db.session import connect_to_db +from app.schemas.workstream import WorkstreamDetailModel, WorkstreamListModel, WorkstreamBase +from .author import Author +from app.schemas.author import AuthorListModel +from app.schemas.output import OutputListModel +from .output import Output + +class Workstream: + + def get_all(self, skip: int = 0, limit: int = 20) -> WorkstreamListModel: + """Return a list of all workstreams + + Arguments + --------- + skip: int, default=0 + number of records to skip + limit: int, default=20 + number of records to return + + Returns + ------- + app.schema.workstream.WorkstreamListModel + """ + return {'results': self.get_workstreams(skip=skip, limit=limit), + 'meta': {'count': {'total': self.count_members()}, + 'skip': skip, + 'limit': limit}} + + def get(self, + id: str, + skip: int = 0, + limit: int = 20) -> WorkstreamDetailModel: + """Return a list of members for a workstream + + Arguments + --------- + id: str + id of the workstream + skip: int, default=0 + number of records to skip + limit: int, default=20 + number of records to return + + Returns + ------- + app.schema.workstream.WorkstreamDetailModel + + """ + workstream = self.get_workstream_detail(id) + if workstream: + members = self.get_members([id] + workstream.pop('children'), + skip=skip, + limit=limit) # typing: AuthorListModel + return workstream | {'members': members} + else: + return {'members': {}} + + @connect_to_db + def get_workstream_detail(self, id: str, db: Driver) -> dict: + query = """MATCH (p:Workstream) + WHERE p.id = $id + OPTIONAL MATCH (p)<-[:unit_of]-(b:Workstream) + RETURN p.id as id, p.name as name, collect(b.id) as children + """ + records, _, _ = db.execute_query(query, id=id) + if records: + return records[0].data() + else: + raise KeyError("No records returned for {id}") + + def get_outputs(self, id: str, skip, limit) -> OutputListModel: + output = Output() + return output.get_outputs(skip, limit) + + @connect_to_db + def get_workstreams(self, + db: Driver, + skip: int = 0, + limit: int = 20 + ) -> list[WorkstreamBase]: + query = """MATCH (p:Workstream)-[]-(:Author) + OPTIONAL MATCH (u:Workstream)<-[:unit_of]-(p) + RETURN DISTINCT u.id as unit_id, u.name as unit_name, p.id as id, p.name as name + ORDER BY unit_name, name + SKIP $skip + LIMIT $limit""" + records, _, _ = db.execute_query(query, skip=skip, limit=limit) + if len(records) == 0: + raise KeyError("No records returned") + else: + return [x.data() for x in records] + + def get_members(self, + id: list[str], + skip: int = 0, + limit: int = 20) -> AuthorListModel: + author = Author() + return author.get_authors(skip, limit, id) + + @connect_to_db + def count_members(self, db: Driver) -> int: + query = """MATCH (p:Workstream) + RETURN count(DISTINCT p) as count""" + records, _, _ = db.execute_query(query) + return records[0].data()['count'] diff --git a/app/db/__init__.py b/app/db/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/app/db/session.py b/app/db/session.py new file mode 100644 index 0000000..6590dba --- /dev/null +++ b/app/db/session.py @@ -0,0 +1,26 @@ +from functools import wraps +from app.core.config import settings + + +from neo4j import GraphDatabase + +MG_HOST = settings.MG_HOST +MG_PORT = settings.MG_PORT + + +def connect_to_db(f): + @wraps(f) + def with_connection_(*args, **kwargs): + + try: + URI = f"bolt://{MG_HOST}:{MG_PORT}" + AUTH = ("", "") + with GraphDatabase.driver(URI, auth=AUTH) as db: + db.verify_connectivity() + return f(*args, db, **kwargs) + except Exception: + raise + finally: + db.close() + + return with_connection_ diff --git a/app/ingest.py b/app/ingest.py deleted file mode 100644 index 9ab6478..0000000 --- a/app/ingest.py +++ /dev/null @@ -1,64 +0,0 @@ -from gqlalchemy import Memgraph, Node, Relationship, Field -from typing import Optional -from csv import DictReader -from os.path import join -from os import environ - -MG_HOST = environ.get('MG_HOST', '127.0.0.1') -MG_PORT = int(environ.get('MG_PORT', 7687)) - -db = Memgraph(host=MG_HOST, port=MG_PORT) -db.drop_database() - - -class Author(Node): - uuid: str = Field(unique=True, index=True, db=db) - orcid: Optional[str] - last_name: Optional[str] - first_name: Optional[str] - - -class Output(Node): - uuid: str = Field(unique=True, index=True, db=db) - - -class Article(Output): - doi: Optional[str] - title: Optional[str] - abstract: Optional[str] - - -class author_of(Relationship): - pass - - -author_objects = {} -with open(join('data', 'authors.csv')) as authors_csv: - reader = DictReader(authors_csv) - for author in reader: - print(author) - author_objects[author['uuid']] = Author(uuid=author['uuid'], - first_name=author['First Name'], - last_name=author['Last Name'], - orcid=author['Orcid']).save(db) - -output_objects = {} -with open(join('data', 'papers.csv')) as papers_csv: - reader = DictReader(papers_csv) - for output in reader: - print(output) - output_objects[output['paper_uuid']] = Article(uuid=output['paper_uuid'], - doi=output['DOI'], - title=output['title'], - abstract=output['Abstract']).save(db) - -with open(join('data', 'relations.csv')) as relations_csv: - reader = DictReader(relations_csv) - for rel in reader: - author_uuid = rel['uuid'] - paper_uuid = rel['paper_uuid'] - - loaded_author = Author(uuid=author_uuid).load(db=db) - loaded_output = Article(uuid=paper_uuid).load(db=db) - - author_of(_start_node_id=loaded_author._id, _end_node_id=loaded_output._id).save(db) diff --git a/app/main.py b/app/main.py new file mode 100644 index 0000000..11c03fb --- /dev/null +++ b/app/main.py @@ -0,0 +1,227 @@ +from fastapi.logger import logger +from fastapi import FastAPI, Request, Query, Path, HTTPException +from fastapi.responses import HTMLResponse +from fastapi.staticfiles import StaticFiles +from fastapi.templating import Jinja2Templates + +from typing import Annotated +from uuid import UUID + +from app.crud.author import Author +from app.crud.country import Country +from app.crud.output import Output +from app.crud.workstream import Workstream +from app.schemas.query import (FilterWorkstream, FilterParams, FilterBase, FilterOutputList) + +from app.api import author, output, country, workstream + +import logging + +# Add console handler to the fastapi logger +console_handler = logging.StreamHandler() +logger.addHandler(console_handler) + +# Use a nice format for the log messages +formatter = logging.Formatter( + "%(asctime)s [%(levelname)s] %(message)s" +) +console_handler.setFormatter(formatter) + +# Obtain access loggers for uvicorn +uvicorn_access_logger = logging.getLogger("uvicorn.access") +logger.handlers = uvicorn_access_logger.handlers + +app = FastAPI() + +app.include_router(author.router) +app.include_router(output.router) +app.include_router(country.router) +app.include_router(workstream.router) + +templates = Jinja2Templates(directory="app/templates") +app.mount("/static", StaticFiles(directory="app/static"), name="static") + + +@app.get("/", response_class=HTMLResponse) +@app.get("/index", response_class=HTMLResponse) +def index(request: Request): + countries = Country().get_countries(skip=0, limit=200) + return templates.TemplateResponse( + request, + "index.html", + {"title": "Home"} | countries + ) + + +@app.get("/countries/{id}", response_class=HTMLResponse) +def country(request: Request, + id: Annotated[str, Path(examples=['KEN'], title="Country identifier", pattern="^([A-Z]{3})$")], + query: Annotated[FilterParams, Query()] + ): + country_model = Country() + try: + country = country_model.get_country(id, query.skip, query.limit, query.result_type) + except KeyError: + raise HTTPException(status_code=404, + detail=f"Country with id '{id}' not found") + except Exception as e: + raise HTTPException(status_code=500, + detail=f"Database error: {str(e)}") from e + else: + return templates.TemplateResponse( + request, + "country.html", + {"title": "Country"} | country + ) + + +@app.get("/countries", response_class=HTMLResponse) +def country_list(request: Request, + query: Annotated[FilterBase, Query()] + + ): + country_model = Country() + try: + entity = country_model.get_countries(query.skip, query.limit) + except Exception as e: + raise HTTPException(status_code=500, + detail=f"Server error: {str(e)}") from e + else: + return templates.TemplateResponse( + request, + "country_list.html", + {"title": "Countries"} | entity + ) + + +@app.get("/authors/{id}", response_class=HTMLResponse) +def author(request: Request, + id: Annotated[UUID, Path(title="Unique author identifier")], + query: Annotated[FilterParams, Query()]): + author = Author() + try: + entity = author.get_author( + id, + result_type=query.result_type, + skip=query.skip, + limit=query.limit) + except KeyError: + raise HTTPException(status_code=404, + detail=f"Author '{id}' not found") + else: + return templates.TemplateResponse( + request, + "author.html", + {"title": "Author"} | entity # Merges dicts + ) + + +@app.get("/authors", response_class=HTMLResponse) +def author_list(request: Request, + query: Annotated[FilterWorkstream, Query()]): + authors = Author() + try: + entity = authors.get_authors(skip=query.skip, + limit=query.limit, + workstream=query.workstream) + except KeyError as ex: + raise HTTPException(status_code=404, + detail=f"Authors not found") + else: + return templates.TemplateResponse( + request, + "authors.html", + {"title": "Author List"} | entity # Merges dicts + ) + + +@app.get("/outputs", response_class=HTMLResponse) +def output_list(request: Request, + query: Annotated[FilterOutputList, Query()] + ): + + model = Output() + try: + package = model.get_outputs(skip=query.skip, + limit=query.limit, + result_type=query.result_type, + country=query.country) + except ValueError as e: + raise HTTPException(status_code=500, detail=str(e)) from e + else: + return templates.TemplateResponse( + request, + "outputs.html", + {"title": "Output List"} | package + ) + + +@app.get("/outputs/{id}", response_class=HTMLResponse) +def output(request: Request, + id: Annotated[UUID, Path(title="Unique output identifier")] + ): + output_model = Output() + try: + entity = output_model.get_output(id) + except KeyError as e: + raise HTTPException( + status_code=404, detail=f"Output with id {id} not found" + ) from e + except Exception as e: + logger.error(f"Error in api_output: {str(e)}") + raise HTTPException(status_code=400, detail=str(e)) from e + else: + return templates.TemplateResponse( + request, + "output.html", + {"title": "Output"} | entity) + + +@app.get("/workstreams", response_class=HTMLResponse) +def workstream_list(request: Request): + model = Workstream() + try: + all = model.get_all() + except KeyError as e: + raise HTTPException(status_code=500, + detail=f"Database error: {str(e)}") from e + else: + try: + entity = model.get(all['results'][0]['id']) + except KeyError as e: + raise HTTPException(status_code=500, + detail=f"Database error: {str(e)}") from e + + else: + return templates.TemplateResponse( + request, + "workstreams.html", + {"title": "Workstream"} | entity | all + ) + + +@app.get("/workstreams/{id}", response_class=HTMLResponse) +def workstream(request: Request, + id: str, + query: Annotated[FilterBase, Query()] + ): + model = Workstream() + all = model.get_all() + try: + entity = model.get(id, skip=query.skip, limit=query.limit) + except KeyError as e: + raise HTTPException(status_code=404, + detail=f"Workstream '{id}' not found") + + else: + return templates.TemplateResponse( + request, + "workstreams.html", + {"title": "Workstreams"} | entity | all + ) + + +if __name__ == "__main__": + logger.setLevel(logging.DEBUG) +else: + logger.setLevel(uvicorn_access_logger.level) diff --git a/app/model.py b/app/model.py deleted file mode 100644 index 480782e..0000000 --- a/app/model.py +++ /dev/null @@ -1,329 +0,0 @@ -from neo4j import GraphDatabase, Driver -import os -from functools import wraps - -MG_HOST = os.environ.get('MG_HOST', '127.0.0.1') -MG_PORT = int(os.environ.get('MG_PORT', 7687)) - -print(f"Host is {MG_HOST} and port is {MG_PORT}") - - -def connect_to_db(f): - @wraps(f) - def with_connection_(*args, **kwargs): - - try: - URI = f"bolt://{MG_HOST}:{MG_PORT}" - AUTH = ("", "") - with GraphDatabase.driver(URI, auth=AUTH) as db: - db.verify_connectivity() - result = f(*args, db, **kwargs) - except Exception as e: - raise ValueError(e) - finally: - db.close() - return result - return with_connection_ - - -class OutputList: - - @connect_to_db - def get(self, db): - """ - """ - query = """ - MATCH (o:Article) - OPTIONAL MATCH (o)-[:REFERS_TO]->(c:Country) - CALL - { - WITH o - MATCH (a:Author)-[b:author_of]->(o) - RETURN a - ORDER BY b.rank - } - RETURN o as outputs, collect(DISTINCT c) as countries, collect(DISTINCT a) as authors; - """ - records, summary, keys = db.execute_query(query) - return [x.data() for x in records] - - @connect_to_db - def count(self, db: Driver): - """Returns counts of the result types - - Arguments - --------- - db : Driver - """ - query = """ - MATCH (a:Author)-[b:author_of]->(o:Article) - RETURN o.result_type as result_type, count(o) as count - """ - records, summary, keys = db.execute_query(query) - return {x.data()['result_type']: x.data()['count'] for x in records} - - @connect_to_db - def filter_type(self, db: Driver, result_type: str): - """Returns all outputs with ordered authors filtering on result type - - Arguments - --------- - db - result_type: str - """ - query = """ - MATCH (o:Article) - WHERE o.result_type = $result_type - OPTIONAL MATCH (o)-[:REFERS_TO]->(c:Country) - CALL - { - WITH o - MATCH (a:Author)-[b:author_of]->(o) - RETURN a - ORDER BY b.rank - } - RETURN o as outputs, - collect(DISTINCT c) as countries, - collect(DISTINCT a) as authors; - """ - records, summary, keys = db.execute_query(query, - result_type=result_type) - articles = [x.data() for x in records] - - return articles - - -class AuthorList: - - @connect_to_db - def get(self, db): - """ - - Notes - ----- - - """ - query = """MATCH (a:Author) - OPTIONAL MATCH (a)-[:member_of]->(p:Partner) - OPTIONAL MATCH (a)-[:member_of]->(u:Workstream) - RETURN a.first_name as first_name, a.last_name as last_name, a.uuid as uuid, a.orcid as orcid, collect(p.id, p.name) as affiliation, collect(u.id, u.name) as workstreams - ORDER BY last_name; - """ - records, summary, keys = db.execute_query(query) - - return records - - -class Author: - - @connect_to_db - def get(self, id, db, type=None): - """ - - Notes - ----- - MATCH (a:Author) - RETURN a.first_name as first_name, a.last_name as last_name, - p.name as affiliation; - MATCH (a:Author)-[r:author_of]->(p:Article) - OPTIONAL MATCH (a:Author)-[:member_of]->(p:Partner) - WHERE a.uuid = $uuid - RETURN *; - - """ - author_query = """MATCH (a:Author) WHERE a.uuid = $uuid - OPTIONAL MATCH (a)-[:member_of]->(p:Partner) - OPTIONAL MATCH (a)-[:member_of]->(u:Workstream) - RETURN a.uuid as uuid, a.orcid as orcid, - a.first_name as first_name, a.last_name as last_name, - collect(p.id, p.name) as affiliations, - collect(u.id, u.name) as workstreams; - """ - author, summary, keys = db.execute_query(author_query, uuid=id) - results = author[0].data() - - collab_query = """MATCH (a:Author)-[r:author_of]->(p:Output)<-[s:author_of]-(b:Author) - WHERE a.uuid = $uuid AND b.uuid <> $uuid - RETURN DISTINCT b.uuid as uuid, b.first_name as first_name, b.last_name as last_name, b.orcid as orcid - LIMIT 5""" - colabs, summary, keys = db.execute_query(collab_query, uuid=id) - - results['collaborators'] = colabs - - if type and type in ['publication', 'dataset', 'software', 'other']: - publications_query = """MATCH (a:Author)-[:author_of]->(p:Output) - WHERE (a.uuid) = $uuid AND (p.result_type = $type) - CALL { - WITH p - MATCH (b:Author)-[r:author_of]->(p) - RETURN b - ORDER BY r.rank - } - OPTIONAL MATCH (p)-[:REFERS_TO]->(c:Country) - RETURN p as outputs, - collect(DISTINCT c) as countries, - collect(DISTINCT b) as authors - ORDER BY outputs.publication_year DESCENDING;""" - else: - publications_query = """MATCH (a:Author)-[:author_of]->(p:Output) - WHERE a.uuid = $uuid - CALL { - WITH p - MATCH (b:Author)-[r:author_of]->(p) - RETURN b - ORDER BY r.rank - } - OPTIONAL MATCH (p)-[:REFERS_TO]->(c:Country) - RETURN p as outputs, - collect(DISTINCT c) as countries, - collect(DISTINCT b) as authors - ORDER BY outputs.publication_year DESCENDING;""" - result, summary, keys = db.execute_query(publications_query, - uuid=id, - type=type) - results['outputs'] = [x.data() for x in result] - - return results - - @connect_to_db - def count(self, id: str, db: Driver): - """Returns counts of the result types - - Arguments - --------- - db : Driver - """ - query = """ - MATCH (a:Author)-[b:author_of]->(o:Article) - WHERE (a.uuid) = $uuid - RETURN o.result_type as result_type, count(o) as count - """ - records, summary, keys = db.execute_query(query, uuid=id) - return {x.data()['result_type']: x.data()['count'] for x in records} - - -class Output: - - @connect_to_db - def get(self, id: str, db): - """ - """ - query = """MATCH (p:Article) - WHERE p.uuid = $uuid - OPTIONAL MATCH (p)-[:REFERS_TO]->(c:Country) - RETURN DISTINCT p as output, collect(DISTINCT c) as countries;""" - records, summary, keys = db.execute_query(query, uuid=id) - print(records[0].data()) - results = dict() - results = records[0].data()['output'] - results['countries'] = records[0].data()['countries'] - - authors_query = """MATCH (a:Author)-[r:author_of]->(p:Article) - WHERE p.uuid = $uuid - RETURN a.uuid as uuid, a.first_name as first_name, a.last_name as last_name, a.orcid as orcid;""" - records, summary, keys = db.execute_query(authors_query, uuid=id) - - results['authors'] = [x.data() for x in records] - - return results - - -class Nodes: - - @connect_to_db - def get(self, db): - - query = """MATCH (a:Author) - RETURN a.uuid as id, 0 as group, a.first_name + " " + a.last_name as name, a.orcid as url - UNION ALL - MATCH (b:Article) - RETURN b.uuid as id, 1 as group, b.title as name, "https://doi.org/" + b.doi as url - """ - results, summary, keys = db.execute_query(query) - return [x.data() for x in results] - - -class Edges: - - @connect_to_db - def get(self, db): - - query = """MATCH (p:Article)<-[author_of]-(a:Author) - RETURN p.uuid as target, a.uuid as source - """ - results, summary, keys = db.execute_query(query) - return [x.data() for x in results] - - -class CountryList: - - @connect_to_db - def get(self, db: Driver): - - query = """MATCH (c:Country)<-[:REFERS_TO]-(p:Article) - RETURN DISTINCT c - """ - results, summary, keys = db.execute_query(query) - return [x.data() for x in results] - - -class Country: - - @connect_to_db - def get(self, id: str, db: Driver, result_type=None): - - if result_type: - query = """ - MATCH (o:Output)-[:REFERS_TO]->(c:Country) - WHERE c.id = $id AND (o.result_type = $result_type) - CALL { - WITH o - MATCH (a:Author)-[r:author_of]->(o) - RETURN a - ORDER BY r.rank - } - OPTIONAL MATCH (o)-[:REFERS_TO]->(d:Country) - RETURN o as outputs, - collect(DISTINCT a) as authors, - collect(DISTINCT d) as countries; - """ - results, _, _ = db.execute_query(query, id=id, - result_type=result_type) - else: - query = """ - MATCH (o:Output)-[:REFERS_TO]->(c:Country) - WHERE c.id = $id - CALL { - WITH o - MATCH (a:Author)-[r:author_of]->(o) - RETURN a - ORDER BY r.rank - } - OPTIONAL MATCH (o)-[:REFERS_TO]->(d:Country) - RETURN o as outputs, - collect(DISTINCT a) as authors, - collect(DISTINCT d) as countries; - """ - results, _, _ = db.execute_query(query, id=id, - result_type=result_type) - outputs = [x.data() for x in results] - query = """MATCH (c:Country) WHERE c.id = $id RETURN c as country;""" - results, _, _ = db.execute_query(query, id=id) - country = results[0].data()['country'] - return outputs, country - - @connect_to_db - def count(self, id: str, db: Driver): - """Returns counts of the result types - - Arguments - --------- - db : Driver - """ - query = """ - MATCH (o:Article)-[:REFERS_TO]->(c:Country) - WHERE c.id = $id - RETURN o.result_type as result_type, count(o) as count - """ - records, _, _ = db.execute_query(query, id=id) - return {x.data()['result_type']: x.data()['count'] for x in records} diff --git a/app/schemas/__init__.py b/app/schemas/__init__.py new file mode 100644 index 0000000..c63aef4 --- /dev/null +++ b/app/schemas/__init__.py @@ -0,0 +1,28 @@ +from typing import Dict, List, Optional + +from pydantic import BaseModel, Field, HttpUrl +from uuid import UUID + + +class AuthorBase(BaseModel): + """ + Base data model representing an academic author or contributor + with their associated metadata. + """ + + uuid: UUID = Field(..., + description="Unique identifier for the author") + first_name: str = Field(..., min_length=1) + last_name: str = Field(..., min_length=1) + orcid: Optional[HttpUrl] = \ + Field(None, description="Author's ORCID identifier") + + +class CountryBaseModel(BaseModel): + id: str + name: str + + +class WorkstreamBase(BaseModel): + id: Optional[str] = None + name: Optional[str] = None \ No newline at end of file diff --git a/app/schemas/affiliation.py b/app/schemas/affiliation.py new file mode 100644 index 0000000..fdfbe4a --- /dev/null +++ b/app/schemas/affiliation.py @@ -0,0 +1,9 @@ +from pydantic import BaseModel +from typing import Optional + + +class AffiliationModel(BaseModel): + id: Optional[str] = None + name: Optional[str] = None + ror: Optional[str] = None + ccg_partner: Optional[bool] = None diff --git a/app/schemas/author.py b/app/schemas/author.py new file mode 100644 index 0000000..25b1fdf --- /dev/null +++ b/app/schemas/author.py @@ -0,0 +1,33 @@ +from typing import Dict, List, Optional + +from pydantic import BaseModel, Field, HttpUrl +from uuid import UUID + +from . import AuthorBase +from .affiliation import AffiliationModel +from .output import OutputListModel +from .meta import MetaAuthor +from . import WorkstreamBase + + +class AuthorColabModel(AuthorBase): + """ + An academic author or contributor + with their collaborators, workstreams and affiliations + """ + workstreams: List[WorkstreamBase] = None + affiliations: List[AffiliationModel] = None + + +class AuthorListModel(BaseModel): + """ + A list of authors + """ + meta: MetaAuthor + results: List[AuthorColabModel] + + +class AuthorOutputModel(AuthorColabModel): + """An author with collaborators, workstreams, affiliations and outputs""" + collaborators: List[AuthorBase] = None + outputs: OutputListModel diff --git a/app/schemas/country.py b/app/schemas/country.py new file mode 100644 index 0000000..8e31000 --- /dev/null +++ b/app/schemas/country.py @@ -0,0 +1,23 @@ +from typing import Dict, List, Optional +from pydantic import BaseModel, Field +from . meta import Pagination +from . output import OutputListModel +from . import CountryBaseModel + + +class CountryOutputListModel(OutputListModel, CountryBaseModel): + """Data model representing country outputs""" + + + +class CountryNodeModel(CountryBaseModel): + dbpedia: Optional[str] = None + latitude: Optional[float] = None + longitude: Optional[float] = None + official_name: Optional[str] = None + + +class CountryList(BaseModel): + + meta: Pagination + results: list[CountryNodeModel] diff --git a/app/schemas/graph.py b/app/schemas/graph.py new file mode 100644 index 0000000..e69de29 diff --git a/app/schemas/meta.py b/app/schemas/meta.py new file mode 100644 index 0000000..0a15ab1 --- /dev/null +++ b/app/schemas/meta.py @@ -0,0 +1,68 @@ +from typing import Dict, List, Optional +from pydantic import BaseModel, Field + + +class CountPublication(BaseModel): + """Represents count of the outputs + ```json + {'total': 245684392, + 'publication': 1234, + 'software': 5678, + 'dataset': 1234, + 'other': 39494}, + ``` + """ + + total: int = 0 + publication: int = 0 + software: int = 0 + dataset: int = 0 + other: int = 0 + + +class CountAuthor(BaseModel): + """Represents a count of the authors or countries""" + total: int = 0 + + +class Pagination(BaseModel): + """Used for lists""" + + count: CountAuthor + skip: int + limit: int + + +class MetaPublication(BaseModel): + """ + Base data model representing an academic author or contributor + with their associated metadata. + + ```json + "count": {'total': 245684392, + 'publication': 1234, + 'software': 5678, + 'dataset': 1234, + 'other': 39494} + ``` + + """ + count: CountPublication + skip: int + limit: int + result_type: str + + +class MetaAuthor(BaseModel): + """ + Base data model representing an academic author or contributor + with their associated metadata. + + ```json + "count": {'total': 42} + ``` + + """ + count: CountAuthor | None + skip: int + limit: int diff --git a/app/schemas/output.py b/app/schemas/output.py new file mode 100644 index 0000000..b617a52 --- /dev/null +++ b/app/schemas/output.py @@ -0,0 +1,85 @@ +from pydantic import BaseModel, Field, HttpUrl +from typing import List, Optional +from uuid import UUID +from . import AuthorBase +from . import CountryBaseModel +from .meta import MetaPublication +from .topic import TopicBaseModel + + +class DateTimeComponents(BaseModel): + _Date__ordinal: Optional[int] = None + _Date__year: Optional[int] = None + _Date__month: Optional[int] = None + _Date__day: Optional[int] = None + + +class TimeComponents(BaseModel): + _Time__ticks: Optional[int] = None + _Time__hour: Optional[int] = None + _Time__minute: Optional[int] = None + _Time__second: Optional[int] = None + _Time__nanosecond: Optional[int] = None + _Time__tzinfo: Optional[str] = None + + +class CitedByDateTime(BaseModel): + _DateTime__date: Optional[DateTimeComponents] = None + _DateTime__time: Optional[TimeComponents] = None + + +class OutputModel(BaseModel): + """Schema for an Output + + An output should look like this: + + ```json + { + "uuid": "f05b1fc5-f831-4755-966f-06de074ab51c", # required + "doi": "http://doi.org/10.5281/zenodo.7015450", # required + "title": 'An example title', # required + "abstract": 'A long abstract', # optional + "journal": "Applied JSON", # optional + "cited_by_count_date": '2024-03-01', # optional + "cited_by_count": 34, # optional + "openalex": "https://openalex.org/W4393416420", # optional + "publication_day": 03, # optional + "publication_month": 12, # optional + "publication_year": 2023, # optional + "publisher": "Elsevier", # optional + "result_type": 'publication', # required + "countries": ['KEN', 'BEN'], # optional + "author": [ + { + "uuid": "c613b25b-967c-4586-9101-ece3a901fd9c", + "first_name": "Will", + "last_name": "Usher", + "orcid": "https://orcid.org/0000-0001-9367-1791", + } + ] # required + } + ``` + """ + uuid: UUID + doi: str = Field(pattern=r"^10\.\d{4,9}/[-._;()/:a-zA-Z0-9]+$") + title: str + result_type: str + authors: List[AuthorBase] + abstract: Optional[str] = None + journal: Optional[str] = None # Only academic publications have a journal + cited_by_count_date: Optional[CitedByDateTime] = None + cited_by_count: Optional[int] = None + publication_day: Optional[int] = None + publication_month: Optional[int] = None + publication_year: Optional[int] = None + publisher: Optional[str] = None + countries: Optional[List[CountryBaseModel]] = Field(default_factory=list) + topics: Optional[List[TopicBaseModel]] = Field(default_factory=list) + + +class OutputListModel(BaseModel): + """Represents a list of outputs including metadata + + """ + meta: MetaPublication + results: List[OutputModel] diff --git a/app/schemas/query.py b/app/schemas/query.py new file mode 100644 index 0000000..5745a4c --- /dev/null +++ b/app/schemas/query.py @@ -0,0 +1,28 @@ +from typing import Literal, List + +from pydantic import BaseModel, Field, field_validator +from fastapi import HTTPException + + +class FilterBase(BaseModel): + skip: int = Field(default=0, ge=0, title="Skip", description="Number of records to skip") + limit: int = Field(default=20, ge=1, title="Limit", description="Number of records to return") + + +class FilterParams(FilterBase): + result_type: Literal["publication", "software", "dataset", "other"] = "publication" + + +class FilterCountry(FilterParams): + country: str | None = Field(default=None, examples=['KEN'], pattern="^([A-Z]{3})$") + + +class FilterOutputList(FilterCountry): + pass + +class FilterWorkstream(FilterBase): + workstream: List[str] | None = Field(default=None) + + +class FilterAuthorDetail(FilterCountry): + pass \ No newline at end of file diff --git a/app/schemas/topic.py b/app/schemas/topic.py new file mode 100644 index 0000000..4e24a3c --- /dev/null +++ b/app/schemas/topic.py @@ -0,0 +1,36 @@ +"""The topics schema mimics the OpenAlex implementation of topics + +See the OpenAlex documentation: https://docs.openalex.org/api-entities/topics/topic-object + +""" +from typing import Dict, List +from pydantic import BaseModel, HttpUrl +from uuid import UUID + + +class DomainModel(BaseModel): + id: int + display_name: str + + +class FieldModel(BaseModel): + id: int + display_name: str + + +class TopicBaseModel(BaseModel): + """The topics schema mimics the OpenAlex implementation of topics + + See the OpenAlex documentation: + https://docs.openalex.org/api-entities/topics/topic-object + + """ + id: UUID + openalex_id: HttpUrl + description: str + display_name: str + domain: DomainModel + field: FieldModel + subfield: FieldModel + ids: Dict[str, HttpUrl] + keywords: List[str] diff --git a/app/schemas/workstream.py b/app/schemas/workstream.py new file mode 100644 index 0000000..49c5ad2 --- /dev/null +++ b/app/schemas/workstream.py @@ -0,0 +1,14 @@ +from pydantic import BaseModel +from .output import OutputListModel +from .meta import MetaAuthor +from .author import AuthorListModel +from . import WorkstreamBase + + +class WorkstreamDetailModel(WorkstreamBase): + members: AuthorListModel + + +class WorkstreamListModel(BaseModel): + meta: MetaAuthor + results: list[WorkstreamBase] diff --git a/app/static/js/map.js b/app/static/js/map.js index d3580d2..794b379 100644 --- a/app/static/js/map.js +++ b/app/static/js/map.js @@ -18,30 +18,30 @@ function draw_map(country) { var fill - if (partner_countries.indexOf(country.c.id) >= 0) { + if (partner_countries.indexOf(country.id) >= 0) { fill = PARTNER_COLOUR - } else if (affiliated_countries.indexOf(country.c.id) >= 0) { + } else if (affiliated_countries.indexOf(country.id) >= 0) { fill = AFFILIATE_COLOUR - } else if ((demonstrator_countries.indexOf(country.c.id) >= 0)) { + } else if ((demonstrator_countries.indexOf(country.id) >= 0)) { fill = DEMONSTRATOR_COLOUR } else { fill = HIGHLIGHT_COLOUR } // Above should be replaced with information drawn from the backend database - const id = "#" + country.c.id; + const id = "#" + country.id; const svg_map = d3.select(id), map_width = +svg_map.attr("width"), map_height = +svg_map.attr("height"); - const lat = country.c.latitude; - const lon = country.c.longitude; + const lat = country.latitude; + const lon = country.longitude; // Load external data and boot d3.json("https://raw.githubusercontent.com/holtzy/D3-graph-gallery/master/DATA/world.geojson").then( function(data){ // Filter data - country = data.features.filter(d => {console.log(d.properties.name); return d.properties.name==country.c.name}) + country = data.features.filter(d => {return d.properties.name==country.name}) // Initial Map and projection const initial_projection = d3.geoMercator() diff --git a/app/static/js/world.js b/app/static/js/world.js index a23581c..7538d2e 100644 --- a/app/static/js/world.js +++ b/app/static/js/world.js @@ -27,8 +27,6 @@ const projection = d3.geoNaturalEarth1() .scale(width / 1.1 / Math.PI) .translate([width / 2, height / 2]) -const countries = country_data; - // Load external data and draw base map d3.json("https://raw.githubusercontent.com/holtzy/D3-graph-gallery/master/DATA/world.geojson").then( function(data) { @@ -55,7 +53,7 @@ d3.json("https://raw.githubusercontent.com/holtzy/D3-graph-gallery/master/DATA/w } // Add interactions for CCG output available countries - let flat_countries = d3.map(countries, d => d.c.id); + let flat_countries = d3.map(countries, d => d.id); addCountryInteractions(flat_countries, HIGHLIGHT_COLOUR); // Add interactions for partner countries diff --git a/app/templates/author.html b/app/templates/author.html index 006dd25..c0b4959 100644 --- a/app/templates/author.html +++ b/app/templates/author.html @@ -5,11 +5,11 @@
-

{{ author.first_name }} {{ author.last_name }} - {% if author.orcid %} - - ORCID logo - +

{{ first_name }} {{ last_name }} + {% if orcid %} + + ORCID logo + {% endif %}

@@ -21,19 +21,17 @@

{{ author.first_name }} {{ author.last_name }}
- {% if author.affiliations %} + {% if affiliations %}
Affiliations
- {% for id, value in author.affiliations|dictsort %} - {{ value }} + {% for affiliation in affiliations %} + {{ affiliation.name }} {% endfor %} {% endif %}
- {% if author.workstreams %} + {% if workstreams %}
Workstreams
- {% for id, value in author.workstreams|dictsort %} - {{ value }} - {% endfor %} + {% include 'workstream_list.html' %} {% endif %}
@@ -41,17 +39,26 @@

Workstreams
- {% if author.collaborators %} + {% if collaborators %}
-
Collaborators
+
Top Collaborators for {{ outputs.meta.result_type|title }}s
    - {% for colab in author.collaborators %} + {% for colab in collaborators %}
  • {{ colab.first_name}} {{ colab.last_name }} - {% if colab.orcid %} + {% if colab.orcid %} - ORCID logo - {% endif %}
    • + ORCID logo {% endif %} + {% if colab.affiliations %} + {% for value in colab.affiliations %} + {{ value.name }} + {% endfor %} + {% endif %} + {% if colab.workstreams %} + {% for value in colab.workstreams %} + {{ value.name }} + {% endfor %} + {% endif %} {% endfor %}
@@ -60,8 +67,10 @@
Collaborators
- {% set outputs = author.outputs %} - {% include "output_list.j2" %} + {% set results = outputs.results %} + {% set meta = outputs.meta %} + {% include 'output_list.j2' %} +
diff --git a/app/templates/author_list.html b/app/templates/author_list.html new file mode 100644 index 0000000..fedfcee --- /dev/null +++ b/app/templates/author_list.html @@ -0,0 +1,21 @@ +{% for author in results %} +
+
+ {{ author.first_name}} {{ author.last_name }} + {% if author.orcid %} + + ORCID logo + {% endif %} + {% if author.affiliations %} + {% for value in author.affiliations %} + {{ value.name }} + {% endfor %} + {% endif %} + {% if author.workstreams %} + {% set workstreams = author.workstreams %} + {% include 'workstream_list.html' %} + + {% endif %} +
+
+{% endfor %} \ No newline at end of file diff --git a/app/templates/authors.html b/app/templates/authors.html index 10de8ad..1ac396e 100644 --- a/app/templates/authors.html +++ b/app/templates/authors.html @@ -1,26 +1,9 @@ {% extends "base.html" %} {% block content %} -{% for author in authors %} -
-
- {{ author.first_name}} {{ author.last_name }} - {% if author.orcid %} - - ORCID logo - {% endif %} - {% if author.affiliation %} - {% for id, value in author.affiliation|dictsort %} - {{ value }} - {% endfor %} - {% endif %} - {% if author.workstreams %} - {% for id, value in author.workstreams|dictsort %} - {{ value }} - {% endfor %} - {% endif %} -
-
- {% endfor %} + +{% include 'author_list.html' %} + +{% include 'pagination.html' %} {% endblock %} \ No newline at end of file diff --git a/app/templates/base.html b/app/templates/base.html index e387dd4..ed34a22 100644 --- a/app/templates/base.html +++ b/app/templates/base.html @@ -15,7 +15,7 @@ Climate Compatible Growth Research Index - + - - -
- Climate Compatible Growth Research Index @@ -43,27 +35,14 @@
  • Authors
  • Outputs
  • Countries
    • +
  • Workstreams
    • - -
    - {% with messages = get_flashed_messages() %} - {% if messages %} - {% for message in messages %} - - {% endfor %} - {% endif %} - {% endwith %} {% block content %}{% endblock %}
    diff --git a/app/templates/country.html b/app/templates/country.html index ace5a80..7f65de7 100644 --- a/app/templates/country.html +++ b/app/templates/country.html @@ -1,8 +1,13 @@ {% extends "base.html" %} {% block content %} -

    {{ country.name }}

    - {% include 'output_list.j2' %} +
    + +

    {{ name }}

    + +{% include 'output_list.j2' %} + +
    {% endblock %} \ No newline at end of file diff --git a/app/templates/country_list.html b/app/templates/country_list.html index ea4ca42..a1ed7f8 100644 --- a/app/templates/country_list.html +++ b/app/templates/country_list.html @@ -1,6 +1,9 @@ {% extends "base.html" %} {% block content %} + + +

    CCG works in the following countries

    @@ -8,22 +11,25 @@

    CCG works in the following countries

    - {% for country in countries %} + {% for country in results %}
    -
    - +
    +
    -
    {{country.c.name}}
    +
    {{country.name}}
    - View Outputs + View Outputs
    {% endfor %} - +
    + +{% include 'pagination.html' %} + {% endblock %} \ No newline at end of file diff --git a/app/templates/index.html b/app/templates/index.html index 7280da3..40fea8a 100644 --- a/app/templates/index.html +++ b/app/templates/index.html @@ -1,7 +1,6 @@ {% extends "base.html" %} {% block content %} - + +

    CCG works in the following countries

    -
    diff --git a/app/templates/output.html b/app/templates/output.html index f556bb4..b279a5d 100644 --- a/app/templates/output.html +++ b/app/templates/output.html @@ -8,20 +8,20 @@
    - {% if output.result_type %} - {{ output.result_type }} + {% if result_type %} + {{ result_type }} {% endif %} -

    {{ output.title }}

    +

    {{ title }}

    - {%if output.countries %} + {%if countries %}
    Countries
    - {% for country in output.countries %} + {% for country in countries %} {{ country.name }} {% endfor %}
    @@ -32,9 +32,9 @@
    Countries
    Metadata
    - {% if output.publication_year %}Publication Year: {{ output.publication_year }}{% endif %} - {% if output.journal%}

    Journal: {{ output.journal }}

    {% endif %} - http://doi.org/{{ output.doi }} + {% if publication_year %}Publication Year: {{ publication_year }}{% endif %} + {% if journal%}

    Journal: {{ journal }}

    {% endif %} + http://doi.org/{{ doi }}
    @@ -44,8 +44,8 @@
    Metadata
    Metrics
    - - + +
    @@ -55,10 +55,10 @@
    Metrics
    Authors
    - {% for author in output.authors %} + {% for author in authors %} {{ author.first_name}} {{ author.last_name }} {% if author.orcid %} - logo + logo {% endif %} {% endfor %} @@ -72,7 +72,7 @@
    Authors
    Abstract
    - {{ output.abstract }} + {{ abstract }}
    diff --git a/app/templates/output_list.j2 b/app/templates/output_list.j2 index 7e6fdea..2c6d278 100644 --- a/app/templates/output_list.j2 +++ b/app/templates/output_list.j2 @@ -1,53 +1,43 @@ - -
    Filter by result type:
    - {% if outputs %} - {% for output in outputs %} -
    + {% if results %} + {% for output in results %} +
    -
    {{ output.outputs.title }} +
    {{ output.title }} {% if output.countries %} {% for country in output.countries %} {{ country.name }} {% endfor %} {% endif %} - {% if output.outputs.result_type %} - {{ output.outputs.result_type }} + {% if output.result_type %} + {{ output.result_type }} {% endif %}
    {% for author in output.authors %} {{ author.first_name}} {{ author.last_name }} {% if author.orcid %} - ORCID logo + + ORCID logo + {% endif %} {% endfor %} - {% if output.outputs.publication_year %}({{ output.outputs.publication_year }}){% endif %} - {% if output.outputs.journal %}{{ output.outputs.journal }}{% endif %} + {% if output.publication_year %}({{ output.publication_year }}){% endif %} + {% if output.journal %}{{ output.journal }}{% endif %}
    - View Record - View at Publisher + View Record + View at Publisher
    @@ -59,4 +49,34 @@
    {% endif %} +
    + +{% set result_type = meta.result_type %} +{% set limit = meta.limit %} +{% set skip = meta.skip %} +{% set count = meta.count %} + +
    +
    \ No newline at end of file diff --git a/app/templates/outputs.html b/app/templates/outputs.html index 55addd1..b98bf95 100644 --- a/app/templates/outputs.html +++ b/app/templates/outputs.html @@ -2,11 +2,10 @@ {% block content %} - - - -
    +
    {% include 'output_list.j2' %} +
    + {% endblock %} \ No newline at end of file diff --git a/app/templates/pagination.html b/app/templates/pagination.html new file mode 100644 index 0000000..094b81a --- /dev/null +++ b/app/templates/pagination.html @@ -0,0 +1,24 @@ +
    + +
    \ No newline at end of file diff --git a/app/templates/workstream.html b/app/templates/workstream.html new file mode 100644 index 0000000..3ae1e0a --- /dev/null +++ b/app/templates/workstream.html @@ -0,0 +1,13 @@ +{% block content %} + +
    +
    +
    +

    {{ name }}

    +
    +
    + {% set results = members.results %} + {% include 'author_list.html' %} +
    + +{% endblock %} \ No newline at end of file diff --git a/app/templates/workstream_list.html b/app/templates/workstream_list.html new file mode 100644 index 0000000..d0bfdb7 --- /dev/null +++ b/app/templates/workstream_list.html @@ -0,0 +1,3 @@ +{% for workstream in workstreams %} +{{ workstream.name }} +{% endfor %} \ No newline at end of file diff --git a/app/templates/workstreams.html b/app/templates/workstreams.html new file mode 100644 index 0000000..cd2154a --- /dev/null +++ b/app/templates/workstreams.html @@ -0,0 +1,18 @@ +{% extends "base.html" %} + +{% block content %} + +
    +
    +
    +

    Apply Filter

    + {% set workstreams = results %} + {% include 'workstream_list.html' %} +
    +
    + {% include 'workstream.html' %} +
    +
    +
    + + {% endblock %} diff --git a/app/test_html.py b/app/test_html.py new file mode 100644 index 0000000..4a36928 --- /dev/null +++ b/app/test_html.py @@ -0,0 +1,139 @@ +from fastapi.testclient import TestClient + +from app.main import app + +client = TestClient(app) + + +class TestOutput: + + def test_output_list(self): + response = client.get("/outputs") + print("response=" + str(response.text)) + assert response.status_code == 200 + + def test_output_list_limit(self): + response = client.get("/outputs?limit=1") + assert response.status_code == 200 + + def test_output_list_limit_skip(self): + response = client.get("/outputs?limit=1&skip=1") + assert response.status_code == 200 + + def test_output_list_limit_skip_result_type(self): + response = client.get("/outputs?limit=1&skip=1&result_type=publication") + assert response.status_code == 200 + + def test_output_list_limit_skip_result_type_wrong(self): + response = client.get("/outputs?limit=1&skip=1&result_type=notallowed") + assert response.status_code == 422 + + def test_output_error_id_wrong_format(self): + response = client.get("/outputs/blabla") + assert response.status_code == 422 + + def test_output_error_not_exist(self): + response = client.get("/outputs/97c945d6-e172-4e7f-8a3a-02a7a51ae62b") + assert response.status_code == 404 + + + +class TestAuthor: + + def test_author_list(self): + response = client.get("/authors") + assert response.status_code == 200 + + def test_author_not_exist(self): + response = client.get("/authors/8119878e-1875-4cdf-a6a8-a9025057ddc6") + assert response.status_code == 404 + + def test_author_list(self): + response = client.get("/authors") + assert response.status_code == 200 + + def test_author_list_limit(self): + response = client.get("/authors?limit=1") + assert response.status_code == 200 + + def test_author_list_limit_skip(self): + response = client.get("/authors?limit=1&skip=20") + assert response.status_code == 200 + + def test_author_list_skip_negative(self): + response = client.get("/authors?skip=-1") + assert response.status_code == 422 + + def test_author_list_limit_zero(self): + response = client.get("/authors?limit=0") + assert response.status_code == 422 + + def test_author_error_on_author_id_wrong_format(self): + response = client.get("/authors/blabla") + assert response.status_code == 422 + + def test_author_error_on_author_id_not_exist(self): + response = client.get("/authors/97c945d6-e172-4e7f-8a3a-02a7a51ae62b") + assert response.status_code == 404 + + def test_author_workstream(self): + response = client.get("/authors?workstream=ws7a") + assert response.status_code == 200 + + def test_author_multiple_workstream(self): + response = client.get("/authors?workstream=ws7a&workstream=ws5a") + assert response.status_code == 200 + + +class TestCountry: + + def test_country_list(self): + response = client.get("/countries") + assert response.status_code == 200 + + def test_country_list_skip(self): + response = client.get("/countries?skip=1") + assert response.status_code == 200 + + def test_country_list_limit(self): + response = client.get("/countries?limit=1") + assert response.status_code == 200 + + def test_country_list_limit_illegal(self): + response = client.get("/countries?limit=0") + assert response.status_code == 422 + + def test_country_raise_warning_on_bad_id(self): + response = client.get("/countries/blabla") + assert response.status_code == 422 + + def test_country_detail_skip(self): + response = client.get("/countries/KEN?skip=1") + assert response.status_code == 200 + + def test_country_detail_limit(self): + response = client.get("/countries/KEN?limit=1") + assert response.status_code == 200 + + def test_country_detail_limit_illegal(self): + response = client.get("/countries/KEN?limit=0") + assert response.status_code == 422 + + def test_country_error_on_not_exist(self): + """Meets + """ + response = client.get("/countries/XXX") + assert response.status_code == 404 + + +class TestWorkstream: + + def test_workstream_list(self): + response = client.get("/workstreams") + assert response.status_code == 200 + + def test_workstream_error_on_not_exist(self): + """Meets + """ + response = client.get("/workstreams/XXX") + assert response.status_code == 404 diff --git a/app/test_main.py b/app/test_main.py new file mode 100644 index 0000000..048f240 --- /dev/null +++ b/app/test_main.py @@ -0,0 +1,145 @@ +from fastapi.testclient import TestClient + +from app.main import app + +client = TestClient(app) + + +class TestOutput: + def test_output_list(self): + response = client.get("/api/outputs") + print("response=" + str(response.text)) + assert response.status_code == 200 + + def test_output_list_limit(self): + response = client.get("/api/outputs?limit=1") + assert response.status_code == 200 + + def test_output_list_limit_skip(self): + response = client.get("/api/outputs?limit=1&skip=1") + assert response.status_code == 200 + + def test_output_list_limit_skip_result_type(self): + response = client.get("/api/outputs?limit=1&skip=1&result_type=publication") + assert response.status_code == 200 + + def test_output_list_limit_skip_result_type_wrong(self): + response = client.get("/api/outputs?limit=1&skip=1&result_type=notallowed") + assert response.status_code == 422 + + def test_output_error_id_wrong_format(self): + response = client.get("/api/outputs/blabla") + assert response.status_code == 422 + + def test_output_error_not_exist(self): + response = client.get("/api/outputs/97c945d6-e172-4e7f-8a3a-02a7a51ae62b") + assert response.status_code == 404 + + +class TestAuthor: + + def test_author_list(self): + response = client.get("/api/authors") + assert response.status_code == 200 + + def test_author_list_limit(self): + response = client.get("/api/authors?limit=1") + assert response.status_code == 200 + + def test_author_list_limit_skip(self): + response = client.get("/api/authors?limit=1&skip=20") + assert response.status_code == 200 + + def test_author_list_skip_negative(self): + response = client.get("/api/authors?skip=-1") + assert response.status_code == 422 + + def test_author_list_limit_zero(self): + response = client.get("/api/authors?limit=0") + assert response.status_code == 422 + + def test_author_error_on_author_id_wrong_format(self): + response = client.get("/api/authors/blabla") + assert response.status_code == 422 + + def test_author_error_on_author_id_not_exist(self): + response = client.get("/api/authors/97c945d6-e172-4e7f-8a3a-02a7a51ae62b") + assert response.status_code == 404 + + def test_author_workstream(self): + response = client.get("/api/authors?workstream=ws7a") + assert response.status_code == 200 + + def test_author_multiple_workstream(self): + response = client.get("/api/authors?workstream=ws7a&workstream=ws5a") + assert response.status_code == 200 + + +class TestCountry: + + def test_country_list(self): + response = client.get("/api/countries") + assert response.status_code == 200 + + def test_country_list_skip(self): + response = client.get("/api/countries?skip=1") + assert response.status_code == 200 + + def test_country_list_limit(self): + response = client.get("/api/countries?limit=1") + assert response.status_code == 200 + + def test_country_list_limit_illegal(self): + response = client.get("/api/countries?limit=0") + assert response.status_code == 422 + + def test_country_raise_warning_on_bad_id(self): + response = client.get("/api/countries/blabla") + assert response.status_code == 422 + + def test_country_detail_skip(self): + response = client.get("/api/countries/KEN?skip=1") + assert response.status_code == 200 + + def test_country_detail_limit(self): + response = client.get("/api/countries/KEN?limit=1") + assert response.status_code == 200 + + def test_country_detail_limit_illegal(self): + response = client.get("/api/countries/KEN?limit=0") + assert response.status_code == 422 + + def test_country_error_on_not_exist(self): + """Meets + """ + response = client.get("/api/countries/XXX") + assert response.status_code == 404 + + +class TestWorkstream: + + def test_workstream_list(self): + response = client.get("/api/workstreams") + assert response.status_code == 200 + + def test_workstream_list_limit(self): + response = client.get("/api/workstreams?limit=1") + assert response.status_code == 200 + + def test_workstream_list_limit_illegal(self): + response = client.get("/api/workstreams?limit=0") + assert response.status_code == 422 + + def test_workstream_list_skip(self): + response = client.get("/api/workstreams?skip=1") + assert response.status_code == 200 + + def test_workstream_list_skip_illegal(self): + response = client.get("/api/workstreams?skip=-1") + assert response.status_code == 422 + + def test_workstream_error_on_not_exist(self): + """Meets + """ + response = client.get("/api/workstreams/XXX") + assert response.status_code == 404 diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 0000000..0089281 --- /dev/null +++ b/compose.yaml @@ -0,0 +1,45 @@ +services: + research-index: + build: . + ports: + - 8000:8000 + volumes: + - ri_log:/var/log/research_index.log + environment: + MG_HOST: memgraph + MG_PORT: '7687' + LOG_LEVEL: "DEBUG" + depends_on: + - memgraph + restart: unless-stopped + develop: + watch: + - action: sync+restart + path: . + target: . + + memgraph: + image: memgraph/memgraph:latest + ports: + - 7687:7687 + - 7444:7444 + volumes: + - mg_lib:/var/lib/memgraph + - mg_log:/var/log/memgraph + command: ["--log-level=TRACE"] + healthcheck: + test: ["CMD-SHELL", "echo 'RETURN 0;' | mgconsole || exit 1"] + interval: 30s + timeout: 10s + retries: 3 + deploy: + resources: + limits: + memory: 4G + restart: unless-stopped + +volumes: + mg_lib: + mg_log: + mg_etc: + ri_log: \ No newline at end of file diff --git a/nginx/nginx.conf b/nginx/nginx.conf new file mode 100644 index 0000000..e69de29 diff --git a/requirements.txt b/requirements.txt index 5d7895d..ff35458 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,42 @@ -flask +annotated-types +anyio +blinker +certifi +click +dnspython +email_validator +exceptiongroup +fastapi +fastapi-cli +Flask +h11 +httpcore +httptools +httpx +idna +itsdangerous +Jinja2 +markdown-it-py +MarkupSafe +mdurl neo4j +packaging +pydantic +pydantic_core +Pygments +python-dotenv +python-multipart +pytz +PyYAML +rich +shellingham +sniffio +starlette +typer +typing_extensions +uvicorn gunicorn +uvloop +watchfiles +websockets +Werkzeug