Merge pull request #21 from OKN-CollabNext/implement-search

Super simple search feature

.github/workflows/deploy.yaml

@@ -29,6 +29,8 @@ jobs:
       - run: poetry install
       - run: yarn install --frozen-lockfile
       - run: poetry run yarn build
+        env:
+          BUILD_ENV: production
       - name: Deploy to Observable Cloud
         # This parameter to `--message` will use the latest commit message
         run: yarn deploy -- --message "$(git log -1 --pretty=%s)"

collabnext/openalex/institutions.py

@@ -1,20 +1,40 @@
-from pyalex import Institution, Institutions
 import json
+import os
 import sys
 
+from pyalex import Institution, Institutions
+
+BUILD_ENV = os.getenv("BUILD_ENV") or "development"
+
 
-def get_institutions(institutions_file_path: str = "docs/data/institutions.json") -> list[Institution]:
+def get_institutions() -> list[Institution]:
     institutions = []
-
+
+    institutions_file_path = (
+        "docs/data/institutions.json"
+        if BUILD_ENV == "production"
+        else "docs/data/institutions-dev.json"
+    )
+
     # Load institutions from JSON file
-    try:        
-        institutions = json.load(open(institutions_file_path))        
+    try:
+        institutions = json.load(open(institutions_file_path))
     except Exception as e:
-        print("\nError loading institutions from JSON file", institutions_file_path, ":", e, "\n", file=sys.stderr)
+        print(
+            "\nError loading institutions from JSON file",
+            institutions_file_path,
+            ":",
+            e,
+            "\n",
+            file=sys.stderr,
+        )
 
     # Get 5 random institutions in case of error
     if institutions is None or len(institutions) == 0:
-        print("No institutions found in JSON file, fetching random institutions\n", file=sys.stderr)
+        print(
+            "No institutions found in JSON file, fetching random institutions\n",
+            file=sys.stderr,
+        )
         institutions = [Institutions().random() for _ in range(5)]
-    
+
     return institutions

collabnext/openalex/utils.py

@@ -1,11 +1,18 @@
-def clamp_author_nodes_to_edges(
-    author_nodes: list[dict], edges: list[dict]
-) -> list[dict]:
-    author_ids = {x["start"] for x in edges}
-    return [x for x in author_nodes if x["id"] in author_ids]
-
-def clamp_author_edges_to_nodes(
-    author_edges: list[dict], nodes: list[dict]
-) -> list[dict]:
-    author_ids = {x["id"] for x in nodes}
-    return [x for x in author_edges if x["start"] in author_ids]
+def clamp_nodes_to_edge_start(nodes: list[dict], edges: list[dict]) -> list[dict]:
+    edge_ids = {x["start"] for x in edges}
+    return [x for x in nodes if x["id"] in edge_ids]
+
+
+def clamp_nodes_to_edge_end(nodes: list[dict], edges: list[dict]) -> list[dict]:
+    edge_ids = {x["end"] for x in edges}
+    return [x for x in nodes if x["id"] in edge_ids]
+
+
+def clamp_edge_start_to_nodes(edges: list[dict], nodes: list[dict]) -> list[dict]:
+    node_ids = {x["id"] for x in nodes}
+    return [x for x in edges if x["start"] in node_ids]
+
+
+def clamp_edge_end_to_nodes(edges: list[dict], nodes: list[dict]) -> list[dict]:
+    node_ids = {x["id"] for x in nodes}
+    return [x for x in edges if x["end"] in node_ids]

observable/docs/data/graph.sqlite.py

@@ -22,8 +22,9 @@
 )
 from collabnext.openalex.topics import get_work_topics
 from collabnext.openalex.utils import (
-    clamp_author_edges_to_nodes,
-    clamp_author_nodes_to_edges,
+    clamp_edge_start_to_nodes,
+    clamp_nodes_to_edge_end,
+    clamp_nodes_to_edge_start,
 )
 from collabnext.openalex.works import (
     clamp_works_to_institutions,
@@ -62,32 +63,54 @@
 # Create author -> instutition edges
 author_institution_edges = make_author_institution_edges(authors, institutions)
 
+
 # Create work nodes
 work_nodes = make_work_nodes(works)
 
 # Create author -> work edges
 author_work_edges = make_author_work_edges(authors, works)
 
+
 # Clamp author nodes to those with works
-author_nodes = clamp_author_nodes_to_edges(author_nodes, author_work_edges)
+author_nodes = clamp_nodes_to_edge_start(author_nodes, author_work_edges)
+
+# Clamp work nodes to those with authors
+work_nodes = clamp_nodes_to_edge_end(work_nodes, author_work_edges)
+
+# Clamp author institution edges to those with work
+author_institution_edges = clamp_edge_start_to_nodes(
+    author_institution_edges, author_nodes
+)
+
 
 # Create topic nodes
 topic_nodes = make_topic_nodes(topics)
 
 # Create work-topic edges
 work_topic_edges = make_work_topic_edges(works, topics)
 
+# Clamp work nodes to those with topics
+work_nodes = clamp_nodes_to_edge_start(work_nodes, work_topic_edges)
+
+# Clamp topic nodes to those with works
+topic_nodes = clamp_nodes_to_edge_end(topic_nodes, work_topic_edges)
+
+
 # Infer author-topic edges
 author_topic_edges = infer_author_topic_edges(author_work_edges, work_topic_edges)
 
-# Clamp author nodes to author topic edges
-author_nodes = clamp_author_nodes_to_edges(author_nodes, author_topic_edges)
+# Clamp author nodes to those with topics
+author_nodes = clamp_nodes_to_edge_start(author_nodes, author_topic_edges)
+
+# Clamp topic nodes to those with authors
+topic_nodes = clamp_nodes_to_edge_end(topic_nodes, author_topic_edges)
 
-# Clamp author -> institution edges to those with topics
-author_institution_edges = clamp_author_edges_to_nodes(
+# Clamp author institution edges to those with topics
+author_institution_edges = clamp_edge_start_to_nodes(
     author_institution_edges, author_nodes
 )
 
+
 # Group all nodes and edges together
 nodes = [*institution_nodes, *author_nodes, *work_nodes, *topic_nodes]
 edges = [

observable/docs/index.md

@@ -1,7 +1,6 @@
 ---
 tilte: CollabNext Challenge
 style: ./styles.css
-
 ---
 
 # CollabNext Challenge
@@ -12,9 +11,8 @@ style: ./styles.css
   <p>With a sample of 5 HBCUs as our example, our app provides and interface for the user to explore a visual interactive representation of data from OpenAlex. Our app represents a scalable starting point towards addressing the broader systemic issue of diversity and inclusion in research data.</p>
 </div>
 
-
 ```js
-const query = view(Inputs.text({placeholder: "Search"}));
+const query = view(Inputs.text({ placeholder: "Search" }));
 ```
 
 ```js
@@ -23,14 +21,44 @@ const db = FileAttachment("data/graph.sqlite").sqlite();
 ```
 
 ```js
-const nodes = db.query(
+const institutions = db.query(
+  `
+  SELECT
+    *
+  FROM
+    nodes
+  WHERE
+    nodes.type = 'INSTITUTION'
+  `
+);
+const authors = db.query(
+  query
+    ? `
+  SELECT
+    *
+  FROM
+    nodes
+  WHERE
+    nodes.type = 'AUTHOR'
+    AND nodes.name like '%${query}%'
+  `
+    : `
+  SELECT
+    *
+  FROM
+    nodes
+  WHERE
+    nodes.type = 'AUTHOR'
+  `
+);
+const topics = db.query(
   `
   SELECT
     *
   FROM
     nodes
   WHERE
-    nodes.type <> 'WORK'
+    nodes.type = 'TOPIC'
   `
 );
 ```
@@ -123,62 +151,61 @@ orb.data.setDefaultStyle({
   },
 });
 
-const loaderOverlay = document.getElementById('loader-overlay');
-const graphContainer = document.getElementById('graph');
-const details = document.querySelector('.details');
+const loaderOverlay = document.getElementById("loader-overlay");
+const graphContainer = document.getElementById("graph");
+const details = document.querySelector(".details");
 
 // Show loader overlay
-loaderOverlay.style.display = 'flex';
+loaderOverlay.style.display = "flex";
 
 // Initialize nodes and edges
-orb.data.setup({ nodes, edges });
+orb.data.setup({ nodes: [...institutions, ...topics, ...authors], edges });
 
 // Render and recenter the view
 orb.view.render(() => {
-  loaderOverlay.style.display = 'none';
-  details.style.display = 'block';
+  loaderOverlay.style.display = "none";
+  details.style.display = "block";
   orb.view.recenter();
 });
 ```
 
 ```js
 let selectedNode;
 
-orb.events.on('node-click', (event) => {
-  getData(event)
+orb.events.on("node-click", (event) => {
+  getData(event);
 });
 
-
 function getData(event) {
   selectedNode = event.node.data;
-  updateDetails(selectedNode)
+  updateDetails(selectedNode);
 }
 
-const details = document.querySelector('.details')
+const details = document.querySelector(".details");
 
 function updateDetails(selectedNode) {
-  details.innerHTML = '';
-  let html = '';
+  details.innerHTML = "";
+  let html = "";
 
   if (selectedNode) {
-    details.style.display = 'block'
+    details.style.display = "block";
     html += `<h2>${selectedNode.label}</h2>`;
 
-    if (selectedNode.type === 'INSTITUTION') {
+    if (selectedNode.type === "INSTITUTION") {
       html += `<p><b>Homepage:</b> <a href="${selectedNode.homepage}">${selectedNode.homepage}</a></p>`;
       html += `<p><b>Works:</b> ${selectedNode.works_count}</p>`;
       html += `<p><b>Cited by:</b> ${selectedNode.cited_by_count}</p>`;
       html += `<a href="${selectedNode.id}" target="_blank">View on OpenAlex</a>`;
-    } else if (selectedNode.type === 'AUTHOR') {
+    } else if (selectedNode.type === "AUTHOR") {
       html += `<p><b>Works:</b> ${selectedNode.works_count}</p>`;
       html += `<p><b>Cited by:</b> ${selectedNode.cited_by_count}</p>`;
       html += `<a href="${selectedNode.id}" target="_blank">View on OpenAlex</a>`;
-    } else if (selectedNode.type === 'TOPIC') {
+    } else if (selectedNode.type === "TOPIC") {
       html += `<p><b>Subfield:</b> ${selectedNode.subfield}</p>`;
       html += `<p><b>Domain:</b> ${selectedNode.domain}</p>`;
       html += `<a href="${selectedNode.id}" target="_blank">View on OpenAlex</a>`;
     }
-  } 
+  }
   details.innerHTML = html;
 }
 ```

tasks.py

@@ -58,6 +58,6 @@ def touch(c):
 
 
 @task
-def fetch(c):
+def fetch(c, count: int = 5):
     with cwd("."):
-        c.run("python scripts/fetch_custom_institutions.py hbcus 5")
+        c.run(f"python scripts/fetch_custom_institutions.py hbcus {count}")