Merge pull request #69 from EBISPOT/dev

update stable branch with plenty of updates from dev branch

.github/workflows/docker.yml

@@ -0,0 +1,38 @@
+name: Build & publish OXO images
+
+on:
+  push:
+    branches: [ "dev", "stable" ]
+
+env:
+  REGISTRY: ghcr.io
+  IMAGE_NAME: ${{ github.repository }}
+
+jobs:
+  build-and-push-image:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v2
+
+      - name: Log in to the Container registry
+        uses: docker/login-action@f054a8b539a109f9f41c372932f1ae047eff08c9
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Build and push OXO Docker image
+        run: |
+          docker build -t ghcr.io/ebispot/oxo-loader:${{ github.sha }} -f ./oxo-loader/Dockerfile .
+          docker build -t ghcr.io/ebispot/oxo-indexer:${{ github.sha }} -f ./oxo-indexer/Dockerfile .
+          docker build -t ghcr.io/ebispot/oxo-web:${{ github.sha }} -f ./oxo-web/Dockerfile .
+          docker tag ghcr.io/ebispot/oxo-loader:${{ github.sha }} ghcr.io/ebispot/oxo-loader:${{ github.ref_name }}
+          docker tag ghcr.io/ebispot/oxo-indexer:${{ github.sha }} ghcr.io/ebispot/oxo-indexer:${{ github.ref_name }}
+          docker tag ghcr.io/ebispot/oxo-web:${{ github.sha }} ghcr.io/ebispot/oxo-web:${{ github.ref_name }}
+          docker push --all-tags ghcr.io/ebispot/oxo-loader
+          docker push --all-tags ghcr.io/ebispot/oxo-indexer
+          docker push --all-tags ghcr.io/ebispot/oxo-web

README.md

@@ -11,7 +11,9 @@ OxO is comprised of three components:
 * The indexer (oxo-indexer/), which indexes terms and mappings found in neo4j in solr
 * The Web application (oxo-web/), which provides the user interface
 
-The preferred method of deployment for OxO is using Docker. First, create the necessary volumes:
+The preferred method of deployment for OxO is using Docker.  If you would like to deploy **the entire OntoTools stack** (OLS, OxO, and ZOOMA), check out the [OntoTools Docker Config](https://github.com/EBISPOT/ontotools-docker-config) repository. If you would like to deploy **OxO only**, read on.
+
+First, create the necessary volumes:
 
     docker volume create --name=oxo-neo4j-data
     docker volume create --name=oxo-neo4j-import
@@ -59,4 +61,16 @@ Then run the indexer:
 
 The Web application is a standard WAR and can be deployed using e.g. Tomcat.
 
+## Customisation
+
+It is possible to customise several branding options in `oxo-web/src/main/resources/application.properties`:
+
+* `oxo.customisation.debrand` — If set to true, removes the EBI header and footer, documentation, and about page
+* `oxo.customisation.title` — A custom title for your instance, e.g. "My OxO Instance"
+* `oxo.customisation.short-title` — A shorter version of the custom title, e.g. "MYOxO"
+* `oxo.customisation.description` — A description of the instance
+* `oxo.customisation.org` — The organisation hosting your instance
+
+
+
 

docker-compose.yml

@@ -7,9 +7,9 @@ services:
         ports:
           - 8983:8983
         volumes:
-          - oxo-solr-data:/mnt/solr-data
+          - oxo-solr-data:/var/solr
           - ./solr-config:/mnt/solr-config
-        command: ["solr", "-Dsolr.solr.home=/mnt/solr-config", "-Dsolr.data.dir=/mnt/solr-data", "-Dsolr.log=/mnt/solr-data/logs", "-f"]
+        command: ["-Dsolr.solr.home=/mnt/solr-config", "-Dsolr.data.dir=/var/solr", "-Dsolr.log=/var/solr/logs", "-f"]
     neo4j:
         image: neo4j:3.1.1
         environment:

oxo-indexer/Dockerfile

@@ -5,6 +5,11 @@ COPY pom.xml /opt/oxo/pom.xml
 COPY oxo-web /opt/oxo/oxo-web
 COPY oxo-model /opt/oxo/oxo-model
 COPY oxo-indexer /opt/oxo/oxo-indexer
+COPY lib /opt/oxo/lib
+RUN mvn install:install-file -DcreateChecksum=true -Dpackaging=jar -Dfile=/opt/oxo/lib/sping-security-orcid-stateless-0.0.8.jar -DgroupId=uk.ac.ebi.spot -DartifactId=sping-security-orcid-stateless -Dversion=0.0.8
+RUN mvn install:install-file -DcreateChecksum=true -Dpackaging=jar -Dfile=/opt/oxo/lib/spring-security-core-4.1.4.RELEASE.jar -DgroupId=org.springframework.security -DartifactId=spring-security-core -Dversion=4.1.4.RELEASE
+RUN mvn install:install-file -DcreateChecksum=true -Dpackaging=jar -Dfile=/opt/oxo/lib/spring-security-oauth2-2.0.13.RELEASE.jar -DgroupId=org.springframework.security.oauth -DartifactId=spring-security-oauth2 -Dversion=2.0.13.RELEASE
+RUN mvn install:install-file -DcreateChecksum=true -Dpackaging=jar -Dfile=/opt/oxo/lib/spring-data-jpa-1.10.8.RELEASE.jar -DgroupId=org.springframework.data -DartifactId=spring-data-jpa -Dversion=1.10.8.RELEASE
 RUN cd /opt/oxo && mvn clean package -DskipTests
 
 FROM openjdk:8-jre-alpine

oxo-indexer/application.properties

@@ -1,13 +1,13 @@
-logging.level.org.springframework: ERROR
-logging.level.org.neo4j.ogm: ERROR
-logging.level.uk.ac.ebi.spot: INFO
+logging.level.org.springframework=ERROR
+logging.level.org.neo4j.ogm=ERROR
+logging.level.uk.ac.ebi.spot=INFO
 
 oxo.neo.driver=org.neo4j.ogm.drivers.http.driver.HttpDriver
 #oxo.neo.driver=org.neo4j.ogm.drivers.bolt.driver.BoltDriver
 oxo.neo.uri=http://neo4j:dba@localhost:7474
 #bolt://neo4j:dba@localhost:7688
 
-spring.data.solr.host: http://localhost:8983/solr
+spring.data.solr.host=http://localhost:8983/solr
 
-oxo.indexer.chunks 100000
+oxo.indexer.chunks=100000
 

oxo-indexer/pom.xml

@@ -32,6 +32,30 @@
             <version>0.0.17-SNAPSHOT</version>
         </dependency>
 
+        <dependency>
+            <groupId>uk.ac.ebi.spot</groupId>
+            <artifactId>sping-security-orcid-stateless</artifactId>
+            <version>0.0.8</version>
+        </dependency>
+
+        <dependency>
+            <groupId>org.springframework.security</groupId>
+            <artifactId>spring-security-core</artifactId>
+            <version>4.1.4.RELEASE</version>
+        </dependency>
+
+        <dependency>
+            <groupId>org.springframework.security.oauth</groupId>
+            <artifactId>spring-security-oauth2</artifactId>
+            <version>2.0.13.RELEASE</version>
+        </dependency>
+
+        <dependency>
+            <groupId>org.springframework.data</groupId>
+            <artifactId>spring-data-jpa</artifactId>
+            <version>1.10.8.RELEASE</version>
+        </dependency>
+
     </dependencies>
     <packaging>jar</packaging>
 

oxo-indexer/src/main/java/uk/ac/ebi/spot/indexer/SolrIndexer.java

@@ -2,34 +2,21 @@
 
 import org.apache.solr.client.solrj.SolrClient;
 import org.apache.solr.client.solrj.impl.HttpSolrClient;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.beans.factory.annotation.Qualifier;
 import org.springframework.boot.CommandLineRunner;
 import org.springframework.boot.SpringApplication;
-import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
 import org.springframework.boot.autoconfigure.SpringBootApplication;
-import org.springframework.boot.autoconfigure.domain.EntityScan;
 import org.springframework.boot.context.properties.EnableConfigurationProperties;
 import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.ComponentScan;
-import org.springframework.context.annotation.Configuration;
-import org.springframework.context.annotation.ScopedProxyMode;
 import org.springframework.core.env.Environment;
 import org.springframework.data.neo4j.repository.config.EnableNeo4jRepositories;
 import org.springframework.data.solr.core.SolrTemplate;
 import org.springframework.data.solr.repository.config.EnableSolrRepositories;
-import org.springframework.data.solr.server.support.EmbeddedSolrServerFactoryBean;
-import org.springframework.stereotype.Component;
 import uk.ac.ebi.spot.index.Document;
-import uk.ac.ebi.spot.index.DocumentRepository;
-import uk.ac.ebi.spot.model.Datasource;
-import uk.ac.ebi.spot.service.DatasourceService;
 import uk.ac.ebi.spot.service.TermService;
 
 import javax.annotation.Resource;
-import java.net.MalformedURLException;
 
 /**
  * @author Simon Jupp
@@ -40,9 +27,8 @@
 @SpringBootApplication
 @EnableNeo4jRepositories(basePackages = "uk.ac.ebi.spot.repository")
 @EnableSolrRepositories(basePackages = "uk.ac.ebi.spot.index", basePackageClasses = {Document.class})
-@EnableAutoConfiguration
 @EnableConfigurationProperties
-@ComponentScan({"uk.ac.ebi" })
+@ComponentScan({"uk.ac.ebi"})
 public class SolrIndexer implements CommandLineRunner {
 
     @Autowired

oxo-loader/OlsDatasetExtractor.py

@@ -191,6 +191,46 @@
 datasources[prefPrefix] = OxoClient.Datasource(prefPrefix, namespace, title, desc, sourceType, urlSyntax, altPrefixes, licence, versionInfo)
 
 
+print("Adding ctv3 as datasource")
+prefPrefix="CTV3"
+namespace=None
+title="CTV3"
+desc="CTV3"
+sourceType="DATABASE"
+urlSyntax=None
+altPrefixes=["ctv3", "CTV3"]
+licence=None
+versionInfo=1
+datasources[prefPrefix] = OxoClient.Datasource(prefPrefix, namespace, title, desc, sourceType, urlSyntax, altPrefixes, licence, versionInfo)
+
+
+print("Adding ICD10CM as datasource")
+prefPrefix="ICD10CM"
+namespace=None
+title="ICD10CM"
+desc="ICD10CM"
+sourceType="DATABASE"
+urlSyntax=None
+altPrefixes=["icd10cm", "ICD10CM"]
+licence=None
+versionInfo=1
+datasources[prefPrefix] = OxoClient.Datasource(prefPrefix, namespace, title, desc, sourceType, urlSyntax, altPrefixes, licence, versionInfo)
+
+
+print("Adding ICDO as datasource")
+prefPrefix="ICDO"
+namespace=None
+title="ICDO"
+desc="ICDO"
+sourceType="DATABASE"
+urlSyntax=None
+altPrefixes=["icdo", "ICDO"]
+licence=None
+versionInfo=1
+datasources[prefPrefix] = OxoClient.Datasource(prefPrefix, namespace, title, desc, sourceType, urlSyntax, altPrefixes, licence, versionInfo)
+
+
+
 
 # print OxO loading csv file
 import OxoCsvBuilder

oxo-loader/OlsMappingExtractor.py

@@ -156,7 +156,11 @@ def processSolrDocs(url):
                 print("unknown prefix " + fromPrefix + " so skipping")
                 continue
 
-            fromPrefix = prefixToPreferred[fromPrefix]
+            if fromOntology == "snomed":
+                fromPrefix = "SNOMED"
+            else:
+                fromPrefix = prefixToPreferred[fromPrefix]
+
             fromCurie = fromPrefix + ":" + fromId
 
             if fromCurie not in terms:
@@ -174,7 +178,7 @@ def processSolrDocs(url):
             for anno in knownAnnotations:
                 if anno in docs:
                     for xref in docs[anno]:
-                        if ":" in xref or "_" in xref:
+                        if fromOntology == "snomed" or ":" in xref or "_" in xref:
                             toPrefix = OXO.getPrefixFromCui(xref)
                             toId = OXO.getIdFromCui(xref)
 
@@ -187,7 +191,7 @@ def processSolrDocs(url):
                                 continue
                             if toPrefix.lower() not in prefixToPreferred:
                                 unknownSource[toPrefix] = 1
-                                # print "Unknown prefix source for "+toPrefix+" so skipping"
+                                print("Unknown prefix source for "+toPrefix+" so skipping")
                                 continue
 
 

oxo-loader/runLoader.sh

@@ -0,0 +1,45 @@
+#!/usr/bin/env bash
+
+set -e
+#module load python-3.10.2-gcc-9.3.0-gswnsij
+pip3 install -r requirements.txt
+
+echo 1	get datasources from OLS
+
+python3 OlsDatasetExtractor.py -c config.ini -i idorg.xml -d datasources.csv
+
+echo 2	Load datasources into neo4j
+
+scp datasources.csv $NEO4J_HOME/import/
+python3 OxoNeo4jLoader.py -W -d datasources.csv -c config.ini
+
+echo 3	get the OLS mappings
+
+python3 OlsMappingExtractor.py -c config.ini -t ols_terms.csv -m ols_mappings.csv
+
+echo 4	load the OLS data into Neo4j
+
+scp ols_terms.csv $NEO4J_HOME/import
+scp ols_mappings.csv $NEO4J_HOME/import
+python3 OxoNeo4jLoader.py -c config.ini -t ols_terms.csv -m ols_mappings.csv
+
+
+# get the umls mappings
+echo 5	get the umls mappings
+
+#bsub  -Is -M 30000 -R "rusage[mem=30000]" "singularity exec -B /nfs/production/parkinso/spot/oxo/dev/loader:/opt/config -B /nfs/production/parkinso/spot/oxo/dev/import:/opt/oxo-loader/data  docker://ebispot/oxo-loader:$OXO_LOADER_VERSION  python /opt/oxo-loader/UmlsMappingExtractor.py -c /opt/config/config.ini -t /opt/oxo-loader/data/umls_terms.csv -m /opt/oxo-loader/data/umls_mappings.csv"
+
+# load the UMLS data into neo4j
+echo 6	load the UMLS data into neo4j
+
+#bsub  -Is -M 30000 -R "rusage[mem=30000]" "singularity exec -B /nfs/production/parkinso/spot/oxo/dev/loader:/opt/config docker://ebispot/oxo-loader:$OXO_LOADER_VERSION  python /opt/oxo-loader/OxoNeo4jLoader.py -c /opt/config/config.ini -t umls_terms.csv -m umls_mappings.csv"
+
+# get the LOINC mappings
+echo 7	get the LOINC mappings
+
+#bsub  -Is -M 30000 -R "rusage[mem=30000]" "singularity exec -B /nfs/production/parkinso/spot/oxo/dev/loader:/opt/config -B /nfs/production/parkinso/spot/oxo/dev/import:/opt/oxo-loader/data  docker://ebispot/oxo-loader:$OXO_LOADER_VERSION  python /opt/oxo-loader/LoincMappingExtractor.py -c /opt/config/config.ini -t /opt/oxo-loader/data/loinc_terms.csv -m /opt/oxo-loader/data/loinc_mappings.csv"
+
+# load the LOINC data into neo4j
+echo 8	load the LOINC data into neo4j
+
+#bsub  -Is -M 30000 -R "rusage[mem=30000]" "singularity exec -B /nfs/production/parkinso/spot/oxo/dev/loader:/opt/config docker://ebispot/oxo-loader:$OXO_LOADER_VERSION  python /opt/oxo-loader/OxoNeo4jLoader.py -c /opt/config/config.ini -t loinc_terms.csv -m loinc_mappings.csv"