From 4d8d2ee1b3a948712f9c8a84da9169f4f202ff01 Mon Sep 17 00:00:00 2001 From: James McLaughlin Date: Fri, 7 Aug 2020 13:02:54 +0100 Subject: [PATCH 1/6] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index ceac797..3df6cfb 100644 --- a/README.md +++ b/README.md @@ -34,7 +34,7 @@ The loader scripts are documented in the README of the oxo-loader/ directory. After using the loader to load data into neo4j, the indexer can be executed using Docker: - docker run --net=host EBISPOT/oxo-indexer:stable + docker run --net=host ebispot/oxo-indexer:dev # OxO without Docker From b40fa24fd7da2ca09343ae52ce27295756725f95 Mon Sep 17 00:00:00 2001 From: James McLaughlin Date: Fri, 17 Jul 2020 12:44:27 +0100 Subject: [PATCH 2/6] add k8 chart --- k8chart/oxo-indexer/Chart.yaml | 5 +++ k8chart/oxo-indexer/templates/_helpers.tpl | 0 .../templates/oxo-indexer-job.yaml | 22 ++++++++++ k8chart/oxo-indexer/values.yaml | 10 +++++ k8chart/oxo-loader/Chart.yaml | 5 +++ k8chart/oxo-loader/templates/_helpers.tpl | 0 .../templates/oxo-loader-configmap.yaml | 33 +++++++++++++++ .../oxo-loader/templates/oxo-loader-job.yaml | 37 +++++++++++++++++ k8chart/oxo-loader/values.yaml | 10 +++++ k8chart/oxo/.helmignore | 22 ++++++++++ k8chart/oxo/Chart.yaml | 5 +++ k8chart/oxo/templates/_helpers.tpl | 0 k8chart/oxo/templates/oxo-configmap.yaml | 6 +++ k8chart/oxo/templates/oxo-ingress.yaml | 14 +++++++ .../oxo/templates/oxo-neo4j-deployment.yaml | 41 +++++++++++++++++++ k8chart/oxo/templates/oxo-neo4j-service.yaml | 18 ++++++++ .../oxo/templates/oxo-proxy-configmap.yaml | 10 +++++ .../oxo/templates/oxo-solr-deployment.yaml | 37 +++++++++++++++++ k8chart/oxo/templates/oxo-solr-service.yaml | 14 +++++++ k8chart/oxo/templates/oxo-web-deployment.yaml | 30 ++++++++++++++ k8chart/oxo/templates/oxo-web-service.yaml | 14 +++++++ k8chart/oxo/values.yaml | 6 +++ k8chart/pvcs/Chart.yaml | 5 +++ k8chart/pvcs/templates/_helpers.tpl | 0 .../pvcs/templates/oxo-neo4j-data-pvc.yaml | 11 +++++ .../pvcs/templates/oxo-neo4jimport-pvc.yaml | 11 +++++ k8chart/pvcs/templates/oxo-solr-data-pvc.yaml | 11 +++++ k8chart/pvcs/values.yaml | 5 +++ oxo-loader/Dockerfile | 1 + oxo-loader/load_all.sh | 14 ++++--- 30 files changed, 391 insertions(+), 6 deletions(-) create mode 100644 k8chart/oxo-indexer/Chart.yaml create mode 100644 k8chart/oxo-indexer/templates/_helpers.tpl create mode 100644 k8chart/oxo-indexer/templates/oxo-indexer-job.yaml create mode 100644 k8chart/oxo-indexer/values.yaml create mode 100644 k8chart/oxo-loader/Chart.yaml create mode 100644 k8chart/oxo-loader/templates/_helpers.tpl create mode 100644 k8chart/oxo-loader/templates/oxo-loader-configmap.yaml create mode 100644 k8chart/oxo-loader/templates/oxo-loader-job.yaml create mode 100644 k8chart/oxo-loader/values.yaml create mode 100644 k8chart/oxo/.helmignore create mode 100644 k8chart/oxo/Chart.yaml create mode 100644 k8chart/oxo/templates/_helpers.tpl create mode 100644 k8chart/oxo/templates/oxo-configmap.yaml create mode 100644 k8chart/oxo/templates/oxo-ingress.yaml create mode 100644 k8chart/oxo/templates/oxo-neo4j-deployment.yaml create mode 100644 k8chart/oxo/templates/oxo-neo4j-service.yaml create mode 100644 k8chart/oxo/templates/oxo-proxy-configmap.yaml create mode 100644 k8chart/oxo/templates/oxo-solr-deployment.yaml create mode 100644 k8chart/oxo/templates/oxo-solr-service.yaml create mode 100644 k8chart/oxo/templates/oxo-web-deployment.yaml create mode 100644 k8chart/oxo/templates/oxo-web-service.yaml create mode 100644 k8chart/oxo/values.yaml create mode 100644 k8chart/pvcs/Chart.yaml create mode 100644 k8chart/pvcs/templates/_helpers.tpl create mode 100644 k8chart/pvcs/templates/oxo-neo4j-data-pvc.yaml create mode 100644 k8chart/pvcs/templates/oxo-neo4jimport-pvc.yaml create mode 100644 k8chart/pvcs/templates/oxo-solr-data-pvc.yaml create mode 100644 k8chart/pvcs/values.yaml diff --git a/k8chart/oxo-indexer/Chart.yaml b/k8chart/oxo-indexer/Chart.yaml new file mode 100644 index 0000000..68a31ae --- /dev/null +++ b/k8chart/oxo-indexer/Chart.yaml @@ -0,0 +1,5 @@ +apiVersion: v1 +appVersion: "1.0" +description: oxo indexer +name: oxo +version: 0.1.0 diff --git a/k8chart/oxo-indexer/templates/_helpers.tpl b/k8chart/oxo-indexer/templates/_helpers.tpl new file mode 100644 index 0000000..e69de29 diff --git a/k8chart/oxo-indexer/templates/oxo-indexer-job.yaml b/k8chart/oxo-indexer/templates/oxo-indexer-job.yaml new file mode 100644 index 0000000..e6e1574 --- /dev/null +++ b/k8chart/oxo-indexer/templates/oxo-indexer-job.yaml @@ -0,0 +1,22 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ .Release.Name }}-job +spec: + template: + metadata: + name: {{ .Release.Name }}-job + spec: + containers: + - name: oxo-indexer + image: {{.Values.image.name}}:{{.Values.image.tag}} + env: + - name: OXO_NEO_DRIVER + value: "org.neo4j.ogm.drivers.http.driver.HttpDriver" + - name: OXO_NEO_URI + value: "http://neo4j:dba@oxo-{{.Values.oxoRelease}}-neo4j:7474" + - name: SPRING_DATA_SOLR_HOST + value: "http://oxo-{{.Values.oxoRelease}}-solr:8983/solr" + - name: OXO_INDEXER_CHUNKS + value: "100000" + restartPolicy: Never diff --git a/k8chart/oxo-indexer/values.yaml b/k8chart/oxo-indexer/values.yaml new file mode 100644 index 0000000..cbe3006 --- /dev/null +++ b/k8chart/oxo-indexer/values.yaml @@ -0,0 +1,10 @@ + +image: + name: ebispot/oxo-indexer + tag: stable + + +oxoRelease: dev + + + diff --git a/k8chart/oxo-loader/Chart.yaml b/k8chart/oxo-loader/Chart.yaml new file mode 100644 index 0000000..db31749 --- /dev/null +++ b/k8chart/oxo-loader/Chart.yaml @@ -0,0 +1,5 @@ +apiVersion: v1 +appVersion: "1.0" +description: oxo loader +name: oxo +version: 0.1.0 diff --git a/k8chart/oxo-loader/templates/_helpers.tpl b/k8chart/oxo-loader/templates/_helpers.tpl new file mode 100644 index 0000000..e69de29 diff --git a/k8chart/oxo-loader/templates/oxo-loader-configmap.yaml b/k8chart/oxo-loader/templates/oxo-loader-configmap.yaml new file mode 100644 index 0000000..44da8e1 --- /dev/null +++ b/k8chart/oxo-loader/templates/oxo-loader-configmap.yaml @@ -0,0 +1,33 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ .Release.Name }}-configmap +data: + config.ini: |- + [Basics] + oxoUrl=http://oxo-{{ .Values.oxoRelease }}-web:8080 + oxoAPIkey=key + olsSolrBaseUrl=http://ves-hx-7e:8993/solr + solrChunks=5000 + neoURL=bolt://oxo-{{ .Values.oxoRelease }}-neo4j:7687 + neoUser=neo4j + neoPass=dba + olsurl=https://www.ebi.ac.uk/ols/api + oboDbxrefUrl=https://raw.githubusercontent.com/geneontology/go-site/master/metadata/db-xrefs.yaml + + [Paths] + exportFileDatasources=datasources.csv + exportFileTerms=/path/terms.csv + exportFileMappings=/path/mappings.csv + idorgDataLocation = /path/idorg.xml + + [SQLumls] + user=username + password=password + host=mysql-name + db=dbName + port=4570 + + [LOINC] + Part=/path/Part.csv + PartRelatedCodeMapping=/path/PartRelatedCodeMapping.csv diff --git a/k8chart/oxo-loader/templates/oxo-loader-job.yaml b/k8chart/oxo-loader/templates/oxo-loader-job.yaml new file mode 100644 index 0000000..4e46eec --- /dev/null +++ b/k8chart/oxo-loader/templates/oxo-loader-job.yaml @@ -0,0 +1,37 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ .Release.Name }}-job +spec: + template: + metadata: + name: {{ .Release.Name }}-job + spec: + containers: + - name: oxo-loader + image: {{.Values.image.name}}:{{.Values.image.tag}} + command: ['bash', '/opt/oxo-loader/load_all.sh'] + volumeMounts: + - name: oxoloader-config + mountPath: "/opt/oxo-config" + readOnly: true + - name: oxo-{{.Values.oxoRelease}}-neo4jimport + mountPath: "/mnt/neo4jimport" + env: + - name: NEO4IMPORT + value: '/mnt/neo4jimport' + - name: CONFIG_INI + value: '/opt/oxo-config/config.ini' + - name: IDORG_XML + value: '/opt/oxo-loader/idorg.xml' + envFrom: + - configMapRef: + name: oxo-{{ .Values.oxoRelease }}-proxy-configmap + restartPolicy: Never + volumes: + - name: oxoloader-config + configMap: + name: {{ .Release.Name }}-configmap + - name: oxo-{{.Values.oxoRelease}}-neo4jimport + persistentVolumeClaim: + claimName: oxo-{{.Values.oxoRelease}}-neo4jimport diff --git a/k8chart/oxo-loader/values.yaml b/k8chart/oxo-loader/values.yaml new file mode 100644 index 0000000..957d79f --- /dev/null +++ b/k8chart/oxo-loader/values.yaml @@ -0,0 +1,10 @@ + +image: + name: ebispot/oxo-loader + tag: stable + + +oxoRelease: dev + + + diff --git a/k8chart/oxo/.helmignore b/k8chart/oxo/.helmignore new file mode 100644 index 0000000..50af031 --- /dev/null +++ b/k8chart/oxo/.helmignore @@ -0,0 +1,22 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/k8chart/oxo/Chart.yaml b/k8chart/oxo/Chart.yaml new file mode 100644 index 0000000..966811a --- /dev/null +++ b/k8chart/oxo/Chart.yaml @@ -0,0 +1,5 @@ +apiVersion: v1 +appVersion: "1.0" +description: oxo deployment +name: oxo +version: 0.1.0 diff --git a/k8chart/oxo/templates/_helpers.tpl b/k8chart/oxo/templates/_helpers.tpl new file mode 100644 index 0000000..e69de29 diff --git a/k8chart/oxo/templates/oxo-configmap.yaml b/k8chart/oxo/templates/oxo-configmap.yaml new file mode 100644 index 0000000..4220e37 --- /dev/null +++ b/k8chart/oxo/templates/oxo-configmap.yaml @@ -0,0 +1,6 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ .Release.Name }}-configmap +data: + myvalue: "Hello World" \ No newline at end of file diff --git a/k8chart/oxo/templates/oxo-ingress.yaml b/k8chart/oxo/templates/oxo-ingress.yaml new file mode 100644 index 0000000..100b01b --- /dev/null +++ b/k8chart/oxo/templates/oxo-ingress.yaml @@ -0,0 +1,14 @@ +apiVersion: networking.k8s.io/v1beta1 +kind: Ingress +metadata: + name: {{ .Release.Name }}-ingress + annotations: + nginx.ingress.kubernetes.io/rewrite-target: / +spec: + rules: + - http: + paths: + - path: / + backend: + serviceName: {{ .Release.Name }}-web + servicePort: 8080 diff --git a/k8chart/oxo/templates/oxo-neo4j-deployment.yaml b/k8chart/oxo/templates/oxo-neo4j-deployment.yaml new file mode 100644 index 0000000..f9f34bf --- /dev/null +++ b/k8chart/oxo/templates/oxo-neo4j-deployment.yaml @@ -0,0 +1,41 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ .Release.Name }}-neo4j + labels: + app: {{ .Release.Name }}-neo4j +spec: + replicas: 1 + selector: + matchLabels: + app: {{ .Release.Name }}-neo4j + template: + metadata: + labels: + app: {{ .Release.Name }}-neo4j + spec: + containers: + - name: neo4j + image: neo4j:3.1.1 + env: + # - name: NEO4J_HEAP_MEMORY + # value: 10g + # - name: NEO4J_dbms_memory_heap_maxSize + # value: 8g + - name: NEO4J_AUTH + value: neo4j/dba + ports: + - containerPort: 7474 + - containerPort: 7687 + volumeMounts: + - name: {{.Release.Name}}-neo4jimport + mountPath: "/var/lib/neo4j/import" + - name: {{.Release.Name}}-neo4j-data + mountPath: "/data" + volumes: + - name: {{.Release.Name}}-neo4jimport + persistentVolumeClaim: + claimName: {{.Release.Name}}-neo4jimport + - name: {{.Release.Name}}-neo4j-data + persistentVolumeClaim: + claimName: {{.Release.Name}}-neo4j-data diff --git a/k8chart/oxo/templates/oxo-neo4j-service.yaml b/k8chart/oxo/templates/oxo-neo4j-service.yaml new file mode 100644 index 0000000..55f6916 --- /dev/null +++ b/k8chart/oxo/templates/oxo-neo4j-service.yaml @@ -0,0 +1,18 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ .Release.Name }}-neo4j + labels: + app: {{ .Release.Name }}-neo4j +spec: + ports: + - port: 7474 + targetPort: 7474 + name: http + protocol: TCP + - port: 7687 + targetPort: 7687 + name: bolt + protocol: TCP + selector: + app: {{ .Release.Name }}-neo4j \ No newline at end of file diff --git a/k8chart/oxo/templates/oxo-proxy-configmap.yaml b/k8chart/oxo/templates/oxo-proxy-configmap.yaml new file mode 100644 index 0000000..89fb0f5 --- /dev/null +++ b/k8chart/oxo/templates/oxo-proxy-configmap.yaml @@ -0,0 +1,10 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ .Release.Name }}-proxy-configmap +data: + HTTP_PROXY: http://hx-wwwcache.ebi.ac.uk:3128 + HTTPS_PROXY: http://hx-wwwcache.ebi.ac.uk:3128 + http_proxy: http://hx-wwwcache.ebi.ac.uk:3128 + https_proxy: http://hx-wwwcache.ebi.ac.uk:3128 + no_proxy: localhost,.cluster.local diff --git a/k8chart/oxo/templates/oxo-solr-deployment.yaml b/k8chart/oxo/templates/oxo-solr-deployment.yaml new file mode 100644 index 0000000..9476771 --- /dev/null +++ b/k8chart/oxo/templates/oxo-solr-deployment.yaml @@ -0,0 +1,37 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ .Release.Name }}-solr + labels: + app: {{ .Release.Name }}-solr +spec: + replicas: 1 + selector: + matchLabels: + app: {{ .Release.Name }}-solr + template: + metadata: + labels: + app: {{ .Release.Name }}-solr + spec: + # initContainers: + # - name: init-solr + # image: solr:5.5.3-alpine + # command: ['solr', 'create_core', '-c', 'mapping'] + # volumeMounts: + # - name: {{.Release.Name}}-solr-data + # mountPath: "/var/solr" + containers: + - name: solr + image: solr:5.5.3-alpine + command: ['solr-precreate', 'mapping'] + ports: + - containerPort: 8983 + volumeMounts: + - name: {{.Release.Name}}-solr-data + mountPath: "/var/solr" + volumes: + - name: {{.Release.Name}}-solr-data + persistentVolumeClaim: + claimName: {{.Release.Name}}-solr-data + diff --git a/k8chart/oxo/templates/oxo-solr-service.yaml b/k8chart/oxo/templates/oxo-solr-service.yaml new file mode 100644 index 0000000..1f18b61 --- /dev/null +++ b/k8chart/oxo/templates/oxo-solr-service.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ .Release.Name }}-solr + labels: + app: {{ .Release.Name }}-solr +spec: + ports: + - port: 8983 + targetPort: 8983 + name: http + protocol: TCP + selector: + app: {{ .Release.Name }}-solr \ No newline at end of file diff --git a/k8chart/oxo/templates/oxo-web-deployment.yaml b/k8chart/oxo/templates/oxo-web-deployment.yaml new file mode 100644 index 0000000..a5f1b85 --- /dev/null +++ b/k8chart/oxo/templates/oxo-web-deployment.yaml @@ -0,0 +1,30 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ .Release.Name }}-web + labels: + app: {{ .Release.Name }}-web +spec: + replicas: 1 + selector: + matchLabels: + app: {{ .Release.Name }}-web + template: + metadata: + labels: + app: {{ .Release.Name }}-web + spec: + containers: + - name: web + image: {{.Values.image.name}}:{{.Values.image.tag}} + ports: + - containerPort: 8080 + env: + - name: spring.datasource.url + value: jdbc:hsqldb:file:/mnt/hsqldb + - name: oxo.neo.driver + value: org.neo4j.ogm.drivers.http.driver.HttpDriver + - name: oxo.neo.uri + value: http://neo4j:dba@{{ .Release.Name }}-neo4j:7474 + - name: spring.data.solr.host + value: http://{{ .Release.Name }}-solr:8983/solr \ No newline at end of file diff --git a/k8chart/oxo/templates/oxo-web-service.yaml b/k8chart/oxo/templates/oxo-web-service.yaml new file mode 100644 index 0000000..ad794ff --- /dev/null +++ b/k8chart/oxo/templates/oxo-web-service.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ .Release.Name }}-web + labels: + app: {{ .Release.Name }}-web +spec: + ports: + - port: 8080 + targetPort: 8080 + name: http + protocol: TCP + selector: + app: {{ .Release.Name }}-web \ No newline at end of file diff --git a/k8chart/oxo/values.yaml b/k8chart/oxo/values.yaml new file mode 100644 index 0000000..3b596d8 --- /dev/null +++ b/k8chart/oxo/values.yaml @@ -0,0 +1,6 @@ + +image: + name: ebispot/oxo-web + tag: stable + + diff --git a/k8chart/pvcs/Chart.yaml b/k8chart/pvcs/Chart.yaml new file mode 100644 index 0000000..1daa838 --- /dev/null +++ b/k8chart/pvcs/Chart.yaml @@ -0,0 +1,5 @@ +apiVersion: v1 +appVersion: "1.0" +description: oxo PVCs +name: oxo +version: 0.1.0 diff --git a/k8chart/pvcs/templates/_helpers.tpl b/k8chart/pvcs/templates/_helpers.tpl new file mode 100644 index 0000000..e69de29 diff --git a/k8chart/pvcs/templates/oxo-neo4j-data-pvc.yaml b/k8chart/pvcs/templates/oxo-neo4j-data-pvc.yaml new file mode 100644 index 0000000..77e657f --- /dev/null +++ b/k8chart/pvcs/templates/oxo-neo4j-data-pvc.yaml @@ -0,0 +1,11 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + creationTimestamp: null + name: oxo-{{ .Values.oxoRelease }}-neo4j-data +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Gi diff --git a/k8chart/pvcs/templates/oxo-neo4jimport-pvc.yaml b/k8chart/pvcs/templates/oxo-neo4jimport-pvc.yaml new file mode 100644 index 0000000..0d70abf --- /dev/null +++ b/k8chart/pvcs/templates/oxo-neo4jimport-pvc.yaml @@ -0,0 +1,11 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + creationTimestamp: null + name: oxo-{{ .Values.oxoRelease }}-neo4jimport +spec: + accessModes: + - ReadWriteMany + resources: + requests: + storage: 10Gi diff --git a/k8chart/pvcs/templates/oxo-solr-data-pvc.yaml b/k8chart/pvcs/templates/oxo-solr-data-pvc.yaml new file mode 100644 index 0000000..6c85001 --- /dev/null +++ b/k8chart/pvcs/templates/oxo-solr-data-pvc.yaml @@ -0,0 +1,11 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + creationTimestamp: null + name: oxo-{{ .Values.oxoRelease }}-solr-data +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Gi diff --git a/k8chart/pvcs/values.yaml b/k8chart/pvcs/values.yaml new file mode 100644 index 0000000..7ee536d --- /dev/null +++ b/k8chart/pvcs/values.yaml @@ -0,0 +1,5 @@ + +oxoRelease: dev + + + diff --git a/oxo-loader/Dockerfile b/oxo-loader/Dockerfile index a58953d..d34f20c 100644 --- a/oxo-loader/Dockerfile +++ b/oxo-loader/Dockerfile @@ -5,5 +5,6 @@ ADD oxo-loader /opt/oxo-loader RUN apk add --no-cache bash mariadb-dev build-base #RUN apk --update add mysql mysql-client RUN cd /opt/oxo-loader && pip install -r requirements.txt +RUN chmod +x /opt/oxo-loader/load_all.sh CMD bash diff --git a/oxo-loader/load_all.sh b/oxo-loader/load_all.sh index d4f9370..0a42419 100644 --- a/oxo-loader/load_all.sh +++ b/oxo-loader/load_all.sh @@ -1,8 +1,10 @@ -python /app/OlsDatasetExtractor.py -c /app/config/config.ini -i /app/config/idorg.xml -d /app/data/datasources.csv -python /app/OxoNeo4jLoader.py -W -d datasources.csv -c /app/config/config.ini +#!/usr/bin/env bash -python /app/OlsMappingExtractor.py -c /app/config/config.ini -t /app/data/ols_terms.csv -m /app/data/ols_mappings.csv -python /app/OxoNeo4jLoader.py -c /app/config/config.ini -t ols_terms.csv -m ols_mappings.csv +python /opt/oxo-loader/OlsDatasetExtractor.py -c $CONFIG_INI -i $IDORG_XML -d $NEO4JIMPORT/datasources.csv +python /opt/oxo-loader/OxoNeo4jLoader.py -W -d datasources.csv -c $CONFIG_INI -python /app/UmlsMappingExtractor.py -c /app/config/config.ini -t /app/data/umls_terms.csv -m /app/data/umls_mappings.csv -python /app/OxoNeo4jLoader.py -c /app/config/config.ini -t umls_terms.csv -m umls_mappings.csv +python /opt/oxo-loader/OlsMappingExtractor.py -c $CONFIG_INI -t $NEO4JIMPORT/ols_terms.csv -m $NEO4JIMPORT/ols_mappings.csv +python /opt/oxo-loader/OxoNeo4jLoader.py -c $CONFIG_INI -t ols_terms.csv -m ols_mappings.csv + +#python /app/UmlsMappingExtractor.py -c /app/config/config.ini -t $NEO4JIMPORT/umls_terms.csv -m $NEO4JIMPORT/umls_mappings.csv +#python /app/OxoNeo4jLoader.py -c $CONFIG_INI -t umls_terms.csv -m umls_mappings.csv From 0bf36ccd57d3d620f580e3489caf10f2a76eaae4 Mon Sep 17 00:00:00 2001 From: James McLaughlin Date: Fri, 17 Jul 2020 12:45:05 +0100 Subject: [PATCH 3/6] update gitlab ci for k8s --- .gitlab-ci.yml | 80 ++++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 68 insertions(+), 12 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 326c1bd..6090b9d 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,6 +1,13 @@ +stages: + - build + - deploy + - dataload + - indexing + build: image: docker:18-git + when: manual stage: build only: - dev @@ -9,17 +16,66 @@ build: services: - docker:18-dind script: + - echo docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY - - docker build -t ${OXO_IMAGE_PREFIX}/oxo-loader:$CI_COMMIT_REF_NAME-$CI_BUILD_ID -f ./oxo-loader/Dockerfile . - - docker build -t ${OXO_IMAGE_PREFIX}/oxo-indexer:$CI_COMMIT_REF_NAME-$CI_BUILD_ID -f ./oxo-indexer/Dockerfile . - - docker build -t ${OXO_IMAGE_PREFIX}/oxo-web:$CI_COMMIT_REF_NAME-$CI_BUILD_ID -f ./oxo-web/Dockerfile . - - docker tag ${OXO_IMAGE_PREFIX}/oxo-loader:$CI_COMMIT_REF_NAME-$CI_BUILD_ID ${OXO_IMAGE_PREFIX}/oxo-loader:$CI_COMMIT_REF_NAME - - docker tag ${OXO_IMAGE_PREFIX}/oxo-indexer:$CI_COMMIT_REF_NAME-$CI_BUILD_ID ${OXO_IMAGE_PREFIX}/oxo-indexer:$CI_COMMIT_REF_NAME - - docker tag ${OXO_IMAGE_PREFIX}/oxo-web:$CI_COMMIT_REF_NAME-$CI_BUILD_ID ${OXO_IMAGE_PREFIX}/oxo-web:$CI_COMMIT_REF_NAME - - docker push ${OXO_IMAGE_PREFIX}/oxo-loader:$CI_COMMIT_REF_NAME-$CI_BUILD_ID - - docker push ${OXO_IMAGE_PREFIX}/oxo-loader:$CI_COMMIT_REF_NAME - - docker push ${OXO_IMAGE_PREFIX}/oxo-indexer:$CI_COMMIT_REF_NAME-$CI_BUILD_ID - - docker push ${OXO_IMAGE_PREFIX}/oxo-indexer:$CI_COMMIT_REF_NAME - - docker push ${OXO_IMAGE_PREFIX}/oxo-web:$CI_COMMIT_REF_NAME-$CI_BUILD_ID - - docker push ${OXO_IMAGE_PREFIX}/oxo-web:$CI_COMMIT_REF_NAME + - docker build -t $CI_REGISTRY/$OXO_IMAGE_PREFIX/oxo-loader:$CI_COMMIT_SHA -f ./oxo-loader/Dockerfile . + - docker build -t $CI_REGISTRY/$OXO_IMAGE_PREFIX/oxo-indexer:$CI_COMMIT_SHA -f ./oxo-indexer/Dockerfile . + - docker build -t $CI_REGISTRY/$OXO_IMAGE_PREFIX/oxo-web:$CI_COMMIT_SHA -f ./oxo-web/Dockerfile . + - docker tag $CI_REGISTRY/$OXO_IMAGE_PREFIX/oxo-loader:$CI_COMMIT_SHA $CI_REGISTRY/$OXO_IMAGE_PREFIX/oxo-loader:$CI_COMMIT_REF_NAME + - docker tag $CI_REGISTRY/$OXO_IMAGE_PREFIX/oxo-indexer:$CI_COMMIT_SHA $CI_REGISTRY/$OXO_IMAGE_PREFIX/oxo-indexer:$CI_COMMIT_REF_NAME + - docker tag $CI_REGISTRY/$OXO_IMAGE_PREFIX/oxo-web:$CI_COMMIT_SHA $CI_REGISTRY/$OXO_IMAGE_PREFIX/oxo-web:$CI_COMMIT_REF_NAME + - docker push $CI_REGISTRY/$OXO_IMAGE_PREFIX/oxo-loader:$CI_COMMIT_SHA + - docker push $CI_REGISTRY/$OXO_IMAGE_PREFIX/oxo-loader:$CI_COMMIT_REF_NAME + - docker push $CI_REGISTRY/$OXO_IMAGE_PREFIX/oxo-indexer:$CI_COMMIT_SHA + - docker push $CI_REGISTRY/$OXO_IMAGE_PREFIX/oxo-indexer:$CI_COMMIT_REF_NAME + - docker push $CI_REGISTRY/$OXO_IMAGE_PREFIX/oxo-web:$CI_COMMIT_SHA + - docker push $CI_REGISTRY/$OXO_IMAGE_PREFIX/oxo-web:$CI_COMMIT_REF_NAME + +deploy: + image: dtzar/helm-kubectl:2.13.1 + when: manual + stage: deploy + only: + - dev + script: + - echo "Deploy to $CI_COMMIT_REF_NAME server" + - mkdir -p /root/.kube + - echo ${KUBECONF_DEV} | base64 -d > /root/.kube/config + - helm init + - helm delete --purge oxo-$CI_COMMIT_REF_NAME || true + - sleep 10 + - helm install --namespace oxo --debug --name oxo-$CI_COMMIT_REF_NAME-pvcs ./k8chart/pvcs --wait || true + # - helm install --namespace oxo --debug --set image.tag=$CI_COMMIT_SHA --name oxo-$CI_COMMIT_REF_NAME ./k8chart/oxo --wait + - helm install --namespace oxo --debug --set image.tag=dev --name oxo-$CI_COMMIT_REF_NAME ./k8chart/oxo --wait + environment: + name: dev + +runLoader: + image: dtzar/helm-kubectl:2.13.1 + when: manual + stage: dataload + script: + - echo "Run $CI_COMMIT_REF_NAME loader" + - mkdir -p /root/.kube + - echo ${KUBECONF_DEV} | base64 -d > /root/.kube/config + - helm init + - helm delete --purge oxo-loader-$CI_COMMIT_REF_NAME || true + - sleep 10 + # - helm install --namespace oxo --debug --set image.tag=$CI_COMMIT_SHA,oxoRelease=$CI_COMMIT_REF_NAME --name oxo-loader-$CI_COMMIT_REF_NAME ./k8chart/oxo-loader --wait + - helm install --namespace oxo --debug --set image.tag=dev,oxoRelease=$CI_COMMIT_REF_NAME --name oxo-loader-$CI_COMMIT_REF_NAME ./k8chart/oxo-loader --wait + - kubectl wait --for=condition=complete --timeout=24h --namespace oxo job/oxo-loader-$CI_COMMIT_REF_NAME-job + +runIndexer: + image: dtzar/helm-kubectl:2.13.1 + when: manual + stage: indexing + script: + - echo "Run $CI_COMMIT_REF_NAME indexer" + - mkdir -p /root/.kube + - echo ${KUBECONF_DEV} | base64 -d > /root/.kube/config + - helm init + - helm delete --purge oxo-indexer-$CI_COMMIT_REF_NAME || true + - sleep 10 + - helm install --namespace oxo --debug --set image.tag=dev,oxoRelease=$CI_COMMIT_REF_NAME --name oxo-indexer-$CI_COMMIT_REF_NAME ./k8chart/oxo-indexer --wait + - kubectl wait --for=condition=complete --timeout=24h --namespace oxo job/oxo-indexer-$CI_COMMIT_REF_NAME-job From 0ba7fd14fc27002e16b9c8ede99f1b451fd56212 Mon Sep 17 00:00:00 2001 From: James McLaughlin Date: Thu, 23 Jul 2020 16:22:19 +0100 Subject: [PATCH 4/6] k8s deployment updates --- .gitlab-ci.yml | 4 +-- job_wait.sh | 29 +++++++++++++++++++ .../templates/oxo-indexer-job.yaml | 12 ++++++-- .../templates/oxo-loader-configmap.yaml | 6 ++-- .../oxo-loader/templates/oxo-loader-job.yaml | 10 +++++-- .../oxo/templates/oxo-neo4j-deployment.yaml | 1 + .../oxo/templates/oxo-solr-deployment.yaml | 1 + k8chart/oxo/templates/oxo-web-deployment.yaml | 1 + oxo-web/src/main/resources/static/css/oxo.css | 6 +++- 9 files changed, 60 insertions(+), 10 deletions(-) create mode 100755 job_wait.sh diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 6090b9d..3d90094 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -63,7 +63,7 @@ runLoader: - sleep 10 # - helm install --namespace oxo --debug --set image.tag=$CI_COMMIT_SHA,oxoRelease=$CI_COMMIT_REF_NAME --name oxo-loader-$CI_COMMIT_REF_NAME ./k8chart/oxo-loader --wait - helm install --namespace oxo --debug --set image.tag=dev,oxoRelease=$CI_COMMIT_REF_NAME --name oxo-loader-$CI_COMMIT_REF_NAME ./k8chart/oxo-loader --wait - - kubectl wait --for=condition=complete --timeout=24h --namespace oxo job/oxo-loader-$CI_COMMIT_REF_NAME-job + - ./job_wait.sh oxo job/oxo-loader-$CI_COMMIT_REF_NAME-job runIndexer: image: dtzar/helm-kubectl:2.13.1 @@ -77,5 +77,5 @@ runIndexer: - helm delete --purge oxo-indexer-$CI_COMMIT_REF_NAME || true - sleep 10 - helm install --namespace oxo --debug --set image.tag=dev,oxoRelease=$CI_COMMIT_REF_NAME --name oxo-indexer-$CI_COMMIT_REF_NAME ./k8chart/oxo-indexer --wait - - kubectl wait --for=condition=complete --timeout=24h --namespace oxo job/oxo-indexer-$CI_COMMIT_REF_NAME-job + - ./job_wait.sh oxo job/oxo-indexer-$CI_COMMIT_REF_NAME-job diff --git a/job_wait.sh b/job_wait.sh new file mode 100755 index 0000000..20c1f6a --- /dev/null +++ b/job_wait.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash + + +# https://stackoverflow.com/questions/55073453/wait-for-kubernetes-job-to-complete-on-either-failure-success-using-command-line + + +# wait for completion as background process - capture PID +kubectl wait --timeout=2h --for=condition=complete -n $1 $2 & +completion_pid=$! + +# wait for failure as background process - capture PID +kubectl wait --timeout=2h --for=condition=failed -n $1 $2 && exit 1 & +failure_pid=$! + +# capture exit code of the first subprocess to exit +wait -n $completion_pid $failure_pid + +# store exit code in variable +exit_code=$? + +if (( $exit_code == 0 )); then + echo "Job completed" +else + echo "Job failed with exit code ${exit_code}, exiting..." +fi + +kubectl logs -n $1 $2 + +exit $exit_code diff --git a/k8chart/oxo-indexer/templates/oxo-indexer-job.yaml b/k8chart/oxo-indexer/templates/oxo-indexer-job.yaml index e6e1574..e230cf3 100644 --- a/k8chart/oxo-indexer/templates/oxo-indexer-job.yaml +++ b/k8chart/oxo-indexer/templates/oxo-indexer-job.yaml @@ -3,13 +3,22 @@ kind: Job metadata: name: {{ .Release.Name }}-job spec: + backoffLimit: 0 + restartPolicy: Never template: metadata: name: {{ .Release.Name }}-job spec: + restartPolicy: Never containers: - name: oxo-indexer image: {{.Values.image.name}}:{{.Values.image.tag}} + command: [ 'java', '-Xmx10g', '-jar', 'oxo-indexer.jar' ] + resources: + limits: + memory: "15Gi" + requests: + memory: "15Gi" env: - name: OXO_NEO_DRIVER value: "org.neo4j.ogm.drivers.http.driver.HttpDriver" @@ -18,5 +27,4 @@ spec: - name: SPRING_DATA_SOLR_HOST value: "http://oxo-{{.Values.oxoRelease}}-solr:8983/solr" - name: OXO_INDEXER_CHUNKS - value: "100000" - restartPolicy: Never + value: "5000" diff --git a/k8chart/oxo-loader/templates/oxo-loader-configmap.yaml b/k8chart/oxo-loader/templates/oxo-loader-configmap.yaml index 44da8e1..19dcd05 100644 --- a/k8chart/oxo-loader/templates/oxo-loader-configmap.yaml +++ b/k8chart/oxo-loader/templates/oxo-loader-configmap.yaml @@ -5,11 +5,11 @@ metadata: data: config.ini: |- [Basics] - oxoUrl=http://oxo-{{ .Values.oxoRelease }}-web:8080 + oxoUrl=http://oxo-{{ .Values.oxoRelease }}-web.oxo.svc.cluster.local:8080 oxoAPIkey=key - olsSolrBaseUrl=http://ves-hx-7e:8993/solr + olsSolrBaseUrl=http://ves-hx-7e.ebi.ac.uk:8993/solr solrChunks=5000 - neoURL=bolt://oxo-{{ .Values.oxoRelease }}-neo4j:7687 + neoURL=bolt://oxo-{{ .Values.oxoRelease }}-neo4j.oxo.svc.cluster.local:7687 neoUser=neo4j neoPass=dba olsurl=https://www.ebi.ac.uk/ols/api diff --git a/k8chart/oxo-loader/templates/oxo-loader-job.yaml b/k8chart/oxo-loader/templates/oxo-loader-job.yaml index 4e46eec..be1e914 100644 --- a/k8chart/oxo-loader/templates/oxo-loader-job.yaml +++ b/k8chart/oxo-loader/templates/oxo-loader-job.yaml @@ -2,14 +2,21 @@ apiVersion: batch/v1 kind: Job metadata: name: {{ .Release.Name }}-job +backoffLimit: 0 +concurrencyPolicy: Forbid +failedJobsHistoryLimit: 1 +restartPolicy: Never spec: + restartPolicy: Never template: metadata: name: {{ .Release.Name }}-job spec: + restartPolicy: Never containers: - name: oxo-loader image: {{.Values.image.name}}:{{.Values.image.tag}} + imagePullPolicy: Always command: ['bash', '/opt/oxo-loader/load_all.sh'] volumeMounts: - name: oxoloader-config @@ -18,7 +25,7 @@ spec: - name: oxo-{{.Values.oxoRelease}}-neo4jimport mountPath: "/mnt/neo4jimport" env: - - name: NEO4IMPORT + - name: NEO4JIMPORT value: '/mnt/neo4jimport' - name: CONFIG_INI value: '/opt/oxo-config/config.ini' @@ -27,7 +34,6 @@ spec: envFrom: - configMapRef: name: oxo-{{ .Values.oxoRelease }}-proxy-configmap - restartPolicy: Never volumes: - name: oxoloader-config configMap: diff --git a/k8chart/oxo/templates/oxo-neo4j-deployment.yaml b/k8chart/oxo/templates/oxo-neo4j-deployment.yaml index f9f34bf..21c9213 100644 --- a/k8chart/oxo/templates/oxo-neo4j-deployment.yaml +++ b/k8chart/oxo/templates/oxo-neo4j-deployment.yaml @@ -17,6 +17,7 @@ spec: containers: - name: neo4j image: neo4j:3.1.1 + imagePullPolicy: Always env: # - name: NEO4J_HEAP_MEMORY # value: 10g diff --git a/k8chart/oxo/templates/oxo-solr-deployment.yaml b/k8chart/oxo/templates/oxo-solr-deployment.yaml index 9476771..942df34 100644 --- a/k8chart/oxo/templates/oxo-solr-deployment.yaml +++ b/k8chart/oxo/templates/oxo-solr-deployment.yaml @@ -24,6 +24,7 @@ spec: containers: - name: solr image: solr:5.5.3-alpine + imagePullPolicy: Always command: ['solr-precreate', 'mapping'] ports: - containerPort: 8983 diff --git a/k8chart/oxo/templates/oxo-web-deployment.yaml b/k8chart/oxo/templates/oxo-web-deployment.yaml index a5f1b85..16ff71d 100644 --- a/k8chart/oxo/templates/oxo-web-deployment.yaml +++ b/k8chart/oxo/templates/oxo-web-deployment.yaml @@ -17,6 +17,7 @@ spec: containers: - name: web image: {{.Values.image.name}}:{{.Values.image.tag}} + imagePullPolicy: Always ports: - containerPort: 8080 env: diff --git a/oxo-web/src/main/resources/static/css/oxo.css b/oxo-web/src/main/resources/static/css/oxo.css index 6e02c4f..ee4110c 100644 --- a/oxo-web/src/main/resources/static/css/oxo.css +++ b/oxo-web/src/main/resources/static/css/oxo.css @@ -155,4 +155,8 @@ a .ontology-source { .listingblock .content .highlight{ white-space: pre !important; -} \ No newline at end of file +} + +#mapping-vis .link { + stroke: red; +} From 5ca43cb948d05fc5af57ec95b5e9808121cb6a29 Mon Sep 17 00:00:00 2001 From: James McLaughlin Date: Mon, 27 Jul 2020 12:31:35 +0100 Subject: [PATCH 5/6] Restore broken api docs --- oxo-web/src/test/java/ApiDocumentation.java | 36 ++++++++------------- 1 file changed, 13 insertions(+), 23 deletions(-) diff --git a/oxo-web/src/test/java/ApiDocumentation.java b/oxo-web/src/test/java/ApiDocumentation.java index 412173d..39e6194 100644 --- a/oxo-web/src/test/java/ApiDocumentation.java +++ b/oxo-web/src/test/java/ApiDocumentation.java @@ -11,8 +11,6 @@ import org.neo4j.ogm.response.model.QueryResultModel; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.SpringApplicationConfiguration; -import org.springframework.boot.test.autoconfigure.restdocs.AutoConfigureRestDocs; -import org.springframework.boot.test.autoconfigure.web.servlet.WebMvcTest; import org.springframework.boot.test.context.SpringBootTest; import org.springframework.boot.test.mock.mockito.MockBean; import org.springframework.data.domain.Page; @@ -32,14 +30,8 @@ import org.springframework.test.web.servlet.setup.MockMvcBuilders; import org.springframework.web.context.WebApplicationContext; import uk.ac.ebi.spot.OxoWebApp; -import uk.ac.ebi.spot.controller.api.MappingController; -import uk.ac.ebi.spot.model.Datasource; -import uk.ac.ebi.spot.model.Mapping; -import uk.ac.ebi.spot.model.SourceType; -import uk.ac.ebi.spot.model.Term; -import uk.ac.ebi.spot.service.DatasourceService; -import uk.ac.ebi.spot.service.MappingService; -import uk.ac.ebi.spot.service.TermService; +import uk.ac.ebi.spot.model.*; +import uk.ac.ebi.spot.service.*; import javax.servlet.RequestDispatcher; @@ -70,7 +62,6 @@ @RunWith(SpringJUnit4ClassRunner.class) @SpringBootTest(classes = OxoWebApp.class) @WebAppConfiguration -/*@Ignore*/ public class ApiDocumentation { @Rule @@ -103,7 +94,8 @@ public class ApiDocumentation { @Before public void setUp() { - System.out.print("Start the Tests"); + Mockito.when(neo4jTemplate.query(Mockito.anyString(), Mockito.anyMap(), Mockito.anyBoolean())).thenReturn(new QueryResultModel(null, null)); + this.document = document("{method-name}" , preprocessRequest(prettyPrint()), @@ -112,16 +104,16 @@ public void setUp() { this.mockMvc = MockMvcBuilders.webAppContextSetup(this.context) .apply(documentationConfiguration(this.restDocumentation).uris() - .withScheme("https") - .withHost("www.ebi.ac.uk/spot/oxo") - .withPort(80) + .withScheme("https") + .withHost("www.ebi.ac.uk") + .withPort(443) ) .alwaysDo(this.document) .build(); } - /* + @Test public void pageExample () throws Exception { @@ -153,7 +145,8 @@ public void pageExample () throws Exception { this.mockMvc.perform(requestBuilder) .andExpect(status().isOk()); - }*/ + + } @Test @@ -179,8 +172,6 @@ public void errorExample() throws Exception { ; } - - /* @Test public void apiExample () throws Exception { @@ -197,7 +188,7 @@ public void apiExample () throws Exception { ); this.mockMvc.perform(get("/spot/oxo/api").contextPath("/spot/oxo").accept(MediaType.APPLICATION_JSON)) .andExpect(status().isOk()); - }*/ + } @Test @@ -234,7 +225,6 @@ public void mappingsListExample () throws Exception { } - /* @Test public void mappingExample () throws Exception { @@ -374,5 +364,5 @@ public void searchByIdsToJson () throws Exception { ObjectMapper mapper = new ObjectMapper(); this.mockMvc.perform(post("/spot/oxo/api/search").contextPath("/spot/oxo").contentType(MediaType.APPLICATION_JSON).accept(MediaType.APPLICATION_JSON).content(mapper.writeValueAsString(searchRequest))) .andExpect(status().isOk()); - }*/ -} + } +} \ No newline at end of file From d63b7157e02a3ba6501b50948ff7df243091db08 Mon Sep 17 00:00:00 2001 From: James McLaughlin Date: Mon, 27 Jul 2020 12:36:22 +0100 Subject: [PATCH 6/6] make api doc headers readable --- oxo-web/src/main/resources/static/css/oxo.css | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/oxo-web/src/main/resources/static/css/oxo.css b/oxo-web/src/main/resources/static/css/oxo.css index ee4110c..dd6d4aa 100644 --- a/oxo-web/src/main/resources/static/css/oxo.css +++ b/oxo-web/src/main/resources/static/css/oxo.css @@ -160,3 +160,7 @@ a .ontology-source { #mapping-vis .link { stroke: red; } + +#docs-content h1 { + color: black !important; +}