Merge pull request #1 from EBISPOT/master

Merge changes from OLS main branch

.gitlab-ci.yml

@@ -1,6 +1,13 @@
 
+stages:
+  - build
+  - deploy
+  - dataload
+  - indexing
+
 build:
   image: docker:18-git
+  when: manual
   stage: build
   only:
     - dev
@@ -9,17 +16,66 @@ build:
   services:
     - docker:18-dind
   script:
+    - echo docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
     - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
-    - docker build -t ${OXO_IMAGE_PREFIX}/oxo-loader:$CI_COMMIT_REF_NAME-$CI_BUILD_ID -f  ./oxo-loader/Dockerfile .
-    - docker build -t ${OXO_IMAGE_PREFIX}/oxo-indexer:$CI_COMMIT_REF_NAME-$CI_BUILD_ID -f  ./oxo-indexer/Dockerfile .
-    - docker build -t ${OXO_IMAGE_PREFIX}/oxo-web:$CI_COMMIT_REF_NAME-$CI_BUILD_ID -f  ./oxo-web/Dockerfile .
-    - docker tag ${OXO_IMAGE_PREFIX}/oxo-loader:$CI_COMMIT_REF_NAME-$CI_BUILD_ID ${OXO_IMAGE_PREFIX}/oxo-loader:$CI_COMMIT_REF_NAME
-    - docker tag ${OXO_IMAGE_PREFIX}/oxo-indexer:$CI_COMMIT_REF_NAME-$CI_BUILD_ID ${OXO_IMAGE_PREFIX}/oxo-indexer:$CI_COMMIT_REF_NAME
-    - docker tag ${OXO_IMAGE_PREFIX}/oxo-web:$CI_COMMIT_REF_NAME-$CI_BUILD_ID ${OXO_IMAGE_PREFIX}/oxo-web:$CI_COMMIT_REF_NAME
-    - docker push ${OXO_IMAGE_PREFIX}/oxo-loader:$CI_COMMIT_REF_NAME-$CI_BUILD_ID
-    - docker push ${OXO_IMAGE_PREFIX}/oxo-loader:$CI_COMMIT_REF_NAME
-    - docker push ${OXO_IMAGE_PREFIX}/oxo-indexer:$CI_COMMIT_REF_NAME-$CI_BUILD_ID
-    - docker push ${OXO_IMAGE_PREFIX}/oxo-indexer:$CI_COMMIT_REF_NAME
-    - docker push ${OXO_IMAGE_PREFIX}/oxo-web:$CI_COMMIT_REF_NAME-$CI_BUILD_ID
-    - docker push ${OXO_IMAGE_PREFIX}/oxo-web:$CI_COMMIT_REF_NAME
+    - docker build -t $CI_REGISTRY/$OXO_IMAGE_PREFIX/oxo-loader:$CI_COMMIT_SHA -f  ./oxo-loader/Dockerfile .
+    - docker build -t $CI_REGISTRY/$OXO_IMAGE_PREFIX/oxo-indexer:$CI_COMMIT_SHA -f  ./oxo-indexer/Dockerfile .
+    - docker build -t $CI_REGISTRY/$OXO_IMAGE_PREFIX/oxo-web:$CI_COMMIT_SHA -f  ./oxo-web/Dockerfile .
+    - docker tag $CI_REGISTRY/$OXO_IMAGE_PREFIX/oxo-loader:$CI_COMMIT_SHA $CI_REGISTRY/$OXO_IMAGE_PREFIX/oxo-loader:$CI_COMMIT_REF_NAME
+    - docker tag $CI_REGISTRY/$OXO_IMAGE_PREFIX/oxo-indexer:$CI_COMMIT_SHA $CI_REGISTRY/$OXO_IMAGE_PREFIX/oxo-indexer:$CI_COMMIT_REF_NAME
+    - docker tag $CI_REGISTRY/$OXO_IMAGE_PREFIX/oxo-web:$CI_COMMIT_SHA $CI_REGISTRY/$OXO_IMAGE_PREFIX/oxo-web:$CI_COMMIT_REF_NAME
+    - docker push $CI_REGISTRY/$OXO_IMAGE_PREFIX/oxo-loader:$CI_COMMIT_SHA
+    - docker push $CI_REGISTRY/$OXO_IMAGE_PREFIX/oxo-loader:$CI_COMMIT_REF_NAME
+    - docker push $CI_REGISTRY/$OXO_IMAGE_PREFIX/oxo-indexer:$CI_COMMIT_SHA
+    - docker push $CI_REGISTRY/$OXO_IMAGE_PREFIX/oxo-indexer:$CI_COMMIT_REF_NAME
+    - docker push $CI_REGISTRY/$OXO_IMAGE_PREFIX/oxo-web:$CI_COMMIT_SHA
+    - docker push $CI_REGISTRY/$OXO_IMAGE_PREFIX/oxo-web:$CI_COMMIT_REF_NAME
+
+deploy:
+  image: dtzar/helm-kubectl:2.13.1
+  when: manual
+  stage: deploy
+  only:
+    - dev
+  script:
+    - echo "Deploy to $CI_COMMIT_REF_NAME server"
+    - mkdir -p /root/.kube
+    - echo ${KUBECONF_DEV} | base64 -d > /root/.kube/config
+    - helm init
+    - helm delete --purge oxo-$CI_COMMIT_REF_NAME || true
+    - sleep 10
+    - helm install --namespace oxo --debug --name oxo-$CI_COMMIT_REF_NAME-pvcs ./k8chart/pvcs --wait || true
+    # - helm install --namespace oxo --debug --set image.tag=$CI_COMMIT_SHA --name oxo-$CI_COMMIT_REF_NAME ./k8chart/oxo --wait
+    - helm install --namespace oxo --debug --set image.tag=dev --name oxo-$CI_COMMIT_REF_NAME ./k8chart/oxo --wait
+  environment:
+    name: dev
+
+runLoader:
+  image: dtzar/helm-kubectl:2.13.1
+  when: manual
+  stage: dataload
+  script:
+    - echo "Run $CI_COMMIT_REF_NAME loader"
+    - mkdir -p /root/.kube
+    - echo ${KUBECONF_DEV} | base64 -d > /root/.kube/config
+    - helm init
+    - helm delete --purge oxo-loader-$CI_COMMIT_REF_NAME || true
+    - sleep 10
+    # - helm install --namespace oxo --debug --set image.tag=$CI_COMMIT_SHA,oxoRelease=$CI_COMMIT_REF_NAME --name oxo-loader-$CI_COMMIT_REF_NAME ./k8chart/oxo-loader --wait
+    - helm install --namespace oxo --debug --set image.tag=dev,oxoRelease=$CI_COMMIT_REF_NAME --name oxo-loader-$CI_COMMIT_REF_NAME ./k8chart/oxo-loader --wait
+    - ./job_wait.sh oxo job/oxo-loader-$CI_COMMIT_REF_NAME-job
+
+runIndexer:
+  image: dtzar/helm-kubectl:2.13.1
+  when: manual
+  stage: indexing
+  script:
+    - echo "Run $CI_COMMIT_REF_NAME indexer"
+    - mkdir -p /root/.kube
+    - echo ${KUBECONF_DEV} | base64 -d > /root/.kube/config
+    - helm init
+    - helm delete --purge oxo-indexer-$CI_COMMIT_REF_NAME || true
+    - sleep 10
+    - helm install --namespace oxo --debug --set image.tag=dev,oxoRelease=$CI_COMMIT_REF_NAME --name oxo-indexer-$CI_COMMIT_REF_NAME ./k8chart/oxo-indexer --wait
+    - ./job_wait.sh oxo job/oxo-indexer-$CI_COMMIT_REF_NAME-job
 

README.md

@@ -34,7 +34,7 @@ The loader scripts are documented in the README of the oxo-loader/ directory.
 After using the loader to load data into neo4j, the indexer can be executed
 using Docker:
 
-    docker run --net=host EBISPOT/oxo-indexer:stable
+    docker run --net=host ebispot/oxo-indexer:dev
 
 
 # OxO without Docker

job_wait.sh

@@ -0,0 +1,29 @@
+#!/usr/bin/env bash
+
+
+# https://stackoverflow.com/questions/55073453/wait-for-kubernetes-job-to-complete-on-either-failure-success-using-command-line
+
+
+# wait for completion as background process - capture PID
+kubectl wait --timeout=2h --for=condition=complete -n $1 $2 &
+completion_pid=$!
+
+# wait for failure as background process - capture PID
+kubectl wait --timeout=2h --for=condition=failed -n $1 $2 && exit 1 &
+failure_pid=$! 
+
+# capture exit code of the first subprocess to exit
+wait -n $completion_pid $failure_pid
+
+# store exit code in variable
+exit_code=$?
+
+if (( $exit_code == 0 )); then
+  echo "Job completed"
+else
+  echo "Job failed with exit code ${exit_code}, exiting..."
+fi
+
+kubectl logs -n $1 $2
+
+exit $exit_code

k8chart/oxo-indexer/Chart.yaml

@@ -0,0 +1,5 @@
+apiVersion: v1
+appVersion: "1.0"
+description: oxo indexer
+name: oxo
+version: 0.1.0

k8chart/oxo-indexer/templates/oxo-indexer-job.yaml

@@ -0,0 +1,30 @@
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: {{ .Release.Name }}-job
+spec:
+  backoffLimit: 0
+  restartPolicy: Never
+  template:
+    metadata:
+      name: {{ .Release.Name }}-job
+    spec:
+      restartPolicy: Never
+      containers:
+      - name: oxo-indexer
+        image: {{.Values.image.name}}:{{.Values.image.tag}}
+        command: [ 'java', '-Xmx10g', '-jar', 'oxo-indexer.jar' ]
+        resources:
+          limits:
+            memory: "15Gi"
+          requests:
+            memory: "15Gi"
+        env:
+        - name: OXO_NEO_DRIVER
+          value: "org.neo4j.ogm.drivers.http.driver.HttpDriver"
+        - name: OXO_NEO_URI
+          value: "http://neo4j:dba@oxo-{{.Values.oxoRelease}}-neo4j:7474"
+        - name: SPRING_DATA_SOLR_HOST
+          value: "http://oxo-{{.Values.oxoRelease}}-solr:8983/solr"
+        - name: OXO_INDEXER_CHUNKS
+          value: "5000"

k8chart/oxo-indexer/values.yaml

@@ -0,0 +1,10 @@
+
+image:
+  name: ebispot/oxo-indexer
+  tag: stable
+
+
+oxoRelease: dev
+
+
+

k8chart/oxo-loader/Chart.yaml

@@ -0,0 +1,5 @@
+apiVersion: v1
+appVersion: "1.0"
+description: oxo loader
+name: oxo
+version: 0.1.0

k8chart/oxo-loader/templates/oxo-loader-configmap.yaml

@@ -0,0 +1,33 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ .Release.Name }}-configmap
+data:
+  config.ini: |-
+    [Basics]
+    oxoUrl=http://oxo-{{ .Values.oxoRelease }}-web.oxo.svc.cluster.local:8080
+    oxoAPIkey=key
+    olsSolrBaseUrl=http://ves-hx-7e.ebi.ac.uk:8993/solr
+    solrChunks=5000
+    neoURL=bolt://oxo-{{ .Values.oxoRelease }}-neo4j.oxo.svc.cluster.local:7687
+    neoUser=neo4j
+    neoPass=dba
+    olsurl=https://www.ebi.ac.uk/ols/api
+    oboDbxrefUrl=https://raw.githubusercontent.com/geneontology/go-site/master/metadata/db-xrefs.yaml
+
+    [Paths]
+    exportFileDatasources=datasources.csv
+    exportFileTerms=/path/terms.csv
+    exportFileMappings=/path/mappings.csv
+    idorgDataLocation = /path/idorg.xml
+
+    [SQLumls]
+    user=username
+    password=password
+    host=mysql-name
+    db=dbName
+    port=4570
+
+    [LOINC]
+    Part=/path/Part.csv
+    PartRelatedCodeMapping=/path/PartRelatedCodeMapping.csv

k8chart/oxo-loader/templates/oxo-loader-job.yaml

@@ -0,0 +1,43 @@
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: {{ .Release.Name }}-job
+backoffLimit: 0
+concurrencyPolicy: Forbid
+failedJobsHistoryLimit: 1
+restartPolicy: Never
+spec:
+  restartPolicy: Never
+  template:
+    metadata:
+      name: {{ .Release.Name }}-job
+    spec:
+      restartPolicy: Never
+      containers:
+      - name: oxo-loader
+        image:  {{.Values.image.name}}:{{.Values.image.tag}}
+        imagePullPolicy: Always
+        command: ['bash', '/opt/oxo-loader/load_all.sh']
+        volumeMounts:
+        - name: oxoloader-config
+          mountPath: "/opt/oxo-config"
+          readOnly: true
+        - name: oxo-{{.Values.oxoRelease}}-neo4jimport
+          mountPath: "/mnt/neo4jimport"
+        env:
+        - name: NEO4JIMPORT
+          value: '/mnt/neo4jimport'
+        - name: CONFIG_INI
+          value: '/opt/oxo-config/config.ini'
+        - name: IDORG_XML
+          value: '/opt/oxo-loader/idorg.xml'
+        envFrom:
+          - configMapRef:
+              name: oxo-{{ .Values.oxoRelease }}-proxy-configmap
+      volumes:
+      - name: oxoloader-config
+        configMap:
+          name: {{ .Release.Name }}-configmap
+      - name: oxo-{{.Values.oxoRelease}}-neo4jimport
+        persistentVolumeClaim:
+          claimName: oxo-{{.Values.oxoRelease}}-neo4jimport

k8chart/oxo-loader/values.yaml

@@ -0,0 +1,10 @@
+
+image:
+  name: ebispot/oxo-loader
+  tag: stable
+
+
+oxoRelease: dev
+
+
+

k8chart/oxo/.helmignore

@@ -0,0 +1,22 @@
+# Patterns to ignore when building packages.
+# This supports shell glob matching, relative path matching, and
+# negation (prefixed with !). Only one pattern per line.
+.DS_Store
+# Common VCS dirs
+.git/
+.gitignore
+.bzr/
+.bzrignore
+.hg/
+.hgignore
+.svn/
+# Common backup files
+*.swp
+*.bak
+*.tmp
+*~
+# Various IDEs
+.project
+.idea/
+*.tmproj
+.vscode/

k8chart/oxo/Chart.yaml

@@ -0,0 +1,5 @@
+apiVersion: v1
+appVersion: "1.0"
+description: oxo deployment
+name: oxo
+version: 0.1.0

k8chart/oxo/templates/oxo-configmap.yaml

@@ -0,0 +1,6 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ .Release.Name }}-configmap
+data:
+  myvalue: "Hello World"

k8chart/oxo/templates/oxo-ingress.yaml

@@ -0,0 +1,14 @@
+apiVersion: networking.k8s.io/v1beta1
+kind: Ingress
+metadata:
+  name: {{ .Release.Name }}-ingress
+  annotations:
+    nginx.ingress.kubernetes.io/rewrite-target: /
+spec:
+  rules:
+  - http:
+      paths:
+      - path: /
+        backend:
+          serviceName: {{ .Release.Name }}-web
+          servicePort: 8080

k8chart/oxo/templates/oxo-neo4j-deployment.yaml

@@ -0,0 +1,42 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name:  {{ .Release.Name }}-neo4j
+  labels:
+    app: {{ .Release.Name }}-neo4j
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: {{ .Release.Name }}-neo4j
+  template:
+    metadata:
+      labels:
+        app: {{ .Release.Name }}-neo4j
+    spec:
+      containers:
+        - name: neo4j
+          image: neo4j:3.1.1
+          imagePullPolicy: Always
+          env:
+              # - name: NEO4J_HEAP_MEMORY
+              # value: 10g
+              # - name: NEO4J_dbms_memory_heap_maxSize
+              # value: 8g
+            - name: NEO4J_AUTH
+              value: neo4j/dba
+          ports:
+            - containerPort: 7474
+            - containerPort: 7687
+          volumeMounts:
+            - name: {{.Release.Name}}-neo4jimport
+              mountPath: "/var/lib/neo4j/import"
+            - name: {{.Release.Name}}-neo4j-data
+              mountPath: "/data"
+      volumes:
+      - name: {{.Release.Name}}-neo4jimport
+        persistentVolumeClaim:
+          claimName: {{.Release.Name}}-neo4jimport
+      - name: {{.Release.Name}}-neo4j-data
+        persistentVolumeClaim:
+          claimName: {{.Release.Name}}-neo4j-data

k8chart/oxo/templates/oxo-neo4j-service.yaml

@@ -0,0 +1,18 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ .Release.Name }}-neo4j
+  labels:
+    app: {{ .Release.Name }}-neo4j
+spec:
+  ports:
+      - port: 7474
+        targetPort: 7474
+        name: http
+        protocol: TCP
+      - port: 7687
+        targetPort: 7687
+        name: bolt
+        protocol: TCP
+  selector:
+    app: {{ .Release.Name }}-neo4j