Merge branch 'master' into development

.gitignore

@@ -37,3 +37,4 @@ processed_files/
 queries.txt
 
 graph_comparison.csv
+

Capfile

@@ -27,14 +27,12 @@ install_plugin Capistrano::SCM::Git
 #   https://github.com/capistrano/passenger
 #
 # require "capistrano/rvm"
-require "capistrano/rbenv"
+# require "capistrano/rbenv"
 # require "capistrano/chruby"
-require "capistrano/bundler"
+# require "capistrano/bundler"
 # require "capistrano/rails/assets"
 # require "capistrano/rails/migrations"
 # require "capistrano/passenger"
-require 'capistrano/locally'
-#require 'new_relic/recipes' # announce deployments in NewRelic
 
 # Load custom tasks from `lib/capistrano/tasks` if you have any defined
 Dir.glob("lib/capistrano/tasks/*.rake").each { |r| import r }

Gemfile

@@ -23,7 +23,7 @@ gem 'request_store'
 gem 'parallel'
 gem 'json-ld'
 gem 'ffi', '~> 1.16.3'
-gem 'activesupport', '~> 3.2'
+gem 'activesupport', '~> 3.2.22.5'
 
 # Monitoring
 gem 'cube-ruby', require: 'cube'
@@ -41,7 +41,7 @@ group :test do
   gem 'test-unit-minitest'
 end
 
-group :deployment do
+group :development do
   # bcrypt_pbkdf and ed35519 is required for capistrano deployments when using ed25519 keys; see https://github.com/miloserdow/capistrano-deploy/issues/42
   gem 'bcrypt_pbkdf', '>= 1.0', '< 2.0', require: false
   gem 'capistrano', '~> 3', require: false

Gemfile.lock

@@ -89,7 +89,7 @@ GEM
     binding_of_caller (1.0.1)
       debug_inspector (>= 1.2.0)
     builder (3.3.0)
-    capistrano (3.19.2)
+    capistrano (3.19.1)
       airbrussh (>= 1.0.0)
       i18n
       rake (>= 10.0.0)
@@ -332,7 +332,7 @@ PLATFORMS
   x86_64-linux
 
 DEPENDENCIES
-  activesupport (~> 3.2)
+  activesupport (~> 3.2.22.5)
   bcrypt_pbkdf (>= 1.0, < 2.0)
   binding_of_caller (~> 1.0)
   capistrano (~> 3)

bin/migrations/compare_counts.rb

@@ -13,6 +13,7 @@
 PROCESSED_DIR = ARGV[0] || './processed_files'
 profile = ARGV[1]
 
+
 case profile
 when 'ag'
   # AllegroGraph backend
@@ -103,10 +104,9 @@ def compare_graphs_with_files(graph_triples)
       graph_filename = graphs_files[graph_uri]
 
       next csv << [graph_uri, triples_count, "Graph not found", "N/A"] unless graph_filename
-
+      
       # Construct the expected file name based on the graph URI
       file_name = "#{PROCESSED_DIR}/#{graph_filename}"
-
       # puts "count lines of the file #{file_name} for the graph #{graph_uri}"
       if File.exist?(file_name)
         file_lines_count = count_file_lines(file_name)

config/deploy.rb

@@ -1,105 +1,26 @@
-set :author, "ontoportal-lirmm"
-set :application, "ncbo_cron"
-set :repo_url, "https://github.com/#{fetch(:author)}/#{fetch(:application)}.git"
-
-set :deploy_via, :remote_cache
-
-# Default branch is :master
-# ask :branch, `git rev-parse --abbrev-ref HEAD`.chomp
-
-# Default deploy_to directory is /var/www/my_app_name
-set :deploy_to, "/srv/ontoportal/#{fetch(:application)}"
-
-# Default value for :log_level is :debug
-set :log_level, :debug
-
-# Default value for :linked_files is []
-# append :linked_files, "config/database.yml", 'config/master.key'
-
-# Default value for linked_dirs is []
-# set :linked_dirs, %w{log tmp/pids tmp/cache tmp/sockets vendor/bundle public/system}
-set :linked_dirs, %w{log vendor/bundle tmp/pids tmp/sockets public/system}
+set :repo_url, "[email protected]:biodivportal/ncbo_cron.git"
+set :user, 'ontoportal'
 
+set :deploy_to, '/srv/ontoportal/ncbo_cron_deployments'
 
-# Default value for keep_releases is 5
-set :keep_releases, 5
-set :config_folder_path, "#{fetch(:application)}/#{fetch(:stage)}"
 
+set :stages, %w[appliance]
+set :default_stage, 'appliance'
+set :stage, 'appliance'
+set :application, 'cron'
 
-# If you want to restart using `touch tmp/restart.txt`, add this to your config/deploy.rb:
+# SSH parameters
+set :ssh_port, 22
+set :pty, true
 
-SSH_JUMPHOST = ENV.include?('SSH_JUMPHOST') ? ENV['SSH_JUMPHOST'] : 'jumpbox.hostname.com'
-SSH_JUMPHOST_USER = ENV.include?('SSH_JUMPHOST_USER') ? ENV['SSH_JUMPHOST_USER'] : 'username'
-
-JUMPBOX_PROXY = "#{SSH_JUMPHOST_USER}@#{SSH_JUMPHOST}"
-set :ssh_options, {
-  user: 'ontoportal',
-  forward_agent: 'true',
-  keys: %w(config/deploy_id_rsa),
-  auth_methods: %w(publickey),
-  # use ssh proxy if API servers are on a private network
-  proxy: Net::SSH::Proxy::Command.new("ssh #{JUMPBOX_PROXY} -W %h:%p")
-}
-
-# private git repo for configuraiton
-PRIVATE_CONFIG_REPO = ENV.include?('PRIVATE_CONFIG_REPO') ? ENV['PRIVATE_CONFIG_REPO'] : 'https://[email protected]/your_organization/ontoportal-configs.git'
-desc "Check if agent forwarding is working"
-task :forwarding do
-  on roles(:all) do |h|
-    if test("env | grep SSH_AUTH_SOCK")
-      info "Agent forwarding is up to #{h}"
-    else
-      error "Agent forwarding is NOT up to #{h}"
-    end
-  end
-end
-
-# Smoke test for checking if the service is up
-desc 'Smoke test: Check if ncbo_cron service is running'
-task :smoke_test do
-  on roles(:app), in: :sequence, wait: 5 do
-    # Check if the service is running using systemctl
-    result = `systemctl is-active ncbo_cron`
-    if result.strip == 'active'
-      info "ncbo_cron service is up and running!"
-    else
-      error "ncbo_cron service failed to start."
-    end
-  end
-end
-
-namespace :deploy do
-
-  desc 'Incorporate the private repository content'
-  # Get cofiguration from repo if PRIVATE_CONFIG_REPO env var is set
-  # or get config from local directory if LOCAL_CONFIG_PATH env var is set
-  task :get_config do
-    if defined?(PRIVATE_CONFIG_REPO)
-      TMP_CONFIG_PATH = "/tmp/#{SecureRandom.hex(15)}".freeze
-      on roles(:app) do
-        execute "git clone -q #{PRIVATE_CONFIG_REPO} #{TMP_CONFIG_PATH}"
-        execute "rsync -av #{TMP_CONFIG_PATH}/#{fetch(:config_folder_path)}/ #{release_path}/"
-        execute "rm -rf #{TMP_CONFIG_PATH}"
-      end
-    elsif defined?(LOCAL_CONFIG_PATH)
-      on roles(:app) do
-        execute "rsync -av #{LOCAL_CONFIG_PATH}/#{fetch(:application)}/ #{release_path}/"
-      end
-    end
-  end
+# Source code
+set :repository_cache, "git_cache"
+set :deploy_via, :remote_cache
+set :ssh_options, { :forward_agent => true }
 
-  desc 'Restart application'
-  task :restart do
-    on roles(:app), in: :sequence, wait: 5 do
-      # Your restart mechanism here, for example:
-      # execute :touch, release_path.join('tmp/restart.txt')
-      execute 'sudo systemctl restart ncbo_cron'
-      execute 'sleep 5'
-    end
-  end
+# Linked files and directories
+append :linked_files, "config/config.rb"
+append :linked_dirs, 'logs', '.bundle'
+set :keep_releases, 2
 
-  after :updating, :get_config
-  after :publishing, :restart
-  after :restart, :smoke_test
 
-end

config/deploy/production.rb

@@ -0,0 +1,13 @@
+set :branch, 'master'
+set :server, '192.168.0.22'
+
+server fetch(:server), user: fetch(:user), roles: %w{web app}
+
+set :ssh_options, {
+  user: 'ontoportal',
+  forward_agent: 'true',
+  #keys: %w(config/deploy_id_rsa),
+  auth_methods: %w(publickey),
+  # use ssh proxy if UI servers are on a private network
+  proxy: Net::SSH::Proxy::Command.new('ssh [email protected] -W %h:%p')
+}

config/deploy/test.rb

@@ -1,16 +1,14 @@
-# Simple Role Syntax
-# ==================
-# Supports bulk-adding hosts to roles, the primary
-# server in each group is considered to be the first
-# unless any hosts have the primary property set.
-# Don't declare `role :all`, it's a meta role
-role :app, %w{testportal.lirmm.fr}
-set :branch, ENV.include?('BRANCH') ? ENV['BRANCH'] : 'development'
-# Extended Server Syntax
-# ======================
-# This can be used to drop a more detailed server
-# definition into the server list. The second argument
-# something that quacks like a hash can be used to set
-# extended properties on the server.
-#server 'example.com', user: 'deploy', roles: %w{web app}, my_property: :my_value
-set :log_level, :error
+set :branch, 'master'
+set :server, 'biodivportal.gfbio.dev'
+
+server fetch(:server), user: fetch(:user), roles: %w{web app}
+
+set :ssh_options, {
+  user: 'ontoportal',
+  forward_agent: 'true',
+  port: 30082
+  #keys: %w(config/deploy_id_rsa),
+  #auth_methods: %w(publickey),
+  # use ssh proxy if UI servers are on a private network
+  #proxy: Net::SSH::Proxy::Command.new('ssh [email protected] -W %h:%p')
+}

start_ontoportal_services.sh

@@ -0,0 +1,85 @@
+#!/usr/bin/env bash
+profile=$1
+acronym=$2
+set -e
+
+
+if [ -z "$profile" ]; then
+  echo "Usage: $0 <acronym> <profile>"
+  exit 1
+fi
+
+BACKEND_TYPE=$profile
+if [ "$BACKEND_TYPE" == "ag" ]; then
+  # AllegroGraph backend
+  export GOO_BACKEND_NAME="allegrograph"
+  export GOO_PORT="10035"
+  export GOO_PATH_QUERY="/repositories/ontoportal_test"
+  export GOO_PATH_DATA="/repositories/ontoportal_test/statements"
+  export GOO_PATH_UPDATE="/repositories/ontoportal_test/statements"
+  export COMPOSE_PROFILES="ag"
+
+elif [ "$BACKEND_TYPE" == "fs" ]; then
+  # 4store backend
+  export GOO_PORT="9000"
+  export COMPOSE_PROFILES="fs"
+
+elif [ "$BACKEND_TYPE" == "vo" ]; then
+  # Virtuoso backend
+  export GOO_BACKEND_NAME="virtuoso"
+  export GOO_PORT="8890"
+  export GOO_PATH_QUERY="/sparql"
+  export GOO_PATH_DATA="/sparql"
+  export GOO_PATH_UPDATE="/sparql"
+  export COMPOSE_PROFILES="vo"
+
+elif [ "$BACKEND_TYPE" == "gb" ]; then
+  # Graphdb backend
+  export GOO_BACKEND_NAME="graphdb"
+  export GOO_PORT="7200"
+  export GOO_PATH_QUERY="/repositories/ontoportal"
+  export GOO_PATH_DATA="/repositories/ontoportal/statements"
+  export GOO_PATH_UPDATE="/repositories/ontoportal/statements"
+else
+  echo "Error: Unknown backend type. Please set BACKEND_TYPE to 'ag', 'fs', or 'vo'."
+fi
+
+echo "###########################################################################"
+echo "Stop and remove all containers, networks, and volumes and start fresh"
+docker compose --profile fs --profile vo --profile gb --profile ag down  --volumes --remove-orphans && docker compose --profile "$profile" up -d
+
+echo "Waiting for all Docker services to start..."
+
+while true; do
+    # Get the status of all containers
+    container_status=$(docker compose --profile "$profile" ps -a --format '{{.Names}} {{.State}}')
+
+    all_running=true
+    while read -r container state; do
+        if [ "$state" != "running" ] && [ "$state" != "exited" ]; then
+            all_running=false
+            break
+        fi
+    done <<< "$container_status"
+
+    # If all containers are running, exit the loop
+    if [ "$all_running" = true ]; then
+        echo "All containers are running!"
+        break
+    fi
+
+    # Wait before checking again
+    sleep 2
+done
+
+if [ -z "$acronym" ]; then
+  exit 0
+fi
+
+echo "###########################################################################"
+echo "Create a new user and make it an admin"
+bundle exec rake user:create[admin,[email protected],password]
+bundle exec rake user:adminify[admin]
+echo "###########################################################################"
+echo "Create a new ontology $acronym and import it from a remote server"
+bin/ncbo_ontology_import --admin-user admin -o "$acronym" --from https://data.stageportal.lirmm.fr --from-apikey 82602563-4750-41be-9654-36f46056a0db

test/data/virtuoso.ini

@@ -0,0 +1,61 @@
+[Database]
+DatabaseFile               = ./database/virtuoso.db
+ErrorLogFile               = ./database/virtuoso.log
+TransactionFile            = ./database/virtuoso.trx
+xa_persistent_file         = ./database/virtuoso.pxa
+MaxCheckpointRemap         = 200000
+CheckpointInterval         = 60
+NumberOfBuffers            = 2450000	; Each buffer is 8KB, so ~19GB total
+MaxDirtyBuffers            = 1837500	; About 75% of NumberOfBuffers
+TransactionAfterImageLimit = 50000000
+; NumberOfBuffers            = 1000000
+MaxStaticCursorRows        = 5000
+Striping                   = 0
+TempStorage                = .
+ErrorLogLevel              = 7
+
+[HTTPServer]
+ServerPort           = 8890
+ServerRoot           = ./var/lib/virtuoso/vsp
+MaxClientConnections = 200
+MaxKeepAlives        = 10
+KeepAliveTimeout     = 10
+ServerThreads        = 50
+HttpTimeout          = 300
+MaxBody              = 20000000
+EnableGzip           = 1
+GzipMimeType         = text/html, text/xml, text/plain, text/css, application/xml, application/xhtml+xml, application/rss+xml, application/javascript, application/x-javascript, image/svg+xml
+HTTPLogFile          = ./http17012025.log
+
+[Parameters]
+ServerPort                 = 1111
+NumOfThreads               = 100
+MaxMem                     = 20000000000	; 5GB memory
+ResultSetMaxRows           = 10000
+DirsAllowed                = ., ./vad, ./virtuoso, ../migration-to-virtuoso,../migration-to-virtuoso/processed_files
+MaxQueryCostEstimationTime = 6000
+MaxQueryExecutionTime      = 6000
+DynamicLocal               = 1
+LogEnable                  = 2	; Enable SPARQL query logging
+TraceOn                    = errors
+LogFile                    = virtuoso.log
+NumberOfBuffers            = 2450000	; Each buffer is 8KB, so ~19GB total
+MaxDirtyBuffers            = 1837500	; About 75% of NumberOfBuffers
+
+[VDB]
+ArrayOptimization     = 0
+NumArrayParams        = 0
+VDBDisconnectTimeout  = 1000
+KeepAliveTimeout      = 60
+RetryCount            = 3
+ThreadCleanupInterval = 600
+
+[Replication]
+ServerName   = virtuoso
+ServerEnable = 1
+
+[SPARQL]
+ResultSetMaxRows      = 1000000000000
+MaxQueryExecutionTime = 6000
+DefaultGraph          = http://localhost:8890/sparql
+MaxSortedTopRows      = 10000