From abfdc71982d6718d0c5413e1bc9cd1bb924f039e Mon Sep 17 00:00:00 2001
From: Syphax <gs_bouazzouni@esi.dz>
Date: Sat, 1 Feb 2025 13:00:20 +0100
Subject: [PATCH] add benchmarking tests

---
 bin/migrations/count_graph_triples.rb         |  60 ++++++
 bin/migrations/virtuoso/import_to_virtuoso.sh |   0
 test/benchmarks/data_benchs.rb                |  84 +++++++++
 test/benchmarks/examples.md                   |   6 +
 test/benchmarks/import_all_metadata_file.sh   |  17 ++
 .../import_and_fetch_all_triples_nt_file.rb   |  77 ++++++++
 test/benchmarks/metadata_benchs.rb            | 178 ++++++++++++++++++
 .../parse_and_do_ontoportal_operations.rb     |  74 ++++++++
 test/benchmarks/run_metadata_benchs.rb        |  55 ++++++
 .../benchmarks/start_ontoportal_services.sh   |  19 +-
 10 files changed, 568 insertions(+), 2 deletions(-)
 create mode 100755 bin/migrations/count_graph_triples.rb
 create mode 100644 bin/migrations/virtuoso/import_to_virtuoso.sh
 create mode 100644 test/benchmarks/data_benchs.rb
 create mode 100644 test/benchmarks/examples.md
 create mode 100755 test/benchmarks/import_all_metadata_file.sh
 create mode 100644 test/benchmarks/import_and_fetch_all_triples_nt_file.rb
 create mode 100644 test/benchmarks/metadata_benchs.rb
 create mode 100644 test/benchmarks/parse_and_do_ontoportal_operations.rb
 create mode 100644 test/benchmarks/run_metadata_benchs.rb
 rename start_ontoportal_services.sh => test/benchmarks/start_ontoportal_services.sh (80%)

diff --git a/bin/migrations/count_graph_triples.rb b/bin/migrations/count_graph_triples.rb
new file mode 100755
index 00000000..db8aef65
--- /dev/null
+++ b/bin/migrations/count_graph_triples.rb
@@ -0,0 +1,60 @@
+# require 'bundler/setup'
+require 'pry'
+require 'benchmark'
+require 'ncbo_annotator'
+require 'ncbo_cron'
+require 'ontologies_linked_data'
+
+graph = ARGV[1]
+profile = ARGV[2]
+
+if graph.nil?
+  puts "Error: Missing arguments. Please provide the graph name."
+  exit(1)
+end
+
+case profile
+when 'ag'
+  # AllegroGraph backend
+  ENV['GOO_BACKEND_NAME'] = 'allegrograph'
+  ENV['GOO_PORT'] = '10035'
+  ENV['GOO_PATH_QUERY'] = '/repositories/ontoportal_test'
+  ENV['GOO_PATH_DATA'] = '/repositories/ontoportal_test/statements'
+  ENV['GOO_PATH_UPDATE'] = '/repositories/ontoportal_test/statements'
+  ENV['COMPOSE_PROFILES'] = 'ag'
+
+when 'fs'
+  # 4store backend
+  ENV['GOO_PORT'] = '9000'
+  ENV['COMPOSE_PROFILES'] = 'fs'
+
+when 'vo'
+  # Virtuoso backend
+  ENV['GOO_BACKEND_NAME'] = 'virtuoso'
+  ENV['GOO_PORT'] = '8890'
+  ENV['GOO_PATH_QUERY'] = '/sparql'
+  ENV['GOO_PATH_DATA'] = '/sparql'
+  ENV['GOO_PATH_UPDATE'] = '/sparql'
+  ENV['COMPOSE_PROFILES'] = 'vo'
+
+when 'gb'
+  # Graphdb backend
+  ENV['GOO_BACKEND_NAME'] = 'graphdb'
+  ENV['GOO_PORT'] = '7200'
+  ENV['GOO_PATH_QUERY'] = '/repositories/ontoportal'
+  ENV['GOO_PATH_DATA'] = '/repositories/ontoportal/statements'
+  ENV['GOO_PATH_UPDATE'] = '/repositories/ontoportal/statements'
+
+else
+  puts "Will import to default config set in config/config.rb"
+end
+
+require_relative '../../config/config'
+count = 0
+time = Benchmark.realtime do
+  rs = Goo.sparql_query_client.query("SELECT (COUNT(?s) as ?count) FROM <#{graph_uri}> WHERE { ?s ?p ?o }")
+  rs = rs.solutions.first
+  count = rs[:count].to_i  if rs
+end
+
+puts 'Imported triples in ' + format("%.4f", time) + 's with total count: ' + count.to_s
diff --git a/bin/migrations/virtuoso/import_to_virtuoso.sh b/bin/migrations/virtuoso/import_to_virtuoso.sh
new file mode 100644
index 00000000..e69de29b
diff --git a/test/benchmarks/data_benchs.rb b/test/benchmarks/data_benchs.rb
new file mode 100644
index 00000000..0cf455b2
--- /dev/null
+++ b/test/benchmarks/data_benchs.rb
@@ -0,0 +1,84 @@
+require 'ontologies_linked_data'
+module Benchmarks
+
+  def self.do_all_benchmarks(sub)
+    Benchmarks.bench("fetch triples") do
+      Benchmarks.paginate_all_triples(sub)
+    end
+
+    Benchmarks.bench("get ontology Concept Roots") do
+      Benchmarks.ontology_roots(sub)
+    end
+
+    Benchmarks.bench("concept children") do
+      Benchmarks.concept_children("http://terminologies.gfbio.org/ITIS/Taxa_0", sub)
+    end
+
+    Benchmarks.bench("concept path to root") do
+      Benchmarks.concept_tree("http://terminologies.gfbio.org/ITIS/Taxa_6007", sub)
+    end
+  end
+
+  def self.bench(label, &block)
+    time = Benchmark.realtime do
+      block.call
+    end
+    puts "Time to #{label}: " + time.round(2).to_s
+  end
+
+  def self.import_nt_file(sub, file_path)
+    Goo.sparql_data_client.delete_graph(sub.id)
+    Goo.sparql_data_client.append_triples_no_bnodes(sub.id, file_path, nil)
+  end
+
+  def self.paginate_all_triples(sub)
+    page = 1
+    pagesize = 10000
+    count = 1
+    total_count = 0
+    while count > 0 && page < 100
+      puts "Starting query for page #{page}"
+      offset = " OFFSET #{(page - 1) * pagesize}"
+      rs = Goo.sparql_query_client.query("SELECT ?s ?p ?o FROM <#{sub.id}> WHERE {  ?s ?p ?o  } LIMIT #{pagesize} #{offset}")
+      count = rs.each_solution.size
+      total_count += count
+      page += 1
+    end
+    puts "Total triples: " + total_count.to_s
+  end
+
+  def self.ontology_roots(sub)
+    load_attrs = LinkedData::Models::Class.goo_attrs_to_load([:all])
+    roots = []
+    time = Benchmark.realtime do
+      roots = sub.roots(load_attrs)
+    end
+    puts "Time to find roots: " + time.round(2).to_s
+    Goo.log_debug_file('roots')
+    time = Benchmark.realtime do
+      LinkedData::Models::Class.in(sub).models(roots).include(:unmapped).all
+    end
+    puts "Time to load roots: " + time.round(2).to_s
+    Goo.log_debug_file('roots')
+    puts "Roots count: " + roots.length.to_s
+  end
+
+  def self.concept_children(uri, sub)
+    page, size = [1, 100]
+    cls = LinkedData::Models::Class.find(RDF::URI.new("http://terminologies.gfbio.org/ITIS/Taxa_0")).in(sub).first
+    ld = LinkedData::Models::Class.goo_attrs_to_load([:all])
+    children = sub.children(cls, includes_param: ld, page: page, size: size)
+    puts "Children count: " + children.length.to_s
+  end
+
+  def self.concept_tree(uri, sub)
+    cls = LinkedData::Models::Class.find("http://terminologies.gfbio.org/ITIS/Taxa_6007").in(sub).first
+    display_attrs = [:prefLabel, :hasChildren, :children, :obsolete, :subClassOf]
+    extra_include = display_attrs + [:hasChildren, :isInActiveScheme, :isInActiveScheme]
+
+    roots = sub.roots(extra_include)
+    # path = cls.path_to_root(roots)
+    cls.tree(roots: roots)
+  end
+
+end
diff --git a/test/benchmarks/examples.md b/test/benchmarks/examples.md
new file mode 100644
index 00000000..ae1c11c7
--- /dev/null
+++ b/test/benchmarks/examples.md
@@ -0,0 +1,6 @@
+# Benchmarks
+## Import all portal metadata
+
+## Import AGROVOC and query all triples by pages
+## Parse ITIS and do ontoportal operations
+ruby test/benchmarks/parse_and_do_ontoportal_operations.rb ITIS fs 47a57aa3-7b54-4f34-b695-dbb5f5b7363e https://data.biodivportal.gfbio.dev 
diff --git a/test/benchmarks/import_all_metadata_file.sh b/test/benchmarks/import_all_metadata_file.sh
new file mode 100755
index 00000000..d3f3077b
--- /dev/null
+++ b/test/benchmarks/import_all_metadata_file.sh
@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+path_graphs_files=$1
+profile=$2
+set -e
+
+
+if [ -z "$profile" ]; then
+  echo "Usage: $0 <path to path_graphs_files> <profile>"
+  exit 1
+fi
+echo "###########################################################################"
+./test/benchmarks/start_ontoportal_services.sh "$profile"
+./bin/migrations/import_metadata_graphs_to_store "$path_graphs_files" "$profile"
+echo 'All metadata graphs imported successfully.'
+echo "###########################################################################"
+
+ruby bin/migrations/compare_counts.rb "$path_graphs_files" "$profile"
diff --git a/test/benchmarks/import_and_fetch_all_triples_nt_file.rb b/test/benchmarks/import_and_fetch_all_triples_nt_file.rb
new file mode 100644
index 00000000..a2350165
--- /dev/null
+++ b/test/benchmarks/import_and_fetch_all_triples_nt_file.rb
@@ -0,0 +1,77 @@
+
+# Documentation:
+# This script is used to import a large NT file into the triple store
+# and then fetch all the triples by paginating through the triples.
+# The script is used to compare the performance of the import and fetch of different backends.
+
+profile = ARGV[0]
+file_path = ARGV[1]
+acronym = ARGV[2] || 'STY' # Default to STY
+pwd = File.dirname(__FILE__)
+system("#{pwd}/start_ontoportal_services.sh #{profile} #{acronym}")
+
+if $?.exitstatus != 0
+  puts "Error occurred during script execution."
+  exit(1)
+end
+
+if file_path == nil
+  puts "Error: Missing arguments. Please provide the file path."
+  exit(1)
+end
+
+puts "Finished parsing file"
+case profile
+when 'ag'
+  # AllegroGraph backend
+  ENV['GOO_BACKEND_NAME'] = 'allegrograph'
+  ENV['GOO_PORT'] = '10035'
+  ENV['GOO_PATH_QUERY'] = '/repositories/ontoportal_test'
+  ENV['GOO_PATH_DATA'] = '/repositories/ontoportal_test/statements'
+  ENV['GOO_PATH_UPDATE'] = '/repositories/ontoportal_test/statements'
+  ENV['COMPOSE_PROFILES'] = 'ag'
+
+when 'fs'
+  # 4store backend
+  ENV['GOO_PORT'] = '9000'
+  ENV['COMPOSE_PROFILES'] = 'fs'
+
+when 'vo'
+  # Virtuoso backend
+  ENV['GOO_BACKEND_NAME'] = 'virtuoso'
+  ENV['GOO_PORT'] = '8890'
+  ENV['GOO_PATH_QUERY'] = '/sparql'
+  ENV['GOO_PATH_DATA'] = '/sparql'
+  ENV['GOO_PATH_UPDATE'] = '/sparql'
+  ENV['COMPOSE_PROFILES'] = 'vo'
+
+when 'gb'
+  # Graphdb backend
+  ENV['GOO_BACKEND_NAME'] = 'graphdb'
+  ENV['GOO_PORT'] = '7200'
+  ENV['GOO_PATH_QUERY'] = '/repositories/ontoportal'
+  ENV['GOO_PATH_DATA'] = '/repositories/ontoportal/statements'
+  ENV['GOO_PATH_UPDATE'] = '/repositories/ontoportal/statements'
+
+else
+  puts "Error: Unknown backend type. Please set BACKEND_TYPE to 'ag', 'fs', 'vo', or 'gb'."
+end
+
+require 'bundler/setup'
+require 'pry'
+require 'benchmark'
+require 'ncbo_annotator'
+require 'ncbo_cron'
+require 'ontologies_linked_data'
+require_relative '../../config/config'
+require_relative 'data_benchs'
+
+puts "Starting to fetch triples"
+sub = LinkedData::Models::Ontology.find(acronym).first.latest_submission(status: :any)
+sub.bring_remaining
+
+Benchmarks.bench('Append triples') do
+  Benchmarks.import_nt_file(sub, file_path)
+end
+
+Benchmarks.do_all_benchmarks(sub)
diff --git a/test/benchmarks/metadata_benchs.rb b/test/benchmarks/metadata_benchs.rb
new file mode 100644
index 00000000..ecb91c63
--- /dev/null
+++ b/test/benchmarks/metadata_benchs.rb
@@ -0,0 +1,178 @@
+require 'ontologies_linked_data'
+require_relative 'data_benchs'
+module Benchmarks
+  module Metadata
+
+    def self.do_all_benchmarks
+      # Benchmarks.bench("Fetch all ontologies (display=all)") do
+      #   self.all_ontologies
+      # end
+
+      Benchmarks.bench("Fetch all submissions (display=all)") do
+        Goo.logger.info("Fetching all submissions")
+        self.all_submissions
+      end
+      # Benchmarks.bench("Old all submission query") do
+      #   query = <<-SPARQL
+      #     SELECT DISTINCT ?id ?attributeProperty ?attributeObject FROM <http://data.bioontology.org/metadata/OntologySubmission> WHERE { ?id a <http://data.bioontology.org/metadata/OntologySubmission> . OPTIONAL { { ?id ?attributeProperty ?attributeObject . FILTER(?attributeProperty = <http://data.bioontology.org/metadata/submissionId> || ?attributeProperty = <http://data.bioontology.org/metadata/prefLabelProperty> || ?attributeProperty = <http://data.bioontology.org/metadata/definitionProperty> || ?attributeProperty = <http://data.bioontology.org/metadata/synonymProperty> || ?attributeProperty = <http://data.bioontology.org/metadata/authorProperty> || ?attributeProperty = <http://data.bioontology.org/metadata/classType> || ?attributeProperty = <http://data.bioontology.org/metadata/hierarchyProperty> || ?attributeProperty = <http://data.bioontology.org/metadata/obsoleteProperty> || ?attributeProperty = <http://data.bioontology.org/metadata/obsoleteParent> || ?attributeProperty = <http://data.bioontology.org/metadata/createdProperty> || ?attributeProperty = <http://data.bioontology.org/metadata/modifiedProperty> || ?attributeProperty = <http://omv.ontoware.org/2005/05/ontology#URI> || ?attributeProperty = <http://www.w3.org/2002/07/owl#versionIRI> || ?attributeProperty = <http://omv.ontoware.org/2005/05/ontology#version> || ?attributeProperty = <http://omv.ontoware.org/2005/05/ontology#status> || ?attributeProperty = <http://www.w3.org/2002/07/owl#deprecated> || ?attributeProperty = <http://omv.ontoware.org/2005/05/ontology#hasOntologyLanguage> || ?attributeProperty = <http://omv.ontoware.org/2005/05/ontology#hasFormalityLevel> || ?attributeProperty = <http://omv.ontoware.org/2005/05/ontology#hasOntologySyntax> || ?attributeProperty = <http://omv.ontoware.org/2005/05/ontology#naturalLanguage> || ?attributeProperty = <http://omv.ontoware.org/2005/05/ontology#isOfType> || ?attributeProperty = <http://purl.org/dc/terms/identifier> || ?attributeProperty = <http://omv.ontoware.org/2005/05/ontology#description> || ?attributeProperty = <http://xmlns.com/foaf/0.1/homepage> || ?attributeProperty = <http://omv.ontoware.org/2005/05/ontology#documentation> || ?attributeProperty = <http://omv.ontoware.org/2005/05/ontology#notes> || ?attributeProperty = <http://omv.ontoware.org/2005/05/ontology#keywords> || ?attributeProperty = <http://www.w3.org/2004/02/skos/core#hiddenLabel> || ?attributeProperty = <http://purl.org/dc/terms/alternative> || ?attributeProperty = <http://purl.org/dc/terms/abstract> || ?attributeProperty = <http://data.bioontology.org/metadata/publication> || ?attributeProperty = <http://omv.ontoware.org/2005/05/ontology#hasLicense> || ?attributeProperty = <http://creativecommons.org/ns#useGuidelines> || ?attributeProperty = <http://creativecommons.org/ns#morePermissions> || ?attributeProperty = <http://schema.org/copyrightHolder> || ?attributeProperty = <http://data.bioontology.org/metadata/released> || ?attributeProperty = <http://purl.org/dc/terms/valid> || ?attributeProperty = <http://purl.org/pav/curatedOn> || ?attributeProperty = <http://omv.ontoware.org/2005/05/ontology#creationDate> || ?attributeProperty = <http://omv.ontoware.org/2005/05/ontology#modificationDate> || ?attributeProperty = <http://data.bioontology.org/metadata/contact> || ?attributeProperty = <http://omv.ontoware.org/2005/05/ontology#hasCreator> || ?attributeProperty = <http://omv.ontoware.org/2005/05/ontology#hasContributor> || ?attributeProperty = <http://purl.org/pav/curatedBy> || ?attributeProperty = <http://purl.org/dc/terms/publisher> || ?attributeProperty = <http://xmlns.com/foaf/0.1/fundedBy> || ?attributeProperty = <http://omv.ontoware.org/2005/05/ontology#endorsedBy> || ?attributeProperty = <http://schema.org/translator> || ?attributeProperty = <http://purl.org/dc/terms/audience> || ?attributeProperty = <http://usefulinc.com/ns/doap#repository> || ?attributeProperty = <http://usefulinc.com/ns/doap#bugDatabase> || ?attributeProperty = <http://usefulinc.com/ns/doap#mailingList> || ?attributeProperty = <http://purl.org/vocommons/voaf#toDoList> || ?attributeProperty = <http://schema.org/award> || ?attributeProperty = <http://omv.ontoware.org/2005/05/ontology#knownUsage> || ?attributeProperty = <http://omv.ontoware.org/2005/05/ontology#designedForOntologyTask> || ?attributeProperty = <http://omv.ontoware.org/2005/05/ontology#hasDomain> || ?attributeProperty = <http://purl.org/dc/terms/coverage> || ?attributeProperty = <http://purl.org/vocab/vann/example> || ?attributeProperty = <http://omv.ontoware.org/2005/05/ontology#conformsToKnowledgeRepresentationParadigm> || ?attributeProperty = <http://omv.ontoware.org/2005/05/ontology#usedOntologyEngineeringMethodology> || ?attributeProperty = <http://omv.ontoware.org/2005/05/ontology#usedOntologyEngineeringTool> || ?attributeProperty = <http://purl.org/dc/terms/accrualMethod> || ?attributeProperty = <http://purl.org/dc/terms/accrualPeriodicity> || ?attributeProperty = <http://purl.org/dc/terms/accrualPolicy> || ?attributeProperty = <http://www.isibang.ac.in/ns/mod#competencyQuestion> || ?attributeProperty = <http://www.w3.org/ns/prov#wasGeneratedBy> || ?attributeProperty = <http://www.w3.org/ns/prov#wasInvalidatedBy> || ?attributeProperty = <http://data.bioontology.org/metadata/pullLocation> || ?attributeProperty = <http://purl.org/dc/terms/isFormatOf> || ?attributeProperty = <http://purl.org/dc/terms/hasFormat> || ?attributeProperty = <http://rdfs.org/ns/void#dataDump> || ?attributeProperty = <http://data.bioontology.org/metadata/csvDump> || ?attributeProperty = <http://rdfs.org/ns/void#uriLookupEndpoint> || ?attributeProperty = <http://rdfs.org/ns/void#openSearchDescription> || ?attributeProperty = <http://purl.org/dc/terms/source> || ?attributeProperty = <http://www.w3.org/ns/sparql-service-description#endpoint> || ?attributeProperty = <http://schema.org/includedInDataCatalog> || ?attributeProperty = <http://omv.ontoware.org/2005/05/ontology#hasPriorVersion> || ?attributeProperty = <http://purl.org/dc/terms/hasPart> || ?attributeProperty = <http://kannel.open.ac.uk/ontology#ontologyRelatedTo> || ?attributeProperty = <http://kannel.open.ac.uk/ontology#similarTo> || ?attributeProperty = <http://kannel.open.ac.uk/ontology#comesFromTheSameDomain> || ?attributeProperty = <http://kannel.open.ac.uk/ontology#isAlignedTo> || ?attributeProperty = <http://omv.ontoware.org/2005/05/ontology#isBackwardCompatibleWith> || ?attributeProperty = <http://omv.ontoware.org/2005/05/ontology#isIncompatibleWith> || ?attributeProperty = <http://kannel.open.ac.uk/ontology#hasDisparateModelling> || ?attributeProperty = <http://purl.org/vocommons/voaf#hasDisjunctionsWith> || ?attributeProperty = <http://purl.org/vocommons/voaf#generalizes> || ?attributeProperty = <http://kannel.open.ac.uk/ontology#explanationEvolution> || ?attributeProperty = <http://omv.ontoware.org/2005/05/ontology#useImports> || ?attributeProperty = <http://purl.org/vocommons/voaf#usedBy> || ?attributeProperty = <http://schema.org/workTranslation> || ?attributeProperty = <http://schema.org/translationOfWork> || ?attributeProperty = <http://rdfs.org/ns/void#uriRegexPattern> || ?attributeProperty = <http://purl.org/vocab/vann/preferredNamespaceUri> || ?attributeProperty = <http://purl.org/vocab/vann/preferredNamespacePrefix> || ?attributeProperty = <http://identifiers.org/idot/exampleIdentifier> || ?attributeProperty = <http://omv.ontoware.org/2005/05/ontology#keyClasses> || ?attributeProperty = <http://purl.org/vocommons/voaf#metadataVoc> || ?attributeProperty = <http://data.bioontology.org/metadata/uploadFilePath> || ?attributeProperty = <http://data.bioontology.org/metadata/diffFilePath> || ?attributeProperty = <http://data.bioontology.org/metadata/masterFileName> || ?attributeProperty = <http://schema.org/associatedMedia> || ?attributeProperty = <http://xmlns.com/foaf/0.1/depiction> || ?attributeProperty = <http://xmlns.com/foaf/0.1/logo> || ?attributeProperty = <http://data.bioontology.org/metadata/metrics> || ?attributeProperty = <http://data.bioontology.org/metadata/submissionStatus> || ?attributeProperty = <http://data.bioontology.org/metadata/missingImports> || ?attributeProperty = <http://data.bioontology.org/metadata/ontology>)  } } }
+      #   SPARQL
+      #   Goo.sparql_query_client.query(query)
+      # end
+      #
+      # Benchmarks.bench("New all submission query") do
+      #   attribute_properties = [
+      #     "http://data.bioontology.org/metadata/submissionId",
+      #     "http://data.bioontology.org/metadata/prefLabelProperty",
+      #     "http://data.bioontology.org/metadata/definitionProperty",
+      #     "http://data.bioontology.org/metadata/synonymProperty",
+      #     "http://data.bioontology.org/metadata/authorProperty",
+      #     "http://data.bioontology.org/metadata/classType",
+      #     "http://data.bioontology.org/metadata/hierarchyProperty",
+      #     "http://data.bioontology.org/metadata/obsoleteProperty",
+      #     "http://data.bioontology.org/metadata/obsoleteParent",
+      #     "http://data.bioontology.org/metadata/createdProperty",
+      #     "http://data.bioontology.org/metadata/modifiedProperty",
+      #     "http://omv.ontoware.org/2005/05/ontology#URI",
+      #     "http://www.w3.org/2002/07/owl#versionIRI",
+      #     "http://omv.ontoware.org/2005/05/ontology#version",
+      #     "http://omv.ontoware.org/2005/05/ontology#status",
+      #     "http://www.w3.org/2002/07/owl#deprecated",
+      #     "http://omv.ontoware.org/2005/05/ontology#hasOntologyLanguage",
+      #     "http://omv.ontoware.org/2005/05/ontology#hasFormalityLevel",
+      #     "http://omv.ontoware.org/2005/05/ontology#hasOntologySyntax",
+      #     "http://omv.ontoware.org/2005/05/ontology#naturalLanguage",
+      #     "http://omv.ontoware.org/2005/05/ontology#isOfType",
+      #     "http://purl.org/dc/terms/identifier",
+      #     "http://omv.ontoware.org/2005/05/ontology#description",
+      #     "http://xmlns.com/foaf/0.1/homepage",
+      #     "http://omv.ontoware.org/2005/05/ontology#documentation",
+      #     "http://omv.ontoware.org/2005/05/ontology#notes",
+      #     "http://omv.ontoware.org/2005/05/ontology#keywords",
+      #     "http://www.w3.org/2004/02/skos/core#hiddenLabel",
+      #     "http://purl.org/dc/terms/alternative",
+      #     "http://purl.org/dc/terms/abstract",
+      #     "http://data.bioontology.org/metadata/publication",
+      #     "http://omv.ontoware.org/2005/05/ontology#hasLicense",
+      #     "http://creativecommons.org/ns#useGuidelines",
+      #     "http://creativecommons.org/ns#morePermissions",
+      #     "http://schema.org/copyrightHolder",
+      #     "http://data.bioontology.org/metadata/released",
+      #     "http://purl.org/dc/terms/valid",
+      #     "http://purl.org/pav/curatedOn",
+      #     "http://omv.ontoware.org/2005/05/ontology#creationDate",
+      #     "http://omv.ontoware.org/2005/05/ontology#modificationDate",
+      #     "http://data.bioontology.org/metadata/contact",
+      #     "http://omv.ontoware.org/2005/05/ontology#hasCreator",
+      #     "http://omv.ontoware.org/2005/05/ontology#hasContributor",
+      #     "http://purl.org/pav/curatedBy",
+      #     "http://purl.org/dc/terms/publisher",
+      #     "http://xmlns.com/foaf/0.1/fundedBy",
+      #     "http://omv.ontoware.org/2005/05/ontology#endorsedBy",
+      #     "http://schema.org/translator",
+      #     "http://purl.org/dc/terms/audience",
+      #     "http://usefulinc.com/ns/doap#repository",
+      #     "http://usefulinc.com/ns/doap#bugDatabase",
+      #     "http://usefulinc.com/ns/doap#mailingList",
+      #     "http://purl.org/vocommons/voaf#toDoList",
+      #     "http://schema.org/award",
+      #     "http://omv.ontoware.org/2005/05/ontology#knownUsage",
+      #     "http://omv.ontoware.org/2005/05/ontology#designedForOntologyTask",
+      #     "http://omv.ontoware.org/2005/05/ontology#hasDomain",
+      #     "http://purl.org/dc/terms/coverage",
+      #     "http://purl.org/vocab/vann/example",
+      #     "http://omv.ontoware.org/2005/05/ontology#conformsToKnowledgeRepresentationParadigm",
+      #     "http://omv.ontoware.org/2005/05/ontology#usedOntologyEngineeringMethodology",
+      #     "http://omv.ontoware.org/2005/05/ontology#usedOntologyEngineeringTool",
+      #     "http://purl.org/dc/terms/accrualMethod",
+      #     "http://purl.org/dc/terms/accrualPeriodicity",
+      #     "http://purl.org/dc/terms/accrualPolicy",
+      #     "http://www.isibang.ac.in/ns/mod#competencyQuestion",
+      #     "http://www.w3.org/ns/prov#wasGeneratedBy",
+      #     "http://www.w3.org/ns/prov#wasInvalidatedBy",
+      #     "http://data.bioontology.org/metadata/pullLocation",
+      #     "http://purl.org/dc/terms/isFormatOf",
+      #     "http://purl.org/dc/terms/hasFormat",
+      #     "http://rdfs.org/ns/void#dataDump",
+      #     "http://data.bioontology.org/metadata/csvDump",
+      #     "http://rdfs.org/ns/void#uriLookupEndpoint",
+      #     "http://rdfs.org/ns/void#openSearchDescription",
+      #     "http://purl.org/dc/terms/source",
+      #     "http://www.w3.org/ns/sparql-service-description#endpoint",
+      #     "http://schema.org/includedInDataCatalog",
+      #     "http://omv.ontoware.org/2005/05/ontology#hasPriorVersion",
+      #     "http://purl.org/dc/terms/hasPart",
+      #     "http://kannel.open.ac.uk/ontology#ontologyRelatedTo",
+      #     "http://kannel.open.ac.uk/ontology#similarTo",
+      #     "http://kannel.open.ac.uk/ontology#comesFromTheSameDomain",
+      #     "http://kannel.open.ac.uk/ontology#isAlignedTo",
+      #     "http://omv.ontoware.org/2005/05/ontology#isBackwardCompatibleWith",
+      #     "http://omv.ontoware.org/2005/05/ontology#isIncompatibleWith",
+      #     "http://kannel.open.ac.uk/ontology#hasDisparateModelling",
+      #     "http://purl.org/vocommons/voaf#hasDisjunctionsWith",
+      #     "http://purl.org/vocommons/voaf#generalizes",
+      #     "http://kannel.open.ac.uk/ontology#explanationEvolution",
+      #     "http://omv.ontoware.org/2005/05/ontology#useImports",
+      #     "http://purl.org/vocommons/voaf#usedBy",
+      #     "http://schema.org/workTranslation",
+      #     "http://schema.org/translationOfWork",
+      #     "http://rdfs.org/ns/void#uriRegexPattern",
+      #     "http://purl.org/vocab/vann/preferredNamespaceUri",
+      #     "http://purl.org/vocab/vann/preferredNamespacePrefix",
+      #     "http://identifiers.org/idot/exampleIdentifier",
+      #     "http://omv.ontoware.org/2005/05/ontology#keyClasses",
+      #     "http://purl.org/vocommons/voaf#metadataVoc",
+      #     "http://data.bioontology.org/metadata/uploadFilePath",
+      #     "http://data.bioontology.org/metadata/diffFilePath",
+      #     "http://data.bioontology.org/metadata/masterFileName",
+      #     "http://schema.org/associatedMedia",
+      #     "http://xmlns.com/foaf/0.1/depiction",
+      #     "http://xmlns.com/foaf/0.1/logo",
+      #     "http://data.bioontology.org/metadata/metrics",
+      #     "http://data.bioontology.org/metadata/submissionStatus",
+      #     "http://data.bioontology.org/metadata/missingImports",
+      #     "http://data.bioontology.org/metadata/ontology"
+      #   ]
+      #   total_count = 0
+      #   chunks = attribute_properties.each_slice(100).to_a
+      #   # Process each chunk and construct the query
+      #   chunks.each_with_index do |chunk, index|
+      #     query = <<-SPARQL
+      #       SELECT ?id ?attributeProperty ?attributeObject
+      #       FROM <http://data.bioontology.org/metadata/OntologySubmission>
+      #       WHERE {
+      #         ?id a <http://data.bioontology.org/metadata/OntologySubmission> .
+      #
+      #           ?id ?attributeProperty ?attributeObject .
+      #           VALUES ?attributeProperty {
+      #             #{chunk.map { |ap| "<#{ap}>" }.join(" ")}
+      #           }
+      #       }
+      #     SPARQL
+      #
+      #     count = 0
+      #     page = 1
+      #     size = 50_000
+      #     while count > 0 || page == 1
+      #       r = Goo.sparql_query_client.query("#{query} LIMIT #{size} OFFSET #{(page-1)*size}")
+      #       count = r.length
+      #       total_count += count
+      #       page+=1
+      #     end
+      #   end
+      #   puts "Total count: #{total_count}"
+      # end
+    end
+
+    def self.all_ontologies
+      attr_ontology = LinkedData::Models::Ontology.attributes(:all)
+      LinkedData::Models::Ontology.where.include(attr_ontology).all.count
+    end
+
+    def self.all_submissions
+      attr_ontology = LinkedData::Models::Ontology.attributes(:all)
+      attr = LinkedData::Models::OntologySubmission.attributes(:all)
+      attr << { ontology: attr_ontology }
+      LinkedData::Models::OntologySubmission.where.include(attr).all.count
+    end
+  end
+end
diff --git a/test/benchmarks/parse_and_do_ontoportal_operations.rb b/test/benchmarks/parse_and_do_ontoportal_operations.rb
new file mode 100644
index 00000000..8276cb90
--- /dev/null
+++ b/test/benchmarks/parse_and_do_ontoportal_operations.rb
@@ -0,0 +1,74 @@
+require 'benchmark'
+acronym = ARGV[0]
+profile = ARGV[1]
+api_key = ARGV[2] || '1de0a270-29c5-4dda-b043-7c3580628cd5'
+api_url = ARGV[3] || 'http://data.stageportal.lirmm.fr'
+pwd = File.dirname(__FILE__)
+
+system("bash #{pwd}/start_ontoportal_services.sh #{profile} #{acronym} #{api_key} #{api_url}")
+if $?.exitstatus != 0
+  puts "Error occurred during running services script execution."
+  exit(1)
+end
+
+case profile
+when 'ag'
+  # AllegroGraph backend
+  ENV['GOO_BACKEND_NAME'] = 'allegrograph'
+  ENV['GOO_PORT'] = '10035'
+  ENV['GOO_PATH_QUERY'] = '/repositories/ontoportal_test'
+  ENV['GOO_PATH_DATA'] = '/repositories/ontoportal_test/statements'
+  ENV['GOO_PATH_UPDATE'] = '/repositories/ontoportal_test/statements'
+  ENV['COMPOSE_PROFILES'] = 'ag'
+
+when 'fs'
+  # 4store backend
+  ENV['GOO_PORT'] = '9000'
+  ENV['COMPOSE_PROFILES'] = 'fs'
+
+when 'vo'
+  # Virtuoso backend
+  ENV['GOO_BACKEND_NAME'] = 'virtuoso'
+  ENV['GOO_PORT'] = '8890'
+  ENV['GOO_PATH_QUERY'] = '/sparql'
+  ENV['GOO_PATH_DATA'] = '/sparql'
+  ENV['GOO_PATH_UPDATE'] = '/sparql'
+  ENV['COMPOSE_PROFILES'] = 'vo'
+
+when 'gb'
+  # Graphdb backend
+  ENV['GOO_BACKEND_NAME'] = 'graphdb'
+  ENV['GOO_PORT'] = '7200'
+  ENV['GOO_PATH_QUERY'] = '/repositories/ontoportal'
+  ENV['GOO_PATH_DATA'] = '/repositories/ontoportal/statements'
+  ENV['GOO_PATH_UPDATE'] = '/repositories/ontoportal/statements'
+
+else
+  puts "Error: Unknown backend type. Please set BACKEND_TYPE to 'ag', 'fs', 'vo', or 'gb'."
+end
+
+puts "Parsing file for #{acronym} and #{profile}"
+time = Benchmark.realtime do
+  system("#{pwd}/../../bin/ncbo_ontology_process -o #{acronym} -t process_rdf")
+end
+puts "Time to parse file: " + time.round(2).to_s + 's'
+
+if $?.exitstatus != 0
+  puts "Error occurred during script execution."
+  exit(1)
+end
+puts "Finished parsing file"
+
+require 'bundler/setup'
+require 'pry'
+require 'benchmark'
+require 'ncbo_annotator'
+require 'ncbo_cron'
+require 'ontologies_linked_data'
+require_relative '../../config/config'
+require_relative './metadata_benchs'
+require_relative './data_benchs'
+Goo.sparql_query_client.cache.redis_cache.flushdb
+sub = LinkedData::Models::Ontology.find(acronym).first.latest_submission(status: :any)
+
+Benchmarks.do_all_benchmarks(sub)
diff --git a/test/benchmarks/run_metadata_benchs.rb b/test/benchmarks/run_metadata_benchs.rb
new file mode 100644
index 00000000..c67cca6b
--- /dev/null
+++ b/test/benchmarks/run_metadata_benchs.rb
@@ -0,0 +1,55 @@
+require 'benchmark'
+profile = ARGV[0]
+pwd = File.dirname(__FILE__)
+
+case profile
+when 'ag'
+  # AllegroGraph backend
+  ENV['GOO_BACKEND_NAME'] = 'allegrograph'
+  ENV['GOO_PORT'] = '10035'
+  ENV['GOO_PATH_QUERY'] = '/repositories/ontoportal_test'
+  ENV['GOO_PATH_DATA'] = '/repositories/ontoportal_test/statements'
+  ENV['GOO_PATH_UPDATE'] = '/repositories/ontoportal_test/statements'
+  ENV['COMPOSE_PROFILES'] = 'ag'
+
+when 'fs'
+  # 4store backend
+  ENV['GOO_PORT'] = '9000'
+  ENV['COMPOSE_PROFILES'] = 'fs'
+
+when 'vo'
+  # Virtuoso backend
+  ENV['GOO_BACKEND_NAME'] = 'virtuoso'
+  ENV['GOO_PORT'] = '8890'
+  ENV['GOO_PATH_QUERY'] = '/sparql'
+  ENV['GOO_PATH_DATA'] = '/sparql'
+  ENV['GOO_PATH_UPDATE'] = '/sparql'
+  ENV['COMPOSE_PROFILES'] = 'vo'
+
+when 'gb'
+  # Graphdb backend
+  ENV['GOO_BACKEND_NAME'] = 'graphdb'
+  ENV['GOO_PORT'] = '7200'
+  ENV['GOO_PATH_QUERY'] = '/repositories/ontoportal'
+  ENV['GOO_PATH_DATA'] = '/repositories/ontoportal/statements'
+  ENV['GOO_PATH_UPDATE'] = '/repositories/ontoportal/statements'
+
+else
+  puts "Error: Unknown backend type. Please set BACKEND_TYPE to 'ag', 'fs', 'vo', or 'gb'."
+end
+
+
+
+require 'bundler/setup'
+require 'pry'
+require 'benchmark'
+require 'ncbo_annotator'
+require 'ncbo_cron'
+require 'ontologies_linked_data'
+require_relative '../../config/config'
+require_relative './metadata_benchs'
+
+Goo.sparql_query_client.cache.redis_cache.flushdb
+
+
+Benchmarks::Metadata.do_all_benchmarks
diff --git a/start_ontoportal_services.sh b/test/benchmarks/start_ontoportal_services.sh
similarity index 80%
rename from start_ontoportal_services.sh
rename to test/benchmarks/start_ontoportal_services.sh
index 9a8a9823..e2c4cd99 100755
--- a/start_ontoportal_services.sh
+++ b/test/benchmarks/start_ontoportal_services.sh
@@ -1,8 +1,23 @@
 #!/usr/bin/env bash
 profile=$1
 acronym=$2
+from_apikey=$3
+import_from_api=$4
 set -e
 
+if [ -z "$import_from_api" ]; then
+  import_from_api=https://data.stageportal.lirmm.fr
+fi
+
+if [ -z "$from_apikey" ]; then
+  from_apikey="1de0a270-29c5-4dda-b043-7c3580628cd5"
+fi
+
+# Documentation:
+# The below script is used to start the OntoPortal services with a specific backend type.
+# The script takes two arguments: acronym and profile.
+# The acronym is the name of the ontology to be imported.
+# The profile is the type of backend to be used.
 
 if [ -z "$profile" ]; then
   echo "Usage: $0 <acronym> <profile>"
@@ -41,7 +56,7 @@ elif [ "$BACKEND_TYPE" == "gb" ]; then
   export GOO_PATH_DATA="/repositories/ontoportal/statements"
   export GOO_PATH_UPDATE="/repositories/ontoportal/statements"
 else
-  echo "Error: Unknown backend type. Please set BACKEND_TYPE to 'ag', 'fs', or 'vo'."
+  echo "Error: Unknown backend type $profile. Please set BACKEND_TYPE to 'ag', 'fs', or 'vo'."
 fi
 
 echo "###########################################################################"
@@ -82,4 +97,4 @@ bundle exec rake user:create[admin,admin@nodomain.org,password]
 bundle exec rake user:adminify[admin]
 echo "###########################################################################"
 echo "Create a new ontology $acronym and import it from a remote server"
-bin/ncbo_ontology_import --admin-user admin -o "$acronym" --from https://data.stageportal.lirmm.fr --from-apikey 82602563-4750-41be-9654-36f46056a0db
+bin/ncbo_ontology_import --admin-user admin -o "$acronym" --from "$import_from_api" --from-apikey "$from_apikey"