Cleanup and optimize DB schema

The database schema was lacking many integrity checks and indexes.
Correct it by first applying a migration that removes all old/stale
data, then creating those indexes.

The driving reason for this is the very slow performance of processing
(specially aggregation) on the new mezuro.org servers. It will hopefully
remove (or at least heavily improve) the superlinear slowdown when the
number of metrics rises, as observed in #207.

Additionally, remove the timestamp columns from kalibro_modules,
module_results and metric_results: they are not used in any way, and
there are millions of rows containing them. It's possible and probably
desirable to just look at the timestamps in the processing.

CHANGELOG.rdoc

@@ -4,6 +4,7 @@ KalibroProcessor is the processing web service for Mezuro.
 
 == Unreleased
 
+* Optimize database structure by adding foreign keys and indexes where needed
 * Introduce performance tests for Aggregator
 * Update KolektiMetricfu
 

db/migrate/20160715192850_clean_inconsistencies.rb

@@ -0,0 +1,71 @@
+class CleanInconsistencies < ActiveRecord::Migration
+  def self.up
+    # Unset project reference for repositories with non-existing projects
+    execute <<-SQL
+      UPDATE repositories AS r
+      SET project_id = NULL
+      WHERE project_id = 0 OR NOT EXISTS (
+        SELECT 1 FROM projects AS p WHERE p.id = r.project_id
+      )
+    SQL
+
+    # Delete processings with non-existing repositories
+    execute <<-SQL
+      DELETE FROM processings AS p
+      WHERE NOT EXISTS(
+        SELECT 1 FROM repositories AS r WHERE r.id = p.repository_id
+      )
+    SQL
+
+    # Delete process times with non-existing processings
+    execute <<-SQL
+      DELETE FROM process_times AS t
+      WHERE NOT EXISTS (
+        SELECT 1 FROM processings AS p WHERE p.id = t.processing_id
+      )
+    SQL
+
+    # Delete module results with non-existing processings
+    execute <<-SQL
+      DELETE FROM module_results AS m
+      WHERE NOT EXISTS (
+        SELECT 1 FROM processings AS p WHERE p.id = m.processing_id
+      )
+    SQL
+
+    # Delete kalibro modules with non-existing module results
+    execute <<-SQL
+      DELETE FROM kalibro_modules AS k
+      WHERE NOT EXISTS (
+        SELECT 1 FROM module_results AS m WHERE m.id = k.module_result_id
+      )
+    SQL
+
+    # Delete metric results with non-existing module results
+    execute <<-SQL
+      DELETE FROM metric_results AS met
+      WHERE NOT EXISTS (
+        SELECT 1 FROM module_results AS mod WHERE mod.id = met.module_result_id
+      )
+    SQL
+
+    # Delete duplicate metric_results. Group them by (module_result, metric_configuration),
+    # then delete all but the one with the highest ID
+    # The double wrapping on the inner query is necessary because window functions
+    # cannot be used in WHERE in PostgreSQL.
+    execute <<-SQL
+      DELETE FROM metric_results
+      WHERE id IN (
+        SELECT t.id FROM (
+          SELECT id, ROW_NUMBER() OVER (PARTITION BY module_result_id, metric_configuration_id ORDER BY id DESC) AS rnum
+          FROM metric_results
+        ) AS t
+        WHERE t.rnum > 1
+      )
+    SQL
+  end
+
+  def self.down
+    raise ActiveRecord::IrreversibleMigration
+  end
+end

db/migrate/20160715192900_add_indexes_to_kalibro_modules.rb

@@ -0,0 +1,6 @@
+class AddIndexesToKalibroModules < ActiveRecord::Migration
+  def change
+    add_foreign_key :kalibro_modules, :module_results, on_delete: :cascade
+    add_index :kalibro_modules, [:long_name, :granularity]
+  end
+end

db/migrate/20160715192901_add_indexes_to_module_results.rb

@@ -0,0 +1,6 @@
+class AddIndexesToModuleResults < ActiveRecord::Migration
+  def change
+    add_foreign_key :module_results, :module_results, column: 'parent_id'
+    add_foreign_key :module_results, :processings, on_delete: :cascade
+  end
+end

db/migrate/20160715192902_add_indexes_to_metric_results.rb

@@ -0,0 +1,8 @@
+class AddIndexesToMetricResults < ActiveRecord::Migration
+  def change
+    add_foreign_key :metric_results, :module_results, on_delete: :cascade
+    add_index :metric_results, :type
+    add_index :metric_results, [:module_result_id, :metric_configuration_id],
+              unique: true, name: 'metric_results_module_res_metric_cfg_uniq_idx'
+  end
+end

db/migrate/20160715192903_add_indexes_to_processings.rb

@@ -0,0 +1,6 @@
+class AddIndexesToProcessings < ActiveRecord::Migration
+  def change
+    add_foreign_key :processings, :repositories
+    add_foreign_key :processings, :module_results, column: 'root_module_result_id'
+  end
+end

db/migrate/20160715192904_add_indexes_to_process_times.rb

@@ -0,0 +1,5 @@
+class AddIndexesToProcessTimes < ActiveRecord::Migration
+  def change
+    add_foreign_key :process_times, :processings, on_delete: :cascade
+  end
+end

db/migrate/20160715192905_add_indexes_to_repositories.rb

@@ -0,0 +1,5 @@
+class AddIndexesToRepositories < ActiveRecord::Migration
+  def change
+    add_foreign_key :repositories, :projects
+  end
+end

db/migrate/20160716000521_remove_timestamps_from_module_results.rb

@@ -0,0 +1,6 @@
+class RemoveTimestampsFromModuleResults < ActiveRecord::Migration
+  def change
+    remove_column :module_results, :created_at, :string
+    remove_column :module_results, :updated_at, :string
+  end
+end

db/migrate/20160716000528_remove_timestamps_from_metric_results.rb

@@ -0,0 +1,6 @@
+class RemoveTimestampsFromMetricResults < ActiveRecord::Migration
+  def change
+    remove_column :metric_results, :created_at, :string
+    remove_column :metric_results, :updated_at, :string
+  end
+end

db/migrate/20160716000530_remove_timestamps_from_kalibro_modules.rb

@@ -0,0 +1,6 @@
+class RemoveTimestampsFromKalibroModules < ActiveRecord::Migration
+  def change
+    remove_column :kalibro_modules, :created_at, :string
+    remove_column :kalibro_modules, :updated_at, :string
+  end
+end

db/schema.rb

@@ -11,7 +11,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 20151002172231) do
+ActiveRecord::Schema.define(version: 20160716000530) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
@@ -33,33 +33,31 @@
   add_index "delayed_jobs", ["priority", "run_at"], name: "delayed_jobs_priority", using: :btree
 
   create_table "kalibro_modules", force: :cascade do |t|
-    t.string   "long_name",        limit: 255
-    t.string   "granularity",      limit: 255
-    t.datetime "created_at"
-    t.datetime "updated_at"
-    t.integer  "module_result_id"
+    t.string  "long_name",        limit: 255
+    t.string  "granularity",      limit: 255
+    t.integer "module_result_id"
   end
 
+  add_index "kalibro_modules", ["long_name", "granularity"], name: "index_kalibro_modules_on_long_name_and_granularity", using: :btree
+
   create_table "metric_results", force: :cascade do |t|
-    t.integer  "module_result_id"
-    t.integer  "metric_configuration_id"
-    t.float    "value"
-    t.datetime "created_at"
-    t.datetime "updated_at"
-    t.string   "type",                              default: "MetricResult", null: false
-    t.integer  "line_number"
-    t.text     "message"
-    t.integer  "related_hotspot_metric_results_id"
+    t.integer "module_result_id"
+    t.integer "metric_configuration_id"
+    t.float   "value"
+    t.string  "type",                              default: "MetricResult", null: false
+    t.integer "line_number"
+    t.text    "message"
+    t.integer "related_hotspot_metric_results_id"
   end
 
+  add_index "metric_results", ["module_result_id", "metric_configuration_id"], name: "metric_results_module_res_metric_cfg_uniq_idx", unique: true, using: :btree
   add_index "metric_results", ["related_hotspot_metric_results_id"], name: "index_metric_results_on_related_hotspot_metric_results_id", using: :btree
+  add_index "metric_results", ["type"], name: "index_metric_results_on_type", using: :btree
 
   create_table "module_results", force: :cascade do |t|
-    t.float    "grade"
-    t.integer  "parent_id"
-    t.datetime "created_at"
-    t.datetime "updated_at"
-    t.integer  "processing_id"
+    t.float   "grade"
+    t.integer "parent_id"
+    t.integer "processing_id"
   end
 
   add_index "module_results", ["parent_id"], name: "index_module_results_on_parent_id", using: :btree
@@ -106,5 +104,13 @@
     t.string   "branch",                               default: "master", null: false
   end
 
+  add_foreign_key "kalibro_modules", "module_results"
+  add_foreign_key "metric_results", "module_results"
   add_foreign_key "metric_results", "related_hotspot_metric_results", column: "related_hotspot_metric_results_id"
+  add_foreign_key "module_results", "module_results", column: "parent_id"
+  add_foreign_key "module_results", "processings"
+  add_foreign_key "process_times", "processings"
+  add_foreign_key "processings", "module_results", column: "root_module_result_id"
+  add_foreign_key "processings", "repositories"
+  add_foreign_key "repositories", "projects"
 end