workers

Gemfile

@@ -48,6 +48,11 @@ group :production do
   gem 'unicorn'
 end
 
+group :development do
+  gem 'quiet_assets' # skip logs for serving assets in development.log
+  gem 'thin' # to avoid wobrick content-length confusion
+end
+
 # test suite
 group :development, :test do
   gem 'capistrano'

Gemfile.lock

@@ -41,8 +41,8 @@ GEM
     activesupport (3.2.8)
       i18n (~> 0.6)
       multi_json (~> 1.0)
-    annotator (0.0.2)
-      rails (~> 3.0)
+    annotator (0.0.8.1)
+      activerecord (~> 3.0)
     arbre (1.0.1)
       activesupport (>= 3.0.0)
     arel (3.0.2)
@@ -81,6 +81,7 @@ GEM
     coffee-script-source (1.2.0)
     columnize (0.3.6)
     commonjs (0.2.6)
+    daemons (1.1.9)
     debugger (1.2.1)
       columnize (>= 0.3.1)
       debugger-linecache (~> 1.1.1)
@@ -160,8 +161,8 @@ GEM
       activerecord (~> 3.1)
       activesupport (~> 3.1)
       polyamorous (~> 0.5.0)
-    mime-types (1.18)
-    multi_json (1.3.6)
+    mime-types (1.19)
+    multi_json (1.3.7)
     multipart-post (1.1.5)
     net-scp (1.0.4)
       net-ssh (>= 1.99.1)
@@ -181,6 +182,8 @@ GEM
     pusher (0.10.0)
       multi_json (~> 1.0)
       signature (~> 0.1.4)
+    quiet_assets (1.0.1)
+      railties (~> 3.1)
     rack (1.4.1)
     rack-cache (1.2)
       rack (>= 0.4)
@@ -251,6 +254,10 @@ GEM
     sqlite3 (1.3.5)
     therubyracer (0.10.2)
       libv8 (~> 3.3.10)
+    thin (1.5.0)
+      daemons (>= 1.0.9)
+      eventmachine (>= 0.12.6)
+      rack (>= 1.0.0)
     thor (0.16.0)
     tilt (1.3.3)
     tinder (1.9.1)
@@ -263,7 +270,7 @@ GEM
       multi_json (~> 1.0)
       multipart-post (~> 1.1)
       twitter-stream (~> 0.1)
-    treetop (1.4.10)
+    treetop (1.4.12)
       polyglot
       polyglot (>= 0.3.1)
     twitter-bootstrap-rails (2.1.3)
@@ -275,7 +282,7 @@ GEM
       eventmachine (>= 0.12.8)
       http_parser.rb (~> 0.5.1)
       simple_oauth (~> 0.1.4)
-    tzinfo (0.3.33)
+    tzinfo (0.3.35)
     unicorn (4.3.1)
       kgio (~> 2.6)
       rack
@@ -312,6 +319,7 @@ DEPENDENCIES
   open4
   pg
   pusher
+  quiet_assets
   rails (= 3.2.8)
   rspec-mocks
   rspec-rails
@@ -321,6 +329,7 @@ DEPENDENCIES
   simplecov
   sql_queries_count
   sqlite3
+  thin
   tinder
   twitter-bootstrap-rails
   unicorn

app/assets/javascripts/application.js

@@ -5,6 +5,8 @@
 //= require jquery_ujs
 //= require twitter/bootstrap
 //= require pusher.min
+//= require moment
+//= require livestamp
 //= require moci
 //= require webs
 

app/assets/javascripts/webs.js.coffee.erb

@@ -20,7 +20,25 @@ Webs =
   receive: (event, data) ->
     Moci.debug("Got project event '" + event + "' with data: " + JSON.stringify(data))
     switch event
+      when 'worker'
+        tr = $('table.workers tr.worker_'+data['id'])
+        if tr.length == 0
+          q = $.ajax
+            url: '/workers/'+data['id']+'/tr_worker'
+            dataType: 'html'
+            async: false
+            success: (data,t,j) ->
+              $(data).prependTo('table.workers tbody')
+        else
+          if data['destroyed?'] == true
+            tr.remove()
+          else
+            tr.find('.last_seen_at span').attr('data-livestamp',data['last_seen_at'])
+            tr.find('.state').html(data['state'])
+            tr.find('.task').html(data['task']['name'])
+
       when 'test_suite_run'
+        # TODO we probably want to switch to js view rendering with these on the long run
         if $('.last_runs .test_suite_run_'+data['id']).length == 0
           q = $.ajax
             url: '/p/' + Project.slug + '/tr_last_run?'+$.param({test_suite_run_id: data['id']})
@@ -34,6 +52,7 @@ Webs =
           tr.removeClass('state_running')
           tr.removeClass('state_finished')
           tr.addClass('state_'+data['state'])
+
        when 'test_unit_run'
          tsr_id = data['test_suite_run']['id']
          tr_details = $('.last_runs .details.test_suite_run_'+tsr_id)
@@ -82,6 +101,10 @@ $(document).ready ->
     Moci.debug("Subscribing to project channel " + channel)
     Webs.subscribe(channel)
     setInterval LastTestSuiteRuns.update_progress, 500
+  if $('input#private_webs_channel').length != 0
+    Moci.debug("Subscribing to private channel")
+    Webs.subscribe($('input#private_webs_channel').val())
+
 
 
 

app/assets/stylesheets/moci.sass

@@ -69,6 +69,8 @@ table.last_runs
       color:  darken($fail-color, 50%)
     border: 1px solid darken($fail-color, 20%)
     background-color: $fail-color
+.workers tr.dead
+  opacity: 0.5
 
 .orange
   color: orange

app/controllers/workers_controller.rb

@@ -0,0 +1,12 @@
+class WorkersController < ApplicationController
+  before_filter :authenticate_admin!
+
+  def index
+    @workers = Worker.order('worker_type_id', 'last_seen_at DESC').all
+  end
+
+  def tr_worker
+    worker = Worker.find params[:id]
+    render worker
+  end
+end

app/helpers/workers_helper.rb

@@ -0,0 +1,2 @@
+module WorkersHelper
+end

app/models/commit.rb

@@ -10,6 +10,7 @@
 # * preparation_log [text] - FIXME: remove me, it's present in ProjectInstanceCommit
 # * project_id [integer] - belongs_to Project
 # * skipped [boolean] - commit can be marked as skipped if we don't want moci to run it
+# * slug [string] - TODO: document me
 # * updated_at [datetime] - last update time
 class Commit < ActiveRecord::Base
 

app/models/project.rb

@@ -6,6 +6,7 @@
 # * project_type [string, default=Base] - same as class name that will be used as project handler
 #   (Moci::ProjectHandler::..)
 # * public [boolean] - if true, it can be viewed by everybody (without singing in)
+# * slug [string] - TODO: document me
 # * updated_at [datetime] - last update time
 # * vcs_type [string, default=Base] - VCS type e.g. Git, Mercurial (see Moci::VCS::Base)
 class Project < ActiveRecord::Base

app/models/user.rb

@@ -1,3 +1,5 @@
+require 'digest/md5'
+
 # Attributes:
 # * id [integer, primary, not null] - primary key
 # * admin [boolean, not null] - TODO: document me
@@ -36,4 +38,10 @@ class User < ActiveRecord::Base
 
   has_many :manage_project_permissions, :class_name => 'ProjectPermission', :conditions => {:name => 'manage'}
   has_many :projects_can_manage, :through => :manage_project_permissions, :source => :project
+
+  scope :admin, where(admin: true)
+
+  def webs_channel
+    Digest::MD5.hexdigest(id.to_s + created_at.to_i.to_s + encrypted_password.to_s)
+  end
 end

app/models/worker.rb

@@ -0,0 +1,62 @@
+# Attributes:
+# * id [integer, primary, not null] - primary key
+# * created_at [datetime] - creation time
+# * last_seen_at [datetime] - columne periodically updated by worker process
+# * pid [integer, not null] - worker process pid
+# * state [string] - current worker state [idle, working, waiting - for lock on instance]
+# * task [text] - currently executed task in form of JSON
+# * worker_type_id [integer, not null] - see Worker::TYPE
+class Worker < ActiveRecord::Base
+
+  # Master worker spawns slave and these are workers that are actually running some tests
+  TYPES = {
+    0 => :master,
+    1 => :slave
+  }
+
+  # How often should worker report that it is alive
+  PING_FREQUENCY = 30
+
+  # Current task, stored as serialized hash:
+  # * name - human readable task to display in UI
+  # * task_id - currently processed queue element
+  # * project_instance_id - instance used
+  serialize :task, Hash
+
+  validates :worker_type_id, inclusion: {in: TYPES.keys}
+
+  scope :alive, lambda { where('last_seen_at >= ?', Time.now - PING_FREQUENCY*2) }
+  scope :dead, lambda { where('last_seen_at < ?', Time.now - PING_FREQUENCY*2) }
+
+  scope :slave, where(worker_type_id: TYPES.invert[:slave])
+  scope :master, where(worker_type_id: TYPES.invert[:master])
+
+  def self.cleanup
+    dead.delete_all
+  end
+
+  def as_json(options=nil)
+    super(only: [:id, :last_seen_at, :pid, :state, :task],
+          methods: [:worker_type, :destroyed?])
+  end
+
+  def alive?
+    last_seen_at > Time.now - PING_FREQUENCY*2
+  end
+
+  def worker_type
+    TYPES[worker_type_id]
+  end
+
+  def worker_type=(name)
+    self.worker_type_id = TYPES.invert[name.to_sym]
+  end
+
+  # Live web notifications
+  # FIXME pusher is extremaly slow, especially if we keep using it we
+  # may want to send these notifications in some bg process. It's most painfull
+  # on destroy when we are waiting 2s for process to finish because of tihs pusher
+  after_save { Webs.notify :worker, self }
+  after_destroy { Webs.notify :worker, self }
+
+end

app/views/common/_sidebar.html.haml

@@ -13,3 +13,5 @@
   - else
     %li.nav-header No project selected
     %li= link_to "Latest test runs", test_suite_runs_path
+  %li.divider
+  %li= link_to "Workers", workers_path

app/views/layouts/application.html.haml

@@ -8,16 +8,16 @@
     = stylesheet_link_tag 'application', media: 'all'
     = javascript_include_tag 'application'
     = csrf_meta_tag
-    -# Le HTML5 shim, for IE6-8 support of HTML5 elements 
+    -# Le HTML5 shim, for IE6-8 support of HTML5 elements
     <!--[if lt IE 9]>
-    %script(src="http://html5shim.googlecode.com/svn/trunk/html5.js") 
+    %script(src="http://html5shim.googlecode.com/svn/trunk/html5.js")
     <![endif]-->
     = yield :head
   %body
     %header.navbar.navbar-fixed-top
       %nav.navbar-inner
         .container-fluid
-          %a(class="brand" href="/") 
+          %a(class="brand" href="/")
             %i.icon-th
             Moci
           %ul.nav
@@ -33,6 +33,7 @@
             - if user_signed_in?
               - if current_user.admin?
                 %li= link_to 'admin panel', '/admin'
+              %input#private_webs_channel.hidden{:value => current_user.webs_channel}
               %li= link_to 'log out', destroy_user_session_path, :method => :delete
               %li.avatar= image_tag current_user.gravatar_url(size: 24), class: :gravatar, title: current_user.email
             - else

app/views/workers/_worker.html.haml

@@ -0,0 +1,9 @@
+%tr{:class => [(worker.alive? ? 'alive' : 'dead'), "worker_#{worker.id}"]}
+  %td.worker_type= worker.worker_type
+  %td.pid= worker.pid
+  %td.state= worker.state
+  %td.task= worker.task[:name] || 'n/d'
+  %td.last_seen_at
+    %span{:'data-livestamp' => worker.last_seen_at.to_i}
+
+

app/views/workers/index.html.haml

@@ -0,0 +1,15 @@
+%h2 Workers
+
+%table.table.table-hover.table-condensed.workers
+  %thead
+    %tr
+      %th Type
+      %th PID
+      %th State
+      %th Task
+      %th Last seen
+  %tbody
+    = render @workers
+
+
+

config/moci.yml.example

@@ -4,4 +4,8 @@ pusher:
   app_id: ''
   key: ''
   secret: ''
+# number of slave workers to spawn
+# TODO since it can be changed live it should be possible to change it from web,
+# or this should just be upper limit and number should be adjusted manually
+number_of_workers: 3
 rvm_source: $HOME/.rvm/scripts/rvm

config/routes.rb

@@ -25,6 +25,10 @@
     resources :test_suite_runs
   end
 
+  resources :workers, :only => [:index] do
+    member { get :tr_worker }
+  end
+
   # You can have the root of your site routed with "root"
   # just remember to delete public/index.html.
   root :to => "test_suite_runs#index"

db/migrate/20121105090057_create_workers.rb

@@ -0,0 +1,12 @@
+class CreateWorkers < ActiveRecord::Migration
+  def change
+    create_table :workers do |t|
+      t.integer :pid, :unsigned => true, :null => false
+      t.integer :worker_type_id, :null => false, :size => 4
+      t.string :state
+      t.text :task
+      t.timestamp :last_seen_at
+      t.timestamp :created_at
+    end
+  end
+end