From 3d0e0b4b5b329f56edf83b4694b551c8a529027f Mon Sep 17 00:00:00 2001 From: Heiko Bernloehr Date: Wed, 16 Dec 2015 21:45:18 +0100 Subject: Merge and send to computation service. --- Gemfile | 6 +++- Gemfile.lock | 48 +++++++++++++++++++++++++ config/application.rb | 1 + config/initializers/appcfg.rb | 6 ++++ config/initializers/spring.rb | 1 + lib/http_ecs.rb | 25 +++++++++++++ lib/job_event.rb | 81 +++++++++++++++++++++++++++++++++++++++++++ lib/main_loop.rb | 66 +++++++++++++++++++++++++++++++++++ 8 files changed, 233 insertions(+), 1 deletion(-) create mode 100644 config/initializers/appcfg.rb create mode 100644 config/initializers/spring.rb create mode 100644 lib/http_ecs.rb create mode 100644 lib/job_event.rb create mode 100644 lib/main_loop.rb diff --git a/Gemfile b/Gemfile index a3f2279..4beb20a 100644 --- a/Gemfile +++ b/Gemfile @@ -12,7 +12,7 @@ gem 'uglifier', '>= 1.3.0' # Use CoffeeScript for .js.coffee assets and views gem 'coffee-rails', '~> 4.0.0' # See https://github.com/sstephenson/execjs#readme for more supported runtimes -# gem 'therubyracer', platforms: :ruby +gem 'therubyracer', platforms: :ruby # Use jquery as the JavaScript library gem 'jquery-rails' @@ -26,6 +26,10 @@ gem 'sdoc', '~> 0.4.0', group: :doc # Spring speeds up development by keeping your application running in the background. Read more: https://github.com/rails/spring gem 'spring', group: :development +gem 'daemons-rails' + +gem 'rest-client' + gem 'minitest-rails', group: [:development, :test] gem 'haml-rails', group: [:development, :test] # Use ActiveModel has_secure_password diff --git a/Gemfile.lock b/Gemfile.lock index a256940..85de409 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -36,9 +36,30 @@ GEM coffee-script-source execjs coffee-script-source (1.10.0) + daemons (1.2.3) + daemons-rails (1.2.1) + daemons + multi_json (~> 1.0) + domain_name (0.5.25) + unf (>= 0.0.5, < 1.0.0) erubis (2.7.0) execjs (2.6.0) + haml (4.0.7) + tilt + haml-rails (0.9.0) + actionpack (>= 4.0.1) + activesupport (>= 4.0.1) + haml (>= 4.0.6, < 5.0) + html2haml (>= 1.0.1) + railties (>= 4.0.1) hike (1.2.3) + html2haml (2.0.0) + erubis (~> 2.7.0) + haml (~> 4.0.0) + nokogiri (~> 1.6.0) + ruby_parser (~> 3.5) + http-cookie (1.0.2) + domain_name (~> 0.5) i18n (0.7.0) jbuilder (2.3.2) activesupport (>= 3.0.0, < 5) @@ -47,11 +68,19 @@ GEM railties (>= 3.0, < 5.0) thor (>= 0.14, < 2.0) json (1.8.3) + libv8 (3.16.14.13) mail (2.6.3) mime-types (>= 1.16, < 3) mime-types (2.99) + mini_portile2 (2.0.0) minitest (5.8.3) + minitest-rails (2.2.0) + minitest (~> 5.7) + railties (~> 4.1) multi_json (1.11.2) + netrc (0.11.0) + nokogiri (1.6.7) + mini_portile2 (~> 2.0.0.rc2) rack (1.5.5) rack-test (0.6.3) rack (>= 1.0) @@ -73,6 +102,13 @@ GEM rake (10.4.2) rdoc (4.2.0) json (~> 1.4) + ref (2.0.0) + rest-client (1.8.0) + http-cookie (>= 1.0.2, < 2.0) + mime-types (>= 1.16, < 3.0) + netrc (~> 0.7) + ruby_parser (3.7.2) + sexp_processor (~> 4.1) sass (3.2.19) sass-rails (4.0.5) railties (>= 4.0.0, < 5.0) @@ -82,6 +118,7 @@ GEM sdoc (0.4.1) json (~> 1.7, >= 1.7.7) rdoc (~> 4.0) + sexp_processor (4.6.0) spring (1.5.0) sprockets (2.12.4) hike (~> 1.2) @@ -93,6 +130,9 @@ GEM activesupport (>= 3.0) sprockets (>= 2.8, < 4.0) sqlite3 (1.3.11) + therubyracer (0.12.2) + libv8 (~> 3.16.14.0) + ref thor (0.19.1) thread_safe (0.3.5) tilt (1.4.1) @@ -103,19 +143,27 @@ GEM uglifier (2.7.2) execjs (>= 0.3.0) json (>= 1.8.0) + unf (0.1.4) + unf_ext + unf_ext (0.0.7.1) PLATFORMS ruby DEPENDENCIES coffee-rails (~> 4.0.0) + daemons-rails + haml-rails jbuilder (~> 2.0) jquery-rails + minitest-rails rails (= 4.1.8) + rest-client sass-rails (~> 4.0.3) sdoc (~> 0.4.0) spring sqlite3 + therubyracer turbolinks uglifier (>= 1.3.0) diff --git a/config/application.rb b/config/application.rb index 224d539..0b5d4d8 100644 --- a/config/application.rb +++ b/config/application.rb @@ -19,5 +19,6 @@ module Vipevalservice # The default locale is :en and all translations from config/locales/*.rb,yml are auto loaded. # config.i18n.load_path += Dir[Rails.root.join('my', 'locales', '*.{rb,yml}').to_s] # config.i18n.default_locale = :de + config.autoload_paths += %W(#{config.root}/lib) end end diff --git a/config/initializers/appcfg.rb b/config/initializers/appcfg.rb new file mode 100644 index 0000000..0a55ac2 --- /dev/null +++ b/config/initializers/appcfg.rb @@ -0,0 +1,6 @@ +APP_CONFIG = YAML.load(ERB.new(IO.read(Rails.root.join('config', 'appcfg.yml'))).result)[Rails.env] + +#Type.load_config_data +#Resource.load_config_data + +#HTTPI.log = false diff --git a/config/initializers/spring.rb b/config/initializers/spring.rb new file mode 100644 index 0000000..98b75f6 --- /dev/null +++ b/config/initializers/spring.rb @@ -0,0 +1 @@ +Spring.watch "config/appcfg.yml" diff --git a/lib/http_ecs.rb b/lib/http_ecs.rb new file mode 100644 index 0000000..2070dc0 --- /dev/null +++ b/lib/http_ecs.rb @@ -0,0 +1,25 @@ +class HttpEcs + + require 'rest_client' + include Singleton + + attr_reader :connection + + CONTENT_TYPE_URI_LIST = { 'Content-Type' => 'text/uri-list' } + CONTENT_TYPE_JSON = { 'Content-Type' => 'application/json' } + + def initialize + @connection= RestClient::Resource.new( + APP_CONFIG['ecs']['url'], + :ssl_ca_file => APP_CONFIG['ecs']['ssl_ca_file'], + :verify_ssl => APP_CONFIG['ecs']['verify_ssl'], + :headers => {"Content-Type" => :json, "Accept" => :json, "Authorization" => "Basic "+Base64.urlsafe_encode64(APP_CONFIG['ecs']['login']+":"+APP_CONFIG['ecs']['password'])} + ) + if APP_CONFIG['proxy'].blank? + RestClient.proxy = "" + else + RestClient.proxy = APP_CONFIG['proxy'] + end + end + +end diff --git a/lib/job_event.rb b/lib/job_event.rb new file mode 100644 index 0000000..0d0202d --- /dev/null +++ b/lib/job_event.rb @@ -0,0 +1,81 @@ +class JobEvent + def initialize + @ecs=HttpEcs.instance + end + + ## + # Process a *sys/events* job event. + # This is the entrypoint of job event processing. + def process(jobev) + Rails.logger.info "***** JobEvent#process: eventbody=#{jobev}" + case jobev[0]['status'] + when "created","updated" + job=JSON.parse(@ecs.connection[jobev[0]['ressource']].delete) + exercise = JSON.parse fetch_exercise(job) + evaluation = JSON.parse fetch_evaluation(job) + solution = JSON.parse fetch_solution(job) + exercise,solution = merge(exercise, evaluation, solution, job["EvaluationJob"]["identifier"]) + computation_backend = job["EvaluationJob"]["target"]["mid"] + compute(exercise, solution, computation_backend) + end + end + + private + + ## + # Fetch an exercise from ECS. + def fetch_exercise(job) + # URI#path returns the path with leading "/" + path= URI(job["EvaluationJob"]["resources"]["exercise"]).path[1..-1] + exercise= @ecs.connection[path].get # FIXME change to delete + Rails.logger.info "***** JobEvent#fetch_exercise: #{path} = #{exercise}" + exercise + end + + ## + # Fetch an evaluation from ECS. + def fetch_evaluation(job) + # URI#path returns the path with leading "/" + path= URI(job["EvaluationJob"]["resources"]["evaluation"]).path[1..-1] + evaluation= @ecs.connection[path].get # FIXME change to delete + Rails.logger.info "***** JobEvent#fetch_evaluation: #{path} = #{evaluation}" + evaluation + end + + ## + # Fetch a solution from ECS. + def fetch_solution(job) + # URI#path returns the path with leading "/" + path= URI(job["EvaluationJob"]["resources"]["solution"]).path[1..-1] + solution= @ecs.connection[path].get # FIXME change to delete + Rails.logger.info "***** JobEvent#fetch_solution: #{path} = #{solution}" + solution + end + + ## + # Substitute evaluation code snippets with appropriate exercise code + # snippets. + def merge(exercise, evaluation, solution, jobid) + evaluation["Evaluation"]["elements"].each do |ev| + exercise["Exercise"]["elements"].map! do |ex| + if ex["identifier"] == ev["identifier"] + ex=ev + else + ex + end + end + end + solution["Solution"]["evaluationJobID"]= jobid + Rails.logger.info "***** JobEvent#merge exercise: #{exercise.to_json}" + Rails.logger.info "***** JobEvent#merge solution: #{solution.to_json}" + return exercise, solution + end + + ## + # Calls the computation backend with membership_id *mid*. + def compute(exercise, solution, mid) + @ecs.connection[APP_CONFIG["resources"]["exercises"]["name"]].post exercise.to_json, {"X-EcsReceiverMemberships" => mid} + @ecs.connection[APP_CONFIG["resources"]["solutions"]["name"]].post solution.to_json, {"X-EcsReceiverMemberships" => mid} + end + +end diff --git a/lib/main_loop.rb b/lib/main_loop.rb new file mode 100644 index 0000000..473ec6e --- /dev/null +++ b/lib/main_loop.rb @@ -0,0 +1,66 @@ +## +# Loops for getting new events from *sys/events*. +# This class is a singelton, because it should be +# the only looping thing. +class MainLoop + include Singleton + +# class << self +# attr_accessor :exception_tries +# end +# +# attr_accessor :ecs + + @exception_tries= 0 + + def initialize + Rails.logger.info "*** VIP merge and compute service started ***" + @ecs= HttpEcs.instance + end + + def start + loop do + evbody=JSON::parse((ev=read_event).body) + if evbody.blank? + #Rails.logger.info "MainLoop#start: " + sleep(1) + else + if evbody[0]["ressource"].start_with?(APP_CONFIG["eventtypes"]["job"]["name"]) + Rails.logger.info "received \"#{APP_CONFIG["eventtypes"]["job"]["name"]}\" event type" + JobEvent.new.process(evbody) + elsif evbody[0]["ressource"].start_with?(APP_CONFIG["eventtypes"]["result"]["name"]) + Rails.logger.info "received \"#{APP_CONFIG["eventtypes"]["result"]["name"]}\" event type" + process_result_event(evbody) + else + Rails.logger.info "Unknown event type" + end + end + end + rescue => e + Rails.logger.error "MainLoop#start:Exception: #{e.class}: #{e.message}" + Rails.logger.error Rails.backtrace_cleaner.clean(e.backtrace) + #retry if MainLoop.try_ones_more? + end + + private + + def self.try_ones_more? + if MainLoop.exception_tries < 1 + sleep(2**MainLoop.exception_tries) + MainLoop.exception_tries+= 1 + true + else + false + end + end + + def read_event + @ecs.connection["sys/events/fifo"].post "" + end + + def process_result_event(ev) + # TODO remove exercise referenced through solution embedded in result + end + +end + -- cgit v1.2.3