diff --git a/.ameba.yml b/.ameba.yml new file mode 100644 index 0000000..f16ec1a --- /dev/null +++ b/.ameba.yml @@ -0,0 +1,4 @@ +Naming/BlockParameterName: + Description: Disallows non-descriptive block parameter names + Enabled: false + Severity: Convention diff --git a/docker-compose.yml b/docker-compose.yml index 2de8af4..38aec2d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -51,6 +51,7 @@ services: environment: PLACEOS_BUILD_LOCAL: "true" PLACEOS_ENABLE_TRACE: "true" + BUILD_SERVICE_DISABLED: "true" redis: image: eqalpha/keydb diff --git a/shard.lock b/shard.lock index f5ca061..653669b 100644 --- a/shard.lock +++ b/shard.lock @@ -7,7 +7,7 @@ shards: action-controller: git: https://github.com/spider-gazelle/action-controller.git - version: 4.10.1 + version: 7.5.1 active-model: git: https://github.com/spider-gazelle/active-model.git @@ -15,7 +15,7 @@ shards: ameba: git: https://github.com/crystal-ameba/ameba.git - version: 1.6.1 + version: 1.6.4 ansi-escapes: git: https://github.com/gtramontina/ansi-escapes.cr.git @@ -39,7 +39,7 @@ shards: bindata: git: https://github.com/spider-gazelle/bindata.git - version: 2.0.0 + version: 2.1.0 clip: git: https://github.com/erdnaxeli/clip.git @@ -47,7 +47,7 @@ shards: connect-proxy: git: https://github.com/spider-gazelle/connect-proxy.git - version: 2.0.0 + version: 2.0.2 crc16: git: https://github.com/maiha/crc16.cr.git @@ -63,7 +63,7 @@ shards: csuuid: git: https://github.com/wyhaines/csuuid.cr.git - version: 1.0.0+git.commit.a4cf9615c6518cf27c68a1755a8c2ac4ae4fe987 + version: 1.0.2 db: git: https://github.com/crystal-lang/crystal-db.git @@ -71,7 +71,7 @@ shards: debug: git: https://github.com/sija/debug.cr.git - version: 2.0.2 + version: 2.0.4 defined: git: https://github.com/wyhaines/defined.cr.git @@ -83,19 +83,15 @@ shards: email: git: https://github.com/arcage/crystal-email.git - version: 0.7.0 - - etcd: - git: https://github.com/place-labs/crystal-etcd.git - version: 1.2.5 + version: 0.7.1 eventbus: git: https://github.com/spider-gazelle/eventbus.git - version: 0.9.9+git.commit.ca8ef0c5e21ee15da079edd5bcea39bee7e07f26 + version: 1.0.0+git.commit.af63536d718348885a553dc4aa6debccc2946289 exception_page: git: https://github.com/crystal-loot/exception_page.git - version: 0.4.1 + version: 0.5.0 exec_from: git: https://github.com/place-labs/exec_from.git @@ -107,7 +103,7 @@ shards: git-repository: git: https://github.com/place-labs/git-repository.git - version: 1.3.1 + version: 1.4.1 google: git: https://github.com/placeos/google.git @@ -119,11 +115,11 @@ shards: habitat: git: https://github.com/luckyframework/habitat.git - version: 0.4.8 + version: 0.4.9 - hound-dog: - git: https://github.com/place-labs/hound-dog.git - version: 2.9.1 + hot_topic: + git: https://github.com/jgaskins/hot_topic.git + version: 0.1.0+git.commit.3c901e77b6e000930398738260a2944b6f5785dc http-params-serializable: git: https://github.com/place-labs/http-params-serializable.git @@ -151,7 +147,7 @@ shards: lucky_router: git: https://github.com/luckyframework/lucky_router.git - version: 0.5.2 + version: 0.6.0 lz4: git: https://github.com/naqvis/lz4.cr.git @@ -167,23 +163,15 @@ shards: nbchannel: git: https://github.com/wyhaines/nbchannel.cr.git - version: 0.1.0+git.commit.a8f5be6aa198abfa9f1893e1156640b8ea526094 + version: 0.1.0 neuroplastic: git: https://github.com/spider-gazelle/neuroplastic.git - version: 1.13.0 + version: 1.13.1 office365: git: https://github.com/placeos/office365.git - version: 1.23.3 - - open_api: - git: https://github.com/elbywan/open_api.cr.git - version: 1.3.0 - - openapi-generator: - git: https://github.com/place-labs/openapi-generator.git - version: 2.1.0+git.commit.a65ffc2f7dcc6a393e7d1f9229650b520d9525be + version: 1.25.4 openssl_ext: git: https://github.com/spider-gazelle/openssl_ext.git @@ -191,15 +179,15 @@ shards: opentelemetry-api: git: https://github.com/wyhaines/opentelemetry-api.cr.git - version: 0.5.0 + version: 0.5.1 opentelemetry-instrumentation: git: https://github.com/wyhaines/opentelemetry-instrumentation.cr.git - version: 0.5.3+git.commit.cd3994b22d9f7a0d68752698974d3873a1b2fce2 + version: 0.5.5+git.commit.83eace832dad20a71231e71a42258538b05fb193 opentelemetry-sdk: git: https://github.com/wyhaines/opentelemetry-sdk.cr.git - version: 0.6.1 + version: 0.6.2 pars: # Overridden git: https://github.com/spider-gazelle/pars.git @@ -215,11 +203,11 @@ shards: pg: git: https://github.com/will/crystal-pg.git - version: 0.28.0 + version: 0.29.0 pg-orm: git: https://github.com/spider-gazelle/pg-orm.git - version: 1.1.0+git.commit.e35e4c535be1ab47b6afb821723ceafb637fa4e8 + version: 1.1.2+git.commit.9b340ee269cd4a10ed6c5b51235cbaf45fc380e1 pinger: git: https://github.com/spider-gazelle/pinger.git @@ -227,15 +215,15 @@ shards: place_calendar: git: https://github.com/placeos/calendar.git - version: 4.20.1 + version: 4.22.1 placeos-build: git: https://github.com/placeos/build.git - version: 1.3.0 + version: 1.5.1 placeos-compiler: git: https://github.com/placeos/compiler.git - version: 4.9.3 + version: 4.9.4 placeos-core-client: git: https://github.com/placeos/core-client.git @@ -243,7 +231,7 @@ shards: placeos-driver: git: https://github.com/placeos/driver.git - version: 6.9.18 + version: 7.3.0 placeos-log-backend: git: https://github.com/place-labs/log-backend.git @@ -251,7 +239,7 @@ shards: placeos-models: git: https://github.com/placeos/models.git - version: 9.37.0 + version: 9.60.1 pool: git: https://github.com/ysbaddaden/pool.git @@ -259,15 +247,15 @@ shards: promise: git: https://github.com/spider-gazelle/promise.git - version: 3.0.0 + version: 3.2.0 protobuf: git: https://github.com/jeromegn/protobuf.cr.git - version: 2.3.0 + version: 2.3.1 raven: git: https://github.com/sija/raven.cr.git - version: 1.9.3+git.commit.990a7650fb027cbe65705fc6dad228cddc76c503 + version: 1.9.4+git.commit.a43f45229d1ee8030f8f07b101cb159896273195 redis: git: https://github.com/stefanwille/crystal-redis.git @@ -277,6 +265,10 @@ shards: git: https://github.com/caspiano/redis-cluster.cr.git version: 0.8.5 + redis_service_manager: + git: https://github.com/place-labs/redis_service_manager.git + version: 3.2.0 + rendezvous-hash: git: https://github.com/caspiano/rendezvous-hash.git version: 0.3.1 @@ -299,7 +291,7 @@ shards: shards: git: https://github.com/crystal-lang/shards.git - version: 0.17.4 + version: 0.18.0 simple_retry: git: https://github.com/spider-gazelle/simple_retry.git @@ -307,11 +299,11 @@ shards: splay_tree_map: git: https://github.com/wyhaines/splay_tree_map.cr.git - version: 0.2.2 + version: 0.3.0 ssh2: git: https://github.com/spider-gazelle/ssh2.cr.git - version: 1.6.1 + version: 1.7.0 tasker: git: https://github.com/spider-gazelle/tasker.git @@ -319,7 +311,7 @@ shards: time-ext: git: https://github.com/wyhaines/time-ext.cr.git - version: 0.1.0+git.commit.175f658235fb6cdc9c804cb96da510fec27f4cd6 + version: 1.0.1 tokenizer: git: https://github.com/spider-gazelle/tokenizer.git @@ -327,7 +319,7 @@ shards: tracer: git: https://github.com/wyhaines/tracer.cr.git - version: 0.3.1 + version: 0.3.2 ulid: # Overridden git: https://github.com/place-labs/ulid.git diff --git a/spec/build_spec.cr b/spec/build_spec.cr index 24c2ecb..e342bce 100644 --- a/spec/build_spec.cr +++ b/spec/build_spec.cr @@ -4,9 +4,8 @@ module PlaceOS::Drivers::Api describe Build do describe "GET /build" do it "should list drivers" do - context = Api::Build.with_request("GET", "/build?repository=private_drivers", &.index) - - drivers = Array(String).from_json(context.response.output.to_s) + resp = client.get(Build.base_route + "?repository=private_drivers") + drivers = Array(String).from_json(resp.body) drivers.should_not be_empty drivers.should contain("drivers/place/private_helper.cr") end @@ -14,25 +13,24 @@ module PlaceOS::Drivers::Api describe "POST /build" do it "should build a driver" do - Api::Build - .with_request("POST", "/build?commit=#{DRIVER_COMMIT}&repository=private_drivers&driver=drivers/place/private_helper.cr", &.create) - .response - .status_code.should eq(201) + params = HTTP::Params.encode({"commit" => DRIVER_COMMIT, "repository" => "private_drivers", "driver" => "drivers/place/private_helper.cr"}) + resp = client.post(Build.base_route + "?#{params}") + resp.status_code.should eq(201) end end describe "GET /build/driver/:driver" do it "should list compiled versions" do - Api::Build - .with_request("POST", "/build?commit=#{DRIVER_COMMIT}&repository=private_drivers&driver=src/place/private_helper.cr", &.create) - .response.status + params = HTTP::Params.encode({"commit" => DRIVER_COMMIT, "repository" => "private_drivers", "driver" => "drivers/place/private_helper.cr"}) + resp = client.post(Build.base_route + "?#{params}") + resp.status .success? .should be_true - context = Api::Build.with_request("GET", "/build/src%2Fplace%2Fprivate_helper.cr", route_params: {"driver" => "src/place/private_helper.cr"}, &.show) - + params = HTTP::Params.encode({"driver" => "drivers/place/private_helper.cr"}) + resp = client.get(Build.base_route + "/#{params}") Array(PlaceOS::Model::Executable) - .from_json(context.response.output.to_s) + .from_json(resp.body) .tap(&.should_not be_empty) .first .filename @@ -42,8 +40,11 @@ module PlaceOS::Drivers::Api describe "GET /build/driver/:driver/commits" do it "should list commits" do - context = Api::Build.with_request("GET", "/build/src%2Fplace%2Fprivate_helper.cr/commits?repository=private_drivers", route_params: {"driver" => "src/place/private_helper.cr"}, &.commits) - Array(PlaceOS::Compiler::Git::Commit).from_json(context.response.output.to_s).should_not be_empty + resp = client.get(Build.base_route + "/src%2Fplace%2Fprivate_helper.cr/commits?repository=private_drivers") + resp.status + .success? + .should be_true + Array(PlaceOS::Compiler::Git::Commit).from_json(resp.body).should_not be_empty end end end diff --git a/spec/compiler_spec.cr b/spec/compiler_spec.cr index 03b5898..1459533 100644 --- a/spec/compiler_spec.cr +++ b/spec/compiler_spec.cr @@ -8,10 +8,9 @@ module PlaceOS::Drivers PlaceOS::Compiler.clone_and_install("private_drivers", "https://github.com/placeos/private-drivers.git") File.file?(File.expand_path("./repositories/private_drivers/drivers/place/private_helper.cr")).should be_true - Api::Build - .with_request("POST", "/build?repository=private_drivers&driver=drivers/place/private_helper.cr", &.create) - .response - .status_code.should eq(201) + params = HTTP::Params.encode({"repository" => "private_drivers", "driver" => "drivers/place/private_helper.cr"}) + resp = client.post(Api::Build.base_route + "?#{params}") + resp.status_code.should eq(201) end end end diff --git a/spec/spec_helper.cr b/spec/spec_helper.cr index ac185c4..fa6055f 100644 --- a/spec/spec_helper.cr +++ b/spec/spec_helper.cr @@ -1,3 +1,4 @@ +require "action-controller/spec_helper" require "spec" # Your application config @@ -5,43 +6,33 @@ require "spec" require "../src/config" # Helper methods for testing controllers (curl, with_server, context) -require "../lib/action-controller/spec/curl_context" +# require "../lib/action-controller/spec/curl_context" require "placeos-compiler" -class MockServer - include ActionController::Router +module PlaceOS::Api::SpecClient + # Can't use ivars at top level, hence this hack + private CLIENT = ActionController::SpecHelper.client - def initialize - init_routes - end - - private def init_routes - {% for klass in ActionController::Base::CONCRETE_CONTROLLERS %} - {{klass}}.__init_routes__(self) - {% end %} + def client + CLIENT end end -abstract class PlaceOS::Drivers::Api::Application < ActionController::Base - def self.with_request(verb, path, expect_failure = false, route_params = nil) - io = IO::Memory.new - context = context(verb.upcase, path) - - context.route_params = route_params if route_params - - MOCK_SERVER.route_handler.search_route(verb, path, "#{verb.downcase}#{path}", context) - context.response.output = io +include PlaceOS::Api::SpecClient - yield new(context) - - context.response.status.success?.should(expect_failure ? be_false : be_true) - context +abstract class ActionController::Base + macro inherited + macro finished + {% begin %} + def self.base_route + NAMESPACE[0] + end + {% end %} + end end end -MOCK_SERVER = MockServer.new - DRIVER_COMMIT = "265518b" SPEC_COMMIT = "d1b51ac" @@ -53,3 +44,40 @@ Spec.before_suite do "https://github.com/placeos/private-drivers" ) end + +::CURL_CONTEXT__ = [] of ActionController::Server + +macro with_server(&block) + if ::CURL_CONTEXT__.empty? + %app = ActionController::Server.new + ::CURL_CONTEXT__ << %app + %channel = Channel(Nil).new(1) + + Spec.before_each do + %app.reload + %app.socket.bind_tcp("127.0.0.1", 6000, true) + begin + File.delete("/tmp/spider-socket.sock") + rescue + end + %app.socket.bind_unix "/tmp/spider-socket.sock" + spawn do + %channel.send(nil) + %app.run + end + %channel.receive + end + + Spec.after_each do + %app.close + end + end + + %app = ::CURL_CONTEXT__[0] + + {% if block.args.size > 0 %} + {{block.args[0].id}} = %app + {% end %} + + {{block.body}} +end diff --git a/spec/test_spec.cr b/spec/test_spec.cr index fc79d22..634a7d7 100644 --- a/spec/test_spec.cr +++ b/spec/test_spec.cr @@ -4,8 +4,9 @@ module PlaceOS::Drivers describe Api::Test do describe "GET /test" do it "should list driver specs" do - context = Api::Test.with_request("GET", "/test?repository=private_drivers", &.index) - drivers = Array(String).from_json(context.response.output.to_s) + resp = client.get(Api::Test.base_route + "?repository=private_drivers") + resp.status.success?.should be_true + drivers = Array(String).from_json(resp.body) drivers.should_not be_empty drivers.should contain("drivers/place/private_helper_spec.cr") end @@ -13,7 +14,11 @@ module PlaceOS::Drivers describe "POST /test" do it "should build and test a driver" do - Api::Test.with_request("POST", "/test?spec_commit=#{SPEC_COMMIT}&commit=#{DRIVER_COMMIT}&repository=private_drivers&driver=drivers/place/private_helper.cr&spec=drivers/place/private_helper_spec.cr&force=true", &.create) + params = HTTP::Params.encode({"spec_commit" => SPEC_COMMIT, "commit" => DRIVER_COMMIT, "repository" => "private_drivers", "driver" => "drivers/place/private_helper.cr", "spec" => "drivers/place/private_helper_spec.cr", "enforce" => "true"}) + resp = client.post(Api::Test.base_route + "?#{params}") + resp.status + .success? + .should be_true end end end diff --git a/src/controllers/application.cr b/src/controllers/application.cr index 9ac123d..338425c 100644 --- a/src/controllers/application.cr +++ b/src/controllers/application.cr @@ -5,7 +5,9 @@ require "uuid" module PlaceOS::Drivers::Api abstract class Application < ActionController::Base - before_action :set_request_id + macro inherited + Log = ::Log.for({{ @type }}) + end class_getter binary_store = PlaceOS::Build::Filesystem.new @@ -26,10 +28,24 @@ module PlaceOS::Drivers::Api ########################################################################### - # Support request tracking + getter request_id : String do + request.headers["X-Request-ID"]? || UUID.random.to_s + end + + # This makes it simple to match client requests with server side logs. + # When building microservices this ID should be propagated to upstream services. + @[AC::Route::Filter(:before_action)] def set_request_id - Log.context.set(client_ip: client_ip) - response.headers["X-Request-ID"] = Log.context.metadata[:request_id].as_s + Log.context.set( + client_ip: client_ip, + request_id: request_id + ) + response.headers["X-Request-ID"] = request_id + end + + @[AC::Route::Filter(:before_action)] + def set_date_header + response.headers["Date"] = HTTP.format_time(Time.utc) end getter working_directory : String = Path["./repositories"].expand.to_s @@ -43,7 +59,7 @@ module PlaceOS::Drivers::Api Compiler::Git.repository_path(repository, working_directory) end - def with_temporary_repository + def with_temporary_repository(&) temporary_working_directory = File.join(Dir.tempdir, UUID.random.to_s) Dir.mkdir_p(temporary_working_directory) @@ -81,7 +97,7 @@ module PlaceOS::Drivers::Api PlaceOS::Build::Client.client do |client| client.repository_path = repository_path - client.compile(file: driver, url: "local", commit: commit, force_recompile: force_recompile) do |key, io| + client.compile(file: driver, url: "local", repository_path: Path[working_directory, repository].to_s, commit: commit, force_recompile: force_recompile) do |key, io| binary_store.write(key, io) end end diff --git a/src/controllers/build.cr b/src/controllers/build.cr index 1c76c00..4884936 100644 --- a/src/controllers/build.cr +++ b/src/controllers/build.cr @@ -7,15 +7,8 @@ module PlaceOS::Drivers::Api class Build < Application base "/build" - id_param :driver - - # Build a drvier, optionally based on the version specified - # - def create - compilation_response(build_driver(driver, commit, force?)) - end - # List the available files + @[AC::Route::GET("/")] def index compiled = params["compiled"]? result = PlaceOS::Build::Client.client do |client| @@ -29,11 +22,20 @@ module PlaceOS::Drivers::Api render json: result end + @[AC::Route::GET("/:driver")] def show entrypoint = route_params["driver"] render json: PlaceOS::Build::Client.client(&.query(file: entrypoint)) end + # Build a drvier, optionally based on the version specified + # + @[AC::Route::POST("/")] + def create + result = build_driver(driver, commit, force?) + compilation_response(result) + end + # grab the list of available repositories get "/repositories", :list_repositories do render json: PlaceOS::Compiler.repositories diff --git a/src/controllers/test.cr b/src/controllers/test.cr index 6902cb0..8f52250 100644 --- a/src/controllers/test.cr +++ b/src/controllers/test.cr @@ -4,8 +4,6 @@ module PlaceOS::Drivers::Api class Test < Application base "/test" - id_param :driver - @driver_path : String? = nil @spec_path : String? = nil @@ -22,7 +20,7 @@ module PlaceOS::Drivers::Api getter spec_commit : String? { params["spec_commit"]? } # Specs available - def index + get "/", :index do result = Dir.cd(repository_path) do Dir.glob("drivers/**/*_spec.cr") end @@ -41,7 +39,7 @@ module PlaceOS::Drivers::Api end # Run the spec and return success if the exit status is 0 - def create + post "/", :create do @driver_path, @spec_path = { {driver, commit}, {spec, spec_commit}, @@ -205,14 +203,14 @@ module PlaceOS::Drivers::Api select when channel.receive when closed_channel.receive? - process.not_nil!.signal(:kill) + process.try &.signal(:kill) channel.receive when timeout(5.minutes) - process.not_nil!.signal(:kill) + process.try &.signal(:kill) channel.receive end - exit_code = status.not_nil!.exit_code + exit_code = status.try &.exit_code || 1 io << "spec runner exited with #{exit_code}\n" Log.error { memory.to_s } unless exit_code.zero? diff --git a/src/report/cli.cr b/src/report/cli.cr index 6a313d1..4173a0a 100644 --- a/src/report/cli.cr +++ b/src/report/cli.cr @@ -246,7 +246,7 @@ module PlaceOS::Drivers nil end - def self.with_runner_client + def self.with_runner_client(&) HTTP::Client.new(Settings.host, Settings.port) do |client| yield client end diff --git a/test b/test index 9b8e319..63cff26 100755 --- a/test +++ b/test @@ -5,7 +5,7 @@ set -e # this function is called when Ctrl-C is sent function trap_ctrlc () { - docker-compose down + docker compose down exit 2 } @@ -15,8 +15,8 @@ trap "trap_ctrlc" 2 exit_code="0" -docker-compose run --rm test $@ || exit_code="$?" +docker compose run --rm test $@ || exit_code="$?" -docker-compose down +docker compose down exit ${exit_code}