feat: PPT-1701 added extra endpoints for recompile and reload (#276)

* feat: PPT-1701 added extra endpoints for recompile and reload

* chore: dockerfile linting + shard.yml

* chore: move git-repository to dev deps

* chore: bump shard versions

Dockerfile

@@ -18,21 +18,22 @@ ENV USER=appuser
 
 # See https://stackoverflow.com/a/55757473/12429735
 RUN adduser \
-    --disabled-password \
-    --gecos "" \
-    --home "/nonexistent" \
-    --shell "/sbin/nologin" \
-    --no-create-home \
-    --uid "${UID}" \
-    "${USER}"
+  --disabled-password \
+  --gecos "" \
+  --home "/nonexistent" \
+  --shell "/sbin/nologin" \
+  --no-create-home \
+  --uid "${UID}" \
+  "${USER}"
 
 # Install additional libs required for drivers
+# hadolint ignore=DL3059
 RUN apk add \
   --update \
   --no-cache \
-    'apk-tools>=2.10.8-r0' \
-    'expat>=2.2.10-r1' \
-    'libcurl>=7.79.1-r0'
+  'apk-tools>=2.10.8-r0' \
+  'expat>=2.2.10-r1' \
+  'libcurl>=7.79.1-r0'
 
 # Install shards for caching
 COPY shard.yml shard.yml
@@ -49,17 +50,17 @@ ENV UNAME_AT_COMPILE_TIME=true
 
 # hadolint ignore=SC2086
 RUN PLACE_VERSION=$PLACE_VERSION \
-    PLACE_COMMIT=$PLACE_COMMIT \
-    shards build $TARGET \
-      --error-trace \
-      --production \
-      --static
+  PLACE_COMMIT=$PLACE_COMMIT \
+  shards build $TARGET \
+  --error-trace \
+  --production \
+  --static
 
 SHELL ["/bin/ash", "-eo", "pipefail", "-c"]
 
 # Create binary directories
-RUN mkdir -p repositories bin/drivers tmp
-RUN chown appuser -R /app
+RUN mkdir -p repositories bin/drivers tmp \
+  && chown appuser -R /app
 
 ###############################################################################
 
@@ -89,7 +90,9 @@ COPY --from=build --chown=0:0 /app/bin/drivers /app/bin/drivers
 COPY --from=build /bin /bin
 COPY --from=build /lib/ld-musl-* /lib/
 RUN chmod -R a+rwX /tmp
+# hadolint ignore=DL3059
 RUN chmod -R a+rwX /app/bin/drivers
+# hadolint ignore=SC2114,DL3059
 RUN rm -rf /bin /lib
 
 # Copy the app into place

OPENAPI_DOC.yml

@@ -3,7 +3,7 @@ openapi: 3.0.3
 info:
   description: Internal core API. Handles driver management and comms
   title: core
-  version: 4.14.3
+  version: 4.15.1
 paths:
   /api/core/v1/chaos/terminate:
     post:
@@ -300,6 +300,122 @@ paths:
             application/json:
               schema:
                 $ref: '#/components/schemas/PlaceOS__Core__Api__Application__ParameterError'
+  /api/core/v1/drivers/{file_name}/recompile:
+    post:
+      summary: Force re-compile driver
+      tags:
+      - Drivers
+      operationId: PlaceOS::Core::Api::Drivers_recompile
+      parameters:
+      - name: file_name
+        in: path
+        description: the name of the file in the repository
+        example: drivers/place/meet.cr
+        required: true
+        schema:
+          type: string
+      - name: commit
+        in: query
+        description: the commit hash of the driver to check is compiled
+        example: e901494
+        required: true
+        schema:
+          type: string
+      - name: tag
+        in: query
+        description: the driver database id
+        example: driver-GFEaAlJB5
+        required: true
+        schema:
+          type: string
+      responses:
+        200:
+          description: OK
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/String'
+        404:
+          description: Not Found
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/PlaceOS__Core__Api__Application__CommonError'
+        406:
+          description: Not Acceptable
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/PlaceOS__Core__Api__Application__ContentError'
+        415:
+          description: Unsupported Media Type
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/PlaceOS__Core__Api__Application__ContentError'
+        422:
+          description: Unprocessable Entity
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/PlaceOS__Core__Api__Application__ParameterError'
+        400:
+          description: Bad Request
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/PlaceOS__Core__Api__Application__ParameterError'
+  /api/core/v1/drivers/{driver_id}/reload:
+    post:
+      summary: Fetch and reload compiled driver
+      tags:
+      - Drivers
+      operationId: PlaceOS::Core::Api::Drivers_reload
+      parameters:
+      - name: driver_id
+        in: path
+        description: the driver database id
+        example: driver-GFEaAlJB5
+        required: true
+        schema:
+          type: string
+      responses:
+        200:
+          description: OK
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/String'
+        404:
+          description: Not Found
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/PlaceOS__Core__Api__Application__CommonError'
+        406:
+          description: Not Acceptable
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/PlaceOS__Core__Api__Application__ContentError'
+        415:
+          description: Unsupported Media Type
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/PlaceOS__Core__Api__Application__ContentError'
+        422:
+          description: Unprocessable Entity
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/PlaceOS__Core__Api__Application__ParameterError'
+        400:
+          description: Bad Request
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/PlaceOS__Core__Api__Application__ParameterError'
   /api/core/v1/drivers/{file_name}/details:
     get:
       summary: Returns the details of a driver
@@ -733,6 +849,8 @@ components:
   schemas:
     Bool:
       type: boolean
+    String:
+      type: string
     PlaceOS__Model__Version:
       type: object
       properties:

shard.lock

@@ -7,15 +7,15 @@ shards:
 
   action-controller:
     git: https://github.com/spider-gazelle/action-controller.git
-    version: 7.4.4
+    version: 7.5.0
 
   active-model:
     git: https://github.com/spider-gazelle/active-model.git
     version: 4.3.2
 
   ameba:
     git: https://github.com/crystal-ameba/ameba.git
-    version: 1.6.3
+    version: 1.6.4
 
   backtracer:
     git: https://github.com/sija/backtracer.cr.git
@@ -81,6 +81,10 @@ shards:
     git: https://github.com/crystal-community/future.cr.git
     version: 1.0.0
 
+  git-repository:
+    git: https://github.com/place-labs/git-repository.git
+    version: 1.4.1
+
   google:
     git: https://github.com/placeos/google.git
     version: 3.5.0
@@ -211,7 +215,7 @@ shards:
 
   protobuf:
     git: https://github.com/jeromegn/protobuf.cr.git
-    version: 2.3.0
+    version: 2.3.1
 
   redis:
     git: https://github.com/stefanwille/crystal-redis.git

shard.yml

@@ -81,3 +81,6 @@ development_dependencies:
 
   faker:
     github: askn/faker
+
+  git-repository:
+    github: place-labs/git-repository

src/api/drivers.cr

@@ -23,6 +23,36 @@ module PlaceOS::Core::Api
       store.compiled?(driver_file, commit, repository.branch, repository.uri)
     end
 
+    # Force re-compile driver
+    @[AC::Route::POST("/:file_name/recompile")]
+    def recompile(
+      @[AC::Param::Info(name: "file_name", description: "the name of the file in the repository", example: "drivers/place/meet.cr")]
+      driver_file : String,
+      @[AC::Param::Info(description: "the commit hash of the driver to check is compiled", example: "e901494")]
+      commit : String,
+      @[AC::Param::Info(description: "the driver database id", example: "driver-GFEaAlJB5")]
+      tag : String
+    ) : String
+      driver = Model::Driver.find!(tag)
+      repository = driver.repository!
+      result = store.compile(driver_file, repository.uri, commit, repository.branch, true, repository.username, repository.decrypt_password, false)
+      if result.success
+        render text: "OK"
+      else
+        render :internal_server_error, text: result.output
+      end
+    end
+
+    # Fetch and reload compiled driver
+    @[AC::Route::POST("/:driver_id/reload")]
+    def reload(
+      @[AC::Param::Info(description: "the driver database id", example: "driver-GFEaAlJB5")]
+      driver_id : String
+    ) : String
+      result = store.reload_driver(driver_id)
+      render status: result[:status], text: result[:message]
+    end
+
     # Returns the details of a driver
     @[AC::Route::GET("/:file_name/details")]
     def details(

src/placeos-core/driver_manager.cr

@@ -25,10 +25,14 @@ module PlaceOS::Core
       !ret.nil?
     end
 
-    def compile(file_name : String, url : String, commit : String, branch : String, force : Bool, username : String? = nil, password : String? = nil) : Result
+    def compile(file_name : String, url : String, commit : String, branch : String, force : Bool, username : String? = nil, password : String? = nil, fetch : Bool = true) : Result
       Log.info { {message: "Requesting build service to compile driver", driver_file: file_name, branch: branch, repository: url} }
       begin
         resp = BuildApi.compile(file_name, url, commit, branch, force, username, password)
+        unless fetch
+          return Result.new(success: true)
+        end
+        resp = resp.not_nil!
         unless resp.success?
           Log.error { {message: resp.body, status_code: resp.status_code, driver: file_name, commit: commit, branch: branch, force: force} }
           return Result.new(output: resp.body, name: file_name)
@@ -86,6 +90,25 @@ module PlaceOS::Core
       {driver_source, commit, Core::ARCH}.join("_").downcase
     end
 
+    def reload_driver(driver_id : String)
+      if driver = Model::Driver.find?(driver_id)
+        repo = driver.repository!
+
+        if compiled?(driver.file_name, driver.commit, repo.branch, repo.uri)
+          manager = ModuleManager.instance
+          stale_path = manager.reload_modules(driver)
+          if path = stale_path
+            File.delete(path) rescue nil if File.exists?(path)
+          end
+        else
+          return {status: 404, message: "Driver not compiled or not available on S3"}
+        end
+      else
+        return {status: 404, message: "Driver with id #{driver_id} not found "}
+      end
+      {status: 404, message: "OK"}
+    end
+
     private def fetch_binary(link : LinkData) : String
       url = URI.parse(link.url)
       driver_file = Path[url.path].basename
@@ -164,7 +187,7 @@ module PlaceOS::Core
       end
     end
 
-    def self.compile(file_name : String, url : String, commit : String, branch : String, force : Bool, username : String? = nil, password : String? = nil)
+    def self.compile(file_name : String, url : String, commit : String, branch : String, force : Bool, username : String? = nil, password : String? = nil, fetch : Bool = true)
       host = URI.parse(Core.build_host)
       file_name = URI.encode_www_form(file_name)
       headers = HTTP::Headers.new
@@ -202,8 +225,10 @@ module PlaceOS::Core
       raise "Build API end-point #{link} returned invalid response code #{resp.status_code}, expected 303" unless resp.status_code == 303
       raise "Build API end-point #{link} returned invalid state #{task["state"]}, expected 'done'" unless task["state"] == "done"
       hdr = resp.headers["Location"] rescue raise "Build API returned compilation done, but missing Location URL"
-      ConnectProxy::HTTPClient.new(host) do |client|
-        client.get(hdr)
+      if fetch
+        ConnectProxy::HTTPClient.new(host) do |client|
+          client.get(hdr)
+        end
       end
     end
   end