Merge remote-tracking branch 'origin/dev' into feature/openvino

LlamaCPP/llama_adapter.py

@@ -1,6 +1,7 @@
 import threading
 from queue import Empty, Queue
 import json
+import time
 import traceback
 from typing import Dict, List, Callable
 #from model_downloader import NotEnoughDiskSpaceException, DownloadException
@@ -92,7 +93,13 @@ def text_conversation(self, params: LLMParams):
         return self.generator()
 
 
-    def stream_function(self, stream):
+    def stream_function(self, stream):  
+        num_tokens = 0
+        start_time = time.time()
+        is_first = True
+        first_token_time = 0.0
+        last_token_time = 0.0
+
         for output in stream:
             if self.llm_interface.stop_generate:
                 self.llm_interface.stop_generate = False
@@ -104,6 +111,26 @@ def stream_function(self, stream):
             else:
                 # openai style
                 self.text_out_callback(output["choices"][0]["delta"].get("content",""))
+                num_tokens += 1
+
+                if is_first:
+                    first_token_time = time.time()
+                    is_first = False
+
+        last_token_time = time.time()
+
+        metrics_data = {
+            "type": "metrics",
+            "num_tokens": num_tokens,
+            "total_time": last_token_time - start_time,
+            "overall_tokens_per_second": num_tokens / (last_token_time - start_time),
+            "second_plus_tokens_per_second": (num_tokens - 1) / (last_token_time - first_token_time),
+            "first_token_latency": first_token_time - start_time,
+            "after_token_latency": (last_token_time - first_token_time) / (num_tokens - 1) if num_tokens > 1 else None
+        }
+
+        self.put_msg(metrics_data)
+
         self.put_msg({"type": "finish"})
 
     def text_conversation_run(

LlamaCPP/llama_params.py

@@ -5,13 +5,15 @@ class LLMParams:
     device: int
     enable_rag: bool 
     model_repo_id: str
+    max_tokens: int
     generation_parameters: Dict[str, Any]
 
     def __init__(
-        self, prompt: list, device: int, enable_rag: bool, model_repo_id: str, **kwargs
+        self, prompt: list, device: int, enable_rag: bool, model_repo_id: str, max_tokens: int, **kwargs
     ) -> None:
         self.prompt = prompt
         self.device = device
         self.enable_rag = enable_rag
         self.model_repo_id = model_repo_id
+        self.max_tokens = max_tokens
         self.generation_parameters = kwargs

WebUI/build/build-config.json

@@ -32,8 +32,17 @@
       "filter": ["!__pycache__/", "!.cache/", "!db/", "!llm_cache/"]
     },
     {
-      "from": "external/intel_extension_for_pytorch-2.3.110+xpu-cp311-cp311-win_amd64.whl",
-      "to": "intel_extension_for_pytorch-2.3.110+xpu-cp311-cp311-win_amd64.whl"
+      "from": "external/OpenVINO",
+      "to": "OpenVINO",
+      "filter": ["!__pycache__/", "!.cache/", "!db/", "!llm_cache/"]
+    },
+    {
+      "from": "external/intel_extension_for_pytorch-2.3.110+bmg-cp311-cp311-win_amd64.whl",
+      "to": "intel_extension_for_pytorch-2.3.110+bmg-cp311-cp311-win_amd64.whl"
+    },
+    {
+      "from": "external/intel_extension_for_pytorch-2.3.110+arl_h-cp311-cp311-win_amd64.whl",
+      "to": "intel_extension_for_pytorch-2.3.110+arl_h-cp311-cp311-win_amd64.whl"
     },
     {
       "from": "external/llama_cpp_python-0.3.2-cp311-cp311-win_amd64.whl",

WebUI/build/scripts/fetch-python-package-resources.js

@@ -52,7 +52,7 @@ function fetchFile(url) {
       })
     })
     .on('error', (err) => {
-      console.error(`Error downloading ${embeddablePythonUrl}: ${err}`)
+      console.error(`Error downloading ${url}: ${err}`)
     })
 }
 

WebUI/electron/subprocesses/aiBackendService.ts

@@ -21,63 +21,62 @@ export class AiBackendService extends LongLivedPythonApiService {
   async *set_up(): AsyncIterable<SetupProgress> {
     this.setStatus('installing')
     this.appLogger.info('setting up service', this.name)
-    const self = this
 
     try {
       yield {
-        serviceName: self.name,
+        serviceName: this.name,
         step: 'start',
         status: 'executing',
         debugMessage: 'starting to set up environment',
       }
       // lsLevelZero will ensure uv and pip are installed
       await this.lsLevelZero.ensureInstalled()
 
-      const deviceArch = await self.lsLevelZero.detectDevice()
+      const deviceArch = await this.lsLevelZero.detectDevice()
       yield {
-        serviceName: self.name,
+        serviceName: this.name,
         step: `Detecting intel device`,
         status: 'executing',
         debugMessage: `detected intel hardware ${deviceArch}`,
       }
 
       yield {
-        serviceName: self.name,
+        serviceName: this.name,
         step: `install dependencies`,
         status: 'executing',
         debugMessage: `installing dependencies`,
       }
       const deviceSpecificRequirements = existingFileOrError(
-        path.join(self.serviceDir, `requirements-${deviceArch}.txt`),
+        path.join(this.serviceDir, `requirements-${deviceArch}.txt`),
       )
       await this.pip.run(['install', '-r', deviceSpecificRequirements])
-      if (deviceArch === 'bmg') {
-        const intelSpecificExtension = existingFileOrError(self.customIntelExtensionForPytorch)
+      if (deviceArch === 'bmg' || deviceArch === 'arl_h') {
+        const intelSpecificExtension = existingFileOrError(this.getIPEXWheelPath(deviceArch))
         await this.pip.run(['install', intelSpecificExtension])
       }
 
-      const commonRequirements = existingFileOrError(path.join(self.serviceDir, 'requirements.txt'))
+      const commonRequirements = existingFileOrError(path.join(this.serviceDir, 'requirements.txt'))
       await this.uvPip.run(['install', '-r', commonRequirements])
       yield {
-        serviceName: self.name,
+        serviceName: this.name,
         step: `install dependencies`,
         status: 'executing',
         debugMessage: `dependencies installed`,
       }
 
       this.setStatus('notYetStarted')
       yield {
-        serviceName: self.name,
+        serviceName: this.name,
         step: 'end',
         status: 'success',
         debugMessage: `service set up completely`,
       }
     } catch (e) {
-      self.appLogger.warn(`Set up of service failed due to ${e}`, self.name, true)
-      self.appLogger.warn(`Aborting set up of ${self.name} service environment`, self.name, true)
+      this.appLogger.warn(`Set up of service failed due to ${e}`, this.name, true)
+      this.appLogger.warn(`Aborting set up of ${this.name} service environment`, this.name, true)
       this.setStatus('installationFailed')
       yield {
-        serviceName: self.name,
+        serviceName: this.name,
         step: 'end',
         status: 'failed',
         debugMessage: `Failed to setup python environment due to ${e}`,

WebUI/electron/subprocesses/comfyUIBackendService.ts

@@ -38,19 +38,18 @@ export class ComfyUiBackendService extends LongLivedPythonApiService {
   async *set_up(): AsyncIterable<SetupProgress> {
     this.appLogger.info('setting up service', this.name)
     this.setStatus('installing')
-    const self = this
 
-    async function checkServiceDir(): Promise<boolean> {
-      if (!filesystem.existsSync(self.serviceDir)) {
+    const checkServiceDir = async (): Promise<boolean> => {
+      if (!filesystem.existsSync(this.serviceDir)) {
         return false
       }
 
       // Check if it's a valid git repo
       try {
-        await self.git.run(['-C', self.serviceDir, 'status'])
+        await this.git.run(['-C', this.serviceDir, 'status'])
       } catch (_e) {
         try {
-          filesystem.removeSync(self.serviceDir)
+          filesystem.removeSync(this.serviceDir)
         } finally {
           return false
         }
@@ -59,31 +58,31 @@ export class ComfyUiBackendService extends LongLivedPythonApiService {
       return true
     }
 
-    async function setupComfyUiBaseService(): Promise<void> {
+    const setupComfyUiBaseService = async (): Promise<void> => {
       if (await checkServiceDir()) {
-        self.appLogger.info('comfyUI already cloned, skipping', self.name)
+        this.appLogger.info('comfyUI already cloned, skipping', this.name)
       } else {
-        await self.git.run(['clone', self.remoteUrl, self.serviceDir])
-        await self.git.run(['-C', self.serviceDir, 'checkout', self.revision], {}, self.serviceDir)
+        await this.git.run(['clone', this.remoteUrl, this.serviceDir])
+        await this.git.run(['-C', this.serviceDir, 'checkout', this.revision], {}, this.serviceDir)
       }
 
       // Check whether all requirements are installed
       const requirementsTextPath = existingFileOrError(
-        path.join(self.serviceDir, 'requirements.txt'),
+        path.join(this.serviceDir, 'requirements.txt'),
       )
       try {
-        await self.uvPip.checkRequirementsTxt(requirementsTextPath)
+        await this.uvPip.checkRequirementsTxt(requirementsTextPath)
       } catch (_e) {
-        await self.uvPip.run(['install', '-r', requirementsTextPath])
+        await this.uvPip.run(['install', '-r', requirementsTextPath])
       }
     }
 
-    async function configureComfyUI(): Promise<void> {
+    const configureComfyUI = async (): Promise<void> => {
       try {
-        self.appLogger.info('Configuring extra model paths for comfyUI', self.name)
-        const extraModelPathsYaml = path.join(self.serviceDir, 'extra_model_paths.yaml')
+        this.appLogger.info('Configuring extra model paths for comfyUI', this.name)
+        const extraModelPathsYaml = path.join(this.serviceDir, 'extra_model_paths.yaml')
         const extraModelsYaml = `aipg:
-  base_path: ${path.resolve(self.baseDir, 'service/models/stable_diffusion')}
+  base_path: ${path.resolve(this.baseDir, 'service/models/stable_diffusion')}
   checkpoints: checkpoints
   clip: checkpoints
   vae: checkpoints
@@ -93,85 +92,85 @@ export class ComfyUiBackendService extends LongLivedPythonApiService {
           encoding: 'utf-8',
           flag: 'w',
         })
-        self.appLogger.info(
+        this.appLogger.info(
           `Configured extra model paths for comfyUI at ${extraModelPathsYaml} as ${extraModelsYaml} `,
-          self.name,
+          this.name,
         )
       } catch (_e) {
-        self.appLogger.error('Failed to configure extra model paths for comfyUI', self.name)
+        this.appLogger.error('Failed to configure extra model paths for comfyUI', this.name)
         throw new Error('Failed to configure extra model paths for comfyUI')
       }
     }
 
     try {
       yield {
-        serviceName: self.name,
+        serviceName: this.name,
         step: 'start',
         status: 'executing',
         debugMessage: 'starting to set up comfyUI environment',
       }
 
-      await self.lsLevelZero.ensureInstalled()
-      await self.git.ensureInstalled()
+      await this.lsLevelZero.ensureInstalled()
+      await this.git.ensureInstalled()
 
       yield {
-        serviceName: self.name,
+        serviceName: this.name,
         step: `Detecting intel device`,
         status: 'executing',
         debugMessage: `Trying to identify intel hardware`,
       }
-      const deviceArch = await self.lsLevelZero.detectDevice()
+      const deviceArch = await this.lsLevelZero.detectDevice()
       yield {
-        serviceName: self.name,
+        serviceName: this.name,
         step: `Detecting intel device`,
         status: 'executing',
         debugMessage: `detected intel hardware ${deviceArch}`,
       }
 
       yield {
-        serviceName: self.name,
+        serviceName: this.name,
         step: `install dependencies`,
         status: 'executing',
         debugMessage: `installing dependencies`,
       }
       const deviceSpecificRequirements = existingFileOrError(
         path.join(aiBackendServiceDir(), `requirements-${deviceArch}.txt`),
       )
-      await self.uvPip.pip.run(['install', '-r', deviceSpecificRequirements])
-      if (deviceArch === 'bmg') {
-        const intelSpecificExtension = existingFileOrError(self.customIntelExtensionForPytorch)
-        await self.uvPip.pip.run(['install', intelSpecificExtension])
+      await this.uvPip.pip.run(['install', '-r', deviceSpecificRequirements])
+      if (deviceArch === 'bmg' || deviceArch === 'arl_h') {
+        const intelSpecificExtension = existingFileOrError(this.getIPEXWheelPath(deviceArch))
+        await this.uvPip.pip.run(['install', intelSpecificExtension])
       }
       yield {
-        serviceName: self.name,
+        serviceName: this.name,
         step: `install dependencies`,
         status: 'executing',
         debugMessage: `dependencies installed`,
       }
 
       yield {
-        serviceName: self.name,
+        serviceName: this.name,
         step: `install comfyUI`,
         status: 'executing',
         debugMessage: `installing comfyUI base repo`,
       }
       await setupComfyUiBaseService()
       yield {
-        serviceName: self.name,
+        serviceName: this.name,
         step: `install comfyUI`,
         status: 'executing',
         debugMessage: `installation of comfyUI base repo complete`,
       }
 
       yield {
-        serviceName: self.name,
+        serviceName: this.name,
         step: `configure comfyUI`,
         status: 'executing',
         debugMessage: `configuring comfyUI base repo`,
       }
       await configureComfyUI()
       yield {
-        serviceName: self.name,
+        serviceName: this.name,
         step: `configure comfyUI`,
         status: 'executing',
         debugMessage: `configured comfyUI base repo`,
@@ -180,17 +179,17 @@ export class ComfyUiBackendService extends LongLivedPythonApiService {
 
       this.setStatus('notYetStarted')
       yield {
-        serviceName: self.name,
+        serviceName: this.name,
         step: 'end',
         status: 'success',
         debugMessage: `service set up completely`,
       }
     } catch (e) {
-      self.appLogger.warn(`Set up of service failed due to ${e}`, self.name, true)
-      self.appLogger.warn(`Aborting set up of ${self.name} service environment`, self.name, true)
+      this.appLogger.warn(`Set up of service failed due to ${e}`, this.name, true)
+      this.appLogger.warn(`Aborting set up of ${this.name} service environment`, this.name, true)
       this.setStatus('installationFailed')
       yield {
-        serviceName: self.name,
+        serviceName: this.name,
         step: 'end',
         status: 'failed',
         debugMessage: `Failed to setup comfyUI service due to ${e}`,

WebUI/electron/subprocesses/deviceArch.ts

@@ -59,8 +59,8 @@ const ID2ARCH: { [key: number]: Arch } = {
 
   // // arl
   // 0x7D67: "arl",
-  // 0x7D51: "arl",
-  // 0x7DD1: "arl",
+  0x7d51: 'arl_h',
+  0x7dd1: 'arl_h',
   // 0x7D41: "arl",
 }
 
@@ -71,8 +71,10 @@ export function getDeviceArch(deviceId: number): Arch {
 export function getArchPriority(arch: Arch): number {
   switch (arch) {
     case 'bmg':
-      return 4
+      return 5
     case 'acm':
+      return 4
+    case 'arl_h':
       return 3
     case 'lnl':
       return 2
@@ -83,4 +85,4 @@ export function getArchPriority(arch: Arch): number {
   }
 }
 
-export type Arch = 'bmg' | 'acm' | 'lnl' | 'mtl' | 'unknown'
+export type Arch = 'bmg' | 'acm' | 'arl_h' | 'lnl' | 'mtl' | 'unknown'