From 6665c9960e2c07a69197aaa673ab78368e7453e4 Mon Sep 17 00:00:00 2001 From: Robin Huang Date: Tue, 25 Jun 2024 19:30:33 -0700 Subject: [PATCH] Update. --- action.yml | 41 +++------------------ queue_prompt.py | 95 +++++++++++++++++++++++++++++++++---------------- 2 files changed, 69 insertions(+), 67 deletions(-) diff --git a/action.yml b/action.yml index 6274c8f..3fcdcdf 100644 --- a/action.yml +++ b/action.yml @@ -98,12 +98,6 @@ runs: comfy --version comfy --skip-prompt --no-enable-telemetry env - - name: '[Unix] Install Comfy' - if: ${{ inputs.os != 'windows' }} - shell: bash -el {0} - run: | - comfy install --skip-manager - - name: '[Unix-Mac-Only] Install Pytorch nightly' if: ${{ inputs.os == 'macos' }} shell: bash -el {0} @@ -136,27 +130,14 @@ runs: run: | cd ${{ github.action_path }} ls "${{ github.workspace }}" - # python3 download-models.py raw '${{ inputs.models-json }}' "${{ github.workspace }}/models/" - - - name: '[Unix] Run Python application quick test' - if: ${{ inputs.os != 'windows' }} - shell: bash -el {0} - run: | - python main.py --quick-test-for-ci + python3 download-models.py raw '${{ inputs.models-json }}' "${{ github.workspace }}/models/" - - name: '[Unix] Run Python application' + - name: '[Unix] Run Python application in background' if: ${{ inputs.os != 'windows' }} shell: bash -el {0} run: | python3 main.py --force-fp16 > application.log 2>&1 & - - name: '[Unix] Check if the server is running' - if: ${{ inputs.os != 'windows' }} - run: | - cd ${{ github.action_path }} - python3 poll_server_start.py > application.log 2>&1 - shell: bash -el {0} - - name: '[Unix] Get Commit Details' id: unix_get_commit_details if: ${{ inputs.os != 'windows' }} @@ -177,22 +158,8 @@ runs: if: ${{ inputs.os != 'windows' }} shell: bash -el {0} run: | - echo "Full path to the JSON file: $inputs.workflow_filenames" - python3 queue_prompt.py --comfy_workflow_names $inputs.workflow_filenames --github_action_workflow_name ${{ github.workflow }} --os ${{ runner.os }} --run_id ${{ github.run_id }} --gsc_bucket_name your-bucket-name --workspace_path /path/to/workspace --output_file_prefix output - - - name: '[Unix] Auth to GCS' - uses: "google-github-actions/auth@v2" - if: ${{ inputs.os != 'windows' }} - with: - credentials_json: "${{ inputs.google_credentials }}" - - - name: '[Unix] Upload log file to GCS' - if: ${{ ( success() || failure() ) && inputs.os != 'windows' }} - id: unix_upload-log-files - uses: google-github-actions/upload-cloud-storage@v2 - with: - path: ${{ github.workspace }}/application.log - destination: ${{ inputs.gcs_bucket_name }}/logs/${{ github.job }}-${{ inputs.os }}-${{ inputs.workflow_name }}-run${{ github.run_id }} + echo "Running workflows: $inputs.workflow_filenames" + python3 queue_prompt.py --comfy_workflow_names $inputs.workflow_filenames --github_action_workflow_name ${{ github.workflow }} --os ${{ runner.os }} --run_id ${{ github.run_id }} --gsc_bucket_name ${{ inputs.gcs_bucket_name }} --workspace_path ${{ github.workspace }} --output_file_prefix output --repo ${{ github.repository }} --commit_hash $COMMIT_HASH --commit_time $TIMESTAMP --commit_message $MESSAGE --branch_name ${{ github.ref_name }} --api_endpoint ${{ inputs.api_endpoint }} - name: '[Unix] Debug print out commit timestamp and commit message' if: ${{ inputs.os != 'windows' }} diff --git a/queue_prompt.py b/queue_prompt.py index 3e16412..795829f 100644 --- a/queue_prompt.py +++ b/queue_prompt.py @@ -8,38 +8,46 @@ def read_json_file(file_path): - with open(file_path, 'r', encoding='utf-8') as file: + with open(file_path, "r", encoding="utf-8") as file: return json.load(file) + def post_json_to_server(json_data, url): return requests.post(url, json=json_data) + def get_status(prompt_id, url): response = requests.get(f"{url}/{prompt_id}") if response.status_code == 200: return response.json() return None + def is_completed(status_response, prompt_id): # Check if the expected fields exist in the response return ( status_response and prompt_id in status_response - and 'status' in status_response[prompt_id] - and status_response[prompt_id]['status'].get('completed', False) + and "status" in status_response[prompt_id] + and status_response[prompt_id]["status"].get("completed", False) ) def upload_to_gcs(bucket_name: str, destination_blob_name: str, source_file_name: str): + print( + f"Uploading file {source_file_name} to GCS bucket {bucket_name} as {destination_blob_name}" + ) storage_client = storage.Client() bucket = storage_client.get_bucket(bucket_name) blob = bucket.blob(destination_blob_name) blob.upload_from_filename(source_file_name) + print(f"File {source_file_name} uploaded to {destination_blob_name}") -def send_payload_to_api(args, output_files_gcs_paths, workflow_name, - start_time, end_time): +def send_payload_to_api( + args, output_files_gcs_paths, workflow_name, start_time, end_time +): # Create the payload as a dictionary payload = { @@ -55,25 +63,27 @@ def send_payload_to_api(args, output_files_gcs_paths, workflow_name, "bucket_name": args.gsc_bucket_name, "workflow_name": workflow_name, "start_time": start_time, - "end_time": end_time + "end_time": end_time, } # Convert payload dictionary to a JSON string payload_json = json.dumps(payload) # Send POST request - headers = {'Content-Type': 'application/json'} + headers = {"Content-Type": "application/json"} response = requests.post(args.api_endpoint, headers=headers, data=payload_json) # Write response to application.log log_file_path = "./application.log" - with open(log_file_path, 'w', encoding='utf-8') as log_file: + with open(log_file_path, "w", encoding="utf-8") as log_file: log_file.write("\n##### Comfy CI Post Response #####\n") log_file.write(response.text) # Check the response code if response.status_code != 200: - print(f"API request failed with status code {response.status_code} and response body") + print( + f"API request failed with status code {response.status_code} and response body" + ) print(response.text) exit(1) else: @@ -81,47 +91,72 @@ def send_payload_to_api(args, output_files_gcs_paths, workflow_name, return response.status_code + def main(args): # Split the workflow file names using "," - workflow_files = args.comfy_workflow_names.split(',') - + workflow_files = args.comfy_workflow_names.split(",") + print("Running workflows ") counter = 1 for workflow_file_name in workflow_files: # Construct the file path file_path = f"workflows/{workflow_file_name}" + print(f"Running workflow {file_path}") start_time = int(datetime.datetime.now().timestamp()) subprocess.run( - ["comfycli", "run", "--workflow", file_path], + ["comfy", "--here", "run", "--workflow", file_path], check=True, ) + print(f"Workflow {file_path} completed") end_time = int(datetime.datetime.now().timestamp()) - #TODO: add support for multiple file outputs + # TODO: add support for multiple file outputs gs_path = f"output-files/{args.github_action_workflow_name}-{args.os}-{workflow_file_name}-run-${args.run_id}" - upload_to_gcs(args.gsc_bucket_name, gs_path, f"{args.workspace_path}/output/{args.output_file_prefix}_{counter:05}_.png") + upload_to_gcs( + args.gsc_bucket_name, + gs_path, + f"{args.workspace_path}/output/{args.output_file_prefix}_{counter:05}_.png", + ) send_payload_to_api(args, gs_path, workflow_file_name, start_time, end_time) counter += 1 if __name__ == "__main__": - parser = argparse.ArgumentParser(description='Send a JSON file contents to a server as a prompt.') - parser.add_argument('--api-endpoint', type=str, help='API endpoint.') - parser.add_argument('--comfy-workflow-names', type=str, help='List of comfy workflow names.') - parser.add_argument('--github-action-workflow-name', type=str, help='Github action workflow name.') - parser.add_argument('--os', type=str, help='Operating system.') - parser.add_argument('--run-id', type=str, help='Github Run ID.') - parser.add_argument('--repo', type=str, help='Github repo.') - parser.add_argument('--cuda-version', type=str, help='CUDA version.') - parser.add_argument('--commit-hash', type=str, help='Commit hash.') - parser.add_argument('--commit-time', type=str, help='Commit time.') - parser.add_argument('--commit-message', type=str, help='Commit message.') - parser.add_argument('--branch-name', type=str, help='Branch name.') - parser.add_argument('--gsc-bucket-name', type=str, help='Name of the GCS bucket to store the output files in.') - parser.add_argument('--workspace-path', type=str, help='Workspace (ComfyUI repo) path, likely ${HOME}/action_runners/_work/ComfyUI/ComfyUI/.') - parser.add_argument('--action-path', type=str, help='Action path., likely ${HOME}/action_runners/_work/comfy-action/.') - parser.add_argument('--output-file-prefix', type=str, help='Output file prefix.') + parser = argparse.ArgumentParser( + description="Send a JSON file contents to a server as a prompt." + ) + parser.add_argument("--api-endpoint", type=str, help="API endpoint.") + parser.add_argument( + "--comfy-workflow-names", type=str, help="List of comfy workflow names." + ) + parser.add_argument( + "--github-action-workflow-name", type=str, help="Github action workflow name." + ) + parser.add_argument("--os", type=str, help="Operating system.") + parser.add_argument("--run-id", type=str, help="Github Run ID.") + parser.add_argument("--repo", type=str, help="Github repo.") + parser.add_argument("--cuda-version", type=str, help="CUDA version.") + parser.add_argument("--commit-hash", type=str, help="Commit hash.") + parser.add_argument("--commit-time", type=str, help="Commit time.") + parser.add_argument("--commit-message", type=str, help="Commit message.") + parser.add_argument("--branch-name", type=str, help="Branch name.") + parser.add_argument( + "--gsc-bucket-name", + type=str, + help="Name of the GCS bucket to store the output files in.", + ) + parser.add_argument( + "--workspace-path", + type=str, + help="Workspace (ComfyUI repo) path, likely ${HOME}/action_runners/_work/ComfyUI/ComfyUI/.", + ) + parser.add_argument( + "--action-path", + type=str, + help="Action path., likely ${HOME}/action_runners/_work/comfy-action/.", + ) + parser.add_argument("--output-file-prefix", type=str, help="Output file prefix.") args = parser.parse_args() main(args)