From 0a0c5f98d7cb8a6752608c750749806effcd9efd Mon Sep 17 00:00:00 2001 From: dilverse <109769432+dilverse@users.noreply.github.com> Date: Wed, 23 Oct 2024 13:18:39 -0700 Subject: [PATCH 1/8] Add prompt_object API --- minio/api.py | 86 +++++++++++++++++++++++++++++++++++----------------- 1 file changed, 58 insertions(+), 28 deletions(-) diff --git a/minio/api.py b/minio/api.py index ec64422f9..23f3b1be3 100644 --- a/minio/api.py +++ b/minio/api.py @@ -28,6 +28,7 @@ from __future__ import absolute_import, annotations import itertools +import json import os import tarfile from collections.abc import Iterable @@ -50,7 +51,7 @@ from urllib3.util import Timeout -from . import __title__, __version__, time +from . import time from .commonconfig import (COPY, REPLACE, ComposeSource, CopySource, SnowballObject, Tags) from .credentials import Credentials, StaticProvider @@ -306,6 +307,7 @@ def _url_open( headers=http_headers, preload_content=preload_content, ) + # print(response.status, response.data.decode()) if self._trace_stream: self._trace_stream.write(f"HTTP/1.1 {response.status}\n") @@ -334,8 +336,8 @@ def _url_open( if ( method != "HEAD" and "application/xml" not in response.headers.get( - "content-type", "", - ).split(";") + "content-type", "", + ).split(";") ): if self._trace_stream: self._trace_stream.write("----------END-HTTP----------\n") @@ -1125,7 +1127,7 @@ def fget_object( etag = queryencode(cast(str, stat.etag)) # Write to a temporary file "file_path.part.minio" before saving. tmp_file_path = ( - tmp_file_path or f"{file_path}.{etag}.part.minio" + tmp_file_path or f"{file_path}.{etag}.part.minio" ) response = None @@ -1145,7 +1147,7 @@ def fget_object( progress.set_meta(object_name=object_name, total_length=length) with open(tmp_file_path, "wb") as tmp_file: - for data in response.stream(amt=1024*1024): + for data in response.stream(amt=1024 * 1024): size = tmp_file.write(data) if progress: progress.update(size) @@ -1258,6 +1260,34 @@ def get_object( preload_content=False, ) + def prompt_object( + self, + bucket_name: str, + object_name: str, + prompt: str, + lambda_arn: str | None = None, + **kwargs: Any | None, + ) -> BaseHTTPResponse: + check_bucket_name(bucket_name, s3_check=self._base_url.is_aws_host) + + check_object_name(object_name) + + extra_query_params = {} + extra_query_params["lambdaArn"] = lambda_arn or "" + + prompt_body = kwargs + prompt_body["prompt"] = prompt + + body = json.dumps(kwargs) + return self._execute( + "POST", + bucket_name, + object_name, + query_params=extra_query_params, + body=body, + preload_content=False, + ) + def copy_object( self, bucket_name: str, @@ -1592,11 +1622,11 @@ def compose_object( part_number += 1 if src.length is not None: headers["x-amz-copy-source-range"] = ( - f"bytes={offset}-{offset+src.length-1}" + f"bytes={offset}-{offset + src.length - 1}" ) elif src.offset is not None: headers["x-amz-copy-source-range"] = ( - f"bytes={offset}-{offset+size-1}" + f"bytes={offset}-{offset + size - 1}" ) etag, _ = self._upload_part_copy( bucket_name, @@ -1756,20 +1786,20 @@ def _upload_part_task(self, args): return args[5], self._upload_part(*args) def put_object( - self, - bucket_name: str, - object_name: str, - data: BinaryIO, - length: int, - content_type: str = "application/octet-stream", - metadata: DictType | None = None, - sse: Sse | None = None, - progress: ProgressType | None = None, - part_size: int = 0, - num_parallel_uploads: int = 3, - tags: Tags | None = None, - retention: Retention | None = None, - legal_hold: bool = False + self, + bucket_name: str, + object_name: str, + data: BinaryIO, + length: int, + content_type: str = "application/octet-stream", + metadata: DictType | None = None, + sse: Sse | None = None, + progress: ProgressType | None = None, + part_size: int = 0, + num_parallel_uploads: int = 3, + tags: Tags | None = None, + retention: Retention | None = None, + legal_hold: bool = False ) -> ObjectWriteResult: """ Uploads data from a stream to an object in a bucket. @@ -1909,7 +1939,7 @@ def put_object( parts = [Part(0, "")] * part_count while not result.empty(): part_number, etag = result.get() - parts[part_number-1] = Part(part_number, etag) + parts[part_number - 1] = Part(part_number, etag) upload_result = self._complete_multipart_upload( bucket_name, object_name, cast(str, upload_id), parts, @@ -2074,10 +2104,10 @@ def stat_object( ) def remove_object( - self, - bucket_name: str, - object_name: str, - version_id: str | None = None + self, + bucket_name: str, + object_name: str, + version_id: str | None = None ): """ Remove an object. @@ -3047,7 +3077,7 @@ def upload_snowball_objects( object_name, cast(BinaryIO, fileobj), length, - metadata=cast(Union[DictType, None], metadata), + metadata=cast(Union[DictType, None], metadata), sse=sse, tags=tags, retention=retention, @@ -3066,7 +3096,7 @@ def _list_objects( max_keys: int | None = None, # all prefix: str | None = None, # all start_after: str | None = None, - # all: v1:marker, versioned:key_marker + # all: v1:marker, versioned:key_marker version_id_marker: str | None = None, # versioned use_api_v1: bool = False, include_version: bool = False, From 0c279ab76491b805e7c18b2f290f0bccc9a924e9 Mon Sep 17 00:00:00 2001 From: dilverse <109769432+dilverse@users.noreply.github.com> Date: Thu, 24 Oct 2024 13:14:49 -0700 Subject: [PATCH 2/8] prompt_object enhancement * using DictType for `kwargs` * Added initial docstring for prompt_object --- minio/api.py | 32 +++++++++++++++++++++++++++----- 1 file changed, 27 insertions(+), 5 deletions(-) diff --git a/minio/api.py b/minio/api.py index 23f3b1be3..ff863e7ec 100644 --- a/minio/api.py +++ b/minio/api.py @@ -336,8 +336,8 @@ def _url_open( if ( method != "HEAD" and "application/xml" not in response.headers.get( - "content-type", "", - ).split(";") + "content-type", "", + ).split(";") ): if self._trace_stream: self._trace_stream.write("----------END-HTTP----------\n") @@ -1127,7 +1127,7 @@ def fget_object( etag = queryencode(cast(str, stat.etag)) # Write to a temporary file "file_path.part.minio" before saving. tmp_file_path = ( - tmp_file_path or f"{file_path}.{etag}.part.minio" + tmp_file_path or f"{file_path}.{etag}.part.minio" ) response = None @@ -1266,8 +1266,30 @@ def prompt_object( object_name: str, prompt: str, lambda_arn: str | None = None, - **kwargs: Any | None, + **kwargs: DictType | None, ) -> BaseHTTPResponse: + """ + Prompt an object using natural language. + + :param bucket_name: Name of the bucket. + :param object_name: Object name in the bucket. + :param prompt: Prompt the Object to interact with the AI model. + request. + :param lambda_arn: Lambda ARN to use for prompt. + :param kwargs: Extra parameters for advanced usage. + :return: :class:`urllib3.response.BaseHTTPResponse` object. + + Example:: + # prompt an object. + response = None + try: + response = client.get_object("my-bucket", "my-object", "Describe the object for me") + # Read data from response. + finally: + if response: + response.close() + response.release_conn() + """ check_bucket_name(bucket_name, s3_check=self._base_url.is_aws_host) check_object_name(object_name) @@ -1284,7 +1306,7 @@ def prompt_object( bucket_name, object_name, query_params=extra_query_params, - body=body, + body=body.encode(), preload_content=False, ) From bf6616c07e10be53697d6fd5ada4f37069ad1e46 Mon Sep 17 00:00:00 2001 From: dilverse <109769432+dilverse@users.noreply.github.com> Date: Thu, 24 Oct 2024 13:47:59 -0700 Subject: [PATCH 3/8] Fix lint errors --- minio/api.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/minio/api.py b/minio/api.py index ad13928fd..0eea22576 100644 --- a/minio/api.py +++ b/minio/api.py @@ -336,8 +336,8 @@ def _url_open( if ( method != "HEAD" and "application/xml" not in response.headers.get( - "content-type", "", - ).split(";") + "content-type", "", + ).split(";") ): if self._trace_stream: self._trace_stream.write("----------END-HTTP----------\n") @@ -1127,7 +1127,7 @@ def fget_object( etag = queryencode(cast(str, stat.etag)) # Write to a temporary file "file_path.part.minio" before saving. tmp_file_path = ( - tmp_file_path or f"{file_path}.{etag}.part.minio" + tmp_file_path or f"{file_path}.{etag}.part.minio" ) response = None @@ -1283,7 +1283,9 @@ def prompt_object( # prompt an object. response = None try: - response = client.get_object("my-bucket", "my-object", "Describe the object for me") + response = client.get_object("my-bucket", + "my-object", + "Describe the object for me") # Read data from response. finally: if response: From 34015a301c7af1be3891184806949f5df17ca612 Mon Sep 17 00:00:00 2001 From: dilverse <109769432+dilverse@users.noreply.github.com> Date: Thu, 24 Oct 2024 15:03:36 -0700 Subject: [PATCH 4/8] Add support for customer encryption key and versioned objects --- minio/api.py | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/minio/api.py b/minio/api.py index 0eea22576..40dbbdade 100644 --- a/minio/api.py +++ b/minio/api.py @@ -1266,6 +1266,9 @@ def prompt_object( object_name: str, prompt: str, lambda_arn: str | None = None, + request_headers: DictType | None = None, + ssec: SseCustomerKey | None = None, + version_id: str | None = None, **kwargs: DictType | None, ) -> BaseHTTPResponse: """ @@ -1276,6 +1279,9 @@ def prompt_object( :param prompt: Prompt the Object to interact with the AI model. request. :param lambda_arn: Lambda ARN to use for prompt. + :param request_headers: Any additional headers to be added with POST + :param ssec: Server-side encryption customer key. + :param version_id: Version-ID of the object. :param kwargs: Extra parameters for advanced usage. :return: :class:`urllib3.response.BaseHTTPResponse` object. @@ -1283,8 +1289,8 @@ def prompt_object( # prompt an object. response = None try: - response = client.get_object("my-bucket", - "my-object", + response = client.get_object( + "my-bucket", "my-object", "Describe the object for me") # Read data from response. finally: @@ -1293,11 +1299,16 @@ def prompt_object( response.release_conn() """ check_bucket_name(bucket_name, s3_check=self._base_url.is_aws_host) - check_object_name(object_name) + check_ssec(ssec) + + headers = cast(DictType, ssec.headers() if ssec else {}) + headers.update(request_headers or {}) - extra_query_params = {} - extra_query_params["lambdaArn"] = lambda_arn or "" + extra_query_params = {"lambdaArn": lambda_arn or ""} + + if version_id: + extra_query_params["versionId"] = version_id prompt_body = kwargs prompt_body["prompt"] = prompt @@ -1307,6 +1318,7 @@ def prompt_object( "POST", bucket_name, object_name, + headers=cast(DictType, headers), query_params=extra_query_params, body=body.encode(), preload_content=False, From 6f6b887e2dcb98936852f4428ef5ea6a26cdbda2 Mon Sep 17 00:00:00 2001 From: dilverse <109769432+dilverse@users.noreply.github.com> Date: Thu, 24 Oct 2024 15:35:44 -0700 Subject: [PATCH 5/8] Use prompt_body instead of kwargs --- minio/api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/minio/api.py b/minio/api.py index 40dbbdade..54cc8d2e3 100644 --- a/minio/api.py +++ b/minio/api.py @@ -1313,7 +1313,7 @@ def prompt_object( prompt_body = kwargs prompt_body["prompt"] = prompt - body = json.dumps(kwargs) + body = json.dumps(prompt_body) return self._execute( "POST", bucket_name, From 3026797970ae531b03921dac2c07fbfdc3841586 Mon Sep 17 00:00:00 2001 From: dilverse <109769432+dilverse@users.noreply.github.com> Date: Thu, 24 Oct 2024 15:47:48 -0700 Subject: [PATCH 6/8] Fix lint errors --- Makefile | 2 +- minio/api.py | 7 +++---- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index b4463ef77..8474e857c 100644 --- a/Makefile +++ b/Makefile @@ -3,7 +3,7 @@ default: tests getdeps: @echo "Installing required dependencies" - @pip install --user --upgrade autopep8 certifi pytest pylint urllib3 argon2-cffi pycryptodome typing-extensions mypy + @pip install --upgrade autopep8 certifi pytest pylint urllib3 argon2-cffi pycryptodome typing-extensions mypy check: getdeps @echo "Running checks" diff --git a/minio/api.py b/minio/api.py index 54cc8d2e3..d694319ae 100644 --- a/minio/api.py +++ b/minio/api.py @@ -307,7 +307,6 @@ def _url_open( headers=http_headers, preload_content=preload_content, ) - # print(response.status, response.data.decode()) if self._trace_stream: self._trace_stream.write(f"HTTP/1.1 {response.status}\n") @@ -336,8 +335,8 @@ def _url_open( if ( method != "HEAD" and "application/xml" not in response.headers.get( - "content-type", "", - ).split(";") + "content-type", "", + ).split(";") ): if self._trace_stream: self._trace_stream.write("----------END-HTTP----------\n") @@ -1127,7 +1126,7 @@ def fget_object( etag = queryencode(cast(str, stat.etag)) # Write to a temporary file "file_path.part.minio" before saving. tmp_file_path = ( - tmp_file_path or f"{file_path}.{etag}.part.minio" + tmp_file_path or f"{file_path}.{etag}.part.minio" ) response = None From 1d0a017de8c09d92b2dbda68efe8b5aadf9fefb7 Mon Sep 17 00:00:00 2001 From: dilverse <109769432+dilverse@users.noreply.github.com> Date: Thu, 24 Oct 2024 15:49:51 -0700 Subject: [PATCH 7/8] Added make file changes --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 8474e857c..b4463ef77 100644 --- a/Makefile +++ b/Makefile @@ -3,7 +3,7 @@ default: tests getdeps: @echo "Installing required dependencies" - @pip install --upgrade autopep8 certifi pytest pylint urllib3 argon2-cffi pycryptodome typing-extensions mypy + @pip install --user --upgrade autopep8 certifi pytest pylint urllib3 argon2-cffi pycryptodome typing-extensions mypy check: getdeps @echo "Running checks" From 734b9c360f93d3031b4fce1c1d646d9633485d5f Mon Sep 17 00:00:00 2001 From: dilverse <109769432+dilverse@users.noreply.github.com> Date: Thu, 24 Oct 2024 18:27:55 -0700 Subject: [PATCH 8/8] Fix Lint errors and type casting errors --- minio/api.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/minio/api.py b/minio/api.py index d694319ae..7c4384751 100644 --- a/minio/api.py +++ b/minio/api.py @@ -35,7 +35,7 @@ from datetime import datetime, timedelta from io import BytesIO from random import random -from typing import BinaryIO, Iterator, TextIO, Tuple, Union, cast +from typing import Any, BinaryIO, Iterator, TextIO, Tuple, Union, cast from urllib.parse import urlunsplit from xml.etree import ElementTree as ET @@ -1268,7 +1268,7 @@ def prompt_object( request_headers: DictType | None = None, ssec: SseCustomerKey | None = None, version_id: str | None = None, - **kwargs: DictType | None, + **kwargs: Any | None, ) -> BaseHTTPResponse: """ Prompt an object using natural language. @@ -1318,7 +1318,7 @@ def prompt_object( bucket_name, object_name, headers=cast(DictType, headers), - query_params=extra_query_params, + query_params=cast(DictType, extra_query_params), body=body.encode(), preload_content=False, )